From b861e47e52eb718cff056bc720e2c282f99eb86d Mon Sep 17 00:00:00 2001 From: rnichi1 <71671446+rnichi1@users.noreply.github.com> Date: Sun, 2 Feb 2025 17:12:13 +0100 Subject: [PATCH 1/7] Add llm correction --- build.gradle.kts | 2 + .../ifi/access/config/ModelMapperConfig.java | 44 ++++- .../ifi/access/controller/CourseController.kt | 1 + .../kotlin/ch/uzh/ifi/access/model/Task.kt | 36 ++++ .../uzh/ifi/access/model/dto/AssistantDTO.kt | 80 ++++++++ .../access/model/dto/AssistantResponseDTO.kt | 45 +++++ .../uzh/ifi/access/model/dto/LLMConfigDTO.kt | 16 ++ .../ch/uzh/ifi/access/model/dto/TaskDTO.kt | 3 +- .../access/service/CourseConfigImporter.kt | 68 ++++++- .../uzh/ifi/access/service/CourseService.kt | 172 ++++++++++++++++-- .../db/migration/V2_1__add_llm_fields.sql | 12 ++ .../migration/V2_2__add_llm_model_field.sql | 2 + .../db/migration/V2_3__add_llm_max_points.sql | 2 + 13 files changed, 456 insertions(+), 27 deletions(-) create mode 100644 src/main/kotlin/ch/uzh/ifi/access/model/dto/AssistantDTO.kt create mode 100644 src/main/kotlin/ch/uzh/ifi/access/model/dto/AssistantResponseDTO.kt create mode 100644 src/main/kotlin/ch/uzh/ifi/access/model/dto/LLMConfigDTO.kt create mode 100644 src/main/resources/db/migration/V2_1__add_llm_fields.sql create mode 100644 src/main/resources/db/migration/V2_2__add_llm_model_field.sql create mode 100644 src/main/resources/db/migration/V2_3__add_llm_max_points.sql diff --git a/build.gradle.kts b/build.gradle.kts index cf65aeae..f22f0ce7 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -66,6 +66,8 @@ dependencies { implementation("org.apache.httpcomponents.client5:httpclient5:5.2.1") implementation("org.apache.tika:tika-core:2.8.0") implementation("org.flywaydb:flyway-core") + implementation("com.squareup.okhttp3:okhttp:4.9.2") + implementation("com.fasterxml.jackson.module:jackson-module-kotlin:2.12.+") implementation("io.github.oshai:kotlin-logging-jvm:5.1.0") testImplementation("org.springframework.security:spring-security-test") testImplementation("org.springframework.boot:spring-boot-starter-test") { diff --git a/src/main/java/ch/uzh/ifi/access/config/ModelMapperConfig.java b/src/main/java/ch/uzh/ifi/access/config/ModelMapperConfig.java index 57d7fdb8..0beb780f 100644 --- a/src/main/java/ch/uzh/ifi/access/config/ModelMapperConfig.java +++ b/src/main/java/ch/uzh/ifi/access/config/ModelMapperConfig.java @@ -5,16 +5,15 @@ import ch.uzh.ifi.access.model.Submission; import ch.uzh.ifi.access.model.Task; import ch.uzh.ifi.access.model.constants.Visibility; -import ch.uzh.ifi.access.model.dto.AssignmentDTO; -import ch.uzh.ifi.access.model.dto.CourseDTO; -import ch.uzh.ifi.access.model.dto.SubmissionDTO; -import ch.uzh.ifi.access.model.dto.TaskDTO; +import ch.uzh.ifi.access.model.dto.*; import org.apache.commons.lang3.ObjectUtils; import org.modelmapper.ModelMapper; import org.modelmapper.convention.MatchingStrategies; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import java.util.Objects; + @Configuration public class ModelMapperConfig { // TODO: convert to Kotlin. This is very tricky, because ModelMapper is very Java-specific. Maybe replace it. @@ -30,11 +29,46 @@ public ModelMapper modelMapper() { modelMapper.typeMap(AssignmentDTO.class, Assignment.class) .addMappings(mapping -> mapping.skip(AssignmentDTO::getTasks, Assignment::setTasks)); modelMapper.typeMap(TaskDTO.class, Task.class) - .addMappings(mapping -> mapping.skip(TaskDTO::getFiles, Task::setFiles)); + .addMappings(mapper -> { + mapper.skip(Task::setLlmSubmission); + mapper.skip(Task::setLlmSolution); + mapper.skip(Task::setLlmRubrics); + mapper.skip(Task::setLlmCot); + mapper.skip(Task::setLlmVoting); + mapper.skip(Task::setLlmExamples); + mapper.skip(Task::setLlmPrompt); + mapper.skip(Task::setLlmPre); + mapper.skip(Task::setLlmPost); + mapper.skip(Task::setLlmModel); + mapper.skip(Task::setLlmTemperature); + mapper.skip(Task::setLlmMaxPoints); + }); modelMapper.typeMap(SubmissionDTO.class, Submission.class) .addMappings(mapping -> mapping.skip(Submission::setFiles)); modelMapper.createTypeMap(String.class, Visibility.class) .setConverter(context -> Visibility.valueOf(context.getSource().toUpperCase())); + modelMapper.addConverter((context) -> { + TaskDTO source = (TaskDTO) context.getSource(); + Task destination = (Task) context.getDestination(); + + if (source.getLlm() != null) { + LLMConfigDTO llm = source.getLlm(); + destination.setLlmSubmission(llm.getSubmission()); + destination.setLlmSolution(llm.getSolution()); + destination.setLlmRubrics(llm.getRubrics()); + destination.setLlmCot(llm.getCot()); + destination.setLlmVoting(llm.getVoting()); + destination.setLlmExamples(llm.getExamples()); + destination.setLlmPrompt(llm.getPrompt()); + destination.setLlmPre(llm.getPre()); + destination.setLlmPost(llm.getPost()); + destination.setLlmModel(llm.getModel()); + destination.setLlmTemperature(llm.getTemperature()); + destination.setLlmMaxPoints(llm.getMaxPoints()); + } + + return destination; + }, TaskDTO.class, Task.class); return modelMapper; } } diff --git a/src/main/kotlin/ch/uzh/ifi/access/controller/CourseController.kt b/src/main/kotlin/ch/uzh/ifi/access/controller/CourseController.kt index ca133a4e..efd44657 100644 --- a/src/main/kotlin/ch/uzh/ifi/access/controller/CourseController.kt +++ b/src/main/kotlin/ch/uzh/ifi/access/controller/CourseController.kt @@ -14,6 +14,7 @@ import org.springframework.web.bind.annotation.* import org.springframework.web.server.ResponseStatusException + @RestController class CourseRootController( private val courseService: CourseService, diff --git a/src/main/kotlin/ch/uzh/ifi/access/model/Task.kt b/src/main/kotlin/ch/uzh/ifi/access/model/Task.kt index 079833b4..dde45e19 100644 --- a/src/main/kotlin/ch/uzh/ifi/access/model/Task.kt +++ b/src/main/kotlin/ch/uzh/ifi/access/model/Task.kt @@ -71,6 +71,42 @@ class Task { val attemptRefill: Int? get() = if (Objects.nonNull(attemptWindow)) Math.toIntExact(attemptWindow!!.toSeconds()) else null + @Column(nullable = false) + var llmSubmission: String? = null + + @Column + var llmSolution: String? = null + + @Column + var llmRubrics: String? = null + + @Column(nullable = false) + var llmCot: Boolean = false + + @Column(nullable = false) + var llmVoting: Int = 1 + + @Column + var llmExamples: String? = null + + @Column + var llmPrompt: String? = null + + @Column + var llmPre: String? = null + + @Column + var llmPost: String? = null + + @Column + var llmTemperature: Double? = 0.2 + + @Column + var llmModel: String? = null + + @Column + var llmMaxPoints: Double? = null + fun createFile(): TaskFile { val newTaskFile = TaskFile() files.add(newTaskFile) diff --git a/src/main/kotlin/ch/uzh/ifi/access/model/dto/AssistantDTO.kt b/src/main/kotlin/ch/uzh/ifi/access/model/dto/AssistantDTO.kt new file mode 100644 index 00000000..ff712317 --- /dev/null +++ b/src/main/kotlin/ch/uzh/ifi/access/model/dto/AssistantDTO.kt @@ -0,0 +1,80 @@ +package ch.uzh.ifi.access.model.dto + +import com.fasterxml.jackson.annotation.JsonProperty +import lombok.Data + + +enum class LLMType { + gpt, + claude +} + +@Data +class RubricDTO ( + @JsonProperty("id") + var id: String, + + @JsonProperty("title") + var title: String, + + @JsonProperty("points") + var points: Double +) + +@Data +class FewShotExampleDTO ( + @JsonProperty("answer") + var answer: String, + + @JsonProperty("points") + var points: String // Stringified JSON for flexibility +) + +@Data +class AssistantDTO ( + @JsonProperty("question") + var question: String, + + @JsonProperty("answer") + var answer: String, + + @JsonProperty("rubrics") + var rubrics: List? = null, + + @JsonProperty("modelSolution") + var modelSolution: String? = null, + + @JsonProperty("maxPoints") + var maxPoints: Double? = 1.0, + + @JsonProperty("minPoints") + var minPoints: Double? = 0.0, + + @JsonProperty("pointStep") + var pointStep: Double? = 0.5, + + @JsonProperty("chainOfThought") + var chainOfThought: Boolean? = true, + + @JsonProperty("votingCount") + var votingCount: Int? = 1, + + @JsonProperty("temperature") + var temperature: Double? = 0.2, + + @JsonProperty("fewShotExamples") + var fewShotExamples: List? = null, + + @JsonProperty("prePrompt") + var prePrompt: String? = null, + + @JsonProperty("prompt") + var prompt: String? = null, + + @JsonProperty("postPrompt") + var postPrompt: String? = null, + + @JsonProperty("llmType") + var llmType: LLMType? = null + +) \ No newline at end of file diff --git a/src/main/kotlin/ch/uzh/ifi/access/model/dto/AssistantResponseDTO.kt b/src/main/kotlin/ch/uzh/ifi/access/model/dto/AssistantResponseDTO.kt new file mode 100644 index 00000000..c091d4da --- /dev/null +++ b/src/main/kotlin/ch/uzh/ifi/access/model/dto/AssistantResponseDTO.kt @@ -0,0 +1,45 @@ +package ch.uzh.ifi.access.model.dto + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties +import com.fasterxml.jackson.annotation.JsonProperty +import lombok.Data + +enum class Status { + correct, + incorrect, + incomplete +} + +@JsonIgnoreProperties(ignoreUnknown = true) +@Data +class AssistantResponseDTO ( + @JsonProperty("status") + var status: Status, + + @JsonProperty("feedback") + var feedback: String, + + @JsonProperty("hint") + var hint: String? = null, + + @JsonProperty("points") + var points: Double, + + @JsonProperty("votingResult") + var votingResult: String +) + +@JsonIgnoreProperties(ignoreUnknown = true) +@Data +class AssistantEvaluationResponseDTO ( + @JsonProperty("status") + var status: String, + + @JsonProperty("result") + var result: AssistantResponseDTO? +) + +class TaskIdDTO( + @JsonProperty("jobId") + var jobId: String +) \ No newline at end of file diff --git a/src/main/kotlin/ch/uzh/ifi/access/model/dto/LLMConfigDTO.kt b/src/main/kotlin/ch/uzh/ifi/access/model/dto/LLMConfigDTO.kt new file mode 100644 index 00000000..0adfe7e4 --- /dev/null +++ b/src/main/kotlin/ch/uzh/ifi/access/model/dto/LLMConfigDTO.kt @@ -0,0 +1,16 @@ +package ch.uzh.ifi.access.model.dto + +data class LLMConfigDTO( + var submission: String? = null, + var solution: String? = null, + var rubrics: String? = null, + var cot: Boolean = false, + var voting: Int = 1, + var examples: String? = null, + var prompt: String? = null, + var pre: String? = null, + var post: String? = null, + var temperature: Double? = null, + var model: String? = null, + var maxPoints: Double? = null +) \ No newline at end of file diff --git a/src/main/kotlin/ch/uzh/ifi/access/model/dto/TaskDTO.kt b/src/main/kotlin/ch/uzh/ifi/access/model/dto/TaskDTO.kt index 7afb6430..8e2adb6e 100644 --- a/src/main/kotlin/ch/uzh/ifi/access/model/dto/TaskDTO.kt +++ b/src/main/kotlin/ch/uzh/ifi/access/model/dto/TaskDTO.kt @@ -11,5 +11,6 @@ class TaskDTO( var maxAttempts: Int? = null, var refill: Int? = null, var evaluator: TaskEvaluatorDTO? = null, - var files: TaskFilesDTO? = null + var files: TaskFilesDTO? = null, + var llm: LLMConfigDTO? = null, ) diff --git a/src/main/kotlin/ch/uzh/ifi/access/service/CourseConfigImporter.kt b/src/main/kotlin/ch/uzh/ifi/access/service/CourseConfigImporter.kt index 96ffa878..fc810e94 100644 --- a/src/main/kotlin/ch/uzh/ifi/access/service/CourseConfigImporter.kt +++ b/src/main/kotlin/ch/uzh/ifi/access/service/CourseConfigImporter.kt @@ -2,6 +2,7 @@ package ch.uzh.ifi.access.service import ch.uzh.ifi.access.model.dto.* import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.databind.node.NullNode import com.fasterxml.jackson.dataformat.toml.TomlMapper import org.springframework.stereotype.Service @@ -12,7 +13,8 @@ import java.time.LocalDateTime @Service class CourseConfigImporter( private val tomlMapper: TomlMapper, - private val fileService: FileService + private val fileService: FileService, + private val objectMapper: ObjectMapper // Needed for JSON serialization ) { fun JsonNode?.asTextOrNull(): String? { @@ -62,7 +64,6 @@ class CourseConfigImporter( course.globalFiles = files return course - } fun readAssignmentConfig(path: Path): AssignmentDTO { @@ -86,7 +87,37 @@ class CourseConfigImporter( } return assignment + } + + private fun readRubricsFromToml(path: Path, rubricsFile: String): String? { + val rubricsPath = path.resolve(rubricsFile) + if (!Files.exists(rubricsPath)) return null + + val rubricsConfig: JsonNode = tomlMapper.readTree(Files.readString(rubricsPath)) + val rubricsList = rubricsConfig["rubrics"]?.map { rubric -> + RubricDTO( + id = rubric["id"].asText(), + title = rubric["title"].asText(), + points = rubric["points"].asDouble() + ) + } ?: emptyList() + return objectMapper.writeValueAsString(rubricsList) // Convert to JSON string + } + + private fun readExamplesFromToml(path: Path, examplesFile: String): String? { + val examplesPath = path.resolve(examplesFile) + if (!Files.exists(examplesPath)) return null + + val examplesConfig: JsonNode = tomlMapper.readTree(Files.readString(examplesPath)) + val examplesList = examplesConfig["examples"]?.map { example -> + FewShotExampleDTO( + answer = example["answer"].asText(), + points = example["points"].asText() + ) + } ?: emptyList() + + return objectMapper.writeValueAsString(examplesList) // Convert to JSON string } fun readTaskConfig(path: Path): TaskDTO { @@ -130,15 +161,44 @@ class CourseConfigImporter( "solution" -> files.solution = filenames "persist" -> files.persist = filenames } + + val llmConfig = config["llm"] + if (llmConfig != null && !llmConfig.isNull) { + val submissionContent = llmConfig["submission"]?.asTextOrNull()?.let { Files.readString(path.resolve(it)) } + ?: throw InvalidCourseException() + + val solutionContent = llmConfig["solution"]?.asTextOrNull()?.let { Files.readString(path.resolve(it)) } + val rubricsJson = llmConfig["rubrics"]?.asTextOrNull()?.let { readRubricsFromToml(path, it) } + val examplesJson = llmConfig["examples"]?.asTextOrNull()?.let { readExamplesFromToml(path, it) } + val promptContent = llmConfig["prompt"]?.asTextOrNull()?.let { Files.readString(path.resolve(it)) } + val preContent = llmConfig["pre"]?.asTextOrNull()?.let { Files.readString(path.resolve(it)) } + val postContent = llmConfig["post"]?.asTextOrNull()?.let { Files.readString(path.resolve(it)) } + val temperature = llmConfig["temperature"]?.asDouble() + val model = llmConfig["model"]?.asTextOrNull() + + task.llm = LLMConfigDTO( + submission = submissionContent, + solution = solutionContent, + rubrics = rubricsJson, + cot = llmConfig["cot"]?.asBoolean() ?: false, + voting = llmConfig["voting"]?.asInt() ?: 1, + examples = examplesJson, + prompt = promptContent, + pre = preContent, + post = postContent, + temperature = temperature, + model = model, + maxPoints = llmConfig["max_points"].asDouble() + ) + } } task.files = files return task - } - } class InvalidCourseException : Throwable() { + } \ No newline at end of file diff --git a/src/main/kotlin/ch/uzh/ifi/access/service/CourseService.kt b/src/main/kotlin/ch/uzh/ifi/access/service/CourseService.kt index b5d4e4a0..74f778bf 100644 --- a/src/main/kotlin/ch/uzh/ifi/access/service/CourseService.kt +++ b/src/main/kotlin/ch/uzh/ifi/access/service/CourseService.kt @@ -7,6 +7,7 @@ import ch.uzh.ifi.access.model.dao.Results import ch.uzh.ifi.access.model.dto.* import ch.uzh.ifi.access.projections.* import ch.uzh.ifi.access.repository.* +import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.databind.json.JsonMapper import com.github.dockerjava.api.DockerClient import com.github.dockerjava.api.command.PullImageResultCallback @@ -17,10 +18,17 @@ import com.github.dockerjava.api.model.HostConfig import io.github.oshai.kotlinlogging.KotlinLogging import jakarta.transaction.Transactional import jakarta.xml.bind.DatatypeConverter +import okhttp3.Request +import okhttp3.OkHttpClient +import okhttp3.MediaType.Companion.toMediaTypeOrNull +import okhttp3.RequestBody +import okhttp3.RequestBody.Companion.toRequestBody +import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import org.apache.commons.collections4.ListUtils import org.apache.commons.io.FileUtils import org.apache.tika.Tika import org.keycloak.representations.idm.UserRepresentation +import org.keycloak.util.JsonSerialization.mapper import org.modelmapper.ModelMapper import org.springframework.cache.annotation.CacheEvict import org.springframework.cache.annotation.Cacheable @@ -99,36 +107,40 @@ class CourseService( private val evaluationRepository: EvaluationRepository, private val dockerClient: DockerClient, private val modelMapper: ModelMapper, + private val objectMapper: ObjectMapper, private val jsonMapper: JsonMapper, private val courseLifecycle: CourseLifecycle, private val roleService: RoleService, private val fileService: FileService, - private val tika: Tika + private val tika: Tika, + private val assistantServerUrl: String ) { private val logger = KotlinLogging.logger {} + private val client = OkHttpClient() + private fun verifyUserId(@Nullable userId: String?): String { return userId ?: SecurityContextHolder.getContext().authentication.name } fun getCourseBySlug(courseSlug: String): Course { return courseRepository.getBySlug(courseSlug) ?: - throw ResponseStatusException(HttpStatus.NOT_FOUND, "No course found with the URL $courseSlug") + throw ResponseStatusException(HttpStatus.NOT_FOUND, "No course found with the URL $courseSlug") } fun getCourseWorkspaceBySlug(courseSlug: String): CourseWorkspace { return courseRepository.findBySlug(courseSlug) ?: - throw ResponseStatusException(HttpStatus.NOT_FOUND, "No course found with the URL $courseSlug") + throw ResponseStatusException(HttpStatus.NOT_FOUND, "No course found with the URL $courseSlug") } fun getTaskById(taskId: Long): Task { return taskRepository.findById(taskId).get() ?: - throw ResponseStatusException(HttpStatus.NOT_FOUND, "No task found with the ID $taskId") + throw ResponseStatusException(HttpStatus.NOT_FOUND, "No task found with the ID $taskId") } fun getTaskFileById(fileId: Long): TaskFile { return taskFileRepository.findById(fileId).get() ?: - throw ResponseStatusException(HttpStatus.NOT_FOUND, "No task file found with the ID $fileId") + throw ResponseStatusException(HttpStatus.NOT_FOUND, "No task file found with the ID $fileId") } fun getCoursesOverview(): List { //return courseRepository.findCoursesBy() @@ -142,7 +154,7 @@ class CourseService( fun getCourseSummary(courseSlug: String): CourseSummary { return courseRepository.findCourseBySlug(courseSlug) ?: - throw ResponseStatusException(HttpStatus.NOT_FOUND, "No course found with the URL $courseSlug") + throw ResponseStatusException(HttpStatus.NOT_FOUND, "No course found with the URL $courseSlug") } fun enabledTasksOnly(tasks: List): List { @@ -156,23 +168,23 @@ class CourseService( fun getAssignment(courseSlug: String?, assignmentSlug: String): AssignmentWorkspace { return assignmentRepository.findByCourse_SlugAndSlug(courseSlug, assignmentSlug) ?: - throw ResponseStatusException( HttpStatus.NOT_FOUND, - "No assignment found with the URL $assignmentSlug" ) + throw ResponseStatusException( HttpStatus.NOT_FOUND, + "No assignment found with the URL $assignmentSlug" ) } fun getAssignmentBySlug(courseSlug: String?, assignmentSlug: String): Assignment { return assignmentRepository.getByCourse_SlugAndSlug(courseSlug, assignmentSlug) ?: throw ResponseStatusException( - HttpStatus.NOT_FOUND, - "No assignment found with the URL $assignmentSlug" - ) + HttpStatus.NOT_FOUND, + "No assignment found with the URL $assignmentSlug" + ) } fun getTask(courseSlug: String?, assignmentSlug: String?, taskSlug: String?, userId: String?): TaskWorkspace { val workspace = taskRepository.findByAssignment_Course_SlugAndAssignment_SlugAndSlug(courseSlug, assignmentSlug, taskSlug) ?: throw ResponseStatusException( HttpStatus.NOT_FOUND, - "No task found with the URL: $courseSlug/$assignmentSlug/$taskSlug" ) + "No task found with the URL: $courseSlug/$assignmentSlug/$taskSlug" ) workspace.setUserId(userId) return workspace } @@ -293,8 +305,8 @@ class CourseService( assignmentSlug, taskSlug ) ?: throw ResponseStatusException( - HttpStatus.NOT_FOUND, "No task found with the URL $taskSlug" - ) + HttpStatus.NOT_FOUND, "No task found with the URL $taskSlug" + ) } fun getGradingFiles(taskId: Long?): List { @@ -397,6 +409,77 @@ class CourseService( } } + // Evaluates a submission using the assistant API + fun evaluateSubmissionWithAssistant(submission: AssistantDTO): AssistantResponseDTO? { + val url = "$assistantServerUrl/evaluate" + + val requestBodyJson = mapper.writeValueAsString(submission) + logger.debug { "Request body: $requestBodyJson" } + + val requestBody = requestBodyJson.toRequestBody("application/json".toMediaTypeOrNull()) + val request = Request.Builder() + .url(url) + .post(requestBody) + .build() + + client.newCall(request).execute().use { response -> + if (!response.isSuccessful) { + throw RuntimeException("Failed to get response from assistant backend: ${response.message}") + } + + val responseBody = response.body?.string() ?: throw RuntimeException("Empty response body") + val taskId = mapper.readValue(responseBody, TaskIdDTO::class.java).jobId + + if (taskId.isNullOrBlank()) { + throw RuntimeException("Invalid taskId received from assistant backend") + } + + // Polling loop + var attempts = 0 + val maxAttempts = 20 // Adjust as needed + val delayMillis = 2000L // 2 seconds delay per attempt + + while (attempts < maxAttempts) { + val statusUrl = "$assistantServerUrl/evaluate/$taskId" + val statusRequest = Request.Builder().url(statusUrl).get().build() + client.newCall(statusRequest).execute().use { statusResponse -> + if (!statusResponse.isSuccessful) { + throw RuntimeException("Failed to fetch evaluation status: ${statusResponse.message}") + } + + val statusBody = statusResponse.body?.string() + val statusResponseDTO = mapper.readValue(statusBody, AssistantEvaluationResponseDTO::class.java) + + when (statusResponseDTO.status) { + "completed" -> return statusResponseDTO.result // Return completed result + "not_found" -> throw RuntimeException("Task not found in assistant backend") + "active" -> { + // Keep polling + logger.debug { "Task $taskId still active, retrying..." } + } + else -> throw RuntimeException("Unexpected status: ${statusResponseDTO.status}") + } + } + Thread.sleep(delayMillis) // Wait before next attempt + attempts++ + } + throw RuntimeException("Evaluation timed out for taskId: $taskId") + } + } + + fun parseJsonOrEmpty(json: String?, objectMapper: ObjectMapper, clazz: Class>): List { + return try { + if (!json.isNullOrBlank()) { + objectMapper.readValue(json, clazz).toList() + } else { + emptyList() + } + } catch (e: Exception) { + emptyList() // Fallback to empty list if parsing fails + } + } + + @Caching(evict = [ CacheEvict(value = ["getStudent"], key = "#courseSlug + '-' + #submissionDTO.userId"), CacheEvict(value = ["getStudentWithPoints"], key = "#courseSlug + '-' + #submissionDTO.userId"), @@ -425,6 +508,7 @@ class CourseService( ) } val submission = submissionRepository.saveAndFlush(newSubmission) + //pruneSubmissions(evaluation) submissionDTO.files.stream().filter { fileDTO -> fileDTO.content != null } .forEach { fileDTO: SubmissionFileDTO -> createSubmissionFile(submission, fileDTO) } @@ -471,7 +555,7 @@ class CourseService( // TODO: make this size configurable in task config.toml? val resultFileSizeLimit = convertSizeToBytes("100K") val persistentFileCopyCommands = task.persistentResultFilePaths.joinToString("\n") { path -> -""" + """ # Check if results file exceeds permissible size limit if [[ -f "$path" ]]; then actual_size=${'$'}(stat -c%s "$path") @@ -485,7 +569,7 @@ cp "$path" "/submission/${'$'}file_dir" """ } val command = ( -""" + """ # copy submitted files to tmpfs /bin/cp -R /submission/* /workspace/; # run command (the cwd is set to /workspace already) @@ -504,7 +588,7 @@ fi $persistentFileCopyCommands exit ${'$'}exit_code; """ - ) + ) val container = containerCmd .withLabels(mapOf("userId" to submission.userId)).withWorkingDir("/workspace") .withCmd("/bin/bash", "-c", command) @@ -607,9 +691,63 @@ exit ${'$'}exit_code; FileUtils.deleteQuietly(submissionDir.toFile()) } } + + // Evaluate with Assistant API + try { + // Collect the student's submitted code + val studentCode = submission.files.joinToString("\n\n") { file -> + val filename = file.taskFile?.name ?: "Unnamed File" + "// File: $filename\n${file.content ?: ""}" + } + + // TODO: This could be handled nicer in the future + val llmType = when (task.llmModel?.lowercase()) { + "claude" -> LLMType.claude + else -> LLMType.gpt // default to gpt if not specified + } + + val assistantResponse = evaluateSubmissionWithAssistant( + AssistantDTO( + question = task.llmPrompt ?: task.instructions ?: "No instructions provided", + answer = studentCode, + llmType = llmType, + chainOfThought = task.llmCot, + votingCount = task.llmVoting, + rubrics = parseJsonOrEmpty(task.llmRubrics, objectMapper, Array::class.java), + prePrompt = task.llmPre, + postPrompt = task.llmPost, + prompt = task.llmPrompt, + temperature = task.llmTemperature, + fewShotExamples = parseJsonOrEmpty(task.llmExamples, objectMapper, Array::class.java), + maxPoints = task.llmMaxPoints, + modelSolution = task.llmSolution, + ) + ) + + + + if (assistantResponse != null) { + logger.debug { "Assistant internal feedback: ${assistantResponse.feedback}" } + logger.debug { "Assistant scoring: ${assistantResponse.points}" } + } + + // Incorporate the assistant feedback into the submission + assistantResponse?.let { + if (it.hint != null && it.hint != "" && it.status !== Status.correct) { + newSubmission.logs += "\nHint: ${it.hint}" + } + val newPoints = newSubmission.points?.plus(it.points) + newSubmission.points = minOf(newPoints!!, newSubmission.maxPoints!!) + } + } catch (e: Exception) { + // print error + logger.error { "Failed to evaluate submission with assistant: ${e.message}" } + } + } catch (e: Exception) { newSubmission.output = "Uncaught ${e::class.simpleName}: ${e.message}. Please report this as a bug and provide as much detail as possible." } + submissionRepository.save(newSubmission) } diff --git a/src/main/resources/db/migration/V2_1__add_llm_fields.sql b/src/main/resources/db/migration/V2_1__add_llm_fields.sql new file mode 100644 index 00000000..595916c5 --- /dev/null +++ b/src/main/resources/db/migration/V2_1__add_llm_fields.sql @@ -0,0 +1,12 @@ +ALTER TABLE task +ADD COLUMN IF NOT EXISTS llm_submission TEXT, +ADD COLUMN IF NOT EXISTS llm_solution TEXT, +ADD COLUMN IF NOT EXISTS llm_rubrics TEXT, +ADD COLUMN IF NOT EXISTS llm_cot BOOLEAN NOT NULL DEFAULT FALSE, +ADD COLUMN IF NOT EXISTS llm_voting INTEGER NOT NULL DEFAULT 1, +ADD COLUMN IF NOT EXISTS llm_examples TEXT, +ADD COLUMN IF NOT EXISTS llm_prompt TEXT, +ADD COLUMN IF NOT EXISTS llm_pre TEXT, +ADD COLUMN IF NOT EXISTS llm_post TEXT, +ADD COLUMN IF NOT EXISTS llm_point_step DOUBLE PRECISION NOT NULL DEFAULT 0.5, +ADD COLUMN IF NOT EXISTS llm_temperature DOUBLE PRECISION NOT NULL DEFAULT 0.2; \ No newline at end of file diff --git a/src/main/resources/db/migration/V2_2__add_llm_model_field.sql b/src/main/resources/db/migration/V2_2__add_llm_model_field.sql new file mode 100644 index 00000000..28cc0388 --- /dev/null +++ b/src/main/resources/db/migration/V2_2__add_llm_model_field.sql @@ -0,0 +1,2 @@ +ALTER TABLE task +ADD COLUMN IF NOT EXISTS llm_model TEXT; \ No newline at end of file diff --git a/src/main/resources/db/migration/V2_3__add_llm_max_points.sql b/src/main/resources/db/migration/V2_3__add_llm_max_points.sql new file mode 100644 index 00000000..40078dec --- /dev/null +++ b/src/main/resources/db/migration/V2_3__add_llm_max_points.sql @@ -0,0 +1,2 @@ +ALTER TABLE task +ADD COLUMN IF NOT EXISTS llm_max_points INTEGER; \ No newline at end of file From 4dc25c427ae252a4356aac01f1191aa7682d149f Mon Sep 17 00:00:00 2001 From: rnichi1 <71671446+rnichi1@users.noreply.github.com> Date: Thu, 6 Feb 2025 09:48:13 +0100 Subject: [PATCH 2/7] Add missin Bean to SecurityConfig --- src/main/kotlin/ch/uzh/ifi/access/config/SecurityConfig.kt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/kotlin/ch/uzh/ifi/access/config/SecurityConfig.kt b/src/main/kotlin/ch/uzh/ifi/access/config/SecurityConfig.kt index 061f1faa..b94e15fe 100644 --- a/src/main/kotlin/ch/uzh/ifi/access/config/SecurityConfig.kt +++ b/src/main/kotlin/ch/uzh/ifi/access/config/SecurityConfig.kt @@ -122,6 +122,11 @@ class SecurityConfig(private val env: Environment) { return Path.of(env.getProperty("WORKING_DIR", "/workspace/data")) } + @Bean + fun assistantServerUrl(): String { + return env.getProperty("ASSISTANT_SERVER_URL", "http://localhost:4000") + } + @Bean fun accessRealm(): RealmResource { val keycloakClient = Keycloak.getInstance( From 7a9cd39c16f109bc8ff03074da6595070efc3c67 Mon Sep 17 00:00:00 2001 From: rnichi1 <71671446+rnichi1@users.noreply.github.com> Date: Thu, 13 Feb 2025 19:46:52 +0100 Subject: [PATCH 3/7] Add llm family as replacement for model llmModel is now used for more granular control over which model to use. --- src/main/java/ch/uzh/ifi/access/config/ModelMapperConfig.java | 1 + src/main/kotlin/ch/uzh/ifi/access/model/Task.kt | 3 +++ src/main/kotlin/ch/uzh/ifi/access/model/dto/AssistantDTO.kt | 4 +++- src/main/kotlin/ch/uzh/ifi/access/model/dto/LLMConfigDTO.kt | 1 + .../kotlin/ch/uzh/ifi/access/service/CourseConfigImporter.kt | 2 ++ src/main/kotlin/ch/uzh/ifi/access/service/CourseService.kt | 3 ++- src/main/resources/db/migration/V2_4__add_llm_family.sql | 2 ++ 7 files changed, 14 insertions(+), 2 deletions(-) create mode 100644 src/main/resources/db/migration/V2_4__add_llm_family.sql diff --git a/src/main/java/ch/uzh/ifi/access/config/ModelMapperConfig.java b/src/main/java/ch/uzh/ifi/access/config/ModelMapperConfig.java index 0beb780f..e91939fb 100644 --- a/src/main/java/ch/uzh/ifi/access/config/ModelMapperConfig.java +++ b/src/main/java/ch/uzh/ifi/access/config/ModelMapperConfig.java @@ -63,6 +63,7 @@ public ModelMapper modelMapper() { destination.setLlmPre(llm.getPre()); destination.setLlmPost(llm.getPost()); destination.setLlmModel(llm.getModel()); + destination.setLlmModelFamily(llm.getModelFamily()); destination.setLlmTemperature(llm.getTemperature()); destination.setLlmMaxPoints(llm.getMaxPoints()); } diff --git a/src/main/kotlin/ch/uzh/ifi/access/model/Task.kt b/src/main/kotlin/ch/uzh/ifi/access/model/Task.kt index dde45e19..3322216e 100644 --- a/src/main/kotlin/ch/uzh/ifi/access/model/Task.kt +++ b/src/main/kotlin/ch/uzh/ifi/access/model/Task.kt @@ -104,6 +104,9 @@ class Task { @Column var llmModel: String? = null + @Column + var llmModelFamily: String? = null + @Column var llmMaxPoints: Double? = null diff --git a/src/main/kotlin/ch/uzh/ifi/access/model/dto/AssistantDTO.kt b/src/main/kotlin/ch/uzh/ifi/access/model/dto/AssistantDTO.kt index ff712317..62f18832 100644 --- a/src/main/kotlin/ch/uzh/ifi/access/model/dto/AssistantDTO.kt +++ b/src/main/kotlin/ch/uzh/ifi/access/model/dto/AssistantDTO.kt @@ -75,6 +75,8 @@ class AssistantDTO ( var postPrompt: String? = null, @JsonProperty("llmType") - var llmType: LLMType? = null + var llmType: LLMType? = null, + @JsonProperty("llmModel") + var llmModel: String? = null ) \ No newline at end of file diff --git a/src/main/kotlin/ch/uzh/ifi/access/model/dto/LLMConfigDTO.kt b/src/main/kotlin/ch/uzh/ifi/access/model/dto/LLMConfigDTO.kt index 0adfe7e4..cbde82ce 100644 --- a/src/main/kotlin/ch/uzh/ifi/access/model/dto/LLMConfigDTO.kt +++ b/src/main/kotlin/ch/uzh/ifi/access/model/dto/LLMConfigDTO.kt @@ -12,5 +12,6 @@ data class LLMConfigDTO( var post: String? = null, var temperature: Double? = null, var model: String? = null, + var modelFamily: String? = null, var maxPoints: Double? = null ) \ No newline at end of file diff --git a/src/main/kotlin/ch/uzh/ifi/access/service/CourseConfigImporter.kt b/src/main/kotlin/ch/uzh/ifi/access/service/CourseConfigImporter.kt index fc810e94..0529b3d9 100644 --- a/src/main/kotlin/ch/uzh/ifi/access/service/CourseConfigImporter.kt +++ b/src/main/kotlin/ch/uzh/ifi/access/service/CourseConfigImporter.kt @@ -175,6 +175,7 @@ class CourseConfigImporter( val postContent = llmConfig["post"]?.asTextOrNull()?.let { Files.readString(path.resolve(it)) } val temperature = llmConfig["temperature"]?.asDouble() val model = llmConfig["model"]?.asTextOrNull() + val modelFamily = llmConfig["model_family"]?.asTextOrNull() task.llm = LLMConfigDTO( submission = submissionContent, @@ -188,6 +189,7 @@ class CourseConfigImporter( post = postContent, temperature = temperature, model = model, + modelFamily = modelFamily, maxPoints = llmConfig["max_points"].asDouble() ) } diff --git a/src/main/kotlin/ch/uzh/ifi/access/service/CourseService.kt b/src/main/kotlin/ch/uzh/ifi/access/service/CourseService.kt index ee10ce7c..330e4313 100644 --- a/src/main/kotlin/ch/uzh/ifi/access/service/CourseService.kt +++ b/src/main/kotlin/ch/uzh/ifi/access/service/CourseService.kt @@ -730,7 +730,7 @@ exit ${'$'}exit_code; } // TODO: This could be handled nicer in the future - val llmType = when (task.llmModel?.lowercase()) { + val llmType = when (task.llmModelFamily?.lowercase()) { "claude" -> LLMType.claude else -> LLMType.gpt // default to gpt if not specified } @@ -750,6 +750,7 @@ exit ${'$'}exit_code; fewShotExamples = parseJsonOrEmpty(task.llmExamples, objectMapper, Array::class.java), maxPoints = task.llmMaxPoints, modelSolution = task.llmSolution, + llmModel = task.llmModel, ) ) diff --git a/src/main/resources/db/migration/V2_4__add_llm_family.sql b/src/main/resources/db/migration/V2_4__add_llm_family.sql new file mode 100644 index 00000000..a30cfd82 --- /dev/null +++ b/src/main/resources/db/migration/V2_4__add_llm_family.sql @@ -0,0 +1,2 @@ +ALTER TABLE task +ADD COLUMN IF NOT EXISTS llm_model_family TEXT; \ No newline at end of file From 8093e6d5a5b14164e27591fe28e1314bee90ac5b Mon Sep 17 00:00:00 2001 From: rnichi1 <71671446+rnichi1@users.noreply.github.com> Date: Sat, 15 Feb 2025 11:50:42 +0100 Subject: [PATCH 4/7] change example format and take correct submission for llm --- .../uzh/ifi/access/model/dto/AssistantDTO.kt | 2 +- .../access/service/CourseConfigImporter.kt | 9 ++-- .../uzh/ifi/access/service/CourseService.kt | 54 +++++++++++-------- 3 files changed, 38 insertions(+), 27 deletions(-) diff --git a/src/main/kotlin/ch/uzh/ifi/access/model/dto/AssistantDTO.kt b/src/main/kotlin/ch/uzh/ifi/access/model/dto/AssistantDTO.kt index 62f18832..b4a30861 100644 --- a/src/main/kotlin/ch/uzh/ifi/access/model/dto/AssistantDTO.kt +++ b/src/main/kotlin/ch/uzh/ifi/access/model/dto/AssistantDTO.kt @@ -27,7 +27,7 @@ class FewShotExampleDTO ( var answer: String, @JsonProperty("points") - var points: String // Stringified JSON for flexibility + var points: Map ) @Data diff --git a/src/main/kotlin/ch/uzh/ifi/access/service/CourseConfigImporter.kt b/src/main/kotlin/ch/uzh/ifi/access/service/CourseConfigImporter.kt index 0529b3d9..23e4dcfb 100644 --- a/src/main/kotlin/ch/uzh/ifi/access/service/CourseConfigImporter.kt +++ b/src/main/kotlin/ch/uzh/ifi/access/service/CourseConfigImporter.kt @@ -1,6 +1,7 @@ package ch.uzh.ifi.access.service import ch.uzh.ifi.access.model.dto.* +import com.fasterxml.jackson.core.type.TypeReference import com.fasterxml.jackson.databind.JsonNode import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.databind.node.NullNode @@ -113,7 +114,7 @@ class CourseConfigImporter( val examplesList = examplesConfig["examples"]?.map { example -> FewShotExampleDTO( answer = example["answer"].asText(), - points = example["points"].asText() + points = objectMapper.convertValue(example["points"], object : TypeReference>() {}) ) } ?: emptyList() @@ -164,9 +165,7 @@ class CourseConfigImporter( val llmConfig = config["llm"] if (llmConfig != null && !llmConfig.isNull) { - val submissionContent = llmConfig["submission"]?.asTextOrNull()?.let { Files.readString(path.resolve(it)) } - ?: throw InvalidCourseException() - + val submissionFileName = llmConfig["submission"]?.asTextOrNull() val solutionContent = llmConfig["solution"]?.asTextOrNull()?.let { Files.readString(path.resolve(it)) } val rubricsJson = llmConfig["rubrics"]?.asTextOrNull()?.let { readRubricsFromToml(path, it) } val examplesJson = llmConfig["examples"]?.asTextOrNull()?.let { readExamplesFromToml(path, it) } @@ -178,7 +177,7 @@ class CourseConfigImporter( val modelFamily = llmConfig["model_family"]?.asTextOrNull() task.llm = LLMConfigDTO( - submission = submissionContent, + submission = submissionFileName, solution = solutionContent, rubrics = rubricsJson, cot = llmConfig["cot"]?.asBoolean() ?: false, diff --git a/src/main/kotlin/ch/uzh/ifi/access/service/CourseService.kt b/src/main/kotlin/ch/uzh/ifi/access/service/CourseService.kt index 330e4313..d6f189ef 100644 --- a/src/main/kotlin/ch/uzh/ifi/access/service/CourseService.kt +++ b/src/main/kotlin/ch/uzh/ifi/access/service/CourseService.kt @@ -482,6 +482,10 @@ class CourseService( when (statusResponseDTO.status) { "completed" -> return statusResponseDTO.result // Return completed result "not_found" -> throw RuntimeException("Task not found in assistant backend") + "delayed" -> { + // Keep polling + logger.debug { "Task $taskId delayed, retrying..." } + } "active" -> { // Keep polling logger.debug { "Task $taskId still active, retrying..." } @@ -724,10 +728,8 @@ exit ${'$'}exit_code; // Evaluate with Assistant API try { // Collect the student's submitted code - val studentCode = submission.files.joinToString("\n\n") { file -> - val filename = file.taskFile?.name ?: "Unnamed File" - "// File: $filename\n${file.content ?: ""}" - } + val llmSubmissionFile = submission.files + .firstOrNull { file -> file.taskFile?.path == task.llmSubmission } // Find the specific file // TODO: This could be handled nicer in the future val llmType = when (task.llmModelFamily?.lowercase()) { @@ -735,24 +737,34 @@ exit ${'$'}exit_code; else -> LLMType.gpt // default to gpt if not specified } - val assistantResponse = evaluateSubmissionWithAssistant( - AssistantDTO( - question = task.llmPrompt ?: task.instructions ?: "No instructions provided", - answer = studentCode, - llmType = llmType, - chainOfThought = task.llmCot, - votingCount = task.llmVoting, - rubrics = parseJsonOrEmpty(task.llmRubrics, objectMapper, Array::class.java), - prePrompt = task.llmPre, - postPrompt = task.llmPost, - prompt = task.llmPrompt, - temperature = task.llmTemperature, - fewShotExamples = parseJsonOrEmpty(task.llmExamples, objectMapper, Array::class.java), - maxPoints = task.llmMaxPoints, - modelSolution = task.llmSolution, - llmModel = task.llmModel, + + + var assistantResponse: AssistantResponseDTO? = null + + if (llmSubmissionFile?.content != null) { + assistantResponse = evaluateSubmissionWithAssistant( + AssistantDTO( + question = task.llmPrompt ?: task.instructions ?: "No instructions provided", + answer = llmSubmissionFile.content!!, + llmType = llmType, + chainOfThought = task.llmCot, + votingCount = task.llmVoting, + rubrics = parseJsonOrEmpty(task.llmRubrics, objectMapper, Array::class.java), + prePrompt = task.llmPre, + postPrompt = task.llmPost, + prompt = task.llmPrompt, + temperature = task.llmTemperature, + fewShotExamples = parseJsonOrEmpty( + task.llmExamples, + objectMapper, + Array::class.java + ), + maxPoints = task.llmMaxPoints, + modelSolution = task.llmSolution, + llmModel = task.llmModel, + ) ) - ) + } From 3d9cd715e327decd2e76b719784c978424f7164e Mon Sep 17 00:00:00 2001 From: rnichi1 <71671446+rnichi1@users.noreply.github.com> Date: Sat, 15 Feb 2025 12:40:56 +0100 Subject: [PATCH 5/7] Add a guard to not always run assistant scoring --- .../kotlin/ch/uzh/ifi/access/model/Task.kt | 2 +- .../uzh/ifi/access/model/dto/AssistantDTO.kt | 10 +- .../uzh/ifi/access/service/CourseService.kt | 100 +++++++++--------- 3 files changed, 51 insertions(+), 61 deletions(-) diff --git a/src/main/kotlin/ch/uzh/ifi/access/model/Task.kt b/src/main/kotlin/ch/uzh/ifi/access/model/Task.kt index 3322216e..81269c33 100644 --- a/src/main/kotlin/ch/uzh/ifi/access/model/Task.kt +++ b/src/main/kotlin/ch/uzh/ifi/access/model/Task.kt @@ -99,7 +99,7 @@ class Task { var llmPost: String? = null @Column - var llmTemperature: Double? = 0.2 + var llmTemperature: Double? = null @Column var llmModel: String? = null diff --git a/src/main/kotlin/ch/uzh/ifi/access/model/dto/AssistantDTO.kt b/src/main/kotlin/ch/uzh/ifi/access/model/dto/AssistantDTO.kt index b4a30861..1576ec28 100644 --- a/src/main/kotlin/ch/uzh/ifi/access/model/dto/AssistantDTO.kt +++ b/src/main/kotlin/ch/uzh/ifi/access/model/dto/AssistantDTO.kt @@ -3,12 +3,6 @@ package ch.uzh.ifi.access.model.dto import com.fasterxml.jackson.annotation.JsonProperty import lombok.Data - -enum class LLMType { - gpt, - claude -} - @Data class RubricDTO ( @JsonProperty("id") @@ -31,7 +25,7 @@ class FewShotExampleDTO ( ) @Data -class AssistantDTO ( +class AssistantDTO( @JsonProperty("question") var question: String, @@ -75,7 +69,7 @@ class AssistantDTO ( var postPrompt: String? = null, @JsonProperty("llmType") - var llmType: LLMType? = null, + var llmType: String? = null, @JsonProperty("llmModel") var llmModel: String? = null diff --git a/src/main/kotlin/ch/uzh/ifi/access/service/CourseService.kt b/src/main/kotlin/ch/uzh/ifi/access/service/CourseService.kt index d6f189ef..4dcb291f 100644 --- a/src/main/kotlin/ch/uzh/ifi/access/service/CourseService.kt +++ b/src/main/kotlin/ch/uzh/ifi/access/service/CourseService.kt @@ -725,65 +725,61 @@ exit ${'$'}exit_code; } } - // Evaluate with Assistant API - try { - // Collect the student's submitted code - val llmSubmissionFile = submission.files - .firstOrNull { file -> file.taskFile?.path == task.llmSubmission } // Find the specific file - - // TODO: This could be handled nicer in the future - val llmType = when (task.llmModelFamily?.lowercase()) { - "claude" -> LLMType.claude - else -> LLMType.gpt // default to gpt if not specified - } - - - - var assistantResponse: AssistantResponseDTO? = null - - if (llmSubmissionFile?.content != null) { - assistantResponse = evaluateSubmissionWithAssistant( - AssistantDTO( - question = task.llmPrompt ?: task.instructions ?: "No instructions provided", - answer = llmSubmissionFile.content!!, - llmType = llmType, - chainOfThought = task.llmCot, - votingCount = task.llmVoting, - rubrics = parseJsonOrEmpty(task.llmRubrics, objectMapper, Array::class.java), - prePrompt = task.llmPre, - postPrompt = task.llmPost, - prompt = task.llmPrompt, - temperature = task.llmTemperature, - fewShotExamples = parseJsonOrEmpty( - task.llmExamples, - objectMapper, - Array::class.java - ), - maxPoints = task.llmMaxPoints, - modelSolution = task.llmSolution, - llmModel = task.llmModel, + // Only evaluate with assistant if the submission file was defined in the config + if(task.llmSubmission != null) { + // Evaluate with Assistant API + try { + // Collect the student's submitted code + val llmSubmissionFile = submission.files + .firstOrNull { file -> file.taskFile?.path == task.llmSubmission } // Find the specific file + + var assistantResponse: AssistantResponseDTO? = null + + if (llmSubmissionFile?.content != null) { + assistantResponse = evaluateSubmissionWithAssistant( + AssistantDTO( + question = task.llmPrompt ?: task.instructions ?: "No instructions provided", + answer = llmSubmissionFile.content!!, + llmType = task.llmModelFamily, + chainOfThought = task.llmCot, + votingCount = task.llmVoting, + rubrics = parseJsonOrEmpty(task.llmRubrics, objectMapper, Array::class.java), + prePrompt = task.llmPre, + postPrompt = task.llmPost, + prompt = task.llmPrompt, + temperature = task.llmTemperature, + fewShotExamples = parseJsonOrEmpty( + task.llmExamples, + objectMapper, + Array::class.java + ), + maxPoints = task.llmMaxPoints, + modelSolution = task.llmSolution, + llmModel = task.llmModel, + ) ) - ) - } + } - if (assistantResponse != null) { - logger.debug { "Assistant internal feedback: ${assistantResponse.feedback}" } - logger.debug { "Assistant scoring: ${assistantResponse.points}" } - } + if (assistantResponse != null) { + logger.debug { "Assistant internal feedback: ${assistantResponse.feedback}" } + logger.debug { "Assistant scoring: ${assistantResponse.points}" } + } - // Incorporate the assistant feedback into the submission - assistantResponse?.let { - if (it.hint != null && it.hint != "" && it.status !== Status.correct) { - newSubmission.logs += "\nHint: ${it.hint}" + // Incorporate the assistant feedback into the submission + assistantResponse?.let { + if (it.hint != null && it.hint != "" && it.status !== Status.correct) { + newSubmission.logs += "\nHint: ${it.hint}" + } + val newPoints = newSubmission.points?.plus(it.points) + newSubmission.points = minOf(newPoints!!, newSubmission.maxPoints!!) + evaluation.update(newSubmission.points) } - val newPoints = newSubmission.points?.plus(it.points) - newSubmission.points = minOf(newPoints!!, newSubmission.maxPoints!!) + } catch (e: Exception) { + // print error + logger.error { "Failed to evaluate submission with assistant: ${e.message}" } } - } catch (e: Exception) { - // print error - logger.error { "Failed to evaluate submission with assistant: ${e.message}" } } } catch (e: Exception) { From 60d48b08afe8e207a419661316efac450e635482 Mon Sep 17 00:00:00 2001 From: rnichi1 <71671446+rnichi1@users.noreply.github.com> Date: Tue, 18 Feb 2025 21:01:53 +0100 Subject: [PATCH 6/7] fix: import of courses --- .../ifi/access/config/ModelMapperConfig.java | 38 +------------------ .../ch/uzh/ifi/access/model/dto/TaskDTO.kt | 14 ++++++- .../access/service/CourseConfigImporter.kt | 29 +++++++------- .../db/migration/V2_1__add_llm_fields.sql | 24 ++++++------ .../migration/V2_2__add_llm_model_field.sql | 2 - .../db/migration/V2_3__add_llm_max_points.sql | 2 - .../db/migration/V2_4__add_llm_family.sql | 2 - 7 files changed, 41 insertions(+), 70 deletions(-) delete mode 100644 src/main/resources/db/migration/V2_2__add_llm_model_field.sql delete mode 100644 src/main/resources/db/migration/V2_3__add_llm_max_points.sql delete mode 100644 src/main/resources/db/migration/V2_4__add_llm_family.sql diff --git a/src/main/java/ch/uzh/ifi/access/config/ModelMapperConfig.java b/src/main/java/ch/uzh/ifi/access/config/ModelMapperConfig.java index e91939fb..499d6407 100644 --- a/src/main/java/ch/uzh/ifi/access/config/ModelMapperConfig.java +++ b/src/main/java/ch/uzh/ifi/access/config/ModelMapperConfig.java @@ -29,47 +29,11 @@ public ModelMapper modelMapper() { modelMapper.typeMap(AssignmentDTO.class, Assignment.class) .addMappings(mapping -> mapping.skip(AssignmentDTO::getTasks, Assignment::setTasks)); modelMapper.typeMap(TaskDTO.class, Task.class) - .addMappings(mapper -> { - mapper.skip(Task::setLlmSubmission); - mapper.skip(Task::setLlmSolution); - mapper.skip(Task::setLlmRubrics); - mapper.skip(Task::setLlmCot); - mapper.skip(Task::setLlmVoting); - mapper.skip(Task::setLlmExamples); - mapper.skip(Task::setLlmPrompt); - mapper.skip(Task::setLlmPre); - mapper.skip(Task::setLlmPost); - mapper.skip(Task::setLlmModel); - mapper.skip(Task::setLlmTemperature); - mapper.skip(Task::setLlmMaxPoints); - }); + .addMappings(mapping -> mapping.skip(TaskDTO::getFiles, Task::setFiles)); modelMapper.typeMap(SubmissionDTO.class, Submission.class) .addMappings(mapping -> mapping.skip(Submission::setFiles)); modelMapper.createTypeMap(String.class, Visibility.class) .setConverter(context -> Visibility.valueOf(context.getSource().toUpperCase())); - modelMapper.addConverter((context) -> { - TaskDTO source = (TaskDTO) context.getSource(); - Task destination = (Task) context.getDestination(); - - if (source.getLlm() != null) { - LLMConfigDTO llm = source.getLlm(); - destination.setLlmSubmission(llm.getSubmission()); - destination.setLlmSolution(llm.getSolution()); - destination.setLlmRubrics(llm.getRubrics()); - destination.setLlmCot(llm.getCot()); - destination.setLlmVoting(llm.getVoting()); - destination.setLlmExamples(llm.getExamples()); - destination.setLlmPrompt(llm.getPrompt()); - destination.setLlmPre(llm.getPre()); - destination.setLlmPost(llm.getPost()); - destination.setLlmModel(llm.getModel()); - destination.setLlmModelFamily(llm.getModelFamily()); - destination.setLlmTemperature(llm.getTemperature()); - destination.setLlmMaxPoints(llm.getMaxPoints()); - } - - return destination; - }, TaskDTO.class, Task.class); return modelMapper; } } diff --git a/src/main/kotlin/ch/uzh/ifi/access/model/dto/TaskDTO.kt b/src/main/kotlin/ch/uzh/ifi/access/model/dto/TaskDTO.kt index 8e2adb6e..94e7cd73 100644 --- a/src/main/kotlin/ch/uzh/ifi/access/model/dto/TaskDTO.kt +++ b/src/main/kotlin/ch/uzh/ifi/access/model/dto/TaskDTO.kt @@ -12,5 +12,17 @@ class TaskDTO( var refill: Int? = null, var evaluator: TaskEvaluatorDTO? = null, var files: TaskFilesDTO? = null, - var llm: LLMConfigDTO? = null, + var llmSubmission: String? = null, + var llmSolution: String? = null, + var llmRubrics: String? = null, + var llmCot: Boolean = false, + var llmVoting: Int = 1, + var llmExamples: String? = null, + var llmPrompt: String? = null, + var llmPre: String? = null, + var llmPost: String? = null, + var llmTemperature: Double? = null, + var llmModel: String? = null, + var llmModelFamily: String? = null, + var llmMaxPoints: Double? = null, ) diff --git a/src/main/kotlin/ch/uzh/ifi/access/service/CourseConfigImporter.kt b/src/main/kotlin/ch/uzh/ifi/access/service/CourseConfigImporter.kt index 23e4dcfb..cd227359 100644 --- a/src/main/kotlin/ch/uzh/ifi/access/service/CourseConfigImporter.kt +++ b/src/main/kotlin/ch/uzh/ifi/access/service/CourseConfigImporter.kt @@ -176,22 +176,21 @@ class CourseConfigImporter( val model = llmConfig["model"]?.asTextOrNull() val modelFamily = llmConfig["model_family"]?.asTextOrNull() - task.llm = LLMConfigDTO( - submission = submissionFileName, - solution = solutionContent, - rubrics = rubricsJson, - cot = llmConfig["cot"]?.asBoolean() ?: false, - voting = llmConfig["voting"]?.asInt() ?: 1, - examples = examplesJson, - prompt = promptContent, - pre = preContent, - post = postContent, - temperature = temperature, - model = model, - modelFamily = modelFamily, - maxPoints = llmConfig["max_points"].asDouble() - ) + task.llmSubmission = submissionFileName + task.llmSolution = solutionContent + task.llmRubrics = rubricsJson + task.llmCot = llmConfig["cot"]?.asBoolean() ?: false + task.llmVoting = llmConfig["voting"]?.asInt() ?: 1 + task.llmExamples = examplesJson + task.llmPrompt = promptContent + task.llmPre = preContent + task.llmPost = postContent + task.llmTemperature = temperature + task.llmModel = model + task.llmModelFamily = modelFamily + task.llmMaxPoints = llmConfig["max_points"].asDouble() } + } task.files = files diff --git a/src/main/resources/db/migration/V2_1__add_llm_fields.sql b/src/main/resources/db/migration/V2_1__add_llm_fields.sql index 595916c5..405fb841 100644 --- a/src/main/resources/db/migration/V2_1__add_llm_fields.sql +++ b/src/main/resources/db/migration/V2_1__add_llm_fields.sql @@ -1,12 +1,14 @@ ALTER TABLE task -ADD COLUMN IF NOT EXISTS llm_submission TEXT, -ADD COLUMN IF NOT EXISTS llm_solution TEXT, -ADD COLUMN IF NOT EXISTS llm_rubrics TEXT, -ADD COLUMN IF NOT EXISTS llm_cot BOOLEAN NOT NULL DEFAULT FALSE, -ADD COLUMN IF NOT EXISTS llm_voting INTEGER NOT NULL DEFAULT 1, -ADD COLUMN IF NOT EXISTS llm_examples TEXT, -ADD COLUMN IF NOT EXISTS llm_prompt TEXT, -ADD COLUMN IF NOT EXISTS llm_pre TEXT, -ADD COLUMN IF NOT EXISTS llm_post TEXT, -ADD COLUMN IF NOT EXISTS llm_point_step DOUBLE PRECISION NOT NULL DEFAULT 0.5, -ADD COLUMN IF NOT EXISTS llm_temperature DOUBLE PRECISION NOT NULL DEFAULT 0.2; \ No newline at end of file + ADD COLUMN IF NOT EXISTS llm_submission TEXT, + ADD COLUMN IF NOT EXISTS llm_solution TEXT, + ADD COLUMN IF NOT EXISTS llm_rubrics TEXT, + ADD COLUMN IF NOT EXISTS llm_cot BOOLEAN, + ADD COLUMN IF NOT EXISTS llm_voting INTEGER, + ADD COLUMN IF NOT EXISTS llm_examples TEXT, + ADD COLUMN IF NOT EXISTS llm_prompt TEXT, + ADD COLUMN IF NOT EXISTS llm_pre TEXT, + ADD COLUMN IF NOT EXISTS llm_post TEXT, + ADD COLUMN IF NOT EXISTS llm_temperature DOUBLE PRECISION, + ADD COLUMN IF NOT EXISTS llm_model TEXT, + ADD COLUMN IF NOT EXISTS llm_model_family TEXT, + ADD COLUMN IF NOT EXISTS llm_max_points DOUBLE PRECISION; diff --git a/src/main/resources/db/migration/V2_2__add_llm_model_field.sql b/src/main/resources/db/migration/V2_2__add_llm_model_field.sql deleted file mode 100644 index 28cc0388..00000000 --- a/src/main/resources/db/migration/V2_2__add_llm_model_field.sql +++ /dev/null @@ -1,2 +0,0 @@ -ALTER TABLE task -ADD COLUMN IF NOT EXISTS llm_model TEXT; \ No newline at end of file diff --git a/src/main/resources/db/migration/V2_3__add_llm_max_points.sql b/src/main/resources/db/migration/V2_3__add_llm_max_points.sql deleted file mode 100644 index 40078dec..00000000 --- a/src/main/resources/db/migration/V2_3__add_llm_max_points.sql +++ /dev/null @@ -1,2 +0,0 @@ -ALTER TABLE task -ADD COLUMN IF NOT EXISTS llm_max_points INTEGER; \ No newline at end of file diff --git a/src/main/resources/db/migration/V2_4__add_llm_family.sql b/src/main/resources/db/migration/V2_4__add_llm_family.sql deleted file mode 100644 index a30cfd82..00000000 --- a/src/main/resources/db/migration/V2_4__add_llm_family.sql +++ /dev/null @@ -1,2 +0,0 @@ -ALTER TABLE task -ADD COLUMN IF NOT EXISTS llm_model_family TEXT; \ No newline at end of file From 91582846728f2e8117509461cd16d4c5a7b6d0fb Mon Sep 17 00:00:00 2001 From: rnichi1 <71671446+rnichi1@users.noreply.github.com> Date: Tue, 18 Feb 2025 21:20:09 +0100 Subject: [PATCH 7/7] fix: send example points as string --- .../ch/uzh/ifi/access/model/dto/AssistantDTO.kt | 2 +- .../uzh/ifi/access/service/CourseConfigImporter.kt | 2 +- .../ch/uzh/ifi/access/service/CourseService.kt | 12 +++++++++--- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/main/kotlin/ch/uzh/ifi/access/model/dto/AssistantDTO.kt b/src/main/kotlin/ch/uzh/ifi/access/model/dto/AssistantDTO.kt index 1576ec28..d0f2a5a7 100644 --- a/src/main/kotlin/ch/uzh/ifi/access/model/dto/AssistantDTO.kt +++ b/src/main/kotlin/ch/uzh/ifi/access/model/dto/AssistantDTO.kt @@ -21,7 +21,7 @@ class FewShotExampleDTO ( var answer: String, @JsonProperty("points") - var points: Map + var points: String ) @Data diff --git a/src/main/kotlin/ch/uzh/ifi/access/service/CourseConfigImporter.kt b/src/main/kotlin/ch/uzh/ifi/access/service/CourseConfigImporter.kt index cd227359..4d750753 100644 --- a/src/main/kotlin/ch/uzh/ifi/access/service/CourseConfigImporter.kt +++ b/src/main/kotlin/ch/uzh/ifi/access/service/CourseConfigImporter.kt @@ -114,7 +114,7 @@ class CourseConfigImporter( val examplesList = examplesConfig["examples"]?.map { example -> FewShotExampleDTO( answer = example["answer"].asText(), - points = objectMapper.convertValue(example["points"], object : TypeReference>() {}) + points = objectMapper.convertValue(example["points"], object : TypeReference>() {}).toString() ) } ?: emptyList() diff --git a/src/main/kotlin/ch/uzh/ifi/access/service/CourseService.kt b/src/main/kotlin/ch/uzh/ifi/access/service/CourseService.kt index 4dcb291f..47e513c4 100644 --- a/src/main/kotlin/ch/uzh/ifi/access/service/CourseService.kt +++ b/src/main/kotlin/ch/uzh/ifi/access/service/CourseService.kt @@ -53,6 +53,7 @@ import java.util.function.Consumer import java.util.stream.Stream import javax.crypto.Mac import javax.crypto.spec.SecretKeySpec +import kotlin.math.log @Service class CourseServiceForCaching( @@ -725,21 +726,26 @@ exit ${'$'}exit_code; } } + // Only evaluate with assistant if the submission file was defined in the config if(task.llmSubmission != null) { // Evaluate with Assistant API try { // Collect the student's submitted code val llmSubmissionFile = submission.files - .firstOrNull { file -> file.taskFile?.path == task.llmSubmission } // Find the specific file + .firstOrNull { file -> file.taskFile?.path == "/${task.llmSubmission}" } // Find the specific file + + submission.files.map { + logger.debug { "Submission file: ${it.taskFile?.path}" } + } var assistantResponse: AssistantResponseDTO? = null if (llmSubmissionFile?.content != null) { assistantResponse = evaluateSubmissionWithAssistant( AssistantDTO( - question = task.llmPrompt ?: task.instructions ?: "No instructions provided", - answer = llmSubmissionFile.content!!, + question = task.instructions ?: "No instructions provided", + answer = llmSubmissionFile.content ?: "No answer provided", llmType = task.llmModelFamily, chainOfThought = task.llmCot, votingCount = task.llmVoting,