Add LLM Assistant Functionality for Scoring

build.gradle.kts

@@ -66,6 +66,8 @@ dependencies {
     implementation("org.apache.httpcomponents.client5:httpclient5:5.2.1")
     implementation("org.apache.tika:tika-core:2.8.0")
     implementation("org.flywaydb:flyway-core")
+    implementation("com.squareup.okhttp3:okhttp:4.9.2")
+    implementation("com.fasterxml.jackson.module:jackson-module-kotlin:2.12.+")
     implementation("io.github.oshai:kotlin-logging-jvm:5.1.0")
     testImplementation("org.springframework.security:spring-security-test")
     testImplementation("org.springframework.boot:spring-boot-starter-test") {

src/main/java/ch/uzh/ifi/access/config/ModelMapperConfig.java

@@ -5,16 +5,15 @@
 import ch.uzh.ifi.access.model.Submission;
 import ch.uzh.ifi.access.model.Task;
 import ch.uzh.ifi.access.model.constants.Visibility;
-import ch.uzh.ifi.access.model.dto.AssignmentDTO;
-import ch.uzh.ifi.access.model.dto.CourseDTO;
-import ch.uzh.ifi.access.model.dto.SubmissionDTO;
-import ch.uzh.ifi.access.model.dto.TaskDTO;
+import ch.uzh.ifi.access.model.dto.*;
 import org.apache.commons.lang3.ObjectUtils;
 import org.modelmapper.ModelMapper;
 import org.modelmapper.convention.MatchingStrategies;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 
+import java.util.Objects;
+
 @Configuration
 public class ModelMapperConfig {
     // TODO: convert to Kotlin. This is very tricky, because ModelMapper is very Java-specific. Maybe replace it.

src/main/kotlin/ch/uzh/ifi/access/config/SecurityConfig.kt

@@ -122,6 +122,11 @@ class SecurityConfig(private val env: Environment) {
         return Path.of(env.getProperty("WORKING_DIR", "/workspace/data"))
     }
 
+    @Bean
+    fun assistantServerUrl(): String {
+        return env.getProperty("ASSISTANT_SERVER_URL", "http://localhost:4000")
+    }
+
     @Bean
     fun accessRealm(): RealmResource {
         val keycloakClient = Keycloak.getInstance(

src/main/kotlin/ch/uzh/ifi/access/controller/CourseController.kt

@@ -17,6 +17,7 @@ import java.time.LocalDateTime
 import java.util.concurrent.Semaphore
 
 
+
 @RestController
 class CourseRootController(
     private val courseService: CourseService,

src/main/kotlin/ch/uzh/ifi/access/model/Task.kt

@@ -71,6 +71,45 @@ class Task {
     val attemptRefill: Int?
         get() = if (Objects.nonNull(attemptWindow)) Math.toIntExact(attemptWindow!!.toSeconds()) else null
 
+    @Column(nullable = false)
+    var llmSubmission: String? = null
+
+    @Column
+    var llmSolution: String? = null
+
+    @Column
+    var llmRubrics: String? = null
+
+    @Column(nullable = false)
+    var llmCot: Boolean = false
+
+    @Column(nullable = false)
+    var llmVoting: Int = 1
+
+    @Column
+    var llmExamples: String? = null
+
+    @Column
+    var llmPrompt: String? = null
+
+    @Column
+    var llmPre: String? = null
+
+    @Column
+    var llmPost: String? = null
+
+    @Column
+    var llmTemperature: Double? = null
+
+    @Column
+    var llmModel: String? = null
+
+    @Column
+    var llmModelFamily: String? = null
+
+    @Column
+    var llmMaxPoints: Double? = null
+
     fun createFile(): TaskFile {
         val newTaskFile = TaskFile()
         files.add(newTaskFile)

src/main/kotlin/ch/uzh/ifi/access/model/dto/AssistantDTO.kt

@@ -0,0 +1,76 @@
+package ch.uzh.ifi.access.model.dto
+
+import com.fasterxml.jackson.annotation.JsonProperty
+import lombok.Data
+
+@Data
+class RubricDTO (
+    @JsonProperty("id")
+    var id: String,
+
+    @JsonProperty("title")
+    var title: String,
+
+    @JsonProperty("points")
+    var points: Double
+)
+
+@Data
+class FewShotExampleDTO (
+    @JsonProperty("answer")
+    var answer: String,
+
+    @JsonProperty("points")
+    var points: String
+)
+
+@Data
+class AssistantDTO(
+    @JsonProperty("question")
+    var question: String,
+
+    @JsonProperty("answer")
+    var answer: String,
+
+    @JsonProperty("rubrics")
+    var rubrics: List<RubricDTO>? = null,
+
+    @JsonProperty("modelSolution")
+    var modelSolution: String? = null,
+
+    @JsonProperty("maxPoints")
+    var maxPoints: Double? = 1.0,
+
+    @JsonProperty("minPoints")
+    var minPoints: Double? = 0.0,
+
+    @JsonProperty("pointStep")
+    var pointStep: Double? = 0.5,
+
+    @JsonProperty("chainOfThought")
+    var chainOfThought: Boolean? = true,
+
+    @JsonProperty("votingCount")
+    var votingCount: Int? = 1,
+
+    @JsonProperty("temperature")
+    var temperature: Double? = 0.2,
+
+    @JsonProperty("fewShotExamples")
+    var fewShotExamples: List<FewShotExampleDTO>? = null,
+
+    @JsonProperty("prePrompt")
+    var prePrompt: String? = null,
+
+    @JsonProperty("prompt")
+    var prompt: String? = null,
+
+    @JsonProperty("postPrompt")
+    var postPrompt: String? = null,
+
+    @JsonProperty("llmType")
+    var llmType: String? = null,
+
+    @JsonProperty("llmModel")
+    var llmModel: String? = null
+)

src/main/kotlin/ch/uzh/ifi/access/model/dto/AssistantResponseDTO.kt

@@ -0,0 +1,45 @@
+package ch.uzh.ifi.access.model.dto
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties
+import com.fasterxml.jackson.annotation.JsonProperty
+import lombok.Data
+
+enum class Status {
+    correct,
+    incorrect,
+    incomplete
+}
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+@Data
+class AssistantResponseDTO (
+    @JsonProperty("status")
+    var status: Status,
+
+    @JsonProperty("feedback")
+    var feedback: String,
+
+    @JsonProperty("hint")
+    var hint: String? = null,
+
+    @JsonProperty("points")
+    var points: Double,
+
+    @JsonProperty("votingResult")
+    var votingResult: String
+)
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+@Data
+class AssistantEvaluationResponseDTO (
+    @JsonProperty("status")
+    var status: String,
+
+    @JsonProperty("result")
+    var result: AssistantResponseDTO?
+)
+
+class TaskIdDTO(
+    @JsonProperty("jobId")
+    var jobId: String
+)

src/main/kotlin/ch/uzh/ifi/access/model/dto/LLMConfigDTO.kt

@@ -0,0 +1,17 @@
+package ch.uzh.ifi.access.model.dto
+
+data class LLMConfigDTO(
+    var submission: String? = null,
+    var solution: String? = null,
+    var rubrics: String? = null,
+    var cot: Boolean = false,
+    var voting: Int = 1,
+    var examples: String? = null,
+    var prompt: String? = null,
+    var pre: String? = null,
+    var post: String? = null,
+    var temperature: Double? = null,
+    var model: String? = null,
+    var modelFamily: String? = null,
+    var maxPoints: Double? = null
+) 

src/main/kotlin/ch/uzh/ifi/access/model/dto/TaskDTO.kt

@@ -11,5 +11,18 @@ class TaskDTO(
     var maxAttempts: Int? = null,
     var refill: Int? = null,
     var evaluator: TaskEvaluatorDTO? = null,
-    var files: TaskFilesDTO? = null
+    var files: TaskFilesDTO? = null,
+    var llmSubmission: String? = null,
+    var llmSolution: String? = null,
+    var llmRubrics: String? = null,
+    var llmCot: Boolean = false,
+    var llmVoting: Int = 1,
+    var llmExamples: String? = null,
+    var llmPrompt: String? = null,
+    var llmPre: String? = null,
+    var llmPost: String? = null,
+    var llmTemperature: Double? = null,
+    var llmModel: String? = null,
+    var llmModelFamily: String? = null,
+    var llmMaxPoints: Double? = null,
 )

src/main/kotlin/ch/uzh/ifi/access/service/CourseConfigImporter.kt

@@ -1,7 +1,9 @@
 package ch.uzh.ifi.access.service
 
 import ch.uzh.ifi.access.model.dto.*
+import com.fasterxml.jackson.core.type.TypeReference
 import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.ObjectMapper
 import com.fasterxml.jackson.databind.node.NullNode
 import com.fasterxml.jackson.dataformat.toml.TomlMapper
 import org.springframework.stereotype.Service
@@ -12,7 +14,8 @@ import java.time.LocalDateTime
 @Service
 class CourseConfigImporter(
     private val tomlMapper: TomlMapper,
-    private val fileService: FileService
+    private val fileService: FileService,
+    private val objectMapper: ObjectMapper // Needed for JSON serialization
 ) {
 
     fun JsonNode?.asTextOrNull(): String? {
@@ -62,7 +65,6 @@ class CourseConfigImporter(
         course.globalFiles = files
 
         return course
-
     }
 
     fun readAssignmentConfig(path: Path): AssignmentDTO {
@@ -86,7 +88,37 @@ class CourseConfigImporter(
         }
 
         return assignment
+    }
+
+    private fun readRubricsFromToml(path: Path, rubricsFile: String): String? {
+        val rubricsPath = path.resolve(rubricsFile)
+        if (!Files.exists(rubricsPath)) return null
+
+        val rubricsConfig: JsonNode = tomlMapper.readTree(Files.readString(rubricsPath))
+        val rubricsList = rubricsConfig["rubrics"]?.map { rubric ->
+            RubricDTO(
+                id = rubric["id"].asText(),
+                title = rubric["title"].asText(),
+                points = rubric["points"].asDouble()
+            )
+        } ?: emptyList()
+
+        return objectMapper.writeValueAsString(rubricsList) // Convert to JSON string
+    }
+
+    private fun readExamplesFromToml(path: Path, examplesFile: String): String? {
+        val examplesPath = path.resolve(examplesFile)
+        if (!Files.exists(examplesPath)) return null
 
+        val examplesConfig: JsonNode = tomlMapper.readTree(Files.readString(examplesPath))
+        val examplesList = examplesConfig["examples"]?.map { example ->
+            FewShotExampleDTO(
+                answer = example["answer"].asText(),
+                points = objectMapper.convertValue(example["points"], object : TypeReference<Map<String, Double>>() {}).toString()
+            )
+        } ?: emptyList()
+
+        return objectMapper.writeValueAsString(examplesList) // Convert to JSON string
     }
 
     fun readTaskConfig(path: Path): TaskDTO {
@@ -130,15 +162,43 @@ class CourseConfigImporter(
                 "solution" -> files.solution = filenames
                 "persist" -> files.persist = filenames
             }
+
+        val llmConfig = config["llm"]
+        if (llmConfig != null && !llmConfig.isNull) {
+                val submissionFileName = llmConfig["submission"]?.asTextOrNull()
+                val solutionContent = llmConfig["solution"]?.asTextOrNull()?.let { Files.readString(path.resolve(it)) }
+                val rubricsJson = llmConfig["rubrics"]?.asTextOrNull()?.let { readRubricsFromToml(path, it) }
+                val examplesJson = llmConfig["examples"]?.asTextOrNull()?.let { readExamplesFromToml(path, it) }
+                val promptContent = llmConfig["prompt"]?.asTextOrNull()?.let { Files.readString(path.resolve(it)) }
+                val preContent = llmConfig["pre"]?.asTextOrNull()?.let { Files.readString(path.resolve(it)) }
+                val postContent = llmConfig["post"]?.asTextOrNull()?.let { Files.readString(path.resolve(it)) }
+                val temperature = llmConfig["temperature"]?.asDouble()
+                val model = llmConfig["model"]?.asTextOrNull()
+                val modelFamily = llmConfig["model_family"]?.asTextOrNull()
+
+                task.llmSubmission = submissionFileName
+                task.llmSolution = solutionContent
+                task.llmRubrics = rubricsJson
+                task.llmCot = llmConfig["cot"]?.asBoolean() ?: false
+                task.llmVoting = llmConfig["voting"]?.asInt() ?: 1
+                task.llmExamples = examplesJson
+                task.llmPrompt = promptContent
+                task.llmPre = preContent
+                task.llmPost = postContent
+                task.llmTemperature = temperature
+                task.llmModel = model
+                task.llmModelFamily = modelFamily
+                task.llmMaxPoints = llmConfig["max_points"].asDouble()
+            }
+
         }
         task.files = files
 
         return task
-
     }
-
 }
 
 class InvalidCourseException : Throwable() {
 
+
 }