Add llm specification for automated scoring

01_intro/hello_world/config.toml

@@ -30,3 +30,5 @@ solution = [
   "solution/script.py",
 ]
 
+
+

03_classes/carpark_multiple_inheritance/config.toml

@@ -45,4 +45,3 @@ solution = [
   "solution/electric_car.py",
   "solution/hybrid_car.py",
 ]
-

04_llm/config.toml

@@ -0,0 +1,25 @@
+# Mandatory. This slug will be used for the URL of the assignment.
+# ACCESS will refuse to import or update the course if the course/assignment slug is already taken by another assignment.
+slug = "llm"
+
+# Mandatory.
+# Assignments will be invisible to regular users until the start date.
+# Users will be able to submit solutions between the start and end date.
+# After the end date, users will be able to see the assignment and run code, but they may not submit solutions.
+# They will also see files marked as "solution" in individual tasks.
+start = 2023-01-01T13:00:00
+end = 2028-01-01T13:00:00
+
+# Mandatory. List of directory paths containing the tasks.
+# ACCESS will show tasks in the order in which they are listed here.
+"tasks" = [
+  "string_manipulation",
+]
+
+# Information for at least one language must be specified.
+[information.en]
+title = "String Manipulation in Python"
+
+[information.de]
+title = "Stringmanipulation in Python"
+

04_llm/string_manipulation/config.toml

@@ -0,0 +1,48 @@
+slug = "string_manipulation"
+
+
+max_attempts = 6
+refill = 43200 # 12 hours
+max_points = 2
+
+[information.en]
+title = "String Manipulation in Python"
+instructions_file = "instructions_en.md"
+
+[evaluator]
+docker_image = "python:latest"
+run_command = "python -m task.script"
+test_command = "python -m unittest discover -v task"
+grade_command = "python -m grading.tests"
+
+[llm]
+submission = "task/explanation.md"
+rubrics = 'rubrics/rubrics.toml' # This file contains the rubrics for the task to guide the model
+examples = 'grading/examples.toml' # This file contains examples of the task for the model to learn from
+solution = 'solution/explanation.md' # This is the solution file used to guide the model
+cot = true # This will add "think step by step" to the context prompt in order to encourage the model to think step by step
+voting = 3 # This allows for the results to be evaluated 3 times, and the most common result is chosen. This setting will increase the time it takes to grade the task!
+post = "grading/post.md" # Adds further instruction at the end of the context prompt
+# pre = "grading/pre.md" # Adds further instruction in front of the context prompt
+# prompt = "grading/prompt.md" # replaces the context prompt with the content of the file. This is only for advanced usages!
+temperature = 0.2 # Decides the randomness of the gpt model
+model_family = "claude" # gpt or claude
+model = "claude-3-5-sonnet-latest" # gpt-4o, gpt-4o-mini, claude-3-5-sonnet-latest, etc.
+max_points = 1 # Max points for the sub-task that is passed to the model
+
+[files]
+visible = [
+  "task/script.py",
+  "task/explanation.md",
+]
+editable = [
+  "task/script.py",
+  "task/explanation.md",
+]
+grading = [
+  "grading/tests.py",
+]
+solution = [
+  "solution/script.py",
+  "solution/explanation.md",
+]

04_llm/string_manipulation/grading/examples.toml

@@ -0,0 +1,7 @@
+[[examples]]
+answer = "Reversing word order and reversing characters both have O(n) complexity, but character reversal requires more operations per word, making it slightly less efficient in practice."
+points = { time_complexity_mentioned = 0.5, asymptotically_equivalent = 0.5 }
+
+[[examples]]
+answer = "Both operations have O(n) complexity because they process each character in the string."
+points = { time_complexity_mentioned = 0.5, asymptotically_equivalent = 0 }

04_llm/string_manipulation/grading/post.md

@@ -0,0 +1 @@
+The student answer will not contain any code. This is expected, since you only need to grade the explanation according to the rubrics!

04_llm/string_manipulation/grading/tests.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+
+# Scaffolding necessary to set up ACCESS test
+import sys
+try: from universal.harness import *
+except: sys.path.append("../../universal/"); from harness import *
+
+# Grading test suite starts here
+
+script = grading_import("task", "script")
+
+class GradingTests(AccessTestCase):
+
+    def _test(self, sentence, expected):
+        actual = script.reverse_words(sentence)
+        self.hint(f"Reversal not correct for sentence='{sentence}'... expected result is '{expected}'!")
+        self.assertEqual(expected, actual)
+
+    def test_case1(self):
+        self._test("Hello World", "World Hello")
+
+    def test_case2(self):
+        self._test("  This   is  a  test  ", "test a is This")
+
+    def test_case3(self):
+        self._test("Python", "Python")
+
+    def test_case4(self):
+        self._test("", "")
+
+    def test_case5(self):
+        self._test("Hello, World!", "World! Hello,")
+
+    def test_case6(self):
+        self._test("123 456 789", "789 456 123")
+
+TestRunner().run(AccessTestSuite(1, [GradingTests]))

04_llm/string_manipulation/instructions_en.md

@@ -0,0 +1,11 @@
+# String Manipulation in Python
+
+## Task Description
+
+Your task is to implement a Python function called `reverse_words`. This function should take a single string input and return a string with the words in reverse order. For example:
+
+- Input: `"Hello World"`
+- Output: `"World Hello"`
+
+Additionally, reflect on the differences in complexity between reversing the order of words in a sentence and reversing the characters within each word in the `explanation.md` file provided. 
+Which operation do you think is more computationally efficient, and why? Consider factors such as string manipulation methods and time complexity in your explanation. Provide examples to support your reasoning.

04_llm/string_manipulation/rubrics/rubrics.toml

@@ -0,0 +1,9 @@
+[[rubrics]]
+id = "time_complexity_mentioned"
+title = "Mentioned the time complexity of both operations is O(n)"
+points = 0.5
+
+[[rubrics]]
+id = "asymptotically_equivalent"
+title = "Explained that both are asymptotically equivalent but in practice, reversing characters is slower"
+points = 0.5

04_llm/string_manipulation/solution/explanation.md

@@ -0,0 +1,16 @@
+Reversing the order of words in a sentence and reversing the characters within each word have different complexities.
+
+ Time Complexity of Both Operations:
+     Reversing the word order involves splitting the string into a list of words O(n), reversing the list(O(n), and joining it back into a string O(n). This results in an overall complexity of O(n).
+     Reversing characters within each word requires iterating through each word and reversing it (O(m) per word, where m is the word length). Since this must be done for all words, the total complexity remains O(n).
+
+ Which is More Efficient and Why:
+     Both operations have an O(n) complexity, but reversing words is generally more efficient in practice because it operates at a higher level (list reversal), whereas reversing characters requires more fine-grained string manipulation.
+     If implemented using in-place reversal, reversing characters within each word can introduce additional overhead compared to simple list manipulation.
+
+Example:
+
+    "Hello World" → "World Hello" (word order reversal)
+    "Hello World" → "olleH dlroW" (character reversal)
+
+While both methods scale similarly, reversing characters involves additional operations per word, making it slightly more complex in practical execution.

04_llm/string_manipulation/solution/script.py

@@ -0,0 +1,3 @@
+def reverse_words(sentence):
+    # Split the sentence into words, reverse the list, and join it back into a string
+    return ' '.join(sentence.split()[::-1])

04_llm/string_manipulation/task/explanation.md

@@ -0,0 +1 @@
+[comment]: <> (Add your solution here:)

04_llm/string_manipulation/task/script.py

@@ -0,0 +1,7 @@
+# Task: Implement a function `reverse_words` that takes a string
+# and returns the string with the order of words reversed.
+# Example: "Hello World" -> "World Hello"
+
+def reverse_words(sentence):
+    # TODO: Implement this function
+    pass

config.toml

@@ -12,7 +12,8 @@ logo = "python.svg"
 "assignments" = [
   "01_intro",
   "02_basics",
-  "03_classes"
+  "03_classes",
+  "04_llm",
 ]
 
 # Mandatory. Determines who can see the course in ACCESS at any given time.