Skip to content

feat: only edit doc 2 & add excerpt to the moderator #81

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
simonmeoni wants to merge 1 commit into
base: ht+sm/conflict-maybe-lrec
Choose a base branch
from
Open

feat: only edit doc 2 & add excerpt to the moderator #81

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 9 additions & 3 deletions lib/conflicts/conflicts/agents/doctor_agent.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,8 @@ def __init__(self, client, model, cfg):
def __call__(
self,
document_pair: DocumentPair,
propositions1: PropositionResult = None,
propositions2: PropositionResult = None,
propositions1: PropositionResult,
propositions2: PropositionResult,
) -> ConflictResult:
"""
Analyze documents and determine the best conflict type to introduce
Expand Down Expand Up @@ -100,7 +100,12 @@ def __call__(
parsed_response = self._parse_json_response(response)

# Validate required fields
required_fields = ["conflict_type", "reasoning", "modification_instructions"]
required_fields = [
"conflict_type",
"reasoning",
"modification_instructions",
"highlighted_text_doc1",
]
for field in required_fields:
if field not in parsed_response:
raise ValueError(f"Missing required field '{field}' in Doctor Agent response")
Expand All @@ -117,6 +122,7 @@ def __call__(
conflict_type=parsed_response["conflict_type"],
reasoning=parsed_response["reasoning"],
modification_instructions=parsed_response["modification_instructions"],
highlighted_text_doc1=parsed_response["highlighted_text_doc1"],
editor_instructions=parsed_response.get("editor_instructions", []),
proposition_conflicts=parsed_response.get("proposition_conflicts", []),
)
Expand Down
30 changes: 13 additions & 17 deletions lib/conflicts/conflicts/agents/editor_agent.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,22 +45,20 @@ def __call__(
if attempt == max_retries - 1:
self.logger.error(f"All {max_retries} attempts failed: {e}")
return EditorResult(
modified_document1=document_pair.doc1_text,
modified_document2=document_pair.doc2_text,
changes_made=f"Failed to create conflict after {max_retries} attempts: {e}",
change_info_1="No changes made - all attempts failed",
change_info_2="No changes made - all attempts failed",
)
self.logger.warning(f"Attempt {attempt + 1} failed: {e}, retrying...")

def _perform_modification(
self, document_pair: DocumentPair, conflict_instructions: ConflictResult
self, document_pair: DocumentPair, conflict_result: ConflictResult
) -> EditorResult:
"""Perform a single modification attempt"""
prompt = self._build_prompt(document_pair, conflict_instructions)
prompt = self._build_prompt(document_pair, conflict_result)
response = self._execute_prompt(prompt, self.cfg.model.editor_temperature)
parsed_result = self._parse_and_validate_response(response, document_pair)
return self._create_result(parsed_result, document_pair)
return self._create_result(parsed_result, document_pair, conflict_result)

def _build_prompt(
self, document_pair: DocumentPair, conflict_instructions: ConflictResult
Expand Down Expand Up @@ -126,24 +124,23 @@ def _parse_and_validate_response(self, response: str, document_pair: DocumentPai
self.logger.error(f"Response was: {response}")
raise

if (
parsed_result["modified_doc_1"].strip() == document_pair.doc1_text.strip()
and parsed_result["modified_doc_2"].strip() == document_pair.doc2_text.strip()
):
if parsed_result["modified_doc_2"].strip() == document_pair.doc2_text.strip():
raise ValueError("No modifications were applied to the documents")

return parsed_result

def _create_result(self, parsed_result: dict, document_pair: DocumentPair) -> EditorResult:
def _create_result(
self,
parsed_result: dict,
document_pair: DocumentPair,
conflict_instructions: ConflictResult,
) -> EditorResult:
"""Create and log the final result"""
result = EditorResult(
modified_document1=parsed_result["modified_doc_1"],
modified_document2=parsed_result["modified_doc_2"],
changes_made=f"Applied {parsed_result['conflict_type']} conflict modifications",
change_info_1=parsed_result.get("change_info_1"),
change_info_2=parsed_result.get("change_info_2"),
original_excerpt_1=parsed_result.get("original_excerpt_1"),
modified_excerpt_1=parsed_result.get("modified_excerpt_1"),
original_excerpt_1=conflict_instructions.highlighted_text_doc1,
original_excerpt_2=parsed_result.get("original_excerpt_2"),
modified_excerpt_2=parsed_result.get("modified_excerpt_2"),
)
Expand All @@ -152,10 +149,9 @@ def _create_result(self, parsed_result: dict, document_pair: DocumentPair) -> Ed
self.logger.info(f"Conflict type: {parsed_result['conflict_type']}")

# Log document length changes
orig_len1, orig_len2 = len(document_pair.doc1_text), len(document_pair.doc2_text)
mod_len1, mod_len2 = len(result.modified_document1), len(result.modified_document2)
orig_len2 = len(document_pair.doc2_text)
mod_len2 = len(result.modified_document2)

self.logger.debug(f"Document 1 length: {orig_len1} -> {mod_len1} ({mod_len1-orig_len1:+d})")
self.logger.debug(f"Document 2 length: {orig_len2} -> {mod_len2} ({mod_len2-orig_len2:+d})")

return result
15 changes: 6 additions & 9 deletions lib/conflicts/conflicts/agents/moderator_agent.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ def __init__(self, client, model, cfg, min_validation_score: int = 4):
self.min_score = min_validation_score

def __call__(
self, original_pair: DocumentPair, modified_docs: EditorResult, conflict_type: str
self, original_pair: DocumentPair, editor_result: EditorResult, conflict_type: str
) -> ValidationResult:
"""
Validate the modifications made to clinical documents
Expand All @@ -40,10 +40,11 @@ def __call__(

try:
prompt = self.system_prompt.format(
context_document_1=self._truncate_document(modified_docs.modified_document1),
context_document_2=self._truncate_document(modified_docs.modified_document2),
conflict_1=modified_docs.change_info_1 or "No change info available",
conflict_2=modified_docs.change_info_2 or "No change info available",
context_document_1=self._truncate_document(original_pair.doc1_text),
context_document_2=self._truncate_document(editor_result.modified_document2),
excerpt_2=editor_result.modified_excerpt_2,
excerpt_1=editor_result.original_excerpt_1,
conflict_2=editor_result.change_info_2 or "No change info available",
)
print(prompt)

Expand Down Expand Up @@ -109,15 +110,11 @@ def detailed_validation_check(
"""
checks = {
"documents_modified": {
"doc1_changed": original_pair.doc1_text != modified_docs.modified_document1,
"doc2_changed": original_pair.doc2_text != modified_docs.modified_document2,
"any_changed": False,
},
"length_analysis": {
"doc1_original_length": len(original_pair.doc1_text),
"doc1_modified_length": len(modified_docs.modified_document1),
"doc1_length_change": len(modified_docs.modified_document1)
- len(original_pair.doc1_text),
"doc2_original_length": len(original_pair.doc2_text),
"doc2_modified_length": len(modified_docs.modified_document2),
"doc2_length_change": len(modified_docs.modified_document2)
Expand Down
26 changes: 13 additions & 13 deletions lib/conflicts/conflicts/core/base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,7 @@ def save_to_parquet(self, output_path: str = None):
def save_validated_documents(
self,
original_pair: DocumentPair,
modified_docs: EditorResult,
editor_result: EditorResult,
conflict_type: str,
validation_result: ValidationResult,
) -> int:
Expand All @@ -178,8 +178,8 @@ def save_validated_documents(

# Create document data
doc_data = DocumentData(
doc_1=modified_docs.modified_document1,
doc_2=modified_docs.modified_document2,
doc_1=original_pair.doc1_text,
doc_2=editor_result.modified_document2,
orig_doc_1=original_pair.doc1_text,
orig_doc_2=original_pair.doc2_text,
created_at=datetime.now().isoformat(),
Expand All @@ -192,12 +192,12 @@ def save_validated_documents(

# Add annotation for excerpt 1 if exists
if (
modified_docs.modified_excerpt_1
and not pd.isna(modified_docs.modified_excerpt_1)
and modified_docs.modified_excerpt_1.strip()
editor_result.original_excerpt_1
and not pd.isna(editor_result.original_excerpt_1)
and editor_result.original_excerpt_1.strip()
):
start_pos, end_pos = self.find_text_positions(
modified_docs.modified_document1, modified_docs.modified_excerpt_1
original_pair.doc1_text, editor_result.original_excerpt_1
)
if start_pos is not None:
annotation = Annotation(
Expand All @@ -210,20 +210,20 @@ def save_validated_documents(
value=AnnotationValue(
start=start_pos,
end=end_pos,
text=modified_docs.modified_excerpt_1,
text=editor_result.original_excerpt_1,
labels=["Conflict"],
),
)
annotations.append(annotation)

# Add annotation for excerpt 2 if exists
if (
modified_docs.modified_excerpt_2
and not pd.isna(modified_docs.modified_excerpt_2)
and modified_docs.modified_excerpt_2.strip()
editor_result.modified_excerpt_2
and not pd.isna(editor_result.modified_excerpt_2)
and editor_result.modified_excerpt_2.strip()
):
start_pos, end_pos = self.find_text_positions(
modified_docs.modified_document2, modified_docs.modified_excerpt_2
editor_result.modified_document2, editor_result.modified_excerpt_2
)
if start_pos is not None:
annotation = Annotation(
Expand All @@ -236,7 +236,7 @@ def save_validated_documents(
value=AnnotationValue(
start=start_pos,
end=end_pos,
text=modified_docs.modified_excerpt_2,
text=editor_result.modified_excerpt_2,
labels=["Conflict"],
),
)
Expand Down
20 changes: 3 additions & 17 deletions lib/conflicts/conflicts/core/document_operations.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -140,22 +140,12 @@ def parse_response(
data = _extract_json_from_response(response)

# Check if we have the expected edit operations format
required_fields = ["doc1", "doc2", "conflict_type"]
if all(field in data for field in required_fields):
# Support both old format (doc1 + doc2) and new format (doc2 only)
if "doc2" in data and "conflict_type" in data:
logging.info("Found edit operations format, applying operations to documents")

# Apply edit operations to documents
try:
(
modified_doc_1,
change_description_1,
orig_excerpt_1,
mod_excerpt_1,
) = apply_edit_operation(
original_doc_1,
data["doc1"],
min_text_length,
)
(
modified_doc_2,
change_description_2,
Expand All @@ -178,18 +168,14 @@ def parse_response(
conflict_type = data["conflict_type"]

return {
"modified_doc_1": modified_doc_1,
"modified_doc_2": modified_doc_2,
"conflict_type": conflict_type,
"change_info_1": change_description_1,
"change_info_2": change_description_2,
"original_excerpt_1": orig_excerpt_1,
"modified_excerpt_1": mod_excerpt_1,
"original_excerpt_2": orig_excerpt_2,
"modified_excerpt_2": mod_excerpt_2,
}
else:
raise ValueError(f"Response missing required fields: {required_fields}")
raise ValueError("Response missing required fields: doc2 and conflict_type")

except json.JSONDecodeError as e:
logging.error(f"Failed to parse response as JSON: {e}")
Expand Down
4 changes: 1 addition & 3 deletions lib/conflicts/conflicts/core/models.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ class ConflictResult:
conflict_type: str
reasoning: str
modification_instructions: str
highlighted_text_doc1: str
editor_instructions: Optional[list[str]] = None
proposition_conflicts: Optional[list[dict]] = None

Expand All @@ -33,13 +34,10 @@ class ConflictResult:
class EditorResult:
"""Result from the Editor Agent"""

modified_document1: str
modified_document2: str
changes_made: str
change_info_1: Optional[str] = None
change_info_2: Optional[str] = None
original_excerpt_1: Optional[str] = None
modified_excerpt_1: Optional[str] = None
original_excerpt_2: Optional[str] = None
modified_excerpt_2: Optional[str] = None

Expand Down
10 changes: 6 additions & 4 deletions lib/conflicts/prompts/doctor_agent_system.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,17 +28,19 @@ Analyze these documents and their propositions to choose the most appropriate co
4. What type of conflict would be realistic and educational given the temporal context
5. Which conflict type best matches the available clinical information and temporal recommendations

IMPORTANT: Only modify Document 2 to introduce conflicts. Document 1 should remain unchanged but you must identify and highlight specific sentences or paragraphs in Document 1 that need to be countered when editing Document 2.

Create specific counterfactual modifications that will create conflicts between the documents, then provide clear step-by-step instructions for the Editor Agent.

Respond with JSON format:
{{
"conflict_type": "chosen_conflict_type_key",
"reasoning": "explanation of why this conflict type was chosen, including temporal considerations",
"modification_instructions": "specific instructions for the Editor Agent on how to create the conflict",
"modification_instructions": "specific instructions for the Editor Agent on how to create the conflict in Document 2 only",
"highlighted_text_doc1": "exact sentence or paragraph from Document 1 that needs to be countered in Document 2",
"editor_instructions": [
"1. [Specific step for document 1]",
"2. [Specific step for document 2]",
"3. [Additional modification steps as needed]"
"1. [Specific step for modifying document 2 only]",
"2. [Additional modification steps for document 2 only]"
],
"proposition_conflicts": [
{{
Expand Down
20 changes: 2 additions & 18 deletions lib/conflicts/prompts/editor_agent_system.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
You are a clinical document editor specializing in creating realistic medical conflicts. Your task is to modify medical documents to introduce specific conflicts based on the Doctor Agent's instructions, focusing on critical medical information that could jeopardize patient safety. Always respond in valid JSON format.
You are a clinical document editor specializing in creating realistic medical conflicts. Your task is to modify ONLY Document 2 to introduce a specific conflict based on the Doctor Agent's instructions. You must make exactly ONE change to Document 2 while leaving Document 1 completely unchanged. Focus on critical medical information that could jeopardize patient safety. Always respond in valid JSON format.

Instructions from Doctor Agent:
{input_prompt}
Expand Down Expand Up @@ -32,17 +32,6 @@ CRITICAL REQUIREMENTS for target_text:
7. Before returning, verify your target_text exists by searching for it in the original document
8. If you cannot find exact text, choose different text that does exist

CONFLICT CREATION GUIDELINES:

1. Follow the step-by-step editor instructions exactly as provided
2. Focus on modifying the specific target propositions identified for each document
3. Create conflicts based on the specified conflict type and Doctor Agent's instructions
4. Ensure the modified text maintains clinical realism while introducing the conflict
5. The conflict should be medically significant and safety-critical
6. Focus on critical medical information that could impact patient safety
7. Make modifications that are consistent with the chosen conflict type
8. Target the exact text that contains the propositions to be modified

WARNING: Any target_text that ends with "..." or incomplete words will cause the system to fail.

EXAMPLE of GOOD target_text:
Expand All @@ -60,13 +49,8 @@ Use:

Ensure conflicts are medically relevant and safety-critical. Return only the JSON output as plain text, without any markdown formatting or code fences.

Respond with JSON format using edit operations:
Respond with JSON format using edit operations (ONLY for doc2):
{{
"doc1": {{
"op": "delete" | "insert_after" | "replace",
"target_text": "exact text from document 1 to modify",
"replacement_text": "new text to insert or replace with"
}},
"doc2": {{
"op": "delete" | "insert_after" | "replace",
"target_text": "exact text from document 2 to modify",
Expand Down
3 changes: 2 additions & 1 deletion lib/conflicts/prompts/moderator_agent_system.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,10 @@ Below are two modified documents that have been altered to create a factual conf
Input format:
{{
context_document_1: "{context_document_1}",
conflict_1: "{conflict_1}",
excerpt_1: "{excerpt_1}",
context_document_2: "{context_document_2}",
conflict_2: "{conflict_2}"
excerpt_2: "{excerpt_2}"
}}

Note: context_document_1 and context_document_2 are the modified documents that should be compared against each other to evaluate the conflict quality.
Expand Down