From 7ab45e94111865eb3253220d45bc755debbff343 Mon Sep 17 00:00:00 2001 From: Simon Meoni Date: Fri, 19 Sep 2025 17:41:45 +0200 Subject: [PATCH] feat: only edit doc 2 & add execerpt to the moderator --- .../conflicts/agents/doctor_agent.py | 12 ++++++-- .../conflicts/agents/editor_agent.py | 30 ++++++++----------- .../conflicts/agents/moderator_agent.py | 15 ++++------ lib/conflicts/conflicts/core/base.py | 26 ++++++++-------- .../conflicts/core/document_operations.py | 20 ++----------- lib/conflicts/conflicts/core/models.py | 4 +-- lib/conflicts/prompts/doctor_agent_system.txt | 10 ++++--- lib/conflicts/prompts/editor_agent_system.txt | 20 ++----------- .../prompts/moderator_agent_system.txt | 3 +- 9 files changed, 55 insertions(+), 85 deletions(-) diff --git a/lib/conflicts/conflicts/agents/doctor_agent.py b/lib/conflicts/conflicts/agents/doctor_agent.py index c7c5ca1..3ba524e 100644 --- a/lib/conflicts/conflicts/agents/doctor_agent.py +++ b/lib/conflicts/conflicts/agents/doctor_agent.py @@ -34,8 +34,8 @@ def __init__(self, client, model, cfg): def __call__( self, document_pair: DocumentPair, - propositions1: PropositionResult = None, - propositions2: PropositionResult = None, + propositions1: PropositionResult, + propositions2: PropositionResult, ) -> ConflictResult: """ Analyze documents and determine the best conflict type to introduce @@ -100,7 +100,12 @@ def __call__( parsed_response = self._parse_json_response(response) # Validate required fields - required_fields = ["conflict_type", "reasoning", "modification_instructions"] + required_fields = [ + "conflict_type", + "reasoning", + "modification_instructions", + "highlighted_text_doc1", + ] for field in required_fields: if field not in parsed_response: raise ValueError(f"Missing required field '{field}' in Doctor Agent response") @@ -117,6 +122,7 @@ def __call__( conflict_type=parsed_response["conflict_type"], reasoning=parsed_response["reasoning"], modification_instructions=parsed_response["modification_instructions"], + highlighted_text_doc1=parsed_response["highlighted_text_doc1"], editor_instructions=parsed_response.get("editor_instructions", []), proposition_conflicts=parsed_response.get("proposition_conflicts", []), ) diff --git a/lib/conflicts/conflicts/agents/editor_agent.py b/lib/conflicts/conflicts/agents/editor_agent.py index 71ebd98..a33c5a4 100644 --- a/lib/conflicts/conflicts/agents/editor_agent.py +++ b/lib/conflicts/conflicts/agents/editor_agent.py @@ -45,22 +45,20 @@ def __call__( if attempt == max_retries - 1: self.logger.error(f"All {max_retries} attempts failed: {e}") return EditorResult( - modified_document1=document_pair.doc1_text, modified_document2=document_pair.doc2_text, changes_made=f"Failed to create conflict after {max_retries} attempts: {e}", - change_info_1="No changes made - all attempts failed", change_info_2="No changes made - all attempts failed", ) self.logger.warning(f"Attempt {attempt + 1} failed: {e}, retrying...") def _perform_modification( - self, document_pair: DocumentPair, conflict_instructions: ConflictResult + self, document_pair: DocumentPair, conflict_result: ConflictResult ) -> EditorResult: """Perform a single modification attempt""" - prompt = self._build_prompt(document_pair, conflict_instructions) + prompt = self._build_prompt(document_pair, conflict_result) response = self._execute_prompt(prompt, self.cfg.model.editor_temperature) parsed_result = self._parse_and_validate_response(response, document_pair) - return self._create_result(parsed_result, document_pair) + return self._create_result(parsed_result, document_pair, conflict_result) def _build_prompt( self, document_pair: DocumentPair, conflict_instructions: ConflictResult @@ -126,24 +124,23 @@ def _parse_and_validate_response(self, response: str, document_pair: DocumentPai self.logger.error(f"Response was: {response}") raise - if ( - parsed_result["modified_doc_1"].strip() == document_pair.doc1_text.strip() - and parsed_result["modified_doc_2"].strip() == document_pair.doc2_text.strip() - ): + if parsed_result["modified_doc_2"].strip() == document_pair.doc2_text.strip(): raise ValueError("No modifications were applied to the documents") return parsed_result - def _create_result(self, parsed_result: dict, document_pair: DocumentPair) -> EditorResult: + def _create_result( + self, + parsed_result: dict, + document_pair: DocumentPair, + conflict_instructions: ConflictResult, + ) -> EditorResult: """Create and log the final result""" result = EditorResult( - modified_document1=parsed_result["modified_doc_1"], modified_document2=parsed_result["modified_doc_2"], changes_made=f"Applied {parsed_result['conflict_type']} conflict modifications", - change_info_1=parsed_result.get("change_info_1"), change_info_2=parsed_result.get("change_info_2"), - original_excerpt_1=parsed_result.get("original_excerpt_1"), - modified_excerpt_1=parsed_result.get("modified_excerpt_1"), + original_excerpt_1=conflict_instructions.highlighted_text_doc1, original_excerpt_2=parsed_result.get("original_excerpt_2"), modified_excerpt_2=parsed_result.get("modified_excerpt_2"), ) @@ -152,10 +149,9 @@ def _create_result(self, parsed_result: dict, document_pair: DocumentPair) -> Ed self.logger.info(f"Conflict type: {parsed_result['conflict_type']}") # Log document length changes - orig_len1, orig_len2 = len(document_pair.doc1_text), len(document_pair.doc2_text) - mod_len1, mod_len2 = len(result.modified_document1), len(result.modified_document2) + orig_len2 = len(document_pair.doc2_text) + mod_len2 = len(result.modified_document2) - self.logger.debug(f"Document 1 length: {orig_len1} -> {mod_len1} ({mod_len1-orig_len1:+d})") self.logger.debug(f"Document 2 length: {orig_len2} -> {mod_len2} ({mod_len2-orig_len2:+d})") return result diff --git a/lib/conflicts/conflicts/agents/moderator_agent.py b/lib/conflicts/conflicts/agents/moderator_agent.py index 24539ae..4df61a0 100644 --- a/lib/conflicts/conflicts/agents/moderator_agent.py +++ b/lib/conflicts/conflicts/agents/moderator_agent.py @@ -23,7 +23,7 @@ def __init__(self, client, model, cfg, min_validation_score: int = 4): self.min_score = min_validation_score def __call__( - self, original_pair: DocumentPair, modified_docs: EditorResult, conflict_type: str + self, original_pair: DocumentPair, editor_result: EditorResult, conflict_type: str ) -> ValidationResult: """ Validate the modifications made to clinical documents @@ -40,10 +40,11 @@ def __call__( try: prompt = self.system_prompt.format( - context_document_1=self._truncate_document(modified_docs.modified_document1), - context_document_2=self._truncate_document(modified_docs.modified_document2), - conflict_1=modified_docs.change_info_1 or "No change info available", - conflict_2=modified_docs.change_info_2 or "No change info available", + context_document_1=self._truncate_document(original_pair.doc1_text), + context_document_2=self._truncate_document(editor_result.modified_document2), + excerpt_2=editor_result.modified_excerpt_2, + excerpt_1=editor_result.original_excerpt_1, + conflict_2=editor_result.change_info_2 or "No change info available", ) print(prompt) @@ -109,15 +110,11 @@ def detailed_validation_check( """ checks = { "documents_modified": { - "doc1_changed": original_pair.doc1_text != modified_docs.modified_document1, "doc2_changed": original_pair.doc2_text != modified_docs.modified_document2, "any_changed": False, }, "length_analysis": { "doc1_original_length": len(original_pair.doc1_text), - "doc1_modified_length": len(modified_docs.modified_document1), - "doc1_length_change": len(modified_docs.modified_document1) - - len(original_pair.doc1_text), "doc2_original_length": len(original_pair.doc2_text), "doc2_modified_length": len(modified_docs.modified_document2), "doc2_length_change": len(modified_docs.modified_document2) diff --git a/lib/conflicts/conflicts/core/base.py b/lib/conflicts/conflicts/core/base.py index 156b73c..00b2ee2 100644 --- a/lib/conflicts/conflicts/core/base.py +++ b/lib/conflicts/conflicts/core/base.py @@ -169,7 +169,7 @@ def save_to_parquet(self, output_path: str = None): def save_validated_documents( self, original_pair: DocumentPair, - modified_docs: EditorResult, + editor_result: EditorResult, conflict_type: str, validation_result: ValidationResult, ) -> int: @@ -178,8 +178,8 @@ def save_validated_documents( # Create document data doc_data = DocumentData( - doc_1=modified_docs.modified_document1, - doc_2=modified_docs.modified_document2, + doc_1=original_pair.doc1_text, + doc_2=editor_result.modified_document2, orig_doc_1=original_pair.doc1_text, orig_doc_2=original_pair.doc2_text, created_at=datetime.now().isoformat(), @@ -192,12 +192,12 @@ def save_validated_documents( # Add annotation for excerpt 1 if exists if ( - modified_docs.modified_excerpt_1 - and not pd.isna(modified_docs.modified_excerpt_1) - and modified_docs.modified_excerpt_1.strip() + editor_result.original_excerpt_1 + and not pd.isna(editor_result.original_excerpt_1) + and editor_result.original_excerpt_1.strip() ): start_pos, end_pos = self.find_text_positions( - modified_docs.modified_document1, modified_docs.modified_excerpt_1 + original_pair.doc1_text, editor_result.original_excerpt_1 ) if start_pos is not None: annotation = Annotation( @@ -210,7 +210,7 @@ def save_validated_documents( value=AnnotationValue( start=start_pos, end=end_pos, - text=modified_docs.modified_excerpt_1, + text=editor_result.original_excerpt_1, labels=["Conflict"], ), ) @@ -218,12 +218,12 @@ def save_validated_documents( # Add annotation for excerpt 2 if exists if ( - modified_docs.modified_excerpt_2 - and not pd.isna(modified_docs.modified_excerpt_2) - and modified_docs.modified_excerpt_2.strip() + editor_result.modified_excerpt_2 + and not pd.isna(editor_result.modified_excerpt_2) + and editor_result.modified_excerpt_2.strip() ): start_pos, end_pos = self.find_text_positions( - modified_docs.modified_document2, modified_docs.modified_excerpt_2 + editor_result.modified_document2, editor_result.modified_excerpt_2 ) if start_pos is not None: annotation = Annotation( @@ -236,7 +236,7 @@ def save_validated_documents( value=AnnotationValue( start=start_pos, end=end_pos, - text=modified_docs.modified_excerpt_2, + text=editor_result.modified_excerpt_2, labels=["Conflict"], ), ) diff --git a/lib/conflicts/conflicts/core/document_operations.py b/lib/conflicts/conflicts/core/document_operations.py index 5777ee7..2b809bf 100644 --- a/lib/conflicts/conflicts/core/document_operations.py +++ b/lib/conflicts/conflicts/core/document_operations.py @@ -140,22 +140,12 @@ def parse_response( data = _extract_json_from_response(response) # Check if we have the expected edit operations format - required_fields = ["doc1", "doc2", "conflict_type"] - if all(field in data for field in required_fields): + # Support both old format (doc1 + doc2) and new format (doc2 only) + if "doc2" in data and "conflict_type" in data: logging.info("Found edit operations format, applying operations to documents") # Apply edit operations to documents try: - ( - modified_doc_1, - change_description_1, - orig_excerpt_1, - mod_excerpt_1, - ) = apply_edit_operation( - original_doc_1, - data["doc1"], - min_text_length, - ) ( modified_doc_2, change_description_2, @@ -178,18 +168,14 @@ def parse_response( conflict_type = data["conflict_type"] return { - "modified_doc_1": modified_doc_1, "modified_doc_2": modified_doc_2, "conflict_type": conflict_type, - "change_info_1": change_description_1, "change_info_2": change_description_2, - "original_excerpt_1": orig_excerpt_1, - "modified_excerpt_1": mod_excerpt_1, "original_excerpt_2": orig_excerpt_2, "modified_excerpt_2": mod_excerpt_2, } else: - raise ValueError(f"Response missing required fields: {required_fields}") + raise ValueError("Response missing required fields: doc2 and conflict_type") except json.JSONDecodeError as e: logging.error(f"Failed to parse response as JSON: {e}") diff --git a/lib/conflicts/conflicts/core/models.py b/lib/conflicts/conflicts/core/models.py index a618582..de7387f 100644 --- a/lib/conflicts/conflicts/core/models.py +++ b/lib/conflicts/conflicts/core/models.py @@ -25,6 +25,7 @@ class ConflictResult: conflict_type: str reasoning: str modification_instructions: str + highlighted_text_doc1: str editor_instructions: Optional[list[str]] = None proposition_conflicts: Optional[list[dict]] = None @@ -33,13 +34,10 @@ class ConflictResult: class EditorResult: """Result from the Editor Agent""" - modified_document1: str modified_document2: str changes_made: str - change_info_1: Optional[str] = None change_info_2: Optional[str] = None original_excerpt_1: Optional[str] = None - modified_excerpt_1: Optional[str] = None original_excerpt_2: Optional[str] = None modified_excerpt_2: Optional[str] = None diff --git a/lib/conflicts/prompts/doctor_agent_system.txt b/lib/conflicts/prompts/doctor_agent_system.txt index 3a02ad4..bd0b575 100644 --- a/lib/conflicts/prompts/doctor_agent_system.txt +++ b/lib/conflicts/prompts/doctor_agent_system.txt @@ -28,17 +28,19 @@ Analyze these documents and their propositions to choose the most appropriate co 4. What type of conflict would be realistic and educational given the temporal context 5. Which conflict type best matches the available clinical information and temporal recommendations +IMPORTANT: Only modify Document 2 to introduce conflicts. Document 1 should remain unchanged but you must identify and highlight specific sentences or paragraphs in Document 1 that need to be countered when editing Document 2. + Create specific counterfactual modifications that will create conflicts between the documents, then provide clear step-by-step instructions for the Editor Agent. Respond with JSON format: {{ "conflict_type": "chosen_conflict_type_key", "reasoning": "explanation of why this conflict type was chosen, including temporal considerations", - "modification_instructions": "specific instructions for the Editor Agent on how to create the conflict", + "modification_instructions": "specific instructions for the Editor Agent on how to create the conflict in Document 2 only", + "highlighted_text_doc1": "exact sentence or paragraph from Document 1 that needs to be countered in Document 2", "editor_instructions": [ - "1. [Specific step for document 1]", - "2. [Specific step for document 2]", - "3. [Additional modification steps as needed]" + "1. [Specific step for modifying document 2 only]", + "2. [Additional modification steps for document 2 only]" ], "proposition_conflicts": [ {{ diff --git a/lib/conflicts/prompts/editor_agent_system.txt b/lib/conflicts/prompts/editor_agent_system.txt index d116900..1f51107 100644 --- a/lib/conflicts/prompts/editor_agent_system.txt +++ b/lib/conflicts/prompts/editor_agent_system.txt @@ -1,4 +1,4 @@ -You are a clinical document editor specializing in creating realistic medical conflicts. Your task is to modify medical documents to introduce specific conflicts based on the Doctor Agent's instructions, focusing on critical medical information that could jeopardize patient safety. Always respond in valid JSON format. +You are a clinical document editor specializing in creating realistic medical conflicts. Your task is to modify ONLY Document 2 to introduce a specific conflict based on the Doctor Agent's instructions. You must make exactly ONE change to Document 2 while leaving Document 1 completely unchanged. Focus on critical medical information that could jeopardize patient safety. Always respond in valid JSON format. Instructions from Doctor Agent: {input_prompt} @@ -32,17 +32,6 @@ CRITICAL REQUIREMENTS for target_text: 7. Before returning, verify your target_text exists by searching for it in the original document 8. If you cannot find exact text, choose different text that does exist -CONFLICT CREATION GUIDELINES: - -1. Follow the step-by-step editor instructions exactly as provided -2. Focus on modifying the specific target propositions identified for each document -3. Create conflicts based on the specified conflict type and Doctor Agent's instructions -4. Ensure the modified text maintains clinical realism while introducing the conflict -5. The conflict should be medically significant and safety-critical -6. Focus on critical medical information that could impact patient safety -7. Make modifications that are consistent with the chosen conflict type -8. Target the exact text that contains the propositions to be modified - WARNING: Any target_text that ends with "..." or incomplete words will cause the system to fail. EXAMPLE of GOOD target_text: @@ -60,13 +49,8 @@ Use: Ensure conflicts are medically relevant and safety-critical. Return only the JSON output as plain text, without any markdown formatting or code fences. -Respond with JSON format using edit operations: +Respond with JSON format using edit operations (ONLY for doc2): {{ - "doc1": {{ - "op": "delete" | "insert_after" | "replace", - "target_text": "exact text from document 1 to modify", - "replacement_text": "new text to insert or replace with" - }}, "doc2": {{ "op": "delete" | "insert_after" | "replace", "target_text": "exact text from document 2 to modify", diff --git a/lib/conflicts/prompts/moderator_agent_system.txt b/lib/conflicts/prompts/moderator_agent_system.txt index 0f95668..a74f4f8 100644 --- a/lib/conflicts/prompts/moderator_agent_system.txt +++ b/lib/conflicts/prompts/moderator_agent_system.txt @@ -11,9 +11,10 @@ Below are two modified documents that have been altered to create a factual conf Input format: {{ context_document_1: "{context_document_1}", - conflict_1: "{conflict_1}", + excerpt_1: "{excerpt_1}", context_document_2: "{context_document_2}", conflict_2: "{conflict_2}" + excerpt_2: "{excerpt_2}" }} Note: context_document_1 and context_document_2 are the modified documents that should be compared against each other to evaluate the conflict quality.