diff --git a/pyproject.toml b/pyproject.toml index 6393b96..b9b2420 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,6 +78,9 @@ target-version = "py312" select = ["E", "F", "I", "W"] ignore = ["E501"] +[tool.ruff.lint.per-file-ignores] +"scripts/*.py" = ["E402"] # Allow imports after sys.path modification + [tool.mypy] python_version = "3.12" ignore_missing_imports = true diff --git a/scripts/generate_agent_report.py b/scripts/generate_agent_report.py index a57dea1..261fbb8 100755 --- a/scripts/generate_agent_report.py +++ b/scripts/generate_agent_report.py @@ -73,24 +73,30 @@ def generate_agent_report(summary: dict, db_stats: dict) -> dict: if report["status"] == "failed": agent_summary = summary.get("agent_summary", {}) for issue in agent_summary.get("blocking_issues", []): - next_steps.append({ - "type": "fix", - "priority": "high", - "description": issue, - }) + next_steps.append( + { + "type": "fix", + "priority": "high", + "description": issue, + } + ) for recommendation in agent_summary.get("recommendations", []): - next_steps.append({ - "type": "investigate", - "priority": "medium", - "description": recommendation, - }) + next_steps.append( + { + "type": "investigate", + "priority": "medium", + "description": recommendation, + } + ) else: - next_steps.append({ - "type": "proceed", - "priority": "low", - "description": "All tests passed. Ready for next development phase.", - }) + next_steps.append( + { + "type": "proceed", + "priority": "low", + "description": "All tests passed. Ready for next development phase.", + } + ) report["next_steps"] = next_steps @@ -103,7 +109,9 @@ def generate_human_report(summary: dict, db_stats: dict) -> str: lines.append("# PowerSchool Portal - Test Report") lines.append("") lines.append(f"**Generated:** {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}") - lines.append(f"**Branch:** {os.getenv('CI_COMMIT_REF_NAME', os.getenv('GITHUB_REF_NAME', 'local'))}") + lines.append( + f"**Branch:** {os.getenv('CI_COMMIT_REF_NAME', os.getenv('GITHUB_REF_NAME', 'local'))}" + ) lines.append(f"**Pipeline:** {os.getenv('CI_PIPELINE_ID', os.getenv('GITHUB_RUN_ID', 'N/A'))}") lines.append("") diff --git a/scripts/generate_test_summary.py b/scripts/generate_test_summary.py index 66d0ccd..dfdf2eb 100755 --- a/scripts/generate_test_summary.py +++ b/scripts/generate_test_summary.py @@ -43,11 +43,13 @@ def parse_junit_xml(xml_path: Path) -> dict: error = testcase.find("error") result_element = failure if failure is not None else error if result_element is not None: - failures.append({ - "name": testcase.get("name"), - "classname": testcase.get("classname"), - "message": result_element.get("message", ""), - }) + failures.append( + { + "name": testcase.get("name"), + "classname": testcase.get("classname"), + "message": result_element.get("message", ""), + } + ) return { "tests": int(testsuite.get("tests", 0)), @@ -112,9 +114,7 @@ def get_database_stats(db_path: Path) -> dict: # Get days absent try: - cursor.execute( - "SELECT days_absent FROM attendance_summary WHERE term = 'YTD' LIMIT 1" - ) + cursor.execute("SELECT days_absent FROM attendance_summary WHERE term = 'YTD' LIMIT 1") row = cursor.fetchone() stats["days_absent"] = row[0] if row else None except sqlite3.OperationalError: @@ -165,9 +165,7 @@ def validate_ground_truth(stats: dict) -> dict: return validations -def generate_agent_summary( - test_results: dict, db_stats: dict, validations: dict -) -> dict: +def generate_agent_summary(test_results: dict, db_stats: dict, validations: dict) -> dict: """Generate agent-friendly summary with recommendations.""" summary = { "ready_for_next_phase": True, @@ -196,9 +194,7 @@ def generate_agent_summary( "Missing assignments not detected correctly. " f"Found: {db_stats.get('missing_assignments', 0)}, Expected: >= 2" ) - summary["recommendations"].append( - "Check scraper parser for assignment status detection" - ) + summary["recommendations"].append("Check scraper parser for assignment status detection") summary["ready_for_next_phase"] = False if not validations.get("attendance_rate_correct"): @@ -208,15 +204,11 @@ def generate_agent_summary( "Attendance rate not extracted. Check attendance parser." ) else: - summary["warnings"].append( - f"Attendance rate {rate}% outside expected range (85-92%)" - ) + summary["warnings"].append(f"Attendance rate {rate}% outside expected range (85-92%)") summary["recommendations"].append("Verify attendance scraper targets correct elements") if not validations.get("courses_found"): - summary["warnings"].append( - f"Found {db_stats.get('courses', 0)} courses, expected >= 8" - ) + summary["warnings"].append(f"Found {db_stats.get('courses', 0)} courses, expected >= 8") summary["recommendations"].append("Check schedule scraper for missing courses") # Database health @@ -262,17 +254,13 @@ def main(): summary["ground_truth_validation"] = validations # Calculate overall test stats - total_tests = sum( - r.get("tests", 0) for r in test_results.values() if isinstance(r, dict) - ) + total_tests = sum(r.get("tests", 0) for r in test_results.values() if isinstance(r, dict)) total_failures = sum( r.get("failures", 0) + r.get("errors", 0) for r in test_results.values() if isinstance(r, dict) ) - total_skipped = sum( - r.get("skipped", 0) for r in test_results.values() if isinstance(r, dict) - ) + total_skipped = sum(r.get("skipped", 0) for r in test_results.values() if isinstance(r, dict)) summary["totals"] = { "tests": total_tests, @@ -285,15 +273,11 @@ def main(): all_tests_passed = total_failures == 0 summary["status"] = ( - "passed" - if (all_tests_passed and validations.get("all_passed")) - else "failed" + "passed" if (all_tests_passed and validations.get("all_passed")) else "failed" ) # Generate agent summary - summary["agent_summary"] = generate_agent_summary( - test_results, db_stats, validations - ) + summary["agent_summary"] = generate_agent_summary(test_results, db_stats, validations) # Output as JSON print(json.dumps(summary, indent=2, default=str)) diff --git a/scripts/recon.py b/scripts/recon.py index d608d1f..c35fa28 100755 --- a/scripts/recon.py +++ b/scripts/recon.py @@ -156,18 +156,29 @@ def analyze_page_structure(html: str, name: str) -> dict: # Find key elements by common IDs/classes key_selectors = [ - "#grades-table", ".grades", "#assignments", ".assignment", - "#attendance", ".attendance", "#schedule", ".schedule", - ".student-info", "#student-name", ".course", ".class" + "#grades-table", + ".grades", + "#assignments", + ".assignment", + "#attendance", + ".attendance", + "#schedule", + ".schedule", + ".student-info", + "#student-name", + ".course", + ".class", ] for selector in key_selectors: elements = soup.select(selector) if elements: - analysis["key_elements"].append({ - "selector": selector, - "count": len(elements), - "sample_text": elements[0].get_text(strip=True)[:100] if elements else "" - }) + analysis["key_elements"].append( + { + "selector": selector, + "count": len(elements), + "sample_text": elements[0].get_text(strip=True)[:100] if elements else "", + } + ) return analysis @@ -192,7 +203,7 @@ def run_recon(): browser = p.chromium.launch(headless=False, slow_mo=500) context = browser.new_context( viewport={"width": 1280, "height": 900}, - user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36" + user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36", ) page = context.new_page() diff --git a/scripts/recon_enhanced.py b/scripts/recon_enhanced.py index 9b319f9..1453a9d 100755 --- a/scripts/recon_enhanced.py +++ b/scripts/recon_enhanced.py @@ -33,12 +33,7 @@ def get_student_info(page: Page) -> dict: # Look for student name in header try: # Try multiple selectors - selectors = [ - "#userName", - ".student-name", - "#student-name", - "span.studentName" - ] + selectors = ["#userName", ".student-name", "#student-name", "span.studentName"] for sel in selectors: elem = page.query_selector(sel) if elem: @@ -206,10 +201,16 @@ def scrape_assignments_page(page: Page, show_all: bool = True) -> list: "percent": percent, "letter_grade": letter_grade, "codes": codes, - "status": "Missing" if "Missing" in codes else "Collected" if "Collected" in codes else "Unknown" + "status": "Missing" + if "Missing" in codes + else "Collected" + if "Collected" in codes + else "Unknown", } assignments.append(assignment) - print(f" Assignment: {assignment_name} ({course}) - Score: {score}, Status: {codes}") + print( + f" Assignment: {assignment_name} ({course}) - Score: {score}, Status: {codes}" + ) print(f" Found {len(assignments)} assignments") @@ -268,7 +269,10 @@ def scrape_schedule_page(page: Page) -> list: def scrape_attendance_dashboard(page: Page) -> dict: """Scrape the attendance dashboard for detailed attendance data.""" print("\n=== SCRAPING ATTENDANCE DASHBOARD ===") - page.goto(f"{BASE_URL}/guardian/mba_attendance_monitor/guardian_dashboard.html", wait_until="networkidle") + page.goto( + f"{BASE_URL}/guardian/mba_attendance_monitor/guardian_dashboard.html", + wait_until="networkidle", + ) # Wait for Angular app to load data page.wait_for_timeout(5000) @@ -334,7 +338,9 @@ def scrape_attendance_dashboard(page: Page) -> dict: except Exception as e: print(f" Note: Could not extract Angular data: {e}") - print(f" Attendance: Rate={data['rate']}%, Present={data['days_present']}, Absent={data['days_absent']}, Tardies={data['tardies']}") + print( + f" Attendance: Rate={data['rate']}%, Present={data['days_present']}, Absent={data['days_absent']}, Tardies={data['tardies']}" + ) except Exception as e: print(f" Error extracting attendance: {e}") @@ -371,7 +377,7 @@ def run_enhanced_recon(): browser = p.chromium.launch(headless=False, slow_mo=300) context = browser.new_context( viewport={"width": 1280, "height": 900}, - user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36" + user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36", ) page = context.new_page() diff --git a/scripts/scrape_full.py b/scripts/scrape_full.py index e3cf0bb..6f98039 100755 --- a/scripts/scrape_full.py +++ b/scripts/scrape_full.py @@ -545,12 +545,8 @@ def run_full_scrape(headless: bool = False, student_name: str | None = None): import argparse parser = argparse.ArgumentParser(description="Scrape PowerSchool parent portal") - parser.add_argument( - "--headless", action="store_true", help="Run browser in headless mode" - ) - parser.add_argument( - "--student", "-s", type=str, help="Scrape specific student only" - ) + parser.add_argument("--headless", action="store_true", help="Run browser in headless mode") + parser.add_argument("--student", "-s", type=str, help="Scrape specific student only") args = parser.parse_args() run_full_scrape(headless=args.headless, student_name=args.student) diff --git a/scripts/validate_ground_truth.py b/scripts/validate_ground_truth.py index c82be99..af03c79 100755 --- a/scripts/validate_ground_truth.py +++ b/scripts/validate_ground_truth.py @@ -78,16 +78,12 @@ def validate(db_path: Path) -> Tuple[bool, list, list]: if missing_count < expected_min: errors.append( - f"Missing assignments: found {missing_count}, " - f"expected at least {expected_min}" + f"Missing assignments: found {missing_count}, expected at least {expected_min}" ) else: # Check for specific known missing assignments for expected in GROUND_TRUTH["missing_assignments"]: - found = any( - expected["name"].lower() in (m[1] or "").lower() - for m in missing - ) + found = any(expected["name"].lower() in (m[1] or "").lower() for m in missing) if not found: warnings.append( f"Expected missing assignment not found: {expected['name']}" @@ -110,8 +106,7 @@ def validate(db_path: Path) -> Tuple[bool, list, list]: # Allow 5% tolerance for rate changes if abs(rate - expected_rate) > 5.0: warnings.append( - f"Attendance rate: got {rate:.1f}%, " - f"expected ~{expected_rate}%" + f"Attendance rate: got {rate:.1f}%, expected ~{expected_rate}%" ) # Check days (allow some variance) @@ -119,16 +114,14 @@ def validate(db_path: Path) -> Tuple[bool, list, list]: expected_present = GROUND_TRUTH["days_present"] if abs(present - expected_present) > 5: warnings.append( - f"Days present: got {present}, " - f"expected ~{expected_present}" + f"Days present: got {present}, expected ~{expected_present}" ) if absent is not None: expected_absent = GROUND_TRUTH["days_absent"] if abs(absent - expected_absent) > 3: warnings.append( - f"Days absent: got {absent}, " - f"expected ~{expected_absent}" + f"Days absent: got {absent}, expected ~{expected_absent}" ) else: errors.append("No attendance summary found") @@ -144,15 +137,15 @@ def validate(db_path: Path) -> Tuple[bool, list, list]: expected_min = GROUND_TRUTH["courses_min"] if course_count < expected_min: - errors.append( - f"Courses: found {course_count}, expected >= {expected_min}" - ) + errors.append(f"Courses: found {course_count}, expected >= {expected_min}") except sqlite3.OperationalError as e: errors.append(f"Error counting courses: {e}") # Check teachers try: - cursor.execute("SELECT DISTINCT teacher_name FROM courses WHERE teacher_name IS NOT NULL") + cursor.execute( + "SELECT DISTINCT teacher_name FROM courses WHERE teacher_name IS NOT NULL" + ) teachers = [t[0] for t in cursor.fetchall()] # Check at least 3 expected teachers are found @@ -166,9 +159,7 @@ def validate(db_path: Path) -> Tuple[bool, list, list]: warnings.append(f"Teacher not found: {expected_teacher}") if teachers_found < 3: - warnings.append( - f"Only {teachers_found} of expected teachers found" - ) + warnings.append(f"Only {teachers_found} of expected teachers found") except sqlite3.OperationalError as e: warnings.append(f"Could not check teachers: {e}")