29736 - highlight potential issues - status tab (bcgov#3669)

* restructured cell highlighting functions and update fetch colin_extract logic
* remove extra blank line at the end of code cells

Author: eason-pan-bc

data-tool/notebooks/corps_onboarding_process_flow/migration_status_tracking.ipynb

@@ -270,7 +270,9 @@
     "    c.corp_type_cd AS \"{COLUMN_NAMES['corp_type']}\",\n",
     "    CASE\n",
     "        WHEN cp.processed_status = 'COMPLETED' THEN 'Migrated'\n",
+    "        WHEN cp.processed_status = 'FAILED' THEN 'Failed'\n",
     "        WHEN cp.processed_status IS NULL THEN 'Pending'\n",
+    "        ELSE 'Pending'\n",
     "    END AS \"{COLUMN_NAMES['status']}\",\n",
     "    cp.create_date::date AS \"{COLUMN_NAMES['date']}\"\n",
     "FROM\n",
@@ -291,14 +293,16 @@
     "    g.id IN ({mig_group_ids})\n",
     "    AND (\n",
     "        (cp.processed_status = 'COMPLETED' AND cp.environment = 'prod')\n",
+    "        OR (cp.processed_status = 'FAILED' AND cp.environment = 'prod')\n",
     "        OR cp.processed_status IS NULL\n",
     "    )\n",
     "ORDER BY\n",
     "    g.display_name, \n",
     "    b.display_name,\n",
     "    CASE\n",
     "        WHEN cp.processed_status = 'COMPLETED' THEN 0\n",
-    "        ELSE 1\n",
+    "        WHEN cp.processed_status = 'FAILED' THEN 1\n",
+    "        ELSE 2\n",
     "    END, \n",
     "    cp.create_date DESC,\n",
     "    cn.corp_name;\n",
@@ -319,7 +323,7 @@
     "\n",
     "# Display results\n",
     "with pd.option_context('display.max_rows', None):\n",
-    "    display(colin_extract_df)\n"
+    "    display(colin_extract_df)"
    ]
   },
   {
@@ -431,7 +435,7 @@
     "\n",
     "# Display results\n",
     "with pd.option_context('display.max_rows', None):\n",
-    "    display(lear_combined_df)\n"
+    "    display(lear_combined_df)"
    ]
   },
   {
@@ -548,7 +552,7 @@
     "\n",
     "# Display results\n",
     "with pd.option_context('display.max_rows', None):\n",
-    "    display(colin_oracle_combined_df)\n"
+    "    display(colin_oracle_combined_df)"
    ]
   },
   {
@@ -660,7 +664,7 @@
     "    raise\n",
     "\n",
     "with pd.option_context('display.max_rows', None):\n",
-    "    display(batch_summary_df)\n"
+    "    display(batch_summary_df)"
    ]
   },
   {
@@ -672,6 +676,37 @@
     "Generate formatted Excel file with the merged migration tracking data."
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Define highlighting rules\n",
+    "HIGHLIGHTING_RULES = [\n",
+    "    {\n",
+    "        'column_name': COLUMN_NAMES['affiliated'],\n",
+    "        'condition_value': 'N',\n",
+    "        'fill_color': CONFIG['excel_export']['filled_color']\n",
+    "    },\n",
+    "    {\n",
+    "        'column_name': COLUMN_NAMES['banner_updated_in_colin'], \n",
+    "        'condition_value': 'N',\n",
+    "        'fill_color': CONFIG['excel_export']['filled_color']\n",
+    "    },\n",
+    "    {\n",
+    "        'column_name': COLUMN_NAMES['status'],\n",
+    "        'condition_value': 'Failed', \n",
+    "        'fill_color': CONFIG['excel_export']['filled_color']\n",
+    "    },\n",
+    "    {\n",
+    "        'column_name': COLUMN_NAMES['frozen_in_colin'],\n",
+    "        'condition_value': 'N',\n",
+    "        'fill_color': CONFIG['excel_export']['filled_color']\n",
+    "    }\n",
+    "]"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -680,22 +715,54 @@
    "source": [
     "from openpyxl.styles import Font, PatternFill, Alignment\n",
     "\n",
+    "def apply_cell_highlighting(worksheet, highlighting_rules):\n",
+    "    \"\"\"\n",
+    "    Apply conditional highlighting to worksheet cells based on rules.\n",
+    "    \n",
+    "    Args:\n",
+    "        worksheet: The openpyxl worksheet\n",
+    "        highlighting_rules: List of dicts with column_name, condition_value, fill_color\n",
+    "    \n",
+    "    Returns:\n",
+    "        int: Total number of cells highlighted\n",
+    "    \"\"\"\n",
+    "    highlighted_count = 0\n",
+    "    \n",
+    "    # Find column indices for all highlighting rules\n",
+    "    column_indices = {}\n",
+    "    for col_idx, cell in enumerate(worksheet[1], 1):\n",
+    "        for rule in highlighting_rules:\n",
+    "            if cell.value == rule['column_name']:\n",
+    "                column_indices[rule['column_name']] = col_idx\n",
+    "                break\n",
+    "    \n",
+    "    # Apply highlighting based on rules\n",
+    "    for row_num, row in enumerate(worksheet.iter_rows(), 1):\n",
+    "        if row_num == 1:  # Skip header row\n",
+    "            continue\n",
+    "            \n",
+    "        for col_idx, cell in enumerate(row, 1):\n",
+    "            # Check each highlighting rule\n",
+    "            for rule in highlighting_rules:\n",
+    "                if col_idx == column_indices.get(rule['column_name']) and cell.value == rule['condition_value']:\n",
+    "                    fill = PatternFill(start_color=rule['fill_color'], end_color=rule['fill_color'], fill_type='solid')\n",
+    "                    cell.fill = fill\n",
+    "                    highlighted_count += 1\n",
+    "    \n",
+    "    return highlighted_count\n",
+    "\n",
+    "\n",
     "def format_worksheet(worksheet) -> None:\n",
     "    \"\"\"Format the given worksheet.\"\"\"\n",
+    "    \n",
     "    # Define display styles\n",
     "    header_font = Font(size=CONFIG['excel_export']['font_size'], bold=True)\n",
     "    normal_font = Font(size=CONFIG['excel_export']['font_size'])\n",
-    "    red_fill = PatternFill(start_color=CONFIG['excel_export']['filled_color'], end_color=CONFIG['excel_export']['filled_color'], fill_type='solid')\n",
     "\n",
-    "    # Find the Affiliated column index\n",
-    "    affiliated_col_idx = None\n",
-    "    for col_idx, cell in enumerate(worksheet[1], 1):\n",
-    "        if cell.value == COLUMN_NAMES['affiliated']:\n",
-    "            affiliated_col_idx = col_idx\n",
-    "            break\n",
+    "    # Apply cell highlighting\n",
+    "    highlighted_count = apply_cell_highlighting(worksheet, HIGHLIGHTING_RULES)\n",
     "\n",
-    "    # Format and highlight rows\n",
-    "    highlighted_count = 0\n",
+    "    # Format rows (excluding highlighting which is now handled separately)\n",
     "    for row_num, row in enumerate(worksheet.iter_rows(), 1):\n",
     "        for col_idx, cell in enumerate(row, 1):\n",
     "            if row_num == 1:\n",
@@ -705,11 +772,6 @@
     "                # Data rows\n",
     "                cell.font = normal_font\n",
     "                cell.alignment = Alignment(horizontal='left')\n",
-    "                \n",
-    "                # Highlight Affiliated column if value is 'N'\n",
-    "                if col_idx == affiliated_col_idx and cell.value == 'N':\n",
-    "                    cell.fill = red_fill\n",
-    "                    highlighted_count += 1\n",
     "    \n",
     "    # Freeze header row\n",
     "    worksheet.freeze_panes = 'A2'\n",
@@ -737,7 +799,9 @@
     "        adjusted_width = min(max_length + 10, CONFIG['excel_export']['max_column_width'])\n",
     "        worksheet.column_dimensions[column_letter].width = adjusted_width\n",
     "\n",
-    "    print(f\"Red highlighting applied to {highlighted_count} cells in '{COLUMN_NAMES['affiliated']}' column with 'N'\")"
+    "    # Print highlighting summary\n",
+    "    rule_names = [rule['column_name'] for rule in HIGHLIGHTING_RULES]\n",
+    "    print(f\"In {worksheet.title} tab:\\n Red highlighting applied to {highlighted_count} cells across columns: {', '.join(rule_names)}\")"
    ]
   },
   {
@@ -798,7 +862,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "3.8.17",
+   "display_name": "Python 3",
    "language": "python",
    "name": "python3"
   },
@@ -812,7 +876,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.17"
+   "version": "3.10.12"
   }
  },
  "nbformat": 4,