adding migration verifier

migration/mongosync_insights/mongosync_insights.py

@@ -2,6 +2,7 @@
 from flask import Flask, render_template, request, make_response
 from mongosync_plot_logs import upload_file
 from mongosync_plot_metadata import plotMetrics, gatherMetrics, gatherPartitionsMetrics, gatherEndpointMetrics
+from migration_verifier import plotVerifierMetrics, gatherVerifierMetrics
 from pymongo.errors import InvalidURI, PyMongoError
 from pymongo.uri_parser import parse_uri 
 from app_config import (
@@ -99,9 +100,19 @@ def home_page():
     else:
         progress_endpoint_form = f"<p><b>Mongosync Progress Endpoint: </b>{PROGRESS_ENDPOINT_URL}</p>"
 
+    # Migration verifier connection string form
+    if not CONNECTION_STRING:
+        verifier_connection_string_form = '''<label for="verifierConnectionString">Verifier MongoDB Connection String:</label>  
+                                    <input type="text" id="verifierConnectionString" name="verifierConnectionString" size="47" autocomplete="off"
+                                        placeholder="mongodb+srv://usr:pwd@cluster0.mongodb.net/"><br><br>'''
+    else:
+        sanitized_connection = sanitize_for_display(CONNECTION_STRING)
+        verifier_connection_string_form = f"<p><b>Connecting to Verifier DB at: </b>{sanitized_connection}</p>"
+
     return render_template('home.html', 
                            connection_string_form=connection_string_form,
                            progress_endpoint_form=progress_endpoint_form,
+                           verifier_connection_string_form=verifier_connection_string_form,
                            max_file_size_gb=max_file_size_gb)
 
 
@@ -244,6 +255,91 @@ def getEndpointData():
 
     return gatherEndpointMetrics(endpoint_url)
 
+@app.route('/renderVerifier', methods=['POST'])
+def renderVerifier():
+    """Render the migration verifier monitoring page."""
+    # Get connection string from env var or form
+    if CONNECTION_STRING:
+        TARGET_MONGO_URI = CONNECTION_STRING
+    else:
+        TARGET_MONGO_URI = request.form.get('verifierConnectionString')
+        if TARGET_MONGO_URI:
+            TARGET_MONGO_URI = TARGET_MONGO_URI.strip() if TARGET_MONGO_URI.strip() else None
+
+    # Get database name from form (default: migration_verification_metadata)
+    db_name = request.form.get('verifierDbName', 'migration_verification_metadata')
+    if db_name:
+        db_name = db_name.strip() if db_name.strip() else 'migration_verification_metadata'
+
+    if not TARGET_MONGO_URI:
+        logger.error("No connection string provided for migration verifier")
+        return render_template('error.html',
+                             error_title="No Connection String",
+                             error_message="Please provide a MongoDB Connection String for the migration verifier database.")
+
+    # Test MongoDB connection
+    try:
+        validate_connection(TARGET_MONGO_URI)
+    except InvalidURI as e:
+        logger.error(f"Invalid connection string format: {e}")
+        return render_template('error.html',
+                            error_title="Invalid Connection String",
+                            error_message="The connection string format is invalid. Please check your MongoDB connection string and try again.")
+    except PyMongoError as e:
+        logger.error(f"Failed to connect: {e}")
+        return render_template('error.html',
+                            error_title="Connection Failed",
+                            error_message="Could not connect to MongoDB. Please verify your credentials, network connectivity, and that the cluster is accessible.")
+    except Exception as e:
+        logger.error(f"Unexpected error during connection validation: {e}")
+        return render_template('error.html',
+                            error_title="Connection Error",
+                            error_message="An unexpected error occurred. Please try again.")
+
+    # Store credentials in server-side in-memory session store
+    session_data = {
+        'verifier_connection_string': TARGET_MONGO_URI,
+        'verifier_db_name': db_name
+    }
+    session_id = session_store.create_session(session_data)
+
+    # Render the verifier metrics page
+    response = make_response(plotVerifierMetrics(db_name=db_name))
+
+    # Set session ID in a secure cookie
+    response.set_cookie(
+        SESSION_COOKIE_NAME,
+        session_id,
+        httponly=True,
+        secure=SECURE_COOKIES,
+        samesite='Strict',
+        max_age=SESSION_TIMEOUT
+    )
+
+    return response
+
+@app.route('/get_verifier_data', methods=['POST'])
+def getVerifierData():
+    """Get migration verifier metrics data for AJAX refresh."""
+    # Get connection string from env var or in-memory session store
+    if CONNECTION_STRING:
+        connection_string = CONNECTION_STRING
+    else:
+        session_id = request.cookies.get(SESSION_COOKIE_NAME)
+        session_data = session_store.get_session(session_id)
+        connection_string = session_data.get('verifier_connection_string')
+
+    if not connection_string:
+        logger.error("No connection string available for verifier metrics refresh")
+        return {"error": "No connection string available. Please refresh the page and re-enter your credentials."}, 400
+
+    # Get database name from session
+    session_id = request.cookies.get(SESSION_COOKIE_NAME)
+    session_data = session_store.get_session(session_id)
+    db_name = session_data.get('verifier_db_name', 'migration_verification_metadata')
+
+    return gatherVerifierMetrics(connection_string, db_name)
+
 if __name__ == '__main__':
     # Log startup information
     app_info = get_app_info()

migration/mongosync_insights/templates/CONFIGURATION.md

@@ -0,0 +1,269 @@
+# Configuration Management Guide
+
+This document explains the configuration management system for Mongosync Insights using environment variables.
+
+## Prerequisites
+
+**Python 3.11+** and **libmagic** system library are required to run Mongosync Insights. See [README.md](README.md) for complete installation instructions including system dependencies.
+
+## Configuration Overview
+
+Mongosync Insights is configured entirely through **environment variables**. No configuration files are used.
+
+### **Configuration Priority**
+
+1. **Environment Variables** (highest priority)
+2. **Default Values** (lowest priority)
+
+All configuration can be set using `export` commands before running the application, or through your system's environment configuration.
+
+## Environment Variables Reference
+
+### Server Configuration
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `MI_HOST` | `127.0.0.1` | Server host address (use `0.0.0.0` for all interfaces) |
+| `MI_PORT` | `3030` | Server port number |
+| `LOG_LEVEL` | `INFO` | Logging level (DEBUG, INFO, WARNING, ERROR) |
+| `MI_LOG_FILE` | `insights.log` | Path to log file |
+
+### MongoDB Connection
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `MI_CONNECTION_STRING` | _(empty)_ | MongoDB connection string (optional, can be provided via UI) |
+| `MI_INTERNAL_DB_NAME` | `mongosync_reserved_for_internal_use` | MongoDB internal database name |
+| `MI_POOL_SIZE` | `10` | MongoDB connection pool size |
+| `MI_TIMEOUT_MS` | `5000` | MongoDB connection timeout in milliseconds |
+
+### Live Monitoring Settings
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `MI_REFRESH_TIME` | `10` | Live monitoring refresh interval in seconds |
+| `MI_PROGRESS_ENDPOINT_URL` | _(empty)_ | Mongosync progress endpoint URL (optional, can be provided via UI) |
+
+### File Upload Settings
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `MI_MAX_FILE_SIZE` | `10737418240` | Max upload file size in bytes (10GB) |
+
+### UI Customization
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `MI_MAX_PARTITIONS_DISPLAY` | `10` | Maximum partitions to display in UI |
+| `MI_PLOT_WIDTH` | `1450` | Plot width in pixels |
+| `MI_PLOT_HEIGHT` | `1800` | Plot height in pixels |
+
+### Security Settings
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `MI_SECURE_COOKIES` | `true` | Enable secure cookies (requires HTTPS) |
+| `MI_SESSION_TIMEOUT` | `3600` | Session timeout in seconds (1 hour default) |
+| `MI_SSL_ENABLED` | `false` | Enable HTTPS/SSL in Flask application |
+| `MI_SSL_CERT` | `/etc/letsencrypt/live/your-domain/fullchain.pem` | Path to SSL certificate file |
+| `MI_SSL_KEY` | `/etc/letsencrypt/live/your-domain/privkey.pem` | Path to SSL private key file |
+
+> **Note**: For detailed HTTPS setup instructions, see [HTTPS_SETUP.md](HTTPS_SETUP.md)
+>
+
+### Connection String Validation
+
+> **Note**: For connection string handling information, see [VALIDATION.md](VALIDATION.md)
+
+---
+
+## 🚀 Usage Examples
+
+### Example 1: Basic Local Development
+
+Default settings - no environment variables needed:
+
+```bash
+# Run with all defaults
+python3 mongosync_insights.py
+
+# Access at: http://127.0.0.1:3030
+```
+
+### Example 2: Custom Port and Host
+
+Run on a different port and allow external connections:
+
+```bash
+# Set custom port and host
+export MI_PORT=8080
+export MI_HOST=0.0.0.0
+
+# Run the application
+python3 mongosync_insights.py
+
+# Access at: http://your-ip:8080
+```
+
+### Example 3: Pre-configured MongoDB Connection
+
+Set the MongoDB connection string to avoid entering it in the UI:
+
+```bash
+# Set connection string
+export MI_CONNECTION_STRING="mongodb+srv://user:pass@cluster.mongodb.net/"
+
+# Optional: Adjust refresh rate
+export MI_REFRESH_TIME=5
+
+# Run the application
+python3 mongosync_insights.py
+```
+
+### Example 3b: Combined Monitoring (Metadata + Progress Endpoint)
+
+Pre-configure both the connection string and progress endpoint for comprehensive monitoring:
+
+```bash
+# Set MongoDB connection string for metadata access
+export MI_CONNECTION_STRING="mongodb+srv://user:pass@cluster.mongodb.net/"
+
+# Set mongosync progress endpoint URL
+export MI_PROGRESS_ENDPOINT_URL="localhost:27182/api/v1/progress"
+
+# Faster refresh for active migrations
+export MI_REFRESH_TIME=5
+
+# Run the application
+python3 mongosync_insights.py
+```
+
+### Example 4: Debug Mode with Logging
+
+Enable detailed logging for troubleshooting:
+
+```bash
+# Enable debug logging
+export LOG_LEVEL=DEBUG
+export MI_LOG_FILE=/var/log/mongosync-insights/debug.log
+
+# Run the application
+python3 mongosync_insights.py
+
+# Tail the log in another terminal
+tail -f /var/log/mongosync-insights/debug.log
+```
+
+### Example 5: Production Configuration with HTTPS
+
+Secure production setup with HTTPS:
+
+```bash
+# Server configuration
+export MI_HOST=127.0.0.1  # Behind reverse proxy
+export MI_PORT=3030
+export LOG_LEVEL=INFO
+
+# Security settings
+export MI_SSL_ENABLED=false  # Nginx handles SSL
+export MI_SECURE_COOKIES=true
+
+# MongoDB connection
+export MI_CONNECTION_STRING="mongodb+srv://user:pass@production-cluster.mongodb.net/"
+
+# Performance settings
+export MI_POOL_SIZE=20
+export MI_TIMEOUT_MS=10000
+
+# Run the application
+python3 mongosync_insights.py
+```
+
+See [HTTPS_SETUP.md](HTTPS_SETUP.md) for complete production deployment guide.
+
+### Example 6: Custom Upload Size and UI Settings
+
+Adjust file upload limits and plot dimensions:
+
+```bash
+# Allow larger log files (20GB)
+export MI_MAX_FILE_SIZE=21474836480
+
+# Customize plot dimensions
+export MI_PLOT_WIDTH=1920
+export MI_PLOT_HEIGHT=2400
+export MI_MAX_PARTITIONS_DISPLAY=20
+
+# Run the application
+python3 mongosync_insights.py
+```
+
+---
+
+## Troubleshooting
+
+### Environment Variables Not Taking Effect
+
+**Problem**: Changed environment variables but application uses defaults
+
+**Solution**: 
+```bash
+# Verify variables are set
+env | grep MI_
+
+# Use -E flag with sudo to preserve environment
+sudo -E python3 mongosync_insights.py
+```
+
+### Connection String Not Working
+
+**Problem**: `MI_CONNECTION_STRING` is set but application still asks for it
+
+**Solution**: 
+- Verify the connection string format: `mongodb+srv://user:pass@cluster.mongodb.net/`
+- Check for extra quotes or spaces
+- Test connection string with `mongosh` first
+
+### Log File Permission Denied
+
+**Problem**: Cannot write to log file location
+
+**Solution**:
+```bash
+# Use writable location
+export MI_LOG_FILE=$HOME/mongosync-insights.log
+
+# Or create directory with proper permissions
+sudo mkdir -p /var/log/mongosync-insights
+sudo chown $USER:$USER /var/log/mongosync-insights
+export MI_LOG_FILE=/var/log/mongosync-insights/insights.log
+```
+
+---
+
+## Related Documentation
+
+- **[README.md](README.md)** - Getting started and installation guide
+- **[HTTPS_SETUP.md](HTTPS_SETUP.md)** - Enable HTTPS/SSL for secure deployments
+
+### License
+
+[Apache 2.0](http://www.apache.org/licenses/LICENSE-2.0)
+
+DISCLAIMER
+----------
+Please note: all tools/ scripts in this repo are released for use "AS IS" **without any warranties of any kind**,
+including, but not limited to their installation, use, or performance.  We disclaim any and all warranties, either 
+express or implied, including but not limited to any warranty of noninfringement, merchantability, and/ or fitness 
+for a particular purpose.  We do not warrant that the technology will meet your requirements, that the operation 
+thereof will be uninterrupted or error-free, or that any errors will be corrected.
+
+Any use of these scripts and tools is **at your own risk**.  There is no guarantee that they have been through 
+thorough testing in a comparable environment and we are not responsible for any damage or data loss incurred with 
+their use.
+
+You are responsible for reviewing and testing any scripts you run *thoroughly* before use in any non-testing 
+environment.
+
+Thanks,  
+The MongoDB Support Team