diff --git a/application/single_app/config.py b/application/single_app/config.py index 03f34724..8e668c0c 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -88,7 +88,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.236.007" +VERSION = "0.236.011" SECRET_KEY = os.getenv('SECRET_KEY', 'dev-secret-key-change-in-production') diff --git a/application/single_app/functions_activity_logging.py b/application/single_app/functions_activity_logging.py index 9870eca9..89b52f44 100644 --- a/application/single_app/functions_activity_logging.py +++ b/application/single_app/functions_activity_logging.py @@ -1215,3 +1215,75 @@ def has_user_accepted_agreement_today( ) debug_print(f"⚠️ Error checking user agreement acceptance: {str(e)}") return False + + +def log_retention_policy_force_push( + admin_user_id: str, + admin_email: str, + scopes: list, + results: dict, + total_updated: int +) -> None: + """ + Log retention policy force push action to activity_logs container. + + This creates a permanent audit record when an admin forces organization + default retention policies to be applied to all workspaces. + + Args: + admin_user_id (str): User ID of the admin performing the force push + admin_email (str): Email of the admin performing the force push + scopes (list): List of workspace types affected (e.g., ['personal', 'group', 'public']) + results (dict): Breakdown of updates per workspace type + total_updated (int): Total number of workspaces/users updated + """ + + try: + # Create force push activity record + force_push_activity = { + 'id': str(uuid.uuid4()), + 'user_id': admin_user_id, # Partition key + 'activity_type': 'retention_policy_force_push', + 'timestamp': datetime.utcnow().isoformat(), + 'created_at': datetime.utcnow().isoformat(), + 'admin': { + 'user_id': admin_user_id, + 'email': admin_email + }, + 'force_push_details': { + 'scopes': scopes, + 'results': results, + 'total_updated': total_updated, + 'executed_at': datetime.utcnow().isoformat() + }, + 'workspace_type': 'admin', + 'workspace_context': { + 'action': 'retention_policy_force_push' + } + } + + # Save to activity_logs container for permanent audit trail + cosmos_activity_logs_container.create_item(body=force_push_activity) + + # Also log to Application Insights for monitoring + log_event( + message=f"Retention policy force push executed by {admin_email} for scopes: {', '.join(scopes)}. Total updated: {total_updated}", + extra=force_push_activity, + level=logging.INFO + ) + + debug_print(f"✅ Retention policy force push logged: {scopes} by {admin_email}, updated {total_updated}") + + except Exception as e: + # Log error but don't break the force push flow + log_event( + message=f"Error logging retention policy force push: {str(e)}", + extra={ + 'admin_user_id': admin_user_id, + 'scopes': scopes, + 'total_updated': total_updated, + 'error': str(e) + }, + level=logging.ERROR + ) + debug_print(f"⚠️ Warning: Failed to log retention policy force push: {str(e)}") diff --git a/application/single_app/functions_retention_policy.py b/application/single_app/functions_retention_policy.py index 6c59ef64..6ce6dee0 100644 --- a/application/single_app/functions_retention_policy.py +++ b/application/single_app/functions_retention_policy.py @@ -82,6 +82,47 @@ def get_all_public_workspaces(): return [] +def resolve_retention_value(value, workspace_type, retention_type, settings=None): + """ + Resolve a retention value, handling 'default' by looking up organization defaults. + + Args: + value: The retention value ('none', 'default', or a number/string of days) + workspace_type: 'personal', 'group', or 'public' + retention_type: 'conversation' or 'document' + settings: Optional pre-loaded settings dict (to avoid repeated lookups) + + Returns: + str or int: 'none' if no deletion, or the number of days as int + """ + if value is None or value == 'default' or value == '': + # Look up the organization default + if settings is None: + settings = get_settings() + + setting_key = f'default_retention_{retention_type}_{workspace_type}' + default_value = settings.get(setting_key, 'none') + + # If the org default is also 'none', return 'none' + if default_value == 'none' or default_value is None: + return 'none' + + # Return the org default as the effective value + try: + return int(default_value) + except (ValueError, TypeError): + return 'none' + + # User/workspace has their own explicit value + if value == 'none': + return 'none' + + try: + return int(value) + except (ValueError, TypeError): + return 'none' + + def execute_retention_policy(workspace_scopes=None, manual_execution=False): """ Execute retention policy for specified workspace scopes. @@ -185,6 +226,9 @@ def process_personal_retention(): # Get all user settings all_users = get_all_user_settings() + # Pre-load settings once for efficiency + settings = get_settings() + for user in all_users: user_id = user.get('id') if not user_id: @@ -194,10 +238,15 @@ def process_personal_retention(): user_settings = user.get('settings', {}) retention_settings = user_settings.get('retention_policy', {}) - conversation_retention_days = retention_settings.get('conversation_retention_days', 'none') - document_retention_days = retention_settings.get('document_retention_days', 'none') + # Get raw values (may be 'default', 'none', or a number) + raw_conversation_days = retention_settings.get('conversation_retention_days') + raw_document_days = retention_settings.get('document_retention_days') - # Skip if both are set to "none" + # Resolve to effective values (handles 'default' -> org default lookup) + conversation_retention_days = resolve_retention_value(raw_conversation_days, 'personal', 'conversation', settings) + document_retention_days = resolve_retention_value(raw_document_days, 'personal', 'document', settings) + + # Skip if both resolve to "none" if conversation_retention_days == 'none' and document_retention_days == 'none': continue @@ -273,6 +322,9 @@ def process_group_retention(): # Get all groups all_groups = get_all_groups() + # Pre-load settings once for efficiency + settings = get_settings() + for group in all_groups: group_id = group.get('id') if not group_id: @@ -281,10 +333,15 @@ def process_group_retention(): # Get group's retention settings retention_settings = group.get('retention_policy', {}) - conversation_retention_days = retention_settings.get('conversation_retention_days', 'none') - document_retention_days = retention_settings.get('document_retention_days', 'none') + # Get raw values (may be 'default', 'none', or a number) + raw_conversation_days = retention_settings.get('conversation_retention_days') + raw_document_days = retention_settings.get('document_retention_days') + + # Resolve to effective values (handles 'default' -> org default lookup) + conversation_retention_days = resolve_retention_value(raw_conversation_days, 'group', 'conversation', settings) + document_retention_days = resolve_retention_value(raw_document_days, 'group', 'document', settings) - # Skip if both are set to "none" + # Skip if both resolve to "none" if conversation_retention_days == 'none' and document_retention_days == 'none': continue @@ -359,6 +416,9 @@ def process_public_retention(): # Get all public workspaces all_workspaces = get_all_public_workspaces() + # Pre-load settings once for efficiency + settings = get_settings() + for workspace in all_workspaces: workspace_id = workspace.get('id') if not workspace_id: @@ -367,10 +427,15 @@ def process_public_retention(): # Get workspace's retention settings retention_settings = workspace.get('retention_policy', {}) - conversation_retention_days = retention_settings.get('conversation_retention_days', 'none') - document_retention_days = retention_settings.get('document_retention_days', 'none') + # Get raw values (may be 'default', 'none', or a number) + raw_conversation_days = retention_settings.get('conversation_retention_days') + raw_document_days = retention_settings.get('document_retention_days') + + # Resolve to effective values (handles 'default' -> org default lookup) + conversation_retention_days = resolve_retention_value(raw_conversation_days, 'public', 'conversation', settings) + document_retention_days = resolve_retention_value(raw_document_days, 'public', 'document', settings) - # Skip if both are set to "none" + # Skip if both resolve to "none" if conversation_retention_days == 'none' and document_retention_days == 'none': continue diff --git a/application/single_app/functions_settings.py b/application/single_app/functions_settings.py index a0575f54..cbf10b4f 100644 --- a/application/single_app/functions_settings.py +++ b/application/single_app/functions_settings.py @@ -276,6 +276,15 @@ def get_settings(use_cosmos=False): 'retention_conversation_max_days': 3650, # ~10 years 'retention_document_min_days': 1, 'retention_document_max_days': 3650, # ~10 years + # Default retention policies for each workspace type + # 'none' means no automatic deletion (users can still set their own) + # Numeric values (e.g., 30, 60, 90, 180, 365, 730) represent days + 'default_retention_conversation_personal': 'none', + 'default_retention_document_personal': 'none', + 'default_retention_conversation_group': 'none', + 'default_retention_document_group': 'none', + 'default_retention_conversation_public': 'none', + 'default_retention_document_public': 'none', } try: diff --git a/application/single_app/requirements.txt b/application/single_app/requirements.txt index 187b4a98..8aefdb06 100644 --- a/application/single_app/requirements.txt +++ b/application/single_app/requirements.txt @@ -4,7 +4,7 @@ azure-monitor-query==1.4.1 Flask==2.2.5 Flask-WTF==1.2.1 gunicorn -Werkzeug==3.1.4 +Werkzeug==3.1.5 requests==2.32.4 openai==1.67 docx2txt==0.8 @@ -49,7 +49,7 @@ azure-monitor-opentelemetry==1.6.13 psycopg2-binary==2.9.10 cython pyyaml==6.0.2 -aiohttp==3.12.15 +aiohttp==3.13.3 html2text==2025.4.15 matplotlib==3.10.7 azure-cognitiveservices-speech==1.47.0 \ No newline at end of file diff --git a/application/single_app/route_backend_retention_policy.py b/application/single_app/route_backend_retention_policy.py index 70d5cc76..60935f60 100644 --- a/application/single_app/route_backend_retention_policy.py +++ b/application/single_app/route_backend_retention_policy.py @@ -3,7 +3,8 @@ from config import * from functions_authentication import * from functions_settings import * -from functions_retention_policy import execute_retention_policy +from functions_retention_policy import execute_retention_policy, get_all_user_settings, get_all_groups, get_all_public_workspaces +from functions_activity_logging import log_retention_policy_force_push from swagger_wrapper import swagger_route, get_auth_security from functions_debug import debug_print @@ -106,6 +107,75 @@ def update_retention_policy_settings(): }), 500 + @app.route('/api/retention-policy/defaults/', methods=['GET']) + @swagger_route(security=get_auth_security()) + @login_required + @user_required + def get_retention_policy_defaults(workspace_type): + """ + Get organization default retention policy settings for a specific workspace type. + + Args: + workspace_type: One of 'personal', 'group', or 'public' + + Returns: + JSON with default_conversation_days and default_document_days for the workspace type + """ + try: + # Validate workspace type + if workspace_type not in ['personal', 'group', 'public']: + return jsonify({ + 'success': False, + 'error': f'Invalid workspace type: {workspace_type}' + }), 400 + + settings = get_settings() + + # Get the default values for the specified workspace type + default_conversation = settings.get(f'default_retention_conversation_{workspace_type}', 'none') + default_document = settings.get(f'default_retention_document_{workspace_type}', 'none') + + # Get human-readable labels for the values + def get_retention_label(value): + if value == 'none' or value is None: + return 'No automatic deletion' + try: + days = int(value) + if days == 1: + return '1 day' + elif days == 21: + return '21 days (3 weeks)' + elif days == 90: + return '90 days (3 months)' + elif days == 180: + return '180 days (6 months)' + elif days == 365: + return '365 days (1 year)' + elif days == 730: + return '730 days (2 years)' + else: + return f'{days} days' + except (ValueError, TypeError): + return 'No automatic deletion' + + return jsonify({ + 'success': True, + 'workspace_type': workspace_type, + 'default_conversation_days': default_conversation, + 'default_document_days': default_document, + 'default_conversation_label': get_retention_label(default_conversation), + 'default_document_label': get_retention_label(default_document) + }) + + except Exception as e: + debug_print(f"Error fetching retention policy defaults: {e}") + log_event(f"Fetching retention policy defaults failed: {e}", level=logging.ERROR) + return jsonify({ + 'success': False, + 'error': 'Failed to fetch retention policy defaults' + }), 500 + + @app.route('/api/admin/retention-policy/execute', methods=['POST']) @swagger_route(security=get_auth_security()) @login_required @@ -155,6 +225,165 @@ def manual_execute_retention_policy(): }), 500 + @app.route('/api/admin/retention-policy/force-push', methods=['POST']) + @swagger_route(security=get_auth_security()) + @login_required + @admin_required + def force_push_retention_defaults(): + """ + Force push organization default retention policies to all users/groups/workspaces. + This resets all custom retention policies to use the organization default ('default' value). + + Body: + scopes (list): List of workspace types to push defaults to: 'personal', 'group', 'public' + """ + try: + data = request.get_json() + scopes = data.get('scopes', []) + + if not scopes: + return jsonify({ + 'success': False, + 'error': 'No workspace scopes provided' + }), 400 + + # Validate scopes + valid_scopes = ['personal', 'group', 'public'] + invalid_scopes = [s for s in scopes if s not in valid_scopes] + if invalid_scopes: + return jsonify({ + 'success': False, + 'error': f'Invalid workspace scopes: {", ".join(invalid_scopes)}' + }), 400 + + details = {} + total_updated = 0 + + # Force push to personal workspaces (user settings) + if 'personal' in scopes: + debug_print("Force pushing retention defaults to personal workspaces...") + all_users = get_all_user_settings() + personal_count = 0 + + for user in all_users: + user_id = user.get('id') + if not user_id: + continue + + try: + # Update user's retention policy to use 'default' + user_settings = user.get('settings', {}) + user_settings['retention_policy'] = { + 'conversation_retention_days': 'default', + 'document_retention_days': 'default' + } + user['settings'] = user_settings + + cosmos_user_settings_container.upsert_item(user) + personal_count += 1 + except Exception as e: + debug_print(f"Error updating user {user_id}: {e}") + log_event(f"Error updating user {user_id} during force push: {e}", level=logging.ERROR) + continue + + details['personal'] = personal_count + total_updated += personal_count + debug_print(f"Updated {personal_count} personal workspaces") + + # Force push to group workspaces + if 'group' in scopes: + debug_print("Force pushing retention defaults to group workspaces...") + from functions_group import cosmos_groups_container + all_groups = get_all_groups() + group_count = 0 + + for group in all_groups: + group_id = group.get('id') + if not group_id: + continue + + try: + # Update group's retention policy to use 'default' + group['retention_policy'] = { + 'conversation_retention_days': 'default', + 'document_retention_days': 'default' + } + + cosmos_groups_container.upsert_item(group) + group_count += 1 + except Exception as e: + debug_print(f"Error updating group {group_id}: {e}") + log_event(f"Error updating group {group_id} during force push: {e}", level=logging.ERROR) + continue + + details['group'] = group_count + total_updated += group_count + debug_print(f"Updated {group_count} group workspaces") + + # Force push to public workspaces + if 'public' in scopes: + debug_print("Force pushing retention defaults to public workspaces...") + from functions_public_workspaces import cosmos_public_workspaces_container + all_workspaces = get_all_public_workspaces() + public_count = 0 + + for workspace in all_workspaces: + workspace_id = workspace.get('id') + if not workspace_id: + continue + + try: + # Update workspace's retention policy to use 'default' + workspace['retention_policy'] = { + 'conversation_retention_days': 'default', + 'document_retention_days': 'default' + } + + cosmos_public_workspaces_container.upsert_item(workspace) + public_count += 1 + except Exception as e: + debug_print(f"Error updating public workspace {workspace_id}: {e}") + log_event(f"Error updating public workspace {workspace_id} during force push: {e}", level=logging.ERROR) + continue + + details['public'] = public_count + total_updated += public_count + debug_print(f"Updated {public_count} public workspaces") + + # Log to activity logs for audit trail + admin_user_id = session.get('user', {}).get('oid', 'unknown') + admin_email = session.get('user', {}).get('preferred_username', session.get('user', {}).get('email', 'unknown')) + log_retention_policy_force_push( + admin_user_id=admin_user_id, + admin_email=admin_email, + scopes=scopes, + results=details, + total_updated=total_updated + ) + + log_event("retention_policy_force_push", { + "scopes": scopes, + "updated_count": total_updated, + "details": details + }) + + return jsonify({ + 'success': True, + 'message': f'Defaults pushed to {total_updated} items', + 'updated_count': total_updated, + 'scopes': scopes, + 'details': details + }) + + except Exception as e: + debug_print(f"Error force pushing retention defaults: {e}") + log_event(f"Force push retention defaults failed: {e}", level=logging.ERROR) + return jsonify({ + 'success': False, + 'error': f'Failed to push retention defaults' + }), 500 + + @app.route('/api/retention-policy/user', methods=['POST']) @swagger_route(security=get_auth_security()) @login_required diff --git a/application/single_app/route_frontend_admin_settings.py b/application/single_app/route_frontend_admin_settings.py index 66c6a45a..411805cc 100644 --- a/application/single_app/route_frontend_admin_settings.py +++ b/application/single_app/route_frontend_admin_settings.py @@ -537,6 +537,14 @@ def admin_settings(): enable_retention_policy_public = form_data.get('enable_retention_policy_public') == 'on' retention_policy_execution_hour = int(form_data.get('retention_policy_execution_hour', 2)) + # Default retention policy values for each workspace type + default_retention_conversation_personal = form_data.get('default_retention_conversation_personal', 'none') + default_retention_document_personal = form_data.get('default_retention_document_personal', 'none') + default_retention_conversation_group = form_data.get('default_retention_conversation_group', 'none') + default_retention_document_group = form_data.get('default_retention_document_group', 'none') + default_retention_conversation_public = form_data.get('default_retention_conversation_public', 'none') + default_retention_document_public = form_data.get('default_retention_document_public', 'none') + # Validate execution hour (0-23) if retention_policy_execution_hour < 0 or retention_policy_execution_hour > 23: retention_policy_execution_hour = 2 # Default to 2 AM @@ -696,6 +704,12 @@ def is_valid_url(url): 'enable_retention_policy_public': enable_retention_policy_public, 'retention_policy_execution_hour': retention_policy_execution_hour, 'retention_policy_next_run': retention_policy_next_run, + 'default_retention_conversation_personal': default_retention_conversation_personal, + 'default_retention_document_personal': default_retention_document_personal, + 'default_retention_conversation_group': default_retention_conversation_group, + 'default_retention_document_group': default_retention_document_group, + 'default_retention_conversation_public': default_retention_conversation_public, + 'default_retention_document_public': default_retention_document_public, # User Agreement 'enable_user_agreement': enable_user_agreement, diff --git a/application/single_app/static/js/workspace-manager.js b/application/single_app/static/js/workspace-manager.js index d109351f..7b540af7 100644 --- a/application/single_app/static/js/workspace-manager.js +++ b/application/single_app/static/js/workspace-manager.js @@ -411,6 +411,9 @@ window.WorkspaceManager = { // Load workspace members for ownership transfer dropdown await WorkspaceManager.loadWorkspaceMembersForTransfer(workspaceId); + // Load retention settings if enabled + await WorkspaceManager.loadRetentionSettings(workspace); + // Store current workspace ID for saving changes document.getElementById('publicWorkspaceManagementModal').setAttribute('data-workspace-id', workspaceId); @@ -485,6 +488,54 @@ window.WorkspaceManager = { } }, + // Load retention settings for public workspace + loadRetentionSettings: async function(workspace) { + const convSelect = document.getElementById('publicConversationRetention'); + const docSelect = document.getElementById('publicDocumentRetention'); + + // Check if retention policy elements exist (feature might be disabled) + if (!convSelect || !docSelect) { + return; + } + + try { + // Fetch organization defaults for public workspace retention + const orgDefaultsResp = await fetch('/api/retention-policy/defaults/public'); + const orgData = await orgDefaultsResp.json(); + + if (orgData.success) { + const convDefaultOption = convSelect.querySelector('option[value="default"]'); + const docDefaultOption = docSelect.querySelector('option[value="default"]'); + + if (convDefaultOption) { + convDefaultOption.textContent = `Using organization default (${orgData.default_conversation_label})`; + } + if (docDefaultOption) { + docDefaultOption.textContent = `Using organization default (${orgData.default_document_label})`; + } + } + } catch (error) { + console.error('Error loading public retention defaults:', error); + } + + // Set current values from workspace + if (workspace.retention_policy) { + let convRetention = workspace.retention_policy.conversation_retention_days; + let docRetention = workspace.retention_policy.document_retention_days; + + // If undefined, use 'default' + if (convRetention === undefined) convRetention = 'default'; + if (docRetention === undefined) docRetention = 'default'; + + convSelect.value = convRetention; + docSelect.value = docRetention; + } else { + // Set to organization default if no retention policy set + convSelect.value = 'default'; + docSelect.value = 'default'; + } + }, + // Save workspace changes saveWorkspaceChanges: async function() { const workspaceId = document.getElementById('publicWorkspaceManagementModal').getAttribute('data-workspace-id'); @@ -556,6 +607,26 @@ window.WorkspaceManager = { } } + // Save retention policy settings if enabled + const convRetentionSelect = document.getElementById('publicConversationRetention'); + const docRetentionSelect = document.getElementById('publicDocumentRetention'); + + if (convRetentionSelect && docRetentionSelect) { + const retentionResponse = await fetch(`/api/retention-policy/public/${workspaceId}`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + conversation_retention_days: convRetentionSelect.value, + document_retention_days: docRetentionSelect.value + }) + }); + + if (!retentionResponse.ok) { + console.error('Failed to save retention policy'); + // Don't throw - allow other changes to succeed + } + } + showToast('Workspace updated successfully!', 'success'); // Close modal and refresh table diff --git a/application/single_app/templates/admin_settings.html b/application/single_app/templates/admin_settings.html index e153568d..6410b4ef 100644 --- a/application/single_app/templates/admin_settings.html +++ b/application/single_app/templates/admin_settings.html @@ -2289,6 +2289,142 @@
+ +
+
Default Retention Policies
+

Set organization-wide default retention periods for each workspace type. Users can override these defaults with their own preferences. Setting a default here means new users/workspaces will start with this retention period.

+ + +
+
+ +
+
+ + +
+
+ + +
+
+ + +
+
+ +
+
+ + +
+
+ + +
+
+ + +
+
+ +
+
+ + +
+
+ + +
+
+ + +
+
+
+ + Force Push: Override all user/group/workspace custom retention policies with the organization defaults above. Users will then use the organization default until they set their own preference. +
+ +
+
+
+
@@ -3562,6 +3698,71 @@
Execution Results:
+ + + {% include '_video_indexer_info.html' %} @@ -4046,6 +4247,177 @@
Retention Policy
- Configure automatic deletion of aged conversations and documents. Set to "No automatic deletion" to keep items indefinitely. + Configure automatic deletion of aged conversations and documents. You can use the organization default or set a custom retention period.
+ + + + + @@ -2259,6 +2269,59 @@
Member Management
+ + {% if app_settings.enable_retention_policy_public %} +
+
+
Retention Policy
+
+
+
+ + Configure automatic deletion of aged conversations and documents. You can use the organization default or set a custom retention period. +
+
+
+ + +
+
+ + +
+
+
+
+
+
+ {% endif %} +
@@ -3186,22 +3249,43 @@
Retention Policy Sett
- Default: Set to "No automatic deletion" means items are never automatically deleted. Choose a retention period to enable automatic cleanup. + Default: You can use the organization default or set your own retention period. Choose "No automatic deletion" to keep items indefinitely.
@@ -316,6 +316,7 @@
Retention Policy Sett
+ @@ -1055,24 +1057,52 @@