From 2435af3bf993ffe220ae05dcbf537cd9eaf5956b Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Wed, 21 Jan 2026 14:46:32 -0500 Subject: [PATCH 1/5] Added default retention policy --- application/single_app/config.py | 2 +- .../single_app/functions_retention_policy.py | 83 +++++++- application/single_app/functions_settings.py | 9 + application/single_app/requirements.txt | 4 +- .../route_backend_retention_policy.py | 69 +++++++ .../route_frontend_admin_settings.py | 14 ++ .../single_app/static/js/workspace-manager.js | 71 +++++++ .../single_app/templates/admin_settings.html | 150 +++++++++++++++ .../single_app/templates/control_center.html | 112 +++++++++-- application/single_app/templates/profile.html | 66 +++++-- .../CONTROL_CENTER_APPLICATION_ROLES.md | 154 +++++++++++++++ .../v0.236.008/PRIVATE_NETWORKING_SUPPORT.md | 179 ++++++++++++++++++ ...EIGN_CLOUD_COGNITIVE_SERVICES_SCOPE_FIX.md | 94 +++++++++ .../USER_SEARCH_TOAST_INLINE_MESSAGES_FIX.md | 68 +++++++ docs/explanation/release_notes.md | 40 ++++ 15 files changed, 1071 insertions(+), 44 deletions(-) create mode 100644 docs/explanation/features/v0.236.008/CONTROL_CENTER_APPLICATION_ROLES.md create mode 100644 docs/explanation/features/v0.236.008/PRIVATE_NETWORKING_SUPPORT.md create mode 100644 docs/explanation/fixes/v0.236.008/SOVEREIGN_CLOUD_COGNITIVE_SERVICES_SCOPE_FIX.md create mode 100644 docs/explanation/fixes/v0.236.008/USER_SEARCH_TOAST_INLINE_MESSAGES_FIX.md diff --git a/application/single_app/config.py b/application/single_app/config.py index 03f34724..f2cc8297 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -88,7 +88,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.236.007" +VERSION = "0.236.009" SECRET_KEY = os.getenv('SECRET_KEY', 'dev-secret-key-change-in-production') diff --git a/application/single_app/functions_retention_policy.py b/application/single_app/functions_retention_policy.py index 6c59ef64..6ce6dee0 100644 --- a/application/single_app/functions_retention_policy.py +++ b/application/single_app/functions_retention_policy.py @@ -82,6 +82,47 @@ def get_all_public_workspaces(): return [] +def resolve_retention_value(value, workspace_type, retention_type, settings=None): + """ + Resolve a retention value, handling 'default' by looking up organization defaults. + + Args: + value: The retention value ('none', 'default', or a number/string of days) + workspace_type: 'personal', 'group', or 'public' + retention_type: 'conversation' or 'document' + settings: Optional pre-loaded settings dict (to avoid repeated lookups) + + Returns: + str or int: 'none' if no deletion, or the number of days as int + """ + if value is None or value == 'default' or value == '': + # Look up the organization default + if settings is None: + settings = get_settings() + + setting_key = f'default_retention_{retention_type}_{workspace_type}' + default_value = settings.get(setting_key, 'none') + + # If the org default is also 'none', return 'none' + if default_value == 'none' or default_value is None: + return 'none' + + # Return the org default as the effective value + try: + return int(default_value) + except (ValueError, TypeError): + return 'none' + + # User/workspace has their own explicit value + if value == 'none': + return 'none' + + try: + return int(value) + except (ValueError, TypeError): + return 'none' + + def execute_retention_policy(workspace_scopes=None, manual_execution=False): """ Execute retention policy for specified workspace scopes. @@ -185,6 +226,9 @@ def process_personal_retention(): # Get all user settings all_users = get_all_user_settings() + # Pre-load settings once for efficiency + settings = get_settings() + for user in all_users: user_id = user.get('id') if not user_id: @@ -194,10 +238,15 @@ def process_personal_retention(): user_settings = user.get('settings', {}) retention_settings = user_settings.get('retention_policy', {}) - conversation_retention_days = retention_settings.get('conversation_retention_days', 'none') - document_retention_days = retention_settings.get('document_retention_days', 'none') + # Get raw values (may be 'default', 'none', or a number) + raw_conversation_days = retention_settings.get('conversation_retention_days') + raw_document_days = retention_settings.get('document_retention_days') - # Skip if both are set to "none" + # Resolve to effective values (handles 'default' -> org default lookup) + conversation_retention_days = resolve_retention_value(raw_conversation_days, 'personal', 'conversation', settings) + document_retention_days = resolve_retention_value(raw_document_days, 'personal', 'document', settings) + + # Skip if both resolve to "none" if conversation_retention_days == 'none' and document_retention_days == 'none': continue @@ -273,6 +322,9 @@ def process_group_retention(): # Get all groups all_groups = get_all_groups() + # Pre-load settings once for efficiency + settings = get_settings() + for group in all_groups: group_id = group.get('id') if not group_id: @@ -281,10 +333,15 @@ def process_group_retention(): # Get group's retention settings retention_settings = group.get('retention_policy', {}) - conversation_retention_days = retention_settings.get('conversation_retention_days', 'none') - document_retention_days = retention_settings.get('document_retention_days', 'none') + # Get raw values (may be 'default', 'none', or a number) + raw_conversation_days = retention_settings.get('conversation_retention_days') + raw_document_days = retention_settings.get('document_retention_days') + + # Resolve to effective values (handles 'default' -> org default lookup) + conversation_retention_days = resolve_retention_value(raw_conversation_days, 'group', 'conversation', settings) + document_retention_days = resolve_retention_value(raw_document_days, 'group', 'document', settings) - # Skip if both are set to "none" + # Skip if both resolve to "none" if conversation_retention_days == 'none' and document_retention_days == 'none': continue @@ -359,6 +416,9 @@ def process_public_retention(): # Get all public workspaces all_workspaces = get_all_public_workspaces() + # Pre-load settings once for efficiency + settings = get_settings() + for workspace in all_workspaces: workspace_id = workspace.get('id') if not workspace_id: @@ -367,10 +427,15 @@ def process_public_retention(): # Get workspace's retention settings retention_settings = workspace.get('retention_policy', {}) - conversation_retention_days = retention_settings.get('conversation_retention_days', 'none') - document_retention_days = retention_settings.get('document_retention_days', 'none') + # Get raw values (may be 'default', 'none', or a number) + raw_conversation_days = retention_settings.get('conversation_retention_days') + raw_document_days = retention_settings.get('document_retention_days') + + # Resolve to effective values (handles 'default' -> org default lookup) + conversation_retention_days = resolve_retention_value(raw_conversation_days, 'public', 'conversation', settings) + document_retention_days = resolve_retention_value(raw_document_days, 'public', 'document', settings) - # Skip if both are set to "none" + # Skip if both resolve to "none" if conversation_retention_days == 'none' and document_retention_days == 'none': continue diff --git a/application/single_app/functions_settings.py b/application/single_app/functions_settings.py index a0575f54..cbf10b4f 100644 --- a/application/single_app/functions_settings.py +++ b/application/single_app/functions_settings.py @@ -276,6 +276,15 @@ def get_settings(use_cosmos=False): 'retention_conversation_max_days': 3650, # ~10 years 'retention_document_min_days': 1, 'retention_document_max_days': 3650, # ~10 years + # Default retention policies for each workspace type + # 'none' means no automatic deletion (users can still set their own) + # Numeric values (e.g., 30, 60, 90, 180, 365, 730) represent days + 'default_retention_conversation_personal': 'none', + 'default_retention_document_personal': 'none', + 'default_retention_conversation_group': 'none', + 'default_retention_document_group': 'none', + 'default_retention_conversation_public': 'none', + 'default_retention_document_public': 'none', } try: diff --git a/application/single_app/requirements.txt b/application/single_app/requirements.txt index 187b4a98..8aefdb06 100644 --- a/application/single_app/requirements.txt +++ b/application/single_app/requirements.txt @@ -4,7 +4,7 @@ azure-monitor-query==1.4.1 Flask==2.2.5 Flask-WTF==1.2.1 gunicorn -Werkzeug==3.1.4 +Werkzeug==3.1.5 requests==2.32.4 openai==1.67 docx2txt==0.8 @@ -49,7 +49,7 @@ azure-monitor-opentelemetry==1.6.13 psycopg2-binary==2.9.10 cython pyyaml==6.0.2 -aiohttp==3.12.15 +aiohttp==3.13.3 html2text==2025.4.15 matplotlib==3.10.7 azure-cognitiveservices-speech==1.47.0 \ No newline at end of file diff --git a/application/single_app/route_backend_retention_policy.py b/application/single_app/route_backend_retention_policy.py index 70d5cc76..99a4ae4e 100644 --- a/application/single_app/route_backend_retention_policy.py +++ b/application/single_app/route_backend_retention_policy.py @@ -106,6 +106,75 @@ def update_retention_policy_settings(): }), 500 + @app.route('/api/retention-policy/defaults/', methods=['GET']) + @swagger_route(security=get_auth_security()) + @login_required + @user_required + def get_retention_policy_defaults(workspace_type): + """ + Get organization default retention policy settings for a specific workspace type. + + Args: + workspace_type: One of 'personal', 'group', or 'public' + + Returns: + JSON with default_conversation_days and default_document_days for the workspace type + """ + try: + # Validate workspace type + if workspace_type not in ['personal', 'group', 'public']: + return jsonify({ + 'success': False, + 'error': f'Invalid workspace type: {workspace_type}' + }), 400 + + settings = get_settings() + + # Get the default values for the specified workspace type + default_conversation = settings.get(f'default_retention_conversation_{workspace_type}', 'none') + default_document = settings.get(f'default_retention_document_{workspace_type}', 'none') + + # Get human-readable labels for the values + def get_retention_label(value): + if value == 'none' or value is None: + return 'No automatic deletion' + try: + days = int(value) + if days == 1: + return '1 day' + elif days == 21: + return '21 days (3 weeks)' + elif days == 90: + return '90 days (3 months)' + elif days == 180: + return '180 days (6 months)' + elif days == 365: + return '365 days (1 year)' + elif days == 730: + return '730 days (2 years)' + else: + return f'{days} days' + except (ValueError, TypeError): + return 'No automatic deletion' + + return jsonify({ + 'success': True, + 'workspace_type': workspace_type, + 'default_conversation_days': default_conversation, + 'default_document_days': default_document, + 'default_conversation_label': get_retention_label(default_conversation), + 'default_document_label': get_retention_label(default_document) + }) + + except Exception as e: + debug_print(f"Error fetching retention policy defaults: {e}") + log_event(f"Fetching retention policy defaults failed: {e}", level=logging.ERROR) + return jsonify({ + 'success': False, + 'error': 'Failed to fetch retention policy defaults' + }), 500 + + @app.route('/api/admin/retention-policy/execute', methods=['POST']) @swagger_route(security=get_auth_security()) @login_required diff --git a/application/single_app/route_frontend_admin_settings.py b/application/single_app/route_frontend_admin_settings.py index 66c6a45a..411805cc 100644 --- a/application/single_app/route_frontend_admin_settings.py +++ b/application/single_app/route_frontend_admin_settings.py @@ -537,6 +537,14 @@ def admin_settings(): enable_retention_policy_public = form_data.get('enable_retention_policy_public') == 'on' retention_policy_execution_hour = int(form_data.get('retention_policy_execution_hour', 2)) + # Default retention policy values for each workspace type + default_retention_conversation_personal = form_data.get('default_retention_conversation_personal', 'none') + default_retention_document_personal = form_data.get('default_retention_document_personal', 'none') + default_retention_conversation_group = form_data.get('default_retention_conversation_group', 'none') + default_retention_document_group = form_data.get('default_retention_document_group', 'none') + default_retention_conversation_public = form_data.get('default_retention_conversation_public', 'none') + default_retention_document_public = form_data.get('default_retention_document_public', 'none') + # Validate execution hour (0-23) if retention_policy_execution_hour < 0 or retention_policy_execution_hour > 23: retention_policy_execution_hour = 2 # Default to 2 AM @@ -696,6 +704,12 @@ def is_valid_url(url): 'enable_retention_policy_public': enable_retention_policy_public, 'retention_policy_execution_hour': retention_policy_execution_hour, 'retention_policy_next_run': retention_policy_next_run, + 'default_retention_conversation_personal': default_retention_conversation_personal, + 'default_retention_document_personal': default_retention_document_personal, + 'default_retention_conversation_group': default_retention_conversation_group, + 'default_retention_document_group': default_retention_document_group, + 'default_retention_conversation_public': default_retention_conversation_public, + 'default_retention_document_public': default_retention_document_public, # User Agreement 'enable_user_agreement': enable_user_agreement, diff --git a/application/single_app/static/js/workspace-manager.js b/application/single_app/static/js/workspace-manager.js index d109351f..7b540af7 100644 --- a/application/single_app/static/js/workspace-manager.js +++ b/application/single_app/static/js/workspace-manager.js @@ -411,6 +411,9 @@ window.WorkspaceManager = { // Load workspace members for ownership transfer dropdown await WorkspaceManager.loadWorkspaceMembersForTransfer(workspaceId); + // Load retention settings if enabled + await WorkspaceManager.loadRetentionSettings(workspace); + // Store current workspace ID for saving changes document.getElementById('publicWorkspaceManagementModal').setAttribute('data-workspace-id', workspaceId); @@ -485,6 +488,54 @@ window.WorkspaceManager = { } }, + // Load retention settings for public workspace + loadRetentionSettings: async function(workspace) { + const convSelect = document.getElementById('publicConversationRetention'); + const docSelect = document.getElementById('publicDocumentRetention'); + + // Check if retention policy elements exist (feature might be disabled) + if (!convSelect || !docSelect) { + return; + } + + try { + // Fetch organization defaults for public workspace retention + const orgDefaultsResp = await fetch('/api/retention-policy/defaults/public'); + const orgData = await orgDefaultsResp.json(); + + if (orgData.success) { + const convDefaultOption = convSelect.querySelector('option[value="default"]'); + const docDefaultOption = docSelect.querySelector('option[value="default"]'); + + if (convDefaultOption) { + convDefaultOption.textContent = `Using organization default (${orgData.default_conversation_label})`; + } + if (docDefaultOption) { + docDefaultOption.textContent = `Using organization default (${orgData.default_document_label})`; + } + } + } catch (error) { + console.error('Error loading public retention defaults:', error); + } + + // Set current values from workspace + if (workspace.retention_policy) { + let convRetention = workspace.retention_policy.conversation_retention_days; + let docRetention = workspace.retention_policy.document_retention_days; + + // If undefined, use 'default' + if (convRetention === undefined) convRetention = 'default'; + if (docRetention === undefined) docRetention = 'default'; + + convSelect.value = convRetention; + docSelect.value = docRetention; + } else { + // Set to organization default if no retention policy set + convSelect.value = 'default'; + docSelect.value = 'default'; + } + }, + // Save workspace changes saveWorkspaceChanges: async function() { const workspaceId = document.getElementById('publicWorkspaceManagementModal').getAttribute('data-workspace-id'); @@ -556,6 +607,26 @@ window.WorkspaceManager = { } } + // Save retention policy settings if enabled + const convRetentionSelect = document.getElementById('publicConversationRetention'); + const docRetentionSelect = document.getElementById('publicDocumentRetention'); + + if (convRetentionSelect && docRetentionSelect) { + const retentionResponse = await fetch(`/api/retention-policy/public/${workspaceId}`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + conversation_retention_days: convRetentionSelect.value, + document_retention_days: docRetentionSelect.value + }) + }); + + if (!retentionResponse.ok) { + console.error('Failed to save retention policy'); + // Don't throw - allow other changes to succeed + } + } + showToast('Workspace updated successfully!', 'success'); // Close modal and refresh table diff --git a/application/single_app/templates/admin_settings.html b/application/single_app/templates/admin_settings.html index e153568d..5517956c 100644 --- a/application/single_app/templates/admin_settings.html +++ b/application/single_app/templates/admin_settings.html @@ -2289,6 +2289,129 @@
+ +
+
Default Retention Policies
+

Set organization-wide default retention periods for each workspace type. Users can override these defaults with their own preferences. Setting a default here means new users/workspaces will start with this retention period.

+ + +
+
+ +
+
+ + +
+
+ + +
+
+ + +
+
+ +
+
+ + +
+
+ + +
+
+ + +
+
+ +
+
+ + +
+
+ + +
+
+
+
@@ -4046,6 +4169,33 @@
Retention Policy
- Configure automatic deletion of aged conversations and documents. Set to "No automatic deletion" to keep items indefinitely. + Configure automatic deletion of aged conversations and documents. You can use the organization default or set a custom retention period.
+ + + + + @@ -2259,6 +2269,59 @@
Member Management
+ + {% if app_settings.enable_retention_policy_public %} +
+
+
Retention Policy
+
+
+
+ + Configure automatic deletion of aged conversations and documents. You can use the organization default or set a custom retention period. +
+
+
+ + +
+
+ + +
+
+
+
+
+
+ {% endif %} +
@@ -3186,22 +3249,43 @@
await GroupManager.loadGroupMembersForTransfer(groupId); // Load retention settings if enabled - if (group.retention_policy) { - const convSelect = document.getElementById('groupConversationRetention'); - const docSelect = document.getElementById('groupDocumentRetention'); + const convSelect = document.getElementById('groupConversationRetention'); + const docSelect = document.getElementById('groupDocumentRetention'); + + // Fetch organization defaults for group retention + try { + const orgDefaultsResp = await fetch('/api/retention-policy/defaults/group'); + const orgData = await orgDefaultsResp.json(); - if (convSelect) { - convSelect.value = group.retention_policy.conversation_retention_days || 'none'; - } - if (docSelect) { - docSelect.value = group.retention_policy.document_retention_days || 'none'; + if (orgData.success) { + const convDefaultOption = convSelect?.querySelector('option[value="default"]'); + const docDefaultOption = docSelect?.querySelector('option[value="default"]'); + + if (convDefaultOption) { + convDefaultOption.textContent = `Using organization default (${orgData.default_conversation_label})`; + } + if (docDefaultOption) { + docDefaultOption.textContent = `Using organization default (${orgData.default_document_label})`; + } } + } catch (error) { + console.error('Error loading group retention defaults:', error); + } + + if (group.retention_policy) { + let convRetention = group.retention_policy.conversation_retention_days; + let docRetention = group.retention_policy.document_retention_days; + + // If undefined, use 'default' + if (convRetention === undefined) convRetention = 'default'; + if (docRetention === undefined) docRetention = 'default'; + + if (convSelect) convSelect.value = convRetention; + if (docSelect) docSelect.value = docRetention; } else { - // Set defaults if no retention policy - const convSelect = document.getElementById('groupConversationRetention'); - const docSelect = document.getElementById('groupDocumentRetention'); - if (convSelect) convSelect.value = 'none'; - if (docSelect) docSelect.value = 'none'; + // Set to organization default if no retention policy set + if (convSelect) convSelect.value = 'default'; + if (docSelect) docSelect.value = 'default'; } // Store current group ID for saving changes diff --git a/application/single_app/templates/profile.html b/application/single_app/templates/profile.html index d444cf3b..e5a62887 100644 --- a/application/single_app/templates/profile.html +++ b/application/single_app/templates/profile.html @@ -308,7 +308,7 @@
Retention Policy Sett
- Default: Set to "No automatic deletion" means items are never automatically deleted. Choose a retention period to enable automatic cleanup. + Default: You can use the organization default or set your own retention period. Choose "No automatic deletion" to keep items indefinitely.
@@ -316,6 +316,7 @@
Retention Policy Sett
+ @@ -1055,24 +1057,52 @@
+ + +
+
+
+ + Force Push: Override all user/group/workspace custom retention policies with the organization defaults above. Users will then use the organization default until they set their own preference. +
+ +
+
@@ -3685,6 +3698,71 @@
Execution Results:
+ + + {% include '_video_indexer_info.html' %} @@ -4196,6 +4274,150 @@