diff --git a/dashboard_viewer/dashboard_viewer/settings.py b/dashboard_viewer/dashboard_viewer/settings.py index f80c7bb..dd449cf 100644 --- a/dashboard_viewer/dashboard_viewer/settings.py +++ b/dashboard_viewer/dashboard_viewer/settings.py @@ -345,6 +345,11 @@ def locks_make_key(key, key_prefix, version): # noqa "Css for the img tag displaying the app logo", str, ), + "REFRESH_MATERIALIZED_TIMEOUT": ( + 7200000, + "Refresh materialized view statement timeout in POSTGRES (in ms). Default: 2hrs", + int, + ), } CONSTANCE_CONFIG_FIELDSETS = OrderedDict( @@ -361,7 +366,7 @@ def locks_make_key(key, key_prefix, version): # noqa ("Uploader Settings", ("UPLOADER_ALLOW_EDIT_DRAFT_STATUS",)), ( "Superset", - ("SUPERSET_HOST", "DATABASE_DASHBOARD_IDENTIFIER", "DATABASE_FILTER_ID"), + ("SUPERSET_HOST", "DATABASE_DASHBOARD_IDENTIFIER", "DATABASE_FILTER_ID", "REFRESH_MATERIALIZED_TIMEOUT"), ), ("Tabs (Deprecated)", ("TABS_LOGO_CONTAINER_CSS", "TABS_LOGO_IMG_CSS")), ] diff --git a/dashboard_viewer/materialized_queries_manager/utils.py b/dashboard_viewer/materialized_queries_manager/utils.py index 0e5c3e9..3807efc 100644 --- a/dashboard_viewer/materialized_queries_manager/utils.py +++ b/dashboard_viewer/materialized_queries_manager/utils.py @@ -1,7 +1,7 @@ from django.core.cache import caches from django.db import connections from redis_rw_lock import RWLock - +import constance from materialized_queries_manager.models import MaterializedQuery @@ -24,6 +24,7 @@ def refresh(logger, db_id=None, query_set=None): for i, materialized_query in enumerate(to_refresh): try: + cursor.execute("SET statement_timeout = " + str(constance.config.REFRESH_MATERIALIZED_TIMEOUT) + ";") logger.info( "Refreshing materialized view %s (%d/%d) [%s]", materialized_query.matviewname, @@ -37,6 +38,6 @@ def refresh(logger, db_id=None, query_set=None): except: # noqa logger.exception( "Some unexpected error happen while refreshing materialized query %s. [%s]", - materialized_query.name, + materialized_query.matviewname, "command" if not db_id else f"datasource {db_id}", ) diff --git a/dashboard_viewer/uploader/tasks.py b/dashboard_viewer/uploader/tasks.py index e54628e..2d7b930 100644 --- a/dashboard_viewer/uploader/tasks.py +++ b/dashboard_viewer/uploader/tasks.py @@ -91,7 +91,7 @@ def upload_results_file(pending_upload_id: int): ) logger.info( - "Creating an upload history record [datasource %d, pending upload %d]", + "Saving data source release date [datasource %d, pending upload %d]", data_source.id, pending_upload_id, ) @@ -99,6 +99,12 @@ def upload_results_file(pending_upload_id: int): data_source.release_date = data["source_release_date"] data_source.save() + logger.info( + "Creating an upload history record [datasource %d, pending upload %d]", + data_source.id, + pending_upload_id, + ) + pending_upload.uploaded_file.seek(0) UploadHistory.objects.create( data_source=data_source, @@ -111,7 +117,21 @@ def upload_results_file(pending_upload_id: int): pending_upload_id=pending_upload.id, ) + logger.info( + "Deleting pending upload files [datasource %d, pending upload %d, pending upload file %s]", + data_source.id, + pending_upload_id, + pending_upload.uploaded_file.file + ) + pending_upload.uploaded_file.delete() + + logger.info( + "Deleting pending upload [datasource %d, pending upload %d]", + data_source.id, + pending_upload_id + ) + pending_upload.delete() finally: