Skip to content

Improve logging in upload, add a refresh timeout for MaterializedViews #285

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
joselfrias wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Improve logging in upload, add a refresh timeout for MaterializedViews #285

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion dashboard_viewer/dashboard_viewer/settings.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -345,6 +345,11 @@ def locks_make_key(key, key_prefix, version): # noqa
"Css for the img tag displaying the app logo",
str,
),
"REFRESH_MATERIALIZED_TIMEOUT": (
7200000,
"Refresh materialized view statement timeout in POSTGRES (in ms). Default: 2hrs",
int,
),
}

CONSTANCE_CONFIG_FIELDSETS = OrderedDict(
Expand All @@ -361,7 +366,7 @@ def locks_make_key(key, key_prefix, version): # noqa
("Uploader Settings", ("UPLOADER_ALLOW_EDIT_DRAFT_STATUS",)),
(
"Superset",
("SUPERSET_HOST", "DATABASE_DASHBOARD_IDENTIFIER", "DATABASE_FILTER_ID"),
("SUPERSET_HOST", "DATABASE_DASHBOARD_IDENTIFIER", "DATABASE_FILTER_ID", "REFRESH_MATERIALIZED_TIMEOUT"),
),
("Tabs (Deprecated)", ("TABS_LOGO_CONTAINER_CSS", "TABS_LOGO_IMG_CSS")),
]
Expand Down
5 changes: 3 additions & 2 deletions dashboard_viewer/materialized_queries_manager/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
from django.core.cache import caches
from django.db import connections
from redis_rw_lock import RWLock

import constance
from materialized_queries_manager.models import MaterializedQuery


Expand All @@ -24,6 +24,7 @@ def refresh(logger, db_id=None, query_set=None):

for i, materialized_query in enumerate(to_refresh):
try:
cursor.execute("SET statement_timeout = " + str(constance.config.REFRESH_MATERIALIZED_TIMEOUT) + ";")
logger.info(
"Refreshing materialized view %s (%d/%d) [%s]",
materialized_query.matviewname,
Expand All @@ -37,6 +38,6 @@ def refresh(logger, db_id=None, query_set=None):
except: # noqa
logger.exception(
"Some unexpected error happen while refreshing materialized query %s. [%s]",
materialized_query.name,
materialized_query.matviewname,
"command" if not db_id else f"datasource {db_id}",
)
22 changes: 21 additions & 1 deletion dashboard_viewer/uploader/tasks.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,14 +91,20 @@ def upload_results_file(pending_upload_id: int):
)

logger.info(
"Creating an upload history record [datasource %d, pending upload %d]",
"Saving data source release date [datasource %d, pending upload %d]",
data_source.id,
pending_upload_id,
)

data_source.release_date = data["source_release_date"]
data_source.save()

logger.info(
"Creating an upload history record [datasource %d, pending upload %d]",
data_source.id,
pending_upload_id,
)

pending_upload.uploaded_file.seek(0)
UploadHistory.objects.create(
data_source=data_source,
Expand All @@ -111,7 +117,21 @@ def upload_results_file(pending_upload_id: int):
pending_upload_id=pending_upload.id,
)

logger.info(
"Deleting pending upload files [datasource %d, pending upload %d, pending upload file %s]",
data_source.id,
pending_upload_id,
pending_upload.uploaded_file.file
)

pending_upload.uploaded_file.delete()

logger.info(
"Deleting pending upload [datasource %d, pending upload %d]",
data_source.id,
pending_upload_id
)

pending_upload.delete()

finally:
Expand Down