diff --git a/dashboard_viewer/dashboard_viewer/routers.py b/dashboard_viewer/dashboard_viewer/routers.py index 278d65ec..1a82e7c0 100644 --- a/dashboard_viewer/dashboard_viewer/routers.py +++ b/dashboard_viewer/dashboard_viewer/routers.py @@ -9,7 +9,7 @@ class AchillesRouter: achilles database. The rest will be stored on the default database """ - achilles_apps = ["uploader", "materialized_queries_manager"] + achilles_apps = ["uploader", "materialized_queries_manager", "updates"] achilles_db = "achilles" def db_for_read(self, model, **_): diff --git a/dashboard_viewer/dashboard_viewer/settings.py b/dashboard_viewer/dashboard_viewer/settings.py index a76a0ac7..acf67c4c 100644 --- a/dashboard_viewer/dashboard_viewer/settings.py +++ b/dashboard_viewer/dashboard_viewer/settings.py @@ -79,6 +79,7 @@ "martor", "rest_framework", "sass_processor", + "updates", "materialized_queries_manager", "tabsManager", "uploader", diff --git a/dashboard_viewer/docker-init.sh b/dashboard_viewer/docker-init.sh index 824d8cb4..f92b2f98 100755 --- a/dashboard_viewer/docker-init.sh +++ b/dashboard_viewer/docker-init.sh @@ -7,7 +7,7 @@ wait-for-it "$POSTGRES_ACHILLES_HOST:$POSTGRES_ACHILLES_PORT" # Apply django migrations python manage.py migrate -python manage.py migrate --database=achilles uploader +python manage.py migrate --database=achilles python manage.py populate_countries # Create an user for the admin app diff --git a/dashboard_viewer/requirements.txt b/dashboard_viewer/requirements.txt index 52910ea2..75838eb8 100644 --- a/dashboard_viewer/requirements.txt +++ b/dashboard_viewer/requirements.txt @@ -10,6 +10,7 @@ django-redis==4.12.1 # acess redis through a programmatic A django-sass-processor==0.8.2 # automate scss devolopment django==2.2.17 djangorestframework==3.12.2 # expose tabs content through an API +Jinja2==2.11.3 # to render custom code to publish upload updates libsass==0.20.1 # to compile scss files into css gunicorn==20.0.4 # for production deployment martor==1.5.8 # markdown editor in admin app @@ -36,6 +37,7 @@ django-appconf==1.0.4 idna==2.10 kombu==5.0.2 Markdown==3.3.3 +MarkupSafe==1.1.1 numpy==1.20.0 packaging==20.9 prompt-toolkit==3.0.14 diff --git a/dashboard_viewer/updates/__init__.py b/dashboard_viewer/updates/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dashboard_viewer/updates/admin.py b/dashboard_viewer/updates/admin.py new file mode 100644 index 00000000..e833012d --- /dev/null +++ b/dashboard_viewer/updates/admin.py @@ -0,0 +1,41 @@ +from django import forms +from django.contrib import admin + +from . import models + + +@admin.register(models.RequestsGroupLog) +class RequestGroupLogAdmin(admin.ModelAdmin): + list_display = ("group", "trigger_upload", "success_count", "time") + + def has_add_permission(self, request): + return False + + def has_change_permission(self, request, obj=None): + return False + + +@admin.register(models.RequestLog) +class RequestLogAdmin(admin.ModelAdmin): + list_display = ("group", "request", "success") + + def has_add_permission(self, request): + return False + + def has_change_permission(self, request, obj=None): + return False + + +admin.site.register(models.RequestsGroup) + + +class RequestAdminForm(forms.ModelForm): + class Meta: + model = models.Request + fields = "__all__" + + +@admin.register(models.Request) +class RequestAdmin(admin.ModelAdmin): + list_display = ("group", "order") + form = RequestAdminForm diff --git a/dashboard_viewer/updates/apps.py b/dashboard_viewer/updates/apps.py new file mode 100644 index 00000000..6cc446ce --- /dev/null +++ b/dashboard_viewer/updates/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class UpdatesConfig(AppConfig): + name = "updates" diff --git a/dashboard_viewer/updates/fixtures/achilles_results_dist.csv b/dashboard_viewer/updates/fixtures/achilles_results_dist.csv new file mode 100644 index 00000000..c032dcee --- /dev/null +++ b/dashboard_viewer/updates/fixtures/achilles_results_dist.csv @@ -0,0 +1,4 @@ +analysis_id,stratum_1,stratum_2,stratum_3,stratum_4,stratum_5,count_value,min_value,max_value,avg_value,stdev_value,median_value,p10_value,p25_value,p75_value,p90_value +0,test,NULL,NULL,NULL,NULL,1171,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL +2000000,0.151157 secs,NULL,NULL,NULL,NULL,6,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL +103,NULL,NULL,NULL,NULL,NULL,1171,0,61,14.1289496157131,12.4272078334908,14,0,2,21,32 diff --git a/dashboard_viewer/updates/fixtures/active.json b/dashboard_viewer/updates/fixtures/active.json new file mode 100644 index 00000000..cad0868e --- /dev/null +++ b/dashboard_viewer/updates/fixtures/active.json @@ -0,0 +1,28 @@ +[ +{ + "model": "updates.requestsgroup", + "pk": 1, + "fields": { + "name": "test", + "active": true + } +}, +{ + "model": "updates.request", + "pk": 1, + "fields": { + "group": 1, + "request_arguments_template": "{\"url\": \"http://localhost:8383\",\"method\": \"get\",\"data\": {\"patient_count\": \"{{ achilles_results.0.0.count_value }}\"}}", + "order": 1 + } +}, +{ + "model": "updates.request", + "pk": 2, + "fields": { + "group": 1, + "request_arguments_template": "{\"url\": \"https://stackoverflow.com?response={{ responses|last|attr(\"text\") }}\",\"method\": \"GET\"}", + "order": 2 + } +} +] diff --git a/dashboard_viewer/updates/fixtures/base.json b/dashboard_viewer/updates/fixtures/base.json new file mode 100644 index 00000000..c2f2699e --- /dev/null +++ b/dashboard_viewer/updates/fixtures/base.json @@ -0,0 +1,24 @@ +[ +{ + "model": "uploader.country", + "pk": 1, + "fields": { + "country": "test", + "continent": "test" + } +}, +{ + "model": "uploader.datasource", + "pk": 1, + "fields": { + "name": "test", + "acronym": "test", + "release_date": null, + "database_type": "test", + "country": 1, + "latitude": 35.0524837066247, + "longitude": -103.095703125, + "link": "" + } +} +] diff --git a/dashboard_viewer/updates/fixtures/data.json b/dashboard_viewer/updates/fixtures/data.json new file mode 100644 index 00000000..da8ecf2c --- /dev/null +++ b/dashboard_viewer/updates/fixtures/data.json @@ -0,0 +1,84 @@ +[ + { + "model": "uploader.uploadhistory", + "pk": 1, + "fields": { + "data_source": 1, + "upload_date": "2021-05-10T14:46:11.609Z", + "r_package_version": "nan", + "generation_date": "nan", + "cdm_release_date": null, + "cdm_version": null, + "vocabulary_version": null + } + }, + { + "model": "uploader.achillesresults", + "pk": 1, + "fields": { + "data_source": 1, + "analysis_id": 0, + "stratum_1": "test", + "stratum_2": null, + "stratum_3": null, + "stratum_4": null, + "stratum_5": null, + "count_value": 1171, + "min_value": null, + "max_value": null, + "avg_value": null, + "stdev_value": null, + "median_value": null, + "p10_value": null, + "p25_value": null, + "p75_value": null, + "p90_value": null + } + }, + { + "model": "uploader.achillesresults", + "pk": 2, + "fields": { + "data_source": 1, + "analysis_id": 2000000, + "stratum_1": "0.151157 secs", + "stratum_2": null, + "stratum_3": null, + "stratum_4": null, + "stratum_5": null, + "count_value": 6, + "min_value": null, + "max_value": null, + "avg_value": null, + "stdev_value": null, + "median_value": null, + "p10_value": null, + "p25_value": null, + "p75_value": null, + "p90_value": null + } + }, + { + "model": "uploader.achillesresults", + "pk": 3, + "fields": { + "data_source": 1, + "analysis_id": 103, + "stratum_1": null, + "stratum_2": null, + "stratum_3": null, + "stratum_4": null, + "stratum_5": null, + "count_value": 1171, + "min_value": 0, + "max_value": 61, + "avg_value": 14.1289496157, + "stdev_value": 12.4272078335, + "median_value": 14, + "p10_value": 0, + "p25_value": 2, + "p75_value": 21, + "p90_value": 32 + } + } +] \ No newline at end of file diff --git a/dashboard_viewer/updates/fixtures/not_active.json b/dashboard_viewer/updates/fixtures/not_active.json new file mode 100644 index 00000000..8ae3b50d --- /dev/null +++ b/dashboard_viewer/updates/fixtures/not_active.json @@ -0,0 +1,28 @@ +[ +{ + "model": "updates.requestsgroup", + "pk": 1, + "fields": { + "name": "test", + "active": false + } +}, +{ + "model": "updates.request", + "pk": 1, + "fields": { + "group": 1, + "request_arguments_template": "{\"url\": \"http://localhost:8383\",\"method\": \"get\",\"data\": {\"patient_count\": \"{{ achilles_results.0.0.count_value }}\"}}", + "order": 1 + } +}, +{ + "model": "updates.request", + "pk": 2, + "fields": { + "group": 1, + "request_arguments_template": "{\"url\": \"https://stackoverflow.com?response={{ responses|last|attr(\"text\") }}\",\"method\": \"GET\"}", + "order": 2 + } +} +] diff --git a/dashboard_viewer/updates/migrations/0001_initial.py b/dashboard_viewer/updates/migrations/0001_initial.py new file mode 100644 index 00000000..95b3e048 --- /dev/null +++ b/dashboard_viewer/updates/migrations/0001_initial.py @@ -0,0 +1,143 @@ +# Generated by Django 2.2.17 on 2021-05-10 17:29 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ("uploader", "0009_auto_20210112_1719"), + ] + + operations = [ + migrations.CreateModel( + name="Request", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "request_arguments_template", + models.TextField( + blank=True, + help_text='Jinja2 template that must render valid JSON. This JSON will then be converted to a python dict which will be used to define the arguments of the Session.request method. Note that both the method and url arguments are required. To build the template you have access to the uploaded records, grouped by analysis id (achilles_results) (ex: {1: [{"stratum_1":10, ...}, {{"stratum_1":11, ...}}], ...}), the django model object of the target Data Source (data_source) and the responses to the previous requests of the same group (responses). An example of a valid template to send the number of patients of a datasource: {"url": "http://server.com" , "method": "post", "data": {"patient_count": "{{ achilles_results.0.0.count_value }}"}}', + ), + ), + ( + "success_condition_template", + models.TextField( + blank=True, + help_text='A Jinja2 template that must render a boolean expression that indicates that this request was successful and the next requests of the associated request group can be performed. On this template, you have access to the returned requests.Response object (response). Ex: {{ response.status_code }} == 200', + null=True, + ), + ), + ("order", models.IntegerField()), + ], + options={ + "ordering": ("order",), + }, + ), + migrations.CreateModel( + name="RequestsGroup", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("name", models.CharField(max_length=255)), + ( + "active", + models.BooleanField( + default=True, + help_text="Allows to deactivate this group of requests from being processed without having to delete associated records", + ), + ), + ], + ), + migrations.CreateModel( + name="RequestsGroupLog", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("success_count", models.IntegerField(default=0)), + ("time", models.DateTimeField(auto_now_add=True)), + ( + "group", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="updates.RequestsGroup", + ), + ), + ( + "trigger_upload", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="uploader.UploadHistory", + ), + ), + ], + ), + migrations.CreateModel( + name="RequestLog", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("request_arguments_template_render", models.TextField(null=True)), + ("success_condition_template_render", models.TextField(null=True)), + ("success", models.BooleanField(default=True)), + ("exception", models.TextField(null=True)), + ( + "group", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="requests", + to="updates.RequestsGroupLog", + ), + ), + ( + "request", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="updates.Request", + ), + ), + ], + ), + migrations.AddField( + model_name="request", + name="group", + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="requests", + to="updates.RequestsGroup", + ), + ), + ] diff --git a/dashboard_viewer/updates/migrations/__init__.py b/dashboard_viewer/updates/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dashboard_viewer/updates/models.py b/dashboard_viewer/updates/models.py new file mode 100644 index 00000000..2fcfeaba --- /dev/null +++ b/dashboard_viewer/updates/models.py @@ -0,0 +1,80 @@ +from django.db import models +from uploader.models import UploadHistory + + +class RequestsGroup(models.Model): + name = models.CharField(max_length=255) + active = models.BooleanField( + default=True, + help_text="Allows to deactivate this group of requests from being " + "processed without having to delete associated records", + ) + + def __repr__(self): + return self.__str__() + + def __str__(self): + return self.name + + +class Request(models.Model): + group = models.ForeignKey( + RequestsGroup, on_delete=models.CASCADE, related_name="requests" + ) + request_arguments_template = models.TextField( + blank=True, + help_text="Jinja2 template that must render valid JSON. This JSON will then be converted to " + 'a python dict which will be used to define the arguments of the Session' + ".request method. Note that both the method and url arguments are " + "required. To build the template you have access to the uploaded records, grouped by " + 'analysis id (achilles_results) (ex: {1: [{"stratum_1":10, ...}, {{"stratum_1":' + "11, ...}}], ...}), the django model object of the target Data Source (data_source)" + " and the responses to the previous requests of the same group (responses). An" + ' example of a valid template to send the number of patients of a datasource: {"url":' + ' "http://server.com" , "method": "post", "data": {"patient_count": "{{ achilles_results' + '.0.0.count_value }}"}}', + ) + success_condition_template = models.TextField( + blank=True, + null=True, + help_text="A Jinja2 template that must render a boolean expression that indicates that this " + "request was successful and the next requests of the associated request group can " + 'be performed. On this template, you have access to the returned requests' + ".Response object (response). Ex: {{ response.status_code }} == 200", + ) + order = models.IntegerField() + + class Meta: + ordering = ("order",) + + def __repr__(self): + return self.__str__() + + def __str__(self): + return f"Group: {self.group} - Order: {self.order}" + + +class RequestsGroupLog(models.Model): + group = models.ForeignKey(RequestsGroup, on_delete=models.CASCADE) + trigger_upload = models.ForeignKey(UploadHistory, on_delete=models.CASCADE) + success_count = models.IntegerField(default=0) + time = models.DateTimeField(auto_now_add=True) + + def __repr__(self): + return self.__str__() + + def __str__(self): + return f"Group: {self.group.name} - Data Source: {self.trigger_upload.data_source.acronym} - Requests Time: {self.time}" + + +class RequestLog(models.Model): + group = models.ForeignKey( + RequestsGroupLog, on_delete=models.CASCADE, related_name="requests" + ) + request = models.ForeignKey(Request, on_delete=models.CASCADE) + request_arguments_template_render = models.TextField(null=True) + success_condition_template_render = models.TextField(null=True) + success = models.BooleanField(default=True) + exception = models.TextField(null=True) diff --git a/dashboard_viewer/updates/tasks.py b/dashboard_viewer/updates/tasks.py new file mode 100644 index 00000000..cf0d28b2 --- /dev/null +++ b/dashboard_viewer/updates/tasks.py @@ -0,0 +1,76 @@ +import json +import traceback + +import pandas +import requests +from celery import shared_task +from jinja2 import Template +from uploader.models import DataSource, UploadHistory + +from .models import RequestLog, RequestsGroup, RequestsGroupLog + + +@shared_task +def send_updates(db_id: int, upload_history_id: int, achilles_results: str): + data_source = DataSource.objects.get(id=db_id) + + responses = [] + context = { + "achilles_results": { + analysis_id: rows[rows.columns[1:]].to_dict("records") + for analysis_id, rows in pandas.read_json(achilles_results).groupby( + "analysis_id" + ) + }, + "data_source": data_source, + "responses": responses, + } + + trigger_upload = UploadHistory.objects.get(id=upload_history_id) + + for group in RequestsGroup.objects.filter(active=True): + group_log = RequestsGroupLog.objects.create( + group=group, trigger_upload=trigger_upload + ) + + responses.clear() + + with requests.Session() as session: + for request in group.requests.all(): + request_log = RequestLog(group=group_log, request=request) + + try: + template = Template(request.request_arguments_template) # noqa + + render_result = template.render(**context) # noqa + request_log.request_arguments_template_render = render_result + + response = session.request(**json.loads(render_result)) + + if request.success_condition_template is not None: + template = Template(request.success_condition_template) # noqa + render_result = template.render(response=response) # noqa + + request_log.success_condition_template_render = render_result + + if not bool( + eval( # noqa - we trust the admins + request_log.success_condition_template_render + ) + ): + raise AssertionError("Success condition not met") + + group_log.success_count += 1 + except: # noqa - we are logging them + group_log.success = False + + request_log.success = False + request_log.exception = traceback.format_exc() + + break + finally: + request_log.save() + + responses.append(response) + + group_log.save() diff --git a/dashboard_viewer/updates/tests.py b/dashboard_viewer/updates/tests.py new file mode 100644 index 00000000..d9866c0c --- /dev/null +++ b/dashboard_viewer/updates/tests.py @@ -0,0 +1,210 @@ +import json +import os +from unittest.mock import patch + +from django.test import Client, TestCase +from uploader import models as uploader_models + +from . import models +from .tasks import send_updates + + +class NoActiveRequestGroupsTestCase(TestCase): + databases = "__all__" + + fixtures = ("base", "not_active") + + @classmethod + def setUpClass(cls): + super(NoActiveRequestGroupsTestCase, cls).setUpClass() + cls.client = Client() + + def _upload_and_check(self): + with open( + f"{os.path.dirname(__file__)}/fixtures/achilles_results_dist.csv" + ) as fp: + self.client.post("/uploader/test/", {"results_file": fp}, follow=True) + + self.assertEqual(0, models.RequestLog.objects.count()) + self.assertEqual(0, models.RequestsGroupLog.objects.count()) + self.assertEqual(1, uploader_models.UploadHistory.objects.count()) + + @patch("uploader.tasks.update_achilles_results_data.delay") + def test_exist_but_not_active(self, _): + self._upload_and_check() + + @patch("uploader.tasks.update_achilles_results_data.delay") + def test_none(self, _): + models.RequestsGroup.objects.all().delete() + + self._upload_and_check() + + +class BaseInitialDataTestCase(TestCase): + databases = "__all__" + + fixtures = ("base", "active", "data") + + @staticmethod + def _generate_json_data(): + data = uploader_models.AchillesResults.objects.all() + key_to_exclude = ("_state", "id", "data_source_id") + data = [ + {k: v for k, v in record.__dict__.items() if k not in key_to_exclude} + for record in data + ] + + return json.dumps(data) + + +class LogsTestCase(BaseInitialDataTestCase): + def _execute_and_check(self, error_message_portion, render_empty=False): + task = send_updates.delay(1, 1, self._generate_json_data()) + task.wait(timeout=None) + + self.assertEqual(1, models.RequestsGroupLog.objects.count()) + group_log = models.RequestsGroupLog.objects.get() + self.assertEqual(0, group_log.success_count) + + self.assertEqual(1, models.RequestLog.objects.count()) + failed_request_log = models.RequestLog.objects.get() + self.assertFalse(failed_request_log.success) + if render_empty: + self.assertIsNone(failed_request_log.request_arguments_template_render) + else: + self.assertIsNotNone(failed_request_log.request_arguments_template_render) + self.assertIsNotNone(failed_request_log.exception) + self.assertIn(error_message_portion, failed_request_log.exception) + + def test_invalid_json_render(self): + first_request = models.Request.objects.get(order=1) + first_request.request_arguments_template = ( + first_request.request_arguments_template[1:] + ) + first_request.save() + + self._execute_and_check("JSONDecodeError") + + def test_missing_required_parameters(self): + first_request = models.Request.objects.get(order=1) + first_request.request_arguments_template = '{"method": "get","data": {"patient_count": "{{ achilles_results.0.0.count_value }}"}}' + first_request.save() + + self._execute_and_check("missing 1 required positional argument") + + @patch("requests.Session.request") + def test_invalid_condition_expression(self, _): + first_request = models.Request.objects.get(order=1) + first_request.success_condition_template = "def function():" + first_request.save() + + self._execute_and_check("SyntaxError: invalid syntax") + + def test_invalid_template(self): + first_request = models.Request.objects.get(order=1) + first_request.request_arguments_template = "{{ asdf }" + first_request.save() + + self._execute_and_check("TemplateSyntaxError", render_empty=True) + + +class ContextsAvailableTestCase(BaseInitialDataTestCase): + @patch("requests.Session.request") + def test_previous_answers_available(self, session_request): + session_request.return_value.text = "LAST_RESPONSE_CONTENT" + + task = send_updates.delay(1, 1, self._generate_json_data()) + task.wait(timeout=None) + + self.assertEqual(1, models.RequestsGroupLog.objects.count()) + group_log = models.RequestsGroupLog.objects.get() + self.assertEqual(2, group_log.success_count) + requests_logs = group_log.requests + self.assertEqual(2, requests_logs.count()) + + first_request_log = requests_logs.get(request__order=1) + self.assertTrue(first_request_log.success) + self.assertIsNone(first_request_log.exception) + self.assertIsNotNone(first_request_log.request_arguments_template_render) + + second_request_log = requests_logs.get(request__order=2) + self.assertTrue(second_request_log.success) + self.assertIsNone(second_request_log.exception) + self.assertIsNotNone(second_request_log.request_arguments_template_render) + self.assertIn( + "LAST_RESPONSE_CONTENT", + second_request_log.request_arguments_template_render, + ) + + @patch("requests.Session.request") + def test_response_available(self, session_request): + session_request.return_value.status_code = 400 + + first_request = models.Request.objects.get(order=1) + first_request.success_condition_template = "{{ response.status_code }} == 200" + first_request.save() + + task = send_updates.delay(1, 1, self._generate_json_data()) + task.wait(timeout=None) + + self.assertEqual(1, models.RequestsGroupLog.objects.count()) + group_log = models.RequestsGroupLog.objects.get() + self.assertEqual(0, group_log.success_count) + requests_logs = group_log.requests + self.assertEqual(1, requests_logs.count()) + + first_request_log = requests_logs.get(request__order=1) + self.assertIsNotNone(first_request_log.success_condition_template_render) + self.assertIn("400 == 200", first_request_log.success_condition_template_render) + + +class SuccessConditionTestSuit(BaseInitialDataTestCase): + @patch("requests.Session.request") + def test_stop_on_failed_success_condition(self, session_request): + session_request.return_value.status_code = 400 + + first_request = models.Request.objects.get(order=1) + first_request.success_condition_template = "{{ response.status_code }} == 200" + first_request.save() + + task = send_updates.delay(1, 1, self._generate_json_data()) + task.wait(timeout=None) + + self.assertEqual(1, models.RequestsGroupLog.objects.count()) + group_log = models.RequestsGroupLog.objects.get() + self.assertEqual(0, group_log.success_count) + requests_logs = group_log.requests + self.assertEqual(1, requests_logs.count()) + + request_log = requests_logs.get(request__order=1) + self.assertFalse(request_log.success) + self.assertIsNotNone(request_log.request_arguments_template_render) + self.assertIsNotNone(request_log.success_condition_template_render) + self.assertIsNotNone(request_log.exception) + self.assertIn("Success condition not met", request_log.exception) + + @patch("requests.Session.request") + def test_exec_all_if_good_condition(self, session_request): + session_request.return_value.status_code = 200 + + first_request = models.Request.objects.get(order=1) + first_request.success_condition_template = "{{ response.status_code }} == 200" + first_request.save() + + task = send_updates.delay(1, 1, self._generate_json_data()) + task.wait(timeout=None) + + self.assertEqual(1, models.RequestsGroupLog.objects.count()) + group_log = models.RequestsGroupLog.objects.get() + self.assertEqual(2, group_log.success_count) + requests_logs = group_log.requests + self.assertEqual(2, requests_logs.count()) + + for i, request_log in enumerate(requests_logs.all()): + self.assertTrue(request_log.success) + self.assertIsNotNone(request_log.request_arguments_template_render) + if i == 0: + self.assertIsNotNone(request_log.success_condition_template_render) + else: + self.assertIsNone(request_log.success_condition_template_render) + self.assertIsNone(request_log.exception) diff --git a/dashboard_viewer/uploader/models.py b/dashboard_viewer/uploader/models.py index 7de5888b..4091ee4f 100644 --- a/dashboard_viewer/uploader/models.py +++ b/dashboard_viewer/uploader/models.py @@ -95,7 +95,7 @@ def __repr__(self): return self.__str__() def __str__(self): - return f"{self.data_source.name} - {self.upload_date}" + return f"Data Source: {self.data_source.name} - Upload Date: {self.upload_date}" class AchillesResults(models.Model): diff --git a/dashboard_viewer/uploader/views.py b/dashboard_viewer/uploader/views.py index 37fc75a4..b607cfcf 100644 --- a/dashboard_viewer/uploader/views.py +++ b/dashboard_viewer/uploader/views.py @@ -12,6 +12,8 @@ from django.shortcuts import redirect, render from django.utils.html import format_html, mark_safe from django.views.decorators.csrf import csrf_exempt +from updates.models import RequestsGroup +from updates.tasks import send_updates from .forms import AchillesResultsForm, SourceForm from .models import Country, DataSource, UploadHistory @@ -232,11 +234,13 @@ def upload_achilles_results(request, *args, **kwargs): data = _extract_data_from_uploaded_file(request) if data: + achilles_results_json = data["achilles_results"].to_json() + # launch an asynchronous task to insert the new data update_achilles_results_data.delay( obj_data_source.id, upload_history[0].id if len(upload_history) > 0 else None, - data["achilles_results"].to_json(), + achilles_results_json, ) obj_data_source.release_date = data["source_release_date"] @@ -254,6 +258,11 @@ def upload_achilles_results(request, *args, **kwargs): latest_upload.save() upload_history = [latest_upload] + upload_history + if RequestsGroup.objects.filter(active=True).count() > 0: + send_updates.delay( + obj_data_source.id, latest_upload.id, achilles_results_json + ) + # save the achilles result file to disk data_source_storage_path = os.path.join( settings.BASE_DIR,