From cad8782853b71a0b139f6e84095314200827c1a4 Mon Sep 17 00:00:00 2001 From: Adam McMaster Date: Tue, 24 Jun 2025 15:24:00 +0100 Subject: [PATCH 1/5] TOM Toolkit setup --- zooniverse/zoonitom/.gitignore | 3 + zooniverse/zoonitom/manage.py | 22 ++ zooniverse/zoonitom/requirements.txt | 1 + zooniverse/zoonitom/static/.keep | 0 .../zoonitom/static/tom_common/css/custom.css | 32 ++ zooniverse/zoonitom/zoo/__init__.py | 0 zooniverse/zoonitom/zoo/admin.py | 3 + zooniverse/zoonitom/zoo/apps.py | 6 + .../zoonitom/zoo/migrations/__init__.py | 0 zooniverse/zoonitom/zoo/models.py | 18 + zooniverse/zoonitom/zoo/tests.py | 3 + zooniverse/zoonitom/zoo/views.py | 3 + zooniverse/zoonitom/zoonitom/__init__.py | 0 zooniverse/zoonitom/zoonitom/asgi.py | 16 + zooniverse/zoonitom/zoonitom/settings.py | 363 ++++++++++++++++++ zooniverse/zoonitom/zoonitom/urls.py | 20 + zooniverse/zoonitom/zoonitom/wsgi.py | 16 + 17 files changed, 506 insertions(+) create mode 100644 zooniverse/zoonitom/.gitignore create mode 100755 zooniverse/zoonitom/manage.py create mode 100644 zooniverse/zoonitom/requirements.txt create mode 100644 zooniverse/zoonitom/static/.keep create mode 100644 zooniverse/zoonitom/static/tom_common/css/custom.css create mode 100644 zooniverse/zoonitom/zoo/__init__.py create mode 100644 zooniverse/zoonitom/zoo/admin.py create mode 100644 zooniverse/zoonitom/zoo/apps.py create mode 100644 zooniverse/zoonitom/zoo/migrations/__init__.py create mode 100644 zooniverse/zoonitom/zoo/models.py create mode 100644 zooniverse/zoonitom/zoo/tests.py create mode 100644 zooniverse/zoonitom/zoo/views.py create mode 100644 zooniverse/zoonitom/zoonitom/__init__.py create mode 100644 zooniverse/zoonitom/zoonitom/asgi.py create mode 100644 zooniverse/zoonitom/zoonitom/settings.py create mode 100644 zooniverse/zoonitom/zoonitom/urls.py create mode 100644 zooniverse/zoonitom/zoonitom/wsgi.py diff --git a/zooniverse/zoonitom/.gitignore b/zooniverse/zoonitom/.gitignore new file mode 100644 index 0000000..d944388 --- /dev/null +++ b/zooniverse/zoonitom/.gitignore @@ -0,0 +1,3 @@ +*.sqlite3 +tmp/ +data/ \ No newline at end of file diff --git a/zooniverse/zoonitom/manage.py b/zooniverse/zoonitom/manage.py new file mode 100755 index 0000000..dd16be1 --- /dev/null +++ b/zooniverse/zoonitom/manage.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +"""Django's command-line utility for administrative tasks.""" +import os +import sys + + +def main(): + """Run administrative tasks.""" + os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'zoonitom.settings') + try: + from django.core.management import execute_from_command_line + except ImportError as exc: + raise ImportError( + "Couldn't import Django. Are you sure it's installed and " + "available on your PYTHONPATH environment variable? Did you " + "forget to activate a virtual environment?" + ) from exc + execute_from_command_line(sys.argv) + + +if __name__ == '__main__': + main() diff --git a/zooniverse/zoonitom/requirements.txt b/zooniverse/zoonitom/requirements.txt new file mode 100644 index 0000000..8b93e99 --- /dev/null +++ b/zooniverse/zoonitom/requirements.txt @@ -0,0 +1 @@ +tomtoolkit diff --git a/zooniverse/zoonitom/static/.keep b/zooniverse/zoonitom/static/.keep new file mode 100644 index 0000000..e69de29 diff --git a/zooniverse/zoonitom/static/tom_common/css/custom.css b/zooniverse/zoonitom/static/tom_common/css/custom.css new file mode 100644 index 0000000..889b89c --- /dev/null +++ b/zooniverse/zoonitom/static/tom_common/css/custom.css @@ -0,0 +1,32 @@ +/*! + * Adapted from Bootstrap v4.6.2 (https://getbootstrap.com/) + * Set defaults for custom css + */ +:root { + --primary: #007bff; + --secondary: #6c757d; + --success: #28a745; + --info: #17a2b8; + --warning: #ffc107; + --danger: #dc3545; + --light: #f8f9fa; + --dark: #343a40; +} + +body { + font-size: 1rem; + font-weight: 400; + line-height: 1.5; + color: #212529; + background-color: #fff; +} + +a { + color: #007bff; + background-color: transparent; +} + +a:hover { + color: #0056b3; + text-decoration: underline; +} diff --git a/zooniverse/zoonitom/zoo/__init__.py b/zooniverse/zoonitom/zoo/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/zooniverse/zoonitom/zoo/admin.py b/zooniverse/zoonitom/zoo/admin.py new file mode 100644 index 0000000..8c38f3f --- /dev/null +++ b/zooniverse/zoonitom/zoo/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/zooniverse/zoonitom/zoo/apps.py b/zooniverse/zoonitom/zoo/apps.py new file mode 100644 index 0000000..44bcffe --- /dev/null +++ b/zooniverse/zoonitom/zoo/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class ZooConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'zoo' diff --git a/zooniverse/zoonitom/zoo/migrations/__init__.py b/zooniverse/zoonitom/zoo/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/zooniverse/zoonitom/zoo/models.py b/zooniverse/zoonitom/zoo/models.py new file mode 100644 index 0000000..e970a64 --- /dev/null +++ b/zooniverse/zoonitom/zoo/models.py @@ -0,0 +1,18 @@ +# from django.db import models +# +# from tom_targets.base_models import BaseTarget +# +# +# class UserDefinedTarget(BaseTarget): +# """ +# A target with fields defined by a user. +# """ +# +# class Meta: +# verbose_name = "target" +# permissions = ( +# ('view_target', 'View Target'), +# ('add_target', 'Add Target'), +# ('change_target', 'Change Target'), +# ('delete_target', 'Delete Target'), +# ) diff --git a/zooniverse/zoonitom/zoo/tests.py b/zooniverse/zoonitom/zoo/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/zooniverse/zoonitom/zoo/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/zooniverse/zoonitom/zoo/views.py b/zooniverse/zoonitom/zoo/views.py new file mode 100644 index 0000000..91ea44a --- /dev/null +++ b/zooniverse/zoonitom/zoo/views.py @@ -0,0 +1,3 @@ +from django.shortcuts import render + +# Create your views here. diff --git a/zooniverse/zoonitom/zoonitom/__init__.py b/zooniverse/zoonitom/zoonitom/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/zooniverse/zoonitom/zoonitom/asgi.py b/zooniverse/zoonitom/zoonitom/asgi.py new file mode 100644 index 0000000..55364d8 --- /dev/null +++ b/zooniverse/zoonitom/zoonitom/asgi.py @@ -0,0 +1,16 @@ +""" +ASGI config for zoonitom project. + +It exposes the ASGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/4.2/howto/deployment/asgi/ +""" + +import os + +from django.core.asgi import get_asgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'zoonitom.settings') + +application = get_asgi_application() diff --git a/zooniverse/zoonitom/zoonitom/settings.py b/zooniverse/zoonitom/zoonitom/settings.py new file mode 100644 index 0000000..c32e671 --- /dev/null +++ b/zooniverse/zoonitom/zoonitom/settings.py @@ -0,0 +1,363 @@ +""" +Django settings for your TOM project. + +Originally generated by 'django-admin startproject' using Django 2.1.1. +Generated by ./manage.py tom_setup on June 24, 2025, 2:19 p.m. + +For more information on this file, see +https://docs.djangoproject.com/en/2.1/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/2.1/ref/settings/ +""" +import logging.config +import os +import tempfile + + +# Build paths inside the project like this: os.path.join(BASE_DIR, ...) +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/2.1/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = 'g+2zgo%6jr_gw%v_*tull1)g3a=b!oyql^exrptsm!^+$q^x00' + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = True + +ALLOWED_HOSTS = [] + + +# Application definition + +TOM_NAME = 'zoonitom' + +INSTALLED_APPS = [ + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', + 'django.contrib.sites', + 'django_extensions', + 'django_tasks', + 'django_tasks.backends.database', + 'guardian', + 'tom_common', + 'django_comments', + 'bootstrap4', + 'crispy_bootstrap4', + 'crispy_forms', + 'rest_framework', + 'rest_framework.authtoken', + 'django_filters', + 'django_gravatar', + 'django_htmx', + 'tom_targets', + 'tom_alerts', + 'tom_catalogs', + 'tom_observations', + 'tom_dataproducts', + 'zoo', +] + +SITE_ID = 1 + +MIDDLEWARE = [ + 'django.middleware.security.SecurityMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', + 'django_htmx.middleware.HtmxMiddleware', + 'tom_common.middleware.Raise403Middleware', + 'tom_common.middleware.ExternalServiceMiddleware', + 'tom_common.middleware.AuthStrategyMiddleware', +] + +ROOT_URLCONF = 'zoonitom.urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [os.path.join(BASE_DIR, 'templates')], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + ], + }, + }, +] + +CRISPY_TEMPLATE_PACK = 'bootstrap4' + +WSGI_APPLICATION = 'zoonitom.wsgi.application' + + +# Database +# https://docs.djangoproject.com/en/2.1/ref/settings/#databases + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), + } +} + +DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' + +# Password validation +# https://docs.djangoproject.com/en/2.1/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + }, +] + +LOGIN_URL = '/accounts/login/' +LOGIN_REDIRECT_URL = '/' +LOGOUT_REDIRECT_URL = '/' + +AUTHENTICATION_BACKENDS = ( + 'django.contrib.auth.backends.ModelBackend', + 'guardian.backends.ObjectPermissionBackend', +) + +# Internationalization +# https://docs.djangoproject.com/en/2.1/topics/i18n/ + +LANGUAGE_CODE = 'en-us' + +TIME_ZONE = 'UTC' + +USE_I18N = True + +USE_L10N = False + +USE_TZ = True + +DATETIME_FORMAT = 'Y-m-d H:i:s' +DATE_FORMAT = 'Y-m-d' + + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/2.1/howto/static-files/ + +STATIC_URL = '/static/' +STATIC_ROOT = os.path.join(BASE_DIR, '_static') +STATICFILES_DIRS = [os.path.join(BASE_DIR, 'static')] +MEDIA_ROOT = os.path.join(BASE_DIR, 'data') +MEDIA_URL = '/data/' + +LOGGING = { + 'version': 1, + 'disable_existing_loggers': False, + 'handlers': { + 'console': { + 'class': 'logging.StreamHandler', + } + }, + 'loggers': { + '': { + 'handlers': ['console'], + 'level': 'INFO' + } + } +} + +# Caching +# https://docs.djangoproject.com/en/dev/topics/cache/#filesystem-caching + +CACHES = { + 'default': { + 'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache', + 'LOCATION': tempfile.gettempdir() + } +} + +TASKS = { + "default": { + # "BACKEND": "django_tasks.backends.database.DatabaseBackend" + "BACKEND": "django_tasks.backends.immediate.ImmediateBackend" + } +} + +# TOM Specific configuration +TARGET_TYPE = 'SIDEREAL' + +# Set to the full path of a custom target model to extend the BaseTarget Model with custom fields. +# TARGET_MODEL_CLASS = 'zoo.models.UserDefinedTarget' + +# Define MATCH_MANAGERS here. This is a dictionary that contains a dotted module path to the desired match manager +# for a given model. +# For example: +# MATCH_MANAGERS = { +# "Target": "custom_code.match_managers.CustomTargetMatchManager" +# } +MATCH_MANAGERS = {} + +FACILITIES = { + 'LCO': { + 'portal_url': 'https://observe.lco.global', + 'api_key': '', + }, + 'GEM': { + 'portal_url': { + 'GS': 'https://139.229.34.15:8443', + 'GN': 'https://128.171.88.221:8443', + }, + 'api_key': { + 'GS': '', + 'GN': '', + }, + 'user_email': '', + 'programs': { + 'GS-YYYYS-T-NNN': { + 'MM': 'Std: Some descriptive text', + 'NN': 'Rap: Some descriptive text' + }, + 'GN-YYYYS-T-NNN': { + 'QQ': 'Std: Some descriptive text', + 'PP': 'Rap: Some descriptive text', + }, + }, + }, +} + +# Define the valid data product types for your TOM. +# This is a dictionary of tuples to be used as ChoiceField options, with the first element being the type and the +# second being the display name. +# Be careful when removing items, as previously valid types will no +# longer be valid, and may cause issues unless the offending records are modified. +DATA_PRODUCT_TYPES = { + 'photometry': ('photometry', 'Photometry'), + 'fits_file': ('fits_file', 'FITS File'), + 'spectroscopy': ('spectroscopy', 'Spectroscopy'), + 'image_file': ('image_file', 'Image File') +} + +DATA_PROCESSORS = { + 'photometry': 'tom_dataproducts.processors.photometry_processor.PhotometryProcessor', + 'spectroscopy': 'tom_dataproducts.processors.spectroscopy_processor.SpectroscopyProcessor', +} + +TOM_FACILITY_CLASSES = [ + 'tom_observations.facilities.lco.LCOFacility', + 'tom_observations.facilities.gemini.GEMFacility', + 'tom_observations.facilities.soar.SOARFacility', +] + +TOM_ALERT_CLASSES = [ + 'tom_alerts.brokers.alerce.ALeRCEBroker', + # 'tom_alerts.brokers.antares.ANTARESBroker', + 'tom_alerts.brokers.gaia.GaiaBroker', + 'tom_alerts.brokers.lasair.LasairBroker', + 'tom_alerts.brokers.tns.TNSBroker', + # 'tom_alerts.brokers.fink.FinkBroker', +] + +BROKERS = { + 'TNS': { + 'api_key': '', + 'bot_id': '', + 'bot_name': '', + }, + 'LASAIR': { + 'api_key': '', + } +} + +# Include or exclude specific dot separated harvester classes. If not set, all harvesters will be included based on +# app configurations. If INCLUDE_HARVESTER_CLASSES is set, only those harvesters will be included. If +# EXCLUDE_HARVESTER_CLASSES is set, all harvesters except those will be included. +# INCLUDE_HARVESTER_CLASSES = ['app.example.harvesters.ExampleHarvester'] +# EXCLUDE_HARVESTER_CLASSES = ['app.example.harvesters.ExampleHarvester'] + +HARVESTERS = { + 'TNS': { + 'api_key': '' + } +} + +# Define extra target fields here. Types can be any of "number", "string", "boolean" or "datetime" +# See https://tomtoolkit.github.io/docs/target_fields for documentation on this feature +# For example: +# EXTRA_FIELDS = [ +# {'name': 'redshift', 'type': 'number'}, +# {'name': 'discoverer', 'type': 'string'}, +# {'name': 'eligible', 'type': 'boolean'}, +# {'name': 'discovery_date', 'type': 'datetime'} +# ] +EXTRA_FIELDS = [] + +# Authentication strategy can either be LOCKED (required login for all views) +# or READ_ONLY (read only access to views) +AUTH_STRATEGY = 'READ_ONLY' + +# Row-level data permissions restrict users from viewing certain objects unless they are a member of the group to which +# the object belongs. Setting this value to True will allow all `ObservationRecord`, `DataProduct`, and `ReducedDatum` +# objects to be seen by everyone. Setting it to False will allow users to specify which groups can access +# `ObservationRecord`, `DataProduct`, and `ReducedDatum` objects. +TARGET_PERMISSIONS_ONLY = True + +# URLs that should be allowed access even with AUTH_STRATEGY = LOCKED +# for example: OPEN_URLS = ['/', '/about'] +OPEN_URLS = [] + +HOOKS = { + 'target_post_save': 'tom_common.hooks.target_post_save', + 'observation_change_state': 'tom_common.hooks.observation_change_state', + 'data_product_post_upload': 'tom_dataproducts.hooks.data_product_post_upload', + 'data_product_post_save': 'tom_dataproducts.hooks.data_product_post_save', + 'multiple_data_products_post_save': 'tom_dataproducts.hooks.multiple_data_products_post_save', +} + +AUTO_THUMBNAILS = False + +THUMBNAIL_MAX_SIZE = (0, 0) + +THUMBNAIL_DEFAULT_SIZE = (200, 200) + +HINTS_ENABLED = False +HINT_LEVEL = 20 + +REST_FRAMEWORK = { + 'DEFAULT_PERMISSION_CLASSES': [ + ], + 'TEST_REQUEST_DEFAULT_FORMAT': 'json', + 'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.LimitOffsetPagination', + 'PAGE_SIZE': 100 +} + +# Default Plotly theme setting, can set to any valid theme: +# 'plotly', 'plotly_white', 'plotly_dark', 'ggplot2', 'seaborn', 'simple_white', 'none' +PLOTLY_THEME = 'plotly_white' + +# Setting for displaying pagination information (e.g., "(0-0 of 0)"). +# Set this to False if you have a particularly large DB and paginated views are slow. +SHOW_PAGINATION_INFO = True + +try: + from local_settings import * # noqa +except ImportError: + pass diff --git a/zooniverse/zoonitom/zoonitom/urls.py b/zooniverse/zoonitom/zoonitom/urls.py new file mode 100644 index 0000000..9fdf673 --- /dev/null +++ b/zooniverse/zoonitom/zoonitom/urls.py @@ -0,0 +1,20 @@ +"""django URL Configuration + +The `urlpatterns` list routes URLs to views. For more information please see: + https://docs.djangoproject.com/en/2.2/topics/http/urls/ +Examples: +Function views + 1. Add an import: from my_app import views + 2. Add a URL to urlpatterns: path('', views.home, name='home') +Class-based views + 1. Add an import: from other_app.views import Home + 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') +Including another URLconf + 1. Import the include() function: from django.urls import include, path + 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) +""" +from django.urls import path, include + +urlpatterns = [ + path('', include('tom_common.urls')), +] diff --git a/zooniverse/zoonitom/zoonitom/wsgi.py b/zooniverse/zoonitom/zoonitom/wsgi.py new file mode 100644 index 0000000..c55311d --- /dev/null +++ b/zooniverse/zoonitom/zoonitom/wsgi.py @@ -0,0 +1,16 @@ +""" +WSGI config for zoonitom project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/4.2/howto/deployment/wsgi/ +""" + +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'zoonitom.settings') + +application = get_wsgi_application() From eacaecfe0a7664eb6eed7d5f2992a65b638d63ff Mon Sep 17 00:00:00 2001 From: Adam McMaster Date: Tue, 24 Jun 2025 16:00:42 +0100 Subject: [PATCH 2/5] Start setting up Zooniverse client --- zooniverse/zoonitom/requirements.txt | 1 + zooniverse/zoonitom/zoonitom/client.py | 17 ++ zooniverse/zoonitom/zoonitom/settings.py | 287 +++++++++++------------ 3 files changed, 160 insertions(+), 145 deletions(-) create mode 100644 zooniverse/zoonitom/zoonitom/client.py diff --git a/zooniverse/zoonitom/requirements.txt b/zooniverse/zoonitom/requirements.txt index 8b93e99..0380ecf 100644 --- a/zooniverse/zoonitom/requirements.txt +++ b/zooniverse/zoonitom/requirements.txt @@ -1 +1,2 @@ tomtoolkit +panoptes-client \ No newline at end of file diff --git a/zooniverse/zoonitom/zoonitom/client.py b/zooniverse/zoonitom/zoonitom/client.py new file mode 100644 index 0000000..4241d4a --- /dev/null +++ b/zooniverse/zoonitom/zoonitom/client.py @@ -0,0 +1,17 @@ +from django.conf import settings + +from panoptes_client import Panoptes, Project, Workflow + +if ( + settings.ZOONIVERSE_CLIENT_ID + and settings.ZOONIVERSE_CLIENT_SECRET + and not Panoptes.client().logged_in() +): + Panoptes.connect( + client_id=settings.ZOONIVERSE_CLIENT_ID, + client_secret=settings.ZOONIVERSE_CLIENT_SECRET, + ) + + +project = Project(settings.ZOONIVERSE_PROJECT_ID) +workflow = Workflow(settings.ZOONIVERSE_WORKFLOW_ID) diff --git a/zooniverse/zoonitom/zoonitom/settings.py b/zooniverse/zoonitom/zoonitom/settings.py index c32e671..5982b71 100644 --- a/zooniverse/zoonitom/zoonitom/settings.py +++ b/zooniverse/zoonitom/zoonitom/settings.py @@ -10,6 +10,7 @@ For the full list of settings and their values, see https://docs.djangoproject.com/en/2.1/ref/settings/ """ + import logging.config import os import tempfile @@ -23,7 +24,7 @@ # See https://docs.djangoproject.com/en/2.1/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = 'g+2zgo%6jr_gw%v_*tull1)g3a=b!oyql^exrptsm!^+$q^x00' +SECRET_KEY = "g+2zgo%6jr_gw%v_*tull1)g3a=b!oyql^exrptsm!^+$q^x00" # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True @@ -33,122 +34,122 @@ # Application definition -TOM_NAME = 'zoonitom' +TOM_NAME = "zoonitom" INSTALLED_APPS = [ - 'django.contrib.admin', - 'django.contrib.auth', - 'django.contrib.contenttypes', - 'django.contrib.sessions', - 'django.contrib.messages', - 'django.contrib.staticfiles', - 'django.contrib.sites', - 'django_extensions', - 'django_tasks', - 'django_tasks.backends.database', - 'guardian', - 'tom_common', - 'django_comments', - 'bootstrap4', - 'crispy_bootstrap4', - 'crispy_forms', - 'rest_framework', - 'rest_framework.authtoken', - 'django_filters', - 'django_gravatar', - 'django_htmx', - 'tom_targets', - 'tom_alerts', - 'tom_catalogs', - 'tom_observations', - 'tom_dataproducts', - 'zoo', + "django.contrib.admin", + "django.contrib.auth", + "django.contrib.contenttypes", + "django.contrib.sessions", + "django.contrib.messages", + "django.contrib.staticfiles", + "django.contrib.sites", + "django_extensions", + "django_tasks", + "django_tasks.backends.database", + "guardian", + "tom_common", + "django_comments", + "bootstrap4", + "crispy_bootstrap4", + "crispy_forms", + "rest_framework", + "rest_framework.authtoken", + "django_filters", + "django_gravatar", + "django_htmx", + "tom_targets", + "tom_alerts", + "tom_catalogs", + "tom_observations", + "tom_dataproducts", + "zoo", ] SITE_ID = 1 MIDDLEWARE = [ - 'django.middleware.security.SecurityMiddleware', - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.common.CommonMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'django.middleware.clickjacking.XFrameOptionsMiddleware', - 'django_htmx.middleware.HtmxMiddleware', - 'tom_common.middleware.Raise403Middleware', - 'tom_common.middleware.ExternalServiceMiddleware', - 'tom_common.middleware.AuthStrategyMiddleware', + "django.middleware.security.SecurityMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", + "django.middleware.common.CommonMiddleware", + "django.middleware.csrf.CsrfViewMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", + "django_htmx.middleware.HtmxMiddleware", + "tom_common.middleware.Raise403Middleware", + "tom_common.middleware.ExternalServiceMiddleware", + "tom_common.middleware.AuthStrategyMiddleware", ] -ROOT_URLCONF = 'zoonitom.urls' +ROOT_URLCONF = "zoonitom.urls" TEMPLATES = [ { - 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [os.path.join(BASE_DIR, 'templates')], - 'APP_DIRS': True, - 'OPTIONS': { - 'context_processors': [ - 'django.template.context_processors.debug', - 'django.template.context_processors.request', - 'django.contrib.auth.context_processors.auth', - 'django.contrib.messages.context_processors.messages', + "BACKEND": "django.template.backends.django.DjangoTemplates", + "DIRS": [os.path.join(BASE_DIR, "templates")], + "APP_DIRS": True, + "OPTIONS": { + "context_processors": [ + "django.template.context_processors.debug", + "django.template.context_processors.request", + "django.contrib.auth.context_processors.auth", + "django.contrib.messages.context_processors.messages", ], }, }, ] -CRISPY_TEMPLATE_PACK = 'bootstrap4' +CRISPY_TEMPLATE_PACK = "bootstrap4" -WSGI_APPLICATION = 'zoonitom.wsgi.application' +WSGI_APPLICATION = "zoonitom.wsgi.application" # Database # https://docs.djangoproject.com/en/2.1/ref/settings/#databases DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), + "default": { + "ENGINE": "django.db.backends.sqlite3", + "NAME": os.path.join(BASE_DIR, "db.sqlite3"), } } -DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' +DEFAULT_AUTO_FIELD = "django.db.models.AutoField" # Password validation # https://docs.djangoproject.com/en/2.1/ref/settings/#auth-password-validators AUTH_PASSWORD_VALIDATORS = [ { - 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", }, { - 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator", }, { - 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator", }, { - 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator", }, ] -LOGIN_URL = '/accounts/login/' -LOGIN_REDIRECT_URL = '/' -LOGOUT_REDIRECT_URL = '/' +LOGIN_URL = "/accounts/login/" +LOGIN_REDIRECT_URL = "/" +LOGOUT_REDIRECT_URL = "/" AUTHENTICATION_BACKENDS = ( - 'django.contrib.auth.backends.ModelBackend', - 'guardian.backends.ObjectPermissionBackend', + "django.contrib.auth.backends.ModelBackend", + "guardian.backends.ObjectPermissionBackend", ) # Internationalization # https://docs.djangoproject.com/en/2.1/topics/i18n/ -LANGUAGE_CODE = 'en-us' +LANGUAGE_CODE = "en-us" -TIME_ZONE = 'UTC' +TIME_ZONE = "UTC" USE_I18N = True @@ -156,42 +157,37 @@ USE_TZ = True -DATETIME_FORMAT = 'Y-m-d H:i:s' -DATE_FORMAT = 'Y-m-d' +DATETIME_FORMAT = "Y-m-d H:i:s" +DATE_FORMAT = "Y-m-d" # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/2.1/howto/static-files/ -STATIC_URL = '/static/' -STATIC_ROOT = os.path.join(BASE_DIR, '_static') -STATICFILES_DIRS = [os.path.join(BASE_DIR, 'static')] -MEDIA_ROOT = os.path.join(BASE_DIR, 'data') -MEDIA_URL = '/data/' +STATIC_URL = "/static/" +STATIC_ROOT = os.path.join(BASE_DIR, "_static") +STATICFILES_DIRS = [os.path.join(BASE_DIR, "static")] +MEDIA_ROOT = os.path.join(BASE_DIR, "data") +MEDIA_URL = "/data/" LOGGING = { - 'version': 1, - 'disable_existing_loggers': False, - 'handlers': { - 'console': { - 'class': 'logging.StreamHandler', + "version": 1, + "disable_existing_loggers": False, + "handlers": { + "console": { + "class": "logging.StreamHandler", } }, - 'loggers': { - '': { - 'handlers': ['console'], - 'level': 'INFO' - } - } + "loggers": {"": {"handlers": ["console"], "level": "INFO"}}, } # Caching # https://docs.djangoproject.com/en/dev/topics/cache/#filesystem-caching CACHES = { - 'default': { - 'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache', - 'LOCATION': tempfile.gettempdir() + "default": { + "BACKEND": "django.core.cache.backends.filebased.FileBasedCache", + "LOCATION": tempfile.gettempdir(), } } @@ -203,7 +199,7 @@ } # TOM Specific configuration -TARGET_TYPE = 'SIDEREAL' +TARGET_TYPE = "SIDEREAL" # Set to the full path of a custom target model to extend the BaseTarget Model with custom fields. # TARGET_MODEL_CLASS = 'zoo.models.UserDefinedTarget' @@ -217,28 +213,28 @@ MATCH_MANAGERS = {} FACILITIES = { - 'LCO': { - 'portal_url': 'https://observe.lco.global', - 'api_key': '', + "LCO": { + "portal_url": "https://observe.lco.global", + "api_key": "", }, - 'GEM': { - 'portal_url': { - 'GS': 'https://139.229.34.15:8443', - 'GN': 'https://128.171.88.221:8443', + "GEM": { + "portal_url": { + "GS": "https://139.229.34.15:8443", + "GN": "https://128.171.88.221:8443", }, - 'api_key': { - 'GS': '', - 'GN': '', + "api_key": { + "GS": "", + "GN": "", }, - 'user_email': '', - 'programs': { - 'GS-YYYYS-T-NNN': { - 'MM': 'Std: Some descriptive text', - 'NN': 'Rap: Some descriptive text' + "user_email": "", + "programs": { + "GS-YYYYS-T-NNN": { + "MM": "Std: Some descriptive text", + "NN": "Rap: Some descriptive text", }, - 'GN-YYYYS-T-NNN': { - 'QQ': 'Std: Some descriptive text', - 'PP': 'Rap: Some descriptive text', + "GN-YYYYS-T-NNN": { + "QQ": "Std: Some descriptive text", + "PP": "Rap: Some descriptive text", }, }, }, @@ -250,41 +246,41 @@ # Be careful when removing items, as previously valid types will no # longer be valid, and may cause issues unless the offending records are modified. DATA_PRODUCT_TYPES = { - 'photometry': ('photometry', 'Photometry'), - 'fits_file': ('fits_file', 'FITS File'), - 'spectroscopy': ('spectroscopy', 'Spectroscopy'), - 'image_file': ('image_file', 'Image File') + "photometry": ("photometry", "Photometry"), + "fits_file": ("fits_file", "FITS File"), + "spectroscopy": ("spectroscopy", "Spectroscopy"), + "image_file": ("image_file", "Image File"), } DATA_PROCESSORS = { - 'photometry': 'tom_dataproducts.processors.photometry_processor.PhotometryProcessor', - 'spectroscopy': 'tom_dataproducts.processors.spectroscopy_processor.SpectroscopyProcessor', + "photometry": "tom_dataproducts.processors.photometry_processor.PhotometryProcessor", + "spectroscopy": "tom_dataproducts.processors.spectroscopy_processor.SpectroscopyProcessor", } TOM_FACILITY_CLASSES = [ - 'tom_observations.facilities.lco.LCOFacility', - 'tom_observations.facilities.gemini.GEMFacility', - 'tom_observations.facilities.soar.SOARFacility', + "tom_observations.facilities.lco.LCOFacility", + "tom_observations.facilities.gemini.GEMFacility", + "tom_observations.facilities.soar.SOARFacility", ] TOM_ALERT_CLASSES = [ - 'tom_alerts.brokers.alerce.ALeRCEBroker', + "tom_alerts.brokers.alerce.ALeRCEBroker", # 'tom_alerts.brokers.antares.ANTARESBroker', - 'tom_alerts.brokers.gaia.GaiaBroker', - 'tom_alerts.brokers.lasair.LasairBroker', - 'tom_alerts.brokers.tns.TNSBroker', + "tom_alerts.brokers.gaia.GaiaBroker", + "tom_alerts.brokers.lasair.LasairBroker", + "tom_alerts.brokers.tns.TNSBroker", # 'tom_alerts.brokers.fink.FinkBroker', ] BROKERS = { - 'TNS': { - 'api_key': '', - 'bot_id': '', - 'bot_name': '', + "TNS": { + "api_key": "", + "bot_id": "", + "bot_name": "", + }, + "LASAIR": { + "api_key": "", }, - 'LASAIR': { - 'api_key': '', - } } # Include or exclude specific dot separated harvester classes. If not set, all harvesters will be included based on @@ -293,11 +289,7 @@ # INCLUDE_HARVESTER_CLASSES = ['app.example.harvesters.ExampleHarvester'] # EXCLUDE_HARVESTER_CLASSES = ['app.example.harvesters.ExampleHarvester'] -HARVESTERS = { - 'TNS': { - 'api_key': '' - } -} +HARVESTERS = {"TNS": {"api_key": ""}} # Define extra target fields here. Types can be any of "number", "string", "boolean" or "datetime" # See https://tomtoolkit.github.io/docs/target_fields for documentation on this feature @@ -312,7 +304,7 @@ # Authentication strategy can either be LOCKED (required login for all views) # or READ_ONLY (read only access to views) -AUTH_STRATEGY = 'READ_ONLY' +AUTH_STRATEGY = "READ_ONLY" # Row-level data permissions restrict users from viewing certain objects unless they are a member of the group to which # the object belongs. Setting this value to True will allow all `ObservationRecord`, `DataProduct`, and `ReducedDatum` @@ -325,11 +317,11 @@ OPEN_URLS = [] HOOKS = { - 'target_post_save': 'tom_common.hooks.target_post_save', - 'observation_change_state': 'tom_common.hooks.observation_change_state', - 'data_product_post_upload': 'tom_dataproducts.hooks.data_product_post_upload', - 'data_product_post_save': 'tom_dataproducts.hooks.data_product_post_save', - 'multiple_data_products_post_save': 'tom_dataproducts.hooks.multiple_data_products_post_save', + "target_post_save": "tom_common.hooks.target_post_save", + "observation_change_state": "tom_common.hooks.observation_change_state", + "data_product_post_upload": "tom_dataproducts.hooks.data_product_post_upload", + "data_product_post_save": "tom_dataproducts.hooks.data_product_post_save", + "multiple_data_products_post_save": "tom_dataproducts.hooks.multiple_data_products_post_save", } AUTO_THUMBNAILS = False @@ -342,22 +334,27 @@ HINT_LEVEL = 20 REST_FRAMEWORK = { - 'DEFAULT_PERMISSION_CLASSES': [ - ], - 'TEST_REQUEST_DEFAULT_FORMAT': 'json', - 'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.LimitOffsetPagination', - 'PAGE_SIZE': 100 + "DEFAULT_PERMISSION_CLASSES": [], + "TEST_REQUEST_DEFAULT_FORMAT": "json", + "DEFAULT_PAGINATION_CLASS": "rest_framework.pagination.LimitOffsetPagination", + "PAGE_SIZE": 100, } # Default Plotly theme setting, can set to any valid theme: # 'plotly', 'plotly_white', 'plotly_dark', 'ggplot2', 'seaborn', 'simple_white', 'none' -PLOTLY_THEME = 'plotly_white' +PLOTLY_THEME = "plotly_white" # Setting for displaying pagination information (e.g., "(0-0 of 0)"). # Set this to False if you have a particularly large DB and paginated views are slow. SHOW_PAGINATION_INFO = True +# Zooniverse configuration. Set in environment or override in local_settings.py +ZOONIVERSE_CLIENT_ID = os.environ.get("ZOONIVERSE_CLIENT_ID") +ZOONIVERSE_CLIENT_SECRET = os.environ.get("ZOONIVERSE_CLIENT_SECRET") +ZOONIVERSE_PROJECT_ID = os.environ.get("ZOONIVERSE_PROJECT_ID") +ZOONIVERSE_WORKFLOW_ID = os.environ.get("ZOONIVERSE_WORKFLOW_ID") + try: - from local_settings import * # noqa + from local_settings import * # noqa except ImportError: pass From a00be978b52792a59d756cfe0723dc0798dbf2ec Mon Sep 17 00:00:00 2001 From: Adam McMaster Date: Thu, 26 Jun 2025 14:26:22 +0100 Subject: [PATCH 3/5] Move Zooniverse client config into zoo app --- zooniverse/zoonitom/{zoonitom => zoo}/client.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename zooniverse/zoonitom/{zoonitom => zoo}/client.py (100%) diff --git a/zooniverse/zoonitom/zoonitom/client.py b/zooniverse/zoonitom/zoo/client.py similarity index 100% rename from zooniverse/zoonitom/zoonitom/client.py rename to zooniverse/zoonitom/zoo/client.py From 492563f63107669d61a887eaaad81aff950ad1e2 Mon Sep 17 00:00:00 2001 From: Adam McMaster Date: Thu, 26 Jun 2025 17:08:52 +0100 Subject: [PATCH 4/5] Add initial zooniverse models and data import --- zooniverse/zoonitom/zoo/client.py | 2 +- zooniverse/zoonitom/zoo/data_import.py | 155 ++++++++++++++++++ .../zoonitom/zoo/migrations/0001_initial.py | 77 +++++++++ ..._alter_zooniverseclassification_user_id.py | 18 ++ zooniverse/zoonitom/zoo/models.py | 83 ++++++++-- zooniverse/zoonitom/zoonitom/settings.py | 2 +- 6 files changed, 317 insertions(+), 20 deletions(-) create mode 100644 zooniverse/zoonitom/zoo/data_import.py create mode 100644 zooniverse/zoonitom/zoo/migrations/0001_initial.py create mode 100644 zooniverse/zoonitom/zoo/migrations/0002_alter_zooniverseclassification_user_id.py diff --git a/zooniverse/zoonitom/zoo/client.py b/zooniverse/zoonitom/zoo/client.py index 4241d4a..118c4e3 100644 --- a/zooniverse/zoonitom/zoo/client.py +++ b/zooniverse/zoonitom/zoo/client.py @@ -5,7 +5,7 @@ if ( settings.ZOONIVERSE_CLIENT_ID and settings.ZOONIVERSE_CLIENT_SECRET - and not Panoptes.client().logged_in() + and not Panoptes.client().logged_in ): Panoptes.connect( client_id=settings.ZOONIVERSE_CLIENT_ID, diff --git a/zooniverse/zoonitom/zoo/data_import.py b/zooniverse/zoonitom/zoo/data_import.py new file mode 100644 index 0000000..da7ca7d --- /dev/null +++ b/zooniverse/zoonitom/zoo/data_import.py @@ -0,0 +1,155 @@ +import json +import logging + +from csv import DictReader + +from dateutil.parser import parse as date_parse + +from zoo.client import project, workflow +from zoo.models import ( + ZooniverseClassification, + ZooniverseSubject, + ZooniverseTarget, + ZooniverseSurvey, +) + +logger = logging.getLogger(__name__) + + +def generate_subject_export(): + return project.generate_export("subjects") + + +def generate_classification_export(): + return workflow.generate_export("classifications") + + +def get_subject_export(): + return project.get_export("subjects").csv_dictreader() + + +def get_classification_export(): + return workflow.get_export("classifications").csv_dictreader() + + +def import_classifications(): + """ + Downloads the latest workflow classifications export and creates new ZooniverseClassification + objects based on it. + """ + existing_classifications = ZooniverseClassification.objects.all().values_list( + "classification_id", flat=True + ) + existing_subjects = ZooniverseSubject.objects.all().values_list( + "subject_id", flat=True + ) + for c in get_classification_export(): + classification_id = int(c["classification_id"]) + subject_id = int(c["subject_ids"]) + user_id = c["user_id"] + if len(user_id) == 0: + user_id = None + else: + user_id = int(user_id) + + if classification_id in existing_classifications: + continue + + if subject_id not in existing_subjects: + logger.warning( + f"Skipping classification {classification_id} for unknown subject {subject_id}" + ) + continue + + subject = ZooniverseSubject.objects.get(subject_id=subject_id) + + annotation = json.loads(c["annotation"]) + timestamp = date_parse(c["created_at"]) + + ZooniverseClassification.objects.create( + classification_id=classification_id, + subject=subject, + user_id=user_id, + timestamp=timestamp, + annotation=annotation, + ) + + +def import_subjects( + target_identifier=None, + survey=None, + survey_identifier=None, + sequence=None, + sequence_identifier=None, +): + """ + Downloads the latest subjects export and creates new ZooniverseSubject objects. + + Options: + - target_identifier: The metadata key name which gives the target/object ID. + Any subjects which don't have this metadata key will be skipped. + - survey: If this and survey_identifier are both provided, filters subjects + to just the ones in the specified survey. If survey_identifier is not provided, + assumes all subjects are in the specified survey. + - survey_identifier: The metadata key name which gives the survey name. + - sequence: If this and sequence_identifier are both provided, filters subjects + to just the ones in the specified sequence. Has no effect if sequence_identifier + is not provided. + - sequence_identifier: the metadata key name which gives the sequence name (i.e. + the data release number, sector name, or other grouping). + """ + if survey is not None: + survey = ZooniverseSurvey.objects.get_or_create(name=survey)[0] + + existing_subjects = ZooniverseSubject.objects.all() + if survey is not None: + existing_subjects = existing_subjects.filter(target__survey=survey) + existing_subjects = existing_subjects.values_list("subject_id", flat=True) + + count = 0 + for s in get_subject_export(): + if count > 100: + break + subject_id = int(s["subject_id"]) + + if subject_id in existing_subjects: + continue + + locations = json.loads(s["locations"]) + metadata = json.loads(s["metadata"]) + + if survey_identifier is not None: + survey_name = metadata.get(survey_identifier, None) + if survey_name is None: + continue + if survey is None: + survey = ZooniverseSurvey.objects.get_or_create(name=survey_name)[0] + else: + if survey.name != survey_name: + continue + + target = None + if target_identifier is not None: + target_name = metadata.get(target_identifier, None) + if target_name is None: + continue + target = ZooniverseTarget.objects.get_or_create( + survey=survey, identifier=target_name + )[0] + + sequence_name = None + if sequence_identifier is not None: + sequence_name = metadata.get(sequence_identifier, None) + if sequence_name is None: + continue + if sequence is not None and sequence != sequence_name: + continue + + ZooniverseSubject.objects.create( + subject_id=subject_id, + metadata=s["metadata"], + data_url=locations["0"], + target=target, + sequence=sequence_name, + ) + count += 1 diff --git a/zooniverse/zoonitom/zoo/migrations/0001_initial.py b/zooniverse/zoonitom/zoo/migrations/0001_initial.py new file mode 100644 index 0000000..b3f289d --- /dev/null +++ b/zooniverse/zoonitom/zoo/migrations/0001_initial.py @@ -0,0 +1,77 @@ +# Generated by Django 4.2.23 on 2025-06-26 13:40 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='ZooniverseClassification', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('classification_id', models.BigIntegerField(unique=True)), + ('user_id', models.BigIntegerField()), + ('timestamp', models.DateTimeField()), + ('annotation', models.JSONField()), + ('created', models.DateTimeField(auto_now_add=True)), + ('updated', models.DateTimeField(auto_now=True)), + ], + ), + migrations.CreateModel( + name='ZooniverseSurvey', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=50)), + ('created', models.DateTimeField(auto_now_add=True)), + ('updated', models.DateTimeField(auto_now=True)), + ], + ), + migrations.CreateModel( + name='ZooniverseTarget', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('identifier', models.CharField(max_length=128)), + ('created', models.DateTimeField(auto_now_add=True)), + ('updated', models.DateTimeField(auto_now=True)), + ('survey', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='zoo.zooniversesurvey')), + ], + ), + migrations.CreateModel( + name='ZooniverseTargetReduction', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('reduced_annotations', models.JSONField()), + ('created', models.DateTimeField(auto_now_add=True)), + ('updated', models.DateTimeField(auto_now=True)), + ('classifications', models.ManyToManyField(to='zoo.zooniverseclassification')), + ('target', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='zoo.zooniversetarget')), + ], + ), + migrations.CreateModel( + name='ZooniverseSubject', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('subject_id', models.BigIntegerField(unique=True)), + ('sequence', models.CharField(blank=True, help_text='Sector, data release, etc.', max_length=50, null=True)), + ('data_url', models.URLField()), + ('start_time', models.DateTimeField(blank=True, help_text='Earliest time in the light curve', null=True)), + ('end_time', models.DateTimeField(blank=True, help_text='Latest time in the light curve', null=True)), + ('metadata', models.JSONField()), + ('created', models.DateTimeField(auto_now_add=True)), + ('updated', models.DateTimeField(auto_now=True)), + ('target', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='zoo.zooniversetarget')), + ], + ), + migrations.AddField( + model_name='zooniverseclassification', + name='subject', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='zoo.zooniversesubject'), + ), + ] diff --git a/zooniverse/zoonitom/zoo/migrations/0002_alter_zooniverseclassification_user_id.py b/zooniverse/zoonitom/zoo/migrations/0002_alter_zooniverseclassification_user_id.py new file mode 100644 index 0000000..e19e2c2 --- /dev/null +++ b/zooniverse/zoonitom/zoo/migrations/0002_alter_zooniverseclassification_user_id.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.23 on 2025-06-26 14:50 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('zoo', '0001_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='zooniverseclassification', + name='user_id', + field=models.BigIntegerField(blank=True, null=True), + ), + ] diff --git a/zooniverse/zoonitom/zoo/models.py b/zooniverse/zoonitom/zoo/models.py index e970a64..dd5afa1 100644 --- a/zooniverse/zoonitom/zoo/models.py +++ b/zooniverse/zoonitom/zoo/models.py @@ -1,18 +1,65 @@ -# from django.db import models -# -# from tom_targets.base_models import BaseTarget -# -# -# class UserDefinedTarget(BaseTarget): -# """ -# A target with fields defined by a user. -# """ -# -# class Meta: -# verbose_name = "target" -# permissions = ( -# ('view_target', 'View Target'), -# ('add_target', 'Add Target'), -# ('change_target', 'Change Target'), -# ('delete_target', 'Delete Target'), -# ) +from django.db import models + + +class ZooniverseSurvey(models.Model): + name = models.CharField(max_length=50) + + created = models.DateTimeField(auto_now_add=True) + updated = models.DateTimeField(auto_now=True) + + +class ZooniverseTarget(models.Model): + survey = models.ForeignKey(ZooniverseSurvey, on_delete=models.CASCADE) + identifier = models.CharField(max_length=128) + + created = models.DateTimeField(auto_now_add=True) + updated = models.DateTimeField(auto_now=True) + + +class ZooniverseSubject(models.Model): + subject_id = models.BigIntegerField(unique=True) + target = models.ForeignKey( + ZooniverseTarget, on_delete=models.CASCADE, null=True, blank=True + ) + + sequence = models.CharField( + max_length=50, help_text="Sector, data release, etc.", null=True, blank=True + ) + data_url = models.URLField() + start_time = models.DateTimeField( + null=True, blank=True, help_text="Earliest time in the light curve" + ) + end_time = models.DateTimeField( + null=True, blank=True, help_text="Latest time in the light curve" + ) + + metadata = models.JSONField() + + created = models.DateTimeField(auto_now_add=True) + updated = models.DateTimeField(auto_now=True) + + +class ZooniverseClassification(models.Model): + classification_id = models.BigIntegerField(unique=True) + subject = models.ForeignKey(ZooniverseSubject, on_delete=models.CASCADE) + + user_id = models.BigIntegerField(null=True, blank=True) + timestamp = models.DateTimeField() + annotation = models.JSONField() + + created = models.DateTimeField(auto_now_add=True) + updated = models.DateTimeField(auto_now=True) + + +class ZooniverseTargetReduction(models.Model): + """ + Reduced classifications for targets. + """ + + target = models.ForeignKey(ZooniverseTarget, on_delete=models.CASCADE) + classifications = models.ManyToManyField(ZooniverseClassification) + + reduced_annotations = models.JSONField() + + created = models.DateTimeField(auto_now_add=True) + updated = models.DateTimeField(auto_now=True) diff --git a/zooniverse/zoonitom/zoonitom/settings.py b/zooniverse/zoonitom/zoonitom/settings.py index 5982b71..a2c176e 100644 --- a/zooniverse/zoonitom/zoonitom/settings.py +++ b/zooniverse/zoonitom/zoonitom/settings.py @@ -355,6 +355,6 @@ ZOONIVERSE_WORKFLOW_ID = os.environ.get("ZOONIVERSE_WORKFLOW_ID") try: - from local_settings import * # noqa + from .local_settings import * # noqa except ImportError: pass From ecaa9b80d9e3af520871b0aba80862b16f6c90a8 Mon Sep 17 00:00:00 2001 From: Adam McMaster Date: Fri, 27 Jun 2025 16:04:10 +0100 Subject: [PATCH 5/5] Make a start on views --- .../zoo/zooniversetarget_detail.html | 31 +++++++++++++++++++ .../templates/zoo/zooniversetarget_list.html | 16 ++++++++++ zooniverse/zoonitom/zoo/data_import.py | 6 ++-- zooniverse/zoonitom/zoo/models.py | 19 ++++++++++++ zooniverse/zoonitom/zoo/urls.py | 29 +++++++++++++++++ zooniverse/zoonitom/zoo/views.py | 13 +++++++- zooniverse/zoonitom/zoonitom/urls.py | 4 ++- 7 files changed, 113 insertions(+), 5 deletions(-) create mode 100644 zooniverse/zoonitom/templates/zoo/zooniversetarget_detail.html create mode 100644 zooniverse/zoonitom/templates/zoo/zooniversetarget_list.html create mode 100644 zooniverse/zoonitom/zoo/urls.py diff --git a/zooniverse/zoonitom/templates/zoo/zooniversetarget_detail.html b/zooniverse/zoonitom/templates/zoo/zooniversetarget_detail.html new file mode 100644 index 0000000..84ef7d1 --- /dev/null +++ b/zooniverse/zoonitom/templates/zoo/zooniversetarget_detail.html @@ -0,0 +1,31 @@ +{% extends "tom_common/base.html" %} + +{% block content %} +

Zooniverse Target Detail

+ +
+

{{ object.identifier }}

+
    +
  • Survey: {{ object.survey }}
    • +
  • Created: {{ object.created }}
    • +
  • Updated: {{ object.updated }}
    • + +
+

Subjects

+
    + {% for subject in object.zooniversesubject_set.all %} +
  • {{ subject.subject_id }} (Zooniverse Talk) +
      + {% for k, v in subject.annotation_totals.items %} +
    • {{ k }}: {{ v }}
      • + {% endfor %} +
    +
    • + {% empty %} +
  • No subjects found.
    • + {% endfor %} +
+
+ +Back to list +{% endblock %} \ No newline at end of file diff --git a/zooniverse/zoonitom/templates/zoo/zooniversetarget_list.html b/zooniverse/zoonitom/templates/zoo/zooniversetarget_list.html new file mode 100644 index 0000000..c515449 --- /dev/null +++ b/zooniverse/zoonitom/templates/zoo/zooniversetarget_list.html @@ -0,0 +1,16 @@ +{% extends "tom_common/base.html" %} + +{% block content %} +

Zooniverse Target List

+
    + {% for object in object_list %} +
  • + + {{ object }} + +
    • + {% empty %} +
  • No targets available.
    • + {% endfor %} +
+{% endblock %} \ No newline at end of file diff --git a/zooniverse/zoonitom/zoo/data_import.py b/zooniverse/zoonitom/zoo/data_import.py index da7ca7d..22960b5 100644 --- a/zooniverse/zoonitom/zoo/data_import.py +++ b/zooniverse/zoonitom/zoo/data_import.py @@ -21,7 +21,7 @@ def generate_subject_export(): def generate_classification_export(): - return workflow.generate_export("classifications") + return project.generate_export("classifications") def get_subject_export(): @@ -29,7 +29,7 @@ def get_subject_export(): def get_classification_export(): - return workflow.get_export("classifications").csv_dictreader() + return project.get_export("classifications").csv_dictreader() def import_classifications(): @@ -63,7 +63,7 @@ def import_classifications(): subject = ZooniverseSubject.objects.get(subject_id=subject_id) - annotation = json.loads(c["annotation"]) + annotation = json.loads(c["annotations"]) timestamp = date_parse(c["created_at"]) ZooniverseClassification.objects.create( diff --git a/zooniverse/zoonitom/zoo/models.py b/zooniverse/zoonitom/zoo/models.py index dd5afa1..53f2479 100644 --- a/zooniverse/zoonitom/zoo/models.py +++ b/zooniverse/zoonitom/zoo/models.py @@ -1,5 +1,8 @@ from django.db import models +from zoo.client import project +from collections import Counter + class ZooniverseSurvey(models.Model): name = models.CharField(max_length=50) @@ -7,6 +10,9 @@ class ZooniverseSurvey(models.Model): created = models.DateTimeField(auto_now_add=True) updated = models.DateTimeField(auto_now=True) + def __str__(self): + return self.name + class ZooniverseTarget(models.Model): survey = models.ForeignKey(ZooniverseSurvey, on_delete=models.CASCADE) @@ -15,6 +21,9 @@ class ZooniverseTarget(models.Model): created = models.DateTimeField(auto_now_add=True) updated = models.DateTimeField(auto_now=True) + def __str__(self): + return self.identifier + class ZooniverseSubject(models.Model): subject_id = models.BigIntegerField(unique=True) @@ -38,6 +47,16 @@ class ZooniverseSubject(models.Model): created = models.DateTimeField(auto_now_add=True) updated = models.DateTimeField(auto_now=True) + def talk_url(self): + return f"https://www.zooniverse.org/projects/{project.slug}/talk/subjects/{self.subject_id}" + + def annotation_totals(self): + annotations = self.zooniverseclassification_set.values_list( + "annotation", flat=True + ) + annotations = [a[0]["value"] for a in annotations] + return dict(Counter(annotations)) + class ZooniverseClassification(models.Model): classification_id = models.BigIntegerField(unique=True) diff --git a/zooniverse/zoonitom/zoo/urls.py b/zooniverse/zoonitom/zoo/urls.py new file mode 100644 index 0000000..3eb0033 --- /dev/null +++ b/zooniverse/zoonitom/zoo/urls.py @@ -0,0 +1,29 @@ +"""django URL Configuration + +The `urlpatterns` list routes URLs to views. For more information please see: + https://docs.djangoproject.com/en/2.2/topics/http/urls/ +Examples: +Function views + 1. Add an import: from my_app import views + 2. Add a URL to urlpatterns: path('', views.home, name='home') +Class-based views + 1. Add an import: from other_app.views import Home + 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') +Including another URLconf + 1. Import the include() function: from django.urls import include, path + 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) +""" + +from django.urls import path, include + +from zoo.views import ZooniverseTargetListView, ZooniverseTargetDetailView + + +urlpatterns = [ + path("", ZooniverseTargetListView.as_view(), name="zooniversetarget_list"), + path( + "/", + ZooniverseTargetDetailView.as_view(), + name="zooniverse_target_detail", + ), +] diff --git a/zooniverse/zoonitom/zoo/views.py b/zooniverse/zoonitom/zoo/views.py index 91ea44a..8096f65 100644 --- a/zooniverse/zoonitom/zoo/views.py +++ b/zooniverse/zoonitom/zoo/views.py @@ -1,3 +1,14 @@ from django.shortcuts import render +from django.views.generic.detail import DetailView +from django.views.generic.list import ListView -# Create your views here. +from zoo.models import ZooniverseSubject, ZooniverseClassification, ZooniverseTarget + + +class ZooniverseTargetDetailView(DetailView): + model = ZooniverseTarget + + +class ZooniverseTargetListView(ListView): + model = ZooniverseTarget + paginate_by = 100 diff --git a/zooniverse/zoonitom/zoonitom/urls.py b/zooniverse/zoonitom/zoonitom/urls.py index 9fdf673..fe4b533 100644 --- a/zooniverse/zoonitom/zoonitom/urls.py +++ b/zooniverse/zoonitom/zoonitom/urls.py @@ -13,8 +13,10 @@ 1. Import the include() function: from django.urls import include, path 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) """ + from django.urls import path, include urlpatterns = [ - path('', include('tom_common.urls')), + path("", include("tom_common.urls")), + path("zoo/", include("zoo.urls")), ]