From eed0eb73246e2b5771087e5b68a836fff5bf44f1 Mon Sep 17 00:00:00 2001 From: Ashwini Sukale Date: Mon, 10 Mar 2025 14:13:48 +0530 Subject: [PATCH 01/50] client id changes --- rorapi/middleware.py | 43 +++++++++++++++++++++ rorapi/settings.py | 5 +++ rorapi/v2/models/client.py | 12 ++++++ rorapi/v2/serializers/client_serializers.py | 7 ++++ rorapi/v2/tests.py | 11 ++++++ rorapi/v2/urls.py | 7 ++++ rorapi/v2/views/client_views.py | 27 +++++++++++++ 7 files changed, 112 insertions(+) create mode 100644 rorapi/middleware.py create mode 100644 rorapi/v2/models/client.py create mode 100644 rorapi/v2/serializers/client_serializers.py create mode 100644 rorapi/v2/tests.py create mode 100644 rorapi/v2/urls.py create mode 100644 rorapi/v2/views/client_views.py diff --git a/rorapi/middleware.py b/rorapi/middleware.py new file mode 100644 index 0000000..a4800af --- /dev/null +++ b/rorapi/middleware.py @@ -0,0 +1,43 @@ +from django.core.cache import cache +from django.http import JsonResponse +from rorapi.v2.models.client import Client +from django.utils.timezone import now +import os + +# Toggle Behavior-Based Rate Limiting +ENABLE_BEHAVIORAL_LIMITING = os.getenv("ENABLE_BEHAVIORAL_LIMITING", "False") == "True" + +class ClientRateLimitMiddleware: + def __init__(self, get_response): + self.get_response = get_response + + def __call__(self, request): + client_id = request.headers.get('Client-Id') + ip_address = request.META.get('REMOTE_ADDR') + rate_limit = 50 # Default rate limit + + if client_id: + try: + client = Client.objects.get(client_id=client_id) + rate_limit = 2000 # Higher limit for registered users + + # Behavior-based throttling (if enabled) + if ENABLE_BEHAVIORAL_LIMITING: + client.request_count += 1 + client.last_request_at = now() + client.save() + + if client.request_count > 500 and (now() - client.last_request_at).seconds < 300: + return JsonResponse({"error": "Rate limit exceeded due to excessive requests"}, status=429) + + except Client.DoesNotExist: + rate_limit = 50 # Invalid client ID gets the lower limit + + cache_key = f"rate_limit_{client_id or ip_address}" + request_count = cache.get(cache_key, 0) + + if request_count >= rate_limit: + return JsonResponse({"error": "Rate limit exceeded"}, status=429) + + cache.set(cache_key, request_count + 1, timeout=300) # Reset every 5 min + return self.get_response(request) diff --git a/rorapi/settings.py b/rorapi/settings.py index a8f3504..87cd611 100644 --- a/rorapi/settings.py +++ b/rorapi/settings.py @@ -72,6 +72,7 @@ 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', 'django_prometheus.middleware.PrometheusAfterMiddleware', + 'rorapi.middleware.ClientRateLimitMiddleware', ] ROOT_URLCONF = 'rorapi.common.urls' @@ -273,3 +274,7 @@ GRID_REMOVED_IDS = [] LAUNCH_DARKLY_KEY = os.environ.get('LAUNCH_DARKLY_KEY') + +# Toggle for behavior-based rate limiting +import os +ENABLE_BEHAVIORAL_LIMITING = os.getenv("ENABLE_BEHAVIORAL_LIMITING", "False") == "True" diff --git a/rorapi/v2/models/client.py b/rorapi/v2/models/client.py new file mode 100644 index 0000000..d1d1899 --- /dev/null +++ b/rorapi/v2/models/client.py @@ -0,0 +1,12 @@ +from django.db import models +import uuid + +class Client(models.Model): + email = models.EmailField() + client_id = models.UUIDField(default=uuid.uuid4, unique=True, editable=False) + created_at = models.DateTimeField(auto_now_add=True) + last_request_at = models.DateTimeField(null=True, blank=True) + request_count = models.IntegerField(default=0) + + def __str__(self): + return f"{self.email} - {self.client_id}" diff --git a/rorapi/v2/serializers/client_serializers.py b/rorapi/v2/serializers/client_serializers.py new file mode 100644 index 0000000..39e5866 --- /dev/null +++ b/rorapi/v2/serializers/client_serializers.py @@ -0,0 +1,7 @@ +from rest_framework import serializers +from ..models.client import Client + +class ClientSerializer(serializers.ModelSerializer): + class Meta: + model = Client + fields = ['email'] diff --git a/rorapi/v2/tests.py b/rorapi/v2/tests.py new file mode 100644 index 0000000..5e2c9ef --- /dev/null +++ b/rorapi/v2/tests.py @@ -0,0 +1,11 @@ +from django.test import TestCase +from .models.client import Client + +class ClientTests(TestCase): + def test_client_registration(self): + client = Client.objects.create(email='test@example.com') + self.assertIsNotNone(client.client_id) + + def test_rate_limiting(self): + response = self.client.get('/client-id/', HTTP_CLIENT_ID="INVALID_ID") + self.assertEqual(response.status_code, 429) diff --git a/rorapi/v2/urls.py b/rorapi/v2/urls.py new file mode 100644 index 0000000..54a123b --- /dev/null +++ b/rorapi/v2/urls.py @@ -0,0 +1,7 @@ +from django.urls import path +from .views.client_views import ClientRegistrationView, ValidateClientView + +urlpatterns = [ + path('client-id/', ClientRegistrationView.as_view(), name='client-registration'), + path('validate-client-id//', ValidateClientView.as_view(), name='validate-client-id'), +] diff --git a/rorapi/v2/views/client_views.py b/rorapi/v2/views/client_views.py new file mode 100644 index 0000000..18fdba1 --- /dev/null +++ b/rorapi/v2/views/client_views.py @@ -0,0 +1,27 @@ +from rest_framework import status +from rest_framework.response import Response +from rest_framework.views import APIView +from ..models.client import Client +from ..serializers.client_serializers import ClientSerializer +from django.core.mail import send_mail +from django.utils.timezone import now + +class ClientRegistrationView(APIView): + def post(self, request): + serializer = ClientSerializer(data=request.data) + if serializer.is_valid(): + client = serializer.save() + send_mail( + 'Your ROR API Client ID', + f'Thank you for registering. Your Client ID is: {client.client_id}', + 'support@ror.org', + [client.email], + fail_silently=False, + ) + return Response({'client_id': client.client_id}, status=status.HTTP_201_CREATED) + return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) + +class ValidateClientView(APIView): + def get(self, request, client_id): + client_exists = Client.objects.filter(client_id=client_id).exists() + return Response({'valid': client_exists}, status=status.HTTP_200_OK) From 89a7eeba177b41f288111e707300100daa57af01 Mon Sep 17 00:00:00 2001 From: kaysiz Date: Mon, 10 Mar 2025 14:23:39 +0200 Subject: [PATCH 02/50] remove duplicate import --- rorapi/common/views.py | 1 - 1 file changed, 1 deletion(-) diff --git a/rorapi/common/views.py b/rorapi/common/views.py index ebcfa21..62506cd 100644 --- a/rorapi/common/views.py +++ b/rorapi/common/views.py @@ -37,7 +37,6 @@ import os import update_address as ua from rorapi.management.commands.generaterorid import check_ror_id -from rorapi.management.commands.generaterorid import check_ror_id from rorapi.management.commands.indexror import process_files from django.core import management import rorapi.management.commands.indexrordump From 16c6ff835abe010e78269c67c8eba3134e436d06 Mon Sep 17 00:00:00 2001 From: kaysiz Date: Mon, 10 Mar 2025 15:13:02 +0200 Subject: [PATCH 03/50] Add method to generate 32 character client_id --- rorapi/management/commands/generaterorid.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/rorapi/management/commands/generaterorid.py b/rorapi/management/commands/generaterorid.py index 2ea42a5..af07be0 100644 --- a/rorapi/management/commands/generaterorid.py +++ b/rorapi/management/commands/generaterorid.py @@ -26,3 +26,10 @@ def check_ror_id(version): check_ror_id(version) return ror_id + +def generate_ror_client_id(): + """Generates a random ROR client ID. + """ + + n = random.randint(0, 2**160 - 1) + return base32_crockford.encode(n).lower().zfill(32) From cf5205cf2f44b57794b28658b49a53b85add32a6 Mon Sep 17 00:00:00 2001 From: kaysiz Date: Tue, 11 Mar 2025 09:16:31 +0200 Subject: [PATCH 04/50] Add mysql db for local dev --- Dockerfile | 3 ++- docker-compose.yml | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 9ebc84e..09b18d6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,7 @@ RUN mv /etc/apt/sources.list.d /etc/apt/sources.list.d.bak && \ mv /etc/apt/sources.list.d.bak /etc/apt/sources.list.d && \ apt-get upgrade -y -o Dpkg::Options::="--force-confold" && \ apt-get clean && \ - apt-get install ntp wget unzip tzdata python3-pip libmagic1 -y && \ + apt-get install ntp wget unzip tzdata python3-pip libmagic1 default-libmysqlclient-dev libcairo2-dev pkg-config -y && \ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* # Enable Passenger and Nginx and remove the default site @@ -54,6 +54,7 @@ RUN pip3 install --no-cache-dir -r requirements.txt RUN pip3 install yapf # collect static files for Django +ENV DJANGO_SKIP_DB_CHECK=True RUN python manage.py collectstatic --noinput # Expose web diff --git a/docker-compose.yml b/docker-compose.yml index 0cd1b0b..1d74476 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,6 +19,21 @@ services: timeout: 1s volumes: - ./esdata:/usr/share/elasticsearch/data + db: + image: mysql:8.0 + volumes: + - mysql_data:/var/lib/mysql + environment: + - MYSQL_DATABASE=rorapi + - MYSQL_USER=ror_user + - MYSQL_PASSWORD=password + - MYSQL_ROOT_PASSWORD=password + ports: + - "3306:3306" + healthcheck: + test: ["CMD", "mysqladmin", "ping", "-h", "localhost"] + timeout: 20s + retries: 10 web: container_name: rorapiweb env_file: .env @@ -31,3 +46,6 @@ services: - ./rorapi:/home/app/webapp/rorapi depends_on: - elasticsearch7 + - db +volumes: + mysql_data: From 9a7033bab9cb6683b30081460ee3e3d208724348 Mon Sep 17 00:00:00 2001 From: kaysiz Date: Tue, 11 Mar 2025 09:16:57 +0200 Subject: [PATCH 05/50] Add mysql library --- requirements.txt | 74 +++++++++++++++++++++++++++++++++++------------- 1 file changed, 54 insertions(+), 20 deletions(-) diff --git a/requirements.txt b/requirements.txt index 5844ea8..5e13d9e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,27 +1,61 @@ +attrs==24.3.0 +base32-crockford==0.3.0 +boto3==1.35.99 +botocore==1.35.99 +certifi==2019.11.28 +chardet==3.0.4 +coreapi==2.3.3 +coreschema==0.0.4 +dbus-python==1.2.16 Django==2.2.28 -elasticsearch_dsl==7.4.1 +django-cors-headers==3.1.0 +django-prometheus==1.0.15 +djangorestframework==3.11.2 +elasticsearch==7.10.1 +elasticsearch-dsl==7.4.1 +expiringdict==1.2.2 +fuzzywuzzy==0.17.0 geonamescache==1.3.0 -requests==2.22.0 -requests-aws4auth==0.9 +idna==2.8 +importlib_resources==6.4.5 +iso639-lang==2.5.1 +itypes==1.2.0 +Jinja2==3.1.5 +jmespath==1.0.1 +jsonschema==3.2.0 +launchdarkly-server-sdk==7.6.1 +MarkupSafe==2.1.5 mock==3.0.5 -base32_crockford==0.3.0 -elasticsearch==7.10.1 -djangorestframework==3.11.2 -coreapi==2.3.3 -django-prometheus==1.0.15 -sentry-sdk==0.12.2 +mysqlclient==2.2.7 +numpy==1.22.0 +pandas==1.4.1 +patsy==1.0.1 +platformdirs==4.3.6 +prometheus_client==0.21.1 +PyGObject==3.36.0 +pyRFC3339==2.0.1 +pyrsistent==0.20.0 +python-apt==2.0.1+ubuntu0.20.4.1 +python-dateutil==2.9.0.post0 python-dotenv==0.10.3 -django-cors-headers==3.1.0 -unidecode==1.1.1 -fuzzywuzzy==0.17.0 python-Levenshtein==0.12.1 +python-magic==0.4.27 +pytz==2024.2 +requests==2.22.0 +requests-aws4auth==0.9 +requests-unixsocket==0.2.0 +s3transfer==0.10.4 +scipy==1.10.1 +semver==2.13.0 +sentry-sdk==0.12.2 +six==1.14.0 +sqlparse==0.5.3 statsmodels==0.10.2 -boto3 -pandas==1.4.1 -numpy==1.22 titlecase==2.3 -update_address @ git+https://github.com/ror-community/update_address.git@v2-1-locations -launchdarkly-server-sdk==7.6.1 -jsonschema==3.2.0 -python-magic -iso639-lang \ No newline at end of file +tomli==2.2.1 +Unidecode==1.1.1 +update-address @ git+https://github.com/ror-community/update_address.git@5b088906e788a8539c8bd1a1b25829610dd2e331 +uritemplate==4.1.1 +urllib3==1.25.8 +yapf==0.43.0 +zipp==3.20.2 \ No newline at end of file From f22d8cbb7ad9f7cdb40770dcb22c70c3a1a564c5 Mon Sep 17 00:00:00 2001 From: kaysiz Date: Tue, 11 Mar 2025 09:17:27 +0200 Subject: [PATCH 06/50] Add db connection settings --- rorapi/settings.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/rorapi/settings.py b/rorapi/settings.py index 87cd611..86295c1 100644 --- a/rorapi/settings.py +++ b/rorapi/settings.py @@ -11,6 +11,7 @@ """ import os +import sys import json import sentry_sdk import boto3 @@ -106,7 +107,23 @@ # Database # https://docs.djangoproject.com/en/2.2/ref/settings/#databases -DATABASES = {} +if 'collectstatic' in sys.argv and os.environ.get('DJANGO_SKIP_DB_CHECK') == 'True': + DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.dummy' + } + } +else: + DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.mysql', + 'NAME': os.environ.get('DB_NAME', 'rorapi'), + 'USER': os.environ.get('DB_USER', 'root'), + 'PASSWORD': os.environ.get('DB_PASSWORD', 'password'), + 'HOST': os.environ.get('DB_HOST', 'db'), + 'PORT': os.environ.get('DB_PORT', '3306'), + } +} # Password validation # https://docs.djangoproject.com/en/2.2/ref/settings/#auth-password-validators From d2175320a64e39691d5c0304f97fd5609d5962bc Mon Sep 17 00:00:00 2001 From: kaysiz Date: Tue, 11 Mar 2025 09:19:08 +0200 Subject: [PATCH 07/50] move model to models.py to fix circular ddependency issue --- rorapi/v2/models.py | 28 +++++++++++++++++++++++++++- rorapi/v2/models/client.py | 12 ------------ 2 files changed, 27 insertions(+), 13 deletions(-) delete mode 100644 rorapi/v2/models/client.py diff --git a/rorapi/v2/models.py b/rorapi/v2/models.py index e9c7d1a..a29174a 100644 --- a/rorapi/v2/models.py +++ b/rorapi/v2/models.py @@ -1,4 +1,5 @@ from geonamescache.mappers import country +from django.db import models from rorapi.common.models import TypeBucket, CountryBucket, StatusBucket, Entity from rorapi.v2.record_constants import continent_code_to_name @@ -130,4 +131,29 @@ class MatchingResult: def __init__(self, data): self.number_of_results = len(data) - self.items = [MatchedOrganization(x) for x in data] \ No newline at end of file + self.items = [MatchedOrganization(x) for x in data] + + +class Client(models.Model): + # Required field + email = models.EmailField(max_length=255) + + # Optional fields + name = models.CharField(max_length=255, blank=True) + institution_name = models.CharField(max_length=255, blank=True) + institution_ror = models.URLField(max_length=255, blank=True) + country_code = models.CharField(max_length=2, blank=True) + ror_use = models.TextField(max_length=500, blank=True) + + # System fields + client_id = models.CharField( + max_length=32, + unique=True, + editable=False + ) + created_at = models.DateTimeField(auto_now_add=True) + last_request_at = models.DateTimeField(null=True, blank=True) + request_count = models.IntegerField(default=0) + + def __str__(self): + return f"{self.email} - {self.client_id}" \ No newline at end of file diff --git a/rorapi/v2/models/client.py b/rorapi/v2/models/client.py deleted file mode 100644 index d1d1899..0000000 --- a/rorapi/v2/models/client.py +++ /dev/null @@ -1,12 +0,0 @@ -from django.db import models -import uuid - -class Client(models.Model): - email = models.EmailField() - client_id = models.UUIDField(default=uuid.uuid4, unique=True, editable=False) - created_at = models.DateTimeField(auto_now_add=True) - last_request_at = models.DateTimeField(null=True, blank=True) - request_count = models.IntegerField(default=0) - - def __str__(self): - return f"{self.email} - {self.client_id}" From 9c20eecea0de551f1f6710d4d4ff787399e1ed95 Mon Sep 17 00:00:00 2001 From: kaysiz Date: Tue, 11 Mar 2025 09:19:59 +0200 Subject: [PATCH 08/50] move serializer to serializers.py to fix circular dependnecy --- rorapi/v2/serializers.py | 7 +++++++ rorapi/v2/serializers/client_serializers.py | 7 ------- 2 files changed, 7 insertions(+), 7 deletions(-) delete mode 100644 rorapi/v2/serializers/client_serializers.py diff --git a/rorapi/v2/serializers.py b/rorapi/v2/serializers.py index 62327a5..7e9eb1a 100644 --- a/rorapi/v2/serializers.py +++ b/rorapi/v2/serializers.py @@ -1,4 +1,5 @@ from rest_framework import serializers +from rorapi.v2.models import Client from rorapi.common.serializers import BucketSerializer, OrganizationRelationshipsSerializer class AggregationsSerializer(serializers.Serializer): @@ -87,3 +88,9 @@ class MatchedOrganizationSerializer(serializers.Serializer): class MatchingResultSerializer(serializers.Serializer): number_of_results = serializers.IntegerField() items = MatchedOrganizationSerializer(many=True) + + +class ClientSerializer(serializers.ModelSerializer): + class Meta: + model = Client + fields = ['email'] \ No newline at end of file diff --git a/rorapi/v2/serializers/client_serializers.py b/rorapi/v2/serializers/client_serializers.py deleted file mode 100644 index 39e5866..0000000 --- a/rorapi/v2/serializers/client_serializers.py +++ /dev/null @@ -1,7 +0,0 @@ -from rest_framework import serializers -from ..models.client import Client - -class ClientSerializer(serializers.ModelSerializer): - class Meta: - model = Client - fields = ['email'] From 95d657999ac83d5b89a5a40dbbfe20d4e83514d2 Mon Sep 17 00:00:00 2001 From: kaysiz Date: Tue, 11 Mar 2025 09:20:41 +0200 Subject: [PATCH 09/50] refactor and update imports --- rorapi/v2/urls.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rorapi/v2/urls.py b/rorapi/v2/urls.py index 54a123b..adebebd 100644 --- a/rorapi/v2/urls.py +++ b/rorapi/v2/urls.py @@ -1,5 +1,6 @@ from django.urls import path -from .views.client_views import ClientRegistrationView, ValidateClientView +from rorapi.v2.views import ClientRegistrationView, ValidateClientView + urlpatterns = [ path('client-id/', ClientRegistrationView.as_view(), name='client-registration'), From 3ca2ff3549395372da6588ce0880eff59779e615 Mon Sep 17 00:00:00 2001 From: kaysiz Date: Tue, 11 Mar 2025 09:21:32 +0200 Subject: [PATCH 10/50] Add migrations for client table --- rorapi/migrations/0001_create_client_model.py | 30 +++++++++++++++++++ rorapi/migrations/__init__.py | 0 2 files changed, 30 insertions(+) create mode 100644 rorapi/migrations/0001_create_client_model.py create mode 100644 rorapi/migrations/__init__.py diff --git a/rorapi/migrations/0001_create_client_model.py b/rorapi/migrations/0001_create_client_model.py new file mode 100644 index 0000000..b7b558f --- /dev/null +++ b/rorapi/migrations/0001_create_client_model.py @@ -0,0 +1,30 @@ +# Generated by Django 2.2.28 on 2025-03-11 07:13 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='Client', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('email', models.EmailField(max_length=255)), + ('name', models.CharField(blank=True, max_length=255)), + ('institution_name', models.CharField(blank=True, max_length=255)), + ('institution_ror', models.URLField(blank=True, max_length=255)), + ('country_code', models.CharField(blank=True, max_length=2)), + ('ror_use', models.TextField(blank=True, max_length=500)), + ('client_id', models.CharField(editable=False, max_length=32, unique=True)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('last_request_at', models.DateTimeField(blank=True, null=True)), + ('request_count', models.IntegerField(default=0)), + ], + ), + ] diff --git a/rorapi/migrations/__init__.py b/rorapi/migrations/__init__.py new file mode 100644 index 0000000..e69de29 From f5cce3e6f2bba71b645e33ffaad1ffd1e63c18a8 Mon Sep 17 00:00:00 2001 From: kaysiz Date: Tue, 11 Mar 2025 09:22:15 +0200 Subject: [PATCH 11/50] rename views file and make folder a package --- rorapi/v2/views/__init__.py | 1 + rorapi/v2/views/{client_views.py => client.py} | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 rorapi/v2/views/__init__.py rename rorapi/v2/views/{client_views.py => client.py} (91%) diff --git a/rorapi/v2/views/__init__.py b/rorapi/v2/views/__init__.py new file mode 100644 index 0000000..d87d8eb --- /dev/null +++ b/rorapi/v2/views/__init__.py @@ -0,0 +1 @@ +from .client import ClientRegistrationView, ValidateClientView \ No newline at end of file diff --git a/rorapi/v2/views/client_views.py b/rorapi/v2/views/client.py similarity index 91% rename from rorapi/v2/views/client_views.py rename to rorapi/v2/views/client.py index 18fdba1..feda669 100644 --- a/rorapi/v2/views/client_views.py +++ b/rorapi/v2/views/client.py @@ -1,8 +1,8 @@ from rest_framework import status from rest_framework.response import Response from rest_framework.views import APIView -from ..models.client import Client -from ..serializers.client_serializers import ClientSerializer +from rorapi.v2.models import Client +from rorapi.v2.serializers import ClientSerializer from django.core.mail import send_mail from django.utils.timezone import now From 76f96754e0c90dd2e9c38f754b499599e220050e Mon Sep 17 00:00:00 2001 From: Ashwini Sukale Date: Thu, 20 Mar 2025 15:35:13 +0530 Subject: [PATCH 12/50] Send email and added more validation --- rorapi/settings.py | 9 +++++++ rorapi/v2/serializers.py | 8 ++++++- rorapi/v2/views/{client.py => views.py} | 31 ++++++++++++++++++++----- 3 files changed, 41 insertions(+), 7 deletions(-) rename rorapi/v2/views/{client.py => views.py} (55%) diff --git a/rorapi/settings.py b/rorapi/settings.py index 86295c1..40a50c4 100644 --- a/rorapi/settings.py +++ b/rorapi/settings.py @@ -295,3 +295,12 @@ # Toggle for behavior-based rate limiting import os ENABLE_BEHAVIORAL_LIMITING = os.getenv("ENABLE_BEHAVIORAL_LIMITING", "False") == "True" + +# Email settings for Django +EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend' +EMAIL_HOST = 'smtp.mailtrap.io' # Change this to your SMTP server +EMAIL_PORT = 587 +EMAIL_USE_TLS = True +EMAIL_HOST_USER = 'your-email@example.com' # Your SMTP username +EMAIL_HOST_PASSWORD = 'your-email-password' # Your SMTP password +DEFAULT_FROM_EMAIL = 'support@ror.org' \ No newline at end of file diff --git a/rorapi/v2/serializers.py b/rorapi/v2/serializers.py index 7e9eb1a..4cfc2a4 100644 --- a/rorapi/v2/serializers.py +++ b/rorapi/v2/serializers.py @@ -93,4 +93,10 @@ class MatchingResultSerializer(serializers.Serializer): class ClientSerializer(serializers.ModelSerializer): class Meta: model = Client - fields = ['email'] \ No newline at end of file + fields = ['email', 'name', 'institution_name', 'institution_ror', 'country_code', 'ror_use'] + + def validate_email(self, value): + """Validate the email format and ensure it's unique.""" + if Client.objects.filter(email=value).exists(): + raise serializers.ValidationError("A client with this email already exists.") + return value \ No newline at end of file diff --git a/rorapi/v2/views/client.py b/rorapi/v2/views/views.py similarity index 55% rename from rorapi/v2/views/client.py rename to rorapi/v2/views/views.py index feda669..b28f5f8 100644 --- a/rorapi/v2/views/client.py +++ b/rorapi/v2/views/views.py @@ -1,27 +1,46 @@ from rest_framework import status from rest_framework.response import Response from rest_framework.views import APIView -from rorapi.v2.models import Client -from rorapi.v2.serializers import ClientSerializer from django.core.mail import send_mail from django.utils.timezone import now +from rorapi.v2.models import Client +from rorapi.v2.serializers import ClientSerializer class ClientRegistrationView(APIView): def post(self, request): + # Initialize serializer with request data serializer = ClientSerializer(data=request.data) + + # Check if the data is valid if serializer.is_valid(): + # Save the client instance client = serializer.save() + + # Send a registration email to the client + subject = 'Your ROR API Client ID' + message = f'Thank you for registering. Your Client ID is: {client.client_id}' + from_email = 'support@ror.org' + recipient_list = [client.email] + send_mail( - 'Your ROR API Client ID', - f'Thank you for registering. Your Client ID is: {client.client_id}', - 'support@ror.org', - [client.email], + subject, + message, + from_email, + recipient_list, fail_silently=False, ) + + # Return response with client ID return Response({'client_id': client.client_id}, status=status.HTTP_201_CREATED) + + # Return validation errors if serializer is not valid return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) + class ValidateClientView(APIView): def get(self, request, client_id): + # Check if the client_id exists in the database client_exists = Client.objects.filter(client_id=client_id).exists() + + # Return response indicating whether client ID is valid return Response({'valid': client_exists}, status=status.HTTP_200_OK) From 65461abe441dc096e82e254ff4a3df96e191013d Mon Sep 17 00:00:00 2001 From: Ashwini Sukale Date: Mon, 24 Mar 2025 18:37:09 +0530 Subject: [PATCH 13/50] Some refactoring in api structure --- rorapi/common/urls.py | 3 ++- rorapi/common/views.py | 29 ++++++++++++++++++++++++ rorapi/middleware.py | 2 +- rorapi/v2/models.py | 19 +++++++++++++--- rorapi/v2/urls.py | 2 +- rorapi/v2/views/client.py | 46 +++++++++++++++++++++++++++++++++++++++ rorapi/v2/views/views.py | 46 --------------------------------------- 7 files changed, 95 insertions(+), 52 deletions(-) create mode 100644 rorapi/v2/views/client.py delete mode 100644 rorapi/v2/views/views.py diff --git a/rorapi/common/urls.py b/rorapi/common/urls.py index b9aa57c..813c51d 100644 --- a/rorapi/common/urls.py +++ b/rorapi/common/urls.py @@ -3,7 +3,7 @@ from rest_framework.documentation import include_docs_urls from . import views from rorapi.common.views import ( - HeartbeatView,GenerateAddress,GenerateId,IndexData,IndexDataDump,BulkUpdate) + HeartbeatView,GenerateAddress,GenerateId,IndexData,IndexDataDump,BulkUpdate,ClientRegistrationView) urlpatterns = [ # Health check @@ -14,6 +14,7 @@ path('generateaddress/', GenerateAddress.as_view()), url(r"^generateid$", GenerateId.as_view()), re_path(r"^(?P(v1|v2))\/bulkupdate$", BulkUpdate.as_view()), + re_path(r"^(?P(v1|v2))\/register$", ClientRegistrationView.as_view()), url(r"^(?P(v1|v2))\/indexdata/(?P.*)", IndexData.as_view()), url(r"^(?P(v1|v2))\/indexdatadump\/(?Pv(\d+\.)?(\d+\.)?(\*|\d+)-\d{4}-\d{2}-\d{2}-ror-data)\/(?P(test|prod))$", IndexDataDump.as_view()), url(r"^(?P(v1|v2))\/", include(views.organizations_router.urls)), diff --git a/rorapi/common/views.py b/rorapi/common/views.py index 62506cd..24f38d6 100644 --- a/rorapi/common/views.py +++ b/rorapi/common/views.py @@ -40,6 +40,35 @@ from rorapi.management.commands.indexror import process_files from django.core import management import rorapi.management.commands.indexrordump +from django.core.mail import send_mail +from django.utils.timezone import now +from rorapi.v2.models import Client +from rorapi.v2.serializers import ClientSerializer + +class ClientRegistrationView(APIView): + def post(self, request, version='v2'): + serializer = ClientSerializer(data=request.data) + if serializer.is_valid(): + client = serializer.save() + + # Send a registration email to the client + subject = 'Your ROR API Client ID' + message = f'Thank you for registering. Your Client ID is: {client.client_id}' + from_email = 'support@ror.org' + recipient_list = [client.email] + + send_mail( + subject, + message, + from_email, + recipient_list, + fail_silently=False, + ) + + return Response({'client_id': client.client_id}, status=status.HTTP_201_CREATED) + + # Return validation errors if serializer is not valid + return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) class OurTokenPermission(BasePermission): """ diff --git a/rorapi/middleware.py b/rorapi/middleware.py index a4800af..3e114c7 100644 --- a/rorapi/middleware.py +++ b/rorapi/middleware.py @@ -1,6 +1,6 @@ from django.core.cache import cache from django.http import JsonResponse -from rorapi.v2.models.client import Client +from rorapi.v2.models import Client from django.utils.timezone import now import os diff --git a/rorapi/v2/models.py b/rorapi/v2/models.py index a29174a..18589aa 100644 --- a/rorapi/v2/models.py +++ b/rorapi/v2/models.py @@ -1,4 +1,6 @@ from geonamescache.mappers import country +import random +import string from django.db import models from rorapi.common.models import TypeBucket, CountryBucket, StatusBucket, Entity from rorapi.v2.record_constants import continent_code_to_name @@ -135,8 +137,8 @@ def __init__(self, data): class Client(models.Model): - # Required field - email = models.EmailField(max_length=255) + # Required fields + email = models.EmailField(max_length=255, unique=True) # Optional fields name = models.CharField(max_length=255, blank=True) @@ -156,4 +158,15 @@ class Client(models.Model): request_count = models.IntegerField(default=0) def __str__(self): - return f"{self.email} - {self.client_id}" \ No newline at end of file + return f"{self.email} - {self.client_id}" + + @staticmethod + def generate_client_id(): + """Generate a unique 32-character client ID""" + return ''.join(random.choices(string.ascii_uppercase + string.digits, k=32)) + + def save(self, *args, **kwargs): + # Ensure client_id is generated before saving + if not self.client_id: # Only generate if it's empty + self.client_id = self.generate_client_id() + super().save(*args, **kwargs) diff --git a/rorapi/v2/urls.py b/rorapi/v2/urls.py index adebebd..d5fd35c 100644 --- a/rorapi/v2/urls.py +++ b/rorapi/v2/urls.py @@ -3,6 +3,6 @@ urlpatterns = [ - path('client-id/', ClientRegistrationView.as_view(), name='client-registration'), + #path('register/', ClientRegistrationView.as_view(), name='client-registration'), path('validate-client-id//', ValidateClientView.as_view(), name='validate-client-id'), ] diff --git a/rorapi/v2/views/client.py b/rorapi/v2/views/client.py new file mode 100644 index 0000000..92689e5 --- /dev/null +++ b/rorapi/v2/views/client.py @@ -0,0 +1,46 @@ +from rest_framework import status +from rest_framework.response import Response +from rest_framework.views import APIView +from django.core.mail import send_mail +from django.utils.timezone import now +from rorapi.v2.models import Client +from rorapi.v2.serializers import ClientSerializer + +# class ClientRegistrationView(APIView): +# def post(self, request): +# # Initialize serializer with request data +# serializer = ClientSerializer(data=request.data) + +# # Check if the data is valid +# if serializer.is_valid(): +# # Save the client instance +# client = serializer.save() + +# # Send a registration email to the client +# subject = 'Your ROR API Client ID' +# message = f'Thank you for registering. Your Client ID is: {client.client_id}' +# from_email = 'support@ror.org' +# recipient_list = [client.email] + +# send_mail( +# subject, +# message, +# from_email, +# recipient_list, +# fail_silently=False, +# ) + +# # Return response with client ID +# return Response({'client_id': client.client_id}, status=status.HTTP_201_CREATED) + +# # Return validation errors if serializer is not valid +# return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) + + +class ValidateClientView(APIView): + def get(self, request, client_id): + # Check if the client_id exists in the database + client_exists = Client.objects.filter(client_id=client_id).exists() + + # Return response indicating whether client ID is valid + return Response({'valid': client_exists}, status=status.HTTP_200_OK) diff --git a/rorapi/v2/views/views.py b/rorapi/v2/views/views.py deleted file mode 100644 index b28f5f8..0000000 --- a/rorapi/v2/views/views.py +++ /dev/null @@ -1,46 +0,0 @@ -from rest_framework import status -from rest_framework.response import Response -from rest_framework.views import APIView -from django.core.mail import send_mail -from django.utils.timezone import now -from rorapi.v2.models import Client -from rorapi.v2.serializers import ClientSerializer - -class ClientRegistrationView(APIView): - def post(self, request): - # Initialize serializer with request data - serializer = ClientSerializer(data=request.data) - - # Check if the data is valid - if serializer.is_valid(): - # Save the client instance - client = serializer.save() - - # Send a registration email to the client - subject = 'Your ROR API Client ID' - message = f'Thank you for registering. Your Client ID is: {client.client_id}' - from_email = 'support@ror.org' - recipient_list = [client.email] - - send_mail( - subject, - message, - from_email, - recipient_list, - fail_silently=False, - ) - - # Return response with client ID - return Response({'client_id': client.client_id}, status=status.HTTP_201_CREATED) - - # Return validation errors if serializer is not valid - return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) - - -class ValidateClientView(APIView): - def get(self, request, client_id): - # Check if the client_id exists in the database - client_exists = Client.objects.filter(client_id=client_id).exists() - - # Return response indicating whether client ID is valid - return Response({'valid': client_exists}, status=status.HTTP_200_OK) From 0d7de832439dad6ab2c4ad331ecff76fda3926e3 Mon Sep 17 00:00:00 2001 From: Ashwini Sukale Date: Tue, 25 Mar 2025 16:34:50 +0530 Subject: [PATCH 14/50] Moved the urls to common urls and now it is working --- rorapi/common/urls.py | 3 ++- rorapi/common/views.py | 22 +++++++++++++------ rorapi/v2/urls.py | 8 ------- rorapi/v2/views/client.py | 46 --------------------------------------- 4 files changed, 17 insertions(+), 62 deletions(-) delete mode 100644 rorapi/v2/urls.py delete mode 100644 rorapi/v2/views/client.py diff --git a/rorapi/common/urls.py b/rorapi/common/urls.py index 813c51d..277e0bd 100644 --- a/rorapi/common/urls.py +++ b/rorapi/common/urls.py @@ -3,7 +3,7 @@ from rest_framework.documentation import include_docs_urls from . import views from rorapi.common.views import ( - HeartbeatView,GenerateAddress,GenerateId,IndexData,IndexDataDump,BulkUpdate,ClientRegistrationView) + HeartbeatView,GenerateAddress,GenerateId,IndexData,IndexDataDump,BulkUpdate,ClientRegistrationView,ValidateClientView) urlpatterns = [ # Health check @@ -15,6 +15,7 @@ url(r"^generateid$", GenerateId.as_view()), re_path(r"^(?P(v1|v2))\/bulkupdate$", BulkUpdate.as_view()), re_path(r"^(?P(v1|v2))\/register$", ClientRegistrationView.as_view()), + path('validate-client-id//', ValidateClientView.as_view()), url(r"^(?P(v1|v2))\/indexdata/(?P.*)", IndexData.as_view()), url(r"^(?P(v1|v2))\/indexdatadump\/(?Pv(\d+\.)?(\d+\.)?(\*|\d+)-\d{4}-\d{2}-\d{2}-ror-data)\/(?P(test|prod))$", IndexDataDump.as_view()), url(r"^(?P(v1|v2))\/", include(views.organizations_router.urls)), diff --git a/rorapi/common/views.py b/rorapi/common/views.py index 24f38d6..8d58402 100644 --- a/rorapi/common/views.py +++ b/rorapi/common/views.py @@ -57,19 +57,27 @@ def post(self, request, version='v2'): from_email = 'support@ror.org' recipient_list = [client.email] - send_mail( - subject, - message, - from_email, - recipient_list, - fail_silently=False, - ) + # send_mail( + # subject, + # message, + # from_email, + # recipient_list, + # fail_silently=False, + # ) return Response({'client_id': client.client_id}, status=status.HTTP_201_CREATED) # Return validation errors if serializer is not valid return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) +class ValidateClientView(APIView): + def get(self, request, client_id): + # Check if the client_id exists in the database + client_exists = Client.objects.filter(client_id=client_id).exists() + + # Return response indicating whether client ID is valid + return Response({'valid': client_exists}, status=status.HTTP_200_OK) + class OurTokenPermission(BasePermission): """ Allows access only to using our token and user name. diff --git a/rorapi/v2/urls.py b/rorapi/v2/urls.py deleted file mode 100644 index d5fd35c..0000000 --- a/rorapi/v2/urls.py +++ /dev/null @@ -1,8 +0,0 @@ -from django.urls import path -from rorapi.v2.views import ClientRegistrationView, ValidateClientView - - -urlpatterns = [ - #path('register/', ClientRegistrationView.as_view(), name='client-registration'), - path('validate-client-id//', ValidateClientView.as_view(), name='validate-client-id'), -] diff --git a/rorapi/v2/views/client.py b/rorapi/v2/views/client.py deleted file mode 100644 index 92689e5..0000000 --- a/rorapi/v2/views/client.py +++ /dev/null @@ -1,46 +0,0 @@ -from rest_framework import status -from rest_framework.response import Response -from rest_framework.views import APIView -from django.core.mail import send_mail -from django.utils.timezone import now -from rorapi.v2.models import Client -from rorapi.v2.serializers import ClientSerializer - -# class ClientRegistrationView(APIView): -# def post(self, request): -# # Initialize serializer with request data -# serializer = ClientSerializer(data=request.data) - -# # Check if the data is valid -# if serializer.is_valid(): -# # Save the client instance -# client = serializer.save() - -# # Send a registration email to the client -# subject = 'Your ROR API Client ID' -# message = f'Thank you for registering. Your Client ID is: {client.client_id}' -# from_email = 'support@ror.org' -# recipient_list = [client.email] - -# send_mail( -# subject, -# message, -# from_email, -# recipient_list, -# fail_silently=False, -# ) - -# # Return response with client ID -# return Response({'client_id': client.client_id}, status=status.HTTP_201_CREATED) - -# # Return validation errors if serializer is not valid -# return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) - - -class ValidateClientView(APIView): - def get(self, request, client_id): - # Check if the client_id exists in the database - client_exists = Client.objects.filter(client_id=client_id).exists() - - # Return response indicating whether client ID is valid - return Response({'valid': client_exists}, status=status.HTTP_200_OK) From e4fbc2b85fffbdd65ab1239600192359f9ef8497 Mon Sep 17 00:00:00 2001 From: kaysiz Date: Wed, 26 Mar 2025 15:28:14 +0200 Subject: [PATCH 15/50] add migration for update of the of the email attribute on client --- rorapi/migrations/0002_auto_20250326_1054.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 rorapi/migrations/0002_auto_20250326_1054.py diff --git a/rorapi/migrations/0002_auto_20250326_1054.py b/rorapi/migrations/0002_auto_20250326_1054.py new file mode 100644 index 0000000..0d06ed9 --- /dev/null +++ b/rorapi/migrations/0002_auto_20250326_1054.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.28 on 2025-03-26 10:54 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('rorapi', '0001_create_client_model'), + ] + + operations = [ + migrations.AlterField( + model_name='client', + name='email', + field=models.EmailField(max_length=255, unique=True), + ), + ] From d3d44ea69c50ce10b431fe78ccb986081e6cfa44 Mon Sep 17 00:00:00 2001 From: Ashwini Sukale Date: Thu, 27 Mar 2025 08:59:57 +0530 Subject: [PATCH 16/50] Rate limit part will be done next release --- rorapi/middleware.py | 43 ------------------------------------- rorapi/settings.py | 3 +-- rorapi/v2/views/__init__.py | 1 - 3 files changed, 1 insertion(+), 46 deletions(-) delete mode 100644 rorapi/middleware.py delete mode 100644 rorapi/v2/views/__init__.py diff --git a/rorapi/middleware.py b/rorapi/middleware.py deleted file mode 100644 index 3e114c7..0000000 --- a/rorapi/middleware.py +++ /dev/null @@ -1,43 +0,0 @@ -from django.core.cache import cache -from django.http import JsonResponse -from rorapi.v2.models import Client -from django.utils.timezone import now -import os - -# Toggle Behavior-Based Rate Limiting -ENABLE_BEHAVIORAL_LIMITING = os.getenv("ENABLE_BEHAVIORAL_LIMITING", "False") == "True" - -class ClientRateLimitMiddleware: - def __init__(self, get_response): - self.get_response = get_response - - def __call__(self, request): - client_id = request.headers.get('Client-Id') - ip_address = request.META.get('REMOTE_ADDR') - rate_limit = 50 # Default rate limit - - if client_id: - try: - client = Client.objects.get(client_id=client_id) - rate_limit = 2000 # Higher limit for registered users - - # Behavior-based throttling (if enabled) - if ENABLE_BEHAVIORAL_LIMITING: - client.request_count += 1 - client.last_request_at = now() - client.save() - - if client.request_count > 500 and (now() - client.last_request_at).seconds < 300: - return JsonResponse({"error": "Rate limit exceeded due to excessive requests"}, status=429) - - except Client.DoesNotExist: - rate_limit = 50 # Invalid client ID gets the lower limit - - cache_key = f"rate_limit_{client_id or ip_address}" - request_count = cache.get(cache_key, 0) - - if request_count >= rate_limit: - return JsonResponse({"error": "Rate limit exceeded"}, status=429) - - cache.set(cache_key, request_count + 1, timeout=300) # Reset every 5 min - return self.get_response(request) diff --git a/rorapi/settings.py b/rorapi/settings.py index 40a50c4..5e8607b 100644 --- a/rorapi/settings.py +++ b/rorapi/settings.py @@ -72,8 +72,7 @@ 'django.contrib.auth.middleware.AuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', - 'django_prometheus.middleware.PrometheusAfterMiddleware', - 'rorapi.middleware.ClientRateLimitMiddleware', + 'django_prometheus.middleware.PrometheusAfterMiddleware' ] ROOT_URLCONF = 'rorapi.common.urls' diff --git a/rorapi/v2/views/__init__.py b/rorapi/v2/views/__init__.py deleted file mode 100644 index d87d8eb..0000000 --- a/rorapi/v2/views/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .client import ClientRegistrationView, ValidateClientView \ No newline at end of file From 9f8e5243da1ac3c7220e00798231ac4114c7d5e9 Mon Sep 17 00:00:00 2001 From: Ashwini Sukale Date: Thu, 27 Mar 2025 09:09:10 +0530 Subject: [PATCH 17/50] moved the variables to env file --- docker-compose.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 1d74476..253e3c3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -23,11 +23,8 @@ services: image: mysql:8.0 volumes: - mysql_data:/var/lib/mysql - environment: - - MYSQL_DATABASE=rorapi - - MYSQL_USER=ror_user - - MYSQL_PASSWORD=password - - MYSQL_ROOT_PASSWORD=password + env_file: + - .env ports: - "3306:3306" healthcheck: From 927d54e1a5a4a0e02d7aff267a5b320a7f333697 Mon Sep 17 00:00:00 2001 From: Ashwini Sukale Date: Tue, 15 Apr 2025 12:05:44 +0530 Subject: [PATCH 18/50] Addressed code review comments --- rorapi/tests/tests_unit/tests_client.py | 11 +++++++ rorapi/v2/models.py | 2 +- rorapi/v2/serializers.py | 41 ++++++++++++++++++++++++- 3 files changed, 52 insertions(+), 2 deletions(-) create mode 100644 rorapi/tests/tests_unit/tests_client.py diff --git a/rorapi/tests/tests_unit/tests_client.py b/rorapi/tests/tests_unit/tests_client.py new file mode 100644 index 0000000..eb1f9d0 --- /dev/null +++ b/rorapi/tests/tests_unit/tests_client.py @@ -0,0 +1,11 @@ +from django.test import TestCase +from rorapi.v2.models.client import Client + +class ClientTests(TestCase): + def test_client_registration(self): + client = Client.objects.create(email='test@example.com') + self.assertIsNotNone(client.client_id) + + def test_rate_limiting(self): + response = self.client.get('/client-id/', HTTP_CLIENT_ID="INVALID_ID") + self.assertEqual(response.status_code, 429) \ No newline at end of file diff --git a/rorapi/v2/models.py b/rorapi/v2/models.py index 18589aa..01fa59e 100644 --- a/rorapi/v2/models.py +++ b/rorapi/v2/models.py @@ -138,7 +138,7 @@ def __init__(self, data): class Client(models.Model): # Required fields - email = models.EmailField(max_length=255, unique=True) + email = models.EmailField(max_length=255) # Optional fields name = models.CharField(max_length=255, blank=True) diff --git a/rorapi/v2/serializers.py b/rorapi/v2/serializers.py index 4cfc2a4..0d09b23 100644 --- a/rorapi/v2/serializers.py +++ b/rorapi/v2/serializers.py @@ -1,4 +1,7 @@ from rest_framework import serializers +import bleach +import pycountry +import re from rorapi.v2.models import Client from rorapi.common.serializers import BucketSerializer, OrganizationRelationshipsSerializer @@ -99,4 +102,40 @@ def validate_email(self, value): """Validate the email format and ensure it's unique.""" if Client.objects.filter(email=value).exists(): raise serializers.ValidationError("A client with this email already exists.") - return value \ No newline at end of file + return value + + def validate_name(self, value): + """Sanitize name and validate length.""" + value = bleach.clean(value) # Sanitize to strip HTML + if len(value) > 255: + raise serializers.ValidationError("Name cannot be longer than 255 characters.") + return value + + def validate_institution_name(self, value): + """Sanitize institution name and validate length.""" + value = bleach.clean(value) # Sanitize to strip HTML + if len(value) > 255: + raise serializers.ValidationError("Institution name cannot be longer than 255 characters.") + return value + + def validate_institution_ror(self, value): + """Validate and format institution ROR to match 'https://ror.org/XXXXX'.""" + value = bleach.clean(value) # Sanitize to strip HTML + ror_regex = r'https://ror\.org/[A-Za-z0-9]+' + if not re.match(ror_regex, value): + raise serializers.ValidationError("Institution ROR must be in the format 'https://ror.org/XXXXX'.") + return value + + def validate_country_code(self, value): + """Validate that the country code is a valid ISO 3166-1 alpha-2 country code.""" + value = value.strip().upper() # Normalize to uppercase + if len(value) != 2 or not pycountry.countries.get(alpha_2=value): + raise serializers.ValidationError(f"{value} is not a valid ISO 3166-1 alpha-2 country code.") + return value + + def validate_ror_use(self, value): + """Sanitize ror_use and validate length.""" + value = bleach.clean(value) # Sanitize to strip HTML + if len(value) > 500: + raise serializers.ValidationError("ROR use cannot be longer than 500 characters.") + return value From 724df3ae1b92c19a03d6593c5778ed06a1d301e4 Mon Sep 17 00:00:00 2001 From: Ashwini Sukale Date: Tue, 15 Apr 2025 17:46:33 +0530 Subject: [PATCH 19/50] Allow null values --- requirements.txt | 2 + rorapi/migrations/0003_auto_20250415_1207.py | 43 +++++++++++++ rorapi/v2/models.py | 10 +-- rorapi/v2/serializers.py | 68 +++++++++++++------- 4 files changed, 95 insertions(+), 28 deletions(-) create mode 100644 rorapi/migrations/0003_auto_20250415_1207.py diff --git a/requirements.txt b/requirements.txt index 3cdd26e..38183fe 100644 --- a/requirements.txt +++ b/requirements.txt @@ -63,3 +63,5 @@ launchdarkly-server-sdk==7.6.1 jsonschema==3.2.0 python-magic iso639-lang +bleach==6.0.0 +pycountry==22.3.5 diff --git a/rorapi/migrations/0003_auto_20250415_1207.py b/rorapi/migrations/0003_auto_20250415_1207.py new file mode 100644 index 0000000..0697d02 --- /dev/null +++ b/rorapi/migrations/0003_auto_20250415_1207.py @@ -0,0 +1,43 @@ +# Generated by Django 2.2.28 on 2025-04-15 12:07 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('rorapi', '0002_auto_20250326_1054'), + ] + + operations = [ + migrations.AlterField( + model_name='client', + name='country_code', + field=models.CharField(blank=True, max_length=2, null=True), + ), + migrations.AlterField( + model_name='client', + name='email', + field=models.EmailField(max_length=255), + ), + migrations.AlterField( + model_name='client', + name='institution_name', + field=models.CharField(blank=True, max_length=255, null=True), + ), + migrations.AlterField( + model_name='client', + name='institution_ror', + field=models.URLField(blank=True, max_length=255, null=True), + ), + migrations.AlterField( + model_name='client', + name='name', + field=models.CharField(blank=True, max_length=255, null=True), + ), + migrations.AlterField( + model_name='client', + name='ror_use', + field=models.TextField(blank=True, max_length=500, null=True), + ), + ] diff --git a/rorapi/v2/models.py b/rorapi/v2/models.py index 01fa59e..1b5a82f 100644 --- a/rorapi/v2/models.py +++ b/rorapi/v2/models.py @@ -141,11 +141,11 @@ class Client(models.Model): email = models.EmailField(max_length=255) # Optional fields - name = models.CharField(max_length=255, blank=True) - institution_name = models.CharField(max_length=255, blank=True) - institution_ror = models.URLField(max_length=255, blank=True) - country_code = models.CharField(max_length=2, blank=True) - ror_use = models.TextField(max_length=500, blank=True) + name = models.CharField(max_length=255, blank=True, null=True) + institution_name = models.CharField(max_length=255, blank=True, null=True) + institution_ror = models.URLField(max_length=255, blank=True, null=True) + country_code = models.CharField(max_length=2, blank=True, null=True) + ror_use = models.TextField(max_length=500, blank=True, null=True) # System fields client_id = models.CharField( diff --git a/rorapi/v2/serializers.py b/rorapi/v2/serializers.py index 0d09b23..002fd8e 100644 --- a/rorapi/v2/serializers.py +++ b/rorapi/v2/serializers.py @@ -97,45 +97,67 @@ class ClientSerializer(serializers.ModelSerializer): class Meta: model = Client fields = ['email', 'name', 'institution_name', 'institution_ror', 'country_code', 'ror_use'] + extra_kwargs = { + 'name': {'required': False, 'allow_null': True}, + 'institution_name': {'required': False, 'allow_null': True}, + 'institution_ror': {'required': False, 'allow_null': True}, + 'country_code': {'required': False, 'allow_null': True}, + 'ror_use': {'required': False, 'allow_null': True}, + } def validate_email(self, value): """Validate the email format and ensure it's unique.""" - if Client.objects.filter(email=value).exists(): - raise serializers.ValidationError("A client with this email already exists.") + if value is None: + raise serializers.ValidationError("Email cannot be null.") return value def validate_name(self, value): - """Sanitize name and validate length.""" - value = bleach.clean(value) # Sanitize to strip HTML - if len(value) > 255: - raise serializers.ValidationError("Name cannot be longer than 255 characters.") + """Sanitize name and validate length. Reject empty string.""" + if value is not None: + if value == "": + raise serializers.ValidationError("Name cannot be an empty string.") + value = bleach.clean(value) # Sanitize to strip HTML + if len(value) > 255: + raise serializers.ValidationError("Name cannot be longer than 255 characters.") return value def validate_institution_name(self, value): - """Sanitize institution name and validate length.""" - value = bleach.clean(value) # Sanitize to strip HTML - if len(value) > 255: - raise serializers.ValidationError("Institution name cannot be longer than 255 characters.") + """Sanitize institution name and validate length. Reject empty string.""" + if value is not None: + if value == "": + raise serializers.ValidationError("Institution name cannot be an empty string.") + value = bleach.clean(value) # Sanitize to strip HTML + if len(value) > 255: + raise serializers.ValidationError("Institution name cannot be longer than 255 characters.") return value def validate_institution_ror(self, value): - """Validate and format institution ROR to match 'https://ror.org/XXXXX'.""" - value = bleach.clean(value) # Sanitize to strip HTML - ror_regex = r'https://ror\.org/[A-Za-z0-9]+' - if not re.match(ror_regex, value): - raise serializers.ValidationError("Institution ROR must be in the format 'https://ror.org/XXXXX'.") + """Validate and format institution ROR to match 'https://ror.org/XXXXX'. Reject empty string.""" + if value is not None: + if value == "": + raise serializers.ValidationError("Institution ROR cannot be an empty string.") + value = bleach.clean(value) # Sanitize to strip HTML + ror_regex = r'https://ror\.org/[A-Za-z0-9]+' + if not re.match(ror_regex, value): + raise serializers.ValidationError("Institution ROR must be in the format 'https://ror.org/XXXXX'.") return value def validate_country_code(self, value): - """Validate that the country code is a valid ISO 3166-1 alpha-2 country code.""" - value = value.strip().upper() # Normalize to uppercase - if len(value) != 2 or not pycountry.countries.get(alpha_2=value): - raise serializers.ValidationError(f"{value} is not a valid ISO 3166-1 alpha-2 country code.") + """Validate that the country code is a valid ISO 3166-1 alpha-2 country code. Reject empty string.""" + if value is not None: + if value == "": + raise serializers.ValidationError("Country code cannot be an empty string.") + value = value.strip().upper() # Normalize to uppercase + if len(value) != 2 or not pycountry.countries.get(alpha_2=value): + raise serializers.ValidationError(f"{value} is not a valid ISO 3166-1 alpha-2 country code.") return value def validate_ror_use(self, value): - """Sanitize ror_use and validate length.""" - value = bleach.clean(value) # Sanitize to strip HTML - if len(value) > 500: - raise serializers.ValidationError("ROR use cannot be longer than 500 characters.") + """Sanitize ror_use and validate length. Reject empty string.""" + if value is not None: + if value == "": + raise serializers.ValidationError("ROR use cannot be an empty string.") + value = bleach.clean(value) # Sanitize to strip HTML + if len(value) > 500: + raise serializers.ValidationError("ROR use cannot be longer than 500 characters.") return value From 30de33699713c34de0274bea991cdae7adc329f2 Mon Sep 17 00:00:00 2001 From: Ashwini Sukale Date: Wed, 16 Apr 2025 20:22:59 +0530 Subject: [PATCH 20/50] client id changes (#422) --- Dockerfile | 3 +- README.md | 4 +- docker-compose.yml | 15 ++++ requirements.txt | 72 ++++++++++++++---- rorapi/common/urls.py | 4 +- rorapi/common/views.py | 38 +++++++++- rorapi/management/commands/generaterorid.py | 7 ++ rorapi/migrations/0001_create_client_model.py | 30 ++++++++ rorapi/migrations/0002_auto_20250326_1054.py | 18 +++++ rorapi/migrations/0003_auto_20250415_1207.py | 43 +++++++++++ rorapi/migrations/__init__.py | 0 rorapi/settings.py | 34 ++++++++- rorapi/tests/tests_unit/tests_client.py | 11 +++ rorapi/v2/models.py | 41 +++++++++- rorapi/v2/serializers.py | 74 +++++++++++++++++++ rorapi/v2/tests.py | 11 +++ 16 files changed, 382 insertions(+), 23 deletions(-) create mode 100644 rorapi/migrations/0001_create_client_model.py create mode 100644 rorapi/migrations/0002_auto_20250326_1054.py create mode 100644 rorapi/migrations/0003_auto_20250415_1207.py create mode 100644 rorapi/migrations/__init__.py create mode 100644 rorapi/tests/tests_unit/tests_client.py create mode 100644 rorapi/v2/tests.py diff --git a/Dockerfile b/Dockerfile index 9ebc84e..09b18d6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,7 @@ RUN mv /etc/apt/sources.list.d /etc/apt/sources.list.d.bak && \ mv /etc/apt/sources.list.d.bak /etc/apt/sources.list.d && \ apt-get upgrade -y -o Dpkg::Options::="--force-confold" && \ apt-get clean && \ - apt-get install ntp wget unzip tzdata python3-pip libmagic1 -y && \ + apt-get install ntp wget unzip tzdata python3-pip libmagic1 default-libmysqlclient-dev libcairo2-dev pkg-config -y && \ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* # Enable Passenger and Nginx and remove the default site @@ -54,6 +54,7 @@ RUN pip3 install --no-cache-dir -r requirements.txt RUN pip3 install yapf # collect static files for Django +ENV DJANGO_SKIP_DB_CHECK=True RUN python manage.py collectstatic --noinput # Expose web diff --git a/README.md b/README.md index 953426d..dd58935 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,9 @@ Commands for indexing ROR data, generating new ROR IDs and other internal operat ROUTE_USER=[USER] TOKEN=[TOKEN] -Replace values in [] with valid credential values. GITHUB_TOKEN is needed in order to index an existing data dump locally. ROUTE_USER and TOKEN are only needed in order to use generate-id functionality locally. AWS_* and DATA_STORE are only needed in order to use incremental indexing from S3 functionality locally. +ROR staff should replace values in [] with valid credential values. External users do not need to add these values but should comment out this line https://github.com/ror-community/ror-api/blob/8a5a5ae8b483564c966a7184349c581dcae756ef/rorapi/management/commands/setup.py#L13 so that there is no attempt to send a Github token when retrieving a data dump for indexing. + +- Optionally, uncomment [line 24 in docker-compose.yml](https://github.com/ror-community/ror-api/blob/master/docker-compose.yml#L24) in order to pull the rorapi image from Dockerhub rather than creating it from local code ## Start ror-api locally 1. Start Docker Desktop diff --git a/docker-compose.yml b/docker-compose.yml index 0cd1b0b..253e3c3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,6 +19,18 @@ services: timeout: 1s volumes: - ./esdata:/usr/share/elasticsearch/data + db: + image: mysql:8.0 + volumes: + - mysql_data:/var/lib/mysql + env_file: + - .env + ports: + - "3306:3306" + healthcheck: + test: ["CMD", "mysqladmin", "ping", "-h", "localhost"] + timeout: 20s + retries: 10 web: container_name: rorapiweb env_file: .env @@ -31,3 +43,6 @@ services: - ./rorapi:/home/app/webapp/rorapi depends_on: - elasticsearch7 + - db +volumes: + mysql_data: diff --git a/requirements.txt b/requirements.txt index f28d73b..38183fe 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,27 +1,67 @@ +attrs==24.3.0 +base32-crockford==0.3.0 +boto3==1.35.99 +botocore==1.35.99 +certifi==2019.11.28 +chardet==3.0.4 +coreapi==2.3.3 +coreschema==0.0.4 +dbus-python==1.2.16 Django==2.2.28 -elasticsearch_dsl==7.4.1 +django-cors-headers==3.1.0 +django-prometheus==1.0.15 +djangorestframework==3.11.2 +elasticsearch==7.10.1 +elasticsearch-dsl==7.4.1 +expiringdict==1.2.2 +fuzzywuzzy==0.17.0 geonamescache==1.3.0 -requests==2.22.0 -requests-aws4auth==0.9 +idna==2.8 +importlib_resources==6.4.5 +iso639-lang==2.5.1 +itypes==1.2.0 +Jinja2==3.1.5 +jmespath==1.0.1 +jsonschema==3.2.0 +launchdarkly-server-sdk==7.6.1 +MarkupSafe==2.1.5 mock==3.0.5 -base32_crockford==0.3.0 -elasticsearch==7.10.1 -djangorestframework==3.11.2 -coreapi==2.3.3 -django-prometheus==1.0.15 -sentry-sdk==0.12.2 +mysqlclient==2.2.7 +numpy==1.22.0 +pandas==1.4.1 +patsy==1.0.1 +platformdirs==4.3.6 +prometheus_client==0.21.1 +PyGObject==3.36.0 +pyRFC3339==2.0.1 +pyrsistent==0.20.0 +python-apt==2.0.1+ubuntu0.20.4.1 +python-dateutil==2.9.0.post0 python-dotenv==0.10.3 -django-cors-headers==3.1.0 -unidecode==1.1.1 -fuzzywuzzy==0.17.0 python-Levenshtein==0.12.1 +python-magic==0.4.27 +pytz==2024.2 +requests==2.22.0 +requests-aws4auth==0.9 +requests-unixsocket==0.2.0 +s3transfer==0.10.4 +scipy==1.10.1 +semver==2.13.0 +sentry-sdk==0.12.2 +six==1.14.0 +sqlparse==0.5.3 statsmodels==0.10.2 -boto3 -pandas==1.4.1 -numpy==1.22 titlecase==2.3 +tomli==2.2.1 +Unidecode==1.1.1 +uritemplate==4.1.1 +urllib3==1.25.8 +yapf==0.43.0 +zipp==3.20.2 update_address @ git+https://github.com/ror-community/update_address.git launchdarkly-server-sdk==7.6.1 jsonschema==3.2.0 python-magic -iso639-lang \ No newline at end of file +iso639-lang +bleach==6.0.0 +pycountry==22.3.5 diff --git a/rorapi/common/urls.py b/rorapi/common/urls.py index b9aa57c..277e0bd 100644 --- a/rorapi/common/urls.py +++ b/rorapi/common/urls.py @@ -3,7 +3,7 @@ from rest_framework.documentation import include_docs_urls from . import views from rorapi.common.views import ( - HeartbeatView,GenerateAddress,GenerateId,IndexData,IndexDataDump,BulkUpdate) + HeartbeatView,GenerateAddress,GenerateId,IndexData,IndexDataDump,BulkUpdate,ClientRegistrationView,ValidateClientView) urlpatterns = [ # Health check @@ -14,6 +14,8 @@ path('generateaddress/', GenerateAddress.as_view()), url(r"^generateid$", GenerateId.as_view()), re_path(r"^(?P(v1|v2))\/bulkupdate$", BulkUpdate.as_view()), + re_path(r"^(?P(v1|v2))\/register$", ClientRegistrationView.as_view()), + path('validate-client-id//', ValidateClientView.as_view()), url(r"^(?P(v1|v2))\/indexdata/(?P.*)", IndexData.as_view()), url(r"^(?P(v1|v2))\/indexdatadump\/(?Pv(\d+\.)?(\d+\.)?(\*|\d+)-\d{4}-\d{2}-\d{2}-ror-data)\/(?P(test|prod))$", IndexDataDump.as_view()), url(r"^(?P(v1|v2))\/", include(views.organizations_router.urls)), diff --git a/rorapi/common/views.py b/rorapi/common/views.py index ebcfa21..8d58402 100644 --- a/rorapi/common/views.py +++ b/rorapi/common/views.py @@ -37,10 +37,46 @@ import os import update_address as ua from rorapi.management.commands.generaterorid import check_ror_id -from rorapi.management.commands.generaterorid import check_ror_id from rorapi.management.commands.indexror import process_files from django.core import management import rorapi.management.commands.indexrordump +from django.core.mail import send_mail +from django.utils.timezone import now +from rorapi.v2.models import Client +from rorapi.v2.serializers import ClientSerializer + +class ClientRegistrationView(APIView): + def post(self, request, version='v2'): + serializer = ClientSerializer(data=request.data) + if serializer.is_valid(): + client = serializer.save() + + # Send a registration email to the client + subject = 'Your ROR API Client ID' + message = f'Thank you for registering. Your Client ID is: {client.client_id}' + from_email = 'support@ror.org' + recipient_list = [client.email] + + # send_mail( + # subject, + # message, + # from_email, + # recipient_list, + # fail_silently=False, + # ) + + return Response({'client_id': client.client_id}, status=status.HTTP_201_CREATED) + + # Return validation errors if serializer is not valid + return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) + +class ValidateClientView(APIView): + def get(self, request, client_id): + # Check if the client_id exists in the database + client_exists = Client.objects.filter(client_id=client_id).exists() + + # Return response indicating whether client ID is valid + return Response({'valid': client_exists}, status=status.HTTP_200_OK) class OurTokenPermission(BasePermission): """ diff --git a/rorapi/management/commands/generaterorid.py b/rorapi/management/commands/generaterorid.py index 2ea42a5..af07be0 100644 --- a/rorapi/management/commands/generaterorid.py +++ b/rorapi/management/commands/generaterorid.py @@ -26,3 +26,10 @@ def check_ror_id(version): check_ror_id(version) return ror_id + +def generate_ror_client_id(): + """Generates a random ROR client ID. + """ + + n = random.randint(0, 2**160 - 1) + return base32_crockford.encode(n).lower().zfill(32) diff --git a/rorapi/migrations/0001_create_client_model.py b/rorapi/migrations/0001_create_client_model.py new file mode 100644 index 0000000..b7b558f --- /dev/null +++ b/rorapi/migrations/0001_create_client_model.py @@ -0,0 +1,30 @@ +# Generated by Django 2.2.28 on 2025-03-11 07:13 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='Client', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('email', models.EmailField(max_length=255)), + ('name', models.CharField(blank=True, max_length=255)), + ('institution_name', models.CharField(blank=True, max_length=255)), + ('institution_ror', models.URLField(blank=True, max_length=255)), + ('country_code', models.CharField(blank=True, max_length=2)), + ('ror_use', models.TextField(blank=True, max_length=500)), + ('client_id', models.CharField(editable=False, max_length=32, unique=True)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('last_request_at', models.DateTimeField(blank=True, null=True)), + ('request_count', models.IntegerField(default=0)), + ], + ), + ] diff --git a/rorapi/migrations/0002_auto_20250326_1054.py b/rorapi/migrations/0002_auto_20250326_1054.py new file mode 100644 index 0000000..0d06ed9 --- /dev/null +++ b/rorapi/migrations/0002_auto_20250326_1054.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.28 on 2025-03-26 10:54 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('rorapi', '0001_create_client_model'), + ] + + operations = [ + migrations.AlterField( + model_name='client', + name='email', + field=models.EmailField(max_length=255, unique=True), + ), + ] diff --git a/rorapi/migrations/0003_auto_20250415_1207.py b/rorapi/migrations/0003_auto_20250415_1207.py new file mode 100644 index 0000000..0697d02 --- /dev/null +++ b/rorapi/migrations/0003_auto_20250415_1207.py @@ -0,0 +1,43 @@ +# Generated by Django 2.2.28 on 2025-04-15 12:07 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('rorapi', '0002_auto_20250326_1054'), + ] + + operations = [ + migrations.AlterField( + model_name='client', + name='country_code', + field=models.CharField(blank=True, max_length=2, null=True), + ), + migrations.AlterField( + model_name='client', + name='email', + field=models.EmailField(max_length=255), + ), + migrations.AlterField( + model_name='client', + name='institution_name', + field=models.CharField(blank=True, max_length=255, null=True), + ), + migrations.AlterField( + model_name='client', + name='institution_ror', + field=models.URLField(blank=True, max_length=255, null=True), + ), + migrations.AlterField( + model_name='client', + name='name', + field=models.CharField(blank=True, max_length=255, null=True), + ), + migrations.AlterField( + model_name='client', + name='ror_use', + field=models.TextField(blank=True, max_length=500, null=True), + ), + ] diff --git a/rorapi/migrations/__init__.py b/rorapi/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/rorapi/settings.py b/rorapi/settings.py index a8f3504..5e8607b 100644 --- a/rorapi/settings.py +++ b/rorapi/settings.py @@ -11,6 +11,7 @@ """ import os +import sys import json import sentry_sdk import boto3 @@ -71,7 +72,7 @@ 'django.contrib.auth.middleware.AuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', - 'django_prometheus.middleware.PrometheusAfterMiddleware', + 'django_prometheus.middleware.PrometheusAfterMiddleware' ] ROOT_URLCONF = 'rorapi.common.urls' @@ -105,7 +106,23 @@ # Database # https://docs.djangoproject.com/en/2.2/ref/settings/#databases -DATABASES = {} +if 'collectstatic' in sys.argv and os.environ.get('DJANGO_SKIP_DB_CHECK') == 'True': + DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.dummy' + } + } +else: + DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.mysql', + 'NAME': os.environ.get('DB_NAME', 'rorapi'), + 'USER': os.environ.get('DB_USER', 'root'), + 'PASSWORD': os.environ.get('DB_PASSWORD', 'password'), + 'HOST': os.environ.get('DB_HOST', 'db'), + 'PORT': os.environ.get('DB_PORT', '3306'), + } +} # Password validation # https://docs.djangoproject.com/en/2.2/ref/settings/#auth-password-validators @@ -273,3 +290,16 @@ GRID_REMOVED_IDS = [] LAUNCH_DARKLY_KEY = os.environ.get('LAUNCH_DARKLY_KEY') + +# Toggle for behavior-based rate limiting +import os +ENABLE_BEHAVIORAL_LIMITING = os.getenv("ENABLE_BEHAVIORAL_LIMITING", "False") == "True" + +# Email settings for Django +EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend' +EMAIL_HOST = 'smtp.mailtrap.io' # Change this to your SMTP server +EMAIL_PORT = 587 +EMAIL_USE_TLS = True +EMAIL_HOST_USER = 'your-email@example.com' # Your SMTP username +EMAIL_HOST_PASSWORD = 'your-email-password' # Your SMTP password +DEFAULT_FROM_EMAIL = 'support@ror.org' \ No newline at end of file diff --git a/rorapi/tests/tests_unit/tests_client.py b/rorapi/tests/tests_unit/tests_client.py new file mode 100644 index 0000000..eb1f9d0 --- /dev/null +++ b/rorapi/tests/tests_unit/tests_client.py @@ -0,0 +1,11 @@ +from django.test import TestCase +from rorapi.v2.models.client import Client + +class ClientTests(TestCase): + def test_client_registration(self): + client = Client.objects.create(email='test@example.com') + self.assertIsNotNone(client.client_id) + + def test_rate_limiting(self): + response = self.client.get('/client-id/', HTTP_CLIENT_ID="INVALID_ID") + self.assertEqual(response.status_code, 429) \ No newline at end of file diff --git a/rorapi/v2/models.py b/rorapi/v2/models.py index e9c7d1a..1b5a82f 100644 --- a/rorapi/v2/models.py +++ b/rorapi/v2/models.py @@ -1,4 +1,7 @@ from geonamescache.mappers import country +import random +import string +from django.db import models from rorapi.common.models import TypeBucket, CountryBucket, StatusBucket, Entity from rorapi.v2.record_constants import continent_code_to_name @@ -130,4 +133,40 @@ class MatchingResult: def __init__(self, data): self.number_of_results = len(data) - self.items = [MatchedOrganization(x) for x in data] \ No newline at end of file + self.items = [MatchedOrganization(x) for x in data] + + +class Client(models.Model): + # Required fields + email = models.EmailField(max_length=255) + + # Optional fields + name = models.CharField(max_length=255, blank=True, null=True) + institution_name = models.CharField(max_length=255, blank=True, null=True) + institution_ror = models.URLField(max_length=255, blank=True, null=True) + country_code = models.CharField(max_length=2, blank=True, null=True) + ror_use = models.TextField(max_length=500, blank=True, null=True) + + # System fields + client_id = models.CharField( + max_length=32, + unique=True, + editable=False + ) + created_at = models.DateTimeField(auto_now_add=True) + last_request_at = models.DateTimeField(null=True, blank=True) + request_count = models.IntegerField(default=0) + + def __str__(self): + return f"{self.email} - {self.client_id}" + + @staticmethod + def generate_client_id(): + """Generate a unique 32-character client ID""" + return ''.join(random.choices(string.ascii_uppercase + string.digits, k=32)) + + def save(self, *args, **kwargs): + # Ensure client_id is generated before saving + if not self.client_id: # Only generate if it's empty + self.client_id = self.generate_client_id() + super().save(*args, **kwargs) diff --git a/rorapi/v2/serializers.py b/rorapi/v2/serializers.py index 62327a5..002fd8e 100644 --- a/rorapi/v2/serializers.py +++ b/rorapi/v2/serializers.py @@ -1,4 +1,8 @@ from rest_framework import serializers +import bleach +import pycountry +import re +from rorapi.v2.models import Client from rorapi.common.serializers import BucketSerializer, OrganizationRelationshipsSerializer class AggregationsSerializer(serializers.Serializer): @@ -87,3 +91,73 @@ class MatchedOrganizationSerializer(serializers.Serializer): class MatchingResultSerializer(serializers.Serializer): number_of_results = serializers.IntegerField() items = MatchedOrganizationSerializer(many=True) + + +class ClientSerializer(serializers.ModelSerializer): + class Meta: + model = Client + fields = ['email', 'name', 'institution_name', 'institution_ror', 'country_code', 'ror_use'] + extra_kwargs = { + 'name': {'required': False, 'allow_null': True}, + 'institution_name': {'required': False, 'allow_null': True}, + 'institution_ror': {'required': False, 'allow_null': True}, + 'country_code': {'required': False, 'allow_null': True}, + 'ror_use': {'required': False, 'allow_null': True}, + } + + def validate_email(self, value): + """Validate the email format and ensure it's unique.""" + if value is None: + raise serializers.ValidationError("Email cannot be null.") + return value + + def validate_name(self, value): + """Sanitize name and validate length. Reject empty string.""" + if value is not None: + if value == "": + raise serializers.ValidationError("Name cannot be an empty string.") + value = bleach.clean(value) # Sanitize to strip HTML + if len(value) > 255: + raise serializers.ValidationError("Name cannot be longer than 255 characters.") + return value + + def validate_institution_name(self, value): + """Sanitize institution name and validate length. Reject empty string.""" + if value is not None: + if value == "": + raise serializers.ValidationError("Institution name cannot be an empty string.") + value = bleach.clean(value) # Sanitize to strip HTML + if len(value) > 255: + raise serializers.ValidationError("Institution name cannot be longer than 255 characters.") + return value + + def validate_institution_ror(self, value): + """Validate and format institution ROR to match 'https://ror.org/XXXXX'. Reject empty string.""" + if value is not None: + if value == "": + raise serializers.ValidationError("Institution ROR cannot be an empty string.") + value = bleach.clean(value) # Sanitize to strip HTML + ror_regex = r'https://ror\.org/[A-Za-z0-9]+' + if not re.match(ror_regex, value): + raise serializers.ValidationError("Institution ROR must be in the format 'https://ror.org/XXXXX'.") + return value + + def validate_country_code(self, value): + """Validate that the country code is a valid ISO 3166-1 alpha-2 country code. Reject empty string.""" + if value is not None: + if value == "": + raise serializers.ValidationError("Country code cannot be an empty string.") + value = value.strip().upper() # Normalize to uppercase + if len(value) != 2 or not pycountry.countries.get(alpha_2=value): + raise serializers.ValidationError(f"{value} is not a valid ISO 3166-1 alpha-2 country code.") + return value + + def validate_ror_use(self, value): + """Sanitize ror_use and validate length. Reject empty string.""" + if value is not None: + if value == "": + raise serializers.ValidationError("ROR use cannot be an empty string.") + value = bleach.clean(value) # Sanitize to strip HTML + if len(value) > 500: + raise serializers.ValidationError("ROR use cannot be longer than 500 characters.") + return value diff --git a/rorapi/v2/tests.py b/rorapi/v2/tests.py new file mode 100644 index 0000000..5e2c9ef --- /dev/null +++ b/rorapi/v2/tests.py @@ -0,0 +1,11 @@ +from django.test import TestCase +from .models.client import Client + +class ClientTests(TestCase): + def test_client_registration(self): + client = Client.objects.create(email='test@example.com') + self.assertIsNotNone(client.client_id) + + def test_rate_limiting(self): + response = self.client.get('/client-id/', HTTP_CLIENT_ID="INVALID_ID") + self.assertEqual(response.status_code, 429) From 7b83b60bd9eadbd4c344f0a7537d926446c76f6d Mon Sep 17 00:00:00 2001 From: kaysiz Date: Wed, 16 Apr 2025 17:19:36 +0200 Subject: [PATCH 21/50] clean up the requirements.tx file --- requirements.txt | 69 +++++++++++------------------------------------- 1 file changed, 16 insertions(+), 53 deletions(-) diff --git a/requirements.txt b/requirements.txt index 38183fe..8438c8a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,67 +1,30 @@ -attrs==24.3.0 -base32-crockford==0.3.0 -boto3==1.35.99 -botocore==1.35.99 -certifi==2019.11.28 -chardet==3.0.4 -coreapi==2.3.3 -coreschema==0.0.4 -dbus-python==1.2.16 Django==2.2.28 -django-cors-headers==3.1.0 -django-prometheus==1.0.15 -djangorestframework==3.11.2 -elasticsearch==7.10.1 -elasticsearch-dsl==7.4.1 -expiringdict==1.2.2 -fuzzywuzzy==0.17.0 +elasticsearch_dsl==7.4.1 geonamescache==1.3.0 -idna==2.8 -importlib_resources==6.4.5 -iso639-lang==2.5.1 -itypes==1.2.0 -Jinja2==3.1.5 -jmespath==1.0.1 -jsonschema==3.2.0 -launchdarkly-server-sdk==7.6.1 -MarkupSafe==2.1.5 -mock==3.0.5 -mysqlclient==2.2.7 -numpy==1.22.0 -pandas==1.4.1 -patsy==1.0.1 -platformdirs==4.3.6 -prometheus_client==0.21.1 -PyGObject==3.36.0 -pyRFC3339==2.0.1 -pyrsistent==0.20.0 -python-apt==2.0.1+ubuntu0.20.4.1 -python-dateutil==2.9.0.post0 -python-dotenv==0.10.3 -python-Levenshtein==0.12.1 -python-magic==0.4.27 -pytz==2024.2 requests==2.22.0 requests-aws4auth==0.9 -requests-unixsocket==0.2.0 -s3transfer==0.10.4 -scipy==1.10.1 -semver==2.13.0 +mock==3.0.5 +base32_crockford==0.3.0 +elasticsearch==7.10.1 +djangorestframework==3.11.2 +coreapi==2.3.3 +django-prometheus==1.0.15 sentry-sdk==0.12.2 -six==1.14.0 -sqlparse==0.5.3 +python-dotenv==0.10.3 +django-cors-headers==3.1.0 +unidecode==1.1.1 +fuzzywuzzy==0.17.0 +python-Levenshtein==0.12.1 statsmodels==0.10.2 +boto3 +pandas==1.4.1 +numpy==1.22 titlecase==2.3 -tomli==2.2.1 -Unidecode==1.1.1 -uritemplate==4.1.1 -urllib3==1.25.8 -yapf==0.43.0 -zipp==3.20.2 update_address @ git+https://github.com/ror-community/update_address.git launchdarkly-server-sdk==7.6.1 jsonschema==3.2.0 python-magic iso639-lang +mysqlclient==2.2.7 bleach==6.0.0 pycountry==22.3.5 From 304cf7e7fa28284d0be82330afb447ea20ccb2e2 Mon Sep 17 00:00:00 2001 From: kudakwashe siziva <9620622+kaysiz@users.noreply.github.com> Date: Wed, 16 Apr 2025 17:37:05 +0200 Subject: [PATCH 22/50] Client ID registration (#429) --- requirements.txt | 69 +++++++++++------------------------------------- 1 file changed, 16 insertions(+), 53 deletions(-) diff --git a/requirements.txt b/requirements.txt index 38183fe..8438c8a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,67 +1,30 @@ -attrs==24.3.0 -base32-crockford==0.3.0 -boto3==1.35.99 -botocore==1.35.99 -certifi==2019.11.28 -chardet==3.0.4 -coreapi==2.3.3 -coreschema==0.0.4 -dbus-python==1.2.16 Django==2.2.28 -django-cors-headers==3.1.0 -django-prometheus==1.0.15 -djangorestframework==3.11.2 -elasticsearch==7.10.1 -elasticsearch-dsl==7.4.1 -expiringdict==1.2.2 -fuzzywuzzy==0.17.0 +elasticsearch_dsl==7.4.1 geonamescache==1.3.0 -idna==2.8 -importlib_resources==6.4.5 -iso639-lang==2.5.1 -itypes==1.2.0 -Jinja2==3.1.5 -jmespath==1.0.1 -jsonschema==3.2.0 -launchdarkly-server-sdk==7.6.1 -MarkupSafe==2.1.5 -mock==3.0.5 -mysqlclient==2.2.7 -numpy==1.22.0 -pandas==1.4.1 -patsy==1.0.1 -platformdirs==4.3.6 -prometheus_client==0.21.1 -PyGObject==3.36.0 -pyRFC3339==2.0.1 -pyrsistent==0.20.0 -python-apt==2.0.1+ubuntu0.20.4.1 -python-dateutil==2.9.0.post0 -python-dotenv==0.10.3 -python-Levenshtein==0.12.1 -python-magic==0.4.27 -pytz==2024.2 requests==2.22.0 requests-aws4auth==0.9 -requests-unixsocket==0.2.0 -s3transfer==0.10.4 -scipy==1.10.1 -semver==2.13.0 +mock==3.0.5 +base32_crockford==0.3.0 +elasticsearch==7.10.1 +djangorestframework==3.11.2 +coreapi==2.3.3 +django-prometheus==1.0.15 sentry-sdk==0.12.2 -six==1.14.0 -sqlparse==0.5.3 +python-dotenv==0.10.3 +django-cors-headers==3.1.0 +unidecode==1.1.1 +fuzzywuzzy==0.17.0 +python-Levenshtein==0.12.1 statsmodels==0.10.2 +boto3 +pandas==1.4.1 +numpy==1.22 titlecase==2.3 -tomli==2.2.1 -Unidecode==1.1.1 -uritemplate==4.1.1 -urllib3==1.25.8 -yapf==0.43.0 -zipp==3.20.2 update_address @ git+https://github.com/ror-community/update_address.git launchdarkly-server-sdk==7.6.1 jsonschema==3.2.0 python-magic iso639-lang +mysqlclient==2.2.7 bleach==6.0.0 pycountry==22.3.5 From be7e23432c501743f6ad7a983dd774d70b228ea9 Mon Sep 17 00:00:00 2001 From: kaysiz Date: Wed, 16 Apr 2025 18:06:20 +0200 Subject: [PATCH 23/50] Add config for github workflow --- .github/workflows/dev.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 6199f77..733e790 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -13,6 +13,10 @@ jobs: ELASTIC7_HOST: "localhost" ELASTIC7_PORT: "9200" ELASTIC_PASSWORD: "changeme" + DB_NAME: rorapi + DB_USER: ror_user + DB_HOST: db + DB_PASSWORD: password AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_REGION: ${{ secrets.AWS_REGION }} @@ -30,6 +34,15 @@ jobs: http.cors.allow-origin: "*" ports: - 9200:9200 + db: + image: mysql:8.0 + env: + DB_NAME: rorapi + DB_USER: ror_user + DB_HOST: rorapi + DB_PASSWORD: password + ports: + - 3306:3306 steps: - name: Checkout ror-api code uses: actions/checkout@v2 From 953ad9bb0297fdb89a2164e2cc996714845915b6 Mon Sep 17 00:00:00 2001 From: kaysiz Date: Wed, 16 Apr 2025 18:06:37 +0200 Subject: [PATCH 24/50] Add config for github workflow --- .github/workflows/dev.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 733e790..229e34c 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -17,6 +17,7 @@ jobs: DB_USER: ror_user DB_HOST: db DB_PASSWORD: password + DB_PORT: 3306 AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_REGION: ${{ secrets.AWS_REGION }} From 324963daade4d7674e4ce2248ded612ba8a0d9f8 Mon Sep 17 00:00:00 2001 From: kaysiz Date: Wed, 16 Apr 2025 18:11:09 +0200 Subject: [PATCH 25/50] add config for db for staging github workflow --- .github/workflows/staging.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml index 57bef2f..e3ec120 100644 --- a/.github/workflows/staging.yml +++ b/.github/workflows/staging.yml @@ -10,6 +10,11 @@ jobs: ELASTIC_PASSWORD: "changeme" ELASTIC7_HOST: "localhost" ELASTIC7_PORT: "9200" + DB_NAME: rorapi + DB_USER: ror_user + DB_HOST: db + DB_PASSWORD: password + DB_PORT: 3306 AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_REGION: ${{ secrets.AWS_REGION }} @@ -27,6 +32,15 @@ jobs: http.cors.allow-origin: "*" ports: - 9200:9200 + db: + image: mysql:8.0 + env: + DB_NAME: rorapi + DB_USER: ror_user + DB_HOST: rorapi + DB_PASSWORD: password + ports: + - 3306:3306 steps: - name: Checkout ror-api code uses: actions/checkout@v2 From 7a3d6d275df96969366599bce998b76b5d20f325 Mon Sep 17 00:00:00 2001 From: kaysiz Date: Wed, 16 Apr 2025 18:11:31 +0200 Subject: [PATCH 26/50] add config for db for release github workflow --- .github/workflows/release.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bfd0346..438e4ee 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,6 +9,11 @@ jobs: ELASTIC_PASSWORD: "changeme" ELASTIC7_HOST: "localhost" ELASTIC7_PORT: "9200" + DB_NAME: rorapi + DB_USER: ror_user + DB_HOST: db + DB_PASSWORD: password + DB_PORT: 3306 AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_REGION: ${{ secrets.AWS_REGION }} @@ -26,6 +31,15 @@ jobs: http.cors.allow-origin: "*" ports: - 9200:9200 + db: + image: mysql:8.0 + env: + DB_NAME: rorapi + DB_USER: ror_user + DB_HOST: rorapi + DB_PASSWORD: password + ports: + - 3306:3306 steps: - name: Checkout ror-api code uses: actions/checkout@v2 From e54532e029ba5994c3e8fe9cd344485957dcf904 Mon Sep 17 00:00:00 2001 From: kudakwashe siziva <9620622+kaysiz@users.noreply.github.com> Date: Wed, 16 Apr 2025 18:13:50 +0200 Subject: [PATCH 27/50] Client ID registration (#430) --- .github/workflows/dev.yml | 14 ++++++++++++++ .github/workflows/release.yml | 14 ++++++++++++++ .github/workflows/staging.yml | 14 ++++++++++++++ 3 files changed, 42 insertions(+) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 6199f77..229e34c 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -13,6 +13,11 @@ jobs: ELASTIC7_HOST: "localhost" ELASTIC7_PORT: "9200" ELASTIC_PASSWORD: "changeme" + DB_NAME: rorapi + DB_USER: ror_user + DB_HOST: db + DB_PASSWORD: password + DB_PORT: 3306 AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_REGION: ${{ secrets.AWS_REGION }} @@ -30,6 +35,15 @@ jobs: http.cors.allow-origin: "*" ports: - 9200:9200 + db: + image: mysql:8.0 + env: + DB_NAME: rorapi + DB_USER: ror_user + DB_HOST: rorapi + DB_PASSWORD: password + ports: + - 3306:3306 steps: - name: Checkout ror-api code uses: actions/checkout@v2 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bfd0346..438e4ee 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,6 +9,11 @@ jobs: ELASTIC_PASSWORD: "changeme" ELASTIC7_HOST: "localhost" ELASTIC7_PORT: "9200" + DB_NAME: rorapi + DB_USER: ror_user + DB_HOST: db + DB_PASSWORD: password + DB_PORT: 3306 AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_REGION: ${{ secrets.AWS_REGION }} @@ -26,6 +31,15 @@ jobs: http.cors.allow-origin: "*" ports: - 9200:9200 + db: + image: mysql:8.0 + env: + DB_NAME: rorapi + DB_USER: ror_user + DB_HOST: rorapi + DB_PASSWORD: password + ports: + - 3306:3306 steps: - name: Checkout ror-api code uses: actions/checkout@v2 diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml index 57bef2f..e3ec120 100644 --- a/.github/workflows/staging.yml +++ b/.github/workflows/staging.yml @@ -10,6 +10,11 @@ jobs: ELASTIC_PASSWORD: "changeme" ELASTIC7_HOST: "localhost" ELASTIC7_PORT: "9200" + DB_NAME: rorapi + DB_USER: ror_user + DB_HOST: db + DB_PASSWORD: password + DB_PORT: 3306 AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_REGION: ${{ secrets.AWS_REGION }} @@ -27,6 +32,15 @@ jobs: http.cors.allow-origin: "*" ports: - 9200:9200 + db: + image: mysql:8.0 + env: + DB_NAME: rorapi + DB_USER: ror_user + DB_HOST: rorapi + DB_PASSWORD: password + ports: + - 3306:3306 steps: - name: Checkout ror-api code uses: actions/checkout@v2 From 8ae50c519a40d40106ad812efbf28ecf100dfd73 Mon Sep 17 00:00:00 2001 From: kaysiz Date: Wed, 16 Apr 2025 18:22:41 +0200 Subject: [PATCH 28/50] fix mysql setup --- .github/workflows/dev.yml | 16 ++++++++-------- .github/workflows/release.yml | 16 ++++++++-------- .github/workflows/staging.yml | 16 ++++++++-------- 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 229e34c..04a7f0b 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -13,10 +13,10 @@ jobs: ELASTIC7_HOST: "localhost" ELASTIC7_PORT: "9200" ELASTIC_PASSWORD: "changeme" - DB_NAME: rorapi - DB_USER: ror_user - DB_HOST: db - DB_PASSWORD: password + DB_NAME: rorapi" + DB_USER: "ror_user" + DB_HOST: "db" + DB_PASSWORD: "password" DB_PORT: 3306 AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} @@ -38,10 +38,10 @@ jobs: db: image: mysql:8.0 env: - DB_NAME: rorapi - DB_USER: ror_user - DB_HOST: rorapi - DB_PASSWORD: password + MYSQL_DATABASE: rorapi + MYSQL_USER: ror_user + MYSQL_PASSWORD: password + MYSQL_ROOT_PASSWORD: password ports: - 3306:3306 steps: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 438e4ee..8a1a47b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,10 +9,10 @@ jobs: ELASTIC_PASSWORD: "changeme" ELASTIC7_HOST: "localhost" ELASTIC7_PORT: "9200" - DB_NAME: rorapi - DB_USER: ror_user - DB_HOST: db - DB_PASSWORD: password + DB_NAME: "rorapi" + DB_USER: "ror_user" + DB_HOST: "db" + DB_PASSWORD: "password" DB_PORT: 3306 AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} @@ -34,10 +34,10 @@ jobs: db: image: mysql:8.0 env: - DB_NAME: rorapi - DB_USER: ror_user - DB_HOST: rorapi - DB_PASSWORD: password + MYSQL_DATABASE: rorapi + MYSQL_USER: ror_user + MYSQL_PASSWORD: password + MYSQL_ROOT_PASSWORD: password ports: - 3306:3306 steps: diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml index e3ec120..b5058c1 100644 --- a/.github/workflows/staging.yml +++ b/.github/workflows/staging.yml @@ -10,10 +10,10 @@ jobs: ELASTIC_PASSWORD: "changeme" ELASTIC7_HOST: "localhost" ELASTIC7_PORT: "9200" - DB_NAME: rorapi - DB_USER: ror_user - DB_HOST: db - DB_PASSWORD: password + DB_NAME: rorapi" + DB_USER: "ror_user" + DB_HOST: "db" + DB_PASSWORD: "password" DB_PORT: 3306 AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} @@ -35,10 +35,10 @@ jobs: db: image: mysql:8.0 env: - DB_NAME: rorapi - DB_USER: ror_user - DB_HOST: rorapi - DB_PASSWORD: password + MYSQL_DATABASE: rorapi + MYSQL_USER: ror_user + MYSQL_PASSWORD: password + MYSQL_ROOT_PASSWORD: password ports: - 3306:3306 steps: From 7d3facc89b36039eeea5c4241d9c4b71f068eecd Mon Sep 17 00:00:00 2001 From: kaysiz Date: Wed, 16 Apr 2025 18:53:38 +0200 Subject: [PATCH 29/50] fix mysql setup --- .github/workflows/dev.yml | 2 +- .github/workflows/release.yml | 2 +- .github/workflows/staging.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 75d50eb..3e7398e 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -15,7 +15,7 @@ jobs: ELASTIC_PASSWORD: "changeme" DB_NAME: "rorapi" DB_USER: "ror_user" - DB_HOST: "db" + DB_HOST: "localhost" DB_PASSWORD: "password" DB_PORT: 3306 AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index fc50ebf..933866a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,7 +11,7 @@ jobs: ELASTIC7_PORT: "9200" DB_NAME: "rorapi" DB_USER: "ror_user" - DB_HOST: "db" + DB_HOST: "localhost" DB_PASSWORD: "password" DB_PORT: 3306 AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml index 0b8d073..4e13388 100644 --- a/.github/workflows/staging.yml +++ b/.github/workflows/staging.yml @@ -12,7 +12,7 @@ jobs: ELASTIC7_PORT: "9200" DB_NAME: "rorapi" DB_USER: "ror_user" - DB_HOST: "db" + DB_HOST: "localhost" DB_PASSWORD: "password" DB_PORT: 3306 AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} From 141b74eece28b59496fb97e6593a7309f00aed7a Mon Sep 17 00:00:00 2001 From: kudakwashe siziva <9620622+kaysiz@users.noreply.github.com> Date: Wed, 16 Apr 2025 18:56:50 +0200 Subject: [PATCH 30/50] fix mysql setup (#432) --- .github/workflows/dev.yml | 2 +- .github/workflows/release.yml | 2 +- .github/workflows/staging.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 75d50eb..3e7398e 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -15,7 +15,7 @@ jobs: ELASTIC_PASSWORD: "changeme" DB_NAME: "rorapi" DB_USER: "ror_user" - DB_HOST: "db" + DB_HOST: "localhost" DB_PASSWORD: "password" DB_PORT: 3306 AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index fc50ebf..933866a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,7 +11,7 @@ jobs: ELASTIC7_PORT: "9200" DB_NAME: "rorapi" DB_USER: "ror_user" - DB_HOST: "db" + DB_HOST: "localhost" DB_PASSWORD: "password" DB_PORT: 3306 AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml index 0b8d073..4e13388 100644 --- a/.github/workflows/staging.yml +++ b/.github/workflows/staging.yml @@ -12,7 +12,7 @@ jobs: ELASTIC7_PORT: "9200" DB_NAME: "rorapi" DB_USER: "ror_user" - DB_HOST: "db" + DB_HOST: "localhost" DB_PASSWORD: "password" DB_PORT: 3306 AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} From 9a018a5d2266b019d5f1ebfab506108124bdde9f Mon Sep 17 00:00:00 2001 From: kaysiz Date: Wed, 16 Apr 2025 19:13:37 +0200 Subject: [PATCH 31/50] skip db look up for dev --- .github/workflows/dev.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 3e7398e..8bc0b57 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -18,6 +18,7 @@ jobs: DB_HOST: "localhost" DB_PASSWORD: "password" DB_PORT: 3306 + DJANGO_SKIP_DB_CHECK: "True" AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_REGION: ${{ secrets.AWS_REGION }} From ae8b65633fc4b61b5ce63f77123ba98fb7be6e20 Mon Sep 17 00:00:00 2001 From: kaysiz Date: Wed, 16 Apr 2025 19:15:57 +0200 Subject: [PATCH 32/50] skip db look up for dev --- .github/workflows/dev.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 8bc0b57..f0c5c46 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -15,7 +15,7 @@ jobs: ELASTIC_PASSWORD: "changeme" DB_NAME: "rorapi" DB_USER: "ror_user" - DB_HOST: "localhost" + DB_HOST: "127.0.0.1" DB_PASSWORD: "password" DB_PORT: 3306 DJANGO_SKIP_DB_CHECK: "True" From 882dc4eddaee765a4ae126aa2c0fd0da7cfa67b1 Mon Sep 17 00:00:00 2001 From: kudakwashe siziva <9620622+kaysiz@users.noreply.github.com> Date: Wed, 16 Apr 2025 19:17:12 +0200 Subject: [PATCH 33/50] Client id ror (#433) * fix mysql setup * skip db look up for dev * skip db look up for dev --- .github/workflows/dev.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 3e7398e..f0c5c46 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -15,9 +15,10 @@ jobs: ELASTIC_PASSWORD: "changeme" DB_NAME: "rorapi" DB_USER: "ror_user" - DB_HOST: "localhost" + DB_HOST: "127.0.0.1" DB_PASSWORD: "password" DB_PORT: 3306 + DJANGO_SKIP_DB_CHECK: "True" AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_REGION: ${{ secrets.AWS_REGION }} From dd079ce1cb316835189d1b303f3e91a192c8ad3d Mon Sep 17 00:00:00 2001 From: lizkrznarich Date: Wed, 16 Apr 2025 12:19:01 -0500 Subject: [PATCH 34/50] update test db config --- .github/workflows/dev.yml | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index f0c5c46..f672aab 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -13,12 +13,6 @@ jobs: ELASTIC7_HOST: "localhost" ELASTIC7_PORT: "9200" ELASTIC_PASSWORD: "changeme" - DB_NAME: "rorapi" - DB_USER: "ror_user" - DB_HOST: "127.0.0.1" - DB_PASSWORD: "password" - DB_PORT: 3306 - DJANGO_SKIP_DB_CHECK: "True" AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_REGION: ${{ secrets.AWS_REGION }} @@ -39,10 +33,10 @@ jobs: db: image: mysql:8.0 env: - MYSQL_DATABASE: "rorapi" - MYSQL_USER: "ror_user" - MYSQL_PASSWORD: "password" - MYSQL_ROOT_PASSWORD: "password" + MYSQL_DATABASE: rorapi + MYSQL_USER: ror_user + MYSQL_PASSWORD: password + MYSQL_ROOT_PASSWORD: password ports: - 3306:3306 steps: From 7d697bcf0b19393ee291a6263938dfaec0a340d7 Mon Sep 17 00:00:00 2001 From: lizkrznarich Date: Wed, 16 Apr 2025 12:41:00 -0500 Subject: [PATCH 35/50] update db config --- .github/workflows/dev.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index f672aab..3bfa57f 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -18,6 +18,7 @@ jobs: AWS_REGION: ${{ secrets.AWS_REGION }} GITHUB_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} LAUNCH_DARKLY_KEY: ${{ secrets.LAUNCH_DARKLY_KEY_DEV }} + DB_HOST: 127.0.0.1 # Will not work with 'localhost', since that will try a Unix socket connection (!) services: elasticsearch7: image: docker.elastic.co/elasticsearch/elasticsearch:7.10.0 @@ -33,12 +34,13 @@ jobs: db: image: mysql:8.0 env: - MYSQL_DATABASE: rorapi - MYSQL_USER: ror_user - MYSQL_PASSWORD: password - MYSQL_ROOT_PASSWORD: password + MYSQL_DATABASE: "rorapi" + MYSQL_USER: "ror_user" + MYSQL_PASSWORD: "password" + MYSQL_ROOT_PASSWORD: "password" ports: - 3306:3306 + options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3 steps: - name: Checkout ror-api code uses: actions/checkout@v2 From 7cf4d4b96f14bed3580f88d5af0632fb0ac95140 Mon Sep 17 00:00:00 2001 From: kaysiz Date: Wed, 16 Apr 2025 19:54:46 +0200 Subject: [PATCH 36/50] fix failing tests --- rorapi/tests/tests_unit/tests_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rorapi/tests/tests_unit/tests_client.py b/rorapi/tests/tests_unit/tests_client.py index eb1f9d0..1990431 100644 --- a/rorapi/tests/tests_unit/tests_client.py +++ b/rorapi/tests/tests_unit/tests_client.py @@ -1,5 +1,5 @@ from django.test import TestCase -from rorapi.v2.models.client import Client +from rorapi.v2.models import Client class ClientTests(TestCase): def test_client_registration(self): From 1c7288f2704e544c288f29ce499fe1ac47cdfaa6 Mon Sep 17 00:00:00 2001 From: kaysiz Date: Wed, 16 Apr 2025 19:56:59 +0200 Subject: [PATCH 37/50] Update config for staging and release --- .github/workflows/release.yml | 7 ++----- .github/workflows/staging.yml | 7 ++----- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 933866a..fede267 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,11 +9,7 @@ jobs: ELASTIC_PASSWORD: "changeme" ELASTIC7_HOST: "localhost" ELASTIC7_PORT: "9200" - DB_NAME: "rorapi" - DB_USER: "ror_user" - DB_HOST: "localhost" - DB_PASSWORD: "password" - DB_PORT: 3306 + DB_HOST: 127.0.0.1 AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_REGION: ${{ secrets.AWS_REGION }} @@ -40,6 +36,7 @@ jobs: MYSQL_ROOT_PASSWORD: "password" ports: - 3306:3306 + options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3 steps: - name: Checkout ror-api code uses: actions/checkout@v2 diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml index 4e13388..42907f1 100644 --- a/.github/workflows/staging.yml +++ b/.github/workflows/staging.yml @@ -10,11 +10,7 @@ jobs: ELASTIC_PASSWORD: "changeme" ELASTIC7_HOST: "localhost" ELASTIC7_PORT: "9200" - DB_NAME: "rorapi" - DB_USER: "ror_user" - DB_HOST: "localhost" - DB_PASSWORD: "password" - DB_PORT: 3306 + DB_HOST: 127.0.0.1 AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_REGION: ${{ secrets.AWS_REGION }} @@ -41,6 +37,7 @@ jobs: MYSQL_ROOT_PASSWORD: "password" ports: - 3306:3306 + options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3 steps: - name: Checkout ror-api code uses: actions/checkout@v2 From 8d906bd2024c384bf650a9672d9a61f5ab7c11b2 Mon Sep 17 00:00:00 2001 From: lizkrznarich Date: Wed, 16 Apr 2025 13:01:17 -0500 Subject: [PATCH 38/50] fix test package import --- rorapi/tests/tests_unit/tests_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rorapi/tests/tests_unit/tests_client.py b/rorapi/tests/tests_unit/tests_client.py index eb1f9d0..1990431 100644 --- a/rorapi/tests/tests_unit/tests_client.py +++ b/rorapi/tests/tests_unit/tests_client.py @@ -1,5 +1,5 @@ from django.test import TestCase -from rorapi.v2.models.client import Client +from rorapi.v2.models import Client class ClientTests(TestCase): def test_client_registration(self): From 8ac04575be01f05c44c775e6a6882973080af289 Mon Sep 17 00:00:00 2001 From: lizkrznarich Date: Wed, 16 Apr 2025 13:09:26 -0500 Subject: [PATCH 39/50] remove unneeded test --- rorapi/tests/tests_unit/tests_client.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/rorapi/tests/tests_unit/tests_client.py b/rorapi/tests/tests_unit/tests_client.py index 1990431..d70ccc9 100644 --- a/rorapi/tests/tests_unit/tests_client.py +++ b/rorapi/tests/tests_unit/tests_client.py @@ -4,8 +4,4 @@ class ClientTests(TestCase): def test_client_registration(self): client = Client.objects.create(email='test@example.com') - self.assertIsNotNone(client.client_id) - - def test_rate_limiting(self): - response = self.client.get('/client-id/', HTTP_CLIENT_ID="INVALID_ID") - self.assertEqual(response.status_code, 429) \ No newline at end of file + self.assertIsNotNone(client.client_id) \ No newline at end of file From 3116c8e9a7f4e69c971d2172c8bc015901444abf Mon Sep 17 00:00:00 2001 From: kaysiz Date: Thu, 24 Apr 2025 11:59:51 +0200 Subject: [PATCH 40/50] Setup email sending for client api registration --- requirements.txt | 1 + rorapi/settings.py | 11 ++++------- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/requirements.txt b/requirements.txt index 8438c8a..a44a9b8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,3 +28,4 @@ iso639-lang mysqlclient==2.2.7 bleach==6.0.0 pycountry==22.3.5 +django-ses==3.5.0 \ No newline at end of file diff --git a/rorapi/settings.py b/rorapi/settings.py index 5e8607b..e1a7f8e 100644 --- a/rorapi/settings.py +++ b/rorapi/settings.py @@ -296,10 +296,7 @@ ENABLE_BEHAVIORAL_LIMITING = os.getenv("ENABLE_BEHAVIORAL_LIMITING", "False") == "True" # Email settings for Django -EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend' -EMAIL_HOST = 'smtp.mailtrap.io' # Change this to your SMTP server -EMAIL_PORT = 587 -EMAIL_USE_TLS = True -EMAIL_HOST_USER = 'your-email@example.com' # Your SMTP username -EMAIL_HOST_PASSWORD = 'your-email-password' # Your SMTP password -DEFAULT_FROM_EMAIL = 'support@ror.org' \ No newline at end of file +EMAIL_BACKEND = 'django_ses.SESBackend' +AWS_ACCESS_KEY_ID = os.environ.get('AWS_ACCESS_KEY_ID') +AWS_SECRET_ACCESS_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY') +AWS_SES_REGION_NAME = os.environ.get('AWS_REGION', 'eu-west-1') From 75fb990c4d54e3089c5122f9ec132344c74d351d Mon Sep 17 00:00:00 2001 From: kaysiz Date: Thu, 24 Apr 2025 12:12:15 +0200 Subject: [PATCH 41/50] Add content for client registration email --- rorapi/common/views.py | 67 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 54 insertions(+), 13 deletions(-) diff --git a/rorapi/common/views.py b/rorapi/common/views.py index 8d58402..e431c0d 100644 --- a/rorapi/common/views.py +++ b/rorapi/common/views.py @@ -40,7 +40,7 @@ from rorapi.management.commands.indexror import process_files from django.core import management import rorapi.management.commands.indexrordump -from django.core.mail import send_mail +from django.core.mail import EmailMultiAlternatives from django.utils.timezone import now from rorapi.v2.models import Client from rorapi.v2.serializers import ClientSerializer @@ -51,19 +51,62 @@ def post(self, request, version='v2'): if serializer.is_valid(): client = serializer.save() - # Send a registration email to the client - subject = 'Your ROR API Client ID' - message = f'Thank you for registering. Your Client ID is: {client.client_id}' + subject = 'ROR API client ID' from_email = 'support@ror.org' recipient_list = [client.email] - # send_mail( - # subject, - # message, - # from_email, - # recipient_list, - # fail_silently=False, - # ) + text_content = """ + Thank you for registering for a ROR API client ID! + + Your ROR API client ID is: + + {client.client_id} + + This client ID is not used for authentication or authorization, and is therefore not secret and can be sent as plain text. + + In order to receive a rate limit of 2000 requests per 5 minute period, please include this client ID with your ROR API requests, in a custom HTTP header named Client-Id, for example: + + curl -H "Client-Id: {client.client_id}" https://api.ror.org/organizations?query=oxford + + Requests without a valid client ID are subject to a rate limit of 50 requests per 5 minute period. + + We do not provide a way to recover or revoke a lost client ID. If you lose track of your client ID, please register a new client ID. For more information about ROR API client IDs, see https://ror.readme.io + + If you have questions, please see ROR documentation or contact us at support@ror.org + + Cheers, + The ROR Team + support@ror.org + https://ror.org + """ + + html_content = """ +

Thank you for registering for a ROR API client ID!

+ +

Your ROR API client ID is:

+ + 
{client.client_id}
+ +

This client ID is not used for authentication or authorization, and is therefore not secret and can be sent as plain text.

+ +

In order to receive a rate limit of 2000 requests per 5 minute period, please include this client ID with your ROR API requests, in a custom HTTP header named Client-Id, for example:

+ + 
curl -H "Client-Id: {client.client_id}" https://api.ror.org/organizations?query=oxford
+ +

Requests without a valid client ID are subject to a rate limit of 50 requests per 5 minute period.

+ +

We do not provide a way to recover or revoke a lost client ID. If you lose track of your client ID, please register a new one.

+ +

For more information about ROR API client IDs, see our documentation.

+ +

If you have questions, please see the ROR documentation or contact us at support@ror.org.

+ +

Cheers,
The ROR Team
support@ror.org
https://ror.org

+ """ + + msg = EmailMultiAlternatives(subject, text_content, from_email, recipient_list) + msg.attach_alternative(html_content, "text/html") + msg.send() return Response({'client_id': client.client_id}, status=status.HTTP_201_CREATED) @@ -72,10 +115,8 @@ def post(self, request, version='v2'): class ValidateClientView(APIView): def get(self, request, client_id): - # Check if the client_id exists in the database client_exists = Client.objects.filter(client_id=client_id).exists() - # Return response indicating whether client ID is valid return Response({'valid': client_exists}, status=status.HTTP_200_OK) class OurTokenPermission(BasePermission): From e036ffa4b24f5552181492cd6a7a9250664e61df Mon Sep 17 00:00:00 2001 From: kaysiz Date: Thu, 24 Apr 2025 15:06:05 +0200 Subject: [PATCH 42/50] fix variable substitution on template strings --- rorapi/common/views.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/rorapi/common/views.py b/rorapi/common/views.py index e431c0d..3fc17ab 100644 --- a/rorapi/common/views.py +++ b/rorapi/common/views.py @@ -60,13 +60,13 @@ def post(self, request, version='v2'): Your ROR API client ID is: - {client.client_id} + {} This client ID is not used for authentication or authorization, and is therefore not secret and can be sent as plain text. In order to receive a rate limit of 2000 requests per 5 minute period, please include this client ID with your ROR API requests, in a custom HTTP header named Client-Id, for example: - curl -H "Client-Id: {client.client_id}" https://api.ror.org/organizations?query=oxford + curl -H "Client-Id: {}" https://api.ror.org/organizations?query=oxford Requests without a valid client ID are subject to a rate limit of 50 requests per 5 minute period. @@ -78,20 +78,20 @@ def post(self, request, version='v2'): The ROR Team support@ror.org https://ror.org - """ + """.format(client.client_id, client.client_id) html_content = """

Thank you for registering for a ROR API client ID!

Your ROR API client ID is:

- 
{client.client_id}
+ 
{}

This client ID is not used for authentication or authorization, and is therefore not secret and can be sent as plain text.

In order to receive a rate limit of 2000 requests per 5 minute period, please include this client ID with your ROR API requests, in a custom HTTP header named Client-Id, for example:

- 
curl -H "Client-Id: {client.client_id}" https://api.ror.org/organizations?query=oxford
+ 
curl -H "Client-Id: {}" https://api.ror.org/organizations?query=oxford

Requests without a valid client ID are subject to a rate limit of 50 requests per 5 minute period.

@@ -102,7 +102,7 @@ def post(self, request, version='v2'):

If you have questions, please see the ROR documentation or contact us at support@ror.org.

Cheers,
The ROR Team
support@ror.org
https://ror.org

- """ + """.format(client.client_id, client.client_id) msg = EmailMultiAlternatives(subject, text_content, from_email, recipient_list) msg.attach_alternative(html_content, "text/html") From af95edc86f85816594102b8b53d6c862e76a1420 Mon Sep 17 00:00:00 2001 From: kaysiz Date: Thu, 24 Apr 2025 19:01:56 +0200 Subject: [PATCH 43/50] refactor the email templates --- rorapi/common/views.py | 74 +++++++++++++++++++++--------------------- 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/rorapi/common/views.py b/rorapi/common/views.py index 3fc17ab..b4aa66b 100644 --- a/rorapi/common/views.py +++ b/rorapi/common/views.py @@ -55,63 +55,63 @@ def post(self, request, version='v2'): from_email = 'support@ror.org' recipient_list = [client.email] - text_content = """ - Thank you for registering for a ROR API client ID! + html_content = self._get_html_content(client.client_id) + text_content = self._get_text_content(client.client_id) - Your ROR API client ID is: + msg = EmailMultiAlternatives(subject, text_content, from_email, recipient_list) + msg.attach_alternative(html_content, "text/html") + msg.send() - {} + return Response({'client_id': client.client_id}, status=status.HTTP_201_CREATED) - This client ID is not used for authentication or authorization, and is therefore not secret and can be sent as plain text. + return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) - In order to receive a rate limit of 2000 requests per 5 minute period, please include this client ID with your ROR API requests, in a custom HTTP header named Client-Id, for example: + def _get_text_content(self, client_id): + return f""" + Thank you for registering for a ROR API client ID! - curl -H "Client-Id: {}" https://api.ror.org/organizations?query=oxford + Your ROR API client ID is: + {client_id} - Requests without a valid client ID are subject to a rate limit of 50 requests per 5 minute period. + This client ID is not used for authentication or authorization, and is therefore not secret and can be sent as plain text. - We do not provide a way to recover or revoke a lost client ID. If you lose track of your client ID, please register a new client ID. For more information about ROR API client IDs, see https://ror.readme.io + In order to receive a rate limit of 2000 requests per 5 minute period, please include this client ID with your ROR API requests, in a custom HTTP header named Client-Id, for example: - If you have questions, please see ROR documentation or contact us at support@ror.org + curl -H "Client-Id: {client_id}" https://api.ror.org/organizations?query=oxford - Cheers, - The ROR Team - support@ror.org - https://ror.org - """.format(client.client_id, client.client_id) + Requests without a valid client ID are subject to a rate limit of 50 requests per 5 minute period. - html_content = """ -

Thank you for registering for a ROR API client ID!

+ We do not provide a way to recover or revoke a lost client ID. If you lose track of your client ID, please register a new client ID. For more information about ROR API client IDs, see https://ror.readme.io -

Your ROR API client ID is:

+ If you have questions, please see ROR documentation or contact us at support@ror.org - 
{}
+ Cheers, + The ROR Team + support@ror.org + https://ror.org + """ -

This client ID is not used for authentication or authorization, and is therefore not secret and can be sent as plain text.

+ def _get_html_content(self, client_id): + return f""" +
+

Thank you for registering for a ROR API client ID!

+

Your ROR API client ID is:

+ 
{client_id}
+

This client ID is not used for authentication or authorization, and is therefore not secret and can be sent as plain text.

In order to receive a rate limit of 2000 requests per 5 minute period, please include this client ID with your ROR API requests, in a custom HTTP header named Client-Id, for example:

- - 
curl -H "Client-Id: {}" https://api.ror.org/organizations?query=oxford
- + 
curl -H "Client-Id: {client_id}" https://api.ror.org/organizations?query=oxford

Requests without a valid client ID are subject to a rate limit of 50 requests per 5 minute period.

-

We do not provide a way to recover or revoke a lost client ID. If you lose track of your client ID, please register a new one.

-

For more information about ROR API client IDs, see our documentation.

-

If you have questions, please see the ROR documentation or contact us at support@ror.org.

+

Cheers,
+ The ROR Team
+ support@ror.org
+ https://ror.org

+
+ """ -

Cheers,
The ROR Team
support@ror.org
https://ror.org

- """.format(client.client_id, client.client_id) - - msg = EmailMultiAlternatives(subject, text_content, from_email, recipient_list) - msg.attach_alternative(html_content, "text/html") - msg.send() - - return Response({'client_id': client.client_id}, status=status.HTTP_201_CREATED) - - # Return validation errors if serializer is not valid - return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) class ValidateClientView(APIView): def get(self, request, client_id): From ddb5b557e20365c90f2231dccff20a85a87e19e0 Mon Sep 17 00:00:00 2001 From: Adam Date: Thu, 1 May 2025 14:22:42 -0700 Subject: [PATCH 44/50] Add fix for https://github.com/ror-community/ror-roadmap/issues/310 --- rorapi/common/csv_update.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rorapi/common/csv_update.py b/rorapi/common/csv_update.py index 3e41716..df12c45 100644 --- a/rorapi/common/csv_update.py +++ b/rorapi/common/csv_update.py @@ -127,7 +127,7 @@ def update_record_from_csv(csv_data, version): temp_ext_ids = [i for i in temp_ext_ids if i['type'] != t] else: - if not temp_preferred in temp_all: + if temp_preferred is not None and temp_preferred not in temp_all: errors.append("Changes to external ID object with type {} result in preferred value '{}' not in all values '{}'".format(t, temp_preferred, ", ".join(temp_all))) # remove all of type and replace with new obj new_ext_id_obj = { From 92ce69d62d14f261d2bc475a669d47386bbde7c5 Mon Sep 17 00:00:00 2001 From: Adam Date: Thu, 1 May 2025 15:25:10 -0700 Subject: [PATCH 45/50] Add fix for https://github.com/ror-community/ror-roadmap/issues/307 --- rorapi/management/commands/generaterorid.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rorapi/management/commands/generaterorid.py b/rorapi/management/commands/generaterorid.py index af07be0..a743981 100644 --- a/rorapi/management/commands/generaterorid.py +++ b/rorapi/management/commands/generaterorid.py @@ -23,7 +23,7 @@ def check_ror_id(version): ror_id = get_ror_id(generate_ror_id()) errors, organization = retrieve_organization(ror_id, version) if errors is None: - check_ror_id(version) + return check_ror_id(version) return ror_id From 35a21c52c41346b0ad96d05bb229a3bc151471c6 Mon Sep 17 00:00:00 2001 From: Adam Date: Thu, 1 May 2025 15:26:06 -0700 Subject: [PATCH 46/50] Add unit tests for https://github.com/ror-community/ror-roadmap/issues/308 --- .../tests_unit/tests_generaterorid_v1.py | 35 +++++++++++++++++++ .../tests_unit/tests_generaterorid_v2.py | 35 +++++++++++++++++++ 2 files changed, 70 insertions(+) create mode 100644 rorapi/tests/tests_unit/tests_generaterorid_v1.py create mode 100644 rorapi/tests/tests_unit/tests_generaterorid_v2.py diff --git a/rorapi/tests/tests_unit/tests_generaterorid_v1.py b/rorapi/tests/tests_unit/tests_generaterorid_v1.py new file mode 100644 index 0000000..d9fe5d8 --- /dev/null +++ b/rorapi/tests/tests_unit/tests_generaterorid_v1.py @@ -0,0 +1,35 @@ +from django.test import SimpleTestCase +from unittest.mock import patch +from rorapi.management.commands import generaterorid +from rorapi.common.models import Errors +from rorapi.settings import ROR_API + +DUPLICATE_ID_RAW = "duplicateid" +UNIQUE_ID_RAW = "uniqueid" +DUPLICATE_ROR_ID = f"{ROR_API['ID_PREFIX']}{DUPLICATE_ID_RAW}" +UNIQUE_ROR_ID = f"{ROR_API['ID_PREFIX']}{UNIQUE_ID_RAW}" +TEST_VERSION = 'v1' + +class GenerateRorIdCommandTestCase(SimpleTestCase): + + @patch('rorapi.management.commands.generaterorid.get_ror_id') + @patch('rorapi.management.commands.generaterorid.retrieve_organization') + @patch('rorapi.management.commands.generaterorid.generate_ror_id') + def test_check_ror_id_handles_collision_and_returns_unique( + self, mock_generate_ror_id, mock_retrieve_organization, mock_get_ror_id + ): + mock_generate_ror_id.side_effect = [ + DUPLICATE_ROR_ID, + UNIQUE_ROR_ID + ] + + mock_get_ror_id.side_effect = lambda x: x + + mock_retrieve_organization.side_effect = [ + (None, {'id': DUPLICATE_ROR_ID, 'name': 'Mock Duplicate Org'}), + (Errors(f"ROR ID '{UNIQUE_ROR_ID}' does not exist"), None) + ] + + result_ror_id = generaterorid.check_ror_id(TEST_VERSION) + + self.assertEqual(result_ror_id, UNIQUE_ROR_ID) \ No newline at end of file diff --git a/rorapi/tests/tests_unit/tests_generaterorid_v2.py b/rorapi/tests/tests_unit/tests_generaterorid_v2.py new file mode 100644 index 0000000..95f20b9 --- /dev/null +++ b/rorapi/tests/tests_unit/tests_generaterorid_v2.py @@ -0,0 +1,35 @@ +from django.test import SimpleTestCase +from unittest.mock import patch +from rorapi.management.commands import generaterorid +from rorapi.common.models import Errors +from rorapi.settings import ROR_API + +DUPLICATE_ID_RAW = "duplicateid" +UNIQUE_ID_RAW = "uniqueid" +DUPLICATE_ROR_ID = f"{ROR_API['ID_PREFIX']}{DUPLICATE_ID_RAW}" +UNIQUE_ROR_ID = f"{ROR_API['ID_PREFIX']}{UNIQUE_ID_RAW}" +TEST_VERSION = 'v2' + +class GenerateRorIdCommandTestCase(SimpleTestCase): + + @patch('rorapi.management.commands.generaterorid.get_ror_id') + @patch('rorapi.management.commands.generaterorid.retrieve_organization') + @patch('rorapi.management.commands.generaterorid.generate_ror_id') + def test_check_ror_id_handles_collision_and_returns_unique( + self, mock_generate_ror_id, mock_retrieve_organization, mock_get_ror_id + ): + mock_generate_ror_id.side_effect = [ + DUPLICATE_ROR_ID, + UNIQUE_ROR_ID + ] + + mock_get_ror_id.side_effect = lambda x: x + + mock_retrieve_organization.side_effect = [ + (None, {'id': DUPLICATE_ROR_ID, 'name': 'Mock Duplicate Org'}), + (Errors(f"ROR ID '{UNIQUE_ROR_ID}' does not exist"), None) + ] + + result_ror_id = generaterorid.check_ror_id(TEST_VERSION) + + self.assertEqual(result_ror_id, UNIQUE_ROR_ID) \ No newline at end of file From 70d24e089faa52461832d81c951015a2ab5b23b7 Mon Sep 17 00:00:00 2001 From: kudakwashe siziva <9620622+kaysiz@users.noreply.github.com> Date: Thu, 8 May 2025 09:00:17 +0200 Subject: [PATCH 47/50] Client ID Registration: QA feedback (#441) * Update readme url for client id reg * Update the email set for client id registration to api@ror.org --- rorapi/common/views.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/rorapi/common/views.py b/rorapi/common/views.py index b4aa66b..5a3ba44 100644 --- a/rorapi/common/views.py +++ b/rorapi/common/views.py @@ -52,7 +52,7 @@ def post(self, request, version='v2'): client = serializer.save() subject = 'ROR API client ID' - from_email = 'support@ror.org' + from_email = 'api@ror.org' recipient_list = [client.email] html_content = self._get_html_content(client.client_id) @@ -81,13 +81,13 @@ def _get_text_content(self, client_id): Requests without a valid client ID are subject to a rate limit of 50 requests per 5 minute period. - We do not provide a way to recover or revoke a lost client ID. If you lose track of your client ID, please register a new client ID. For more information about ROR API client IDs, see https://ror.readme.io + We do not provide a way to recover or revoke a lost client ID. If you lose track of your client ID, please register a new client ID. For more information about ROR API client IDs, see https://ror.readme.io/docs/client-id - If you have questions, please see ROR documentation or contact us at support@ror.org + If you have questions, please see ROR documentation or contact us at api@ror.org Cheers, The ROR Team - support@ror.org + api@ror.org https://ror.org """ @@ -103,11 +103,11 @@ def _get_html_content(self, client_id): 
curl -H "Client-Id: {client_id}" https://api.ror.org/organizations?query=oxford

Requests without a valid client ID are subject to a rate limit of 50 requests per 5 minute period.

We do not provide a way to recover or revoke a lost client ID. If you lose track of your client ID, please register a new one.

-

For more information about ROR API client IDs, see our documentation.

-

If you have questions, please see the ROR documentation or contact us at support@ror.org.

+

For more information about ROR API client IDs, see our documentation.

+

If you have questions, please see the ROR documentation or contact us at api@ror.org.

Cheers,
The ROR Team
- support@ror.org
+ api@ror.org
https://ror.org

""" From 80f1d43137e2a9c8f03fed1ceb8bfa94015c2b5d Mon Sep 17 00:00:00 2001 From: kudakwashe siziva <9620622+kaysiz@users.noreply.github.com> Date: Thu, 8 May 2025 15:43:06 +0200 Subject: [PATCH 48/50] Update name for from_email for client id registration (#442) --- rorapi/common/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rorapi/common/views.py b/rorapi/common/views.py index 5a3ba44..31c3971 100644 --- a/rorapi/common/views.py +++ b/rorapi/common/views.py @@ -52,7 +52,7 @@ def post(self, request, version='v2'): client = serializer.save() subject = 'ROR API client ID' - from_email = 'api@ror.org' + from_email = "ROR API Support " recipient_list = [client.email] html_content = self._get_html_content(client.client_id) From 215456c4a190951d5d1c4abacb4789b29a232625 Mon Sep 17 00:00:00 2001 From: kudakwashe siziva <9620622+kaysiz@users.noreply.github.com> Date: Thu, 8 May 2025 19:26:51 +0200 Subject: [PATCH 49/50] update support email address (#443) --- rorapi/common/views.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rorapi/common/views.py b/rorapi/common/views.py index 31c3971..554e22b 100644 --- a/rorapi/common/views.py +++ b/rorapi/common/views.py @@ -83,11 +83,11 @@ def _get_text_content(self, client_id): We do not provide a way to recover or revoke a lost client ID. If you lose track of your client ID, please register a new client ID. For more information about ROR API client IDs, see https://ror.readme.io/docs/client-id - If you have questions, please see ROR documentation or contact us at api@ror.org + If you have questions, please see ROR documentation or contact us at support@ror.org Cheers, The ROR Team - api@ror.org + support@ror.org https://ror.org """ @@ -104,10 +104,10 @@ def _get_html_content(self, client_id):

Requests without a valid client ID are subject to a rate limit of 50 requests per 5 minute period.

We do not provide a way to recover or revoke a lost client ID. If you lose track of your client ID, please register a new one.

For more information about ROR API client IDs, see our documentation.

-

If you have questions, please see the ROR documentation or contact us at api@ror.org.

+

If you have questions, please see the ROR documentation or contact us at support@ror.org.

Cheers,
The ROR Team
- api@ror.org
+ support@ror.org
https://ror.org

""" From 4cdaa04b93f6c1bbf463f6efb107b1d0c185bf5f Mon Sep 17 00:00:00 2001 From: kudakwashe siziva <9620622+kaysiz@users.noreply.github.com> Date: Wed, 14 May 2025 19:58:44 +0200 Subject: [PATCH 50/50] Client ID registration (#444) --- .github/workflows/dev.yml | 11 +++ .github/workflows/release.yml | 11 +++ .github/workflows/staging.yml | 11 +++ Dockerfile | 3 +- README.md | 4 +- docker-compose.yml | 15 ++++ requirements.txt | 6 +- rorapi/common/urls.py | 4 +- rorapi/common/views.py | 79 ++++++++++++++++++- rorapi/management/commands/generaterorid.py | 7 ++ rorapi/migrations/0001_create_client_model.py | 30 +++++++ rorapi/migrations/0002_auto_20250326_1054.py | 18 +++++ rorapi/migrations/0003_auto_20250415_1207.py | 43 ++++++++++ rorapi/migrations/__init__.py | 0 rorapi/settings.py | 31 +++++++- rorapi/tests/tests_unit/tests_client.py | 7 ++ rorapi/v2/models.py | 41 +++++++++- rorapi/v2/serializers.py | 74 +++++++++++++++++ rorapi/v2/tests.py | 11 +++ 19 files changed, 398 insertions(+), 8 deletions(-) create mode 100644 rorapi/migrations/0001_create_client_model.py create mode 100644 rorapi/migrations/0002_auto_20250326_1054.py create mode 100644 rorapi/migrations/0003_auto_20250415_1207.py create mode 100644 rorapi/migrations/__init__.py create mode 100644 rorapi/tests/tests_unit/tests_client.py create mode 100644 rorapi/v2/tests.py diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 6199f77..3bfa57f 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -18,6 +18,7 @@ jobs: AWS_REGION: ${{ secrets.AWS_REGION }} GITHUB_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} LAUNCH_DARKLY_KEY: ${{ secrets.LAUNCH_DARKLY_KEY_DEV }} + DB_HOST: 127.0.0.1 # Will not work with 'localhost', since that will try a Unix socket connection (!) services: elasticsearch7: image: docker.elastic.co/elasticsearch/elasticsearch:7.10.0 @@ -30,6 +31,16 @@ jobs: http.cors.allow-origin: "*" ports: - 9200:9200 + db: + image: mysql:8.0 + env: + MYSQL_DATABASE: "rorapi" + MYSQL_USER: "ror_user" + MYSQL_PASSWORD: "password" + MYSQL_ROOT_PASSWORD: "password" + ports: + - 3306:3306 + options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3 steps: - name: Checkout ror-api code uses: actions/checkout@v2 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bfd0346..fede267 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,6 +9,7 @@ jobs: ELASTIC_PASSWORD: "changeme" ELASTIC7_HOST: "localhost" ELASTIC7_PORT: "9200" + DB_HOST: 127.0.0.1 AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_REGION: ${{ secrets.AWS_REGION }} @@ -26,6 +27,16 @@ jobs: http.cors.allow-origin: "*" ports: - 9200:9200 + db: + image: mysql:8.0 + env: + MYSQL_DATABASE: "rorapi" + MYSQL_USER: "ror_user" + MYSQL_PASSWORD: "password" + MYSQL_ROOT_PASSWORD: "password" + ports: + - 3306:3306 + options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3 steps: - name: Checkout ror-api code uses: actions/checkout@v2 diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml index 57bef2f..42907f1 100644 --- a/.github/workflows/staging.yml +++ b/.github/workflows/staging.yml @@ -10,6 +10,7 @@ jobs: ELASTIC_PASSWORD: "changeme" ELASTIC7_HOST: "localhost" ELASTIC7_PORT: "9200" + DB_HOST: 127.0.0.1 AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_REGION: ${{ secrets.AWS_REGION }} @@ -27,6 +28,16 @@ jobs: http.cors.allow-origin: "*" ports: - 9200:9200 + db: + image: mysql:8.0 + env: + MYSQL_DATABASE: "rorapi" + MYSQL_USER: "ror_user" + MYSQL_PASSWORD: "password" + MYSQL_ROOT_PASSWORD: "password" + ports: + - 3306:3306 + options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3 steps: - name: Checkout ror-api code uses: actions/checkout@v2 diff --git a/Dockerfile b/Dockerfile index 9ebc84e..09b18d6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,7 @@ RUN mv /etc/apt/sources.list.d /etc/apt/sources.list.d.bak && \ mv /etc/apt/sources.list.d.bak /etc/apt/sources.list.d && \ apt-get upgrade -y -o Dpkg::Options::="--force-confold" && \ apt-get clean && \ - apt-get install ntp wget unzip tzdata python3-pip libmagic1 -y && \ + apt-get install ntp wget unzip tzdata python3-pip libmagic1 default-libmysqlclient-dev libcairo2-dev pkg-config -y && \ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* # Enable Passenger and Nginx and remove the default site @@ -54,6 +54,7 @@ RUN pip3 install --no-cache-dir -r requirements.txt RUN pip3 install yapf # collect static files for Django +ENV DJANGO_SKIP_DB_CHECK=True RUN python manage.py collectstatic --noinput # Expose web diff --git a/README.md b/README.md index 953426d..dd58935 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,9 @@ Commands for indexing ROR data, generating new ROR IDs and other internal operat ROUTE_USER=[USER] TOKEN=[TOKEN] -Replace values in [] with valid credential values. GITHUB_TOKEN is needed in order to index an existing data dump locally. ROUTE_USER and TOKEN are only needed in order to use generate-id functionality locally. AWS_* and DATA_STORE are only needed in order to use incremental indexing from S3 functionality locally. +ROR staff should replace values in [] with valid credential values. External users do not need to add these values but should comment out this line https://github.com/ror-community/ror-api/blob/8a5a5ae8b483564c966a7184349c581dcae756ef/rorapi/management/commands/setup.py#L13 so that there is no attempt to send a Github token when retrieving a data dump for indexing. + +- Optionally, uncomment [line 24 in docker-compose.yml](https://github.com/ror-community/ror-api/blob/master/docker-compose.yml#L24) in order to pull the rorapi image from Dockerhub rather than creating it from local code ## Start ror-api locally 1. Start Docker Desktop diff --git a/docker-compose.yml b/docker-compose.yml index 0cd1b0b..253e3c3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,6 +19,18 @@ services: timeout: 1s volumes: - ./esdata:/usr/share/elasticsearch/data + db: + image: mysql:8.0 + volumes: + - mysql_data:/var/lib/mysql + env_file: + - .env + ports: + - "3306:3306" + healthcheck: + test: ["CMD", "mysqladmin", "ping", "-h", "localhost"] + timeout: 20s + retries: 10 web: container_name: rorapiweb env_file: .env @@ -31,3 +43,6 @@ services: - ./rorapi:/home/app/webapp/rorapi depends_on: - elasticsearch7 + - db +volumes: + mysql_data: diff --git a/requirements.txt b/requirements.txt index f28d73b..a44a9b8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,4 +24,8 @@ update_address @ git+https://github.com/ror-community/update_address.git launchdarkly-server-sdk==7.6.1 jsonschema==3.2.0 python-magic -iso639-lang \ No newline at end of file +iso639-lang +mysqlclient==2.2.7 +bleach==6.0.0 +pycountry==22.3.5 +django-ses==3.5.0 \ No newline at end of file diff --git a/rorapi/common/urls.py b/rorapi/common/urls.py index b9aa57c..277e0bd 100644 --- a/rorapi/common/urls.py +++ b/rorapi/common/urls.py @@ -3,7 +3,7 @@ from rest_framework.documentation import include_docs_urls from . import views from rorapi.common.views import ( - HeartbeatView,GenerateAddress,GenerateId,IndexData,IndexDataDump,BulkUpdate) + HeartbeatView,GenerateAddress,GenerateId,IndexData,IndexDataDump,BulkUpdate,ClientRegistrationView,ValidateClientView) urlpatterns = [ # Health check @@ -14,6 +14,8 @@ path('generateaddress/', GenerateAddress.as_view()), url(r"^generateid$", GenerateId.as_view()), re_path(r"^(?P(v1|v2))\/bulkupdate$", BulkUpdate.as_view()), + re_path(r"^(?P(v1|v2))\/register$", ClientRegistrationView.as_view()), + path('validate-client-id//', ValidateClientView.as_view()), url(r"^(?P(v1|v2))\/indexdata/(?P.*)", IndexData.as_view()), url(r"^(?P(v1|v2))\/indexdatadump\/(?Pv(\d+\.)?(\d+\.)?(\*|\d+)-\d{4}-\d{2}-\d{2}-ror-data)\/(?P(test|prod))$", IndexDataDump.as_view()), url(r"^(?P(v1|v2))\/", include(views.organizations_router.urls)), diff --git a/rorapi/common/views.py b/rorapi/common/views.py index ebcfa21..554e22b 100644 --- a/rorapi/common/views.py +++ b/rorapi/common/views.py @@ -37,10 +37,87 @@ import os import update_address as ua from rorapi.management.commands.generaterorid import check_ror_id -from rorapi.management.commands.generaterorid import check_ror_id from rorapi.management.commands.indexror import process_files from django.core import management import rorapi.management.commands.indexrordump +from django.core.mail import EmailMultiAlternatives +from django.utils.timezone import now +from rorapi.v2.models import Client +from rorapi.v2.serializers import ClientSerializer + +class ClientRegistrationView(APIView): + def post(self, request, version='v2'): + serializer = ClientSerializer(data=request.data) + if serializer.is_valid(): + client = serializer.save() + + subject = 'ROR API client ID' + from_email = "ROR API Support " + recipient_list = [client.email] + + html_content = self._get_html_content(client.client_id) + text_content = self._get_text_content(client.client_id) + + msg = EmailMultiAlternatives(subject, text_content, from_email, recipient_list) + msg.attach_alternative(html_content, "text/html") + msg.send() + + return Response({'client_id': client.client_id}, status=status.HTTP_201_CREATED) + + return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) + + def _get_text_content(self, client_id): + return f""" + Thank you for registering for a ROR API client ID! + + Your ROR API client ID is: + {client_id} + + This client ID is not used for authentication or authorization, and is therefore not secret and can be sent as plain text. + + In order to receive a rate limit of 2000 requests per 5 minute period, please include this client ID with your ROR API requests, in a custom HTTP header named Client-Id, for example: + + curl -H "Client-Id: {client_id}" https://api.ror.org/organizations?query=oxford + + Requests without a valid client ID are subject to a rate limit of 50 requests per 5 minute period. + + We do not provide a way to recover or revoke a lost client ID. If you lose track of your client ID, please register a new client ID. For more information about ROR API client IDs, see https://ror.readme.io/docs/client-id + + If you have questions, please see ROR documentation or contact us at support@ror.org + + Cheers, + The ROR Team + support@ror.org + https://ror.org + """ + + + def _get_html_content(self, client_id): + return f""" +
+

Thank you for registering for a ROR API client ID!

+

Your ROR API client ID is:

+ 
{client_id}
+

This client ID is not used for authentication or authorization, and is therefore not secret and can be sent as plain text.

+

In order to receive a rate limit of 2000 requests per 5 minute period, please include this client ID with your ROR API requests, in a custom HTTP header named Client-Id, for example:

+ 
curl -H "Client-Id: {client_id}" https://api.ror.org/organizations?query=oxford
+

Requests without a valid client ID are subject to a rate limit of 50 requests per 5 minute period.

+

We do not provide a way to recover or revoke a lost client ID. If you lose track of your client ID, please register a new one.

+

For more information about ROR API client IDs, see our documentation.

+

If you have questions, please see the ROR documentation or contact us at support@ror.org.

+

Cheers,
+ The ROR Team
+ support@ror.org
+ https://ror.org

+
+ """ + + +class ValidateClientView(APIView): + def get(self, request, client_id): + client_exists = Client.objects.filter(client_id=client_id).exists() + + return Response({'valid': client_exists}, status=status.HTTP_200_OK) class OurTokenPermission(BasePermission): """ diff --git a/rorapi/management/commands/generaterorid.py b/rorapi/management/commands/generaterorid.py index 2ea42a5..af07be0 100644 --- a/rorapi/management/commands/generaterorid.py +++ b/rorapi/management/commands/generaterorid.py @@ -26,3 +26,10 @@ def check_ror_id(version): check_ror_id(version) return ror_id + +def generate_ror_client_id(): + """Generates a random ROR client ID. + """ + + n = random.randint(0, 2**160 - 1) + return base32_crockford.encode(n).lower().zfill(32) diff --git a/rorapi/migrations/0001_create_client_model.py b/rorapi/migrations/0001_create_client_model.py new file mode 100644 index 0000000..b7b558f --- /dev/null +++ b/rorapi/migrations/0001_create_client_model.py @@ -0,0 +1,30 @@ +# Generated by Django 2.2.28 on 2025-03-11 07:13 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='Client', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('email', models.EmailField(max_length=255)), + ('name', models.CharField(blank=True, max_length=255)), + ('institution_name', models.CharField(blank=True, max_length=255)), + ('institution_ror', models.URLField(blank=True, max_length=255)), + ('country_code', models.CharField(blank=True, max_length=2)), + ('ror_use', models.TextField(blank=True, max_length=500)), + ('client_id', models.CharField(editable=False, max_length=32, unique=True)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('last_request_at', models.DateTimeField(blank=True, null=True)), + ('request_count', models.IntegerField(default=0)), + ], + ), + ] diff --git a/rorapi/migrations/0002_auto_20250326_1054.py b/rorapi/migrations/0002_auto_20250326_1054.py new file mode 100644 index 0000000..0d06ed9 --- /dev/null +++ b/rorapi/migrations/0002_auto_20250326_1054.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.28 on 2025-03-26 10:54 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('rorapi', '0001_create_client_model'), + ] + + operations = [ + migrations.AlterField( + model_name='client', + name='email', + field=models.EmailField(max_length=255, unique=True), + ), + ] diff --git a/rorapi/migrations/0003_auto_20250415_1207.py b/rorapi/migrations/0003_auto_20250415_1207.py new file mode 100644 index 0000000..0697d02 --- /dev/null +++ b/rorapi/migrations/0003_auto_20250415_1207.py @@ -0,0 +1,43 @@ +# Generated by Django 2.2.28 on 2025-04-15 12:07 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('rorapi', '0002_auto_20250326_1054'), + ] + + operations = [ + migrations.AlterField( + model_name='client', + name='country_code', + field=models.CharField(blank=True, max_length=2, null=True), + ), + migrations.AlterField( + model_name='client', + name='email', + field=models.EmailField(max_length=255), + ), + migrations.AlterField( + model_name='client', + name='institution_name', + field=models.CharField(blank=True, max_length=255, null=True), + ), + migrations.AlterField( + model_name='client', + name='institution_ror', + field=models.URLField(blank=True, max_length=255, null=True), + ), + migrations.AlterField( + model_name='client', + name='name', + field=models.CharField(blank=True, max_length=255, null=True), + ), + migrations.AlterField( + model_name='client', + name='ror_use', + field=models.TextField(blank=True, max_length=500, null=True), + ), + ] diff --git a/rorapi/migrations/__init__.py b/rorapi/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/rorapi/settings.py b/rorapi/settings.py index a8f3504..e1a7f8e 100644 --- a/rorapi/settings.py +++ b/rorapi/settings.py @@ -11,6 +11,7 @@ """ import os +import sys import json import sentry_sdk import boto3 @@ -71,7 +72,7 @@ 'django.contrib.auth.middleware.AuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', - 'django_prometheus.middleware.PrometheusAfterMiddleware', + 'django_prometheus.middleware.PrometheusAfterMiddleware' ] ROOT_URLCONF = 'rorapi.common.urls' @@ -105,7 +106,23 @@ # Database # https://docs.djangoproject.com/en/2.2/ref/settings/#databases -DATABASES = {} +if 'collectstatic' in sys.argv and os.environ.get('DJANGO_SKIP_DB_CHECK') == 'True': + DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.dummy' + } + } +else: + DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.mysql', + 'NAME': os.environ.get('DB_NAME', 'rorapi'), + 'USER': os.environ.get('DB_USER', 'root'), + 'PASSWORD': os.environ.get('DB_PASSWORD', 'password'), + 'HOST': os.environ.get('DB_HOST', 'db'), + 'PORT': os.environ.get('DB_PORT', '3306'), + } +} # Password validation # https://docs.djangoproject.com/en/2.2/ref/settings/#auth-password-validators @@ -273,3 +290,13 @@ GRID_REMOVED_IDS = [] LAUNCH_DARKLY_KEY = os.environ.get('LAUNCH_DARKLY_KEY') + +# Toggle for behavior-based rate limiting +import os +ENABLE_BEHAVIORAL_LIMITING = os.getenv("ENABLE_BEHAVIORAL_LIMITING", "False") == "True" + +# Email settings for Django +EMAIL_BACKEND = 'django_ses.SESBackend' +AWS_ACCESS_KEY_ID = os.environ.get('AWS_ACCESS_KEY_ID') +AWS_SECRET_ACCESS_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY') +AWS_SES_REGION_NAME = os.environ.get('AWS_REGION', 'eu-west-1') diff --git a/rorapi/tests/tests_unit/tests_client.py b/rorapi/tests/tests_unit/tests_client.py new file mode 100644 index 0000000..d70ccc9 --- /dev/null +++ b/rorapi/tests/tests_unit/tests_client.py @@ -0,0 +1,7 @@ +from django.test import TestCase +from rorapi.v2.models import Client + +class ClientTests(TestCase): + def test_client_registration(self): + client = Client.objects.create(email='test@example.com') + self.assertIsNotNone(client.client_id) \ No newline at end of file diff --git a/rorapi/v2/models.py b/rorapi/v2/models.py index e9c7d1a..1b5a82f 100644 --- a/rorapi/v2/models.py +++ b/rorapi/v2/models.py @@ -1,4 +1,7 @@ from geonamescache.mappers import country +import random +import string +from django.db import models from rorapi.common.models import TypeBucket, CountryBucket, StatusBucket, Entity from rorapi.v2.record_constants import continent_code_to_name @@ -130,4 +133,40 @@ class MatchingResult: def __init__(self, data): self.number_of_results = len(data) - self.items = [MatchedOrganization(x) for x in data] \ No newline at end of file + self.items = [MatchedOrganization(x) for x in data] + + +class Client(models.Model): + # Required fields + email = models.EmailField(max_length=255) + + # Optional fields + name = models.CharField(max_length=255, blank=True, null=True) + institution_name = models.CharField(max_length=255, blank=True, null=True) + institution_ror = models.URLField(max_length=255, blank=True, null=True) + country_code = models.CharField(max_length=2, blank=True, null=True) + ror_use = models.TextField(max_length=500, blank=True, null=True) + + # System fields + client_id = models.CharField( + max_length=32, + unique=True, + editable=False + ) + created_at = models.DateTimeField(auto_now_add=True) + last_request_at = models.DateTimeField(null=True, blank=True) + request_count = models.IntegerField(default=0) + + def __str__(self): + return f"{self.email} - {self.client_id}" + + @staticmethod + def generate_client_id(): + """Generate a unique 32-character client ID""" + return ''.join(random.choices(string.ascii_uppercase + string.digits, k=32)) + + def save(self, *args, **kwargs): + # Ensure client_id is generated before saving + if not self.client_id: # Only generate if it's empty + self.client_id = self.generate_client_id() + super().save(*args, **kwargs) diff --git a/rorapi/v2/serializers.py b/rorapi/v2/serializers.py index 62327a5..002fd8e 100644 --- a/rorapi/v2/serializers.py +++ b/rorapi/v2/serializers.py @@ -1,4 +1,8 @@ from rest_framework import serializers +import bleach +import pycountry +import re +from rorapi.v2.models import Client from rorapi.common.serializers import BucketSerializer, OrganizationRelationshipsSerializer class AggregationsSerializer(serializers.Serializer): @@ -87,3 +91,73 @@ class MatchedOrganizationSerializer(serializers.Serializer): class MatchingResultSerializer(serializers.Serializer): number_of_results = serializers.IntegerField() items = MatchedOrganizationSerializer(many=True) + + +class ClientSerializer(serializers.ModelSerializer): + class Meta: + model = Client + fields = ['email', 'name', 'institution_name', 'institution_ror', 'country_code', 'ror_use'] + extra_kwargs = { + 'name': {'required': False, 'allow_null': True}, + 'institution_name': {'required': False, 'allow_null': True}, + 'institution_ror': {'required': False, 'allow_null': True}, + 'country_code': {'required': False, 'allow_null': True}, + 'ror_use': {'required': False, 'allow_null': True}, + } + + def validate_email(self, value): + """Validate the email format and ensure it's unique.""" + if value is None: + raise serializers.ValidationError("Email cannot be null.") + return value + + def validate_name(self, value): + """Sanitize name and validate length. Reject empty string.""" + if value is not None: + if value == "": + raise serializers.ValidationError("Name cannot be an empty string.") + value = bleach.clean(value) # Sanitize to strip HTML + if len(value) > 255: + raise serializers.ValidationError("Name cannot be longer than 255 characters.") + return value + + def validate_institution_name(self, value): + """Sanitize institution name and validate length. Reject empty string.""" + if value is not None: + if value == "": + raise serializers.ValidationError("Institution name cannot be an empty string.") + value = bleach.clean(value) # Sanitize to strip HTML + if len(value) > 255: + raise serializers.ValidationError("Institution name cannot be longer than 255 characters.") + return value + + def validate_institution_ror(self, value): + """Validate and format institution ROR to match 'https://ror.org/XXXXX'. Reject empty string.""" + if value is not None: + if value == "": + raise serializers.ValidationError("Institution ROR cannot be an empty string.") + value = bleach.clean(value) # Sanitize to strip HTML + ror_regex = r'https://ror\.org/[A-Za-z0-9]+' + if not re.match(ror_regex, value): + raise serializers.ValidationError("Institution ROR must be in the format 'https://ror.org/XXXXX'.") + return value + + def validate_country_code(self, value): + """Validate that the country code is a valid ISO 3166-1 alpha-2 country code. Reject empty string.""" + if value is not None: + if value == "": + raise serializers.ValidationError("Country code cannot be an empty string.") + value = value.strip().upper() # Normalize to uppercase + if len(value) != 2 or not pycountry.countries.get(alpha_2=value): + raise serializers.ValidationError(f"{value} is not a valid ISO 3166-1 alpha-2 country code.") + return value + + def validate_ror_use(self, value): + """Sanitize ror_use and validate length. Reject empty string.""" + if value is not None: + if value == "": + raise serializers.ValidationError("ROR use cannot be an empty string.") + value = bleach.clean(value) # Sanitize to strip HTML + if len(value) > 500: + raise serializers.ValidationError("ROR use cannot be longer than 500 characters.") + return value diff --git a/rorapi/v2/tests.py b/rorapi/v2/tests.py new file mode 100644 index 0000000..5e2c9ef --- /dev/null +++ b/rorapi/v2/tests.py @@ -0,0 +1,11 @@ +from django.test import TestCase +from .models.client import Client + +class ClientTests(TestCase): + def test_client_registration(self): + client = Client.objects.create(email='test@example.com') + self.assertIsNotNone(client.client_id) + + def test_rate_limiting(self): + response = self.client.get('/client-id/', HTTP_CLIENT_ID="INVALID_ID") + self.assertEqual(response.status_code, 429)