diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9d2f4423..48ba9a56 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,9 +8,9 @@ jobs: strategy: matrix: - python-version: ["3.7", "3.8"] - django-version: ["2.1.1", "3.1.4"] - database-engine: ["postgres", "mysql"] + python-version: [ "3.7", "3.8" ] + django-version: [ "2.1.1", "3.1.4" ] + database-engine: [ "postgres", "mysql", "mssql"] services: postgres: @@ -37,6 +37,15 @@ jobs: ports: - 3306:3306 + + mssqldb: + image: "mcr.microsoft.com/mssql/server:2017-latest" + env: + ACCEPT_EULA: y + SA_PASSWORD: '~Test123' + ports: + - 1433:1433 + steps: - name: Checkout code uses: actions/checkout@v2 @@ -73,11 +82,16 @@ jobs: mysql_tzinfo_to_sql /usr/share/zoneinfo | mysql --protocol=TCP -h localhost -u root -prootpassword mysql if: matrix.database-engine == 'mysql' + - name: Prepare mssql database + run: | + /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P '~Test123' -m-1 -Q 'CREATE DATABASE [binder-test];' + if: matrix.database-engine == 'mssql' + - name: Run tests run: | .venv/bin/coverage run --include="binder/*" setup.py test env: - BINDER_TEST_MYSQL: ${{ matrix.database-engine == 'mysql' && 1 || 0 }} + BINDER_TEST_DATABASE_ENGINE: ${{ matrix.database-engine }} CY_RUNNING_INSIDE_CI: 1 - name: Upload coverage report diff --git a/README.md b/README.md index da9d2020..7c5527da 100644 --- a/README.md +++ b/README.md @@ -14,11 +14,19 @@ There are two ways to run the tests: - Run with docker `docker-compose run binder ./setup.py test` - Access the test database directly by with `docker-compose run db psql -h db -U postgres`. - It may be possible to recreate the test database (for example when you added/changed models). One way of achieving this is to just remove all the docker images that were build `docker-compose rm`. The database will be created during the setup in `tests/__init__.py`. - + The tests are set up in such a way that there is no need to keep migration files. The setup procedure in `tests/__init__.py` handles the preparation of the database by directly calling some build-in Django commands. To only run a selection of the tests, use the `-s` flag like `./setup.py test -s tests.test_some_specific_test`. +### Running test other database systems +Locally the tests can be run for MySQL or MSSQL by overriding the `BINDER_TEST_DATABASE_ENGINE` environment variable: + +- `docker-compose run -e BINDER_TEST_DATABASE_ENGINE=mysql binder ./setup.py test` for MySQL +- `docker-compose run -e BINDER_TEST_DATABASE_ENGINE=mssql binder ./setup.py test` for MsSQL + + + ## MySQL support MySQL is supported, but only with the goal to replace it with diff --git a/binder/models.py b/binder/models.py index 3764054b..7c526754 100644 --- a/binder/models.py +++ b/binder/models.py @@ -405,6 +405,7 @@ class Meta: def save(self, *args, **kwargs): self.full_clean() # Never allow saving invalid models! + return super().save(*args, **kwargs) diff --git a/binder/views.py b/binder/views.py index c12c1760..56e3446c 100644 --- a/binder/views.py +++ b/binder/views.py @@ -315,11 +315,18 @@ class ModelView(View): # } virtual_relations = {} + @property + def database_type(self): + return { + 'mysql': 'mysql', + 'microsoft': 'mssql', + }.get(connections[self.model.objects.db].vendor, 'postgres') + @property def AggStrategy(self): - if connections[self.model.objects.db].vendor == 'mysql': + if self.database_type == 'mysql': return GroupConcat - if connections[self.model.objects.db].vendor == 'microsoft': + if self.database_type == 'mssql': return StringAgg return OrderableArrayAgg @@ -869,6 +876,14 @@ def _follow_related(self, fieldspec): # permission scoping. This will be done when fetching the actual # objects. def _get_with_ids(self, pks, request, include_annotations, with_map, where_map): + if self.database_type == 'mssql': + with_ids = {} + for key, sub_with_map in with_map.items(): + with_ids.update(self._base_get_with_ids(pks, request, include_annotations, {key: sub_with_map}, where_map)) + return with_ids + return self._base_get_with_ids( pks, request, include_annotations, with_map, where_map) + + def _base_get_with_ids(self, pks, request, include_annotations, with_map, where_map): result = {} annotations = {} diff --git a/docker-compose.yml b/docker-compose.yml index 5e890a8a..dcdacd3d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,9 +4,28 @@ services: db: image: postgres:11.5 binder: - build: . + build: + context: ./ + dockerfile: docker/binder/Dockerfile command: tail -f /dev/null volumes: - .:/binder depends_on: - db + - mysql + - mssqldb + mysql: + image: mysql + environment: + MYSQL_ROOT_PASSWORD: rootpassword + ports: + - 3306:3306 + mssqldb: + build: + context: ./ + dockerfile: docker/mssqldb/Dockerfile + environment: + ACCEPT_EULA: y + SA_PASSWORD: '~Test123' + ports: + - 1433:1433 \ No newline at end of file diff --git a/Dockerfile b/docker/binder/Dockerfile similarity index 58% rename from Dockerfile rename to docker/binder/Dockerfile index 41f16d7d..bc1c2d94 100644 --- a/Dockerfile +++ b/docker/binder/Dockerfile @@ -1,5 +1,6 @@ -FROM python:3 +FROM python:3-buster ENV PYTHONUNBUFFERED 1 RUN mkdir /binder WORKDIR /binder ADD . /binder +RUN ./install-mssql-driver.sh diff --git a/docker/mssqldb/Dockerfile b/docker/mssqldb/Dockerfile new file mode 100644 index 00000000..2e1ed3ab --- /dev/null +++ b/docker/mssqldb/Dockerfile @@ -0,0 +1,12 @@ +FROM mcr.microsoft.com/mssql/server:2017-latest +WORKDIR / + +ENV ACCEPT_EULA Y +ENV SA_PASSWORD "~Test123" + +COPY docker/mssqldb/init /init +COPY docker/mssqldb/bootstrap.sql /bootstrap.sql +COPY docker/mssqldb/entrypoint.sh /entrypoint.sh + +EXPOSE 1433 +CMD /entrypoint.sh \ No newline at end of file diff --git a/docker/mssqldb/bootstrap.sql b/docker/mssqldb/bootstrap.sql new file mode 100644 index 00000000..9b64f644 --- /dev/null +++ b/docker/mssqldb/bootstrap.sql @@ -0,0 +1,4 @@ +USE [master] +GO +CREATE DATABASE [binder-test] +GO \ No newline at end of file diff --git a/docker/mssqldb/entrypoint.sh b/docker/mssqldb/entrypoint.sh new file mode 100755 index 00000000..962beb49 --- /dev/null +++ b/docker/mssqldb/entrypoint.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +#start SQL Server, start the script to create the DB and import the data, start the app +nohup /init & +/opt/mssql/bin/sqlservr diff --git a/docker/mssqldb/init b/docker/mssqldb/init new file mode 100755 index 00000000..1f63253b --- /dev/null +++ b/docker/mssqldb/init @@ -0,0 +1,7 @@ +#!/bin/sh + +#wait for the SQL Server to come up +sleep 20s + +echo "Running CREATE DATABASE" +/opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P '~Test123' -m-1 -i bootstrap.sql \ No newline at end of file diff --git a/install-mssql-driver.sh b/install-mssql-driver.sh new file mode 100755 index 00000000..05888a87 --- /dev/null +++ b/install-mssql-driver.sh @@ -0,0 +1,14 @@ +curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - + +#Download appropriate package for the OS version +#Choose only ONE of the following, corresponding to your OS version + +#Debian 10 +curl https://packages.microsoft.com/config/debian/10/prod.list > /etc/apt/sources.list.d/mssql-release.list + +apt-get update +ACCEPT_EULA=Y apt-get install -y msodbcsql17 +# optional: for bcp and sqlcmd +ACCEPT_EULA=Y apt-get install -y mssql-tools +echo 'export PATH="$PATH:/opt/mssql-tools/bin"' >> ~/.bashrc +apt-get install -y unixodbc-dev \ No newline at end of file diff --git a/project/packages.pip b/project/packages.pip index 88f0be8c..c49e8afd 100644 --- a/project/packages.pip +++ b/project/packages.pip @@ -3,4 +3,6 @@ Pillow django-request-id psycopg2 requests -openpyxl \ No newline at end of file +openpyxl +django-pyodbc-azure +django-pyodbc \ No newline at end of file diff --git a/setup.py b/setup.py index 31a0f90f..99184980 100755 --- a/setup.py +++ b/setup.py @@ -9,6 +9,14 @@ # allow setup.py to be run from any path os.chdir(os.path.normpath(os.path.join(os.path.abspath(__file__), os.pardir))) +test_require_database_engine = { + 'mysql': ['mysqlclient >= 1.3.12', 'psycopg2 >= 2.7'], + 'mssql': ['django-pyodbc-azure >= 2.1.0.0', 'django-pyodbc >= 1.1.3', 'django-hijack == 2.1.10', 'psycopg2 >= 2.7'], + # Alternative mssql setup with django 3 setup. For later + # 'mssql': ['mssql-django==1.0rc1', 'psycopg2 >= 2.7'] +}.get(os.environ.get('BINDER_TEST_DATABASE_ENGINE'), ['psycopg2 >= 2.7']) + + setup( name='django-binder', version='1.5.0', @@ -44,11 +52,7 @@ ], tests_require=[ 'django-hijack >= 2.1.10', - ( - 'mysqlclient >= 1.3.12' - if os.environ.get('BINDER_TEST_MYSQL', '0') == '1' else - 'psycopg2 >= 2.7' - ), + *test_require_database_engine, "openpyxl >= 3.0.0" ], ) diff --git a/tests/__init__.py b/tests/__init__.py index 4dcaf8bd..bea87d5d 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -3,18 +3,12 @@ from django.core.management import call_command import os -if ( - os.path.exists('/.dockerenv') and - 'CY_RUNNING_INSIDE_CI' not in os.environ -): - db_settings = { - 'ENGINE': 'django.db.backends.postgresql', - 'NAME': 'postgres', - 'USER': 'postgres', - 'HOST': 'db', - 'PORT': 5432, - } -elif os.environ.get('BINDER_TEST_MYSQL', '0') == '1': +if os.environ.get('BINDER_TEST_DATABASE_ENGINE') == 'mssql': + os.system("/opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P '~Test123' -m-1 -Q 'CREATE DATABASE [binder-test];'") + + + +if os.environ.get('BINDER_TEST_DATABASE_ENGINE') == 'mysql': db_settings = { 'ENGINE': 'django.db.backends.mysql', 'NAME': 'binder-test', @@ -23,6 +17,26 @@ 'USER': 'root', 'PASSWORD': 'rootpassword', } +if os.environ.get('BINDER_TEST_DATABASE_ENGINE') == 'mssql': + db_settings = { + 'ENGINE': 'sql_server.pyodbc', + 'NAME': 'binder-test', + 'TIME_ZONE': 'UTC', + 'HOST': 'mssqldb', + 'USER': 'sa', + 'PASSWORD': '~Test123', + 'OPTIONS': { + 'driver': 'ODBC Driver 17 for SQL Server', + }, + } +elif (os.path.exists('/.dockerenv') and 'CY_RUNNING_INSIDE_CI' not in os.environ): + db_settings = { + 'ENGINE': 'django.db.backends.postgresql', + 'NAME': 'postgres', + 'USER': 'postgres', + 'HOST': 'db', + 'PORT': 5432, + } else: db_settings = { 'ENGINE': 'django.db.backends.postgresql_psycopg2', @@ -31,6 +45,7 @@ 'USER': 'postgres', } + settings.configure(**{ 'DEBUG': True, 'SECRET_KEY': 'testy mctestface', @@ -74,6 +89,12 @@ 'level': 'DEBUG', 'class': 'logging.StreamHandler', }, + 'file': { + 'level': 'DEBUG', + 'class': 'logging.handlers.WatchedFileHandler', + 'filename': 'backend.log', + 'filters': [], + }, }, 'loggers': { # We override only this one to avoid logspam @@ -83,7 +104,13 @@ 'handlers': ['console'], 'level': 'ERROR', }, - } + # 'django.db.backends': { + # 'handlers': ['console', 'file'], + # 'level': 'DEBUG', + # 'propagate': True, + # }, + } + }, 'BINDER_PERMISSION': { 'default': [ @@ -110,7 +137,6 @@ 'admin': [] } }) - setup() # Do the dance to ensure the models are synched to the DB. diff --git a/tests/filters/test_time_filters.py b/tests/filters/test_time_filters.py index 76b923ef..b1e702e4 100644 --- a/tests/filters/test_time_filters.py +++ b/tests/filters/test_time_filters.py @@ -6,7 +6,7 @@ import os # This is not possible in -if os.environ.get('BINDER_TEST_MYSQL', '0') != '1': +if os.environ.get('BINDER_TEST_DATABASE_ENGINE') not in ['mssql', 'mysql']: class TimeFiltersTest(TestCase): def setUp(self): diff --git a/tests/test_multi_put.py b/tests/test_multi_put.py index b664443c..dfa4f90f 100644 --- a/tests/test_multi_put.py +++ b/tests/test_multi_put.py @@ -176,6 +176,8 @@ def test_put_relations_from_referencing_side(self): } response = self.client.put('/animal/', data=json.dumps(with_model_data), content_type='application/json') + print(response.content) + self.assertEqual(response.status_code, 200) returned_data = jsonloads(response.content) diff --git a/tests/test_nulls_last.py b/tests/test_nulls_last.py index bd459014..e67b81bc 100644 --- a/tests/test_nulls_last.py +++ b/tests/test_nulls_last.py @@ -26,7 +26,7 @@ def test_order_by_nulls_last(self): self._load_test_data() # MySQL has different defaults when no nulls option is selected... - if os.environ.get('BINDER_TEST_MYSQL', '0') != '0': + if os.environ.get('BINDER_TEST_DATABASE_ENGINE') in ['mssql', 'mysql']: self._assert_order('last_seen', ['4', '5', '1', '2', '3']) self._assert_order('-last_seen', ['3', '2', '1', '4', '5']) else: @@ -46,7 +46,7 @@ def test_order_by_nulls_last_on_annotation(self): self._load_test_data() # MySQL has different defaults when no nulls option is selected... - if os.environ.get('BINDER_TEST_MYSQL', '0') != '0': + if os.environ.get('BINDER_TEST_DATABASE_ENGINE') in ['mssql', 'mysql']: self._assert_order('last_present', ['4', '5', '1', '2', '3']) self._assert_order('-last_present', ['3', '2', '1', '4', '5']) else: diff --git a/tests/test_postgres_fields.py b/tests/test_postgres_fields.py index 4713d218..b5a9a828 100644 --- a/tests/test_postgres_fields.py +++ b/tests/test_postgres_fields.py @@ -7,12 +7,12 @@ from binder.json import jsonloads from django.contrib.auth.models import User -if os.environ.get('BINDER_TEST_MYSQL', '0') == '0': +if os.environ.get('BINDER_TEST_DATABASE_ENGINE') not in ['mysql', 'mssql']: from .testapp.models import FeedingSchedule, Animal, Zoo # TODO: Currently these only really test filtering. Move to test/filters? @unittest.skipIf( - os.environ.get('BINDER_TEST_MYSQL', '0') != '0', + os.environ.get('BINDER_TEST_DATABASE_ENGINE') in ['mysql', 'mssql'], "Only available with PostgreSQL" ) class PostgresFieldsTest(TestCase): diff --git a/tests/testapp/models/__init__.py b/tests/testapp/models/__init__.py index d83931a2..ae3d5b5d 100644 --- a/tests/testapp/models/__init__.py +++ b/tests/testapp/models/__init__.py @@ -7,7 +7,7 @@ from .contact_person import ContactPerson from .costume import Costume # This is a Postgres-specific model -if os.environ.get('BINDER_TEST_MYSQL', '0') != '1': +if os.environ.get('BINDER_TEST_DATABASE_ENGINE') not in ['mysql', 'mssql']: from .feeding_schedule import FeedingSchedule from .gate import Gate from .nickname import Nickname, NullableNickname diff --git a/tests/testapp/models/animal.py b/tests/testapp/models/animal.py index 85d844e0..d2d22f8e 100644 --- a/tests/testapp/models/animal.py +++ b/tests/testapp/models/animal.py @@ -11,6 +11,7 @@ class Concat(Func): # From the api docs: an animal with a name. class Animal(LoadedValuesMixin, BinderModel): + # id = models.IntegerField(primary_key=True, null=True, blank=True) name = models.TextField(max_length=64) zoo = models.ForeignKey('Zoo', on_delete=models.CASCADE, related_name='animals', blank=True, null=True) zoo_of_birth = models.ForeignKey('Zoo', on_delete=models.CASCADE, related_name='+', blank=True, null=True) # might've been born outside captivity diff --git a/tests/testapp/models/zoo.py b/tests/testapp/models/zoo.py index a6b1737c..8d931404 100644 --- a/tests/testapp/models/zoo.py +++ b/tests/testapp/models/zoo.py @@ -17,13 +17,14 @@ def delete_files(sender, instance=None, **kwargs): # From the api docs: a zoo with a name. It also has a founding date, # which is nullable (representing "unknown"). class Zoo(BinderModel): + # id = models.IntegerField(primary_key=True, null=True, blank=True) name = models.TextField() founding_date = models.DateField(null=True, blank=True) floor_plan = models.ImageField(upload_to='floor-plans', null=True, blank=True) # It is important that this m2m relationship is defined on this model, one of the tests depends on this fact contacts = models.ManyToManyField('ContactPerson', blank=True, related_name='zoos') # We assume that a person can only be the director of one zoo, one of the tests depends on this fact - director = models.OneToOneField('ContactPerson', on_delete=models.SET_NULL, blank=True, null=True, related_name='managing_zoo') + director = models.ForeignKey('ContactPerson', on_delete=models.SET_NULL, blank=True, null=True, related_name='managing_zoo') most_popular_animals = models.ManyToManyField('Animal', blank=True, related_name='+') opening_time = models.TimeField(default=datetime.time(9, 0, 0)) diff --git a/tests/testapp/views/__init__.py b/tests/testapp/views/__init__.py index 7dd1f4f0..6f4a0531 100644 --- a/tests/testapp/views/__init__.py +++ b/tests/testapp/views/__init__.py @@ -10,7 +10,7 @@ from .country import CountryView # This has a Postgres-specific model -if os.environ.get('BINDER_TEST_MYSQL', '0') != '1': +if os.environ.get('BINDER_TEST_DATABASE_ENGINE') not in ['mysql', 'mssql']: from .feeding_schedule import FeedingScheduleView from .gate import GateView from .lion import LionView