From 9fa6ee1f7e3c6efa572910279e31ec5aa1472d76 Mon Sep 17 00:00:00 2001 From: KhadijaMahanga Date: Wed, 23 Jan 2019 12:51:27 +0300 Subject: [PATCH 1/6] Docker setup --- .gitignore | 1 + CHECKS | 5 +++++ Dockerfile | 42 ++++++++++++++++++++++++++++++++++++++++++ docker-compose.yml | 32 ++++++++++++++++++++++++++++++++ docker-entrypoint.sh | 30 ++++++++++++++++++++++++++++++ runtime.txt | 1 + 6 files changed, 111 insertions(+) create mode 100644 CHECKS create mode 100644 Dockerfile create mode 100644 docker-compose.yml create mode 100755 docker-entrypoint.sh create mode 100644 runtime.txt diff --git a/.gitignore b/.gitignore index d9e8b03f..15e74fdf 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,7 @@ # This is the name that the install notes suggest be used for the virtualenv .venv +env/ .DS_Store .vagrant diff --git a/CHECKS b/CHECKS new file mode 100644 index 00000000..037f0ce8 --- /dev/null +++ b/CHECKS @@ -0,0 +1,5 @@ +WAIT=10 +ATTEMPTS=3 + +/ + diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..cf25a20f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,42 @@ +FROM python:2.7-stretch +ENV DEBIAN_FRONTEND noninteractive + +# Upgrade OS Dependencies +RUN apt-get update && apt-get upgrade -y +# Install Postgresql Client +RUN apt-get install postgresql-client -y + +# Upgrade pip + setuptools +RUN pip install -q -U pip setuptools + +# Install gunicorn with gevent +RUN pip install -q -U gunicorn[gevent] + +# GDAL Installs +RUN apt-get install gdal-bin python-gdal libgdal-dev -y +RUN pip install -q GDAL==2.1.3 --global-option=build_ext --global-option="-I/usr/include/gdal" + +# Set env variables used in this Dockerfile +#Django settings +ENV DJANGO_SETTINGS_MODULE=project.settings + +# Local directory with project source +ENV APP_SRC=. +# Directory in container for all project files +ENV APP_SRVHOME=/src +# Directory in container for project source files +ENV APP_SRVPROJ=/src/mapit + +# Add application source code to SRCDIR +ADD $APP_SRC $APP_SRVPROJ +WORKDIR $APP_SRVPROJ + +# Install dependencies +RUN pip install -q -e . + +# Server +EXPOSE 8000 + +COPY ./docker-entrypoint.sh / +ENTRYPOINT ["/docker-entrypoint.sh"] +CMD [ "--name", "mapit", "--reload", "mapit.wsgi:application" ] diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..586f6271 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,32 @@ +version: '3.3' + +services: + db: + image: mdillon/postgis:9.5 + ports: + - "54321:5432" + environment: + - POSTGRES_USER=mapit + - POSTGRES_PASSWORD=mapit + - POSTGRES_DB=mapit + - PGUSER=mapit + - PGPASSWORD=mapit + + web: + build: . + ports: + - "8000:8000" + depends_on: + - db + volumes: + - ./:/src/mapit_africa + environment: + - DJANGO_SETTINGS_MODULE=project.settings + - DATABASE_URL=postgresql://mapit:mapit@db:5432/mapit + - DJANGO_SECRET_KEY=somethingsecret + - PGHOST=db + - PGDATABASE=mapit + - PGUSER=mapit + - PGPASSWORD=mapit + - PYTHONDONTWRITEBYTECODE="True" + # - DJANGO_DEBUG=False # For testing deploys diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh new file mode 100755 index 00000000..6a46dfff --- /dev/null +++ b/docker-entrypoint.sh @@ -0,0 +1,30 @@ +#!/bin/bash +set -e + +until psql $DATABASE_URL -c '\l'; do + >&2 echo "Postgres is unavailable - sleeping" + sleep 1 +done + +>&2 echo "Postgres is up - continuing" + +./bin/mapit_make_css +python manage.py collectstatic --clear --noinput # Collect static files + +python manage.py migrate mapit --noinput # Apply database migrations + +# Prepare log files and start outputting logs to stdout +touch /src/logs/gunicorn.log +touch /src/logs/access.log +tail -n 0 -f /src/logs/*.log & + +# Start Gunicorn processes +echo Starting Gunicorn. +exec gunicorn \ + --bind 0.0.0.0:8000 \ + --workers 3 \ + --worker-class gevent \ + --log-level=info \ + --log-file=/src/logs/gunicorn.log \ + --access-logfile=/src/logs/access.log \ + project.wsgi:application diff --git a/runtime.txt b/runtime.txt new file mode 100644 index 00000000..5b664547 --- /dev/null +++ b/runtime.txt @@ -0,0 +1 @@ +python-2.7.15 \ No newline at end of file From 8c87165e7ca74c7d4ceb6d9724f57d093c4c9cd4 Mon Sep 17 00:00:00 2001 From: KhadijaMahanga Date: Wed, 23 Jan 2019 13:09:09 +0300 Subject: [PATCH 2/6] docker - add subdirectories for logs and media --- Dockerfile | 5 +++++ project/settings.py | 6 +++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index cf25a20f..72736e1b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,6 +27,11 @@ ENV APP_SRVHOME=/src # Directory in container for project source files ENV APP_SRVPROJ=/src/mapit +# Create application subdirectories +WORKDIR $APP_SRVHOME +RUN mkdir media static logs +VOLUME ["$APP_SRVHOME/media/", "$APP_SRVHOME/logs/"] + # Add application source code to SRCDIR ADD $APP_SRC $APP_SRVPROJ WORKDIR $APP_SRVPROJ diff --git a/project/settings.py b/project/settings.py index 45c86219..bdc17e5e 100644 --- a/project/settings.py +++ b/project/settings.py @@ -75,9 +75,9 @@ 'ENGINE': 'django.contrib.gis.db.backends.postgis', 'NAME': config.get('MAPIT_DB_NAME', 'mapit'), 'USER': config.get('MAPIT_DB_USER', 'mapit'), - 'PASSWORD': config.get('MAPIT_DB_PASS', ''), - 'HOST': config.get('MAPIT_DB_HOST', ''), - 'PORT': config.get('MAPIT_DB_PORT', ''), + 'PASSWORD': config.get('MAPIT_DB_PASS', 'mapit'), + 'HOST': config.get('MAPIT_DB_HOST', 'db'), + 'PORT': config.get('MAPIT_DB_PORT', '5432'), } } From a9385bf19d7e45a1f2a427df842544c0673e7ccb Mon Sep 17 00:00:00 2001 From: KhadijaMahanga Date: Wed, 23 Jan 2019 14:36:35 +0300 Subject: [PATCH 3/6] static files --- docker-compose.yml | 2 +- project/settings.py | 2 +- project/urls.py | 4 +++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 586f6271..c46e7338 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -23,7 +23,7 @@ services: environment: - DJANGO_SETTINGS_MODULE=project.settings - DATABASE_URL=postgresql://mapit:mapit@db:5432/mapit - - DJANGO_SECRET_KEY=somethingsecret + - DJANGO_SECRET_KEY=guutyfghvbkllkjhg - PGHOST=db - PGDATABASE=mapit - PGUSER=mapit diff --git a/project/settings.py b/project/settings.py index bdc17e5e..9cc2035d 100644 --- a/project/settings.py +++ b/project/settings.py @@ -82,7 +82,7 @@ } # Make this unique, and don't share it with anybody. -SECRET_KEY = config.get('DJANGO_SECRET_KEY', '') +SECRET_KEY = config.get('DJANGO_SECRET_KEY', 'gu^&xc)hoibh3x&s+9009jbn4d$!nq0lz+syx-^x8%z24!kfs4') ALLOWED_HOSTS = ['*'] diff --git a/project/urls.py b/project/urls.py index bca42fa9..f515b470 100644 --- a/project/urls.py +++ b/project/urls.py @@ -1,9 +1,11 @@ from django.conf.urls import include, url +from django.conf.urls.static import static from django.contrib import admin +import settings handler500 = 'mapit.shortcuts.json_500' urlpatterns = [ url(r'^', include('mapit.urls')), url(r'^admin/', admin.site.urls), -] +] + static(settings.STATIC_URL, document_root=settings.STATIC_ROOT) From 5accfc589ca05a26281ed0cad180c2e2d7d330a8 Mon Sep 17 00:00:00 2001 From: KhadijaMahanga Date: Fri, 25 Jan 2019 11:32:46 +0300 Subject: [PATCH 4/6] get environment variable --- Dockerfile | 2 +- docker-compose.yml | 19 +++++++++++++------ docker-entrypoint.sh | 2 +- project/settings.py | 34 +++++++++++++++++++++------------- 4 files changed, 36 insertions(+), 21 deletions(-) diff --git a/Dockerfile b/Dockerfile index 72736e1b..cc58cd37 100644 --- a/Dockerfile +++ b/Dockerfile @@ -44,4 +44,4 @@ EXPOSE 8000 COPY ./docker-entrypoint.sh / ENTRYPOINT ["/docker-entrypoint.sh"] -CMD [ "--name", "mapit", "--reload", "mapit.wsgi:application" ] +CMD [ "--name", "mapit", "--reload", "project.wsgi:application" ] diff --git a/docker-compose.yml b/docker-compose.yml index c46e7338..4d80efe0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,14 +19,21 @@ services: depends_on: - db volumes: - - ./:/src/mapit_africa + - ./:/src/mapit environment: - DJANGO_SETTINGS_MODULE=project.settings - DATABASE_URL=postgresql://mapit:mapit@db:5432/mapit - - DJANGO_SECRET_KEY=guutyfghvbkllkjhg - - PGHOST=db - - PGDATABASE=mapit - - PGUSER=mapit - - PGPASSWORD=mapit + - DJANGO_SECRET_KEY=somethingsecret + - MAPIT_DB_HOST=db + - MAPIT_DB_NAME=mapit + - MAPIT_DB_USER=mapit + - MAPIT_DB_PASS=mapit + - AREA_SRID= + - COUNTRY= + - RATE_LIMIT= + - GOOGLE_ANALYTICS= + - DEBUG= + - BUGS_EMAIL= + - EMAIL_SUBJECT_PREFIX= - PYTHONDONTWRITEBYTECODE="True" # - DJANGO_DEBUG=False # For testing deploys diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 6a46dfff..26b2b0a8 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -11,7 +11,7 @@ done ./bin/mapit_make_css python manage.py collectstatic --clear --noinput # Collect static files -python manage.py migrate mapit --noinput # Apply database migrations +python manage.py migrate --noinput # Apply database migrations # Prepare log files and start outputting logs to stdout touch /src/logs/gunicorn.log diff --git a/project/settings.py b/project/settings.py index 9cc2035d..4dca47e9 100644 --- a/project/settings.py +++ b/project/settings.py @@ -20,22 +20,22 @@ # An EPSG code for what the areas are stored as, e.g. 27700 is OSGB, 4326 for # WGS84. Optional, defaults to 4326. -MAPIT_AREA_SRID = int(config.get('AREA_SRID', 4326)) +MAPIT_AREA_SRID = int(config.get('AREA_SRID', os.environ.get('AREA_SRID', 4326))) # Country is currently one of GB, NO, IT, KE, SA, or ZA. # Optional; country specific things won't happen if not set. -MAPIT_COUNTRY = config.get('COUNTRY', '') +MAPIT_COUNTRY = config.get('COUNTRY', os.environ.get('COUNTRY', '')) # A dictionary of IP addresses, User Agents, or functions that should be # excluded from rate limiting. Optional. -MAPIT_RATE_LIMIT = config.get('RATE_LIMIT', {}) +MAPIT_RATE_LIMIT = config.get('RATE_LIMIT', os.environ.get('RATE_LIMIT', {})) # A GA code for analytics -GOOGLE_ANALYTICS = config.get('GOOGLE_ANALYTICS', '') +GOOGLE_ANALYTICS = config.get('GOOGLE_ANALYTICS', os.environ.get('GOOGLE_ANALYTICS', '')) # Django settings for mapit project. -DEBUG = config.get('DEBUG', True) +DEBUG = config.get('DEBUG', os.environ.get('DEBUG',True)) # (Note that even if DEBUG is true, output_json still sets a # Cache-Control header with max-age of 28 days.) @@ -59,30 +59,38 @@ except ImportError: pass CACHE_MIDDLEWARE_SECONDS = 86400 - CACHE_MIDDLEWARE_KEY_PREFIX = config.get('MAPIT_DB_NAME') + CACHE_MIDDLEWARE_KEY_PREFIX = config.get('MAPIT_DB_NAME', os.environ.get('MAPIT_DB_NAME')) if config.get('BUGS_EMAIL'): SERVER_EMAIL = config['BUGS_EMAIL'] ADMINS = ( - ('mySociety bugs', config['BUGS_EMAIL']), + ('mySociety bugs', SERVER_EMAIL), + ) +if os.environ.get('BUGS_EMAIL'): + SERVER_EMAIL = os.environ['BUGS_EMAIL'] + ADMINS = ( + ('mySociety bugs', SERVER_EMAIL), ) if config.get('EMAIL_SUBJECT_PREFIX'): EMAIL_SUBJECT_PREFIX = config['EMAIL_SUBJECT_PREFIX'] +if os.environ.get('EMAIL_SUBJECT_PREFIX'): + EMAIL_SUBJECT_PREFIX = os.environ['EMAIL_SUBJECT_PREFIX'] + DATABASES = { 'default': { 'ENGINE': 'django.contrib.gis.db.backends.postgis', - 'NAME': config.get('MAPIT_DB_NAME', 'mapit'), - 'USER': config.get('MAPIT_DB_USER', 'mapit'), - 'PASSWORD': config.get('MAPIT_DB_PASS', 'mapit'), - 'HOST': config.get('MAPIT_DB_HOST', 'db'), - 'PORT': config.get('MAPIT_DB_PORT', '5432'), + 'NAME': config.get('MAPIT_DB_NAME', os.environ.get('MAPIT_DB_NAME')), + 'USER': config.get('MAPIT_DB_USER', os.environ.get('MAPIT_DB_USER')), + 'PASSWORD': config.get('MAPIT_DB_PASS', os.environ.get('MAPIT_DB_PASS')), + 'HOST': config.get('MAPIT_DB_HOST', os.environ.get('MAPIT_DB_HOST')), + 'PORT': config.get('MAPIT_DB_PORT', os.environ.get('MAPIT_DB_PORT')), } } # Make this unique, and don't share it with anybody. -SECRET_KEY = config.get('DJANGO_SECRET_KEY', 'gu^&xc)hoibh3x&s+9009jbn4d$!nq0lz+syx-^x8%z24!kfs4') +SECRET_KEY = config.get('DJANGO_SECRET_KEY', os.environ.get('DJANGO_SECRET_KEY')) ALLOWED_HOSTS = ['*'] From 884d044f83ab201625f200e0f7d4e46f85502e51 Mon Sep 17 00:00:00 2001 From: KhadijaMahanga Date: Fri, 25 Jan 2019 12:18:01 +0300 Subject: [PATCH 5/6] fix travis build import error --- project/urls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/urls.py b/project/urls.py index f515b470..ef51c2c9 100644 --- a/project/urls.py +++ b/project/urls.py @@ -1,7 +1,7 @@ from django.conf.urls import include, url from django.conf.urls.static import static from django.contrib import admin -import settings +import project.settings as settings handler500 = 'mapit.shortcuts.json_500' From 2a73113395839bc42a033451c21c28728d98b7ef Mon Sep 17 00:00:00 2001 From: KhadijaMahanga Date: Fri, 25 Jan 2019 12:23:42 +0300 Subject: [PATCH 6/6] fix tox commands failed --- project/settings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/project/settings.py b/project/settings.py index 4dca47e9..750fbe94 100644 --- a/project/settings.py +++ b/project/settings.py @@ -35,7 +35,7 @@ # Django settings for mapit project. -DEBUG = config.get('DEBUG', os.environ.get('DEBUG',True)) +DEBUG = config.get('DEBUG', os.environ.get('DEBUG', True)) # (Note that even if DEBUG is true, output_json still sets a # Cache-Control header with max-age of 28 days.) @@ -66,7 +66,7 @@ ADMINS = ( ('mySociety bugs', SERVER_EMAIL), ) -if os.environ.get('BUGS_EMAIL'): +if os.environ.get('BUGS_EMAIL'): SERVER_EMAIL = os.environ['BUGS_EMAIL'] ADMINS = ( ('mySociety bugs', SERVER_EMAIL),