From 1a9e183a611e9c7065e2502ffb765991f11d0152 Mon Sep 17 00:00:00 2001 From: jrhizor Date: Wed, 15 Feb 2023 10:43:17 -0800 Subject: [PATCH 01/11] setup example --- .github/workflows/build.yaml | 38 +++++ .gitignore | 260 +++++++++++++++++++++-------------- README.md | 14 +- requirements.txt | 2 + test_db.py | 38 +++++ 5 files changed, 249 insertions(+), 103 deletions(-) create mode 100644 .github/workflows/build.yaml create mode 100644 requirements.txt create mode 100644 test_db.py diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml new file mode 100644 index 0000000..fbb3cb0 --- /dev/null +++ b/.github/workflows/build.yaml @@ -0,0 +1,38 @@ +name: Build + +on: + push: + +jobs: + test: + runs-on: ubuntu-latest + + services: + db: + image: postgres:latest + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + ports: + - 5432/tcp + options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 3 + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Install Python + uses: actions/setup-python@v2 + with: + python-version: 3.10 + + - name: Install Dependencies + run: | + pip install -r requirements.txt + + - name: Test + env: + DATABASE_URL: postgresql://postgres:postgres@localhost:5432/postgres + run: | + python -m pytest diff --git a/.gitignore b/.gitignore index 6704566..68bc17f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,104 +1,160 @@ -# Logs -logs +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: *.log -npm-debug.log* -yarn-debug.log* -yarn-error.log* -lerna-debug.log* - -# Diagnostic reports (https://nodejs.org/api/report.html) -report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json - -# Runtime data -pids -*.pid -*.seed -*.pid.lock - -# Directory for instrumented libs generated by jscoverage/JSCover -lib-cov - -# Coverage directory used by tools like istanbul -coverage -*.lcov - -# nyc test coverage -.nyc_output - -# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) -.grunt - -# Bower dependency directory (https://bower.io/) -bower_components - -# node-waf configuration -.lock-wscript - -# Compiled binary addons (https://nodejs.org/api/addons.html) -build/Release - -# Dependency directories -node_modules/ -jspm_packages/ - -# TypeScript v1 declaration files -typings/ - -# TypeScript cache -*.tsbuildinfo - -# Optional npm cache directory -.npm - -# Optional eslint cache -.eslintcache - -# Microbundle cache -.rpt2_cache/ -.rts2_cache_cjs/ -.rts2_cache_es/ -.rts2_cache_umd/ - -# Optional REPL history -.node_repl_history - -# Output of 'npm pack' -*.tgz - -# Yarn Integrity file -.yarn-integrity - -# dotenv environment variables file +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments .env -.env.test - -# parcel-bundler cache (https://parceljs.org/) -.cache - -# Next.js build output -.next - -# Nuxt.js build / generate output -.nuxt -dist - -# Gatsby files -.cache/ -# Comment in the public line in if your project uses Gatsby and *not* Next.js -# https://nextjs.org/blog/next-9-1#public-directory-support -# public - -# vuepress build output -.vuepress/dist - -# Serverless directories -.serverless/ - -# FuseBox cache -.fusebox/ - -# DynamoDB Local files -.dynamodb/ - -# TernJS port file -.tern-port +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ diff --git a/README.md b/README.md index d1ef5dc..dce8884 100644 --- a/README.md +++ b/README.md @@ -1 +1,13 @@ -# example-datasette \ No newline at end of file +# example-datasette + +``` +docker run --name example-datasette-db -p 5432:5432 -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=postgres -d postgres +``` + +``` +python3 -m venv env +pip install -r requirements.txt +source env/bin/activate +export DATABASE_URL=postgresql://postgres:postgres@localhost:5432/postgres +python -m pytest +``` diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..689a8cb --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +psycopg2-binary == 2.9.5 +pytest == 7.2.1 diff --git a/test_db.py b/test_db.py new file mode 100644 index 0000000..4bc5156 --- /dev/null +++ b/test_db.py @@ -0,0 +1,38 @@ +import os +import pytest +import psycopg2 + +@pytest.fixture(scope="module") +def test_db(): + conn = psycopg2.connect(os.environ.get('DATABASE_URL')) + yield conn + conn.close() + +# define a test function to test the database content +def test_database_content(test_db): + # create a cursor object to interact with the database + cur = test_db.cursor() + + # create table + cur.execute("CREATE TABLE IF NOT EXISTS users (id SERIAL PRIMARY KEY, name VARCHAR(255) NOT NULL, email VARCHAR(255) NOT NULL)"); + + # ensure that the users table is empty + cur.execute("SELECT COUNT(*) FROM users") + assert cur.fetchone()[0] == 0 + + # insert a new user + cur.execute("INSERT INTO users (name, email) VALUES (%s, %s)", ("John Doe", "johndoe@example.com")) + test_db.commit() + + # retrieve all users and validate that the new user is in the table + cur.execute("SELECT COUNT(*) FROM users WHERE name = %s AND email = %s", ("John Doe", "johndoe@example.com")) + assert cur.fetchone()[0] == 1 + + # delete the user and ensure that the users table is empty again + cur.execute("DELETE FROM users WHERE name = %s", ("John Doe",)) + test_db.commit() + cur.execute("SELECT COUNT(*) FROM users") + assert cur.fetchone()[0] == 0 + + # close the cursor + cur.close() From 930fea52f02fbe21507d304c1f007d08b59f1ad4 Mon Sep 17 00:00:00 2001 From: jrhizor Date: Wed, 15 Feb 2023 10:44:37 -0800 Subject: [PATCH 02/11] use 3.10 --- .github/workflows/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index fbb3cb0..5c11449 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -25,7 +25,7 @@ jobs: - name: Install Python uses: actions/setup-python@v2 with: - python-version: 3.10 + python-version: "3.10" - name: Install Dependencies run: | From d943009f80cc407b3262adcc6b391187b8fc90a9 Mon Sep 17 00:00:00 2001 From: jrhizor Date: Wed, 15 Feb 2023 10:54:52 -0800 Subject: [PATCH 03/11] add stoat and db capture --- .github/workflows/build.yaml | 13 +++++++++++++ .stoat/comment.hbs | 5 +++++ .stoat/config.yaml | 10 ++++++++++ test_db.py | 3 +++ 4 files changed, 31 insertions(+) create mode 100644 .stoat/comment.hbs create mode 100644 .stoat/config.yaml diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 5c11449..43ea9db 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -36,3 +36,16 @@ jobs: DATABASE_URL: postgresql://postgres:postgres@localhost:5432/postgres run: | python -m pytest + + - name: Export DB on Failure + if: !success() + env: + DATABASE_URL: postgresql://postgres:postgres@localhost:5432/postgres + run: | + pip install 'db-to-sqlite[postgresql]' + db-to-sqlite "$DATABASE_URL" postgres.db --all + + - name: Run Stoat Action + uses: stoat-dev/stoat-action@v0 + if: always() + diff --git a/.stoat/comment.hbs b/.stoat/comment.hbs new file mode 100644 index 0000000..b19c21b --- /dev/null +++ b/.stoat/comment.hbs @@ -0,0 +1,5 @@ +{{#if plugins.static_hosting.postgres_dump.link}} +❌ [View Database](https://lite.datasette.io/?url={{ plugins.static_hosting.postgres_dump.link }}) +{{else}} +✅ All Tests Passed +{{/if}} diff --git a/.stoat/config.yaml b/.stoat/config.yaml new file mode 100644 index 0000000..574edd0 --- /dev/null +++ b/.stoat/config.yaml @@ -0,0 +1,10 @@ +--- +version: 1 +enabled: true +comment_template_file: ".stoat/comment.hbs" +plugins: + job_runtime: + enabled: true + static_hosting: + postgres_dump: + path: postgres.db diff --git a/test_db.py b/test_db.py index 4bc5156..a72d4af 100644 --- a/test_db.py +++ b/test_db.py @@ -34,5 +34,8 @@ def test_database_content(test_db): cur.execute("SELECT COUNT(*) FROM users") assert cur.fetchone()[0] == 0 + # force failure so we can test stoat functionality of reading the dumped db + assert false + # close the cursor cur.close() From d35ba9002e676c17b6d58b0fec9e26da0f55346f Mon Sep 17 00:00:00 2001 From: jrhizor Date: Wed, 15 Feb 2023 10:56:58 -0800 Subject: [PATCH 04/11] use correct service hostname --- .github/workflows/build.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 43ea9db..0882dc9 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest services: - db: + postgres: image: postgres:latest env: POSTGRES_USER: postgres @@ -33,14 +33,14 @@ jobs: - name: Test env: - DATABASE_URL: postgresql://postgres:postgres@localhost:5432/postgres + DATABASE_URL: postgresql://postgres:postgres@postgres:5432/postgres run: | python -m pytest - name: Export DB on Failure if: !success() env: - DATABASE_URL: postgresql://postgres:postgres@localhost:5432/postgres + DATABASE_URL: postgresql://postgres:postgres@postgres:5432/postgres run: | pip install 'db-to-sqlite[postgresql]' db-to-sqlite "$DATABASE_URL" postgres.db --all From 02ec0974a6ea266a0291d1b18a89c4178b82392a Mon Sep 17 00:00:00 2001 From: jrhizor Date: Wed, 15 Feb 2023 10:58:12 -0800 Subject: [PATCH 05/11] use failure check --- .github/workflows/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 0882dc9..f4a1acd 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -38,7 +38,7 @@ jobs: python -m pytest - name: Export DB on Failure - if: !success() + if: ${{ failure() }} env: DATABASE_URL: postgresql://postgres:postgres@postgres:5432/postgres run: | From 3ddece5940af5a5a5e8278710f7a324e5cd39601 Mon Sep 17 00:00:00 2001 From: jrhizor Date: Wed, 15 Feb 2023 11:48:56 -0800 Subject: [PATCH 06/11] try with localhost --- .github/workflows/build.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index f4a1acd..87e902f 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -33,14 +33,14 @@ jobs: - name: Test env: - DATABASE_URL: postgresql://postgres:postgres@postgres:5432/postgres + DATABASE_URL: postgresql://postgres:postgres@localhost:5432/postgres run: | python -m pytest - name: Export DB on Failure if: ${{ failure() }} env: - DATABASE_URL: postgresql://postgres:postgres@postgres:5432/postgres + DATABASE_URL: postgresql://postgres:postgres@localhost:5432/postgres run: | pip install 'db-to-sqlite[postgresql]' db-to-sqlite "$DATABASE_URL" postgres.db --all From a1127fd70ef5371312aa346674ea6c67bcf98997 Mon Sep 17 00:00:00 2001 From: jrhizor Date: Wed, 15 Feb 2023 11:50:54 -0800 Subject: [PATCH 07/11] try 127.0.0.1 --- .github/workflows/build.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 87e902f..d50206e 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -33,14 +33,14 @@ jobs: - name: Test env: - DATABASE_URL: postgresql://postgres:postgres@localhost:5432/postgres + DATABASE_URL: postgresql://postgres:postgres@127.0.0.1:5432/postgres run: | python -m pytest - name: Export DB on Failure if: ${{ failure() }} env: - DATABASE_URL: postgresql://postgres:postgres@localhost:5432/postgres + DATABASE_URL: postgresql://postgres:postgres@127.0.0.1:5432/postgres run: | pip install 'db-to-sqlite[postgresql]' db-to-sqlite "$DATABASE_URL" postgres.db --all From a17beef767d1a028f84a2566c12940c4047e891b Mon Sep 17 00:00:00 2001 From: jrhizor Date: Wed, 15 Feb 2023 11:52:50 -0800 Subject: [PATCH 08/11] change port def --- .github/workflows/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index d50206e..09b0929 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -15,7 +15,7 @@ jobs: POSTGRES_PASSWORD: postgres POSTGRES_DB: postgres ports: - - 5432/tcp + - 5432:5432 options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 3 steps: From c66cfca0e5c238ae4ea8cf14c95327baa47848aa Mon Sep 17 00:00:00 2001 From: jrhizor Date: Wed, 15 Feb 2023 11:59:21 -0800 Subject: [PATCH 09/11] fix --- test_db.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test_db.py b/test_db.py index a72d4af..c6371f4 100644 --- a/test_db.py +++ b/test_db.py @@ -28,14 +28,14 @@ def test_database_content(test_db): cur.execute("SELECT COUNT(*) FROM users WHERE name = %s AND email = %s", ("John Doe", "johndoe@example.com")) assert cur.fetchone()[0] == 1 + # force failure so we can test stoat functionality of reading the dumped db + assert false + # delete the user and ensure that the users table is empty again cur.execute("DELETE FROM users WHERE name = %s", ("John Doe",)) test_db.commit() cur.execute("SELECT COUNT(*) FROM users") assert cur.fetchone()[0] == 0 - # force failure so we can test stoat functionality of reading the dumped db - assert false - # close the cursor cur.close() From b31f51ee20616f0d0bd384a406e2c3223b19cc2a Mon Sep 17 00:00:00 2001 From: jrhizor Date: Wed, 15 Feb 2023 19:53:55 -0800 Subject: [PATCH 10/11] add another table --- test_db.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/test_db.py b/test_db.py index c6371f4..9780f61 100644 --- a/test_db.py +++ b/test_db.py @@ -15,24 +15,32 @@ def test_database_content(test_db): # create table cur.execute("CREATE TABLE IF NOT EXISTS users (id SERIAL PRIMARY KEY, name VARCHAR(255) NOT NULL, email VARCHAR(255) NOT NULL)"); + cur.execute("CREATE TABLE IF NOT EXISTS companies (id SERIAL PRIMARY KEY, name VARCHAR(255) NOT NULL)"); # ensure that the users table is empty cur.execute("SELECT COUNT(*) FROM users") assert cur.fetchone()[0] == 0 - # insert a new user - cur.execute("INSERT INTO users (name, email) VALUES (%s, %s)", ("John Doe", "johndoe@example.com")) + # insert new users + cur.execute("INSERT INTO users (name, email) VALUES (%s, %s)", ("John", "johndoe@example.com")) + cur.execute("INSERT INTO users (name, email) VALUES (%s, %s)", ("Jared", "jared@example.com")) + cur.execute("INSERT INTO users (name, email) VALUES (%s, %s)", ("Liren", "liren@example.com")) + test_db.commit() + + # insert new companies + cur.execute("INSERT INTO companies (name) VALUES (%s)", ("Google")) + cur.execute("INSERT INTO companies (name) VALUES (%s)", ("Facebook")) test_db.commit() # retrieve all users and validate that the new user is in the table - cur.execute("SELECT COUNT(*) FROM users WHERE name = %s AND email = %s", ("John Doe", "johndoe@example.com")) + cur.execute("SELECT COUNT(*) FROM users WHERE name = %s AND email = %s", ("John", "johndoe@example.com")) assert cur.fetchone()[0] == 1 # force failure so we can test stoat functionality of reading the dumped db assert false # delete the user and ensure that the users table is empty again - cur.execute("DELETE FROM users WHERE name = %s", ("John Doe",)) + cur.execute("DELETE FROM users WHERE name = %s", ("John",)) test_db.commit() cur.execute("SELECT COUNT(*) FROM users") assert cur.fetchone()[0] == 0 From 06d12d83bce1c9f386bf51cd82e05c86f97747c3 Mon Sep 17 00:00:00 2001 From: jrhizor Date: Wed, 15 Feb 2023 20:04:36 -0800 Subject: [PATCH 11/11] fix adding companies --- test_db.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test_db.py b/test_db.py index 9780f61..0ebf0f9 100644 --- a/test_db.py +++ b/test_db.py @@ -16,6 +16,7 @@ def test_database_content(test_db): # create table cur.execute("CREATE TABLE IF NOT EXISTS users (id SERIAL PRIMARY KEY, name VARCHAR(255) NOT NULL, email VARCHAR(255) NOT NULL)"); cur.execute("CREATE TABLE IF NOT EXISTS companies (id SERIAL PRIMARY KEY, name VARCHAR(255) NOT NULL)"); + cur.execute("TRUNCATE users, companies"); # ensure that the users table is empty cur.execute("SELECT COUNT(*) FROM users") @@ -28,8 +29,8 @@ def test_database_content(test_db): test_db.commit() # insert new companies - cur.execute("INSERT INTO companies (name) VALUES (%s)", ("Google")) - cur.execute("INSERT INTO companies (name) VALUES (%s)", ("Facebook")) + cur.execute("INSERT INTO companies (name) VALUES (%s)", ("Google",)) + cur.execute("INSERT INTO companies (name) VALUES (%s)", ("Facebook",)) test_db.commit() # retrieve all users and validate that the new user is in the table