From e5e34f6f2c1bdc7f067f6733884f24f0d07f7aa8 Mon Sep 17 00:00:00 2001 From: Edwin Grubbs Date: Tue, 9 Jul 2019 19:10:56 -0500 Subject: [PATCH 01/31] Pipelines on a branch [skip ci] --- azure-pipelines-1.yml | 59 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 azure-pipelines-1.yml diff --git a/azure-pipelines-1.yml b/azure-pipelines-1.yml new file mode 100644 index 0000000..b9e9aa0 --- /dev/null +++ b/azure-pipelines-1.yml @@ -0,0 +1,59 @@ +# Python Django +# Test a Django project on multiple versions of Python. +# Add steps that analyze code, save build artifacts, deploy, and more: +# https://docs.microsoft.com/azure/devops/pipelines/languages/python + +trigger: +- master + +pool: + vmImage: 'ubuntu-latest' +strategy: + matrix: + Python37: + PYTHON_VERSION: '3.7' + maxParallel: 3 + +steps: +- task: UsePythonVersion@0 + inputs: + versionSpec: '$(PYTHON_VERSION)' + architecture: 'x64' + +- task: PythonScript@0 + displayName: 'Export project path' + inputs: + scriptSource: 'inline' + script: | + """Search all subdirectories for `manage.py`.""" + from glob import iglob + from os import path + # Python >= 3.5 + manage_py = next(iglob(path.join('**', 'manage.py'), recursive=True), None) + if not manage_py: + raise SystemExit('Could not find a Django project') + project_location = path.dirname(path.abspath(manage_py)) + print('Found Django project in', project_location) + print('##vso[task.setvariable variable=projectRoot]{}'.format(project_location)) + +- script: | + sudo apt install wget + wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh + bash Miniconda3-latest-Linux-x86_64.sh + source $HOME/miniconda3/bin/activate + conda env update -f environment.yml + python -m pip install --upgrade pip setuptools wheel + pip install -r requirements.txt + pip install unittest-xml-reporting + displayName: 'Install prerequisites' + +- script: | + pushd '$(projectRoot)' + python manage.py test --testrunner xmlrunner.extra.djangotestrunner.XMLTestRunner --no-input + displayName: 'Run tests' + +- task: PublishTestResults@2 + inputs: + testResultsFiles: "**/TEST-*.xml" + testRunTitle: 'Python $(PYTHON_VERSION)' + condition: succeededOrFailed() From 07a4a723ccf091f41e8d373ca8319e70c70777ad Mon Sep 17 00:00:00 2001 From: Edwin Grubbs Date: Tue, 9 Jul 2019 19:19:18 -0500 Subject: [PATCH 02/31] Update azure-pipelines-1.yml for Azure Pipelines --- azure-pipelines-1.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/azure-pipelines-1.yml b/azure-pipelines-1.yml index b9e9aa0..e238b48 100644 --- a/azure-pipelines-1.yml +++ b/azure-pipelines-1.yml @@ -14,6 +14,9 @@ strategy: PYTHON_VERSION: '3.7' maxParallel: 3 +variables: + DEBIAN_FRONTEND: 'noninteractive' + steps: - task: UsePythonVersion@0 inputs: @@ -37,7 +40,7 @@ steps: print('##vso[task.setvariable variable=projectRoot]{}'.format(project_location)) - script: | - sudo apt install wget + DEBIAN_FRONTEND=noninteractive sudo apt install -y wget wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh bash Miniconda3-latest-Linux-x86_64.sh source $HOME/miniconda3/bin/activate From 5d2bf09cc842bf049e4cb89de9df88a7d8bd7e2c Mon Sep 17 00:00:00 2001 From: Edwin Grubbs Date: Tue, 9 Jul 2019 19:28:23 -0500 Subject: [PATCH 03/31] Run Miniconda in batch mode to eliminate prompts --- azure-pipelines-1.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines-1.yml b/azure-pipelines-1.yml index e238b48..8229217 100644 --- a/azure-pipelines-1.yml +++ b/azure-pipelines-1.yml @@ -42,7 +42,7 @@ steps: - script: | DEBIAN_FRONTEND=noninteractive sudo apt install -y wget wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh - bash Miniconda3-latest-Linux-x86_64.sh + bash Miniconda3-latest-Linux-x86_64.sh -b source $HOME/miniconda3/bin/activate conda env update -f environment.yml python -m pip install --upgrade pip setuptools wheel From 730778413789fa601cad7193d0bfdc97eaf3ea93 Mon Sep 17 00:00:00 2001 From: Edwin Grubbs Date: Tue, 9 Jul 2019 19:31:26 -0500 Subject: [PATCH 04/31] Fixed conda env update --- azure-pipelines-1.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines-1.yml b/azure-pipelines-1.yml index 8229217..467f14e 100644 --- a/azure-pipelines-1.yml +++ b/azure-pipelines-1.yml @@ -44,7 +44,7 @@ steps: wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh bash Miniconda3-latest-Linux-x86_64.sh -b source $HOME/miniconda3/bin/activate - conda env update -f environment.yml + conda env update -f server/environment.yml python -m pip install --upgrade pip setuptools wheel pip install -r requirements.txt pip install unittest-xml-reporting From 2908a711711300a397d234ad2eee9381fed42bb8 Mon Sep 17 00:00:00 2001 From: Edwin Grubbs Date: Tue, 9 Jul 2019 19:38:40 -0500 Subject: [PATCH 05/31] No requirements.txt for pip --- azure-pipelines-1.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/azure-pipelines-1.yml b/azure-pipelines-1.yml index 467f14e..a7c2904 100644 --- a/azure-pipelines-1.yml +++ b/azure-pipelines-1.yml @@ -46,7 +46,6 @@ steps: source $HOME/miniconda3/bin/activate conda env update -f server/environment.yml python -m pip install --upgrade pip setuptools wheel - pip install -r requirements.txt pip install unittest-xml-reporting displayName: 'Install prerequisites' From 1982b2ba0228393fd9432f061af75d80913a7036 Mon Sep 17 00:00:00 2001 From: Edwin Grubbs Date: Tue, 9 Jul 2019 19:46:46 -0500 Subject: [PATCH 06/31] activate conda env --- azure-pipelines-1.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/azure-pipelines-1.yml b/azure-pipelines-1.yml index a7c2904..cbc299d 100644 --- a/azure-pipelines-1.yml +++ b/azure-pipelines-1.yml @@ -44,6 +44,7 @@ steps: wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh bash Miniconda3-latest-Linux-x86_64.sh -b source $HOME/miniconda3/bin/activate + conda activate /home/vsts/miniconda3 conda env update -f server/environment.yml python -m pip install --upgrade pip setuptools wheel pip install unittest-xml-reporting From 01e3ed1d8e22446e4e7c623c78aa7366654294cc Mon Sep 17 00:00:00 2001 From: Edwin Grubbs Date: Tue, 9 Jul 2019 20:02:10 -0500 Subject: [PATCH 07/31] Fixed conda activate and quieted wget and conda env update --- azure-pipelines-1.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/azure-pipelines-1.yml b/azure-pipelines-1.yml index cbc299d..73333d0 100644 --- a/azure-pipelines-1.yml +++ b/azure-pipelines-1.yml @@ -41,11 +41,12 @@ steps: - script: | DEBIAN_FRONTEND=noninteractive sudo apt install -y wget - wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh + wget -q https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh bash Miniconda3-latest-Linux-x86_64.sh -b source $HOME/miniconda3/bin/activate - conda activate /home/vsts/miniconda3 - conda env update -f server/environment.yml + conda env update -q -f server/environment.yml + source /home/vsts/miniconda3/etc/profile.d/conda.sh + conda activate python -m pip install --upgrade pip setuptools wheel pip install unittest-xml-reporting displayName: 'Install prerequisites' From 89cbfd860ec20fd4ce8d5e99dbaa4eadd7c9c708 Mon Sep 17 00:00:00 2001 From: Edwin Grubbs Date: Tue, 9 Jul 2019 20:13:25 -0500 Subject: [PATCH 08/31] Eliminated duplicate azure-pipelines-1.yml --- azure-pipelines-1.yml | 63 ------------------------------------------- azure-pipelines.yml | 13 ++++++--- 2 files changed, 10 insertions(+), 66 deletions(-) delete mode 100644 azure-pipelines-1.yml diff --git a/azure-pipelines-1.yml b/azure-pipelines-1.yml deleted file mode 100644 index 73333d0..0000000 --- a/azure-pipelines-1.yml +++ /dev/null @@ -1,63 +0,0 @@ -# Python Django -# Test a Django project on multiple versions of Python. -# Add steps that analyze code, save build artifacts, deploy, and more: -# https://docs.microsoft.com/azure/devops/pipelines/languages/python - -trigger: -- master - -pool: - vmImage: 'ubuntu-latest' -strategy: - matrix: - Python37: - PYTHON_VERSION: '3.7' - maxParallel: 3 - -variables: - DEBIAN_FRONTEND: 'noninteractive' - -steps: -- task: UsePythonVersion@0 - inputs: - versionSpec: '$(PYTHON_VERSION)' - architecture: 'x64' - -- task: PythonScript@0 - displayName: 'Export project path' - inputs: - scriptSource: 'inline' - script: | - """Search all subdirectories for `manage.py`.""" - from glob import iglob - from os import path - # Python >= 3.5 - manage_py = next(iglob(path.join('**', 'manage.py'), recursive=True), None) - if not manage_py: - raise SystemExit('Could not find a Django project') - project_location = path.dirname(path.abspath(manage_py)) - print('Found Django project in', project_location) - print('##vso[task.setvariable variable=projectRoot]{}'.format(project_location)) - -- script: | - DEBIAN_FRONTEND=noninteractive sudo apt install -y wget - wget -q https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh - bash Miniconda3-latest-Linux-x86_64.sh -b - source $HOME/miniconda3/bin/activate - conda env update -q -f server/environment.yml - source /home/vsts/miniconda3/etc/profile.d/conda.sh - conda activate - python -m pip install --upgrade pip setuptools wheel - pip install unittest-xml-reporting - displayName: 'Install prerequisites' - -- script: | - pushd '$(projectRoot)' - python manage.py test --testrunner xmlrunner.extra.djangotestrunner.XMLTestRunner --no-input - displayName: 'Run tests' - -- task: PublishTestResults@2 - inputs: - testResultsFiles: "**/TEST-*.xml" - testRunTitle: 'Python $(PYTHON_VERSION)' - condition: succeededOrFailed() diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 6eccd83..73333d0 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -10,12 +10,13 @@ pool: vmImage: 'ubuntu-latest' strategy: matrix: - Python35: - PYTHON_VERSION: '3.5' Python37: PYTHON_VERSION: '3.7' maxParallel: 3 +variables: + DEBIAN_FRONTEND: 'noninteractive' + steps: - task: UsePythonVersion@0 inputs: @@ -39,8 +40,14 @@ steps: print('##vso[task.setvariable variable=projectRoot]{}'.format(project_location)) - script: | + DEBIAN_FRONTEND=noninteractive sudo apt install -y wget + wget -q https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh + bash Miniconda3-latest-Linux-x86_64.sh -b + source $HOME/miniconda3/bin/activate + conda env update -q -f server/environment.yml + source /home/vsts/miniconda3/etc/profile.d/conda.sh + conda activate python -m pip install --upgrade pip setuptools wheel - pip install -r requirements.txt pip install unittest-xml-reporting displayName: 'Install prerequisites' From 2249cbee6fe681c637fd0aab8eaf40169e3947ad Mon Sep 17 00:00:00 2001 From: Edwin Grubbs Date: Tue, 9 Jul 2019 20:19:15 -0500 Subject: [PATCH 09/31] Added tests and trying to activate conda right before tests. --- .gitignore | 4 ++++ azure-pipelines.yml | 4 +++- server/citrial/tests.py | 9 +++++++++ 3 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 .gitignore create mode 100644 server/citrial/tests.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8819bda --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +*.swp +*.swo +*.pyc +__pycache__ diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 73333d0..3790312 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -44,7 +44,7 @@ steps: wget -q https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh bash Miniconda3-latest-Linux-x86_64.sh -b source $HOME/miniconda3/bin/activate - conda env update -q -f server/environment.yml + conda env update -q -f server/environment.yml > /dev/null source /home/vsts/miniconda3/etc/profile.d/conda.sh conda activate python -m pip install --upgrade pip setuptools wheel @@ -53,6 +53,8 @@ steps: - script: | pushd '$(projectRoot)' + source /home/vsts/miniconda3/etc/profile.d/conda.sh + conda activate python manage.py test --testrunner xmlrunner.extra.djangotestrunner.XMLTestRunner --no-input displayName: 'Run tests' diff --git a/server/citrial/tests.py b/server/citrial/tests.py new file mode 100644 index 0000000..4404c7f --- /dev/null +++ b/server/citrial/tests.py @@ -0,0 +1,9 @@ +from unittest import TestCase + + +class TestPractice(TestCase): + def testGood(self): + self.assertEqual(1, 1) + + def testBad(self): + self.assertEqual(1, 0) From 535b7d3486be34344b32bfb8b1cfbe6b4313f3e8 Mon Sep 17 00:00:00 2001 From: Edwin Grubbs Date: Tue, 9 Jul 2019 23:50:11 -0500 Subject: [PATCH 10/31] Switched to pytest --- azure-pipelines.yml | 3 ++- server/environment.yml | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 3790312..be6116c 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -55,7 +55,8 @@ steps: pushd '$(projectRoot)' source /home/vsts/miniconda3/etc/profile.d/conda.sh conda activate - python manage.py test --testrunner xmlrunner.extra.djangotestrunner.XMLTestRunner --no-input + #python manage.py test --testrunner xmlrunner.extra.djangotestrunner.XMLTestRunner --no-input + pytest -vv displayName: 'Run tests' - task: PublishTestResults@2 diff --git a/server/environment.yml b/server/environment.yml index c270c0e..2c0344c 100644 --- a/server/environment.yml +++ b/server/environment.yml @@ -10,6 +10,7 @@ dependencies: - gunicorn - pytest - pytest-django + - pytest-azurepipelines - pandas - xlrd - ujson From 050e42cbd5097890fe25df465636a42e9df50b53 Mon Sep 17 00:00:00 2001 From: Edwin Grubbs Date: Wed, 10 Jul 2019 11:57:34 -0500 Subject: [PATCH 11/31] Changed azure-pipelines.yml to find output xml from pytest-azurepipelines --- azure-pipelines.yml | 2 +- server/citrial/tests.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index be6116c..c866ef6 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -61,6 +61,6 @@ steps: - task: PublishTestResults@2 inputs: - testResultsFiles: "**/TEST-*.xml" + testResultsFiles: "**/test-*.xml" testRunTitle: 'Python $(PYTHON_VERSION)' condition: succeededOrFailed() diff --git a/server/citrial/tests.py b/server/citrial/tests.py index 4404c7f..b1d991c 100644 --- a/server/citrial/tests.py +++ b/server/citrial/tests.py @@ -7,3 +7,7 @@ def testGood(self): def testBad(self): self.assertEqual(1, 0) + + +def test_pytest_testfinder(): + assert 1 == 0 From 0347b84d558285abed6499a9973eafca1af2598b Mon Sep 17 00:00:00 2001 From: Edwin Grubbs Date: Wed, 10 Jul 2019 12:27:14 -0500 Subject: [PATCH 12/31] Added pytest.ini and pytest-cov --- azure-pipelines.yml | 6 ++++++ server/citrial/models.py | 10 ++++++++++ server/citrial/tests.py | 2 ++ server/environment.yml | 1 + server/pytest.ini | 8 ++++++++ 5 files changed, 27 insertions(+) create mode 100644 server/citrial/models.py create mode 100644 server/pytest.ini diff --git a/azure-pipelines.yml b/azure-pipelines.yml index c866ef6..8fbc322 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -64,3 +64,9 @@ steps: testResultsFiles: "**/test-*.xml" testRunTitle: 'Python $(PYTHON_VERSION)' condition: succeededOrFailed() + +- task: PublishCodeCoverageResults@1 + inputs: + codeCoverageTool: pytest-cov + summaryFileLocation: '$(System.DefaultWorkingDirectory)/**/coverage.xml' + reportDirectory: '$(System.DefaultWorkingDirectory)/**/htmlcov' diff --git a/server/citrial/models.py b/server/citrial/models.py new file mode 100644 index 0000000..c6cd7f0 --- /dev/null +++ b/server/citrial/models.py @@ -0,0 +1,10 @@ + + + +def foo(): + x = 1 + return x + +def bar(): + y = 2 + return y diff --git a/server/citrial/tests.py b/server/citrial/tests.py index b1d991c..955b2cb 100644 --- a/server/citrial/tests.py +++ b/server/citrial/tests.py @@ -10,4 +10,6 @@ def testBad(self): def test_pytest_testfinder(): + from citrial.models import foo + foo() assert 1 == 0 diff --git a/server/environment.yml b/server/environment.yml index 2c0344c..856dc3e 100644 --- a/server/environment.yml +++ b/server/environment.yml @@ -11,6 +11,7 @@ dependencies: - pytest - pytest-django - pytest-azurepipelines + - pytest-cov - pandas - xlrd - ujson diff --git a/server/pytest.ini b/server/pytest.ini new file mode 100644 index 0000000..bf8a970 --- /dev/null +++ b/server/pytest.ini @@ -0,0 +1,8 @@ +[pytest] +DJANGO_SETTINGS_MODULE=citrial.settings + +# Use conftest.py to ignore directories or files. +# https://docs.pytest.org/en/latest/example/pythoncollection.html#customizing-test-collection +python_files = tests.py test_*.py *_tests.py +#addopts = -q -r fExXp --disable-warnings --disable-pytest-warnings --showlocals --no-migrations +addopts = --durations=0 -r fExXp --capture=no --doctest-modules --disable-warnings --disable-pytest-warnings --showlocals --reuse-db --cov=citrial --cov-report=html From b66205c1c66c294dfa0b7233b5d7d5983608c017 Mon Sep 17 00:00:00 2001 From: Edwin Grubbs Date: Wed, 10 Jul 2019 12:40:46 -0500 Subject: [PATCH 13/31] Report code coverage even if tests fail. --- azure-pipelines.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 8fbc322..3e67f4a 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -68,5 +68,6 @@ steps: - task: PublishCodeCoverageResults@1 inputs: codeCoverageTool: pytest-cov - summaryFileLocation: '$(System.DefaultWorkingDirectory)/**/coverage.xml' - reportDirectory: '$(System.DefaultWorkingDirectory)/**/htmlcov' + summaryFileLocation: '**/coverage.xml' + reportDirectory: '**/htmlcov' + condition: succeededOrFailed() From e531063cb4aec9f6c476febb82f90154d5233c05 Mon Sep 17 00:00:00 2001 From: Edwin Grubbs Date: Wed, 10 Jul 2019 12:54:11 -0500 Subject: [PATCH 14/31] Changed azure-pipelines.yml code coverage tool specification. --- azure-pipelines.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 3e67f4a..bcfec35 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -67,7 +67,8 @@ steps: - task: PublishCodeCoverageResults@1 inputs: - codeCoverageTool: pytest-cov + # pytest-cov creates a coverage.xml file in the same format as Cobertura. + codeCoverageTool: Cobertura summaryFileLocation: '**/coverage.xml' reportDirectory: '**/htmlcov' condition: succeededOrFailed() From 89120139a9d55903eb6564db940308dcf13a21ca Mon Sep 17 00:00:00 2001 From: Edwin Grubbs Date: Thu, 11 Jul 2019 14:46:45 -0500 Subject: [PATCH 15/31] Changed pool to TradeMetrics-Build --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index bcfec35..dfa9a4f 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -7,7 +7,7 @@ trigger: - master pool: - vmImage: 'ubuntu-latest' + name: TradeMetrics-Build strategy: matrix: Python37: From 8cbebcfff81736c73a6fc30b21bf5ee6ade845f5 Mon Sep 17 00:00:00 2001 From: Edwin Grubbs Date: Thu, 11 Jul 2019 14:55:34 -0500 Subject: [PATCH 16/31] Removed matrix config from azure-pipelines.yml --- azure-pipelines.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index dfa9a4f..9b0c2be 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -8,11 +8,6 @@ trigger: pool: name: TradeMetrics-Build -strategy: - matrix: - Python37: - PYTHON_VERSION: '3.7' - maxParallel: 3 variables: DEBIAN_FRONTEND: 'noninteractive' From 5e5dd78a35e4a46ea2461c364e2be59a66b36469 Mon Sep 17 00:00:00 2001 From: Edwin Grubbs Date: Thu, 11 Jul 2019 14:57:59 -0500 Subject: [PATCH 17/31] Removed versionSpec --- azure-pipelines.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 9b0c2be..b141d39 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -9,13 +9,17 @@ trigger: pool: name: TradeMetrics-Build +strategy: + matrix: + Python37: + PYTHON_VERSION: '3.7' + variables: DEBIAN_FRONTEND: 'noninteractive' steps: - task: UsePythonVersion@0 inputs: - versionSpec: '$(PYTHON_VERSION)' architecture: 'x64' - task: PythonScript@0 From fc25283ae50191b62787650b5b727b7d8c25992b Mon Sep 17 00:00:00 2001 From: Edwin Grubbs Date: Thu, 11 Jul 2019 15:00:50 -0500 Subject: [PATCH 18/31] Added back maxParallel --- azure-pipelines.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b141d39..3e1b06c 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -13,6 +13,7 @@ strategy: matrix: Python37: PYTHON_VERSION: '3.7' + maxParallel: 3 variables: DEBIAN_FRONTEND: 'noninteractive' From 26f017247993e65172abd458a76ccdb9033eb500 Mon Sep 17 00:00:00 2001 From: Edwin Grubbs Date: Thu, 11 Jul 2019 15:04:45 -0500 Subject: [PATCH 19/31] Removed task to check the version --- azure-pipelines.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 3e1b06c..cc18a12 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -19,9 +19,6 @@ variables: DEBIAN_FRONTEND: 'noninteractive' steps: -- task: UsePythonVersion@0 - inputs: - architecture: 'x64' - task: PythonScript@0 displayName: 'Export project path' From 9b554b376ae13a15c62936c64fbbab56a65e6f4d Mon Sep 17 00:00:00 2001 From: Edwin Grubbs Date: Thu, 11 Jul 2019 15:14:26 -0500 Subject: [PATCH 20/31] Do not use inline script to find manage.py. --- azure-pipelines.yml | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index cc18a12..488c4e9 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -20,22 +20,6 @@ variables: steps: -- task: PythonScript@0 - displayName: 'Export project path' - inputs: - scriptSource: 'inline' - script: | - """Search all subdirectories for `manage.py`.""" - from glob import iglob - from os import path - # Python >= 3.5 - manage_py = next(iglob(path.join('**', 'manage.py'), recursive=True), None) - if not manage_py: - raise SystemExit('Could not find a Django project') - project_location = path.dirname(path.abspath(manage_py)) - print('Found Django project in', project_location) - print('##vso[task.setvariable variable=projectRoot]{}'.format(project_location)) - - script: | DEBIAN_FRONTEND=noninteractive sudo apt install -y wget wget -q https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh @@ -49,7 +33,7 @@ steps: displayName: 'Install prerequisites' - script: | - pushd '$(projectRoot)' + pushd 'server' source /home/vsts/miniconda3/etc/profile.d/conda.sh conda activate #python manage.py test --testrunner xmlrunner.extra.djangotestrunner.XMLTestRunner --no-input From c97f932a512c0affc335b7a7e668137396e0119c Mon Sep 17 00:00:00 2001 From: Edwin Grubbs Date: Thu, 11 Jul 2019 15:28:20 -0500 Subject: [PATCH 21/31] sudo not installed --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 488c4e9..b30b06a 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -21,7 +21,7 @@ variables: steps: - script: | - DEBIAN_FRONTEND=noninteractive sudo apt install -y wget + DEBIAN_FRONTEND=noninteractive apt install -y wget wget -q https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh bash Miniconda3-latest-Linux-x86_64.sh -b source $HOME/miniconda3/bin/activate From 363d8dfab7d9f4b3b2dc9137bca59fdecfaadea7 Mon Sep 17 00:00:00 2001 From: Edwin Grubbs Date: Thu, 11 Jul 2019 15:40:04 -0500 Subject: [PATCH 22/31] Added bzip2 (bunzip2) needed for Miniconda. --- azure-pipelines.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b30b06a..bd5f654 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -21,7 +21,8 @@ variables: steps: - script: | - DEBIAN_FRONTEND=noninteractive apt install -y wget + DEBIAN_FRONTEND=noninteractive apt update + DEBIAN_FRONTEND=noninteractive apt install -y wget bzip2 wget -q https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh bash Miniconda3-latest-Linux-x86_64.sh -b source $HOME/miniconda3/bin/activate From 78207c77492b440444b5f253a7e3df2054765e0b Mon Sep 17 00:00:00 2001 From: Edwin Grubbs Date: Thu, 11 Jul 2019 15:44:36 -0500 Subject: [PATCH 23/31] Clean out /root/miniconda3 --- azure-pipelines.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index bd5f654..a265ffb 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -24,6 +24,7 @@ steps: DEBIAN_FRONTEND=noninteractive apt update DEBIAN_FRONTEND=noninteractive apt install -y wget bzip2 wget -q https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh + rm -Rf /root/miniconda3 bash Miniconda3-latest-Linux-x86_64.sh -b source $HOME/miniconda3/bin/activate conda env update -q -f server/environment.yml > /dev/null From 4706ea70aaee0167b7481270355a97e04a788a02 Mon Sep 17 00:00:00 2001 From: Edwin Grubbs Date: Tue, 8 Oct 2019 16:07:27 -0500 Subject: [PATCH 24/31] Attempt to change the bash interpreter used by the azure pipelines `script` task by setting the PATH env var --- azure-pipelines.yml | 8 ++++++++ fakebin/bash | 11 +++++++++++ server/pytest.ini | 2 +- 3 files changed, 20 insertions(+), 1 deletion(-) create mode 100755 fakebin/bash diff --git a/azure-pipelines.yml b/azure-pipelines.yml index a265ffb..d7d9c45 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -17,9 +17,17 @@ strategy: variables: DEBIAN_FRONTEND: 'noninteractive' + PATH: '$System.DefaultWorkingDirectory)/fakebin' steps: +- script: | + echo "Test script" + date + ls + pwd + displayName: 'Test script task' + - script: | DEBIAN_FRONTEND=noninteractive apt update DEBIAN_FRONTEND=noninteractive apt install -y wget bzip2 diff --git a/fakebin/bash b/fakebin/bash new file mode 100755 index 0000000..2ef4f38 --- /dev/null +++ b/fakebin/bash @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -x +echo "OVERRIDE bash interpreter" +echo "ARGS: $0 $*" +echo "PWD: $PWD" +echo "PATH=$PATH" +which bash +PATH=/bin:/usr/bin +echo "new PATH=$PATH" +which bash diff --git a/server/pytest.ini b/server/pytest.ini index bf8a970..c21c5db 100644 --- a/server/pytest.ini +++ b/server/pytest.ini @@ -5,4 +5,4 @@ DJANGO_SETTINGS_MODULE=citrial.settings # https://docs.pytest.org/en/latest/example/pythoncollection.html#customizing-test-collection python_files = tests.py test_*.py *_tests.py #addopts = -q -r fExXp --disable-warnings --disable-pytest-warnings --showlocals --no-migrations -addopts = --durations=0 -r fExXp --capture=no --doctest-modules --disable-warnings --disable-pytest-warnings --showlocals --reuse-db --cov=citrial --cov-report=html +addopts = --durations=0 -r fExXp --tb=line --capture=no --doctest-modules --disable-warnings --disable-pytest-warnings --showlocals --reuse-db --cov=citrial --cov-report=html From 08d23b5a7c5344cb6c1352c38e93427c935564d0 Mon Sep 17 00:00:00 2001 From: Edwin Grubbs Date: Tue, 8 Oct 2019 16:13:48 -0500 Subject: [PATCH 25/31] Use vmImage instead of a self-hosted agent --- azure-pipelines.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index d7d9c45..60f660c 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -7,7 +7,9 @@ trigger: - master pool: - name: TradeMetrics-Build + #name: TradeMetrics-Build + vmImage: 'ubuntu-latest' + strategy: matrix: From 5172324b0d5983620ba79da04679f42f50140494 Mon Sep 17 00:00:00 2001 From: Edwin Grubbs Date: Tue, 8 Oct 2019 16:45:10 -0500 Subject: [PATCH 26/31] Test env vars --- azure-pipelines.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 60f660c..fb4eca4 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -19,16 +19,26 @@ strategy: variables: DEBIAN_FRONTEND: 'noninteractive' - PATH: '$System.DefaultWorkingDirectory)/fakebin' steps: - script: | + env | sort + echo "TESTPATH=$TESTPATH" + echo "DEBIAN_FRONTEND=$DEBIAN_FRONTEND" + displayName: 'Display env' + env: + TESTPATH: '$System.DefaultWorkingDirectory)/fakebin' + +- script: | + env | sort echo "Test script" date ls pwd displayName: 'Test script task' + env: + PATH: '$System.DefaultWorkingDirectory)/fakebin' - script: | DEBIAN_FRONTEND=noninteractive apt update From 788c67d0711db725222972c48ee91327220bc1ed Mon Sep 17 00:00:00 2001 From: Edwin Grubbs Date: Tue, 8 Oct 2019 16:46:35 -0500 Subject: [PATCH 27/31] Fixed path --- azure-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index fb4eca4..533cc97 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -28,7 +28,7 @@ steps: echo "DEBIAN_FRONTEND=$DEBIAN_FRONTEND" displayName: 'Display env' env: - TESTPATH: '$System.DefaultWorkingDirectory)/fakebin' + TESTPATH: '$(System.DefaultWorkingDirectory)/fakebin' - script: | env | sort @@ -38,7 +38,7 @@ steps: pwd displayName: 'Test script task' env: - PATH: '$System.DefaultWorkingDirectory)/fakebin' + PATH: '$(System.DefaultWorkingDirectory)/fakebin' - script: | DEBIAN_FRONTEND=noninteractive apt update From 30c3b13cce6ac2dbc0844c61bb6c21646fabd816 Mon Sep 17 00:00:00 2001 From: Edwin Grubbs Date: Tue, 8 Oct 2019 16:49:11 -0500 Subject: [PATCH 28/31] Fixed path again --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 533cc97..8aed441 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -38,7 +38,7 @@ steps: pwd displayName: 'Test script task' env: - PATH: '$(System.DefaultWorkingDirectory)/fakebin' + PATH: '$(System.DefaultWorkingDirectory)/fakebin:/usr/share/rust/.cargo/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin' - script: | DEBIAN_FRONTEND=noninteractive apt update From 1ffd93e24c227a562bf1e852575eb8f28d772991 Mon Sep 17 00:00:00 2001 From: Edwin Grubbs Date: Wed, 9 Oct 2019 09:01:59 -0500 Subject: [PATCH 29/31] Try to diagnose "Arguments too long" error related to env vars --- azure-pipelines.yml | 11 ++++++++--- fakebin/bash | 3 ++- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 8aed441..365c965 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -23,7 +23,13 @@ variables: steps: - script: | - env | sort + set -x + # The total size of the env variables must be less than ARG_MAX + # or you will get an "Argument list too long" error. + # This is normally 2097152 bytes (2,048 KB). A single env variable + # is must be no greater than 1/16 of ARG_MAX or 128 KB. + getconf ARG_MAX + env | wc -c echo "TESTPATH=$TESTPATH" echo "DEBIAN_FRONTEND=$DEBIAN_FRONTEND" displayName: 'Display env' @@ -31,14 +37,13 @@ steps: TESTPATH: '$(System.DefaultWorkingDirectory)/fakebin' - script: | - env | sort echo "Test script" date ls pwd displayName: 'Test script task' env: - PATH: '$(System.DefaultWorkingDirectory)/fakebin:/usr/share/rust/.cargo/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin' + PATH: '$(System.DefaultWorkingDirectory)/fakebin:/usr/share/rust/.cargo/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin' - script: | DEBIAN_FRONTEND=noninteractive apt update diff --git a/fakebin/bash b/fakebin/bash index 2ef4f38..bfc9acf 100755 --- a/fakebin/bash +++ b/fakebin/bash @@ -1,10 +1,11 @@ -#!/usr/bin/env bash +#!/bin/bash set -x echo "OVERRIDE bash interpreter" echo "ARGS: $0 $*" echo "PWD: $PWD" echo "PATH=$PATH" +set | sort which bash PATH=/bin:/usr/bin echo "new PATH=$PATH" From 28a128a8a01eae7ccd95e04eec363a10edcbd011 Mon Sep 17 00:00:00 2001 From: Edwin Grubbs Date: Wed, 9 Oct 2019 10:39:41 -0500 Subject: [PATCH 30/31] Test fakebin/bash --- azure-pipelines.yml | 3 +++ fakebin/bash | 8 +++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 365c965..41c0022 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -41,6 +41,9 @@ steps: date ls pwd + which env + echo "GLOB TEST:" * + env | sort displayName: 'Test script task' env: PATH: '$(System.DefaultWorkingDirectory)/fakebin:/usr/share/rust/.cargo/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin' diff --git a/fakebin/bash b/fakebin/bash index bfc9acf..4532d28 100755 --- a/fakebin/bash +++ b/fakebin/bash @@ -5,8 +5,14 @@ echo "OVERRIDE bash interpreter" echo "ARGS: $0 $*" echo "PWD: $PWD" echo "PATH=$PATH" -set | sort which bash PATH=/bin:/usr/bin echo "new PATH=$PATH" which bash +cd /usr +echo "Do not execute, just evaluate:" +/bin/bash -nv $* +echo "Execute without GLOB support:" +/bin/bash -xf $* +echo "Execute with GLOB support:" +/bin/bash -x $* From ebb91839bb9c5a74e8022e83a9bf497f43d8a1c3 Mon Sep 17 00:00:00 2001 From: Edwin Grubbs Date: Wed, 9 Oct 2019 11:25:59 -0500 Subject: [PATCH 31/31] Move bash GNU long options before the short options --- fakebin/bash | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/fakebin/bash b/fakebin/bash index 4532d28..0da6a26 100755 --- a/fakebin/bash +++ b/fakebin/bash @@ -10,9 +10,20 @@ PATH=/bin:/usr/bin echo "new PATH=$PATH" which bash cd /usr +echo "Arg count: $#" +start_args=${@:1:$[ $# - 1 ]} +echo "All but last arg: $start_args" +last_arg=${@:$#:$[ $# + 1 ]} +echo "Last arg: $last_arg" + +# Bash does not allow "GNU long options" such as "--norc" +# to be after short options such as "-x". echo "Do not execute, just evaluate:" -/bin/bash -nv $* +/bin/bash $start_args -nv $last_arg + echo "Execute without GLOB support:" -/bin/bash -xf $* +/bin/bash $start_args -xf $last_arg + echo "Execute with GLOB support:" -/bin/bash -x $* +/bin/bash $start_args -x $last_arg +