diff --git a/.dockerignore b/.dockerignore index ae23cfe60..efaff757a 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,7 +5,6 @@ docs /infrastructure /blueprint test -/tools/deployment-cli-tools .github .git .vscode diff --git a/application-templates/base/test/api/test_st.py b/application-templates/base/test/api/test_st.py index 902e2bd5e..20ebd222b 100644 --- a/application-templates/base/test/api/test_st.py +++ b/application-templates/base/test/api/test_st.py @@ -1,17 +1,14 @@ import os -from pprint import pprint import schemathesis as st -from schemathesis.checks import response_schema_conformance, not_a_server_error - -from cloudharness_test import apitest_init # include to perform default authorization +from cloudharness_test import apitest_init # include to register default hooks +from cloudharness_test import apitest_auth_hooks # include to register authentication hooks app_url = os.environ.get("APP_URL", "http://samples.ch.local/api") -schema = st.from_uri(app_url + "/openapi.json") +schema = st.openapi.from_url(app_url + "/openapi.json") -@schema.parametrize(endpoint="/ping") +@schema.include(path="/ping", method="GET").parametrize() def test_ping(case): response = case.call() - pprint(response.__dict__) assert response.status_code == 200, "this api errors on purpose" diff --git a/application-templates/django-base/api/test_st.py b/application-templates/django-base/api/test_st.py index a7527b804..e466808e5 100644 --- a/application-templates/django-base/api/test_st.py +++ b/application-templates/django-base/api/test_st.py @@ -1,23 +1,20 @@ import os -from pprint import pprint import schemathesis as st -from schemathesis.checks import response_schema_conformance, not_a_server_error - -from cloudharness_test import apitest_init # include to perform default authorization +from cloudharness_test import apitest_init # include to register default hooks +from cloudharness_test import apitest_auth_hooks # include to register authentication hooks app_url = os.environ.get("APP_URL", "http://samples.ch.local/api") try: - schema = st.from_uri(app_url + "/openapi.json") + schema = st.openapi.from_url(app_url + "/openapi.json") except: # support alternative schema location - schema = st.from_uri(app_url.replace("/api", "") + "/openapi.json") + schema = st.openapi.from_url(app_url.replace("/api", "") + "/openapi.json") -@schema.parametrize(endpoint="/ping") +@schema.include(path="/ping", method="GET").parametrize() def test_ping(case): response = case.call() - pprint(response.__dict__) assert response.status_code == 200, "this api errors on purpose" diff --git a/applications/common/api/openapi.yaml b/applications/common/api/openapi.yaml index 33c2b91a9..0ded9c81a 100644 --- a/applications/common/api/openapi.yaml +++ b/applications/common/api/openapi.yaml @@ -32,13 +32,16 @@ paths: description: Sentry not configured for the given application '404': content: + application/json: + schema: + type: object application/problem+json: schema: type: object text/html: schema: type: string - description: Sentry not configured for the given application + description: Application not found operationId: getdsn summary: Gets the Sentry DSN for a given application description: Gets the Sentry DSN for a given application diff --git a/applications/common/deploy/values.yaml b/applications/common/deploy/values.yaml index eec864503..5a0f9f627 100644 --- a/applications/common/deploy/values.yaml +++ b/applications/common/deploy/values.yaml @@ -27,4 +27,6 @@ harness: enabled: true autotest: true checks: - - all \ No newline at end of file + - all + runParams: + - "--suppress-health-check=filter_too_much" \ No newline at end of file diff --git a/applications/common/schemathesis.toml b/applications/common/schemathesis.toml new file mode 100644 index 000000000..377648b55 --- /dev/null +++ b/applications/common/schemathesis.toml @@ -0,0 +1,2 @@ +[phases.coverage] +unexpected-methods = ["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"] \ No newline at end of file diff --git a/applications/common/server/common/controllers/sentry_controller.py b/applications/common/server/common/controllers/sentry_controller.py index 4d0727825..e63bf3c09 100644 --- a/applications/common/server/common/controllers/sentry_controller.py +++ b/applications/common/server/common/controllers/sentry_controller.py @@ -26,7 +26,7 @@ def getdsn(appname): # noqa: E501 try: ch_app = applications.get_configuration(appname) except applications.ConfigurationCallException as e: - return {"error": f"Application `{appname}` does not exist"}, 400 + return {"error": f"Application `{appname}` does not exist"}, 404 if ch_app.is_sentry_enabled(): if global_dsn: # if a global dsn env var is set and not empty then use this diff --git a/applications/common/server/common/openapi/openapi.yaml b/applications/common/server/common/openapi/openapi.yaml index 02e7b77fb..d74212f78 100644 --- a/applications/common/server/common/openapi/openapi.yaml +++ b/applications/common/server/common/openapi/openapi.yaml @@ -57,13 +57,16 @@ paths: description: Sentry not configured for the given application "404": content: + application/json: + schema: + type: object application/problem+json: schema: type: object text/html: schema: type: string - description: Sentry not configured for the given application + description: Application not found summary: Gets the Sentry DSN for a given application tags: - Sentry diff --git a/applications/samples/api/openapi.yaml b/applications/samples/api/openapi.yaml index f51b328df..9462b1c60 100644 --- a/applications/samples/api/openapi.yaml +++ b/applications/samples/api/openapi.yaml @@ -54,6 +54,8 @@ paths: schema: type: string description: Check if token is valid + "400": + description: Bad request "401": description: "invalid token, unauthorized" security: @@ -74,6 +76,8 @@ paths: schema: type: string description: Check if token is valid + "400": + description: Bad request "401": description: "invalid token, unauthorized" security: diff --git a/applications/samples/backend/samples/controllers/auth_controller.py b/applications/samples/backend/samples/controllers/auth_controller.py index e144a976c..554e92f90 100644 --- a/applications/samples/backend/samples/controllers/auth_controller.py +++ b/applications/samples/backend/samples/controllers/auth_controller.py @@ -15,6 +15,8 @@ def valid_token(): # noqa: E501 """ from cloudharness.middleware import get_authentication_token token = get_authentication_token() + if not token: + return 'Unauthorized', 401 return 'OK!' @@ -29,5 +31,7 @@ def valid_cookie(): # noqa: E501 from cloudharness.middleware import get_authentication_token from cloudharness.auth import decode_token token = get_authentication_token() + if not token: + return 'Unauthorized', 401 assert decode_token(token) return 'OK' diff --git a/applications/samples/backend/samples/controllers/resource_controller.py b/applications/samples/backend/samples/controllers/resource_controller.py index 4f17d3da5..4ed23dabf 100644 --- a/applications/samples/backend/samples/controllers/resource_controller.py +++ b/applications/samples/backend/samples/controllers/resource_controller.py @@ -44,7 +44,7 @@ def delete_sample_resource(sampleresource_id): # noqa: E501 except resource_service.ResourceNotFound: return "Resource not found", 404 except ValueError: - return "sampleresource_id must be integer", 400 + return "Resource not found", 404 return 'OK', 204 @@ -64,7 +64,7 @@ def get_sample_resource(sampleresource_id): # noqa: E501 except resource_service.ResourceNotFound: return "Resource not found", 404 except ValueError: - return "sampleresource_id must be integer", 400 + return "Resource not found", 404 def get_sample_resources(): # noqa: E501 @@ -101,4 +101,4 @@ def update_sample_resource(sampleresource_id, sample_resource=None): # noqa: E5 except resource_service.ResourceNotFound: return "Resource not found", 404 except ValueError: - return "sampleresource_id must be integer", 400 + return "Resource not found", 404 diff --git a/applications/samples/backend/samples/controllers/security_controller_.py b/applications/samples/backend/samples/controllers/security_controller_.py index c052e680f..1ee232e9a 100644 --- a/applications/samples/backend/samples/controllers/security_controller_.py +++ b/applications/samples/backend/samples/controllers/security_controller_.py @@ -12,4 +12,6 @@ def info_from_bearerAuth(token): :return: Decoded token information or None if token is invalid :rtype: dict | None """ + if token is None: + return None return {'uid': 'user_id'} diff --git a/applications/samples/deploy/values.yaml b/applications/samples/deploy/values.yaml index ed04be7c7..93f9289e6 100644 --- a/applications/samples/deploy/values.yaml +++ b/applications/samples/deploy/values.yaml @@ -81,16 +81,15 @@ harness: checks: - all runParams: - - "--skip-deprecated-operations" + - "--exclude-deprecated" - "--exclude-operation-id=submit_sync" - "--exclude-operation-id=submit_sync_with_results" - "--exclude-operation-id=error" - - "--hypothesis-suppress-health-check=too_slow" - - "--hypothesis-deadline=180000" + - "--suppress-health-check=too_slow" + - "--suppress-health-check=filter_too_much" - "--request-timeout=180000" - - "--hypothesis-max-examples=2" - - "--show-trace" - - "--exclude-checks=ignored_auth" # ignored_auth is not working on schemathesis 3.36.0 + - "--max-examples=2" + - "--exclude-checks=ignored_auth" dockerfile: buildArgs: diff --git a/applications/samples/schemathesis.toml b/applications/samples/schemathesis.toml new file mode 100644 index 000000000..1b6ff3fc6 --- /dev/null +++ b/applications/samples/schemathesis.toml @@ -0,0 +1,12 @@ +[phases.coverage] +unexpected-methods = ["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"] + +[[operations]] +include-operation-id = "valid_token" +[operations.checks] +negative_data_rejection.enabled = false + +[[operations]] +include-operation-id = "valid_cookie" +[operations.checks] +negative_data_rejection.enabled = false \ No newline at end of file diff --git a/applications/samples/test/api/test_st.py b/applications/samples/test/api/test_st.py index e67cb3e55..f78874617 100644 --- a/applications/samples/test/api/test_st.py +++ b/applications/samples/test/api/test_st.py @@ -1,36 +1,44 @@ import os from pprint import pprint import schemathesis as st -from schemathesis.checks import response_schema_conformance, not_a_server_error +from schemathesis.specs.openapi.checks import response_schema_conformance -from cloudharness_test import apitest_init # include to perform default authorization +from cloudharness_test import apitest_init # include to register default hooks +from cloudharness_test import apitest_auth_hooks # include to register authentication hooks app_url = os.environ.get("APP_URL", "http://samples.ch.local/api") -schema = st.from_uri(app_url + "/openapi.json") +schema = st.openapi.from_url(app_url + "/openapi.json") -@schema.parametrize(endpoint="/error") +@schema.include(path="/error", method="GET").parametrize() def test_api(case): response = case.call() - pprint(response.__dict__) - assert response.status_code >= 500, "this api errors on purpose" + if case.method == "GET": + # Assert that this endpoint returns a 500 error as expected + assert response.status_code == 500, f"Expected 500 error, got {response.status_code}. This api errors on purpose." + elif case.method == "OPTIONS": + # OPTIONS requests typically return 200 OK + assert response.status_code == 200, f"Expected 200 OK for OPTIONS, got {response.status_code}." + else: + # Other methods should return 405 (Method Not Allowed) + assert response.status_code == 405, f"Expected 405 (Method Not Allowed) for {case.method}, got {response.status_code}." -@schema.parametrize(endpoint="/valid") + +@schema.include(path="/valid", method="GET").parametrize() def test_bearer(case): response = case.call() - case.validate_response(response, checks=(response_schema_conformance,)) -@schema.parametrize(endpoint="/valid-cookie") +@schema.include(path="/valid-cookie", method="GET").parametrize() def test_cookie(case): response = case.call() case.validate_response(response, checks=(response_schema_conformance,)) -@schema.parametrize(endpoint="/sampleresources", method="POST") +@schema.include(path="/sampleresources", method="POST").parametrize() def test_response(case): response = case.call() case.validate_response(response, checks=(response_schema_conformance,)) diff --git a/applications/volumemanager/api/openapi.yaml b/applications/volumemanager/api/openapi.yaml index 50fcc805e..25dc5468d 100644 --- a/applications/volumemanager/api/openapi.yaml +++ b/applications/volumemanager/api/openapi.yaml @@ -28,10 +28,15 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/PersistentVolumeClaim' + type: string + example: "Saved!" description: Save successful. "400": - description: The Persistent Volume Claim already exists. + description: Bad request - invalid or missing required fields. + "401": + description: Unauthorized - No authorization token provided. + "500": + description: Internal server error. security: - bearerAuth: [] summary: Create a Persistent Volume Claim in Kubernetes @@ -49,6 +54,9 @@ paths: required: true schema: type: string + pattern: '^[a-z0-9]([-a-z0-9]*[a-z0-9])?$' + minLength: 1 + maxLength: 253 style: simple responses: "200": @@ -57,6 +65,10 @@ paths: schema: $ref: '#/components/schemas/PersistentVolumeClaim' description: The Persistent Volume Claim. + "400": + description: Bad request - invalid path parameter or headers. + "401": + description: Unauthorized - No authorization token provided. "404": description: The Persistent Volume Claim was not found. security: @@ -69,21 +81,29 @@ components: schemas: PersistentVolumeClaimCreate: example: - size: 2Gi (see also https://github.com/kubernetes/community/blob/master/contributors/design-proposals/scheduling/resources.md#resource-quantities) + size: 2Gi name: pvc-1 properties: name: description: Unique name for the Persisten Volume Claim to create. example: pvc-1 type: string + minLength: 1 + pattern: '^[a-z0-9]([-a-z0-9]*[a-z0-9])?$' size: description: The size of the Persistent Volume Claim to create. - example: 2Gi (see also https://github.com/kubernetes/community/blob/master/contributors/design-proposals/scheduling/resources.md#resource-quantities) + example: 2Gi type: string + minLength: 1 + pattern: '^[1-9][0-9]*(Ei|Pi|Ti|Gi|Mi|Ki|E|P|T|G|M|K)?$' + required: + - name + - size type: object + additionalProperties: false PersistentVolumeClaim: example: - size: 2Gi (see also https://github.com/kubernetes/community/blob/master/contributors/design-proposals/scheduling/resources.md#resource-quantities) + size: 2Gi name: pvc-1 namespace: ch accessmode: ReadWriteMany @@ -102,7 +122,7 @@ components: type: string size: description: The size of the Persistent Volume Claim. - example: 2Gi (see also https://github.com/kubernetes/community/blob/master/contributors/design-proposals/scheduling/resources.md#resource-quantities) + example: 2Gi type: string type: object securitySchemes: diff --git a/applications/volumemanager/deploy/values.yaml b/applications/volumemanager/deploy/values.yaml index a30e43d11..bd1296e1e 100644 --- a/applications/volumemanager/deploy/values.yaml +++ b/applications/volumemanager/deploy/values.yaml @@ -21,4 +21,7 @@ harness: enabled: true autotest: true checks: - - all \ No newline at end of file + - all + runParams: + - "--phases=examples" + - "--max-examples=1" \ No newline at end of file diff --git a/applications/volumemanager/schemathesis.toml b/applications/volumemanager/schemathesis.toml new file mode 100644 index 000000000..fc9bd19e8 --- /dev/null +++ b/applications/volumemanager/schemathesis.toml @@ -0,0 +1,5 @@ +[phases.coverage] +unexpected-methods = ["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"] + +[checks] +negative_data_rejection.enabled = false diff --git a/applications/volumemanager/server/volumemanager/controllers/rest_controller.py b/applications/volumemanager/server/volumemanager/controllers/rest_controller.py index 5eb30a4f5..23b289ecb 100644 --- a/applications/volumemanager/server/volumemanager/controllers/rest_controller.py +++ b/applications/volumemanager/server/volumemanager/controllers/rest_controller.py @@ -1,6 +1,8 @@ import connexion import six import flask +import re +from kubernetes.client.rest import ApiException from cloudharness.service.pvc import create_persistent_volume_claim, get_persistent_volume_claim @@ -23,13 +25,21 @@ def pvc_name_get(name): # noqa: E501 if not pvc: return f"Persistent Volume Claim with name {name} not found.", 404 - pvc = PersistentVolumeClaim( + # Extract access mode safely + access_mode = pvc.status.access_modes[0] if pvc.status and pvc.status.access_modes else '' + + # Extract size safely + size = '' + if pvc.status and pvc.status.capacity: + size = pvc.status.capacity.get('storage', '') + + pvc_response = PersistentVolumeClaim( name=pvc.metadata.name, namespace=pvc.metadata.namespace, - accessmode=pvc.status.access_modes[0], - size=pvc.status.capacity.get('storage', '') + accessmode=access_mode, + size=size ) - return pvc + return pvc_response def pvc_post(): # noqa: E501 @@ -44,8 +54,22 @@ def pvc_post(): # noqa: E501 """ if connexion.request.is_json: persistent_volume_claim_create = PersistentVolumeClaimCreate.from_dict(connexion.request.get_json()) # noqa: E501 - create_persistent_volume_claim( - name=persistent_volume_claim_create.name, - size=persistent_volume_claim_create.size, - logger=flask.current_app.logger) - return 'Saved!' + + # Validate required fields + if not persistent_volume_claim_create.name or not persistent_volume_claim_create.size: + return {'description': 'Name and size are required and cannot be empty.'}, 400 + try: + create_persistent_volume_claim( + name=persistent_volume_claim_create.name, + size=persistent_volume_claim_create.size, + logger=flask.current_app.logger) + except ApiException as e: + flask.current_app.logger.error(f"Kubernetes API error creating PVC: {e}") + # Return 400 for client errors (bad request to k8s), 500 for server errors + if e.status >= 400 and e.status < 500: + return {'description': f'Invalid PVC configuration: {e.reason}'}, 400 + return {'description': f'Failed to create Persistent Volume Claim: {e.reason}'}, 500 + except Exception as e: + flask.current_app.logger.error(f"Error creating PVC: {e}") + return {'description': f'Failed to create Persistent Volume Claim: {str(e)}'}, 500 + return 'Saved!', 201 diff --git a/applications/volumemanager/server/volumemanager/openapi/openapi.yaml b/applications/volumemanager/server/volumemanager/openapi/openapi.yaml index f27b03d89..821b753cd 100644 --- a/applications/volumemanager/server/volumemanager/openapi/openapi.yaml +++ b/applications/volumemanager/server/volumemanager/openapi/openapi.yaml @@ -31,6 +31,10 @@ paths: description: Save successful. "400": description: The Persistent Volume Claim already exists. + "401": + description: Unauthorized - No authorization token provided. + "500": + description: Internal server error. security: - bearerAuth: [] summary: Create a Persistent Volume Claim in Kubernetes @@ -56,6 +60,8 @@ paths: schema: $ref: '#/components/schemas/PersistentVolumeClaim' description: The Persistent Volume Claim. + "401": + description: Unauthorized - No authorization token provided. "404": description: The Persistent Volume Claim was not found. security: diff --git a/applications/workflows/api/openapi.yaml b/applications/workflows/api/openapi.yaml index b2d674d60..bd4e6fc56 100644 --- a/applications/workflows/api/openapi.yaml +++ b/applications/workflows/api/openapi.yaml @@ -52,7 +52,6 @@ paths: parameters: - name: name schema: - pattern: '^[0-9A-Za-z\s\-]+$' type: string in: path required: true @@ -79,12 +78,11 @@ paths: parameters: - examples: e1: - value: '"my-operation"' + value: my-operation e2: value: my-operation-122a name: name schema: - pattern: '^[0-9A-Za-z\s\-]+$' type: string in: path required: true @@ -101,11 +99,7 @@ paths: schema: $ref: "#/components/schemas/OperationStatus" in: query - - examples: - example1: - value: >- - "eyJ2IjoibWV0YS5rOHMuaW8vdjEiLCJydiI6NDUzMDMzOCwic3RhcnQiOiJoZWxsby13b3JsZC05YnE2ZFx1MDAwMCJ8" - name: previous_search_token + - name: previous_search_token description: continue previous search (pagination chunks) schema: type: string diff --git a/applications/workflows/deploy/values.yaml b/applications/workflows/deploy/values.yaml index 15c8706a8..d9b4531e5 100644 --- a/applications/workflows/deploy/values.yaml +++ b/applications/workflows/deploy/values.yaml @@ -18,3 +18,5 @@ harness: autotest: true checks: - all + runParams: + - "--suppress-health-check=filter_too_much" \ No newline at end of file diff --git a/applications/workflows/schemathesis.toml b/applications/workflows/schemathesis.toml new file mode 100644 index 000000000..414ad9395 --- /dev/null +++ b/applications/workflows/schemathesis.toml @@ -0,0 +1,10 @@ +[phases.coverage] +unexpected-methods = ["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"] + +[checks] +negative_data_rejection.enabled = false + + +[[operations]] +include-name = "GET /operations" +parameters = { previous_search_token = "" } diff --git a/applications/workflows/server/workflows_api/controllers/create_and_access_controller.py b/applications/workflows/server/workflows_api/controllers/create_and_access_controller.py index 561b20af3..a148016e0 100644 --- a/applications/workflows/server/workflows_api/controllers/create_and_access_controller.py +++ b/applications/workflows/server/workflows_api/controllers/create_and_access_controller.py @@ -65,6 +65,9 @@ def list_operations(status=None, previous_search_token=None, limit=None): # noq :rtype: OperationSearchResult """ + if previous_search_token == "": + previous_search_token = None + try: return workflow_service.list_operations(status, continue_token=previous_search_token, limit=limit) except BadParam as e: @@ -90,3 +93,6 @@ def log_operation(name): # noqa: E501 return workflow_service.log_operation(name) except OperationNotFound as e: return (f'{name} not found', 404) + except OperationException as e: + log.error(f'Unhandled remote exception while retrieving workflow logs for {name}', exc_info=e) + return f'Unexpected error', e.status diff --git a/applications/workflows/server/workflows_api/openapi/openapi.yaml b/applications/workflows/server/workflows_api/openapi/openapi.yaml index 638b9f9a7..b19f29641 100644 --- a/applications/workflows/server/workflows_api/openapi/openapi.yaml +++ b/applications/workflows/server/workflows_api/openapi/openapi.yaml @@ -30,9 +30,6 @@ paths: $ref: '#/components/schemas/OperationStatus' style: form - description: continue previous search (pagination chunks) - examples: - example1: - value: '"eyJ2IjoibWV0YS5rOHMuaW8vdjEiLCJydiI6NDUzMDMzOCwic3RhcnQiOiJoZWxsby13b3JsZC05YnE2ZFx1MDAwMCJ8"' explode: true in: query name: previous_search_token @@ -96,7 +93,6 @@ paths: name: name required: true schema: - pattern: "^[0-9A-Za-z\\s\\-]+$" type: string style: simple responses: @@ -124,7 +120,7 @@ paths: parameters: - examples: e1: - value: '"my-operation"' + value: my-operation e2: value: my-operation-122a explode: false @@ -132,7 +128,6 @@ paths: name: name required: true schema: - pattern: "^[0-9A-Za-z\\s\\-]+$" type: string style: simple responses: diff --git a/deployment/codefresh-test.yaml b/deployment/codefresh-test.yaml index e9699e95a..91bfca977 100644 --- a/deployment/codefresh-test.yaml +++ b/deployment/codefresh-test.yaml @@ -13,13 +13,26 @@ steps: repo: '${{CF_REPO_OWNER}}/${{CF_REPO_NAME}}' revision: '${{CF_BRANCH}}' git: github + post_main_clone: + title: Post main clone + type: parallel + stage: prepare + steps: + clone_cloud_harness: + title: Cloning cloud-harness repository... + type: git-clone + stage: prepare + repo: https://github.com/MetaCell/cloud-harness.git + revision: '${{CLOUDHARNESS_BRANCH}}' + working_directory: . + git: github prepare_deployment: title: Prepare helm chart image: python:3.12 stage: prepare working_directory: . commands: - - bash ./install.sh + - bash cloud-harness/install.sh - harness-deployment . -n test-${{NAMESPACE_BASENAME}} -d ${{DOMAIN}} -r ${{REGISTRY}} -rs ${{REGISTRY_SECRET}} -e test --write-env --cache-url '${{IMAGE_CACHE_URL}}' -N -i samples @@ -38,7 +51,7 @@ steps: type: parallel stage: build steps: - accounts: + test-e2e: type: build stage: build dockerfile: Dockerfile @@ -46,65 +59,66 @@ steps: buildkit: true build_arguments: - NOCACHE=${{CF_BUILD_ID}} - image_name: cloud-harness/accounts - title: Accounts - working_directory: ./applications/accounts + image_name: cloud-harness/test-e2e + title: Test e2e + working_directory: ./test/test-e2e tags: - - '${{ACCOUNTS_TAG}}' + - '${{TEST_E2E_TAG}}' - '${{DEPLOYMENT_PUBLISH_TAG}}-dev' - '${{CF_BRANCH_TAG_NORMALIZED_LOWER_CASE}}' + - latest when: condition: any: - buildDoesNotExist: includes('${{ACCOUNTS_TAG_EXISTS}}', '{{ACCOUNTS_TAG_EXISTS}}') + buildDoesNotExist: includes('${{TEST_E2E_TAG_EXISTS}}', '{{TEST_E2E_TAG_EXISTS}}') == true - forceNoCache: includes('${{ACCOUNTS_TAG_FORCE_BUILD}}', '{{ACCOUNTS_TAG_FORCE_BUILD}}') + forceNoCache: includes('${{TEST_E2E_TAG_FORCE_BUILD}}', '{{TEST_E2E_TAG_FORCE_BUILD}}') == false - cloudharness-base: + cloudharness-frontend-build: type: build stage: build - dockerfile: infrastructure/base-images/cloudharness-base/Dockerfile + dockerfile: infrastructure/base-images/cloudharness-frontend-build/Dockerfile registry: '${{CODEFRESH_REGISTRY}}' buildkit: true build_arguments: - NOCACHE=${{CF_BUILD_ID}} - image_name: cloud-harness/cloudharness-base - title: Cloudharness base + image_name: cloud-harness/cloudharness-frontend-build + title: Cloudharness frontend build working_directory: ./. tags: - - '${{CLOUDHARNESS_BASE_TAG}}' + - '${{CLOUDHARNESS_FRONTEND_BUILD_TAG}}' - '${{DEPLOYMENT_PUBLISH_TAG}}-dev' - '${{CF_BRANCH_TAG_NORMALIZED_LOWER_CASE}}' when: condition: any: - buildDoesNotExist: includes('${{CLOUDHARNESS_BASE_TAG_EXISTS}}', '{{CLOUDHARNESS_BASE_TAG_EXISTS}}') - == true - forceNoCache: includes('${{CLOUDHARNESS_BASE_TAG_FORCE_BUILD}}', '{{CLOUDHARNESS_BASE_TAG_FORCE_BUILD}}') - == false - cloudharness-frontend-build: + buildDoesNotExist: includes('${{CLOUDHARNESS_FRONTEND_BUILD_TAG_EXISTS}}', + '{{CLOUDHARNESS_FRONTEND_BUILD_TAG_EXISTS}}') == true + forceNoCache: includes('${{CLOUDHARNESS_FRONTEND_BUILD_TAG_FORCE_BUILD}}', + '{{CLOUDHARNESS_FRONTEND_BUILD_TAG_FORCE_BUILD}}') == false + cloudharness-base: type: build stage: build - dockerfile: infrastructure/base-images/cloudharness-frontend-build/Dockerfile + dockerfile: infrastructure/base-images/cloudharness-base/Dockerfile registry: '${{CODEFRESH_REGISTRY}}' buildkit: true build_arguments: - NOCACHE=${{CF_BUILD_ID}} - image_name: cloud-harness/cloudharness-frontend-build - title: Cloudharness frontend build + image_name: cloud-harness/cloudharness-base + title: Cloudharness base working_directory: ./. tags: - - '${{CLOUDHARNESS_FRONTEND_BUILD_TAG}}' + - '${{CLOUDHARNESS_BASE_TAG}}' - '${{DEPLOYMENT_PUBLISH_TAG}}-dev' - '${{CF_BRANCH_TAG_NORMALIZED_LOWER_CASE}}' when: condition: any: - buildDoesNotExist: includes('${{CLOUDHARNESS_FRONTEND_BUILD_TAG_EXISTS}}', - '{{CLOUDHARNESS_FRONTEND_BUILD_TAG_EXISTS}}') == true - forceNoCache: includes('${{CLOUDHARNESS_FRONTEND_BUILD_TAG_FORCE_BUILD}}', - '{{CLOUDHARNESS_FRONTEND_BUILD_TAG_FORCE_BUILD}}') == false - test-e2e: + buildDoesNotExist: includes('${{CLOUDHARNESS_BASE_TAG_EXISTS}}', '{{CLOUDHARNESS_BASE_TAG_EXISTS}}') + == true + forceNoCache: includes('${{CLOUDHARNESS_BASE_TAG_FORCE_BUILD}}', '{{CLOUDHARNESS_BASE_TAG_FORCE_BUILD}}') + == false + accounts: type: build stage: build dockerfile: Dockerfile @@ -112,27 +126,26 @@ steps: buildkit: true build_arguments: - NOCACHE=${{CF_BUILD_ID}} - image_name: cloud-harness/test-e2e - title: Test e2e - working_directory: ./test/test-e2e + image_name: cloud-harness/accounts + title: Accounts + working_directory: ./applications/accounts tags: - - '${{TEST_E2E_TAG}}' + - '${{ACCOUNTS_TAG}}' - '${{DEPLOYMENT_PUBLISH_TAG}}-dev' - '${{CF_BRANCH_TAG_NORMALIZED_LOWER_CASE}}' - - latest when: condition: any: - buildDoesNotExist: includes('${{TEST_E2E_TAG_EXISTS}}', '{{TEST_E2E_TAG_EXISTS}}') + buildDoesNotExist: includes('${{ACCOUNTS_TAG_EXISTS}}', '{{ACCOUNTS_TAG_EXISTS}}') == true - forceNoCache: includes('${{TEST_E2E_TAG_FORCE_BUILD}}', '{{TEST_E2E_TAG_FORCE_BUILD}}') + forceNoCache: includes('${{ACCOUNTS_TAG_FORCE_BUILD}}', '{{ACCOUNTS_TAG_FORCE_BUILD}}') == false title: Build parallel step 1 build_application_images_1: type: parallel stage: build steps: - jupyterhub: + cloudharness-flask: type: build stage: build dockerfile: Dockerfile @@ -141,19 +154,19 @@ steps: build_arguments: - NOCACHE=${{CF_BUILD_ID}} - CLOUDHARNESS_BASE=${{REGISTRY}}/cloud-harness/cloudharness-base:${{CLOUDHARNESS_BASE_TAG}} - image_name: cloud-harness/jupyterhub - title: Jupyterhub - working_directory: ./applications/jupyterhub + image_name: cloud-harness/cloudharness-flask + title: Cloudharness flask + working_directory: ./infrastructure/common-images/cloudharness-flask tags: - - '${{JUPYTERHUB_TAG}}' + - '${{CLOUDHARNESS_FLASK_TAG}}' - '${{DEPLOYMENT_PUBLISH_TAG}}-dev' - '${{CF_BRANCH_TAG_NORMALIZED_LOWER_CASE}}' when: condition: any: - buildDoesNotExist: includes('${{JUPYTERHUB_TAG_EXISTS}}', '{{JUPYTERHUB_TAG_EXISTS}}') + buildDoesNotExist: includes('${{CLOUDHARNESS_FLASK_TAG_EXISTS}}', '{{CLOUDHARNESS_FLASK_TAG_EXISTS}}') == true - forceNoCache: includes('${{JUPYTERHUB_TAG_FORCE_BUILD}}', '{{JUPYTERHUB_TAG_FORCE_BUILD}}') + forceNoCache: includes('${{CLOUDHARNESS_FLASK_TAG_FORCE_BUILD}}', '{{CLOUDHARNESS_FLASK_TAG_FORCE_BUILD}}') == false workflows-extract-download: type: build @@ -178,29 +191,6 @@ steps: '{{WORKFLOWS_EXTRACT_DOWNLOAD_TAG_EXISTS}}') == true forceNoCache: includes('${{WORKFLOWS_EXTRACT_DOWNLOAD_TAG_FORCE_BUILD}}', '{{WORKFLOWS_EXTRACT_DOWNLOAD_TAG_FORCE_BUILD}}') == false - workflows-notify-queue: - type: build - stage: build - dockerfile: Dockerfile - registry: '${{CODEFRESH_REGISTRY}}' - buildkit: true - build_arguments: - - NOCACHE=${{CF_BUILD_ID}} - - CLOUDHARNESS_BASE=${{REGISTRY}}/cloud-harness/cloudharness-base:${{CLOUDHARNESS_BASE_TAG}} - image_name: cloud-harness/workflows-notify-queue - title: Workflows notify queue - working_directory: ./applications/workflows/tasks/notify-queue - tags: - - '${{WORKFLOWS_NOTIFY_QUEUE_TAG}}' - - '${{DEPLOYMENT_PUBLISH_TAG}}-dev' - - '${{CF_BRANCH_TAG_NORMALIZED_LOWER_CASE}}' - when: - condition: - any: - buildDoesNotExist: includes('${{WORKFLOWS_NOTIFY_QUEUE_TAG_EXISTS}}', - '{{WORKFLOWS_NOTIFY_QUEUE_TAG_EXISTS}}') == true - forceNoCache: includes('${{WORKFLOWS_NOTIFY_QUEUE_TAG_FORCE_BUILD}}', - '{{WORKFLOWS_NOTIFY_QUEUE_TAG_FORCE_BUILD}}') == false samples-print-file: type: build stage: build @@ -224,7 +214,7 @@ steps: == true forceNoCache: includes('${{SAMPLES_PRINT_FILE_TAG_FORCE_BUILD}}', '{{SAMPLES_PRINT_FILE_TAG_FORCE_BUILD}}') == false - workflows-send-result-event: + samples-sum: type: build stage: build dockerfile: Dockerfile @@ -233,45 +223,44 @@ steps: build_arguments: - NOCACHE=${{CF_BUILD_ID}} - CLOUDHARNESS_BASE=${{REGISTRY}}/cloud-harness/cloudharness-base:${{CLOUDHARNESS_BASE_TAG}} - image_name: cloud-harness/workflows-send-result-event - title: Workflows send result event - working_directory: ./applications/workflows/tasks/send-result-event + image_name: cloud-harness/samples-sum + title: Samples sum + working_directory: ./applications/samples/tasks/sum tags: - - '${{WORKFLOWS_SEND_RESULT_EVENT_TAG}}' + - '${{SAMPLES_SUM_TAG}}' - '${{DEPLOYMENT_PUBLISH_TAG}}-dev' - '${{CF_BRANCH_TAG_NORMALIZED_LOWER_CASE}}' when: condition: any: - buildDoesNotExist: includes('${{WORKFLOWS_SEND_RESULT_EVENT_TAG_EXISTS}}', - '{{WORKFLOWS_SEND_RESULT_EVENT_TAG_EXISTS}}') == true - forceNoCache: includes('${{WORKFLOWS_SEND_RESULT_EVENT_TAG_FORCE_BUILD}}', - '{{WORKFLOWS_SEND_RESULT_EVENT_TAG_FORCE_BUILD}}') == false - test-api: + buildDoesNotExist: includes('${{SAMPLES_SUM_TAG_EXISTS}}', '{{SAMPLES_SUM_TAG_EXISTS}}') + == true + forceNoCache: includes('${{SAMPLES_SUM_TAG_FORCE_BUILD}}', '{{SAMPLES_SUM_TAG_FORCE_BUILD}}') + == false + samples-secret: type: build stage: build - dockerfile: test/test-api/Dockerfile + dockerfile: Dockerfile registry: '${{CODEFRESH_REGISTRY}}' buildkit: true build_arguments: - NOCACHE=${{CF_BUILD_ID}} - CLOUDHARNESS_BASE=${{REGISTRY}}/cloud-harness/cloudharness-base:${{CLOUDHARNESS_BASE_TAG}} - image_name: cloud-harness/test-api - title: Test api - working_directory: ./. + image_name: cloud-harness/samples-secret + title: Samples secret + working_directory: ./applications/samples/tasks/secret tags: - - '${{TEST_API_TAG}}' + - '${{SAMPLES_SECRET_TAG}}' - '${{DEPLOYMENT_PUBLISH_TAG}}-dev' - '${{CF_BRANCH_TAG_NORMALIZED_LOWER_CASE}}' - - latest when: condition: any: - buildDoesNotExist: includes('${{TEST_API_TAG_EXISTS}}', '{{TEST_API_TAG_EXISTS}}') + buildDoesNotExist: includes('${{SAMPLES_SECRET_TAG_EXISTS}}', '{{SAMPLES_SECRET_TAG_EXISTS}}') == true - forceNoCache: includes('${{TEST_API_TAG_FORCE_BUILD}}', '{{TEST_API_TAG_FORCE_BUILD}}') + forceNoCache: includes('${{SAMPLES_SECRET_TAG_FORCE_BUILD}}', '{{SAMPLES_SECRET_TAG_FORCE_BUILD}}') == false - samples-secret: + workflows-send-result-event: type: build stage: build dockerfile: Dockerfile @@ -280,21 +269,21 @@ steps: build_arguments: - NOCACHE=${{CF_BUILD_ID}} - CLOUDHARNESS_BASE=${{REGISTRY}}/cloud-harness/cloudharness-base:${{CLOUDHARNESS_BASE_TAG}} - image_name: cloud-harness/samples-secret - title: Samples secret - working_directory: ./applications/samples/tasks/secret + image_name: cloud-harness/workflows-send-result-event + title: Workflows send result event + working_directory: ./applications/workflows/tasks/send-result-event tags: - - '${{SAMPLES_SECRET_TAG}}' + - '${{WORKFLOWS_SEND_RESULT_EVENT_TAG}}' - '${{DEPLOYMENT_PUBLISH_TAG}}-dev' - '${{CF_BRANCH_TAG_NORMALIZED_LOWER_CASE}}' when: condition: any: - buildDoesNotExist: includes('${{SAMPLES_SECRET_TAG_EXISTS}}', '{{SAMPLES_SECRET_TAG_EXISTS}}') - == true - forceNoCache: includes('${{SAMPLES_SECRET_TAG_FORCE_BUILD}}', '{{SAMPLES_SECRET_TAG_FORCE_BUILD}}') - == false - cloudharness-flask: + buildDoesNotExist: includes('${{WORKFLOWS_SEND_RESULT_EVENT_TAG_EXISTS}}', + '{{WORKFLOWS_SEND_RESULT_EVENT_TAG_EXISTS}}') == true + forceNoCache: includes('${{WORKFLOWS_SEND_RESULT_EVENT_TAG_FORCE_BUILD}}', + '{{WORKFLOWS_SEND_RESULT_EVENT_TAG_FORCE_BUILD}}') == false + jupyterhub: type: build stage: build dockerfile: Dockerfile @@ -303,21 +292,21 @@ steps: build_arguments: - NOCACHE=${{CF_BUILD_ID}} - CLOUDHARNESS_BASE=${{REGISTRY}}/cloud-harness/cloudharness-base:${{CLOUDHARNESS_BASE_TAG}} - image_name: cloud-harness/cloudharness-flask - title: Cloudharness flask - working_directory: ./infrastructure/common-images/cloudharness-flask + image_name: cloud-harness/jupyterhub + title: Jupyterhub + working_directory: ./applications/jupyterhub tags: - - '${{CLOUDHARNESS_FLASK_TAG}}' + - '${{JUPYTERHUB_TAG}}' - '${{DEPLOYMENT_PUBLISH_TAG}}-dev' - '${{CF_BRANCH_TAG_NORMALIZED_LOWER_CASE}}' when: condition: any: - buildDoesNotExist: includes('${{CLOUDHARNESS_FLASK_TAG_EXISTS}}', '{{CLOUDHARNESS_FLASK_TAG_EXISTS}}') + buildDoesNotExist: includes('${{JUPYTERHUB_TAG_EXISTS}}', '{{JUPYTERHUB_TAG_EXISTS}}') == true - forceNoCache: includes('${{CLOUDHARNESS_FLASK_TAG_FORCE_BUILD}}', '{{CLOUDHARNESS_FLASK_TAG_FORCE_BUILD}}') + forceNoCache: includes('${{JUPYTERHUB_TAG_FORCE_BUILD}}', '{{JUPYTERHUB_TAG_FORCE_BUILD}}') == false - samples-sum: + workflows-notify-queue: type: build stage: build dockerfile: Dockerfile @@ -326,49 +315,49 @@ steps: build_arguments: - NOCACHE=${{CF_BUILD_ID}} - CLOUDHARNESS_BASE=${{REGISTRY}}/cloud-harness/cloudharness-base:${{CLOUDHARNESS_BASE_TAG}} - image_name: cloud-harness/samples-sum - title: Samples sum - working_directory: ./applications/samples/tasks/sum + image_name: cloud-harness/workflows-notify-queue + title: Workflows notify queue + working_directory: ./applications/workflows/tasks/notify-queue tags: - - '${{SAMPLES_SUM_TAG}}' + - '${{WORKFLOWS_NOTIFY_QUEUE_TAG}}' - '${{DEPLOYMENT_PUBLISH_TAG}}-dev' - '${{CF_BRANCH_TAG_NORMALIZED_LOWER_CASE}}' when: condition: any: - buildDoesNotExist: includes('${{SAMPLES_SUM_TAG_EXISTS}}', '{{SAMPLES_SUM_TAG_EXISTS}}') - == true - forceNoCache: includes('${{SAMPLES_SUM_TAG_FORCE_BUILD}}', '{{SAMPLES_SUM_TAG_FORCE_BUILD}}') - == false - title: Build parallel step 2 - build_application_images_2: - type: parallel - stage: build - steps: - samples: + buildDoesNotExist: includes('${{WORKFLOWS_NOTIFY_QUEUE_TAG_EXISTS}}', + '{{WORKFLOWS_NOTIFY_QUEUE_TAG_EXISTS}}') == true + forceNoCache: includes('${{WORKFLOWS_NOTIFY_QUEUE_TAG_FORCE_BUILD}}', + '{{WORKFLOWS_NOTIFY_QUEUE_TAG_FORCE_BUILD}}') == false + test-api: type: build stage: build - dockerfile: Dockerfile + dockerfile: test/test-api/Dockerfile registry: '${{CODEFRESH_REGISTRY}}' buildkit: true build_arguments: - NOCACHE=${{CF_BUILD_ID}} - - CLOUDHARNESS_FRONTEND_BUILD=${{REGISTRY}}/cloud-harness/cloudharness-frontend-build:${{CLOUDHARNESS_FRONTEND_BUILD_TAG}} - - CLOUDHARNESS_FLASK=${{REGISTRY}}/cloud-harness/cloudharness-flask:${{CLOUDHARNESS_FLASK_TAG}} - image_name: cloud-harness/samples - title: Samples - working_directory: ./applications/samples + - CLOUDHARNESS_BASE=${{REGISTRY}}/cloud-harness/cloudharness-base:${{CLOUDHARNESS_BASE_TAG}} + image_name: cloud-harness/test-api + title: Test api + working_directory: ./. tags: - - '${{SAMPLES_TAG}}' + - '${{TEST_API_TAG}}' - '${{DEPLOYMENT_PUBLISH_TAG}}-dev' - '${{CF_BRANCH_TAG_NORMALIZED_LOWER_CASE}}' + - latest when: condition: any: - buildDoesNotExist: includes('${{SAMPLES_TAG_EXISTS}}', '{{SAMPLES_TAG_EXISTS}}') + buildDoesNotExist: includes('${{TEST_API_TAG_EXISTS}}', '{{TEST_API_TAG_EXISTS}}') == true - forceNoCache: includes('${{SAMPLES_TAG_FORCE_BUILD}}', '{{SAMPLES_TAG_FORCE_BUILD}}') + forceNoCache: includes('${{TEST_API_TAG_FORCE_BUILD}}', '{{TEST_API_TAG_FORCE_BUILD}}') == false + title: Build parallel step 2 + build_application_images_2: + type: parallel + stage: build + steps: common: type: build stage: build @@ -415,6 +404,30 @@ steps: == true forceNoCache: includes('${{VOLUMEMANAGER_TAG_FORCE_BUILD}}', '{{VOLUMEMANAGER_TAG_FORCE_BUILD}}') == false + samples: + type: build + stage: build + dockerfile: Dockerfile + registry: '${{CODEFRESH_REGISTRY}}' + buildkit: true + build_arguments: + - NOCACHE=${{CF_BUILD_ID}} + - CLOUDHARNESS_FRONTEND_BUILD=${{REGISTRY}}/cloud-harness/cloudharness-frontend-build:${{CLOUDHARNESS_FRONTEND_BUILD_TAG}} + - CLOUDHARNESS_FLASK=${{REGISTRY}}/cloud-harness/cloudharness-flask:${{CLOUDHARNESS_FLASK_TAG}} + image_name: cloud-harness/samples + title: Samples + working_directory: ./applications/samples + tags: + - '${{SAMPLES_TAG}}' + - '${{DEPLOYMENT_PUBLISH_TAG}}-dev' + - '${{CF_BRANCH_TAG_NORMALIZED_LOWER_CASE}}' + when: + condition: + any: + buildDoesNotExist: includes('${{SAMPLES_TAG_EXISTS}}', '{{SAMPLES_TAG_EXISTS}}') + == true + forceNoCache: includes('${{SAMPLES_TAG_FORCE_BUILD}}', '{{SAMPLES_TAG_FORCE_BUILD}}') + == false workflows: type: build stage: build @@ -472,13 +485,13 @@ steps: commands: - kubectl config use-context ${{CLUSTER_NAME}} - kubectl config set-context --current --namespace=test-${{NAMESPACE_BASENAME}} - - kubectl rollout status deployment/volumemanager - - kubectl rollout status deployment/workflows - - kubectl rollout status deployment/samples - - kubectl rollout status deployment/samples-gk - kubectl rollout status deployment/accounts - kubectl rollout status deployment/common - kubectl rollout status deployment/argo-gk + - kubectl rollout status deployment/volumemanager + - kubectl rollout status deployment/samples + - kubectl rollout status deployment/samples-gk + - kubectl rollout status deployment/workflows - sleep 60 tests_api: stage: qa @@ -489,18 +502,16 @@ steps: commands: - echo $APP_NAME scale: - volumemanager_api_test: - title: volumemanager api test + workflows_api_test: + title: workflows api test volumes: - - '${{CF_REPO_NAME}}/applications/volumemanager:/home/test' + - '${{CF_REPO_NAME}}/applications/workflows:/home/test' - '${{CF_REPO_NAME}}/deployment/helm/values.yaml:/opt/cloudharness/resources/allvalues.yaml' environment: - - APP_URL=https://volumemanager.${{DOMAIN}}/api - - USERNAME=volumes@testuser.com - - PASSWORD=test + - APP_URL=https://workflows.${{DOMAIN}}/api + - SCHEMATHESIS_HOOKS=cloudharness_test.apitest_init commands: - - st --pre-run cloudharness_test.apitest_init run api/openapi.yaml --base-url - https://volumemanager.${{DOMAIN}}/api -c all + - harness-test $CH_VALUES_PATH -i workflows -a samples_api_test: title: samples api test volumes: @@ -510,33 +521,32 @@ steps: - APP_URL=https://samples.${{DOMAIN}}/api - USERNAME=sample@testuser.com - PASSWORD=test + - SCHEMATHESIS_HOOKS=cloudharness_test.apitest_init commands: - - st --pre-run cloudharness_test.apitest_init run api/openapi.yaml --base-url - https://samples.${{DOMAIN}}/api -c all --skip-deprecated-operations --exclude-operation-id=submit_sync - --exclude-operation-id=submit_sync_with_results --exclude-operation-id=error - --hypothesis-suppress-health-check=too_slow --hypothesis-deadline=180000 - --request-timeout=180000 --hypothesis-max-examples=2 --show-trace --exclude-checks=ignored_auth + - harness-test $CH_VALUES_PATH -i samples -a - pytest -v test/api - common_api_test: - title: common api test + volumemanager_api_test: + title: volumemanager api test volumes: - - '${{CF_REPO_NAME}}/applications/common:/home/test' + - '${{CF_REPO_NAME}}/applications/volumemanager:/home/test' - '${{CF_REPO_NAME}}/deployment/helm/values.yaml:/opt/cloudharness/resources/allvalues.yaml' environment: - - APP_URL=https://common.${{DOMAIN}}/api + - APP_URL=https://volumemanager.${{DOMAIN}}/api + - USERNAME=volumes@testuser.com + - PASSWORD=test + - SCHEMATHESIS_HOOKS=cloudharness_test.apitest_init commands: - - st --pre-run cloudharness_test.apitest_init run api/openapi.yaml --base-url - https://common.${{DOMAIN}}/api -c all - workflows_api_test: - title: workflows api test + - harness-test $CH_VALUES_PATH -i volumemanager -a + common_api_test: + title: common api test volumes: - - '${{CF_REPO_NAME}}/applications/workflows:/home/test' + - '${{CF_REPO_NAME}}/applications/common:/home/test' - '${{CF_REPO_NAME}}/deployment/helm/values.yaml:/opt/cloudharness/resources/allvalues.yaml' environment: - - APP_URL=https://workflows.${{DOMAIN}}/api + - APP_URL=https://common.${{DOMAIN}}/api + - SCHEMATHESIS_HOOKS=cloudharness_test.apitest_init commands: - - st --pre-run cloudharness_test.apitest_init run api/openapi.yaml --base-url - https://workflows.${{DOMAIN}}/api -c all + - harness-test $CH_VALUES_PATH -i common -a hooks: on_fail: exec: @@ -553,12 +563,6 @@ steps: - npx puppeteer browsers install chrome - yarn test scale: - jupyterhub_e2e_test: - title: jupyterhub e2e test - volumes: - - '${{CF_REPO_NAME}}/applications/jupyterhub/test/e2e:/home/test/__tests__/jupyterhub' - environment: - - APP_URL=https://hub.${{DOMAIN}} samples_e2e_test: title: samples e2e test volumes: @@ -567,6 +571,14 @@ steps: - APP_URL=https://samples.${{DOMAIN}} - USERNAME=sample@testuser.com - PASSWORD=test + - SCHEMATHESIS_HOOKS=cloudharness_test.apitest_init + jupyterhub_e2e_test: + title: jupyterhub e2e test + volumes: + - '${{CF_REPO_NAME}}/applications/jupyterhub/test/e2e:/home/test/__tests__/jupyterhub' + environment: + - APP_URL=https://hub.${{DOMAIN}} + - SCHEMATHESIS_HOOKS=cloudharness_test.apitest_init hooks: on_fail: exec: diff --git a/docs/testing.md b/docs/testing.md index 0a941f901..fab39c151 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -78,12 +78,10 @@ harness: checks: - all runParams: - - "--skip-deprecated-operations" - - "--hypothesis-suppress-health-check=too_slow" - - "--hypothesis-deadline=60000" + - "--exclude-deprecated" + - "--suppress-health-check=too_slow" - "--request-timeout=60000" - - "--hypothesis-max-examples=2" - - "--show-trace" + - "--max-examples=2" ``` See [the model documentation](model/ApiTestsConfig.md) for more insights about test parameters. diff --git a/libraries/cloudharness-utils/cloudharness_utils/testing/api.py b/libraries/cloudharness-utils/cloudharness_utils/testing/api.py index 6506804bc..7a62240aa 100644 --- a/libraries/cloudharness-utils/cloudharness_utils/testing/api.py +++ b/libraries/cloudharness-utils/cloudharness_utils/testing/api.py @@ -1,4 +1,5 @@ import os +import logging from ruamel.yaml import YAML @@ -11,18 +12,46 @@ def get_api_filename(app_dir): return os.path.join(app_dir, "api", "openapi.yaml") -def get_schemathesis_command(api_filename, app_config: ApplicationHarnessConfig, app_domain: str): - return ["st", "--pre-run", "cloudharness_test.apitest_init", "run", api_filename, *get_schemathesis_params(app_config, app_domain)] +def get_schemathesis_command(api_filename, app_config: ApplicationHarnessConfig, app_domain: str, app_env: dict | None = None): + """ + Build the schemathesis command for running API tests. + Extended to support runtime authentication header generation directly in the command instead of relying on hooks. + """ + return ["st", "run", api_filename, *get_schemathesis_params(app_config, app_domain, app_env)] -def get_schemathesis_params(app_config: ApplicationHarnessConfig, app_domain: str): - params = ["--base-url", app_domain] + +def _get_auth_headers(app_env: dict): + from cloudharness.auth import get_token + + """Return schemathesis CLI flags for auth.""" + if not app_env: + return [] + username = app_env.get("USERNAME") + password = app_env.get("PASSWORD") + if not (username and password): + return [] + try: + token = get_token(username, password) + if not token: + logging.warning("Token retrieval returned empty token for user %s", username) + return [] + return ["--header", f"Authorization: Bearer {token}", "--header", f"Cookie: kc-access={token}"] + except Exception as e: + logging.warning("Failed to retrieve bearer token for user %s: %s", username, e) + return [] + + +def get_schemathesis_params(app_config: ApplicationHarnessConfig, app_domain: str, app_env: dict | None = None): + params = ["--url", app_domain] api_config: ApiTestsConfig = app_config.test.api if api_config.checks: for c in api_config.checks: params += ["-c", c] - return [*params, *api_config.run_params] + params.extend(api_config.run_params) + params.extend(_get_auth_headers(app_env or {})) + return params def get_urls_from_api_file(api_filename): diff --git a/libraries/cloudharness-utils/cloudharness_utils/testing/util.py b/libraries/cloudharness-utils/cloudharness_utils/testing/util.py index 7bb665584..f89194f83 100644 --- a/libraries/cloudharness-utils/cloudharness_utils/testing/util.py +++ b/libraries/cloudharness-utils/cloudharness_utils/testing/util.py @@ -18,8 +18,14 @@ def get_app_environment(app_config: ApplicationHarnessConfig, app_domain, use_lo password = get_user_password(main_user) my_env["USERNAME"] = main_user.username my_env["PASSWORD"] = password + test_config: ApplicationTestConfig = app_config.test + api_config = test_config.api e2e_config: E2ETestsConfig = test_config.e2e + + if api_config.enabled or api_config.autotest: + my_env["SCHEMATHESIS_HOOKS"] = "cloudharness_test.apitest_init" + if not e2e_config.smoketest: my_env["SKIP_SMOKETEST"] = "true" if e2e_config.ignore_console_errors: diff --git a/test/test-api/.dockerignore b/test/test-api/.dockerignore new file mode 100644 index 000000000..9c06dbcc3 --- /dev/null +++ b/test/test-api/.dockerignore @@ -0,0 +1,23 @@ +**/node_modules +.tox +docs +/applications +/infrastructure +/blueprint +/test +.github +.git +.vscode +/deployment +skaffold.yaml +*.egg-info +__pycache__ +.hypothesis +.coverage +.pytest_cache +/application-templates +/deployment-configuration +/cloud-harness +.openapi-generator +docker-compose.yaml +.history \ No newline at end of file diff --git a/test/test-api/Dockerfile b/test/test-api/Dockerfile index 06f439420..6f2dccba4 100644 --- a/test/test-api/Dockerfile +++ b/test/test-api/Dockerfile @@ -2,14 +2,19 @@ ARG CLOUDHARNESS_BASE FROM $CLOUDHARNESS_BASE +ENV CH_VALUES_PATH=/codefresh/volume/cloud-harness + +# Install cloudharness-utils first (required by other packages) COPY libraries/cloudharness-utils/requirements.txt /libraries/cloudharness-utils/requirements.txt RUN pip install -r /libraries/cloudharness-utils/requirements.txt --no-cache-dir -COPY tools/cloudharness-test/requirements.txt /tools/cloudharness-test/requirements.txt -RUN pip install -r /tools/cloudharness-test/requirements.txt --no-cache-dir - COPY libraries/cloudharness-utils /libraries/cloudharness-utils RUN pip install -e /libraries/cloudharness-utils +# Install deployment-cli-tools (depends on cloudharness-utils) +COPY tools/deployment-cli-tools /tools/deployment-cli-tools +RUN pip install -e /tools/deployment-cli-tools + +# Install cloudharness-test (depends on both above) COPY tools/cloudharness-test /tools/cloudharness-test RUN pip install -e /tools/cloudharness-test \ No newline at end of file diff --git a/tools/cloudharness-test/cloudharness_test/api.py b/tools/cloudharness-test/cloudharness_test/api.py index af80f6bd3..09ed8e2db 100644 --- a/tools/cloudharness-test/cloudharness_test/api.py +++ b/tools/cloudharness-test/cloudharness_test/api.py @@ -69,7 +69,7 @@ def run_api_tests(root_paths, helm_values: HarnessMainConfig, base_domain, inclu if api_config.autotest: logging.info("Running auto api tests") - cmd = get_schemathesis_command(api_filename, app_config, app_domain) + cmd = get_schemathesis_command(api_filename, app_config, app_domain, app_env) logging.info("Running: %s", " ".join(cmd)) result = subprocess.run(cmd, env=app_env, cwd=app_dir) diff --git a/tools/cloudharness-test/cloudharness_test/apitest_auth_hooks.py b/tools/cloudharness-test/cloudharness_test/apitest_auth_hooks.py new file mode 100644 index 000000000..fe5e14bca --- /dev/null +++ b/tools/cloudharness-test/cloudharness_test/apitest_auth_hooks.py @@ -0,0 +1,63 @@ +import os +import logging +import schemathesis as st +from cloudharness.auth import get_token + + +@st.auth() +class TokenAuth: + """ + Schemathesis authentication hook that retrieves a bearer token + using Keycloak credentials and sets both Authorization header and Cookie. + + Requires USERNAME and PASSWORD environment variables to be set. + """ + + def get(self, case, ctx): + """ + Retrieve the authentication token using username and password from environment. + + Args: + case: Schemathesis test case + ctx: Schemathesis hook context + + Returns: + str: The bearer token + + Raises: + ValueError: If USERNAME or PASSWORD environment variables are not set + Exception: If token retrieval fails + """ + username = os.environ.get("USERNAME") + password = os.environ.get("PASSWORD") + + if not username or not password: + logging.warning("USERNAME and/or PASSWORD environment variables not set. Skipping authentication.") + return None + + try: + token = get_token(username, password) + if not token: + logging.warning("Token retrieval returned empty token for user %s", username) + return None + logging.info("Successfully retrieved authentication token for user %s", username) + return token + except Exception as e: + logging.error("Failed to retrieve bearer token for user %s: %s", username, e) + raise + + def set(self, case, data, context): + """ + Set the authentication token in the request headers and cookies. + + Args: + case: Schemathesis test case + data: The authentication token + context: Schemathesis hook context + """ + if not data: + return + + case.headers = case.headers or {} + case.headers["Authorization"] = f"Bearer {data}" + case.headers["Cookie"] = f"kc-access={data}" diff --git a/tools/cloudharness-test/cloudharness_test/apitest_init.py b/tools/cloudharness-test/cloudharness_test/apitest_init.py index 64d3544e1..f246dc0e8 100644 --- a/tools/cloudharness-test/cloudharness_test/apitest_init.py +++ b/tools/cloudharness-test/cloudharness_test/apitest_init.py @@ -4,9 +4,6 @@ import schemathesis as st from schemathesis.hooks import HookContext -from cloudharness.auth import get_token - -st.experimental.OPEN_API_3_1.enable() if "APP_URL" or "APP_SCHEMA_FILE" in os.environ: app_schema = os.environ.get("APP_SCHEMA_FILE", None) @@ -22,9 +19,9 @@ # First, attempt to load the local file if provided if app_schema: try: - schema = st.from_file(app_schema) + schema = st.openapi.from_file(app_schema) logging.info("Successfully loaded schema from local file: %s", app_schema) - except st.exceptions.SchemaError: + except st.errors.LoaderError: logging.exception("The local schema file %s cannot be loaded. Attempting loading from URL", app_schema) # If no schema from file, then loop over URL candidates @@ -36,44 +33,16 @@ for candidate in candidates: try: logging.info("Attempting to load schema from URI: %s", candidate) - schema = st.from_uri(candidate) + schema = st.openapi.from_url(candidate) logging.info("Successfully loaded schema from %s", candidate) break # Exit loop on successful load - except st.exceptions.SchemaError as e: + except st.errors.LoaderError as e: logging.warning("Failed to load schema from %s: %s", candidate, e) except Exception as e: logging.error("Unexpected error when loading schema from %s: %s", candidate, e) if not schema: raise Exception("Cannot setup API tests: No valid schema found. Check your deployment and configuration.") - if "USERNAME" in os.environ and "PASSWORD" in os.environ: - logging.info("Setting token from username and password") - - @st.auth.register() - class TokenAuth: - def get(self, context): - - username = os.environ["USERNAME"] - password = os.environ["PASSWORD"] - - return get_token(username, password) - - def set(self, case, data, context): - case.headers = case.headers or {} - case.headers["Authorization"] = f"Bearer {data}" - case.headers["Cookie"] = f"kc-access={data}" - else: - @st.auth.register() - class TokenAuth: - def get(self, context): - - return "" - - def set(self, case, data, context): - case.headers = case.headers or {} - case.headers["Authorization"] = f"Bearer {data}" - case.headers["Cookie"] = f"kc-access={data}" - UNSAFE_VALUES = ("%", ) @st.hook diff --git a/tools/cloudharness-test/harness-test b/tools/cloudharness-test/harness-test index fa69ddf88..770ea03dd 100644 --- a/tools/cloudharness-test/harness-test +++ b/tools/cloudharness-test/harness-test @@ -17,8 +17,6 @@ yaml = YAML(typ='safe') HERE = os.path.dirname(os.path.realpath(__file__)).replace(os.path.sep, '/') ROOT = os.path.dirname(os.path.dirname(HERE)).replace(os.path.sep, '/') -HELM_DIR = os.path.join('./deployment/helm') - if __name__ == "__main__": import argparse @@ -35,8 +33,8 @@ if __name__ == "__main__": parser.add_argument('-i', '--include', dest='include', action="append", default=[], help='Specify the applications to include and exclude the rest. ' 'Omit to test all application included by harness-deployment.') - parser.add_argument('-c', '--helm-chart', dest='helm_chart_path', action="store", default=HELM_DIR, - help=f'Specify helm chart base path (default `{HELM_DIR}`') + parser.add_argument('-c', '--helm-chart', dest='helm_chart_path', action="store", default=None, + help='Specify helm chart base path (default: deployment/helm relative to first path)') parser.add_argument('-e', '--e2e', dest='run_e2e', action="store_const", default=None, const=True, help=f'Run only end to end tests (default: run both api and end to end tests') parser.add_argument('-a', '--api', dest='run_api', action="store_const", default=None, const=True, @@ -47,12 +45,14 @@ if __name__ == "__main__": args, unknown = parser.parse_known_args(sys.argv[1:]) root_paths = [os.path.join(os.getcwd(), path) for path in args.paths] + HELM_DIR = os.path.join(root_paths[0], 'deployment/helm') if unknown: print('There are unknown args. Make sure to call the script with the accepted args. Try --help') print(f'unknown: {unknown}') else: - helm_values_path = os.path.join(HELM_DIR, 'values.yaml') + helm_chart_dir = args.helm_chart_path or HELM_DIR + helm_values_path = os.path.join(helm_chart_dir, 'values.yaml') if not os.path.exists(helm_values_path): logging.error( "Could not find helm installation. Have you run harness-deployment already?") diff --git a/tools/cloudharness-test/requirements.txt b/tools/cloudharness-test/requirements.txt index a9240f4e4..8ff49955a 100644 --- a/tools/cloudharness-test/requirements.txt +++ b/tools/cloudharness-test/requirements.txt @@ -1,3 +1,3 @@ openapi-spec-validator==0.5.1 -schemathesis<4.0.0 +schemathesis>=4.3.5 cloudharness_model diff --git a/tools/cloudharness-test/setup.py b/tools/cloudharness-test/setup.py index 01064fa88..4c375643b 100644 --- a/tools/cloudharness-test/setup.py +++ b/tools/cloudharness-test/setup.py @@ -24,7 +24,7 @@ 'requests', 'cloudharness_model', 'cloudharness', - 'schemathesis<4.0.0', + 'schemathesis>=4.3.5', ] diff --git a/tools/deployment-cli-tools/ch_cli_tools/codefresh.py b/tools/deployment-cli-tools/ch_cli_tools/codefresh.py index e644470ab..85dd07b13 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/codefresh.py +++ b/tools/deployment-cli-tools/ch_cli_tools/codefresh.py @@ -13,7 +13,7 @@ from .configurationgenerator import KEY_APPS, KEY_TASK_IMAGES, KEY_TEST_IMAGES from .utils import check_image_exists_in_registry, clean_image_name, find_dockerfiles_paths, get_app_relative_to_base_path, guess_build_dependencies_from_dockerfile, \ get_image_name, get_template, dict_merge, app_name_from_path, clean_path -from cloudharness_utils.testing.api import get_api_filename, get_schemathesis_command, get_urls_from_api_file +from cloudharness_utils.testing.api import get_api_filename, get_urls_from_api_file logging.getLogger().setLevel(logging.INFO) @@ -450,13 +450,19 @@ def codefresh_template_spec(template_path, **kwargs): def api_tests_commands(app_config: ApplicationHarnessConfig, run_custom_tests, api_url): + """Return commands to execute API tests for Codefresh pipeline. + + Harness-test is now the unified entrypoint; it internally builds schemathesis command with headers. + We invoke harness-test with -a flag targeting only API tests & include specific app. + Custom pytest tests run separately inside the same container. + """ api_config: ApiTestsConfig = app_config.test.api commands = [] + app_name = app_config.name if api_config.autotest: - commands.append(" ".join(get_schemathesis_command( - get_api_filename(""), app_config, api_url))) + commands.append(f"harness-test $CH_VALUES_PATH -i {app_name} -a") if run_custom_tests: - commands.append(f"pytest -v test/api") + commands.append("pytest -v test/api") return commands diff --git a/tools/deployment-cli-tools/tests/test_codefresh.py b/tools/deployment-cli-tools/tests/test_codefresh.py index b1b8e1c52..740ebad9e 100644 --- a/tools/deployment-cli-tools/tests/test_codefresh.py +++ b/tools/deployment-cli-tools/tests/test_codefresh.py @@ -240,12 +240,14 @@ def test_create_codefresh_configuration_tests(): assert len(test_step["commands"]) == 2, "Both default and custom api tests should be run" - st_cmd = test_step["commands"][0] - assert "--pre-run cloudharness_test.apitest_init" in st_cmd, "Prerun hook must be specified in schemathesis command" - assert "api/openapi.yaml" in st_cmd, "Openapi file must be passed to the schemathesis command" - - assert "-c all" in st_cmd, "Default check loaded is `all` on schemathesis command" - assert "--hypothesis-deadline=" in st_cmd, "Custom parameters are loaded from values.yaml" + # First command should run harness-test with api flag, specific app, and helm chart path + harness_test_cmd = test_step["commands"][0] + assert "harness-test" in harness_test_cmd, "harness-test should be used for api tests" + assert "-c $CH_VALUES_PATH/deployment/helm" in harness_test_cmd, "Helm chart path should be specified with -c flag" + assert "-i samples" in harness_test_cmd, "App name should be included with -i flag" + assert "-a" in harness_test_cmd, "API tests should be run with -a flag" + # Second command should run custom pytest tests + assert "pytest -v test/api" in test_step["commands"][1], "Custom pytest tests should be run" test_step = api_steps["common_api_test"] for volume in test_step["volumes"]: