Update ANYRUN.py

.circleci/config.yml

@@ -1,78 +1,217 @@
-version: 2.0
+version: 2
 jobs:
   build:
     docker:
-      - image: ronykoz/content-build-node911:latest
+      - image: devdemisto/content-build-2and3:2.0.0.877  # disable-secrets-detection
+    resource_class: medium+
     environment:
-      CONTENT_VERSION: "18.11.2"
-      GIT_SHA1: "3ac083b891d2f146ffbad6c7ce0a3be9e4f94b92"
+      CONTENT_VERSION: "19.8.2"
+      SERVER_VERSION: "4.5.0"
+      GIT_SHA1: "177b7e574fc936df19170c13650ea2581e4920f6" # guardrails-disable-line disable-secrets-detection
     steps:
       - checkout
+      - setup_remote_docker
       - run:
           name: Prepare Environment
+          when: always
           command: |
               echo 'export CIRCLE_ARTIFACTS="/home/circleci/project/artifacts"' >> $BASH_ENV
+              echo 'export PATH="/home/circleci/.pyenv/shims:/home/circleci/.local/bin:/home/circleci/.pyenv/bin:${PATH}"' >> $BASH_ENV # disable-secrets-detection
+              echo 'export PYTHONPATH="/home/circleci/project:${PYTHONPATH}"' >> $BASH_ENV
+              echo "=== sourcing $BASH_ENV ==="
               source $BASH_ENV
               sudo mkdir -p -m 777 $CIRCLE_ARTIFACTS
-      - run:
-          name: Install dependencies
-          command: |
               chmod +x ./Tests/scripts/*
+              chmod +x ./Tests/lastest_server_build_scripts/*
+              pyenv versions
+              python --version
+              python3 --version
+              echo "Parameters: NIGHTLY: $NIGHTLY, NON_AMI_RUN: $NON_AMI_RUN, SERVER_BRANCH_NAME: $SERVER_BRANCH_NAME"
       - add_ssh_keys:
           fingerprints:
-              - "02:df:a5:6a:53:9a:f5:5d:bd:a6:fc:b2:db:9b:c9:47"
+              - "02:df:a5:6a:53:9a:f5:5d:bd:a6:fc:b2:db:9b:c9:47" # disable-secrets-detection
+              - "f5:25:6a:e5:ac:4b:84:fb:60:54:14:82:f1:e9:6c:f9" # disable-secrets-detection
+      - run:
+          name: Create ID Set
+          when: always
+          command: |
+            python ./Tests/scripts/update_id_set.py -r
+      - run:
+          name: Infrastucture testing
+          when: always
+          command: |
+            pytest ./Tests/scripts/hook_validations/tests/ -v
+            pytest ./Tests/scripts/infrastructure_tests/ -v
+            pytest ./Tests/scripts/test_configure_tests.py -v
       - run:
           name: Validate Files and Yaml
+          when: always
           command: |
-              python ./Tests/scripts/validate_files_structure.py -c true
+              # Run flake8 on all excluding Integraions and Scripts (they will be handled in linting)
+              ./Tests/scripts/pyflake.sh *.py
+              find . -maxdepth 1 -type d -not \( -path . -o -path ./Integrations -o -path ./Scripts -o -path ./Beta_Integrations  \) | xargs ./Tests/scripts/pyflake.sh
+
+              [ -n "${BACKWARD_COMPATIBILITY}" ] && CHECK_BACKWARD=false || CHECK_BACKWARD=true
+              python ./Tests/scripts/validate_files.py -c true -b $CHECK_BACKWARD
       - run:
           name: Configure Test Filter
+          when: always
           command: |
               [ -n "${NIGHTLY}" ] && IS_NIGHTLY=true || IS_NIGHTLY=false
               python ./Tests/scripts/configure_tests.py -n $IS_NIGHTLY
+      - run:
+          name: Spell Checks
+          command: |
+            python ./Tests/scripts/circleci_spell_checker.py $CIRCLE_BRANCH
       - run:
           name: Build Content Descriptor
-          command: ./setContentDescriptor.sh $CIRCLE_BUILD_NUM $GIT_SHA1 $CONTENT_VERSION
+          when: always
+          command: python release_notes.py $CONTENT_VERSION $GIT_SHA1 $CIRCLE_BUILD_NUM $SERVER_VERSION $GITHUB_TOKEN
       - run:
           name: Common Server Documentation
+          when: always
           command: ./Documentation/commonServerDocs.sh
       - run:
           name: Create Content Artifacts
+          when: always
           command: python content_creator.py $CIRCLE_ARTIFACTS
       - store_artifacts:
           path: artifacts
           destination: artifacts
+      - run:
+          name: Run Unit Testing and Lint
+          when: always
+          command: SKIP_GIT_COMPARE_FILTER=${NIGHTLY} ./Tests/scripts/run_all_pkg_dev_tasks.sh
+      - run:
+          name: Download Artifacts
+          when: always
+          command: |
+            if ./Tests/scripts/is_ami.sh ;
+              then
+                echo "Using AMI - Not downloading artifacts"
+
+              else
+                ./Tests/scripts/server_get_artifact.sh $SERVER_CI_TOKEN
+            fi
       - run:
           name: Download Configuration
+          when: always
           command: |
-              ./Tests/scripts/download_demisto_conf.sh
+            if ./Tests/scripts/is_ami.sh ;
+              then
+                ./Tests/scripts/download_demisto_conf.sh
+
+              else
+                ./Tests/lastest_server_build_scripts/download_demisto_conf.sh
+            fi
       - run:
           name: Create Instance
+          when: always
           command: |
-              ./Tests/scripts/create_instance.sh instance.json
+            pip install --upgrade awscli
+            pip install ansible ansible-runner rsa paramiko boto3 requests
+
+            if ./Tests/scripts/is_ami.sh ;
+              then
+                if [ -n "${NIGHTLY}" ] ;
+                  then
+                    export IFRA_ENV_TYPE=Content-Master
+
+                  else
+                    export IFRA_ENV_TYPE=Content-Env
+                fi
+                python ./Tests/scripts/awsinstancetool/aws_instance_tool.py -envType $IFRA_ENV_TYPE
+
+              else
+                python ./Tests/scripts/awsinstancetool/aws_instance_tool.py -envType CustomBuild
+            fi
       - run:
           name: Setup Instance
+          when: always
+          command: |
+            if ./Tests/scripts/is_ami.sh ;
+              then
+                python ./Tests/scripts/run_content_installation.py
+                python ./Tests/scripts/wait_until_server_ready.py -c $(cat secret_conf_path) -v $CONTENT_VERSION
+
+              else
+                ./Tests/lastest_server_build_scripts/run_installer_on_instance.sh
+                python ./Tests/scripts/wait_until_server_ready.py -c $(cat secret_conf_path) -v $CONTENT_VERSION --non-ami
+            fi
+      - run:
+          name: Run Tests - Latest GA
+          shell: /bin/bash
+          when: always
+          command: |
+            if ./Tests/scripts/is_ami.sh ;
+              then
+                ./Tests/scripts/run_tests.sh "Demisto GA"
+
+              else
+                ./Tests/lastest_server_build_scripts/run_tests.sh
+            fi
+      - run:
+          name: Run Tests - One Before GA
+          shell: /bin/bash
+          when: always
+          command: |
+            if ./Tests/scripts/is_ami.sh ;
+              then
+                ./Tests/scripts/run_tests.sh "Demisto one before GA"
+
+              else
+                echo "Not AMI run, can't run on this version"
+            fi
+      - run:
+          name: Run Tests - Two Before GA
+          shell: /bin/bash
+          when: always
           command: |
-              ./Tests/scripts/run_installer_on_instance.sh
-              ./Tests/scripts/wait_until_server_ready.sh
+            if ./Tests/scripts/is_ami.sh ;
+              then
+                ./Tests/scripts/run_tests.sh "Demisto two before GA"
+
+              else
+                echo "Not AMI run, can't run on this version"
+            fi
       - run:
-          name: Run Tests
+          name: Run Tests - Server Master
           shell: /bin/bash
-          command: ./Tests/scripts/run_tests.sh
+          when: always
+          command: |
+            if ./Tests/scripts/is_ami.sh ;
+              then
+                ./Tests/scripts/run_tests.sh "Server Master"
+
+              else
+                echo "Not AMI run, can't run on this version"
+            fi
       - run:
           name: Slack Notifier
           shell: /bin/bash
           command: ./Tests/scripts/slack_notifier.sh
           when: always
+      - run:
+          name: Validate Docker Images
+          shell: /bin/bash
+          command: ./Tests/scripts/validate_docker_images.sh
+          when: always
       - run:
           name: Instance Test
           command: ./Tests/scripts/instance_test.sh
           when: always
       - run:
           name: Destroy Instances
-          command: ./Tests/scripts/destroy_instances.sh $CIRCLE_ARTIFACTS
+          command: python ./Tests/scripts/destroy_instances.py $CIRCLE_ARTIFACTS
           when: always
       - store_artifacts:
           path: artifacts
           destination: artifacts
           when: always
+
+
+workflows:
+  version: 2
+  commit:
+    jobs:
+      - build

.github/config.yml

@@ -0,0 +1,6 @@
+newPRWelcomeComment: >
+  Hi and welcome to the Demisto Content project!
+  Thank you and congrats on your first pull request, we will review it soon!
+  Until then you can check out our [documentation](https://github.com/demisto/content/tree/master/docs) for more details.
+  We would be thrilled to see you get involved in our [Slack DFIR community](https://go.demisto.com/join-our-slack-community) for discussions.
+  Hope you have a great time here :)

.github/pull_request_template.md

@@ -1,3 +1,5 @@
+<!-- REMINDER: THIS IS A PUBLIC REPO DO NOT POST HERE SECRETS/SENSITIVE DATA -->
+
 ## Status
 Ready/In Progress/In Hold(Reason for hold)
 
@@ -30,5 +32,11 @@ x.x.x
 - [ ] Documentation (with link to it)
 - [ ] Code Review
 
+## Dependencies
+Mention the dependencies of the entity you changed as given from the precommit hooks in checkboxes, and tick after tested them.
+- [ ] Dependency 1
+- [ ] Dependency 2
+- [ ] Dependency 3
+
 ## Additional changes
 Describe additional changes done, for example adding a function to common server.

.gitignore

@@ -1,3 +1,21 @@
 .DS_Store
 .idea
-_site
+.vscode
+_site
+TestData/EmailWithNonUnicodeAttachmentName.eml
+TestData/EmailWithNonUnicodeSubject.eml
+*.pyc
+.pytest_cache
+
+CommonServerPython.py
+!Scripts/CommonServerPython/CommonServerPython.py
+CommonServerUserPython.py
+demistomock.py
+Tests/filter_file.txt
+Tests/id_set.json
+.mypy_cache
+Scripts/*/*_unified.yml
+Integrations/*/*_unified.yml
+Beta_Integrations/*/*_unified.yml
+conftest.py
+!Tests/scripts/dev_envs/pytest/conftest.py

.guardrails/ignore

@@ -0,0 +1 @@
+Integrations/Active_Directory_Query/key.pem

.hooks/pre-commit

@@ -1,15 +1,37 @@
 #!/bin/bash
 
 # validating that each modified file has a valid schema, release notes, proper prefix & suffix
-echo "Validating files"
-python Tests/scripts/validate_files_structure.py
+echo "Validating files..."
+if [[ -z "${WINDIR}" ]]
+    then
+        PYTHONPATH="`pwd`:${PYTHONPATH}" python Tests/scripts/validate_files.py -t true
+    else
+        python Tests/scripts/validate_files.py
+fi
+
+RES=$?
+
+echo ""
+if [[ -n "$CONTENT_PRECOMMIT_RUN_DEV_TASKS" ]]; then
+    echo "Running content dev tasks (flake8, mypy, pylint, pytst) as env variable CONTENT_PRECOMMIT_RUN_DEV_TASKS is set."
+    ./Tests/scripts/run_all_pkg_dev_tasks.sh
+    RES=$(($RES + $?))
+else    
+    echo "Skipping running dev tasks (flake8, mypy, pylint, pytest). If you want to run this as part of the precommit hook"
+    echo 'set CONTENT_PRECOMMIT_RUN_DEV_TASKS=1. You can add the following line to ~/.zshrc:'
+    echo 'echo "export CONTENT_PRECOMMIT_RUN_DEV_TASKS=1" >> ~/.zshrc'
+    echo ""
+    echo 'Or if you want to manually run dev tasks: ./Tests/scripts/pkg_dev_test_tasks.py -d <integration/script dir>'
+    echo 'Example: ./Tests/scripts/pkg_dev_test_tasks.py -d Scripts/ParseEmailFiles'
+fi
 
-if [[ $? -ne 0 ]]
-then
+if [[ $RES -ne 0 ]]
+  then
     echo "Please fix the aforementioned errors and then commit again"
     exit 1
 fi
 
+
 # prevent push to master
 if [ -z "$1"  ]; then
     protected_branch='master'
@@ -19,3 +41,5 @@ if [ -z "$1"  ]; then
         exit 1
     fi
 fi
+
+echo ""