diff --git a/.circleci/config.yml b/.circleci/config.yml index 4de7cc8ad994..af28cb31bbb7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,78 +1,274 @@ -version: 2.0 +version: 2 jobs: build: docker: - - image: ronykoz/content-build-node911:latest + - image: devdemisto/content-build-2and3:2.0.0.2832 # disable-secrets-detection + resource_class: medium+ environment: - CONTENT_VERSION: "18.11.2" - GIT_SHA1: "3ac083b891d2f146ffbad6c7ce0a3be9e4f94b92" + CONTENT_VERSION: "19.10.1" + SERVER_VERSION: "5.0.0" + GIT_SHA1: "8c2c76794ce844ded59c4c5e9484858c7c3c16d3" # guardrails-disable-line disable-secrets-detection steps: + - run: + name: Look It's a New CircleCI Step + when: always + command: | + echo 'blah blah blah' + echo 'YAY go Content' - checkout + - setup_remote_docker - run: name: Prepare Environment + when: always command: | echo 'export CIRCLE_ARTIFACTS="/home/circleci/project/artifacts"' >> $BASH_ENV + echo 'export PATH="/home/circleci/.pyenv/shims:/home/circleci/.local/bin:/home/circleci/.pyenv/bin:${PATH}"' >> $BASH_ENV # disable-secrets-detection + echo 'export PYTHONPATH="/home/circleci/project:${PYTHONPATH}"' >> $BASH_ENV + echo "=== sourcing $BASH_ENV ===" source $BASH_ENV sudo mkdir -p -m 777 $CIRCLE_ARTIFACTS - - run: - name: Install dependencies - command: | chmod +x ./Tests/scripts/* + chmod +x ./Tests/lastest_server_build_scripts/* + pyenv versions + python --version + python3 --version + echo "Parameters: NIGHTLY: $NIGHTLY, NON_AMI_RUN: $NON_AMI_RUN, SERVER_BRANCH_NAME: $SERVER_BRANCH_NAME" - add_ssh_keys: fingerprints: - - "02:df:a5:6a:53:9a:f5:5d:bd:a6:fc:b2:db:9b:c9:47" + - "02:df:a5:6a:53:9a:f5:5d:bd:a6:fc:b2:db:9b:c9:47" # disable-secrets-detection + - "f5:25:6a:e5:ac:4b:84:fb:60:54:14:82:f1:e9:6c:f9" # disable-secrets-detection + - run: + name: Create ID Set + when: always + command: | + python ./Tests/scripts/update_id_set.py -r + - run: + name: Infrastucture testing + when: always + command: | + pytest ./Tests/scripts/hook_validations/tests/ -v + pytest ./Tests/scripts/infrastructure_tests/ -v + pytest ./Tests/scripts/test_configure_tests.py -v - run: name: Validate Files and Yaml + when: always command: | - python ./Tests/scripts/validate_files_structure.py -c true + # Run flake8 on all excluding Integraions and Scripts (they will be handled in linting) + ./Tests/scripts/pyflake.sh *.py + find . -maxdepth 1 -type d -not \( -path . -o -path ./Integrations -o -path ./Scripts -o -path ./Beta_Integrations \) | xargs ./Tests/scripts/pyflake.sh + + [ -n "${BACKWARD_COMPATIBILITY}" ] && CHECK_BACKWARD=false || CHECK_BACKWARD=true + python ./Tests/scripts/validate_files.py -c true -b $CHECK_BACKWARD - run: name: Configure Test Filter + when: always command: | [ -n "${NIGHTLY}" ] && IS_NIGHTLY=true || IS_NIGHTLY=false python ./Tests/scripts/configure_tests.py -n $IS_NIGHTLY + - run: + name: Spell Checks + command: | + python ./Tests/scripts/circleci_spell_checker.py $CIRCLE_BRANCH - run: name: Build Content Descriptor - command: ./setContentDescriptor.sh $CIRCLE_BUILD_NUM $GIT_SHA1 $CONTENT_VERSION + when: always + command: | + if [ -n "${GITHUB_TOKEN}" ] ; + then + python3 release_notes.py $CONTENT_VERSION $GIT_SHA1 $CIRCLE_BUILD_NUM $SERVER_VERSION --github-token $GITHUB_TOKEN + + else + python3 release_notes.py $CONTENT_VERSION $GIT_SHA1 $CIRCLE_BUILD_NUM $SERVER_VERSION + fi - run: name: Common Server Documentation + when: always command: ./Documentation/commonServerDocs.sh - run: name: Create Content Artifacts + when: always command: python content_creator.py $CIRCLE_ARTIFACTS - store_artifacts: path: artifacts destination: artifacts + - run: + name: Run Unit Testing and Lint + when: always + command: SKIP_GIT_COMPARE_FILTER=${NIGHTLY} ./Tests/scripts/run_all_pkg_dev_tasks.sh + - run: + name: Download Artifacts + when: always + command: | + if [[ $CIRCLE_BRANCH =~ pull/[0-9]+ ]]; then + echo "Skipping instance tests for forked PRs" + exit 0 + fi + if ./Tests/scripts/is_ami.sh ; + then + echo "Using AMI - Not downloading artifacts" + + else + ./Tests/scripts/server_get_artifact.sh $SERVER_CI_TOKEN + cp demistoserver.sh ./Tests/scripts/awsinstancetool/ansibleinstall/demistoserver.sh + fi - run: name: Download Configuration + when: always command: | - ./Tests/scripts/download_demisto_conf.sh + if [[ $CIRCLE_BRANCH =~ pull/[0-9]+ ]]; then + echo "Skipping instance tests for forked PRs" + exit 0 + fi + if ./Tests/scripts/is_ami.sh ; + then + ./Tests/scripts/download_demisto_conf.sh + + else + ./Tests/lastest_server_build_scripts/download_demisto_conf.sh + fi - run: name: Create Instance + when: always command: | - ./Tests/scripts/create_instance.sh instance.json + if [[ $CIRCLE_BRANCH =~ pull/[0-9]+ ]]; then + echo "Skipping instance tests for forked PRs" + exit 0 + fi + if ./Tests/scripts/is_ami.sh ; + then + if [ -n "${NIGHTLY}" ] ; + then + export IFRA_ENV_TYPE=Content-Master + + else + export IFRA_ENV_TYPE=Content-Env + fi + python ./Tests/scripts/awsinstancetool/aws_instance_tool.py -envType $IFRA_ENV_TYPE -outfile ./env_results.json + + else + python ./Tests/scripts/awsinstancetool/aws_instance_tool.py -envType CustomBuild -outfile ./env_results.json + fi - run: name: Setup Instance + when: always + command: | + if [[ $CIRCLE_BRANCH =~ pull/[0-9]+ ]]; then + echo "Skipping instance tests for forked PRs" + exit 0 + fi + if ./Tests/scripts/is_ami.sh ; + then + python ./Tests/scripts/run_content_installation.py + python ./Tests/scripts/wait_until_server_ready.py -c $(cat secret_conf_path) -v $CONTENT_VERSION + + else + ./Tests/lastest_server_build_scripts/run_installer_on_instance.sh + python ./Tests/scripts/wait_until_server_ready.py -c $(cat secret_conf_path) -v $CONTENT_VERSION --non-ami + fi + - run: + name: Run Tests - Latest GA + shell: /bin/bash + when: always + command: | + if [[ $CIRCLE_BRANCH =~ pull/[0-9]+ ]]; then + echo "Skipping instance tests for forked PRs" + exit 0 + fi + if ./Tests/scripts/is_ami.sh ; + then + ./Tests/scripts/run_tests.sh "Demisto GA" + + else + ./Tests/lastest_server_build_scripts/run_tests.sh + fi + - run: + name: Run Tests - One Before GA + shell: /bin/bash + when: always command: | - ./Tests/scripts/run_installer_on_instance.sh - ./Tests/scripts/wait_until_server_ready.sh + if [[ $CIRCLE_BRANCH =~ pull/[0-9]+ ]]; then + echo "Skipping instance tests for forked PRs" + exit 0 + fi + if ./Tests/scripts/is_ami.sh ; + then + ./Tests/scripts/run_tests.sh "Demisto one before GA" + + else + echo "Not AMI run, can't run on this version" + fi - run: - name: Run Tests + name: Run Tests - Two Before GA shell: /bin/bash - command: ./Tests/scripts/run_tests.sh + when: always + command: | + if [[ $CIRCLE_BRANCH =~ pull/[0-9]+ ]]; then + echo "Skipping instance tests for forked PRs" + exit 0 + fi + if ./Tests/scripts/is_ami.sh ; + then + ./Tests/scripts/run_tests.sh "Demisto two before GA" + + else + echo "Not AMI run, can't run on this version" + fi + - run: + name: Run Tests - Server Master + shell: /bin/bash + when: always + command: | + if [[ $CIRCLE_BRANCH =~ pull/[0-9]+ ]]; then + echo "Skipping instance tests for forked PRs" + exit 0 + fi + if ./Tests/scripts/is_ami.sh ; + then + ./Tests/scripts/run_tests.sh "Server Master" + + else + echo "Not AMI run, can't run on this version" + fi - run: name: Slack Notifier shell: /bin/bash - command: ./Tests/scripts/slack_notifier.sh + command: | + if [[ $CIRCLE_BRANCH =~ pull/[0-9]+ ]]; then + echo "Skipping instance tests for forked PRs" + exit 0 + fi + ./Tests/scripts/slack_notifier.sh ./env_results.json + when: always + - run: + name: Validate Docker Images + shell: /bin/bash + command: ./Tests/scripts/validate_docker_images.sh when: always - run: name: Instance Test - command: ./Tests/scripts/instance_test.sh + command: | + if [[ $CIRCLE_BRANCH =~ pull/[0-9]+ ]]; then + echo "Skipping instance tests for forked PRs" + exit 0 + fi + ./Tests/scripts/instance_test.sh when: always - run: name: Destroy Instances - command: ./Tests/scripts/destroy_instances.sh $CIRCLE_ARTIFACTS + command: | + if [[ $CIRCLE_BRANCH =~ pull/[0-9]+ ]]; then + echo "Skipping instance tests for forked PRs" + exit 0 + fi + python ./Tests/scripts/destroy_instances.py $CIRCLE_ARTIFACTS ./env_results.json when: always - store_artifacts: path: artifacts destination: artifacts when: always + +workflows: + version: 2 + commit: + jobs: + - build diff --git a/.github/config.yml b/.github/config.yml new file mode 100644 index 000000000000..1529de87d451 --- /dev/null +++ b/.github/config.yml @@ -0,0 +1,6 @@ +newPRWelcomeComment: > + Hi and welcome to the Demisto Content project! + Thank you and congrats on your first pull request, we will review it soon! + Until then you can check out our [documentation](https://github.com/demisto/content/tree/master/docs) for more details. + We would be thrilled to see you get involved in our [Slack DFIR community](https://go.demisto.com/join-our-slack-community) for discussions. + Hope you have a great time here :) \ No newline at end of file diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index e33bc8b903d2..2e4241f659ac 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,3 +1,5 @@ + + ## Status Ready/In Progress/In Hold(Reason for hold) @@ -30,5 +32,16 @@ x.x.x - [ ] Documentation (with link to it) - [ ] Code Review +## Dependencies +Mention the dependencies of the entity you changed as given from the precommit hooks in checkboxes, and tick after tested them. +- [ ] Dependency 1 +- [ ] Dependency 2 +- [ ] Dependency 3 + ## Additional changes Describe additional changes done, for example adding a function to common server. + +## Technical writer review +Mention and link to the files that require a technical writer review. +- [ ] [YAML file](link) +- [ ] [CHANGELOG](link) \ No newline at end of file diff --git a/.gitignore b/.gitignore index fe72f9f370d2..d6050235005c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,21 @@ .DS_Store .idea -_site \ No newline at end of file +.vscode +_site +TestData/EmailWithNonUnicodeAttachmentName.eml +TestData/EmailWithNonUnicodeSubject.eml +*.pyc +.pytest_cache + +CommonServerPython.py +!Scripts/CommonServerPython/CommonServerPython.py +CommonServerUserPython.py +demistomock.py +Tests/filter_file.txt +Tests/id_set.json +.mypy_cache +Scripts/*/*_unified.yml +Integrations/*/*_unified.yml +Beta_Integrations/*/*_unified.yml +conftest.py +!Tests/scripts/dev_envs/pytest/conftest.py diff --git a/.guardrails/ignore b/.guardrails/ignore new file mode 100644 index 000000000000..6ef4617bd08f --- /dev/null +++ b/.guardrails/ignore @@ -0,0 +1 @@ +Integrations/Active_Directory_Query/key.pem diff --git a/.hooks/pre-commit b/.hooks/pre-commit index 013b9a5e750c..f166bb2318d4 100755 --- a/.hooks/pre-commit +++ b/.hooks/pre-commit @@ -1,15 +1,37 @@ #!/bin/bash # validating that each modified file has a valid schema, release notes, proper prefix & suffix -echo "Validating files" -python Tests/scripts/validate_files_structure.py +echo "Validating files..." +if [[ -z "${WINDIR}" ]] + then + PYTHONPATH="`pwd`:${PYTHONPATH}" python Tests/scripts/validate_files.py -t true + else + python Tests/scripts/validate_files.py +fi + +RES=$? + +echo "" +if [[ -n "$CONTENT_PRECOMMIT_RUN_DEV_TASKS" ]]; then + echo "Running content dev tasks (flake8, mypy, pylint, pytst) as env variable CONTENT_PRECOMMIT_RUN_DEV_TASKS is set." + ./Tests/scripts/run_all_pkg_dev_tasks.sh + RES=$(($RES + $?)) +else + echo "Skipping running dev tasks (flake8, mypy, pylint, pytest). If you want to run this as part of the precommit hook" + echo 'set CONTENT_PRECOMMIT_RUN_DEV_TASKS=1. You can add the following line to ~/.zshrc:' + echo 'echo "export CONTENT_PRECOMMIT_RUN_DEV_TASKS=1" >> ~/.zshrc' + echo "" + echo 'Or if you want to manually run dev tasks: ./Tests/scripts/pkg_dev_test_tasks.py -d ' + echo 'Example: ./Tests/scripts/pkg_dev_test_tasks.py -d Scripts/ParseEmailFiles' +fi -if [[ $? -ne 0 ]] -then +if [[ $RES -ne 0 ]] + then echo "Please fix the aforementioned errors and then commit again" exit 1 fi + # prevent push to master if [ -z "$1" ]; then protected_branch='master' @@ -19,3 +41,5 @@ if [ -z "$1" ]; then exit 1 fi fi + +echo "" diff --git a/Beta_Integrations/AWS-Athena/AWS-Athena.py b/Beta_Integrations/AWS-Athena/AWS-Athena.py new file mode 100644 index 000000000000..d5e5f3055936 --- /dev/null +++ b/Beta_Integrations/AWS-Athena/AWS-Athena.py @@ -0,0 +1,247 @@ +import demistomock as demisto +from CommonServerPython import * +from CommonServerUserPython import * +import boto3 +import json +from datetime import datetime, date +from botocore.config import Config +from botocore.parsers import ResponseParserError +import urllib3.util + +# Disable insecure warnings +urllib3.disable_warnings() + +"""PARAMETERS""" +AWS_DEFAULT_REGION = demisto.params().get('defaultRegion') +AWS_ROLE_ARN = demisto.params().get('roleArn') +AWS_ROLE_SESSION_NAME = demisto.params().get('roleSessionName') +AWS_ROLE_SESSION_DURATION = demisto.params().get('sessionDuration') +AWS_ROLE_POLICY = None +AWS_ACCESS_KEY_ID = demisto.params().get('access_key') +AWS_SECRET_ACCESS_KEY = demisto.params().get('secret_key') +VERIFY_CERTIFICATE = not demisto.params().get('insecure', True) +proxies = handle_proxy(proxy_param_name='proxy', checkbox_default_value=False) +config = Config( + connect_timeout=1, + retries=dict( + max_attempts=5 + ), + proxies=proxies +) + + +"""HELPER FUNCTIONS""" + + +class DatetimeEncoder(json.JSONEncoder): + # pylint: disable=method-hidden + def default(self, obj): + if isinstance(obj, datetime): + return obj.strftime('%Y-%m-%dT%H:%M:%S') + elif isinstance(obj, date): + return obj.strftime('%Y-%m-%d') + # Let the base class default method raise the TypeError + return json.JSONEncoder.default(self, obj) + + +def aws_session(service='athena', region=None, roleArn=None, roleSessionName=None, roleSessionDuration=None, + rolePolicy=None): + kwargs = {} + if roleArn and roleSessionName is not None: + kwargs.update({ + 'RoleArn': roleArn, + 'RoleSessionName': roleSessionName, + }) + elif AWS_ROLE_ARN and AWS_ROLE_SESSION_NAME is not None: + kwargs.update({ + 'RoleArn': AWS_ROLE_ARN, + 'RoleSessionName': AWS_ROLE_SESSION_NAME, + }) + + if roleSessionDuration is not None: + kwargs.update({'DurationSeconds': int(roleSessionDuration)}) + elif AWS_ROLE_SESSION_DURATION is not None: + kwargs.update({'DurationSeconds': int(AWS_ROLE_SESSION_DURATION)}) + + if rolePolicy is not None: + kwargs.update({'Policy': rolePolicy}) + elif AWS_ROLE_POLICY is not None: + kwargs.update({'Policy': AWS_ROLE_POLICY}) + if kwargs and AWS_ACCESS_KEY_ID is None: + + if AWS_ACCESS_KEY_ID is None: + sts_client = boto3.client('sts', config=config, verify=VERIFY_CERTIFICATE) + sts_response = sts_client.assume_role(**kwargs) + if region is not None: + client = boto3.client( + service_name=service, + region_name=region, + aws_access_key_id=sts_response['Credentials']['AccessKeyId'], + aws_secret_access_key=sts_response['Credentials']['SecretAccessKey'], + aws_session_token=sts_response['Credentials']['SessionToken'], + verify=VERIFY_CERTIFICATE, + config=config + ) + else: + client = boto3.client( + service_name=service, + region_name=AWS_DEFAULT_REGION, + aws_access_key_id=sts_response['Credentials']['AccessKeyId'], + aws_secret_access_key=sts_response['Credentials']['SecretAccessKey'], + aws_session_token=sts_response['Credentials']['SessionToken'], + verify=VERIFY_CERTIFICATE, + config=config + ) + elif AWS_ACCESS_KEY_ID and AWS_ROLE_ARN: + sts_client = boto3.client( + service_name='sts', + aws_access_key_id=AWS_ACCESS_KEY_ID, + aws_secret_access_key=AWS_SECRET_ACCESS_KEY, + verify=VERIFY_CERTIFICATE, + config=config + ) + kwargs.update({ + 'RoleArn': AWS_ROLE_ARN, + 'RoleSessionName': AWS_ROLE_SESSION_NAME, + }) + sts_response = sts_client.assume_role(**kwargs) + client = boto3.client( + service_name=service, + region_name=AWS_DEFAULT_REGION, + aws_access_key_id=sts_response['Credentials']['AccessKeyId'], + aws_secret_access_key=sts_response['Credentials']['SecretAccessKey'], + aws_session_token=sts_response['Credentials']['SessionToken'], + verify=VERIFY_CERTIFICATE, + config=config + ) + else: + if region is not None: + client = boto3.client( + service_name=service, + region_name=region, + aws_access_key_id=AWS_ACCESS_KEY_ID, + aws_secret_access_key=AWS_SECRET_ACCESS_KEY, + verify=VERIFY_CERTIFICATE, + config=config + ) + else: + client = boto3.client( + service_name=service, + region_name=AWS_DEFAULT_REGION, + aws_access_key_id=AWS_ACCESS_KEY_ID, + aws_secret_access_key=AWS_SECRET_ACCESS_KEY, + verify=VERIFY_CERTIFICATE, + config=config + ) + + return client + + +def start_query_execution_command(args): + client = aws_session( + region=args.get('region'), + roleArn=args.get('roleArn'), + roleSessionName=args.get('roleSessionName'), + roleSessionDuration=args.get('roleSessionDuration'), + ) + data = [] + kwargs = {'QueryString': args.get('QueryString')} + if args.get('ClientRequestToken') is not None: + kwargs.update({'ClientRequestToken': args.get('ClientRequestToken')}) + if args.get('Database') is not None: + kwargs.update({'QueryExecutionContext': {'Database': args.get('Database')}}) + if args.get('OutputLocation') is not None: + kwargs.update({'ResultConfiguration': {'OutputLocation': args.get('OutputLocation')}}) + if args.get('EncryptionOption') is not None: + kwargs.update({'ResultConfiguration': {'EncryptionConfiguration': {'EncryptionOption': args.get('EncryptionOption')}}}) + if args.get('KmsKey') is not None: + kwargs.update({'ResultConfiguration': {'EncryptionConfiguration': {'KmsKey': args.get('KmsKey')}}}) + if args.get('WorkGroup') is not None: + kwargs.update({'WorkGroup': args.get('WorkGroup')}) + + response = client.start_query_execution(**kwargs) + + data.append({ + 'QueryString': args.get('QueryString'), + 'QueryExecutionId': response['QueryExecutionId'] + }) + ec = {'AWS.Athena.Query': data} + human_readable = tableToMarkdown('AWS Athena Query', data) + return_outputs(human_readable, ec) + + +def stop_query_command(args): + client = aws_session( + region=args.get('region'), + roleArn=args.get('roleArn'), + roleSessionName=args.get('roleSessionName'), + roleSessionDuration=args.get('roleSessionDuration'), + ) + + response = client.stop_query_execution(QueryExecutionId=args.get('QueryExecutionId')) + if response['ResponseMetadata']['HTTPStatusCode'] == 200: + demisto.results("The Query {query} was Deleted ".format(query=args.get('QueryExecutionId'))) + + +def get_query_execution_command(args): + client = aws_session( + region=args.get('region'), + roleArn=args.get('roleArn'), + roleSessionName=args.get('roleSessionName'), + roleSessionDuration=args.get('roleSessionDuration'), + ) + kwargs = {'QueryExecutionId': args.get('QueryExecutionId')} + response = client.get_query_execution(**kwargs) + try: + raw = json.loads(json.dumps(response, cls=DatetimeEncoder)) + except ValueError as e: + return_error('Could not decode/encode the raw response - {err_msg}'.format(err_msg=e)) + ec = {'AWS.Athena.Query(val.QueryExecutionId === obj.QueryExecutionId)': raw} + human_readable = tableToMarkdown('AWS Athena Query', raw) + return_outputs(human_readable, ec) + + +def get_query_results_command(args): + client = aws_session( + region=args.get('region'), + roleArn=args.get('roleArn'), + roleSessionName=args.get('roleSessionName'), + roleSessionDuration=args.get('roleSessionDuration'), + ) + kwargs = {'QueryExecutionId': args.get('QueryExecutionId')} + response = client.get_query_results(**kwargs) + ec = {'AWS.Athena.Query(val.QueryExecutionId === obj.QueryExecutionId)': response} + human_readable = tableToMarkdown('AWS Athena Query', response) + return_outputs(human_readable, ec) + + +"""COMMAND BLOCK""" +try: + LOG('Command being called is {command}'.format(command=demisto.command())) + if demisto.command() == 'test-module': + client = aws_session() + response = client.list_named_queries() + if response['ResponseMetadata']['HTTPStatusCode'] == 200: + demisto.results('ok') + + elif demisto.command() == 'aws-athena-start-query': + start_query_execution_command(demisto.args()) + + elif demisto.command() == 'aws-athena-stop-query': + stop_query_command(demisto.args()) + + elif demisto.command() == 'aws-athena-get-query-execution': + get_query_execution_command(demisto.args()) + + elif demisto.command() == 'aws-athena-get-query-results': + get_query_results_command(demisto.args()) + + +except ResponseParserError as e: + return_error('Could not connect to the AWS endpoint. Please check that the region is valid.\n {error}'.format( + error=type(e))) + LOG(e) + +except Exception as e: + return_error('Error has occurred in the AWS Athena Integration: {error}\n {message}'.format( + error=type(e), message=e)) diff --git a/Beta_Integrations/AWS-Athena/AWS-Athena.yml b/Beta_Integrations/AWS-Athena/AWS-Athena.yml new file mode 100644 index 000000000000..3fd9729f9144 --- /dev/null +++ b/Beta_Integrations/AWS-Athena/AWS-Athena.yml @@ -0,0 +1,148 @@ +commonfields: + id: AWS - Athena - Beta + version: -1 +name: AWS - Athena - Beta +display: AWS - Athena (Beta) +category: IT Services +description: Amazon Web Services Athena +configuration: +- display: Role Arn + name: roleArn + defaultvalue: "" + type: 0 + required: false +- display: Role Session Name + name: roleSessionName + defaultvalue: "" + type: 0 + required: false +- display: AWS Default Region + name: defaultRegion + defaultvalue: "" + type: 0 + required: false +- display: Role Session Duration + name: sessionDuration + defaultvalue: "" + type: 0 + required: false +- display: Access Key + name: access_key + defaultvalue: "" + type: 0 + required: false +- display: Secret Key + name: secret_key + defaultvalue: "" + type: 4 + required: false +- display: Trust any cert (Not Secure) + name: insecure + defaultvalue: "" + type: 8 + required: false +- display: Use system proxy + name: proxy + defaultvalue: "" + type: 8 + required: false +script: + script: '' + type: python + commands: + - name: aws-athena-start-query + arguments: + - name: QueryString + required: true + description: The SQL query statements to be executed. + - name: ClientRequestToken + auto: PREDEFINED + predefined: + - private + - public-read + - public-read-write + - authenticated-read + description: A unique case-sensitive string used to ensure the request to create + the query is idempotent (executes only once). + - name: Database + description: The name of the database. + - name: OutputLocation + description: he location in Amazon S3 where your query results are stored, such + as s3://path/to/query/bucket/. + - name: EncryptionOption + description: Indicates whether Amazon S3 server-side encryption with Amazon + S3-managed keys (SSE-S3 ), server-side encryption with KMS-managed keys (SSE-KMS + ), or client-side encryption with KMS-managed keys (CSE-KMS) is used. + - name: KmsKey + description: For SSE-KMS and CSE-KMS , this is the KMS key ARN or ID. + - name: WorkGroup + description: The name of the workgroup in which the query is being started. + - name: roleArn + description: The Amazon Resource Name (ARN) of the role to assume. + - name: roleSessionName + description: An identifier for the assumed role session. + - name: roleSessionDuration + description: The duration, in seconds, of the role session. The value can range + from 900 seconds (15 minutes) up to the maximum session duration setting for + the role. + - name: region + description: The AWS Region, if not specified the default region will be used. + description: Start Athena Query. + - name: aws-athena-stop-query + arguments: + - name: QueryExecutionId + required: true + description: The unique ID of the query execution to stop. This field is auto-populated + if not provided. + - name: region + description: The AWS Region, if not specified the default region will be used. + - name: roleArn + description: The Amazon Resource Name (ARN) of the role to assume. + - name: roleSessionName + description: An identifier for the assumed role session. + - name: roleSessionDuration + description: The duration, in seconds, of the role session. The value can range + from 900 seconds (15 minutes) up to the maximum session duration setting for + the role. + description: Stops a query execution. Requires you to have access to the workgroup + in which the query ran. + - name: aws-athena-get-query-execution + arguments: + - name: region + description: The AWS Region, if not specified the default region will be used. + - name: roleArn + description: The Amazon Resource Name (ARN) of the role to assume. + - name: roleSessionName + description: An identifier for the assumed role session. + - name: roleSessionDuration + description: The duration, in seconds, of the role session. The value can range + from 900 seconds (15 minutes) up to the maximum session duration setting for + the role. + - name: QueryExecutionId + required: true + description: The unique ID of the query execution. + description: Returns information about a single execution of a query if you have + access to the workgroup in which the query ran. + - name: aws-athena-get-query-results + arguments: + - name: QueryExecutionId + required: true + description: The unique ID of the query execution. + - name: region + description: The AWS Region, if not specified the default region will be used. + - name: roleArn + description: The Amazon Resource Name (ARN) of the role to assume. + - name: roleSessionName + description: An identifier for the assumed role session. + - name: roleSessionDuration + description: The duration, in seconds, of the role session. The value can range + from 900 seconds (15 minutes) up to the maximum session duration setting for + the role. + description: Returns the results of a single query execution specified by QueryExecutionId + if you have access to the workgroup in which the query ran. + dockerimage: demisto/boto3py3:1.0.0.1030 + runonce: false + subtype: python3 +beta: true +tests: +- Beta-Athena-Test diff --git a/Beta_Integrations/AWS-Athena/AWS-Athena_CHANGELOG.md b/Beta_Integrations/AWS-Athena/AWS-Athena_CHANGELOG.md new file mode 100644 index 000000000000..594507ae86ea --- /dev/null +++ b/Beta_Integrations/AWS-Athena/AWS-Athena_CHANGELOG.md @@ -0,0 +1,6 @@ +## [Unreleased] + + +## [19.9.1] - 2019-09-18 +#### New Integration +Amazon Web Services Athena \ No newline at end of file diff --git a/Beta_Integrations/AWS-Athena/AWS-Athena_description.md b/Beta_Integrations/AWS-Athena/AWS-Athena_description.md new file mode 100644 index 000000000000..267a5aa9626c --- /dev/null +++ b/Beta_Integrations/AWS-Athena/AWS-Athena_description.md @@ -0,0 +1,18 @@ +Before you can use the AWS Athena integration in Demisto, you need to perform several configuration steps in your AWS environment. + +### Prerequisites +- Attach an instance profile with the required permissions to the Demisto server or engine that is running +on your AWS environment. +- Instance profile requires minimum permission: sts:AssumeRole. +- Instance profile requires permission to assume the roles needed by the AWS integrations. + +### Configure AWS Settings +1. Create an IAM Role for the Instance Profile. +2. Attach a Role to the Instance Profile. +3. Configure the Necessary IAM Roles that the AWS Integration Can Assume. + +For detailed instructions, [see the AWS Integrations Configuration Guide](https://support.demisto.com/hc/en-us/articles/360005686854-AWS-Integrations-Configuration-Guide). + +Command descriptions, input descriptions, and output descriptions are taken from the Amazon ACM documentation. For more information, see the [Amazon Athena documention](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/athena.html). + + Note: This is a beta Integration, which lets you implement and test pre-release software. Since the integration is beta, it might contain bugs. Updates to the integration during the beta phase might include non-backward compatible features. We appreciate your feedback on the quality and usability of the integration to help us identify issues, fix them, and continually improve. \ No newline at end of file diff --git a/Beta_Integrations/AWS-Athena/AWS-Athena_image.png b/Beta_Integrations/AWS-Athena/AWS-Athena_image.png new file mode 100644 index 000000000000..3ec06e277d29 Binary files /dev/null and b/Beta_Integrations/AWS-Athena/AWS-Athena_image.png differ diff --git a/Beta_Integrations/AWS-Athena/CHANGELOG.md b/Beta_Integrations/AWS-Athena/CHANGELOG.md new file mode 100644 index 000000000000..2a400fe6ac2b --- /dev/null +++ b/Beta_Integrations/AWS-Athena/CHANGELOG.md @@ -0,0 +1,3 @@ +## [Unreleased] +* Packaged AWS Athena to follow standardization of integrations. +* Bugfix for Proxy/Insecure issues. \ No newline at end of file diff --git a/Beta_Integrations/Blueliv/Blueliv.png b/Beta_Integrations/Blueliv/Blueliv.png new file mode 100644 index 000000000000..8e69f04ffb3a Binary files /dev/null and b/Beta_Integrations/Blueliv/Blueliv.png differ diff --git a/Beta_Integrations/Blueliv/Blueliv.py b/Beta_Integrations/Blueliv/Blueliv.py new file mode 100644 index 000000000000..26175601fc50 --- /dev/null +++ b/Beta_Integrations/Blueliv/Blueliv.py @@ -0,0 +1,100 @@ +import demistomock as demisto +from CommonServerPython import * +from CommonServerUserPython import * +''' IMPORTS ''' + +from sdk.blueliv_api import BluelivAPI + +''' GLOBALS/PARAMS ''' + +TOKEN = demisto.params().get('token') +URL = demisto.params()['url'] +SERVER = URL[:-1] if URL.endswith('/') else URL + +if not demisto.params().get('proxy', False): + del os.environ['HTTP_PROXY'] + del os.environ['HTTPS_PROXY'] + del os.environ['http_proxy'] + del os.environ['https_proxy'] + +''' HELPER FUNCTIONS ''' + + +def verify_response_code(response): + + if response.status_code != 200: + raise ValueError(response.error_msg) + + +''' COMMANDS + REQUESTS FUNCTIONS ''' + + +def test_module(): + + response = api.crime_servers.last('all') + verify_response_code(response) + demisto.results('ok') + + +def get_botips_feed_command(): + + response = api.bot_ips.recent('full') + verify_response_code(response) + human_readable = tableToMarkdown('Bot IP feed', response.items) + return_outputs(human_readable, {}) + + +def get_crimeservers_feed_command(): + + response = api.crime_servers.last('all') + verify_response_code(response) + human_readable = tableToMarkdown('Crimeservers feed', response.items) + return_outputs(human_readable, {}) + + +def get_malware_feed_command(): + + response = api.malwares.recent('all') + verify_response_code(response) + human_readable = tableToMarkdown('Malware feed', response.items) + return_outputs(human_readable, {}) + + +def get_attackingips_feed_command(): + + response = api.attacking_ips.recent('all') + verify_response_code(response) + human_readable = tableToMarkdown('Attacking IPs feed', response.items) + return_outputs(human_readable, {}) + + +def get_hacktivism_feed_command(): + + response = api.hacktivism_ops.last('all') + verify_response_code(response) + human_readable = tableToMarkdown('Hacktivism feed', response.items) + return_outputs(human_readable, {}) + + +''' COMMANDS MANAGER / SWITCH PANEL ''' + +COMMANDS = { + 'test-module': test_module, + 'blueliv-get-botips-feed': get_botips_feed_command, + 'blueliv-get-crimeservers-feed': get_crimeservers_feed_command, + 'blueliv-get-malware-feed': get_malware_feed_command, + 'blueliv-get-attackingips-feed': get_attackingips_feed_command, + 'blueliv-get-hacktivism-feed': get_hacktivism_feed_command +} + +try: + api = BluelivAPI( + base_url=SERVER, + token=TOKEN + ) + LOG('Command being called is {}'.format(demisto.command())) + command_func = COMMANDS.get(demisto.command()) + if command_func is not None: + command_func() +except Exception as e: + return_error(str(e)) diff --git a/Beta_Integrations/Blueliv/Blueliv.yml b/Beta_Integrations/Blueliv/Blueliv.yml new file mode 100644 index 000000000000..0193f76363b7 --- /dev/null +++ b/Beta_Integrations/Blueliv/Blueliv.yml @@ -0,0 +1,51 @@ +category: Data Enrichment & Threat Intelligence +commonfields: + id: Blueliv_Beta + version: -1 +configuration: +- defaultvalue: https://api.blueliv.com + display: Server URL (e.g., https://api.blueliv.com) + name: url + required: true + type: 0 +- display: API Token + name: token + required: true + type: 4 +- display: Use system proxy settings + name: proxy + type: 8 + required: false +description: Blueliv reduces risk through actionable, dynamic and targeted threat intelligence, trusted by your organization. +display: Blueliv (Beta) +name: Blueliv_Beta +script: + commands: + - description: Data set collection that gives the latest STIX Indicators about bot + ips gathered by Blueliv. + execution: false + name: blueliv-get-botips-feed + - description: Data set collection that gives the latest STIX Indicators about known + malicious servers gathered by Blueliv. + execution: false + name: blueliv-get-crimeservers-feed + - description: Data set collection that gives the latest STIX Indicators about malware + hashes gathered and analyzed by Blueliv. + execution: false + name: blueliv-get-malware-feed + - description: Data set collection that gives the latest STIX Indicators about attacking + IPs gathered and analyzed by Blueliv. + execution: false + name: blueliv-get-attackingips-feed + - description: 'Data related to the number of hacktivism tweets recently created. + Blueliv provides two types of feeds: the first one contains the most popular + hacktivism hashtags and the second one contains the countries where more number + of hacktivism tweets are coming from.' + execution: false + name: blueliv-get-hacktivism-feed + dockerimage: demisto/blueliv:1.0.0.165 + isfetch: false + runonce: false + script: '' + type: python +beta: true diff --git a/Beta_Integrations/Blueliv/Blueliv_description.md b/Beta_Integrations/Blueliv/Blueliv_description.md new file mode 100644 index 000000000000..cf9944213322 --- /dev/null +++ b/Beta_Integrations/Blueliv/Blueliv_description.md @@ -0,0 +1 @@ +Note: This is a beta Integration, which lets you implement and test pre-release software. Since the integration is beta, it might contain bugs. Updates to the integration during the beta phase might include non-backward compatible features. We appreciate your feedback on the quality and usability of the integration to help us identify issues, fix them, and continually improve. \ No newline at end of file diff --git a/Beta_Integrations/MailListener_-_POP3/MailListener_-_POP3.py b/Beta_Integrations/MailListener_-_POP3/MailListener_-_POP3.py new file mode 100644 index 000000000000..5e277f4883cb --- /dev/null +++ b/Beta_Integrations/MailListener_-_POP3/MailListener_-_POP3.py @@ -0,0 +1,363 @@ +import demistomock as demisto + +from CommonServerPython import * +from CommonServerUserPython import * + +import poplib +import base64 +import quopri +from email.parser import Parser +from htmlentitydefs import name2codepoint +from HTMLParser import HTMLParser, HTMLParseError + + +''' GLOBALS/PARAMS ''' +SERVER = demisto.params().get('server', '') +EMAIL = demisto.params().get('email', '') +PASSWORD = demisto.params().get('password', '') +PORT = int(demisto.params().get('port', '995')) +SSL = demisto.params().get('ssl') +FETCH_TIME = demisto.params().get('fetch_time', '7 days') + +# pop3 server connection object. +pop3_server_conn = None # type: ignore + +TIME_REGEX = re.compile(r'^([\w,\d: ]*) (([+-]{1})(\d{2}):?(\d{2}))?[\s\w\(\)]*$') +DATE_FORMAT = '%Y-%m-%dT%H:%M:%SZ' + + +def connect_pop3_server(): + global pop3_server_conn + + if pop3_server_conn is None: + if SSL: + pop3_server_conn = poplib.POP3_SSL(SERVER, PORT) # type: ignore + else: + pop3_server_conn = poplib.POP3(SERVER, PORT) # type: ignore + + pop3_server_conn.getwelcome() # type: ignore + pop3_server_conn.user(EMAIL) # type: ignore + pop3_server_conn.pass_(PASSWORD) # type: ignore + + +def close_pop3_server_connection(): + global pop3_server_conn + if pop3_server_conn is not None: + pop3_server_conn.quit() + pop3_server_conn = None + + +def get_user_emails(): + _, mails_list, _ = pop3_server_conn.list() # type: ignore + + mails = [] + index = '' + + for mail in mails_list: + try: + index = mail.split(' ')[0] + (resp_message, lines, octets) = pop3_server_conn.retr(index) # type: ignore + msg_content = unicode(b'\r\n'.join(lines), errors='ignore').encode("utf-8") + msg = Parser().parsestr(msg_content) + msg['index'] = index + mails.append(msg) + except Exception as e: + demisto.error("Failed to get email with index " + index + 'from the server.') + raise e + + return mails + + +def get_attachment_name(headers): + name = headers.get('content-description', '') + + if re.match(r'^.+\..{3,5}$', name): + return name + + content_disposition = headers.get('content-disposition', '') + + if content_disposition: + m = re.search('filename="(.*?)"', content_disposition) + if m: + name = m.group(1) + + if re.match('^.+\..{3,5}$', name): + return name + + extension = re.match(r'.*[\\/]([\d\w]{2,4}).*', headers.get('content-type', 'txt')).group(1) # type: ignore + + return name + '.' + extension + + +def parse_base64(text): + if re.match("^=?.*?=$", text): + res = re.search('=\?.*?\?[A-Z]{1}\?(.*?)\?=', text, re.IGNORECASE) + if res: + res = res.group(1) + return base64.b64decode(res) # type: ignore + return text + + +class TextExtractHtmlParser(HTMLParser): + def __init__(self): + HTMLParser.__init__(self) + self._texts = [] # type: list + self._ignore = False + + def handle_starttag(self, tag, attrs): + if tag in ('p', 'br') and not self._ignore: + self._texts.append('\n') + elif tag in ('script', 'style'): + self._ignore = True + + def handle_startendtag(self, tag, attrs): + if tag in ('br', 'tr') and not self._ignore: + self._texts.append('\n') + + def handle_endtag(self, tag): + if tag in ('p', 'tr'): + self._texts.append('\n') + elif tag in ('script', 'style'): + self._ignore = False + + def handle_data(self, data): + if data and not self._ignore: + stripped = data.strip() + if stripped: + self._texts.append(re.sub(r'\s+', ' ', stripped)) + + def handle_entityref(self, name): + if not self._ignore and name in name2codepoint: + self._texts.append(unichr(name2codepoint[name])) + + def handle_charref(self, name): + if not self._ignore: + if name.startswith('x'): + c = unichr(int(name[1:], 16)) + else: + c = unichr(int(name)) + self._texts.append(c) + + def get_text(self): + return "".join(self._texts) + + +def html_to_text(html): + parser = TextExtractHtmlParser() + try: + parser.feed(html) + parser.close() + except HTMLParseError: + pass + return parser.get_text() + + +def get_email_context(email_data): + context_headers = email_data._headers + context_headers = [{'Name': v[0], 'Value': v[1]} + for v in context_headers] + headers = dict([(h['Name'].lower(), h['Value']) for h in context_headers]) + + context = { + 'Mailbox': EMAIL, + 'ID': email_data.get('Message-ID', 'None'), + 'Labels': ', '.join(email_data.get('labelIds', '')), + 'Headers': context_headers, + 'Format': headers.get('content-type', '').split(';')[0], + 'Subject': parse_base64(headers.get('subject')), + 'Body': email_data._payload, + 'From': headers.get('from'), + 'To': headers.get('to'), + 'Cc': headers.get('cc', []), + 'Bcc': headers.get('bcc', []), + 'Date': headers.get('date', ''), + 'Html': None, + } + + if 'text/html' in context['Format']: + context['Html'] = context['Body'] + context['Body'] = html_to_text(context['Body']) + + if 'multipart' in context['Format']: + context['Body'], context['Html'], context['Attachments'] = parse_mail_parts(email_data._payload) + context['Attachment Names'] = ', '.join( + [attachment['Name'] for attachment in context['Attachments']]) + + raw = dict(email_data) + raw['Body'] = context['Body'] + context['RawData'] = json.dumps(raw) + return context, headers + + +def parse_mail_parts(parts): + body = unicode("", "utf-8") + html = unicode("", "utf-8") + + attachments = [] # type: ignore + for part in parts: + context_headers = part._headers + context_headers = [{'Name': v[0], 'Value': v[1]} + for v in context_headers] + headers = dict([(h['Name'].lower(), h['Value']) for h in context_headers]) + + content_type = headers.get('content-type', 'text/plain') + + is_attachment = headers.get('content-disposition', '').startswith('attachment')\ + or headers.get('x-attachment-id') or "image" in content_type + + if 'multipart' in content_type or isinstance(part._payload, list): + part_body, part_html, part_attachments = parse_mail_parts(part._payload) + body += part_body + html += part_html + attachments.extend(part_attachments) + elif not is_attachment: + if headers.get('content-transfer-encoding') == 'base64': + text = base64.b64decode(part._payload).decode('utf-8') + elif headers.get('content-transfer-encoding') == 'quoted-printable': + decoded_string = quopri.decodestring(part._payload) + text = unicode(decoded_string, "utf-8") + else: + text = quopri.decodestring(part._payload) + + if not isinstance(text, unicode): + text = text.decode('unicode-escape') + + if 'text/html' in content_type: + html += text + else: + body += text + + else: + attachments.append({ + 'ID': headers.get('x-attachment-id', 'None'), + 'Name': get_attachment_name(headers), + 'Data': part._payload + }) + + return body, html, attachments + + +def parse_time(t): + base_time, _, _, _, _ = TIME_REGEX.findall(t)[0] + return datetime.strptime(base_time, '%a, %d %b %Y %H:%M:%S').isoformat() + 'Z' + + +def create_incident_labels(parsed_msg, headers): + labels = [ + {'type': 'Email/ID', 'value': parsed_msg['ID']}, + {'type': 'Email/subject', 'value': parsed_msg['Subject']}, + {'type': 'Email/text', 'value': parsed_msg['Body']}, + {'type': 'Email/from', 'value': parsed_msg['From']}, + {'type': 'Email/html', 'value': parsed_msg['Html']}, + ] + labels.extend([{'type': 'Email/to', 'value': to} + for to in headers.get('To', '').split(',')]) + labels.extend([{'type': 'Email/cc', 'value': cc} + for cc in headers.get('Cc', '').split(',')]) + labels.extend([{'type': 'Email/bcc', 'value': bcc} + for bcc in headers.get('Bcc', '').split(',')]) + for key, val in headers.items(): + labels.append({'type': 'Email/Header/' + key, 'value': val}) + + return labels + + +@logger +def mail_to_incident(msg): + parsed_msg, headers = get_email_context(msg) + + file_names = [] + for attachment in parsed_msg.get('Attachments', []): + file_data = base64.urlsafe_b64decode(attachment['Data'].encode('ascii')) + + # save the attachment + file_result = fileResult(attachment['Name'], file_data) + + # check for error + if file_result['Type'] == entryTypes['error']: + demisto.error(file_result['Contents']) + raise Exception(file_result['Contents']) + + file_names.append({ + 'path': file_result['FileID'], + 'name': attachment['Name'], + }) + + return { + 'name': parsed_msg['Subject'], + 'details': parsed_msg['Body'], + 'labels': create_incident_labels(parsed_msg, headers), + 'occurred': parse_time(parsed_msg['Date']), + 'attachment': file_names, + 'rawJSON': parsed_msg['RawData'] + } + + +def fetch_incidents(): + last_run = demisto.getLastRun() + last_fetch = last_run.get('time') + + # handle first time fetch + if last_fetch is None: + last_fetch, _ = parse_date_range(FETCH_TIME, date_format=DATE_FORMAT) + + last_fetch = datetime.strptime(last_fetch, DATE_FORMAT) + current_fetch = last_fetch + + incidents = [] + messages = get_user_emails() + + for msg in messages: + try: + incident = mail_to_incident(msg) + except Exception as e: + demisto.error("failed to create incident from email, index = {}, subject = {}, date = {}".format( + msg['index'], msg['subject'], msg['date'])) + raise e + + temp_date = datetime.strptime( + incident['occurred'], DATE_FORMAT) + + # update last run + if temp_date > last_fetch: + last_fetch = temp_date + timedelta(seconds=1) + + # avoid duplication due to weak time query + if temp_date > current_fetch: + incidents.append(incident) + + demisto.setLastRun({'time': last_fetch.isoformat().split('.')[0] + 'Z'}) + + return demisto.incidents(incidents) + + +def test_module(): + resp_message, _, _ = pop3_server_conn.list() # type: ignore + if "OK" in resp_message: + demisto.results('ok') + + +''' COMMANDS MANAGER / SWITCH PANEL ''' + + +def main(): + try: + handle_proxy() + connect_pop3_server() + if demisto.command() == 'test-module': + # This is the call made when pressing the integration test button. + test_module() + if demisto.command() == 'fetch-incidents': + fetch_incidents() + sys.exit(0) + except Exception as e: + LOG(str(e)) + LOG.print_log() + raise e + finally: + close_pop3_server_connection() + + +# python2 uses __builtin__ python3 uses builtins +if __name__ == "__builtin__" or __name__ == "builtins": + main() diff --git a/Beta_Integrations/MailListener_-_POP3/MailListener_-_POP3.yml b/Beta_Integrations/MailListener_-_POP3/MailListener_-_POP3.yml new file mode 100644 index 000000000000..c63098e9494b --- /dev/null +++ b/Beta_Integrations/MailListener_-_POP3/MailListener_-_POP3.yml @@ -0,0 +1,55 @@ +category: Messaging +commonfields: + id: MailListener - POP3 Beta + version: -1 +configuration: +- display: Server URL (e.g. example.com) + name: server + required: true + type: 0 +- defaultvalue: '995' + display: Port + name: port + required: false + type: 0 +- display: Email + name: email + required: true + type: 0 +- display: Password + name: password + required: true + type: 4 +- defaultvalue: 'True' + display: Use SSL connection + name: ssl + required: false + type: 8 +- display: Use system proxy + name: proxy + required: false + type: 8 +- display: Fetch incidents + name: isFetch + required: false + type: 8 +- defaultvalue: 3 days + display: First fetch timestamp (