diff --git a/.circleci/config.yml b/.circleci/config.yml index 4de7cc8ad994..643f143a99dc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,78 +1,217 @@ -version: 2.0 +version: 2 jobs: build: docker: - - image: ronykoz/content-build-node911:latest + - image: devdemisto/content-build-2and3:2.0.0.877 # disable-secrets-detection + resource_class: medium+ environment: - CONTENT_VERSION: "18.11.2" - GIT_SHA1: "3ac083b891d2f146ffbad6c7ce0a3be9e4f94b92" + CONTENT_VERSION: "19.8.2" + SERVER_VERSION: "4.5.0" + GIT_SHA1: "177b7e574fc936df19170c13650ea2581e4920f6" # guardrails-disable-line disable-secrets-detection steps: - checkout + - setup_remote_docker - run: name: Prepare Environment + when: always command: | echo 'export CIRCLE_ARTIFACTS="/home/circleci/project/artifacts"' >> $BASH_ENV + echo 'export PATH="/home/circleci/.pyenv/shims:/home/circleci/.local/bin:/home/circleci/.pyenv/bin:${PATH}"' >> $BASH_ENV # disable-secrets-detection + echo 'export PYTHONPATH="/home/circleci/project:${PYTHONPATH}"' >> $BASH_ENV + echo "=== sourcing $BASH_ENV ===" source $BASH_ENV sudo mkdir -p -m 777 $CIRCLE_ARTIFACTS - - run: - name: Install dependencies - command: | chmod +x ./Tests/scripts/* + chmod +x ./Tests/lastest_server_build_scripts/* + pyenv versions + python --version + python3 --version + echo "Parameters: NIGHTLY: $NIGHTLY, NON_AMI_RUN: $NON_AMI_RUN, SERVER_BRANCH_NAME: $SERVER_BRANCH_NAME" - add_ssh_keys: fingerprints: - - "02:df:a5:6a:53:9a:f5:5d:bd:a6:fc:b2:db:9b:c9:47" + - "02:df:a5:6a:53:9a:f5:5d:bd:a6:fc:b2:db:9b:c9:47" # disable-secrets-detection + - "f5:25:6a:e5:ac:4b:84:fb:60:54:14:82:f1:e9:6c:f9" # disable-secrets-detection + - run: + name: Create ID Set + when: always + command: | + python ./Tests/scripts/update_id_set.py -r + - run: + name: Infrastucture testing + when: always + command: | + pytest ./Tests/scripts/hook_validations/tests/ -v + pytest ./Tests/scripts/infrastructure_tests/ -v + pytest ./Tests/scripts/test_configure_tests.py -v - run: name: Validate Files and Yaml + when: always command: | - python ./Tests/scripts/validate_files_structure.py -c true + # Run flake8 on all excluding Integraions and Scripts (they will be handled in linting) + ./Tests/scripts/pyflake.sh *.py + find . -maxdepth 1 -type d -not \( -path . -o -path ./Integrations -o -path ./Scripts -o -path ./Beta_Integrations \) | xargs ./Tests/scripts/pyflake.sh + + [ -n "${BACKWARD_COMPATIBILITY}" ] && CHECK_BACKWARD=false || CHECK_BACKWARD=true + python ./Tests/scripts/validate_files.py -c true -b $CHECK_BACKWARD - run: name: Configure Test Filter + when: always command: | [ -n "${NIGHTLY}" ] && IS_NIGHTLY=true || IS_NIGHTLY=false python ./Tests/scripts/configure_tests.py -n $IS_NIGHTLY + - run: + name: Spell Checks + command: | + python ./Tests/scripts/circleci_spell_checker.py $CIRCLE_BRANCH - run: name: Build Content Descriptor - command: ./setContentDescriptor.sh $CIRCLE_BUILD_NUM $GIT_SHA1 $CONTENT_VERSION + when: always + command: python release_notes.py $CONTENT_VERSION $GIT_SHA1 $CIRCLE_BUILD_NUM $SERVER_VERSION $GITHUB_TOKEN - run: name: Common Server Documentation + when: always command: ./Documentation/commonServerDocs.sh - run: name: Create Content Artifacts + when: always command: python content_creator.py $CIRCLE_ARTIFACTS - store_artifacts: path: artifacts destination: artifacts + - run: + name: Run Unit Testing and Lint + when: always + command: SKIP_GIT_COMPARE_FILTER=${NIGHTLY} ./Tests/scripts/run_all_pkg_dev_tasks.sh + - run: + name: Download Artifacts + when: always + command: | + if ./Tests/scripts/is_ami.sh ; + then + echo "Using AMI - Not downloading artifacts" + + else + ./Tests/scripts/server_get_artifact.sh $SERVER_CI_TOKEN + fi - run: name: Download Configuration + when: always command: | - ./Tests/scripts/download_demisto_conf.sh + if ./Tests/scripts/is_ami.sh ; + then + ./Tests/scripts/download_demisto_conf.sh + + else + ./Tests/lastest_server_build_scripts/download_demisto_conf.sh + fi - run: name: Create Instance + when: always command: | - ./Tests/scripts/create_instance.sh instance.json + pip install --upgrade awscli + pip install ansible ansible-runner rsa paramiko boto3 requests + + if ./Tests/scripts/is_ami.sh ; + then + if [ -n "${NIGHTLY}" ] ; + then + export IFRA_ENV_TYPE=Content-Master + + else + export IFRA_ENV_TYPE=Content-Env + fi + python ./Tests/scripts/awsinstancetool/aws_instance_tool.py -envType $IFRA_ENV_TYPE + + else + python ./Tests/scripts/awsinstancetool/aws_instance_tool.py -envType CustomBuild + fi - run: name: Setup Instance + when: always + command: | + if ./Tests/scripts/is_ami.sh ; + then + python ./Tests/scripts/run_content_installation.py + python ./Tests/scripts/wait_until_server_ready.py -c $(cat secret_conf_path) -v $CONTENT_VERSION + + else + ./Tests/lastest_server_build_scripts/run_installer_on_instance.sh + python ./Tests/scripts/wait_until_server_ready.py -c $(cat secret_conf_path) -v $CONTENT_VERSION --non-ami + fi + - run: + name: Run Tests - Latest GA + shell: /bin/bash + when: always + command: | + if ./Tests/scripts/is_ami.sh ; + then + ./Tests/scripts/run_tests.sh "Demisto GA" + + else + ./Tests/lastest_server_build_scripts/run_tests.sh + fi + - run: + name: Run Tests - One Before GA + shell: /bin/bash + when: always + command: | + if ./Tests/scripts/is_ami.sh ; + then + ./Tests/scripts/run_tests.sh "Demisto one before GA" + + else + echo "Not AMI run, can't run on this version" + fi + - run: + name: Run Tests - Two Before GA + shell: /bin/bash + when: always command: | - ./Tests/scripts/run_installer_on_instance.sh - ./Tests/scripts/wait_until_server_ready.sh + if ./Tests/scripts/is_ami.sh ; + then + ./Tests/scripts/run_tests.sh "Demisto two before GA" + + else + echo "Not AMI run, can't run on this version" + fi - run: - name: Run Tests + name: Run Tests - Server Master shell: /bin/bash - command: ./Tests/scripts/run_tests.sh + when: always + command: | + if ./Tests/scripts/is_ami.sh ; + then + ./Tests/scripts/run_tests.sh "Server Master" + + else + echo "Not AMI run, can't run on this version" + fi - run: name: Slack Notifier shell: /bin/bash command: ./Tests/scripts/slack_notifier.sh when: always + - run: + name: Validate Docker Images + shell: /bin/bash + command: ./Tests/scripts/validate_docker_images.sh + when: always - run: name: Instance Test command: ./Tests/scripts/instance_test.sh when: always - run: name: Destroy Instances - command: ./Tests/scripts/destroy_instances.sh $CIRCLE_ARTIFACTS + command: python ./Tests/scripts/destroy_instances.py $CIRCLE_ARTIFACTS when: always - store_artifacts: path: artifacts destination: artifacts when: always + + +workflows: + version: 2 + commit: + jobs: + - build diff --git a/.github/config.yml b/.github/config.yml new file mode 100644 index 000000000000..1529de87d451 --- /dev/null +++ b/.github/config.yml @@ -0,0 +1,6 @@ +newPRWelcomeComment: > + Hi and welcome to the Demisto Content project! + Thank you and congrats on your first pull request, we will review it soon! + Until then you can check out our [documentation](https://github.com/demisto/content/tree/master/docs) for more details. + We would be thrilled to see you get involved in our [Slack DFIR community](https://go.demisto.com/join-our-slack-community) for discussions. + Hope you have a great time here :) \ No newline at end of file diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index e33bc8b903d2..bcca119bcacf 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,3 +1,5 @@ + + ## Status Ready/In Progress/In Hold(Reason for hold) @@ -30,5 +32,11 @@ x.x.x - [ ] Documentation (with link to it) - [ ] Code Review +## Dependencies +Mention the dependencies of the entity you changed as given from the precommit hooks in checkboxes, and tick after tested them. +- [ ] Dependency 1 +- [ ] Dependency 2 +- [ ] Dependency 3 + ## Additional changes Describe additional changes done, for example adding a function to common server. diff --git a/.gitignore b/.gitignore index fe72f9f370d2..d6050235005c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,21 @@ .DS_Store .idea -_site \ No newline at end of file +.vscode +_site +TestData/EmailWithNonUnicodeAttachmentName.eml +TestData/EmailWithNonUnicodeSubject.eml +*.pyc +.pytest_cache + +CommonServerPython.py +!Scripts/CommonServerPython/CommonServerPython.py +CommonServerUserPython.py +demistomock.py +Tests/filter_file.txt +Tests/id_set.json +.mypy_cache +Scripts/*/*_unified.yml +Integrations/*/*_unified.yml +Beta_Integrations/*/*_unified.yml +conftest.py +!Tests/scripts/dev_envs/pytest/conftest.py diff --git a/.guardrails/ignore b/.guardrails/ignore new file mode 100644 index 000000000000..6ef4617bd08f --- /dev/null +++ b/.guardrails/ignore @@ -0,0 +1 @@ +Integrations/Active_Directory_Query/key.pem diff --git a/.hooks/pre-commit b/.hooks/pre-commit index 013b9a5e750c..f166bb2318d4 100755 --- a/.hooks/pre-commit +++ b/.hooks/pre-commit @@ -1,15 +1,37 @@ #!/bin/bash # validating that each modified file has a valid schema, release notes, proper prefix & suffix -echo "Validating files" -python Tests/scripts/validate_files_structure.py +echo "Validating files..." +if [[ -z "${WINDIR}" ]] + then + PYTHONPATH="`pwd`:${PYTHONPATH}" python Tests/scripts/validate_files.py -t true + else + python Tests/scripts/validate_files.py +fi + +RES=$? + +echo "" +if [[ -n "$CONTENT_PRECOMMIT_RUN_DEV_TASKS" ]]; then + echo "Running content dev tasks (flake8, mypy, pylint, pytst) as env variable CONTENT_PRECOMMIT_RUN_DEV_TASKS is set." + ./Tests/scripts/run_all_pkg_dev_tasks.sh + RES=$(($RES + $?)) +else + echo "Skipping running dev tasks (flake8, mypy, pylint, pytest). If you want to run this as part of the precommit hook" + echo 'set CONTENT_PRECOMMIT_RUN_DEV_TASKS=1. You can add the following line to ~/.zshrc:' + echo 'echo "export CONTENT_PRECOMMIT_RUN_DEV_TASKS=1" >> ~/.zshrc' + echo "" + echo 'Or if you want to manually run dev tasks: ./Tests/scripts/pkg_dev_test_tasks.py -d ' + echo 'Example: ./Tests/scripts/pkg_dev_test_tasks.py -d Scripts/ParseEmailFiles' +fi -if [[ $? -ne 0 ]] -then +if [[ $RES -ne 0 ]] + then echo "Please fix the aforementioned errors and then commit again" exit 1 fi + # prevent push to master if [ -z "$1" ]; then protected_branch='master' @@ -19,3 +41,5 @@ if [ -z "$1" ]; then exit 1 fi fi + +echo "" diff --git a/Beta_Integrations/Blueliv/Blueliv.png b/Beta_Integrations/Blueliv/Blueliv.png new file mode 100644 index 000000000000..8e69f04ffb3a Binary files /dev/null and b/Beta_Integrations/Blueliv/Blueliv.png differ diff --git a/Beta_Integrations/Blueliv/Blueliv.py b/Beta_Integrations/Blueliv/Blueliv.py new file mode 100644 index 000000000000..26175601fc50 --- /dev/null +++ b/Beta_Integrations/Blueliv/Blueliv.py @@ -0,0 +1,100 @@ +import demistomock as demisto +from CommonServerPython import * +from CommonServerUserPython import * +''' IMPORTS ''' + +from sdk.blueliv_api import BluelivAPI + +''' GLOBALS/PARAMS ''' + +TOKEN = demisto.params().get('token') +URL = demisto.params()['url'] +SERVER = URL[:-1] if URL.endswith('/') else URL + +if not demisto.params().get('proxy', False): + del os.environ['HTTP_PROXY'] + del os.environ['HTTPS_PROXY'] + del os.environ['http_proxy'] + del os.environ['https_proxy'] + +''' HELPER FUNCTIONS ''' + + +def verify_response_code(response): + + if response.status_code != 200: + raise ValueError(response.error_msg) + + +''' COMMANDS + REQUESTS FUNCTIONS ''' + + +def test_module(): + + response = api.crime_servers.last('all') + verify_response_code(response) + demisto.results('ok') + + +def get_botips_feed_command(): + + response = api.bot_ips.recent('full') + verify_response_code(response) + human_readable = tableToMarkdown('Bot IP feed', response.items) + return_outputs(human_readable, {}) + + +def get_crimeservers_feed_command(): + + response = api.crime_servers.last('all') + verify_response_code(response) + human_readable = tableToMarkdown('Crimeservers feed', response.items) + return_outputs(human_readable, {}) + + +def get_malware_feed_command(): + + response = api.malwares.recent('all') + verify_response_code(response) + human_readable = tableToMarkdown('Malware feed', response.items) + return_outputs(human_readable, {}) + + +def get_attackingips_feed_command(): + + response = api.attacking_ips.recent('all') + verify_response_code(response) + human_readable = tableToMarkdown('Attacking IPs feed', response.items) + return_outputs(human_readable, {}) + + +def get_hacktivism_feed_command(): + + response = api.hacktivism_ops.last('all') + verify_response_code(response) + human_readable = tableToMarkdown('Hacktivism feed', response.items) + return_outputs(human_readable, {}) + + +''' COMMANDS MANAGER / SWITCH PANEL ''' + +COMMANDS = { + 'test-module': test_module, + 'blueliv-get-botips-feed': get_botips_feed_command, + 'blueliv-get-crimeservers-feed': get_crimeservers_feed_command, + 'blueliv-get-malware-feed': get_malware_feed_command, + 'blueliv-get-attackingips-feed': get_attackingips_feed_command, + 'blueliv-get-hacktivism-feed': get_hacktivism_feed_command +} + +try: + api = BluelivAPI( + base_url=SERVER, + token=TOKEN + ) + LOG('Command being called is {}'.format(demisto.command())) + command_func = COMMANDS.get(demisto.command()) + if command_func is not None: + command_func() +except Exception as e: + return_error(str(e)) diff --git a/Beta_Integrations/Blueliv/Blueliv.yml b/Beta_Integrations/Blueliv/Blueliv.yml new file mode 100644 index 000000000000..0193f76363b7 --- /dev/null +++ b/Beta_Integrations/Blueliv/Blueliv.yml @@ -0,0 +1,51 @@ +category: Data Enrichment & Threat Intelligence +commonfields: + id: Blueliv_Beta + version: -1 +configuration: +- defaultvalue: https://api.blueliv.com + display: Server URL (e.g., https://api.blueliv.com) + name: url + required: true + type: 0 +- display: API Token + name: token + required: true + type: 4 +- display: Use system proxy settings + name: proxy + type: 8 + required: false +description: Blueliv reduces risk through actionable, dynamic and targeted threat intelligence, trusted by your organization. +display: Blueliv (Beta) +name: Blueliv_Beta +script: + commands: + - description: Data set collection that gives the latest STIX Indicators about bot + ips gathered by Blueliv. + execution: false + name: blueliv-get-botips-feed + - description: Data set collection that gives the latest STIX Indicators about known + malicious servers gathered by Blueliv. + execution: false + name: blueliv-get-crimeservers-feed + - description: Data set collection that gives the latest STIX Indicators about malware + hashes gathered and analyzed by Blueliv. + execution: false + name: blueliv-get-malware-feed + - description: Data set collection that gives the latest STIX Indicators about attacking + IPs gathered and analyzed by Blueliv. + execution: false + name: blueliv-get-attackingips-feed + - description: 'Data related to the number of hacktivism tweets recently created. + Blueliv provides two types of feeds: the first one contains the most popular + hacktivism hashtags and the second one contains the countries where more number + of hacktivism tweets are coming from.' + execution: false + name: blueliv-get-hacktivism-feed + dockerimage: demisto/blueliv:1.0.0.165 + isfetch: false + runonce: false + script: '' + type: python +beta: true diff --git a/Beta_Integrations/Blueliv/Blueliv_description.md b/Beta_Integrations/Blueliv/Blueliv_description.md new file mode 100644 index 000000000000..cf9944213322 --- /dev/null +++ b/Beta_Integrations/Blueliv/Blueliv_description.md @@ -0,0 +1 @@ +Note: This is a beta Integration, which lets you implement and test pre-release software. Since the integration is beta, it might contain bugs. Updates to the integration during the beta phase might include non-backward compatible features. We appreciate your feedback on the quality and usability of the integration to help us identify issues, fix them, and continually improve. \ No newline at end of file diff --git a/Beta_Integrations/Exabeam/Exabeam.py b/Beta_Integrations/Exabeam/Exabeam.py new file mode 100644 index 000000000000..8c19c0a8237d --- /dev/null +++ b/Beta_Integrations/Exabeam/Exabeam.py @@ -0,0 +1,582 @@ +import demistomock as demisto +from CommonServerPython import * +from CommonServerUserPython import * + +''' IMPORTS ''' +import requests +from datetime import datetime + +# disable insecure warnings +requests.packages.urllib3.disable_warnings() + +if not demisto.params()['proxy']: + del os.environ['HTTP_PROXY'] + del os.environ['HTTPS_PROXY'] + del os.environ['http_proxy'] + del os.environ['https_proxy'] + +''' GLOBALS ''' +URL = demisto.params()['url'] +if URL[-1] != '/': + URL += '/' +URL_LOGIN = URL + 'api/' +URL_UBA = URL + 'uba/api/' +SESSION = requests.session() +SESSION.headers.update({'Accept': 'application/json'}) +if demisto.params()['insecure']: + SESSION.verify = False + +''' HELPERS ''' + + +def convert_unix_to_date(d): + """ Convert millise since epoch to date formatted MM/DD/YYYY HH:MI:SS """ + if d: + dt = datetime.utcfromtimestamp(d / 1000) + return dt.strftime('%m/%d/%Y %H:%M:%S') + return 'N/A' + + +def convert_date_to_unix(d): + """ Convert a given date to millis since epoch """ + return int((d - datetime.utcfromtimestamp(0)).total_seconds() * 1000) + + +def login(): + """ Login using the credentials and store the cookie """ + http_request('POST', URL_LOGIN + 'auth/login', data={ + 'username': demisto.params()['credentials']['identifier'], + 'password': demisto.params()['credentials']['password'] + }) + + +def logout(): + """ Logout from the session """ + http_request('GET', URL_LOGIN + 'auth/logout', None) + + +def http_request(method, path, data): + """ Do the actual HTTP request """ + if method == 'GET': + response = SESSION.get(path, params=data) + else: + response = SESSION.post(path, data=data) + if response.status_code != requests.codes.ok: + text = response.text + if text: + try: + res = response.json() + text = 'Code: [%s], Error: [%s]' % (res.get('_apiErrorCode'), res.get('internalError')) + except Exception: + pass + return_error('Error in API call to Exabeam [%d] - %s' % (response.status_code, text)) + if not response.text: + return {} + return response.json() + + +def get_watchlist_id(): + """ Return watchlist id based on given parameters """ + if not demisto.args()['id'] and not demisto.args()['title']: + logout() + return_error('Please provide either ID or title') + wid = demisto.args()['id'] + if not wid: + watchlist = http_request('GET', URL_UBA + 'watchlist', None) + for item in watchlist: + if item.get('title').lower() == demisto.args()['title'].lower(): + watchlist_id = item.get('watchlistId') + break + if not watchlist_id: + logout() + return_error('Unable to find watchlist with the given title') + return watchlist + + +''' FUNCTIONS ''' + + +def exabeam_users(): + """ Return user statistics """ + res = http_request('GET', URL_UBA + 'kpi/count/users', None) + demisto.results({ + 'Type': entryTypes['note'], + 'ContentsFormat': formats['json'], + 'Contents': res, + 'HumanReadable': tableToMarkdown('User statistics', [res], ['highRisk', 'recent', 'total']) + }) + + +def exabeam_assets(): + """ Return asset statistics """ + res = http_request('GET', URL_UBA + 'kpi/count/assets', None) + demisto.results({ + 'Type': entryTypes['note'], + 'ContentsFormat': formats['json'], + 'Contents': res, + 'HumanReadable': tableToMarkdown('Asset statistics', [res], ['highRisk', 'recent', 'total']) + }) + + +def exabeam_sessions(): + """ Return session statistics """ + res = http_request('GET', URL_UBA + 'kpi/count/sessions', None) + demisto.results({ + 'Type': entryTypes['note'], + 'ContentsFormat': formats['json'], + 'Contents': res, + 'HumanReadable': tableToMarkdown('Session statistics', [res], ['highRisk', 'recent', 'total']) + }) + + +def exabeam_events(): + """ Return event statistics """ + res = http_request('GET', URL_UBA + 'kpi/count/events', None) + demisto.results({ + 'Type': entryTypes['note'], + 'ContentsFormat': formats['json'], + 'Contents': res, + 'HumanReadable': tableToMarkdown('Event statistics', [res], ['recent', 'total']) + }) + + +def exabeam_anomalies(): + res = http_request('GET', URL_UBA + 'kpi/count/anomalies', None) + demisto.results({ + 'Type': entryTypes['note'], + 'ContentsFormat': formats['json'], + 'Contents': res, + 'HumanReadable': tableToMarkdown('Anomalies statistics', [res], ['recent', 'total']) + }) + + +def exabeam_notable(): + """ Return notable users in a specific period of time """ + res = http_request( + 'GET', + URL_UBA + 'users/notable', + { + 'numberOfResults': demisto.args()['number-of-results'], + 'unit': demisto.args()['unit'], + 'num': demisto.args()['num'] + } + ) + + if res.get('users'): + users = [{ + 'Highest': u['highestRiskScore'], + 'Name': u['userFullName'], + 'Username': demisto.get(u, 'user.username'), + 'Email': demisto.get(u, 'user.info.email'), + 'Department': demisto.get(u, 'user.info.department'), + 'DN': demisto.get(u, 'user.info.dn'), + 'RiskScore': demisto.get(u, 'user.riskScore'), + 'NotableSessionIDs': u.get('notableSessionIds', []) + } for u in res['users']] + + demisto.results({ + 'Type': entryTypes['note'], + 'ContentsFormat': formats['json'], + 'Contents': res, + 'HumanReadable': tableToMarkdown('Notables', users, + ['Name', 'Username', 'Email', 'Department', 'DN', 'RiskScore', 'Highest', + 'NotableSessionIDs']), + 'EntryContext': {'Exabeam.Notable': res['users']} + }) + + else: + demisto.results('No notable users found in the requested period') + + +def exabeam_lockouts(): + """ Return lockouts """ + res = http_request( + 'GET', + URL_UBA + 'lockouts/accountLockouts', + { + 'numberOfResults': demisto.getArg('number-of-results'), + 'unit': demisto.getArg('unit'), + 'num': demisto.getArg('num') + }) + if res.get('lockouts'): + lockouts = [{ + 'Name': demisto.get(l, 'user.info.fullName'), + 'Username': demisto.get(l, 'user.username'), + 'Email': demisto.get(l, 'user.info.email'), + 'Department': demisto.get(l, 'user.info.department'), + 'DN': demisto.get(l, 'user.info.dn'), + 'Title': demisto.get(l, 'user.info.title'), + 'RiskScore': demisto.get(l, 'user.riskScore'), + 'Executive': demisto.get(l, 'isUserExecutive'), + 'LockoutTime': convert_unix_to_date(demisto.get(l, 'firstLockoutEvent.time')), + 'Host': demisto.get(l, 'firstLockoutEvent.host'), + 'LockoutRisk': demisto.get(l, 'lockoutInfo.riskScore'), + 'LoginHost': demisto.get(l, 'lockoutInfo.loginHost') + } for l in res['lockouts']] + + demisto.results({ + 'Type': entryTypes['note'], + 'ContentsFormat': formats['json'], + 'Contents': res, + 'HumanReadable': tableToMarkdown('Lockouts', lockouts, + ['User', 'Username', 'Email', 'Department', 'DN', 'Title', 'RiskScore', + 'Executive', 'LockoutTime', 'Host', 'LockoutRisk', 'LoginHost']), + 'EntryContext': {'Exabeam.Lockout': res['lockouts']} + }) + else: + demisto.results('No lockouts found in the requested period') + + +def exabeam_timeline(): + """ Returns session, triggered rules and events of a user """ + res = http_request('GET', URL_UBA + 'user/%s/timeline/entities/all' % demisto.args()['username'], None) + risk_score = 0 + session = '' + for entity in res.get('entities', []): + if entity.get('tp') == 'session' and entity.get('rs', 0) > risk_score: + risk_score = entity.get('rs', 0) + session = entity.get('id') + if session: + session_info = http_request('GET', URL_UBA + 'session/%s/info' % session, None) + si = session_info.get('sessionInfo') + if not si: + return_error('Unable to find session info') + session_data = { + 'Username': si.get('username'), + 'RiskScore': si.get('riskScore'), + 'InitialRiskScore': si.get('initialRiskScore'), + 'NumOfReasons': si.get('numOfReasons'), + 'LoginHost': si.get('loginHost'), + 'Zones': ','.join(si.get('zones', [])), + 'Assets': si.get('numOfAssets'), + 'Events': si.get('numOfEvents'), + 'SecurityEvents': si.get('numOfSecurityEvents') + } + md = tableToMarkdown( + 'Session %s from %s to %s' % (session, convert_unix_to_date(si.get('startTime')), + convert_unix_to_date(si.get('endTime'))), [session_data], + ['Username', 'RiskScore', 'InitialRiskScore', 'NumOfReasons', 'LoginHost', 'Zones', 'Assets', 'Events', + 'SecurityEvents']) + + triggered_rules_data = [{ + 'ID': tr.get('ruleId'), + 'Type': tr.get('ruleType'), + 'Name': demisto.get(session_info, 'rules.%s.ruleName' % (tr.get('ruleId'))), + 'EventID': tr.get('eventId'), + 'SessionID': tr.get('sessionId'), + 'Source': demisto.get(session_info, 'triggeredRuleEvents.%s.fields.source' % (tr.get('eventId'))), + 'Domain': demisto.get(session_info, 'triggeredRuleEvents.%s.fields.domain' % (tr.get('eventId'))), + 'Host': demisto.get(session_info, 'triggeredRuleEvents.%s.fields.host' % (tr.get('eventId'))), + 'DestIP': demisto.get(session_info, 'triggeredRuleEvents.%s.fields.dest_ip' % (tr.get('eventId'))), + 'EventType': demisto.get(session_info, 'triggeredRuleEvents.%s.fields.event_type' % (tr.get('eventId'))) + } for tr in session_info.get('triggeredRules')] + + md += '\n' + tableToMarkdown('Triggered Rules', + triggered_rules_data, + ['ID', 'Type', 'Name', 'EventID', 'SessionID', 'EventType', 'Source', 'Domain', + 'Host', 'DestIP']) + session_data['TriggeredRules'] = triggered_rules_data + events = http_request( + 'GET', + URL_UBA + 'timeline/events/start', + { + 'username': demisto.args()['username'], + 'sequenceTypes': 'session', + 'startSequenceType': 'session', + 'startSequenceId': session, + 'preferredNumberOfEvents': 200 + } + ) + + events_data = [{ + 'Type': ev.get('tp'), + 'Count': ev.get('c'), + 'Start': convert_unix_to_date(ev.get('ts')), + 'End': convert_unix_to_date(ev.get('te')), + 'Sources': [es.get('fields', {}).get('source') for es in ev.get('es')] + } for ev in events.get('aggregatedEvents', [])] + + md += '\n' + tableToMarkdown('Timeline', events_data, ['Type', 'Count', 'Start', 'End']) + session_data['Events'] = events_data + + demisto.results({ + 'Type': entryTypes['note'], + 'ContentsFormat': formats['json'], + 'Contents': events, + 'HumanReadable': md, + 'EntryContext': {'Exabeam.Timeline': session_data} + }) + else: + demisto.results('No risk score exists for the given user') + + +def exabeam_session_entities(): + """ Returns session entities for a given user, can be filtered by container-type, container-id """ + res = http_request( + 'GET', + URL_UBA + 'user/%s/timeline/entities' % demisto.args()['username'], + { + 'numberOfResults': demisto.args()['number-of-results'], + 'unit': demisto.args()['unit'], + 'num': demisto.args()['num'], + 'endContainerType': demisto.args()['container-type'], + 'endContainerId': demisto.args()['container-id'] + } + ) + + demisto.results({ + 'Type': entryTypes['note'], + 'ContentsFormat': formats['json'], + 'Contents': res + }) + + +def exabeam_user_info(): + """ Returns user info """ + username = demisto.args()['username'] + res = http_request('GET', URL_UBA + 'user/%s/info' % username, None) + if res.get('username'): + u = { + 'Username': res['username'], + 'AccountNames': ','.join(res.get('accountNames', [])), + 'Executive': res['isExecutive'], + 'WatchList': res['isOnWatchlist'], + 'Name': demisto.get(res, 'userInfo.info.fullName'), + 'ID': demisto.get(res, 'userInfo.info.accountId'), + 'Department': demisto.get(res, 'userInfo.info.department'), + 'DN': demisto.get(res, 'userInfo.info.dn'), + 'Email': demisto.get(res, 'userInfo.info.email'), + 'Type': demisto.get(res, 'userInfo.info.employeeType'), + 'Groups': demisto.get(res, 'userInfo.info.group'), + 'SID': demisto.get(res, 'userInfo.info.sid'), + 'Title': demisto.get(res, 'userInfo.info.title'), + 'RiskScore': demisto.get(res, 'userInfo.riskScore'), + 'AverageRiskScore': demisto.get(res, 'userInfo.averageRiskScore'), + 'Labels': demisto.get(res, 'userInfo.labels'), + 'FirstSeen': convert_unix_to_date(demisto.get(res, 'userInfo.firstSeen')), + 'LastSeen': convert_unix_to_date(demisto.get(res, 'userInfo.lastSeen')), + 'LastSessionID': demisto.get(res, 'userInfo.lastSessionId'), + 'PastScores': ','.join(map(str, demisto.get(res, 'userInfo.pastScores'))) + } + + md = tableToMarkdown('User info', [u], ['Name', 'Username', 'Email', 'Department', 'DN', 'Groups', + 'Title', 'RiskScore', 'AverageRiskScore', 'Executive', 'WatchList', + 'AccountNames', 'ID', + 'Type', 'SID', 'Labels', 'FirstSeen', 'LastSeen', 'LastSessionID', + 'PastScores']) + + if demisto.get(res, 'userInfo.info.photo'): + md += '\n![Photo](data:image/png;base64,' + demisto.get(res, 'userInfo.info.photo') + ')\n' + + # Let's get the sessions as well + notable_res = http_request( + 'GET', + URL_UBA + 'users/notable', + { + 'numberOfResults': 100, + 'unit': 'd', + 'num': 7 + } + ) + if notable_res.get('users'): + for un in notable_res['users']: + if demisto.get(un, 'user.username') == username: + u['NotableList'] = True + md += '\n## User is on the notable list\n' + notable_session_ids = un.get('notableSessionIds', []) + if notable_session_ids: + u['NoteableSessionIDs'] = notable_session_ids + session_res = http_request( + 'GET', + URL_UBA + 'user/%s/riskTimeline/data' % username, + { + 'unit': 'd', + 'num': 7, + 'endTimeSequenceType': 'session', + 'endTimeSequenceId': notable_session_ids[0] + } + ) + if session_res.get('sessions'): + md += '\n' + tableToMarkdown('Sessions', session_res['sessions']) + + demisto.results({ + 'Type': entryTypes['note'], + 'ContentsFormat': formats['json'], + 'Contents': res, + 'HumanReadable': md, + 'EntryContext': { + 'Account(val.Email && val.Email === obj.Email || val.ID && val.ID === obj.ID ||' + ' val.Username && val.Username === obj.Username)': u} + }) + + else: + demisto.results('No username with [' + username + '] found') + + +def exabeam_triggered_rules(): + """ Return triggered rules for a given container """ + res = http_request( + 'GET', + URL_UBA + 'triggeredRules', + { + 'containerType': demisto.args()['container-type'], + 'containerId': demisto.args()['container-id'] + } + ) + + demisto.results({ + 'Type': entryTypes['note'], + 'ContentsFormat': formats['json'], + 'Contents': res + }) + + +def exabeam_watchlists(): + """ Retrieve current list of watchlists """ + res = http_request('GET', URL_UBA + 'watchlist', None) + + demisto.results({ + 'Type': entryTypes['note'], + 'ContentsFormat': formats['json'], + 'Contents': res, + 'HumanReadable': tableToMarkdown('Watchlists', res, ['title', 'watchlistId']), + 'EntryContext': {'Exabeam.Watchlists': res} + }) + + +def exabeam_watchlist(): + watchlist_id = get_watchlist_id() + res = http_request('GET', URL_UBA + 'watchlist/%s/' % watchlist_id, {'numberOfResults': demisto.args()['num']}) + + users = [{ + 'Name': demisto.get(u, 'user.info.fullName'), + 'Department': demisto.get(u, 'user.info.department'), + 'Username': u.get('username'), + 'RiskScore': demisto.get(u, 'user.riskScore'), + 'IsExecutive': u.get('isExecutive') + } for u in res.get('users', [])] + + demisto.results({ + 'Type': entryTypes['note'], + 'ContentsFormat': formats['json'], + 'Contents': res, + 'HumanReadable': tableToMarkdown( + 'Watchlist %s [%s] - %d users' % (res.get('title'), res.get('category'), res.get('totalNumberOfUsers')), + users, + ['Name', 'Department', 'Username', 'RiskScore', 'IsExecutive']), + 'EntryContext': {'Exabeam.Watchlist.%s' % res.get('title'): users} + }) + + +def exabeam_watchlist_add(): + """ Adds a user to a given watchlist """ + watchlist_id = get_watchlist_id() + username = demisto.args()['username'] + res = http_request( + 'PUT', + URL_UBA + 'watchlist/%s/add' % watchlist_id, + { + 'items[]': username, + 'category': 'Users' + } + ) + + if res.get('numberAdded') == 1: + md = 'User %s added to watchlist %s' % (username, res.get('title')) + else: + md = 'User %s was already on watchlist %s' % (username, res.get('title')) + + demisto.results({ + 'Type': entryTypes['note'], + 'ContentsFormat': formats['json'], + 'Contents': res, + 'HumanReadable': md + }) + + +def exabeam_watchlist_remove(): + watchlist_id = get_watchlist_id() + username = demisto.args()['username'] + res = http_request('PUT', URL_UBA + 'watchlist/%s/remove' % watchlist_id, { + 'items[]': username, + 'category': 'Users', + 'watchlistId': watchlist_id + }) + if res.get('numberRemoved') == 1: + md = 'User %s removed from watchlist %s' % (username, res.get('title')) + else: + md = 'User %s was not on watchlist %s' % (username, res.get('title')) + + demisto.results({ + 'Type': entryTypes['note'], + 'ContentsFormat': formats['json'], + 'Contents': res, + 'HumanReadable': md + }) + + +''' EXECUTION ''' +login() + +LOG('command is %s' % (demisto.command(),)) + +try: + if demisto.command() == 'test-module': + demisto.results('ok') + + elif demisto.command() == 'xb-users': + exabeam_users() + + elif demisto.command() == 'xb-assets': + exabeam_assets() + + elif demisto.command() == 'xb-sessions': + exabeam_sessions() + + elif demisto.command() == 'xb-events': + exabeam_events() + + elif demisto.command() == 'xb-anomalies': + exabeam_anomalies() + + elif demisto.command() == 'xb-notable': + exabeam_notable() + + elif demisto.command() == 'xb-lockouts': + exabeam_lockouts() + + elif demisto.command() == 'xb-timeline': + exabeam_timeline() + + elif demisto.command() == 'xb-session-entities': + exabeam_session_entities() + + # elif demisto.command() == 'xb-userinfo': + # exabeam_userinfo() + + # elif demisto.command() == 'xb-triggered-rules': + # exabeam_triggerred_rules() + + elif demisto.command() == 'xb-watchlists': + exabeam_watchlists() + + elif demisto.command() == 'xb-watchlist': + exabeam_watchlist() + + elif demisto.command() == 'xb-watchlist-add': + exabeam_watchlist_add() + + elif demisto.command() == 'xb-watchlist-remove': + exabeam_watchlist_remove() + + else: + logout() + return_error('Unrecognized command: ' + demisto.command()) + + +except Exception as e: + LOG(e.message) + LOG.print_log() + return_error(e.message) + +logout() diff --git a/Beta_Integrations/Exabeam/Exabeam.yml b/Beta_Integrations/Exabeam/Exabeam.yml new file mode 100644 index 000000000000..dbf9d0fdbc68 --- /dev/null +++ b/Beta_Integrations/Exabeam/Exabeam.yml @@ -0,0 +1,458 @@ +category: Analytics & SIEM +commonfields: + id: Exabeam + version: -1 +configuration: +- display: Server URL + name: url + required: true + type: 0 +- display: Credentials + name: credentials + required: true + type: 9 +- defaultvalue: 'true' + display: Use system proxy settings + name: proxy + required: false + type: 8 +- defaultvalue: 'false' + display: Do not validate server certificate (insecure) + name: insecure + required: false + type: 8 +description: Exabeam Beta integration +display: Exabeam (Beta) +name: Exabeam (Beta) +script: + commands: + - deprecated: false + description: Return the total number of users managed by Exabeam + execution: false + name: xb-users + - deprecated: false + description: Return the total number of assets managed by Exabeam + execution: false + name: xb-assets + - deprecated: false + description: Return the total number of tracked sessions by Exabeam + execution: false + name: xb-sessions + - deprecated: false + description: Return the total number of events processed by Exabeam + execution: false + name: xb-events + - deprecated: false + description: Display anomaly statistics + execution: false + name: xb-anomalies + - arguments: + - default: true + defaultValue: '100' + description: Number of records to return + isArray: false + name: number-of-results + required: false + secret: false + - auto: PREDEFINED + default: false + defaultValue: d + description: The unit of the num argument. Can be d for days, w for weeks, M + for months + isArray: false + name: unit + predefined: + - d + - M + required: false + secret: false + - default: false + defaultValue: '1' + description: The number of units (days, weeks, etc.) + isArray: false + name: num + required: false + secret: false + deprecated: false + description: Display the notable users + execution: false + name: xb-notable + outputs: + - contextPath: Exabeam.Notable.Highest + description: Highest risk score of the user + type: number + - contextPath: Exabeam.Notable.Name + description: User full name + type: string + - contextPath: Exabeam.Notable.Username + description: User name + type: string + - contextPath: Exabeam.Notable.Email + description: User email + type: string + - contextPath: Exabeam.Notable.Department + description: User department + type: string + - contextPath: Exabeam.Notable.DN + description: User dn + type: string + - contextPath: Exabeam.Notable.RiskScore + description: User risk score + type: number + - contextPath: Exabeam.Notable.NotableSessionIDs + description: User notable session Ids + type: string + - arguments: + - default: true + defaultValue: '100' + description: Number of records to return + isArray: false + name: number-of-results + required: false + secret: false + - auto: PREDEFINED + default: false + defaultValue: d + description: The unit of the num argument. Can be d for days, w for weeks, M + for months + isArray: false + name: unit + predefined: + - d + - M + required: false + secret: false + - default: false + defaultValue: '1' + description: The number of units (days, weeks, etc.) + isArray: false + name: num + required: false + secret: false + deprecated: false + description: List all the Exabeam lockout users. + execution: false + name: xb-lockouts + outputs: + - contextPath: Exabeam.Lockout.isUserExecutive + description: Is the user an executive + type: boolean + - contextPath: Exabeam.Lockout.user.username + description: Username of user + type: string + - contextPath: Exabeam.Lockout.user.riskScore + description: Risk score of user + type: number + - contextPath: Exabeam.Lockout.user.firstSeen + description: When did we first see the user + type: date + - contextPath: Exabeam.Lockout.user.lastSeen + description: When did we last see the user + type: date + - contextPath: Exabeam.Lockout.user.lastSessionId + description: Last session id of the user + type: string + - contextPath: Exabeam.Lockout.user.info.department + description: User department + type: string + - contextPath: Exabeam.Lockout.user.info.dn + description: User DN + type: string + - contextPath: Exabeam.Lockout.user.info.email + description: User email + type: string + - contextPath: Exabeam.Lockout.user.info.fullName + description: User full name + type: string + - contextPath: Exabeam.Lockout.user.info.group + description: User groups + type: string + - contextPath: Exabeam.Lockout.user.info.location + description: User location + type: string + - contextPath: Exabeam.Lockout.user.info.manager + description: Users' manager + type: string + - contextPath: Exabeam.Lockout.user.info.sid + description: User identifier + type: string + - contextPath: Exabeam.Lockout.user.info.title + description: User title + type: string + - contextPath: Exabeam.Lockout.lockoutInfo.lockoutId + description: ID of the lockout + type: string + - contextPath: Exabeam.Lockout.lockoutInfo.loginHost + description: The login host for lockout + type: Unknown + - contextPath: Exabeam.Lockout.lockoutInfo.riskScore + description: Risk score for lockout + type: number + - contextPath: Exabeam.Lockout.lockoutInfo.isRisky + description: Is this risky + type: boolean + - arguments: + - default: true + description: The username to act upon + isArray: false + name: username + required: true + secret: false + deprecated: false + description: Display the timeline events for a given user + execution: false + name: xb-timeline + - arguments: + - default: true + description: The username to act upon + isArray: false + name: username + required: true + secret: false + - default: true + defaultValue: '100' + description: Number of records to return + isArray: false + name: number-of-results + required: false + secret: false + - auto: PREDEFINED + default: false + defaultValue: d + description: The unit of the num argument. Can be d for days, w for weeks, M + for months + isArray: false + name: unit + predefined: + - d + - w + - M + required: false + secret: false + - default: false + defaultValue: '1' + description: The number of units (days, weeks, etc.) + isArray: false + name: num + required: false + secret: false + - default: false + defaultValue: session + description: Container type for the filter - accepts container types like session, + etc. + isArray: false + name: container-type + required: false + secret: false + - default: false + description: The container ID we want to filter by + isArray: false + name: container-id + required: false + secret: false + deprecated: false + description: Display the session entities for a given user filter by container + execution: false + name: xb-session-entities + - arguments: + - default: true + description: The username to act upon + isArray: false + name: username + required: true + secret: false + deprecated: false + description: Display information about the given user + execution: false + name: xb-userinfo + outputs: + - contextPath: Account.Username + description: Username of user + type: string + - contextPath: Account.AccountNames + description: All account names we know about + type: string + - contextPath: Account.Executive + description: Is this user an executive + type: boolean + - contextPath: Account.WatchList + description: Is this user on a watch list + type: boolean + - contextPath: Account.Name + description: Name of he user + type: string + - contextPath: Account.ID + description: Account ID of the user + type: string + - contextPath: Account.Department + description: Department of user + type: string + - contextPath: Account.DN + description: DN of user + type: string + - contextPath: Account.Email + description: Email of user + type: string + - contextPath: Account.Type + description: Type of account + type: string + - contextPath: Account.Groups + description: Groups for the user + type: string + - contextPath: Account.SID + description: SID of the user + type: string + - contextPath: Account.Title + description: Title of the user + type: string + - contextPath: Account.RiskScore + description: Risk score of the user + type: number + - contextPath: Account.AverageRiskScore + description: Average risk score of the user + type: number + - contextPath: Account.Labels + description: Any labels assigned to the user + type: string + - contextPath: Account.FirstSeen + description: First time user was seen + type: date + - contextPath: Account.LastSeen + description: Last time user was seen + type: date + - contextPath: Account.LastSessionID + description: Last session ID of the user + type: string + - contextPath: Account.PastScores + description: All past scores of the user + type: number + - contextPath: Account.LoginHost + description: The last session login host + type: string + - contextPath: Account.LoginLabel + description: Last session login label + type: string + - contextPath: Account.NotableList + description: Is the user on the notable list + type: boolean + - contextPath: Account.NotableSessionIDs + description: List of session IDs + type: string + - arguments: + - default: false + defaultValue: session + description: Container type for the filter - accepts container types like session, + etc. + isArray: false + name: container-type + required: false + secret: false + - default: false + description: The container ID we want to filter by + isArray: false + name: container-id + required: false + secret: false + deprecated: false + description: Display the triggered rules for a given container + execution: false + name: xb-triggered-rules + - deprecated: false + description: Retrieve the list of watchlists we currently have + execution: false + name: xb-watchlists + outputs: + - contextPath: Exabeam.Watchlists + description: Watchlists + type: Unknown + - arguments: + - default: true + description: Watchlist ID to retrieve data from + isArray: false + name: id + required: false + secret: false + - default: false + description: Watchlist title to retrieve data from + isArray: false + name: title + required: false + secret: false + - default: false + defaultValue: '100' + description: Number of users to retrieve + isArray: false + name: num + required: false + secret: false + deprecated: false + description: Retrieve the users on a given watchlist. You must provide either + id or title. + execution: false + name: xb-watchlist + outputs: + - contextPath: Exabeam.Watchlists + description: Watchlists + type: Unknown + - arguments: + - default: true + description: Watchlist ID to add user to + isArray: false + name: id + required: false + secret: false + - default: false + description: Watchlist title to add user to + isArray: false + name: title + required: false + secret: false + - default: false + description: The username to act upon + isArray: false + name: username + required: true + secret: false + - default: false + defaultValue: '7' + description: How many days should we watch the given user + isArray: false + name: watch-until-days + required: false + secret: false + deprecated: false + description: Add a user to the watchlist. You must provide either id or title. + execution: true + name: xb-watchlist-add + - arguments: + - default: true + description: Watchlist ID to remove user from + isArray: false + name: id + required: false + secret: false + - default: false + description: Watchlist title to remove user from + isArray: false + name: title + required: false + secret: false + - default: false + description: The username to act upon + isArray: false + name: username + required: true + secret: false + deprecated: false + description: Remove a user from the watchlist. You must provide either id or title. + execution: true + name: xb-watchlist-remove + isfetch: false + longRunning: false + longRunningPort: false + runonce: false + script: '-' + type: python +tests: +- No test +beta: true diff --git a/Beta_Integrations/Exabeam/Exabeam_description.md b/Beta_Integrations/Exabeam/Exabeam_description.md new file mode 100644 index 000000000000..e91ea3697156 --- /dev/null +++ b/Beta_Integrations/Exabeam/Exabeam_description.md @@ -0,0 +1 @@ +Note: This is a beta Integration, which lets you implement and test pre-release software. Since the integration is beta, it might contain bugs. Updates to the integration during the beta phase might include non-backward compatible features. We appreciate your feedback on the quality and usability of the integration to help us identify issues, fix them, and continually improve. \ No newline at end of file diff --git a/Beta_Integrations/Exabeam/Exabeam_image.png b/Beta_Integrations/Exabeam/Exabeam_image.png new file mode 100644 index 000000000000..9642aa150340 Binary files /dev/null and b/Beta_Integrations/Exabeam/Exabeam_image.png differ diff --git a/Beta_Integrations/MailListener_-_POP3/MailListener_-_POP3.py b/Beta_Integrations/MailListener_-_POP3/MailListener_-_POP3.py new file mode 100644 index 000000000000..5e277f4883cb --- /dev/null +++ b/Beta_Integrations/MailListener_-_POP3/MailListener_-_POP3.py @@ -0,0 +1,363 @@ +import demistomock as demisto + +from CommonServerPython import * +from CommonServerUserPython import * + +import poplib +import base64 +import quopri +from email.parser import Parser +from htmlentitydefs import name2codepoint +from HTMLParser import HTMLParser, HTMLParseError + + +''' GLOBALS/PARAMS ''' +SERVER = demisto.params().get('server', '') +EMAIL = demisto.params().get('email', '') +PASSWORD = demisto.params().get('password', '') +PORT = int(demisto.params().get('port', '995')) +SSL = demisto.params().get('ssl') +FETCH_TIME = demisto.params().get('fetch_time', '7 days') + +# pop3 server connection object. +pop3_server_conn = None # type: ignore + +TIME_REGEX = re.compile(r'^([\w,\d: ]*) (([+-]{1})(\d{2}):?(\d{2}))?[\s\w\(\)]*$') +DATE_FORMAT = '%Y-%m-%dT%H:%M:%SZ' + + +def connect_pop3_server(): + global pop3_server_conn + + if pop3_server_conn is None: + if SSL: + pop3_server_conn = poplib.POP3_SSL(SERVER, PORT) # type: ignore + else: + pop3_server_conn = poplib.POP3(SERVER, PORT) # type: ignore + + pop3_server_conn.getwelcome() # type: ignore + pop3_server_conn.user(EMAIL) # type: ignore + pop3_server_conn.pass_(PASSWORD) # type: ignore + + +def close_pop3_server_connection(): + global pop3_server_conn + if pop3_server_conn is not None: + pop3_server_conn.quit() + pop3_server_conn = None + + +def get_user_emails(): + _, mails_list, _ = pop3_server_conn.list() # type: ignore + + mails = [] + index = '' + + for mail in mails_list: + try: + index = mail.split(' ')[0] + (resp_message, lines, octets) = pop3_server_conn.retr(index) # type: ignore + msg_content = unicode(b'\r\n'.join(lines), errors='ignore').encode("utf-8") + msg = Parser().parsestr(msg_content) + msg['index'] = index + mails.append(msg) + except Exception as e: + demisto.error("Failed to get email with index " + index + 'from the server.') + raise e + + return mails + + +def get_attachment_name(headers): + name = headers.get('content-description', '') + + if re.match(r'^.+\..{3,5}$', name): + return name + + content_disposition = headers.get('content-disposition', '') + + if content_disposition: + m = re.search('filename="(.*?)"', content_disposition) + if m: + name = m.group(1) + + if re.match('^.+\..{3,5}$', name): + return name + + extension = re.match(r'.*[\\/]([\d\w]{2,4}).*', headers.get('content-type', 'txt')).group(1) # type: ignore + + return name + '.' + extension + + +def parse_base64(text): + if re.match("^=?.*?=$", text): + res = re.search('=\?.*?\?[A-Z]{1}\?(.*?)\?=', text, re.IGNORECASE) + if res: + res = res.group(1) + return base64.b64decode(res) # type: ignore + return text + + +class TextExtractHtmlParser(HTMLParser): + def __init__(self): + HTMLParser.__init__(self) + self._texts = [] # type: list + self._ignore = False + + def handle_starttag(self, tag, attrs): + if tag in ('p', 'br') and not self._ignore: + self._texts.append('\n') + elif tag in ('script', 'style'): + self._ignore = True + + def handle_startendtag(self, tag, attrs): + if tag in ('br', 'tr') and not self._ignore: + self._texts.append('\n') + + def handle_endtag(self, tag): + if tag in ('p', 'tr'): + self._texts.append('\n') + elif tag in ('script', 'style'): + self._ignore = False + + def handle_data(self, data): + if data and not self._ignore: + stripped = data.strip() + if stripped: + self._texts.append(re.sub(r'\s+', ' ', stripped)) + + def handle_entityref(self, name): + if not self._ignore and name in name2codepoint: + self._texts.append(unichr(name2codepoint[name])) + + def handle_charref(self, name): + if not self._ignore: + if name.startswith('x'): + c = unichr(int(name[1:], 16)) + else: + c = unichr(int(name)) + self._texts.append(c) + + def get_text(self): + return "".join(self._texts) + + +def html_to_text(html): + parser = TextExtractHtmlParser() + try: + parser.feed(html) + parser.close() + except HTMLParseError: + pass + return parser.get_text() + + +def get_email_context(email_data): + context_headers = email_data._headers + context_headers = [{'Name': v[0], 'Value': v[1]} + for v in context_headers] + headers = dict([(h['Name'].lower(), h['Value']) for h in context_headers]) + + context = { + 'Mailbox': EMAIL, + 'ID': email_data.get('Message-ID', 'None'), + 'Labels': ', '.join(email_data.get('labelIds', '')), + 'Headers': context_headers, + 'Format': headers.get('content-type', '').split(';')[0], + 'Subject': parse_base64(headers.get('subject')), + 'Body': email_data._payload, + 'From': headers.get('from'), + 'To': headers.get('to'), + 'Cc': headers.get('cc', []), + 'Bcc': headers.get('bcc', []), + 'Date': headers.get('date', ''), + 'Html': None, + } + + if 'text/html' in context['Format']: + context['Html'] = context['Body'] + context['Body'] = html_to_text(context['Body']) + + if 'multipart' in context['Format']: + context['Body'], context['Html'], context['Attachments'] = parse_mail_parts(email_data._payload) + context['Attachment Names'] = ', '.join( + [attachment['Name'] for attachment in context['Attachments']]) + + raw = dict(email_data) + raw['Body'] = context['Body'] + context['RawData'] = json.dumps(raw) + return context, headers + + +def parse_mail_parts(parts): + body = unicode("", "utf-8") + html = unicode("", "utf-8") + + attachments = [] # type: ignore + for part in parts: + context_headers = part._headers + context_headers = [{'Name': v[0], 'Value': v[1]} + for v in context_headers] + headers = dict([(h['Name'].lower(), h['Value']) for h in context_headers]) + + content_type = headers.get('content-type', 'text/plain') + + is_attachment = headers.get('content-disposition', '').startswith('attachment')\ + or headers.get('x-attachment-id') or "image" in content_type + + if 'multipart' in content_type or isinstance(part._payload, list): + part_body, part_html, part_attachments = parse_mail_parts(part._payload) + body += part_body + html += part_html + attachments.extend(part_attachments) + elif not is_attachment: + if headers.get('content-transfer-encoding') == 'base64': + text = base64.b64decode(part._payload).decode('utf-8') + elif headers.get('content-transfer-encoding') == 'quoted-printable': + decoded_string = quopri.decodestring(part._payload) + text = unicode(decoded_string, "utf-8") + else: + text = quopri.decodestring(part._payload) + + if not isinstance(text, unicode): + text = text.decode('unicode-escape') + + if 'text/html' in content_type: + html += text + else: + body += text + + else: + attachments.append({ + 'ID': headers.get('x-attachment-id', 'None'), + 'Name': get_attachment_name(headers), + 'Data': part._payload + }) + + return body, html, attachments + + +def parse_time(t): + base_time, _, _, _, _ = TIME_REGEX.findall(t)[0] + return datetime.strptime(base_time, '%a, %d %b %Y %H:%M:%S').isoformat() + 'Z' + + +def create_incident_labels(parsed_msg, headers): + labels = [ + {'type': 'Email/ID', 'value': parsed_msg['ID']}, + {'type': 'Email/subject', 'value': parsed_msg['Subject']}, + {'type': 'Email/text', 'value': parsed_msg['Body']}, + {'type': 'Email/from', 'value': parsed_msg['From']}, + {'type': 'Email/html', 'value': parsed_msg['Html']}, + ] + labels.extend([{'type': 'Email/to', 'value': to} + for to in headers.get('To', '').split(',')]) + labels.extend([{'type': 'Email/cc', 'value': cc} + for cc in headers.get('Cc', '').split(',')]) + labels.extend([{'type': 'Email/bcc', 'value': bcc} + for bcc in headers.get('Bcc', '').split(',')]) + for key, val in headers.items(): + labels.append({'type': 'Email/Header/' + key, 'value': val}) + + return labels + + +@logger +def mail_to_incident(msg): + parsed_msg, headers = get_email_context(msg) + + file_names = [] + for attachment in parsed_msg.get('Attachments', []): + file_data = base64.urlsafe_b64decode(attachment['Data'].encode('ascii')) + + # save the attachment + file_result = fileResult(attachment['Name'], file_data) + + # check for error + if file_result['Type'] == entryTypes['error']: + demisto.error(file_result['Contents']) + raise Exception(file_result['Contents']) + + file_names.append({ + 'path': file_result['FileID'], + 'name': attachment['Name'], + }) + + return { + 'name': parsed_msg['Subject'], + 'details': parsed_msg['Body'], + 'labels': create_incident_labels(parsed_msg, headers), + 'occurred': parse_time(parsed_msg['Date']), + 'attachment': file_names, + 'rawJSON': parsed_msg['RawData'] + } + + +def fetch_incidents(): + last_run = demisto.getLastRun() + last_fetch = last_run.get('time') + + # handle first time fetch + if last_fetch is None: + last_fetch, _ = parse_date_range(FETCH_TIME, date_format=DATE_FORMAT) + + last_fetch = datetime.strptime(last_fetch, DATE_FORMAT) + current_fetch = last_fetch + + incidents = [] + messages = get_user_emails() + + for msg in messages: + try: + incident = mail_to_incident(msg) + except Exception as e: + demisto.error("failed to create incident from email, index = {}, subject = {}, date = {}".format( + msg['index'], msg['subject'], msg['date'])) + raise e + + temp_date = datetime.strptime( + incident['occurred'], DATE_FORMAT) + + # update last run + if temp_date > last_fetch: + last_fetch = temp_date + timedelta(seconds=1) + + # avoid duplication due to weak time query + if temp_date > current_fetch: + incidents.append(incident) + + demisto.setLastRun({'time': last_fetch.isoformat().split('.')[0] + 'Z'}) + + return demisto.incidents(incidents) + + +def test_module(): + resp_message, _, _ = pop3_server_conn.list() # type: ignore + if "OK" in resp_message: + demisto.results('ok') + + +''' COMMANDS MANAGER / SWITCH PANEL ''' + + +def main(): + try: + handle_proxy() + connect_pop3_server() + if demisto.command() == 'test-module': + # This is the call made when pressing the integration test button. + test_module() + if demisto.command() == 'fetch-incidents': + fetch_incidents() + sys.exit(0) + except Exception as e: + LOG(str(e)) + LOG.print_log() + raise e + finally: + close_pop3_server_connection() + + +# python2 uses __builtin__ python3 uses builtins +if __name__ == "__builtin__" or __name__ == "builtins": + main() diff --git a/Beta_Integrations/MailListener_-_POP3/MailListener_-_POP3.yml b/Beta_Integrations/MailListener_-_POP3/MailListener_-_POP3.yml new file mode 100644 index 000000000000..c63098e9494b --- /dev/null +++ b/Beta_Integrations/MailListener_-_POP3/MailListener_-_POP3.yml @@ -0,0 +1,55 @@ +category: Messaging +commonfields: + id: MailListener - POP3 Beta + version: -1 +configuration: +- display: Server URL (e.g. example.com) + name: server + required: true + type: 0 +- defaultvalue: '995' + display: Port + name: port + required: false + type: 0 +- display: Email + name: email + required: true + type: 0 +- display: Password + name: password + required: true + type: 4 +- defaultvalue: 'True' + display: Use SSL connection + name: ssl + required: false + type: 8 +- display: Use system proxy + name: proxy + required: false + type: 8 +- display: Fetch incidents + name: isFetch + required: false + type: 8 +- defaultvalue: 3 days + display: First fetch timestamp (