diff --git a/.github/workflows/pre-submit-test.yaml b/.github/workflows/pre-submit-test.yaml index ef869976..7b2e4d89 100644 --- a/.github/workflows/pre-submit-test.yaml +++ b/.github/workflows/pre-submit-test.yaml @@ -20,7 +20,7 @@ jobs: if: ${{ startsWith(github.ref, 'refs/heads/linux_ex2/') }} steps: - run: echo "🎉 The job was automatically triggered by a ${{ github.event_name }} event." - PythonKata1Test: + PythonKataTest: runs-on: ubuntu-latest if: ${{ startsWith(github.ref, 'refs/heads/python_katas_') }} steps: @@ -29,6 +29,6 @@ jobs: uses: actions/setup-python@v2 with: python-version: 3.8 - - name: Katas Tests results + - name: Kata 1 Tests results run: | - python -m python_katas.kata_1.test \ No newline at end of file + python -m python_katas.kata_1.test diff --git a/.gitignore b/.gitignore index 903ac86f..5e8658f8 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,10 @@ __pycache__/ *.so +# ignore .pem file +*.pem +**/*.weights + # Distribution / packaging .Python env/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..6b4e71fc --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,7 @@ +# Further reading in https://pre-commit.com/index.html +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.3.0 + hooks: + - id: detect-aws-credentials + - id: detect-private-key \ No newline at end of file diff --git a/01_encryption/asymmetric/message.encrypted b/01_encryption/asymmetric/message.encrypted new file mode 100644 index 00000000..e49db6f3 Binary files /dev/null and b/01_encryption/asymmetric/message.encrypted differ diff --git a/01_encryption/symmetric/decrypted_secret b/01_encryption/symmetric/decrypted_secret new file mode 100644 index 00000000..072cec70 --- /dev/null +++ b/01_encryption/symmetric/decrypted_secret @@ -0,0 +1 @@ +plaintext secret \ No newline at end of file diff --git a/01_encryption/symmetric/encrypted_secret b/01_encryption/symmetric/encrypted_secret new file mode 100644 index 00000000..0836d6ee Binary files /dev/null and b/01_encryption/symmetric/encrypted_secret differ diff --git a/02_linux_ex1/secretGenerator.tar.gz b/02_linux_ex1/secretGenerator.tar.gz new file mode 100644 index 00000000..d5cbb53e Binary files /dev/null and b/02_linux_ex1/secretGenerator.tar.gz differ diff --git a/02_linux_ex1/src/CONTENT_TO_HASH b/02_linux_ex1/src/CONTENT_TO_HASH new file mode 100644 index 00000000..175bf609 --- /dev/null +++ b/02_linux_ex1/src/CONTENT_TO_HASH @@ -0,0 +1,5 @@ +With the sun in my hand +Gonna throw the sun +Way across the land- +Cause I’m tired, +Tired as I can be \ No newline at end of file diff --git a/02_linux_ex1/src/generateSecret.sh b/02_linux_ex1/src/generateSecret.sh new file mode 100644 index 00000000..0f871636 --- /dev/null +++ b/02_linux_ex1/src/generateSecret.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# practice dir creation +if [ ! -d "secretDir" ]; then + echo "Failed to generate secret. The directory 'secretDir' must exist before." + exit 1 +fi + +# practice dir deletion and file move +if [ -d "maliciousFiles" ]; then + echo "Failed to generate secret. The directory 'maliciousFiles' contains some malicious files... it must be removed before." + exit 1 +fi + +# practice file creation +if [ ! -f "secretDir/.secret" ]; then + echo "Failed to generate secret. The directory 'secretDir' must contain a file '.secret' in which the secret will be stored." + exit 1 +fi + +# practice change permissions +OCTAL_PERMISSIONS=$(stat -c "%a" secretDir/.secret) +if [ "$OCTAL_PERMISSIONS" != "600" ]; then + echo "Failed to generate secret. The file 'secretDir/.secret' must have read and write permission only." + exit 1 +fi + +# practice file linking understanding +if [ -L 'important.link' ] && [ ! -e 'important.link' ]; then + echo "Failed to generate secret. Secret can not be generated when broken file link exists. Please do something..." + exit 1 +fi + +cat ./CONTENT_TO_HASH | xargs | md5sum > secretDir/.secret && echo "Done! Your secret was stored in secretDir/.secret" diff --git a/02_linux_ex1/src/important.link b/02_linux_ex1/src/important.link new file mode 100644 index 00000000..e69de29b diff --git a/02_linux_ex1/src/maliciousFiles/amIMaliciousOrNot.whoKnows b/02_linux_ex1/src/maliciousFiles/amIMaliciousOrNot.whoKnows new file mode 100644 index 00000000..e69de29b diff --git a/02_linux_ex1/src/maliciousFiles/someFileIsLinkingToMe.BeAware b/02_linux_ex1/src/maliciousFiles/someFileIsLinkingToMe.BeAware new file mode 100644 index 00000000..e69de29b diff --git a/02_linux_ex1/src/yourSolution.sh b/02_linux_ex1/src/yourSolution.sh new file mode 100644 index 00000000..139597f9 --- /dev/null +++ b/02_linux_ex1/src/yourSolution.sh @@ -0,0 +1,2 @@ + + diff --git a/02_linux_ex1/whatIdo b/02_linux_ex1/whatIdo new file mode 100755 index 00000000..da7d24bc Binary files /dev/null and b/02_linux_ex1/whatIdo differ diff --git a/02_linux_ex1/whatIdo.c b/02_linux_ex1/whatIdo.c new file mode 100644 index 00000000..b645c55b --- /dev/null +++ b/02_linux_ex1/whatIdo.c @@ -0,0 +1,31 @@ +#include +#include +#include +#include +#include + +struct stat st = {0}; + + + +int main() { + if (stat("./welcomeToDevOpsJan22", &st) == -1) { + mkdir("./welcomeToDevOpsJan22", 0700); + } + + int num; + FILE *fptr; + + // use appropriate location if you are using MacOS or Linux + fptr = fopen("welcomeToDevOpsJan22/goodLuck","w"); + + if(fptr == NULL) + { + printf("Error!"); + exit(1); + } + + fprintf(fptr,"%s","There you go... tell me what I do..."); + fclose(fptr); + return 0; +} \ No newline at end of file diff --git a/05_simple_webserver/app.py b/05_simple_webserver/app.py index 553c8032..ec83d3fa 100644 --- a/05_simple_webserver/app.py +++ b/05_simple_webserver/app.py @@ -1,11 +1,15 @@ +import os from flask import Flask, send_file, request app = Flask(__name__) +hostname = os.environ.get('HOSTNAME', None) + # Try by: curl localhost:8080 @app.route('/', methods=['GET']) def index(): + print(request.headers) return 'Hello world\n' @@ -23,5 +27,20 @@ def profile_picture(): return send_file('images/profile-1.jpg', mimetype='image/gif') +@app.route('/status') +def status(): + return 'OK' + + +@app.route('/load-test') +def load_test_endpoint(): + x = 6 + for i in range(1, 10): + x *= i + x /= i + + return f'Done {hostname}' + + if __name__ == '__main__': app.run(debug=True, port=8080, host='0.0.0.0') diff --git a/05_simple_webserver/simple_webserver.wsgi b/05_simple_webserver/simple_webserver.wsgi new file mode 100644 index 00000000..5cb01cb6 --- /dev/null +++ b/05_simple_webserver/simple_webserver.wsgi @@ -0,0 +1,4 @@ +import sys +sys.path.insert(0, '/var/www/simple_webserver') + +from app import app as application \ No newline at end of file diff --git a/07_git_exercises/dima_ex1/README b/07_git_exercises/dima_ex1/README new file mode 100644 index 00000000..2970042e --- /dev/null +++ b/07_git_exercises/dima_ex1/README @@ -0,0 +1,143 @@ +#my git_ex1 exercise answers : + +#Git Basics : + +1. echo "1" > abc.txt +2. "red color" because the file not in index + +3. "green color" after adding file to index (git add abc.txt && git commit -m "adding abc.txt file") + +4. echo "2" >> abc.txt +5. "blue color" + +6. git diff or git diff main + +7. because i dont have staged files for the next commit + +8. its a invalid argument because i dont have stage2 HEAD(or any hash or branch) in my repository + +9. git add abc.txt +10. prints nothing because all my files is staged in index ( git diff only shows unstaged changes (--staged/--cached option showing staged changes)) + +11. git diff ---cached or --staged in more recent versions of git + +12. echo "3" >> abc.txt +13. No, the output is different because git diff main shows unstaged changes also and git diff --staged shows only the staged changes ! + +14. Because the same reason as above, the green color for "changes to be committed" and red color for "changes not stage for commit" + +15. git reset -- abc.txt for remove the staged files in index and git restore -- abc.txt for unstaged files in working tree + +#Resolve conflicts: + +1. git branch: + output: + bugfix/fix_readme_typo + bugfix/open_kibana_port + dev + feature/data_retention_policy + feature/elasticsearch_helm_chart + feature/upgrade_angular_version + feature/version1 + feature/version2 +* main + reset_question + +2. git checkout -b feature/lambda_migration (the git checkout command accepts a -b argument that acts as a convenience method which will create new branch and switch to it immediately) + output: + Switched to a new branch 'feature/lambda_migration' + +3. git merge feature/version1 + output: + Merge made by the 'recursive' strategy. + .env | 0 + app.py | 4 ++-- + config.json | 0 + 3 files changed, 2 insertions(+), 2 deletions(-) + create mode 100644 .env + create mode 100644 config.json + +4. git log: + output: + commit 84c58db9790c249a2b12cda2a0f2a1726e272e8d (HEAD -> feature/lambda_migration) + Merge: d89dca1 8019018 + Author: dmitriyshub + Date: Tue May 24 12:37:21 2022 +0300 + + Merge branch 'feature/version2' into feature/lambda_migration + + # Conflicts: + # app.py + + commit d89dca1c732a45479f956147da71cd096bb5fc73 + + # Conflicts: + # app.py + + commit d89dca1c732a45479f956147da71cd096bb5fc73 + Merge: d14ba66 e0f83f5 + Author: dmitriyshub + Date: Tue May 24 12:26:26 2022 +0300 + + Merge branch 'feature/version1' into feature/lambda_migration + + Answer: Yes we have one commit for each merge + +#Cherry Picking: + +1. git checkout main && git checkout -b feature/lambda_migration2 + +4. config.json and .env files + +5. yes because the commits must be related with each other by order + +#Changes in working tree and switch branches: + +1. git branch +2. nano take.txt, save file after changes. git add take.txt +3. git checkout dev + output: + error: Your local changes to the following files would be overwritten by checkout: + take.txt + Please commit your changes or stash them before you switch branches. + Aborting + answer: suggested approaches from git - commit or stash + git stash saves the local modification away and reverts the working directory to match the HEAD commit +4. No , the text overwritten to a b c on separate lines +5. No, my local uncommited changes will be overwritten and i will lose them + +# Reset: + +1. git checkout reset_question +2. + 1: git reset --soft HEAD~1 + output1: + On branch reset_question + Changes to be committed: + (use "git restore --staged ..." to unstage) + new file: 10.txt + answer1: + the command will remove the last commit from the current branch,but the file changes will stay in working tree + + 2: git reset --mixed HEAD~1 + output2: + On branch reset_question + Untracked files: + (use "git add ..." to include in what will be committed) + 10.txt + 9.txt + + nothing added to commit but untracked files present (use "git add" to track) + + answer2: + the coomand will still kepp the changes in your working tree but not in the index + + 3: git reset --hard HEAD~1 + output3: + HEAD is now at 17a0b5d 7 + ansswer3: + lose all uncommited changes and all untracked files in addition to the changes introduced in the last commit + + 4. revert the last git commit, it will record a new commit with the changes introduced by reverting the last commit + +#end diff --git a/07_git_exercises/shlomigd_ex1/README b/07_git_exercises/shlomigd_ex1/README new file mode 100644 index 00000000..ea6f5160 --- /dev/null +++ b/07_git_exercises/shlomigd_ex1/README @@ -0,0 +1,68 @@ +Git Basics: + +1. cat > abc.txt + 1 + ctrl d + +2.Red. +3.Green. + git status + git add abc.txt + git status + git commit abc.txt -m "file created" +4. echo "2" >> abc.txt +5.Blue. +6.git diff main. +7.Because we didn't add the file abc.txt to the index area. +8.Because stage2 is not an option of the command git diff. +9. git add abc.txt +10.git diff prints nothing because there is no differences between the index and the working copy. +11. git diff --staged. +12. echo "3" >> abc.txt +13.No,because the git diff --staged command will compare the index with the master and the command git diff main will also compare the changes in the working copy. +14.Because one change in the file is ready to be committed and another change is not staged for commit. +15. git reset --hard + +Resolve conflicts: + +1. git branch +6. Yes, the commits are: +[git_ex1_repo] git -c credential.helper= -c core.quotepath=false -c log.showSignature=false commit -F C:\Users\shlomi\PycharmProjects\git_ex1_repo\.git\MERGE_MSG -- +[feature/lambda_migration 798ff58] Merge branch 'feature/version2' into feature/lambda_migration + +Cherry picking: + +4. file config.json and file .env. +5.Yes,because if the first commit create a file and the second one update the file and you will choose to cherry pick only the second one it will fail + and sometimes you create a function that calls another function and if and if this function is missing it will fail. + +Changes in working tree and switch branches: + +1. git branch +2. cat > take.txt + shlomi shlomi + gedasi gedasi + ctrl d + git add take.txt +3. git checkout dev + error: Your local changes to the following files would be overwritten by checkout: + take.txt + Please commit your changes or stash them before you switch branches. + Aborting + +One approach suggested by git is to commit the changes before switching to another branch and the second one is to stash the changes. + +5. No. +6. No, the file take.txt doesn't exist. + The Force Checkout removes all uncommitted changes. + +Reset: + +1. git checkout reset_question +2.1 The command removed the last commit, now file 10.txt ready to be committed. +2.2 The command removed the last commit,removes files from the index and keep the changes in the working copy, + The file 10.txt removed from the index and now the file is in the working copy with other files. +2.3 The command removed the last commit and all the untracked and uncommitted changes. +2.4 The command removes the last commit by adding a new commit to cancel the changes. +3. The notation HEAD~1 in the git reset command means to reset the last commit. + diff --git a/08_python_class/bank_account.py b/08_python_class/bank_account.py index 435065e4..36e0e7e0 100644 --- a/08_python_class/bank_account.py +++ b/08_python_class/bank_account.py @@ -18,6 +18,10 @@ class Account: currency = '$' + @staticmethod + def usd_to_ils_convert(): + return 3.77 + def __init__(self, name, balance): self.owner = name self.balance = balance diff --git a/09_backoff_algorithm/robust_client.py b/09_backoff_algorithm/robust_client.py new file mode 100644 index 00000000..6d0c2c5d --- /dev/null +++ b/09_backoff_algorithm/robust_client.py @@ -0,0 +1,84 @@ +import time +from random import random +import requests +from loguru import logger + +server_url = 'http://localhost:8081/get-data' + + +def exponential_backoff_retry(count, max_sec=20): + """ + Retry in random time within an exponential increasing range of seconds + + :param count: number of failures, starting with 1 + :param max_sec: the maximum seconds to wait independently the retry count + :return: seconds to sleep before next retry + """ + return min(max_sec, (2 ** (count - 1) - 1) * random()) + + +def exponential_retry(count, max_sec=20): + """ + Retry in exponential increasing number of seconds + some small random change + + :param count: number of failures, starting with 1 + :param max_sec: the maximum seconds to wait independently the retry count + :return: seconds to sleep before next retry + + """ + return min(max_sec, (2 ** (count - 1) - 1) + random()) + + +def linear_retry(count): + """ + Retry in number of seconds linearly dependent of the retry count + + :param count: number of failures, starting with 1 + :return: seconds to sleep before next retry + """ + return 2 * count + + +def constant_retry(): + """ + Retry in constant value of seconds + + :return: seconds to sleep before next retry + """ + return 5 + + +def get_data_from_server(): + retry = 0 + + while retry < 10: + try: + logger.info(f'Getting user data from external server ({retry})') + data = requests.get(server_url) + break + except requests.exceptions.ConnectionError as err: + retry += 1 + time_to_sleep = exponential_backoff_retry(retry) + logger.error(f'Failed due to ..., sleeping {time_to_sleep}sec') + + time.sleep(time_to_sleep) + + +def get_data_from_server_recursive(retry): + if retry > 10: + return + + try: + logger.info(f'Getting user data from external server ({retry})') + data = requests.get(server_url) + return data + except requests.exceptions.ConnectionError as err: + retry += 1 + time_to_sleep = exponential_backoff_retry(retry) + logger.error(f'Failed due to ..., sleeping {time_to_sleep}sec') + time.sleep(time_to_sleep) + return get_data_from_server_recursive(retry) + + +if __name__ == '__main__': + get_data_from_server() diff --git a/09_backoff_algorithm/unstable_webserver.py b/09_backoff_algorithm/unstable_webserver.py new file mode 100644 index 00000000..f04d1a62 --- /dev/null +++ b/09_backoff_algorithm/unstable_webserver.py @@ -0,0 +1,16 @@ +from flask import Flask, abort +from random import random + +app = Flask(__name__) + + +@app.route('/get-data') +def get_data(): + if random() < 0.2: + return 'your data...' + else: + abort(500, 'Server failed due to an internal error') + + +if __name__ == '__main__': + app.run(debug=True, port=8081, host='0.0.0.0') diff --git a/10_docker_compose/Dockerfile b/10_docker_compose/Dockerfile new file mode 100644 index 00000000..6ca9cb01 --- /dev/null +++ b/10_docker_compose/Dockerfile @@ -0,0 +1,10 @@ +FROM python:3.7-alpine +WORKDIR /code +ENV FLASK_APP=app.py +ENV FLASK_RUN_HOST=0.0.0.0 +RUN apk add --no-cache gcc musl-dev linux-headers +COPY requirements.txt requirements.txt +RUN pip install -r requirements.txt +EXPOSE 5000 +COPY . . +CMD ["flask", "run"] \ No newline at end of file diff --git a/10_docker_compose/app.py b/10_docker_compose/app.py new file mode 100644 index 00000000..dca2eaca --- /dev/null +++ b/10_docker_compose/app.py @@ -0,0 +1,27 @@ +import time + +import redis +from flask import Flask + +app = Flask(__name__) +cache = redis.Redis(host='redis', port=6379) + +"https://we-are.bookmyshow.com/understanding-expose-in-dockerfile-266938b6a33d" + +def get_hit_count(): + retries = 5 + while True: + try: + return cache.incr('hits') + except redis.exceptions.ConnectionError as exc: + if retries == 0: + raise exc + retries -= 1 + time.sleep(0.5) + + +@app.route('/') +def hello(): + count = get_hit_count() + return 'Hello World! I have been seen {} times.\n'.format(count) + diff --git a/10_docker_compose/docker-compose.yaml b/10_docker_compose/docker-compose.yaml new file mode 100644 index 00000000..b7dba7d7 --- /dev/null +++ b/10_docker_compose/docker-compose.yaml @@ -0,0 +1,15 @@ +# The version key is mandatory, and it’s always the first line at the root of the file. +# This defines the version of the Compose file format (basically the API). You should +# normally use the latest version. +version: "3.9" +services: + web: + build: . + ports: + - "8000:5000" + # you might want this container to be running only after redis is up (use depends_on: ) + redis: + image: "redis:alpine" + command: redis-server --save 60 1 + volumes: + - /redis_data:/data diff --git a/10_docker_compose/requirements.txt b/10_docker_compose/requirements.txt new file mode 100644 index 00000000..eadf80f9 --- /dev/null +++ b/10_docker_compose/requirements.txt @@ -0,0 +1,2 @@ +flask +redis \ No newline at end of file diff --git a/11_aws_demos/DNS.md b/11_aws_demos/DNS.md new file mode 100644 index 00000000..362bb78d --- /dev/null +++ b/11_aws_demos/DNS.md @@ -0,0 +1,47 @@ +# Route 53 + +## Resolving google.com + +We will resolve google.com step by step using the `nslookup` command. + +1. First, get the list of the root-level DNS servers by `nslookup -type=NS .`. +2. Pick one of the root-level domain names. We will query this server to get the hostname of the *.com* top-level domain by: + `nslookup -type=NS com ` +3. Now that we have a list of *.com* TLD servers, pick on of them to query the hostname of the authoritative DNS of *google.com*: + `nslookup -type=NS google.com ` +4. Finally, as we know the hostname of the authoritative DNS servers of *google.com*, we can query one of them to retrieve the IP address of *google.com*: + `nslookup -type=A google.com ` + +## Creating a public hosted zone + +1. Open the Route 53 console at [https://console\.aws\.amazon\.com/route53/](https://console.aws.amazon.com/route53/). + +2. If you're new to Route 53, choose **Get started** under **DNS management**\. + + If you're already using Route 53, choose **Hosted zones** in the navigation pane\. + +3. Choose **Create hosted zone**\. + +4. In the **Create Hosted Zone** pane, enter the name of the domain that you want to route traffic for\. + +5. For **Type**, accept the default value of **Public Hosted Zone**\. + +6. Choose **Create**\. + +7. Inspired by the above usage of `nslookup` try to query one of the authoritative DNS server created in the hosted zone. + +## Add records to registered domain + + +1. In the navigation pane, choose **Hosted zones**\. + +2. Choose `devops-int-college.com` as the name of the hosted zone that you want to create records in\. + +3. Choose **Create record**\. + +4. Define an A record for a custom subdomain of yours (e.g. `my-name.devops-int-college.com`), the record value is an IP address on a running EC2 instance. + +5. Choose **Create records**\. + **Note** + Your new records take time to propagate to the Route 53 DNS servers + diff --git a/11_aws_demos/compute_demos.md b/11_aws_demos/compute_demos.md new file mode 100644 index 00000000..24ee2e0d --- /dev/null +++ b/11_aws_demos/compute_demos.md @@ -0,0 +1,63 @@ +# AWS demos + +## Hello World EC2 + +1. Create an EC2 instance, as follows: + 1. `Amazon Linux 2 AMI` AMI. + 2. `t2.micto` instance type (or equivalent medium type from another generation). + 3. Choose your key-pair (create if needed). + 4. In network configurations: + 1. Make sure your instance is provisioned in the default VPC. + 2. Choose the **a** availability zone of your region. e.g. my instance will be provisioned in **us-east-1**, the AZ should be us-east-1**a** +2. Your instance should have a public ip4v address. Connect to your instance via SSH by click on **Connect** button in the instance summary page, then **SSH Client**, follow the instructions there. +3. Connect to your instance using SSH. + +## Create and mount EBS volume + +1. In EC2 the navigation pane, choose **Volumes**\. + +2. Choose **Create volume**\. + +3. For **Volume type**, choose the type of volume to create, SSD gp2\. For more information, see [Amazon EBS volume types](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-volume-types.html). + +4. For **Size**, enter the size of the volume, 10GiB\. For more information, see [Constraints on the size and configuration of an EBS volume](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/volume_constraints.html). + +5. For **Availability Zone**, choose the Availability Zone in which to create the volume\. A volume can be attached only to an instance that is in the same Availability Zone\. + +6. For **Snapshot ID**, keep the default value \(**Don't create volume from a snapshot**\)\. + +7. Assign custom tags to the volume, in the **Tags** section, choose **Add tag**, and then enter a tag key and value pair\. + +8. Choose **Create volume**\. + **Note** + The volume is ready for use when the **Volume state** is **available**\. + +9. To use the volume, attach it to an instance\. For more information, see [Attach an Amazon EBS volume to an instance](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-attaching-volume.html). + +10. Connect to your instance over SSH. +11. [Format and mount the attached volume](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-using-volumes.html) +12. and write some data to the mounter EBS. + + +## Create an encrypted EBS and migrate disks + +1. In KMS, [create encryption key](https://docs.aws.amazon.com/kms/latest/developerguide/create-keys.html#create-symmetric-cmk). Make sure your IAM user can administer this key and delete it. +2. [Create a volume snapshot](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-creating-snapshot.html#ebs-create-snapshot) of the EBS you provisioned and mounted in the previous section. +3. Create an **encrypted EBS from the EBS snapshot**. Use the encrypted keys you’ve just created in KMS. +4. Attach and mount the encrypted volume to your instance, as follows: + 1. Generate new UUID for the encrypted disk by: + ```shell + sudo xfs_admin -U generate + ``` + 2. Copy the generated uuid, and add the following entry to `/etc/fstab`: + ```shell + UUID= /data xfs defaults,nofail 0 2 + ``` + while `` is your generated device UUID. + + Make sure the data from the unencrypted volume has been migrated successfully to the encrypted volume. + +#### Discussion + +5. In KMS page, disable your encryption key. What happened to the data in your instance? +6. Stop the machine and start it again, [what happened](https://docs.aws.amazon.com/kms/latest/developerguide/services-ebs.html#ebs-cmk) to the data in your instance? \ No newline at end of file diff --git a/11_aws_demos/databases.md b/11_aws_demos/databases.md new file mode 100644 index 00000000..3d5b478f --- /dev/null +++ b/11_aws_demos/databases.md @@ -0,0 +1,514 @@ +# Databases + +## What is relational database? + +Refer [this](https://code.tutsplus.com/tutorials/relational-databases-for-dummies--net-30244) article to read the source from which this section has been written. + +A database stores data in an organized way so that it can be searched and retrieved later. It should contain one or more tables. A table is much like a spreadsheet, in that it's made up of rows and columns. All rows have the same columns, and each column contains the data itself. If it helps, think of your tables in the same way that you would a table in Excel. + + +![](https://cdn.tutsplus.com/cdn-cgi/image/width=850/net/authors/lalith-polepeddi/relational-databases-for-dummies-fig1.png) + + + +Data can be inserted, retrieved, updated, and deleted from a table. The word, created, is generally used instead of inserted, so, collectively, these four functions are affectionately abbreviated as CRUD. + +A relational database is a type of database that organizes data into tables, and links them, based on defined relationships. These relationships enable you to retrieve and combine data from one or more tables with a single **query**. + +Let's get some Twitter data + +| full_name | username | text | created_at | following_username | +|-----------------------|-------------------|----------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------|------------------------------------| +| Boris Hadjur | _DreamLead | What do you think about #emailing #campaigns #traffic in #USA? Is it a good market nowadays? do you have #databases? | Tue, 12 Feb 2013 08:43:09 +0000 | Scootmedia, MetiersInternet | +| Gunnar Svalander | GunnarSvalander | Bill Gates Talks Databases, Free Software on Reddit https://t.co/ShX4hZlA #billgates #databases | Tue, 12 Feb 2013 07:31:06 +0000 | klout, zillow | +| GE Software | GEsoftware | RT @KirkDBorne: Readings in #Databases: excellent reading list, many categories: http://t.co/S6RBUNxq via @rxin Fascinating. | Tue, 12 Feb 2013 07:30:24 +0000 | DayJobDoc, byosko | +| Adrian Burch | adrianburch | RT @tisakovich: @NimbusData at the @Barclays Big Data conference in San Francisco today, talking #virtualization, #databases, and #flash memory. | Tue, 12 Feb 2013 06:58:22 +0000 | CindyCrawford, Arjantim | +| Andy Ryder | AndyRyder5 | http://t.co/D3KOJIvF article about Madden 2013 using AI to prodict the super bowl #databases #bus311 | Tue, 12 Feb 2013 05:29:41 +0000 | MichaelDell, Yahoo | +| Andy Ryder | AndyRyder5 | http://t.co/rBhBXjma an article about privacy settings and facebook #databases #bus311 | Tue, 12 Feb 2013 05:24:17 +0000 | MichaelDell, Yahoo | +| Brett Englebert | Brett_Englebert | #BUS311 University of Minnesota's NCFPD is creating #databases to prevent food fraud. http://t.co/0LsAbKqJ | Tue, 12 Feb 2013 01:49:19 +0000 | RealSkipBayless, stephenasmith | +| Brett Englebert | Brett_Englebert | #BUS311 companies might be protecting their production #databases, but what about their backup files? http://t.co/okJjV3Bm | Tue, 12 Feb 2013 01:31:52 +0000 | RealSkipBayless, stephenasmith | +| Nimbus Data Systems | NimbusData | @NimbusData CEO @tisakovich @BarclaysOnline Big Data conference in San Francisco today, talking #virtualization, #databases,& #flash memory | Mon, 11 Feb 2013 23:15:05 +0000 | dellock6, rohitkilam | +| SSWUG.ORG | SSWUGorg | Don't forget to sign up for our FREE expo this Friday: #Databases, #BI, and #Sharepoint: What You Need to Know! http://t.co/Ijrqrz29 | "Mon, 11 Feb 2013 22:15:37 +0000" | "drsql", "steam_games" +| + +Usig the above data structure has pros and cons: + +* 👍 The data is all in one place; so it's easy to find. +* ❌ The "username" and "following_username" columns are repetitive +* ❌ @AndyRyder5 and @Brett_Englebert each tweeted twice, so the rest of their information has been duplicated. + +Duplicates are problematic because it makes the CRUD operations more challenging + +### Remove Repetitive Data Across Columns + +Let's improve on the above table design by splitting it up into two tables: one just for the following relationships and one for the rest of the information. + +The followings table + +| from_user | to_user | +|-----------------|-----------------| +| _DreamLead | Scootmedia | +| _DreamLead | MetiersInternet | +| GunnarSvalander | klout | +| GunnarSvalander | zillow | +| GEsoftware | DayJobDoc | +| GEsoftware | byosko | +| adrianburch | CindyCrawford | +| adrianburch | Arjantim | +| AndyRyder | MichaelDell | +| AndyRyder | Yahoo | +| Brett_Englebert | RealSkipBayless | +| Brett_Englebert | stephenasmith | +| NimbusData | dellock6 | +| NimbusData | rohitkilam | +| SSWUGorg | drsql | +| SSWUGorg | steam_games | + + +The users table + +| full_name | username | text | created_at | +|-----------------------|-------------------|----------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------| +| Boris Hadjur | _DreamLead | What do you think about #emailing #campaigns #traffic in #USA? Is it a good market nowadays? do you have #databases? | Tue, 12 Feb 2013 08:43:09 +0000 | +| Gunnar Svalander | GunnarSvalander | Bill Gates Talks Databases, Free Software on Reddit http://t.co/ShX4hZlA #billgates #databases | Tue, 12 Feb 2013 07:31:06 +0000 | +| GE Software | GEsoftware | RT @KirkDBorne: Readings in #Databases: excellent reading list, many categories: http://t.co/S6RBUNxq via @rxin Fascinating. | Tue, 12 Feb 2013 07:30:24 +0000 | +| Adrian Burch | adrianburch | RT @tisakovich: @NimbusData at the @Barclays Big Data conference in San Francisco today, talking #virtualization, #databases, and #flash memory. | Tue, 12 Feb 2013 06:58:22 +0000 | +| Andy Ryder | AndyRyder5 | http://t.co/D3KOJIvF article about Madden 2013 using AI to prodict the super bowl #databases #bus311 | Tue, 12 Feb 2013 05:29:41 +0000 | +| Andy Ryder | AndyRyder5 | http://t.co/rBhBXjma an article about privacy settings and facebook #databases #bus311 | Tue, 12 Feb 2013 05:24:17 +0000 | +| Brett Englebert | Brett_Englebert | #BUS311 University of Minnesota's NCFPD is creating #databases to prevent food fraud. http://t.co/0LsAbKqJ | Tue, 12 Feb 2013 01:49:19 +0000 | +| Brett Englebert | Brett_Englebert | #BUS311 companies might be protecting their production #databases, but what about their backup files? http://t.co/okJjV3Bm | Tue, 12 Feb 2013 01:31:52 +0000 | +| Nimbus Data Systems | NimbusData | @NimbusData CEO @tisakovich @BarclaysOnline Big Data conference in San Francisco today, talking #virtualization, #databases,& #flash memory | Mon, 11 Feb 2013 23:15:05 +0000 | +| SSWUG.ORG | SSWUGorg | Don't forget to sign up for our FREE expo this Friday: #Databases, #BI, and #Sharepoint: What You Need to Know! http://t.co/Ijrqrz29 | "Mon, 11 Feb 2013 22:15:37 +0000" | + +### Remove Repetitive Data Across Rows + +We need to pull out the tweets and place them in their own table. + + +| id | text | created_at | username | +|----|----------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------|-------------------| +| 1 | What do you think about #emailing #campaigns #traffic in #USA? Is it a good market nowadays? do you have #databases? | Tue, 12 Feb 2013 08:43:09 +0000 | _DreamLead | +| 2 | Bill Gates Talks Databases, Free Software on Reddit http://t.co/ShX4hZlA #billgates #databases | Tue, 12 Feb 2013 07:31:06 +0000 | GunnarSvalander | +| 3 | RT @KirkDBorne: Readings in #Databases: excellent reading list, many categories: http://t.co/S6RBUNxq via @rxin Fascinating. | Tue, 12 Feb 2013 07:30:24 +0000 | GEsoftware | +| 4 | RT @tisakovich: @NimbusData at the @Barclays Big Data conference in San Francisco today, talking #virtualization, #databases, and #flash memory. | Tue, 12 Feb 2013 06:58:22 +0000 | adrianburch | +| 5 | http://t.co/D3KOJIvF article about Madden 2013 using AI to prodict the super bowl #databases #bus311 | Tue, 12 Feb 2013 05:29:41 +0000 | AndyRyder5 | +| 6 | http://t.co/rBhBXjma an article about privacy settings and facebook #databases #bus311 | Tue, 12 Feb 2013 05:24:17 +0000 | AndyRyder5 | +| 7 | #BUS311 University of Minnesota's NCFPD is creating #databases to prevent food fraud. http://t.co/0LsAbKqJ | Tue, 12 Feb 2013 01:49:19 +0000 | Brett_Englebert | +| 8 | #BUS311 companies might be protecting their production #databases, but what about their backup files? http://t.co/okJjV3Bm | Tue, 12 Feb 2013 01:31:52 +0000 | Brett_Englebert | +| 9 | @NimbusData CEO @tisakovich @BarclaysOnline Big Data conference in San Francisco today, talking #virtualization, #databases,& #flash memory | Mon, 11 Feb 2013 23:15:05 +0000 | NimbusData | +| 10 | Don't forget to sign up for our FREE expo this Friday: #Databases, #BI, and #Sharepoint: What You Need to Know! http://t.co/Ijrqrz29 | Mon, 11 Feb 2013 22:15:37 +0000 | SSWUGorg +| + + + +The new users table + +| full_name | username | +|-----------------------|-------------------| +| Boris Hadjur | _DreamLead | +| Gunnar Svalander | GunnarSvalander | +| GE Software | GEsoftware | +| Adrian Burch | adrianburch | +| Andy Ryder | AndyRyder5 | +| Brett Englebert | Brett_Englebert | +| Nimbus Data Systems | NimbusData | +| SSWUG.ORG | SSWUGorg | + + +So far, the original tweets table has been split into three new tables: **followings**, **tweets**, and **users**. But how is this useful? Repetitive data has been removed, but now the data is spread out across three independent tables. In order to retrieve the data, we need to draw meaningful links between the tables. This way we can express queries like, "what a user has tweeted and who a user is following". + +![](../docs/img/db.png) + +### Structured Query Language (SQL) + +SQL (Structured Query Language) is a database computer language designed for managing +data in relational database management systems (RDBMS). + +What can SQL do? +* CRUD operations againt a given database +* Create new databases and tables +* SQL can set permissions on tables, procedures, and views + +An **SQL query** is a statement representing some operation to perform in the database. + + +## Relational Database Service (RDS) + +### Provision Postgres using RDS + +[https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/CHAP_GettingStarted.CreatingConnecting.PostgreSQL.html](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/CHAP_GettingStarted.CreatingConnecting.PostgreSQL.html) + +### Create tables, insert and retrieve data + +1. First, install the `psql` cli tool, which is the client for Postgres used from bash terminal + + ```shell + sudo apt-get update + sudo apt-get install postgresql-client + ``` + + Start Postgres shell by `psql postgres://:@:5432` + +2. Create a database in Postgres (drop previous is exists) + ```shell + DROP DATABASE IF EXISTS twitter_demo; + CREATE DATABASE twitter_demo; + ``` +3. Switch to `twitter_demo` db by `\c twitter_demo`. +4. Create tables: + ```text + CREATE TABLE users ( + full_name VARCHAR(100), + username VARCHAR(100) PRIMARY KEY + ); + + CREATE TABLE follows ( + from_user VARCHAR(100), + to_user VARCHAR(100) + ); + + CREATE TABLE tweets ( + id SERIAL, + username VARCHAR(100), + text VARCHAR(500), + created_at DATE NOT NULL DEFAULT CURRENT_DATE + ); + ``` + +5. Insert data + ```text + INSERT INTO users (full_name, username) VALUES ('Boris Hadjur', '_DreamLead'); + INSERT INTO users (full_name, username) VALUES ('Gunnar Svalander', 'GunnarSvalander'); + INSERT INTO users (full_name, username) VALUES ('GE Software', 'GEsoftware'); + INSERT INTO users (full_name, username) VALUES ('Adrian Burch', 'adrianburch'); + INSERT INTO users (full_name, username) VALUES ('Andy Ryder', 'AndyRyder5'); + INSERT INTO users (full_name, username) VALUES ('Brett Englebert', 'Brett_Englebert'); + INSERT INTO users (full_name, username) VALUES ('Nimbus Data Systems', 'NimbusData'); + INSERT INTO users (full_name, username) VALUES ('SSWUG.ORG', 'SSWUGorg'); + + + INSERT INTO follows (from_user, to_user) VALUES ('_DreamLead', 'Scootmedia'); + INSERT INTO follows (from_user, to_user) VALUES ('_DreamLead', 'MetiersInternet'); + INSERT INTO follows (from_user, to_user) VALUES ('GunnarSvalander', 'klout'); + INSERT INTO follows (from_user, to_user) VALUES ('GunnarSvalander', 'zillow'); + INSERT INTO follows (from_user, to_user) VALUES ('GEsoftware', 'DayJobDoc'); + INSERT INTO follows (from_user, to_user) VALUES ('GEsoftware', 'byosko'); + INSERT INTO follows (from_user, to_user) VALUES ('adrianburch', 'CindyCrawford'); + INSERT INTO follows (from_user, to_user) VALUES ('adrianburch', 'Arjantim'); + INSERT INTO follows (from_user, to_user) VALUES ('AndyRyder', 'MichaelDell'); + INSERT INTO follows (from_user, to_user) VALUES ('AndyRyder', 'Yahoo'); + INSERT INTO follows (from_user, to_user) VALUES ('Brett_Englebert', 'RealSkipBayless'); + INSERT INTO follows (from_user, to_user) VALUES ('Brett_Englebert', 'stephenasmith'); + INSERT INTO follows (from_user, to_user) VALUES ('NimbusData', 'dellock6'); + INSERT INTO follows (from_user, to_user) VALUES ('NimbusData', 'rohitkilam'); + INSERT INTO follows (from_user, to_user) VALUES ('SSWUGorg', 'drsql'); + INSERT INTO follows (from_user, to_user) VALUES ('SSWUGorg', 'steam_games'); + + + INSERT INTO tweets (text, created_at, username) VALUES ('What do you think about #emailing #campaigns #traffic in #USA? Is it a good market nowadays? do you have #databases?', 'Tue, 12 Feb 2013 08:43:09 +0000', '_DreamdLead'); + INSERT INTO tweets (text, created_at, username) VALUES ('Bill Gates Talks Databases, Free Software on Reddit http://t.co/ShX4hZlA #billgates #databases', 'Tue, 12 Feb 2013 07:31:06 +0000', 'MetiersInternet'); + INSERT INTO tweets (text, created_at, username) VALUES ('RT @KirkDBorne: Readings in #Databases: excellent reading list, many categories: http://t.co/S6RBUNxq via @rxin Fascinating.', 'Tue, 12 Feb 2013 07:30:24 +0000', 'GEsoftware'); + INSERT INTO tweets (text, created_at, username) VALUES ('RT @tisakovich: @NimbusData at the @Barclays Big Data conference in San Francisco today, talking #virtualization, #databases, and #flash memory.', 'Tue, 12 Feb 2013 06:58:22 +0000', 'adrianburch'); + INSERT INTO tweets (text, created_at, username) VALUES ('http://t.co/D3KOJIvF article about Madden 2013 using AI to prodict the super bowl #databases #bus311', 'Tue, 12 Feb 2013 05:29:41 +0000', 'MetiersInternet'); + INSERT INTO tweets (text, created_at, username) VALUES ('http://t.co/rBhBXjma an article about privacy settings and facebook #databases #bus311', 'Tue, 12 Feb 2013 05:24:17 +0000' ,'AndyRyder5'); + INSERT INTO tweets (text, created_at, username) VALUES ('#BUS311 University of Minnesotas NCFPD is creating #databases to prevent food fraud. http://t.co/0LsAbKqJ', 'Tue, 12 Feb 2013 01:49:19 +0000', 'Brett_Englebert'); + INSERT INTO tweets (text, created_at, username) VALUES ('#BUS311 companies might be protecting their production #databases, but what about their backup files? http://t.co/okJjV3Bm', 'Tue, 12 Feb 2013 01:31:52 +0000', 'Scootmedia'); + INSERT INTO tweets (text, created_at, username) VALUES ('@NimbusData CEO @tisakovich @BarclaysOnline Big Data conference in San Francisco today, talking #virtualization, #databases,& #flash memory', 'Mon, 11 Feb 2013 23:15:05 +0000', 'NimbusData'); + INSERT INTO tweets (text, created_at, username) VALUES ('Dont forget to sign up for our FREE expo this Friday: #Databases, #BI, and #Sharepoint: What You Need to Know! http://t.co/Ijrqrz29', 'Mon, 11 Feb 2013 22:15:37 +0000', 'SSWUGorg'); + ``` + +6. Retrieve some data + * Joint table of all users, and all followings for each user: + ```shell + SELECT * FROM users JOIN follows ON users.username=follows.from_user; + ``` + * The tweets of all users that are followed by the user *_DreamLead*: + ```text + SELECT to_user, text, created_at + FROM users u + JOIN follows f + ON u.username=f.from_user + JOIN tweets t + ON f.to_user=t.username + WHERE u.username='_DreamLead'; + ``` + * Use aggregation + ```text + SELECT to_user, COUNT(*) as tweets_count + FROM users u + JOIN follows f + ON u.username=f.from_user + JOIN tweets t + ON f.to_user=t.username + WHERE u.username='_DreamLead' + GROUP BY to_user; + ``` + +### Monitoring and alerting for an RDS database + +1. Open the CloudWatch console at [https://console\.aws\.amazon\.com/cloudwatch/](https://console.aws.amazon.com/cloudwatch/)\. + +2. In the navigation pane, choose **Alarms**, **All alarms**\. + +3. Choose **Create alarm**\. + +4. On the **Specify metric and conditions** page, choose **Select metric**\. + +5. In the search box, enter the name of your RDS database and press Enter\. + +6. Choose **RDS**, **Per\-Database Metrics**\. + +7. In the search box, enter **IOPS** and press Enter, then select **ReadIOPS** and **WriteIOPS** metrics. The graph will show both read and write i/o operations metric for your db. + +8. We would like to base the alarm on the total sum of read + write i/o. From **Add math**, choose **All functions**, **SUM**\. + +9. Choose the **Graphed metrics** tab, and edit the details for **Expression1** to **TotalIOPS**\. + +10. Change the **Period** to **1 minute**\. + +11. Clear selection from all metrics except for **TotalIOPS**\. + +12. Choose **Select metric**\. + +13. On the **Specify metric and conditions** page, enter a number of IOPS in **Define the threshold value**\. +For this tutorial, enter **100**. You can adjust this value for your workload requirements\. + +14. Choose **Next**, and the **Configure actions** page appears\. + +15. Keep **In alarm** selected, choose **Create new topic**, and enter the topic name and a valid email address\. + +16. Choose **Create topic**, and then choose **Next**\. + +17. On the **Add name and description** page, enter the **Alarm name** and **Alarm description**, and then choose **Next**\. + +18. Preview the alarm that you're about to create on the **Preview and create** page, and then choose **Create alarm**\. + +#### Testing your alarm + +It is very important to test all the alarms you set, in production environment if possible. + +19. Connect to an Amazon Linux EC2 instance in the same region of your DB instance. +20. Install PostgreSQL tooling package by +``` +sudo yum install postgresql-server postgresql-contrib +``` +21. Perform a load test of your server and watch the alarm in action +To initialize test tables, execute first: +```shell +PGPASSWORD= pgbench -i -U postgres -h +``` +Then run the full test by: +```shell +PGPASSWORD= pgbench -P 10 -t 10000 -j 10 -c 10 -U postgres -h +``` +While `` is you db password. `` is you RDS database url and `` is an existed table. + +For more information on the `pgbench` command, read [here](https://www.postgresql.org/docs/current/pgbench.html). + +## DynamoDB - Demo I - Basics + Lambda function + +### Create a table + +1. Open the DynamoDB console at [https://console.aws.amazon.com/dynamodb/](https://console.aws.amazon.com/dynamodb/) +2. In the navigation pane on the left side of the console, choose **Dashboard**. +3. On the right side of the console, choose **Create Table**. +4. Enter the table details as follows: + 1. For the table name, enter a unique table name. + 2. For the partition key, enter `Artist`. + 3. Enter `SongTitle` as the sort key. + 4. Choose **Customize settings**. + 5. On **Read/write capacity settings** choose **Provisioned** mode with autoscale capacity with a minimum capacity of **1** and maximum of **10**. +5. Choose **Create** to create the table. + +### Write and read data + + +1. On DynamoDB web console page, choose **PartiQL editor** on the left side menu. +2. The following example creates several new items in the `` table. The [PartiQL](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/ql-reference.html) is + a SQL-compatible query language for DynamoDB. + +```shell +INSERT INTO "" VALUE {'Artist':'No One You Know','SongTitle':'Call Me Today', 'AlbumTitle':'Somewhat Famous', 'Awards':'1'} + +INSERT INTO "" VALUE {'Artist':'No One You Know','SongTitle':'Howdy', 'AlbumTitle':'Somewhat Famous', 'Awards':'2'} + +INSERT INTO "" VALUE {'Artist':'Acme Band','SongTitle':'Happy Day', 'AlbumTitle':'Songs About Life', 'Awards':'10'} + +INSERT INTO "" VALUE {'Artist':'Acme Band','SongTitle':'PartiQL Rocks', 'AlbumTitle':'Another Album Title', 'Awards':'8'} +``` + +Query the data by + +```shell +SELECT * FROM "" WHERE Artist='Acme Band' AND SongTitle='Happy Day' +``` + +### Create and query a global secondary index + +1. In the navigation pane on the left side of the console, choose **Tables**. +2. Choose your table from the table list. +3. Choose the **Indexes** tab for your table. +4. Choose **Create** index. +5. For the **Partition key**, enter `AlbumTitle`. +6. For **Index** name, enter `AlbumTitle-index`. +7. Leave the other settings on their default values and choose **Create** index. + +8. You query the global secondary index through PartiQL by using the Select statement and providing the index name +```shell +SELECT * FROM ""."AlbumTitle-index" WHERE AlbumTitle='Somewhat Famous' +``` + + +### Process new items with DynamoDB Streams and Lambda + +#### Enable Streams + +1. In the navigation pane on the left side of the console, choose **Tables**. +2. Choose your table from the table list. +3. Choose the **Exports and streams** tab for your table. +4. Under **DynamoDB stream details** choose **Enable**. +5. Choose **New and old images** and click **Enable stream**. + +#### Create Lambda execution IAM role + +1. Open the IAM console at [https://console\.aws\.amazon\.com/iam/](https://console.aws.amazon.com/iam/)\. + +2. In the navigation pane, choose **Roles**, **Create role**\. + +3. On the **Trusted entity type** page, choose **AWS service** and the **Lambda** use case\. + +4. On the **Review** page, enter a name for the role and choose **Create role**\. +5. Edit your IAM role with the following inline policy +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": "lambda:InvokeFunction", + "Resource": "arn:aws:lambda:::function:*" + }, + { + "Effect": "Allow", + "Action": [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + "Resource": "arn:aws:logs:::*" + }, + { + "Effect": "Allow", + "Action": [ + "dynamodb:DescribeStream", + "dynamodb:GetRecords", + "dynamodb:GetShardIterator", + "dynamodb:ListStreams" + ], + "Resource": "arn:aws:dynamodb:::table//stream/*" + }, + { + "Effect": "Allow", + "Action": [ + "sns:Publish" + ], + "Resource": [ + "*" + ] + } + ] +} +``` + +Change the following placeholders to the appropriate values: ``, ``, ``, `` + + +The policy has four statements that allow your role to do the following: ++ Run a Lambda function. You create the function later in this tutorial\. ++ Access Amazon CloudWatch Logs\. The Lambda function writes diagnostics to CloudWatch Logs at runtime\. ++ Read data from the DynamoDB stream. ++ Publish messages to Amazon SNS\. + +#### Create a Lambda Function + +1. Open the [Functions page](https://console.aws.amazon.com/lambda/home#/functions) of the Lambda console\. + +2. Choose **Create function**\. + +3. Under **Basic information**, do the following: + + 1. Enter **Function name**. + + 2. For **Runtime**, confirm that **Node\.js 16\.x** is selected\. + + 3. For **Permissions** use your created role. + +4. Choose **Create function**\. +5. Enter your function, copy the content of `14_dynamodb_lambda_func/publishNewSong.js` and paste it in the **Code source**. Change `` to your SNS topic ARN you created in the previous exercise. +6. Click the **Deploy** button. +7. On the same page, click **Add trigger** and choose your Dynamo table as a source trigger. +8. Test your Lambda function by creating new items in the Dynamo table and watch for new emails in your inbox. + + +## DynamoDB - Demo II - integrate Dynamo with the PolyBot + +### Create a table + +In this step, you create a Youtube Bot table in Amazon DynamoDB. The table should store **video** items in the following structure: +```json +{ + "chatId": "TelegramChatId", + "videoId": "YouTubeVideoId", + "url": "VideoWebPageUrl", + "title": "VideoTitle" +} +``` +Users can send `/myvideos` to the bot in order to retrieve a list their video searches. + +Given the above schema, which attributes should represent primary and sort key? + +1. Sign in to the AWS Management Console and open the DynamoDB console at [https://console.aws.amazon.com/dynamodb/](https://console.aws.amazon.com/dynamodb/) +2. In the navigation pane on the left side of the console, choose **Dashboard**. +3. On the right side of the console, choose **Create Table**. +4. Enter the table details as follows: + 1. For the table name, enter a unique table name. + 2. For the partition key, enter `chatId`. + 3. Enter `videoId` as the sort key. + 4. Leave **Default settings** selected. +5. Choose **Create** to create the table. + +### Send and retrieve data from the PolyBot + +1. In the [PolyBot repository](https://github.com/alonitac/PolyBot.git), go to branch `save_user_data`. +2. Observe the code differences compared to branch `main`. +3. Compete the two TODOs in `bot.py` + 1. In **Use dynamo.put_item() to store user video** you should store in `item` dict in you Dynamo table. + 2. In **Use dynamo.query() to retrieve user videos** you should retrieve all videos by `charId` and send the video `url`s to the user chat. + +#### Code reference + +Store data: + +```python +response = dynamo.put_item( + Item=item, + TableName='your-table', +) +``` + +Retrieve data: + +```python + response = dynamo.query( + ExpressionAttributeValues={ + ':chatId': { + 'S': 'variable-containing-the-chat-id', + }, + }, + KeyConditionExpression=f'chatId = :chatId', + TableName='your-table', + ) + + for item in response['Items']: + # send item url and title to client + ``` \ No newline at end of file diff --git a/11_aws_demos/iam_security.md b/11_aws_demos/iam_security.md new file mode 100644 index 00000000..71d9ce5e --- /dev/null +++ b/11_aws_demos/iam_security.md @@ -0,0 +1,65 @@ +# IAM demos + +## Identity-based policies + +### Create IAM role with permissions over S3 and attach it to an EC2 instance + + +1. Open the IAM console at [https://console\.aws\.amazon\.com/iam/](https://console.aws.amazon.com/iam/)\. + +2. In the navigation pane, choose **Roles**, **Create role**\. + +3. On the **Trusted entity type** page, choose **AWS service** and the **EC2** use case\. Choose **Next: Permissions**\. + +4. On the **Attach permissions policy** page, search for **AmazonS3FullAccess** AWS managed policy\. + +5. On the **Review** page, enter a name for the role and choose **Create role**\. +6. Attach the role to your EC2 instance. + + +### Create a policy to access a certain bucket only and attach it to your IAM role. + +1. Open the [IAM console](https://console.aws.amazon.com/iam/). +2. From the console, open the IAM user or role that should have access to only a certain bucket. +3. In the **Permissions** tab, remove the **AmazonS3FullAccess** AWS managed policy. +4. Click **Add permissions** and **Create inline policy** +5. Choose **Import manages policy** and import **AmazonS3FullAccess**, switch to the JSON view of your new policy. +6. Inspired by [policies examples](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_examples.html) in AWS IAM docs, try to change the given policy JSON such that it allows the user to list, read, and write objects with a prefix `images/` +7. Save your policy, and validate your changes using the [IAM Policy Simulator](https://policysim.aws.amazon.com/). + +### Extend your policy + +9. Explore [S3 policy condition key elements](https://docs.aws.amazon.com/AmazonS3/latest/userguide/amazon-s3-policy-keys.html). +10. Try to add a condition to your above policy such that only objects in `STANDARD_IA` storage-class can be accessed. +11. Validate your changes. + +### Tag IAM users and roles to control what they can access + +12. In IAM roles console, choose your role. +13. In **Tags** tab, add a tag with the key `BucketPrefix` and some value according to your choice. +14. Instead of allow operation on prefix `images/`, try to allow access on a dynamic prefix according to the principal tag: +```text +"Resource": ["arn:aws:s3:::/${aws:PrincipalTag/BucketPrefix}/*"] +``` + +### Controlling access to EC2 using resource tags + +In this demo we are going to create a role which can start/stop EC2 instances belong to Development environment only. + +1. In IAM console, **Roles** page, create a new role with **AWS account** trusted entity type. +2. According to the policy described in [Controlling access to AWS resources](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_tags.html#access_tags_control-resources), create an inline policy for your role, that allows the principal assumed this role to start/stop EC2 instances that was tagged + with key `Env` and value `Dev` +3. Create and save the role. +4. Tag some of your EC2 instance with `Env=Dev`. +5. Now we would like to switch our AWS IAM user to assume the created role. + 1. In the IAM console, choose your user name on the navigation bar in the upper right\. It typically looks like this: ***username*@*account\_ID\_number\_or\_alias***\. + 2. Choose **Switch Role** + 3. On the **Switch Role** page, type the account ID number and the role. +6. Test your policy by trying to start/stop EC2 instances with/without appropriate `Env` tag. +7. Switch back to your IAM user. + +#### Force tagging policy for resources + +7. We now want to force a tagging policy in our AWS account. We want all EC2 instances to be tagged with a key `Env` with allowed values of `Dev`, `Test`, or `Prod`. +8. According to the policy described in [Controlling access during AWS requests](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_tags.html#access_tags_control-requests), add a statement to the above inline policy, that enforces the tagging policy for EC2 instances belonging to different environments. +9. Switch to your role and test your policy. diff --git a/11_aws_demos/lambda.md b/11_aws_demos/lambda.md new file mode 100644 index 00000000..0000ff14 --- /dev/null +++ b/11_aws_demos/lambda.md @@ -0,0 +1,89 @@ +# Lambda functions + +## PolyBot workers in Lambda function + +We will modify the PolyBot such that Workers are now Lambda functions and auto-scaled by Lambda's manner. + +![](https://d2908q01vomqb2.cloudfront.net/1b6453892473a467d07372d45eb05abc2031647a/2021/11/09/sqs1.png) + + +1. In the [PolyBot repo](https://github.com/alonitac/PolyBot.git), review branch `microservices_lambda`. Especially review `worker.Dockerfile` and `worker.py` +2. Create private container registry in [ECR](https://docs.aws.amazon.com/AmazonECR/latest/userguide/repository-create.html). +3. In your ECR repository page, click on **View push commands**, follow the instructions in order to build and push the Docker image defined by `Dockerfile.worker`. +4. From the same region of your ECR, [Create a Lambda function](https://docs.aws.amazon.com/lambda/latest/dg/gettingstarted-images.html#configuration-images-create) based on your Docker image. +5. Make sure your Lambda function has the needed permissions on SQS and S3. +6. [Configure your queue as an event source](https://docs.aws.amazon.com/lambda/latest/dg/with-sqs.html#events-sqs-eventsource) for your Lambda. +7. Test your app by running the Bot (either locally or from an EC2 instance), and make sure messaged are being processes by the Workers Lambda functions. + + +## Face blurring using Lambda and Step Function State Machine + +Face blurring is one of the best-known practices when anonymizing both images and videos. +We will implement an event-driven system for face blurring using composition of different Lambda functions and a state machine. + +Here is the high level architecture: + +![](https://d2908q01vomqb2.cloudfront.net/f1f836cb4ea6efb2a0b1b99f41ad8b103eff4b59/2022/01/06/ML-5902-image005-new.png) + +Let's get started. + +**\* Deploy your resources in a region where [Amazon Rekognition is supported](https://docs.aws.amazon.com/general/latest/gr/rekognition.html).** + +### Create the Lambda Functions + +#### Create the "face-detection" function + +1. Open the [Functions page](https://console.aws.amazon.com/lambda/home#/functions) of the Lambda console\. +2. Choose **Create function**\. +3. Choose **Author from scratch**. +4. Create a `Python 3.9` runtime function from scratch. +5. Copy and deploy `face-blur-lambdas/face-detection/*.py` as the function source code (use the console code editor). +6. The IAM role of this function should have the following permissions: `AmazonS3FullAccess`, `AmazonRekognitionFullAccess` and `AWSStepFunctionsFullAccess`. It's recommended to use the same IAM role for all functions! +7. Configure a trigger for an **All object create** events for a given S3 bucket on objects with `.mp4` suffix (create a bucket and enable event notification if needed). +8. Later on, when you create the Step Function state machine, add the following env var to this function: + `STATE_MACHINE_ARN=` + +#### Create the "check-rekognition-job-status" function + +1. Create a `Python 3.9` runtime function from scratch. Choose the same IAM role as the above function. +2. Copy and deploy `face-blur-lambdas/check-rekognition-job-status/lambda_function.py` as the function source code. + +#### Create the "get-rekognized-faces" function + +1. Create a `Python 3.9` runtime function from scratch. Choose the same IAM role as the above function. +2. Copy and deploy `face-blur-lambdas/get-rekognized-faces/lambda_function.py` as the function source code. + +#### Create the "blur-faces" function + +1. Create a **Container image** Lambda function based on the Docker image built from `face-blur-lambdas/blur-faces/Dockerfile`. Use an existing Docker image, or create an ECR and build the image by: + + 2. Open the Amazon ECR console at [https://console\.aws\.amazon\.com/ecr/repositories](https://console.aws.amazon.com/ecr/repositories). + 3. In the navigation pane, choose **Repositories**\. + 4. On the **Repositories** page, choose **Create repository**\. + 5. For **Repository name**, enter a unique name for your repository\. + 6. Choose **Create repository**\. + 7. Select the repository that you created and choose **View push commands** to view the steps to build and push an image to your new repository\. + +2. Add the following env var to this function: + `OUTPUT_BUCKET=` where `` is another bucket to which the processes videos will be uploaded (create one if needed). +3. This function is CPU and RAM intensive since it processes the video frame-by-frame. Make sure this it has enough time and space to finish (in the **General Configuration** tab, increase the timeout to 5 minutes and the memory to 2048MB). + + +### Create Step Function state machine + +1. Open the [Step Functions page](https://console.aws.amazon.com/lambda/home#/stepfunctions) of the Lambda console\. +2. Choose **Create state machine**. +3. Choose **Write your workflow in code** and edit the JSON in the **Definition** pane as follows: + 1. Copy and paste `face-blur-lambdas/state_machine.json` + 2. Change `< check-rekognition-job-status ARN >`, `< get-rekognized-faces ARN >` and `< blur-faces ARN >` according to the corresponding Lambda functions ARN. +4. Click **Next**. +5. Enter a unique name to your state machine. +6. Under **Logging**, enable ALL logging. +7. Choose **Create state machine**. + +### Test the system + +1. Upload a sample short mp4 video to the "input" S3 bucket (you can download [this](https://www.videvo.net/video/people-walking-on-street/2181/) video). +2. Observe the Lambda invocation, as well as the state machine execution flow. +3. Download the processes video from in "output" S3 bucket and watch the results. + diff --git a/11_aws_demos/load_balance_auto_scale.md b/11_aws_demos/load_balance_auto_scale.md new file mode 100644 index 00000000..3ed9a2f9 --- /dev/null +++ b/11_aws_demos/load_balance_auto_scale.md @@ -0,0 +1,206 @@ +# Load Balancing and Auth Scaling + +## Create Application Load Balancer + +### Configure a target group + +Configuring a target group allows you to register targets such as EC2 instances\. + +1. Open the Amazon EC2 console at [https://console\.aws\.amazon\.com/ec2/](https://console.aws.amazon.com/ec2/)\. + +2. In the left navigation pane, under **Load Balancing**, choose **Target Groups**\. + +3. Choose **Create target group**\. + +4. In the **Basic configuration** section, set the following parameters: + + 1. For **Choose a target type**, select **Instance** to specify targets by instance ID + + 2. For **Target group name**, enter a name for the target group\. + + 3. Leave the **Port** and **Protocol** as HTTP 8080. + + 4. For VPC, select your virtual private cloud \(VPC\) + + 5. For **Protocol version**, select **HTTP1**. + +5. In the **Health checks** section, modify the default settings as needed to perform a health checks to the Flask webserver at endpoint `/status` ([05_simple_webserver](../05_simple_webserver/app.py))\. + +6. Choose **Next**\. +7. In the **Register targets** page, add one or more targets by selecting one or more instances, enter one or more ports, and then choose **Include as pending below**\. +8. Choose **Create target group**\. + +### Configure a load balancer and a listener + +To create an Application Load Balancer, you must first provide basic configuration information for your load balancer, such as a name, scheme, and IP address type\. +Then, you provide information about your network, and one or more listeners\. +A listener is a process that checks for connection requests\. It is configured with a protocol and a port for connections from clients to the load balancer\. + +1. In the navigation pane, under **Load Balancing**, choose **Load Balancers**\. + +2. Choose **Create Load Balancer**\. + +3. Under **Application Load Balancer**, choose **Create**\. + +4. **Basic configuration** + + 1. For **Load balancer name**, enter a name for your load balancer\. + + 2. For **Scheme**, choose **Internet\-facing**. + An internet\-facing load balancer routes requests from clients to targets over the internet\. + + 3. For **IP address type**, choose **IPv4**. + +5. **Network mapping** + + 1. For **VPC**, select the VPC that you used for your EC2 instances\. As you selected **Internet\-facing** for **Scheme**, only VPCs with an internet gateway are available for selection\. + + 1. For **Mappings**, select two or more Availability Zones and corresponding subnets\. Enabling multiple Availability Zones increases the fault tolerance of your applications\. + +6. For **Security groups**, select an existing security group, or create a new one\. + + The security group for your load balancer must allow it to communicate with registered targets on both the listener port and the health check port\. The console can create a security group for your load balancer on your behalf with rules that allow this communication\. You can also create a security group and select it instead\. See [recommended rules](https://docs.aws.amazon.com/elasticloadbalancing/latest/application/load-balancer-update-security-groups.html#security-group-recommended-rules) + +7. For Listeners and routing, the default listener accepts HTTP traffic on port 80. Choose different ones port according to your app. For Default action, choose the target group that you created. +9. Review your configuration, and choose **Create load balancer**\. A few default attributes are applied to your load balancer during creation\. You can view and edit them after creating the load balancer\. For more information, see [Load balancer attributes](https://docs.aws.amazon.com/elasticloadbalancing/latest/application/application-load-balancers.html#load-balancer-attributes)\. + + +### Test the load balancer + +Deploy 2 EC2 instances within you VPC, in each instance, run the flask webservers. Perform the load test as in [13_http_load_test/load_test.py](../13_http_load_test/load_test.py). + + +## Application Load Balancer with TLS termination + +### Create TLS certificate + +We would like our load balancers to listen on HTTPS protocol for clients connection. In order to achieve that, we need to create and sign a digital certificate. + +In order to create your own SSL certificate, perform the following in your local machine (`openssl` required): +1. Generate private key as private.pem + ``` + openssl genrsa -out private.pem 2048 + ``` +2. Generate public key as public.pem + ``` + openssl rsa -in private.pem -outform PEM -pubout -out public.pem + ``` +3. Create a CSR (Certificate Signing Request) as certificate.csr + ``` + openssl req -new -key private.pem -out certificate.csr + ``` +4. Create a Self-signed certificate as certificate.crt + ``` + openssl x509 -req -days 365 -in certificate.csr -signkey private.pem -out certificate.crt + ``` + +IAM securely encrypts your private keys and stores the encrypted version in IAM SSL certificate storage. You cannot manage your certificates from the IAM Console. + +5. To upload a server certificate to IAM (make sure your local aws cli is configured with the proper credentials) + ```shell + aws iam upload-server-certificate --server-certificate-name --certificate-body file://certificate.crt --private-key file://private.pem + ``` + +### Add an HTTPS listener to your load balancer + +1. On the navigation pane, under **LOAD BALANCING**, choose **Load Balancers**\. + +2. Select a load balancer, and choose **Listeners**, **Add listener**\. + +3. For **Protocol : port**, choose **HTTPS** and keep the default port or enter a different port\. + +4. For **Default actions**, choose **Add action**, **Forward to** and choose a target group\. + +5. For **Security policy**, we recommend that you keep the default security policy\. + +6. For **Default SSL certificate**, choose **From IAM** and choose the certificate that you uploaded\. + +7. Choose **Save**\. + +8. Test your load balancer over HTTPS. + +### Working with sticky sessions + +1. On the navigation pane, under **Load Balancing**, choose **Target Groups**\. + +2. Choose the name of the target group to open its details page\. + +3. On the **Group details** tab, in the **Attributes** section, choose **Edit**\. + +4. On the **Edit attributes** page, do the following: + + 1. Select **Stickiness**\. + + 2. For **Stickiness type**, select **Load balancer generated cookie**\. + + 3. For **Stickiness duration**, specify a value between 1 second and 7 days\. + + 4. Choose **Save changes**\. + + +## Create AutoScaler + +### Create launch template + +1. Open the Amazon EC2 console at [https://console\.aws\.amazon\.com/ec2/](https://console.aws.amazon.com/ec2/)\. + +2. On the navigation pane, under **Instances**, choose **Launch Templates**\. + +3. Choose **Create launch template**\. Enter a name and provide a description for the initial version of the launch template\. + +4. Under **Auto Scaling guidance**, select the check box to have Amazon EC2 provide guidance to help create a template to use with Amazon EC2 Auto Scaling\. + +5. Under **Launch template contents**, fill out each required field and any optional fields as needed\. + + 1. **Application and OS Images \(Amazon Machine Image\)**: \(Required\) Choose the ID of the AMI for your instances\. + + 2. For **Instance type**, choose a single instance type that's compatible with the AMI that you specified\. + + 3. **Key pair \(login\)**: For **Key pair name**, choose your existing key pair. + + 4. **Network settings**: For **Firewall \(security groups\)**, use one or more security groups, or keep this blank. If you don't specify any security groups in your launch template, Amazon EC2 uses the default security group for the VPC that your Auto Scaling group will launch instances into\. + + 5. For **Resource tags**, specify tags by providing key and value combinations\. + +6. \(Optional\) Configure advanced settings\. + +8. When you are ready to create the launch template, choose **Create launch template**\. + + +### Create an Auto Scaling group using a launch template + +1. On the navigation bar at the top of the screen, choose the same AWS Region that you used when you created the launch template\. + +2. Choose **Create an Auto Scaling group**\. + +3. On the **Choose launch template or configuration** page, do the following: + + 1. For **Auto Scaling group name**, enter a name for your Auto Scaling group\. + + 1. For **Launch template**, choose an existing launch template\. + + 1. For **Launch template version**, choose whether the Auto Scaling group uses the default, the latest, or a specific version of the launch template when scaling out\. + + 1. Verify that your launch template supports all of the options that you are planning to use, and then choose **Next**\. + +4. On the **Choose instance launch options** page, under **Network**, for **VPC**, choose a VPC\. The Auto Scaling group must be created in the same VPC as the security group you specified in your launch template\. + +5. For **Availability Zones and subnets**, choose one or more subnets in the specified VPC\. Use subnets in multiple Availability Zones for high availability\. + +6. Choose **Next** to continue to the next step\. + +7. On the **Configure advanced options** page, configure the following options, and then choose **Next**: + + 1. Register your Amazon EC2 instances with your load balancer. + +8. On the **Configure group size and scaling policies** page, configure the following options, and then choose **Next**: + + 1. For **Desired capacity**, enter the initial number of instances to launch: 0\. + + 1. To automatically scale the size of the Auto Scaling group, choose **Target tracking scaling policy** and follow the directions\. + +9. \(Optional\) To receive notifications, for **Add notification**, configure the notification, and then choose **Next**\. + +10. Choose **Add tag**, provide a tag key and value. + +11. On the **Review** page, choose **Create Auto Scaling group**\. diff --git a/11_aws_demos/network.md b/11_aws_demos/network.md new file mode 100644 index 00000000..b6176f95 --- /dev/null +++ b/11_aws_demos/network.md @@ -0,0 +1,185 @@ +# Networking + +## VPC with a single public subnet + +![publicVPC](https://docs.aws.amazon.com/vpc/latest/userguide/images/case-1_updated.png) + +The configuration for this network includes the following: ++ A virtual private cloud \(VPC\) with a size /16 IPv4 CIDR block: `10.0.0.0/16`. This provides 65,536 private IPv4 addresses\. ++ A subnet with a size /24 IPv4 CIDR block: `10.0.0.0/24`. This provides 256 private IPv4 addresses\. ++ An internet gateway\. This connects the VPC to the internet and to other AWS services\. ++ An instance with a private IPv4 address in the subnet range, which enables the instance to communicate with other instances in the VPC, and a public IPv4 address which enables the instance to connect to the internet and to be reached from the internet\. ++ A custom route table associated with the subnet\. The route table entries enable instances in the subnet to use IPv4 to communicate with other instances in the VPC, and to communicate directly over the internet\. A subnet that's associated with a route table that has a route to an internet gateway is known as a *public subnet*\. + + +### Create a VPC + +1. Open the Amazon VPC console at [https://console\.aws\.amazon\.com/vpc/](https://console.aws.amazon.com/vpc). + +2. In the navigation pane, choose **Your VPCs**, **Create VPC**\. + +3. Under **Resources to create**, choose **VPC only**\. + +4. Specify the following VPC details\. + + **Name tag**: Provide a name for your VPC\. Doing so creates a tag with a key of `Name` and the value that you specify\. + + **IPv4 CIDR block**: Specify an IPv4 CIDR block of `10.0.0.0/16` for your VPC\. The CIDR block size must have a size between /16 and /28\. More information can be found in [RFC 1918](http://www.faqs.org/rfcs/rfc1918.html). + + + **Tenancy**: Choose the default tenancy option for this VPC\. + + **Default** ensures that EC2 instances launched in this VPC use the EC2 instance tenancy attribute specified when the EC2 instance is launched\. + + **Dedicated** ensures that EC2 instances launched in this VPC are run on dedicated tenancy instances regardless of the tenancy attribute specified at launch\. + +5. Choose **Create VPC**\. + + +### Create a subnet in your VPC + +To add a new subnet to your VPC, you must specify an IPv4 CIDR block for the subnet from the range of your VPC\. You can specify the Availability Zone in which you want the subnet to reside\. You can have multiple subnets in the same Availability Zone\. + +1. In the navigation pane, choose **Subnets**\. + +2. Choose **Create subnet**\. + +3. For **VPC ID**: Choose the VPC for the subnet\. + +4. For **Subnet name**, enter a name for your subnet\. Doing so creates a tag with a key of `Name` and the value that you specify\. + +5. For **Availability Zone**, you can choose a Zone for your subnet, or leave the default **No Preference** to let AWS choose one for you\. + +6. For **IPv4 CIDR block**, enter an IPv4 CIDR block for your subnet\: `10.0.0.0/24`\. + +7. Choose **Create subnet**\. + +### Create a custom route table + +1. In the navigation pane, choose **Route Tables**\. + +2. Choose **Create route table**\. + +3. For **Name tag**, enter a name for your route table\. + +4. For **VPC**, choose your VPC\. + +5. Choose **Create**\. + + +### Create and attach an internet gateway + +After you create an internet gateway, attach it to your VPC\. + +**To create an internet gateway and attach it to your VPC** + +1. In the navigation pane, choose **Internet Gateways**, and then choose **Create internet gateway**\. + +2. Name your internet gateway\. + +3. Choose **Create internet gateway**\. + +4. Select the internet gateway that you just created, and then choose **Actions, Attach to VPC**\. + +5. Select your VPC from the list, and then choose **Attach internet gateway**\. + + +### Add Internet Gateway as a target in a custom route table + +When you create a subnet, we automatically associate it with the main route table for the VPC\. By default, the main route table doesn't contain a route to an internet gateway\. The following procedure uses your custom route table and creates a route that sends traffic destined outside the VPC to the internet gateway, and then associates it with your subnet\. + +1. Select the custom route table that you just created\. The details pane displays tabs for working with its routes, associations, and route propagation\. + +2. On the **Routes** tab, choose **Edit routes**, **Add route**, and add the following routes as necessary\. Choose **Save changes** when you're done\. + + For IPv4 traffic, specify `0.0.0.0/0` in the **Destination** box, and select the internet gateway ID in the **Target** list\. + +3. On the **Subnet associations** tab, choose **Edit subnet associations**, select the check box for the subnet, and then choose **Save associations**\. + + +### Test your VPC + +Create an EC2 instance within you VPC, connect to it and access the internet. + + +## VPC with public and private subnets with NAT gateway + +![publicVPC](https://docs.aws.amazon.com/vpc/latest/userguide/images/scenario-ipv6-diagram_1_updated.png) + + +This architecture is suitable if you want to run a public-facing web application, while maintaining back-end servers that aren't publicly accessible. +A common example is web servers in a public subnet and the database in a private subnet. +You can set up security and routing so that the web servers can communicate with the database servers. + +The instances in the public subnet can send outbound traffic directly to the internet, whereas the instances in the private subnet can't. +Instead, the instances in the private subnet can access the internet by using a network address translation (NAT) gateway that resides in the public subnet. +The database can connect to the internet for software updates using the NAT gateway, but the internet cannot establish connections to the database. + + +The configuration for this scenario includes the following: ++ A VPC with a size /16 IPv4 CIDR block: 10\.0\.0\.0/16\. This provides 65,536 private IPv4 addresses\. ++ A public subnet with a size /24 IPv4 CIDR block: 10\.0\.0\.0/24\. This provides 256 private IPv4 addresses\. A public subnet is a subnet that's associated with a route table that has a route to an internet gateway\. ++ A private subnet with a size /24 IPv4 CIDR block: 10\.0\.1\.0/24\. This provides 256 private IPv4 addresses\. ++ An internet gateway\. This connects the VPC to the internet and to other AWS services\. ++ A NAT gateway with its own Elastic IP address\. Instances in the private subnet can send requests to the internet through the NAT gateway over IPv4 \(for example, for software updates\)\. ++ A custom route table associated with the public subnet\. This route table contains an entry that enables instances in the subnet to communicate with other instances in the VPC, and an entry that enables instances in the subnet to communicate directly with the internet over IPv4\. ++ The main route table associated with the private subnet\. The route table contains an entry that enables instances in the subnet to communicate with other instances in the VPC, and an entry that enables instances in the subnet to communicate with the internet through the NAT gateway. + + +### Create another subnet + +As per the above section, create another subnet within your VPC with CIDR block of `10\.0\.1\.0/24`. This subnet should be associated with the main route table. + +### Create and attach a NAT gateway + +NAT Gateway must be associated with an Elastic IP Address (static IP reservation service of AWS) + +#### Allocate an Elastic IP Address + + +1. Open the Amazon EC2 console at [https://console\.aws\.amazon\.com/ec2/](https://console.aws.amazon.com/ec2/)\. + +2. In the navigation pane, choose **Network & Security**, **Elastic IPs**\. + +3. Choose **Allocate Elastic IP address**\. + +4. For **Public IPv4 address pool**, choose **Amazon's pool of IPv4 addresses** + +5. Add a **Name** tag\. + +6. Choose **Allocate**\. + +#### Create a NAT Gateway + + +1. Open the Amazon VPC console at [https://console\.aws\.amazon\.com/vpc/](https://console.aws.amazon.com/vpc/)\. + +2. In the navigation pane, choose **NAT Gateways**\. + +3. Choose **Create NAT Gateway** and do the following: + + 1. Specify a name for the NAT gateway\. This creates a tag where the key is **Name** and the value is the name that you specify\. + + 2. Select the subnet in which to create the NAT gateway\. + + 3. For **Connectivity type**, select **Public** \(the default\) to create a public NAT gateway\. + + 4. For **Elastic IP allocation ID**, select the Elastic IP address to associate with the NAT gateway\. + + 5. Choose **Create a NAT Gateway**\. + +#### Add NAT Gateway as a target in the main route table + +1. Select the main route table of your VPC\. The details pane displays tabs for working with its routes, associations, and route propagation\. + +2. On the **Routes** tab, choose **Edit routes**, **Add route**, and add the following routes as necessary\. Choose **Save changes** when you're done\. + + For IPv4 traffic, specify `0.0.0.0/0` in the **Destination** box, and select your NAT gateway ID in the **Target** list\. + + +### Deploy an app in your VPC + +- Create an EC2 instance in each subnet (public and private) +- Connect to the instance in the public subnet +- From the instance's terminal, connect to the instance in the private subnet +- Deploy `mongo:5.0` Docker container with port `27017` published (`-p`) and with the following [environment variables](https://docs.docker.com/engine/reference/commandline/run/#set-environment-variables--e---env---env-file): + - `MONGO_INITDB_ROOT_USERNAME=mongo` + - `MONGO_INITDB_ROOT_PASSWORD=mongo` +- Deploy `nanajanashia/k8s-demo-app:v1.0` Docker container in the instance resides in the public subnet with port `3000` published in the host, and the following env vars: + - `USER_NAME=mongo` + - `USER_PWD=mongo` + - `DB_URL=` +- Try to access your app form the internet. \ No newline at end of file diff --git a/11_aws_demos/questions.md b/11_aws_demos/questions.md new file mode 100644 index 00000000..0227b69f --- /dev/null +++ b/11_aws_demos/questions.md @@ -0,0 +1,612 @@ +# Questions + +## EC2 + +1. You know that you need 24 CPUs for your production server. You also know + that your compute capacity is going to remain fixed until next year, so you need + to keep the production server up and running during that time. What pricing + option would you go with? + 1. Choose the spot instance + 2. Choose the on-demand instance + 3. Choose the three-year reserved instance + 4. Choose the one-year reserved instance + + +2. You are planning to run a database on an EC2 instance. You know that the database + is pretty heavy on I/O. The DBA told you that you would need a minimum of + 8,000 IOPS. What is the storage option you should choose? + 1. EBS volume with magnetic hard drive + 2. Store all the data files in the ephemeral storage of the server + 3. EBS volume with provisioned IOPS + 4. EBS volume with general-purpose SSD + + +3. You are running your application on a bunch of on-demand servers. On weekends + you have to kick off a large batch job, and you are planning to add capacity. The + batch job you are going to run over the weekend can be restarted if it fails. What is + the best way to secure additional compute resources? + 1. Use the spot instance to add compute for the weekend + 2. Use the on-demand instance to add compute for the weekend + 3. Use the on-demand instance plus PIOPS storage for the weekend resource + 4. Use the on-demand instance plus a general-purpose EBS volume for the weekend resource + + +4. You have a compliance requirement that you should own the entire physical + hardware and no other customer should run any other instance on the physical + hardware. What option should you choose? + 1. Put the hardware inside the VPC so that no other customer can use it + 2. Use a dedicated instance + 3. Reserve the EC2 for one year + 4. Reserve the EC2 for three years + + +5. You have created an instance in EC2, and you want to connect to it. What should + you do to log in to the system for the first time? + 1. Use the username/password combination to log in to the server + 2. Use the key-pair combination (private and public keys) + 3. Use your cell phone to get a text message for secure login + 4. Log in via the root user + + +6. What are the characteristics of AMI that are backed up by the instance store? + (Choose two.) + 1. The data persists even after the instance reboot. + 2. The data is lost when the instance is shut down. + 3. The data persists when the instance is shut down. + 4. The data persists when the instance is terminated. + + +7. How can you make a cluster of an EC2 instance? + 1. By creating all the instances within a VPC + 2. By creating all the instances in a public subnet + 3. By creating all the instances in a private subnet + 4. By creating a placement group + + +8. You need to take a snapshot of the EBS volume. How long will the EBS remain + unavailable? + 1. The volume will be available immediately. + 2. EBS magnetic drive will take more time than SSD volumes. + 3. It depends on the size of the EBS volume. + 4. It depends on the actual data stored in the EBS volume. + + +9. What are the different ways of making an EC2 server available to the public? + 1. Create it inside a public subnet + 2. Create it inside a private subnet and assign a NAT device + 3. Attach an IPv6 IP address + 4. Allocate that with a load balancer and expose the load balancer to the public + + +10. The application workload changes constantly, and to meet that, you keep on + changing the hardware type for the application server. Because of this, you + constantly need to update the web server with the new IP address. How can + you fix this problem? + 1. Add a load balancer + 2. Add an IPv6 IP address + 3. Add an EIP to it + 4. Use a reserved EC2 instance + + +11. Your web application needs four instances to support steady traffic nearly all of the time. On the last + day of each month, the traffic triples. What is a cost-effective way to handle this traffic pattern? + 1. Run 12 Reserved Instances all of the time. + 2. Run four On-Demand Instances constantly, then add eight more On-Demand Instances on the last day of each month. + 3. Run four Reserved Instances constantly, then add eight On-Demand Instances on the last day of each month. + 4. Run four On-Demand Instances constantly, then add eight Reserved Instances on the last day of each month. + + +12. Your order-processing application processes orders extracted from a queue with two Reserved + Instances processing 10 orders/minute. If an order fails during processing, then it is returned to the + queue without penalty. Due to a weekend sale, the queues have several hundred orders backed up. + While the backup is not catastrophic, you would like to drain it so that customers get their + confirmation emails faster. What is a cost-effective way to drain the queue for orders? + 1. Create more queues. + 2. Deploy additional Spot Instances to assist in processing the orders. + 3. Deploy additional Reserved Instances to assist in processing the orders. + 4. Deploy additional On-Demand Instances to assist in processing the orders. + + +13. Which of the following must be specified when launching a new Amazon Elastic Compute Cloud + (Amazon EC2) Windows instance? (Choose 2 answers) + 1. The Amazon EC2 instance ID + 2. Password for the administrator account + 3. Amazon EC2 instance type + 4. Amazon Machine Image (AMI) + + +14. You have purchased an m3.xlarge Linux Reserved instance in us-east-1a. In which ways can you + modify this reservation? (Choose 2 answers) + 1. Change it into two m3.large instances. + 2. Change it to a Windows instance. + 3. Move it to us-east-1b. + 4. Change it to an m4.xlarge. + + +15. Your instance is associated with two security groups. The first allows Remote Desktop Protocol + (RDP) access over port 3389 from Classless Inter-Domain Routing (CIDR) block 72.14.0.0/16. The + second allows HTTP access over port 80 from CIDR block 0.0.0.0/0. What traffic can reach your + instance? + 1. RDP and HTTP access from CIDR block 0.0.0.0/0 + 2. No traffic is allowed. + 3. RDP and HTTP traffic from 72.14.0.0/16 + 4. RDP traffic over port 3389 from 72.14.0.0/16 and HTTP traffic over port 80 from 0.0.00/0 + + +16. Which of the following are features of enhanced networking? (Choose 3 answers) + 1. More Packets Per Second (PPS) + 2. Lower latency + 3. Multiple network interfaces + 4. Border Gateway Protocol (BGP) routing + 5. Less jitter + + +17. You are creating a High-Performance Computing (HPC) cluster and need very low latency and high + bandwidth between instances. What combination of the following will allow this? (Choose 3 + answers) + 1. Use an instance type with 10 Gbps network performance. + 2. Put the instances in a placement group. + 3. Use Dedicated Instances. + 4. Enable enhanced networking on the instances. + 5. Use Reserved Instances. + + +18. Which Amazon Elastic Compute Cloud (Amazon EC2) feature ensures that your instances will not + share a physical host with instances from any other AWS customer? + 1. Amazon Virtual Private Cloud (VPC) + 2. Placement groups + 3. Dedicated Instances + 4. Reserved Instances + + +19. Which of the following are true of instance stores? (Choose 2 answers) + 1. Automatic backups + 2. Data is lost when the instance stops. + 3. Very high IOPS + 4. Charge is based on the total amount of storage provisioned. + + +20. Which of the following are features of Amazon Elastic Block Store (Amazon EBS)? (Choose 2 + answers) + 1. Data stored on Amazon EBS is automatically replicated within an Availability Zone. + 2. Amazon EBS data is automatically backed up to tape. + 3. Amazon EBS volumes can be encrypted transparently to workloads on the attached instance. + 4. Data on an Amazon EBS volume is lost when the attached instance is stopped. + + +21. You are restoring an Amazon Elastic Block Store (Amazon EBS) volume from a snapshot. How long + will it be before the data is available? + 1. It depends on the provisioned size of the volume. + 2. The data will be available immediately. + 3. It depends on the amount of data stored on the volume. + 4. It depends on whether the attached instance is an Amazon EBS-optimized instance. + + +22. You have a workload that requires 15,000 consistent IOPS for data that must be durable. What + combination of the following steps do you need? (Choose 2 answers) + 1. Use an Amazon Elastic Block Store (Amazon EBS)-optimized instance. + 2. Use an instance store. + 3. Use a Provisioned IOPS SSD volume. + 4. Use a magnetic volume. + + +23. Which of the following can be accomplished through bootstrapping? + 1. Install the most current security updates. + 2. Install the current version of the application. + 3. Configure Operating System (OS) services. + 4. All of the above. + + +24. How can you connect to a new Linux instance using SSH? + 1. Decrypt the root password. + 2. Using a certificate + 3. Using the private half of the instance’s key pair + 4. Using Multi-Factor Authentication (MFA) + + +25. VM Import/Export can import existing virtual machines as: + 1. Amazon Elastic Block Store (Amazon EBS) volumes + 2. Amazon Elastic Compute Cloud (Amazon EC2) instances + 3. Amazon Machine Images (AMIs) + 4. Security groups + + +26. Which of the following can be used to address an Amazon Elastic Compute Cloud (Amazon EC2) + instance over the web? (Choose 2 answers) + 1. Windows machine name + 2. Public DNS name + 3. Amazon EC2 instance ID + 4. Elastic IP address + + +27. Using the correctly decrypted Administrator password and RDP, you cannot log in to a Windows + instance you just launched. Which of the following is a possible reason? + 1. There is no security group rule that allows RDP access over port 3389 from your IP address. + 2. The instance is a Reserved Instance. + 3. The instance is not using enhanced networking. + 4. The instance is not an Amazon EBS-optimized instance. + + +28. You have a workload that requires 1 TB of durable block storage at 1,500 IOPS during normal use. + Every night there is an Extract, Transform, Load (ETL) task that requires 3,000 IOPS for 15 minutes. + What is the most appropriate volume type for this workload? + 1. Use a Provisioned IOPS SSD volume at 3,000 IOPS. + 2. Use an instance store. + 3. Use a general-purpose SSD volume. + 4. Use a magnetic volume. + + +29. How are you billed for elastic IP addresses? + 1. Hourly when they are associated with an instance + 2. Hourly when they are not associated with an instance + 3. Based on the data that flows through them + 4. Based on the instance type to which they are attached + + +## S3 + + +1. In what ways does Amazon Simple Storage Service (Amazon S3) object storage differ from block + and file storage? (Choose 2 answers) + A. Amazon S3 stores data in fixed size blocks. + B. Objects are identified by a numbered address. + C. Objects can be any size. + D. Objects contain both data and metadata. + E. Objects are stored in buckets. + + +2. Which of the following are not appropriates use cases for Amazon Simple Storage Service (Amazon + S3)? (Choose 2 answers) + A. Storing web content + B. Storing a file system mounted to an Amazon Elastic Compute Cloud (Amazon EC2) instance + C. Storing backups for a relational database + D. Primary storage for a database + E. Storing logs for analytics + + +3. What are some of the key characteristics of Amazon Simple Storage Service (Amazon S3)? (Choose + 3 answers) + A. All objects have a URL. + B. Amazon S3 can store unlimited amounts of data. + C. Objects are world-readable by default. + D. Amazon S3 uses a REST (Representational State Transfer) Application Program Interface (API). + E. You must pre-allocate the storage in a bucket. + + +4. Which features can be used to restrict access to Amazon Simple Storage Service (Amazon S3) data? + (Choose 3 answers) + A. Enable static website hosting on the bucket. + B. Create a pre-signed URL for an object. + C. Use an Amazon S3 Access Control List (ACL) on a bucket or object. + D. Use a lifecycle policy. + E. Use an Amazon S3 bucket policy. + + +5. Your application stores critical data in Amazon Simple Storage Service (Amazon S3), which must + be protected against inadvertent or intentional deletion. How can this data be protected? (Choose 2 + answers) + A. Use cross-region replication to copy data to another bucket automatically. + B. Set a vault lock. + C. Enable versioning on the bucket. + D. Use a lifecycle policy to migrate data to Amazon Glacier. + E. Enable MFA Delete on the bucket. + + +6. Your company stores documents in Amazon Simple Storage Service (Amazon S3), but it wants to + minimize cost. Most documents are used actively for only about a month, then much less frequently. + However, all data needs to be available within minutes when requested. How can you meet these + requirements? + A. Migrate the data to Amazon S3 Reduced Redundancy Storage (RRS) after 30 days. + B. Migrate the data to Amazon Glacier after 30 days. + C. Migrate the data to Amazon S3 Standard – Infrequent Access (IA) after 30 days. + D. Turn on versioning, then migrate the older version to Amazon Glacier. + + +7. How is data stored in Amazon Simple Storage Service (Amazon S3) for high durability? + A. Data is automatically replicated to other regions. + B. Data is automatically replicated within a region. + C. Data is replicated only if versioning is enabled on the bucket. + D. Data is automatically backed up on tape and restored if needed. + + +8. Based on the following Amazon Simple Storage Service (Amazon S3) URL, which one of the + following statements is correct? https://bucket1.abc.com.s3.amazonaws.com/folderx/myfile.doc + A. The object “myfile.doc” is stored in the folder “folderx” in the bucket “bucket1.abc.com.” + B. The object “myfile.doc” is stored in the bucket “bucket1.abc.com.” + C. The object “folderx/myfile.doc” is stored in the bucket “bucket1.abc.com.” + D. The object “myfile.doc” is stored in the bucket “bucket1.” + + +9. To have a record of who accessed your Amazon Simple Storage Service (Amazon S3) data and from + where, you should do what? + A. Enable versioning on the bucket. + B. Enable website hosting on the bucket. + C. Enable server access logs on the bucket. + D. Create an AWS Identity and Access Management (IAM) bucket policy. + E. Enable Amazon CloudWatch logs. + + +10. What are some reasons to enable cross-region replication on an Amazon Simple Storage Service + (Amazon S3) bucket? (Choose 2 answers) + A. You want a backup of your data in case of accidental deletion. + B. You have a set of users or customers who can access the second bucket with lower latency. + C. For compliance reasons, you need to store data in a location at least 300 miles away from the first region. + D. Your data needs at least five nines of durability. + + +11. Your company requires that all data sent to external storage be encrypted before being sent. Which + Amazon Simple Storage Service (Amazon S3) encryption solution will meet this requirement? + A. Server-Side Encryption (SSE) with AWS-managed keys (SSE-S3) + B. SSE with customer-provided keys (SSE-C) + C. Client-side encryption with customer-managed keys + D. Server-side encryption with AWS Key Management Service (AWS KMS) keys (SSE-KMS) + + +12. What is needed before you can enable cross-region replication on an Amazon Simple Storage + Service (Amazon S3) bucket? (Choose 2 answers) + A. Enable versioning on the bucket. + B. Enable a lifecycle rule to migrate data to the second region. + C. Enable static website hosting. + D. Create an AWS Identity and Access Management (IAM) policy to allow Amazon S3 to replicate objects on your behalf. + + +13. Your company has 100TB of financial records that need to be stored for seven years by law. + Experience has shown that any record more than one-year old is unlikely to be accessed. Which of + the following storage plans meets these needs in the most cost efficient manner? + A. Store the data on Amazon Elastic Block Store (Amazon EBS) volumes attached to t2.micro instances. + B. Store the data on Amazon Simple Storage Service (Amazon S3) with lifecycle policies that change the storage class to Amazon Glacier after one year and delete the object after seven years. + C. Store the data in Amazon DynamoDB and run daily script to delete data older than seven years. + D. Store the data in Amazon Elastic MapReduce (Amazon EMR). + + +14. Amazon Simple Storage Service (S3) bucket policies can restrict access to an Amazon S3 bucket + and objects by which of the following? (Choose 3 answers) + A. Company name + B. IP address range + C. AWS account + D. Country of origin + E. Objects with a specific prefix + + +15. What must be done to host a static website in an Amazon Simple Storage Service (Amazon S3) + bucket? (Choose 3 answers) + A. Configure the bucket for static hosting and specify an index and error document. + B. Create a bucket with the same name as the website. + C. Enable File Transfer Protocol (FTP) on the bucket. + D. Make the objects in the bucket world-readable. + E. Enable HTTP on the bucket. + + +16. You have valuable media files hosted on AWS and want them to be served only to authenticated + users of your web application. You are concerned that your content could be stolen and distributed + for free. How can you protect your content? + A. Use static web hosting. + B. Generate pre-signed URLs for content in the web application. + C. Use AWS Identity and Access Management (IAM) policies to restrict access. + D. Use logging to track your content. + + +17. Amazon Glacier is well-suited to data that is which of the following? (Choose 2 answers) + A. Is infrequently or rarely accessed + B. Must be immediately available when needed + C. Is available after a three- to five-hour restore period + D. Is frequently erased within 30 days + + +18. What is the best way to protect a file in Amazon S3 against accidental delete? + A. Upload the files in multiple buckets so that you can restore from another when a file is deleted + B. Back up the files regularly to a different bucket or in a different region + C. Enable versioning on the S3 bucket + D. Use MFA for deletion + E. Use cross-region replication + + +19. Amazon S3 provides 99.999999999 percent durability. Which of the following + are true statements? (Choose all that apply.) + A. The data is mirrored across multiple AZs within a region. + B. The data is mirrored across multiple regions to provide the durability SLA. + C. The data in Amazon S3 Standard is designed to handle the concurrent loss of two facilities. + D. The data is regularly backed up to AWS Snowball to provide the durability SLA. + E. The data is automatically mirrored to Amazon Glacier to achieve high availability. + + +20. To set up a cross-region replication, what statements are true? (Choose all + that apply.) + A. The source and target bucket should be in a same region. + B. The source and target bucket should be in different region. + C. You must choose different storage classes across different regions. + D. You need to enable versioning and must have an IAM policy in place to replicate. + E. You must have at least ten files in a bucket. + + +21. You want to move all the files older than a month to S3 IA. What is the best way + of doing this? + A. Copy all the files using the S3 copy command + B. Set up a lifecycle rule to move all the files to S3 IA after a month + C. Download the files after a month and re-upload them to another S3 bucket with IA + D. Copy all the files to Amazon Glacier and from Amazon Glacier copy them to S3 IA + + +22. What are the various way you can control access to the data stored in S3? + (Choose all that apply.) + A. By using IAM policy + B. By creating ACLs + C. By encrypting the files in a bucket + D. By making all the files public + E. By creating a separate folder for the secure files + + +23. How much data can you store on S3? + A. 1 petabyte per account + B. 1 exabyte per account + C. 1 petabyte per region + D. 1 exabyte per region + E. Unlimited + + +24. What is the best way to delete multiple objects from S3? + A. Delete the files manually using a console + B. Use multi-object delete + C. Create a policy to delete multiple files + D. Delete all the S3 buckets to delete the files + + +25. I shut down my EC2 instance, and when I started it, I lost all my data. What + could be the reason for this? + A. The data was stored in the local instance store. + B. The data was stored in EBS but was not backed up to S3. + C. I used an HDD-backed EBS volume instead of an SSD-backed EBS volume. + D. I forgot to take a snapshot of the instance store. + + +26. I am running an Oracle database that is very I/O intense. My database administrator + needs a minimum of 3,600 IOPS. If my system is not able to meet that number, my + application won’t perform optimally. How can I make sure my application always + performs optimally? + A. Use Elastic File System since it automatically handles the performance + B. Use Provisioned IOPS SSD to meet the IOPS number + C. Use your database files in an SSD-based EBS volume and your other files in an HDD-based EBS volume + D. Use a general-purpose SSD under a terabyte that has a burst capability + + +27. Your application needs a shared file system that can be accessed from multiple + EC2 instances across different AZs. How would you provision it? + A. Mount the EBS volume across multiple EC2 instances + B. Use an EFS instance and mount the EFS across multiple EC2 instances across + multiple AZs + C. Access S3 from multiple EC2 instances + D. Use EBS with Provisioned IOPS + + +28. You want to run a mapreduce job (a part of the big data workload) for a noncritical + task. Your main goal is to process it in the most cost-effective way. The task is + throughput sensitive but not at all mission critical and can take a longer time. + Which type of storage would you choose? + A. Throughput Optimized HDD (st1) + B. Cold HDD (sc1) + C. General-Purpose SSD (gp2) + D. Provisioned IOPS (io1) + + + +## IAM + + + +1. Which of the following methods will allow an application using an AWS SDK to be authenticated as + a principal to access AWS Cloud services? (Choose 2 answers) + 1. Create an IAM user and store the username and password for the user in the application’s configuration. + 2. Create an IAM user and store both parts of the access key for the user in the application’s configuration. + 3. Run the application on an Amazon EC2 instance with an assigned IAM role. + 4. Make all the API calls over an SSL connection. + + +2. Which of the following are found in an IAM policy? (Choose 2 answers) + 1. Service Name + 2. Region + 3. Action + 4. Password + + +3. Your AWS account administrator left your company today. The administrator had access to the root + user and a personal IAM administrator account. With these accounts, he generated other IAM + accounts and keys. Which of the following should you do today to protect your AWS infrastructure? + (Choose 3 answers) + 1. Change the password and add MFA to the root user. + 2. Put an IP restriction on the root user. + 3. Rotate keys and change passwords for IAM accounts. + 4. Delete all IAM accounts. + 5. Delete the administrator’s personal IAM account. + 6. Relaunch all Amazon EC2 instances with new roles. + + +4. Which of the following actions can be authorized by IAM? (Choose 2 answers) + 1. Installing ASP.NET on a Windows Server + 2. Launching an Amazon Linux EC2 instance + 3. Querying an Oracle database + 4. Adding a message to an Amazon Simple Queue Service (Amazon SQS) queue + + +5. Which of the following are IAM security features? (Choose 2 answers) + 1. Password policies + 2. Amazon DynamoDB global secondary indexes + 3. MFA + 4. Consolidated Billing + + +6. Which of the following are benefits of using Amazon EC2 roles? (Choose 2 answers) + 1. No policies are required. + 2. Credentials do not need to be stored on the Amazon EC2 instance. + 3. Key rotation is not necessary. + 4. Integration with Active Directory is automatic. + + +7. Which of the following are based on temporary security tokens? (Choose 2 answers) + 1. Amazon EC2 roles + 2. MFA + 3. Root user + 4. Federation + + +8. Your security team is very concerned about the vulnerability of the IAM administrator user accounts + (the accounts used to configure all IAM features and accounts). What steps can be taken to lock + down these accounts? (Choose 3 answers) + 1. Add multi-factor authentication (MFA) to the accounts. + 2. Limit logins to a particular U.S. state. + 3. Implement a password policy on the AWS account. + 4. Apply a source IP address condition to the policy that only grants permissions when the user is on the corporate network. + 5. Add a CAPTCHA test to the accounts. + + +9. You want to grant the individuals on your network team the ability to fully manipulate Amazon EC2 + instances. Which of the following accomplish this goal? (Choose 2 answers) + 1. Create a new policy allowing EC2:* actions, and name the policy NetworkTeam. + 2. Assign the managed policy, EC2FullAccess, to a group named NetworkTeam, and assign all the team members’ IAM user accounts to that group. + 3. Create a new policy that grants EC2:* actions on all resources, and assign that policy to each individual’s IAM user account on the network team. + 4. Create a NetworkTeam IAM group, and have each team member log in to the AWS Management Console using the user name/password for the group. + + + +10. What is the format of an IAM policy? + 1. XML + 2. Key/value pairs + 3. JSON + 4. Tab-delimited text + + +11. Can you add an IAM role to an IAM group? + 1. Yes + 2. No + 3. Yes, if there are ten members in the group + 4. Yes, if the group allows adding a role + +12. What happens if you delete an IAM role that is associated with a running EC2 + instance? + 1. Any application running on the instance that is using the role will be denied access immediately. + 2. The application continues to use that role until the EC2 server is shut down. + 3. The application will have the access until the session is alive. + 4. The application will continue to have access. + + +15. For implementing security features, which of the following would you choose? + 1. Username/password + 2. MFA + 3. Using multiple S3 buckets + 4. Login using the root user + + +16. You want EC2 instances to give access without any username or password to S3 + buckets. What is the easiest way of doing this? + 1. By using a VPC S3 endpoint + 2. By using a signed URL + 3. By using roles + 4. By sharing the keys between S3 and EC2 + + +17. Using the shared security model, the customer is responsible for which of the + following? (Choose two.) + 1. The security of the data running inside the database hosted in EC2 + 2. Maintaining the physical security of the data center + 3. Making sure the hypervisor is patched correctly + 4. Making sure the operating system is patched correctly + diff --git a/11_aws_demos/storage_demos.md b/11_aws_demos/storage_demos.md new file mode 100644 index 00000000..485d2d9e --- /dev/null +++ b/11_aws_demos/storage_demos.md @@ -0,0 +1,200 @@ +# Storage demos + +### TL;DR + +1. Create SSE-S3 encrypted bucket in the same region of your EC2 instance. +2. Connect to your instance over SSH, use [aws s3api put-object](https://docs.aws.amazon.com/cli/latest/reference/s3api/put-object.html) command to upload an object to your bucket. +3. Repeat the same with SSE-KMS encrypted bucket. +4. Follow [this](https://aws.amazon.com/premiumsupport/knowledge-center/s3-console-access-certain-bucket/) aws blog to allow your instance to access a certain bucket only. +5. Update you IAM policy to access only a certain "folder" in your bucket. +6. Enable versioning on your bucket bucket. +7. Create lifecycle rule to manage non-current versions of your objects: + 1. Move noncurrent versions of objects to Standard-IA storage class + 2. Permanently delete noncurrent versions of objects after 90 days + +--- + +### Create a Bucket + +1. In S3, Choose **Create bucket**. + + The **Create bucket** wizard opens. + +2. In **Bucket name**, enter a DNS-compliant name for your bucket. + + The bucket name must: + + Be unique across all of Amazon S3. + + Be between 3 and 63 characters long. + + Not contain uppercase characters. + + Start with a lowercase letter or number. + +3. In **Region**, choose the AWS Region where you want the bucket to reside. + + Choose the Region where you provisioned your EC2 instance. + +4. Under **Object Ownership**, leave ACLs disabled. + +5. Enable Default encryption with SSE-S3 encryption type. + +6. Choose **Create bucket**. + +You've created a bucket in Amazon S3. + +### Upload an object to S3 bucket from an EC2 instance + +Disclaimer: This is not going to work. Your EC2 instance has to have permissions to operate in S3. + +1. Connect to your instance over SSH. +2. Read the [examples](https://docs.aws.amazon.com/cli/latest/reference/s3api/put-object.html#examples) in AWS code and write a command to upload (put-object) in your S3 bucket. +3. Got `Unable to locate credentials.` or `Access Denied`? follow the next section... + +### Attach IAM role to your EC2 Instance with permissions over S3 + +You must create an IAM role before you can launch an instance with that role or attach it to an instance\. + +**To create an IAM role using the IAM console** + +1. Open the IAM console at [https://console\.aws\.amazon\.com/iam/](https://console.aws.amazon.com/iam/)\. + +1. In the navigation pane, choose **Roles**, **Create role**\. + +1. On the **Trusted entity type** page, choose **AWS service** and the **EC2** use case\. Choose **Next: Permissions**\. + +1. On the **Attach permissions policy** page, search for **AmazonS3FullAccess** AWS managed policy\. + +1. On the **Review** page, enter a name for the role and choose **Create role**\. + + +**To replace an IAM role for an instance** + +1. In EC2 navigation pane, choose **Instances**. + +1. Select the instance, choose **Actions**, **Security**, **Modify IAM role**. + +1. Choose your created IAM role, click **Save**. + + +### Enable versioning on your bucket bucket + +1. Sign in to the AWS Management Console and open the Amazon S3 console at [https://console\.aws\.amazon\.com/s3/](https://console.aws.amazon.com/s3/)\. + +2. In the **Buckets** list, choose the name of the bucket that you want to enable versioning for\. + +3. Choose **Properties**\. + +4. Under **Bucket Versioning**, choose **Edit**\. + +5. Choose **Enable**, and then choose **Save changes**\. + +6. Upload multiple object with the same key, make sure versioning is working. + +### Create lifecycle rule to manage non-current versions + +1. Choose the **Management** tab, and choose **Create lifecycle rule**\. + +1. In **Lifecycle rule name**, enter a name for your rule\. + +1. Choose the scope of the lifecycle rule (in this demo we will apply this lifecycle rule to all objects in the bucket). + +1. Under **Lifecycle rule actions**, choose the actions that you want your lifecycle rule to perform: + + Transition *noncurrent* versions of objects between storage classes + + Permanently delete *noncurrent* versions of objects + +1. Under **Transition non\-current versions of objects between storage classes**: + + 1. In **Storage class transitions**, choose **Standard\-IA**. + + 1. In **Days after object becomes non\-current**, enter 30. + +1. Under **Permanently delete previous versions of objects**, in **Number of days after objects become previous versions**, enter 90 days. + +1. Choose **Create rule**\. + + If the rule does not contain any errors, Amazon S3 enables it, and you can see it on the **Management** tab under **Lifecycle rules**\. + + +### Objects deletion in bucket versioning enabled + +1. In the **Buckets** list, choose a versioning enabled bucket\. +2. Choose **Upload** and upload an object multiple times under the same key, such that it has non-current versions. +3. In the bucket console, choose the **Objects** tab, and delete the object you have just uploaded. +4. After the deletion action, can you see the object in the bucket's objects list? + +We will examine through AWS CLI what happened. + +4. From your local machine, open a command terminal with [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html)) installed. +```shell +aws --version +``` +5. List the versions of you object. Replace `` by you bucket name and `` by the object key: + ```shell + aws s3api list-object-versions --bucket --prefix + ``` + Can you confirm that you object has not been deleted? Inspect `DeleteMarkers`. +6. Delete the _delete mark_ by: + ```shell + aws s3api delete-object --bucket --key --version-id + ``` +7. Can you see the object in the bucket's object list in the AWS Web Console? Can you confirm that the object was "deleted softly"? +8. How can you **permanently** delete an object (and its non-current versions) from a version-enabled bucket? + + +### Create S3 event notification to a Lambda Function + +#### Create a private Docker container repository in ECR, build the YoloV5 Docker container and push it + + +1. Open the Amazon ECR console at [https://console\.aws\.amazon\.com/ecr/repositories](https://console.aws.amazon.com/ecr/repositories)\. + +2. From the navigation bar, choose the Region to create your repository in\. + +3. In the navigation pane, choose **Repositories**\. + +4. On the **Repositories** page, choose **Create repository**\. + +5. For **Repository name**, enter a unique name for your repository\ (for example `nginx-web-app`\)\. + +6. Choose **Create repository**\. + +7. Select the repository that you created and choose **View push commands** to view the steps to push an image to your new repository\. + +8. Following the instructions in **View push commands**, build, tag and push the Docker container specified in `img-object-detection` directory in our shared Git repo. **You must build and push the container from an EC2 instance located in the same region of your container registry**. + +#### Create the Lambda Function + +1. Open the [Functions page](https://console.aws.amazon.com/lambda/home#/functions) of the Lambda console\. + +2. Choose **Create function**\. +3. Choose **Container image** function type. + +4. Under **Basic information**, do the following: + + 1. For **Function name**, enter `img-object-detection-`\, while `` is your name. + + 2. For **Container image URI**, click on **Browse images** and choose the container you've built. + +5. Choose **Create function**\. + + +#### Enabling Lambda notifications + + +1. Open the Amazon S3 console at [https://console\.aws\.amazon\.com/s3/](https://console.aws.amazon.com/s3/)\. + +2. In the **Buckets** list, choose the name of the bucket that you want to enable events for\. + +3. Choose **Properties**\. + +4. Navigate to the **Event Notifications** section and choose **Create event notification**\. + +5. In the **General configuration** section, specify descriptive event name for your event notification\. Optionally, you can also specify a prefix and a suffix to limit the notifications to objects with keys ending in the specified characters\. + + 1. Enter `img-object-detect` for the **Event name**\. + + 2. Filter event notifications by prefix, enter `images/`. + +6. In the **Event types** section **All object create events**. + +7. In the **Destination** section, choose your **Lambda Function** as the event notification destination\. +8. Choose **Save changes**, and Amazon S3 sends a test message to the event notification destination\. +9. Test your work by uploading an image into `images/`. diff --git a/12_docker_network_analysis_ex/README.md b/12_docker_network_analysis_ex/README.md new file mode 100644 index 00000000..4ca64069 --- /dev/null +++ b/12_docker_network_analysis_ex/README.md @@ -0,0 +1,22 @@ + +# Results summary for server and client on the same physical instance + +| Network | Latency avg | Bandwidth avg | +|---|---|---| +| Original | x | x | +| Docker Bridge | x | x | +| Docker Host | x | x | + + +# Results summary for server and client on dirrefent physical instance + +| Network | Latency avg | Bandwidth avg | +|---|---|---| +| Original | x | x | +| Docker Bridge | x | x | +| Docker Host | x | x | + + + +# Conclusions + diff --git a/13_http_load_test/load_test.py b/13_http_load_test/load_test.py new file mode 100644 index 00000000..6daf8944 --- /dev/null +++ b/13_http_load_test/load_test.py @@ -0,0 +1,39 @@ +import time +from loguru import logger +import aiohttp +import asyncio +from datetime import datetime + +# change target url to your app under test +test_url = 'http://localhost:8080/load-test' + +# test duration +test_duration_sec = 10 + +# http requests per second +requests_per_second = 50 + + +async def single_req(session): + req_start_time = datetime.now() + + async with session.get(test_url) as resp: + logger.info(f'request results: code={resp.status} latency={datetime.now() - req_start_time} response={await resp.text()}') + + +async def test(): + async with aiohttp.ClientSession() as session: + start_test_time = time.time() + + requests = [] + while time.time() - start_test_time < test_duration_sec: + requests.append(asyncio.create_task(single_req(session))) + await asyncio.sleep(1 / requests_per_second) + + await asyncio.gather(*requests) + + logger.info('Test Done') + + +if __name__ == '__main__': + asyncio.run(test()) diff --git a/14_dynamodb_lambda_func/publishNewSong.js b/14_dynamodb_lambda_func/publishNewSong.js new file mode 100644 index 00000000..d0fe6860 --- /dev/null +++ b/14_dynamodb_lambda_func/publishNewSong.js @@ -0,0 +1,28 @@ +'use strict'; +var AWS = require("aws-sdk"); +var sns = new AWS.SNS(); + +exports.handler = (event, context, callback) => { + + event.Records.forEach((record) => { + console.log('Stream record: ', JSON.stringify(record, null, 2)); + + if (record.eventName == 'INSERT') { + var who = JSON.stringify(record.dynamodb.NewImage.Artist.S); + var what = JSON.stringify(record.dynamodb.NewImage.SongTitle.S); + var params = { + Subject: 'A new song from ' + who, + Message: 'The song is: ' + what, + TopicArn: '' + }; + sns.publish(params, function(err, data) { + if (err) { + console.error("Unable to send message. Error JSON:", JSON.stringify(err, null, 2)); + } else { + console.log("Results from sending message: ", JSON.stringify(data, null, 2)); + } + }); + } + }); + callback(null, `Successfully processed ${event.Records.length} records.`); +}; \ No newline at end of file diff --git a/15_terraform/getting_started_aws.md b/15_terraform/getting_started_aws.md new file mode 100644 index 00000000..d9ae372e --- /dev/null +++ b/15_terraform/getting_started_aws.md @@ -0,0 +1,83 @@ +# Get Started on AWS + +[Tutorial reference](https://learn.hashicorp.com/collections/terraform/aws-get-started) + +## Deploy single EC2 instance + +The set of files used to describe infrastructure in Terraform is known as a Terraform configuration. You will write your first configuration to define a single AWS EC2 instance. + +1. Edit the configuration file in `simple_ec2/main.tf`. This is a complete configuration that you can deploy with Terraform. + 1. `` is the region in which you want to deploy your infrastructure. + 2. `` is the AMI you want to provision (you can choose Amazon Linux). + 3. `` is the name of you EC2 instance +2. When you create a new configuration — or check out an existing configuration from version control — you need to initialize the directory with `terraform init`. + Initializing a configuration directory downloads and installs the providers defined in the configuration, which in this case is the `aws` provider. +3. You can make sure your configuration is syntactically valid and internally consistent by using the `terraform validate` command. +4. Apply the configuration now with the `terraform apply` command. + +When you applied your configuration, Terraform wrote data into a file called `terraform.tfstate`. Terraform stores the IDs and properties of the resources it manages in this file, so that it can update or destroy those resources going forward. +The Terraform state file is the only way Terraform can track which resources it manages, and often contains sensitive information, so you must store your state file securely and restrict access to only trusted team members who need to manage your infrastructure. + +5. Inspect the current state using `terraform show`. + +## Change the deployed infrastructure + +1. Now update the `ami` of your instance. Change the `aws_instance.app_server` resource under the provider block in `main.tf` by replacing the current AMI ID with a new one. +2. Run `terraform plan` to create an execution plan, which lets you preview the changes that Terraform plans to make to your infrastructure. +3. After changing the configuration, run `terraform apply` again to see how Terraform will apply this change to the existing resources. + +The prefix `-/+` means that Terraform will destroy and recreate the resource, rather than updating it in-place. +The AWS provider knows that it cannot change the AMI of an instance after it has been created, so Terraform will destroy the old instance and create a new one. + +## Use Variables to parametrize your module + +The current configuration includes a number of hard-coded values. Terraform variables allow you to write configuration that is flexible and easier to re-use. + +1. In the same directory as `main.tf`, create a new file called `variables.tf` with a block defining a new `instance_name` variable. + ``` + variable "instance_name" { + description = "Value of the Name tag for the EC2 instance" + type = string + default = "ExampleAppServerInstance" + } + ``` +2. In `main.tf`, update the `aws_instance` resource block to use the new variable. The `instance_name` variable block will default to its default value ("ExampleAppServerInstance") unless you declare a different value. + ```shell + tags = { + - Name = "ExampleAppServerInstance" + + Name = var.instance_name + } + ``` +3. Apply the configuration. + + +## Capture outputs from your provisioned infrastructure + +1. In the same directory as `main.tf`, create a file called `outputs.tf`. +2. Add the configuration below to `outputs.tf` to define outputs for your EC2 instance's ID and IP address. + ```shell + output "instance_id" { + description = "ID of the EC2 instance" + value = aws_instance.app_server.id + } + + output "instance_public_ip" { + description = "Public IP address of the EC2 instance" + value = aws_instance.app_server.public_ip + } + ``` +3. Apply the configurations to see the output values. +4. You can query the output data later on with the `terraform output` command. + +## Destroy infrastructure + +The `terraform destroy` command terminates resources managed by your Terraform project. +This command is the inverse of `terraform apply` in that it terminates all the resources specified in your Terraform state. +It _does not_ destroy resources running elsewhere that are not managed by the current Terraform project. + +1. Destroy the resources you created by `terraform destroy`. + +The `-` prefix indicates that the instance will be destroyed. +Just like with `apply`, Terraform determines the order to destroy your resources. In this case, Terraform identified a single instance with no other dependencies, +so it destroyed the instance. In more complicated cases with multiple resources, Terraform will destroy them in a suitable order to respect dependencies. + diff --git a/15_terraform/install.md b/15_terraform/install.md new file mode 100644 index 00000000..ab428ac5 --- /dev/null +++ b/15_terraform/install.md @@ -0,0 +1,11 @@ +# Install Terraform + +## Linux + +Linux users can install from: https://learn.hashicorp.com/tutorials/terraform/install-cli?in=terraform/aws-get-started + +## Windows +- Got to https://www.terraform.io/downloads. +- Download the Windows Binary (386 or amd64). +- Unzip and put the `terraform.exe` file in `c:\terraform\terraform.exe`. +- Make sure that terraform binary [is available on your PATH](https://stackoverflow.com/questions/1618280/where-can-i-set-path-to-make-exe-on-windows). diff --git a/15_terraform/simple_ec2/main.tf b/15_terraform/simple_ec2/main.tf new file mode 100644 index 00000000..f23c949c --- /dev/null +++ b/15_terraform/simple_ec2/main.tf @@ -0,0 +1,43 @@ +/* + The terraform {} block contains Terraform settings, including the required providers Terraform will use to provision infrastructure. + Terraform installs providers from the Terraform Registry by default. + In this example configuration, the aws provider's source is defined as hashicorp/aws, +*/ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 4.16" + } + } + + required_version = ">= 1.0.0" +} + +/* + The provider block configures the specified provider, in this case aws. + You can use multiple provider blocks in your Terraform configuration to manage resources from different providers. +*/ +provider "aws" { + region = "eu-central-1" +} + + +/* + Use resource blocks to define components of your infrastructure. + A resource might be a physical or virtual component such as an EC2 instance. + A resource block declares a resource of a given type ("aws_instance") with a given local name ("app_server"). + The name is used to refer to this resource from elsewhere in the same Terraform module, but has no significance outside that module's scope. + The resource type and name together serve as an identifier for a given resource and so must be unique within a module. + + For full description of this resource: https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/instance +*/ +resource "aws_instance" "app_server" { + ami = "ami-0d3848b33d10bb64f" + instance_type = "t2.micro" + + tags = { + Name = "alonit-instance2" + tf = "true" + } +} diff --git a/15_terraform/terraform_on_aws.md b/15_terraform/terraform_on_aws.md new file mode 100644 index 00000000..e945d7e2 --- /dev/null +++ b/15_terraform/terraform_on_aws.md @@ -0,0 +1,446 @@ +# Getting Started with Terraform on AWS + +## Deploy single EC2 instance + +The set of files used to describe infrastructure in Terraform is known as a Terraform configuration. You will write your first configuration to define a single AWS EC2 instance. + +1. Edit the configuration file in `workspace/main.tf`. This is a complete configuration that you can deploy with Terraform. + 1. `` is the region in which you want to deploy your infrastructure. + 2. `` is the AMI you want to provision (you can choose Amazon Linux). + 3. `` is the name of you EC2 instance. + 4. `` is the profile account with which your local credentials are associated. +2. When you create a new configuration — or check out an existing configuration from version control — you need to initialize the directory with `terraform init`. + Initializing a configuration directory downloads and installs the providers defined in the configuration, which in this case is the `aws` provider. +3. You can make sure your configuration is syntactically valid and internally consistent by using the `terraform validate` command. +4. Apply the configuration now with the `terraform apply` command. + +When you applied your configuration, Terraform wrote data into a file called `terraform.tfstate`. Terraform stores the IDs and properties of the resources it manages in this file, so that it can update or destroy those resources going forward. +The Terraform state file is the only way Terraform can track which resources it manages, and often contains sensitive information, so you must store your state file securely, outside your version control. + +5. Inspect the current state using `terraform show`. + +## Explore the `terraform.lock.hcl` file + +When you initialize a Terraform configuration for the first time, Terraform will generate a new `.terraform.lock.hcl` file in the current working directory. +You should include the lock file in your version control repository to ensure that Terraform uses the same provider versions across your team and in ephemeral remote execution environments. + +While initializing your workspace, Terraform read the dependency lock file and download the specified versions. If Terraform did not find a lock file, it would download the latest versions of the providers that fulfill the version constraints you defined in the `required_providers` block. + +1. Change the version constrains of `aws` provider from `4.16` to `~> 4.16`. +2. Initialize the workspace with the `-upgrade` flag to upgrade the provider. + +## Change the deployed infrastructure + +1. Now update the `ami` of your instance. Change the `aws_instance.app_server` resource under the provider block in `main.tf` by replacing the current AMI ID with a new one. +2. Run `terraform plan` to create an execution plan, which lets you preview the changes that Terraform plans to make to your infrastructure. +3. After changing the configuration, run `terraform apply` again to see how Terraform will apply this change to the existing resources. + +The prefix `-/+` means that Terraform will destroy and recreate the resource, rather than updating it in-place. +The AWS provider knows that it cannot change the AMI of an instance after it has been created, so Terraform will destroy the old instance and create a new one. + +## Use Variables to parametrize your configuration + +The current configuration includes a number of hard-coded values. Terraform variables allow you to write configuration that is flexible and easier to re-use. + +1. In the same directory as `main.tf`, create a new file called `variables.tf` with a block defining a new `env` variable. + ```text + variable "env" { + description = "Deployment environment" + type = string + default = "dev" + } + ``` +2. In `main.tf`, update the `aws_instance.app_server` resource block to use the new variable. The `env` variable block will default to its default value ("dev") unless you declare a different value. + ```text + tags = { + - Name = "" + + Name = "-${var.env}" + } + ``` + + and + + ```text + - instance_type = "t2.micro" + + instance_type = var.env == "prod" ? "t2.micro" : "t2.nano" + ``` + + The [conditional expression](https://www.terraform.io/language/expressions/conditionals) (among over many more [expressions](https://www.terraform.io/language/expressions)) uses the value of a boolean expression to select one of two values. + +3. Apply the configuration. + +## Extend your EC2 + +Terraform infers dependencies between resources based on the configuration given, +so that resources are created and destroyed in the correct order. Let's create a security group for our EC2: + +1. Add another variable to `variables.tf` + ```text + variable "resource_alias" { + description = "Your name" + type = string + default = "alonit" + } + ``` + change `alonit` to your alias. + +2. Create a security group + ```text + resource "aws_security_group" "sg_web" { + name = "${var.resource_alias}-${var.env}-sg" + + ingress { + from_port = "8080" + to_port = "8080" + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + + tags = { + Env = var.env + Terraform = true + } + } + ``` +3. Add the following attributes to `aws_instance.app_server`: + ```text + vpc_security_group_ids = [aws_security_group.sg_web.id] + key_name = "" + ``` +4. Apply. + +You can use the `depends_on` meta-argument to handle hidden resource dependencies that Terraform cannot automatically infer (e.g. SQS queue should be ready before an EC2 instance can product or data). + +5. Create the following S3 bucket + ```text + resource "aws_s3_bucket" "data_bucket" { + bucket = "" + + tags = { + Name = "${var.resource_alias}-bucket" + Env = var.env + Terraform = true + } + } + ``` +6. We assume that `aws_instance.app_server` put and retrieve data from `data_bucket`, which is an implicit dependency. + Add the following `depends_on` meta-attribute to `aws_instance.app_server`: + ```text + depends_on = [ + aws_s3_bucket.data_bucket + ] + ``` + +## Add Outputs + +Terraform output values allow you to export structured data about your resources. You can use this data to configure other parts of your infrastructure with automation tools, or as a data source for another Terraform workspace. Outputs are also necessary to share data from a child module to your root module. + +1. Add the following output in `outputs.tf` file: +```text +output "instance_public_ip" { + description = "Public IP address of the EC2 instance" + value = aws_instance.app_server.public_ip +} +``` +2. Apply and see the output values in the stdout. + + +## Modules + +Modules help you to package and reuse resource configurations with Terraform. +Modules are containers for multiple resources that are used together, consists of a collection of `.tf` files **kept together in a directory**. + +#### The root module + +Every Terraform configuration has at least one module, known as its root module, which consists of the resources defined in the .tf files in the main working directory. + +#### Use the Terraform Registry + +1. Open the [Terraform Registry page for the VPC module](https://registry.terraform.io/modules/terraform-aws-modules/vpc/aws). Review the module **Inputs** and **Outputs**. + +2. Add the following VPC `module` block: + ```text + module "app_vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "3.14.0" + + name = "${var.resource_alias}-vpc" + cidr = var.vpc_cidr + + azs = ["", "", "..."] + private_subnets = var.vpc_private_subnets + public_subnets = var.vpc_public_subnets + + enable_nat_gateway = false + + tags = { + Name = "${var.resource_alias}-vpc" + Env = var.env + Terraform = true + } + } + ``` + Make sure you specify a list of `azs` (availability zones) according to your region. +3. Add the following output in `outputs.tf`: + ```text + output "vpc_public_subnets" { + description = "IDs of the VPC's public subnets" + value = module.app_vpc.public_subnets + } + ``` +4. Edit `vpc-vars.tf` so you'll have two private, and two public subnets within your VPC. +5. Apply and inspect your VPC in AWS Console. + +Let's migrate the EC2 and the security group into your VPC + +5. Add the following attribute to `aws_security_group.sg_web` + ```text + vpc_id = module.app_vpc.vpc_id + ``` + +6. Add the following attributes to `aws_instance.app_server`: + ```text + subnet_id = module.app_vpc.public_subnets[0] + ``` + +## Data Sources + +Data sources allow Terraform to use information defined outside your configuration files. +Cloud infrastructure, applications, and services emit data, which Terraform can query and act on using **data sources**. +A data sources fetches information from cloud provider APIs, such as disk image IDs, availability zones etc... + +You will use the [`aws_availability_zones`](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) data source (which is part of the AWS provider) to configure your VPC's Availability Zones (AZs), allowing you to deploy this configuration in any AWS region. + +1. List the AZs which can be accessed by an AWS account within the region configured in the provider. +```text +data "aws_availability_zones" "available_azs" { + state = "available" +} +``` +2. Change the following attribute in `app_vpc` module: +```text +- azs = ["", "", ...] ++ data.aws_availability_zones.available_azs.names +``` +3. Apply. Now your `app_vpc` block is region-agnostic! + +The `aws_instance` configuration also uses a hard-coded AMI ID, which is only valid for the specific region. Use an `aws_ami` data source to load the correct AMI ID for the current region. + +4. Add the following `aws_ami` data source to fetch AMIs from AWS API +```text +data "aws_ami" "amazon_linux_ami" { + most_recent = true + owners = ["amazon"] + + filter { + name = "name" + values = ["amzn2-ami-hvm-*-x86_64-gp2"] + } +} +``` +5. Replace the hard-coded AMI ID with the one loaded from the new data source. +```text +- ami = "" ++ ami = data.aws_ami.amazon_linux_ami.id +``` +6. Add the following output in `outputs.tf`: +```text +output "app_server_ami" { + description = "ID of the EC2 instance AMI" + value = data.aws_ami.amazon_linux_ami +} +``` +7. Apply. + +## Manage resource drifts + +The Terraform state file is a record of all resources Terraform manages. You should not make manual changes to resources controlled by Terraform, because the state file will be out of sync, or "drift," from the real infrastructure. +By default, Terraform compares your state file to real infrastructure whenever you invoke `terraform plan` or `terraform apply`. + +If you suspect that your infrastructure configuration changed outside of the Terraform workflow, you can use a `-refresh-only` flag to inspect what the changes to your state file would be. + +1. Run `terraform plan -refresh-only` to determine the drift between your current state file and actual configuration. + You should be synced: `No changes. Your infrastructure still matches the configuration.` +2. In the AWS Console, **manually** create a new security group within the `module.app_vpc` VPC (allow TCP access on port 22 for all IP addresses), attach this security group to `aws_instance.app_server` EC2 instance. +3. Run `terraform plan -refresh-only` again. As shown in the output, Terraform has detected differences between the infrastructure and the current state, and sees that your EC2 instance has a new security group attached to it. +4. Apply these changes by `terraform apply -refresh-only` to make your state file match your real infrastructure, **but not your Terraform configuration!!!**. + +A refresh-only operation does not attempt to modify your infrastructure to match your Terraform configuration -- it only gives you the option to review and track the drift in your state file. +If you ran `terraform plan` or `terraform apply` without the `-refresh-only` flag now, Terraform would attempt to revert your manual changes. + +Now, you will update your configuration to associate your EC2 instance with both security groups. + +1. First, add the resource definition to your configuration by + ```text + resource "aws_security_group" "sg_ssh" { + name = "" + description = "" + vpc_id = module.app_vpc.vpc_id + + ingress { + from_port = "22" + to_port = "22" + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + } + ``` + Make sure `` is the same as your manually created security group. +2. Add the security group ID to your instance resource. + ```text + - vpc_security_group_ids = [aws_security_group.sg_ssh.id] + + vpc_security_group_ids = [aws_security_group.sg_ssh.id, aws_security_group.sg_web.id] + ``` +3. Run `terraform import` to associate your resource definition with the security group created in the AWS Console: + ```text + terraform import aws_security_group.sg_ssh + ``` + Where `` is your security group id. + + +## Secrets and sensitive data + +Let's create MySQL RDS instance: + +1. Add the following resources to your configuration file: +```text +resource "aws_db_subnet_group" "private_db_subnet" { + subnet_ids = module.app_vpc.public_subnets +} + +resource "aws_db_instance" "database" { + allocated_storage = 5 + db_name = "${var.resource_alias}mysql" + engine = "mysql" + instance_class = "db.t2.micro" + username = "admin" + password = "password" + + db_subnet_group_name = aws_db_subnet_group.private_db_subnet.name + skip_final_snapshot = true +} +``` +2. The database username and password are hard-coded. Refactor this configuration to remove these values: + 1. Explore the variables `db-vars.tf` (comment them in!). Notice that you've declared the variables as `sensitive`. + 2. Now update main.tf to reference these variables: + ```text + - username = "admin" + - password = "password" + + username = var.db_username + + password = var.db_password + ``` + +If you were to run terraform apply now, Terraform would prompt you for values for these new variables since you haven't assigned defaults to them. However, entering values manually is time consuming and error prone. Next, you will use two different methods to set the sensitive variable values, and learn about security considerations of each method. + +**Set values with a `.tfvars` file** + +Terraform supports setting variable values with variable definition (.tfvars) files. You can use multiple variable definition files, and many practitioners use a separate file to set sensitive or secret values. + +3. Create a new file called `secret.tfvars` to assign values to the new variables. + ```text + db_username = "admin" + db_password = "password" + ``` +4. Apply by `terraform apply -var-file="secret.tfvars"` + + +
+Read more about `.tfvars` files + +To set lots of variables, it is more convenient to specify their values in a variable definitions file (with a filename ending in .tfvars) and then specify that file on the command line with `-var-file`: +```text +terraform apply -var-file="testing.tfvars" +``` + +A variable definitions file uses the same basic syntax as Terraform language files, but consists only of variable name assignments: + +```text +image_id = "ami-abc123" +availability_zone_names = [ + "us-east-1a", + "us-west-1c", +] +``` +
+ +**Set values with variables** + +Set the database administrator username and password using environment variables: + +3. Define the following env vars: + ```text + # Linux (or Git Bash) + export TF_VAR_db_username=admin TF_VAR_db_password=password + + # Powershell + $Env:TF_VAR_db_username = "admin"; $Env:TF_VAR_db_password = "password" + + # cmd + set "TF_VAR_db_username=admin" & set "TF_VAR_db_password=password" + ``` +4. Apply regularly by `terraform apply`. + +#### Reference sensitive variables + +Finally, you can use sensitive variables as you would any other variable. +If those variables are sensitive, Terraform will redact these values in command output and log files, and raise an error when it detects that they will be exposed in other ways. + +5. Add the following output in `outputs.tf`. +6. Apply again the changes. Terraform will raise an error, since the output is derived from sensitive variables. +7. Flag the database connection string output as `sensitive`, causing Terraform to hide it. +8. Apply. + +## Protect your infrastructure from being destroyed + +To prevent destroy operations for specific resources, +you can add the `prevent_destroy` attribute to your resource definition. +This [lifecycle](https://www.terraform.io/language/meta-arguments/lifecycle) option prevents Terraform from accidentally removing critical resources. + +1. Add the `lifecycle` meta-argument by: + ``` + lifecycle { + prevent_destroy = true + } + ``` +2. Apply the change, then apply a destroying change and test the rule. + +## Backend configurations + +A backend defines where Terraform stores its [state](https://www.terraform.io/language/state) data files. +This lets multiple people access the state data and work together on that collection of infrastructure resources. +When changing backends, Terraform will give you the option to migrate your state to the new backend. This lets you adopt backends without losing any existing state. +Always backup your state! + +1. To configure a backend, add a nested `backend` block within the top-level `terraform` block. The following example configures the `s3_backend` backend: + ```text + backend "s3" { + bucket = "" + key = "tfstate.json" + region = "" + # optional: dynamodb_table = "" + } + ``` +2. Apply the changes and make sure the state is stored in S3. + +This backend also supports state locking and consistency checking via Dynamo DB, which can be enabled by setting the `dynamodb_table` field to an existing DynamoDB table name. +The table must have a partition key named `LockID` with type of `String`. + +## Destroy infrastructure + +The `terraform destroy` command terminates resources managed by your Terraform project. +This command is the inverse of `terraform apply` in that it terminates all the resources specified in your Terraform state. +It _does not_ destroy resources running elsewhere that are not managed by the current Terraform project. + +1. Destroy the resources you created by `terraform destroy`. + +The `-` prefix indicates that the instance will be destroyed. +Just like with `apply`, Terraform determines the order to destroy your resources. In this case, Terraform identified a single instance with no other dependencies, +so it destroyed the instance. In more complicated cases with multiple resources, Terraform will destroy them in a suitable order to respect dependencies. + +You can destroy specific resource by `terraform destroy -target RESOURCE_TYPE.NAME`. + + +[comment]: <> (# Multi-regional CloudWatch Alarm) + +[comment]: <> (## build your own module) + diff --git a/15_terraform/workspace/db-vars.tf b/15_terraform/workspace/db-vars.tf new file mode 100644 index 00000000..3be8c66a --- /dev/null +++ b/15_terraform/workspace/db-vars.tf @@ -0,0 +1,11 @@ +#variable "db_username" { +# description = "Database administrator username" +# type = string +# sensitive = true +#} +# +#variable "db_password" { +# description = "Database administrator password" +# type = string +# sensitive = true +#} \ No newline at end of file diff --git a/15_terraform/workspace/main.tf b/15_terraform/workspace/main.tf new file mode 100644 index 00000000..1e0998dc --- /dev/null +++ b/15_terraform/workspace/main.tf @@ -0,0 +1,44 @@ +/* + The terraform {} block contains Terraform settings, including the required providers Terraform will use to provision infrastructure. + Terraform installs providers from the Terraform Registry by default. + In this example configuration, the aws provider's source is defined as hashicorp/aws, +*/ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "4.16" + } + } + + required_version = ">= 1.0.0" +} + + +/* + The provider block configures the specified provider, in this case aws. + You can use multiple provider blocks in your Terraform configuration to manage resources from different providers. +*/ +provider "aws" { + region = "" +} + + +/* + Use resource blocks to define components of your infrastructure. + A resource might be a physical or virtual component such as an EC2 instance. + A resource block declares a resource of a given type ("aws_instance") with a given local name ("app_server"). + The name is used to refer to this resource from elsewhere in the same Terraform module, but has no significance outside that module's scope. + The resource type and name together serve as an identifier for a given resource and so must be unique within a module. + + For full description of this resource: https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/instance +*/ +resource "aws_instance" "app_server" { + ami = "" + instance_type = "t2.micro" + + tags = { + Name = "" + Terraform = "true" + } +} diff --git a/15_terraform/workspace/vpc-vars.tf b/15_terraform/workspace/vpc-vars.tf new file mode 100644 index 00000000..be804330 --- /dev/null +++ b/15_terraform/workspace/vpc-vars.tf @@ -0,0 +1,17 @@ +variable "vpc_cidr" { + description = "CIDR block for VPC" + type = string + default = "10.0.0.0/16" +} + +variable "vpc_private_subnets" { + description = "Private subnets for VPC" + type = list(string) + default = ["", ""] +} + +variable "vpc_public_subnets" { + description = "Public subnets for VPC" + type = list(string) + default = [""] +} diff --git a/16_ansible/ansible_tutorial.md b/16_ansible/ansible_tutorial.md new file mode 100644 index 00000000..9dbe8381 --- /dev/null +++ b/16_ansible/ansible_tutorial.md @@ -0,0 +1,277 @@ +# Ansible + +# Build simple inventory and use ad-hoc commands + +Ansible works against multiple managed nodes or “hosts” in your infrastructure at the same time, using a list or group of lists known as **inventory**. +The default location for inventory is a file called `/etc/ansible/hosts` (use `-i` to specify another location). + +1. For this demo, create **two** Amazon Linux EC2 instances in our shared AWS account. Make sure you have access to the `.pem` private key. +2. Build a simple `hosts` inventory file as follows: +```ini + + +``` + +An Ansible ad hoc command uses the `ansible` command-line tool to automate a single task, in not-reusable manner. We will use the `ping` module to ping our hosts. + +3. Run the following command, investigate the returned error and use the `--user` option to fix it. +```shell +ansible -i /path/to/inventory-file --private-key /path/to/private-key-pem-file all -m ping +``` + +4. Let's say the hosts run webserver, we can arrange hosts under groups, and automate tasks for specific group: +```ini +[webserver] +web1 ansible_host= ansible_user= +web2 ansible_host= ansible_user= +``` + +There are two more default groups: `all` and `ungrouped`. The all group contains every host. The ungrouped group contains all hosts that don’t have another group aside from `all`. + +5. Let's check the uptime of all server in `webserver` group: +```shell +ansible -i /path/to/inventory-file --private-key /path/to/private-key-pem-file webserver -m command -a "uptime" +``` + +## Working with Playbooks + +If you need to execute a task with Ansible more than once, write a **playbook** and put it under source control. +Ansible Playbooks offer a repeatable, re-usable and simple configuration management. +Playbooks are expressed in YAML format, composed of one or more ‘plays’ in an **ordered** list. +A playbook 'play' runs one or more tasks. Each task calls an Ansible module from top to bottom. + +In this demo, we will be practicing some security hardening for the webserver hosts. + +To demonstrate the power of Ansible, we first want to create a task that verify the installation of `httpd` (and install it if needed) + +1. Create the following `site.yaml` file, representing an Ansible playbook: +```yaml +--- +- name: Harden web servers + hosts: + tasks: + - name: Ensure httpd is at the latest version + ansible.builtin.yum: + name: httpd-2.4* + state: latest +``` +In this playbook we execute one task using the built-in [`yum` module](https://docs.ansible.com/ansible/2.9/modules/yum_module.html#yum-module). Ansible ships with hundreds of [built-in modules](https://docs.ansible.com/ansible/2.9/modules/list_of_all_modules.html) available for usage. + +2. Apply your playbook using the following `ansible-playbook` command. +```shell +ansible-playbook -i /path/to/inventory-file --private-key /path/to/private-key-pem-file site.yaml +``` + +As the tasks in this playbook require root-privileges, we add the `become: true` to enable execute tasks as a different Linux user: +```yaml +--- +- name: Harden web servers + hosts: webserver + become: yes + become_method: sudo + tasks: + - name: Ensure httpd is at the latest version + become_user: root + ansible.builtin.yum: + name: httpd-2.4* + state: latest +``` +Run the playbook again and make sure the task has been completed successfully. + +We now want to harden the SSH configurations of the hosts. + +3. Add the following task to your notebook +```yaml + - name: Write the sshd config file + ansible.builtin.template: + src: templates/sshd_config.j2 + dest: /etc/ssh/sshd_config +``` + +As well as the following var file import statement in the top-level area of the playbook: + +```yaml + vars_files: + - vars/amazon-linux.yaml +``` + +Ansible uses [Jinja2](https://jinja.palletsprojects.com/en/3.1.x/) templating to enable dynamic expressions and access to [variables](https://docs.ansible.com/ansible/latest/user_guide/playbooks_variables.html#playbooks-variables). +The `templates/sshd_config.j2` and its corresponding variable file `vars/amazon-linux.yaml` can be found in our shared repo. + +5. Run the playbook. Connect to one of the hosts and make sure the `sshd` configuration file has been updated. +6. It's required to restart the `sshd` service after changing the configuration, let's add a `handlers:` entry with a handler that restarts the daemon after a successful configuration change: +```yaml +--- +- name: Harden web servers + hosts: webserver + become: yes + become_method: sudo + vars_files: + - vars/amazon-linux.yaml + + tasks: + - name: Ensure httpd is at the latest version + ansible.builtin.yum: + name: httpd-2.4* + state: latest + + - name: Write the sshd config file + ansible.builtin.template: + src: templates/sshd_config.j2 + dest: /etc/ssh/sshd_config + notify: restart sshd + + handlers: + - name: restart sshd + become_user: root + ansible.builtin.service: + name: sshd + state: restarted + +``` +Note the added `notify:` entry in the **Write the sshd config file** task. + +7. Run the playbook and check the status of the `sshd` service by running `sudo service sshd status` in one of the hosts machine. +8. Now change the `sshd_port:` in `vars/amazon-linux.yaml` to a number other than `22` and run the playbook. +9. You will need update your `hosts` inventory file to use the new port, as follows: +```ini +web1 ansible_host=... ansible_user=... ansible_port=23 +``` + +Make sure Ansible is able to communicate with your hosts after the change. + +10. Let's create another task to make sure the latest version of `auditd` daemon is installed. +```yaml + - name: Ensure auditd is at the latest version + become_user: root + ansible.builtin.yum: + name: audit + state: latest +``` + +11. We now want to add some rules to the `auditd` daemon. For that we will use the `template` built-in module. Add the following task to your playbook: +```yaml + - name: Write the audits config file + ansible.builtin.template: + src: templates/auditd_rules.j2 + dest: /etc/audit/rules.d/audit.rules +``` +Again, the `templates/auditd_rules.j2` file can be found in the shared repo. This file contains two rules: + +- Audit all activities in `/root` directory +- Audit `sudo` usage + +12. Add the following handler under `handlers` entry, as well as `notify: restart auditd` entry in **Write the audits config file** task. +```yaml + - name: restart auditd + become_user: root + ansible.builtin.command: | + service auditd restart +``` +13. Run the playbook and check the status of the auditd service: `sudo service auditd status`. +14. Test that the auditing rule were applied: + 1. Execute some command as root: `sudo su -` + 2. Search for appropriate audit logs: `sudo ausearch -k using_sudo`. + +### **Challenge:** add another task to your playbook that removes unused yum repositories and enables GPG key-checking + +1. All tasks are configures in `yum_hardening.yaml` +2. Add `vars/main.yaml` with the following variables: +```yaml +# Set to false to disable installing and configuring yum. +os_yum_enabled: true + +# List of yum repository files under /etc/yum.repos.d/ which should not be altered. +os_yum_repo_file_whitelist: [] +``` +Include this var file in `site.yaml` +3. Include the yum tasks YAML file in `site.yaml` by adding the following entry under `tasks:`: +```yaml + - include_tasks: yum_hardening.yaml +``` +4. Run the playbook. + +### Validating playbook tasks: `check` mode and `diff` mode + +Ansible provides two modes of execution that validate tasks: check mode and diff mode. +They are useful when you are creating or editing a playbook or role and you want to know what it will do. + +In check mode, Ansible runs without making any changes on remote systems, and report the changes that would have made. +In diff mode, Ansible provides before-and-after comparisons. + +Simply add the `--check` or `--diff` options (both or separated) to the `ansible-playbook` command: + +```shell +ansible-playbook -i ./inventory/hosts site.yaml --check --diff +``` + +## Ansible Facts + +You can retrieve or discover certain variables containing information about your remote systems, which are called **facts**. +For example, with facts variables you can use the IP address of one system as a configuration value on another system. Or you can perform tasks based on the specific host OS. + +To see all available facts, add this task to a play: + +```yaml +- name: Print all available facts + ansible.builtin.debug: + var: ansible_facts +``` + +Or alternatively, run the `-m setup` ad-hoc command: +```shell +ansible -i ./inventory/hosts webserver -m setup +``` + +As the `ansible.builtin.yum` module fits only RedHat family systems (e.g. Amazon Linux), we would like to add a condition for the **** and ** ** tasks, using the `ansible_pkg_mgr` facts variable: + +```shell + - name: Ensure httpd is at the latest version + become_user: root + ansible.builtin.yum: + name: httpd-2.4* + state: latest + when: ansible_facts['pkg_mgr'] == 'yum' +``` + + +## Organize the playbook using Roles + +[Roles](https://docs.ansible.com/ansible/latest/user_guide/playbooks_reuse_roles.html) let you automatically load related vars, templates, tasks, handlers, and other Ansible artifacts based on a known file structure. + +1. Redesign your YAML files according to the following files structure: +```text +# in the root directory of your Ansible playbook +roles/ + sshd/ + tasks/ + main.yaml + handlers/ + main.yaml + templates/ + sshd_config.j2 + vars/ + main.yaml + auditd/ + tasks/ + main.yaml + handlers/ + main.yaml + templates/ + auditd_rules.j2 + vars/ + main.yaml +``` + +By default, Ansible will look in each directory within a role for a `main.yaml` file for relevant content. + +2. In `site.yaml` add the following entry in the top-play level (roles can be included in the task level also): +```yaml +roles: + - sshd + - auditd +``` +Clean `site.yaml` from tasks and handlers according the content you copied to `sshd` and `auditd` roles. + +Ansible will execute roles first, then other tasks in the play. + diff --git a/16_ansible/site.tmp.yaml b/16_ansible/site.tmp.yaml new file mode 100644 index 00000000..3c5ab592 --- /dev/null +++ b/16_ansible/site.tmp.yaml @@ -0,0 +1,15 @@ +--- +- name: Harden web servers + hosts: webserver + become: yes + become_method: sudo + roles: + - sshd + - auditd + tasks: + - name: Ensure httpd is at the latest version + become_user: root + ansible.builtin.yum: + name: httpd-2.4* + state: latest + when: ansible_facts['pkg_mgr'] == 'yum' diff --git a/16_ansible/vars/main.tmp.yaml b/16_ansible/vars/main.tmp.yaml new file mode 100644 index 00000000..b6e062d5 --- /dev/null +++ b/16_ansible/vars/main.tmp.yaml @@ -0,0 +1,5 @@ +# Set to false to disable installing and configuring yum. +os_yum_enabled: true + +# List of yum repository files under /etc/yum.repos.d/ which should not be altered. +os_yum_repo_file_whitelist: [] diff --git a/16_ansible/yum_hardening.yaml b/16_ansible/yum_hardening.yaml new file mode 100644 index 00000000..14c09aa0 --- /dev/null +++ b/16_ansible/yum_hardening.yaml @@ -0,0 +1,32 @@ +--- +# The following tasks set removes unused yum repositories and enables GPG key-checking + +- name: Remove unused repositories + file: + name: '/etc/yum.repos.d/{{ item }}.repo' + state: 'absent' + loop: + - 'CentOS-Debuginfo' + - 'CentOS-Media' + - 'CentOS-Vault' + when: os_yum_enabled | bool + +- name: Get yum repository files + find: + paths: '/etc/yum.repos.d' + patterns: '*.repo' + register: yum_repos + when: os_yum_enabled | bool + +# for the 'default([])' see here: +# https://github.com/dev-sec/ansible-os-hardening/issues/99 and +# https://stackoverflow.com/questions/37067827/ansible-deprecation-warning-for-undefined-variable-despite-when-clause +- name: Activate gpg-check for yum repository files + replace: + path: '{{ item }}' + regexp: '^\s*gpgcheck.*' + replace: 'gpgcheck=1' + mode: '0644' + with_items: + - "{{ yum_repos.files | default([]) | map(attribute='path') | difference(os_yum_repo_file_whitelist | map('regex_replace', '^', '/etc/yum.repos.d/') | list) }}" + when: os_yum_enabled | bool diff --git a/17_jenkins/JenkinsAgent.Dockerfile b/17_jenkins/JenkinsAgent.Dockerfile new file mode 100644 index 00000000..6be8b76d --- /dev/null +++ b/17_jenkins/JenkinsAgent.Dockerfile @@ -0,0 +1,17 @@ +FROM amazonlinux:2 as installer +RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" +RUN yum update -y \ + && yum install -y unzip \ + && unzip awscliv2.zip \ + && ./aws/install --bin-dir /aws-cli-bin/ + +RUN mkdir /snyk && cd /snyk \ + && curl https://static.snyk.io/cli/v1.666.0/snyk-linux -o snyk \ + && chmod +x ./snyk + +FROM jenkins/agent +COPY --from=docker /usr/local/bin/docker /usr/local/bin/ +COPY --from=installer /usr/local/aws-cli/ /usr/local/aws-cli/ +COPY --from=installer /aws-cli-bin/ /usr/local/bin/ +COPY --from=installer /snyk/ /usr/local/bin/ + diff --git a/17_jenkins/botDeploy.yaml b/17_jenkins/botDeploy.yaml new file mode 100644 index 00000000..66c0a0a9 --- /dev/null +++ b/17_jenkins/botDeploy.yaml @@ -0,0 +1,20 @@ +--- +- hosts: bot + vars: + ansible_python_interpreter: /usr/bin/python3 + tasks: + - name: Install docker python package + ansible.builtin.pip: + name: docker + executable: pip3 + + - name: Docker login + ansible.builtin.shell: | + aws ecr get-login-password --region {{ registry_region }} | docker login --username AWS --password-stdin {{ registry_url }} + + - name: Deploy the bot Docker container + community.general.docker_container: + image: "{{ bot_image }}" + name: bot + state: started + restart_policy: always diff --git a/17_jenkins/jenkins_tutorial.md b/17_jenkins/jenkins_tutorial.md new file mode 100644 index 00000000..0d2d7d27 --- /dev/null +++ b/17_jenkins/jenkins_tutorial.md @@ -0,0 +1,413 @@ +# Jenkins + +## Install Jenkins on EC2 + +Jenkins is typically run as a standalone application in its own process with the built-in Java servlet container/application. + +1. Create a ***.small, Amazon Linux** EC2 instance. +2. Connect to your instance, execute `sudo yum update && sudo amazon-linux-extras install epel -y` +3. Download and install Jenkins as described [here](https://www.jenkins.io/doc/tutorials/tutorial-for-installing-jenkins-on-AWS/#downloading-and-installing-jenkins) (no need to install the EC2 plugin). +4. On Jenkins' machine, [install Docker engine](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/create-container-image.html#create-container-image-prerequisites). You may want to add jenkins linux user the docker group, so Jenkins could run docker commands: + ```shell + sudo usermod -a -G docker jenkins + ``` +5. Install Git. +6. Create an Elastic IP and associate it to your Jenkins instance. +7. Open port `8080` and visit your Jenkins server via `http://:8080` and complete the setup steps. +8. In the **Dashboard** page, choose **Manage Jenkins**, then **Manage Plugins**. In the **Available** tab, search and install **Blue Ocean** and **Docker Pipeline** plugins. Then restart jenkins by `http://:8080/safeRestart` + +## CI integration with GitHub + +1. In your PolyBot repository, in branch `main`, create a `Jenkinsfile` in the root directory as the [following template](../Jenkinsfile). Commit and push your changes. +2. To set up a webhook from GitHub to the Jenkins server, on your GitHub repository page, go to **Settings**. From there, click **Webhooks**, then **Add webhook**. +3. In the **Payload URL** field, type `http://:8080/github-webhook/`. In the **Content type** select: `application/json` and leave the **Secret** field empty. +4. Choose the following events to be sent in the webhook: + 1. Pushes + 2. Pull requests + +5. From the main Jenkins dashboard page, choose **New Item**. +6. Enter `BotBuild` as the project name, and choose **Pipeline**. +7. Check **GitHub project** and enter the URL of your GitHub repo. +8. Under **Build Triggers** check **GitHub hook trigger for GITScm polling**. +9. Under **Pipeline** choose **Pipeline script from SCM**. +10. Choose **Git** as your SCM, and enter the repo URL. +11. If you don't have yet credentials to GitHub, choose **Add** and create **Jenkins** credentials. + 1. **Kind** must be **Username and password** + 2. Choose informative **Username** (as **github** or something similar) + 3. The **Password** should be a GitHub Personal Access Token with the following scope: + ```text + repo,read:user,user:email,write:repo_hook + ``` + Click [here](https://github.com/settings/tokens/new?scopes=repo,read:user,user:email,write:repo_hook) to create a token with this scope. + 4. Enter `github` as the credentials **ID**. + 5. Click **Add**. +12. Under **Branches to build** enter `main` as we want this pipeline to be triggered upon changes in branch main. +13. Under **Script Path** write the path to your `Jenkinsfile` defining this pipeline. +14. **Save** the pipeline. +15. Test the integration by add a [`sh` step](https://www.jenkins.io/doc/pipeline/tour/running-multiple-steps/#linux-bsd-and-mac-os) to the `Jenkinsfile`, commit & push and see the triggered activity in Jenkins Blue Ocean. + + +## Build the Bot app + +1. If you don't have yet, create a private registry in [ECR](https://console.aws.amazon.com/ecr/repositories) for the Bot app. + +2. In the registry page, use the **View push commands** to implement a build step in your `Jenkinsfile`. The step may be seen like: + +```text +stage('Build Bot app') { + steps { + sh ''' + aws ecr get-login-pass..... | docker login --username AWS .... + docker build ... + docker tag ... + docker push ... + ''' + } +} +``` + +You can use the timestamp, or the `BUILD_NUMBER` or `BUILD_TAG` environment variables to tag your Docker images, but don't tag the images as `latest`. + +3. Give your EC2 instance an appropriate role to push an image to ECR. + +4. Use the `environment` directive to store global variable (as AWS region and ECR registry URL) and make your pipeline a bit more elegant. + +## Clean the build artifacts from Jenkins server + +1. As for now, we build the Docker images in the system of the Jenkins server itself, which is a very bad idea (why?). +2. Use the `post{ always{} }` directive to cleanup the built Docker images from the disk. + +## Deploy the Bot app + +We would like to trigger a new pipeline (`BotDeploy`) after every successful running of the `botBuild` pipeline. + +1. Add the following stage to the `BotBuild` pipeline, and familiarize yourself with the [Pipeline: Build Step](https://www.jenkins.io/doc/pipeline/steps/pipeline-build-step/): +```text +stage('Trigger Deploy') { + steps { + build job: '', wait: false, parameters: [ + string(name: '', value: "") + ] + } +} +``` +Where: +- `` is the name of a Bot deployment job you will create soon (should be `BotDeploy`). +- `` is a name of a parameter **to your choice** to be used by the `BotDeploy` pipeline as an env var containing the full Docker image URL you built in the `BotBuild` pipeline. +- `` is a full URL to your built Docker image. You can use env vars like: `value: "${IMAGE_NAME}:${IMAGE_TAG}"` or something similar. + +2. In the PolyBot repo, create another `Jenkinsfile` called `botDeploy.Jenkinsfile`. In this pipeline we will define the deployment steps for the Bot app. For now, you can build some empty pipeline with a dummy step. +3. From the main Jenkins dashboard page, create another Jenkins **Pipeline** names `BotDeploy`. +4. Check **GitHub project** and enter the URL of your GitHub repo. +5. Check **This project is parameterized**, and choose **Multi-line String Parameter** with the following parameters: + 1. The **Name** should be the same variable name as you defined in ``. + 2. Leave the rest empty. +6. **Don't trigger** this build as a result of GitHub hook event. +7. Under **Pipeline** define the same configurations as the `BotBuild` build. But the **Script Path** for the Jenkinsfile defining this pipeline should be `botDeploy.Jenkinsfile`. +11. Make sure the `BotBuild` pipeline triggers the `BotDeploy` as expected. + +### Implement the deployment steps in botDeploy.Jenkinsfile + +We will deploy the Bot app to your EC2 instances using Ansible. +In the `botDeploy.Jenkinsfile` create 3 stages, as follows. + +1. A stage to install Ansible and the [`community.general`](https://galaxy.ansible.com/community/general?extIdCarryOver=true) plugins set: +```text +stage("Install Ansible") { + steps { + sh 'python3 -m pip install ansible' + sh '/var/lib/jenkins/.local/bin/ansible-galaxy collection install community.general' + } +} +``` + +2. The `Generate Ansible Inventory` stage: +```text +stage("Generate Ansible Inventory") { + environment { + BOT_EC2_APP_TAG = "" + BOT_EC2_REGION = "" + } + steps { + sh 'aws ec2 describe-instances --region $BOT_EC2_REGION --filters "Name=tag:App,Values=$BOT_EC2_APP_TAG" --query "Reservations[].Instances[]" > hosts.json' + sh 'python3 prepare_ansible_inv.py' + sh ''' + echo "Inventory generated" + cat hosts + ''' + } +} +``` + +Take a closer look on the steps. +We use the `aws ec2 describe-instances` command to dynamically retrieve the details (IP, etc...) of the EC2 instances running the Bot app. +We use a Python script to generate a `hosts` file in the format of [Ansible Inventory](https://docs.ansible.com/ansible/latest/user_guide/intro_inventory.html). The Python script can be found in [`17_jenkins/prepare_ansible_inv.py`](prepare_ansible_inv.py) for your convenience. + +2. The `Ansible Bot Deploy`: +```text +stage('Ansible Bot Deploy') { + environment { + ANSIBLE_HOST_KEY_CHECKING = 'False' + REGISTRY_URL = '' + REGISTRY_REGION = '' + } + + steps { + withCredentials([sshUserPrivateKey(credentialsId: '', usernameVariable: 'ssh_user', keyFileVariable: 'privatekey')]) { + sh ''' + /var/lib/jenkins/.local/bin/ansible-playbook botDeploy.yaml --extra-vars "registry_region=$REGISTRY_REGION registry_url=$REGISTRY_URL bot_image=$BOT_IMAGE" --user=${ssh_user} -i hosts --private-key ${privatekey} + ''' + } + } +} +``` +While `` is an ID of a credentials **you should create** in Jenkins (**Manage Jenkins** -> **Manage Credentials**...). Choose a private key compatible to the Bot EC2 instances. The playbook YAML file can be found under [`17_jenkins/botDeploy.yaml`](botDeploy.yaml). + +4. Reboot your Jenkins server. +5. Test your build and deploy pipeline. + + +## Add more pipeline features + +As out Jenkins Pipelines has become more and more expressive and complex, we would like to and some features to improve our experience. + +1. Let's add the following `options` directive in the top-level of `Jenkinsfile` pipeline ('BotBuild`): +```text +options { + buildDiscarder(logRotator(daysToKeepStr: '30')) + disableConcurrentBuilds() + timestamps() +} +``` + +The `buildDiscarder` option persist data only for the specified recent number of days. +`disableConcurrentBuilds` can help when multiple developers trigger the same pipeline simultaneous, which can cause unwanted outcome as accesses to shared resources. +`timestamps` will add the machine timestamp for executed commands. + +2. To limit the time allocated for Docker to build images, in the same `Jenkinsfile`, add the following `timeout` option in the `Build` stage only: +```text +options { + timeout(time: 10, unit: 'MINUTES') +} +``` + + +## Use Docker as Jenkins agent + +Using Docker to for build and test pipelines you can benefit from: + +- Isolate build activity from each other and from Jenkins server +- Build for different environments +- Using ephemeral containers for better resource utilization + +Let's create a Docker container that will be used as a Build agent for the `BotBuild` pipeline. +Take a look on the Dockerfile under [17_jenkins/JenkinsAgent.Dockerfile](JenkinsAgent.Dockerfile). + +This dockerfile uses [Multi-stage builds](https://docs.docker.com/build/building/multi-stage/). Familiarize yourself with this technique. +The dockerfile starts with [`amazonlinux:2`](https://hub.docker.com/_/amazonlinux) as an `installer` image to which we will install `aws` cli and [`snyk`](https://docs.snyk.io/snyk-cli) (for later usage...). After this image is built, we will copy the relevant artifacts to the other main image: [jenkins/agent](https://hub.docker.com/r/jenkins/agent/). +The `jenkins/agent` is a base image suitable for running Jenkins activities. +In addition, we copy the `docker` **client only** (as we want to build images as part of our pipeline) from [`docekr`](https://hub.docker.com/_/docker), which is a Docker image containing `docker`. Feel confused? [read more...](https://jpetazzo.github.io/2015/09/03/do-not-use-docker-in-docker-for-ci/). + +1. Build the image from a machine with access to ECR. +2. Push your image to a dedicated container registry in ECR. +3. In `Jenkinsifle`, replace `agent any` by: +```text +agent { + docker { + image '' + args '--user root -v /var/run/docker.sock:/var/run/docker.sock' + } +} +``` + +The `-v` mount the socket file that the docker client is using to talk with the docker daemon. In this case the docker client within the container will talk with the docker daemon on Jenkins machine. +The `--user root` runs the container as `root` user, which is necessary to access `/var/run/docker.sock`. + +4. Test your pipeline on the Docker-based agent. + +## Run agents on a separate EC2 instance node + +Jenkins [EC2-plugin](https://plugins.jenkins.io/ec2/) allows Jenkins to start agents on EC2 on demand, and kill them as they get unused. +It'll start instances using the EC2 API and automatically connect them as Jenkins agents. When the load goes down, excess EC2 instances will be terminated. + +1. From Jenkins Plugins page, install `Amazon EC2` plugin. +2. Once you've installed the plugin, navigate to the main **Manage Jenkins** > **Nodes and Clouds** page, and choose **Configure Clouds**. +3. Add an **Amazon EC2** cloud configured as follows: + 1. Give it a **Name** as your choice. + 2. Keep **Amazon EC2 Credentials** `none`. Instead, you should check **Use EC2 instance profile to obtain credentials** give appropriate permissions to Jenkins` server Role (full permissions JSON can be found in the plugin's page). + 3. In **EC2 Key Pair's Private Key** choose your existed SSH key-pair credentials, or create one of you don't have yet. + 4. Under **AMIs** click **Add** and configure the AMI as the below steps. + 5. In **AMI ID**, search the ID of an image named `jenkins-nodes` in the region you are operating from. + 6. For **Instance Type** choose an appropriate `*.micro` type. + 7. Choose an existed security group id for **Security group names**. + 8. Since the above AMI is based on Amazon Linux, **Remote user** is `ec2-user` and **AMI Type** in `unix`. + 9. Under **Labels** choose a label which will be used in your Jenkinsfile. + 10. Set the **Idle termination time** to `10` minutes. + 11. In the **Advanced** configurations, under **Subnet IDs for VPC** choose an existed subnet ID within your VPC. + 12. Set **Instance Cap** to `3` to restrict Jenkins from provisioning too many instances. + 13. Under **Host Key Verification Strategy** choose `off` since we trust Jenkins agents by default. + 14. Save you configurations +4. In order to instruct Jenkins to run the pipeline on the configured nodes, put a `label` property in the `agent{ docker {} }` setting, as follows: +```text + agent { + docker { + label '' + image '...' + args '...' + } + } +``` +5. Test your pipeline. + + + +## Security vulnerability scanning + +The [Snyk](https://docs.snyk.io/products/snyk-container/snyk-cli-for-container-security) Container command line interface helps you find and fix vulnerabilities in container images on your local machine. + +You must first to [Sign up for Snyk account](https://docs.snyk.io/getting-started/create-a-snyk-account). +You don't need to install Snyk on your Jenkins server as it was already installed in the Docker-based agent image. + +1. Get you API token from your [Account Settings](https://app.snyk.io/account) page. +2. Once you've set a `SNYK_TOKEN` environment variable with the API token as a value, you can easily [scan docker images](https://docs.snyk.io/products/snyk-container) for vulnerabilities: +```shell +# will scan ubuntu docker image from DockerHub +snyk container test ubuntu + +# will alarm for `high` issue and above +snyk container test ubuntu --severity-threshold=high + +# will scan a local image my-image:latest. The --file=Dockerfile can add more context to the security scanning. +snyk container test my-image:latest --file=Dockerfile +``` + +3. Create a **Secret text** Jenkins credentials containing the API token. +4. Use the `withCredentials` step, read your Snyk API secret as `SNYK_TOKEN` env var, and perform the security testing using simple `sh` step and `synk` cli. + +Sometimes, Snyk alerts you to a vulnerability that has no update or Snyk patch available, or that you do not believe to be currently exploitable in your application. + +You can ignore a specific vulnerability in a project using the [`snyk ignore`](https://docs.snyk.io/snyk-cli/test-for-vulnerabilities/ignore-vulnerabilities-using-snyk-cli) command: + +```text +snyk ignore --id= +``` + +## Pull Request testing + +It's common practice performing an extensive testing on a Pull Request before the code is being deployed to production systems. +So far we've seen how pipeline can be built and run around a single Git branch (e.g. `main` or `dev`). Now we would like to create a new pipeline which will be triggered on **every PR that is created in GitHub**. +For that we will utilize Jenkins [multi-branch pipeline](https://www.jenkins.io/doc/book/pipeline/multibranch/). + +1. From the main Jenkins dashboard page, choose **New Item**, and create a **Multibranch Pipeline** named `PR-testing`. +2. In the **GitHub** source section, under **Discover branches** configure this pipeline to discover PRs only! +3. This pipeline should execute a Jenkins file called `PR.Jenkinsfile` (we will soon create this file in the PolyBot source code). + +In the [PolyBot](https://github.com/alonitac/PolyBot.git) repo, we will simulate a pull request from branch `microservices` to `main`. + +4. Checkout `microservices` branch. Create the `PR.Jenkinsfile`: +```text +pipeline { + agent any + + stages { + stage('Unittest') { + steps { + echo "testing" + } + } + stage('Functional test') { + steps { + echo "testing" + } + } + } +} +``` +5. Commit the Jenkinsfile and push it. +6. From GitHub website, create a new PR from `microservices` branch to `main`. Watch the triggered activity in the new pipeline. + +We would like to protect branch `main` from being merged by non-tested and reviewed branch. + +7. From GitHub main repo page, go to **Settings**, then **Branches**. +8. **Add rule** for the `main` branch as follows: + 1. Check **Require a pull request before merging**. + 2. Check **Require status checks to pass before merging** and search the `continuous-integration/jenkins/branch` check done by Jenkins. + 3. Save the protection rule. + +Your `main` branch is now protected and no code can be merged to it unless the PR is reviewed by other team member and passed all automatic tests done by Jenkins. + + +### Run unittests + +1. Copy the `tests` directory located in [`17_jenkins/tests`](https://github.com/alonitac/DevOpsJan22/tree/main/17_jenkins) to your PolyBot repo, in branch `microservices`. This is a common name for the directory containing all unittests files. The directory contains a file called `test_autoscaling_metric.py` which implements unittest for the `calc_backlog_per_instance` function in `utils.py` file. **You may change your code a bit according to [https://github.com/alonitac/PolyBot/blob/microservices/utils.py](https://github.com/alonitac/PolyBot/blob/microservices/utils.py)**. +2. Run the unittest locally (you may need to install the following requirements: `pytest`, `unittest2`), check that all tests are passed. +3. The test can be run from the `PR.Jenkinsfile` by the following `sh` step: +```text +sh 'python3 -m pytest --junitxml results.xml tests' +``` +Make sure to install the Python requirements in a previous step (`pip3 install...`) + +4. You can add the following `post` step to display the tests results in the readable form: +```text +post { + always { + junit allowEmptyResults: true, testResults: 'results.xml' + } +} +``` +5. Test your pipeline. + +### Run linting check + +[Pylint](https://pylint.pycqa.org/en/latest/) is a static code analyser for Python. +Pylint analyses your code without actually running it. It checks for errors, enforces a coding standard, and can make suggestions about how the code could be refactored. + +1. Install Pylint `pip install pylint` and add it to `requirements.txt` +2. Generate a default template for `.pylintrc` file which is a configuration file defines how Pylint should behave +```shell +pylint --generate-rcfile > .pylintrc +``` + +3. Lint your code locally by: +```shell +python3 -m pylint *.py +``` +How many warnings do you get? If you need to ignore some unuseful warning, add it to `disable` list in `[MESSAGES CONTROL]` section in your `.pylintrc` file. + +4. Add a "Static code linting" stage in `PR.Jenkinsfile`. +5. Use [`parallel`](https://www.jenkins.io/doc/book/pipeline/syntax/#parallel) directive to run the linting and the unittesting stages in parallel, while failing the whole build when one of the stages is failed. +6. Add Pylint reports to Jenkins pipeline dashboard: + 1. Install the [Warnings Next Generation Plugin](https://www.jenkins.io/doc/pipeline/steps/warnings-ng/). + 2. Change your linting stage to be something like (make sure you understand this change): + ```text + steps { + sh 'python3 -m pylint -f parseable --reports=no *.py > pylint.log' + } + post { + always { + sh 'cat pylint.log' + recordIssues ( + enabledForFailure: true, + aggregatingResults: true, + tools: [pyLint(name: 'Pylint', pattern: '**/pylint.log')] + ) + } + } + ``` + +## Terraform in Jenkins + +In this part you will create a Jenkins pipeline aimed to provision infrastructure using Terraform. +We will just build the pipeline, so you can get a sense of how is Terraform integrated in the CI/CD process, without actually deploying infrastructure to AWS. + +1. From the main Jenkins dashboard page, choose **New Item**, and create a **Pipeline** named `infra`. +2. Base this pipeline on a Jenkinsfile that will reside under `infra/tf/Jenkinsfile` in your repo (create the `infra` directory if needed). +3. Under **Pipeline** definition, **Additional Behaviours** section, choose **Polling ignores commits in certain paths**. +4. In the **Included Regions** textbox, enter `infra/tf`. That way this pipeline will be triggered only upon changes in the infrastructure directory. +5. Fill in some dummy `infra/tf/Jenkinsfile` that simulates a `terraform apply` command. +6. Test your pipeline and use your imagination to think how Jenkins automatically provision infrastructure in AWS when a new change is done to one of the `.tf` files under `infra/tf` dir. + +## (Optional) Shared libraries + +https://www.jenkins.io/blog/2017/02/15/declarative-notifications/ + diff --git a/17_jenkins/prepare_ansible_inv.py b/17_jenkins/prepare_ansible_inv.py new file mode 100644 index 00000000..de359ae1 --- /dev/null +++ b/17_jenkins/prepare_ansible_inv.py @@ -0,0 +1,32 @@ +import json + + +def get_instance_name(tags): + for tag in tags: + if tag['Key'] == 'Name': + return tag['Value'] + + raise RuntimeError('Name was not found') + + +def prepare_ansible_inventory(): + with open('hosts.json') as f: + instances = json.load(f) + + hosts = [] + for instance in instances: + instance_name = get_instance_name(instance['Tags']) + instance_ip = instance['PublicIpAddress'] + + hosts.append( + f"{instance_name} ansible_host={instance_ip}\n" + ) + + with open('hosts', 'w') as f: + f.write('[bot]\n') + f.writelines(hosts) + + +if __name__ == '__main__': + + prepare_ansible_inventory() diff --git a/17_jenkins/tests/test_autoscaling_metric.py b/17_jenkins/tests/test_autoscaling_metric.py new file mode 100644 index 00000000..ca57848d --- /dev/null +++ b/17_jenkins/tests/test_autoscaling_metric.py @@ -0,0 +1,22 @@ +import unittest2 as unittest +from unittest.mock import Mock +from utils import calc_backlog_per_instance + + +class TestBacklogPerInstanceMetric(unittest.TestCase): + def setUp(self): + self.sqs_queue_client = Mock() + self.asg_client = Mock() + + def test_no_worker_full_queue(self): + self.sqs_queue_client.attributes = { + 'ApproximateNumberOfMessages': '100' + } + + self.asg_client.describe_auto_scaling_groups = Mock(return_value={ + 'AutoScalingGroups': [{ + 'DesiredCapacity': 0 + }] + }) + + self.assertEqual(calc_backlog_per_instance(self.sqs_queue_client, self.asg_client, None), 99) diff --git a/18_jenkins_ex1/k8s_helpers/ecr-creds-helper.yaml b/18_jenkins_ex1/k8s_helpers/ecr-creds-helper.yaml new file mode 100644 index 00000000..05cd276e --- /dev/null +++ b/18_jenkins_ex1/k8s_helpers/ecr-creds-helper.yaml @@ -0,0 +1,44 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: ecr-registry-helper + namespace: kube-system +spec: + schedule: "*/10 * * * *" + successfulJobsHistoryLimit: 3 + suspend: false + jobTemplate: + spec: + template: + spec: + serviceAccountName: k0s-admin + restartPolicy: Never + containers: + - name: ecr-registry-helper + image: amazon/aws-cli + imagePullPolicy: IfNotPresent + command: + - /bin/sh + - -c + - |- + curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" + chmod +x ./kubectl + + AWS_REGION=$(curl http://169.254.169.254/latest/meta-data/placement/region) + ACC=$(aws sts get-caller-identity --query "Account") + ECR_TOKEN=$(aws ecr get-login-password --region $AWS_REGION) + AWS_ACCOUNT=${ACC:1:-1} + + for NS in "dev" "prod" "default" + do + ./kubectl delete secret --ignore-not-found ecr-docker-creds -n $NS + ./kubectl create secret docker-registry ecr-docker-creds \ + --docker-server=https://${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com \ + --docker-username=AWS \ + --docker-password="${ECR_TOKEN}" \ + --namespace=$NS + ./kubectl patch serviceaccount default \ + -p "{\"imagePullSecrets\": [{\"name\": \"ecr-docker-creds\"}]}" \ + -n $NS + echo "Secret was successfully updated at $(date)" + done diff --git a/18_jenkins_ex1/k8s_helpers/init-k0s-cluster-amazon-linux.sh b/18_jenkins_ex1/k8s_helpers/init-k0s-cluster-amazon-linux.sh new file mode 100644 index 00000000..ac92424d --- /dev/null +++ b/18_jenkins_ex1/k8s_helpers/init-k0s-cluster-amazon-linux.sh @@ -0,0 +1,106 @@ +GREEN='\033[0;32m' +CYAN='\033[0;36m' +NC='\033[0m' +RED='\033[0;31m' + +if [ -x /usr/local/bin/k0s ]; then + sudo /usr/local/bin/k0s stop &> /dev/null + sudo /usr/local/bin/k0s reset &> /dev/null +else + + curl -sSLf https://get.k0s.sh | sudo sh && echo "${CYAN}k0s downloaded${NC}" + +fi + + +/usr/local/bin/k0s config create > k0s.yaml && printf "\n${CYAN}Cluster config filed created${NC}" +sudo /usr/local/bin/k0s install controller -c k0s.yaml --single && printf "\n\n${CYAN}K0s cluster has been installed${NC}" +sudo /usr/local/bin/k0s start && printf "\n\n${CYAN}K0s cluster has been started successfully!${NC}" + +sleep 10 + +kubectl version --client &> /dev/null + +if [ $? -ne 0 ]; then + printf "\n\n${CYAN}Installing kubectl cli tool...${NC}\n\n" + curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" + sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl +fi + +mkdir -p ~/.kube +sudo /usr/local/bin/k0s kubeconfig admin > ~/.kube/config + +printf "\n\n${CYAN}Installing k8s dashboard${NC}\n\n" + +kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.6.1/aio/deploy/recommended.yaml + +which jq &> /dev/null +if [ $? -ne 0 ]; then + sudo yum install jq -y +fi + +cat >dashboard-admin-service-account.yaml < dashboard-token.yaml + + +#IP=$(curl http://169.254.169.254/latest/meta-data/public-ipv4) + +printf "\n\n${RED}===========README===========${NC}" +printf "\n\n${GREEN}Cluster dashboard address is: https://:30001 ${NC}\n\n" +printf "\n\n${GREEN}Access token is (can be always found in ~/dashboard-token.yaml):\n\n" +cat dashboard-token.yaml +printf "${NC}" + +printf "\n\n${RED}Good Luck!${NC}\n" \ No newline at end of file diff --git a/19_final_project/ideas.md b/19_final_project/ideas.md new file mode 100644 index 00000000..66e257d3 --- /dev/null +++ b/19_final_project/ideas.md @@ -0,0 +1,64 @@ +# Final Project Extension Ideas + +### New functionality to the PolyBot (Python extension) + +Implement some python module for the app. + +- Get photos from users and store them. +- Allow users to ask `my videos` and send them all their YouTube videos link. +- Compress the videos before the upload to S3. +- Develop some async component (`async/await` form) using [asyncio](https://docs.python.org/3/library/asyncio-task.html). + + +### DevSecOps + +Embed DevSecOps tool to the CI/CD pipeline: + +- [safety](https://pyup.io/safety/) to scan vulnerabilities in Python packages. +- [Bandit](https://bandit.readthedocs.io/en/latest/) to find security issues in your Python code. +- [Pre-commit](https://pre-commit.com/) to enforce some policy before committing a new code. +- [Black](https://github.com/psf/black) as a linting tool. +- [Chef InSpec](https://docs.chef.io/inspec/) to apply security and compliance policies. + + +### Jenkins + +- Implement load testing code for the Bot and run it in the PR testing pipeline. +- Create a [Jenkins shared library](https://www.jenkins.io/blog/2017/02/15/declarative-notifications/#moving-notifications-to-shared-library). +- Send email notifications to users + +### AWS + +- Expose some API using [API gateway](https://aws.amazon.com/api-gateway/) +- Implement basic user auth with [Cognito](https://aws.amazon.com/cognito/) +- Protect your service using [WAF](https://aws.amazon.com/waf/) or [Shield](https://aws.amazon.com/shield/). +- Any other shiny service that interesting you... + +### K8S + +- Deploy some interesting Helm Chart in the cluster (Jenkins, RabbitMQ - as an alternative to SQS, OpenVPN client/server). +- Write your app YAMLs as Helm Chart. +- Run some [CronJob](https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/) in the cluster. +- Use [ArgoCD](https://argo-cd.readthedocs.io/en/stable/) to deploy your app. +- Implement some interesting [ArgoWF](https://argoproj.github.io/argo-workflows/). +- Experimenting with [Calico](https://projectcalico.docs.tigera.io/about/about-calico) to implement network security in the cluster. +- Experimenting with [Istio](https://istio.io/) to implement a service mesh. +- Expose your app through a secured HTTPS. +- Implement Pod identity in EKS instead using the EC2 IAM role. + +### Terraform + +- Provision the app infrastructure as a code. +- Built a dedicated "IaaC" pipeline in Jenkins + +### Ansible + +- Use some [devsec.hardening Ansible](https://github.com/dev-sec/ansible-collection-hardening) collection to harden the system + +### Monitoring + +- Deploy [Prometheus](https://prometheus.io/) in K8S. +- Enable backup/restore to from [ElasticSearch to S3](https://www.elastic.co/guide/en/elasticsearch/reference/current/snapshot-restore.html). +- Build some Kibana dashboard +- Improve the logs stream from the k8s cluster to Elasticsearch +- Create some [alerts in Grafana](https://grafana.com/docs/grafana/latest/alerting/) (e.g. high CPU rate, container restarts many times etc...) diff --git a/20_artifact_repositories/fantastic_ascii/.pypirc b/20_artifact_repositories/fantastic_ascii/.pypirc new file mode 100644 index 00000000..29a7cd43 --- /dev/null +++ b/20_artifact_repositories/fantastic_ascii/.pypirc @@ -0,0 +1,12 @@ +[distutils] +index-servers = + pypi + pypi-hosted + +[pypi] +repository = https://upload.pypi.org/legacy/ + +[pypi-hosted] +repository = http://13.51.249.176:8081/repository/pypi-internal-packages/ +username: admin +password: 1234 \ No newline at end of file diff --git a/20_artifact_repositories/fantastic_ascii/LICENSE b/20_artifact_repositories/fantastic_ascii/LICENSE new file mode 100644 index 00000000..a5d7f102 --- /dev/null +++ b/20_artifact_repositories/fantastic_ascii/LICENSE @@ -0,0 +1,20 @@ +Copyright (c) 2012-2022 Scott Chacon and others + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/20_artifact_repositories/fantastic_ascii/package_src/__init__.py b/20_artifact_repositories/fantastic_ascii/package_src/__init__.py new file mode 100644 index 00000000..ec6cf1be --- /dev/null +++ b/20_artifact_repositories/fantastic_ascii/package_src/__init__.py @@ -0,0 +1 @@ +import ascii \ No newline at end of file diff --git a/20_artifact_repositories/fantastic_ascii/package_src/ascii.py b/20_artifact_repositories/fantastic_ascii/package_src/ascii.py new file mode 100644 index 00000000..3ebc3c90 --- /dev/null +++ b/20_artifact_repositories/fantastic_ascii/package_src/ascii.py @@ -0,0 +1,26 @@ +import time + + +def joe_sleep(t): + time.sleep(t) + + +def joe_say(text): + template = r''' + =-=-=-=-=-=-=-=-=-=-=-=-=-==-=-=-=-=-=-=-= + // {message} \\ + =-=-=-=-=-=-=-=-=-=-=-=-=-==-=-=-=-=-=-=-= + \\ + \\ + ---------------- + / \ + / \ + | OO O0 | + | OO OO | + \ - / + \ DDDDDD / + \ DDDD / + \____________/ + '''.format(message=text) + + return template diff --git a/20_artifact_repositories/fantastic_ascii/pyproject.toml b/20_artifact_repositories/fantastic_ascii/pyproject.toml new file mode 100644 index 00000000..7fd26b97 --- /dev/null +++ b/20_artifact_repositories/fantastic_ascii/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" \ No newline at end of file diff --git a/20_artifact_repositories/fantastic_ascii/setup.py b/20_artifact_repositories/fantastic_ascii/setup.py new file mode 100644 index 00000000..1ae1f189 --- /dev/null +++ b/20_artifact_repositories/fantastic_ascii/setup.py @@ -0,0 +1,18 @@ +from setuptools import setup, find_packages + +setup_args = dict( + name='fantastic_ascii', + version='1.1.0', + description='Fantastic ASCII', + license='MIT', + install_requires=[ + 'requests', + 'importlib-metadata; python_version == "3.8"', + ], + author='Matt', + author_email='example@example.com' +) + + +if __name__ == '__main__': + setup(**setup_args) diff --git a/20_artifact_repositories/nexus-tutorial.md b/20_artifact_repositories/nexus-tutorial.md new file mode 100644 index 00000000..b1c61325 --- /dev/null +++ b/20_artifact_repositories/nexus-tutorial.md @@ -0,0 +1,137 @@ +# Nexus Repository Manager + +## Install + +We will deploy the Nexus server via [pre-built Docker image](https://hub.docker.com/r/sonatype/nexus3/). + +> You can deploy the Nexus server on the same VM of Jenkins. + +```shell +sudo mkdir /nexus-data && sudo chmod 777 /nexus-data +docker run -d --rm -p 8081:8081 --name nexus -v /nexus-data:/nexus-data -e INSTALL4J_ADD_VM_PARAMS="-Xms400m -Xmx400m -XX:MaxDirectMemorySize=400m" sonatype/nexus3 +``` + +## Repository Management + +Nexus ships with a great [Official docs](https://help.sonatype.com/repomanager3/nexus-repository-administration/repository-management) and compatible with [many package managers](https://help.sonatype.com/repomanager3/nexus-repository-administration/formats): Java/Maven, npm, NuGet, PyPI, Docker, Helm, Yum, and APT. + +### Repository Types + +#### Proxy repo + +Proxy repository is a repository that is linked to a remote repository. Any request for a component is verified against the local content of the proxy repository. If no local component is found, the request is forwarded to the remote repository. + +#### Hosted repo + +Hosted repository is a repository that stores components in the repository manager as the authoritative location for these components. + +#### Group repo + +Repository group allow you to combine multiple repositories and other repository groups in a single repository. +This in turn means that your users can rely on a single URL for their configuration needs, while the administrators can add more repositories and therefore components to the repository group. + + +## Create a PyPi **proxy** repo + +1. After signing in to your Nexus server as an administrator, click on the **Server configuration** icon. +2. Create a [PyPi repo](https://help.sonatype.com/repomanager3/nexus-repository-administration/formats/pypi-repositories). +3. [Configure](https://help.sonatype.com/repomanager3/nexus-repository-administration/formats/pypi-repositories#PyPIRepositories-Download,searchandinstallpackagesusingpip) `pip` to download packages from your private artifact repository. To do so, create a file `pip.conf` with the following content: +```text +[global] +trusted-host = +index-url = http://:8081/repository//simple +index = http://:8081/repository/ +``` + +While changing `` to the DNS/IP of your server. + +5. Put the `pip.conf` file either in your virtual env folder (`venv`). Alternatively (when installing packages outside a virtual env, e.g. in Jenkins Agent), define a custom location by setting the following env var: `PIP_CONFIG_FILE=`. There are [other methods](https://pip.pypa.io/en/stable/topics/configuration/#location). + +## Repository Health Check + +https://help.sonatype.com/repomanager3/nexus-repository-administration/repository-management/repository-health-check + +## Define s3 as an artifacts storage + +https://help.sonatype.com/repomanager3/nexus-repository-administration/repository-management/configuring-blob-stores#ConfiguringBlobStores-AWSSimpleStorageService(S3) + +## Create a PyPi **hosted** repo, pack and upload a Python library + +1. Create a`pypi (hosted)` format repo. +2. Set the configured S3 as the blob store. + +### Build a Python package + +You can share reusable code (in the form of a library) and programs (e.g., CLI/GUI tools) implemented in Python. +[Setuptools](https://setuptools.pypa.io/en/latest/index.html) is a Python library designed to facilitate packaging Python projects. + +Under `20_artifact_repositories/fantastic_ascii`, you are given a sample source code for a library called "fantastic_ascii". We will pack and publish the code as a Python library. + +The [official quick start](https://setuptools.pypa.io/en/latest/userguide/quickstart.html) of Setuptools is a good starting point of how to do it. The following steps are summarizing the quickstart page: + +1. Install `build` library by `pip install --upgrade build`. +2. In the library source code (`20_artifact_repositories/fantastic_ascii`), create `pyproject.toml`: +```toml +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" +``` +3. In the library source code, create `setup.py`: +```python +from setuptools import setup + +setup_args = dict( + name='fantastic_ascii', + version='1.0.0', + description='Fantastic ASCII', + license='MIT', + install_requires=[ + 'requests', + 'importlib-metadata; python_version == "3.8"', + ], + author='Matt', + author_email='example@example.com' +) + + +if __name__ == '__main__': + setup(**setup_args) + +``` + +The library source code should look like: +```text +fantastic_ascii +├── pyproject.toml +| setup.py +| LICENCE (properly chosen license information, e.g. MIT, BSD-3, GPL-3, MPL-2, etc...) +└── package_src + ├── __init__.py + └── ... (other Python files) +``` + +4. Open a terminal in the library source code, build the package by: `python -m build`. + You now have your distribution ready (e.g. a tar.gz file and a .whl file in the dist directory), which you can upload to your private PyPi repo. + +### Distribute your package using twine + +6. You can use [twine](https://twine.readthedocs.io/en/latest/) to upload the distribution packages. You’ll need to install Twine: `pip install --upgrade twine`. +7. In order to upload your package to the PyPi repo in Nexus, configure the `.pypirc` file [as describe in Nexus docs](https://help.sonatype.com/repomanager3/nexus-repository-administration/formats/pypi-repositories#PyPIRepositories-Uploadtoahostedrepositoryusingtwine). +8. Upload your package by: +```text +python3 -m twine upload --config-file --repository dist/* +``` + +## Jenkins integration + +### Fantastic ASCII Build pipeline + +Create a Jenkins Pipeline that builds the `fantastic_ascii` package. General guidelines: + +- The pipeline is triggered **manually** from Jenkins dashboard. +- The pipeline checks is the package version specified in `setup.py` exists in Nexus. If it doesn't exist, the pipeline builds and upload the package (as done in the two sections above). +- Store Nexus username and password as a Jenkins credential and use them with `withCredentials()`. + +### Install Python dependencies from Nexus repo + +Recall that whenever Docker builds images, it executes `pip install` as part of the build process. Configure Docker to download and install packages from Nexus. diff --git a/20_artifact_repositories/publish.Jenkinsfile b/20_artifact_repositories/publish.Jenkinsfile new file mode 100644 index 00000000..06f7328f --- /dev/null +++ b/20_artifact_repositories/publish.Jenkinsfile @@ -0,0 +1,25 @@ +pipeline { + agent any + stages { +// stage('Git Clone') { +// steps { +// cleanWs() +// git branch: 'main', url: 'https://github.com/alonitac/DevOpsJan22.git' +// } +// +// } + stage('Upload to artifact') { + steps { + dir('20_artifact_repositories/fantastic_ascii/') { + withCredentials([usernamePassword(credentialsId: 'nexus', usernameVariable: 'USER', passwordVariable: 'PASSWORD')]) { + sh ''' + pip3 install build twine virtualenv + python3 -m build + python3 -m twine upload --repository-url -u $USER -p $PASSWORD dist/* + ''' + } + } + } + } + } +} \ No newline at end of file diff --git a/21_k8s/aws_auth.yaml b/21_k8s/aws_auth.yaml new file mode 100644 index 00000000..a620d1f5 --- /dev/null +++ b/21_k8s/aws_auth.yaml @@ -0,0 +1,187 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: aws-auth + namespace: kube-system +data: + mapRoles: | + - groups: + - system:bootstrappers + - system:nodes + rolearn: arn:aws:iam::352708296901:role/alonit-k8s-default-ng + username: system:node:{{EC2PrivateDNSName}} + mapUsers: | + - username: "AdanBhsas" + userarn: "arn:aws:iam::352708296901:user/AdanBhsas" + groups: + - system:masters + + - username: "AlexeyMihaylovDev" + userarn: "arn:aws:iam::352708296901:user/AlexeyMihaylovDev" + groups: + - system:masters + + - username: "Alon_Itach" + userarn: "arn:aws:iam::352708296901:user/Alon_Itach" + groups: + - system:masters + + - username: "Amitpoz" + userarn: "arn:aws:iam::352708296901:user/Amitpoz" + groups: + - system:masters + + - username: "Aviadfri" + userarn: "arn:aws:iam::352708296901:user/Aviadfri" + groups: + - system:masters + + - username: "avihai1002" + userarn: "arn:aws:iam::352708296901:user/avihai1002" + groups: + - system:masters + + - username: "dadyterra" + userarn: "arn:aws:iam::352708296901:user/dadyterra" + groups: + - system:masters + + - username: "Daniel-Reuven" + userarn: "arn:aws:iam::352708296901:user/Daniel-Reuven" + groups: + - system:masters + + - username: "danielbar0101" + userarn: "arn:aws:iam::352708296901:user/danielbar0101" + groups: + - system:masters + + - username: "danielmalka14" + userarn: "arn:aws:iam::352708296901:user/danielmalka14" + groups: + - system:masters + + - username: "Danishain" + userarn: "arn:aws:iam::352708296901:user/Danishain" + groups: + - system:masters + + - username: "Ddady1" + userarn: "arn:aws:iam::352708296901:user/Ddady1" + groups: + - system:masters + + - username: "dmitriyshub" + userarn: "arn:aws:iam::352708296901:user/dmitriyshub" + groups: + - system:masters + + - username: "dorondollev" + userarn: "arn:aws:iam::352708296901:user/dorondollev" + groups: + - system:masters + + - username: "DustyMadDude" + userarn: "arn:aws:iam::352708296901:user/DustyMadDude" + groups: + - system:masters + + - username: "eliranshriki" + userarn: "arn:aws:iam::352708296901:user/eliranshriki" + groups: + - system:masters + + - username: "elkan316" + userarn: "arn:aws:iam::352708296901:user/elkan316" + groups: + - system:masters + + - username: "Gershoz" + userarn: "arn:aws:iam::352708296901:user/Gershoz" + groups: + - system:masters + + - username: "Haimr101" + userarn: "arn:aws:iam::352708296901:user/Haimr101" + groups: + - system:masters + + - username: "itayb12" + userarn: "arn:aws:iam::352708296901:user/itayb12" + groups: + - system:masters + + - username: "JohnSchiff" + userarn: "arn:aws:iam::352708296901:user/JohnSchiff" + groups: + - system:masters + + - username: "kostalubarsky" + userarn: "arn:aws:iam::352708296901:user/kostalubarsky" + groups: + - system:masters + + - username: "netanelmalkiel" + userarn: "arn:aws:iam::352708296901:user/netanelmalkiel" + groups: + - system:masters + + - username: "poratnick" + userarn: "arn:aws:iam::352708296901:user/poratnick" + groups: + - system:masters + + - username: "RamiKandov" + userarn: "arn:aws:iam::352708296901:user/RamiKandov" + groups: + - system:masters + + - username: "shayalon" + userarn: "arn:aws:iam::352708296901:user/shayalon" + groups: + - system:masters + + - username: "Shaykc" + userarn: "arn:aws:iam::352708296901:user/Shaykc" + groups: + - system:masters + + - username: "shaysoso" + userarn: "arn:aws:iam::352708296901:user/shaysoso" + groups: + - system:masters + + - username: "shlomigd" + userarn: "arn:aws:iam::352708296901:user/shlomigd" + groups: + - system:masters + + - username: "talsht" + userarn: "arn:aws:iam::352708296901:user/talsht" + groups: + - system:masters + + - username: "test" + userarn: "arn:aws:iam::352708296901:user/test" + groups: + - system:masters + + - username: "test2" + userarn: "arn:aws:iam::352708296901:user/test2" + groups: + - system:masters + + - username: "xXxARLxXx" + userarn: "arn:aws:iam::352708296901:user/xXxARLxXx" + groups: + - system:masters + + - username: "yakirfrid" + userarn: "arn:aws:iam::352708296901:user/yakirfrid" + groups: + - system:masters + + - username: "zoharn007" + userarn: "arn:aws:iam::352708296901:user/zoharn007" + groups: + - system:masters diff --git a/21_k8s/elastic-fluent/elasticsearch.yaml b/21_k8s/elastic-fluent/elasticsearch.yaml new file mode 100644 index 00000000..b2b755ea --- /dev/null +++ b/21_k8s/elastic-fluent/elasticsearch.yaml @@ -0,0 +1,58 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: elasticsearch +spec: + serviceName: elasticsearch-svc + replicas: 1 + selector: + matchLabels: + app: elasticsearch + template: + metadata: + labels: + app: elasticsearch + spec: + initContainers: + - name: permissions-fix + image: busybox + volumeMounts: + - name: elasticsearch-storage + mountPath: /usr/share/elasticsearch/data + command: [ 'chown' ] + args: [ '1000:1000', '/usr/share/elasticsearch/data' ] + containers: + - name: elasticsearch + image: elasticsearch:7.17.7 + env: + - name: "discovery.type" + value: "single-node" + - name: "ES_JAVA_OPTS" + value: "-Xms256m -Xmx256m" + volumeMounts: + - name: elasticsearch-storage + mountPath: /usr/share/elasticsearch/data + resources: + requests: + cpu: 50m + volumeClaimTemplates: + - metadata: + name: elasticsearch-storage + spec: + accessModes: [ "ReadWriteOnce" ] + storageClassName: standard + resources: + requests: + storage: 5Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: elasticsearch-svc +spec: + selector: + app: elasticsearch + ports: + - name: http + port: 9200 + targetPort: 9200 diff --git a/21_k8s/elastic-fluent/fluent-values.yaml b/21_k8s/elastic-fluent/fluent-values.yaml new file mode 100644 index 00000000..729bee04 --- /dev/null +++ b/21_k8s/elastic-fluent/fluent-values.yaml @@ -0,0 +1,12 @@ +fileConfigs: + 04_outputs.conf: |- + + + diff --git a/21_k8s/elastic-fluent/grafana.yaml b/21_k8s/elastic-fluent/grafana.yaml new file mode 100644 index 00000000..90de3c42 --- /dev/null +++ b/21_k8s/elastic-fluent/grafana.yaml @@ -0,0 +1,52 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: grafana +spec: + replicas: 1 + serviceName: grafana-svc + selector: + matchLabels: + app: grafana + template: + metadata: + name: grafana + labels: + app: grafana + spec: + securityContext: + runAsUser: 472 + runAsGroup: 8020 + fsGroup: 8020 + containers: + - name: grafana + image: grafana/grafana-oss:9.1.8 + ports: + - name: grafana + containerPort: 3000 + env: + - name: GF_SERVER_DOMAIN + value: "grafana-svc" + volumeMounts: + - mountPath: "/var/lib/grafana" + name: grafana-storage + volumeClaimTemplates: + - metadata: + name: grafana-storage + spec: + accessModes: [ "ReadWriteOnce" ] + storageClassName: standard + resources: + requests: + storage: 5Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: grafana-svc +spec: + selector: + app: grafana + ports: + - port: 3000 + targetPort: 3000 \ No newline at end of file diff --git a/21_k8s/k8s-tutorial.md b/21_k8s/k8s-tutorial.md new file mode 100644 index 00000000..09b28997 --- /dev/null +++ b/21_k8s/k8s-tutorial.md @@ -0,0 +1,420 @@ +# Kubernetes Tutorials + +## Install Minikube + +[Minikube](https://kubernetes.io/docs/tasks/tools/#minikube) is a tool that lets you run Kubernetes locally. + +Installation as well as starting a cluster instructions can be found here: https://minikube.sigs.k8s.io/docs/start/ + + +## Install `kubectl` + +1. Download the `kubectl` binary from [Kubernetes](https://kubernetes.io/docs/tasks/tools/install-kubectl-windows/#install-kubectl-binary-with-curl-on-windows) official site. + +2. Put the `kubectl.exe` binary in a directory accessible in your PATH environment variable. + +## Start K8S dashboard + +Kubernetes Dashboard allows you to get easily acclimated to your new cluster. + +1. Execute +```shell +minikube dashboard +``` + +2. To access the dashboard endpoint, open the printed link with a web browser. + +# Kubernetes Tasks + +The Kubernetes documentation contains pages that show how to do individual tasks. +During this module we will walk through core tasks. + +https://kubernetes.io/docs/tasks/ + +## Run a Stateless Application Using a Deployment + +Follow: +https://kubernetes.io/docs/tasks/run-application/run-stateless-application-deployment/ + +### Further reading and doing + +#### Understanding [Kubernetes Objects](https://kubernetes.io/docs/concepts/overview/working-with-objects/kubernetes-objects/) + +Almost every Kubernetes object includes a nested object that govern the object's configuration: the object `spec`. +The `spec` provides a description of the characteristics you want the resource to have: its **desired state**. + +In the .yaml file for the Kubernetes object you want to create, you'll need to set values for the following fields: + +- `apiVersion` - Which version of the Kubernetes API you're using to create this object. +- `kind` - What kind of object you want to create. +- `metadata` - Data that helps uniquely identify the object, including a name string, UID, and optional namespace. +- `spec` - What state you desire for the object. + +**Labels** are key/value pairs that are attached to objects, such as Deployment. +Labels are intended to be used to specify identifying attributes of objects that are meaningful and relevant to users. E.g.: + +- `"release" : "stable"` +- `"environment" : "dev"` +- `"tier" : "backend"` + +Via a **Label Selector**, the client/user can identify a set of objects. + +#### Deploy your own app + +1. Build a simple Flask webserver in a Docker container (can be found in `05_simple_webserver`). +2. Push the image to a **public** container registry (e.g. ERC). +3. Change the `deployment.yaml` manifest according to your image. +4. Apply your changes. +5. You can use [`kubectl port-forward`](https://kubernetes.io/docs/tasks/access-application-cluster/port-forward-access-application-cluster/) command to forward specific pod and port to your local machine, so you can visit the app under the `localhost:` address. This type of connection can be useful for pod debugging and obviously should not be used outside the borders of the development team. + To do so, perform: + +```shell +kubectl port-forward : +``` + +## Use Port Forwarding to Access Applications in a Cluster + +Follow: +https://kubernetes.io/docs/tasks/access-application-cluster/port-forward-access-application-cluster/ + +## Use a Service to Access an Application in the Cluster + +Follow: +https://kubernetes.io/docs/tasks/access-application-cluster/service-access-application-cluster/ + +Notes: + +- _Using a service configuration file_ section (use YAML file instead `kubectl expose` command). +- Use `minikube ip` to get the IP of Minikube "node" and visit the app in `http://:` + +![](../docs/img/service-k8s.png) + + +### Further reading and doing + +Services can be exposed in different ways by specifying a `type` in the ServiceSpec. We will review two types: + +- `ClusterIP` (default) - Exposes the Service on an internal IP in the cluster. This type makes the Service only reachable from within the cluster. +- `NodePort` - Exposes the Service on some port of each **Node** in the cluster. Makes a Service accessible from outside the cluster using `:`. + +## Assign Memory Resources to Containers and Pods + +Follow: +https://kubernetes.io/docs/tasks/configure-pod-container/assign-memory-resource/ + + +## Assign CPU Resources to Containers and Pods + +Follow: +https://kubernetes.io/docs/tasks/configure-pod-container/assign-cpu-resource/ + + +## Configure Liveness, Readiness and Startup Probes + +Follow: +https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/ + +> The following sections can be skipped: +> - Define a TCP liveness probe +> - Define a gRPC liveness probe +> - Use a named port +> - Protect slow starting containers with startup probes + +### Further reading and doing + +Under `21_k8s/zero-downtime-deployment-demo` you will find a simple Flask webserver. + +1. User Docker and ECR (or Nexus, or DockerHub) to build and push the app according to the Dockerfile. +2. In `deployment.yaml` change the Deployment `image:` according to your image URI. +3. Apply your changes. +4. Generate some load on your app by: + +`kubectl run -i --tty load-generator --rm --image=busybox:1.28 --restart=Never -- /bin/sh -c "while sleep 0.2; do (wget -q -O- http://simaple-webserver-service:8080 &); done"` + +5. During the load test, perform a rolling update to a new version of the app (new built Docker image). Change the Python code so it can be seen clearly when you are responded from the new app version. e.g. return `Hello world 2` instead of `Hello world`. +6. Observe how during rolling update, some requests are failing. +7. Use the `/ready` endpoint and add a `readinessProbe` to gain zero-downtime rolling update, which means, all user requests are being served, even during the update. + + +## Configure a Pod to Use a Volume for Storage + +Follow: +https://kubernetes.io/docs/tasks/configure-pod-container/configure-volume-storage/ + +### Further reading and doing + +- Familiarize yourself with the material in [Volumes](https://kubernetes.io/docs/concepts/storage/volumes/) +- [Communicate Between Containers in the Same Pod Using a Shared Volume](https://kubernetes.io/docs/tasks/access-application-cluster/communicate-containers-same-pod-shared-volume/) + + +## Configure a Pod to Use a PersistentVolume for Storage + +Follow: +https://kubernetes.io/docs/tasks/configure-pod-container/configure-persistent-volume-storage/ + +## Distribute Credentials Securely Using Secrets + +Follow: +https://kubernetes.io/docs/tasks/inject-data-application/distribute-credentials-secure/ + + +## ConfigMap + +In this demo we will deploy MySQL server in Kuberenetes cluster using Deployment. + +All Yaml files are under `21_k8s/mysql-configmap-secret-demo`. + + +1. Create a Secret object containing the root username password for MySQL + +`kubectl apply -f mysql-secret.yaml` + +2. Deploy the MySQL deployment by applying `mysql-deployment.yaml` configuration file. + +Now let's say we want to allow maximum of 50 connection to our DB. We would like to find a useful way to "inject" this config to our pre-built `mysql:5.7` image (we surely don't want to build the MySQL image ourselves). +For that, the [ConfigMap](https://kubernetes.io/docs/concepts/configuration/configmap/) object can assist. +In the `mysql` Docker image, custom configurations for the MySQL server can be placed in `/etc/mysql/mysql.conf.d` directory, any file ends with `.cnf` under that directory, will be applied as an additional configurations to MySQL. But how can we "insert" a custom file to the image? keep reading... + +5. Review the ConfigMap object under `mysql-config.yaml`. And apply it. +6. Comment **in** the two snippets in `mysql-deployment.yaml` and apply the changes. +7. Make sure the new configurations applied. + + +### Further reading and doing + +https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap + +## Run a Single-Instance Stateful Application + +Follow: +https://kubernetes.io/docs/tasks/run-application/run-single-instance-stateful-application/ + +## Run a Replicated Stateful Application + +Follow: +https://kubernetes.io/docs/tasks/run-application/run-replicated-stateful-application/ + +## HorizontalPodAutoscaler Walkthrough + +Follow: +https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/ + +## Running Automated Tasks with a CronJob + +Follow: +https://kubernetes.io/docs/tasks/job/automated-tasks-with-cron-jobs/ + + +### Further reading and doing + +- https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/ + +## Helm + +Helm is the package manager for Kubernetes. +The main big 3 concepts of helm are: + +- A **Chart** is a Helm package. It contains all the resource definitions necessary to run an application, tool, or service inside of a Kubernetes cluster. +- A **Repository** is the place where charts can be collected and shared. +- A **Release** is an instance of a chart running in a Kubernetes cluster. + +[Install](https://helm.sh/docs/intro/install/) the Helm cli if you don't have. + +You can familiarize yourself with this tool using [Helm docs](https://helm.sh/docs/intro/using_helm/). + +### Deploy MySQL using Helm + +How relational databases are deployed in real-life applications? + +The following diagram shows a Multi-AZ DB cluster. + +![](../docs/img/mysql-multi-instance.png) + +With a Multi-AZ DB cluster, MySQL replicates data from the writer DB instance to both of the reader DB instances. +When a change is made on the writer DB instance, it's sent to each reader DB instance. +Acknowledgment from at least one reader DB instance is required for a change to be committed. +Reader DB instances act as automatic failover targets and also serve read traffic to increase application read throughput. + +Once you have Helm ready, you can add a chart repository. Check [Artifact Hub](https://artifacthub.io/packages/search?kind=0). + +Let's review the Helm chart written by Bitnami for MySQL provisioning in k8s cluster. + +[https://github.com/bitnami/charts/tree/master/bitnami/mysql/#installing-the-chart](https://github.com/bitnami/charts/tree/master/bitnami/mysql/#installing-the-chart) + +1. Add the Bitnami Helm repo to your local machine: +```shell +# or update if you have it already: `helm repo update bitnami` +helm repo add bitnami https://charts.bitnami.com/bitnami +``` +2. First let's install the chart without any changes +```shell +# helm install / +helm install mysql bitnami/mysql +``` + +Whenever you install a chart, a new release is created. So one chart can be installed multiple times into the same cluster. And each can be independently managed and upgraded. + +During installation, the helm client will print useful information about which resources were created, what the state of the release is, and also whether there are additional configuration steps you can or should take. + +You can always type `helm list` to see what has been released using Helm. + +Now we want to customize the chart according to our business configurations. +To see what options are configurable on a chart, use `helm show values bitnami/mysql` or even better, go to the chart documentation on GitHub. + +We will pass configuration data during the chart upgrade by specify a YAML file with overrides (`-f custom-values.yaml`). This can be specified multiple times and the rightmost file will take precedence. + +3. Review `mysql-helm/values.yaml`, change values or [add parameters](https://github.com/bitnami/charts/tree/master/bitnami/mysql/#parameters) according to your need. +4. Upgrade the `mysql` chart by +```shell +helm upgrade -f mysql-helm/values.yaml mysql bitnami/mysql +``` + +An upgrade takes an existing release and upgrades it according to the information you provide. Because Kubernetes charts can be large and complex, Helm tries to perform the least invasive upgrade. It will only update things that have changed since the last release. + +If something does not go as planned during a release, it is easy to roll back to a previous release using `helm rollback [RELEASE] [REVISION]`: + +```shell +helm rollback mysql 1 +``` + +5. To uninstall this release: +```shell +helm uninstall mysql +``` + +## Stream Pod logs to Elasticsearch databases using FluentD + +### Fluentd introduced + +[Fluentd](https://www.fluentd.org/) is an open source data collector for unified logging layer. +Fluent allows you to unify data collection and consumption for a better use and understanding of data. + +Here is an illustration of how Fluent works in the k8s cluster: + +![](../docs/img/fluent.png) + +Fluentd runs in the cluster as a [DaemonSet](https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/). A DaemonSet ensures that all **nodes** run a copy of a **pod**. That way, Fluentd can collect log information from every containerized applications easily in each k8s node. + +We will deploy the Fluentd chart to collect containers logs to send them to [Elasticsearch](https://www.elastic.co/what-is/elasticsearch) database. + +1. Visit the Fluentd Helm chart at https://github.com/fluent/helm-charts/tree/main/charts/fluentd +2. Add the helm repo +```shell +# or update if you have it already: `helm repo update fluent` +helm repo add fluent https://fluent.github.io/helm-charts +``` + +3. Install the Fluentd chart by: +```shell +helm install fluentd fluent/fluentd +``` + +4. Watch and inspect the running containers under **Workloads** -> **DaemonSet**. Obviously, it doesn't work, as Fluent need to talk to an existed Elasticsearch database. +5. Elasticsearch db can be provisioned by applying `elasticsearch.yaml`. +6. Create a YAML file called `fluentd-helm-values.yaml`. You should override the [following](https://github.com/fluent/helm-charts/blob/main/charts/fluentd/values.yaml#L379) default Helm values, by: +```yaml +fileConfigs: + 04_outputs.conf: |- + +``` +While replacing `` and `` with the hostname of Elasticsearch int the cluster. +7. Finally, upgrade the `fluentd` release by `helm upgrade -f elastic-fluent/fluentd-helm-values.yaml fluentd fluent/fluentd` + + +### Visualize logs with Grafana + +1. Review the objects in `grafana.yaml` and apply. +2. Visit grafana service (default username and password is `admin`) and configure the Elasticsearch database to view all cluster logs. + + +### Fluentd permissions in the cluster + +Have you wondered how does the Fluentd pods have access to other pods logs!? + +This is a great point to learn something about k8s role and access control mechanism ([RBAC](https://kubernetes.io/docs/reference/access-authn-authz/rbac/)). + +#### Role and ClusterRole + +_Role_ or _ClusterRole_ contains rules that represent a set of permissions on the cluster (e.g. This Pod can do that action..). +A Role always sets permissions within a particular _namespace_ +ClusterRole, by contrast, is a non-namespaced resource. + +#### Service account + +A _Service Account_ provides an identity for processes that run in a Pod. +When you create a pod, if you do not specify a service account, it is automatically assigned the `default` service account in the same namespace. + +#### RoleBinding and ClusterRoleBinding + +A role binding grants the permissions defined in a role to a user or set of users. +A RoleBinding may reference any Role in the same namespace. Alternatively, a RoleBinding can reference a ClusterRole and bind that ClusterRole to the namespace of the RoleBinding. + +--- + +Observe the service account used by the fluentd Pods, observe their ClusterRole bound to them. + + + +## Elastic Kubernetes Service (EKS) + +Follow the below docs to create a cluster using the management console: +https://docs.aws.amazon.com/eks/latest/userguide/create-cluster.html + +In order to connect to an EKS cluster, you should execute the following `aws` command from your local machine: + +```shell +aws eks --region update-kubeconfig --name +``` + +Change `` and `` accordingly. + + +### Deploy the k8s dashboard in EKS + +https://docs.aws.amazon.com/eks/latest/userguide/dashboard-tutorial.html + +### Install Ingress and Ingress Controller on EKS + +[Ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/#what-is-ingress) exposes HTTP and HTTPS routes from outside the cluster to services within the cluster. +An Ingress may be configured to give Services externally-reachable URLs, load balance traffic, terminate SSL / TLS, and offer name-based virtual hosting. +In order for the **Ingress** resource to work, the cluster must have an **Ingress Controller** running. + +Kubernetes as a project supports and maintains AWS, GCE, and [nginx](https://github.com/kubernetes/ingress-nginx) ingress controllers. + +1. If working on a shared repo, create your own namespace by: + ```shell + kubectl create ns + ``` +2. Deploy the 2048 game app under `manifests/2048.yaml`, make sure you change `namespace: ` to your namespace name. +3. Deploy the Nginx ingres controller (done only once per cluster). Nginx ships with ready to use HELM charts or YAML manifests for many cloud providers. We will deploy the [Nginx ingress controller behind a Network Load Balancer](https://kubernetes.github.io/ingress-nginx/deploy/#aws). (why Network LB is preferred than Application LB?) + +We want to access the 2048 game application from a domain such as http://test-2048.devops-int-college.com + +4. Add a subdomain A record for the [devops-int-college.com](https://us-east-1.console.aws.amazon.com/route53/v2/hostedzones#ListRecordSets/Z02842682SGSPDJQMJGFT) domain (e.g. test-2048.devops-int-college.com). The record should have an alias to the NLB created by EKS after the ingress controller has been deployed. +5. Inspired by the manifests described in [Nginx ingress docs](https://kubernetes.github.io/ingress-nginx/user-guide/basic-usage/#basic-usage-host-based-routing), create and apply an Ingress resource such that when visiting your registered DNS, the 2048 game will be displayed on screen. + +## Prometheus on K8S + +[Prometheus](https://prometheus.io/docs/introduction/overview/) Prometheus is a monitoring platform that collects metrics from monitored targets by scraping metrics HTTP endpoints on these targets. +Prometheus is shipped with an extensive list of [exporters](https://prometheus.io/docs/instrumenting/exporters/). An exporter is a pluggable piece which allow Prometheus to collect metrics from other system (e.g. databases, cloud services, OS etc..). Some exporters are official, others developed by the community. + +Note: If using a shared k8s cluster, **deploy all resources in your own namespace**! + +1. Deploy Prometheus using the [community Helm chart](https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus). +2. Deploy Grafana (either by Helm or by the manifest under `21_k8s/elastic-fluent/grafana.yaml`). +3. Connect to Grafana (you can utilize the installed Ingress controller or by `kubectl port-forward`). +4. Configure the Prometheus server as a data source. +5. Import one of the following dashboards: + - https://grafana.com/grafana/dashboards/6417-kubernetes-cluster-prometheus/ + - https://grafana.com/grafana/dashboards/315-kubernetes-cluster-monitoring-via-prometheus/ + - https://grafana.com/grafana/dashboards/12740-kubernetes-monitoring/ +6. Deploy the [Prometheus Cloudwatch Exporter](https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-cloudwatch-exporter). +7. Configure Prometheus to scrape metrics from Cloudwatch Exporter (you may find helpful values under `prometheus/values.yaml`). + diff --git a/21_k8s/manifests/2048.yaml b/21_k8s/manifests/2048.yaml new file mode 100644 index 00000000..08fd47f1 --- /dev/null +++ b/21_k8s/manifests/2048.yaml @@ -0,0 +1,32 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: "2048" + namespace: +spec: + selector: + matchLabels: + app: "2048" + replicas: 1 + template: + metadata: + labels: + app: "2048" + spec: + containers: + - image: alexwhen/docker-2048 + name: "2048" + ports: + - containerPort: 80 + protocol: TCP +--- +apiVersion: v1 +kind: Service +metadata: + name: "game-2048" + namespace: +spec: + ports: + - port: 80 + selector: + app: "2048" diff --git a/21_k8s/manifests/cpu-limit.yaml b/21_k8s/manifests/cpu-limit.yaml new file mode 100644 index 00000000..8a4ca66c --- /dev/null +++ b/21_k8s/manifests/cpu-limit.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Pod +metadata: + name: cpu-demo + namespace: cpu-example +spec: + containers: + - name: cpu-demo-ctr + image: vish/stress + resources: + limits: + cpu: "10" + memory: "100Mi" + requests: + cpu: "8" + memory: "100Mi" + args: + - -cpus + - "2" \ No newline at end of file diff --git a/21_k8s/manifests/deployment-demo.yaml b/21_k8s/manifests/deployment-demo.yaml new file mode 100644 index 00000000..e9ea353e --- /dev/null +++ b/21_k8s/manifests/deployment-demo.yaml @@ -0,0 +1,35 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-deployment + labels: + app: nginx +spec: + selector: + matchLabels: + app: nginx + replicas: 2 # tells deployment to run 2 pods matching the template + template: + metadata: + labels: + app: nginx + spec: + containers: + - name: nginx + image: nginx:1.14.2 + ports: + - containerPort: 80 +--- +apiVersion: v1 +kind: Service +metadata: + name: nginx-deployment-service +spec: + type: NodePort + selector: + app: nginx + ports: + - protocol: TCP + port: 80 + targetPort: 80 + nodePort: 30001 \ No newline at end of file diff --git a/21_k8s/manifests/exec-liveness-probe.yaml b/21_k8s/manifests/exec-liveness-probe.yaml new file mode 100644 index 00000000..28cda3df --- /dev/null +++ b/21_k8s/manifests/exec-liveness-probe.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: Pod +metadata: + labels: + test: liveness + name: liveness-exec +spec: + containers: + - name: liveness + image: registry.k8s.io/busybox + args: + - /bin/sh + - -c + - touch /tmp/healthy; sleep 30; rm -f /tmp/healthy; sleep 600 + livenessProbe: + exec: + command: + - cat + - /tmp/healthy + initialDelaySeconds: 5 + periodSeconds: 5 \ No newline at end of file diff --git a/21_k8s/manifests/hpa-example.yaml b/21_k8s/manifests/hpa-example.yaml new file mode 100644 index 00000000..23fe6ac8 --- /dev/null +++ b/21_k8s/manifests/hpa-example.yaml @@ -0,0 +1,49 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: php-apache +spec: + selector: + matchLabels: + run: php-apache +# replicas: 1 + template: + metadata: + labels: + run: php-apache + spec: + containers: + - name: php-apache + image: registry.k8s.io/hpa-example + ports: + - containerPort: 80 + resources: + limits: + cpu: 300m + requests: + cpu: 100m +--- +apiVersion: v1 +kind: Service +metadata: + name: php-apache + labels: + run: php-apache +spec: + ports: + - port: 80 + selector: + run: php-apache +--- +apiVersion: autoscaling/v1 +kind: HorizontalPodAutoscaler +metadata: + name: php-apache +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: php-apache + minReplicas: 1 + maxReplicas: 100 + targetCPUUtilizationPercentage: 50 \ No newline at end of file diff --git a/21_k8s/manifests/ingress.yaml b/21_k8s/manifests/ingress.yaml new file mode 100644 index 00000000..b25c3e07 --- /dev/null +++ b/21_k8s/manifests/ingress.yaml @@ -0,0 +1,18 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: ingress-sdfsdfalonit + namespace: alonit +spec: + rules: + - host: alonit-game.devops-int-college.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: game-2048 + port: + number: 80 + ingressClassName: nginx \ No newline at end of file diff --git a/21_k8s/manifests/mysql-single-instance-volume.yaml b/21_k8s/manifests/mysql-single-instance-volume.yaml new file mode 100644 index 00000000..40ab3e7f --- /dev/null +++ b/21_k8s/manifests/mysql-single-instance-volume.yaml @@ -0,0 +1,26 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: mysql-pv-volume + labels: + type: local +spec: + storageClassName: manual + capacity: + storage: 5Gi + accessModes: + - ReadWriteOnce + hostPath: + path: "/mnt/data" +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: mysql-pv-claim +spec: + storageClassName: manual + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi \ No newline at end of file diff --git a/21_k8s/manifests/mysql-single-instance.yaml b/21_k8s/manifests/mysql-single-instance.yaml new file mode 100644 index 00000000..45b58530 --- /dev/null +++ b/21_k8s/manifests/mysql-single-instance.yaml @@ -0,0 +1,44 @@ +apiVersion: v1 +kind: Service +metadata: + name: mysql +spec: + ports: + - port: 3306 + selector: + app: mysql + clusterIP: None +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mysql +spec: + replicas: 1 + selector: + matchLabels: + app: mysql + strategy: + type: Recreate + template: + metadata: + labels: + app: mysql + spec: + containers: + - image: mysql:5.7 + name: mysql + env: + # Use secret in real usage + - name: MYSQL_ROOT_PASSWORD + value: password + ports: + - containerPort: 3306 + name: mysql + volumeMounts: + - name: mysql-persistent-storage + mountPath: /var/lib/mysql + volumes: + - name: mysql-persistent-storage + persistentVolumeClaim: + claimName: mysql-pv-claim \ No newline at end of file diff --git a/21_k8s/manifests/pv-claim.yaml b/21_k8s/manifests/pv-claim.yaml new file mode 100644 index 00000000..b33f6faa --- /dev/null +++ b/21_k8s/manifests/pv-claim.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: task-pv-claim +spec: + storageClassName: manual + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 3Gi diff --git a/21_k8s/manifests/pv-pod.yaml b/21_k8s/manifests/pv-pod.yaml new file mode 100644 index 00000000..eb10d597 --- /dev/null +++ b/21_k8s/manifests/pv-pod.yaml @@ -0,0 +1,26 @@ +apiVersion: v1 +kind: Pod +metadata: + name: task-pv-pod +spec: + volumes: + - name: task-pv-storage + persistentVolumeClaim: + claimName: task-pv-claim + containers: + - name: task-pv-container + image: nginx + ports: + - containerPort: 80 + name: "http-server" + volumeMounts: + - mountPath: "/usr/share/nginx/html" + name: task-pv-storage + - name: task-pv-container2 + image: nginx + ports: + - containerPort: 80 + name: "http-server" + volumeMounts: + - mountPath: "/usr/share/nginx/html" + name: task-pv-storage diff --git a/21_k8s/manifests/pv-volume.yaml b/21_k8s/manifests/pv-volume.yaml new file mode 100644 index 00000000..458d4903 --- /dev/null +++ b/21_k8s/manifests/pv-volume.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: task-pv-volume + labels: + type: local +spec: + storageClassName: manual + capacity: + storage: 10Gi + accessModes: + - ReadWriteOnce + hostPath: + path: "/mnt/data" \ No newline at end of file diff --git a/21_k8s/manifests/redis-storage.yaml b/21_k8s/manifests/redis-storage.yaml new file mode 100644 index 00000000..70df53bc --- /dev/null +++ b/21_k8s/manifests/redis-storage.yaml @@ -0,0 +1,23 @@ +apiVersion: v1 +kind: Pod +metadata: + name: redis +spec: + initContainers: + - name: git-clone-repo + image: alpine/git:latest + command: [ 'sh', '-c', "git clone --single-branch --depth 1 --branch {{BRANCH}} {{git-server.host}}/repo.git /app" ] + volumeMounts: + - mountPath: /app + name: repo + containers: + - name: secured-image + image: secured-image:0.0.1 + volumeMounts: + - mountPath: /app + name: repo + volumes: + - name: repo + emptyDir: {} + + diff --git a/21_k8s/manifests/secret-as-env-var.yaml b/21_k8s/manifests/secret-as-env-var.yaml new file mode 100644 index 00000000..d99f77cb --- /dev/null +++ b/21_k8s/manifests/secret-as-env-var.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Pod +metadata: + name: env-single-secret +spec: + containers: + - name: envars-test-container + image: nginx + env: + - name: ENV_VAR1 + value: '1234' + - name: SECRET_USERNAME + valueFrom: + secretKeyRef: + name: backend-user + key: backend-username \ No newline at end of file diff --git a/21_k8s/manifests/secrets-demo.yaml b/21_k8s/manifests/secrets-demo.yaml new file mode 100644 index 00000000..aa7e050c --- /dev/null +++ b/21_k8s/manifests/secrets-demo.yaml @@ -0,0 +1,25 @@ +apiVersion: v1 +kind: Secret +metadata: + name: test-secret +data: + username: bXktYXBw + password: Mzk1MjgkdmRnN0pi +--- +apiVersion: v1 +kind: Pod +metadata: + name: secret-test-pod +spec: + containers: + - name: test-container + image: nginx + volumeMounts: + # name must match the volume name below + - name: secret-volume + mountPath: /etc/secret-volume + # The secret data is exposed to Containers in the Pod through a Volume. + volumes: + - name: secret-volume + secret: + secretName: test-secret diff --git a/21_k8s/manifests/socket-liveness-readiness.yaml b/21_k8s/manifests/socket-liveness-readiness.yaml new file mode 100644 index 00000000..5443fe46 --- /dev/null +++ b/21_k8s/manifests/socket-liveness-readiness.yaml @@ -0,0 +1,23 @@ +apiVersion: v1 +kind: Pod +metadata: + name: goproxy + labels: + app: goproxy +spec: + terminationGracePeriodSeconds: 60 + containers: + - name: goproxy + image: registry.k8s.io/goproxy:0.1 + ports: + - containerPort: 8080 + readinessProbe: + tcpSocket: + port: 8080 + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + tcpSocket: + port: 8080 + initialDelaySeconds: 15 + periodSeconds: 20 \ No newline at end of file diff --git a/21_k8s/mysql-configmap-secret-demo/mysql-config.yaml b/21_k8s/mysql-configmap-secret-demo/mysql-config.yaml new file mode 100644 index 00000000..eee6f91f --- /dev/null +++ b/21_k8s/mysql-configmap-secret-demo/mysql-config.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: mysql-config +data: + mysql-custom-config.cnf: | + [mysqld] + max_connections = 50 diff --git a/21_k8s/mysql-configmap-secret-demo/mysql-deployment.yaml b/21_k8s/mysql-configmap-secret-demo/mysql-deployment.yaml new file mode 100644 index 00000000..be4ac75a --- /dev/null +++ b/21_k8s/mysql-configmap-secret-demo/mysql-deployment.yaml @@ -0,0 +1,45 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mysql + labels: + app: mysql +spec: + replicas: 1 + selector: + matchLabels: + app: mysql + template: + metadata: + labels: + app: mysql + spec: + containers: + - name: mysql + image: mysql:5.7 + volumeMounts: + - name: config-volume + mountPath: /etc/mysql/mysql.conf.d + env: + - name: MYSQL_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: mysql-root-password + key: mysql-password + - name: MYSQL_DATABASE + value: 'videos' + volumes: + - name: config-volume + configMap: + name: mysql-config +--- +apiVersion: v1 +kind: Service +metadata: + name: mysql +spec: + selector: + app: mysql + ports: + - port: 3306 + targetPort: 3306 diff --git a/21_k8s/mysql-configmap-secret-demo/mysql-secret.yaml b/21_k8s/mysql-configmap-secret-demo/mysql-secret.yaml new file mode 100644 index 00000000..f50ba686 --- /dev/null +++ b/21_k8s/mysql-configmap-secret-demo/mysql-secret.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Secret +metadata: + name: mysql-root-password +type: Opaque +data: + mysql-password: bXlzcWwtZGItc2VjcmV0 # this is a base64 of `mysql-db-secret` \ No newline at end of file diff --git a/21_k8s/mysql-helm/values.yaml b/21_k8s/mysql-helm/values.yaml new file mode 100644 index 00000000..8bdfb0dd --- /dev/null +++ b/21_k8s/mysql-helm/values.yaml @@ -0,0 +1,13 @@ +primary: + startupProbe: + initialDelaySeconds: 600 + + service: + ports: + mysql: 3307 +secondary: + startupProbe: + initialDelaySeconds: 600 +architecture: standalone + + diff --git a/21_k8s/prometheus/values.yaml b/21_k8s/prometheus/values.yaml new file mode 100644 index 00000000..f341d9f1 --- /dev/null +++ b/21_k8s/prometheus/values.yaml @@ -0,0 +1,5 @@ +extraScrapeConfigs: | + - job_name: 'prometheus-cloudwatch-exporter' + static_configs: + - targets: + - '' \ No newline at end of file diff --git a/21_k8s/zero-downtime-deployment-demo/Dockerfile b/21_k8s/zero-downtime-deployment-demo/Dockerfile new file mode 100644 index 00000000..8f972944 --- /dev/null +++ b/21_k8s/zero-downtime-deployment-demo/Dockerfile @@ -0,0 +1,5 @@ +FROM python:3.8-slim-buster +WORKDIR /app +COPY . . +RUN pip install -r requirements.txt +CMD ["python3", "app.py"] \ No newline at end of file diff --git a/21_k8s/zero-downtime-deployment-demo/app.py b/21_k8s/zero-downtime-deployment-demo/app.py new file mode 100644 index 00000000..4bb4f574 --- /dev/null +++ b/21_k8s/zero-downtime-deployment-demo/app.py @@ -0,0 +1,34 @@ +import signal +import time +from flask import Flask +from loguru import logger + +app = Flask(__name__) + +terminated = False + + +@app.route('/', methods=['GET']) +def index(): + time.sleep(0.1) + return 'Hello world\n' + + +@app.route('/ready') +def status(): + if not terminated: + return 'OK', 200 + else: + return 'NotReady', 500 + + +def signal_handler(signum, frame): + global terminated + logger.info(f'Handling signal {signum}') + terminated = True + + +if __name__ == '__main__': + signal.signal(signal.SIGTERM, signal_handler) + signal.signal(signal.SIGINT, signal_handler) + app.run(debug=True, port=8080, host='0.0.0.0') diff --git a/21_k8s/zero-downtime-deployment-demo/deployment.yaml b/21_k8s/zero-downtime-deployment-demo/deployment.yaml new file mode 100644 index 00000000..1d409ef4 --- /dev/null +++ b/21_k8s/zero-downtime-deployment-demo/deployment.yaml @@ -0,0 +1,38 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: simaple-webserver + labels: + app: simaple-webserver +spec: + selector: + matchLabels: + app: simaple-webserver + replicas: 1 + template: + metadata: + labels: + app: simaple-webserver + spec: + terminationGracePeriodSeconds: 30 + containers: + - name: simaple-webserver + image: public.ecr.aws/r7m7o9d4/flask_webserver:1 + readinessProbe: + httpGet: + path: /ready + port: 8080 + initialDelaySeconds: 5 + periodSeconds: 3 +--- +apiVersion: v1 +kind: Service +metadata: + name: simaple-webserver-service +spec: + selector: + app: simaple-webserver + ports: + - protocol: TCP + port: 8080 + targetPort: 8080 diff --git a/21_k8s/zero-downtime-deployment-demo/requirements.txt b/21_k8s/zero-downtime-deployment-demo/requirements.txt new file mode 100644 index 00000000..344c2531 --- /dev/null +++ b/21_k8s/zero-downtime-deployment-demo/requirements.txt @@ -0,0 +1,2 @@ +Flask +loguru \ No newline at end of file diff --git a/22_elastic/apm-server.yaml b/22_elastic/apm-server.yaml new file mode 100644 index 00000000..4f2e2e0c --- /dev/null +++ b/22_elastic/apm-server.yaml @@ -0,0 +1,101 @@ +apmConfig: + apm-server.yml: | + apm-server: + host: "0.0.0.0:8200" + kibana: + username: elastic + password: elastic + enabled: true + host: "http://elastic-kibana:5601" + + queue: {} + output.elasticsearch: + hosts: ["https://elastic-elasticsearch:9200"] + username: "elastic" + password: "elastic" + ## If SSL is enabled + protocol: https + ssl.certificate_authorities: + - | + -----BEGIN CERTIFICATE----- + MIIDITCCAgmgAwIBAgIQAkRCji6h2CrcupbPYusu0TANBgkqhkiG9w0BAQsFADAb + MRkwFwYDVQQDExBlbGFzdGljc2VhcmNoLWNhMB4XDTIzMDExODEwNDA1NFoXDTI0 + MDExODEwNDA1NFowGzEZMBcGA1UEAxMQZWxhc3RpY3NlYXJjaC1jYTCCASIwDQYJ + KoZIhvcNAQEBBQADggEPADCCAQoCggEBAKzECfYQT5YdB5U17t8siEgTJo/l2ARg + RU0s2ZZySNJIWVXM5U5IHfQV8QlAToClyjTArXEwZs4Pxr7tfZFBtN8qIoUTcyH6 + uS4+H1DrHBHukNmKXeQMJR4bEsF2bCYotYyj0x9nDFKlpWRve7MRGUwAZ3MCbb7p + nYFPdoONoLh+fi9cNZULBMaweeAT8QB3nheZxZc5usBs/ePBdgk3Q7SUzsNNqc6e + tEOlr68qaRV2Gr+rmSRCeIFIRIm/7nr/AgNx3bA+Hmgrxt5N7Gpw/rJ3/krIop+9 + cRKbFRFL1O8i2tDGT+bjACYQCZAlJJGaG/YRIY8/HL3fO6n6gOjXUVsCAwEAAaNh + MF8wDgYDVR0PAQH/BAQDAgKkMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcD + AjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBSWpbQCprTjWJYl2504Mw3qQrCI + AzANBgkqhkiG9w0BAQsFAAOCAQEAVBhMn+7QQjj6zlXrEAq1Vx9LOeDxGz0bqD64 + ehxV3OkCzUdG4I3RRCrHaXVenVoWVz/Kctg7AA3t72WyHJYpzfQE//vSOMEEQG8Y + JDgGdMu38H+H6Fm2wLHpQZy3loO3yig+s1uT+TeACysVEBjSqrXb0wLYfKfr/2BD + 4QxRuo9Q1rdi52332GesWaKCw7E8PCgn/TpEnmY2v3GFuK/cLDkcmKaTlY0M3Kyw + XosAwfvyRxeMYtNK636FiYnu3seWZ00uyxMml3zgVbqm1GhpvT3kHBf8Pnof2Icq + ++s2nkCQw6oglEYcJ8TL636WFShC5L3WXE0Uj4lVsltip9N0vg== + -----END CERTIFICATE----- + - | + -----BEGIN CERTIFICATE----- + MIIDIjCCAgqgAwIBAgIRAMYdiJ53yTb1S3YlZK8UgGwwDQYJKoZIhvcNAQELBQAw + GzEZMBcGA1UEAxMQZWxhc3RpY3NlYXJjaC1jYTAeFw0yMzAxMTgxMjE3MThaFw0y + NDAxMTgxMjE3MThaMBsxGTAXBgNVBAMTEGVsYXN0aWNzZWFyY2gtY2EwggEiMA0G + CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDRw/8q+LxyC9X0I6bhhT5aXXzaH5yR + VnYG1ZZz1nui1ZwNtjulzG4z/kefAZcRzoZmiq+Y/Z35rwVIYvRmZ2CXSCyYkcdb + FoTgpA5RuYkimlOryHfCUCbt5yX4JDlkXXrvRIX1NMxkcq8YMAUzrzVQJDc43LTU + knw9vvcsEqUAz9vGkZ/x79DoYTxqhWA1QajKn5145uIu0RgItq3QTv/R8MY1QYjY + y0eo366ULSIQW7R+OPiwFNgXvpHriIz6TVJpdqrMHsKQi1hsGdNODEeGZRtJ716I + nbnnJWcQupES5+lYjV1fW78Z0kBU4NaS2drBEiydGTC5A0KffGT08HhNAgMBAAGj + YTBfMA4GA1UdDwEB/wQEAwICpDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUH + AwIwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUOcSnlStivqnIAsEY2XZl8E1D + GiAwDQYJKoZIhvcNAQELBQADggEBACoB4HOZv2hh6FNQCkl5Q+hkt2NiU9x43HAC + gX/fSHMnmUm1DE0X+bBaisl8KMMUA1G7cz2mqdDDOLIQeSVJVpHSMFzTMWK2Xrxe + bZFVGy9X/uOdaxTb5vINiwhDjUcD233C4GnUcu4Q0umWJs5hwkleeS8tYVmcGPD+ + SgC2OUTDaqwPOMRmpLJAS8PjZ9zXClL06+hG362vx/+1hhrS489wmR1Cki9pHhrW + kX1Cnj1DxS6Zq9EUar8LatD9Te25S8o5nb6FQOYRiEOp4TmEIrtbX7lejgZixatc + GIQZ+MT5P6+q4d1VgVHQI6IH89ygc1J8gGOi60QFHIzIUBrMXwk= + -----END CERTIFICATE----- + - | + -----BEGIN CERTIFICATE----- + MIIDIjCCAgqgAwIBAgIRAMYdiJ53yTb1S3YlZK8UgGwwDQYJKoZIhvcNAQELBQAw + GzEZMBcGA1UEAxMQZWxhc3RpY3NlYXJjaC1jYTAeFw0yMzAxMTgxMjE3MThaFw0y + NDAxMTgxMjE3MThaMBsxGTAXBgNVBAMTEGVsYXN0aWNzZWFyY2gtY2EwggEiMA0G + CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDRw/8q+LxyC9X0I6bhhT5aXXzaH5yR + VnYG1ZZz1nui1ZwNtjulzG4z/kefAZcRzoZmiq+Y/Z35rwVIYvRmZ2CXSCyYkcdb + FoTgpA5RuYkimlOryHfCUCbt5yX4JDlkXXrvRIX1NMxkcq8YMAUzrzVQJDc43LTU + knw9vvcsEqUAz9vGkZ/x79DoYTxqhWA1QajKn5145uIu0RgItq3QTv/R8MY1QYjY + y0eo366ULSIQW7R+OPiwFNgXvpHriIz6TVJpdqrMHsKQi1hsGdNODEeGZRtJ716I + nbnnJWcQupES5+lYjV1fW78Z0kBU4NaS2drBEiydGTC5A0KffGT08HhNAgMBAAGj + YTBfMA4GA1UdDwEB/wQEAwICpDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUH + AwIwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUOcSnlStivqnIAsEY2XZl8E1D + GiAwDQYJKoZIhvcNAQELBQADggEBACoB4HOZv2hh6FNQCkl5Q+hkt2NiU9x43HAC + gX/fSHMnmUm1DE0X+bBaisl8KMMUA1G7cz2mqdDDOLIQeSVJVpHSMFzTMWK2Xrxe + bZFVGy9X/uOdaxTb5vINiwhDjUcD233C4GnUcu4Q0umWJs5hwkleeS8tYVmcGPD+ + SgC2OUTDaqwPOMRmpLJAS8PjZ9zXClL06+hG362vx/+1hhrS489wmR1Cki9pHhrW + kX1Cnj1DxS6Zq9EUar8LatD9Te25S8o5nb6FQOYRiEOp4TmEIrtbX7lejgZixatc + GIQZ+MT5P6+q4d1VgVHQI6IH89ygc1J8gGOi60QFHIzIUBrMXwk= + -----END CERTIFICATE----- + - | + -----BEGIN CERTIFICATE----- + MIIDITCCAgmgAwIBAgIQSbtAxZF3gzCuau3K0f45aDANBgkqhkiG9w0BAQsFADAb + MRkwFwYDVQQDExBlbGFzdGljc2VhcmNoLWNhMB4XDTIzMDExODEyMzMzMVoXDTI0 + MDExODEyMzMzMVowGzEZMBcGA1UEAxMQZWxhc3RpY3NlYXJjaC1jYTCCASIwDQYJ + KoZIhvcNAQEBBQADggEPADCCAQoCggEBAKyapUZsdo8kRkIBQDayS/TL/VTHhWp9 + b2bllv7MKy3CIAos2vS3Jrn1HuDJ/Qz/vvM3WSMDqwz820nP/BZCMwqdUA+9Naxi + hxxAK1hHK/EcZnqc7unGul7qFkZ+hoXriKA5Y/ffaHsTvWz4AdMBI+X86kPpnE61 + FNa++TLFofphVOK2xckY3k+/68kAC4t0XI7L1Vj04rvbjRk7FLSDTC7O4qChMhkd + JKApN9KHTn5iQea10ztFCaEwIZUvBYMLH8d8VUmdAKZqJJgwM8/ID2JWB4hBAzoX + sQsTQTO4fK+Fwes/PvvVKzHLrbJsofodSkVMue09DIT9Dn6bb6y5uZcCAwEAAaNh + MF8wDgYDVR0PAQH/BAQDAgKkMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcD + AjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBT9zCS3vkpL0ZKLVEPKoqLSISzA + 3DANBgkqhkiG9w0BAQsFAAOCAQEApbRi8KTpLNQMuAz5uXl2z6oA9tHDbUISjm06 + V+r/uAxz0DAORx6M11+CZPLmhUVgs7pO4KONs4s3lmoEreMnDeRB7A5FV8Q5Zh4A + Jnq4c9zv0oby+UT4AkZF/jkJKzgnY38hK35Dz0y3QHcfzNiEuYiHAbX7Xf8PIY81 + O+mjU/IU86NbgYlgCzF/0Ky5ZQZIXevaoB5xlD03DDEnfemj3mumAMGq1i7pjKMm + wPAbrTazIpSoao6padt0mqCnUGjjNQMN2X0rdBtlmKLqUZTuN4wXlLgcTj3GmU6C + dFjPzUErATIRv+tpvs+pmvIwG3oRq/wjzmKHtOqbvhO1IvmSww== + -----END CERTIFICATE----- + + +extraEnvs: {} \ No newline at end of file diff --git a/22_elastic/elastic_tutorial.md b/22_elastic/elastic_tutorial.md new file mode 100644 index 00000000..791a5d93 --- /dev/null +++ b/22_elastic/elastic_tutorial.md @@ -0,0 +1,500 @@ +# The Elastic Stack (Elasticsearch, Logstash, Kibana) + +Let's get yourself familiar with the ELK stack: + +https://www.elastic.co/what-is/elk-stack + +## Elasticsearch on K8S + +**Note:** this chart should be released once per k8s cluster (the same server will be shared by all students). + +https://bitnami.com/stack/elasticsearch/helm + +Deploy with the following values: + +```yaml +coordinating: + replicaCount: 0 + +ingest: + enabled: false + +global: + kibanaEnabled: true +``` + +## Kibana setup + +Visit Kibana by port-forwarding the service: + +```shell +kubectl port-forward svc/ 5601:5601 +``` + +Then go to `https://localhost:5601`. + +Open the **Spaces** tooltip, and create your own namespace in which you will practice: + +![](../docs/img/elastic-spaces.png) + +### Add the sample data + +Sample data sets come with sample visualizations, dashboards, and more to help you explore before you ingest or add your own data. + +1. On the home page, click *Try sample data*. +2. Click *Other sample data sets*. +3. On the *Sample web logs* card, click *Add data*. + +## Kibana Query Language (KQL) + +Before we are experimenting with KQL, read the following [important concepts of Kibana](https://www.elastic.co/guide/en/kibana/current/kibana-concepts-analysts.html). + +Then, read [the KQL short tutorial](https://www.elastic.co/guide/en/kibana/current/kuery-query.html) from Elastic's official docs. + +### Try it yourself + +Open the **Kibana Sample Data Logs** data view under Discover page, and search for the following information: + +- Query all logs with response code 200 or 404 from the last 4 days. +- Query all successful requests from last day, referred from `twitter.com` +- All non-successful requests (both by client or server side) 2023-01-05T00:00:00Z (which means UTC time). +- Search all documents where the `request` field contains the string `elasticsearch-6.3.2.deb` within the last 7 days. +- According to a bad system design, your platform has to block users from download large files (files larger than `9216` bytes \[9KB\]) during every day for 30 minutes. The product manager asks your insights regarding the hour in the day that you should apply this limitation. What hour will you advise? + +### Filters + +- Create a [Filter](https://www.elastic.co/guide/en/kibana/current/kibana-concepts-analysts.html#autocomplete-suggestions) that displays data when `hour_of_day` value is between the working hours (`9-17`). +- What is the maximum requested resource (highest `bytes` request) within the last 7 days, during working hours? Was it responded with `200` status code? + +## Kibana Dashboards + +### Try it yourself - create a new dashboard + +- Open the main menu, then click **Dashboard**. + +- Click **Create dashboard**. + +- Set the time filter to **Last 7 days**. + +- On the dashboard, click **Create visualization**. + +### Panel I: Unique visitors + +- Open the **Visualization type** dropdown, then select **Metric**. + +- From the **Available fields** list, drag **clientip** to the workspace or layer pane. + +In the layer pane, **Unique count of clientip** appears because the editor automatically applies the **Unique count** function to the **clientip** field (**Unique count** is the only numeric function that works with IP addresses). + +- In the layer pane, click **Unique count of clientip**. + +- In the **Name** field, enter `Unique visitors`. + +- Click **Close**. + +- Click **Save** to save the panel. + +### Panel II: Outbound traffic over time + +To visualize the **bytes** field over time: + +- On the dashboard, click **Create visualization**. + +- From the **Available fields** list, drag **bytes** to the workspace. + +The visualization editor creates a bar chart with the **timestamp** and **Median of bytes** fields. + +- To emphasize the change in **Median of bytes** over time, change the visualization type to **Line**. + +- The default minimum time interval is 3 hour, but we would like to get a view over days. To increase the minimum time interval: + + - In the layer pane, click **timestamp**. + + - Change the **Minimum interval** to **1d**, then click **Close**. + +- Click **Save and return** + +### Panel III: Top requested pages + +We will create a visualization that displays the most frequent values of **request.keyword** on your website, ranked by the unique visitors. + +- On the dashboard, click **Create visualization**. + +- From the **Available fields** list, drag **clientip** to the **Vertical axis** field in the layer pane. + +The visualization editor automatically applies the **Unique count** function. + +- Drag **request.keyword** to the workspace. + +Note: The chart labels are unable to display because the **request.keyword** field contains long text fields + +- Open the **Visualization type** dropdown, then select **Table**. + +- In the layer pane, click **Top 5 values of request.keyword**. + +- In the **Number of values** field, enter `10`. + +- In the **Name** field, enter `Page URL`. + +- Click **Close**. + +- Click **Save and return**. + +### Panel IV: Classify request size + +Create a proportional visualization that helps you determine if your users transfer more bytes from requests under 10KB versus over 10Kb. + +- On the dashboard, click **Create visualization**. + +- From the **Available fields** list, drag **bytes** to the **Vertical axis** field in the layer pane. + +- In the layer pane, click **Median of bytes**. + +- Click the **Sum** quick function, then click **Close**. + +- From the **Available fields** list, drag **bytes** to the **Breakdown by** field in the layer pane. + +- In the **Breakdown** layer pane, click **bytes**. + +- Click **Create custom ranges**, enter the following in the **Ranges** field, then press Return: + + - **Ranges** — `0` -> `10240` + + - **Label** — `Below 10KB` + +- Click **Add range**, enter the following, then press Return: + + - **Ranges** — `10240` -> `+∞` + + - **Label** — `Above 10KB` + +- From the **Value format** dropdown, select **Bytes (1024)**, then click **Close**. + +To display the values as a percentage of the sum of all values, use the **Pie** chart. + +- Open the **Visualization Type** dropdown, then select **Pie**. + +- Click **Save and return**. + + + +### Panel V: Distribution of requests along the day + +Create the following visualization: + +![](../docs/img/kibana-dash-1.png) + + +### Panel VII: Website traffic sources + +- On the dashboard, click **Create visualization**. + +- Open the **Visualization type** dropdown, then select **Treemap**. + +- From the **Available fields** list, drag **Records** to the **Size by** field in the layer pane. + +- In the layer pane, click **Add or drag-and-drop a field** for **Group by**. + +Create a filter for each website traffic source: + +- Click **Filters**. + +- Click **All records**, enter the following in the query bar, then press Return: + + - **KQL** - `referer : *facebook.com*` + + - **Label** - `Facebook` + +- Click **Add a filter**, enter the following in the query bar, then press Return: + + - **KQL** - `referer : *twitter.com*` + + - **Label** - `Twitter` + +- Click **Add a filter**, enter the following in the query bar, then press Return: + + - **KQL** - `NOT referer : *twitter.com* OR NOT referer: *facebook.com*` + + - **Label** - `Other` + +- Click **Close**. + +- Click **Save and return**. + +### Panel VI: SLA (Service-level agreement) + +Assume Facebook and Twitter are your two major customers, and your company agreed to serve 99% of the incoming requests originating from Facebook or Twitter. + +Create a visualization which calculates the SLA per client over a single day. The SLA is defined by the following formula: + +```text +1 - [(# of failed requests)/(# of total requests)] +``` + +Failed requests are those with status code `>= 500`. + +![](../docs/img/kibana-dash-2.png) + +Tip - use thew following custom formula: + +```text +1 - (count(kql='response.keyword >= 500') / count(kql='response.keyword: *')) +``` + +## Kibana Alerting System + +### Update the Helm chart to enable alerting + +**Should be applied only once per cluster!!!** + +Review and discuss the Helm values file under `22_elastic/elasticsearch-values.yaml`. + +From now on, you should connect to Kibana either using `elastic` username (the password is stored as s secret in the cluster), or using your own created username. + +#### Alerts [Concepts and terminology](https://www.elastic.co/guide/en/kibana/current/alerting-getting-started.html#_concepts_and_terminology) + +A **rule** specifies a background task that runs on the Kibana server to check for specific conditions. + +A rule consists of three main parts: + + - Conditions: what needs to be detected? + - Schedule: when/how often should detection checks run? + - Actions: what happens when a condition is detected? + +![](../docs/img/kibana-alerting.png) + +When checking for a condition, a rule might identify multiple occurrences of the condition. Kibana tracks each of these **alerts** separately and takes an **action** per alert. + +**Connectors** provide a central place to store connection information for services and integrations. + +1. Anytime a rule’s conditions are met, an alert is created. This example checks for servers with average CPU > 0.9. Three servers meet the condition, so three alerts are created. +2. Alerts create actions as long as they are not muted or throttled. When actions are created, the template that was setup in the rule is filled with actual values. In this example, three actions are created, and the template string {{server}} is replaced with the server name for each alert. +3. Kibana invokes the actions, sending them to a third party integration like an email service. +4. If the third party integration has connection parameters or credentials, Kibana will fetch these from the connector referenced in the action. + +### Create alert + +We will simulate an alert rule that will be triggered according to a custom log. + +The use case: **account deletion**. + +Account deletion is a very sensitive and important operation that your system should perform well. If your system fails to delete an account whenever a given user is asking to do so, it has legal and security implications. + +Let's say that you want to be notified whenever your application fails to delete an accounts properly. + +#### Simulate logs data + +First we need to feed Elasticsearch db with some data to simulate the desired event. + +1. In the Kibana main menu, go to **Management** -> **Dev Tools**. The Dev Tools allows you to execute queries directly again the Elasticsearch database. +2. Create your own index and insert data into it: +```json +POST //_doc +{ + "@timestamp": "2099-01-15T12:49:07.000Z", + "message": "account deletion failed", + "event": { + "dataset": "" + } +} +``` +Change `` to your index name. Change the timestamp according to your needs - at the end, the alert rule will search for logs that have been written recently. + +3. If you want to make sure your data has been successfully written, query all the records in your index: +```json +GET //_search +``` + +4. Define your index as a data view. From the Kibana main menu, under **Management** -> **Stack Management**, choose **Data view** from the sub-menu and create a new data view. + +#### Add your data to Observability Logs + +Observability Logs in Kibana enables you to search, filter, and tail all your logs ingested into Elasticsearch. There is live streaming of logs functionality, filtering using auto-complete, and a logs histogram for quick navigation. + +We will build the alert rule based on logs data that is being fed into Observability Logs. + +5. In the Kibana main menu, under **Observability** -> **Logs**, click **Settings** in the top-right bar. +6. In the settings page, add your index to the data feed. +7. Save and make sure that your data appears in the feed. + +#### Create the alert rule + +8. Under **Observability** -> **Alert**, choose **Manage Rules**, and then **Create Rule**. +9. Follow the rule definition form, and create a rule of type **Log threshold** that will trigger alert when the message `account deletion failed` is being logged into your index. +10. Under Actions, choose either Index or Server log (will be discussed in class). +11. Test the rule - trigger the alert. + + +## Elastic APM + +https://www.elastic.co/guide/en/apm/get-started/7.15/components.html + +https://www.elastic.co/guide/en/apm/agent/python/current/flask-support.html + + +### Backup and restore + +https://www.elastic.co/guide/en/elasticsearch/reference/current/snapshot-restore.html + +## Elasticsearch Database + +Elastic architecture in a nutshell + +![](../docs/img/elastic-arch.png) + +### Cluster Nodes + +Any time that you start an instance of Elasticsearch, you are starting a node. A collection of connected nodes is called a cluster. +Every node in the cluster can handle HTTP and transport traffic by default. The transport layer is used exclusively for communication between nodes; the HTTP layer is used by REST clients. + +All nodes know about all the other nodes in the cluster and can forward client requests to the appropriate node. + +#### Node types + +- **Master node**: A node that has the master role, which makes it eligible to be elected as the master node, which controls the cluster. +- **Data node**: Data nodes hold data and perform data related operations such as CRUD, search, and aggregations. +- **Ingest node**: Ingest nodes are able to apply an ingest pipeline to a document in order to transform and enrich the document before indexing. +- **Coordinating node**: Requests like search requests or bulk-indexing requests may involve data held on different data nodes. A search request, for example, is executed in two phases which are coordinated by the node which receives the client request— the coordinating node. + +The [official Elastic docs](https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-node.html) contains a detailed list of all node types. + +### Query DSL + +#### Add data + +You add data to Elasticsearch as JSON objects called documents. Elasticsearch stores these documents in searchable indices. + +Submit the following indexing request to add a single log entry to the `logs-my_app-default` index: + +```json lines +POST logs-my_app-default/_doc +{ + "@timestamp": "2099-05-06T16:21:15.000Z", + "event": { + "original": "192.0.2.42 - - [06/May/2099:16:21:15 +0000] \"GET /images/bg.jpg HTTP/1.0\" 200 24736" + } +} +``` +Use the `_bulk` endpoint to add multiple documents in one request. + +```json lines +PUT logs-my_app-default/_bulk +{ "create": { } } +{ "@timestamp": "2099-05-07T16:24:32.000Z", "event": { "original": "192.0.2.242 - - [07/May/2020:16:24:32 -0500] \"GET /images/hm_nbg.jpg HTTP/1.0\" 304 0" } } +{ "create": { } } +{ "@timestamp": "2099-05-08T16:25:42.000Z", "event": { "original": "192.0.2.255 - - [08/May/2099:16:25:42 +0000] \"GET /favicon.ico HTTP/1.0\" 200 3638" } } +``` + +#### Search data + +Indexed documents are available for search in near real-time. +The following search matches all log entries in `logs-my_app-default` and sorts them by `@timestamp` in descending order. + +```json lines +GET logs-my_app-default/_search +{ + "query": { + "match_all": { } + }, + "sort": [ + { + "@timestamp": "desc" + } + ] +} +``` + +Parsing the entire `_source` is unwanted for large documents. To exclude it from the response, set the `_source` parameter to `false`. +Instead, use the `fields` parameter to retrieve the fields you want. + +```json lines +GET logs-my_app-default/_search +{ + "query": { + "match_all": { } + }, + "fields": [ + "@timestamp" + ], + "_source": false, + "sort": [ + { + "@timestamp": "desc" + } + ] +} +``` + +#### Search a date range + +To search across a specific time or IP range, use a `range` query. + +```json lines +GET logs-my_app-default/_search +{ + "query": { + "range": { + "@timestamp": { + "gte": "2099-05-05", + "lt": "2099-05-08" + } + } + }, + "fields": [ + "@timestamp" + ], + "_source": false, + "sort": [ + { + "@timestamp": "desc" + } + ] +} +``` + +You can use date math to define relative time ranges: + +```text +- "gte": "2099-05-05", ++ "gte": "now-1d/d", +- "lt": "2099-05-08" ++ "lt": "now/d" +``` + +#### Match query + +https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-match-query.html + + +#### Boolean query + +https://www.elastic.co/guide/en/elasticsearch/reference/7.17/query-dsl-bool-query.html#query-dsl-bool-query + +#### Aggregations + +https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations.html#search-aggregations + +### Try it yourself + +Perform the following queries against the `kibana_sample_data_logs` index: + +- Use **match** query to search for the word "twitter" contained in the field called `message` in the documents. +- Use **match** query to search for the word "twitter" **or** "facebook" contained in the field called `message` in the documents. +- Use **match** query to search for the word "twitter" **and** "facebook" contained in the field called `message` in the documents. +- Use **bool - must** query to search for the documents that contains the word "beats" in `message`, and `bytes` >= 1024. +- Repeat the above query, but now replace `must` with `filter`. Notice the `_score` value. +- Use **bool - must** query to search for the documents that contains the word "beats" in `message`, and `bytes` >= 1024. In addition, add to the bool query the **should** entry to include documents with `referer` that contains `twitter.com` + + + +## Elastic Beats Family + +[Beats](https://www.elastic.co/guide/en/beats/libbeat/current/beats-reference.html) are open source data shippers that you install as agents on your servers to send operational data to Elasticsearch. + +Install Metricbeat using the official chart: https://github.com/elastic/helm-charts/blob/main/metricbeat/README.md +**Note**: You should follow the [TLS example](https://github.com/elastic/helm-charts/blob/main/metricbeat/README.md#how-to-use-metricbeat-with-elasticsearch-with-security-authentication-and-tls-enabled), as the provisioned Elasticsearch cluster uses TLS. + +Metricbeat comes packaged with example Kibana dashboards, visualizations, and searches for visualizing Metricbeat data in Kibana. + +Let's load some [pre-built dashboards](https://www.elastic.co/guide/en/beats/metricbeat/current/load-kibana-dashboards.html#load-dashboards). + diff --git a/22_elastic/elasticsearch-values.yaml b/22_elastic/elasticsearch-values.yaml new file mode 100644 index 00000000..1617fddc --- /dev/null +++ b/22_elastic/elasticsearch-values.yaml @@ -0,0 +1,38 @@ +coordinating: + replicaCount: 1 + +data: + replicaCount: 1 + +master: + replicaCount: 2 + +ingest: + enabled: true + +global: + kibanaEnabled: true + +security: + elasticPassword: elastic + enabled: true + tls: + autoGenerated: true + +kibana: + elasticsearch: + hosts: + - elastic-elasticsearch + port: 9200 + security: + auth: + enabled: true + createSystemUser: true + kibanaPassword: secretPass + elasticsearchPasswordSecret: elastic-elasticsearch + tls: + enabled: true + existingSecret: elastic-elasticsearch-coordinating-crt + usePemCerts: true + extraConfiguration: + "xpack.encryptedSavedObjects.encryptionKey": "23b2676a-fd52-478f-99e1-bfb09250438a" \ No newline at end of file diff --git a/22_elastic/flask-apm-agent.py b/22_elastic/flask-apm-agent.py new file mode 100644 index 00000000..695f33f4 --- /dev/null +++ b/22_elastic/flask-apm-agent.py @@ -0,0 +1,23 @@ +from flask import Flask, send_file, request +from elasticapm.contrib.flask import ElasticAPM + +app = Flask(__name__) + +app.config['ELASTIC_APM'] = { + 'SERVICE_NAME': 'alonit-flask', + 'SECRET_TOKEN': '', + 'SERVER_URL': 'http://localhost:8200', + 'ENVIRONMENT': 'production', +} + +apm = ElasticAPM(app) + + +@app.route('/', methods=['GET']) +def index(): + # print(request.headers) + return 'Hello world\n' + + +if __name__ == '__main__': + app.run(debug=True, port=8080, host='0.0.0.0') diff --git a/22_elastic/metricbeat-values.yaml b/22_elastic/metricbeat-values.yaml new file mode 100644 index 00000000..855f00b4 --- /dev/null +++ b/22_elastic/metricbeat-values.yaml @@ -0,0 +1,111 @@ +daemonset: + extraEnvs: + - name: "ELASTICSEARCH_USERNAME" + value: elastic + - name: "ELASTICSEARCH_PASSWORD" + valueFrom: + secretKeyRef: + name: elastic-elasticsearch + key: elasticsearch-password + metricbeatConfig: + metricbeat.yml: | + metricbeat.modules: + - module: kubernetes + metricsets: + - container + - node + - pod + - system + - volume + period: 10s + host: "${NODE_NAME}" + hosts: ["https://${NODE_NAME}:10250"] + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + ssl.verification_mode: "none" + # If using Red Hat OpenShift remove ssl.verification_mode entry and + # uncomment these settings: + #ssl.certificate_authorities: + #- /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt + processors: + - add_kubernetes_metadata: ~ + - module: kubernetes + enabled: true + metricsets: + - event + - module: system + period: 10s + metricsets: + - cpu + - load + - memory + - network + - process + - process_summary + processes: ['.*'] + process.include_top_n: + by_cpu: 5 + by_memory: 5 + - module: system + period: 1m + metricsets: + - filesystem + - fsstat + processors: + - drop_event.when.regexp: + system.filesystem.mount_point: '^/(sys|cgroup|proc|dev|etc|host|lib)($|/)' + output.elasticsearch: + username: '${ELASTICSEARCH_USERNAME}' + password: '${ELASTICSEARCH_PASSWORD}' + protocol: https + hosts: ["elastic-elasticsearch:9200"] + ssl.certificate_authorities: + - /usr/share/metricbeat/config/certs/ca.crt + secretMounts: + - name: elastic-certificate-pem + secretName: elastic-elasticsearch-coordinating-crt + path: /usr/share/metricbeat/config/certs + resources: + limits: + # Should avoid OOM (Error 137) when running goss tests into the pod + memory: "300Mi" + +deployment: + extraEnvs: + - name: "ELASTICSEARCH_USERNAME" + value: elastic + - name: "ELASTICSEARCH_PASSWORD" + valueFrom: + secretKeyRef: + name: elastic-elasticsearch + key: elasticsearch-password + metricbeatConfig: + metricbeat.yml: | + setup: + kibana: + host: "elastic-kibana:5601" + metricbeat.modules: + - module: kubernetes + enabled: true + metricsets: + - state_node + - state_deployment + - state_replicaset + - state_pod + - state_container + period: 10s + hosts: ["${KUBE_STATE_METRICS_HOSTS}"] + output.elasticsearch: + username: '${ELASTICSEARCH_USERNAME}' + password: '${ELASTICSEARCH_PASSWORD}' + protocol: https + hosts: ["elastic-elasticsearch:9200"] + ssl.certificate_authorities: + - /usr/share/metricbeat/config/certs/ca.crt + secretMounts: + - name: elastic-certificate-pem + secretName: elastic-elasticsearch-coordinating-crt + path: /usr/share/metricbeat/config/certs + resources: + limits: + # Should avoid OOM (Error 137) when running goss tests into the pod + memory: "300Mi" \ No newline at end of file diff --git a/23_argoCD/argo_getting_started.md b/23_argoCD/argo_getting_started.md new file mode 100644 index 00000000..940a1474 --- /dev/null +++ b/23_argoCD/argo_getting_started.md @@ -0,0 +1,18 @@ +# ArgoCD - Continueous deployment system + +## Deployment strategies +- [Overview](https://blog.christianposta.com/deploy/blue-green-deployments-a-b-testing-and-canary-releases/) +- [Blue/Green deployment](https://www.redhat.com/en/topics/devops/what-is-blue-green-deployment) + +## Argo - getting started + +We will introduce ArgoCD by experimenting with the [getting started](https://argo-cd.readthedocs.io/en/stable/getting_started/) guide from their official docs. + +### Further reading + +[Kubernetes Garbage Collection](https://kubernetes.io/docs/concepts/architecture/garbage-collection/#controlling-how-the-garbage-collector-deletes-dependents) + +## Blue-Green deployment using ArgoRollout + +Follow the guide in: +https://github.com/argoproj/argocd-example-apps/tree/master/blue-green \ No newline at end of file diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 00000000..c4d0498d --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,21 @@ +pipeline { + agent any + + stages { + stage('Build') { + steps { + sh 'echo building...' + } + } + stage('Stage II') { + steps { + sh 'echo "stage II..."' + } + } + stage('Stage III ...') { + steps { + sh 'echo echo "stage III..."' + } + } + } +} \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 00000000..ad93e5bf --- /dev/null +++ b/README.md @@ -0,0 +1,6 @@ +🚨 Attention Students: 🚨 + +This repository is now deprecated. +Please visit our new repository for the most up-to-date materials and information: https://github.com/exit-zero-academy/DevOpsTheHardWay.git. + +Thank you! \ No newline at end of file diff --git a/account_management.py b/account_management.py new file mode 100644 index 00000000..8d573b6f --- /dev/null +++ b/account_management.py @@ -0,0 +1,187 @@ +from datetime import datetime, timedelta, timezone +import re +import sys, traceback +import subprocess + +subprocess.call('pip install boto3>=1.24,<2.0 --target /tmp/ --no-cache-dir'.split(), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) +sys.path.insert(0, '/tmp/') + +import boto3 + +UNATTACHED_EBS_LIFESPAN = 7 +LOAD_BALANCER_LIFESPAN = 30 +STOPPED_EC2_LIFESPAN = 30 +DB_LIFESPAN = 30 + +# UNATTACHED_EBS_LIFESPAN = ${UnattachedVolumesLifespan} +# LOAD_BALANCER_LIFESPAN = ${LoadBalancerLifespan} +# STOPPED_EC2_LIFESPAN = ${StoppedEC2InstanceLifespan} +# DB_LIFESPAN = ${DatabaseRDSLifespan} + + +def delete_non_attached_old_ebs(region): + ec2 = boto3.client('ec2', region_name=region['RegionName']) + + unattached_vols = ec2.describe_volumes( + Filters=[ + { + 'Name': 'status', + 'Values': ['available'] + } + ] + ) + + cloud_trail = boto3.client('cloudtrail', region_name=region['RegionName']) + + volume_detachments = cloud_trail.lookup_events( + LookupAttributes=[ + { + 'AttributeKey': 'EventName', + 'AttributeValue': 'DetachVolume' + } + ], + StartTime=datetime.now(timezone.utc) - timedelta(days=UNATTACHED_EBS_LIFESPAN) + ) + + detached_vol_ids = [] + for event in volume_detachments['Events']: + for resource in event.get('Resources', []): + if 'volume' in resource['ResourceType'].lower(): + detached_vol_ids.append(resource['ResourceName']) + + for volume in unattached_vols['Volumes']: + if volume['VolumeId'] not in detached_vol_ids: + print(f'Deleting old unattached EBS: {volume}') + response = ec2.delete_volume(VolumeId=volume['VolumeId']) + + +def delete_old_load_balancers(region): + elb = boto3.client('elbv2', region_name=region['RegionName']) + lbs = elb.describe_load_balancers() + for lb in lbs['LoadBalancers']: + lb_lifetime = datetime.now(timezone.utc) - lb['CreatedTime'] + lb_lifespan = timedelta(days=LOAD_BALANCER_LIFESPAN) + tags = elb.describe_tags(ResourceArns=[lb['LoadBalancerArn']]) + for tag in tags['TagDescriptions']: + for t in tag.get('Tags', []): + if t['Key'].lower().strip() == 'lifespan': + try: + lb_lifespan = timedelta(days=max(int(t['Value']), LOAD_BALANCER_LIFESPAN)) + except: + pass + + if lb_lifetime > lb_lifespan: + print(f'Deleting old LB: {lb}') + response = elb.delete_load_balancer(LoadBalancerArn=lb['LoadBalancerArn']) + + +def stop_and_terminate_ec2(region): + ec2 = boto3.client('ec2', region_name=region['RegionName']) + + instances_to_stop = [] + instances_to_terminate = [] + next_token = '' + while next_token is not None: + descriptions = ec2.describe_instances( + # Filters=[ + # { + # 'Name': 'instance-lifecycle', + # 'Values': ['scheduled'] # TODO exclude stop instances + # } + # ], + NextToken=next_token + ) + + for description in descriptions['Reservations']: + for instance in description['Instances']: + ec2_lifespan = timedelta(days=STOPPED_EC2_LIFESPAN) + + # valid state values: 'pending'|'running'|'shutting-down'|'terminated'|'stopping'|'stopped' + if instance['State']['Name'] in ['running']: + + res = ec2.modify_instance_attribute( + InstanceId=instance['InstanceId'], + DisableApiStop={'Value': False} + ) + instances_to_stop.append(instance['InstanceId']) + + elif instance['State']['Name'] in ['stopped']: + stopped_reason = instance['StateTransitionReason'] + stopped_time = re.findall('.*\((.*)\)', stopped_reason) + if stopped_time: + stopped_time = datetime.strptime(stopped_time[0].split(' ')[0], '%Y-%m-%d') + + for tag in instance.get('Tags', []): + if tag['Key'].lower().strip() == 'lifespan': + try: + ec2_lifespan = timedelta(days=max(int(tag['Value']), STOPPED_EC2_LIFESPAN)) + except: + pass + + if datetime.now() - stopped_time > ec2_lifespan: + res = ec2.modify_instance_attribute( + InstanceId=instance['InstanceId'], + DisableApiTermination={'Value': False} + ) + instances_to_terminate.append(instance['InstanceId']) + + next_token = descriptions.get('NextToken') + + if instances_to_stop: + print(f'Instances to stop: {instances_to_stop}') + response = ec2.stop_instances(InstanceIds=instances_to_stop) + + if instances_to_terminate: + print(f'Instances to terminate: {instances_to_terminate}') + # response = ec2.terminate_instances(InstanceIds=instances_to_terminate) + + +def delete_old_rds_db_instances(region): + rds = boto3.client('rds', region_name=region['RegionName']) + dbs = rds.describe_db_instances() + + for db in dbs['DBInstances']: + db_lifespan = timedelta(days=DB_LIFESPAN) + + tags = rds.list_tags_for_resource( + ResourceName=db['DBInstanceArn'] + ) + for tag in tags.get('TagList', []): + if tag['Key'].lower().strip() == 'lifespan': + try: + db_lifespan = timedelta(days=max(int(tag['Value']), DB_LIFESPAN)) + except: + pass + + if db['DBInstanceStatus'] != 'deleting' and datetime.now(timezone.utc) - db['InstanceCreateTime'] > db_lifespan: + print(f'Deleting RDS DB: {db}') + response = rds.delete_db_instance( + DBInstanceIdentifier=db['DBInstanceIdentifier'], + SkipFinalSnapshot=True + ) + + +def handler(event, context): + + client = boto3.client('ec2') + regions = client.describe_regions() + + for region in regions['Regions']: + + if not (region['RegionName'].startswith('eu') or region['RegionName'].startswith('us')): + continue + + print(f'Working on region {region["RegionName"]}') + + try: + stop_and_terminate_ec2(region) + delete_non_attached_old_ebs(region) + delete_old_load_balancers(region) + delete_old_rds_db_instances(region) + except: + print(f'Error occured during maintenance job') + traceback.print_exc(file=sys.stdout) + + + +# handler(None, None) diff --git a/docs/aws_ex1.md b/docs/aws_ex1.md new file mode 100644 index 00000000..14a3dc87 --- /dev/null +++ b/docs/aws_ex1.md @@ -0,0 +1,55 @@ +# AWS Ex1 - Extend the YouTube Telegram Bot + +**Can be done in pairs (highly recommended)**. + +## Background + +Your goal is to provision the YouTube Telegram Bot in a scalable and durable architecture in AWS. + +In the [The PolyBot](https://github.com/alonitac/PolyBot) repo, review `microservices` branch. This branch contains the following services: + +1. `bot.py` - the Telegram bot you've implemented in the previous exercise. But this time, the bot doesn't download the videos itself, but sends the "job" to an SQS queue. +2. `worker.py` - the worker service continuously reads messages from the queue and process them, which means download the video from YouTube and store it in a dedicated bucket in S3. + +Here is a high level of the bot-workers microservices architecture: + +![](img/botAws.png) + +As you can see, the Telegram messages are served by the Bot service. All it does is sending a message to an SQS queue. Thus, the Bot is a very lightweight service that can serve requests very quickly. In the other side, there are Workers that consume messages from the queue and do the hard work - to download the video from youtube and upload it to S3. The workers are part of an AutoScaling group. The group is scaled in and out by a custom metric that the Bot service writes to CloudWatch. You can call the metric `BacklogPerInstance` as it represents the number of messages in the queue (messages that was not consumed yet) per instance. For example, assume you have 5 workers up and running, and 100 messages in the queue, thus `BacklogPerInstance` equals 20, since each worker instance has to consume ~20 messages to get the queue empty. For more information, [read here](https://docs.aws.amazon.com/autoscaling/ec2/userguide/as-using-sqs-queue.html). + +## Guidelines + +1. Create a VPC with at least 2 public subnets. + +2. Create an S3 bucket which will store the uploaded youtube videos + +3. Create an SQS standard queue. Messages that was not processed yet should reside in the queue for a maximum period of 4 days. The worker has a maximum period of 30 minutes to process a single message. + +4. Create a Launch Template and an AutoScaling Group. Keep the default configurations, we will change it later. **The Minimum and Desired capacity of the ASG should be zero**. + +5. You are given most of the code for the bot and worker services. In branch `microservices` complete the following *TODO*s: + 1. In `worker.py` complete the implementation of `process_msg()` function such that the downloaded videos will be uploaded to S3 (you can delete them from the disk afterwards). + 2. In `utils.py` complete `calc_backlog_per_instance()` such that the value of variable `backlog_per_instance` will be sent as a metric to [CloudWatch](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/cw-example-metrics.html#publish-custom-metrics). + 3. Change `config.json` according to your resources in AWS. + + Except the above two changes, you don't need to change the code (unless you want to add more functionality to the service). + +6. After you've implemented the code changes, it is good idea to test everything locally. Run the `bot.py` service and a single worker `worker.py`. Make sure that when you send a message via Telegram, the Bot service produces a message to the SQS queue, and the Worker consumes the message, downloads the YouTube video and uploads it to S3. + +7. Deploy the Worker service to an EC2 instance (you can run it as a container the same way you did in the last exercise). Create an AMI from that instance and base your Launch Template on that AMI, such that when a new instance is created from the launch template, the worker app in the VM will be up and running automatically. + +8. Deploy the Bot service on a single EC2 instance (this service is not part of the autoscaling group). It should be very similar to the deployment you've done in the previous exercise - in a docker container that restarts automatically on OS reboot. + +9. Use AWS cli to create a [target tracking scaling policy](https://docs.aws.amazon.com/autoscaling/ec2/userguide/as-using-sqs-queue.html#create-sqs-policies-cli) in your Autoscaling Group. `MetricName` and `Namespace` should correspond to the metric your Bot service is firing to CloudWatch. Give the `TargetValue` some value that you can test later (e.g. 10, which means if there are more than 10 messages per worker in the SQS queue, a scale up event wil trigger). + +10. Make sure your services are given restrictive IAM role permissions. + +11. Test your application under load. + +## Submission + +Present you work in a personal meeting (usually 20 minutes before our bi-weekly class). **Final due date is 31/08/22**. + +# Good Luck + +Don't hesitate to ask any questions \ No newline at end of file diff --git a/docs/docker_ex1.md b/docs/docker_ex1.md new file mode 100644 index 00000000..a277e9cf --- /dev/null +++ b/docs/docker_ex1.md @@ -0,0 +1,95 @@ +# Docker network performance analysis + +# Background + +Docker's networking subsystem is pluggable, using drivers. Several drivers +exist by default, and provide core networking functionality: + +- `bridge`: The default network driver. If you don't specify a driver, this is + the type of network you are creating. **Bridge networks are usually used when + your applications run in standalone containers that need to communicate.** See + [bridge networks](https://docs.docker.com/network/bridge/). + +- `host`: For standalone containers, remove network isolation between the + container and the Docker host, and use the host's networking directly. See + [use the host network](https://docs.docker.com/network/host/). + +Your goal in this exercise is to measure the network performance of each network driver, and compare results. + +## Preliminaries + +1. Checkout a new Git branch `docker_ex1/` (change `` to your name). At the end, commit and push your results in this branch. +2. Create an **Ubuntu AMI** ec2 instance. Connect to your instance. +3. [Install docker](https://docs.docker.com/engine/install/ubuntu/). +4. Install [qperf](https://linux.die.net/man/1/qperf) networking tool: + ```shell + sudo apt-get update -y + sudo apt-get install qperf -y + ``` + +## Test network bandwidth and latency + +We will utilize qperf tool to measure bandwidth and latency. The test is fairly simple. You start +a server by `qperf` and invoke a client which will talk with the server and record the results: `qperf tcp_bw tcp_lat` + +### Test Server and Client on the same physical machine. + +#### Test performance without Docker + +4. Start the qperf server by simply: +```shell +qperf -lp 4000 +```` +5. Open another ssh session to the same instance where the server is running, perform: +```shell +qperf 127.0.0.1 -lp 4000 -ip 4001 tcp_bw tcp_lat conf +``` +This command will perform a single network test, and will print results to stdout. + +6. Use the following bash script template to perform the test multiple times and average results. + ```shell + AVG=0 + ITERATIONS=20 + for i in $( seq 0 $ITERATIONS ); do + CURRENT_TEST_RES=$(qperf 127.0.0.1 -lp 4000 -ip 4001 tcp_lat | tail -n 1 | awk '{ print $3 }') + AVG=$(awk "BEGIN{ print $AVG + $CURRENT_TEST_RES }") + done + awk "BEGIN{ print $AVG / $ITERATIONS }" + ``` + The above script perform only the latency test. Change `tcp_lat` (latency test) to `tcp_bw` (bandwidth) to perform the bandwidth test. + + +#### Test performance with Docker Bridge network + +Now the Server will reside in a docker container using the **default Bridge network**, while the client will stay on the host machine. + +1. [pedroperezmsft/qperf](https://hub.docker.com/r/pedroperezmsft/qperf/) is pre-built docker image with qperf tool installed. Run it by: + ```shell + docker run --rm --name qp_server -p 4000:4000 -p 4001:4001 pedroperezmsft/qperf -lp 4000 + ``` +2. Repeat the same test as described above. + +#### Test performance with Docker Host network + +Repeat the same test while the server will reside in a Docker container using **Host network**. + +--- + +Summarize your results in the appropriate table in `12_docker_network_analysis_ex/README.md` file. + +### Test Server and Client on different machines + +Repeat the above experiment, but now the client and server will reside in a different physical machines, on the **same AWS region**, but in **different AZ**. In order to do that, you need to create another Ubuntu EC2 instance, and run the client from it (open relevant ports in the instance's security group). +Summarize your results in the appropriate table in the README file. + + +## Conclusions + +According to your experiment results, conclude the % overhead of Docker Bridge and Host networks comparing the network performance of the original machine. +Write your conclusions separately for client-server communication on the same physical machine, and for client-server in different machines. + + + +# Good Luck + +Don't hesitate to ask any questions \ No newline at end of file diff --git a/docs/img/botAws.png b/docs/img/botAws.png new file mode 100644 index 00000000..a4ae8c4d Binary files /dev/null and b/docs/img/botAws.png differ diff --git a/docs/img/db.png b/docs/img/db.png new file mode 100644 index 00000000..4798e876 Binary files /dev/null and b/docs/img/db.png differ diff --git a/docs/img/elastic-arch.png b/docs/img/elastic-arch.png new file mode 100644 index 00000000..467ca63c Binary files /dev/null and b/docs/img/elastic-arch.png differ diff --git a/docs/img/elastic-spaces.png b/docs/img/elastic-spaces.png new file mode 100644 index 00000000..d8c3cdfa Binary files /dev/null and b/docs/img/elastic-spaces.png differ diff --git a/docs/img/fluent.png b/docs/img/fluent.png new file mode 100644 index 00000000..840b0553 Binary files /dev/null and b/docs/img/fluent.png differ diff --git a/docs/img/kibana-alerting.png b/docs/img/kibana-alerting.png new file mode 100644 index 00000000..37684c79 Binary files /dev/null and b/docs/img/kibana-alerting.png differ diff --git a/docs/img/kibana-dash-1.png b/docs/img/kibana-dash-1.png new file mode 100644 index 00000000..f42b4b4a Binary files /dev/null and b/docs/img/kibana-dash-1.png differ diff --git a/docs/img/kibana-dash-2.png b/docs/img/kibana-dash-2.png new file mode 100644 index 00000000..7c892ef5 Binary files /dev/null and b/docs/img/kibana-dash-2.png differ diff --git a/docs/img/mysql-multi-instance.png b/docs/img/mysql-multi-instance.png new file mode 100644 index 00000000..6049a5a3 Binary files /dev/null and b/docs/img/mysql-multi-instance.png differ diff --git a/docs/img/service-k8s.png b/docs/img/service-k8s.png new file mode 100644 index 00000000..8847d47e Binary files /dev/null and b/docs/img/service-k8s.png differ diff --git a/docs/img/telegramToken.png b/docs/img/telegramToken.png new file mode 100644 index 00000000..f79a2110 Binary files /dev/null and b/docs/img/telegramToken.png differ diff --git a/docs/jenkins_ex1.md b/docs/jenkins_ex1.md new file mode 100644 index 00000000..61e21e0a --- /dev/null +++ b/docs/jenkins_ex1.md @@ -0,0 +1,131 @@ +# CI/CD with Jenkins + +Due date: 20/11/2022 23:59 + + +### Update I + +You should copy the following files from `18_jenkins_ex1/k8s_helpers`: +- ecr-creds-helper.yaml +- init-k0s-cluster-amazon-linux.sh + +### Update III + +There is a missing `-n` flag in `infra/jenkins/dev/BotDeploy.Jenkinsfile`: + +```text +- bash common/replaceInFile.sh $K8S_CONFIGS/bot.yaml TELEGRAM_TOKEN $(echo $TELEGRAM_TOKEN | base64) ++ bash common/replaceInFile.sh $K8S_CONFIGS/bot.yaml TELEGRAM_TOKEN $(echo -n $TELEGRAM_TOKEN | base64) +``` + +## Deploy k8s cluster (via [k0s](https://k0sproject.io/)) + +Installing the k8s dashboard is as easy as executing pre-built bash script. + +1. Create an **Amazon Linux** EC2 `micro` instance. This instance will host a Kubernetes "cluster". Open the following ports: + - `30001` for accessing the k8s dashboard. + - `6443` to communicate with the k8s API. + **Note:** since Jenkins will communicate with k0s using the EC2 instance's private IP, they both have to reside in the same VPC! +2. Copy the files under `18_jenkins_ex1/k8s_helpers` directory (can be found in our shared repo) to the home directory of your EC2 and execute them by `bash init-k0s-cluster-amazon-linux.sh`. +3. Keep the _dashboard url_, and the _login token_ printed on screen. We will use them later on. +4. The PolyBot app will be running as Docker containers inside the Kubernetes cluster. Thus, your EC2 instance in which the k8s cluster is running needs the appropriate permissions, i.e. S3, SQS, etc... Specifically, it must have read permissions for your ECR registries. +5. After a successful installation of the k8s cluster (you can re-run `init-k0s-cluster-amazon-linux.sh` when something is getting wrong), from the EC2 terminal, create Kubernetes namespaces for the Development and Production environments: +```shell +$ kubectl create namespace dev +>> namespace/dev created + +$ kubectl create namespace prod +>> namespace/prod created +``` + +## Prepare the Jenkins server + +Perform the following steps on your existed Jenkins server. + +1. Install the [ECR Credentials helper](https://github.com/awslabs/amazon-ecr-credential-helper) by: +```shell +sudo amazon-linux-extras enable docker +sudo yum install amazon-ecr-credential-helper +sudo -u jenkins mkdir -p /var/lib/jenkins/.docker +echo '{"credsStore": "ecr-login"}' | sudo -u jenkins tee /var/lib/jenkins/.docker/config.json +``` + +2. In your Jenkins server, create `dev` and `prod` folders (New Item -> Folder). All the pipelines will be created in those folders, so no fear to overwrite the pipelines we've created in class. +3. Jenkins needs to talk with the k8s cluster in order to deploy the applications. It does so using the Kubernetes command-line tool, `kubectl`. To configure `kubectl` to work with your k8s cluster, create in Jenkins **Secret file** credentials called `kubeconfig` in the Jenkins global scope. The secret file itself can be found in the EC2 you've installed the k8s cluster under `~/.kube/config`. You can copy & paste this file's content to your local machine and upload to Jenkins. +4. You should create another Telegram bot. One bot will be functioning as a `dev` bot and will be used in Development environment, while the other is a `prod` bot that your customers are using in Production. So 2 bots, 2 tokens. + +> Follow this section to create a new Telegram bot. You should follow until “Generating an authentication token” (not including that section). + + +5. Create a **Secret text** credentials called `telegram-bot-token` in each folder - `dev` and `prod`. Each credential contains the corresponding Telegram token (e.g. for dev folder creds, go to Dashboard -> dev -> Credentials -> dev store -> Global credentials -> Add Credentials). +6. All pipelines are running on a containerized agent (the same Docker image for all pipelines). The agent's Dockerfile can be found under `infra/jenkins/JenkinsAgent.Dockerfile`. You should build it, push in to an ECR registry, and replace `` with your Docker image URI in each Jenkinsfile. + +**Note:** no need to run agents on different **nodes**! All pipelines can be running on the Jenkins server itself. + + +## The Dev and Prod CI/CD pipelines + +All the code in this exercise is already given to you in the [PolyBot repo](https://github.com/alonitac/PolyBot), `main` branch. So no need to write any Python. +Throughout this exercise we will be working with branches `dev` and `main` which representing Development and Production environments accordingly. The app is the old good PolyBot (excluding the autoscaling functionality). +If you are using branch `main` or `dev` for your personal PolyBot implementation, checkout it to another branch for now, so you'll have a backup of your version, and use `main` and `dev` for this exercise. Alternatively, you can work on a new clean fork (a separate repo). Later on, after you are comfortable with the new project structure, you can migrate your code into the bare PolyBot implementation you are given. + +You are going to implement full CI/CD pipelines for the PolyBot (bot and worker) app in Development and Production environments, using Jenkins. + +Read the following guidelines carefully **before** you start the implementation! +Create the following pipelines in Jenkins and complete the corresponding Jenkinsfiles: + +### The `dev` folder pipelines + +The following pipelines should be located under `dev` folder: + +- The `botBuild` Pipeline - responsible to build the Bot app. The Jenkinsfile is **partially** implemented under `infra/jenkins/dev/BotBuild.Jenkinsfile`. Complete the `TODO`s. This pipeline should be triggered upon changes in `common/` and `services/bot/` dirs only. +- The `botDeploy` Pipeline - responsible to deploy the Bot app. The Jenkinsfile is **completely** implemented under `infra/jenkins/dev/BotDeploy.Jenkinsfile`. Don't change it, only review and make sure you understand everything. +- The `workerBuild` Pipeline - responsible to build the Worker app. The Jenkinsfile configured under `infra/jenkins/dev/WorkerBuild.Jenkinsfile`. You should implement it. This pipeline should be triggered upon changes in `common/` and `services/worker/` dirs only. +- The `workerDeploy` Pipeline - responsible to deploy the Worker app. The Jenkinsfile is **completely** implemented under `infra/jenkins/dev/WorkerDeploy.Jenkinsfile`. No need to change. + +#### Notes for `dev` pipelines + +1. All `dev` pipelines should **only** be triggered from a Git branch called `dev` (if you don't have the `dev` branch, check it out initially from `main`). +2. In Dev env, `botBuild` and `workerBuild` should automatically trigger the `botDeploy` and `workerDeploy` pipelines accordingly (see **Trigger Deploy** stages in the Jenkinsfiles). +3. To trigger a pipeline only upon changes to a given directory, in the pipeline configuration, under **Additional Behaviours** section, choose **Polling ignores commits in certain paths**. In the **Included Regions** textbox, enter your paths, line by line. +4. `BotDeploy` and `WorkerDeploy` pipelines use `telegram-bot-token` and `kubeconfig` credentials, make sure you have them configured. + +### The `prod` folder + +The following pipelines should be located under `prod` folder: + +- The `botBuild` Pipeline - responsible to build the Bot app. The Jenkinsfile is configured under `infra/jenkins/prod/BotBuild.Jenkinsfile`. You should implement it. +- The `botDeploy` Pipeline - responsible to deploy the Bot app. The Jenkinsfile is configured under `infra/jenkins/prod/BotDeploy.Jenkinsfile`. You should implement it. +- The `workerBuild` Pipeline - responsible to build the Worker app. The Jenkinsfile is configured under `infra/jenkins/prod/WorkerBuild.Jenkinsfile`. You should implement it. +- The `workerDeploy` Pipeline - responsible to deploy the Worker app. The Jenkinsfile is configured under `infra/jenkins/prod/WorkerDeploy.Jenkinsfile`. You should implement it. +- The `PRTesting` - responsible to execute PR testing. The Jenkinsfile is configured under `infra/jenkins/prod/PullRequest.Jenkinsfile`. This is a **Multi-branch** pipeline that should be triggered upon Pull Request creation as we [did in class](https://github.com/alonitac/DevOpsJan22/blob/main/17_jenkins/jenkins_tutorial.md#pull-request-testing). Make sure that a given PR is blocked from being merged into `main` when this pipeline fails. No need to implement it, leave it like that. + +#### Notes for `prod` pipelines + +1. All `prod`'s folder pipelines should be triggered from a Git branch `main`. +2. In Prod env, `botBuild` and `workerBuild` should **not** trigger the `botDeploy` and `workerDeploy` pipelines automatically. They should be triggered manually (In the pipeline configurations, choose **This project is parameterized**, then either **String Parameter** or **Run Parameter** should do the job). +3. You should protect branch `main` from pushing code into it, as we [did in class](https://github.com/alonitac/DevOpsJan22/blob/main/17_jenkins/jenkins_tutorial.md#pull-request-testing). New code can be merged only through a Pull Request. + +## Experimenting your Pipelines + +1. Enter the k8s dashboard in `https://:30001`. +2. Enter the login token you were provided when created your cluster. In this dashboard you can watch the running applications in k8s. +3. Commit and push your work, make sure all pipelines are completed successfully and your application is running in the cluster in both `dev` and `prod` namespaces. + +### Deploy a new change + +1. From branch `main`, checkout a new feature branch (e.g. `feature/greeting_msg`). +2. Make some change to the bot code, for example add a greeting message for the users. +3. Commit your change. +4. Let's test your change in Dev env, checkout `dev` branch, and merge `feature/greeting_msg` into `dev`. +5. Push `dev`. Make sure the pipelines are running, and the change has been deployed to Dev (talk with the dev bot to check the change). +6. Everything is good? time to deploy to Production. Create a PR from `feature/greeting_msg` into `main`. Let Jenkins approve your PR, ask a friend to review the code (or review it yourself). Finally, complete the PR. +7. Make sure the `prod/botBuild` pipeline is running, and trigger `prod/botDeploy` manually. Check that the change in prod bot has been deployed. + +### Submission + +Your forked PolyBot repo will be tested. Make sure you have implemented all Jenkinsfiles under `main` and `dev` branches. + +# Good Luck + +Don't hesitate to ask any questions diff --git a/docs/python_ex1.md b/docs/python_ex1.md new file mode 100644 index 00000000..5a57b299 --- /dev/null +++ b/docs/python_ex1.md @@ -0,0 +1,91 @@ +# Poly YouTube Telegram Bot +Due date: 20/06/2022 23:59 + +**Can be done in pairs!** + +# Background + +Your goal in this exercise is to design and develop Telegram bot which will serve YouTube content. + +## Part 1 - Create the environment + +1. Fork [The PolyBot](https://github.com/alonitac/PolyBot) repo (learn about [forking a repo](https://docs.github.com/en/get-started/quickstart/fork-a-repo) in GitHub) +2. Clone your **forked repo** locally into PyCharm (Git -> Clone...) +3. [Create Python venv](https://www.jetbrains.com/help/pycharm/creating-virtual-environment.html) for the project +4. Install requirements by `pip install -r requirements.txt` + + +## Part 2 - Create a Telegram Bot + +1. Download and install telegram desktop (you can use your phone app as well). +2. Once installed, follow this section to create a bot. You should follow until “Generating an authentication token” (not including that section) + + ![telegramBot](/img/telegramToken.png) + +At this point, you should have your own bot, as well as the API token. **Never** commit sensitive data like secrets in Git repo. For now, save the token in a file called `.telegramToken` and add this file to `.gitignore` to exclude it completely from Git index. We will later learn that the place to store sensitive data is the cloud (AWS in our case). + + + +## Part 3 - Running a simple “echo” Bot + +### The class _Bot_ +Under `bot.py` you are given a class called `Bot`. This class handles a simple telegram bot, as follows: + +The constructor `__init__` gets `token` arg which is the bot token you have just received from Telegram. Inside the constructor, an Updater object is created, and the function `self._message_handler` is set as main msg handler, this function is getting called whenever a new message will be sent to the bot. + +The default behaviour of Bot class is to “echo” the incoming messages. +Run the program and send a message to the bot via Telegram app, observe the response and get an idea of how `_message_handler` is functioning (it's recommended to run in debug mode with breakpoints). + +## Part 4 - Extending the echo bot + +### The class _QuoteBot_ + +In `bot.py` you are given a class called `QuoteBot` which inherits from `Bot`. Upon incoming messages, this bot echoing the message while quoting the original message, unless the user is asking politely not to quote. +Run this bot and check its behavior. + +## Part 5 - Build your YouTube Bot + +### The class _YoutubeBot_ + +In `bot.py` you are given a class called `YoutubeBot` which inherits from `Bot`. +Upon incoming messages, this class will take the message text, search and download corresponding Youtube video(s), the bot will then send the video file to the user. + +1. Inside `YoutubeBot` class, override `_message_handler` method and implement the functionality that is needed to download video from youtube and send it to the user (utilize `search_download_youtube_video` in `utils.py`). +2. Remember that by inheriting the Bot class, you can use all of its methods (such as send_video or send_text). +3. (Optional) Feel free to add more functionality e.g. implement a logic that caches videos that have already been downloaded, such that it will use the local copy when user requests the same video again. + +## Part 6 - Containerize your app +1. In your root directory of you repo, open `Dockerfile` and fill out the file such that the Bot app can be run as a Docker container +2. Build and run the container locally, make sure it works well. + +## Part 7 - Run your app "as a service" in an Amazon EC2 instance +1. In the course AWS account, create an Amazon Linux free tier EC2 instance. +2. Connect to the VM using SSH +3. Install [docker engine](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/create-container-image.html#create-container-image-install-docker) in it +4. Get your code there by `git clone...` (install git on the VM if needed). +5. Build you app with Docker (`docker build -t ...`) +6. Run your container "as a service", which means (1) in the background, and (2) the container will start running automatically after reboot. (Hint: read about [`--detach`](https://docs.docker.com/engine/reference/commandline/run/#options) and [`--restart`](https://docs.docker.com/engine/reference/commandline/run/#restart-policies---restart) options) +7. **You must validate that everything is running correctly!** Communicate with your bot and check the response, reboot your machine and validate that the bot is up and running after. + +## Submission guidelines +1. Add the below public key to `~/.ssh/authorized_keys` file in your VM (in a separate line), so course staff will be able to connect and see your work. +2. You don't need to keep the VM running. When your work is done, Stop the machine (don't Terminate it!) +3. In your forked repo, in `SUBMISSION` file, write _your mails_, the _Instance id_ and _region_ of your EC2 instance. Don't forget to commit and push it so it is visible to course stuff: +```text +student1@gmail.com +student2@gmail.com + +Instance: i-09bfad1e9a92275f9 +region: eu-north-1 +``` + +No need to send your forked repo link since GitHub is automatically monitoring who forked the repo. + +##### Course public RSA key + +```text +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC8uOWjEcG0wFKSYbgEv/rDS6vyu1oZNfS7AWX35I0ozoNSJXEYiGW8Kw9VYE7TIEDCzBag61DbQyTDVlQpYVCw7uzDMTrgOAGQQIm8USOyFm2STRCeMa1sKivlDYynXhhtMS5k3e0a9Bo0hCbFRvVqjpixG/g/6wVA+vFjeWTo5bKjh9ekoSd3wdOu22PR6GjT0+NK5xlqhjKCnl19BFiIRptqcUkFuCgXqktrcwix0Cq2QhaQvYfIv/VA68OaClCX8wPDNXbO2VHK4170Kg5ubTrqx4ppP7Q0Gasz8CUCSGhf+njmhj3TnqhZ2UFsohyTIH4xV7e7wtNxDxdJ/r+T DevOpsCourseStuff +``` +# Good Luck + +Don't hesitate to ask any questions \ No newline at end of file diff --git a/face-blur-lambdas/blur-faces/Dockerfile b/face-blur-lambdas/blur-faces/Dockerfile new file mode 100644 index 00000000..85a1f807 --- /dev/null +++ b/face-blur-lambdas/blur-faces/Dockerfile @@ -0,0 +1,16 @@ +FROM public.ecr.aws/lambda/python:3.7 +# Install the function's dependencies +# Copy file requirements.txt from your project folder and install +# the requirements in the app directory. + +COPY requirements.txt . +RUN pip install -r requirements.txt + +# Copy helper functions +COPY video_processor.py video_processor.py + +# Copy handler function (from the local app directory) +COPY app.py . + +# Overwrite the command by providing a different command directly in the template. +CMD ["app.lambda_function"] \ No newline at end of file diff --git a/face-blur-lambdas/blur-faces/app.py b/face-blur-lambdas/blur-faces/app.py new file mode 100644 index 00000000..7c507cea --- /dev/null +++ b/face-blur-lambdas/blur-faces/app.py @@ -0,0 +1,66 @@ +import json +import logging +import os + +import boto3 +import botocore +import cv2 + +from video_processor import apply_faces_to_video, integrate_audio + +logger = logging.getLogger() +logger.setLevel(logging.INFO) + +reko = boto3.client('rekognition') +s3 = boto3.client('s3') + +output_bucket = os.environ['OUTPUT_BUCKET'] + +def lambda_function(event, context): + # download file locally to /tmp retrieve metadata + try: + response = event['response'] + # get metadata of file uploaded to Amazon S3 + bucket = event['s3_object_bucket'] + key = event['s3_object_key'] + filename = key.split('/')[-1] + local_filename = '/tmp/{}'.format(filename) + local_filename_output = '/tmp/anonymized-{}'.format(filename) + except KeyError: + error_message = 'Lambda invoked without S3 event data. Event needs to reference a S3 bucket and object key.' + logger.log(logging.ERROR, error_message) + # add_failed(bucket, error_message, failed_records, key) + + try: + s3.download_file(bucket, key, local_filename) + except botocore.exceptions.ClientError: + error_message = 'Lambda role does not have permission to call GetObject for the input S3 bucket, or object does not exist.' + logger.log(logging.ERROR, error_message) + # add_failed(bucket, error_message, failed_records, key) + # continue + + # get timestamps + try: + timestamps = event['timestamps'] + apply_faces_to_video(timestamps, local_filename, local_filename_output, response["VideoMetadata"]) + except Exception as e: + print(e) + # continue + + try: + integrate_audio(local_filename, local_filename_output) + except Exception as e: + print(e) + + # uploaded modified video to Amazon S3 bucket + try: + s3.upload_file(local_filename_output, output_bucket, key) + except boto3.exceptions.S3UploadFailedError: + error_message = 'Lambda role does not have permission to call PutObject for the output S3 bucket.' + # add_failed(bucket, error_message, failed_records, key) + # continue + + return { + 'statusCode': 200, + 'body': json.dumps('Faces in video blurred') + } \ No newline at end of file diff --git a/face-blur-lambdas/blur-faces/requirements.txt b/face-blur-lambdas/blur-faces/requirements.txt new file mode 100644 index 00000000..d5d7a829 --- /dev/null +++ b/face-blur-lambdas/blur-faces/requirements.txt @@ -0,0 +1,20 @@ +boto3==1.18.6 +botocore==1.21.6 +certifi==2021.5.30 +charset-normalizer==2.0.3 +decorator==4.4.2 +idna==3.2 +imageio==2.9.0 +imageio-ffmpeg==0.4.4 +jmespath==0.10.0 +moviepy==1.0.3 +numpy==1.21.6 +opencv-python==4.5.3.56 +Pillow==9.0.1 +proglog==0.1.9 +python-dateutil==2.8.2 +requests==2.26.0 +s3transfer==0.5.0 +six==1.16.0 +tqdm==4.61.2 +urllib3==1.26.6 \ No newline at end of file diff --git a/face-blur-lambdas/blur-faces/video_processor.py b/face-blur-lambdas/blur-faces/video_processor.py new file mode 100644 index 00000000..5647db2e --- /dev/null +++ b/face-blur-lambdas/blur-faces/video_processor.py @@ -0,0 +1,106 @@ +import cv2 +import numpy as np +from moviepy.editor import * +from moviepy.editor import VideoFileClip, AudioFileClip, CompositeAudioClip +import os + + +def anonymize_face_pixelate(image, blocks=10): + """ + Computes a pixelated blur with OpenCV + Args: + image (ndarray): The image to be blurred + blocks (int): Number of pixel blocks (default is 10) + Returns: + image (ndarray): The blurred image + """ + # divide the input image into NxN blocks + (h, w) = image.shape[:2] + x_coordinates, y_coordinates = np.linspace(0, w, blocks + 1, dtype="int"), np.linspace(0, h, blocks + 1, dtype="int") + + # loop over the blocks along x and y axis + for i in range(1, len(y_coordinates)): + for j in range(1, len(x_coordinates)): + # compute the first and last (x, y)-coordinates for the current block + first_x, last_x = x_coordinates[j - 1], x_coordinates[j] + first_y, last_y = y_coordinates[i - 1], y_coordinates[i] + # extract the ROI + roi = image[first_y:last_y, first_x:last_x] + # compute the mean of the ROI + (b, g, r) = [int(x) for x in cv2.mean(roi)[:3]] + # draw a rectangle with the mean RGB values over the ROI in the original image + cv2.rectangle(image, (first_x, first_y), (last_x, last_y), (b, g, r), -1) + + # return the pixelated blurred image + return image + + +def apply_faces_to_video(final_timestamps, local_path_to_video, local_output, video_metadata, color=(255,0,0), thickness=2): + # Extract video info + frame_rate = video_metadata["FrameRate"] + frame_height = video_metadata["FrameHeight"] + frame_width = video_metadata["FrameWidth"] + width_delta = int(frame_width / 250) + height_delta = int(frame_height / 100) + # Set up support for OpenCV + frame_counter = 0 + fourcc = cv2.VideoWriter_fourcc('M', 'J', 'P', 'G') + # Create the file pointers + v = cv2.VideoCapture(local_path_to_video) + print("VideoCapture - local path to video") + out = cv2.VideoWriter( + filename=local_output, + fourcc=fourcc, + fps=int(frame_rate), + frameSize=(frame_width, frame_height) + ) + # Open the video + while v.isOpened(): + has_frame, frame = v.read() + if has_frame: + for t in final_timestamps: + faces = final_timestamps.get(t) + lower_bound = int(int(t) / 1000 * frame_rate) + upper_bound = int(int(t) / 1000 * frame_rate + frame_rate / 2) + 1 + if (frame_counter >= lower_bound) and (frame_counter <= upper_bound): + for f in faces: + x = int(f['Left'] * frame_width) - width_delta + y = int(f['Top'] * frame_height) - height_delta + w = int(f['Width'] * frame_width) + 2 * width_delta + h = int(f['Height'] * frame_height) + 2 * height_delta + + x1, y1 = x, y + x2, y2 = x1 + w, y1 + h + + to_blur = frame[y1:y2, x1:x2] + blurred = anonymize_face_pixelate(to_blur, blocks=10) + frame[y1:y2, x1:x2] = blurred + + # frame = cv2.rectangle(frame, (x,y), (x+w,y+h), (255,0,0), 3) + out.write(frame) + frame_counter += 1 + else: + break + + out.release() + v.release() + cv2.destroyAllWindows() + print(f"Complete. {frame_counter} frames were written.") + + +def integrate_audio(original_video, output_video, audio_path='/tmp/audio.mp3'): + # Extract audio + my_clip = VideoFileClip(original_video) + my_clip.audio.write_audiofile(audio_path) + temp_location = '/tmp/output_video.mp4' + # Join output video with extracted audio + videoclip = VideoFileClip(output_video) + # new_audioclip = CompositeAudioClip([audioclip]) + # videoclip.audio = new_audioclip + videoclip.write_videofile(temp_location, codec='libx264', audio=audio_path, audio_codec='libmp3lame') + + os.rename(temp_location, output_video) + # Delete audio + os.remove(audio_path) + + print('Complete') \ No newline at end of file diff --git a/face-blur-lambdas/check-rekognition-job-status/lambda_function.py b/face-blur-lambdas/check-rekognition-job-status/lambda_function.py new file mode 100644 index 00000000..c8c78a16 --- /dev/null +++ b/face-blur-lambdas/check-rekognition-job-status/lambda_function.py @@ -0,0 +1,20 @@ +import boto3 + +reko = boto3.client('rekognition') +s3 = boto3.client('s3') + +def lambda_handler(event, context): + job_id = event['job_id'] + reko_client = boto3.client('rekognition') + response = reko_client.get_face_detection(JobId=job_id, MaxResults=100) + + return { + "statusCode": 200, + "body": + { + "job_id": job_id, + "job_status": response['JobStatus'], + "s3_object_bucket": event['s3_object_bucket'], + "s3_object_key": event['s3_object_key'] + } + } \ No newline at end of file diff --git a/face-blur-lambdas/face-detection/lambda_function.py b/face-blur-lambdas/face-detection/lambda_function.py new file mode 100644 index 00000000..93213fc8 --- /dev/null +++ b/face-blur-lambdas/face-detection/lambda_function.py @@ -0,0 +1,88 @@ +import json +import logging +import os +import urllib + +import boto3 +import botocore + +from rekognition import check_format_and_size, start_face_detection + +logger = logging.getLogger() +logger.setLevel(logging.INFO) + +reko = boto3.client('rekognition') +s3 = boto3.client('s3') +sfn = boto3.client('stepfunctions') + +state_machine_arn = os.environ['STATE_MACHINE_ARN'] + +def lambda_handler(event, context): + successful_records = [] + failed_records = [] + + for record in event['Records']: + + # verify event has reference to S3 object + try: + # get metadata of file uploaded to Amazon S3 + bucket = record['s3']['bucket']['name'] + key = urllib.parse.unquote_plus(record['s3']['object']['key']) + path_to_file = '{}/{}'.format(bucket, key) + size = int(record['s3']['object']['size']) + filename = key.split('/')[-1] + local_filename = '/tmp/{}'.format(filename) + local_filename_output = '/tmp/{}'.format(filename) + except KeyError: + error_message = 'Lambda invoked without S3 event data. Event needs to reference a S3 bucket and object key.' + logger.log(logging.ERROR, error_message) + # add_failed(bucket, error_message, failed_records, key) + continue + + # verify file and its size + try: + assert check_format_and_size(filename, size) + except: + error_message = 'Unsupported file type. Amazon Rekognition Video support MOV and MP4 lower than 10 GB in size' + logger.log(logging.ERROR, error_message) + continue + + # use Amazon Rekognition to detect faces in video uploaded to Amazon S3 + try: + job_id = start_face_detection(bucket, key, 1, reko) + # response = wait_for_completion(job_id, reko_client=reko) + except reko.exceptions.AccessDeniedException: + error_message = 'Lambda role does not have permission to call DetectFaces in Amazon Rekognition.' + logger.log(logging.ERROR, error_message) + # add_failed(bucket, error_message, failed_records, key) + continue + except reko.exceptions.InvalidS3ObjectException: + error_message = 'Unable to get object metadata from S3. Check object key, region and/or access permissions for input S3 bucket.' + logger.log(logging.ERROR, error_message) + # add_failed(bucket, error_message, failed_records, key) + continue + + input_data = json.dumps({"body":{"job_id": job_id, "s3_object_bucket": bucket, "s3_object_key": key}}) + + response = sfn.start_execution( + stateMachineArn=state_machine_arn, + input=input_data) + + return { + "statusCode": 200, + "body": json.dumps( + { + "job_id": job_id, + "failed_records": failed_records, + "successful_records": successful_records + } + ) + } + + +def add_failed(bucket, error_message, failed_records, key): + failed_records.append({ + "bucket": bucket, + "key": key, + "error_message": error_message + }) \ No newline at end of file diff --git a/face-blur-lambdas/face-detection/rekognition.py b/face-blur-lambdas/face-detection/rekognition.py new file mode 100644 index 00000000..a2a5dfd7 --- /dev/null +++ b/face-blur-lambdas/face-detection/rekognition.py @@ -0,0 +1,19 @@ +import boto3 + +def boto3_client(): + return boto3.client('rekognition') + + +def check_format_and_size(filename, size): + if filename.split('.')[-1] in ['mp4', 'mov']: + if size < 10*1024*1024*1024: + return True + return False + + +def start_face_detection(bucket, video, size, reko_client=None): + assert check_format_and_size(video, size) + if reko_client == None: + reko_client = boto3.client('rekognition') + response = reko_client.start_face_detection(Video={'S3Object': {'Bucket': bucket, 'Name': video}}) + return response['JobId'] \ No newline at end of file diff --git a/face-blur-lambdas/get-rekognized-faces/lambda_function.py b/face-blur-lambdas/get-rekognized-faces/lambda_function.py new file mode 100644 index 00000000..b7164f07 --- /dev/null +++ b/face-blur-lambdas/get-rekognized-faces/lambda_function.py @@ -0,0 +1,49 @@ +import boto3 + +reko = boto3.client('rekognition') +s3 = boto3.client('s3') + +def get_timestamps_and_faces(job_id, reko_client=None): + final_timestamps = {} + next_token = "Y" + first_round = True + while next_token != "": + print('.', end='') + # Set some variables if it's the first iteration + if first_round: + next_token = "" + first_round = False + # Query Reko Video + response = reko_client.get_face_detection(JobId=job_id, MaxResults=100, NextToken=next_token) + # Iterate over every face + for face in response['Faces']: + f = face["Face"]["BoundingBox"] + t = str(face["Timestamp"]) + time_faces = final_timestamps.get(t) + if time_faces == None: + final_timestamps[t] = [] + final_timestamps[t].append(f) + # Check if there is another portion of the response + try: + next_token = response['NextToken'] + except: + break + # Return the final dictionary + print('Complete') + return final_timestamps, response + + +def lambda_handler(event, context): + job_id = event['job_id'] + timestamps, response = get_timestamps_and_faces(job_id, reko) + return { + 'statusCode': 200, + "body": + { + "job_id": job_id, + "response": response, + "s3_object_bucket": event['s3_object_bucket'], + "s3_object_key": event['s3_object_key'], + "timestamps": timestamps + } + } \ No newline at end of file diff --git a/face-blur-lambdas/state_machine.json b/face-blur-lambdas/state_machine.json new file mode 100644 index 00000000..ec0ed0bc --- /dev/null +++ b/face-blur-lambdas/state_machine.json @@ -0,0 +1,104 @@ +{ + "StartAt": "Check Job Status", + "States": { + "Check Job Status": { + "Next": "Job finished?", + "Retry": [ + { + "ErrorEquals": [ + "Lambda.ServiceException", + "Lambda.AWSLambdaException", + "Lambda.SdkClientException" + ], + "IntervalSeconds": 2, + "MaxAttempts": 6, + "BackoffRate": 2 + } + ], + "Type": "Task", + "InputPath": "$.body", + "OutputPath": "$.Payload", + "Resource": "arn:aws:states:::lambda:invoke", + "Parameters": { + "FunctionName": "< check-rekognition-job-status ARN >", + "Payload.$": "$" + } + }, + "Wait 1 Second": { + "Type": "Wait", + "Seconds": 1, + "Next": "Check Job Status" + }, + "Job finished?": { + "Type": "Choice", + "Choices": [ + { + "Variable": "$.body.job_status", + "StringEquals": "IN_PROGRESS", + "Next": "Wait 1 Second" + }, + { + "Variable": "$.body.job_status", + "StringEquals": "SUCCEEDED", + "Next": "Get Timestamps and Faces" + } + ], + "Default": "Execution Failed" + }, + "Execution Failed": { + "Type": "Fail", + "Error": "Could not get job_status = 'SUCCEEDED'", + "Cause": "Face Detection Failed" + }, + "Get Timestamps and Faces": { + "Next": "Blur Faces on Video", + "Retry": [ + { + "ErrorEquals": [ + "Lambda.ServiceException", + "Lambda.AWSLambdaException", + "Lambda.SdkClientException" + ], + "IntervalSeconds": 2, + "MaxAttempts": 6, + "BackoffRate": 2 + } + ], + "Type": "Task", + "InputPath": "$.body", + "OutputPath": "$.Payload", + "Resource": "arn:aws:states:::lambda:invoke", + "Parameters": { + "FunctionName": "< get-rekognized-faces ARN >", + "Payload.$": "$" + } + }, + "Blur Faces on Video": { + "Next": "Execution Succeeded", + "Retry": [ + { + "ErrorEquals": [ + "Lambda.ServiceException", + "Lambda.AWSLambdaException", + "Lambda.SdkClientException" + ], + "IntervalSeconds": 2, + "MaxAttempts": 6, + "BackoffRate": 2 + } + ], + "Type": "Task", + "InputPath": "$.body", + "OutputPath": "$.Payload", + "Resource": "arn:aws:states:::lambda:invoke", + "Parameters": { + "FunctionName": "< blur-faces ARN >", + "Payload.$": "$" + } + }, + "Execution Succeeded": { + "Type": "Succeed" + } + }, + "TimeoutSeconds": 900 +} \ No newline at end of file diff --git a/img-object-detection/Dockerfile b/img-object-detection/Dockerfile new file mode 100644 index 00000000..5fb91b5c --- /dev/null +++ b/img-object-detection/Dockerfile @@ -0,0 +1,10 @@ +FROM daisukekobayashi/darknet:cpu-cv + +RUN apt-get update -y && apt-get install -y software-properties-common python3.9 python3-pip +RUN pip3 install --target /lambda_func boto3 awslambdaric + +WORKDIR /lambda_func +COPY . . + +ENTRYPOINT [ "/usr/bin/python3", "-m", "awslambdaric" ] +CMD [ "app.lambda_handler" ] \ No newline at end of file diff --git a/img-object-detection/Dockerfile2 b/img-object-detection/Dockerfile2 new file mode 100644 index 00000000..76ac2801 --- /dev/null +++ b/img-object-detection/Dockerfile2 @@ -0,0 +1,19 @@ +ARG FUNCTION_DIR="/home/app/" + +FROM python:3-slim-buster AS python-slim-buster +FROM python-slim-buster AS build-image + + +# Include global args in this stage of the build +ARG FUNCTION_DIR +# Create function directory +RUN mkdir -p ${FUNCTION_DIR} +# Copy handler function +COPY . ${FUNCTION_DIR} +RUN python3 -m pip install boto3 awslambdaric --target ${FUNCTION_DIR} + +FROM python-slim-buster +ARG FUNCTION_DIR +WORKDIR ${FUNCTION_DIR} +COPY --from=build-image ${FUNCTION_DIR} ${FUNCTION_DIR} +CMD [ "python3", "-m", "awslambdaric", "app.lambda_handler" ] \ No newline at end of file diff --git a/img-object-detection/app.py b/img-object-detection/app.py new file mode 100644 index 00000000..065e4693 --- /dev/null +++ b/img-object-detection/app.py @@ -0,0 +1,12 @@ +import os + + +def lambda_handler(event, context): + + stream = os.popen(f'cd /tmp && darknet detector demo /lambda_func/cfg/coco.data /lambda_func/cfg/yolov3.cfg /lambda_func/yolov3.weights /lambda_func/maxresdefault.jpg') + output = str(stream.read()) + + # bucket_name = event['Records'][0]['s3']['bucket']['name'] + # key = event['Records'][0]['s3']['object']['key'] + + print(f'Detecting objects in image...') diff --git a/img-object-detection/cfg/coco.data b/img-object-detection/cfg/coco.data new file mode 100644 index 00000000..e47e4802 --- /dev/null +++ b/img-object-detection/cfg/coco.data @@ -0,0 +1,8 @@ +classes= 80 +train = /home/pjreddie/data/coco/trainvalno5k.txt +valid = coco_testdev +#valid = data/coco_val_5k.list +names = /lambda_func/data/coco.names +backup = /home/pjreddie/backup/ +eval=coco + diff --git a/img-object-detection/cfg/yolov3.cfg b/img-object-detection/cfg/yolov3.cfg new file mode 100644 index 00000000..938ffff2 --- /dev/null +++ b/img-object-detection/cfg/yolov3.cfg @@ -0,0 +1,789 @@ +[net] +# Testing +# batch=1 +# subdivisions=1 +# Training +batch=64 +subdivisions=16 +width=608 +height=608 +channels=3 +momentum=0.9 +decay=0.0005 +angle=0 +saturation = 1.5 +exposure = 1.5 +hue=.1 + +learning_rate=0.001 +burn_in=1000 +max_batches = 500200 +policy=steps +steps=400000,450000 +scales=.1,.1 + +[convolutional] +batch_normalize=1 +filters=32 +size=3 +stride=1 +pad=1 +activation=leaky + +# Downsample + +[convolutional] +batch_normalize=1 +filters=64 +size=3 +stride=2 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=32 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=64 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +# Downsample + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=2 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +# Downsample + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=2 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +# Downsample + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=2 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +# Downsample + +[convolutional] +batch_normalize=1 +filters=1024 +size=3 +stride=2 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=1024 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=1024 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=1024 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=1024 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +###################### + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=1024 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=1024 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=1024 +activation=leaky + +[convolutional] +size=1 +stride=1 +pad=1 +filters=255 +activation=linear + + +[yolo] +mask = 6,7,8 +anchors = 10,13, 16,30, 33,23, 30,61, 62,45, 59,119, 116,90, 156,198, 373,326 +classes=80 +num=9 +jitter=.3 +ignore_thresh = .7 +truth_thresh = 1 +random=1 + + +[route] +layers = -4 + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[upsample] +stride=2 + +[route] +layers = -1, 61 + + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=512 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=512 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=512 +activation=leaky + +[convolutional] +size=1 +stride=1 +pad=1 +filters=255 +activation=linear + + +[yolo] +mask = 3,4,5 +anchors = 10,13, 16,30, 33,23, 30,61, 62,45, 59,119, 116,90, 156,198, 373,326 +classes=80 +num=9 +jitter=.3 +ignore_thresh = .7 +truth_thresh = 1 +random=1 + + + +[route] +layers = -4 + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[upsample] +stride=2 + +[route] +layers = -1, 36 + + + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=256 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=256 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=256 +activation=leaky + +[convolutional] +size=1 +stride=1 +pad=1 +filters=255 +activation=linear + + +[yolo] +mask = 0,1,2 +anchors = 10,13, 16,30, 33,23, 30,61, 62,45, 59,119, 116,90, 156,198, 373,326 +classes=80 +num=9 +jitter=.3 +ignore_thresh = .7 +truth_thresh = 1 +random=1 + diff --git a/img-object-detection/data/coco.names b/img-object-detection/data/coco.names new file mode 100644 index 00000000..ca76c80b --- /dev/null +++ b/img-object-detection/data/coco.names @@ -0,0 +1,80 @@ +person +bicycle +car +motorbike +aeroplane +bus +train +truck +boat +traffic light +fire hydrant +stop sign +parking meter +bench +bird +cat +dog +horse +sheep +cow +elephant +bear +zebra +giraffe +backpack +umbrella +handbag +tie +suitcase +frisbee +skis +snowboard +sports ball +kite +baseball bat +baseball glove +skateboard +surfboard +tennis racket +bottle +wine glass +cup +fork +knife +spoon +bowl +banana +apple +sandwich +orange +broccoli +carrot +hot dog +pizza +donut +cake +chair +sofa +pottedplant +bed +diningtable +toilet +tvmonitor +laptop +mouse +remote +keyboard +cell phone +microwave +oven +toaster +sink +refrigerator +book +clock +vase +scissors +teddy bear +hair drier +toothbrush diff --git a/img-object-detection/maxresdefault.jpg b/img-object-detection/maxresdefault.jpg new file mode 100644 index 00000000..bdb6df37 Binary files /dev/null and b/img-object-detection/maxresdefault.jpg differ diff --git a/mkdocs.yml b/mkdocs.yml index 4e14ff99..1e4ad941 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -4,5 +4,8 @@ nav: - Linux Ex1: 'linux_ex1.md' - Linux Ex2: 'linux_ex2.md' - Git Ex1: 'git_ex1.md' + - Python Ex1: 'python_ex1.md' + - AWS Ex1: 'aws_ex1.md' + - Jenkins Ex1: 'jenkins_ex1.md' theme: name: readthedocs diff --git a/python_katas/kata_2/questions.py b/python_katas/kata_2/questions.py index f7f0da48..09fc4c29 100644 --- a/python_katas/kata_2/questions.py +++ b/python_katas/kata_2/questions.py @@ -11,9 +11,34 @@ def valid_parentheses(s): e.g. s = '[[{()}](){}]' -> True - s = '[{]}' -> False - """ - return None + s = ']}' -> False + """ + stack = [] + for char in s: + if char in '([{': + stack.append(char) + elif not stack: # closing char + top = stack.pop(-1) + if top == '(' and char != ')': + return False + if top == '[' and char != ']': + return False + if top == '{' and char != '}': + return False + else: + return False + return True + + # David's solution + while True: + if '()' in s: + s = s.replace('()', '') + elif '{}' in s: + s = s.replace('{}', '') + elif '[]' in s: + s = s.replace('[]', '') + else: + return not s def fibonacci_fixme(n): @@ -56,7 +81,22 @@ def most_frequent_name(file_path): :param file_path: str - absolute or relative file to read names from :return: str - the mose frequent name. If there are many, return one of them """ - return None + d = { } + + most_common = None + + with open(file_path,'r') as file: + for name in file: + # update the name counter + if name in d: + d[name] += 1 + else: + d[name] = 1 + + if most_common is None or d[name] > d[most_common]: + most_common = name + + return most_common def files_backup(dir_path): @@ -278,7 +318,7 @@ def list_flatten(lst): This function gets a list of combination of integers or nested lists e.g. - [1, [], [1, 2, [4, 0, [5], 6], [5, 4], 34, 0, [3]] + [1, [], [1, 2, [4, 0, [5], 6], [5, 4], 34, 0], [3]] The functions should return a flatten list (including all nested lists): [1, 1, 2, 4, 0, 5, 6, 5, 4, 34, 0, 3] diff --git a/python_katas/kata_3/67203.jpeg b/python_katas/kata_3/67203.jpeg new file mode 100644 index 00000000..b46a5050 Binary files /dev/null and b/python_katas/kata_3/67203.jpeg differ diff --git a/python_katas/kata_3/questions.py b/python_katas/kata_3/questions.py index f23f7ae1..0bb9734a 100644 --- a/python_katas/kata_3/questions.py +++ b/python_katas/kata_3/questions.py @@ -1,86 +1,303 @@ -def knapsack(): - pass +from python_katas.kata_3.utils import open_img, save_img +import requests # to be used in simple_http_request() +ISO_FORMAT = '%Y-%m-%dT%H:%M:%SZ' -def time_me(): - pass +def knapsack(items, knapsack_limit=50): + """ + 5 Kata -def youtube_download(): - pass + Consider a thief gets into a home to rob and he carries a knapsack. + There are fixed number of items in the home — each with its own weight and value — + Jewelry, with less weight and highest value vs tables, with less value but a lot heavy. + To add fuel to the fire, the thief has an old knapsack which has limited capacity. + Obviously, he can’t split the table into half or jewelry into 3/4ths. He either takes it or leaves it + Given a set of items, dict of tuples representing the (weight, value), determine the items to include in a collection + so that the total weight is less than or equal to a given limit and the total value is as large as possible. -def visualizer(): - pass + :param items: dict of tuples e.g. {"bed": (100, 15), "iphone13": (1, 1500)} + :param knapsack_limit: + :return: set of items + """ + return None -def job_scheduling(): - pass +def time_me(func): + """ + 2 Kata + Given func - a pointer to sime function which can be executed by func() + Return the number of time it took to execute the function. Since execution time may vary from time to time, + execute func 100 time and return the mean -def valid_git_graph(): - pass + :param func: + :return: + """ + return None -def rotate_img(): - pass # use rotate matrix +def youtube_download(video_id): + """ + 3 Kata + Youtube video url is in the form https://www.youtube.com/watch?v=