From 0150c8570d4331e7092fb1a22f32654cffa8a3b7 Mon Sep 17 00:00:00 2001 From: ekapratama93 Date: Thu, 28 Nov 2019 18:41:17 +0700 Subject: [PATCH 1/5] add autogenerate ip list aws & gcp --- Dockerfile | 6 +++--- Makefile | 4 ++-- README.md | 16 ++++++++++++++++ aws_listip.py | 18 ++++++++++++++++++ gcp_listip.py | 35 +++++++++++++++++++++++++++++++++++ run.sh | 12 ++++++++++++ shared/ips.txt | 3 --- 7 files changed, 86 insertions(+), 8 deletions(-) create mode 100644 aws_listip.py create mode 100644 gcp_listip.py diff --git a/Dockerfile b/Dockerfile index dd54baf..d097b65 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,13 +1,13 @@ -FROM python:3.5-alpine +FROM python:3.7-alpine RUN apk add --no-cache nmap nmap-scripts git -RUN pip install --no-cache-dir xmltodict google-cloud-storage boto3 +RUN pip install --no-cache-dir xmltodict google-cloud-storage google-api-python-client boto3 RUN git clone https://github.com/vulnersCom/nmap-vulners /usr/share/nmap/scripts/vulners && nmap --script-updatedb RUN mkdir /shared -COPY run.sh output_report.py latex_header.tex gcp_push.py aws_push.py / +COPY run.sh output_report.py latex_header.tex gcp_push.py aws_push.py gcp_listip.py aws_listip.py / COPY shared /shared RUN chmod +x /run.sh diff --git a/Makefile b/Makefile index f207923..09e3eec 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ -build : +build : docker build -t flan_scan . container_name = flan_$(shell date +'%s') -start : +start : docker run --name $(container_name) -v $(shell pwd)/shared:/shared flan_scan diff --git a/README.md b/README.md index 5943951..ffdf799 100644 --- a/README.md +++ b/README.md @@ -97,6 +97,22 @@ docker run --name \ ``` +Autogenerate IP List +-------------------- +Flan Scan currenty support autogenerate IP list from your running instances in GCP and AWS. You need additional environment variables to do that which is `generate_ips` which take value of `aws` and/or `gcp`. You can use both by using comma as separator like `aws,gcp`. + +### Example Autogenerate IP List + +```bash +$ docker run --name \ + -v $(pwd)/shared:/shared \ + -e upload=gcp \ + -e bucket= \ + -e GOOGLE_APPLICATION_CREDENTIALS=/shared/key.json \ + -e generate_ips=gcp + flan_scan +``` + Deploying on Kubernetes ----------------------- diff --git a/aws_listip.py b/aws_listip.py new file mode 100644 index 0000000..0634749 --- /dev/null +++ b/aws_listip.py @@ -0,0 +1,18 @@ +import boto3 + + +ec2 = boto3.resource('ec2') + +running_instances = ec2.instances.filter(Filters=[{ + 'Name': 'instance-state-name', + 'Values': ['running']}]) + +ips = [] +for instance in running_instances: + if instance.public_ip_address: + ips.append(instance.public_ip_address) + else: + ips.append(instance.private_ip_address) + +with open('shared/ips.txt', 'a') as f: + f.writelines("%s\n" % ip for ip in ips) diff --git a/gcp_listip.py b/gcp_listip.py new file mode 100644 index 0000000..06ea1f2 --- /dev/null +++ b/gcp_listip.py @@ -0,0 +1,35 @@ +import googleapiclient.discovery +import os + + +project_id = os.getenv("PROJECT_ID") + +client = googleapiclient.discovery.build('compute', 'v1') + +compute = client.instances() +zones = client.zones().list(project_id).execute() + +ips = [] + +for zone in zones['items']: + result = compute.list( + project=project_id, + zone=zone['name'], + filter="status=RUNNING").execute() + + if 'items' not in result: + continue + + for vm in result['items']: + if 'networkInterfaces' in vm: + for network_interface in vm['networkInterfaces']: + if 'accessConfigs' in network_interface: + ips.extend( + [x['natIP'] for x in network_interface['accessConfigs'] + if x['type'] == 'ONE_TO_ONE_NAT'] + ) + else: + ips.append(network_interface['networkIP']) + +with open('shared/ips.txt', 'a') as f: + f.writelines("%s\n" % ip for ip in ips) diff --git a/run.sh b/run.sh index 77ec6d0..fcefbba 100755 --- a/run.sh +++ b/run.sh @@ -30,6 +30,18 @@ function get_filename(){ echo $1 | tr / - } +if [[ -z $generate_ips ]] +then + if [[ $generate_ips == *"aws"* ]] + then + python /aws_listip.py + fi + if [[ $generate_ips == *"gcp"* ]] + then + python /gcp_listip.py + fi +fi + mkdir $root_dir$xml_dir while IFS= read -r line do diff --git a/shared/ips.txt b/shared/ips.txt index 89455e6..e69de29 100644 --- a/shared/ips.txt +++ b/shared/ips.txt @@ -1,3 +0,0 @@ -8.17.207.0/24 -198.41.138.128/25 -198.41.138.0/25 From e2f382c43d4b201ce843f4013eeb613109e812c4 Mon Sep 17 00:00:00 2001 From: ekapratama93 Date: Fri, 29 Nov 2019 17:10:47 +0700 Subject: [PATCH 2/5] fix no region --- aws_listip.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/aws_listip.py b/aws_listip.py index 0634749..47d9afd 100644 --- a/aws_listip.py +++ b/aws_listip.py @@ -1,18 +1,24 @@ import boto3 -ec2 = boto3.resource('ec2') - -running_instances = ec2.instances.filter(Filters=[{ - 'Name': 'instance-state-name', - 'Values': ['running']}]) +session = boto3.Session(region_name="us-east-1") +client = session.client('ec2') +regions = client.describe_regions() ips = [] -for instance in running_instances: - if instance.public_ip_address: - ips.append(instance.public_ip_address) - else: - ips.append(instance.private_ip_address) + +for region in regions['Regions']: + session = boto3.Session(region_name=region['RegionName']) + ec2 = session.resource('ec2') + running_instances = ec2.instances.filter(Filters=[{ + 'Name': 'instance-state-name', + 'Values': ['running']}]) + + for instance in running_instances: + if instance.public_ip_address: + ips.append(instance.public_ip_address) + else: + ips.append(instance.private_ip_address) with open('shared/ips.txt', 'a') as f: f.writelines("%s\n" % ip for ip in ips) From 2975b1b3f6337837a811afb8ed099085a21d3068 Mon Sep 17 00:00:00 2001 From: ekapratama93 Date: Fri, 29 Nov 2019 17:20:05 +0700 Subject: [PATCH 3/5] fix argument positional error --- gcp_listip.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gcp_listip.py b/gcp_listip.py index 06ea1f2..d6f199c 100644 --- a/gcp_listip.py +++ b/gcp_listip.py @@ -7,7 +7,7 @@ client = googleapiclient.discovery.build('compute', 'v1') compute = client.instances() -zones = client.zones().list(project_id).execute() +zones = client.zones().list(project=project_id).execute() ips = [] From 521d0381eab5a67b6ab0c7381f1662d948c5054b Mon Sep 17 00:00:00 2001 From: ekapratama93 Date: Fri, 29 Nov 2019 18:32:07 +0700 Subject: [PATCH 4/5] refactor ip list --- Dockerfile | 2 +- aws_listip.py | 24 ------------------ gcp_listip.py | 35 -------------------------- list_ip.py | 70 +++++++++++++++++++++++++++++++++++++++++++++++++++ run.sh | 19 +++++--------- 5 files changed, 77 insertions(+), 73 deletions(-) delete mode 100644 aws_listip.py delete mode 100644 gcp_listip.py create mode 100644 list_ip.py diff --git a/Dockerfile b/Dockerfile index d097b65..4b18f8c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,7 @@ RUN pip install --no-cache-dir xmltodict google-cloud-storage google-api-python- RUN git clone https://github.com/vulnersCom/nmap-vulners /usr/share/nmap/scripts/vulners && nmap --script-updatedb RUN mkdir /shared -COPY run.sh output_report.py latex_header.tex gcp_push.py aws_push.py gcp_listip.py aws_listip.py / +COPY run.sh output_report.py latex_header.tex gcp_push.py aws_push.py list_ip.py / COPY shared /shared RUN chmod +x /run.sh diff --git a/aws_listip.py b/aws_listip.py deleted file mode 100644 index 47d9afd..0000000 --- a/aws_listip.py +++ /dev/null @@ -1,24 +0,0 @@ -import boto3 - - -session = boto3.Session(region_name="us-east-1") -client = session.client('ec2') -regions = client.describe_regions() - -ips = [] - -for region in regions['Regions']: - session = boto3.Session(region_name=region['RegionName']) - ec2 = session.resource('ec2') - running_instances = ec2.instances.filter(Filters=[{ - 'Name': 'instance-state-name', - 'Values': ['running']}]) - - for instance in running_instances: - if instance.public_ip_address: - ips.append(instance.public_ip_address) - else: - ips.append(instance.private_ip_address) - -with open('shared/ips.txt', 'a') as f: - f.writelines("%s\n" % ip for ip in ips) diff --git a/gcp_listip.py b/gcp_listip.py deleted file mode 100644 index d6f199c..0000000 --- a/gcp_listip.py +++ /dev/null @@ -1,35 +0,0 @@ -import googleapiclient.discovery -import os - - -project_id = os.getenv("PROJECT_ID") - -client = googleapiclient.discovery.build('compute', 'v1') - -compute = client.instances() -zones = client.zones().list(project=project_id).execute() - -ips = [] - -for zone in zones['items']: - result = compute.list( - project=project_id, - zone=zone['name'], - filter="status=RUNNING").execute() - - if 'items' not in result: - continue - - for vm in result['items']: - if 'networkInterfaces' in vm: - for network_interface in vm['networkInterfaces']: - if 'accessConfigs' in network_interface: - ips.extend( - [x['natIP'] for x in network_interface['accessConfigs'] - if x['type'] == 'ONE_TO_ONE_NAT'] - ) - else: - ips.append(network_interface['networkIP']) - -with open('shared/ips.txt', 'a') as f: - f.writelines("%s\n" % ip for ip in ips) diff --git a/list_ip.py b/list_ip.py new file mode 100644 index 0000000..ed6aaa1 --- /dev/null +++ b/list_ip.py @@ -0,0 +1,70 @@ +import boto3 +import googleapiclient.discovery +import os + +def aws_list_ip(): + region_name = os.getenv("AWS_DEFAULT_REGION", "us-east-1") + + session = boto3.Session(region_name=region_name) + client = session.client('ec2') + regions = client.describe_regions() + + ips = [] + + for region in regions['Regions']: + session = boto3.Session(region_name=region['RegionName']) + ec2 = session.resource('ec2') + running_instances = ec2.instances.filter(Filters=[{ + 'Name': 'instance-state-name', + 'Values': ['running']}]) + + for instance in running_instances: + if instance.public_ip_address: + ips.append(instance.public_ip_address) + else: + ips.append(instance.private_ip_address) + return ips + +def gcp_list_ip(): + project_id = os.getenv("PROJECT_ID") + + client = googleapiclient.discovery.build('compute', 'v1') + + compute = client.instances() + zones = client.zones().list(project=project_id).execute() + + ips = [] + + for zone in zones['items']: + result = compute.list( + project=project_id, + zone=zone['name'], + filter="status=RUNNING").execute() + + if 'items' not in result: + continue + + for vm in result['items']: + if 'networkInterfaces' in vm: + for network_interface in vm['networkInterfaces']: + if 'accessConfigs' in network_interface: + ips.extend( + [x['natIP'] for x in network_interface['accessConfigs'] + if x['type'] == 'ONE_TO_ONE_NAT'] + ) + else: + ips.append(network_interface['networkIP']) + return ips + +def write_file(ip_list): + with open('shared/ips.txt', 'a') as f: + f.writelines("%s\n" % ip for ip in ip_list) + + +if __name__ == "__main__": + request = os.getenv("generate_ips") + if "aws" in request: + write_file(aws_list_ip()) + + if "gcp" in request: + write_file(gcp_list_ip()) diff --git a/run.sh b/run.sh index fcefbba..1b9b1da 100755 --- a/run.sh +++ b/run.sh @@ -1,5 +1,10 @@ #!/bin/sh +if [[ ! -z $generate_ips ]] +then + python /list_ip.py +fi + current_time=$(date "+%Y.%m.%d-%H.%M") if [[ -z $upload ]] then @@ -13,7 +18,7 @@ fi xml_dir=xml_files/$current_time report_file=reports/report_$current_time.tex -function upload { +function upload() { if [[ -z $upload ]] then return @@ -30,18 +35,6 @@ function get_filename(){ echo $1 | tr / - } -if [[ -z $generate_ips ]] -then - if [[ $generate_ips == *"aws"* ]] - then - python /aws_listip.py - fi - if [[ $generate_ips == *"gcp"* ]] - then - python /gcp_listip.py - fi -fi - mkdir $root_dir$xml_dir while IFS= read -r line do From 0c5f8747a6f497d9b815fbcfd84c8443a8879997 Mon Sep 17 00:00:00 2001 From: ekapratama93 Date: Fri, 29 Nov 2019 18:36:25 +0700 Subject: [PATCH 5/5] update example --- README.md | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index ffdf799..a7ae88c 100644 --- a/README.md +++ b/README.md @@ -99,18 +99,31 @@ docker run --name \ Autogenerate IP List -------------------- + Flan Scan currenty support autogenerate IP list from your running instances in GCP and AWS. You need additional environment variables to do that which is `generate_ips` which take value of `aws` and/or `gcp`. You can use both by using comma as separator like `aws,gcp`. -### Example Autogenerate IP List +### Example Autogenerate IP List in GCP + +You still need to set `GOOGLE_APPLICATION_CREDENTIALS` but you don't need to mount any volume to the container. ```bash -$ docker run --name \ - -v $(pwd)/shared:/shared \ - -e upload=gcp \ - -e bucket= \ - -e GOOGLE_APPLICATION_CREDENTIALS=/shared/key.json \ - -e generate_ips=gcp - flan_scan +docker run --name \ + -e GOOGLE_APPLICATION_CREDENTIALS=.json \ + -e PROJECT_ID= \ + -e generate_ips=gcp \ + flan_scan +``` + +### Example Autogenerate IP List in AWS + +You still need to set `GOOGLE_APPLICATION_CREDENTIALS` but you don't need to mount any volume to the container. + +```bash +docker run --name \ + -e AWS_ACCESS_KEY_ID= \ + -e AWS_SECRET_ACCESS_KEY= \ + -e generate_ips=aws \ + flan_scan ``` Deploying on Kubernetes