From f1940ba95fbf3de07911d5423fd182ebbe92b300 Mon Sep 17 00:00:00 2001 From: Jon Smith <99849686+JonSmithTLT@users.noreply.github.com> Date: Wed, 22 Feb 2023 22:22:26 -0800 Subject: [PATCH 1/4] Create twitter_listner_proto.py --- .../twitter-scraper/twitter_listner_proto.py | 92 +++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 services/twitter-scraper/twitter_listner_proto.py diff --git a/services/twitter-scraper/twitter_listner_proto.py b/services/twitter-scraper/twitter_listner_proto.py new file mode 100644 index 0000000..ecab356 --- /dev/null +++ b/services/twitter-scraper/twitter_listner_proto.py @@ -0,0 +1,92 @@ +import requests +import os +import json + +bearer_token = os.environ.get('BEARER_TOKEN') + +# required function for authorzation to endpoint + + +def bearer_oauth(r): + r.headers['Authorization'] = f"Bearer {bearer_token}" + r.headers["User-Agent"] = "v2FilteredStreamPython" + return r + +# gets current rules from previous attempts + + +def get_rules(): + response = requests.get( + "https://api.twitter.com/2/tweets/search/stream/rules", auth=bearer_oauth) + if response.status_code != 200: + raise Exception(" Cannot get rules (HTTP {})".format( + response.status_code, response.text)) + print(json.dumps(response.json())) + return response.json() + + +def set_rules(): + # You can adjust the rules if needed + sample_rule = [ + {"value": "#FTX"} + ] + payload = {"add": sample_rule} + response = requests.post( + "https://api.twitter.com/2/tweets/search/stream/rules", + auth=bearer_oauth, + json=payload, + ) + if response.status_code != 201: + raise Exception( + "Cannot add rules (HTTP {}): {}".format( + response.status_code, response.text) + ) + print(json.dumps(response.json())) + + +def get_stream(): + response = requests.get( + "https://api.twitter.com/2/tweets/search/stream", auth=bearer_oauth, stream=True, + ) + print(response.status_code) + if response.status_code != 200: + raise Exception( + "Cannot get stream (HTTP {}): {}".format( + response.status_code, response.text + ) + ) + for response_line in response.iter_lines(): + if response_line: + json_response = json.loads(response_line) + print(json.dumps(json_response, indent=4, sort_keys=True)) + + +def delete_all_rules(rules): + if rules is None or "data" not in rules: + return None + + ids = list(map(lambda rule: rule["id"], rules["data"])) + payload = {"delete": {"ids": ids}} + response = requests.post( + "https://api.twitter.com/2/tweets/search/stream/rules", + auth=bearer_oauth, + json=payload + ) + if response.status_code != 200: + raise Exception( + "Cannot delete rules (HTTP {}): {}".format( + response.status_code, response.text + ) + ) + print(json.dumps(response.json())) + + +def main(): + rules = get_rules() + delete_all_rules(rules) + set_rules() + get_stream() + + +if __name__ == "__main__": + main() From 403fd061c976d2eba2a885fdc5122a6c2d7e7222 Mon Sep 17 00:00:00 2001 From: Jon Smith <99849686+JonSmithTLT@users.noreply.github.com> Date: Wed, 22 Feb 2023 23:30:47 -0800 Subject: [PATCH 2/4] Created Dockerfile and did some experimenting with the listener A docker file to create an image of the listener. doesn't work if run locally without a bearer token in environment variables meaning it only currently works on my machine until auth issues are fixed for all members. --- .build/twitter_listener.Dockerfile | 9 +++++++++ ...witter_listner_proto.py => twitter_listener_proto.py} | 0 2 files changed, 9 insertions(+) create mode 100644 .build/twitter_listener.Dockerfile rename services/twitter-scraper/{twitter_listner_proto.py => twitter_listener_proto.py} (100%) diff --git a/.build/twitter_listener.Dockerfile b/.build/twitter_listener.Dockerfile new file mode 100644 index 0000000..d042751 --- /dev/null +++ b/.build/twitter_listener.Dockerfile @@ -0,0 +1,9 @@ +FROM python:3.9 + +WORKDIR /app + +COPY ../services/twitter-scraper/twitter_listener_proto.py ./ + +RUN pip install requests + +CMD ["python3", "./twitter_listener_proto.py"] \ No newline at end of file diff --git a/services/twitter-scraper/twitter_listner_proto.py b/services/twitter-scraper/twitter_listener_proto.py similarity index 100% rename from services/twitter-scraper/twitter_listner_proto.py rename to services/twitter-scraper/twitter_listener_proto.py From e4499900c7e7a20ef8c432f344ba0b0c1c9e5d79 Mon Sep 17 00:00:00 2001 From: Jon Smith <99849686+JonSmithTLT@users.noreply.github.com> Date: Wed, 22 Feb 2023 23:36:09 -0800 Subject: [PATCH 3/4] AA-20 Forgot to add commit code --- .build/twitter_listener.Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/.build/twitter_listener.Dockerfile b/.build/twitter_listener.Dockerfile index d042751..fd65782 100644 --- a/.build/twitter_listener.Dockerfile +++ b/.build/twitter_listener.Dockerfile @@ -1,3 +1,4 @@ +#For issue AA-20 FROM python:3.9 WORKDIR /app From 1fb2e625a2af7800d50e49af0ee517fc5c6013d1 Mon Sep 17 00:00:00 2001 From: Jon Smith <99849686+JonSmithTLT@users.noreply.github.com> Date: Sun, 26 Feb 2023 06:08:48 -0800 Subject: [PATCH 4/4] AA-20 Added AWS Functions DO NOT RUN WITHOUT PERMISSION. I added functionality regarding AWS. I have set up a real-time data stream. The stream currently goes to an S3 bucket but can be changed to an RDB if needed depending on the direction we choose to go with the project. --- .build/twitter-scraper-dev.Dockerfile | 4 +- .build/twitter_listener.Dockerfile | 2 +- .../twitter-scraper/twitter_listener_proto.py | 58 ++++++++++++++----- 3 files changed, 48 insertions(+), 16 deletions(-) diff --git a/.build/twitter-scraper-dev.Dockerfile b/.build/twitter-scraper-dev.Dockerfile index 22b11a9..47f5f6d 100644 --- a/.build/twitter-scraper-dev.Dockerfile +++ b/.build/twitter-scraper-dev.Dockerfile @@ -3,8 +3,8 @@ WORKDIR /app ENV NODE_ENV=dev -COPY services/twitter-scraper/ ./ +COPY services/twitter-scraper/twitter_listener_proto.py ./ -RUN pip3 install requests schedule +RUN pip3 install requests CMD ["python3", "main.py"] \ No newline at end of file diff --git a/.build/twitter_listener.Dockerfile b/.build/twitter_listener.Dockerfile index fd65782..6920a86 100644 --- a/.build/twitter_listener.Dockerfile +++ b/.build/twitter_listener.Dockerfile @@ -5,6 +5,6 @@ WORKDIR /app COPY ../services/twitter-scraper/twitter_listener_proto.py ./ -RUN pip install requests +RUN pip install requests boto3 CMD ["python3", "./twitter_listener_proto.py"] \ No newline at end of file diff --git a/services/twitter-scraper/twitter_listener_proto.py b/services/twitter-scraper/twitter_listener_proto.py index ecab356..69f8c3d 100644 --- a/services/twitter-scraper/twitter_listener_proto.py +++ b/services/twitter-scraper/twitter_listener_proto.py @@ -1,19 +1,39 @@ -import requests -import os +import boto3 import json +import requests + +from botocore.exceptions import ClientError + +aws_firehose = boto3.client('firehose', region_name='us-west-1') + + +def get_bearer(): + secret_name = "twit_bearer_token" + region_name = "us-west-1" -bearer_token = os.environ.get('BEARER_TOKEN') + session = boto3.session.Session() + client = session.client( + service_name='secretsmanager', + region_name=region_name + ) + + try: + get_secret_value_response = client.get_secret_value( + SecretId=secret_name + ) + except ClientError as e: + raise e + bearer_token = get_secret_value_response['SecretString'] -# required function for authorzation to endpoint + return bearer_token def bearer_oauth(r): - r.headers['Authorization'] = f"Bearer {bearer_token}" + # Sets headers for HTTP requests + r.headers['Authorization'] = f"Bearer {get_bearer()}" r.headers["User-Agent"] = "v2FilteredStreamPython" return r -# gets current rules from previous attempts - def get_rules(): response = requests.get( @@ -21,33 +41,39 @@ def get_rules(): if response.status_code != 200: raise Exception(" Cannot get rules (HTTP {})".format( response.status_code, response.text)) + print("Rules currently in place: ") print(json.dumps(response.json())) return response.json() def set_rules(): # You can adjust the rules if needed - sample_rule = [ + sample_rules = [ {"value": "#FTX"} + # {"value": "#doge"} + # {"value: "meme has:images"} ] - payload = {"add": sample_rule} + payload = {"add": sample_rules} response = requests.post( "https://api.twitter.com/2/tweets/search/stream/rules", auth=bearer_oauth, - json=payload, + json=payload ) if response.status_code != 201: raise Exception( "Cannot add rules (HTTP {}): {}".format( response.status_code, response.text) ) - print(json.dumps(response.json())) + rules = (json.dumps(response.json())) + print(rules) + return rules def get_stream(): response = requests.get( "https://api.twitter.com/2/tweets/search/stream", auth=bearer_oauth, stream=True, ) + print("Tweet stream starting...") print(response.status_code) if response.status_code != 200: raise Exception( @@ -55,10 +81,14 @@ def get_stream(): response.status_code, response.text ) ) + seqID = 1 for response_line in response.iter_lines(): if response_line: json_response = json.loads(response_line) + aws_firehose.put_record(DeliveryStreamName="TWEETS-S3", Record={'Data': json.dumps( + json_response, indent=4, sort_keys=True)}) print(json.dumps(json_response, indent=4, sort_keys=True)) + seqID += 1 def delete_all_rules(rules): @@ -83,9 +113,11 @@ def delete_all_rules(rules): def main(): rules = get_rules() - delete_all_rules(rules) - set_rules() + # rules in this case refers to thE rules you seek to delete and returns current rules left? + delete = delete_all_rules(rules) + rule_set = set_rules() get_stream() + return 0 if __name__ == "__main__":