Finished ETL pipeline

Author: pycoder49

network_data/phisingData.csv

@@ -10,7 +10,7 @@
 1,0,-1,1,1,-1,1,1,-1,1,1,-1,1,0,1,-1,1,1,0,1,1,1,1,1,-1,1,1,1,0,1,1
 1,1,-1,1,1,-1,-1,1,-1,1,1,1,1,0,1,-1,1,1,0,1,1,1,1,1,-1,0,-1,1,0,1,-1
 1,1,1,1,1,-1,0,1,1,1,1,1,-1,0,0,-1,-1,-1,0,1,1,1,1,-1,1,1,1,1,-1,-1,1
-1,1,-1,1,1,-1,1,-1,-1,1,1,1,1,-1,-1,-1,-1,-1,0,1,1,1,1,-1,-1,-1,-1,1,0,-1,-1
+1,1,-1,1,1,-1,1,-1,-1,1,1,1,1,-1,-1,-1,-1,-1,0,1,1,1,1,-1,-1,-1,-1,,0,-1,-1
 -1,1,-1,1,-1,-1,0,0,1,1,1,-1,-1,-1,1,-1,1,1,0,-1,1,-1,1,1,-1,-1,-1,1,0,1,-1
 1,1,-1,1,1,-1,0,-1,1,1,1,1,-1,-1,-1,-1,1,1,0,1,1,1,1,-1,-1,0,-1,1,1,1,-1
 1,1,-1,1,1,1,-1,1,-1,1,1,-1,1,0,1,1,1,1,0,1,1,1,1,1,-1,1,-1,1,-1,1,1

network_security/exceptions/exception.py

@@ -0,0 +1,16 @@
+from network_security.logging import logger
+import sys
+
+class NetworkSecurityException(Exception):
+    def __init__(self, error_message, error_details: sys):
+        self.error_message = error_message
+        _, _, exc_tb = error_details.exc_info()
+
+        self.lineno = exc_tb.tb_lineno
+        self.file_name = exc_tb.tb_frame.f_code.co_filename
+
+    def __str__(self):
+        return "Error occurred in python script name [{0}] at line number [{1}] error message [{2}]".format(
+            self.file_name, self.lineno, str(self.error_message)
+        )
+    

network_security/logging/logger.py

@@ -0,0 +1,18 @@
+from datetime import datetime
+import logging
+import os, sys
+
+LOG_FILE = f"{datetime.now().strftime('%m_%d_%Y_%H_%M_%S')}.log"
+
+logs_path = os.path.join(os.getcwd(), "Logs", LOG_FILE)
+os.makedirs(logs_path, exist_ok=True)
+
+LOGS_FILE_PATH = os.path.join(logs_path, LOG_FILE)
+
+logging.basicConfig(
+    filename = LOGS_FILE_PATH,
+    format = "[ %(asctime)s ] %(lineno)d %(name)s - %(levelname)s - %(message)s",
+    level = logging.INFO
+)
+
+

push_data.py

@@ -0,0 +1,59 @@
+from dotenv import load_dotenv
+from network_security.logging.logger import logging
+from network_security.exceptions.exception import NetworkSecurityException
+
+import os, sys
+import json
+import certifi
+import pandas as pd
+import numpy as np
+import pymongo
+
+load_dotenv()
+
+# MongoDB URL and certification for valid connection
+MONGO_DB_URL = os.getenv("MONGODB_URI")
+ca = certifi.where()
+
+
+# implementing the ETL pipeline
+class NetworkDataExtraction():
+    def __init__(self):
+        try:
+            pass
+        except Exception as e:
+            raise NetworkSecurityException(e, sys)
+        
+    def csv_to_json_convertor(self, file_path):
+        try:
+            data = pd.read_csv(file_path)
+
+            # getting rid of the index
+            data.reset_index(drop=True, inplace=True)
+
+            records = list((json.loads(data.T.to_json()).values()))
+            return records
+        except Exception as e:
+            raise NetworkSecurityException(e, sys)
+        
+    def push_data_to_mongodb(self, records, database, collection):
+        try:
+            self.mongo_client = pymongo.MongoClient(MONGO_DB_URL)
+            db = self.mongo_client[database]
+            collection_obj = db[collection]
+            collection_obj.insert_many(records)
+
+            return len(records)
+
+        except Exception as e:
+            raise NetworkSecurityException(e, sys)
+        
+if __name__ == "__main__":    
+    FILE_PATH = "network_data/phisingData.csv"
+    DATABASE = "aryan"
+    Collection = "NetworkData"
+
+    networkobj = NetworkDataExtraction()
+    
+    records = networkobj.csv_to_json_convertor(file_path=FILE_PATH)
+    len_records = networkobj.push_data_to_mongodb(records=records, database=DATABASE, collection=Collection)

requirements.txt

@@ -3,5 +3,7 @@ pandas
 numpy
 pymongo
 certifi
+pymongo[srv]==3.12
+pymongo
 
 # -e .

test_mongodb.py

@@ -0,0 +1,13 @@
+from pymongo.mongo_client import MongoClient
+
+uri = "mongodb+srv://aryan:aryan@cluster0.mley4zy.mongodb.net/?retryWrites=true&w=majority&appName=Cluster0"
+
+# Create a new client and connect to the server
+client = MongoClient(uri)
+
+# Send a ping to confirm a successful connection
+try:
+    client.admin.command("ping")
+    print("Pinged your deployment. You successfully connected to MongoDB!")
+except Exception as e:
+    print(e)