finished making file structure for data+ingestion

Author: pycoder49

network_security/components/data_ingestion.py

@@ -0,0 +1,29 @@
+"""
+Training Pipeline Constants
+
+This module contains all constants used throughout the training pipeline.
+Constants are organized by functionality with descriptive prefixes.
+"""
+
+    
+"""
+Data Ingestion related constants start with DATA_INGESTION_* prefix
+"""
+DATA_INGESTION_COLLECTION_NAME: str = "NetworkData"
+DATA_INGESTION_DATABASE_NAME: str = "aryan"
+DATA_INGESTION_DIR_NAME: str = "data_ingestion"
+DATA_INGESTION_FEATURE_STORE_DIR: str = "feature_store"
+DATA_INGESTION_INGESTED_DIR: str = "ingested"
+DATA_INGESTION_TRAIN_TEST_SPLIT_RATION: float = 0.2
+
+
+"""
+Defining common constant variables for training pipeline
+"""
+TARGET_COLUMN: str = "Result"
+PIPELINE_NAME: str = "NetworkSecurity"
+ARTIFACT_DIR: str = "Artifacts"
+FILE_NAME: str = "phishingData.csv"
+
+TRAIN_FILE_NAME: str = "train.csv"
+TEST_FILE_NAME: str = "test.csv" 

network_security/constants/training_pipeline/__init__.py

@@ -0,0 +1,39 @@
+from datetime import datetime
+from network_security.constants import training_pipeline
+
+import os, sys
+
+
+class TrainingPipelineConfig:
+    def __init__(self, timestamp = datetime.now()):
+        timestamp = timestamp.strftime("%m_%d_%Y_%H_%M_%S")
+        self.pipeline_name = training_pipeline.PIPELINE_NAME
+        self.artifact_name = training_pipeline.ARTIFACT_DIR
+        self.artifact_dir = os.join(self.artifact_name, timestamp)
+        self.timestamp: str = timestamp
+
+
+class DataIngestionConfig:
+    def __init__(self, training_pipeline_config:TrainingPipelineConfig):
+        self.data_ingestion_dir: str = os.path.join(
+            training_pipeline_config.artifact_dir, 
+            training_pipeline.DATA_INGESTION_DIR_NAME
+        )
+        self.feature_store_file_path: str = os.path.join(
+            self.data_ingestion_dir, 
+            training_pipeline.DATA_INGESTION_FEATURE_STORE_DIR, 
+            training_pipeline.FILE_NAME
+        )
+        self.train_file_path: str = os.path.join(
+            self.data_ingestion_dir, 
+            training_pipeline.DATA_INGESTION_INGESTED_DIR, 
+            training_pipeline.TRAIN_FILE_NAME
+        )
+        self.test_file_path: str = os.path.join(
+            self.data_ingestion_dir, 
+            training_pipeline.DATA_INGESTION_INGESTED_DIR, 
+            training_pipeline.TEST_FILE_NAME
+        )
+        self.train_test_split_ratio: float = training_pipeline.DATA_INGESTION_TRAIN_TEST_SPLIT_RATION
+        self.collection_name: str = training_pipeline.DATA_INGESTION_COLLECTION_NAME
+        self.db_name: str = training_pipeline.DATA_INGESTION_DATABASE_NAME

network_security/entity/config_entity.py

@@ -56,4 +56,5 @@ def push_data_to_mongodb(self, records, database, collection):
     networkobj = NetworkDataExtraction()
 
     records = networkobj.csv_to_json_convertor(file_path=FILE_PATH)
-    len_records = networkobj.push_data_to_mongodb(records=records, database=DATABASE, collection=Collection)
+    len_records = networkobj.push_data_to_mongodb(records=records, database=DATABASE, collection=Collection)
+