FlugHafen pipeline - first commit

.github/workflows/project_feedback.yml

@@ -0,0 +1,31 @@
+name: Run Tests
+run-name: ${{ github.actor }} is running tests
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  test:
+    runs-on: ubuntu-20.04
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.11
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+
+      # project and exercise feedbacks
+      - name: Make test executable
+        run: chmod +x /home/runner/work/made-template/made-template/project/tests.sh
+
+      - name: Run project tests 
+        run: /home/runner/work/made-template/made-template/project/tests.sh

.gitignore

@@ -1,3 +1,8 @@
 .DS_Store
 /data/*
-!/data/.gitkeep
+!/data/.gitkeep
+.mypy_cache/
+temp_dir/
+project/__pycache__/
+*.pyc
+*.sqlite

LICENSE

@@ -0,0 +1,24 @@
+Creative Commons Attribution 4.0 International License (CC BY 4.0)
+
+Copyright (c) [2025][FAU]
+
+You are free to:
+
+- Share — copy and redistribute the material in any medium or format
+- Adapt — remix, transform, and build upon the material for any purpose, even commercially.
+
+The licensor cannot revoke these freedoms as long as you follow the license terms.
+
+Under the following terms:
+
+- Attribution — You must give appropriate credit, provide a link to the license, and indicate if changes were made. You may do so in any reasonable manner, but not in any way that suggests the licensor endorses you or your use.
+
+No additional restrictions — You may not apply legal terms or technological measures that legally restrict others from doing anything the license permits.
+
+Notices:
+
+You do not have to comply with the license for elements of the material in the public domain or where your use is permitted by an exception or limitation.
+
+No warranties are given. The license may not give you all of the permissions necessary for your intended use. For example, other rights such as publicity, privacy, or moral rights may limit how you use the material.
+
+For more information, visit: https://creativecommons.org/licenses/by/4.0/

README.md

@@ -1,37 +1,55 @@
-# Methods of Advanced Data Engineering Template Project
+## A comparative analysis of North America and Latin America & Caribbean’s contribution to renewable energy; and their adoption to climate mitigation efforts from 1960 to 2023
 
-This template project provides some structure for your open data project in the MADE module at FAU.
-This repository contains (a) a data science project that is developed by the student over the course of the semester, and (b) the exercises that are submitted over the course of the semester.
+This report presents a comparative analysis of the contributions and progress of North America and Latin America & Caribbean in renewable energy
+adoption and climate mitigation efforts over the period 1960 to 2023.
 
-To get started, please follow these steps:
-1. Create your own fork of this repository. Feel free to rename the repository right after creation, before you let the teaching instructors know your repository URL. **Do not rename the repository during the semester**.
+## Data License
+The World Bank strives to enhance public access to and use of data that it collects and publishes. The data are organized in datasets listed in The World Bank Data Catalog (the “Datasets”).[Creative Commons Attribution 4.0 International License (CC BY 4.0)](https://www.worldbank.org/en/about/legal/terms-of-use-for-datasets) It spans multiple countries, enabling both regional and country-level analysis. The data spans from 1960 to 2023. 
 
-## Project Work
-Your data engineering project will run alongside lectures during the semester. We will ask you to regularly submit project work as milestones, so you can reasonably pace your work. All project work submissions **must** be placed in the `project` folder.
+| Region                  | Data Source                                                                 |
+|-------------------------|-----------------------------------------------------------------------------|
+| North America           | [Download](https://api.worldbank.org/v2/en/country/NAC?downloadformat=csv) |
+| Latin America & Caribbean | [Download](https://api.worldbank.org/v2/en/country/LCN?downloadformat=csv) |
+| Metadata URL            | [Link](https://data.worldbank.org/country)                                 |
+| Data Type               | csv                                                                        |
 
-### Exporting a Jupyter Notebook
-Jupyter Notebooks can be exported using `nbconvert` (`pip install nbconvert`). For example, to export the example notebook to HTML: `jupyter nbconvert --to html examples/final-report-example.ipynb --embed-images --output final-report.html`
 
 
-## Exercises
-During the semester you will need to complete exercises using [Jayvee](https://github.com/jvalue/jayvee). You **must** place your submission in the `exercises` folder in your repository and name them according to their number from one to five: `exercise<number from 1-5>.jv`.
+### Description of Directories and Files
 
-In regular intervals, exercises will be given as homework to complete during the semester. Details and deadlines will be discussed in the lecture, also see the [course schedule](https://made.uni1.de/).
+- **data**: Contains all the datasets used in the project.
+- **examples**: Includes example files provided by the course facilitators for reference.
+- **exercises**: Contains solutions to exercises from the WinSem2024 course.
+- **project**: This directory holds all the analysis work done during the semester, including reports, scripts, and temporary files.
+  - **analysis-report.pdf**: The final analysis report.
+  - **data-report.pdf**: A report on the data used in the project.
+  - **final_report.ipynb**: The final report in Jupyter Notebook format.
+  - **pipeline.sh**: A shell script for running the data processing pipeline.
+  - **prepare_data.py**: A Python script for preparing the data.
+  - **project-plan-example.md**: An example of a project plan.
+  - **run_tests.py**: A Python script for running tests.
+  - **temp_dir**: A temporary directory for intermediate files.
+  - **tests.sh**: A shell script for running tests.
+  - **version_check.py**: A Python script for checking package versions.
+project.
+- **requirements.txt**: A file listing the dependencies required for the project.
 
-### Exercise Feedback
-We provide automated exercise feedback using a GitHub action (that is defined in `.github/workflows/exercise-feedback.yml`). 
+### Usage
 
-To view your exercise feedback, navigate to Actions → Exercise Feedback in your repository.
+1. Clone the repository:
+   ```bash
+   git clone <repository-url>
+2. Create virtual environment
+	```bash
+	python -m venv <env_name>
+3. Install the dependencies
+	```bash 
+	pip install -r requirements.txt
+4. Run the data pipeline using `project/pipeline.sh`
+4. You can now open `final_report.ipynb` and explore.
 
-The exercise feedback is executed whenever you make a change in files in the `exercise` folder and push your local changes to the repository on GitHub. To see the feedback, open the latest GitHub Action run, open the `exercise-feedback` job and `Exercise Feedback` step. You should see command line output that contains output like this:
 
-```sh
-Found exercises/exercise1.jv, executing model...
-Found output file airports.sqlite, grading...
-Grading Exercise 1
-	Overall points 17 of 17
-	---
-	By category:
-		Shape: 4 of 4
-		Types: 13 of 13
-```
+
+## License
+
+This project is licensed under the Creative Commons Attribution 4.0 International License (CC BY 4.0). You are free to share and adapt the work as long as you provide appropriate credit. For more details, see the [LICENSE](LICENSE) file or visit [Creative Commons](https://creativecommons.org/licenses/by/4.0/).

exercises/exercise1.jv

@@ -0,0 +1,74 @@
+pipeline FlugHafen{
+
+    //1. FlugHafen Pipeline connects the blocks via pipes to extract data from a CSV file
+    // in the web to a SQLite file sink.    
+    FlugHafenHttpExtractor 
+        -> FlugHafenTextFileInterpreter;
+
+    //2. The FlugHafenTextFileInterpreter output is used as input for the FlugHafenCsvFileInterpreter
+    // block which is then used as input for the FlugHafenDataSelector block.
+    FlugHafenTextFileInterpreter
+        -> FlugHafenCsvFileInterpreter
+        // -> FlugHafenDatabaseWriter
+        -> FlugHafenDataSelector
+        -> FlugHafenTableInterpreter
+        -> FlugHafenLoader;
+
+    //3. The FlugHafenHttpExtractor block is of type HttpExtractor and the URL is specified.
+    block FlugHafenHttpExtractor oftype HttpExtractor {
+        // URL of the data source
+        url: "https://opendata.rhein-kreis-neuss.de/api/explore/v2.1/catalog/datasets/rhein-kreis-neuss-flughafen-weltweit/exports/csv?lang=en&timezone=Europe%2FBerlin&use_labels=true&delimiter=%3B";
+    }
+
+    //4. The FlugHafenTextFileInterpreter block is of type TextFileInterpreter.
+    block FlugHafenTextFileInterpreter oftype TextFileInterpreter { }
+
+    //5. Since we only need a specific range of the data, we use the CellRangeSelector block.
+    block FlugHafenDataSelector oftype CellRangeSelector {
+        // The name of the sheet
+        select: range A1:I*;
+    }
+
+    //6. The FlugHafenCsvFileInterpreter block is of type CSVInterpreter and the delimiter is specified.
+    block FlugHafenCsvFileInterpreter oftype CSVInterpreter {
+        // Specify the separator as a semicolon for the CSV
+        delimiter: ';';
+    }
+
+    // block FlugHafenDatabaseWriter oftype DatabaseWriter {
+    //     // The name of the database
+    //     database: "flughafen.db";
+    //     // The name of the table
+    //     table: "flughafen";
+    // }
+
+    //7. The FlugHafenTableInterpreter block is of type TableInterpreter and the necessary columns are specified.
+    block FlugHafenTableInterpreter oftype TableInterpreter {
+        // The first row contains the header
+        header: true;
+        // The columns of the table
+        columns: [
+            "Lfd. Nummmer" oftype integer,
+            "Name des Flughafens" oftype text,
+            "Ort" oftype text,
+            "Land" oftype text,
+            "IATA" oftype text,
+            "ICAO" oftype text,
+            "Latitude" oftype decimal,
+            "Longitude" oftype decimal,
+            "Altitude" oftype integer,
+
+        ];
+    }
+
+    //8. Finally the FlugHafenLoader block is of type SQLiteLoader and the table name and file name are specified.
+    block FlugHafenLoader oftype SQLiteLoader {
+        // The name of the table
+        table: "airports";
+        // The name of the file
+        file: "airports.sqlite";
+     }
+
+
+}
+

exercises/exercise2.jv

@@ -0,0 +1,85 @@
+pipeline TreePlanting{
+
+    //1. TreePlanting Pipeline connects the blocks via pipes to extract data from a CSV file
+    // in the web to a SQLite file sink.    
+    TreePlantingHttpExtractor 
+        -> TreePlantingTextFileInterpreter;
+
+    //2. The TreePlantingTextFileInterpreter output is used as input for the TreePlantingCsvFileInterpreter
+    // block which is then used as input for the TreePlantingDataSelector block.
+    TreePlantingTextFileInterpreter
+        -> TreePlantingCsvFileInterpreter
+        // -> TreePlantingDatabaseWriter
+        -> TreePlantingBaumartDeutschDeleter
+        -> TreePlantingTableInterpreter
+        -> TreePlantingLoader;
+
+    //3. The TreePlantingHttpExtractor block is of type HttpExtractor and the URL is specified.
+    block TreePlantingHttpExtractor oftype HttpExtractor {
+        // URL of the data source
+        url: "https://opendata.rhein-kreis-neuss.de/api/v2/catalog/datasets/stadt-neuss-herbstpflanzung-2023/exports/csv";
+    }
+
+    //4. The TreePlantingTextFileInterpreter block is of type TextFileInterpreter.
+    block TreePlantingTextFileInterpreter oftype TextFileInterpreter { }
+
+    //6. The TreePlantingCsvFileInterpreter block is of type CSVInterpreter and the delimiter is specified.
+    block TreePlantingCsvFileInterpreter oftype CSVInterpreter {
+        // Specify the separator as a semicolon for the CSV
+        delimiter: ';';
+    }
+
+    //5. The TreePlantingBaumartDeutschDeleter block is of type ColumnDeleter and the column to be deleted is specified.
+    block TreePlantingBaumartDeutschDeleter oftype ColumnDeleter {
+        // The name of the column
+        delete: [column E];
+    }
+
+    //7. The TreePlantingTableInterpreter block is of type TableInterpreter and the necessary columns are specified.
+    block TreePlantingTableInterpreter oftype TableInterpreter {
+        // The first row contains the header
+        header: true;
+        // The columns of the table
+        columns: [
+            "lfd_nr" oftype integer,
+            "stadtteil" oftype Vogelsang,
+            "standort" oftype text,
+            "baumart_botanisch" oftype text,
+            "id" oftype GeoCoordinate,
+            "baumfamilie" oftype text,
+
+        ];
+    }
+
+    block TreePlantingLoader oftype SQLiteLoader {
+        // The name of the table
+        table: "trees";
+        // The name of the file
+        file: "trees.sqlite";
+     }
+
+
+    valuetype Vogelsang oftype text {
+        // The value of the column
+        constraints: [
+            // only allow column values that start with "Vogelsang"
+            VogelsangStadteil
+        ];
+    }
+
+    valuetype GeoCoordinate oftype text {
+        // The value of the column
+        constraints: [
+            // only allow column values that match the pattern of a geo coordinate
+            Geopoints
+        ];
+    }
+
+    constraint VogelsangStadteil on text: value matches(/^Vogelsang*/);
+    //8. Finally the TreePlantingLoader block is of type SQLiteLoader and the table name and file name are specified.
+
+    constraint Geopoints on text: value matches(/^\d{1,3}\.\d+,\s*\d{1,3}\.\d+$/);
+
+
+}
+