Merge pull request #10 from SunbirdAI/development

Integration of postgis and postgresql

Author: PatrickCmd

README.md

@@ -34,13 +34,76 @@ Makefile            # Dev workflow commands
 pip install -r requirements.txt
 ```
 
-### 2. Run Tests
+### 2. (Optional) Stand Up PostGIS via Docker
+
+If you want to use the PostGIS-backed analyzer, bring up a local PostGIS container:
+
+```sh
+docker run \
+  --name suntrace-postgis \
+  -e POSTGRES_DB=suntrace \
+  -e POSTGRES_USER=pguser \
+  -e POSTGRES_PASSWORD=pgpass \
+  -p 5432:5432 \
+  -d postgis/postgis:15-3.4
+```
+
+Wait for the database to accept connections, then ensure the PostGIS extensions exist (the analyzer can do this for you, or run once inside the container):
+
+```sh
+docker exec -it suntrace-postgis psql -U pguser -d suntrace -c "CREATE EXTENSION IF NOT EXISTS postgis;"
+docker exec -it suntrace-postgis psql -U pguser -d suntrace -c "CREATE EXTENSION IF NOT EXISTS postgis_topology;"
+```
+
+### 3. Load Project Data into PostGIS
+
+With the container running and data available under `data/`, load all vector layers:
+
+```sh
+python scripts/load_to_postgis.py \
+  --data-dir data \
+  --db-uri postgresql://pguser:pgpass@localhost:5432/suntrace
+```
+
+Requirements for the loader:
+
+- `ogr2ogr` (GDAL) must be installed on the host running the script.
+```sh
+brew install gdal
+```
+- Python deps: `geopandas`, `pandas`, `sqlalchemy`.
+
+> The script scans `data/`, `data/lamwo_sentinel_composites/`, `data/viz_geojsons/`, and `data/sample_region_mudu/`, writing tables such as `public.lamwo_buildings`, `public.lamwo_roads`, `public.lamwo_tile_stats_ee_biomass`, etc. Ensure filenames follow the repository defaults so table names match the analyzer’s expectations.
+
+If you need the joined tile view, run inside a Python shell after the load:
+
+```python
+from utils.GeospatialAnalyzer2 import GeospatialAnalyzer2
+analyzer = GeospatialAnalyzer2()
+analyzer.create_joined_tiles(
+    tile_stats_table='public.lamwo_tile_stats_ee_biomass',
+    plain_tiles_table='public.lamwo_grid'
+)
+```
+
+### 4. Configure the App to Use PostGIS
+
+Set the following in `.env` (already present in this repo by default):
+
+```
+SUNTRACE_USE_POSTGIS=1
+SUNTRACE_DATABASE_URI=postgresql+psycopg://pguser:pgpass@localhost:5432/suntrace
+```
+
+Restart the app after changing the env file. The factory will log which analyzer was initialized.
+
+### 5. Run Tests
 
 ```sh
 make test
 ```
 
-### 3. Start the Application (Local)
+### 6. Start the Application (Local)
 
 ```sh
 uvicorn main:app --reload
@@ -76,7 +139,7 @@ docker logs -f suntracte
 docker-compose up -d --build
 ```
 
-### 5. Access Frontend
+### 7. Access Frontend
 
 Open [http://localhost:8080](http://localhost:8080) in your browser.
 

app/core/config.py

@@ -6,6 +6,7 @@
 from functools import lru_cache
 
 from pydantic_settings import BaseSettings
+from typing import Optional
 
 
 class Settings(BaseSettings):
@@ -50,6 +51,12 @@ class Settings(BaseSettings):
     # Building sample limit for performance
     BUILDING_SAMPLE_LIMIT: int = 2000
 
+    # Database settings
+    SUNTRACE_USE_POSTGIS: bool = False
+    SUNTRACE_DATABASE_URI: Optional[str] = None
+
+
+
     class Config:
         env_file = ".env"
         case_sensitive = True

app/services/geospatial.py

@@ -2,7 +2,6 @@
 Geospatial analysis service
 """
 
-import json
 import time
 
 from uuid import uuid4
@@ -62,7 +61,7 @@ def get_map_layers(self) -> MapLayersResponse:
         try:
             # Get bounds for the map from the plain tiles
             logger.info("Fetching map layers data.")
-            bounds = self.analyzer._plain_tiles_gdf.total_bounds.tolist()
+            bounds = self.analyzer.get_layer_bounds("tiles")
 
             # Calculate center coordinates
             center = Coordinates(
@@ -78,26 +77,20 @@ def get_map_layers(self) -> MapLayersResponse:
             )
 
             # Convert minigrids to GeoJSON
-            candidate_minigrids_geo = json.loads(
-                self.analyzer._candidate_minigrids_gdf.to_crs("EPSG:4326").to_json()
-            )
+            candidate_minigrids_geo = self.analyzer.get_layer_geojson("candidate_minigrids")
 
             # Get a sample of buildings to avoid performance issues
-            total_buildings = len(self.analyzer._buildings_gdf)
-            building_sample = (
-                self.analyzer._buildings_gdf.sample(
-                    min(settings.BUILDING_SAMPLE_LIMIT, total_buildings)
-                )
-                if total_buildings > settings.BUILDING_SAMPLE_LIMIT
-                else self.analyzer._buildings_gdf
+            total_buildings = self.analyzer.get_layer_count("buildings")
+            buildings_geo = self.analyzer.get_layer_geojson(
+                "buildings", sample=settings.BUILDING_SAMPLE_LIMIT
             )
-            buildings_geo = json.loads(building_sample.to_crs("EPSG:4326").to_json())
+            sampled_buildings = len(buildings_geo.get("features", []))
 
             # Create metadata
             metadata = {
                 "total_buildings": total_buildings,
-                "sampled_buildings": len(building_sample),
-                "total_minigrids": len(self.analyzer._candidate_minigrids_gdf),
+                "sampled_buildings": sampled_buildings,
+                "total_minigrids": self.analyzer.get_layer_count("candidate_minigrids"),
                 "coordinate_system": self.analyzer.target_geographic_crs,
             }
             logger.info("Map layers data fetched successfully.")

configs/paths.py

@@ -6,7 +6,7 @@
 
 # Data paths
 DATA_DIR = PROJECT_ROOT / "data"
-TILE_STATS_PATH = DATA_DIR / "Lamwo_Tile_Stats_EE.csv"
+TILE_STATS_PATH = DATA_DIR / "Lamwo_Tile_Stats_EE_biomass.csv"
 PLAIN_TILES_PATH = DATA_DIR / "lamwo_sentinel_composites" / "lamwo_grid.geojson"
 
 VIZUALIZATION_DIR = DATA_DIR / "viz_geojsons"

configs/stats.py

@@ -0,0 +1,14 @@
+"""Shared configuration for analyzer statistic columns."""
+
+DEFAULT_TILE_STAT_COLUMNS = [
+    "ndvi_mean",
+    "ndvi_med",
+    "ndvi_std",
+    "evi_med",
+    "elev_mean",
+    "slope_mean",
+    "par_mean",
+    "rain_total_mm",
+    "rain_mean_mm_day",
+    "cloud_free_days",
+]

main.py

@@ -1,10 +1,11 @@
-"""
-Main FastAPI application entry point
-"""
+"""Main FastAPI application entry point."""
 
+import logging
 import sys
 from pathlib import Path
+
 import uvicorn
+
 from app.api.deps import create_application
 from app.core.config import get_settings
 
@@ -15,6 +16,9 @@
 sys.path.insert(0, str(project_root))
 
 
+logging.basicConfig(level=logging.INFO)
+logging.getLogger("GeospatialAnalyzer2").setLevel(logging.INFO)
+
 settings = get_settings()
 app = create_application()
 

notebooks/Precomputed Aggregates.ipynb

@@ -825,15 +825,6 @@
     "stats_table.toList(10).getInfo()"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "id": "D4Wk_r2_4Woi"
-   },
-   "outputs": [],
-   "source": []
-  },
   {
    "cell_type": "code",
    "execution_count": null,