From a037e6b4fd771e25b870754f8d23e01b9c6160cd Mon Sep 17 00:00:00 2001 From: Felix Delattre Date: Wed, 4 Mar 2026 18:50:29 +0100 Subject: [PATCH] Foo --- README.md | 4 +- deploy.sh | 3 +- metrics_indices.py | 48 +++-- metrics_stats.py | 4 +- postprocessing.py | 17 +- preparation.py | 20 +++ run.py | 53 +++--- webapp/fusion.html | 361 +++++++++++++++++++++++++++++++++++++ webapp/index.html | 70 ++++---- webapp/postprocessed.html | 367 ++++++++++++++++++++++++++++++++++++++ webapp/prepared.html | 352 ++++++++++++++++++++++++++++++++++++ webapp/preselection.html | 34 ++-- 12 files changed, 1237 insertions(+), 96 deletions(-) create mode 100644 webapp/fusion.html create mode 100644 webapp/postprocessed.html create mode 100644 webapp/prepared.html diff --git a/README.md b/README.md index fede942..2ce1af7 100644 --- a/README.md +++ b/README.md @@ -70,14 +70,14 @@ data/ ## Web Viewer -Run a local HTTP server to view the web interface: +Run a local HTTP server from the **webapp** directory: ```bash cd webapp python3 -m http.server 8000 ``` -Then open `http://localhost:8000/webapp` in your browser to visualize NDVI time series and compare S2, S3, and fusion outputs. +Then open `http://localhost:8000/` in your browser. Data is served via the `webapp/data` symlink. ## License diff --git a/deploy.sh b/deploy.sh index 7ed294c..4e5743e 100755 --- a/deploy.sh +++ b/deploy.sh @@ -55,8 +55,9 @@ if [ -f satellite-fusion-web.service ]; then systemctl daemon-reload fi -# Create data directory +# Create data directory and webapp/data symlink mkdir -p $DATA_DIR +ln -sf ../data $APP_DIR/webapp/data ENDSSH echo "Setup complete!" ;; diff --git a/metrics_indices.py b/metrics_indices.py index d0c00b3..a83dae3 100644 --- a/metrics_indices.py +++ b/metrics_indices.py @@ -76,7 +76,7 @@ def _create_timeseries_for_dir(input_dir, output_dir, site_position, source_name continue filename = input_file.name - parts = filename.replace(".geotiff", "").split("_") + parts = filename.replace(".geotiff", "").replace(".tif", "").split("_") date_str = None for part in parts: @@ -316,7 +316,7 @@ def _create_gcc_timeseries_for_dir(input_dir, output_dir, site_position, source_ continue filename = input_file.name - parts = filename.replace(".geotiff", "").split("_") + parts = filename.replace(".geotiff", "").replace(".tif", "").split("_") date_str = None for part in parts: @@ -440,12 +440,15 @@ def _get_bands_from_original(input_file, site_position): return None -def _create_s2_bands_timeseries_for_dir(input_dir, output_dir, site_position): - print(f"[S2-BANDS] Creating timeseries.json...") +def _create_bands_timeseries_for_dir(input_dir, output_dir, site_position, source_name, pattern="*.geotiff"): + print(f"[BANDS-{source_name}] Creating timeseries.json...") timeseries = [] - for f in sorted(input_dir.glob("*.geotiff")): - date_str = f.stem.split("_")[0] - if len(date_str) != 8 or not date_str.isdigit(): + for f in sorted(input_dir.glob(pattern)): + if "DIST_CLOUD" in f.name: + continue + parts = f.name.replace(".geotiff", "").replace(".tif", "").split("_") + date_str = next((p for p in parts if len(p) == 8 and p.isdigit()), None) + if not date_str: continue date = datetime.strptime(date_str, "%Y%m%d").isoformat() bands = _get_bands_from_original(f, site_position) @@ -453,14 +456,33 @@ def _create_s2_bands_timeseries_for_dir(input_dir, output_dir, site_position): timeseries.sort(key=lambda x: x["date"]) output_dir.mkdir(parents=True, exist_ok=True) (output_dir / "timeseries.json").write_text(json.dumps(timeseries, indent=2)) - print(f"[S2-BANDS] Saved: {output_dir / 'timeseries.json'} ({len(timeseries)} entries)") + print(f"[BANDS-{source_name}] Saved: {output_dir / 'timeseries.json'} ({len(timeseries)} entries)") -def create_s2_bands_timeseries_post_process(season, site_position, site_name): +def create_prepared_fusion_timeseries(season, site_position, site_name): + """Generate NDVI, GCC, and band timeseries for prepared S2/S3 and fusion outputs.""" + for strategy in ["aggressive", "nonaggressive"]: + base = Path(f"data/{site_name}/{season}/prepared_{strategy}") + for source in ["s2", "s3"]: + inp = base / source + if inp.exists(): + _create_timeseries_for_dir(inp, base / "ndvi" / source, site_position, f"PREPARED-{source.upper()}-{strategy}", "*.tif") + _create_gcc_timeseries_for_dir(inp, base / "gcc" / source, site_position, f"PREPARED-{source.upper()}-{strategy}", "*.tif") + _create_bands_timeseries_for_dir(inp, base / "bands" / source, site_position, f"PREPARED-{source.upper()}-{strategy}", "*.tif") + for sig, fusion_sub in [(None, "fusion"), (30, "fusion_sigma30")]: + inp = base / fusion_sub + if inp.exists(): + _create_timeseries_for_dir(inp, base / "ndvi" / fusion_sub, site_position, f"FUSION-{strategy}-σ{sig or 20}", "*.tif") + _create_gcc_timeseries_for_dir(inp, base / "gcc" / fusion_sub, site_position, f"FUSION-{strategy}-σ{sig or 20}", "*.tif") + _create_bands_timeseries_for_dir(inp, base / "bands" / fusion_sub, site_position, f"FUSION-{strategy}-σ{sig or 20}", "*.tif") + + +def create_bands_timeseries_post_process(season, site_position, site_name): for strategy in ["aggressive", "nonaggressive"]: for sigma in [20, 30]: processed_dir = f"processed_{strategy}_sigma{sigma}" - input_dir = Path(f"data/{site_name}/{season}/{processed_dir}/s2/") - output_dir = Path(f"data/{site_name}/{season}/{processed_dir}/s2_bands/") - if input_dir.exists(): - _create_s2_bands_timeseries_for_dir(input_dir, output_dir, site_position) + base = Path(f"data/{site_name}/{season}/{processed_dir}") + for source in ["s2", "s3", "fusion"]: + inp, out = base / source, base / "bands" / source + if inp.exists(): + _create_bands_timeseries_for_dir(inp, out, site_position, f"POST-{source.upper()}-{strategy}-σ{sigma}", "*.geotiff") diff --git a/metrics_stats.py b/metrics_stats.py index c770a35..fa82389 100644 --- a/metrics_stats.py +++ b/metrics_stats.py @@ -246,7 +246,7 @@ def calculate_scenario_metrics(season, site_name, strategy, sigma, site_position # Load timeseries fusion_ts_path = base / processed_dir / "gcc" / "fusion" / "timeseries.json" - phenocam_ts_path = base / "raw" / "phenocam" / "timeseries.json" + phenocam_ts_path = base / "raw" / "phenocam" / "phenocam_gcc.json" fusion_ts = load_timeseries(fusion_ts_path) phenocam_ts = load_timeseries(phenocam_ts_path) @@ -270,7 +270,7 @@ def calculate_all_metrics(season, site_name, site_position): base = Path(f"data/{site_name}/{season}") # Load phenocam timeseries once (same for all scenarios) - phenocam_ts_path = base / "raw" / "phenocam" / "timeseries.json" + phenocam_ts_path = base / "raw" / "phenocam" / "phenocam_gcc.json" phenocam_ts = load_timeseries(phenocam_ts_path) if not phenocam_ts: diff --git a/postprocessing.py b/postprocessing.py index eed4bc0..cb07eb3 100644 --- a/postprocessing.py +++ b/postprocessing.py @@ -96,13 +96,20 @@ def process_cropped(season, site_position, site_name, cleaning_strategy="aggress print("[PROCESS] Completed") -def process_all_scenarios(season, site_position, site_name): - """Process all 4 EFAST scenarios.""" +def post_process_all_scenarios(season, site_position, site_name): + """Crop fusion/S2/S3 to valid pixels for all 4 scenarios.""" for strategy in ["aggressive", "nonaggressive"]: for sigma in [None, 30]: process_cropped(season, site_position, site_name, cleaning_strategy=strategy, sigma=sigma) -# Aliases -postprocess = process_cropped -postprocess_all_scenarios = process_all_scenarios +def post_process_timeseries(season, site_position, site_name): + """Generate NDVI, GCC, and S2 bands timeseries for all 4 scenarios.""" + from metrics_indices import ( + create_ndvi_timeseries_post_process, + create_gcc_timeseries_post_process, + create_bands_timeseries_post_process, + ) + create_ndvi_timeseries_post_process(season, site_position, site_name) + create_gcc_timeseries_post_process(season, site_position, site_name) + create_bands_timeseries_post_process(season, site_position, site_name) diff --git a/preparation.py b/preparation.py index 02b2042..f4791bc 100644 --- a/preparation.py +++ b/preparation.py @@ -71,6 +71,25 @@ def _reproject_raster_to_target( rio_shutil.copy(vrt, dst_path, **profile) +def _rescale_dist_cloud_for_small_roi(s2_output_dir): + """Rescale DIST_CLOUD when max distance ≤1 so EFAST fusion gets valid weights. + + EFAST uses wo_i = (distance - 1) / D; values ≤1 yield zero/NaN weights. In small + ROIs (e.g. PhenoCam sites, 7×4 LR grid), distance_transform_edt never exceeds 1. + Scale non-zero values to ≥2 so fusion can produce non-NaN output. + """ + for dc_path in s2_output_dir.glob("*DIST_CLOUD.tif"): + with rasterio.open(dc_path, "r") as src: + d = src.read(1) + d_max = float(np.nanmax(d)) + if d_max <= 1: + # Map (0, 1] -> (0, 2] so (d-1)/15 gives positive weight + d_scaled = np.where(d > 0, 2.0, d).astype(np.float32) + with rasterio.open(dc_path, "r+") as dst: + dst.write(d_scaled, 1) + print(f"[S2-PREP] Rescaled DIST_CLOUD for {dc_path.name} (max was {d_max})") + + def prepare_s2(season, site_position, site_name, cleaning_strategy="aggressive", date_range=None): lat, lon = site_position s2_dir = Path(f"data/{site_name}/{season}/raw/s2/") @@ -120,6 +139,7 @@ def prepare_s2(season, site_position, site_name, cleaning_strategy="aggressive", print(f"[S2-PREP] Computing distance-to-clouds...") distance_to_clouds = _import_distance_to_clouds() distance_to_clouds(s2_output_dir, ratio=RESOLUTION_RATIO) + _rescale_dist_cloud_for_small_roi(s2_output_dir) print("[S2-PREP] Completed") diff --git a/run.py b/run.py index 23de893..73be7b6 100644 --- a/run.py +++ b/run.py @@ -1,16 +1,12 @@ from fusion import run_all_efast_scenarios -# from postprocessing import process_all_scenarios -# from metrics_indices import ( -# create_ndvi_timeseries_post_process, -# create_gcc_timeseries_post_process, -# create_s2_bands_timeseries_post_process, -# ) +from postprocessing import post_process_all_scenarios, post_process_timeseries from acquisition_s2 import download_s2 from acquisition_s3 import download_s3 from acquisition_phenocam import download_phenocam from preselection import create_timeseries from preparation import prepare_s2, prepare_s3 -# from metrics_stats import calculate_all_metrics +from metrics_indices import create_prepared_fusion_timeseries +from metrics_stats import calculate_all_metrics def run_pipeline(season, site_position, site_name): @@ -21,27 +17,26 @@ def run_pipeline(season, site_position, site_name): #download_s3(season, site_position, site_name) #download_phenocam(season, site_position, site_name) - #print(f"Creating preselection timeseries: {site_name}, {season}") - #create_timeseries(season, site_position, site_name) + print(f"Creating preselection timeseries: {site_name}, {season}") + create_timeseries(season, site_position, site_name) - #print(f"Preparing S2 and S3 for fusion: {site_name}, {season}") - #for strategy in ["aggressive", "nonaggressive"]: - # prepare_s2(season, site_position, site_name, cleaning_strategy=strategy) - # prepare_s3(season, site_position, site_name, cleaning_strategy=strategy) + print(f"Preparing S2 and S3 for fusion: {site_name}, {season}") + for strategy in ["aggressive", "nonaggressive"]: + prepare_s2(season, site_position, site_name, cleaning_strategy=strategy) + prepare_s3(season, site_position, site_name, cleaning_strategy=strategy) print(f"Running EFAST fusion for all scenarios: {site_name}, {season}") run_all_efast_scenarios(season, site_position, site_name) + + print(f"Creating prepared/fusion timeseries: {site_name}, {season}") + create_prepared_fusion_timeseries(season, site_position, site_name) + + print(f"Post-processing: {site_name}, {season}") + post_process_all_scenarios(season, site_position, site_name) + post_process_timeseries(season, site_position, site_name) - # print(f"Post-processing data: {site_name}, {season}") - # process_all_scenarios(season, site_position, site_name) - # print(f"Generating NDVI for final outputs: {site_name}, {season}") - # create_ndvi_timeseries_post_process(season, site_position, site_name) - # print(f"Generating GCC for final outputs: {site_name}, {season}") - # create_gcc_timeseries_post_process(season, site_position, site_name) - # print(f"Generating S2 band timeseries: {site_name}, {season}") - # create_s2_bands_timeseries_post_process(season, site_position, site_name) - # print(f"Calculating metrics: {site_name}, {season}") - # calculate_all_metrics(season, site_name, site_position) + print(f"Calculating metrics: {site_name}, {season}") + calculate_all_metrics(season, site_name, site_position) except Exception as e: print(f"Error: {e}") @@ -50,10 +45,10 @@ def run_pipeline(season, site_position, site_name): if __name__ == "__main__": run_pipeline(2024, (47.116171, 11.320308), "innsbruck") - # run_pipeline(2024, (35.3045, 25.0743), "forthgr") - # run_pipeline(2020, (47.116171, 11.320308), "innsbruck") - # run_pipeline(2024, (58.5633, 24.3688), "pitsalu") - # run_pipeline(2023, (64.2437, 19.7673), "vindeln2") - # run_pipeline(2024, (36.7455, -6.0033), "sunflowerjerez1") - # run_pipeline(2024, (42.6558, 26.9837), "institutekarnobat") + run_pipeline(2024, (35.3045, 25.0743), "forthgr") + run_pipeline(2020, (47.116171, 11.320308), "innsbruck") + run_pipeline(2024, (58.5633, 24.3688), "pitsalu") + run_pipeline(2023, (64.2437, 19.7673), "vindeln2") + run_pipeline(2024, (36.7455, -6.0033), "sunflowerjerez1") + run_pipeline(2024, (42.6558, 26.9837), "institutekarnobat") diff --git a/webapp/fusion.html b/webapp/fusion.html new file mode 100644 index 0000000..43b1762 --- /dev/null +++ b/webapp/fusion.html @@ -0,0 +1,361 @@ + + + + Fusion Viewer + + + + + + + +
+ +

Innsbruck

+

2024

+
+ + + + + + + + +
+ +
2024-01-01
+
Fusion RGB (closest available)
+
+
+
+
NDVI
+
GCC
+
B02 (Blue)
+
B03 (Green)
+
B04 (Red)
+
B8A (NIR)
+
+
+ + + diff --git a/webapp/index.html b/webapp/index.html index 3e40604..0891505 100644 --- a/webapp/index.html +++ b/webapp/index.html @@ -1,13 +1,17 @@ - NDVI Viewer + Full + + +
+ +

Innsbruck

+

2024

+
+ + + + + + + + + + +
+ +
2024-01-01
+
Postprocessed RGB (closest available)
+
+
+
+
NDVI
+
GCC
+
B02 (Blue)
+
B03 (Green)
+
B04 (Red)
+
B8A (NIR)
+
+
+ + + diff --git a/webapp/prepared.html b/webapp/prepared.html new file mode 100644 index 0000000..5cdeb92 --- /dev/null +++ b/webapp/prepared.html @@ -0,0 +1,352 @@ + + + + Prepared S2/S3 Viewer + + + + + + + +
+ +

Innsbruck

+

2024

+
+ + + + + + + + +
+ +
2024-01-01
+
Prepared RGB (closest available)
+
+
+
+
NDVI
+
GCC
+
B02 (Blue)
+
B03 (Green)
+
B04 (Red)
+
B8A (NIR)
+
+
+ + + diff --git a/webapp/preselection.html b/webapp/preselection.html index 9476fcc..ed41dd1 100644 --- a/webapp/preselection.html +++ b/webapp/preselection.html @@ -8,7 +8,12 @@