From 290c8f8c577c1223bac79f14fb94a63da809834b Mon Sep 17 00:00:00 2001 From: Felix Delattre Date: Fri, 26 Dec 2025 13:14:52 +0100 Subject: [PATCH] foo --- clouds.py | 6 ++--- download_s2.py | 8 +++---- download_s3.py | 8 +++---- efast.py | 26 +++++++++++----------- ndvi.py | 57 ++++++++++++++++++++++++++++++++++++++--------- run.py | 60 ++++++++++++++++++++++++++++---------------------- 6 files changed, 104 insertions(+), 61 deletions(-) diff --git a/clouds.py b/clouds.py index a075bcc..cf5db4f 100644 --- a/clouds.py +++ b/clouds.py @@ -3,12 +3,12 @@ from pathlib import Path from datetime import datetime -def detect_clouds(year, site_name): - output_file = Path(f"data/{site_name}/{year}/clouds.json") +def detect_clouds(season, site_name): + output_file = Path(f"data/{site_name}/{season}/clouds.json") clouds = {"s2": [], "s3": []} for source in ["s2", "s3"]: - timeseries_file = Path(f"data/{site_name}/{year}/ndvi/{source}/timeseries.json") + timeseries_file = Path(f"data/{site_name}/{season}/raw/ndvi/{source}/timeseries.json") if not timeseries_file.exists(): print(f"[CLOUDS-{source.upper()}] No timeseries.json found") continue diff --git a/download_s2.py b/download_s2.py index bcec0a4..f6e4311 100644 --- a/download_s2.py +++ b/download_s2.py @@ -7,12 +7,12 @@ from rasterio.windows import from_bounds, transform as window_transform from pystac_client import Client -def download_s2(year, site_position, site_name, date_range=None): +def download_s2(season, site_position, site_name, date_range=None): lat, lon = site_position - datetime_range = date_range or f"{year}-01-01/{year}-12-31" - output_dir = f"data/{site_name}/{year}/s2/" + datetime_range = date_range or f"{season}-01-01/{season}-12-31" + output_dir = f"data/{site_name}/{season}/raw/s2/" - print(f"[S2] Starting download: {site_name} ({lat:.6f}, {lon:.6f}), {year}") + print(f"[S2] Starting download: {site_name} ({lat:.6f}, {lon:.6f}), {season}") bbox_size = 0.011 bbox = [ diff --git a/download_s3.py b/download_s3.py index f49ee5d..6920e65 100644 --- a/download_s3.py +++ b/download_s3.py @@ -12,12 +12,12 @@ from rasterio.transform import from_bounds load_dotenv() -def download_s3(year, site_position, site_name, date_range=None): +def download_s3(season, site_position, site_name, date_range=None): lat, lon = site_position - datetime_range = date_range or f"{year}-01-01/{year}-12-31" - output_dir = Path(f"data/{site_name}/{year}/s3/") + datetime_range = date_range or f"{season}-01-01/{season}-12-31" + output_dir = Path(f"data/{site_name}/{season}/raw/s3/") - print(f"[S3] Starting download: {site_name} ({lat:.6f}, {lon:.6f}), {year}") + print(f"[S3] Starting download: {site_name} ({lat:.6f}, {lon:.6f}), {season}") bbox_size = 0.011 bbox = [ diff --git a/efast.py b/efast.py index 6c01ba2..021622a 100644 --- a/efast.py +++ b/efast.py @@ -43,11 +43,11 @@ else: ) -def prepare_s2(year, site_position, site_name, date_range=None): - s2_dir = Path(f"data/{site_name}/{year}/s2/") - s3_dir = Path(f"data/{site_name}/{year}/s3/") - s2_output_dir = Path(f"data/{site_name}/{year}/efast/s2/") - clouds_file = Path(f"data/{site_name}/{year}/clouds.json") +def prepare_s2(season, site_position, site_name, date_range=None): + s2_dir = Path(f"data/{site_name}/{season}/raw/s2/") + s3_dir = Path(f"data/{site_name}/{season}/raw/s3/") + s2_output_dir = Path(f"data/{site_name}/{season}/prepared/s2/") + clouds_file = Path(f"data/{site_name}/{season}/clouds.json") clouds = {"s2": set(), "s3": set()} if clouds_file.exists(): @@ -159,10 +159,10 @@ def prepare_s2(year, site_position, site_name, date_range=None): dst.write(distance_to_cloud_lr, 1) -def prepare_s3(year, site_position, site_name, date_range=None): - s3_dir = Path(f"data/{site_name}/{year}/s3/") - s3_preprocessed_dir = Path(f"data/{site_name}/{year}/efast/s3/") - clouds_file = Path(f"data/{site_name}/{year}/clouds.json") +def prepare_s3(season, site_position, site_name, date_range=None): + s3_dir = Path(f"data/{site_name}/{season}/raw/s3/") + s3_preprocessed_dir = Path(f"data/{site_name}/{season}/prepared/s3/") + clouds_file = Path(f"data/{site_name}/{season}/clouds.json") clouds = {"s3": set()} if clouds_file.exists(): @@ -180,18 +180,18 @@ def prepare_s3(year, site_position, site_name, date_range=None): shutil.copy2(s3_file, output_path) -def run_efast(year, site_position, site_name, date_range=None): +def run_efast(season, site_position, site_name, date_range=None): lat, lon = site_position - datetime_range = date_range or f"{year}-01-01/{year}-12-31" + datetime_range = date_range or f"{season}-01-01/{season}-12-31" - efast_base_dir = Path(f"data/{site_name}/{year}/efast/") + efast_base_dir = Path(f"data/{site_name}/{season}/prepared/") s2_output_dir = efast_base_dir / "s2" s3_output_dir = efast_base_dir / "s3" fusion_output_dir = efast_base_dir / "fusion" fusion_output_dir.mkdir(parents=True, exist_ok=True) - print(f"[EFAST] Starting fusion: {site_name} ({lat:.6f}, {lon:.6f}), {year}") + print(f"[EFAST] Starting fusion: {site_name} ({lat:.6f}, {lon:.6f}), {season}") start_date = datetime.strptime(datetime_range.split("/")[0], "%Y-%m-%d") end_date = datetime.strptime(datetime_range.split("/")[1], "%Y-%m-%d") diff --git a/ndvi.py b/ndvi.py index 58f378a..def1f73 100644 --- a/ndvi.py +++ b/ndvi.py @@ -51,7 +51,7 @@ def _create_timeseries_for_dir(output_dir, site_position, source_name): lon, lat = site_position[1], site_position[0] x, y = transform_coords("EPSG:4326", src.crs, [lon], [lat]) samples = list(src.sample([(x[0], y[0])])) - if samples and len(samples) > 0: + if samples: value = float(samples[0][0]) if value != 0 and not np.isnan(value): ndvi_value = value @@ -68,10 +68,10 @@ def _create_timeseries_for_dir(output_dir, site_position, source_name): print(f"[NDVI-{source_name}] Saved: {timeseries_file} ({len(timeseries)} entries)") -def generate_ndvi(year, site_position, site_name): +def generate_ndvi_raw(season, site_position, site_name): for source in ["s2", "s3"]: - input_dir = Path(f"data/{site_name}/{year}/{source}/") - output_dir = Path(f"data/{site_name}/{year}/ndvi/{source}/") + input_dir = Path(f"data/{site_name}/{season}/raw/{source}/") + output_dir = Path(f"data/{site_name}/{season}/raw/ndvi/{source}/") output_dir.mkdir(parents=True, exist_ok=True) print(f"[NDVI-{source.upper()}] Processing {input_dir}...") @@ -94,15 +94,46 @@ def generate_ndvi(year, site_position, site_name): print(f"[NDVI-{source.upper()}] Completed") -def create_ndvi_timeseries(year, site_position, site_name): +def create_ndvi_timeseries_raw(season, site_position, site_name): for source in ["s2", "s3"]: - output_dir = Path(f"data/{site_name}/{year}/ndvi/{source}/") + output_dir = Path(f"data/{site_name}/{season}/raw/ndvi/{source}/") _create_timeseries_for_dir(output_dir, site_position, source.upper()) -def generate_ndvi_fusion(year, site_position, site_name): - input_dir = Path(f"data/{site_name}/{year}/efast/fusion/") - output_dir = Path(f"data/{site_name}/{year}/ndvi/fusion/") +def generate_ndvi_prepared(season, site_position, site_name): + for source in ["s2", "s3"]: + input_dir = Path(f"data/{site_name}/{season}/prepared/{source}/") + output_dir = Path(f"data/{site_name}/{season}/prepared/ndvi/{source}/") + output_dir.mkdir(parents=True, exist_ok=True) + + print(f"[NDVI-PREPARED-{source.upper()}] Processing {input_dir}...") + + geotiff_files = sorted(input_dir.glob("*.geotiff")) + sorted(input_dir.glob("*.tif")) + if not geotiff_files: + print(f"[NDVI-PREPARED-{source.upper()}] No files found") + continue + + for geotiff_file in geotiff_files: + if geotiff_file.suffix == ".tif": + if "REFL" in geotiff_file.stem: + date_str = geotiff_file.stem.split("_")[1] + output_file = output_dir / f"{date_str}_ndvi.geotiff" + else: + output_file = output_dir / geotiff_file.name.replace(".tif", ".geotiff") + else: + output_file = output_dir / geotiff_file.name + + if output_file.exists(): + print(f"[NDVI-PREPARED-{source.upper()}] Skipping {geotiff_file.name} (exists)") + continue + + _calculate_and_write_ndvi(geotiff_file, output_file) + print(f"[NDVI-PREPARED-{source.upper()}] Saved: {output_file}") + + print(f"[NDVI-PREPARED-{source.upper()}] Completed") + + input_dir = Path(f"data/{site_name}/{season}/prepared/fusion/") + output_dir = Path(f"data/{site_name}/{season}/prepared/ndvi/fusion/") output_dir.mkdir(parents=True, exist_ok=True) print(f"[NDVI-FUSION] Processing {input_dir}...") @@ -126,6 +157,10 @@ def generate_ndvi_fusion(year, site_position, site_name): print(f"[NDVI-FUSION] Completed") -def create_ndvi_timeseries_fusion(year, site_position, site_name): - output_dir = Path(f"data/{site_name}/{year}/ndvi/fusion/") +def create_ndvi_timeseries_prepared(season, site_position, site_name): + for source in ["s2", "s3"]: + output_dir = Path(f"data/{site_name}/{season}/prepared/ndvi/{source}/") + _create_timeseries_for_dir(output_dir, site_position, f"PREPARED-{source.upper()}") + + output_dir = Path(f"data/{site_name}/{season}/prepared/ndvi/fusion/") _create_timeseries_for_dir(output_dir, site_position, "FUSION") diff --git a/run.py b/run.py index 653b053..bef89ec 100644 --- a/run.py +++ b/run.py @@ -1,34 +1,42 @@ from efast import run_efast, prepare_s2, prepare_s3 -from ndvi import generate_ndvi_fusion, create_ndvi_timeseries_fusion +from ndvi import ( + generate_ndvi_raw, + create_ndvi_timeseries_raw, + generate_ndvi_prepared, + create_ndvi_timeseries_prepared, +) +from download_s2 import download_s2 +from download_s3 import download_s3 +from clouds import detect_clouds -year = 2024 -site_position = (47.116171, 11.320308) -site_name = "innsbruck" -# print(f"Downloading data for {site_name}, {year}") -# download_s2(year, site_position, site_name) -# download_s3(year, site_position, site_name) -# print("All downloads completed") +def run_pipeline(season, site_position, site_name): + try: + print(f"Downloading data for {site_name}, {season}") + download_s2(season, site_position, site_name) + download_s3(season, site_position, site_name) -# print(f"Generating NDVI for {site_name}, {year}") -# generate_ndvi(year, site_position, site_name) -# create_ndvi_timeseries(year, site_position, site_name) -# print("All NDVI generation completed") + print(f"Generating NDVI for raw data: {site_name}, {season}") + generate_ndvi_raw(season, site_position, site_name) + create_ndvi_timeseries_raw(season, site_position, site_name) -# print(f"Detecting clouds for {site_name}, {year}") -# detect_clouds(year, site_name) -# print("Cloud detection completed") + print(f"Detecting clouds for {site_name}, {season}") + detect_clouds(season, site_name) -# print(f"Preparing data for EFAST fusion for {site_name}, {year}") -# prepare_s2(year, site_position, site_name) -# prepare_s3(year, site_position, site_name) -# print("Data preparation completed") + print(f"Preparing data for EFAST fusion for {site_name}, {season}") + prepare_s2(season, site_position, site_name) + prepare_s3(season, site_position, site_name) -# print(f"Running EFAST fusion for {site_name}, {year}") -# run_efast(year, site_position, site_name) -# print("EFAST fusion completed") + print(f"Running EFAST fusion for {site_name}, {season}") + run_efast(season, site_position, site_name) -print(f"Generating NDVI for fusion outputs: {site_name}, {year}") -generate_ndvi_fusion(year, site_position, site_name) -create_ndvi_timeseries_fusion(year, site_position, site_name) -print("Fusion NDVI generation completed") + print(f"Generating NDVI for prepared outputs: {site_name}, {season}") + generate_ndvi_prepared(season, site_position, site_name) + create_ndvi_timeseries_prepared(season, site_position, site_name) + except Exception as e: + print(f"Error: {e}") + raise + + +if __name__ == "__main__": + run_pipeline(2024, (47.116171, 11.320308), "innsbruck")