foo
This commit is contained in:
Felix Delattre
parent
0a14711851
commit
290c8f8c57
6 changed files with 104 additions and 61 deletions
|
|
@ -3,12 +3,12 @@ from pathlib import Path
|
|||
from datetime import datetime
|
||||
|
||||
|
||||
def detect_clouds(year, site_name):
|
||||
output_file = Path(f"data/{site_name}/{year}/clouds.json")
|
||||
def detect_clouds(season, site_name):
|
||||
output_file = Path(f"data/{site_name}/{season}/clouds.json")
|
||||
clouds = {"s2": [], "s3": []}
|
||||
|
||||
for source in ["s2", "s3"]:
|
||||
timeseries_file = Path(f"data/{site_name}/{year}/ndvi/{source}/timeseries.json")
|
||||
timeseries_file = Path(f"data/{site_name}/{season}/raw/ndvi/{source}/timeseries.json")
|
||||
if not timeseries_file.exists():
|
||||
print(f"[CLOUDS-{source.upper()}] No timeseries.json found")
|
||||
continue
|
||||
|
|
|
|||
|
|
@ -7,12 +7,12 @@ from rasterio.windows import from_bounds, transform as window_transform
|
|||
from pystac_client import Client
|
||||
|
||||
|
||||
def download_s2(year, site_position, site_name, date_range=None):
|
||||
def download_s2(season, site_position, site_name, date_range=None):
|
||||
lat, lon = site_position
|
||||
datetime_range = date_range or f"{year}-01-01/{year}-12-31"
|
||||
output_dir = f"data/{site_name}/{year}/s2/"
|
||||
datetime_range = date_range or f"{season}-01-01/{season}-12-31"
|
||||
output_dir = f"data/{site_name}/{season}/raw/s2/"
|
||||
|
||||
print(f"[S2] Starting download: {site_name} ({lat:.6f}, {lon:.6f}), {year}")
|
||||
print(f"[S2] Starting download: {site_name} ({lat:.6f}, {lon:.6f}), {season}")
|
||||
|
||||
bbox_size = 0.011
|
||||
bbox = [
|
||||
|
|
|
|||
|
|
@ -12,12 +12,12 @@ from rasterio.transform import from_bounds
|
|||
load_dotenv()
|
||||
|
||||
|
||||
def download_s3(year, site_position, site_name, date_range=None):
|
||||
def download_s3(season, site_position, site_name, date_range=None):
|
||||
lat, lon = site_position
|
||||
datetime_range = date_range or f"{year}-01-01/{year}-12-31"
|
||||
output_dir = Path(f"data/{site_name}/{year}/s3/")
|
||||
datetime_range = date_range or f"{season}-01-01/{season}-12-31"
|
||||
output_dir = Path(f"data/{site_name}/{season}/raw/s3/")
|
||||
|
||||
print(f"[S3] Starting download: {site_name} ({lat:.6f}, {lon:.6f}), {year}")
|
||||
print(f"[S3] Starting download: {site_name} ({lat:.6f}, {lon:.6f}), {season}")
|
||||
|
||||
bbox_size = 0.011
|
||||
bbox = [
|
||||
|
|
|
|||
26
efast.py
26
efast.py
|
|
@ -43,11 +43,11 @@ else:
|
|||
)
|
||||
|
||||
|
||||
def prepare_s2(year, site_position, site_name, date_range=None):
|
||||
s2_dir = Path(f"data/{site_name}/{year}/s2/")
|
||||
s3_dir = Path(f"data/{site_name}/{year}/s3/")
|
||||
s2_output_dir = Path(f"data/{site_name}/{year}/efast/s2/")
|
||||
clouds_file = Path(f"data/{site_name}/{year}/clouds.json")
|
||||
def prepare_s2(season, site_position, site_name, date_range=None):
|
||||
s2_dir = Path(f"data/{site_name}/{season}/raw/s2/")
|
||||
s3_dir = Path(f"data/{site_name}/{season}/raw/s3/")
|
||||
s2_output_dir = Path(f"data/{site_name}/{season}/prepared/s2/")
|
||||
clouds_file = Path(f"data/{site_name}/{season}/clouds.json")
|
||||
|
||||
clouds = {"s2": set(), "s3": set()}
|
||||
if clouds_file.exists():
|
||||
|
|
@ -159,10 +159,10 @@ def prepare_s2(year, site_position, site_name, date_range=None):
|
|||
dst.write(distance_to_cloud_lr, 1)
|
||||
|
||||
|
||||
def prepare_s3(year, site_position, site_name, date_range=None):
|
||||
s3_dir = Path(f"data/{site_name}/{year}/s3/")
|
||||
s3_preprocessed_dir = Path(f"data/{site_name}/{year}/efast/s3/")
|
||||
clouds_file = Path(f"data/{site_name}/{year}/clouds.json")
|
||||
def prepare_s3(season, site_position, site_name, date_range=None):
|
||||
s3_dir = Path(f"data/{site_name}/{season}/raw/s3/")
|
||||
s3_preprocessed_dir = Path(f"data/{site_name}/{season}/prepared/s3/")
|
||||
clouds_file = Path(f"data/{site_name}/{season}/clouds.json")
|
||||
|
||||
clouds = {"s3": set()}
|
||||
if clouds_file.exists():
|
||||
|
|
@ -180,18 +180,18 @@ def prepare_s3(year, site_position, site_name, date_range=None):
|
|||
shutil.copy2(s3_file, output_path)
|
||||
|
||||
|
||||
def run_efast(year, site_position, site_name, date_range=None):
|
||||
def run_efast(season, site_position, site_name, date_range=None):
|
||||
lat, lon = site_position
|
||||
datetime_range = date_range or f"{year}-01-01/{year}-12-31"
|
||||
datetime_range = date_range or f"{season}-01-01/{season}-12-31"
|
||||
|
||||
efast_base_dir = Path(f"data/{site_name}/{year}/efast/")
|
||||
efast_base_dir = Path(f"data/{site_name}/{season}/prepared/")
|
||||
s2_output_dir = efast_base_dir / "s2"
|
||||
s3_output_dir = efast_base_dir / "s3"
|
||||
fusion_output_dir = efast_base_dir / "fusion"
|
||||
|
||||
fusion_output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
print(f"[EFAST] Starting fusion: {site_name} ({lat:.6f}, {lon:.6f}), {year}")
|
||||
print(f"[EFAST] Starting fusion: {site_name} ({lat:.6f}, {lon:.6f}), {season}")
|
||||
|
||||
start_date = datetime.strptime(datetime_range.split("/")[0], "%Y-%m-%d")
|
||||
end_date = datetime.strptime(datetime_range.split("/")[1], "%Y-%m-%d")
|
||||
|
|
|
|||
57
ndvi.py
57
ndvi.py
|
|
@ -51,7 +51,7 @@ def _create_timeseries_for_dir(output_dir, site_position, source_name):
|
|||
lon, lat = site_position[1], site_position[0]
|
||||
x, y = transform_coords("EPSG:4326", src.crs, [lon], [lat])
|
||||
samples = list(src.sample([(x[0], y[0])]))
|
||||
if samples and len(samples) > 0:
|
||||
if samples:
|
||||
value = float(samples[0][0])
|
||||
if value != 0 and not np.isnan(value):
|
||||
ndvi_value = value
|
||||
|
|
@ -68,10 +68,10 @@ def _create_timeseries_for_dir(output_dir, site_position, source_name):
|
|||
print(f"[NDVI-{source_name}] Saved: {timeseries_file} ({len(timeseries)} entries)")
|
||||
|
||||
|
||||
def generate_ndvi(year, site_position, site_name):
|
||||
def generate_ndvi_raw(season, site_position, site_name):
|
||||
for source in ["s2", "s3"]:
|
||||
input_dir = Path(f"data/{site_name}/{year}/{source}/")
|
||||
output_dir = Path(f"data/{site_name}/{year}/ndvi/{source}/")
|
||||
input_dir = Path(f"data/{site_name}/{season}/raw/{source}/")
|
||||
output_dir = Path(f"data/{site_name}/{season}/raw/ndvi/{source}/")
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
print(f"[NDVI-{source.upper()}] Processing {input_dir}...")
|
||||
|
|
@ -94,15 +94,46 @@ def generate_ndvi(year, site_position, site_name):
|
|||
print(f"[NDVI-{source.upper()}] Completed")
|
||||
|
||||
|
||||
def create_ndvi_timeseries(year, site_position, site_name):
|
||||
def create_ndvi_timeseries_raw(season, site_position, site_name):
|
||||
for source in ["s2", "s3"]:
|
||||
output_dir = Path(f"data/{site_name}/{year}/ndvi/{source}/")
|
||||
output_dir = Path(f"data/{site_name}/{season}/raw/ndvi/{source}/")
|
||||
_create_timeseries_for_dir(output_dir, site_position, source.upper())
|
||||
|
||||
|
||||
def generate_ndvi_fusion(year, site_position, site_name):
|
||||
input_dir = Path(f"data/{site_name}/{year}/efast/fusion/")
|
||||
output_dir = Path(f"data/{site_name}/{year}/ndvi/fusion/")
|
||||
def generate_ndvi_prepared(season, site_position, site_name):
|
||||
for source in ["s2", "s3"]:
|
||||
input_dir = Path(f"data/{site_name}/{season}/prepared/{source}/")
|
||||
output_dir = Path(f"data/{site_name}/{season}/prepared/ndvi/{source}/")
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
print(f"[NDVI-PREPARED-{source.upper()}] Processing {input_dir}...")
|
||||
|
||||
geotiff_files = sorted(input_dir.glob("*.geotiff")) + sorted(input_dir.glob("*.tif"))
|
||||
if not geotiff_files:
|
||||
print(f"[NDVI-PREPARED-{source.upper()}] No files found")
|
||||
continue
|
||||
|
||||
for geotiff_file in geotiff_files:
|
||||
if geotiff_file.suffix == ".tif":
|
||||
if "REFL" in geotiff_file.stem:
|
||||
date_str = geotiff_file.stem.split("_")[1]
|
||||
output_file = output_dir / f"{date_str}_ndvi.geotiff"
|
||||
else:
|
||||
output_file = output_dir / geotiff_file.name.replace(".tif", ".geotiff")
|
||||
else:
|
||||
output_file = output_dir / geotiff_file.name
|
||||
|
||||
if output_file.exists():
|
||||
print(f"[NDVI-PREPARED-{source.upper()}] Skipping {geotiff_file.name} (exists)")
|
||||
continue
|
||||
|
||||
_calculate_and_write_ndvi(geotiff_file, output_file)
|
||||
print(f"[NDVI-PREPARED-{source.upper()}] Saved: {output_file}")
|
||||
|
||||
print(f"[NDVI-PREPARED-{source.upper()}] Completed")
|
||||
|
||||
input_dir = Path(f"data/{site_name}/{season}/prepared/fusion/")
|
||||
output_dir = Path(f"data/{site_name}/{season}/prepared/ndvi/fusion/")
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
print(f"[NDVI-FUSION] Processing {input_dir}...")
|
||||
|
|
@ -126,6 +157,10 @@ def generate_ndvi_fusion(year, site_position, site_name):
|
|||
print(f"[NDVI-FUSION] Completed")
|
||||
|
||||
|
||||
def create_ndvi_timeseries_fusion(year, site_position, site_name):
|
||||
output_dir = Path(f"data/{site_name}/{year}/ndvi/fusion/")
|
||||
def create_ndvi_timeseries_prepared(season, site_position, site_name):
|
||||
for source in ["s2", "s3"]:
|
||||
output_dir = Path(f"data/{site_name}/{season}/prepared/ndvi/{source}/")
|
||||
_create_timeseries_for_dir(output_dir, site_position, f"PREPARED-{source.upper()}")
|
||||
|
||||
output_dir = Path(f"data/{site_name}/{season}/prepared/ndvi/fusion/")
|
||||
_create_timeseries_for_dir(output_dir, site_position, "FUSION")
|
||||
|
|
|
|||
60
run.py
60
run.py
|
|
@ -1,34 +1,42 @@
|
|||
from efast import run_efast, prepare_s2, prepare_s3
|
||||
from ndvi import generate_ndvi_fusion, create_ndvi_timeseries_fusion
|
||||
from ndvi import (
|
||||
generate_ndvi_raw,
|
||||
create_ndvi_timeseries_raw,
|
||||
generate_ndvi_prepared,
|
||||
create_ndvi_timeseries_prepared,
|
||||
)
|
||||
from download_s2 import download_s2
|
||||
from download_s3 import download_s3
|
||||
from clouds import detect_clouds
|
||||
|
||||
year = 2024
|
||||
site_position = (47.116171, 11.320308)
|
||||
site_name = "innsbruck"
|
||||
|
||||
# print(f"Downloading data for {site_name}, {year}")
|
||||
# download_s2(year, site_position, site_name)
|
||||
# download_s3(year, site_position, site_name)
|
||||
# print("All downloads completed")
|
||||
def run_pipeline(season, site_position, site_name):
|
||||
try:
|
||||
print(f"Downloading data for {site_name}, {season}")
|
||||
download_s2(season, site_position, site_name)
|
||||
download_s3(season, site_position, site_name)
|
||||
|
||||
# print(f"Generating NDVI for {site_name}, {year}")
|
||||
# generate_ndvi(year, site_position, site_name)
|
||||
# create_ndvi_timeseries(year, site_position, site_name)
|
||||
# print("All NDVI generation completed")
|
||||
print(f"Generating NDVI for raw data: {site_name}, {season}")
|
||||
generate_ndvi_raw(season, site_position, site_name)
|
||||
create_ndvi_timeseries_raw(season, site_position, site_name)
|
||||
|
||||
# print(f"Detecting clouds for {site_name}, {year}")
|
||||
# detect_clouds(year, site_name)
|
||||
# print("Cloud detection completed")
|
||||
print(f"Detecting clouds for {site_name}, {season}")
|
||||
detect_clouds(season, site_name)
|
||||
|
||||
# print(f"Preparing data for EFAST fusion for {site_name}, {year}")
|
||||
# prepare_s2(year, site_position, site_name)
|
||||
# prepare_s3(year, site_position, site_name)
|
||||
# print("Data preparation completed")
|
||||
print(f"Preparing data for EFAST fusion for {site_name}, {season}")
|
||||
prepare_s2(season, site_position, site_name)
|
||||
prepare_s3(season, site_position, site_name)
|
||||
|
||||
# print(f"Running EFAST fusion for {site_name}, {year}")
|
||||
# run_efast(year, site_position, site_name)
|
||||
# print("EFAST fusion completed")
|
||||
print(f"Running EFAST fusion for {site_name}, {season}")
|
||||
run_efast(season, site_position, site_name)
|
||||
|
||||
print(f"Generating NDVI for fusion outputs: {site_name}, {year}")
|
||||
generate_ndvi_fusion(year, site_position, site_name)
|
||||
create_ndvi_timeseries_fusion(year, site_position, site_name)
|
||||
print("Fusion NDVI generation completed")
|
||||
print(f"Generating NDVI for prepared outputs: {site_name}, {season}")
|
||||
generate_ndvi_prepared(season, site_position, site_name)
|
||||
create_ndvi_timeseries_prepared(season, site_position, site_name)
|
||||
except Exception as e:
|
||||
print(f"Error: {e}")
|
||||
raise
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run_pipeline(2024, (47.116171, 11.320308), "innsbruck")
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue