pantierra/efast-phenocam-validation
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Refactored efast.py to leverage efast package functions.

Browse source
This commit is contained in:
Felix Delattre 2026-01-11 00:23:35 +01:00
parent 853c1c6a30
commit 6741433228
3 changed files with 81 additions and 182 deletions
Download patch file Download diff file Expand all files Collapse all files

11
README.md
View file

@ -57,6 +57,17 @@ data/
clouds.json # Cloud detection results clouds.json # Cloud detection results
``` ```
### File Formats
**Sentinel-2 (raw/s2/)**: Multi-band GeoTIFF
- Bands: B02 (blue), B03 (green), B04 (red), B8A (nir)
- Metadata: `VIEWING_ZENITH_ANGLE` tag (degrees)
- Filename: `{YYYYMMDD}_{increment}.geotiff`
**Sentinel-3 (raw/s3/)**: Multi-band GeoTIFF
- Bands: SDR_Oa04 (blue), SDR_Oa06 (green), SDR_Oa08 (red), SDR_Oa17 (nir)
- Filename: `{YYYYMMDD}_{increment}.geotiff`
## Web Viewer ## Web Viewer
Run a local HTTP server to view the web interface: Run a local HTTP server to view the web interface:

251
efast.py
View file

@ -1,36 +1,24 @@
import json import json
import shutil import shutil
import importlib.util
from pathlib import Path from pathlib import Path
from datetime import datetime, timedelta from datetime import datetime, timedelta
from collections import defaultdict
import numpy as np import numpy as np
import rasterio import rasterio
from rasterio.warp import Resampling from rasterio.warp import Resampling
from rasterio.vrt import WarpedVRT from rasterio.vrt import WarpedVRT
from rasterio import shutil as rio_shutil from rasterio import shutil as rio_shutil
from scipy import ndimage
RESOLUTION_RATIO = 21
try: try:
import efast as efast_fusion import efast
from efast.s2_processing import distance_to_clouds
from efast.s3_processing import reproject_and_crop_s3
except ImportError: except ImportError:
import site raise ImportError(
"efast package not found. Install with: pip install git+https://github.com/DHI-GRAS/efast.git"
)
efast_fusion = None RESOLUTION_RATIO = 21
for site_pkg in site.getsitepackages():
candidate = Path(site_pkg) / "efast" / "efast.py"
if candidate.exists():
spec = importlib.util.spec_from_file_location(
"efast_fusion_module", candidate
)
efast_fusion = importlib.util.module_from_spec(spec)
spec.loader.exec_module(efast_fusion)
break
if efast_fusion is None:
raise ImportError(
"efast package not found. Install with: pip install git+https://github.com/DHI-GRAS/efast.git"
)
def _load_clouds(clouds_file): def _load_clouds(clouds_file):
@ -42,27 +30,24 @@ def _load_clouds(clouds_file):
return clouds return clouds
def _reproject_to_target( def _reproject_raster_to_target(src_path, dst_path, target_bounds, target_crs, width, height, resampling=Resampling.cubic):
data, src_transform, src_crs, target_bounds, target_crs, width, height, resampling
):
dst_transform = rasterio.transform.from_bounds( dst_transform = rasterio.transform.from_bounds(
target_bounds.left, target_bounds.left, target_bounds.bottom,
target_bounds.bottom, target_bounds.right, target_bounds.top,
target_bounds.right, width, height
target_bounds.top,
width,
height,
) )
reprojected, _ = rasterio.warp.reproject( with rasterio.open(src_path) as src:
source=data, vrt_options = {
destination=np.zeros((data.shape[0], height, width), dtype=data.dtype), "transform": dst_transform,
src_transform=src_transform, "height": height,
src_crs=src_crs, "width": width,
dst_transform=dst_transform, "crs": target_crs,
dst_crs=target_crs, "resampling": resampling,
resampling=resampling, }
) with WarpedVRT(src, **vrt_options) as vrt:
return reprojected, dst_transform profile = vrt.profile.copy()
profile.update({"dtype": "float32", "nodata": 0})
rio_shutil.copy(vrt, dst_path, driver="GTiff", **profile)
def prepare_s2(season, site_position, site_name, date_range=None): def prepare_s2(season, site_position, site_name, date_range=None):
@ -72,7 +57,6 @@ def prepare_s2(season, site_position, site_name, date_range=None):
clouds_file = Path(f"data/{site_name}/{season}/clouds.json") clouds_file = Path(f"data/{site_name}/{season}/clouds.json")
clouds = _load_clouds(clouds_file) clouds = _load_clouds(clouds_file)
s2_output_dir.mkdir(parents=True, exist_ok=True) s2_output_dir.mkdir(parents=True, exist_ok=True)
s3_files = [f for f in s3_dir.glob("*.geotiff") if f.name not in clouds["s3"]] s3_files = [f for f in s3_dir.glob("*.geotiff") if f.name not in clouds["s3"]]
@ -82,77 +66,29 @@ def prepare_s2(season, site_position, site_name, date_range=None):
with rasterio.open(s3_files[0]) as s3_ref: with rasterio.open(s3_files[0]) as s3_ref:
target_bounds = s3_ref.bounds target_bounds = s3_ref.bounds
target_crs = s3_ref.crs target_crs = s3_ref.crs
s3_width = s3_ref.width s2_width = s3_ref.width * RESOLUTION_RATIO
s3_height = s3_ref.height s2_height = s3_ref.height * RESOLUTION_RATIO
s2_width = s3_width * RESOLUTION_RATIO
s2_height = s3_height * RESOLUTION_RATIO
for s2_file in s2_dir.glob("*.geotiff"): for s2_file in s2_dir.glob("*.geotiff"):
if s2_file.name in clouds["s2"]: if s2_file.name in clouds["s2"]:
continue continue
date_str = s2_file.name.split("_")[0] date_str = s2_file.name.split("_")[0]
refl_dst = s2_output_dir / f"S2A_MSIL2A_{date_str}_REFL.tif" refl_dst = s2_output_dir / f"S2A_MSIL2A_{date_str}_REFL.tif"
if not refl_dst.exists(): if refl_dst.exists():
with rasterio.open(s2_file) as src: continue
data = src.read().astype("float32") / 10000.0
reprojected_data, dst_transform = _reproject_to_target(
data,
src.transform,
src.crs,
target_bounds,
target_crs,
s2_width,
s2_height,
Resampling.cubic,
)
profile = src.profile.copy()
profile.update(
{
"dtype": "float32",
"nodata": 0,
"width": s2_width,
"height": s2_height,
"transform": dst_transform,
"crs": target_crs,
}
)
with rasterio.open(refl_dst, "w", **profile) as dst_file:
dst_file.write(reprojected_data)
dist_cloud_dst = s2_output_dir / f"S2A_MSIL2A_{date_str}_DIST_CLOUD.tif" temp_normalized = s2_output_dir / f"temp_{s2_file.name}"
if not dist_cloud_dst.exists(): with rasterio.open(s2_file) as src:
with rasterio.open(refl_dst) as src: data = src.read().astype("float32") / 10000.0
s2_hr = src.read(1) profile = src.profile.copy()
mask = s2_hr == 0 profile.update({"dtype": "float32", "nodata": 0})
distance_to_cloud_hr = np.clip( with rasterio.open(temp_normalized, "w", **profile) as dst:
ndimage.distance_transform_edt(~mask), 0, 255 dst.write(data)
).astype("float32")
distance_to_cloud_lr, lr_transform = _reproject_to_target( _reproject_raster_to_target(temp_normalized, refl_dst, target_bounds, target_crs, s2_width, s2_height)
distance_to_cloud_hr[np.newaxis, :, :], temp_normalized.unlink()
src.transform,
src.crs,
target_bounds,
target_crs,
s3_width,
s3_height,
Resampling.average,
)
distance_to_cloud_lr = distance_to_cloud_lr[0]
profile = src.profile.copy() distance_to_clouds(s2_output_dir, ratio=RESOLUTION_RATIO)
profile.update(
{
"count": 1,
"dtype": "float32",
"width": s3_width,
"height": s3_height,
"transform": lr_transform,
}
)
with rasterio.open(dist_cloud_dst, "w", **profile) as dst:
dst.write(distance_to_cloud_lr, 1)
def prepare_s3(season, site_position, site_name, date_range=None): def prepare_s3(season, site_position, site_name, date_range=None):
@ -164,72 +100,37 @@ def prepare_s3(season, site_position, site_name, date_range=None):
clouds = _load_clouds(clouds_file) clouds = _load_clouds(clouds_file)
s3_preprocessed_dir.mkdir(parents=True, exist_ok=True) s3_preprocessed_dir.mkdir(parents=True, exist_ok=True)
# Get reference profile from S2 DIST_CLOUD file s3_by_date = defaultdict(list)
dist_cloud_files = list(s2_prepared_dir.glob("*DIST_CLOUD.tif"))
if not dist_cloud_files:
raise ValueError("No S2 DIST_CLOUD files found. Run prepare_s2 first.")
with rasterio.open(dist_cloud_files[0]) as src:
target_profile = src.profile
# Group S3 files by date
s3_by_date = {}
for s3_file in s3_dir.glob("*.geotiff"): for s3_file in s3_dir.glob("*.geotiff"):
if s3_file.name in clouds["s3"]: if s3_file.name not in clouds["s3"]:
continue s3_by_date[s3_file.name.split("_")[0]].append(s3_file)
date_str = s3_file.name.split("_")[0]
if date_str not in s3_by_date: temp_composite_dir = s3_preprocessed_dir / "temp_composites"
s3_by_date[date_str] = [] if temp_composite_dir.exists():
s3_by_date[date_str].append(s3_file) shutil.rmtree(temp_composite_dir)
temp_composite_dir.mkdir()
# Process each date
for date_str, s3_files in s3_by_date.items(): for date_str, s3_files in s3_by_date.items():
output_path = s3_preprocessed_dir / f"composite_{date_str}.tif" composite_path = temp_composite_dir / f"composite_{date_str}.tif"
if output_path.exists():
continue
if len(s3_files) == 1: if len(s3_files) == 1:
# Single file: reproject directly shutil.copy(s3_files[0], composite_path)
with rasterio.open(s3_files[0]) as src:
vrt_options = {
"transform": target_profile["transform"],
"height": target_profile["height"],
"width": target_profile["width"],
"crs": target_profile["crs"],
"resampling": Resampling.cubic,
}
with WarpedVRT(src, **vrt_options) as vrt:
rio_shutil.copy(vrt, output_path, driver="GTiff")
else: else:
# Multiple files: create weighted composite
s3_stack = [] s3_stack = []
for s3_file in s3_files: for s3_file in s3_files:
with rasterio.open(s3_file) as src: with rasterio.open(s3_file) as src:
vrt_options = { data = src.read()
"transform": target_profile["transform"], data[:, np.abs(np.nanmean(data, axis=0)) >= 5] = np.nan
"height": target_profile["height"], s3_stack.append(data)
"width": target_profile["width"], composite = np.nanmean(np.array(s3_stack), axis=0).astype("float32")
"crs": target_profile["crs"], with rasterio.open(s3_files[0]) as src:
"resampling": Resampling.cubic, profile = src.profile.copy()
} profile.update({"count": composite.shape[0], "dtype": "float32"})
with WarpedVRT(src, **vrt_options) as vrt: with rasterio.open(composite_path, "w", **profile) as dst:
data = vrt.read()
# Remove abnormally high values (pixel-wise mean across bands)
pixel_means = np.abs(np.nanmean(data, axis=0))
mask = pixel_means >= 5
data[:, mask] = np.nan
s3_stack.append(data)
s3_stack = np.array(s3_stack)
# Simple mean composite (can be enhanced with temporal weighting)
composite = np.nanmean(s3_stack, axis=0)
composite = composite.astype("float32")
profile = target_profile.copy()
profile.update({"count": composite.shape[0], "dtype": "float32"})
with rasterio.open(output_path, "w", **profile) as dst:
dst.write(composite) dst.write(composite)
reproject_and_crop_s3(temp_composite_dir, s2_prepared_dir, s3_preprocessed_dir)
shutil.rmtree(temp_composite_dir)
def run_efast(season, site_position, site_name, date_range=None): def run_efast(season, site_position, site_name, date_range=None):
lat, lon = site_position lat, lon = site_position
@ -241,7 +142,6 @@ def run_efast(season, site_position, site_name, date_range=None):
fusion_output_dir = efast_base_dir / "fusion" fusion_output_dir = efast_base_dir / "fusion"
fusion_output_dir.mkdir(parents=True, exist_ok=True) fusion_output_dir.mkdir(parents=True, exist_ok=True)
print(f"[EFAST] Starting fusion: {site_name} ({lat:.6f}, {lon:.6f}), {season}") print(f"[EFAST] Starting fusion: {site_name} ({lat:.6f}, {lon:.6f}), {season}")
start_str, end_str = datetime_range.split("/") start_str, end_str = datetime_range.split("/")
@ -251,32 +151,19 @@ def run_efast(season, site_position, site_name, date_range=None):
current_date = start_date current_date = start_date
while current_date <= end_date: while current_date <= end_date:
date_str = current_date.strftime("%Y%m%d") date_str = current_date.strftime("%Y%m%d")
output_file = fusion_output_dir / f"REFL_{date_str}.tif" output_file = fusion_output_dir / f"REFL_{date_str}.tif"
if output_file.exists(): if output_file.exists():
print(f"[EFAST] Skipping {date_str} (exists)") print(f"[EFAST] Skipping {date_str} (exists)")
current_date += timedelta(days=1) else:
continue try:
efast.fusion(
try: current_date, s3_output_dir, s2_output_dir, fusion_output_dir,
efast_fusion.fusion( product="REFL", max_days=30, date_position=2,
current_date, minimum_acquisition_importance=0.0, ratio=RESOLUTION_RATIO,
s3_output_dir, )
s2_output_dir, print(f"[EFAST] Saved: {output_file}" if output_file.exists() else f"[EFAST] No output for {date_str} (insufficient nearby data)")
fusion_output_dir, except Exception as e:
product="REFL", print(f"[EFAST] Error processing {date_str}: {e}")
max_days=30,
date_position=2,
minimum_acquisition_importance=0.0,
ratio=RESOLUTION_RATIO,
)
if output_file.exists():
print(f"[EFAST] Saved: {output_file}")
else:
print(f"[EFAST] No output for {date_str} (insufficient nearby data)")
except Exception as e:
print(f"[EFAST] Error processing {date_str}: {e}")
current_date += timedelta(days=1) current_date += timedelta(days=1)
print("[EFAST] Completed") print("[EFAST] Completed")

1
webapp/cloudy.html
View file

@ -20,6 +20,7 @@
.map-label { font-size: 12px; margin-bottom: 5px; color: #666; } .map-label { font-size: 12px; margin-bottom: 5px; color: #666; }
.map { height: 500px; border: 1px solid #ccc; } .map { height: 500px; border: 1px solid #ccc; }
.leaflet-image-layer { image-rendering: pixelated; } .leaflet-image-layer { image-rendering: pixelated; }
.leaflet-control-attribution { display: none; }
</style> </style>
</head> </head>
<body> <body>