Ran linter.
This commit is contained in:
parent
49ad6bcaab
commit
d29754e4a5
6 changed files with 186 additions and 84 deletions
|
|
@ -81,7 +81,9 @@ def load_candidate_cameras(
|
||||||
sitename = str(camera["Sitename"])
|
sitename = str(camera["Sitename"])
|
||||||
if site_filter is not None and sitename not in site_filter:
|
if site_filter is not None and sitename not in site_filter:
|
||||||
continue
|
continue
|
||||||
if not _overlaps_year(camera.get("date_first"), camera.get("date_last"), evaluation_year):
|
if not _overlaps_year(
|
||||||
|
camera.get("date_first"), camera.get("date_last"), evaluation_year
|
||||||
|
):
|
||||||
continue
|
continue
|
||||||
cameras.append(dict(camera))
|
cameras.append(dict(camera))
|
||||||
cameras.sort(key=lambda item: str(item["Sitename"]))
|
cameras.sort(key=lambda item: str(item["Sitename"]))
|
||||||
|
|
@ -129,7 +131,9 @@ def download_site(
|
||||||
|
|
||||||
csv_url = roi.get("one_day_summary") if roi else None
|
csv_url = roi.get("one_day_summary") if roi else None
|
||||||
if csv_url:
|
if csv_url:
|
||||||
download_one_day_csv(csv_url, site_csv_path(cache_dir, evaluation_year, sitename))
|
download_one_day_csv(
|
||||||
|
csv_url, site_csv_path(cache_dir, evaluation_year, sitename)
|
||||||
|
)
|
||||||
return sitename
|
return sitename
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -199,7 +203,9 @@ def write_manifest(
|
||||||
cache_dir: Path,
|
cache_dir: Path,
|
||||||
evaluation_year: int,
|
evaluation_year: int,
|
||||||
) -> None:
|
) -> None:
|
||||||
rel_sites_dir = sites_dir(cache_dir, evaluation_year).relative_to(output_path.parent)
|
rel_sites_dir = sites_dir(cache_dir, evaluation_year).relative_to(
|
||||||
|
output_path.parent
|
||||||
|
)
|
||||||
payload = {
|
payload = {
|
||||||
"evaluation_year": evaluation_year,
|
"evaluation_year": evaluation_year,
|
||||||
"count": len(sitenames),
|
"count": len(sitenames),
|
||||||
|
|
|
||||||
|
|
@ -203,7 +203,9 @@ def screen_site(
|
||||||
calculations["snr"] = snr
|
calculations["snr"] = snr
|
||||||
if snr is None or snr < snr_threshold:
|
if snr is None or snr < snr_threshold:
|
||||||
calculations["failing_gate"] = "snr"
|
calculations["failing_gate"] = "snr"
|
||||||
calculations["reason"] = "insufficient_snr" if snr is not None else "snr_undefined"
|
calculations["reason"] = (
|
||||||
|
"insufficient_snr" if snr is not None else "snr_undefined"
|
||||||
|
)
|
||||||
return {"response": response, "calculations": calculations}
|
return {"response": response, "calculations": calculations}
|
||||||
|
|
||||||
calculations["passed_gates"].append("snr")
|
calculations["passed_gates"].append("snr")
|
||||||
|
|
@ -331,10 +333,7 @@ def print_summary(results: list[dict[str, Any]], evaluation_year: int) -> None:
|
||||||
print(f" after_{gate}: {after}, fail_at_{gate}: {fails}")
|
print(f" after_{gate}: {after}, fail_at_{gate}: {fails}")
|
||||||
|
|
||||||
print("\nPer-site table")
|
print("\nPer-site table")
|
||||||
print(
|
print(f"{'site':<24} {'n':>4} {'mon':>3} {'snr':>6} {'status':>6} gate reason")
|
||||||
f"{'site':<24} {'n':>4} {'mon':>3} {'snr':>6} "
|
|
||||||
f"{'status':>6} gate reason"
|
|
||||||
)
|
|
||||||
print("-" * 72)
|
print("-" * 72)
|
||||||
for row in sorted(
|
for row in sorted(
|
||||||
results,
|
results,
|
||||||
|
|
|
||||||
|
|
@ -388,10 +388,14 @@ def download_s2_window(
|
||||||
with rasterio.Env(**_GDAL_COG_ENV):
|
with rasterio.Env(**_GDAL_COG_ENV):
|
||||||
with ThreadPoolExecutor(max_workers=max_workers) as pool:
|
with ThreadPoolExecutor(max_workers=max_workers) as pool:
|
||||||
futures = {
|
futures = {
|
||||||
pool.submit(_process_item, item, bbox, bands, output_dir, ratio): item.id
|
pool.submit(
|
||||||
|
_process_item, item, bbox, bands, output_dir, ratio
|
||||||
|
): item.id
|
||||||
for item in items
|
for item in items
|
||||||
}
|
}
|
||||||
with tqdm(total=len(futures), unit="granule", desc="S2 COG window read") as pbar:
|
with tqdm(
|
||||||
|
total=len(futures), unit="granule", desc="S2 COG window read"
|
||||||
|
) as pbar:
|
||||||
for fut in as_completed(futures):
|
for fut in as_completed(futures):
|
||||||
msg = fut.result()
|
msg = fut.result()
|
||||||
if msg:
|
if msg:
|
||||||
|
|
@ -461,9 +465,7 @@ def _netcdf_to_geotiffs(nc_path: Path, output_dir: Path, epsg: int) -> int:
|
||||||
n = date_counts.get(date_str, 0)
|
n = date_counts.get(date_str, 0)
|
||||||
date_counts[date_str] = n + 1
|
date_counts[date_str] = n + 1
|
||||||
|
|
||||||
raw = np.stack(
|
raw = np.stack([nc.variables[b][t_idx, :, :] for b in S3_BANDS], axis=0)
|
||||||
[nc.variables[b][t_idx, :, :] for b in S3_BANDS], axis=0
|
|
||||||
)
|
|
||||||
stacked = (
|
stacked = (
|
||||||
np.ma.filled(raw, fill_value=np.nan).astype("float32")
|
np.ma.filled(raw, fill_value=np.nan).astype("float32")
|
||||||
/ S3_REFLECTANCE_SCALE
|
/ S3_REFLECTANCE_SCALE
|
||||||
|
|
@ -523,7 +525,9 @@ def _download_with_retry(datacube: Any, nc_path: Path) -> None:
|
||||||
delay *= 2
|
delay *= 2
|
||||||
else:
|
else:
|
||||||
print(f"[S3-OEO] All {_S3_DOWNLOAD_RETRIES} download attempts failed")
|
print(f"[S3-OEO] All {_S3_DOWNLOAD_RETRIES} download attempts failed")
|
||||||
raise RuntimeError(f"S3 download failed after {_S3_DOWNLOAD_RETRIES} attempts") from last_exc
|
raise RuntimeError(
|
||||||
|
f"S3 download failed after {_S3_DOWNLOAD_RETRIES} attempts"
|
||||||
|
) from last_exc
|
||||||
|
|
||||||
|
|
||||||
def download_s3_openeo(
|
def download_s3_openeo(
|
||||||
|
|
@ -592,9 +596,7 @@ def _import_distance_to_clouds():
|
||||||
|
|
||||||
return distance_to_clouds
|
return distance_to_clouds
|
||||||
except ImportError as exc:
|
except ImportError as exc:
|
||||||
raise ImportError(
|
raise ImportError("efast not found. Install with: uv sync") from exc
|
||||||
"efast not found. Install with: uv sync"
|
|
||||||
) from exc
|
|
||||||
|
|
||||||
|
|
||||||
def _normalize_s2_grid(s2_dir: Path) -> None:
|
def _normalize_s2_grid(s2_dir: Path) -> None:
|
||||||
|
|
@ -663,9 +665,7 @@ def _import_s3_processing():
|
||||||
|
|
||||||
return s3_processing
|
return s3_processing
|
||||||
except ImportError as exc:
|
except ImportError as exc:
|
||||||
raise ImportError(
|
raise ImportError("efast not found. Install with: uv sync") from exc
|
||||||
"efast not found. Install with: uv sync"
|
|
||||||
) from exc
|
|
||||||
|
|
||||||
|
|
||||||
def _reproject_s3_composites_to_s2_grid(
|
def _reproject_s3_composites_to_s2_grid(
|
||||||
|
|
@ -912,7 +912,9 @@ def main(argv: list[str] | None = None) -> int:
|
||||||
sitename = site["sitename"]
|
sitename = site["sitename"]
|
||||||
site_dir = DATA_DIR / "sentinel_data" / str(year) / sitename
|
site_dir = DATA_DIR / "sentinel_data" / str(year) / sitename
|
||||||
if args.skip_downloaded and site_dir.exists():
|
if args.skip_downloaded and site_dir.exists():
|
||||||
print(f"[Sentinel-3] ({i}/{len(pass_sites)}) {sitename} — skipping (directory exists)")
|
print(
|
||||||
|
f"[Sentinel-3] ({i}/{len(pass_sites)}) {sitename} — skipping (directory exists)"
|
||||||
|
)
|
||||||
continue
|
continue
|
||||||
print(f"[Sentinel-3] ({i}/{len(pass_sites)}) {sitename}")
|
print(f"[Sentinel-3] ({i}/{len(pass_sites)}) {sitename}")
|
||||||
summary = process_site(sitename, site["lat"], site["lon"], year)
|
summary = process_site(sitename, site["lat"], site["lon"], year)
|
||||||
|
|
|
||||||
24
4-fusion.py
24
4-fusion.py
|
|
@ -61,9 +61,7 @@ def _import_efast():
|
||||||
|
|
||||||
return efast_module
|
return efast_module
|
||||||
except ImportError as exc:
|
except ImportError as exc:
|
||||||
raise ImportError(
|
raise ImportError("efast not found. Install with: uv sync") from exc
|
||||||
"efast not found. Install with: uv sync"
|
|
||||||
) from exc
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
@ -174,7 +172,9 @@ def fuse_site(sitename: str, year: int) -> dict[str, Any]:
|
||||||
|
|
||||||
s2_dir = DATA_DIR / "sentinel_data" / str(year) / sitename / "prepared" / "s2"
|
s2_dir = DATA_DIR / "sentinel_data" / str(year) / sitename / "prepared" / "s2"
|
||||||
s3_dir = DATA_DIR / "sentinel_data" / str(year) / sitename / "prepared" / "s3"
|
s3_dir = DATA_DIR / "sentinel_data" / str(year) / sitename / "prepared" / "s3"
|
||||||
gcc_s3_dir = DATA_DIR / "sentinel_data" / str(year) / sitename / "prepared" / "gcc_s3"
|
gcc_s3_dir = (
|
||||||
|
DATA_DIR / "sentinel_data" / str(year) / sitename / "prepared" / "gcc_s3"
|
||||||
|
)
|
||||||
base = DATA_DIR / "fusion" / str(year) / sitename
|
base = DATA_DIR / "fusion" / str(year) / sitename
|
||||||
|
|
||||||
if not s2_dir.is_dir() or not any(s2_dir.glob("*_REFL.tif")):
|
if not s2_dir.is_dir() or not any(s2_dir.glob("*_REFL.tif")):
|
||||||
|
|
@ -228,11 +228,15 @@ def fuse_site(sitename: str, year: int) -> dict[str, Any]:
|
||||||
|
|
||||||
print(f"[{sitename}] ItB: fusing GCC over {len(fusion_dates)} dates...")
|
print(f"[{sitename}] ItB: fusing GCC over {len(fusion_dates)} dates...")
|
||||||
for date in fusion_dates:
|
for date in fusion_dates:
|
||||||
efast.fusion(date, gcc_s3_dir, s2_dir, itb_fusion, product="GCC", **_fusion_kwargs)
|
efast.fusion(
|
||||||
|
date, gcc_s3_dir, s2_dir, itb_fusion, product="GCC", **_fusion_kwargs
|
||||||
|
)
|
||||||
|
|
||||||
# --- BtI: fuse reflectance (3-band, matching S2 B02/B03/B04), then derive GCC ---
|
# --- BtI: fuse reflectance (3-band, matching S2 B02/B03/B04), then derive GCC ---
|
||||||
# S3 composites have 4 bands; strip band 4 (Oa17/NIR) so shapes match S2 REFL.
|
# S3 composites have 4 bands; strip band 4 (Oa17/NIR) so shapes match S2 REFL.
|
||||||
s3_rgb_dir = DATA_DIR / "sentinel_data" / str(year) / sitename / "prepared" / "s3_rgb"
|
s3_rgb_dir = (
|
||||||
|
DATA_DIR / "sentinel_data" / str(year) / sitename / "prepared" / "s3_rgb"
|
||||||
|
)
|
||||||
s3_rgb_dir.mkdir(parents=True, exist_ok=True)
|
s3_rgb_dir.mkdir(parents=True, exist_ok=True)
|
||||||
for p in sorted(s3_dir.glob("composite_*.tif")):
|
for p in sorted(s3_dir.glob("composite_*.tif")):
|
||||||
out = s3_rgb_dir / p.name
|
out = s3_rgb_dir / p.name
|
||||||
|
|
@ -250,7 +254,9 @@ def fuse_site(sitename: str, year: int) -> dict[str, Any]:
|
||||||
|
|
||||||
print(f"[{sitename}] BtI: fusing REFL over {len(fusion_dates)} dates...")
|
print(f"[{sitename}] BtI: fusing REFL over {len(fusion_dates)} dates...")
|
||||||
for date in fusion_dates:
|
for date in fusion_dates:
|
||||||
efast.fusion(date, s3_rgb_dir, s2_dir, bti_fusion, product="REFL", **_fusion_kwargs)
|
efast.fusion(
|
||||||
|
date, s3_rgb_dir, s2_dir, bti_fusion, product="REFL", **_fusion_kwargs
|
||||||
|
)
|
||||||
|
|
||||||
print(f"[{sitename}] BtI: deriving GCC from fused REFL...")
|
print(f"[{sitename}] BtI: deriving GCC from fused REFL...")
|
||||||
compute_gcc_from_refl(bti_fusion, bti_gcc)
|
compute_gcc_from_refl(bti_fusion, bti_gcc)
|
||||||
|
|
@ -318,7 +324,9 @@ def main(argv: list[str] | None = None) -> int:
|
||||||
for i, sitename in enumerate(sites, 1):
|
for i, sitename in enumerate(sites, 1):
|
||||||
fusion_dir = DATA_DIR / "fusion" / str(year) / sitename
|
fusion_dir = DATA_DIR / "fusion" / str(year) / sitename
|
||||||
if args.skip_blended and fusion_dir.exists():
|
if args.skip_blended and fusion_dir.exists():
|
||||||
print(f"[Fusion] ({i}/{len(sites)}) {sitename} — skipping (fusion directory exists)")
|
print(
|
||||||
|
f"[Fusion] ({i}/{len(sites)}) {sitename} — skipping (fusion directory exists)"
|
||||||
|
)
|
||||||
continue
|
continue
|
||||||
print(f"[Fusion] ({i}/{len(sites)}) {sitename}")
|
print(f"[Fusion] ({i}/{len(sites)}) {sitename}")
|
||||||
summary = fuse_site(sitename, year)
|
summary = fuse_site(sitename, year)
|
||||||
|
|
|
||||||
151
5-metrics.py
151
5-metrics.py
|
|
@ -72,7 +72,9 @@ WHITTAKER_LAMBDA = 400.0
|
||||||
SPATIAL_CV_HALF_M = 150
|
SPATIAL_CV_HALF_M = 150
|
||||||
|
|
||||||
# PhenoCam archive image URL pattern
|
# PhenoCam archive image URL pattern
|
||||||
PHENOCAM_IMAGE_URL = "https://phenocam.nau.edu/data/archive/{site}/{year}/{month}/{filename}"
|
PHENOCAM_IMAGE_URL = (
|
||||||
|
"https://phenocam.nau.edu/data/archive/{site}/{year}/{month}/{filename}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
@ -133,7 +135,9 @@ def _date_from_s2_tif(path: Path) -> str | None:
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
def _whittaker_smooth(values: list[float | None], lam: float = WHITTAKER_LAMBDA) -> list[float | None]:
|
def _whittaker_smooth(
|
||||||
|
values: list[float | None], lam: float = WHITTAKER_LAMBDA
|
||||||
|
) -> list[float | None]:
|
||||||
"""Penalised least-squares smoother (Whittaker, 2nd-order differences).
|
"""Penalised least-squares smoother (Whittaker, 2nd-order differences).
|
||||||
|
|
||||||
Masked (None) values are filled via the smooth and then re-set to None in
|
Masked (None) values are filled via the smooth and then re-set to None in
|
||||||
|
|
@ -211,9 +215,7 @@ def _parse_phenocam_csv(
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
def _moving_average(
|
def _moving_average(series: list[dict], value_key: str, window: int) -> list[dict]:
|
||||||
series: list[dict], value_key: str, window: int
|
|
||||||
) -> list[dict]:
|
|
||||||
"""Compute centred moving average; returns new list with ``_smooth`` suffix key."""
|
"""Compute centred moving average; returns new list with ``_smooth`` suffix key."""
|
||||||
if not series:
|
if not series:
|
||||||
return []
|
return []
|
||||||
|
|
@ -222,10 +224,12 @@ def _moving_average(
|
||||||
smoothed = []
|
smoothed = []
|
||||||
for i, pt in enumerate(series):
|
for i, pt in enumerate(series):
|
||||||
chunk = [v for v in vals[max(0, i - half) : i + half + 1] if v is not None]
|
chunk = [v for v in vals[max(0, i - half) : i + half + 1] if v is not None]
|
||||||
smoothed.append({
|
smoothed.append(
|
||||||
|
{
|
||||||
"date": pt["date"],
|
"date": pt["date"],
|
||||||
value_key + "_smooth": (sum(chunk) / len(chunk)) if chunk else None,
|
value_key + "_smooth": (sum(chunk) / len(chunk)) if chunk else None,
|
||||||
})
|
}
|
||||||
|
)
|
||||||
return smoothed
|
return smoothed
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -237,8 +241,10 @@ MATCH_TOLERANCE_DAYS = 5
|
||||||
|
|
||||||
|
|
||||||
def compute_metrics(
|
def compute_metrics(
|
||||||
ref: list[dict], ref_key: str,
|
ref: list[dict],
|
||||||
pred: list[dict], pred_key: str,
|
ref_key: str,
|
||||||
|
pred: list[dict],
|
||||||
|
pred_key: str,
|
||||||
) -> dict | None:
|
) -> dict | None:
|
||||||
"""Compute NSE, RMSE, nRMSE, Pearson r between pred and ref.
|
"""Compute NSE, RMSE, nRMSE, Pearson r between pred and ref.
|
||||||
|
|
||||||
|
|
@ -246,7 +252,9 @@ def compute_metrics(
|
||||||
``MATCH_TOLERANCE_DAYS``. Returns a dict or ``None`` if fewer than
|
``MATCH_TOLERANCE_DAYS``. Returns a dict or ``None`` if fewer than
|
||||||
2 matched pairs exist.
|
2 matched pairs exist.
|
||||||
"""
|
"""
|
||||||
ref_lookup: dict[str, float] = {p["date"]: p[ref_key] for p in ref if p.get(ref_key) is not None}
|
ref_lookup: dict[str, float] = {
|
||||||
|
p["date"]: p[ref_key] for p in ref if p.get(ref_key) is not None
|
||||||
|
}
|
||||||
if not ref_lookup:
|
if not ref_lookup:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
@ -257,9 +265,16 @@ def compute_metrics(
|
||||||
v = pt.get(pred_key)
|
v = pt.get(pred_key)
|
||||||
if v is None:
|
if v is None:
|
||||||
continue
|
continue
|
||||||
nearest = min(ref_dates, key=lambda d: abs((
|
nearest = min(
|
||||||
np.datetime64(pt["date"]) - np.datetime64(d)) / np.timedelta64(1, "D")))
|
ref_dates,
|
||||||
gap = abs((np.datetime64(pt["date"]) - np.datetime64(nearest)) / np.timedelta64(1, "D"))
|
key=lambda d: abs(
|
||||||
|
(np.datetime64(pt["date"]) - np.datetime64(d)) / np.timedelta64(1, "D")
|
||||||
|
),
|
||||||
|
)
|
||||||
|
gap = abs(
|
||||||
|
(np.datetime64(pt["date"]) - np.datetime64(nearest))
|
||||||
|
/ np.timedelta64(1, "D")
|
||||||
|
)
|
||||||
if gap <= MATCH_TOLERANCE_DAYS and nearest in ref_lookup:
|
if gap <= MATCH_TOLERANCE_DAYS and nearest in ref_lookup:
|
||||||
obs.append(ref_lookup[nearest])
|
obs.append(ref_lookup[nearest])
|
||||||
sim.append(v)
|
sim.append(v)
|
||||||
|
|
@ -281,7 +296,13 @@ def compute_metrics(
|
||||||
def _r4(v: float | None) -> float | None:
|
def _r4(v: float | None) -> float | None:
|
||||||
return round(v, 4) if v is not None else None
|
return round(v, 4) if v is not None else None
|
||||||
|
|
||||||
return {"n": len(obs), "rmse": _r4(rmse), "nrmse": _r4(nrmse), "nse": _r4(nse), "r": _r4(float(r))}
|
return {
|
||||||
|
"n": len(obs),
|
||||||
|
"rmse": _r4(rmse),
|
||||||
|
"nrmse": _r4(nrmse),
|
||||||
|
"nse": _r4(nse),
|
||||||
|
"r": _r4(float(r)),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
S2_BAND_NAMES = ["B02", "B03", "B04"]
|
S2_BAND_NAMES = ["B02", "B03", "B04"]
|
||||||
|
|
@ -294,7 +315,9 @@ def _read_multiband_center(
|
||||||
"""Return 3×3 mean per band at (lat, lon). Keys are ``band_names``, values float or None."""
|
"""Return 3×3 mean per band at (lat, lon). Keys are ``band_names``, values float or None."""
|
||||||
try:
|
try:
|
||||||
with rasterio.open(path) as src:
|
with rasterio.open(path) as src:
|
||||||
transformer = Transformer.from_crs(CRS.from_epsg(4326), src.crs, always_xy=True)
|
transformer = Transformer.from_crs(
|
||||||
|
CRS.from_epsg(4326), src.crs, always_xy=True
|
||||||
|
)
|
||||||
x, y = transformer.transform(lon, lat)
|
x, y = transformer.transform(lon, lat)
|
||||||
row, col = rowcol(src.transform, x, y)
|
row, col = rowcol(src.transform, x, y)
|
||||||
h, w = src.height, src.width
|
h, w = src.height, src.width
|
||||||
|
|
@ -353,7 +376,9 @@ def _read_footprint_stats(
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
with rasterio.open(path) as src:
|
with rasterio.open(path) as src:
|
||||||
transformer = Transformer.from_crs(CRS.from_epsg(4326), src.crs, always_xy=True)
|
transformer = Transformer.from_crs(
|
||||||
|
CRS.from_epsg(4326), src.crs, always_xy=True
|
||||||
|
)
|
||||||
x, y = transformer.transform(lon, lat)
|
x, y = transformer.transform(lon, lat)
|
||||||
res = abs(src.transform.a) # pixel size in CRS units (metres for UTM)
|
res = abs(src.transform.a) # pixel size in CRS units (metres for UTM)
|
||||||
half_px = max(1, int(round(half_m / res)))
|
half_px = max(1, int(round(half_m / res)))
|
||||||
|
|
@ -392,7 +417,11 @@ def compute_covariates(
|
||||||
means.append(m)
|
means.append(m)
|
||||||
stds.append(s)
|
stds.append(s)
|
||||||
|
|
||||||
spatial_gcc_cv = round(float(np.mean([s / m for s, m in zip(stds, means)])), 4) if means else None
|
spatial_gcc_cv = (
|
||||||
|
round(float(np.mean([s / m for s, m in zip(stds, means)])), 4)
|
||||||
|
if means
|
||||||
|
else None
|
||||||
|
)
|
||||||
spatial_gcc_std = round(float(np.mean(stds)), 4) if stds else None
|
spatial_gcc_std = round(float(np.mean(stds)), 4) if stds else None
|
||||||
|
|
||||||
# S2 temporal gap statistics
|
# S2 temporal gap statistics
|
||||||
|
|
@ -521,8 +550,22 @@ def export_site(
|
||||||
]
|
]
|
||||||
|
|
||||||
# Band reflectance timeseries (multi-band center-pixel)
|
# Band reflectance timeseries (multi-band center-pixel)
|
||||||
bands_s2 = _multiband_series(sorted(s2_refl_dir.glob("*_REFL.tif")), _date_from_s2_tif, lat, lon, S2_BAND_NAMES, f"{site} S2 bands")
|
bands_s2 = _multiband_series(
|
||||||
bands_s3 = _multiband_series(sorted(s3_comp_dir.glob("composite_*.tif")), _date_from_gcc_tif, lat, lon, S3_BAND_NAMES, f"{site} S3 bands")
|
sorted(s2_refl_dir.glob("*_REFL.tif")),
|
||||||
|
_date_from_s2_tif,
|
||||||
|
lat,
|
||||||
|
lon,
|
||||||
|
S2_BAND_NAMES,
|
||||||
|
f"{site} S2 bands",
|
||||||
|
)
|
||||||
|
bands_s3 = _multiband_series(
|
||||||
|
sorted(s3_comp_dir.glob("composite_*.tif")),
|
||||||
|
_date_from_gcc_tif,
|
||||||
|
lat,
|
||||||
|
lon,
|
||||||
|
S3_BAND_NAMES,
|
||||||
|
f"{site} S3 bands",
|
||||||
|
)
|
||||||
|
|
||||||
# --- Per-metric JSON outputs ---
|
# --- Per-metric JSON outputs ---
|
||||||
_write_json(out_dir / "gcc_phenocam.json", phenocam_series)
|
_write_json(out_dir / "gcc_phenocam.json", phenocam_series)
|
||||||
|
|
@ -545,16 +588,40 @@ def export_site(
|
||||||
s2_valid_dates = {p["date"].replace("-", "") for p in s2_series}
|
s2_valid_dates = {p["date"].replace("-", "") for p in s2_series}
|
||||||
s3_valid_dates = {p["date"].replace("-", "") for p in s3_series}
|
s3_valid_dates = {p["date"].replace("-", "") for p in s3_series}
|
||||||
|
|
||||||
s2_refl = [r for r in _raster_index(sorted(s2_refl_dir.glob("*_REFL.tif")), _date_from_s2_tif, rel_root)
|
s2_refl = [
|
||||||
if r["date"] in s2_valid_dates]
|
r
|
||||||
s3_comp = [r for r in _raster_index(sorted(s3_comp_dir.glob("composite_*.tif")), _date_from_gcc_tif, rel_root)
|
for r in _raster_index(
|
||||||
if r["date"] in s3_valid_dates]
|
sorted(s2_refl_dir.glob("*_REFL.tif")), _date_from_s2_tif, rel_root
|
||||||
s2_gcc = [r for r in _raster_index(sorted(s2_gcc_dir.glob("*_GCC.tif")), _date_from_s2_tif, rel_root)
|
)
|
||||||
if r["date"] in s2_valid_dates]
|
if r["date"] in s2_valid_dates
|
||||||
s3_gcc = [r for r in _raster_index(sorted(s3_gcc_dir.glob("composite_*.tif")), _date_from_gcc_tif, rel_root)
|
]
|
||||||
if r["date"] in s3_valid_dates]
|
s3_comp = [
|
||||||
bti_refl = _raster_index(sorted(bti_refl_dir.glob("REFL_*.tif")), _date_from_gcc_tif, rel_root)
|
r
|
||||||
itb_gcc = _raster_index(sorted(itb_gcc_dir.glob("GCC_*.tif")), _date_from_gcc_tif, rel_root)
|
for r in _raster_index(
|
||||||
|
sorted(s3_comp_dir.glob("composite_*.tif")), _date_from_gcc_tif, rel_root
|
||||||
|
)
|
||||||
|
if r["date"] in s3_valid_dates
|
||||||
|
]
|
||||||
|
s2_gcc = [
|
||||||
|
r
|
||||||
|
for r in _raster_index(
|
||||||
|
sorted(s2_gcc_dir.glob("*_GCC.tif")), _date_from_s2_tif, rel_root
|
||||||
|
)
|
||||||
|
if r["date"] in s2_valid_dates
|
||||||
|
]
|
||||||
|
s3_gcc = [
|
||||||
|
r
|
||||||
|
for r in _raster_index(
|
||||||
|
sorted(s3_gcc_dir.glob("composite_*.tif")), _date_from_gcc_tif, rel_root
|
||||||
|
)
|
||||||
|
if r["date"] in s3_valid_dates
|
||||||
|
]
|
||||||
|
bti_refl = _raster_index(
|
||||||
|
sorted(bti_refl_dir.glob("REFL_*.tif")), _date_from_gcc_tif, rel_root
|
||||||
|
)
|
||||||
|
itb_gcc = _raster_index(
|
||||||
|
sorted(itb_gcc_dir.glob("GCC_*.tif")), _date_from_gcc_tif, rel_root
|
||||||
|
)
|
||||||
|
|
||||||
_write_json(out_dir / "rasters_s2_refl.json", s2_refl)
|
_write_json(out_dir / "rasters_s2_refl.json", s2_refl)
|
||||||
_write_json(out_dir / "rasters_s3_composite.json", s3_comp)
|
_write_json(out_dir / "rasters_s3_composite.json", s3_comp)
|
||||||
|
|
@ -564,19 +631,27 @@ def export_site(
|
||||||
_write_json(out_dir / "rasters_fusion_itb_gcc.json", itb_gcc)
|
_write_json(out_dir / "rasters_fusion_itb_gcc.json", itb_gcc)
|
||||||
|
|
||||||
# --- Site covariates (heterogeneity + observation density) ---
|
# --- Site covariates (heterogeneity + observation density) ---
|
||||||
_write_json(out_dir / "covariates.json", compute_covariates(
|
_write_json(
|
||||||
s2_gcc_paths, s2_series, s3_series, n_gcc_points, lat, lon
|
out_dir / "covariates.json",
|
||||||
))
|
compute_covariates(s2_gcc_paths, s2_series, s3_series, n_gcc_points, lat, lon),
|
||||||
|
)
|
||||||
|
|
||||||
# --- Validation metrics vs PhenoCam gcc_90 ---
|
# --- Validation metrics vs PhenoCam gcc_90 ---
|
||||||
_write_json(out_dir / "metrics.json", {
|
_write_json(
|
||||||
|
out_dir / "metrics.json",
|
||||||
|
{
|
||||||
"bti": compute_metrics(phenocam_series, "gcc_90", bti_series, "gcc"),
|
"bti": compute_metrics(phenocam_series, "gcc_90", bti_series, "gcc"),
|
||||||
"itb": compute_metrics(phenocam_series, "gcc_90", itb_series, "gcc"),
|
"itb": compute_metrics(phenocam_series, "gcc_90", itb_series, "gcc"),
|
||||||
"s2_whittaker": compute_metrics(phenocam_series, "gcc_90", s2_whittaker, "gcc"),
|
"s2_whittaker": compute_metrics(
|
||||||
"s3_smooth": compute_metrics(phenocam_series, "gcc_90", s3_smooth_series, "gcc"),
|
phenocam_series, "gcc_90", s2_whittaker, "gcc"
|
||||||
|
),
|
||||||
|
"s3_smooth": compute_metrics(
|
||||||
|
phenocam_series, "gcc_90", s3_smooth_series, "gcc"
|
||||||
|
),
|
||||||
"s2": compute_metrics(phenocam_series, "gcc_90", s2_series, "gcc"),
|
"s2": compute_metrics(phenocam_series, "gcc_90", s2_series, "gcc"),
|
||||||
"s3": compute_metrics(phenocam_series, "gcc_90", s3_series, "gcc"),
|
"s3": compute_metrics(phenocam_series, "gcc_90", s3_series, "gcc"),
|
||||||
})
|
},
|
||||||
|
)
|
||||||
|
|
||||||
# Remove legacy bundled outputs if present
|
# Remove legacy bundled outputs if present
|
||||||
for legacy in ("timeseries.json", "rasters.json"):
|
for legacy in ("timeseries.json", "rasters.json"):
|
||||||
|
|
@ -682,7 +757,9 @@ def main() -> None:
|
||||||
print(f"Exporting {len(fusion_sites)} site(s) with fusion data for {year}")
|
print(f"Exporting {len(fusion_sites)} site(s) with fusion data for {year}")
|
||||||
for site, meta in tqdm(fusion_sites.items(), desc="Sites"):
|
for site, meta in tqdm(fusion_sites.items(), desc="Sites"):
|
||||||
out_dir = out_base / str(year) / site
|
out_dir = out_base / str(year) / site
|
||||||
ok = export_site(site, year, meta["lat"], meta["lon"], out_dir, meta.get("n_gcc_points"))
|
ok = export_site(
|
||||||
|
site, year, meta["lat"], meta["lon"], out_dir, meta.get("n_gcc_points")
|
||||||
|
)
|
||||||
if ok:
|
if ok:
|
||||||
print(f" ✓ {site}")
|
print(f" ✓ {site}")
|
||||||
else:
|
else:
|
||||||
|
|
|
||||||
|
|
@ -121,12 +121,16 @@ def repair(site_dir: Path) -> None:
|
||||||
refl_path.unlink(missing_ok=True)
|
refl_path.unlink(missing_ok=True)
|
||||||
n_removed += 1
|
n_removed += 1
|
||||||
|
|
||||||
print(f"[repair] {name}: removed {n_removed} minority-shape file-sets (kept {ref_shape[0]}×{ref_shape[1]})")
|
print(
|
||||||
|
f"[repair] {name}: removed {n_removed} minority-shape file-sets (kept {ref_shape[0]}×{ref_shape[1]})"
|
||||||
|
)
|
||||||
|
|
||||||
# --- 3. Remove stale GCC files from prepared/s2 ---------------------------
|
# --- 3. Remove stale GCC files from prepared/s2 ---------------------------
|
||||||
gcc_removed = sum(1 for f in s2_dir.glob("*_GCC.tif") if f.unlink() or True)
|
gcc_removed = sum(1 for f in s2_dir.glob("*_GCC.tif") if f.unlink() or True)
|
||||||
if gcc_removed:
|
if gcc_removed:
|
||||||
print(f"[repair] {name}: removed {gcc_removed} stale GCC files from prepared/s2")
|
print(
|
||||||
|
f"[repair] {name}: removed {gcc_removed} stale GCC files from prepared/s2"
|
||||||
|
)
|
||||||
|
|
||||||
# --- 4. Wipe stale S3 composites ------------------------------------------
|
# --- 4. Wipe stale S3 composites ------------------------------------------
|
||||||
for d in (s3_out, gcc_s3_out):
|
for d in (s3_out, gcc_s3_out):
|
||||||
|
|
@ -136,15 +140,21 @@ def repair(site_dir: Path) -> None:
|
||||||
|
|
||||||
# --- 5. Regenerate S3 composites with the correct reference ---------------
|
# --- 5. Regenerate S3 composites with the correct reference ---------------
|
||||||
if not s3_raw.exists() or not any(s3_raw.glob("S3*.tif")):
|
if not s3_raw.exists() or not any(s3_raw.glob("S3*.tif")):
|
||||||
print(f"[repair] {name}: WARNING — no raw S3 data in {s3_raw}; skipping S3 regeneration.")
|
print(
|
||||||
|
f"[repair] {name}: WARNING — no raw S3 data in {s3_raw}; skipping S3 regeneration."
|
||||||
|
)
|
||||||
return
|
return
|
||||||
|
|
||||||
s2_refl_path = next(iter(sorted(s2_dir.glob("*_REFL.tif"))), None)
|
s2_refl_path = next(iter(sorted(s2_dir.glob("*_REFL.tif"))), None)
|
||||||
if s2_refl_path is None:
|
if s2_refl_path is None:
|
||||||
print(f"[repair] {name}: WARNING — no REFL files left; cannot regenerate S3 composites.")
|
print(
|
||||||
|
f"[repair] {name}: WARNING — no REFL files left; cannot regenerate S3 composites."
|
||||||
|
)
|
||||||
return
|
return
|
||||||
|
|
||||||
print(f"[repair] {name}: regenerating S3 composites (reference: {s2_refl_path.name})...")
|
print(
|
||||||
|
f"[repair] {name}: regenerating S3 composites (reference: {s2_refl_path.name})..."
|
||||||
|
)
|
||||||
step3 = _load_step3()
|
step3 = _load_step3()
|
||||||
s3_out.mkdir(parents=True, exist_ok=True)
|
s3_out.mkdir(parents=True, exist_ok=True)
|
||||||
step3._prepare_s3(s3_raw, s2_refl_path, s3_out)
|
step3._prepare_s3(s3_raw, s2_refl_path, s3_out)
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue