This commit is contained in:
Felix Delattre 2026-04-01 16:17:46 +02:00
parent a037e6b4fd
commit 8e7fb1de18
6 changed files with 133 additions and 52 deletions

1
.gitignore vendored
View file

@ -1,5 +1,6 @@
# Project data # Project data
data/* data/*
webapp/data
# Environment # Environment
.env .env

View file

@ -459,6 +459,37 @@ def _create_bands_timeseries_for_dir(input_dir, output_dir, site_position, sourc
print(f"[BANDS-{source_name}] Saved: {output_dir / 'timeseries.json'} ({len(timeseries)} entries)") print(f"[BANDS-{source_name}] Saved: {output_dir / 'timeseries.json'} ({len(timeseries)} entries)")
def _write_export(ndvi_dir, gcc_dir, bands_dir, export_dir):
"""Merge ndvi, gcc, bands into combined timeseries.json and timeseries.csv."""
def load(p):
p = Path(p)
if not p.exists():
return []
try:
return json.loads((p / "timeseries.json").read_text())
except Exception:
return []
ndvi = {str(t.get("date", ""))[:10]: t for t in load(ndvi_dir)}
gcc = {str(t.get("date", ""))[:10]: t for t in load(gcc_dir)}
bands = {str(t.get("date", ""))[:10]: t for t in load(bands_dir)}
keys = sorted(set(ndvi) | set(gcc) | set(bands))
merged = []
for k in keys:
r = {"date": k, "filename": ""}
for d in [ndvi.get(k, {}), gcc.get(k, {}), bands.get(k, {})]:
r.update({x: d[x] for x in d if x not in ("date",)})
merged.append(r)
export_dir.mkdir(parents=True, exist_ok=True)
(export_dir / "timeseries.json").write_text(json.dumps(merged, indent=2))
cols = ["date", "filename", "ndvi", "greenness_index", "b02", "b03", "b04", "b8a"]
def esc(v):
s = str(v) if v is not None else ""
return f'"{s}"' if "," in s or '"' in s else s
rows = [cols] + [[esc(r.get(c)) for c in cols] for r in merged]
(export_dir / "timeseries.csv").write_text("\n".join(",".join(x) for x in rows))
print(f"[EXPORT] Saved {export_dir / 'timeseries.json'} and timeseries.csv ({len(merged)} entries)")
def create_prepared_fusion_timeseries(season, site_position, site_name): def create_prepared_fusion_timeseries(season, site_position, site_name):
"""Generate NDVI, GCC, and band timeseries for prepared S2/S3 and fusion outputs.""" """Generate NDVI, GCC, and band timeseries for prepared S2/S3 and fusion outputs."""
for strategy in ["aggressive", "nonaggressive"]: for strategy in ["aggressive", "nonaggressive"]:
@ -469,12 +500,14 @@ def create_prepared_fusion_timeseries(season, site_position, site_name):
_create_timeseries_for_dir(inp, base / "ndvi" / source, site_position, f"PREPARED-{source.upper()}-{strategy}", "*.tif") _create_timeseries_for_dir(inp, base / "ndvi" / source, site_position, f"PREPARED-{source.upper()}-{strategy}", "*.tif")
_create_gcc_timeseries_for_dir(inp, base / "gcc" / source, site_position, f"PREPARED-{source.upper()}-{strategy}", "*.tif") _create_gcc_timeseries_for_dir(inp, base / "gcc" / source, site_position, f"PREPARED-{source.upper()}-{strategy}", "*.tif")
_create_bands_timeseries_for_dir(inp, base / "bands" / source, site_position, f"PREPARED-{source.upper()}-{strategy}", "*.tif") _create_bands_timeseries_for_dir(inp, base / "bands" / source, site_position, f"PREPARED-{source.upper()}-{strategy}", "*.tif")
_write_export(base / "ndvi" / source, base / "gcc" / source, base / "bands" / source, base / "export" / source)
for sig, fusion_sub in [(None, "fusion"), (30, "fusion_sigma30")]: for sig, fusion_sub in [(None, "fusion"), (30, "fusion_sigma30")]:
inp = base / fusion_sub inp = base / fusion_sub
if inp.exists(): if inp.exists():
_create_timeseries_for_dir(inp, base / "ndvi" / fusion_sub, site_position, f"FUSION-{strategy}-σ{sig or 20}", "*.tif") _create_timeseries_for_dir(inp, base / "ndvi" / fusion_sub, site_position, f"FUSION-{strategy}-σ{sig or 20}", "*.tif")
_create_gcc_timeseries_for_dir(inp, base / "gcc" / fusion_sub, site_position, f"FUSION-{strategy}-σ{sig or 20}", "*.tif") _create_gcc_timeseries_for_dir(inp, base / "gcc" / fusion_sub, site_position, f"FUSION-{strategy}-σ{sig or 20}", "*.tif")
_create_bands_timeseries_for_dir(inp, base / "bands" / fusion_sub, site_position, f"FUSION-{strategy}-σ{sig or 20}", "*.tif") _create_bands_timeseries_for_dir(inp, base / "bands" / fusion_sub, site_position, f"FUSION-{strategy}-σ{sig or 20}", "*.tif")
_write_export(base / "ndvi" / fusion_sub, base / "gcc" / fusion_sub, base / "bands" / fusion_sub, base / "export" / fusion_sub)
def create_bands_timeseries_post_process(season, site_position, site_name): def create_bands_timeseries_post_process(season, site_position, site_name):
@ -486,3 +519,4 @@ def create_bands_timeseries_post_process(season, site_position, site_name):
inp, out = base / source, base / "bands" / source inp, out = base / source, base / "bands" / source
if inp.exists(): if inp.exists():
_create_bands_timeseries_for_dir(inp, out, site_position, f"POST-{source.upper()}-{strategy}-σ{sigma}", "*.geotiff") _create_bands_timeseries_for_dir(inp, out, site_position, f"POST-{source.upper()}-{strategy}-σ{sigma}", "*.geotiff")
_write_export(base / "ndvi" / source, base / "gcc" / source, base / "bands" / source, base / "export" / source)

20
run.py
View file

@ -17,26 +17,26 @@ def run_pipeline(season, site_position, site_name):
#download_s3(season, site_position, site_name) #download_s3(season, site_position, site_name)
#download_phenocam(season, site_position, site_name) #download_phenocam(season, site_position, site_name)
print(f"Creating preselection timeseries: {site_name}, {season}") # print(f"Creating preselection timeseries: {site_name}, {season}")
create_timeseries(season, site_position, site_name) # create_timeseries(season, site_position, site_name)
print(f"Preparing S2 and S3 for fusion: {site_name}, {season}") # print(f"Preparing S2 and S3 for fusion: {site_name}, {season}")
for strategy in ["aggressive", "nonaggressive"]: # for strategy in ["aggressive", "nonaggressive"]:
prepare_s2(season, site_position, site_name, cleaning_strategy=strategy) # prepare_s2(season, site_position, site_name, cleaning_strategy=strategy)
prepare_s3(season, site_position, site_name, cleaning_strategy=strategy) # prepare_s3(season, site_position, site_name, cleaning_strategy=strategy)
print(f"Running EFAST fusion for all scenarios: {site_name}, {season}") # print(f"Running EFAST fusion for all scenarios: {site_name}, {season}")
run_all_efast_scenarios(season, site_position, site_name) # run_all_efast_scenarios(season, site_position, site_name)
print(f"Creating prepared/fusion timeseries: {site_name}, {season}") print(f"Creating prepared/fusion timeseries: {site_name}, {season}")
create_prepared_fusion_timeseries(season, site_position, site_name) create_prepared_fusion_timeseries(season, site_position, site_name)
print(f"Post-processing: {site_name}, {season}") print(f"Post-processing: {site_name}, {season}")
post_process_all_scenarios(season, site_position, site_name) # post_process_all_scenarios(season, site_position, site_name)
post_process_timeseries(season, site_position, site_name) post_process_timeseries(season, site_position, site_name)
print(f"Calculating metrics: {site_name}, {season}") print(f"Calculating metrics: {site_name}, {season}")
calculate_all_metrics(season, site_name, site_position) # calculate_all_metrics(season, site_name, site_position)
except Exception as e: except Exception as e:
print(f"Error: {e}") print(f"Error: {e}")

View file

@ -12,12 +12,16 @@
.nav a { margin-right: 12px; color: #0066cc; text-decoration: none; } .nav a { margin-right: 12px; color: #0066cc; text-decoration: none; }
.nav a:hover { text-decoration: underline; } .nav a:hover { text-decoration: underline; }
.nav a.active { font-weight: bold; } .nav a.active { font-weight: bold; }
.container { max-width: 900px; margin: 0 auto; padding: 20px; } .container { max-width: 1400px; margin: 0 auto; padding: 20px; }
.header-sticky { position: sticky; top: 0; background: white; z-index: 1000; border-bottom: 1px solid #ccc; padding-bottom: 20px; margin-bottom: 20px; }
.selectors { margin-bottom: 20px; } .selectors { margin-bottom: 20px; }
.selectors select { padding: 5px 10px; font-size: 14px; margin-right: 15px; } .selectors select { padding: 5px 10px; font-size: 14px; margin-right: 15px; }
h1 { margin: 0 0 5px 0; font-size: 22px; } h1 { margin: 0 0 5px 0; font-size: 22px; }
.season-row { padding-bottom: 15px; } .season-row { padding-bottom: 15px; }
h2 { margin: 0; font-size: 16px; color: #666; display: inline; } h2 { margin: 0; font-size: 16px; color: #666; display: inline; }
.download-links { margin-left: 10px; font-size: 14px; }
.download-links a { margin-right: 8px; color: #0066cc; text-decoration: none; }
.download-links a:hover { text-decoration: underline; }
#dateSlider { width: 100%; margin: 15px 0; } #dateSlider { width: 100%; margin: 15px 0; }
#dateDisplay { text-align: center; font-size: 14px; color: #666; } #dateDisplay { text-align: center; font-size: 14px; color: #666; }
.map-label { font-size: 12px; margin-bottom: 3px; color: #666; } .map-label { font-size: 12px; margin-bottom: 3px; color: #666; }
@ -31,6 +35,7 @@
</head> </head>
<body> <body>
<div class="container"> <div class="container">
<div class="header-sticky">
<div class="nav"> <div class="nav">
<a href="index.html">Full</a> <a href="index.html">Full</a>
<a href="preselection.html">Pre-selection</a> <a href="preselection.html">Pre-selection</a>
@ -39,7 +44,7 @@
<a href="postprocessed.html">Postprocessed</a> <a href="postprocessed.html">Postprocessed</a>
</div> </div>
<h1 id="siteName">Innsbruck</h1> <h1 id="siteName">Innsbruck</h1>
<div class="season-row"><h2 id="season">2024</h2></div> <div class="season-row"><h2 id="season">2024</h2><span class="download-links" id="downloadLinks"></span></div>
<div class="selectors"> <div class="selectors">
<label>Site:</label> <label>Site:</label>
<select id="siteSelect"></select> <select id="siteSelect"></select>
@ -58,6 +63,7 @@
</div> </div>
<input type="range" id="dateSlider" min="0" max="365" value="0"> <input type="range" id="dateSlider" min="0" max="365" value="0">
<div id="dateDisplay">2024-01-01</div> <div id="dateDisplay">2024-01-01</div>
</div>
<div class="map-label">Fusion RGB (closest available)</div> <div class="map-label">Fusion RGB (closest available)</div>
<div id="mapDate" class="map-date"></div> <div id="mapDate" class="map-date"></div>
<div id="fusionMap"></div> <div id="fusionMap"></div>
@ -114,6 +120,7 @@
ndviTs = n; gccTs = g; bandsTs = b; ndviTs = n; gccTs = g; bandsTs = b;
} catch { ndviTs = []; gccTs = []; bandsTs = []; } } catch { ndviTs = []; gccTs = []; bandsTs = []; }
drawPlots(); drawPlots();
updateDownloadLinks();
} }
function drawPlot(canvasId, data, key, color) { function drawPlot(canvasId, data, key, color) {
@ -159,6 +166,15 @@
BANDS.forEach(b => drawPlot(`plot_${b.key}`, bandsTs, b.key, b.color)); BANDS.forEach(b => drawPlot(`plot_${b.key}`, bandsTs, b.key, b.color));
} }
function updateDownloadLinks() {
const el = document.getElementById("downloadLinks");
if (!el) return;
const sub = getFusionTimeseriesDir();
const base = `data/${siteName}/${season}/prepared_${strategy}/export/${sub}`;
const name = `${siteName}_${season}_fusion_${strategy}_${sub}`;
el.innerHTML = `<a href="${base}/timeseries.json" download="${name}.json">[JSON]</a><a href="${base}/timeseries.csv" download="${name}.csv">[CSV]</a>`;
}
async function findFusionFile(dateStr) { async function findFusionFile(dateStr) {
const target = new Date(dateStr); const target = new Date(dateStr);
const yearEnd = new Date(parseInt(season), 11, 31); const yearEnd = new Date(parseInt(season), 11, 31);

View file

@ -12,12 +12,16 @@
.nav a { margin-right: 12px; color: #0066cc; text-decoration: none; } .nav a { margin-right: 12px; color: #0066cc; text-decoration: none; }
.nav a:hover { text-decoration: underline; } .nav a:hover { text-decoration: underline; }
.nav a.active { font-weight: bold; } .nav a.active { font-weight: bold; }
.container { max-width: 900px; margin: 0 auto; padding: 20px; } .container { max-width: 1400px; margin: 0 auto; padding: 20px; }
.header-sticky { position: sticky; top: 0; background: white; z-index: 1000; border-bottom: 1px solid #ccc; padding-bottom: 20px; margin-bottom: 20px; }
.selectors { margin-bottom: 20px; } .selectors { margin-bottom: 20px; }
.selectors select { padding: 5px 10px; font-size: 14px; margin-right: 15px; } .selectors select { padding: 5px 10px; font-size: 14px; margin-right: 15px; }
h1 { margin: 0 0 5px 0; font-size: 22px; } h1 { margin: 0 0 5px 0; font-size: 22px; }
.season-row { padding-bottom: 15px; } .season-row { padding-bottom: 15px; }
h2 { margin: 0; font-size: 16px; color: #666; display: inline; } h2 { margin: 0; font-size: 16px; color: #666; display: inline; }
.download-links { margin-left: 10px; font-size: 14px; }
.download-links a { margin-right: 8px; color: #0066cc; text-decoration: none; }
.download-links a:hover { text-decoration: underline; }
#dateSlider { width: 100%; margin: 15px 0; } #dateSlider { width: 100%; margin: 15px 0; }
#dateDisplay { text-align: center; font-size: 14px; color: #666; } #dateDisplay { text-align: center; font-size: 14px; color: #666; }
.map-label { font-size: 12px; margin-bottom: 3px; color: #666; } .map-label { font-size: 12px; margin-bottom: 3px; color: #666; }
@ -31,6 +35,7 @@
</head> </head>
<body> <body>
<div class="container"> <div class="container">
<div class="header-sticky">
<div class="nav"> <div class="nav">
<a href="index.html">Full</a> <a href="index.html">Full</a>
<a href="preselection.html">Pre-selection</a> <a href="preselection.html">Pre-selection</a>
@ -39,7 +44,7 @@
<a href="postprocessed.html" class="active">Postprocessed</a> <a href="postprocessed.html" class="active">Postprocessed</a>
</div> </div>
<h1 id="siteName">Innsbruck</h1> <h1 id="siteName">Innsbruck</h1>
<div class="season-row"><h2 id="season">2024</h2></div> <div class="season-row"><h2 id="season">2024</h2><span class="download-links" id="downloadLinks"></span></div>
<div class="selectors"> <div class="selectors">
<label>Site:</label> <label>Site:</label>
<select id="siteSelect"></select> <select id="siteSelect"></select>
@ -64,6 +69,7 @@
</div> </div>
<input type="range" id="dateSlider" min="0" max="365" value="0"> <input type="range" id="dateSlider" min="0" max="365" value="0">
<div id="dateDisplay">2024-01-01</div> <div id="dateDisplay">2024-01-01</div>
</div>
<div class="map-label">Postprocessed RGB (closest available)</div> <div class="map-label">Postprocessed RGB (closest available)</div>
<div id="mapDate" class="map-date"></div> <div id="mapDate" class="map-date"></div>
<div id="postprocessedMap"></div> <div id="postprocessedMap"></div>
@ -112,6 +118,7 @@
ndviTs = n; gccTs = g; bandsTs = b; ndviTs = n; gccTs = g; bandsTs = b;
} catch { ndviTs = []; gccTs = []; bandsTs = []; } } catch { ndviTs = []; gccTs = []; bandsTs = []; }
drawPlots(); drawPlots();
updateDownloadLinks();
} }
function drawPlot(canvasId, data, key, color) { function drawPlot(canvasId, data, key, color) {
@ -157,6 +164,14 @@
BANDS.forEach(b => drawPlot(`plot_${b.key}`, bandsTs, b.key, b.color)); BANDS.forEach(b => drawPlot(`plot_${b.key}`, bandsTs, b.key, b.color));
} }
function updateDownloadLinks() {
const el = document.getElementById("downloadLinks");
if (!el) return;
const base = `data/${siteName}/${season}/processed_${strategy}_sigma${sigma}/export/${source}`;
const name = `${siteName}_${season}_postprocessed_${strategy}_sigma${sigma}_${source}`;
el.innerHTML = `<a href="${base}/timeseries.json" download="${name}.json">[JSON]</a><a href="${base}/timeseries.csv" download="${name}.csv">[CSV]</a>`;
}
async function findProcessedFile(dateStr) { async function findProcessedFile(dateStr) {
const target = new Date(dateStr); const target = new Date(dateStr);
const yearEnd = new Date(parseInt(season), 11, 31); const yearEnd = new Date(parseInt(season), 11, 31);

View file

@ -12,12 +12,16 @@
.nav a { margin-right: 12px; color: #0066cc; text-decoration: none; } .nav a { margin-right: 12px; color: #0066cc; text-decoration: none; }
.nav a:hover { text-decoration: underline; } .nav a:hover { text-decoration: underline; }
.nav a.active { font-weight: bold; } .nav a.active { font-weight: bold; }
.container { max-width: 900px; margin: 0 auto; padding: 20px; } .container { max-width: 1400px; margin: 0 auto; padding: 20px; }
.header-sticky { position: sticky; top: 0; background: white; z-index: 1000; border-bottom: 1px solid #ccc; padding-bottom: 20px; margin-bottom: 20px; }
.selectors { margin-bottom: 20px; } .selectors { margin-bottom: 20px; }
.selectors select { padding: 5px 10px; font-size: 14px; margin-right: 15px; } .selectors select { padding: 5px 10px; font-size: 14px; margin-right: 15px; }
h1 { margin: 0 0 5px 0; font-size: 22px; } h1 { margin: 0 0 5px 0; font-size: 22px; }
.season-row { padding-bottom: 15px; } .season-row { padding-bottom: 15px; }
h2 { margin: 0; font-size: 16px; color: #666; display: inline; } h2 { margin: 0; font-size: 16px; color: #666; display: inline; }
.download-links { margin-left: 10px; font-size: 14px; }
.download-links a { margin-right: 8px; color: #0066cc; text-decoration: none; }
.download-links a:hover { text-decoration: underline; }
#dateSlider { width: 100%; margin: 15px 0; } #dateSlider { width: 100%; margin: 15px 0; }
#dateDisplay { text-align: center; font-size: 14px; color: #666; } #dateDisplay { text-align: center; font-size: 14px; color: #666; }
.map-label { font-size: 12px; margin-bottom: 3px; color: #666; } .map-label { font-size: 12px; margin-bottom: 3px; color: #666; }
@ -31,6 +35,7 @@
</head> </head>
<body> <body>
<div class="container"> <div class="container">
<div class="header-sticky">
<div class="nav"> <div class="nav">
<a href="index.html">Full</a> <a href="index.html">Full</a>
<a href="preselection.html">Pre-selection</a> <a href="preselection.html">Pre-selection</a>
@ -39,7 +44,7 @@
<a href="postprocessed.html">Postprocessed</a> <a href="postprocessed.html">Postprocessed</a>
</div> </div>
<h1 id="siteName">Innsbruck</h1> <h1 id="siteName">Innsbruck</h1>
<div class="season-row"><h2 id="season">2024</h2></div> <div class="season-row"><h2 id="season">2024</h2><span class="download-links" id="downloadLinks"></span></div>
<div class="selectors"> <div class="selectors">
<label>Site:</label> <label>Site:</label>
<select id="siteSelect"></select> <select id="siteSelect"></select>
@ -58,6 +63,7 @@
</div> </div>
<input type="range" id="dateSlider" min="0" max="365" value="0"> <input type="range" id="dateSlider" min="0" max="365" value="0">
<div id="dateDisplay">2024-01-01</div> <div id="dateDisplay">2024-01-01</div>
</div>
<div class="map-label" id="mapLabel">Prepared RGB (closest available)</div> <div class="map-label" id="mapLabel">Prepared RGB (closest available)</div>
<div id="mapDate" class="map-date"></div> <div id="mapDate" class="map-date"></div>
<div id="preparedMap"></div> <div id="preparedMap"></div>
@ -106,6 +112,7 @@
ndviTs = n; gccTs = g; bandsTs = b; ndviTs = n; gccTs = g; bandsTs = b;
} catch { ndviTs = []; gccTs = []; bandsTs = []; } } catch { ndviTs = []; gccTs = []; bandsTs = []; }
drawPlots(); drawPlots();
updateDownloadLinks();
} }
function drawPlot(canvasId, data, key, color) { function drawPlot(canvasId, data, key, color) {
@ -151,6 +158,14 @@
BANDS.forEach(b => drawPlot(`plot_${b.key}`, bandsTs, b.key, b.color)); BANDS.forEach(b => drawPlot(`plot_${b.key}`, bandsTs, b.key, b.color));
} }
function updateDownloadLinks() {
const el = document.getElementById("downloadLinks");
if (!el) return;
const base = `data/${siteName}/${season}/prepared_${strategy}/export/${source}`;
const name = `${siteName}_${season}_prepared_${strategy}_${source}`;
el.innerHTML = `<a href="${base}/timeseries.json" download="${name}.json">[JSON]</a><a href="${base}/timeseries.csv" download="${name}.csv">[CSV]</a>`;
}
async function findPreparedFile(dateStr) { async function findPreparedFile(dateStr) {
const target = new Date(dateStr); const target = new Date(dateStr);
const yearEnd = new Date(parseInt(season), 11, 31); const yearEnd = new Date(parseInt(season), 11, 31);