Added phenocam download.
This commit is contained in:
Felix Delattre
parent
bf92a399e2
commit
68d894c59f
3 changed files with 185 additions and 5 deletions
146
download_phenocam.py
Normal file
146
download_phenocam.py
Normal file
|
|
@ -0,0 +1,146 @@
|
|||
import requests
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
|
||||
PHENOCAM_API = "https://phenocam.nau.edu/api"
|
||||
|
||||
|
||||
def _find_start_offset(site_name, start_dt, total_count):
|
||||
"""Binary search to find approximate offset for start date."""
|
||||
low, high = 0, total_count - 1
|
||||
limit = 1
|
||||
|
||||
for _ in range(15):
|
||||
mid = (low + high) // 2
|
||||
response = requests.get(
|
||||
f"{PHENOCAM_API}/middayimages/",
|
||||
params={"site": site_name, "limit": limit, "offset": mid},
|
||||
timeout=30
|
||||
)
|
||||
response.raise_for_status()
|
||||
results = response.json().get("results", [])
|
||||
if not results:
|
||||
break
|
||||
|
||||
mid_date_str = results[0].get("imgdate", "")
|
||||
if not mid_date_str:
|
||||
break
|
||||
|
||||
try:
|
||||
mid_date = datetime.strptime(mid_date_str, "%Y-%m-%d")
|
||||
if mid_date < start_dt:
|
||||
low = mid + 1
|
||||
else:
|
||||
high = mid
|
||||
except ValueError:
|
||||
break
|
||||
|
||||
return max(0, low - 100)
|
||||
|
||||
|
||||
def download_phenocam(season, site_position, site_name, date_range=None):
|
||||
lat, lon = site_position
|
||||
datetime_range = date_range or f"{season}-01-01/{season}-12-31"
|
||||
output_dir = Path(f"data/{site_name}/{season}/raw/phenocam/")
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
print(f"[PhenoCam] Starting download: {site_name} ({lat:.6f}, {lon:.6f}), {season}")
|
||||
|
||||
start_date, end_date = datetime_range.split("/")
|
||||
start_dt = datetime.strptime(start_date, "%Y-%m-%d")
|
||||
end_dt = datetime.strptime(end_date, "%Y-%m-%d")
|
||||
|
||||
try:
|
||||
response = requests.get(
|
||||
f"{PHENOCAM_API}/middayimages/",
|
||||
params={"site": site_name, "limit": 1},
|
||||
timeout=30
|
||||
)
|
||||
response.raise_for_status()
|
||||
total_count = response.json().get("count", 0)
|
||||
|
||||
if total_count == 0:
|
||||
print(f"[PhenoCam] No images found for site '{site_name}'")
|
||||
return
|
||||
|
||||
print(f"[PhenoCam] Found {total_count} total images, estimating start offset...")
|
||||
start_offset = _find_start_offset(site_name, start_dt, total_count)
|
||||
|
||||
url = f"{PHENOCAM_API}/middayimages/"
|
||||
params = {"site": site_name, "offset": start_offset}
|
||||
|
||||
print(f"[PhenoCam] Fetching image list from offset {start_offset}...")
|
||||
images = []
|
||||
page = 1
|
||||
max_pages = 500
|
||||
past_end_date = False
|
||||
|
||||
while url and page <= max_pages and not past_end_date:
|
||||
response = requests.get(url, params=params, timeout=30)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
results = data.get("results", [])
|
||||
|
||||
if not results:
|
||||
break
|
||||
|
||||
for img in results:
|
||||
img_date_str = img.get("imgdate", "")
|
||||
if not img_date_str:
|
||||
continue
|
||||
try:
|
||||
img_date = datetime.strptime(img_date_str, "%Y-%m-%d")
|
||||
if img_date > end_dt:
|
||||
past_end_date = True
|
||||
break
|
||||
if start_dt <= img_date <= end_dt:
|
||||
images.append(img)
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
if url and not past_end_date:
|
||||
url = data.get("next")
|
||||
params = None
|
||||
page += 1
|
||||
if page % 50 == 0:
|
||||
print(f"[PhenoCam] Processed {page} pages, found {len(images)} images in range...")
|
||||
except requests.exceptions.HTTPError as e:
|
||||
if e.response.status_code == 404:
|
||||
print(f"[PhenoCam] Site '{site_name}' not found")
|
||||
return
|
||||
raise
|
||||
|
||||
print(f"[PhenoCam] Found {len(images)} images")
|
||||
|
||||
def _download_image(img):
|
||||
date_str = img.get("imgdate", "").replace("-", "")
|
||||
if not date_str:
|
||||
return None
|
||||
|
||||
filepath = output_dir / f"{date_str}.jpg"
|
||||
if filepath.exists():
|
||||
return f"Skipped {date_str}.jpg (exists)"
|
||||
|
||||
img_path = img.get("imgpath")
|
||||
if not img_path:
|
||||
return None
|
||||
|
||||
img_url = f"https://phenocam.nau.edu{img_path}"
|
||||
try:
|
||||
img_response = requests.get(img_url, timeout=30)
|
||||
img_response.raise_for_status()
|
||||
filepath.write_bytes(img_response.content)
|
||||
return f"Saved {date_str}.jpg"
|
||||
except Exception as e:
|
||||
return f"Error downloading {date_str}: {e}"
|
||||
|
||||
with ThreadPoolExecutor(max_workers=5) as executor:
|
||||
futures = [executor.submit(_download_image, img) for img in images]
|
||||
for future in as_completed(futures):
|
||||
result = future.result()
|
||||
if result:
|
||||
print(f"[PhenoCam] {result}")
|
||||
|
||||
print("[PhenoCam] Completed")
|
||||
|
||||
12
run.py
12
run.py
|
|
@ -8,6 +8,7 @@ from ndvi import (
|
|||
)
|
||||
from download_s2 import download_s2
|
||||
from download_s3 import download_s3
|
||||
from download_phenocam import download_phenocam
|
||||
from clouds import detect_clouds
|
||||
|
||||
|
||||
|
|
@ -16,6 +17,7 @@ def run_pipeline(season, site_position, site_name):
|
|||
# print(f"Downloading data for {site_name}, {season}")
|
||||
# download_s2(season, site_position, site_name)
|
||||
# download_s3(season, site_position, site_name)
|
||||
download_phenocam(season, site_position, site_name)
|
||||
|
||||
# print(f"Generating NDVI for raw data: {site_name}, {season}")
|
||||
# generate_ndvi_raw(season, site_position, site_name)
|
||||
|
|
@ -31,11 +33,11 @@ def run_pipeline(season, site_position, site_name):
|
|||
# print(f"Running EFAST fusion for {site_name}, {season}")
|
||||
# run_efast(season, site_position, site_name)
|
||||
|
||||
print(f"Post-processing data: {site_name}, {season}")
|
||||
process_cropped(season, site_position, site_name)
|
||||
print(f"Generating NDVI for final outputs: {site_name}, {season}")
|
||||
generate_ndvi_post_process(season, site_position, site_name)
|
||||
create_ndvi_timeseries_post_process(season, site_position, site_name)
|
||||
# print(f"Post-processing data: {site_name}, {season}")
|
||||
# process_cropped(season, site_position, site_name)
|
||||
# print(f"Generating NDVI for final outputs: {site_name}, {season}")
|
||||
# generate_ndvi_post_process(season, site_position, site_name)
|
||||
# create_ndvi_timeseries_post_process(season, site_position, site_name)
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error: {e}")
|
||||
|
|
|
|||
|
|
@ -20,6 +20,10 @@
|
|||
.map-label { font-size: 12px; margin-bottom: 5px; color: #666; }
|
||||
.map-date { font-size: 11px; margin-top: 5px; color: #999; }
|
||||
.map { height: 500px; border: 1px solid #ccc; }
|
||||
.phenocam-container { margin-top: 20px; width: 100%; }
|
||||
.phenocam-label { font-size: 12px; margin-bottom: 5px; color: #666; }
|
||||
.phenocam-date { font-size: 11px; margin-top: 5px; color: #999; }
|
||||
.phenocam-image { width: 100%; max-height: 400px; object-fit: contain; border: 1px solid #ccc; }
|
||||
.leaflet-image-layer { image-rendering: pixelated; }
|
||||
.leaflet-control-attribution { display: none; }
|
||||
</style>
|
||||
|
|
@ -65,6 +69,11 @@
|
|||
<div id="s3ndvimap" class="map"></div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="phenocam-container">
|
||||
<div class="phenocam-label">PhenoCam Imagery</div>
|
||||
<div id="phenocamdate" class="phenocam-date"></div>
|
||||
<img id="phenocamimage" class="phenocam-image" alt="PhenoCam">
|
||||
</div>
|
||||
</div>
|
||||
<script>
|
||||
proj4.defs("EPSG:32632", "+proj=utm +zone=32 +datum=WGS84 +units=m +no_defs");
|
||||
|
|
@ -335,6 +344,28 @@
|
|||
return null;
|
||||
}
|
||||
|
||||
async function loadPhenoCam(dateStr) {
|
||||
const target = new Date(dateStr);
|
||||
for (let offset = 0; offset < 15; offset++) {
|
||||
for (const dir of [0, -1, 1]) {
|
||||
const d = new Date(target);
|
||||
d.setDate(d.getDate() + offset * dir);
|
||||
const date = d.toISOString().split("T")[0].replace(/-/g, "");
|
||||
const url = `../data/innsbruck/2024/raw/phenocam/${date}.jpg`;
|
||||
try {
|
||||
const res = await fetch(url);
|
||||
if (res.ok) {
|
||||
document.getElementById("phenocamimage").src = url;
|
||||
document.getElementById("phenocamdate").textContent = `${date.slice(0,4)}-${date.slice(4,6)}-${date.slice(6,8)}`;
|
||||
return;
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
document.getElementById("phenocamimage").src = "";
|
||||
document.getElementById("phenocamdate").textContent = "";
|
||||
}
|
||||
|
||||
async function updateImages() {
|
||||
const date = dateFromDays(parseInt(slider.value));
|
||||
dateDisplay.textContent = date;
|
||||
|
|
@ -360,6 +391,7 @@
|
|||
}
|
||||
}
|
||||
}
|
||||
await loadPhenoCam(date);
|
||||
}
|
||||
|
||||
const urlParams = new URLSearchParams(location.search);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue