explore: shard tracks into per-year files for progressive loading

bake_tracks now writes tracks_YYYY.json shards + tracks_index.json manifest
instead of a single monolithic tracks.json. API /api/me/tracks returns the
manifest; /api/me/tracks/{year} serves individual shards. Explore.svelte
fetches the two most recent years eagerly then streams the rest in the
background so the map renders immediately with recent data.
This commit is contained in:
Davide Scaini
2026-05-14 18:34:53 +02:00
parent 8af6b7b04e
commit 5167f2a988
3 changed files with 119 additions and 14 deletions
+47 -6
View File
@@ -1,7 +1,8 @@
"""Pre-bake per-handle GPS tracks for the Explore page.
Reads all activity GeoJSON files for a handle, applies RDP simplification,
and writes a single tracks.json for fast client-side heatmap rendering.
and writes per-year tracks_YYYY.json shards plus a tracks_index.json manifest
for progressive client-side loading.
"""
from __future__ import annotations
@@ -11,7 +12,7 @@ from pathlib import Path
from bincio.extract.simplify import _rdp_mask
_VERSION = 1
_VERSION = 2
_RDP_EPSILON = 0.0001 # ~10 m on the ground
@@ -88,14 +89,54 @@ def bake_tracks(handle: str, data_dir: Path) -> int:
tracks.sort(key=lambda t: t["date"], reverse=True)
out = data_dir / handle / "tracks.json"
out.write_text(
user_dir = data_dir / handle
now = int(time.time())
# Group into per-year buckets
by_year: dict[str, list] = {}
for t in tracks:
year = t["date"][:4] or "0000"
by_year.setdefault(year, []).append(t)
# Remove stale year shards that no longer have data
for old in user_dir.glob("tracks_*.json"):
stem = old.stem # e.g. "tracks_2024" or "tracks_index"
if stem == "tracks_index":
continue
year_part = stem[len("tracks_"):]
if year_part not in by_year:
old.unlink(missing_ok=True)
# Write per-year shards
for year, year_tracks in by_year.items():
shard_path = user_dir / f"tracks_{year}.json"
shard_path.write_text(
json.dumps({
"v": _VERSION,
"handle": handle,
"year": year,
"generated_at": now,
"tracks": year_tracks,
}),
encoding="utf-8",
)
# Write manifest
years_sorted = sorted(by_year.keys(), reverse=True)
index_path = user_dir / "tracks_index.json"
index_path.write_text(
json.dumps({
"v": _VERSION,
"handle": handle,
"generated_at": int(time.time()),
"tracks": tracks,
"generated_at": now,
"total": len(tracks),
"years": years_sorted,
}),
encoding="utf-8",
)
# Remove legacy monolithic file if present
legacy = user_dir / "tracks.json"
legacy.unlink(missing_ok=True)
return len(tracks)