explore: shard tracks into per-year files for progressive loading

bake_tracks now writes tracks_YYYY.json shards + tracks_index.json manifest
instead of a single monolithic tracks.json. API /api/me/tracks returns the
manifest; /api/me/tracks/{year} serves individual shards. Explore.svelte
fetches the two most recent years eagerly then streams the rest in the
background so the map renders immediately with recent data.

bincio/explore.py

@@ -1,7 +1,8 @@
 """Pre-bake per-handle GPS tracks for the Explore page.
 
 Reads all activity GeoJSON files for a handle, applies RDP simplification,
-and writes a single tracks.json for fast client-side heatmap rendering.
+and writes per-year tracks_YYYY.json shards plus a tracks_index.json manifest
+for progressive client-side loading.
 """
 from __future__ import annotations
 
@@ -11,7 +12,7 @@ from pathlib import Path
 
 from bincio.extract.simplify import _rdp_mask
 
-_VERSION     = 1
+_VERSION     = 2
 _RDP_EPSILON = 0.0001   # ~10 m on the ground
 
 
@@ -88,14 +89,54 @@ def bake_tracks(handle: str, data_dir: Path) -> int:
 
     tracks.sort(key=lambda t: t["date"], reverse=True)
 
-    out = data_dir / handle / "tracks.json"
+    user_dir = data_dir / handle
-    out.write_text(
+    now = int(time.time())
+
+    # Group into per-year buckets
+    by_year: dict[str, list] = {}
+    for t in tracks:
+        year = t["date"][:4] or "0000"
+        by_year.setdefault(year, []).append(t)
+
+    # Remove stale year shards that no longer have data
+    for old in user_dir.glob("tracks_*.json"):
+        stem = old.stem  # e.g. "tracks_2024" or "tracks_index"
+        if stem == "tracks_index":
+            continue
+        year_part = stem[len("tracks_"):]
+        if year_part not in by_year:
+            old.unlink(missing_ok=True)
+
+    # Write per-year shards
+    for year, year_tracks in by_year.items():
+        shard_path = user_dir / f"tracks_{year}.json"
+        shard_path.write_text(
             json.dumps({
                 "v":            _VERSION,
                 "handle":       handle,
-            "generated_at": int(time.time()),
+                "year":         year,
-            "tracks":       tracks,
+                "generated_at": now,
+                "tracks":       year_tracks,
             }),
             encoding="utf-8",
         )
+
+    # Write manifest
+    years_sorted = sorted(by_year.keys(), reverse=True)
+    index_path = user_dir / "tracks_index.json"
+    index_path.write_text(
+        json.dumps({
+            "v":            _VERSION,
+            "handle":       handle,
+            "generated_at": now,
+            "total":        len(tracks),
+            "years":        years_sorted,
+        }),
+        encoding="utf-8",
+    )
+
+    # Remove legacy monolithic file if present
+    legacy = user_dir / "tracks.json"
+    legacy.unlink(missing_ok=True)
+
     return len(tracks)

bincio/serve/routers/me.py

@@ -43,23 +43,39 @@ def _wipe_user_activities(user_dir: Path) -> int:
         if d.exists():
             shutil.rmtree(d)
 
-    for name in ("index.json", "athlete.json", ".bincio_cache.json", "tracks.json"):
+    for name in ("index.json", "athlete.json", ".bincio_cache.json", "tracks.json", "tracks_index.json"):
         f = user_dir / name
         if f.exists():
             f.unlink()
             deleted += 1
 
+    for shard in user_dir.glob("tracks_*.json"):
+        shard.unlink(missing_ok=True)
+        deleted += 1
+
     return deleted
 
 
 @router.get("/api/me/tracks")
 async def me_tracks(bincio_session: str | None = Cookie(default=None)) -> Response:
-    """Return the pre-baked tracks.json for the logged-in user (Explore page)."""
+    """Return the tracks manifest (years list + total) for the logged-in user."""
     user = deps._require_user(bincio_session)
-    tracks_path = deps._get_data_dir() / user.handle / "tracks.json"
+    index_path = deps._get_data_dir() / user.handle / "tracks_index.json"
-    if not tracks_path.exists():
+    if not index_path.exists():
         raise HTTPException(404, "Tracks not yet baked — upload an activity first")
-    return Response(content=tracks_path.read_bytes(), media_type="application/json")
+    return Response(content=index_path.read_bytes(), media_type="application/json")
+
+
+@router.get("/api/me/tracks/{year}")
+async def me_tracks_year(year: str, bincio_session: str | None = Cookie(default=None)) -> Response:
+    """Return the pre-baked tracks shard for a specific year."""
+    user = deps._require_user(bincio_session)
+    if not year.isdigit() or len(year) != 4:
+        raise HTTPException(400, "year must be a 4-digit string")
+    shard_path = deps._get_data_dir() / user.handle / f"tracks_{year}.json"
+    if not shard_path.exists():
+        raise HTTPException(404, f"No tracks shard for year {year}")
+    return Response(content=shard_path.read_bytes(), media_type="application/json")
 
 
 @router.get("/api/me/storage")

site/src/components/Explore.svelte

@@ -15,6 +15,9 @@
 
   let tracks: Track[] = [];
   let loading = true;
+  let loadingYears = false;   // background years still fetching
+  let loadedCount = 0;        // years fetched so far
+  let totalYears = 0;         // total years in manifest
   let error = '';
 
   // Filters
@@ -69,8 +72,8 @@
   $: allTypes = [...new Set(tracks.map(t => t.type))].sort();
   $: availableYears = [...new Set(tracks.map(t => t.date.slice(0,4)).filter(Boolean))].sort().reverse();
 
-  let typesInitialized = false;
+  // Auto-select any type that hasn't been seen before (handles progressive loading)
-  $: if (allTypes.length > 0 && !typesInitialized) { selectedTypes = new Set(allTypes); typesInitialized = true; }
+  $: { const s = new Set(selectedTypes); let changed = false; for (const t of allTypes) { if (!s.has(t)) { s.add(t); changed = true; } } if (changed) selectedTypes = s; }
 
   $: filteredTracks = tracks.filter(t => {
     if (!selectedTypes.has(t.type)) return false;
@@ -199,15 +202,59 @@
     map.fitBounds([[w,s],[e,n]], { padding: 40, maxZoom: 14 });
   }
 
+  async function _loadYears(years: string[]) {
+    for (const year of years) {
+      try {
+        const r = await fetch(`/api/me/tracks/${year}`, { credentials: 'include' });
+        if (!r.ok) continue;
+        const d = await r.json();
+        const newTracks: Track[] = d.tracks ?? [];
+        if (newTracks.length > 0) {
+          tracks = [...tracks, ...newTracks];
+          // reactive $: statement picks up the tracks change and updates the map
+        }
+      } catch { /* non-fatal — skip year */ }
+      loadedCount += 1;
+    }
+    loadingYears = false;
+  }
+
   onMount(async () => {
+    // Fetch manifest to discover available years
+    let years: string[] = [];
     try {
       const r = await fetch('/api/me/tracks', { credentials: 'include' });
       if (!r.ok) { error = r.status === 404 ? 'No tracks baked yet — upload activities first.' : `Error ${r.status}`; loading = false; return; }
       const d = await r.json();
-      tracks = d.tracks ?? [];
+      years = d.years ?? [];
     } catch (e: any) { error = e.message ?? 'Failed to load'; loading = false; return; }
+
+    if (years.length === 0) { loading = false; return; }
+
+    totalYears = years.length;
+
+    // Load the two most recent years before showing the map
+    const eager = years.slice(0, 2);
+    const lazy  = years.slice(2);
+
+    for (const year of eager) {
+      try {
+        const r = await fetch(`/api/me/tracks/${year}`, { credentials: 'include' });
+        if (r.ok) {
+          const d = await r.json();
+          tracks = [...tracks, ...(d.tracks ?? [])];
+        }
+      } catch { /* skip */ }
+      loadedCount += 1;
+    }
     loading = false;
 
+    // Stream remaining years in the background
+    if (lazy.length > 0) {
+      loadingYears = true;
+      _loadYears(lazy);
+    }
+
     map = new maplibregl.Map({
       container: mapEl,
       style: { version: 8,
@@ -364,6 +411,7 @@
     </section>
 
     {#if loading}<p class="status">Loading tracks…</p>{/if}
+    {#if !loading && loadingYears}<p class="status">Loading history… {loadedCount}/{totalYears}</p>{/if}
     {#if error}<p class="status error">{error}</p>{/if}
   </aside>