diff --git a/bincio/render/merge.py b/bincio/render/merge.py index 276de88..270b445 100644 --- a/bincio/render/merge.py +++ b/bincio/render/merge.py @@ -412,10 +412,11 @@ _COMBINED_FEED_STRIP = _FEED_STRIP | {"mmp"} def write_combined_feed(data_dir: Path) -> int: - """Build data_dir/feed.json — the N most recent activities across all users. + """Build data_dir/feed.json and per-month data_dir/feed-YYYY-MM.json shards. - The global feed page loads this single file instead of resolving 20+ user - shards recursively. Returns the number of activities written. + feed.json is a BAS shard index (same format as per-user index.json). + Each feed-YYYY-MM.json contains all activities for that month across all users, + sorted newest-first. Returns the number of activities written. """ user_dirs = sorted( p for p in data_dir.iterdir() @@ -458,24 +459,35 @@ def write_combined_feed(data_dir: Path) -> int: all_activities.sort(key=lambda a: a.get("started_at", ""), reverse=True) - # Remove stale feed pages + # Remove stale feed files (sequential pages and old year shards) for f in data_dir.glob("feed*.json"): f.unlink(missing_ok=True) if not all_activities: return 0 - pages = [all_activities[i:i + FEED_PAGE_SIZE] for i in range(0, len(all_activities), FEED_PAGE_SIZE)] - for page_num, page in enumerate(pages): - slim = [{k: v for k, v in a.items() if k not in _COMBINED_FEED_STRIP} for a in page] - fname = "feed.json" if page_num == 0 else f"feed-{page_num + 1}.json" - doc = { - "bas_version": "1.0", - "page": page_num + 1, - "total_pages": len(pages), - "total_activities": len(all_activities), - "activities": slim, - } - (data_dir / fname).write_text(_dumps(doc)) + # Group by YYYY-MM (month), preserving newest-first order within each bucket + by_month: dict[str, list[dict]] = {} + for a in all_activities: + ym = (a.get("started_at") or "")[:7] # "YYYY-MM" + if len(ym) == 7 and ym[4] == "-": + by_month.setdefault(ym, []).append(a) + + months_desc = sorted(by_month.keys(), reverse=True) + + # Write per-month shard files (~150-200 acts each → ~25 KB gzip) + for ym, acts in by_month.items(): + slim = [{k: v for k, v in a.items() if k not in _COMBINED_FEED_STRIP} for a in acts] + doc: dict = {"bas_version": "1.0", "activities": slim} + (data_dir / f"feed-{ym}.json").write_text(_dumps(doc)) + + # Write feed.json as a BAS shard index (same pattern as per-user index.json) + index_doc: dict = { + "bas_version": "1.0", + "total_activities": len(all_activities), + "shards": [{"url": f"feed-{ym}.json"} for ym in months_desc], + "activities": [], + } + (data_dir / "feed.json").write_text(_dumps(index_doc)) return len(all_activities) diff --git a/site/src/components/ActivityFeed.svelte b/site/src/components/ActivityFeed.svelte index 5ccda62..619cd51 100644 --- a/site/src/components/ActivityFeed.svelte +++ b/site/src/components/ActivityFeed.svelte @@ -2,7 +2,7 @@ import { onMount } from 'svelte'; import type { ActivitySummary, BASIndex, Sport } from '../lib/types'; import { formatDistance, formatDuration, formatElevation, formatDate, isUnlisted, sportIcon, sportColor, sportLabel } from '../lib/format'; - import { loadIndexPaged, loadShardActivities, loadCombinedFeed, loadCombinedFeedPage } from '../lib/dataloader'; + import { loadIndexPaged, loadShardActivities, loadCombinedFeed } from '../lib/dataloader'; /** Render preview_coords as an SVG polyline path string. */ function trackPath(coords: [number, number][] | null, w: number, h: number): string { @@ -52,9 +52,6 @@ let error = ''; let mounted = false; let pendingShards: string[] = []; - /** Remaining combined-feed pages (multi-user global feed). */ - let feedNextPage = 0; - let feedTotalPages = 0; /** Grand total from feed.json — shows instance-wide count even before all pages are loaded. */ let totalActivities = 0; /** Logged-in handle — resolved async via bincio:me event. */ @@ -97,26 +94,19 @@ : filtered; $: visible = withSearch.slice(0, shown); $: canShowMore = shown < withSearch.length; - $: hasMore = canShowMore || pendingShards.length > 0 || feedNextPage > 0; + $: hasMore = canShowMore || pendingShards.length > 0; async function loadMore() { if (canShowMore) { shown += PAGE_SIZE; return; } + if (!pendingShards.length) return; loadingMore = true; try { - let fresh: ActivitySummary[] = []; - if (feedNextPage > 0) { - fresh = await loadCombinedFeedPage(base, feedNextPage); - feedNextPage = feedNextPage < feedTotalPages ? feedNextPage + 1 : 0; - } else if (pendingShards.length) { - const url = pendingShards[0]; - pendingShards = pendingShards.slice(1); - fresh = await loadShardActivities(url); - } else { - return; - } + const url = pendingShards[0]; + pendingShards = pendingShards.slice(1); + const fresh = await loadShardActivities(url); const existing = new Map(all.map(a => [a.id, a])); for (const a of fresh) if (!existing.has(a.id)) existing.set(a.id, a); all = [...existing.values()].sort((a, b) => @@ -132,12 +122,11 @@ $: if (sport || datePre || query || customFrom || customTo) shown = PAGE_SIZE; // reset pagination on filter change - // Eager-load shards / feed-pages when a filter needs data not yet in memory. - let loadingAllShards = false; - let loadingAllFeedPages = false; + // Eager-load shards when a filter needs data not yet in memory. + let loadingAllShards = false; function _yearFromShard(url: string): number | null { - const m = url.match(/index-(\d{4})\.json$/); + const m = url.match(/(?:index|feed)-(\d{4})(?:-\d{2})?\.json$/); return m ? parseInt(m[1], 10) : null; } @@ -157,57 +146,28 @@ const yr = _neededYearRange(datePre, customFrom, customTo); const needEager = !!query.trim() || yr !== null; if (needEager && pendingShards.length > 0 && !loadingAllShards) { - loadingAllShards = true; - // When year-specific filter (no search), load only shards that cover - // the needed range; unneeded shards stay in pendingShards for "Load more". - // When search is active, load everything so full-text search works. + // Compute toLoad first — if empty (needed years already loaded, others remain), + // skip the async entirely to avoid an infinite reactive loop. const toLoad = (yr && !query.trim()) ? pendingShards.filter(url => { const y = _yearFromShard(url); return y !== null && y >= yr[0] && y <= yr[1]; }) : [...pendingShards]; - (async () => { - for (const url of toLoad) { - pendingShards = pendingShards.filter(u => u !== url); - try { - const fresh = await loadShardActivities(url); - const existing = new Map(all.map(a => [a.id, a])); - for (const a of fresh) if (!existing.has(a.id)) existing.set(a.id, a); - all = [...existing.values()].sort((a, b) => - (b.started_at ?? '').localeCompare(a.started_at ?? ''), - ); - } catch { /* ignore — partial results still useful */ } - } - loadingAllShards = false; - })(); - } - } - - $: { - const yr = _neededYearRange(datePre, customFrom, customTo); - if ((!!query.trim() || yr !== null) && feedNextPage > 0 && !loadingAllFeedPages) { - loadingAllFeedPages = true; - // Capture at loop start — dateFrom is reactive and may change mid-fetch. - const effectiveFrom = dateFrom; - (async () => { - while (feedNextPage > 0) { - const page = feedNextPage; - feedNextPage = page < feedTotalPages ? page + 1 : 0; - try { - const fresh = await loadCombinedFeedPage(base, page); - const existing = new Map(all.map(a => [a.id, a])); - for (const a of fresh) if (!existing.has(a.id)) existing.set(a.id, a); - all = [...existing.values()].sort((a, b) => - (b.started_at ?? '').localeCompare(a.started_at ?? ''), - ); - // Feed is sorted newest-first. Once the oldest activity in this page - // predates our from-filter, everything needed is already loaded. - if (effectiveFrom && fresh.length > 0) { - const oldest = fresh.reduce((m, a) => (a.started_at ?? '') < (m.started_at ?? '') ? a : m); - if ((oldest.started_at ?? '') < effectiveFrom) { feedNextPage = 0; break; } - } - } catch { /* ignore — partial results still useful */ } - } - loadingAllFeedPages = false; - })(); + if (toLoad.length > 0) { + loadingAllShards = true; + (async () => { + for (const url of toLoad) { + pendingShards = pendingShards.filter(u => u !== url); + try { + const fresh = await loadShardActivities(url); + const existing = new Map(all.map(a => [a.id, a])); + for (const a of fresh) if (!existing.has(a.id)) existing.set(a.id, a); + all = [...existing.values()].sort((a, b) => + (b.started_at ?? '').localeCompare(a.started_at ?? ''), + ); + } catch { /* ignore — partial results still useful */ } + } + loadingAllShards = false; + })(); + } } } @@ -245,8 +205,7 @@ if (combined) { all = combined.activities; totalActivities = combined.totalActivities; - feedTotalPages = combined.remainingPages + 1; - feedNextPage = combined.remainingPages > 0 ? 2 : 0; + pendingShards = combined.pendingShards; loading = false; return; } @@ -367,7 +326,7 @@

Could not load activities: {error}

{:else if withSearch.length === 0}

- {#if loadingAllFeedPages || loadingAllShards}Loading…{:else if query.trim()}No activities match "{query.trim()}".{:else}No activities found.{/if} + {#if loadingAllShards}Loading…{:else if query.trim()}No activities match "{query.trim()}".{:else}No activities found.{/if}

{:else}
@@ -462,10 +421,8 @@ Loading… {:else if canShowMore} Load more ({filtered.length - shown} remaining) - {:else if feedNextPage > 0} - Load more activities {:else} - Load older activities ({pendingShards.length} more {pendingShards.length === 1 ? 'year' : 'years'}) + Load older activities ({pendingShards.length} more {pendingShards.length === 1 ? 'period' : 'periods'}) {/if}
diff --git a/site/src/lib/dataloader.ts b/site/src/lib/dataloader.ts index c1da75e..87c9105 100644 --- a/site/src/lib/dataloader.ts +++ b/site/src/lib/dataloader.ts @@ -58,7 +58,7 @@ function emptyIndex(): BASIndex { // ── Helpers ─────────────────────────────────────────────────────────────────── function isYearShardUrl(url: string): boolean { - return /(?:^|\/)index-\d{4}\.json$/.test(url); + return /(?:^|\/)(?:index-\d{4}|feed-\d{4}-\d{2})\.json$/.test(url); } function rewriteActivityUrls(a: ActivitySummary, shardBase: string): ActivitySummary { @@ -253,50 +253,28 @@ export async function loadShardActivities(shardUrl: string): Promise { +): Promise<{ activities: ActivitySummary[]; pendingShards: string[]; totalActivities: number } | null> { try { - const feed = await fetchJSON(`${baseUrl}data/feed.json`); + const { index, pendingShards } = await loadIndexPaged(baseUrl, `${baseUrl}data/feed.json`); return { - activities: feed.activities ?? [], - remainingPages: (feed.total_pages ?? 1) - 1, - totalActivities: feed.total_activities ?? 0, + activities: index.activities, + pendingShards, + totalActivities: (index as any).total_activities ?? index.activities.length, }; } catch { return null; } } -/** - * Load a subsequent page of the combined feed (feed-2.json, feed-3.json, etc.). - */ -export async function loadCombinedFeedPage( - baseUrl: string, - page: number, -): Promise { - try { - const feed = await fetchJSON(`${baseUrl}data/feed-${page}.json`); - return feed.activities ?? []; - } catch { - return []; - } -} - /** * Load a single activity detail, checking IndexedDB first so locally-converted * activities are available offline.