perf: combined feed index for multi-user global feed
Instead of the browser resolving 20+ user shards recursively (~27 MB), generate a pre-sorted feed.json at merge time with 50 activities per page. The global feed loads one ~30 KB file on first paint; "Load more" fetches subsequent pages (feed-2.json, feed-3.json, etc.). Per-user profile pages still use year-sharded loadIndexPaged as before.
This commit is contained in:
@@ -133,6 +133,11 @@ def _write_root_manifest(data: Path) -> None:
|
|||||||
root.write_text(json.dumps(manifest, indent=2))
|
root.write_text(json.dumps(manifest, indent=2))
|
||||||
console.print(f"Root manifest updated: [cyan]{len(users)}[/cyan] user shard(s)")
|
console.print(f"Root manifest updated: [cyan]{len(users)}[/cyan] user shard(s)")
|
||||||
|
|
||||||
|
if len(users) > 1:
|
||||||
|
from bincio.render.merge import write_combined_feed
|
||||||
|
n = write_combined_feed(data)
|
||||||
|
console.print(f"Combined feed: [cyan]{n}[/cyan] activities across all users")
|
||||||
|
|
||||||
|
|
||||||
def _link_data(site: Path, data: Path) -> None:
|
def _link_data(site: Path, data: Path) -> None:
|
||||||
"""Symlink site/public/data → data root (each user has their own _merged/)."""
|
"""Symlink site/public/data → data root (each user has their own _merged/)."""
|
||||||
|
|||||||
@@ -320,3 +320,85 @@ def _write_year_shards(merged_dir: Path, activities: list[dict], index_meta: dic
|
|||||||
"activities": [],
|
"activities": [],
|
||||||
}
|
}
|
||||||
(merged_dir / "index.json").write_text(json.dumps(root_doc, indent=2, ensure_ascii=False))
|
(merged_dir / "index.json").write_text(json.dumps(root_doc, indent=2, ensure_ascii=False))
|
||||||
|
|
||||||
|
|
||||||
|
FEED_PAGE_SIZE = 50
|
||||||
|
|
||||||
|
# Extra fields stripped from the combined feed — preview_coords is the biggest
|
||||||
|
# contributor (~24% of shard size) but the feed cards need it for thumbnails,
|
||||||
|
# so we keep it. mmp is never displayed in feed cards.
|
||||||
|
_COMBINED_FEED_STRIP = _FEED_STRIP | {"mmp"}
|
||||||
|
|
||||||
|
|
||||||
|
def write_combined_feed(data_dir: Path) -> int:
|
||||||
|
"""Build data_dir/feed.json — the N most recent activities across all users.
|
||||||
|
|
||||||
|
The global feed page loads this single file instead of resolving 20+ user
|
||||||
|
shards recursively. Returns the number of activities written.
|
||||||
|
"""
|
||||||
|
user_dirs = sorted(
|
||||||
|
p for p in data_dir.iterdir()
|
||||||
|
if p.is_dir() and (p / "activities").exists()
|
||||||
|
)
|
||||||
|
|
||||||
|
all_activities: list[dict] = []
|
||||||
|
for user_dir in user_dirs:
|
||||||
|
handle = user_dir.name
|
||||||
|
merged = user_dir / "_merged"
|
||||||
|
index_path = merged / "index.json" if merged.exists() else user_dir / "index.json"
|
||||||
|
if not index_path.exists():
|
||||||
|
continue
|
||||||
|
|
||||||
|
index = json.loads(index_path.read_text(encoding="utf-8"))
|
||||||
|
shards = index.get("shards", [])
|
||||||
|
activities = index.get("activities", [])
|
||||||
|
|
||||||
|
if shards:
|
||||||
|
year_shards = sorted(
|
||||||
|
[s for s in shards if re.match(r"index-\d{4}\.json$", s.get("url", ""))],
|
||||||
|
key=lambda s: s["url"],
|
||||||
|
reverse=True,
|
||||||
|
)
|
||||||
|
base = index_path.parent
|
||||||
|
for shard in year_shards[:2]:
|
||||||
|
shard_path = base / shard["url"]
|
||||||
|
if shard_path.exists():
|
||||||
|
shard_data = json.loads(shard_path.read_text(encoding="utf-8"))
|
||||||
|
for a in shard_data.get("activities", []):
|
||||||
|
a_tagged = {**a, "handle": handle}
|
||||||
|
detail_url = a_tagged.get("detail_url", "")
|
||||||
|
if detail_url and not detail_url.startswith("http") and not detail_url.startswith("/"):
|
||||||
|
merged_rel = f"{handle}/_merged/" if merged.exists() else f"{handle}/"
|
||||||
|
a_tagged["detail_url"] = merged_rel + detail_url
|
||||||
|
track_url = a_tagged.get("track_url", "")
|
||||||
|
if track_url and not track_url.startswith("http") and not track_url.startswith("/"):
|
||||||
|
merged_rel = f"{handle}/_merged/" if merged.exists() else f"{handle}/"
|
||||||
|
a_tagged["track_url"] = merged_rel + track_url
|
||||||
|
all_activities.append(a_tagged)
|
||||||
|
else:
|
||||||
|
for a in activities:
|
||||||
|
all_activities.append({**a, "handle": handle})
|
||||||
|
|
||||||
|
all_activities.sort(key=lambda a: a.get("started_at", ""), reverse=True)
|
||||||
|
|
||||||
|
# Remove stale feed pages
|
||||||
|
for f in data_dir.glob("feed*.json"):
|
||||||
|
f.unlink()
|
||||||
|
|
||||||
|
if not all_activities:
|
||||||
|
return 0
|
||||||
|
|
||||||
|
pages = [all_activities[i:i + FEED_PAGE_SIZE] for i in range(0, len(all_activities), FEED_PAGE_SIZE)]
|
||||||
|
for page_num, page in enumerate(pages):
|
||||||
|
slim = [{k: v for k, v in a.items() if k not in _COMBINED_FEED_STRIP} for a in page]
|
||||||
|
fname = "feed.json" if page_num == 0 else f"feed-{page_num + 1}.json"
|
||||||
|
doc = {
|
||||||
|
"bas_version": "1.0",
|
||||||
|
"page": page_num + 1,
|
||||||
|
"total_pages": len(pages),
|
||||||
|
"total_activities": len(all_activities),
|
||||||
|
"activities": slim,
|
||||||
|
}
|
||||||
|
(data_dir / fname).write_text(json.dumps(doc, indent=2, ensure_ascii=False))
|
||||||
|
|
||||||
|
return len(all_activities)
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
import { onMount } from 'svelte';
|
import { onMount } from 'svelte';
|
||||||
import type { ActivitySummary, BASIndex, Sport } from '../lib/types';
|
import type { ActivitySummary, BASIndex, Sport } from '../lib/types';
|
||||||
import { formatDistance, formatDuration, formatElevation, formatDate, isUnlisted, sportIcon, sportColor, sportLabel } from '../lib/format';
|
import { formatDistance, formatDuration, formatElevation, formatDate, isUnlisted, sportIcon, sportColor, sportLabel } from '../lib/format';
|
||||||
import { loadIndexPaged, loadShardActivities } from '../lib/dataloader';
|
import { loadIndexPaged, loadShardActivities, loadCombinedFeed, loadCombinedFeedPage } from '../lib/dataloader';
|
||||||
|
|
||||||
/** Render preview_coords as an SVG polyline path string. */
|
/** Render preview_coords as an SVG polyline path string. */
|
||||||
function trackPath(coords: [number, number][] | null, w: number, h: number): string {
|
function trackPath(coords: [number, number][] | null, w: number, h: number): string {
|
||||||
@@ -45,6 +45,9 @@
|
|||||||
let error = '';
|
let error = '';
|
||||||
let mounted = false;
|
let mounted = false;
|
||||||
let pendingShards: string[] = [];
|
let pendingShards: string[] = [];
|
||||||
|
/** Remaining combined-feed pages (multi-user global feed). */
|
||||||
|
let feedNextPage = 0;
|
||||||
|
let feedTotalPages = 0;
|
||||||
/** Logged-in handle — resolved async via bincio:me event. */
|
/** Logged-in handle — resolved async via bincio:me event. */
|
||||||
let me: string = '';
|
let me: string = '';
|
||||||
|
|
||||||
@@ -61,20 +64,26 @@
|
|||||||
$: filtered = sport === 'all' ? withPrivacy : withPrivacy.filter(a => a.sport === sport);
|
$: filtered = sport === 'all' ? withPrivacy : withPrivacy.filter(a => a.sport === sport);
|
||||||
$: visible = filtered.slice(0, shown);
|
$: visible = filtered.slice(0, shown);
|
||||||
$: canShowMore = shown < filtered.length;
|
$: canShowMore = shown < filtered.length;
|
||||||
$: hasMore = canShowMore || pendingShards.length > 0;
|
$: hasMore = canShowMore || pendingShards.length > 0 || feedNextPage > 0;
|
||||||
|
|
||||||
async function loadMore() {
|
async function loadMore() {
|
||||||
if (canShowMore) {
|
if (canShowMore) {
|
||||||
shown += PAGE_SIZE;
|
shown += PAGE_SIZE;
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (!pendingShards.length) return;
|
|
||||||
loadingMore = true;
|
loadingMore = true;
|
||||||
const url = pendingShards[0];
|
|
||||||
pendingShards = pendingShards.slice(1);
|
|
||||||
try {
|
try {
|
||||||
const fresh = await loadShardActivities(url);
|
let fresh: ActivitySummary[] = [];
|
||||||
// Merge avoiding duplicates (IDB activities may already be present)
|
if (feedNextPage > 0) {
|
||||||
|
fresh = await loadCombinedFeedPage(base, feedNextPage);
|
||||||
|
feedNextPage = feedNextPage < feedTotalPages ? feedNextPage + 1 : 0;
|
||||||
|
} else if (pendingShards.length) {
|
||||||
|
const url = pendingShards[0];
|
||||||
|
pendingShards = pendingShards.slice(1);
|
||||||
|
fresh = await loadShardActivities(url);
|
||||||
|
} else {
|
||||||
|
return;
|
||||||
|
}
|
||||||
const existing = new Map(all.map(a => [a.id, a]));
|
const existing = new Map(all.map(a => [a.id, a]));
|
||||||
for (const a of fresh) if (!existing.has(a.id)) existing.set(a.id, a);
|
for (const a of fresh) if (!existing.has(a.id)) existing.set(a.id, a);
|
||||||
all = [...existing.values()].sort((a, b) =>
|
all = [...existing.values()].sort((a, b) =>
|
||||||
@@ -82,7 +91,7 @@
|
|||||||
);
|
);
|
||||||
shown += PAGE_SIZE;
|
shown += PAGE_SIZE;
|
||||||
} catch {
|
} catch {
|
||||||
// shard load failed — don't block the user
|
// load failed — don't block the user
|
||||||
} finally {
|
} finally {
|
||||||
loadingMore = false;
|
loadingMore = false;
|
||||||
}
|
}
|
||||||
@@ -109,6 +118,17 @@
|
|||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
const isGlobalFeed = !profileIndexUrl && !filterHandle;
|
||||||
|
if (isGlobalFeed) {
|
||||||
|
const combined = await loadCombinedFeed(base);
|
||||||
|
if (combined) {
|
||||||
|
all = combined.activities;
|
||||||
|
feedTotalPages = combined.remainingPages + 1;
|
||||||
|
feedNextPage = combined.remainingPages > 0 ? 2 : 0;
|
||||||
|
loading = false;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
const indexUrl = profileIndexUrl
|
const indexUrl = profileIndexUrl
|
||||||
? `${base}data/${profileIndexUrl}`
|
? `${base}data/${profileIndexUrl}`
|
||||||
: `${base}data/index.json`;
|
: `${base}data/index.json`;
|
||||||
@@ -263,6 +283,8 @@
|
|||||||
Loading…
|
Loading…
|
||||||
{:else if canShowMore}
|
{:else if canShowMore}
|
||||||
Load more ({filtered.length - shown} remaining)
|
Load more ({filtered.length - shown} remaining)
|
||||||
|
{:else if feedNextPage > 0}
|
||||||
|
Load more activities
|
||||||
{:else}
|
{:else}
|
||||||
Load older activities ({pendingShards.length} more {pendingShards.length === 1 ? 'year' : 'years'})
|
Load older activities ({pendingShards.length} more {pendingShards.length === 1 ? 'year' : 'years'})
|
||||||
{/if}
|
{/if}
|
||||||
|
|||||||
@@ -253,6 +253,49 @@ export async function loadShardActivities(shardUrl: string): Promise<ActivitySum
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
interface FeedPage {
|
||||||
|
page: number;
|
||||||
|
total_pages: number;
|
||||||
|
total_activities: number;
|
||||||
|
activities: ActivitySummary[];
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Load the combined feed (multi-user global feed). Returns the first page of
|
||||||
|
* activities pre-sorted across all users, plus remaining page count.
|
||||||
|
*
|
||||||
|
* Falls back to the full shard-resolution path if feed.json doesn't exist
|
||||||
|
* (single-user installs, older data).
|
||||||
|
*/
|
||||||
|
export async function loadCombinedFeed(
|
||||||
|
baseUrl: string,
|
||||||
|
): Promise<{ activities: ActivitySummary[]; remainingPages: number } | null> {
|
||||||
|
try {
|
||||||
|
const feed = await fetchJSON<FeedPage>(`${baseUrl}data/feed.json`);
|
||||||
|
return {
|
||||||
|
activities: feed.activities ?? [],
|
||||||
|
remainingPages: (feed.total_pages ?? 1) - 1,
|
||||||
|
};
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Load a subsequent page of the combined feed (feed-2.json, feed-3.json, etc.).
|
||||||
|
*/
|
||||||
|
export async function loadCombinedFeedPage(
|
||||||
|
baseUrl: string,
|
||||||
|
page: number,
|
||||||
|
): Promise<ActivitySummary[]> {
|
||||||
|
try {
|
||||||
|
const feed = await fetchJSON<FeedPage>(`${baseUrl}data/feed-${page}.json`);
|
||||||
|
return feed.activities ?? [];
|
||||||
|
} catch {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Load a single activity detail, checking IndexedDB first so locally-converted
|
* Load a single activity detail, checking IndexedDB first so locally-converted
|
||||||
* activities are available offline.
|
* activities are available offline.
|
||||||
|
|||||||
Reference in New Issue
Block a user