perf: year-shard index.json to cut initial load from MBs to ~1 year
merge_all/_merged/index.json is now a shard manifest; activities are
split into index-{year}.json files. The feed loads only the most-recent
year on first paint (~200 activities instead of all of them). Older
years are fetched lazily when the user clicks "Load older activities".
Also strips best_efforts / best_climb_m / source from shard files —
these fields are aggregation inputs only, never read by the feed UI.
This commit is contained in:
+18
-4
@@ -9,6 +9,18 @@ import pytest
|
||||
from bincio.render.merge import apply_sidecar, merge_all, merge_one, parse_sidecar
|
||||
|
||||
|
||||
def _load_merged_activities(merged_dir: Path) -> dict:
|
||||
"""Load all activities from year-sharded merged index. Returns id→dict map."""
|
||||
root = json.loads((merged_dir / "index.json").read_text())
|
||||
all_acts = list(root.get("activities", []))
|
||||
for shard in root.get("shards", []):
|
||||
shard_path = merged_dir / shard["url"]
|
||||
if shard_path.exists():
|
||||
sub = json.loads(shard_path.read_text())
|
||||
all_acts.extend(sub.get("activities", []))
|
||||
return {a["id"]: a for a in all_acts}
|
||||
|
||||
|
||||
# ── parse_sidecar ─────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@@ -176,8 +188,7 @@ def test_merge_all_private_filtered_from_index(data_dir):
|
||||
(edits / "2024-01-01T080000Z-morning-ride.md").write_text("---\nprivate: true\n---\n")
|
||||
merge_all(data_dir)
|
||||
|
||||
index = json.loads((data_dir / "_merged" / "index.json").read_text())
|
||||
activities = {a["id"]: a for a in index["activities"]}
|
||||
activities = _load_merged_activities(data_dir / "_merged")
|
||||
# unlisted activities are kept in the index; filtering is client-side
|
||||
assert "2024-01-01T080000Z-morning-ride" in activities
|
||||
assert activities["2024-01-01T080000Z-morning-ride"]["privacy"] == "unlisted"
|
||||
@@ -191,8 +202,11 @@ def test_merge_all_highlight_sorts_first(data_dir):
|
||||
(edits / "2024-01-01T080000Z-morning-ride.md").write_text("---\nhighlight: true\n---\n")
|
||||
merge_all(data_dir)
|
||||
|
||||
index = json.loads((data_dir / "_merged" / "index.json").read_text())
|
||||
ids = [a["id"] for a in index["activities"]]
|
||||
# Highlighted activity must be first within its year shard
|
||||
merged_dir = data_dir / "_merged"
|
||||
root = json.loads((merged_dir / "index.json").read_text())
|
||||
shard_path = merged_dir / root["shards"][0]["url"]
|
||||
ids = [a["id"] for a in json.loads(shard_path.read_text())["activities"]]
|
||||
assert ids[0] == "2024-01-01T080000Z-morning-ride"
|
||||
|
||||
|
||||
|
||||
@@ -83,10 +83,15 @@ class TestPipeline:
|
||||
merge_all(data_root / "brut")
|
||||
|
||||
for handle in ("dave", "brut"):
|
||||
merged = json.loads((data_root / handle / "_merged" / "index.json").read_text())
|
||||
assert len(merged["activities"]) >= 8, (
|
||||
f"Expected ≥8 merged activities for {handle}"
|
||||
)
|
||||
merged_dir = data_root / handle / "_merged"
|
||||
root = json.loads((merged_dir / "index.json").read_text())
|
||||
# Root index now has year shards; collect all activities across them
|
||||
all_acts: list = list(root.get("activities", []))
|
||||
for shard in root.get("shards", []):
|
||||
sp = merged_dir / shard["url"]
|
||||
if sp.exists():
|
||||
all_acts.extend(json.loads(sp.read_text()).get("activities", []))
|
||||
assert len(all_acts) >= 8, f"Expected ≥8 merged activities for {handle}"
|
||||
|
||||
def test_root_manifest(self, data_root):
|
||||
from bincio.render.cli import _user_dirs, _write_root_manifest
|
||||
|
||||
Reference in New Issue
Block a user