1 — Timeseries split

- writer.py: timeseries is now written to {id}.timeseries.json as a separate file. The detail JSON gets a timeseries_url field instead. finalize_pending and cleanup_pending handle the extra file.
  - merge.py (merge_one): symlinks the .timeseries.json file alongside the detail JSON. merge_all already handles it transparently (the .timeseries.json stem doesn't match any activity
  ID in to_merge, so it falls through to the symlink branch).
  - types.ts: timeseries is now timeseries?: Timeseries | null, and timeseries_url?: string | null added.
  - dataloader.ts: new loadTimeseries(url, detailUrl, base) function that resolves paths correctly in both single- and multi-user modes (uses the fetched detail URL's directory as the base).
  - ActivityDetail.svelte: loads timeseries separately after detail loads; uses detail.timeseries for IDB activities (embedded) or fetches via detail.timeseries_url for server activities. Charts show a pulse placeholder while loading.

 2 — GZip

  - GZipMiddleware (min 1 KB) added to both bincio/serve/server.py and bincio/edit/server.py — all API JSON responses are now gzip-compressed.
  - For static files (the big timeseries JSONs), nginx should be configured with gzip on; gzip_types application/json application/geo+json; — no code change needed on the server side.

  Net effect: opening an activity page now fetches ~1.4 KB (detail) instead of ~586 KB. The timeseries fetches ~60–150 KB gzip-compressed shortly after (it loads concurrently with the map rendering).
This commit is contained in:
Davide Scaini
2026-04-09 14:01:02 +02:00
parent 76b7b9ac7e
commit 8118f6f316
7 changed files with 90 additions and 11 deletions
+2
View File
@@ -9,6 +9,7 @@ from typing import Any
from fastapi import FastAPI, File, HTTPException, Request, UploadFile from fastapi import FastAPI, File, HTTPException, Request, UploadFile
from fastapi.middleware.cors import CORSMiddleware from fastapi.middleware.cors import CORSMiddleware
from fastapi.middleware.gzip import GZipMiddleware
from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse
from bincio.edit.ops import SPORTS, STAT_PANELS, VALID_ACTIVITY_ID from bincio.edit.ops import SPORTS, STAT_PANELS, VALID_ACTIVITY_ID
@@ -21,6 +22,7 @@ strava_client_secret: str = ""
app = FastAPI(title="BincioActivity Edit Server", docs_url=None, redoc_url=None) app = FastAPI(title="BincioActivity Edit Server", docs_url=None, redoc_url=None)
app.add_middleware(GZipMiddleware, minimum_size=1024)
# Allow localhost origins only — this server is never meant to be public # Allow localhost origins only — this server is never meant to be public
app.add_middleware( app.add_middleware(
CORSMiddleware, CORSMiddleware,
+22 -3
View File
@@ -50,6 +50,10 @@ def write_activity(
source = _infer_source(activity) source = _infer_source(activity)
has_gps = metrics.bbox is not None and privacy not in ("no_gps", "private") has_gps = metrics.bbox is not None and privacy not in ("no_gps", "private")
# Build timeseries once — written to a separate file to keep detail JSON small
timeseries = build_timeseries(activity.points, activity.started_at, privacy)
tag = activity.source_hash[-8:] if activity.source_hash else "unknown"
# ── detail JSON ────────────────────────────────────────────────────────── # ── detail JSON ──────────────────────────────────────────────────────────
detail: dict = { detail: dict = {
"bas_version": "1.0", "bas_version": "1.0",
@@ -80,7 +84,7 @@ def write_activity(
"best_efforts": metrics.best_efforts, "best_efforts": metrics.best_efforts,
"best_climb_m": metrics.best_climb_m, "best_climb_m": metrics.best_climb_m,
"laps": [_serialise_lap(lap) for lap in activity.laps], "laps": [_serialise_lap(lap) for lap in activity.laps],
"timeseries": build_timeseries(activity.points, activity.started_at, privacy), "timeseries_url": f"activities/{activity_id}.timeseries.json" if timeseries else None,
"source": source, "source": source,
"source_file": activity.source_file, "source_file": activity.source_file,
"source_hash": activity.source_hash, "source_hash": activity.source_hash,
@@ -92,7 +96,6 @@ def write_activity(
if pending: if pending:
# Write to a unique pending file — no collision possible # Write to a unique pending file — no collision possible
tag = activity.source_hash[-8:] if activity.source_hash else "unknown"
json_path = acts_dir / f"{activity_id}.{tag}.pending.json" json_path = acts_dir / f"{activity_id}.{tag}.pending.json"
else: else:
json_path = acts_dir / f"{activity_id}.json" json_path = acts_dir / f"{activity_id}.json"
@@ -104,9 +107,18 @@ def write_activity(
activity_id = f"{activity_id}-{activity.source_hash[-6:]}" activity_id = f"{activity_id}-{activity.source_hash[-6:]}"
json_path = acts_dir / f"{activity_id}.json" json_path = acts_dir / f"{activity_id}.json"
detail["id"] = activity_id detail["id"] = activity_id
detail["timeseries_url"] = f"activities/{activity_id}.timeseries.json" if timeseries else None
json_path.write_text(json.dumps(detail, indent=2, ensure_ascii=False)) json_path.write_text(json.dumps(detail, indent=2, ensure_ascii=False))
# ── timeseries JSON (separate file — lazy-loaded by the UI) ─────────────
if timeseries:
if pending:
ts_path = acts_dir / f"{activity_id}.{tag}.pending.timeseries.json"
else:
ts_path = acts_dir / f"{activity_id}.timeseries.json"
ts_path.write_text(json.dumps(timeseries, indent=2, ensure_ascii=False))
# ── GeoJSON track ──────────────────────────────────────────────────────── # ── GeoJSON track ────────────────────────────────────────────────────────
if has_gps: if has_gps:
geojson = build_geojson(activity.points, activity_id, epsilon=rdp_epsilon) geojson = build_geojson(activity.points, activity_id, epsilon=rdp_epsilon)
@@ -150,6 +162,7 @@ def finalize_pending(output_dir: Path, activity_id: str, source_hash: str) -> st
pending_json = acts_dir / f"{activity_id}.{tag}.pending.json" pending_json = acts_dir / f"{activity_id}.{tag}.pending.json"
pending_geojson = acts_dir / f"{activity_id}.{tag}.pending.geojson" pending_geojson = acts_dir / f"{activity_id}.{tag}.pending.geojson"
pending_ts = acts_dir / f"{activity_id}.{tag}.pending.timeseries.json"
final_id = activity_id final_id = activity_id
final_json = acts_dir / f"{final_id}.json" final_json = acts_dir / f"{final_id}.json"
@@ -165,6 +178,8 @@ def finalize_pending(output_dir: Path, activity_id: str, source_hash: str) -> st
if final_id != activity_id and pending_json.exists(): if final_id != activity_id and pending_json.exists():
detail = json.loads(pending_json.read_text(encoding="utf-8")) detail = json.loads(pending_json.read_text(encoding="utf-8"))
detail["id"] = final_id detail["id"] = final_id
if detail.get("timeseries_url"):
detail["timeseries_url"] = f"activities/{final_id}.timeseries.json"
pending_json.write_text(json.dumps(detail, indent=2, ensure_ascii=False)) pending_json.write_text(json.dumps(detail, indent=2, ensure_ascii=False))
# Atomic rename: pending → final # Atomic rename: pending → final
@@ -180,6 +195,10 @@ def finalize_pending(output_dir: Path, activity_id: str, source_hash: str) -> st
pending_geojson.write_text(json.dumps(geo, indent=2, ensure_ascii=False)) pending_geojson.write_text(json.dumps(geo, indent=2, ensure_ascii=False))
pending_geojson.rename(final_geojson) pending_geojson.rename(final_geojson)
final_ts = acts_dir / f"{final_id}.timeseries.json"
if pending_ts.exists():
pending_ts.rename(final_ts)
return final_id return final_id
@@ -187,7 +206,7 @@ def cleanup_pending(output_dir: Path, activity_id: str, source_hash: str) -> Non
"""Remove pending files for a losing activity (the one not chosen as canonical).""" """Remove pending files for a losing activity (the one not chosen as canonical)."""
acts_dir = output_dir / "activities" acts_dir = output_dir / "activities"
tag = source_hash[-8:] if source_hash else "unknown" tag = source_hash[-8:] if source_hash else "unknown"
for suffix in (".pending.json", ".pending.geojson"): for suffix in (".pending.json", ".pending.geojson", ".pending.timeseries.json"):
p = acts_dir / f"{activity_id}.{tag}{suffix}" p = acts_dir / f"{activity_id}.{tag}{suffix}"
p.unlink(missing_ok=True) p.unlink(missing_ok=True)
+8
View File
@@ -109,6 +109,14 @@ def merge_one(data_dir: Path, activity_id: str) -> None:
needs_merge = has_sidecar or bool(image_files) needs_merge = has_sidecar or bool(image_files)
# Symlink the timeseries file (never merged — always points to the extract output)
ts_src = acts_dir / f"{activity_id}.timeseries.json"
ts_dest = merged_acts / f"{activity_id}.timeseries.json"
if ts_dest.exists() or ts_dest.is_symlink():
ts_dest.unlink()
if ts_src.exists():
ts_dest.symlink_to(ts_src.resolve())
# Remove the old dest (symlink or file) before writing the new one # Remove the old dest (symlink or file) before writing the new one
if dest.exists() or dest.is_symlink(): if dest.exists() or dest.is_symlink():
dest.unlink() dest.unlink()
+2
View File
@@ -17,6 +17,7 @@ from typing import Any, Optional
from fastapi import Cookie, FastAPI, File, HTTPException, Request, Response, UploadFile from fastapi import Cookie, FastAPI, File, HTTPException, Request, Response, UploadFile
from fastapi.middleware.cors import CORSMiddleware from fastapi.middleware.cors import CORSMiddleware
from fastapi.middleware.gzip import GZipMiddleware
from fastapi.responses import JSONResponse from fastapi.responses import JSONResponse
from bincio.serve.db import ( from bincio.serve.db import (
@@ -61,6 +62,7 @@ def _get_data_dir() -> Path:
app = FastAPI(title="BincioActivity Serve", docs_url=None, redoc_url=None) app = FastAPI(title="BincioActivity Serve", docs_url=None, redoc_url=None)
app.add_middleware(GZipMiddleware, minimum_size=1024)
app.add_middleware( app.add_middleware(
CORSMiddleware, CORSMiddleware,
allow_origin_regex=r"https?://localhost(:\d+)?", allow_origin_regex=r"https?://localhost(:\d+)?",
+17 -6
View File
@@ -2,12 +2,12 @@
import { onMount } from 'svelte'; import { onMount } from 'svelte';
import { marked } from 'marked'; import { marked } from 'marked';
import DOMPurify from 'dompurify'; import DOMPurify from 'dompurify';
import type { ActivitySummary, ActivityDetail, AthleteZones } from '../lib/types'; import type { ActivitySummary, ActivityDetail, AthleteZones, Timeseries } from '../lib/types';
import { formatDistance, formatDuration, formatElevation, formatSpeed, formatDate, formatTime, sportIcon, sportLabel, sportColor } from '../lib/format'; import { formatDistance, formatDuration, formatElevation, formatSpeed, formatDate, formatTime, sportIcon, sportLabel, sportColor } from '../lib/format';
import ActivityMap from './ActivityMap.svelte'; import ActivityMap from './ActivityMap.svelte';
import ActivityCharts from './ActivityCharts.svelte'; import ActivityCharts from './ActivityCharts.svelte';
import EditDrawer from './EditDrawer.svelte'; import EditDrawer from './EditDrawer.svelte';
import { loadActivity } from '../lib/dataloader'; import { loadActivity, loadTimeseries } from '../lib/dataloader';
export let activity: ActivitySummary; export let activity: ActivitySummary;
export let base: string = '/'; export let base: string = '/';
@@ -17,6 +17,8 @@
const editEnabled = editUrl !== '' || import.meta.env.PUBLIC_EDIT_ENABLED === 'true'; const editEnabled = editUrl !== '' || import.meta.env.PUBLIC_EDIT_ENABLED === 'true';
let detail: ActivityDetail | null = null; let detail: ActivityDetail | null = null;
let timeseries: Timeseries | null = null;
let timeseriesLoading = false;
let error = ''; let error = '';
let hoveredIdx: number | null = null; let hoveredIdx: number | null = null;
let editOpen = false; let editOpen = false;
@@ -31,8 +33,17 @@
try { try {
detail = await loadActivity(activity.id, activity.detail_url ?? '', base); detail = await loadActivity(activity.id, activity.detail_url ?? '', base);
if (!detail) throw new Error('Activity not found'); if (!detail) throw new Error('Activity not found');
// Use embedded timeseries (IDB activities) or lazy-fetch from URL
if (detail.timeseries) {
timeseries = detail.timeseries;
} else if (detail.timeseries_url) {
timeseriesLoading = true;
timeseries = await loadTimeseries(detail.timeseries_url, activity.detail_url ?? '', base);
timeseriesLoading = false;
}
} catch (e: any) { } catch (e: any) {
error = e.message; error = e.message;
timeseriesLoading = false;
} }
}); });
@@ -231,7 +242,7 @@
{#if trackUrl} {#if trackUrl}
<ActivityMap <ActivityMap
{trackUrl} {trackUrl}
timeseries={detail?.timeseries ?? null} {timeseries}
bbox={detail?.bbox ?? null} bbox={detail?.bbox ?? null}
accentColor={color} accentColor={color}
bind:hoveredIdx bind:hoveredIdx
@@ -263,11 +274,11 @@
<!-- Charts --> <!-- Charts -->
{#if error} {#if error}
<p class="text-red-400 text-sm mt-4">{error}</p> <p class="text-red-400 text-sm mt-4">{error}</p>
{:else if detail?.timeseries && detail.timeseries.t.length > 0} {:else if timeseries && timeseries.t.length > 0}
<div class="bg-zinc-900 rounded-xl border border-zinc-800 p-4"> <div class="bg-zinc-900 rounded-xl border border-zinc-800 p-4">
<ActivityCharts timeseries={detail.timeseries} bind:hoveredIdx {athlete} /> <ActivityCharts {timeseries} bind:hoveredIdx {athlete} />
</div> </div>
{:else if !detail} {:else if !detail || timeseriesLoading}
<div class="bg-zinc-900 rounded-xl border border-zinc-800 p-4 h-32 animate-pulse"></div> <div class="bg-zinc-900 rounded-xl border border-zinc-800 p-4 h-32 animate-pulse"></div>
{/if} {/if}
+35 -1
View File
@@ -15,7 +15,7 @@
* for anything the user recorded or converted on this device). * for anything the user recorded or converted on this device).
*/ */
import type { ActivityDetail, ActivitySummary, BASIndex } from './types'; import type { ActivityDetail, ActivitySummary, BASIndex, Timeseries } from './types';
import { listLocalActivities } from './localstore'; import { listLocalActivities } from './localstore';
// ── Helpers ─────────────────────────────────────────────────────────────────── // ── Helpers ───────────────────────────────────────────────────────────────────
@@ -170,6 +170,40 @@ export async function loadActivity(
} }
} }
/**
* Fetch the timeseries for an activity. Called lazily when the charts section
* is shown, so the initial detail load stays small (~1 KB instead of ~600 KB).
*
* @param timeseriesUrl Relative path from the detail JSON (e.g. "activities/id.timeseries.json")
* @param detailUrl The URL from which the detail JSON was fetched — used to resolve
* relative paths correctly in both single- and multi-user modes.
* @param baseUrl Site base URL — fallback when detailUrl is empty
*/
export async function loadTimeseries(
timeseriesUrl: string,
detailUrl: string,
baseUrl: string,
): Promise<Timeseries | null> {
try {
let url: string;
if (timeseriesUrl.startsWith('http')) {
url = timeseriesUrl;
} else if (detailUrl.startsWith('http')) {
// detailUrl is absolute — resolve timeseries relative to its directory
const dir = detailUrl.substring(0, detailUrl.lastIndexOf('/') + 1);
// timeseriesUrl is "activities/id.timeseries.json" — strip leading "activities/"
// because dir already ends with "activities/"
const filename = timeseriesUrl.replace(/^activities\//, '');
url = `${dir}${filename}`;
} else {
url = `${baseUrl}data/${timeseriesUrl}`;
}
return await fetchJSON<Timeseries>(url);
} catch {
return null;
}
}
/** /**
* Load athlete profile. Athlete data is not stored locally yet, so this is * Load athlete profile. Athlete data is not stored locally yet, so this is
* always a network fetch with a graceful null on failure. * always a network fetch with a graceful null on failure.
+4 -1
View File
@@ -114,7 +114,10 @@ export interface ActivityDetail extends Omit<ActivitySummary, 'detail_url' | 'tr
start_latlng: [number, number] | null; start_latlng: [number, number] | null;
end_latlng: [number, number] | null; end_latlng: [number, number] | null;
laps: Lap[]; laps: Lap[];
timeseries: Timeseries; /** Embedded timeseries — present for IDB-stored (locally converted) activities. */
timeseries?: Timeseries | null;
/** URL to fetch the timeseries — present for server-extracted activities. */
timeseries_url?: string | null;
mmp: MmpCurve | null; mmp: MmpCurve | null;
strava_id: string | null; strava_id: string | null;
duplicate_of: string | null; duplicate_of: string | null;