From 469a5954ccc0293df836e5d0d58228ea0cd2be66 Mon Sep 17 00:00:00 2001 From: Davide Scaini Date: Fri, 10 Apr 2026 13:01:21 +0200 Subject: [PATCH] "keep data on the server" opt-in/out --- bincio/edit/ops.py | 13 ++++++++++--- bincio/edit/server.py | 17 ++++++++++++++--- bincio/extract/ingest.py | 8 +++++++- bincio/serve/init_cmd.py | 7 ++++++- bincio/serve/server.py | 21 ++++++++++++++++++--- site/src/layouts/Base.astro | 26 ++++++++++++++++++++++---- 6 files changed, 77 insertions(+), 15 deletions(-) diff --git a/bincio/edit/ops.py b/bincio/edit/ops.py index fddddb5..50a1f16 100644 --- a/bincio/edit/ops.py +++ b/bincio/edit/ops.py @@ -9,7 +9,7 @@ from __future__ import annotations import json import re from pathlib import Path -from typing import Any +from typing import Any, Optional # ── Shared constants (imported by edit/server.py and serve/server.py) ───────── @@ -58,13 +58,20 @@ def apply_sidecar_edit(activity_id: str, payload: dict[str, Any], data_dir: Path merge_one(data_dir, activity_id) -def run_strava_sync(data_dir: Path, client_id: str, client_secret: str) -> dict[str, Any]: +def run_strava_sync( + data_dir: Path, + client_id: str, + client_secret: str, + originals_dir: Optional[Path] = None, +) -> dict[str, Any]: """Fetch new Strava activities and write them into data_dir. Args: data_dir: Per-user data directory. client_id: Strava OAuth client ID. client_secret: Strava OAuth client secret. + originals_dir: If set, raw Strava API data (meta + streams) is saved here + as JSON files for potential future reprocessing. Returns: Dict with keys: ok, imported, skipped, error_count, errors. @@ -75,7 +82,7 @@ def run_strava_sync(data_dir: Path, client_id: str, client_secret: str) -> dict[ from bincio.extract.ingest import strava_sync as _strava_sync from bincio.render.merge import merge_all - result = _strava_sync(data_dir, client_id, client_secret) + result = _strava_sync(data_dir, client_id, client_secret, originals_dir=originals_dir) if result["imported"]: merge_all(data_dir) diff --git a/bincio/edit/server.py b/bincio/edit/server.py index 7e7b516..7cb99ed 100644 --- a/bincio/edit/server.py +++ b/bincio/edit/server.py @@ -7,7 +7,7 @@ import shutil from pathlib import Path from typing import Any -from fastapi import FastAPI, File, HTTPException, Request, UploadFile +from fastapi import FastAPI, File, Form, HTTPException, Request, UploadFile from fastapi.middleware.cors import CORSMiddleware from fastapi.middleware.gzip import GZipMiddleware from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse @@ -517,7 +517,10 @@ def _file_suffix(name: str) -> str: @app.post("/api/upload") -async def upload_activity(file: UploadFile = File(...)) -> JSONResponse: +async def upload_activity( + file: UploadFile = File(...), + store_original: bool = Form(False), +) -> JSONResponse: """Accept a FIT/GPX/TCX file, extract it, update index.json, and re-merge.""" dd = _get_data_dir() @@ -536,6 +539,7 @@ async def upload_activity(file: UploadFile = File(...)) -> JSONResponse: staged = staging / name staged.write_bytes(contents) + kept = False try: from bincio.extract.metrics import compute from bincio.extract.parsers.factory import parse_file @@ -563,6 +567,12 @@ async def upload_activity(file: UploadFile = File(...)) -> JSONResponse: existing[activity_id] = summary write_index(list(existing.values()), dd, owner) + if store_original: + originals_dir = dd / "originals" + originals_dir.mkdir(exist_ok=True) + staged.rename(originals_dir / name) + kept = True + from bincio.render.merge import merge_all merge_all(dd) @@ -571,7 +581,8 @@ async def upload_activity(file: UploadFile = File(...)) -> JSONResponse: except Exception as exc: raise HTTPException(422, f"Failed to process activity file: {type(exc).__name__}") finally: - staged.unlink(missing_ok=True) + if not kept: + staged.unlink(missing_ok=True) return JSONResponse({"ok": True, "id": activity_id}) diff --git a/bincio/extract/ingest.py b/bincio/extract/ingest.py index 84406eb..7515251 100644 --- a/bincio/extract/ingest.py +++ b/bincio/extract/ingest.py @@ -10,7 +10,6 @@ from __future__ import annotations import json from pathlib import Path from typing import Any, Optional - from bincio.extract.models import ParsedActivity @@ -67,6 +66,7 @@ def strava_sync( data_dir: Path, client_id: str, client_secret: str, + originals_dir: Optional[Path] = None, ) -> dict[str, Any]: """Fetch new Strava activities and ingest them into data_dir. @@ -119,6 +119,12 @@ def strava_sync( skipped += 1 continue streams = fetch_streams(token["access_token"], meta["id"]) + if originals_dir is not None: + orig_path = originals_dir / f"{activity_id}.json" + orig_path.write_text( + json.dumps({"meta": meta, "streams": streams}, indent=2), + encoding="utf-8", + ) parsed = strava_to_parsed(meta, streams) ingest_parsed(parsed, data_dir, privacy="public", rdp_epsilon=0.0001) imported += 1 diff --git a/bincio/serve/init_cmd.py b/bincio/serve/init_cmd.py index 5122857..5dfbb84 100644 --- a/bincio/serve/init_cmd.py +++ b/bincio/serve/init_cmd.py @@ -24,7 +24,7 @@ def init(data_dir: str, handle: str, password: str, display_name: str, name: str Creates the SQLite database, the admin user, the per-user data directory, and prints a first invite code. Safe to re-run — skips steps already done. """ - from bincio.serve.db import create_invite, create_user, get_user, open_db, set_setting + from bincio.serve.db import create_invite, create_user, get_user, open_db, set_setting, get_setting dd = Path(data_dir).expanduser().resolve() dd.mkdir(parents=True, exist_ok=True) @@ -75,6 +75,11 @@ def init(data_dir: str, handle: str, password: str, display_name: str, name: str else: console.print(" [dim]·[/dim] no user limit (unlimited)") + # ── Original file storage default ───────────────────────────────────────── + if get_setting(db, "store_originals") is None: + set_setting(db, "store_originals", "true") + console.print(" [green]✓[/green] store_originals = true (users can override per upload)") + # ── First invite code ───────────────────────────────────────────────────── code = create_invite(db, handle) diff --git a/bincio/serve/server.py b/bincio/serve/server.py index 983a07d..d6f3f95 100644 --- a/bincio/serve/server.py +++ b/bincio/serve/server.py @@ -15,7 +15,7 @@ import time from pathlib import Path from typing import Any, Optional -from fastapi import Cookie, FastAPI, File, HTTPException, Request, Response, UploadFile +from fastapi import Cookie, FastAPI, File, Form, HTTPException, Request, Response, UploadFile from fastapi.middleware.cors import CORSMiddleware from fastapi.middleware.gzip import GZipMiddleware from fastapi.responses import JSONResponse @@ -164,10 +164,12 @@ async def me(bincio_session: Optional[str] = Cookie(default=None)) -> JSONRespon user = _current_user(bincio_session) if not user: raise HTTPException(404, "Not authenticated") + store_orig = get_setting(_get_db(), "store_originals") return JSONResponse({ "handle": user.handle, "display_name": user.display_name, "is_admin": user.is_admin, + "store_originals_default": store_orig != "false", }) @@ -466,6 +468,7 @@ _SUPPORTED_SUFFIXES = {".fit", ".gpx", ".tcx", ".fit.gz", ".gpx.gz", ".tcx.gz"} @app.post("/api/upload") async def upload_activity( files: list[UploadFile] = File(...), + store_original: bool = Form(False), bincio_session: Optional[str] = Cookie(default=None), ) -> JSONResponse: from bincio.extract.ingest import ingest_parsed @@ -496,6 +499,7 @@ async def upload_activity( staged = staging / name staged.write_bytes(contents) + kept = False try: activity = parse_file(staged) activity_id = make_activity_id(activity) @@ -503,12 +507,18 @@ async def upload_activity( results.append({"name": name, "ok": False, "error": "duplicate"}) continue ingest_parsed(activity, dd, privacy="public") + if store_original: + originals_dir = dd / "originals" + originals_dir.mkdir(exist_ok=True) + staged.rename(originals_dir / name) + kept = True results.append({"name": name, "ok": True, "id": activity_id}) any_added = True except Exception as exc: results.append({"name": name, "ok": False, "error": f"{type(exc).__name__}: {exc}"}) finally: - staged.unlink(missing_ok=True) + if not kept: + staged.unlink(missing_ok=True) if any_added: merge_all(dd) @@ -524,9 +534,14 @@ async def serve_strava_sync(bincio_session: Optional[str] = Cookie(default=None) if not strava_client_id or not strava_client_secret: raise HTTPException(400, "Strava not configured on this server") dd = _get_data_dir() / user.handle + store_orig_setting = get_setting(_get_db(), "store_originals") + store_orig = store_orig_setting == "true" + originals_dir = (dd / "originals" / "strava") if store_orig else None + if originals_dir: + originals_dir.mkdir(parents=True, exist_ok=True) from bincio.edit.ops import run_strava_sync try: - result = run_strava_sync(dd, strava_client_id, strava_client_secret) + result = run_strava_sync(dd, strava_client_id, strava_client_secret, originals_dir=originals_dir) except RuntimeError as e: raise HTTPException(502, str(e)) _trigger_rebuild(user.handle) diff --git a/site/src/layouts/Base.astro b/site/src/layouts/Base.astro index 679daf8..7cc36f7 100644 --- a/site/src/layouts/Base.astro +++ b/site/src/layouts/Base.astro @@ -175,6 +175,7 @@ try { {mobileApp && ( Convert )} + About )} @@ -259,6 +260,17 @@ try {
Drop FIT, GPX, or TCX files
or click to browse
+

@@ -366,6 +378,10 @@ try { // Show logout button const logoutEl = document.getElementById('nav-logout'); if (logoutEl) logoutEl.style.display = ''; + + // Pre-populate the "keep original" checkbox from the instance default + const chk = document.getElementById('upload-keep-original'); + if (chk && user.store_originals_default) chk.checked = true; } catch (_) {} })(); @@ -388,10 +404,11 @@ try { const chooseStrava = document.getElementById('upload-choose-strava'); const backFile = document.getElementById('upload-back-file'); const backStrava = document.getElementById('upload-back-strava'); - const drop = document.getElementById('upload-drop'); - const input = document.getElementById('upload-input'); - const label = document.getElementById('upload-label'); - const fileStatus = document.getElementById('upload-status'); + const drop = document.getElementById('upload-drop'); + const input = document.getElementById('upload-input'); + const label = document.getElementById('upload-label'); + const keepOriginalChk = document.getElementById('upload-keep-original'); + const fileStatus = document.getElementById('upload-status'); const stravaStatus = document.getElementById('strava-status'); const stravaConnect = document.getElementById('strava-connect-area'); const stravaSync = document.getElementById('strava-sync-area'); @@ -449,6 +466,7 @@ try { drop.style.pointerEvents = 'none'; const fd = new FormData(); for (const f of files) fd.append('files', f); + fd.append('store_original', keepOriginalChk?.checked ? 'true' : 'false'); try { const r = await fetch(`${editUrl}/api/upload`, { method: 'POST', credentials: 'include', body: fd }); if (!r.ok) throw new Error(await r.text());