diff --git a/bincio/extract/ingest.py b/bincio/extract/ingest.py index dedb4fc..24daf4e 100644 --- a/bincio/extract/ingest.py +++ b/bincio/extract/ingest.py @@ -76,24 +76,19 @@ def ingest_parsed( return activity_id -def strava_sync( +def strava_sync_iter( data_dir: Path, client_id: str, client_secret: str, originals_dir: Optional[Path] = None, -) -> dict[str, Any]: - """Fetch new Strava activities and ingest them into data_dir. +): + """Generator version of strava_sync — yields progress dicts, then a final summary. - Args: - data_dir: Per-user data directory. - client_id: Strava OAuth client ID. - client_secret: Strava OAuth client secret. - - Returns: - Dict with keys: ok, imported, skipped, error_count, errors. - - Raises: - RuntimeError: If Strava credentials are missing or API calls fail. + Each yielded dict has a ``type`` key: + - ``"fetching"`` — about to fetch the activity list from Strava + - ``"progress"`` — one activity processed; keys: n, total, name, status ("imported"|"skipped"|"error") + - ``"done"`` — final summary; keys: imported, skipped, error_count, errors + - ``"error"`` — fatal error before processing started; key: message """ import time @@ -109,28 +104,36 @@ def strava_sync( from bincio.extract.writer import make_activity_id if not client_id or not client_secret: - raise RuntimeError("Strava not configured (missing client_id or client_secret)") + yield {"type": "error", "message": "Strava not configured"} + return try: token = ensure_fresh(data_dir, client_id, client_secret) except StravaError as e: - raise RuntimeError(str(e)) from e + yield {"type": "error", "message": str(e)} + return + + yield {"type": "fetching"} after: Optional[int] = token.get("last_sync_at") try: activities = fetch_activities(token["access_token"], after=after) except StravaError as e: - raise RuntimeError(str(e)) from e + yield {"type": "error", "message": str(e)} + return + total = len(activities) imported = 0 skipped = 0 errors: list[str] = [] - for meta in activities: + for n, meta in enumerate(activities, 1): + name = meta.get("name", "Untitled") try: activity_id = make_activity_id(strava_meta_to_partial(meta)) if (data_dir / "activities" / f"{activity_id}.json").exists(): skipped += 1 + yield {"type": "progress", "n": n, "total": total, "name": name, "status": "skipped"} continue streams = fetch_streams(token["access_token"], meta["id"]) if originals_dir is not None: @@ -142,16 +145,41 @@ def strava_sync( parsed = strava_to_parsed(meta, streams) ingest_parsed(parsed, data_dir, privacy="public", rdp_epsilon=0.0001) imported += 1 + yield {"type": "progress", "n": n, "total": total, "name": name, "status": "imported"} except Exception as exc: errors.append(f"{meta.get('id')}: {type(exc).__name__}") + yield {"type": "progress", "n": n, "total": total, "name": name, "status": "error"} token["last_sync_at"] = int(time.time()) save_token(data_dir, token) - return { - "ok": True, + yield { + "type": "done", "imported": imported, "skipped": skipped, "error_count": len(errors), "errors": errors[:5], } + + +def strava_sync( + data_dir: Path, + client_id: str, + client_secret: str, + originals_dir: Optional[Path] = None, +) -> dict[str, Any]: + """Fetch new Strava activities and ingest them into data_dir. + + Returns: + Dict with keys: ok, imported, skipped, error_count, errors. + + Raises: + RuntimeError: If Strava credentials are missing or API calls fail. + """ + result: dict[str, Any] = {} + for event in strava_sync_iter(data_dir, client_id, client_secret, originals_dir): + if event["type"] == "error": + raise RuntimeError(event["message"]) + if event["type"] == "done": + result = event + return {"ok": True, **{k: v for k, v in result.items() if k != "type"}} diff --git a/bincio/serve/server.py b/bincio/serve/server.py index 9b8ab9f..3646648 100644 --- a/bincio/serve/server.py +++ b/bincio/serve/server.py @@ -18,7 +18,7 @@ from pathlib import Path from typing import Any, Optional from fastapi import Cookie, FastAPI, File, Form, HTTPException, Request, Response, UploadFile -from fastapi.responses import RedirectResponse +from fastapi.responses import RedirectResponse, StreamingResponse from fastapi.middleware.cors import CORSMiddleware from fastapi.middleware.gzip import GZipMiddleware from fastapi.responses import JSONResponse @@ -712,6 +712,37 @@ async def strava_callback( return RedirectResponse(f"{site_origin}/?strava=connected") +@app.get("/api/strava/sync/stream") +async def serve_strava_sync_stream(bincio_session: Optional[str] = Cookie(default=None)) -> StreamingResponse: + """SSE endpoint — streams per-activity progress then a final summary event.""" + user = _require_user(bincio_session) + if not strava_client_id or not strava_client_secret: + raise HTTPException(400, "Strava not configured on this server") + dd = _get_data_dir() / user.handle + store_orig_setting = get_setting(_get_db(), "store_originals") + store_orig = store_orig_setting == "true" + originals_dir = (dd / "originals" / "strava") if store_orig else None + if originals_dir: + originals_dir.mkdir(parents=True, exist_ok=True) + + from bincio.extract.ingest import strava_sync_iter + + def event_stream(): + try: + for event in strava_sync_iter(dd, strava_client_id, strava_client_secret, originals_dir): + yield f"data: {json.dumps(event)}\n\n" + if event["type"] == "done": + _trigger_rebuild(user.handle) + except Exception as exc: + yield f"data: {json.dumps({'type': 'error', 'message': str(exc)})}\n\n" + + return StreamingResponse( + event_stream(), + media_type="text/event-stream", + headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}, + ) + + @app.post("/api/strava/sync") async def serve_strava_sync(bincio_session: Optional[str] = Cookie(default=None)) -> JSONResponse: user = _require_user(bincio_session) diff --git a/site/src/layouts/Base.astro b/site/src/layouts/Base.astro index 5f33ad2..da29b81 100644 --- a/site/src/layouts/Base.astro +++ b/site/src/layouts/Base.astro @@ -561,26 +561,51 @@ try { } }); - stravaSyncBtn.addEventListener('click', async () => { + stravaSyncBtn.addEventListener('click', () => { stravaSyncBtn.disabled = true; stravaSyncBtn.textContent = 'Syncing…'; stravaStatus.textContent = ''; - try { - const r = await fetch(`${editUrl}/api/strava/sync`, { method: 'POST' }); - if (!r.ok) throw new Error(await r.text()); - const d = await r.json(); - stravaLastSync.textContent = new Date().toLocaleString(); - const errNote = d.error_count ? `, ${d.error_count} errors` : ''; - stravaStatus.textContent = `Done — ${d.imported} imported, ${d.skipped} already up to date${errNote}.`; - stravaStatus.style.color = '#4ade80'; - if (d.imported > 0) setTimeout(() => window.location.reload(), 1500); - } catch (e) { - stravaStatus.textContent = 'Error: ' + e.message; - stravaStatus.style.color = '#f87171'; - } finally { - stravaSyncBtn.disabled = false; - stravaSyncBtn.textContent = 'Sync now'; - } + stravaStatus.style.color = ''; + + const es = new EventSource(`${editUrl}/api/strava/sync/stream`, { withCredentials: true }); + let imported = 0; + + es.onmessage = (e) => { + const d = JSON.parse(e.data); + if (d.type === 'fetching') { + stravaStatus.textContent = 'Fetching activity list from Strava…'; + } else if (d.type === 'progress') { + const pct = Math.round((d.n / d.total) * 100); + const icon = d.status === 'imported' ? '↓' : d.status === 'error' ? '✗' : '·'; + stravaStatus.textContent = `${icon} ${d.n}/${d.total} (${pct}%) — ${d.name}`; + if (d.status === 'imported') imported++; + } else if (d.type === 'done') { + es.close(); + stravaLastSync.textContent = new Date().toLocaleString(); + const errNote = d.error_count ? `, ${d.error_count} errors` : ''; + stravaStatus.textContent = `Done — ${d.imported} imported, ${d.skipped} already up to date${errNote}.`; + stravaStatus.style.color = '#4ade80'; + stravaSyncBtn.disabled = false; + stravaSyncBtn.textContent = 'Sync now'; + if (d.imported > 0) setTimeout(() => window.location.reload(), 1500); + } else if (d.type === 'error') { + es.close(); + stravaStatus.textContent = 'Error: ' + d.message; + stravaStatus.style.color = '#f87171'; + stravaSyncBtn.disabled = false; + stravaSyncBtn.textContent = 'Sync now'; + } + }; + + es.onerror = () => { + es.close(); + if (stravaSyncBtn.disabled) { + stravaStatus.textContent = 'Connection lost. Check logs.'; + stravaStatus.style.color = '#f87171'; + stravaSyncBtn.disabled = false; + stravaSyncBtn.textContent = 'Sync now'; + } + }; }); async function stravaReset(mode) { diff --git a/tests/test_server_imports.py b/tests/test_server_imports.py index 76c55ac..8ac4c7d 100644 --- a/tests/test_server_imports.py +++ b/tests/test_server_imports.py @@ -18,6 +18,7 @@ def test_serve_app_has_routes(): assert "/api/strava/auth-url" in paths assert "/api/strava/callback" in paths assert "/api/strava/sync" in paths + assert "/api/strava/sync/stream" in paths assert "/api/register" in paths