For users uploading:
- POST /api/upload now returns text/event-stream instead of JSON - Per-file progress events stream back as each file is processed: ↓ 3/47 (6%) — morning_ride.fit - Final done event shows the summary: "12 added, 35 duplicates" - The Vite proxy is configured to stream this properly (no buffering) For the admin: - New GET /api/admin/jobs endpoint (admin-only) returns the list of active upload jobs, each with user, started_at, total, done, current (filename being processed) - A pulsing amber badge appears in the nav bar for admins when any user has an active upload running — it shows e.g. "2 uploads running" with a tooltip listing each user's progress (@alice: 12/50 files) - Polls every 5 seconds, disappears automatically when all jobs finish
This commit is contained in:
Davide Scaini
+69
-63
@@ -540,11 +540,11 @@ def _file_suffix(name: str) -> str:
|
||||
async def upload_activity(
|
||||
files: list[UploadFile] = File(...),
|
||||
store_original: bool = Form(False),
|
||||
) -> JSONResponse:
|
||||
"""Accept FIT/GPX/TCX files and/or activities.csv, extract, update index, re-merge.
|
||||
) -> StreamingResponse:
|
||||
"""Accept FIT/GPX/TCX files and/or activities.csv; stream SSE progress while processing.
|
||||
|
||||
activities.csv (Strava export format) can be included in the batch to:
|
||||
- Enrich activity files being uploaded in the same batch (matched by filename)
|
||||
- Enrich activity files in the same batch (matched by filename)
|
||||
- Retroactively update sidecars for existing activities (matched by strava_id)
|
||||
"""
|
||||
from bincio.extract.ingest import ingest_parsed
|
||||
@@ -556,90 +556,96 @@ async def upload_activity(
|
||||
staging = dd / "_uploads"
|
||||
staging.mkdir(exist_ok=True)
|
||||
|
||||
_MAX_UPLOAD_BYTES = 50 * 1024 * 1024 # 50 MB
|
||||
# Read all files into memory now (async), then process synchronously in the generator
|
||||
csv_bytes_list: list[bytes] = []
|
||||
activity_items: list[tuple[str, bytes]] = []
|
||||
|
||||
# Separate CSV files from activity files
|
||||
csv_files: list[UploadFile] = []
|
||||
activity_files: list[UploadFile] = []
|
||||
for f in files:
|
||||
name = Path(f.filename or "").name.lower()
|
||||
if name.endswith(".csv"):
|
||||
csv_files.append(f)
|
||||
fname = Path(f.filename or "").name
|
||||
raw = await f.read()
|
||||
if fname.lower().endswith(".csv"):
|
||||
csv_bytes_list.append(raw)
|
||||
else:
|
||||
activity_files.append(f)
|
||||
activity_items.append((fname, raw))
|
||||
|
||||
# Build metadata from the first CSV found (activities.csv from Strava export)
|
||||
metadata = None
|
||||
if csv_files:
|
||||
if csv_bytes_list:
|
||||
from bincio.extract.strava_csv import StravaMetadata
|
||||
import tempfile
|
||||
csv_upload = csv_files[0]
|
||||
csv_bytes = await csv_upload.read()
|
||||
with tempfile.NamedTemporaryFile(suffix=".csv", delete=False) as tmp:
|
||||
tmp.write(csv_bytes)
|
||||
tmp.write(csv_bytes_list[0])
|
||||
tmp_path = Path(tmp.name)
|
||||
try:
|
||||
metadata = StravaMetadata(tmp_path)
|
||||
finally:
|
||||
tmp_path.unlink(missing_ok=True)
|
||||
|
||||
results = []
|
||||
any_added = False
|
||||
total_files = len(activity_items)
|
||||
|
||||
for file in activity_files:
|
||||
name = Path(file.filename or "upload.fit").name
|
||||
suffix = _file_suffix(name)
|
||||
if suffix not in _SUPPORTED_SUFFIXES:
|
||||
results.append({"name": name, "ok": False, "error": f"Unsupported file type '{Path(name).suffix}'"})
|
||||
continue
|
||||
def event_stream():
|
||||
added = 0
|
||||
duplicates = 0
|
||||
errors = 0
|
||||
any_added = False
|
||||
|
||||
contents = await file.read()
|
||||
if len(contents) > _MAX_UPLOAD_BYTES:
|
||||
results.append({"name": name, "ok": False, "error": "File too large (max 50 MB)"})
|
||||
continue
|
||||
|
||||
staged = staging / name
|
||||
staged.write_bytes(contents)
|
||||
kept = False
|
||||
try:
|
||||
activity = parse_file(staged)
|
||||
|
||||
# Enrich with CSV metadata when available (matched by filename)
|
||||
if metadata is not None:
|
||||
metadata.enrich(name, activity)
|
||||
|
||||
activity_id = make_activity_id(activity)
|
||||
if (dd / "activities" / f"{activity_id}.json").exists():
|
||||
results.append({"name": name, "ok": False, "error": "duplicate"})
|
||||
for n, (name, contents) in enumerate(activity_items, 1):
|
||||
suffix = _file_suffix(name)
|
||||
if suffix not in _SUPPORTED_SUFFIXES:
|
||||
errors += 1
|
||||
yield f"data: {json.dumps({'type': 'progress', 'n': n, 'total': total_files, 'name': name, 'status': 'error', 'detail': 'unsupported type'})}\n\n"
|
||||
continue
|
||||
|
||||
ingest_parsed(activity, dd, privacy="public")
|
||||
if len(contents) > _MAX_UPLOAD_BYTES:
|
||||
errors += 1
|
||||
yield f"data: {json.dumps({'type': 'progress', 'n': n, 'total': total_files, 'name': name, 'status': 'error', 'detail': 'file too large'})}\n\n"
|
||||
continue
|
||||
|
||||
if store_original:
|
||||
originals_dir = dd / "originals"
|
||||
originals_dir.mkdir(exist_ok=True)
|
||||
staged.rename(originals_dir / name)
|
||||
kept = True
|
||||
staged = staging / name
|
||||
staged.write_bytes(contents)
|
||||
kept = False
|
||||
try:
|
||||
activity = parse_file(staged)
|
||||
if metadata is not None:
|
||||
metadata.enrich(name, activity)
|
||||
activity_id = make_activity_id(activity)
|
||||
if (dd / "activities" / f"{activity_id}.json").exists():
|
||||
duplicates += 1
|
||||
yield f"data: {json.dumps({'type': 'progress', 'n': n, 'total': total_files, 'name': name, 'status': 'duplicate'})}\n\n"
|
||||
continue
|
||||
ingest_parsed(activity, dd, privacy="public")
|
||||
if store_original:
|
||||
originals_dir = dd / "originals"
|
||||
originals_dir.mkdir(exist_ok=True)
|
||||
staged.rename(originals_dir / name)
|
||||
kept = True
|
||||
added += 1
|
||||
any_added = True
|
||||
yield f"data: {json.dumps({'type': 'progress', 'n': n, 'total': total_files, 'name': name, 'status': 'imported'})}\n\n"
|
||||
except Exception:
|
||||
errors += 1
|
||||
yield f"data: {json.dumps({'type': 'progress', 'n': n, 'total': total_files, 'name': name, 'status': 'error'})}\n\n"
|
||||
finally:
|
||||
if not kept:
|
||||
staged.unlink(missing_ok=True)
|
||||
|
||||
results.append({"name": name, "ok": True, "id": activity_id})
|
||||
any_added = True
|
||||
except Exception:
|
||||
results.append({"name": name, "ok": False, "error": "Processing failed"})
|
||||
finally:
|
||||
if not kept:
|
||||
staged.unlink(missing_ok=True)
|
||||
csv_updates = 0
|
||||
if metadata is not None:
|
||||
from bincio.extract.strava_csv import apply_csv_to_data_dir
|
||||
csv_updates = apply_csv_to_data_dir(dd, metadata)
|
||||
if csv_updates:
|
||||
yield f"data: {json.dumps({'type': 'csv', 'updates': csv_updates})}\n\n"
|
||||
|
||||
# Retroactively update sidecars for existing activities matched by strava_id
|
||||
csv_updates = 0
|
||||
if metadata is not None:
|
||||
from bincio.extract.strava_csv import apply_csv_to_data_dir
|
||||
csv_updates = apply_csv_to_data_dir(dd, metadata)
|
||||
if any_added or csv_updates:
|
||||
merge_all(dd)
|
||||
|
||||
if any_added or csv_updates:
|
||||
merge_all(dd)
|
||||
yield f"data: {json.dumps({'type': 'done', 'added': added, 'csv_updates': csv_updates, 'duplicates': duplicates, 'errors': errors})}\n\n"
|
||||
|
||||
added = [r for r in results if r["ok"]]
|
||||
return JSONResponse({"ok": True, "added": len(added), "csv_updates": csv_updates, "results": results})
|
||||
return StreamingResponse(
|
||||
event_stream(),
|
||||
media_type="text/event-stream",
|
||||
headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
|
||||
)
|
||||
|
||||
|
||||
@app.post("/api/import-bas")
|
||||
|
||||
Reference in New Issue
Block a user