diff --git a/bincio/dev.py b/bincio/dev.py index 8e7f6df..06e2ef4 100644 --- a/bincio/dev.py +++ b/bincio/dev.py @@ -141,6 +141,11 @@ def _watch_data(data: Path) -> None: continue for prefix, user_dir in prefix_to_user.items(): if path.startswith(prefix): + # Skip new-activity .json writes in activities/ — the upload + # endpoint calls merge_one inline, so firing merge_all here + # too would cause O(N²) full rebuilds during bulk uploads. + if path.startswith(str(user_dir / "activities")) and path.endswith(".json"): + break affected.add(user_dir) break diff --git a/bincio/render/merge.py b/bincio/render/merge.py index fc6c5da..264b6a7 100644 --- a/bincio/render/merge.py +++ b/bincio/render/merge.py @@ -98,6 +98,11 @@ def merge_one(data_dir: Path, activity_id: str) -> None: Use merge_all() for bulk operations (first run, Strava sync, etc.). """ + with _merge_lock(data_dir): + _merge_one_locked(data_dir, activity_id) + + +def _merge_one_locked(data_dir: Path, activity_id: str) -> None: edits_dir = data_dir / "edits" acts_dir = data_dir / "activities" merged_dir = data_dir / "_merged" @@ -311,7 +316,7 @@ def _write_year_shards(merged_dir: Path, activities: list[dict], index_meta: dic # Remove stale year shard files from previous runs for f in merged_dir.glob("index-*.json"): - f.unlink() + f.unlink(missing_ok=True) by_year: dict[str, list[dict]] = defaultdict(list) for a in activities: @@ -398,7 +403,7 @@ def write_combined_feed(data_dir: Path) -> int: # Remove stale feed pages for f in data_dir.glob("feed*.json"): - f.unlink() + f.unlink(missing_ok=True) if not all_activities: return 0 diff --git a/bincio/serve/server.py b/bincio/serve/server.py index 920004c..8d8f474 100644 --- a/bincio/serve/server.py +++ b/bincio/serve/server.py @@ -596,9 +596,12 @@ async def upload_bas_activity( _upsert_index_summary(user_dir, activity_id, activity, geojson_body) - from bincio.render.merge import merge_all, write_combined_feed - merge_all(user_dir) - write_combined_feed(_get_data_dir()) + try: + from bincio.render.merge import merge_one, write_combined_feed + merge_one(user_dir, activity_id) + write_combined_feed(_get_data_dir()) + except Exception as exc: + log.warning("upload/bas[%s]: merge/feed failed (non-fatal): %s", user.handle, exc) log.info("upload/bas[%s]: imported %s", user.handle, activity_id) return JSONResponse({"ok": True, "id": activity_id, "status": "imported"}) @@ -688,10 +691,6 @@ async def upload_raw_activity( _upsert_index_summary(user_dir, act_id, detail, geojson) - from bincio.render.merge import merge_all, write_combined_feed - merge_all(user_dir) - write_combined_feed(_get_data_dir()) - except Exception as exc: log.warning("upload/raw[%s]: extraction failed: %s", user.handle, exc) raise HTTPException(422, f"Could not extract activity: {exc}") from exc @@ -699,6 +698,16 @@ async def upload_raw_activity( tmp_in.unlink(missing_ok=True) shutil.rmtree(tmp_out, ignore_errors=True) + # Merge and update feed — best effort; a race or transient FS error here must + # not turn a successful extraction into a 422 (the file is on disk; the mobile + # would retry indefinitely and the activity would never be marked synced). + try: + from bincio.render.merge import merge_one, write_combined_feed + merge_one(user_dir, act_id) + write_combined_feed(_get_data_dir()) + except Exception as exc: + log.warning("upload/raw[%s]: merge/feed failed (non-fatal): %s", user.handle, exc) + log.info("upload/raw[%s]: imported %s", user.handle, act_id) return JSONResponse({ "ok": True, diff --git a/site/astro.config.mjs b/site/astro.config.mjs index 95e75cd..b18a482 100644 --- a/site/astro.config.mjs +++ b/site/astro.config.mjs @@ -39,6 +39,12 @@ export default defineConfig({ // Proxy /api/* to bincio serve/edit so cookies work same-origin in dev. // In production nginx handles this — same pattern, no code change needed. server: { + watch: { + // public/data is a symlink to the live data dir; Chokidar follows it and + // opens a handle per file, which causes EMFILE during bulk uploads as + // activity count grows. Data files don't need HMR — exclude them. + ignored: ['**/public/data/**'], + }, proxy: { // Both /api/upload and /api/upload/strava-zip return SSE streams in response // to POST requests. Vite's default proxy buffers the full body before forwarding,