upload: add overwrite option to replace existing activities

When 'Overwrite existing activities' is checked, duplicate activities are
re-extracted and replaced instead of silently skipped:
- Deletes {id}.json, .geojson, .timeseries.json from activities/ and _merged/
- Removes the stale index summary and dedup cache entry
- Ingests the new file fresh via ingest_parsed
- Reports 'overwritten' (↺) status in the SSE stream vs 'imported' (↓)
- done event includes 'overwritten' count; UI shows it alongside 'added'
This commit is contained in:
Davide Scaini
2026-04-15 20:17:32 +02:00
parent a33fea91cf
commit 764da09130
2 changed files with 61 additions and 12 deletions
+42 -8
View File
@@ -1129,6 +1129,7 @@ def _file_suffix(name: str) -> str:
async def upload_activity( async def upload_activity(
files: list[UploadFile] = File(...), files: list[UploadFile] = File(...),
store_original: bool = Form(False), store_original: bool = Form(False),
overwrite: bool = Form(False),
bincio_session: Optional[str] = Cookie(default=None), bincio_session: Optional[str] = Cookie(default=None),
) -> StreamingResponse: ) -> StreamingResponse:
"""Accept FIT/GPX/TCX files and/or activities.csv; stream SSE progress while processing. """Accept FIT/GPX/TCX files and/or activities.csv; stream SSE progress while processing.
@@ -1138,9 +1139,9 @@ async def upload_activity(
- Retroactively update sidecars for existing activities (matched by strava_id) - Retroactively update sidecars for existing activities (matched by strava_id)
SSE events: SSE events:
{"type": "progress", "n": N, "total": T, "name": "...", "status": "imported"|"duplicate"|"error"} {"type": "progress", "n": N, "total": T, "name": "...", "status": "imported"|"overwritten"|"duplicate"|"error"}
{"type": "csv", "updates": N} -- only when CSV was included {"type": "csv", "updates": N} -- only when CSV was included
{"type": "done", "added": N, "csv_updates": N, "duplicates": N, "errors": N} {"type": "done", "added": N, "csv_updates": N, "duplicates": N, "overwritten": N, "errors": N}
""" """
from bincio.extract.ingest import ingest_parsed from bincio.extract.ingest import ingest_parsed
from bincio.extract.parsers.factory import parse_file from bincio.extract.parsers.factory import parse_file
@@ -1182,6 +1183,7 @@ async def upload_activity(
def event_stream(): def event_stream():
added = 0 added = 0
overwritten = 0
duplicates = 0 duplicates = 0
errors = 0 errors = 0
any_added = False any_added = False
@@ -1209,19 +1211,51 @@ async def upload_activity(
if metadata is not None: if metadata is not None:
metadata.enrich(name, activity) metadata.enrich(name, activity)
activity_id = make_activity_id(activity) activity_id = make_activity_id(activity)
was_overwrite = False
if (dd / "activities" / f"{activity_id}.json").exists(): if (dd / "activities" / f"{activity_id}.json").exists():
duplicates += 1 if not overwrite:
yield f"data: {json.dumps({'type': 'progress', 'n': n, 'total': total_files, 'name': name, 'status': 'duplicate'})}\n\n" duplicates += 1
continue yield f"data: {json.dumps({'type': 'progress', 'n': n, 'total': total_files, 'name': name, 'status': 'duplicate'})}\n\n"
continue
# Overwrite: delete existing files before re-ingesting.
for ext in (".json", ".geojson", ".timeseries.json"):
(dd / "activities" / f"{activity_id}{ext}").unlink(missing_ok=True)
# Remove stale summary from index so ingest_parsed writes a clean one
index_path = dd / "index.json"
if index_path.exists():
idx = json.loads(index_path.read_text(encoding="utf-8"))
idx["activities"] = [a for a in idx.get("activities", []) if a.get("id") != activity_id]
index_path.write_text(json.dumps(idx, indent=2, ensure_ascii=False))
# Remove from dedup hash cache so the new file isn't blocked
cache_path = dd / ".bincio_cache.json"
if cache_path.exists():
try:
cache = json.loads(cache_path.read_text(encoding="utf-8"))
cache.pop(activity_id, None)
cache_path.write_text(json.dumps(cache, ensure_ascii=False))
except Exception:
pass
# Remove merged copies (merge_all will regenerate them after ingest)
merged_acts = dd / "_merged" / "activities"
if merged_acts.exists():
for ext in (".json", ".geojson", ".timeseries.json"):
p = merged_acts / f"{activity_id}{ext}"
if p.exists() or p.is_symlink():
p.unlink(missing_ok=True)
was_overwrite = True
ingest_parsed(activity, dd, privacy="public") ingest_parsed(activity, dd, privacy="public")
if store_original: if store_original:
originals_dir = dd / "originals" originals_dir = dd / "originals"
originals_dir.mkdir(exist_ok=True) originals_dir.mkdir(exist_ok=True)
staged.rename(originals_dir / name) staged.rename(originals_dir / name)
kept = True kept = True
added += 1 if was_overwrite:
overwritten += 1
else:
added += 1
any_added = True any_added = True
yield f"data: {json.dumps({'type': 'progress', 'n': n, 'total': total_files, 'name': name, 'status': 'imported'})}\n\n" status = 'overwritten' if was_overwrite else 'imported'
yield f"data: {json.dumps({'type': 'progress', 'n': n, 'total': total_files, 'name': name, 'status': status})}\n\n"
except Exception as exc: except Exception as exc:
errors += 1 errors += 1
log.error("upload[%s]: failed to process %s: %s", user.handle, name, exc, exc_info=True) log.error("upload[%s]: failed to process %s: %s", user.handle, name, exc, exc_info=True)
@@ -1243,7 +1277,7 @@ async def upload_activity(
if any_added: if any_added:
_trigger_rebuild(user.handle) _trigger_rebuild(user.handle)
yield f"data: {json.dumps({'type': 'done', 'added': added, 'csv_updates': csv_updates, 'duplicates': duplicates, 'errors': errors})}\n\n" yield f"data: {json.dumps({'type': 'done', 'added': added, 'overwritten': overwritten, 'csv_updates': csv_updates, 'duplicates': duplicates, 'errors': errors})}\n\n"
if job_id: if job_id:
_job_finish(job_id) _job_finish(job_id)
+19 -4
View File
@@ -317,6 +317,17 @@ try {
<span class="text-zinc-600 block mt-0.5">Lets you reprocess if the format changes. See the <a href={`${baseUrl}about/`} class="underline hover:text-zinc-400">About page</a> for details.</span> <span class="text-zinc-600 block mt-0.5">Lets you reprocess if the format changes. See the <a href={`${baseUrl}about/`} class="underline hover:text-zinc-400">About page</a> for details.</span>
</span> </span>
</label> </label>
<label class="flex items-start gap-2 mt-2 cursor-pointer group">
<input
id="upload-overwrite"
type="checkbox"
class="mt-0.5 accent-amber-500 shrink-0"
/>
<span class="text-xs text-zinc-500 group-hover:text-zinc-300 transition-colors leading-snug">
Overwrite existing activities
<span class="text-zinc-600 block mt-0.5">Re-extract and replace any duplicate found on the server. Use to fix a corrupted or mis-parsed activity.</span>
</span>
</label>
<p id="upload-status" class="mt-3 text-xs text-center" style="color: var(--text-5); min-height: 1.25rem"></p> <p id="upload-status" class="mt-3 text-xs text-center" style="color: var(--text-5); min-height: 1.25rem"></p>
</div> </div>
@@ -554,6 +565,7 @@ try {
const input = document.getElementById('upload-input'); const input = document.getElementById('upload-input');
const label = document.getElementById('upload-label'); const label = document.getElementById('upload-label');
const keepOriginalChk = document.getElementById('upload-keep-original'); const keepOriginalChk = document.getElementById('upload-keep-original');
const overwriteChk = document.getElementById('upload-overwrite');
const fileStatus = document.getElementById('upload-status'); const fileStatus = document.getElementById('upload-status');
const stravaStatus = document.getElementById('strava-status'); const stravaStatus = document.getElementById('strava-status');
const stravaConnect = document.getElementById('strava-connect-area'); const stravaConnect = document.getElementById('strava-connect-area');
@@ -629,6 +641,7 @@ try {
const fd = new FormData(); const fd = new FormData();
for (const f of files) fd.append('files', f); for (const f of files) fd.append('files', f);
fd.append('store_original', keepOriginalChk?.checked ? 'true' : 'false'); fd.append('store_original', keepOriginalChk?.checked ? 'true' : 'false');
fd.append('overwrite', overwriteChk?.checked ? 'true' : 'false');
const xhr = new XMLHttpRequest(); const xhr = new XMLHttpRequest();
xhr.open('POST', `${editUrl}/api/upload`); xhr.open('POST', `${editUrl}/api/upload`);
@@ -636,7 +649,7 @@ try {
xhr.setRequestHeader('Accept', 'text/event-stream'); xhr.setRequestHeader('Accept', 'text/event-stream');
let buf = ''; let buf = '';
let added = 0, dupes = 0, errors = 0, csvUpdates = 0; let added = 0, overwrittenCount = 0, dupes = 0, errors = 0, csvUpdates = 0;
xhr.onprogress = () => { xhr.onprogress = () => {
const newText = xhr.responseText.slice(buf.length); const newText = xhr.responseText.slice(buf.length);
@@ -647,23 +660,25 @@ try {
const ev = JSON.parse(line.slice(6)); const ev = JSON.parse(line.slice(6));
if (ev.type === 'progress') { if (ev.type === 'progress') {
const pct = Math.round((ev.n / ev.total) * 100); const pct = Math.round((ev.n / ev.total) * 100);
const icon = ev.status === 'imported' ? '↓' : ev.status === 'duplicate' ? '·' : '✗'; const icon = ev.status === 'imported' ? '↓' : ev.status === 'overwritten' ? '↺' : ev.status === 'duplicate' ? '·' : '✗';
fileStatus.textContent = `${icon} ${ev.n}/${ev.total} (${pct}%) — ${ev.name}`; fileStatus.textContent = `${icon} ${ev.n}/${ev.total} (${pct}%) — ${ev.name}`;
if (ev.status === 'imported') added++; if (ev.status === 'imported') added++;
else if (ev.status === 'overwritten') overwrittenCount++;
else if (ev.status === 'duplicate') dupes++; else if (ev.status === 'duplicate') dupes++;
else errors++; else errors++;
} else if (ev.type === 'csv') { } else if (ev.type === 'csv') {
csvUpdates = ev.updates; csvUpdates = ev.updates;
} else if (ev.type === 'done') { } else if (ev.type === 'done') {
added = ev.added; dupes = ev.duplicates; errors = ev.errors; csvUpdates = ev.csv_updates; added = ev.added; overwrittenCount = ev.overwritten ?? 0; dupes = ev.duplicates; errors = ev.errors; csvUpdates = ev.csv_updates;
const parts = []; const parts = [];
if (added > 0) parts.push(`${added} added`); if (added > 0) parts.push(`${added} added`);
if (overwrittenCount > 0) parts.push(`${overwrittenCount} overwritten`);
if (csvUpdates > 0) parts.push(`${csvUpdates} updated from CSV`); if (csvUpdates > 0) parts.push(`${csvUpdates} updated from CSV`);
if (dupes) parts.push(`${dupes} duplicate${dupes > 1 ? 's' : ''}`); if (dupes) parts.push(`${dupes} duplicate${dupes > 1 ? 's' : ''}`);
if (errors) parts.push(`${errors} failed`); if (errors) parts.push(`${errors} failed`);
if (parts.length === 0) parts.push('nothing to add'); if (parts.length === 0) parts.push('nothing to add');
fileStatus.textContent = parts.join(', '); fileStatus.textContent = parts.join(', ');
const anyGood = added > 0 || csvUpdates > 0; const anyGood = added > 0 || overwrittenCount > 0 || csvUpdates > 0;
fileStatus.style.color = anyGood ? '#4ade80' : '#a1a1aa'; fileStatus.style.color = anyGood ? '#4ade80' : '#a1a1aa';
if (anyGood) setTimeout(() => window.location.reload(), 1200); if (anyGood) setTimeout(() => window.location.reload(), 1200);
else drop.style.pointerEvents = ''; else drop.style.pointerEvents = '';