upload: add overwrite option to replace existing activities
When 'Overwrite existing activities' is checked, duplicate activities are
re-extracted and replaced instead of silently skipped:
- Deletes {id}.json, .geojson, .timeseries.json from activities/ and _merged/
- Removes the stale index summary and dedup cache entry
- Ingests the new file fresh via ingest_parsed
- Reports 'overwritten' (↺) status in the SSE stream vs 'imported' (↓)
- done event includes 'overwritten' count; UI shows it alongside 'added'
This commit is contained in:
+38
-4
@@ -1129,6 +1129,7 @@ def _file_suffix(name: str) -> str:
|
|||||||
async def upload_activity(
|
async def upload_activity(
|
||||||
files: list[UploadFile] = File(...),
|
files: list[UploadFile] = File(...),
|
||||||
store_original: bool = Form(False),
|
store_original: bool = Form(False),
|
||||||
|
overwrite: bool = Form(False),
|
||||||
bincio_session: Optional[str] = Cookie(default=None),
|
bincio_session: Optional[str] = Cookie(default=None),
|
||||||
) -> StreamingResponse:
|
) -> StreamingResponse:
|
||||||
"""Accept FIT/GPX/TCX files and/or activities.csv; stream SSE progress while processing.
|
"""Accept FIT/GPX/TCX files and/or activities.csv; stream SSE progress while processing.
|
||||||
@@ -1138,9 +1139,9 @@ async def upload_activity(
|
|||||||
- Retroactively update sidecars for existing activities (matched by strava_id)
|
- Retroactively update sidecars for existing activities (matched by strava_id)
|
||||||
|
|
||||||
SSE events:
|
SSE events:
|
||||||
{"type": "progress", "n": N, "total": T, "name": "...", "status": "imported"|"duplicate"|"error"}
|
{"type": "progress", "n": N, "total": T, "name": "...", "status": "imported"|"overwritten"|"duplicate"|"error"}
|
||||||
{"type": "csv", "updates": N} -- only when CSV was included
|
{"type": "csv", "updates": N} -- only when CSV was included
|
||||||
{"type": "done", "added": N, "csv_updates": N, "duplicates": N, "errors": N}
|
{"type": "done", "added": N, "csv_updates": N, "duplicates": N, "overwritten": N, "errors": N}
|
||||||
"""
|
"""
|
||||||
from bincio.extract.ingest import ingest_parsed
|
from bincio.extract.ingest import ingest_parsed
|
||||||
from bincio.extract.parsers.factory import parse_file
|
from bincio.extract.parsers.factory import parse_file
|
||||||
@@ -1182,6 +1183,7 @@ async def upload_activity(
|
|||||||
|
|
||||||
def event_stream():
|
def event_stream():
|
||||||
added = 0
|
added = 0
|
||||||
|
overwritten = 0
|
||||||
duplicates = 0
|
duplicates = 0
|
||||||
errors = 0
|
errors = 0
|
||||||
any_added = False
|
any_added = False
|
||||||
@@ -1209,19 +1211,51 @@ async def upload_activity(
|
|||||||
if metadata is not None:
|
if metadata is not None:
|
||||||
metadata.enrich(name, activity)
|
metadata.enrich(name, activity)
|
||||||
activity_id = make_activity_id(activity)
|
activity_id = make_activity_id(activity)
|
||||||
|
was_overwrite = False
|
||||||
if (dd / "activities" / f"{activity_id}.json").exists():
|
if (dd / "activities" / f"{activity_id}.json").exists():
|
||||||
|
if not overwrite:
|
||||||
duplicates += 1
|
duplicates += 1
|
||||||
yield f"data: {json.dumps({'type': 'progress', 'n': n, 'total': total_files, 'name': name, 'status': 'duplicate'})}\n\n"
|
yield f"data: {json.dumps({'type': 'progress', 'n': n, 'total': total_files, 'name': name, 'status': 'duplicate'})}\n\n"
|
||||||
continue
|
continue
|
||||||
|
# Overwrite: delete existing files before re-ingesting.
|
||||||
|
for ext in (".json", ".geojson", ".timeseries.json"):
|
||||||
|
(dd / "activities" / f"{activity_id}{ext}").unlink(missing_ok=True)
|
||||||
|
# Remove stale summary from index so ingest_parsed writes a clean one
|
||||||
|
index_path = dd / "index.json"
|
||||||
|
if index_path.exists():
|
||||||
|
idx = json.loads(index_path.read_text(encoding="utf-8"))
|
||||||
|
idx["activities"] = [a for a in idx.get("activities", []) if a.get("id") != activity_id]
|
||||||
|
index_path.write_text(json.dumps(idx, indent=2, ensure_ascii=False))
|
||||||
|
# Remove from dedup hash cache so the new file isn't blocked
|
||||||
|
cache_path = dd / ".bincio_cache.json"
|
||||||
|
if cache_path.exists():
|
||||||
|
try:
|
||||||
|
cache = json.loads(cache_path.read_text(encoding="utf-8"))
|
||||||
|
cache.pop(activity_id, None)
|
||||||
|
cache_path.write_text(json.dumps(cache, ensure_ascii=False))
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
# Remove merged copies (merge_all will regenerate them after ingest)
|
||||||
|
merged_acts = dd / "_merged" / "activities"
|
||||||
|
if merged_acts.exists():
|
||||||
|
for ext in (".json", ".geojson", ".timeseries.json"):
|
||||||
|
p = merged_acts / f"{activity_id}{ext}"
|
||||||
|
if p.exists() or p.is_symlink():
|
||||||
|
p.unlink(missing_ok=True)
|
||||||
|
was_overwrite = True
|
||||||
ingest_parsed(activity, dd, privacy="public")
|
ingest_parsed(activity, dd, privacy="public")
|
||||||
if store_original:
|
if store_original:
|
||||||
originals_dir = dd / "originals"
|
originals_dir = dd / "originals"
|
||||||
originals_dir.mkdir(exist_ok=True)
|
originals_dir.mkdir(exist_ok=True)
|
||||||
staged.rename(originals_dir / name)
|
staged.rename(originals_dir / name)
|
||||||
kept = True
|
kept = True
|
||||||
|
if was_overwrite:
|
||||||
|
overwritten += 1
|
||||||
|
else:
|
||||||
added += 1
|
added += 1
|
||||||
any_added = True
|
any_added = True
|
||||||
yield f"data: {json.dumps({'type': 'progress', 'n': n, 'total': total_files, 'name': name, 'status': 'imported'})}\n\n"
|
status = 'overwritten' if was_overwrite else 'imported'
|
||||||
|
yield f"data: {json.dumps({'type': 'progress', 'n': n, 'total': total_files, 'name': name, 'status': status})}\n\n"
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
errors += 1
|
errors += 1
|
||||||
log.error("upload[%s]: failed to process %s: %s", user.handle, name, exc, exc_info=True)
|
log.error("upload[%s]: failed to process %s: %s", user.handle, name, exc, exc_info=True)
|
||||||
@@ -1243,7 +1277,7 @@ async def upload_activity(
|
|||||||
if any_added:
|
if any_added:
|
||||||
_trigger_rebuild(user.handle)
|
_trigger_rebuild(user.handle)
|
||||||
|
|
||||||
yield f"data: {json.dumps({'type': 'done', 'added': added, 'csv_updates': csv_updates, 'duplicates': duplicates, 'errors': errors})}\n\n"
|
yield f"data: {json.dumps({'type': 'done', 'added': added, 'overwritten': overwritten, 'csv_updates': csv_updates, 'duplicates': duplicates, 'errors': errors})}\n\n"
|
||||||
|
|
||||||
if job_id:
|
if job_id:
|
||||||
_job_finish(job_id)
|
_job_finish(job_id)
|
||||||
|
|||||||
@@ -317,6 +317,17 @@ try {
|
|||||||
<span class="text-zinc-600 block mt-0.5">Lets you reprocess if the format changes. See the <a href={`${baseUrl}about/`} class="underline hover:text-zinc-400">About page</a> for details.</span>
|
<span class="text-zinc-600 block mt-0.5">Lets you reprocess if the format changes. See the <a href={`${baseUrl}about/`} class="underline hover:text-zinc-400">About page</a> for details.</span>
|
||||||
</span>
|
</span>
|
||||||
</label>
|
</label>
|
||||||
|
<label class="flex items-start gap-2 mt-2 cursor-pointer group">
|
||||||
|
<input
|
||||||
|
id="upload-overwrite"
|
||||||
|
type="checkbox"
|
||||||
|
class="mt-0.5 accent-amber-500 shrink-0"
|
||||||
|
/>
|
||||||
|
<span class="text-xs text-zinc-500 group-hover:text-zinc-300 transition-colors leading-snug">
|
||||||
|
Overwrite existing activities
|
||||||
|
<span class="text-zinc-600 block mt-0.5">Re-extract and replace any duplicate found on the server. Use to fix a corrupted or mis-parsed activity.</span>
|
||||||
|
</span>
|
||||||
|
</label>
|
||||||
<p id="upload-status" class="mt-3 text-xs text-center" style="color: var(--text-5); min-height: 1.25rem"></p>
|
<p id="upload-status" class="mt-3 text-xs text-center" style="color: var(--text-5); min-height: 1.25rem"></p>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
@@ -554,6 +565,7 @@ try {
|
|||||||
const input = document.getElementById('upload-input');
|
const input = document.getElementById('upload-input');
|
||||||
const label = document.getElementById('upload-label');
|
const label = document.getElementById('upload-label');
|
||||||
const keepOriginalChk = document.getElementById('upload-keep-original');
|
const keepOriginalChk = document.getElementById('upload-keep-original');
|
||||||
|
const overwriteChk = document.getElementById('upload-overwrite');
|
||||||
const fileStatus = document.getElementById('upload-status');
|
const fileStatus = document.getElementById('upload-status');
|
||||||
const stravaStatus = document.getElementById('strava-status');
|
const stravaStatus = document.getElementById('strava-status');
|
||||||
const stravaConnect = document.getElementById('strava-connect-area');
|
const stravaConnect = document.getElementById('strava-connect-area');
|
||||||
@@ -629,6 +641,7 @@ try {
|
|||||||
const fd = new FormData();
|
const fd = new FormData();
|
||||||
for (const f of files) fd.append('files', f);
|
for (const f of files) fd.append('files', f);
|
||||||
fd.append('store_original', keepOriginalChk?.checked ? 'true' : 'false');
|
fd.append('store_original', keepOriginalChk?.checked ? 'true' : 'false');
|
||||||
|
fd.append('overwrite', overwriteChk?.checked ? 'true' : 'false');
|
||||||
|
|
||||||
const xhr = new XMLHttpRequest();
|
const xhr = new XMLHttpRequest();
|
||||||
xhr.open('POST', `${editUrl}/api/upload`);
|
xhr.open('POST', `${editUrl}/api/upload`);
|
||||||
@@ -636,7 +649,7 @@ try {
|
|||||||
xhr.setRequestHeader('Accept', 'text/event-stream');
|
xhr.setRequestHeader('Accept', 'text/event-stream');
|
||||||
|
|
||||||
let buf = '';
|
let buf = '';
|
||||||
let added = 0, dupes = 0, errors = 0, csvUpdates = 0;
|
let added = 0, overwrittenCount = 0, dupes = 0, errors = 0, csvUpdates = 0;
|
||||||
|
|
||||||
xhr.onprogress = () => {
|
xhr.onprogress = () => {
|
||||||
const newText = xhr.responseText.slice(buf.length);
|
const newText = xhr.responseText.slice(buf.length);
|
||||||
@@ -647,23 +660,25 @@ try {
|
|||||||
const ev = JSON.parse(line.slice(6));
|
const ev = JSON.parse(line.slice(6));
|
||||||
if (ev.type === 'progress') {
|
if (ev.type === 'progress') {
|
||||||
const pct = Math.round((ev.n / ev.total) * 100);
|
const pct = Math.round((ev.n / ev.total) * 100);
|
||||||
const icon = ev.status === 'imported' ? '↓' : ev.status === 'duplicate' ? '·' : '✗';
|
const icon = ev.status === 'imported' ? '↓' : ev.status === 'overwritten' ? '↺' : ev.status === 'duplicate' ? '·' : '✗';
|
||||||
fileStatus.textContent = `${icon} ${ev.n}/${ev.total} (${pct}%) — ${ev.name}`;
|
fileStatus.textContent = `${icon} ${ev.n}/${ev.total} (${pct}%) — ${ev.name}`;
|
||||||
if (ev.status === 'imported') added++;
|
if (ev.status === 'imported') added++;
|
||||||
|
else if (ev.status === 'overwritten') overwrittenCount++;
|
||||||
else if (ev.status === 'duplicate') dupes++;
|
else if (ev.status === 'duplicate') dupes++;
|
||||||
else errors++;
|
else errors++;
|
||||||
} else if (ev.type === 'csv') {
|
} else if (ev.type === 'csv') {
|
||||||
csvUpdates = ev.updates;
|
csvUpdates = ev.updates;
|
||||||
} else if (ev.type === 'done') {
|
} else if (ev.type === 'done') {
|
||||||
added = ev.added; dupes = ev.duplicates; errors = ev.errors; csvUpdates = ev.csv_updates;
|
added = ev.added; overwrittenCount = ev.overwritten ?? 0; dupes = ev.duplicates; errors = ev.errors; csvUpdates = ev.csv_updates;
|
||||||
const parts = [];
|
const parts = [];
|
||||||
if (added > 0) parts.push(`${added} added`);
|
if (added > 0) parts.push(`${added} added`);
|
||||||
|
if (overwrittenCount > 0) parts.push(`${overwrittenCount} overwritten`);
|
||||||
if (csvUpdates > 0) parts.push(`${csvUpdates} updated from CSV`);
|
if (csvUpdates > 0) parts.push(`${csvUpdates} updated from CSV`);
|
||||||
if (dupes) parts.push(`${dupes} duplicate${dupes > 1 ? 's' : ''}`);
|
if (dupes) parts.push(`${dupes} duplicate${dupes > 1 ? 's' : ''}`);
|
||||||
if (errors) parts.push(`${errors} failed`);
|
if (errors) parts.push(`${errors} failed`);
|
||||||
if (parts.length === 0) parts.push('nothing to add');
|
if (parts.length === 0) parts.push('nothing to add');
|
||||||
fileStatus.textContent = parts.join(', ');
|
fileStatus.textContent = parts.join(', ');
|
||||||
const anyGood = added > 0 || csvUpdates > 0;
|
const anyGood = added > 0 || overwrittenCount > 0 || csvUpdates > 0;
|
||||||
fileStatus.style.color = anyGood ? '#4ade80' : '#a1a1aa';
|
fileStatus.style.color = anyGood ? '#4ade80' : '#a1a1aa';
|
||||||
if (anyGood) setTimeout(() => window.location.reload(), 1200);
|
if (anyGood) setTimeout(() => window.location.reload(), 1200);
|
||||||
else drop.style.pointerEvents = '';
|
else drop.style.pointerEvents = '';
|
||||||
|
|||||||
Reference in New Issue
Block a user