diff --git a/.gitignore b/.gitignore index fa7a52d..f157861 100644 --- a/.gitignore +++ b/.gitignore @@ -39,6 +39,11 @@ todo.md site/public/data site/public/*.whl .claude/settings.local.json +dns.md +ngix_bincio.md +publish.sh +docs/squash-for-github.md +CLAUDE.md # Capacitor native projects # Commit these if you want to track native code changes; diff --git a/CLAUDE.md b/CLAUDE.md deleted file mode 100644 index ff8c7fa..0000000 --- a/CLAUDE.md +++ /dev/null @@ -1,338 +0,0 @@ -# BincioActivity — Context for Claude - -## What this project is - -BincioActivity is a federated, open-source, self-hosted activity stats platform -(think personal Strava). Two-stage pipeline: - -1. **`bincio extract`** (Python): GPX/FIT/TCX → BAS JSON data store -2. **`bincio render`** (Astro/Node): BAS data store → static website - -The BAS (BincioActivity Schema) JSON files are the federation protocol. -Anyone can publish their data as BAS JSON and others can include it. - -## Key design decisions - -- **Unified data layout** — single-user and multi-user share the same structure: activities always live in `{data-root}/{handle}/`. The only difference is the presence of `instance.db` (auth). No mode switching, no migration. -- **No database, no server** — everything is static files; multi-user VPS mode adds SQLite auth only -- **Python with uv** for the extract stage -- **Astro + Svelte + Tailwind + MapLibre GL + Observable Plot** for the site -- **Haversine** (not geopy) for distance calculations (10x faster) -- **Worker initializer pattern** for ProcessPoolExecutor — large shared data - (strava_lookup dict, known_hashes frozenset) is sent once per worker via - `initializer=`, not once per task -- **BAS activity IDs** always use UTC with Z suffix for URL safety -- **TCX files** from Garmin use both `http://` and `https://` namespace URIs — - parser handles both -- **Shard manifest for multi-user** — no activity data duplication; root `index.json` - lists user shard URLs; browser resolves all shards concurrently; same mechanism - handles yearly pagination and remote federation -- **Iterative RDP** implemented inline in `simplify.py` — no `rdp` PyPI package - (not available as a pure-Python wheel for Pyodide) - -## Your data - -- Source: `~/your-activity-data/` - - `activities/` — Strava export (GPX, FIT, TCX, all with .gz variants) - - Any subdirectories with FIT files from Garmin/Karoo devices - - `activities.csv` — Strava metadata (names, descriptions, gear) -- Extracted output: `~/bincio_data/` (or `/tmp/bincio_test/` for testing) - -Configure input paths in `extract_config.yaml`. - -## Project structure - -``` -bincio/ Python package - extract/ - models.py DataPoint, ParsedActivity, LapData - parsers/ GPX, FIT, TCX parsers + factory - sport.py sport name normalisation - metrics.py haversine-based stats computation (single pass) - timeseries.py downsample to 1Hz, build BAS timeseries object - simplify.py RDP track simplification → GeoJSON (iterative, no rdp dep) - dedup.py exact (hash) + near-duplicate detection - strava_csv.py Strava activities.csv importer - writer.py BAS JSON + GeoJSON writer - config.py extract_config.yaml loader - cli.py `bincio extract` CLI - render/ - cli.py `bincio render` CLI (symlinks data, runs astro build/dev) - merge.py sidecar edit overlay (produces _merged/) - edit/ - cli.py `bincio edit` CLI (single-user local only) - server.py FastAPI write API for the edit drawer - serve/ - cli.py `bincio serve` CLI (multi-user VPS) - server.py FastAPI: auth, user mgmt, write API (auth-gated) - db.py SQLite data layer (users, sessions, invites) - init_cmd.py `bincio init` CLI: bootstrap instance.db + admin user -schema/ - bas-v1.schema.json JSON Schema for BAS -SCHEMA.md Human-readable BAS spec -site/ Astro project - src/ - layouts/Base.astro Reads instancePrivate from index.json; injects auth wall - pages/ - index.astro Activity feed (loads index.json client-side) - activity/[id].astro Single activity (SSG, loads detail JSON client-side) - activity/local/ IDB-only activities (converted locally via Pyodide) - stats/index.astro Heatmap + year totals - u/[handle].astro Per-user profile pages (multi-user) - login/index.astro Login form (public page) - register/index.astro Registration with invite code (public page) - invites/index.astro Invite management - convert/index.astro Local file conversion via Pyodide (browser-only) - components/ - ActivityFeed.svelte Card grid, sport filter, pagination - ActivityDetail.svelte Map + stats + charts + photo gallery - ActivityMap.svelte MapLibre GL (gradient track, linked hover dot) - ActivityCharts.svelte Observable Plot (elevation/speed/HR/cadence tabs) - StatsView.svelte Yearly heatmap + totals - EditDrawer.svelte Slide-in edit panel (visible when PUBLIC_EDIT_URL set) - LocalActivityDetail.svelte Detail view for IDB-only (locally converted) activities - lib/ - types.ts BAS TypeScript types - format.ts formatDistance, formatDuration, sportIcon, etc. - localstore.ts IndexedDB store for locally converted activities - dataloader.ts Fetches index.json, resolves shards recursively -``` - -## How to run - -```bash -# Single-user (no login) -cd ~/src/bincio_activity -uv run bincio extract --output /tmp/bincio_test # → /tmp/bincio_test/{handle}/ -uv run bincio dev --data-dir /tmp/bincio_test -# → http://localhost:4321/u/{handle}/ - -# Multi-user (with login) -uv run bincio init --data-dir /tmp/bincio_test --handle dave -uv run bincio extract --output /tmp/bincio_test # → /tmp/bincio_test/dave/ -uv run bincio dev --data-dir /tmp/bincio_test -# → http://localhost:4321 (login required) - -# bincio dev does everything: merges sidecars, writes manifest, -# symlinks public/data, starts bincio serve (if instance.db exists), -# starts astro dev. Ctrl+C stops all. - -# Tests -uv run pytest -``` - -## MapLibre GL + Vite/Astro — known gotchas - -Learnt the hard way during debugging (March 2026): - -- **`maplibregl.workerUrl = ...` is the v3 API and silently no-ops in v4+.** - The v5 API is `maplibregl.setWorkerUrl(url)`, but you don't need it at all in a - normal Vite environment — MapLibre handles the blob worker automatically. - -- **`optimizeDeps: { exclude: ['maplibre-gl'] }` breaks tile loading.** - It prevents Vite from converting MapLibre's UMD bundle to ESM. The UMD bundle - uses AMD `define()` internally; served raw, the tile worker blob fails silently → - black map, no tiles. The correct setting is `include: ['maplibre-gl']`. - -- **`build.target: 'es2022'` (and `optimizeDeps.esbuildOptions.target`) is required.** - MapLibre's dependencies use ES2022 class field syntax. If esbuild downgrades it, - helpers like `__publicField` aren't available inside the serialised worker blob - scope → tile loading fails. This is a known upstream issue (maplibre-gl-js #6680). - -- **Use static imports, not dynamic `await import('maplibre-gl')`, when possible.** - With `client:only="svelte"` in Astro, SSR never runs for the component so there is - no `window is not defined` risk. Static import lets Vite pre-bundle correctly. - -- **Use `client:only="svelte"` (not `client:load`) for the activity detail page.** - `client:load` does SSR + hydration; complex interactive components with MapLibre - can hit hydration mismatch issues. `client:only` mounts fresh on the client only. - -- **MapLibre v5 requires explicit `center` and `zoom` in the Map constructor.** - v4 silently defaulted to `center: [0,0], zoom: 0`. v5 leaves internal projection - state undefined → `Cannot read properties of undefined (reading 'lng')` crashes - on any operation that touches coordinates (markers, resize, render). Always pass - `center` and `zoom` even if you plan to `fitBounds` later. - -- **MapLibre v5 requires `setLngLat()` on markers before `.addTo(map)`.** - v4 tolerated markers without coordinates. v5 calls `Marker._update()` inside - `addTo()`, which needs valid lngLat → same `'lng'` crash. Set a dummy `[0, 0]` - if the real position arrives later (e.g. hover markers). - -## Observable Plot — known gotchas - -- **Curve names are hyphenated, not camelCase.** - Use `"monotone-x"`, not `"monotoneX"`. Plot uses its own curve name registry - (not raw d3 identifiers). Wrong names throw `unknown curve` at runtime. - -The working `astro.config.mjs` Vite section: -```js -vite: { - optimizeDeps: { - include: ['maplibre-gl'], - esbuildOptions: { target: 'es2022' }, - }, - build: { target: 'es2022' }, -}, -``` - -## Activity sidecar edits — design spec - -Users edit activities via **sidecar markdown files** in the data dir. -No database, no server — consistent with the project's static-files-only philosophy. - -### File naming - -``` -~/bincio_data/ - activities/{id}.json ← immutable extract output - edits/{id}.md ← user edits (sidecar) - edits/images/{id}/ ← uploaded photos - _merged/ ← render-time merge output (gitignored-style) -``` - -### Sidecar format - -```markdown ---- -title: "Epic climb up Monte Grappa" -sport: cycling -hide_stats: [cadence] -highlight: true -private: false -gear: "Trek Domane" ---- - -Rode with friends. Legs felt great after the rest week... -``` - -The sidecar `private: true` flag maps to `privacy: "unlisted"` in the merged JSON. -**`unlisted`** means: not shown in the public feed, but the activity detail, GPS track, -and timeseries are all accessible by direct URL (security by obscurity, same model -as the detail JSON itself). Use `no_gps` if the GPS track must not be published. - -### Editing UX: drawer in Astro + `bincio edit` write API - -- `bincio edit --data-dir ~/bincio_data` starts a FastAPI server on port 4041 -- Set `PUBLIC_EDIT_URL=http://localhost:4041` in `site/.env` to enable the edit button -- Clicking Edit on any activity detail page opens a slide-in drawer -- Saving writes the sidecar and triggers `merge_all()` automatically -- `bincio render` always runs `merge_all()` before build/serve and symlinks `public/data` → `_merged/` - -### `PUBLIC_EDIT_URL` as feature flag - -- **Unset** → no Edit button, normal static site -- **Set** → edit drawer enabled; lives in `site/.env` (gitignored) - -## Multi-user VPS architecture - -`bincio serve` is a FastAPI app that owns auth and write ops. nginx proxies `/api/*` to it; static files are served by nginx directly. The Vite dev server replicates this proxy for local testing. - -Key facts: -- Session cookie: `bincio_session`, httpOnly, SameSite=Lax, 30-day max-age -- Rate limiting: 10 login attempts / 15 min / IP (in-memory, resets on restart) -- Invite limits: admins unlimited, regular users 3 each (`_MAX_USER_INVITES` in `db.py`) -- Instance privacy: `instance.private=true` in root `index.json` → `Base.astro` injects a - `fetch('/api/me')` auth wall; `/login/` and `/register/` have `public={true}` to skip it -- Incremental rebuild: `POST /api/activity/{id}` triggers `bincio render --handle {user}` - as a fire-and-forget subprocess (only if `--site-dir` was passed to `bincio serve`) - -### Password reset (no email — out-of-band code) - -There is no email infrastructure. Password resets work via admin-generated one-time codes: - -1. **Admin** opens `/admin/` → clicks **"Reset pwd"** next to the user → a code appears - inline (monospace, click to copy). Valid for **24 hours**, tied to that handle. -2. **Admin** sends the code out-of-band (Signal, Telegram, etc.). -3. **User** goes to `/reset-password/`, enters handle + code + new password → done. - -API: -- `POST /api/admin/users/{handle}/reset-password-code` (admin) → `{code, expires_in_hours: 24}` -- `POST /api/auth/reset-password` (public) → body `{handle, code, password}` - -DB: `reset_codes` table `(code, handle, created_by, created_at, expires_at, used_at)`. -Generating a new code invalidates any prior unused code for the same handle. -Used codes are kept for audit. `change_password()` in `db.py` updates the bcrypt hash. -- Write API in `bincio serve` delegates to `bincio.edit.server._apply_sidecar_edit`; the - Strava sync delegates to `bincio.edit.server.strava_sync` with a temporary data_dir swap - -## Instance settings (stored in `instance.db` `settings` table) - -| Key | Default | Set by | Description | -|-----|---------|--------|-------------| -| `max_users` | — (unlimited) | `bincio init --max-users N` or `bincio serve --max-users N` | Cap on registered users; 0 or absent = unlimited | -| `store_originals` | `true` | `bincio init` (first run only) | Whether uploaded source files and raw Strava API data are kept in `{user_dir}/originals/` | - -`get_setting` / `set_setting` in `db.py` are the read/write accessors. Any new instance-wide flag should use this table rather than a new column. - -## Original file storage - -When a user uploads a FIT/GPX/TCX file the server may keep the source in `{user_dir}/originals/{filename}` rather than always deleting it after extraction. The per-upload `store_original` form field controls the behaviour for a single upload (sent by the UI checkbox). The instance-level `store_originals` setting provides the default that pre-populates the checkbox (read from `GET /api/me` → `store_originals_default`). - -For Strava sync, `store_originals=true` causes `POST /api/strava/sync` to save `{"meta":…,"streams":…}` JSON per activity to `{user_dir}/originals/strava/{activity_id}.json`. - -## Feedback storage - -User feedback submitted via `/feedback/` is stored as flat files under the instance data root (NOT inside a user's own data dir): - -``` -{data_root}/ - _feedback/ - {handle}.json ← append-only list of submissions for that user - {handle}/ - {timestamp}_{token}_{filename} ← attached images -``` - -Each entry in `{handle}.json`: -```json -{ "id": "1712345678_ab12cd34", "handle": "brut", "submitted_at": "...", "text": "...", "images": ["..."] } -``` - -To read feedback on the VPS: -```bash -cat /var/bincio/data/_feedback/brut.json | python3 -m json.tool -ls /var/bincio/data/_feedback/brut/ # attached images -``` - -There is no admin UI for feedback — it is intentionally read via SSH/shell only. - -## About pages - -Static public pages at `/about/` (EN), `/about/it/` (IT), `/about/es/` (ES), `/about/ca/` (CA). All use `public={true}` to bypass the auth wall. Each page: -- Shows a Ko-fi donation button at the top. -- Fetches `GET /api/stats` on load and renders a **community/invitation tree** (member count, each user's display name, membership duration, and who invited them). Hidden silently in single-user mode. -- Contains project description, data storage explanation, early-software caveat, and liability disclaimer. - -## Known issues / next steps - -- `bincio render --watch` mode not yet implemented as a standalone command, but `bincio dev` now watches the data directory via `watchfiles` (bundled with uvicorn) and re-runs `merge_all` automatically when sidecars or activity files change -- Activity IDs in older test data may use `+0000` format (pre-fix); re-run extract to get `Z` format -- Some activities appear with both untitled and titled IDs (near-dedup timing race) -- Remote federation (remote shard URLs in root manifest) is parsed but not yet displayed with attribution in the UI -- The `site/.env` file is gitignored — copy from `site/.env.example` - -## What "good" looks like (not yet done) - -- [ ] Friends/federation pages in site (remote shard attribution) -- [ ] Personal records page -- [ ] Activity search / full-text filter in feed -- [ ] GitHub Actions template for auto-publish -- [ ] Karoo/Garmin Connect importers beyond Strava -- [ ] `bincio render --watch` incremental rebuild on sidecar/data changes -- [ ] Highlight badge in activity feed cards -- [x] Per-instance user limit (`max_users` setting, enforced at registration) -- [x] Original file storage option (per-upload checkbox + `store_originals` instance setting) -- [x] About page — multilingual (EN/IT/ES/CA), Ko-fi button, community invitation tree -- [x] `GET /api/stats` — public endpoint with member count and invitation tree -- [x] `bincio.render.merge` — sidecar parser, `_merged/` output, private filter, highlight sort -- [x] `bincio edit` FastAPI write API (GET/POST activity, image upload/delete, triggers merge) -- [x] `EditDrawer.svelte` — slide-in edit UI in the Astro site -- [x] `PUBLIC_EDIT_URL` feature flag -- [x] Markdown rendering in activity description with image path rewriting -- [x] Photo gallery with lightbox on activity detail page -- [x] `bincio serve` — multi-user VPS server (auth, invites, write API) -- [x] `bincio init` — instance bootstrap (SQLite, admin user, root manifest) -- [x] Login, register, invites pages -- [x] Per-user profile pages (`/u/{handle}/`) -- [x] Instance privacy (auth wall, private-by-default) -- [x] Shard-based combined feed (no duplication, concurrent resolution) -- [x] Local file conversion via Pyodide (`/convert/` page, IDB storage) diff --git a/docs/squash-for-github.md b/docs/squash-for-github.md deleted file mode 100644 index 53a6460..0000000 --- a/docs/squash-for-github.md +++ /dev/null @@ -1,78 +0,0 @@ -# squash-for-github branch strategy - -`squash-for-github` is a curated public-facing branch. It has its own orphan -history (unrelated to `main`) and grows by appending one large squash commit -each time you want to publish a batch of work. - -## When to use - -Whenever `main` has accumulated enough work worth publishing — typically after a -meaningful feature set or before tagging a release. - -## How it works - -`squash-for-github` and `main` have completely unrelated histories (different -root commits). Because of this, `git merge --squash` won't work. Instead, use -`git commit-tree` to create a new commit that carries **main's file tree** but -is **parented to the current squash-for-github tip**. - -## Steps - -1. **Collect commit messages** to write the summary: - - ```bash - git log --oneline main ^squash-for-github - ``` - -2. **Switch to the branch:** - - ```bash - git checkout squash-for-github - ``` - -3. **Create the squash commit** (replace the message with your summary): - - ```bash - NEW=$(git commit-tree main^{tree} -p HEAD -m "feat: your summary here") - git reset --hard $NEW - ``` - - Or as a one-liner with a heredoc for a multi-line message: - - ```bash - git reset --hard $(git commit-tree main^{tree} -p HEAD -m "$(cat <<'EOF' - feat: short title - - - bullet one - - bullet two - EOF - )") - ``` - -4. **Verify:** - - ```bash - git log --oneline squash-for-github | head -5 - ``` - -5. **Push** when ready: - - ```bash - git push origin squash-for-github - # or force-push if you've rewritten history on the remote: - git push --force origin squash-for-github - ``` - -6. **Return to main:** - - ```bash - git checkout main - ``` - -## Why not `git merge --squash`? - -The two branches share no common ancestor, so git refuses with -`fatal: refusing to merge unrelated histories`. `git commit-tree` bypasses this -by directly constructing the commit object: it takes the tree (file snapshot) -from `main`, sets the parent to the current `squash-for-github` tip, and -attaches your custom message — no merge machinery needed. diff --git a/publish.sh b/publish.sh deleted file mode 100755 index 814b3d9..0000000 --- a/publish.sh +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -REMOTE="github-public" -BRANCH="main" -LOCAL_DIR="$(cd "$(dirname "$0")" && pwd)" -PUBLISH_DIR="${LOCAL_DIR}/publish" -MANIFEST="${PUBLISH_DIR}/manifest" -DRY_RUN=false - -for arg in "$@"; do - case "$arg" in - --dry-run) DRY_RUN=true ;; - *) echo "Unknown argument: $arg"; exit 1 ;; - esac -done - -if ! git -C "$LOCAL_DIR" remote get-url "$REMOTE" &>/dev/null; then - echo "ERROR: remote '${REMOTE}' not found." - echo " git remote add ${REMOTE} https://github.com/brutsalvadi/bincio-activity.git" - exit 1 -fi -if [[ -n "$(git -C "$LOCAL_DIR" status --porcelain --untracked-files=no)" ]]; then - echo "ERROR: uncommitted changes. Commit or stash first." - exit 1 -fi -if [[ ! -f "$MANIFEST" ]]; then - echo "ERROR: manifest not found at ${MANIFEST}" - exit 1 -fi - -# Collect file list AND stage into temp dir before touching git state. -# Both must happen here — `git rm -rf .` removes the manifest itself, -# so it can't be re-read during the orphan branch step. -STAGING="$(mktemp -d)" -trap 'rm -rf "$STAGING"' EXIT -FILES=() - -while IFS= read -r relpath || [[ -n "$relpath" ]]; do - [[ -z "$relpath" || "$relpath" == \#* ]] && continue - override="${PUBLISH_DIR}/${relpath}" - original="${LOCAL_DIR}/${relpath}" - dest="${STAGING}/${relpath}" - mkdir -p "$(dirname "$dest")" - if [[ -f "$override" ]]; then - cp "$override" "$dest" - elif [[ -f "$original" ]]; then - cp "$original" "$dest" - else - echo "ERROR: '${relpath}' in manifest but not found (no override, no original)" - exit 1 - fi - FILES+=("$relpath") -done < "$MANIFEST" - -echo "Files to be published:" -printf ' %s\n' "${FILES[@]}" - -if $DRY_RUN; then - echo "" - echo "Dry run complete. No changes made." - exit 0 -fi - -# Create orphan branch, wipe working tree, restore only manifest files -git -C "$LOCAL_DIR" checkout --orphan _public_tmp -git -C "$LOCAL_DIR" rm -rf . --quiet -cp -r "${STAGING}/." "${LOCAL_DIR}/" - -TIMESTAMP="$(date -u +%Y-%m-%dT%H:%M:%SZ)" -# Add only manifest files — never picks up untracked files outside the manifest -for relpath in "${FILES[@]}"; do - git -C "$LOCAL_DIR" add -- "$relpath" -done -git -C "$LOCAL_DIR" commit -m "Published ${TIMESTAMP}" -git -C "$LOCAL_DIR" push --force "$REMOTE" "HEAD:${BRANCH}" - -git -C "$LOCAL_DIR" checkout main -git -C "$LOCAL_DIR" branch -D _public_tmp - -echo "" -echo "Done: $(git -C "$LOCAL_DIR" remote get-url "$REMOTE")"