feat(mobile): batch import + Karoo auto-import from watch folder

- Import tab now accepts multiple files at once (DocumentPicker multiple:true),
  processes them sequentially through Pyodide, and shows a summary with per-file
  errors on completion.

- DB migration v2 adds source_path column (original filesystem path before copy)
  and an index on it, enabling O(1) deduplication for watch-folder imports.

- On Android, if auto_import_path is set, the Import tab scans the directory on
  mount and on AppState 'active' (app foreground), then automatically imports any
  FIT files not yet in the DB. Designed for Karoo: finish a ride, open the app,
  new files import without any manual steps.

- insertActivity now accepts optional source_path; both importBasJson and
  importNativeFile pass it through (null for files picked via DocumentPicker,
  real path for watch-folder files).
This commit is contained in:
Davide Scaini
2026-04-25 21:25:54 +02:00
parent a796bf8cae
commit 2f53fbc359
3 changed files with 204 additions and 58 deletions
+175 -55
View File
@@ -1,62 +1,125 @@
import * as DocumentPicker from 'expo-document-picker'; import * as DocumentPicker from 'expo-document-picker';
import * as FileSystem from 'expo-file-system/legacy'; import * as FileSystem from 'expo-file-system/legacy';
import { useSQLiteContext } from 'expo-sqlite'; import { useSQLiteContext } from 'expo-sqlite';
import { useState } from 'react'; import { useEffect, useRef, useState } from 'react';
import { Pressable, ScrollView, StyleSheet, Text, View } from 'react-native'; import { AppState, PermissionsAndroid, Platform, Pressable, ScrollView, StyleSheet, Text, View } from 'react-native';
import { insertActivity } from '@/db/queries'; import { insertActivity, isSourcePathImported, getSetting } from '@/db/queries';
import { PyodideWebView } from '@/extraction/PyodideWebView'; import { PyodideWebView } from '@/extraction/PyodideWebView';
import { extractFile } from '@/extraction/extractActivity'; import { extractFile } from '@/extraction/extractActivity';
import { useTheme } from '@/ThemeContext'; import { useTheme } from '@/ThemeContext';
const FIT_EXTENSIONS = ['.fit', '.fit.gz']; const FIT_EXTENSIONS = ['.fit', '.fit.gz'];
const OTHER_EXTENSIONS = ['.gpx', '.tcx', '.gpx.gz', '.tcx.gz']; const OTHER_EXTENSIONS = ['.gpx', '.tcx', '.gpx.gz', '.tcx.gz'];
const ALL_NATIVE_EXTENSIONS = [...FIT_EXTENSIONS, ...OTHER_EXTENSIONS]; const ALL_NATIVE_EXTENSIONS = [...FIT_EXTENSIONS, ...OTHER_EXTENSIONS];
type ImportState = type ImportState =
| { status: 'idle' } | { status: 'idle' }
| { status: 'loading'; msg: string } | { status: 'loading'; msg: string; current: number; total: number }
| { status: 'done'; title: string; id: string } | { status: 'done'; count: number; errors: Array<{ name: string; message: string }> }
| { status: 'error'; message: string }; | { status: 'error'; message: string };
export default function ImportScreen() { export default function ImportScreen() {
const db = useSQLiteContext(); const db = useSQLiteContext();
const theme = useTheme(); const theme = useTheme();
const [state, setState] = useState<ImportState>({ status: 'idle' }); const [state, setState] = useState<ImportState>({ status: 'idle' });
const isImporting = useRef(false);
async function pickFile() { // Auto-scan watch folder on mount and when app comes to foreground.
setState({ status: 'loading', msg: 'Picking file…' }); useEffect(() => {
if (Platform.OS !== 'android') return;
runAutoScan();
const sub = AppState.addEventListener('change', (next) => {
if (next === 'active') runAutoScan();
});
return () => sub.remove();
// eslint-disable-next-line react-hooks/exhaustive-deps
}, []);
async function runAutoScan() {
if (isImporting.current) return;
const watchPath = await getSetting(db, 'auto_import_path');
if (!watchPath) return;
const newFiles = await discoverNewFiles(db, watchPath);
if (newFiles.length === 0) return;
isImporting.current = true;
try {
await processBatch(newFiles.map(f => ({ uri: `file://${f}`, name: f.split('/').pop() ?? f, sourcePath: f })));
} finally {
isImporting.current = false;
}
}
async function pickFiles() {
if (isImporting.current) return;
setState({ status: 'loading', msg: 'Picking files…', current: 0, total: 0 });
try { try {
const result = await DocumentPicker.getDocumentAsync({ const result = await DocumentPicker.getDocumentAsync({
type: ['*/*'], type: ['*/*'],
copyToCacheDirectory: true, copyToCacheDirectory: true,
multiple: true,
}); });
if (result.canceled || !result.assets?.[0]) { if (result.canceled || !result.assets?.length) {
setState({ status: 'idle' }); setState({ status: 'idle' });
return; return;
} }
isImporting.current = true;
const asset = result.assets[0]; try {
const name = asset.name ?? ''; await processBatch(result.assets.map(a => ({ uri: a.uri, name: a.name ?? '', sourcePath: null })));
const uri = asset.uri; } finally {
const lower = name.toLowerCase(); isImporting.current = false;
if (lower.endsWith('.json')) {
await importBasJson(uri, name, db);
} else if (ALL_NATIVE_EXTENSIONS.some(ext => lower.endsWith(ext))) {
await importNativeFile(uri, name, db);
} else {
setState({ status: 'error', message: `Unsupported file type: ${name}` });
} }
} catch (e: unknown) { } catch (e: unknown) {
const msg = e instanceof Error ? e.message : String(e); const msg = e instanceof Error ? e.message : String(e);
setState({ status: 'error', message: msg }); setState({ status: 'error', message: msg });
isImporting.current = false;
} }
} }
async function processBatch(files: Array<{ uri: string; name: string; sourcePath: string | null }>) {
const total = files.length;
const errors: Array<{ name: string; message: string }> = [];
let count = 0;
for (let i = 0; i < files.length; i++) {
const { uri, name, sourcePath } = files[i];
const lower = name.toLowerCase();
setState({ status: 'loading', msg: `Processing ${name}`, current: i + 1, total });
try {
if (lower.endsWith('.json')) {
await importBasJson(uri, name, sourcePath, (msg) =>
setState({ status: 'loading', msg, current: i + 1, total }),
);
} else if (ALL_NATIVE_EXTENSIONS.some(ext => lower.endsWith(ext))) {
await importNativeFile(uri, name, sourcePath, (msg) =>
setState({ status: 'loading', msg, current: i + 1, total }),
);
} else {
errors.push({ name, message: 'Unsupported file type' });
continue;
}
count++;
} catch (e: unknown) {
errors.push({ name, message: e instanceof Error ? e.message : String(e) });
}
}
setState({ status: 'done', count, errors });
}
// ── BAS JSON import (no extraction needed) ────────────────────────────────── // ── BAS JSON import (no extraction needed) ──────────────────────────────────
async function importBasJson(uri: string, name: string, dbCtx: typeof db) { async function importBasJson(
setState({ status: 'loading', msg: 'Importing…' }); uri: string,
_name: string,
sourcePath: string | null,
onStatus: (msg: string) => void,
) {
onStatus('Importing…');
const text = await FileSystem.readAsStringAsync(uri); const text = await FileSystem.readAsStringAsync(uri);
const detail = JSON.parse(text); const detail = JSON.parse(text);
@@ -70,23 +133,27 @@ export default function ImportScreen() {
const dest = `${origDir}${detail.id}.json`; const dest = `${origDir}${detail.id}.json`;
await FileSystem.copyAsync({ from: uri, to: dest }); await FileSystem.copyAsync({ from: uri, to: dest });
await insertActivity(dbCtx, { await insertActivity(db, {
id: detail.id, id: detail.id,
source_hash: hash, source_hash: hash,
detail_json: text, detail_json: text,
timeseries_json: null, timeseries_json: null,
geojson: null, geojson: null,
original_path: dest, original_path: dest,
source_path: sourcePath,
origin: 'local', origin: 'local',
}); });
setState({ status: 'done', title: detail.title ?? detail.id, id: detail.id });
} }
// ── FIT / GPX / TCX import via Pyodide extraction ────────────────────────── // ── FIT / GPX / TCX import via Pyodide extraction ──────────────────────────
async function importNativeFile(uri: string, name: string, dbCtx: typeof db) { async function importNativeFile(
setState({ status: 'loading', msg: 'Reading file…' }); uri: string,
name: string,
sourcePath: string | null,
onStatus: (msg: string) => void,
) {
onStatus('Reading file…');
// Read the original file as base64 so we can (a) pass it to the WebView // Read the original file as base64 so we can (a) pass it to the WebView
// and (b) copy it to permanent storage without a second read. // and (b) copy it to permanent storage without a second read.
@@ -97,19 +164,13 @@ export default function ImportScreen() {
// Fetch the bincio wheel here (React Native networking), not inside the // Fetch the bincio wheel here (React Native networking), not inside the
// WebView. WKWebView blocks HTTP requests via ATS; RN native networking // WebView. WKWebView blocks HTTP requests via ATS; RN native networking
// allows local-network HTTP (NSAllowsLocalNetworking=true in Info.plist). // allows local-network HTTP (NSAllowsLocalNetworking=true in Info.plist).
const instanceUrl = await getInstanceUrl(dbCtx); const instanceUrl = await getInstanceUrl(db);
setState({ status: 'loading', msg: 'Fetching Bincio engine…' }); onStatus('Fetching Bincio engine…');
const { base64: wheelBase64, filename: wheelFilename } = await fetchWheelBase64(instanceUrl); const { base64: wheelBase64, filename: wheelFilename } = await fetchWheelBase64(instanceUrl);
const result = await extractFile( const result = await extractFile(name, base64, wheelBase64, wheelFilename, onStatus);
name,
base64,
wheelBase64,
wheelFilename,
(msg) => setState({ status: 'loading', msg }),
);
setState({ status: 'loading', msg: 'Saving…' }); onStatus('Saving…');
// Copy original file to permanent storage (keeps original bytes for future re-extraction) // Copy original file to permanent storage (keeps original bytes for future re-extraction)
const ext = name.includes('.') ? name.slice(name.lastIndexOf('.')) : ''; const ext = name.includes('.') ? name.slice(name.lastIndexOf('.')) : '';
@@ -118,18 +179,16 @@ export default function ImportScreen() {
const dest = `${origDir}${result.id}${ext}`; const dest = `${origDir}${result.id}${ext}`;
await FileSystem.copyAsync({ from: uri, to: dest }); await FileSystem.copyAsync({ from: uri, to: dest });
await insertActivity(dbCtx, { await insertActivity(db, {
id: result.id, id: result.id,
source_hash: result.sourceHash, source_hash: result.sourceHash,
detail_json: JSON.stringify(result.detail), detail_json: JSON.stringify(result.detail),
timeseries_json: result.timeseries ? JSON.stringify(result.timeseries) : null, timeseries_json: result.timeseries ? JSON.stringify(result.timeseries) : null,
geojson: result.geojson ? JSON.stringify(result.geojson) : null, geojson: result.geojson ? JSON.stringify(result.geojson) : null,
original_path: dest, original_path: dest,
source_path: sourcePath,
origin: 'local', origin: 'local',
}); });
const d = result.detail as Record<string, unknown>;
setState({ status: 'done', title: (d.title as string) ?? result.id, id: result.id });
} }
return ( return (
@@ -144,21 +203,27 @@ export default function ImportScreen() {
<Text style={styles.header}>Import</Text> <Text style={styles.header}>Import</Text>
<Text style={styles.body}> <Text style={styles.body}>
Import a FIT, GPX, or TCX file extracted on your device, nothing uploaded. Import FIT, GPX, or TCX files extracted on your device, nothing uploaded.
You can also import a pre-extracted BAS <Text style={[styles.code, { color: theme.accent }]}>.json</Text> file directly. You can also import pre-extracted BAS <Text style={[styles.code, { color: theme.accent }]}>.json</Text> files.
Select multiple files at once to import in batch.
</Text> </Text>
<Pressable <Pressable
style={[styles.button, state.status === 'loading' && styles.buttonDisabled]} style={[styles.button, state.status === 'loading' && styles.buttonDisabled]}
onPress={state.status !== 'loading' ? pickFile : undefined} onPress={state.status !== 'loading' ? pickFiles : undefined}
> >
<Text style={styles.buttonText}> <Text style={styles.buttonText}>
{state.status === 'loading' ? 'Working…' : ' Pick file'} {state.status === 'loading' ? 'Working…' : ' Pick files'}
</Text> </Text>
</Pressable> </Pressable>
{state.status === 'loading' && ( {state.status === 'loading' && (
<View style={styles.statusBox}> <View style={styles.statusBox}>
{state.total > 1 && (
<Text style={styles.statusCounter}>
File {state.current} of {state.total}
</Text>
)}
<Text style={[styles.statusMsg, { color: theme.accent }]}>{state.msg}</Text> <Text style={[styles.statusMsg, { color: theme.accent }]}>{state.msg}</Text>
<Text style={styles.statusHint}> <Text style={styles.statusHint}>
First run downloads ~35 MB (Python runtime + packages). Subsequent runs are instant. First run downloads ~35 MB (Python runtime + packages). Subsequent runs are instant.
@@ -168,7 +233,15 @@ export default function ImportScreen() {
{state.status === 'done' && ( {state.status === 'done' && (
<View style={styles.success}> <View style={styles.success}>
<Text style={styles.successText}> Imported: {state.title}</Text> <Text style={styles.successText}>
Imported {state.count} {state.count === 1 ? 'activity' : 'activities'}
</Text>
{state.errors.map((e, i) => (
<Text key={i} style={styles.batchError}> {e.name}: {e.message}</Text>
))}
<Pressable onPress={() => setState({ status: 'idle' })}>
<Text style={styles.errorRetry}>Import more</Text>
</Pressable>
</View> </View>
)} )}
@@ -176,7 +249,7 @@ export default function ImportScreen() {
<View style={styles.error}> <View style={styles.error}>
<Text style={styles.errorText}>{state.message}</Text> <Text style={styles.errorText}>{state.message}</Text>
<Pressable onPress={() => setState({ status: 'idle' })}> <Pressable onPress={() => setState({ status: 'idle' })}>
<Text style={styles.errorRetry}>Try another file</Text> <Text style={styles.errorRetry}>Try again</Text>
</Pressable> </Pressable>
</View> </View>
)} )}
@@ -198,8 +271,9 @@ export default function ImportScreen() {
<View style={styles.notice}> <View style={styles.notice}>
<Text style={styles.noticeText}> <Text style={styles.noticeText}>
FIT/GPX/TCX extraction runs entirely on your device. FIT/GPX/TCX extraction runs entirely on your device.{'\n'}
A Bincio instance must be reachable on first run to download the extraction engine (~35 MB, then cached). A Bincio instance must be reachable on first run to download the extraction engine (~35 MB, then cached).{'\n\n'}
On Karoo or Android: set a Watch directory in Settings to auto-import new FIT files when the app opens.
</Text> </Text>
</View> </View>
</ScrollView> </ScrollView>
@@ -207,7 +281,51 @@ export default function ImportScreen() {
); );
} }
// ── Helpers ───────────────────────────────────────────────────────────────── // ── Watch-folder helpers ──────────────────────────────────────────────────────
async function requestStoragePermission(): Promise<boolean> {
if (Platform.OS !== 'android') return true;
try {
const granted = await PermissionsAndroid.request(
PermissionsAndroid.PERMISSIONS.READ_EXTERNAL_STORAGE,
);
return granted === PermissionsAndroid.RESULTS.GRANTED;
} catch {
return false;
}
}
async function discoverNewFiles(
db: ReturnType<typeof useSQLiteContext>,
watchPath: string,
): Promise<string[]> {
const ok = await requestStoragePermission();
if (!ok) return [];
// Normalize: strip trailing slash, then use file:// URI for expo-fs
const dir = watchPath.replace(/\/+$/, '');
const uri = dir.startsWith('file://') ? dir : `file://${dir}`;
let entries: string[];
try {
entries = await FileSystem.readDirectoryAsync(uri);
} catch {
return [];
}
const newFiles: string[] = [];
for (const entry of entries) {
const lower = entry.toLowerCase();
if (!lower.endsWith('.fit')) continue;
const fullPath = `${dir}/${entry}`;
if (!isSourcePathImported(db, fullPath)) {
newFiles.push(fullPath);
}
}
return newFiles;
}
// ── Module-level helpers ──────────────────────────────────────────────────────
async function getInstanceUrl(db: ReturnType<typeof useSQLiteContext>): Promise<string> { async function getInstanceUrl(db: ReturnType<typeof useSQLiteContext>): Promise<string> {
const row = db.getFirstSync<{ value: string }>( const row = db.getFirstSync<{ value: string }>(
@@ -279,17 +397,19 @@ const styles = StyleSheet.create({
backgroundColor: '#18181b', borderRadius: 8, borderWidth: 1, backgroundColor: '#18181b', borderRadius: 8, borderWidth: 1,
borderColor: '#27272a', padding: 14, marginBottom: 16, gap: 6, borderColor: '#27272a', padding: 14, marginBottom: 16, gap: 6,
}, },
statusMsg: { color: '#60a5fa', fontSize: 14, textAlign: 'center' }, statusCounter: { color: '#71717a', fontSize: 12, textAlign: 'center' },
statusHint: { color: '#52525b', fontSize: 12, textAlign: 'center', lineHeight: 16 }, statusMsg: { color: '#60a5fa', fontSize: 14, textAlign: 'center' },
statusHint: { color: '#52525b', fontSize: 12, textAlign: 'center', lineHeight: 16 },
success: { success: {
backgroundColor: '#14532d', borderRadius: 8, padding: 12, marginBottom: 16, backgroundColor: '#14532d', borderRadius: 8, padding: 12, marginBottom: 16, gap: 6,
}, },
successText: { color: '#86efac', fontSize: 14 }, successText: { color: '#86efac', fontSize: 14 },
batchError: { color: '#fca5a5', fontSize: 12 },
error: { error: {
backgroundColor: '#450a0a', borderRadius: 8, padding: 12, marginBottom: 16, gap: 8, backgroundColor: '#450a0a', borderRadius: 8, padding: 12, marginBottom: 16, gap: 8,
}, },
errorText: { color: '#fca5a5', fontSize: 14 }, errorText: { color: '#fca5a5', fontSize: 14 },
errorRetry: { color: '#71717a', fontSize: 13, textDecorationLine: 'underline' }, errorRetry: { color: '#71717a', fontSize: 13, textDecorationLine: 'underline', marginTop: 4 },
divider: { height: 1, backgroundColor: '#27272a', marginVertical: 24 }, divider: { height: 1, backgroundColor: '#27272a', marginVertical: 24 },
sectionTitle: { color: '#a1a1aa', fontSize: 12, fontWeight: '600', marginBottom: 12, letterSpacing: 0.5 }, sectionTitle: { color: '#a1a1aa', fontSize: 12, fontWeight: '600', marginBottom: 12, letterSpacing: 0.5 },
formatRow: { flexDirection: 'row', gap: 12, marginBottom: 10 }, formatRow: { flexDirection: 'row', gap: 12, marginBottom: 10 },
+12
View File
@@ -23,4 +23,16 @@ export async function migrateDb(db: SQLiteDatabase): Promise<void> {
value TEXT NOT NULL value TEXT NOT NULL
); );
`); `);
// Migration v2: source_path stores the original filesystem path a file was
// imported from (e.g. /sdcard/Karoo/Rides/ride.fit), used for watch-folder
// deduplication without re-hashing files.
try {
await db.execAsync('ALTER TABLE activities ADD COLUMN source_path TEXT');
await db.execAsync(
'CREATE INDEX IF NOT EXISTS idx_activities_source_path ON activities(source_path)',
);
} catch {
// Column already exists — migration already ran, ignore.
}
} }
+17 -3
View File
@@ -9,6 +9,7 @@ export type ActivityRow = {
timeseries_json: string | null; timeseries_json: string | null;
geojson: string | null; geojson: string | null;
original_path: string | null; original_path: string | null;
source_path: string | null;
synced_at: number | null; synced_at: number | null;
origin: 'local' | 'remote'; origin: 'local' | 'remote';
created_at: number; created_at: number;
@@ -67,12 +68,13 @@ export function useActivity(id: string): ActivityRow | null {
export async function insertActivity( export async function insertActivity(
db: ReturnType<typeof useSQLiteContext>, db: ReturnType<typeof useSQLiteContext>,
row: Pick<ActivityRow, 'id' | 'source_hash' | 'detail_json' | 'timeseries_json' | 'geojson' | 'original_path' | 'origin'>, row: Pick<ActivityRow, 'id' | 'source_hash' | 'detail_json' | 'timeseries_json' | 'geojson' | 'original_path' | 'origin'>
& { source_path?: string | null },
): Promise<void> { ): Promise<void> {
await db.runAsync( await db.runAsync(
`INSERT OR IGNORE INTO activities `INSERT OR IGNORE INTO activities
(id, source_hash, detail_json, timeseries_json, geojson, original_path, origin) (id, source_hash, detail_json, timeseries_json, geojson, original_path, source_path, origin)
VALUES (?, ?, ?, ?, ?, ?, ?)`, VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
[ [
row.id, row.id,
row.source_hash, row.source_hash,
@@ -80,11 +82,23 @@ export async function insertActivity(
row.timeseries_json ?? null, row.timeseries_json ?? null,
row.geojson ?? null, row.geojson ?? null,
row.original_path ?? null, row.original_path ?? null,
row.source_path ?? null,
row.origin, row.origin,
], ],
); );
} }
export function isSourcePathImported(
db: ReturnType<typeof useSQLiteContext>,
sourcePath: string,
): boolean {
const row = db.getFirstSync<{ id: string }>(
'SELECT id FROM activities WHERE source_path = ?',
[sourcePath],
);
return row != null;
}
export async function upsertRemoteActivity( export async function upsertRemoteActivity(
db: ReturnType<typeof useSQLiteContext>, db: ReturnType<typeof useSQLiteContext>,
id: string, id: string,