Davide Scaini
2f53fbc359
- Import tab now accepts multiple files at once (DocumentPicker multiple:true), processes them sequentially through Pyodide, and shows a summary with per-file errors on completion. - DB migration v2 adds source_path column (original filesystem path before copy) and an index on it, enabling O(1) deduplication for watch-folder imports. - On Android, if auto_import_path is set, the Import tab scans the directory on mount and on AppState 'active' (app foreground), then automatically imports any FIT files not yet in the DB. Designed for Karoo: finish a ride, open the app, new files import without any manual steps. - insertActivity now accepts optional source_path; both importBasJson and importNativeFile pass it through (null for files picked via DocumentPicker, real path for watch-folder files).
424 lines
16 KiB
TypeScript
424 lines
16 KiB
TypeScript
import * as DocumentPicker from 'expo-document-picker';
|
||
import * as FileSystem from 'expo-file-system/legacy';
|
||
import { useSQLiteContext } from 'expo-sqlite';
|
||
import { useEffect, useRef, useState } from 'react';
|
||
import { AppState, PermissionsAndroid, Platform, Pressable, ScrollView, StyleSheet, Text, View } from 'react-native';
|
||
import { insertActivity, isSourcePathImported, getSetting } from '@/db/queries';
|
||
import { PyodideWebView } from '@/extraction/PyodideWebView';
|
||
import { extractFile } from '@/extraction/extractActivity';
|
||
import { useTheme } from '@/ThemeContext';
|
||
|
||
const FIT_EXTENSIONS = ['.fit', '.fit.gz'];
|
||
const OTHER_EXTENSIONS = ['.gpx', '.tcx', '.gpx.gz', '.tcx.gz'];
|
||
const ALL_NATIVE_EXTENSIONS = [...FIT_EXTENSIONS, ...OTHER_EXTENSIONS];
|
||
|
||
type ImportState =
|
||
| { status: 'idle' }
|
||
| { status: 'loading'; msg: string; current: number; total: number }
|
||
| { status: 'done'; count: number; errors: Array<{ name: string; message: string }> }
|
||
| { status: 'error'; message: string };
|
||
|
||
export default function ImportScreen() {
|
||
const db = useSQLiteContext();
|
||
const theme = useTheme();
|
||
const [state, setState] = useState<ImportState>({ status: 'idle' });
|
||
const isImporting = useRef(false);
|
||
|
||
// Auto-scan watch folder on mount and when app comes to foreground.
|
||
useEffect(() => {
|
||
if (Platform.OS !== 'android') return;
|
||
runAutoScan();
|
||
|
||
const sub = AppState.addEventListener('change', (next) => {
|
||
if (next === 'active') runAutoScan();
|
||
});
|
||
return () => sub.remove();
|
||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||
}, []);
|
||
|
||
async function runAutoScan() {
|
||
if (isImporting.current) return;
|
||
const watchPath = await getSetting(db, 'auto_import_path');
|
||
if (!watchPath) return;
|
||
|
||
const newFiles = await discoverNewFiles(db, watchPath);
|
||
if (newFiles.length === 0) return;
|
||
|
||
isImporting.current = true;
|
||
try {
|
||
await processBatch(newFiles.map(f => ({ uri: `file://${f}`, name: f.split('/').pop() ?? f, sourcePath: f })));
|
||
} finally {
|
||
isImporting.current = false;
|
||
}
|
||
}
|
||
|
||
async function pickFiles() {
|
||
if (isImporting.current) return;
|
||
setState({ status: 'loading', msg: 'Picking files…', current: 0, total: 0 });
|
||
try {
|
||
const result = await DocumentPicker.getDocumentAsync({
|
||
type: ['*/*'],
|
||
copyToCacheDirectory: true,
|
||
multiple: true,
|
||
});
|
||
if (result.canceled || !result.assets?.length) {
|
||
setState({ status: 'idle' });
|
||
return;
|
||
}
|
||
isImporting.current = true;
|
||
try {
|
||
await processBatch(result.assets.map(a => ({ uri: a.uri, name: a.name ?? '', sourcePath: null })));
|
||
} finally {
|
||
isImporting.current = false;
|
||
}
|
||
} catch (e: unknown) {
|
||
const msg = e instanceof Error ? e.message : String(e);
|
||
setState({ status: 'error', message: msg });
|
||
isImporting.current = false;
|
||
}
|
||
}
|
||
|
||
async function processBatch(files: Array<{ uri: string; name: string; sourcePath: string | null }>) {
|
||
const total = files.length;
|
||
const errors: Array<{ name: string; message: string }> = [];
|
||
let count = 0;
|
||
|
||
for (let i = 0; i < files.length; i++) {
|
||
const { uri, name, sourcePath } = files[i];
|
||
const lower = name.toLowerCase();
|
||
|
||
setState({ status: 'loading', msg: `Processing ${name}…`, current: i + 1, total });
|
||
|
||
try {
|
||
if (lower.endsWith('.json')) {
|
||
await importBasJson(uri, name, sourcePath, (msg) =>
|
||
setState({ status: 'loading', msg, current: i + 1, total }),
|
||
);
|
||
} else if (ALL_NATIVE_EXTENSIONS.some(ext => lower.endsWith(ext))) {
|
||
await importNativeFile(uri, name, sourcePath, (msg) =>
|
||
setState({ status: 'loading', msg, current: i + 1, total }),
|
||
);
|
||
} else {
|
||
errors.push({ name, message: 'Unsupported file type' });
|
||
continue;
|
||
}
|
||
count++;
|
||
} catch (e: unknown) {
|
||
errors.push({ name, message: e instanceof Error ? e.message : String(e) });
|
||
}
|
||
}
|
||
|
||
setState({ status: 'done', count, errors });
|
||
}
|
||
|
||
// ── BAS JSON import (no extraction needed) ──────────────────────────────────
|
||
|
||
async function importBasJson(
|
||
uri: string,
|
||
_name: string,
|
||
sourcePath: string | null,
|
||
onStatus: (msg: string) => void,
|
||
) {
|
||
onStatus('Importing…');
|
||
const text = await FileSystem.readAsStringAsync(uri);
|
||
const detail = JSON.parse(text);
|
||
|
||
if (!detail.id || !detail.started_at) {
|
||
throw new Error('Not a valid BAS activity JSON (missing id or started_at)');
|
||
}
|
||
|
||
const hash = detail.source_hash ?? `${detail.id}-${text.length}`;
|
||
const origDir = `${FileSystem.documentDirectory}originals/`;
|
||
await FileSystem.makeDirectoryAsync(origDir, { intermediates: true });
|
||
const dest = `${origDir}${detail.id}.json`;
|
||
await FileSystem.copyAsync({ from: uri, to: dest });
|
||
|
||
await insertActivity(db, {
|
||
id: detail.id,
|
||
source_hash: hash,
|
||
detail_json: text,
|
||
timeseries_json: null,
|
||
geojson: null,
|
||
original_path: dest,
|
||
source_path: sourcePath,
|
||
origin: 'local',
|
||
});
|
||
}
|
||
|
||
// ── FIT / GPX / TCX import via Pyodide extraction ──────────────────────────
|
||
|
||
async function importNativeFile(
|
||
uri: string,
|
||
name: string,
|
||
sourcePath: string | null,
|
||
onStatus: (msg: string) => void,
|
||
) {
|
||
onStatus('Reading file…');
|
||
|
||
// Read the original file as base64 so we can (a) pass it to the WebView
|
||
// and (b) copy it to permanent storage without a second read.
|
||
const base64 = await FileSystem.readAsStringAsync(uri, {
|
||
encoding: FileSystem.EncodingType.Base64,
|
||
});
|
||
|
||
// Fetch the bincio wheel here (React Native networking), not inside the
|
||
// WebView. WKWebView blocks HTTP requests via ATS; RN native networking
|
||
// allows local-network HTTP (NSAllowsLocalNetworking=true in Info.plist).
|
||
const instanceUrl = await getInstanceUrl(db);
|
||
onStatus('Fetching Bincio engine…');
|
||
const { base64: wheelBase64, filename: wheelFilename } = await fetchWheelBase64(instanceUrl);
|
||
|
||
const result = await extractFile(name, base64, wheelBase64, wheelFilename, onStatus);
|
||
|
||
onStatus('Saving…');
|
||
|
||
// Copy original file to permanent storage (keeps original bytes for future re-extraction)
|
||
const ext = name.includes('.') ? name.slice(name.lastIndexOf('.')) : '';
|
||
const origDir = `${FileSystem.documentDirectory}originals/`;
|
||
await FileSystem.makeDirectoryAsync(origDir, { intermediates: true });
|
||
const dest = `${origDir}${result.id}${ext}`;
|
||
await FileSystem.copyAsync({ from: uri, to: dest });
|
||
|
||
await insertActivity(db, {
|
||
id: result.id,
|
||
source_hash: result.sourceHash,
|
||
detail_json: JSON.stringify(result.detail),
|
||
timeseries_json: result.timeseries ? JSON.stringify(result.timeseries) : null,
|
||
geojson: result.geojson ? JSON.stringify(result.geojson) : null,
|
||
original_path: dest,
|
||
source_path: sourcePath,
|
||
origin: 'local',
|
||
});
|
||
}
|
||
|
||
return (
|
||
<View style={styles.screen}>
|
||
{/* Hidden WebView for Pyodide — mounted here so it lives inside the tab
|
||
(Expo Router keeps tabs mounted after first visit, preserving Pyodide state).
|
||
The 1×1 container clips it out of the scroll layout entirely. */}
|
||
<View style={styles.hiddenEngine}>
|
||
<PyodideWebView />
|
||
</View>
|
||
<ScrollView style={styles.container} contentContainerStyle={styles.content}>
|
||
<Text style={styles.header}>Import</Text>
|
||
|
||
<Text style={styles.body}>
|
||
Import FIT, GPX, or TCX files — extracted on your device, nothing uploaded.
|
||
You can also import pre-extracted BAS <Text style={[styles.code, { color: theme.accent }]}>.json</Text> files.
|
||
Select multiple files at once to import in batch.
|
||
</Text>
|
||
|
||
<Pressable
|
||
style={[styles.button, state.status === 'loading' && styles.buttonDisabled]}
|
||
onPress={state.status !== 'loading' ? pickFiles : undefined}
|
||
>
|
||
<Text style={styles.buttonText}>
|
||
{state.status === 'loading' ? 'Working…' : '＋ Pick files'}
|
||
</Text>
|
||
</Pressable>
|
||
|
||
{state.status === 'loading' && (
|
||
<View style={styles.statusBox}>
|
||
{state.total > 1 && (
|
||
<Text style={styles.statusCounter}>
|
||
File {state.current} of {state.total}
|
||
</Text>
|
||
)}
|
||
<Text style={[styles.statusMsg, { color: theme.accent }]}>{state.msg}</Text>
|
||
<Text style={styles.statusHint}>
|
||
First run downloads ~35 MB (Python runtime + packages). Subsequent runs are instant.
|
||
</Text>
|
||
</View>
|
||
)}
|
||
|
||
{state.status === 'done' && (
|
||
<View style={styles.success}>
|
||
<Text style={styles.successText}>
|
||
✓ Imported {state.count} {state.count === 1 ? 'activity' : 'activities'}
|
||
</Text>
|
||
{state.errors.map((e, i) => (
|
||
<Text key={i} style={styles.batchError}>✗ {e.name}: {e.message}</Text>
|
||
))}
|
||
<Pressable onPress={() => setState({ status: 'idle' })}>
|
||
<Text style={styles.errorRetry}>Import more</Text>
|
||
</Pressable>
|
||
</View>
|
||
)}
|
||
|
||
{state.status === 'error' && (
|
||
<View style={styles.error}>
|
||
<Text style={styles.errorText}>{state.message}</Text>
|
||
<Pressable onPress={() => setState({ status: 'idle' })}>
|
||
<Text style={styles.errorRetry}>Try again</Text>
|
||
</Pressable>
|
||
</View>
|
||
)}
|
||
|
||
<View style={styles.divider} />
|
||
|
||
<Text style={styles.sectionTitle}>Supported formats</Text>
|
||
{([
|
||
['FIT', 'Garmin, Wahoo, Karoo native format'],
|
||
['GPX', 'Most GPS devices and apps'],
|
||
['TCX', 'Garmin Training Center'],
|
||
['BAS JSON', 'Pre-extracted Bincio format (instant)'],
|
||
] as [string, string][]).map(([fmt, desc]) => (
|
||
<View key={fmt} style={styles.formatRow}>
|
||
<Text style={styles.formatName}>{fmt}</Text>
|
||
<Text style={styles.formatDesc}>{desc}</Text>
|
||
</View>
|
||
))}
|
||
|
||
<View style={styles.notice}>
|
||
<Text style={styles.noticeText}>
|
||
FIT/GPX/TCX extraction runs entirely on your device.{'\n'}
|
||
A Bincio instance must be reachable on first run to download the extraction engine (~35 MB, then cached).{'\n\n'}
|
||
On Karoo or Android: set a Watch directory in Settings to auto-import new FIT files when the app opens.
|
||
</Text>
|
||
</View>
|
||
</ScrollView>
|
||
</View>
|
||
);
|
||
}
|
||
|
||
// ── Watch-folder helpers ──────────────────────────────────────────────────────
|
||
|
||
async function requestStoragePermission(): Promise<boolean> {
|
||
if (Platform.OS !== 'android') return true;
|
||
try {
|
||
const granted = await PermissionsAndroid.request(
|
||
PermissionsAndroid.PERMISSIONS.READ_EXTERNAL_STORAGE,
|
||
);
|
||
return granted === PermissionsAndroid.RESULTS.GRANTED;
|
||
} catch {
|
||
return false;
|
||
}
|
||
}
|
||
|
||
async function discoverNewFiles(
|
||
db: ReturnType<typeof useSQLiteContext>,
|
||
watchPath: string,
|
||
): Promise<string[]> {
|
||
const ok = await requestStoragePermission();
|
||
if (!ok) return [];
|
||
|
||
// Normalize: strip trailing slash, then use file:// URI for expo-fs
|
||
const dir = watchPath.replace(/\/+$/, '');
|
||
const uri = dir.startsWith('file://') ? dir : `file://${dir}`;
|
||
|
||
let entries: string[];
|
||
try {
|
||
entries = await FileSystem.readDirectoryAsync(uri);
|
||
} catch {
|
||
return [];
|
||
}
|
||
|
||
const newFiles: string[] = [];
|
||
for (const entry of entries) {
|
||
const lower = entry.toLowerCase();
|
||
if (!lower.endsWith('.fit')) continue;
|
||
const fullPath = `${dir}/${entry}`;
|
||
if (!isSourcePathImported(db, fullPath)) {
|
||
newFiles.push(fullPath);
|
||
}
|
||
}
|
||
return newFiles;
|
||
}
|
||
|
||
// ── Module-level helpers ──────────────────────────────────────────────────────
|
||
|
||
async function getInstanceUrl(db: ReturnType<typeof useSQLiteContext>): Promise<string> {
|
||
const row = db.getFirstSync<{ value: string }>(
|
||
'SELECT value FROM settings WHERE key = ?',
|
||
['instance_url'],
|
||
);
|
||
return (row?.value ?? '').replace(/\/$/, '');
|
||
}
|
||
|
||
// In-memory cache so repeated imports in one session don't re-download the wheel.
|
||
let _cachedWheel: { base64: string; filename: string } | null = null;
|
||
|
||
async function fetchWheelBase64(instanceUrl: string): Promise<{ base64: string; filename: string }> {
|
||
if (_cachedWheel) return _cachedWheel;
|
||
|
||
const base = instanceUrl || 'https://bincio.org';
|
||
|
||
// Ask the instance for the canonical wheel URL (handles both dev and prod layouts).
|
||
let wheelUrl = `${base}/api/wheel/download`;
|
||
let wheelFilename = 'bincio-0.1.0-py3-none-any.whl';
|
||
try {
|
||
const vr = await fetch(`${base}/api/wheel/version`, { signal: AbortSignal.timeout(5000) });
|
||
if (vr.ok) {
|
||
const d = await vr.json() as { api_url?: string; url?: string };
|
||
const path = d.api_url ?? d.url ?? '/api/wheel/download';
|
||
wheelUrl = path.startsWith('http') ? path : `${base}${path}`;
|
||
// Extract the filename from the URL path (last segment after final /)
|
||
const urlBasename = wheelUrl.split('/').pop() ?? '';
|
||
if (urlBasename.endsWith('.whl')) wheelFilename = urlBasename;
|
||
}
|
||
} catch {}
|
||
|
||
// Fetch via React Native networking (supports local HTTP; WKWebView would block it).
|
||
const resp = await fetch(wheelUrl);
|
||
if (!resp.ok) throw new Error(`Could not download Bincio engine (${resp.status}). Is the instance running?`);
|
||
const buf = await resp.arrayBuffer();
|
||
_cachedWheel = { base64: arrayBufferToBase64(buf), filename: wheelFilename };
|
||
return _cachedWheel;
|
||
}
|
||
|
||
function arrayBufferToBase64(buf: ArrayBuffer): string {
|
||
const bytes = new Uint8Array(buf);
|
||
let binary = '';
|
||
// Process in chunks to avoid spread-operator stack overflow on large arrays.
|
||
const CHUNK = 8192;
|
||
for (let i = 0; i < bytes.length; i += CHUNK) {
|
||
binary += String.fromCharCode(...(bytes.subarray(i, i + CHUNK) as unknown as number[]));
|
||
}
|
||
return btoa(binary);
|
||
}
|
||
|
||
// ── Styles ───────────────────────────────────────────────────────────────────
|
||
|
||
const styles = StyleSheet.create({
|
||
screen: { flex: 1, backgroundColor: '#09090b' },
|
||
hiddenEngine: { position: 'absolute', width: 1, height: 1, overflow: 'hidden' },
|
||
container: { flex: 1 },
|
||
content: { padding: 16, paddingTop: 60, paddingBottom: 40 },
|
||
header: { color: '#fff', fontSize: 22, fontWeight: '700', marginBottom: 12 },
|
||
body: { color: '#a1a1aa', fontSize: 14, lineHeight: 20, marginBottom: 24 },
|
||
code: { color: '#60a5fa', fontFamily: 'monospace' },
|
||
button: {
|
||
backgroundColor: '#2563eb', borderRadius: 10,
|
||
paddingVertical: 14, alignItems: 'center', marginBottom: 16,
|
||
},
|
||
buttonDisabled: { opacity: 0.5 },
|
||
buttonText: { color: '#fff', fontWeight: '600', fontSize: 16 },
|
||
statusBox: {
|
||
backgroundColor: '#18181b', borderRadius: 8, borderWidth: 1,
|
||
borderColor: '#27272a', padding: 14, marginBottom: 16, gap: 6,
|
||
},
|
||
statusCounter: { color: '#71717a', fontSize: 12, textAlign: 'center' },
|
||
statusMsg: { color: '#60a5fa', fontSize: 14, textAlign: 'center' },
|
||
statusHint: { color: '#52525b', fontSize: 12, textAlign: 'center', lineHeight: 16 },
|
||
success: {
|
||
backgroundColor: '#14532d', borderRadius: 8, padding: 12, marginBottom: 16, gap: 6,
|
||
},
|
||
successText: { color: '#86efac', fontSize: 14 },
|
||
batchError: { color: '#fca5a5', fontSize: 12 },
|
||
error: {
|
||
backgroundColor: '#450a0a', borderRadius: 8, padding: 12, marginBottom: 16, gap: 8,
|
||
},
|
||
errorText: { color: '#fca5a5', fontSize: 14 },
|
||
errorRetry: { color: '#71717a', fontSize: 13, textDecorationLine: 'underline', marginTop: 4 },
|
||
divider: { height: 1, backgroundColor: '#27272a', marginVertical: 24 },
|
||
sectionTitle: { color: '#a1a1aa', fontSize: 12, fontWeight: '600', marginBottom: 12, letterSpacing: 0.5 },
|
||
formatRow: { flexDirection: 'row', gap: 12, marginBottom: 10 },
|
||
formatName: { color: '#f4f4f5', fontSize: 13, fontWeight: '600', width: 72 },
|
||
formatDesc: { color: '#71717a', fontSize: 13, flex: 1 },
|
||
notice: {
|
||
marginTop: 8, backgroundColor: '#18181b',
|
||
borderRadius: 8, padding: 12, borderWidth: 1, borderColor: '#27272a',
|
||
},
|
||
noticeText: { color: '#71717a', fontSize: 12, lineHeight: 18 },
|
||
});
|