From ec6185395739289c9ee4ae8a8d18db29a167507f Mon Sep 17 00:00:00 2001 From: Jaime Idolpx Date: Wed, 17 Jun 2026 12:05:28 -0400 Subject: [PATCH] feat(SearchLocal, locate-db): enhance scanning process with compression and progress tracking --- package.json | 1 + src/app/components/SearchLocal.tsx | 25 +++++++++++++++++++------ src/app/locate-db.ts | 20 ++++++++++++++++---- 3 files changed, 36 insertions(+), 10 deletions(-) diff --git a/package.json b/package.json index 14d4ed5..8039390 100644 --- a/package.json +++ b/package.json @@ -62,6 +62,7 @@ "cmdk": "1.1.1", "date-fns": "3.6.0", "embla-carousel-react": "8.6.0", + "fflate": "^0.8.3", "input-otp": "1.4.2", "lucide-react": "0.487.0", "motion": "12.23.24", diff --git a/src/app/components/SearchLocal.tsx b/src/app/components/SearchLocal.tsx index 1bf6030..95d4331 100644 --- a/src/app/components/SearchLocal.tsx +++ b/src/app/components/SearchLocal.tsx @@ -117,8 +117,9 @@ function TypeBadge({ type, isDir }: { type: string; isDir: boolean }) { } function scanPhaseLabel(phase: ScanPhase, value: number): string { - if (phase === 'scanning') return `Scanning… ${humanFileSize(value)}`; - if (phase === 'building') return `Building… ${value.toLocaleString()} entries`; + if (phase === 'scanning') return `Scanning… ${humanFileSize(value)}`; + if (phase === 'building') return `Building… ${value.toLocaleString()} entries`; + if (phase === 'compressing') return value ? `Compressed to ${humanFileSize(value)}` : 'Compressing…'; return `Saving… ${humanFileSize(value)}`; } @@ -171,6 +172,7 @@ export default function SearchLocal({ config, setConfig, onClose, onOpenFolder } const [isScanning, setIsScanning] = useState(false); const [scanPhase, setScanPhase] = useState(null); const [scanValue, setScanValue] = useState(0); + const [scanPath, setScanPath] = useState(null); const [results, setResults] = useState(() => _store.results); const [hasSearched, setHasSearched] = useState(() => _store.hasSearched); const [mountEntry, setMountEntry] = useState(null); @@ -269,8 +271,8 @@ export default function SearchLocal({ config, setConfig, onClose, onOpenFolder } setScanPhase('scanning'); setScanValue(0); try { - const { count, bytes } = await buildLocateDb((phase, value) => - flushSync(() => { setScanPhase(phase); setScanValue(value); }) + const { count, bytes } = await buildLocateDb((phase, value, path) => + flushSync(() => { setScanPhase(phase); setScanValue(value); if (path !== undefined) setScanPath(path); }) ); toast.success(`Indexed ${count.toLocaleString()} items · ${humanFileSize(bytes)}`); setDbPhase('ready'); @@ -279,6 +281,7 @@ export default function SearchLocal({ config, setConfig, onClose, onOpenFolder } } finally { setIsScanning(false); setScanPhase(null); + setScanPath(null); } }; @@ -442,10 +445,20 @@ export default function SearchLocal({ config, setConfig, onClose, onOpenFolder } onScroll={e => { _store.scrollTop = (e.currentTarget as HTMLDivElement).scrollTop; }} > {isScanning && scanPhase && ( -
+

{scanPhaseLabel(scanPhase, scanValue)}

-

Scanning /sd recursively…

+ {scanPath ? ( +

+ {scanPath} +

+ ) : ( +

Scanning /sd recursively…

+ )}
)} diff --git a/src/app/locate-db.ts b/src/app/locate-db.ts index b01066e..e5ccffb 100644 --- a/src/app/locate-db.ts +++ b/src/app/locate-db.ts @@ -1,8 +1,11 @@ import sqlite3InitModule from '@sqlite.org/sqlite-wasm'; import wasmUrl from '@sqlite.org/sqlite-wasm/sqlite3.wasm?url'; +import { gzip } from 'fflate'; import { getWebDAVBaseUrl, basename, listDirectory, putFileContents, deletePath } from './webdav'; -const LOCATE_PATH = '/sd/.locate'; +const LOCATE_PATH = '/sd/.locate'; +const LOCATE_GZ_PATH = '/sd/.locate.gz'; +const SCAN_PROGRESS_INTERVAL = 50; // Vite rewrites this `?url` import to the hashed asset path // (e.g. /assets/sqlite3-BVKGSWc-.wasm) and respects the configured base path. const WASM_URL: string = wasmUrl; @@ -159,7 +162,7 @@ export async function openLocateDb( _setLoadState({ phase: 'ready' }); } -export type ScanPhase = 'scanning' | 'building' | 'saving'; +export type ScanPhase = 'scanning' | 'building' | 'saving' | 'compressing'; /** * Recursively scan /sd, build a fresh SQLite locate database in memory, @@ -171,7 +174,7 @@ export type ScanPhase = 'scanning' | 'building' | 'saving'; * saving → bytes of the serialized DB written to the server */ export async function buildLocateDb( - onProgress?: (phase: ScanPhase, value: number) => void, + onProgress?: (phase: ScanPhase, value: number, path?: string) => void, ): Promise<{ count: number; bytes: number }> { const sqlite3 = await getSqlite3(); @@ -204,7 +207,7 @@ export async function buildLocateDb( e.lastModified ? Math.floor(e.lastModified.getTime() / 1000) : 0, e.type === 'folder' ? 1 : 0, ]).stepReset(); - if (i % 250 === 0) onProgress?.('building', i); + if (i % SCAN_PROGRESS_INTERVAL === 0) onProgress?.('building', i, e.path); } } finally { stmt.finalize(); @@ -234,6 +237,15 @@ export async function buildLocateDb( await putFileContents(LOCATE_PATH, dbBytes); onProgress?.('saving', dbBytes.length); + // ── 5. Gzip at level 9 and upload ─────────────────────────────────────── + onProgress?.('compressing', 0); + const gzBytes = await new Promise((resolve, reject) => + gzip(dbBytes, { level: 9 }, (err, data) => err ? reject(err) : resolve(data)), + ); + await deletePath(LOCATE_GZ_PATH).catch(() => {}); + await putFileContents(LOCATE_GZ_PATH, gzBytes); + onProgress?.('compressing', gzBytes.length); + // Invalidate cached DB so the next search reloads the fresh file. resetLocateDb();