feat(SearchLocal, locate-db): enhance scanning process with compression and progress tracking

This commit is contained in:
Jaime Idolpx 2026-06-17 12:05:28 -04:00
parent ef48b167b0
commit ec61853957
3 changed files with 36 additions and 10 deletions

View File

@ -62,6 +62,7 @@
"cmdk": "1.1.1",
"date-fns": "3.6.0",
"embla-carousel-react": "8.6.0",
"fflate": "^0.8.3",
"input-otp": "1.4.2",
"lucide-react": "0.487.0",
"motion": "12.23.24",

View File

@ -117,8 +117,9 @@ function TypeBadge({ type, isDir }: { type: string; isDir: boolean }) {
}
function scanPhaseLabel(phase: ScanPhase, value: number): string {
if (phase === 'scanning') return `Scanning… ${humanFileSize(value)}`;
if (phase === 'building') return `Building… ${value.toLocaleString()} entries`;
if (phase === 'scanning') return `Scanning… ${humanFileSize(value)}`;
if (phase === 'building') return `Building… ${value.toLocaleString()} entries`;
if (phase === 'compressing') return value ? `Compressed to ${humanFileSize(value)}` : 'Compressing…';
return `Saving… ${humanFileSize(value)}`;
}
@ -171,6 +172,7 @@ export default function SearchLocal({ config, setConfig, onClose, onOpenFolder }
const [isScanning, setIsScanning] = useState(false);
const [scanPhase, setScanPhase] = useState<ScanPhase | null>(null);
const [scanValue, setScanValue] = useState(0);
const [scanPath, setScanPath] = useState<string | null>(null);
const [results, setResults] = useState<SearchResult[]>(() => _store.results);
const [hasSearched, setHasSearched] = useState(() => _store.hasSearched);
const [mountEntry, setMountEntry] = useState<SearchResult | null>(null);
@ -269,8 +271,8 @@ export default function SearchLocal({ config, setConfig, onClose, onOpenFolder }
setScanPhase('scanning');
setScanValue(0);
try {
const { count, bytes } = await buildLocateDb((phase, value) =>
flushSync(() => { setScanPhase(phase); setScanValue(value); })
const { count, bytes } = await buildLocateDb((phase, value, path) =>
flushSync(() => { setScanPhase(phase); setScanValue(value); if (path !== undefined) setScanPath(path); })
);
toast.success(`Indexed ${count.toLocaleString()} items · ${humanFileSize(bytes)}`);
setDbPhase('ready');
@ -279,6 +281,7 @@ export default function SearchLocal({ config, setConfig, onClose, onOpenFolder }
} finally {
setIsScanning(false);
setScanPhase(null);
setScanPath(null);
}
};
@ -442,10 +445,20 @@ export default function SearchLocal({ config, setConfig, onClose, onOpenFolder }
onScroll={e => { _store.scrollTop = (e.currentTarget as HTMLDivElement).scrollTop; }}
>
{isScanning && scanPhase && (
<div className="flex flex-col items-center justify-center py-16 gap-3">
<div className="flex flex-col items-center justify-center py-16 gap-3 px-6">
<Loader2 className="w-8 h-8 text-blue-500 animate-spin" />
<p className="text-sm font-medium text-neutral-700">{scanPhaseLabel(scanPhase, scanValue)}</p>
<p className="text-xs text-neutral-400">Scanning /sd recursively</p>
{scanPath ? (
<p
className="text-xs text-neutral-400 w-full text-center truncate"
style={{ direction: 'rtl', unicodeBidi: 'plaintext' }}
title={scanPath}
>
{scanPath}
</p>
) : (
<p className="text-xs text-neutral-400">Scanning /sd recursively</p>
)}
</div>
)}

View File

@ -1,8 +1,11 @@
import sqlite3InitModule from '@sqlite.org/sqlite-wasm';
import wasmUrl from '@sqlite.org/sqlite-wasm/sqlite3.wasm?url';
import { gzip } from 'fflate';
import { getWebDAVBaseUrl, basename, listDirectory, putFileContents, deletePath } from './webdav';
const LOCATE_PATH = '/sd/.locate';
const LOCATE_PATH = '/sd/.locate';
const LOCATE_GZ_PATH = '/sd/.locate.gz';
const SCAN_PROGRESS_INTERVAL = 50;
// Vite rewrites this `?url` import to the hashed asset path
// (e.g. /assets/sqlite3-BVKGSWc-.wasm) and respects the configured base path.
const WASM_URL: string = wasmUrl;
@ -159,7 +162,7 @@ export async function openLocateDb(
_setLoadState({ phase: 'ready' });
}
export type ScanPhase = 'scanning' | 'building' | 'saving';
export type ScanPhase = 'scanning' | 'building' | 'saving' | 'compressing';
/**
* Recursively scan /sd, build a fresh SQLite locate database in memory,
@ -171,7 +174,7 @@ export type ScanPhase = 'scanning' | 'building' | 'saving';
* saving bytes of the serialized DB written to the server
*/
export async function buildLocateDb(
onProgress?: (phase: ScanPhase, value: number) => void,
onProgress?: (phase: ScanPhase, value: number, path?: string) => void,
): Promise<{ count: number; bytes: number }> {
const sqlite3 = await getSqlite3();
@ -204,7 +207,7 @@ export async function buildLocateDb(
e.lastModified ? Math.floor(e.lastModified.getTime() / 1000) : 0,
e.type === 'folder' ? 1 : 0,
]).stepReset();
if (i % 250 === 0) onProgress?.('building', i);
if (i % SCAN_PROGRESS_INTERVAL === 0) onProgress?.('building', i, e.path);
}
} finally {
stmt.finalize();
@ -234,6 +237,15 @@ export async function buildLocateDb(
await putFileContents(LOCATE_PATH, dbBytes);
onProgress?.('saving', dbBytes.length);
// ── 5. Gzip at level 9 and upload ───────────────────────────────────────
onProgress?.('compressing', 0);
const gzBytes = await new Promise<Uint8Array>((resolve, reject) =>
gzip(dbBytes, { level: 9 }, (err, data) => err ? reject(err) : resolve(data)),
);
await deletePath(LOCATE_GZ_PATH).catch(() => {});
await putFileContents(LOCATE_GZ_PATH, gzBytes);
onProgress?.('compressing', gzBytes.length);
// Invalidate cached DB so the next search reloads the fresh file.
resetLocateDb();