feat(SearchLocal, locate-db): enhance scanning process with compression and progress tracking
This commit is contained in:
parent
ef48b167b0
commit
ec61853957
|
|
@ -62,6 +62,7 @@
|
||||||
"cmdk": "1.1.1",
|
"cmdk": "1.1.1",
|
||||||
"date-fns": "3.6.0",
|
"date-fns": "3.6.0",
|
||||||
"embla-carousel-react": "8.6.0",
|
"embla-carousel-react": "8.6.0",
|
||||||
|
"fflate": "^0.8.3",
|
||||||
"input-otp": "1.4.2",
|
"input-otp": "1.4.2",
|
||||||
"lucide-react": "0.487.0",
|
"lucide-react": "0.487.0",
|
||||||
"motion": "12.23.24",
|
"motion": "12.23.24",
|
||||||
|
|
|
||||||
|
|
@ -117,8 +117,9 @@ function TypeBadge({ type, isDir }: { type: string; isDir: boolean }) {
|
||||||
}
|
}
|
||||||
|
|
||||||
function scanPhaseLabel(phase: ScanPhase, value: number): string {
|
function scanPhaseLabel(phase: ScanPhase, value: number): string {
|
||||||
if (phase === 'scanning') return `Scanning… ${humanFileSize(value)}`;
|
if (phase === 'scanning') return `Scanning… ${humanFileSize(value)}`;
|
||||||
if (phase === 'building') return `Building… ${value.toLocaleString()} entries`;
|
if (phase === 'building') return `Building… ${value.toLocaleString()} entries`;
|
||||||
|
if (phase === 'compressing') return value ? `Compressed to ${humanFileSize(value)}` : 'Compressing…';
|
||||||
return `Saving… ${humanFileSize(value)}`;
|
return `Saving… ${humanFileSize(value)}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -171,6 +172,7 @@ export default function SearchLocal({ config, setConfig, onClose, onOpenFolder }
|
||||||
const [isScanning, setIsScanning] = useState(false);
|
const [isScanning, setIsScanning] = useState(false);
|
||||||
const [scanPhase, setScanPhase] = useState<ScanPhase | null>(null);
|
const [scanPhase, setScanPhase] = useState<ScanPhase | null>(null);
|
||||||
const [scanValue, setScanValue] = useState(0);
|
const [scanValue, setScanValue] = useState(0);
|
||||||
|
const [scanPath, setScanPath] = useState<string | null>(null);
|
||||||
const [results, setResults] = useState<SearchResult[]>(() => _store.results);
|
const [results, setResults] = useState<SearchResult[]>(() => _store.results);
|
||||||
const [hasSearched, setHasSearched] = useState(() => _store.hasSearched);
|
const [hasSearched, setHasSearched] = useState(() => _store.hasSearched);
|
||||||
const [mountEntry, setMountEntry] = useState<SearchResult | null>(null);
|
const [mountEntry, setMountEntry] = useState<SearchResult | null>(null);
|
||||||
|
|
@ -269,8 +271,8 @@ export default function SearchLocal({ config, setConfig, onClose, onOpenFolder }
|
||||||
setScanPhase('scanning');
|
setScanPhase('scanning');
|
||||||
setScanValue(0);
|
setScanValue(0);
|
||||||
try {
|
try {
|
||||||
const { count, bytes } = await buildLocateDb((phase, value) =>
|
const { count, bytes } = await buildLocateDb((phase, value, path) =>
|
||||||
flushSync(() => { setScanPhase(phase); setScanValue(value); })
|
flushSync(() => { setScanPhase(phase); setScanValue(value); if (path !== undefined) setScanPath(path); })
|
||||||
);
|
);
|
||||||
toast.success(`Indexed ${count.toLocaleString()} items · ${humanFileSize(bytes)}`);
|
toast.success(`Indexed ${count.toLocaleString()} items · ${humanFileSize(bytes)}`);
|
||||||
setDbPhase('ready');
|
setDbPhase('ready');
|
||||||
|
|
@ -279,6 +281,7 @@ export default function SearchLocal({ config, setConfig, onClose, onOpenFolder }
|
||||||
} finally {
|
} finally {
|
||||||
setIsScanning(false);
|
setIsScanning(false);
|
||||||
setScanPhase(null);
|
setScanPhase(null);
|
||||||
|
setScanPath(null);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
@ -442,10 +445,20 @@ export default function SearchLocal({ config, setConfig, onClose, onOpenFolder }
|
||||||
onScroll={e => { _store.scrollTop = (e.currentTarget as HTMLDivElement).scrollTop; }}
|
onScroll={e => { _store.scrollTop = (e.currentTarget as HTMLDivElement).scrollTop; }}
|
||||||
>
|
>
|
||||||
{isScanning && scanPhase && (
|
{isScanning && scanPhase && (
|
||||||
<div className="flex flex-col items-center justify-center py-16 gap-3">
|
<div className="flex flex-col items-center justify-center py-16 gap-3 px-6">
|
||||||
<Loader2 className="w-8 h-8 text-blue-500 animate-spin" />
|
<Loader2 className="w-8 h-8 text-blue-500 animate-spin" />
|
||||||
<p className="text-sm font-medium text-neutral-700">{scanPhaseLabel(scanPhase, scanValue)}</p>
|
<p className="text-sm font-medium text-neutral-700">{scanPhaseLabel(scanPhase, scanValue)}</p>
|
||||||
<p className="text-xs text-neutral-400">Scanning /sd recursively…</p>
|
{scanPath ? (
|
||||||
|
<p
|
||||||
|
className="text-xs text-neutral-400 w-full text-center truncate"
|
||||||
|
style={{ direction: 'rtl', unicodeBidi: 'plaintext' }}
|
||||||
|
title={scanPath}
|
||||||
|
>
|
||||||
|
{scanPath}
|
||||||
|
</p>
|
||||||
|
) : (
|
||||||
|
<p className="text-xs text-neutral-400">Scanning /sd recursively…</p>
|
||||||
|
)}
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,8 +1,11 @@
|
||||||
import sqlite3InitModule from '@sqlite.org/sqlite-wasm';
|
import sqlite3InitModule from '@sqlite.org/sqlite-wasm';
|
||||||
import wasmUrl from '@sqlite.org/sqlite-wasm/sqlite3.wasm?url';
|
import wasmUrl from '@sqlite.org/sqlite-wasm/sqlite3.wasm?url';
|
||||||
|
import { gzip } from 'fflate';
|
||||||
import { getWebDAVBaseUrl, basename, listDirectory, putFileContents, deletePath } from './webdav';
|
import { getWebDAVBaseUrl, basename, listDirectory, putFileContents, deletePath } from './webdav';
|
||||||
|
|
||||||
const LOCATE_PATH = '/sd/.locate';
|
const LOCATE_PATH = '/sd/.locate';
|
||||||
|
const LOCATE_GZ_PATH = '/sd/.locate.gz';
|
||||||
|
const SCAN_PROGRESS_INTERVAL = 50;
|
||||||
// Vite rewrites this `?url` import to the hashed asset path
|
// Vite rewrites this `?url` import to the hashed asset path
|
||||||
// (e.g. /assets/sqlite3-BVKGSWc-.wasm) and respects the configured base path.
|
// (e.g. /assets/sqlite3-BVKGSWc-.wasm) and respects the configured base path.
|
||||||
const WASM_URL: string = wasmUrl;
|
const WASM_URL: string = wasmUrl;
|
||||||
|
|
@ -159,7 +162,7 @@ export async function openLocateDb(
|
||||||
_setLoadState({ phase: 'ready' });
|
_setLoadState({ phase: 'ready' });
|
||||||
}
|
}
|
||||||
|
|
||||||
export type ScanPhase = 'scanning' | 'building' | 'saving';
|
export type ScanPhase = 'scanning' | 'building' | 'saving' | 'compressing';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Recursively scan /sd, build a fresh SQLite locate database in memory,
|
* Recursively scan /sd, build a fresh SQLite locate database in memory,
|
||||||
|
|
@ -171,7 +174,7 @@ export type ScanPhase = 'scanning' | 'building' | 'saving';
|
||||||
* saving → bytes of the serialized DB written to the server
|
* saving → bytes of the serialized DB written to the server
|
||||||
*/
|
*/
|
||||||
export async function buildLocateDb(
|
export async function buildLocateDb(
|
||||||
onProgress?: (phase: ScanPhase, value: number) => void,
|
onProgress?: (phase: ScanPhase, value: number, path?: string) => void,
|
||||||
): Promise<{ count: number; bytes: number }> {
|
): Promise<{ count: number; bytes: number }> {
|
||||||
const sqlite3 = await getSqlite3();
|
const sqlite3 = await getSqlite3();
|
||||||
|
|
||||||
|
|
@ -204,7 +207,7 @@ export async function buildLocateDb(
|
||||||
e.lastModified ? Math.floor(e.lastModified.getTime() / 1000) : 0,
|
e.lastModified ? Math.floor(e.lastModified.getTime() / 1000) : 0,
|
||||||
e.type === 'folder' ? 1 : 0,
|
e.type === 'folder' ? 1 : 0,
|
||||||
]).stepReset();
|
]).stepReset();
|
||||||
if (i % 250 === 0) onProgress?.('building', i);
|
if (i % SCAN_PROGRESS_INTERVAL === 0) onProgress?.('building', i, e.path);
|
||||||
}
|
}
|
||||||
} finally {
|
} finally {
|
||||||
stmt.finalize();
|
stmt.finalize();
|
||||||
|
|
@ -234,6 +237,15 @@ export async function buildLocateDb(
|
||||||
await putFileContents(LOCATE_PATH, dbBytes);
|
await putFileContents(LOCATE_PATH, dbBytes);
|
||||||
onProgress?.('saving', dbBytes.length);
|
onProgress?.('saving', dbBytes.length);
|
||||||
|
|
||||||
|
// ── 5. Gzip at level 9 and upload ───────────────────────────────────────
|
||||||
|
onProgress?.('compressing', 0);
|
||||||
|
const gzBytes = await new Promise<Uint8Array>((resolve, reject) =>
|
||||||
|
gzip(dbBytes, { level: 9 }, (err, data) => err ? reject(err) : resolve(data)),
|
||||||
|
);
|
||||||
|
await deletePath(LOCATE_GZ_PATH).catch(() => {});
|
||||||
|
await putFileContents(LOCATE_GZ_PATH, gzBytes);
|
||||||
|
onProgress?.('compressing', gzBytes.length);
|
||||||
|
|
||||||
// Invalidate cached DB so the next search reloads the fresh file.
|
// Invalidate cached DB so the next search reloads the fresh file.
|
||||||
resetLocateDb();
|
resetLocateDb();
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue
Block a user