From bbd38746df56e0021d864cf73c75605255bca23c Mon Sep 17 00:00:00 2001 From: Jaime Idolpx Date: Thu, 18 Jun 2026 01:44:06 -0400 Subject: [PATCH] feat(locate-db): enhance database build process with directory tracking and status updates --- src/app/locate-db.ts | 95 ++++++++++++++++++++++++++++++++++++++------ 1 file changed, 83 insertions(+), 12 deletions(-) diff --git a/src/app/locate-db.ts b/src/app/locate-db.ts index 35c1c09..ab54e7b 100644 --- a/src/app/locate-db.ts +++ b/src/app/locate-db.ts @@ -6,6 +6,11 @@ import { getWebDAVBaseUrl, basename, listDirectory, putFileContents, deletePath const LOCATE_PATH = '/sd/.locate'; const LOCATE_GZ_PATH = '/sd/.locate.gz'; const SCAN_PROGRESS_INTERVAL = 50; + +function dirPath(path: string): string { + const i = path.lastIndexOf('/'); + return i > 0 ? path.slice(0, i) : '/'; +} // Vite rewrites this `?url` import to the hashed asset path // (e.g. /assets/sqlite3-BVKGSWc-.wasm) and respects the configured base path. const WASM_URL: string = wasmUrl; @@ -190,10 +195,12 @@ export async function buildLocateDb( let totalBytes = 0; let dirCount = 0; let fileCount = 0; + let lastFolder = '/sd'; while (queue.length > 0) { signal?.throwIfAborted(); const dir = queue.shift()!; + lastFolder = dir; let prevBytes = 0; const batch = await listDirectory(dir, false, bytes => { totalBytes += bytes - prevBytes; @@ -227,28 +234,77 @@ export async function buildLocateDb( signal?.throwIfAborted(); // ── 2. Build in-memory DB ─────────────────────────────────────────────── + const scanStart = Date.now(); const db = new sqlite3.oo1.DB(':memory:', 'ct'); db.exec( - 'CREATE TABLE files (' + - ' path TEXT PRIMARY KEY,' + - ' size INTEGER NOT NULL DEFAULT 0,' + - ' mtime INTEGER NOT NULL DEFAULT 0,' + - ' is_dir INTEGER NOT NULL DEFAULT 0' + + 'CREATE TABLE dirs (' + + ' id INTEGER PRIMARY KEY,' + + ' path TEXT NOT NULL UNIQUE,' + + ' scanned INTEGER NOT NULL DEFAULT 0' + ');' + - 'CREATE INDEX IF NOT EXISTS idx_path ON files(path);', + 'CREATE TABLE files (' + + ' id INTEGER PRIMARY KEY,' + + ' dir_id INTEGER NOT NULL,' + + ' name TEXT NOT NULL,' + + ' size INTEGER NOT NULL DEFAULT 0,' + + ' mtime INTEGER NOT NULL DEFAULT 0,' + + ' is_dir INTEGER NOT NULL DEFAULT 0,' + + ' UNIQUE(dir_id, name)' + + ');' + + 'CREATE INDEX files_dir_idx ON files(dir_id);' + + 'CREATE VIRTUAL TABLE files_fts USING fts5(' + + " path, content='', tokenize=\"unicode61\"" + + ');' + + 'CREATE TABLE status (' + + ' id INTEGER PRIMARY KEY DEFAULT 1,' + + ' total_dirs INTEGER NOT NULL DEFAULT 0,' + + ' total_files INTEGER NOT NULL DEFAULT 0,' + + ' last_scan INTEGER NOT NULL DEFAULT 0,' + + ' duration INTEGER NOT NULL DEFAULT 0,' + + " last_folder TEXT NOT NULL DEFAULT ''" + + ');', ); + // ── 3. Insert all unique directory paths ──────────────────────────────── + const allDirPaths = new Set(['/sd']); + for (const e of entries) { + allDirPaths.add(dirPath(e.path)); + if (e.type === 'folder') allDirPaths.add(e.path); + } db.exec('BEGIN'); - const stmt = db.prepare('INSERT OR REPLACE INTO files VALUES (?,?,?,?)'); + const stmtDir = db.prepare('INSERT OR IGNORE INTO dirs(path, scanned) VALUES (?, 1)'); + try { + for (const p of allDirPaths) stmtDir.bind([p]).stepReset(); + } finally { stmtDir.finalize(); } + db.exec('COMMIT'); + + const dirIdMap = new Map(); + db.exec({ + sql: 'SELECT id, path FROM dirs', + rowMode: 'array', + callback: (row: any[]) => dirIdMap.set(row[1] as string, row[0] as number), + }); + + // ── 4. Insert files + FTS ──────────────────────────────────────────────── + db.exec('BEGIN'); + const stmtFile = db.prepare( + 'INSERT INTO files(dir_id, name, size, mtime, is_dir) VALUES (?,?,?,?,?)', + ); + const stmtFts = db.prepare('INSERT INTO files_fts(rowid, path) VALUES (?,?)'); try { for (let i = 0; i < entries.length; i++) { const e = entries[i]; - stmt.bind([ - e.path, + const dirId = dirIdMap.get(dirPath(e.path)); + if (dirId === undefined) continue; + stmtFile.bind([ + dirId, + basename(e.path) || e.path, e.size, e.lastModified ? Math.floor(e.lastModified.getTime() / 1000) : 0, e.type === 'folder' ? 1 : 0, ]).stepReset(); + const rowid = sqlite3.capi.sqlite3_last_insert_rowid(db.pointer); + stmtFts.bind([rowid, e.path]).stepReset(); if (i % SCAN_PROGRESS_INTERVAL === 0) { signal?.throwIfAborted(); onProgress?.('building', i, e.path, { dirs: dirCount, files: fileCount }); @@ -256,11 +312,24 @@ export async function buildLocateDb( } } } finally { - stmt.finalize(); + stmtFile.finalize(); + stmtFts.finalize(); } db.exec('COMMIT'); signal?.throwIfAborted(); - onProgress?.('building', entries.length); + onProgress?.('building', entries.length, undefined, { dirs: dirCount, files: fileCount }); + + // ── 5. Write status ────────────────────────────────────────────────────── + db.exec({ + sql: 'INSERT OR REPLACE INTO status(id,total_dirs,total_files,last_scan,duration,last_folder) VALUES (1,?,?,?,?,?)', + bind: [ + dirCount, + fileCount, + Math.floor(Date.now() / 1000), + Math.floor((Date.now() - scanStart) / 1000), + lastFolder, + ], + }); // ── 3. Serialize to bytes ──────────────────────────────────────────────── const pSaved = sqlite3.wasm.pstack.pointer; @@ -331,7 +400,9 @@ export function searchLocate(query: string, limit = 500): LocateEntry[] { const needle = toSqlLike(query); const rows: LocateEntry[] = []; _db.exec({ - sql: "SELECT path, size, mtime, is_dir FROM files WHERE path LIKE ? ESCAPE '\\' LIMIT ?", + sql: "SELECT d.path || '/' || f.name, f.size, f.mtime, f.is_dir " + + "FROM files f JOIN dirs d ON f.dir_id = d.id " + + "WHERE (d.path || '/' || f.name) LIKE ? ESCAPE '\\' LIMIT ?", bind: [needle, limit], rowMode: 'array', callback: (row: any[]) => {