feat: persist search index and run providers
This commit is contained in:
parent
60909d3e46
commit
5011ddf89c
|
|
@ -15,6 +15,9 @@
|
|||
var SEARCH_DEBOUNCE_MS = 300;
|
||||
var MAX_FILES = 500;
|
||||
var MAX_RESULTS = 100;
|
||||
var INDEX_STORAGE_KEY = 'search-index';
|
||||
var INDEX_VERSION = 1;
|
||||
var SEARCH_COMMAND_ID = 'verstak.search.searchVaultText';
|
||||
|
||||
function injectStyles() {
|
||||
if (document.getElementById('search-style-injected')) return;
|
||||
|
|
@ -149,27 +152,131 @@
|
|||
return found;
|
||||
}
|
||||
|
||||
async function runSearch(api, rootPath, query) {
|
||||
query = String(query || '').trim();
|
||||
if (query.length < 2) return [];
|
||||
async function buildIndex(api, rootPath) {
|
||||
var entries = await collectEntries(api, rootPath);
|
||||
var results = [];
|
||||
for (var i = 0; i < entries.length && results.length < MAX_RESULTS; i++) {
|
||||
var files = [];
|
||||
for (var i = 0; i < entries.length; i++) {
|
||||
var entry = entries[i];
|
||||
var path = entry.relativePath;
|
||||
if (pathMatches(entry, query)) results.push(scanPath(entry));
|
||||
if (!isTextFile(entry) || results.length >= MAX_RESULTS) continue;
|
||||
if (!isTextFile(entry)) continue;
|
||||
try {
|
||||
var text = await api.files.readText(path);
|
||||
var match = scanText(path, String(text || ''), query);
|
||||
if (match) results.push(match);
|
||||
files.push({
|
||||
path: entry.relativePath,
|
||||
text: String(await api.files.readText(entry.relativePath) || '')
|
||||
});
|
||||
} catch (err) {
|
||||
// Ignore unreadable files; search should remain usable on mixed vaults.
|
||||
}
|
||||
}
|
||||
return {
|
||||
version: INDEX_VERSION,
|
||||
workspaceRootPath: cleanPath(rootPath),
|
||||
updatedAt: new Date().toISOString(),
|
||||
entries: entries,
|
||||
files: files
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeIndex(value, rootPath) {
|
||||
if (!value || value.version !== INDEX_VERSION) return null;
|
||||
if (cleanPath(value.workspaceRootPath) !== cleanPath(rootPath)) return null;
|
||||
if (!Array.isArray(value.entries) || !Array.isArray(value.files)) return null;
|
||||
return value;
|
||||
}
|
||||
|
||||
async function readStoredIndex(api, rootPath) {
|
||||
if (!api.storage || !api.storage.data || typeof api.storage.data.read !== 'function') return null;
|
||||
try {
|
||||
return normalizeIndex(await api.storage.data.read(INDEX_STORAGE_KEY), rootPath);
|
||||
} catch (err) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function writeStoredIndex(api, index) {
|
||||
if (!api.storage || !api.storage.data || typeof api.storage.data.write !== 'function') return;
|
||||
try {
|
||||
await api.storage.data.write(INDEX_STORAGE_KEY, index);
|
||||
} catch (err) {
|
||||
// Storage is an optimization for fast reloads; search can still work in memory.
|
||||
}
|
||||
}
|
||||
|
||||
async function loadOrBuildIndex(api, rootPath, currentIndex) {
|
||||
var normalized = normalizeIndex(currentIndex, rootPath);
|
||||
if (normalized) return normalized;
|
||||
var stored = await readStoredIndex(api, rootPath);
|
||||
if (stored) return stored;
|
||||
var index = await buildIndex(api, rootPath);
|
||||
await writeStoredIndex(api, index);
|
||||
return index;
|
||||
}
|
||||
|
||||
function runLocalSearch(index, query) {
|
||||
query = String(query || '').trim();
|
||||
if (query.length < 2) return [];
|
||||
index = index || { entries: [], files: [] };
|
||||
var results = [];
|
||||
var entries = Array.isArray(index.entries) ? index.entries : [];
|
||||
var files = Array.isArray(index.files) ? index.files : [];
|
||||
for (var i = 0; i < entries.length && results.length < MAX_RESULTS; i++) {
|
||||
var entry = entries[i];
|
||||
if (pathMatches(entry, query)) results.push(scanPath(entry));
|
||||
}
|
||||
for (var j = 0; j < files.length && results.length < MAX_RESULTS; j++) {
|
||||
var file = files[j];
|
||||
var match = scanText(file.path, String(file.text || ''), query);
|
||||
if (match) results.push(match);
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
function normalizeProviderResults(provider, value) {
|
||||
var list = Array.isArray(value) ? value : (value && Array.isArray(value.results) ? value.results : []);
|
||||
return list.map(function (item) {
|
||||
return {
|
||||
path: cleanPath(item.path || item.relativePath || item.title || provider.label || provider.id),
|
||||
type: item.type || 'external',
|
||||
matchType: item.matchType || provider.label || 'External match',
|
||||
sourceLabel: provider.label || provider.id || provider.pluginId,
|
||||
openable: item.openable === true,
|
||||
line: item.line || 0,
|
||||
snippet: item.snippet || item.preview || ''
|
||||
};
|
||||
}).filter(function (item) { return item.path; });
|
||||
}
|
||||
|
||||
async function runExternalProviders(api, rootPath, query, remaining) {
|
||||
var output = { results: [], errors: [] };
|
||||
if (remaining <= 0) return output;
|
||||
if (!api.contributions || typeof api.contributions.list !== 'function') return output;
|
||||
if (!api.commands || typeof api.commands.executeFor !== 'function') return output;
|
||||
var providers = [];
|
||||
try {
|
||||
providers = await api.contributions.list('searchProviders');
|
||||
} catch (err) {
|
||||
output.errors.push(err && err.message ? err.message : String(err));
|
||||
return output;
|
||||
}
|
||||
providers = Array.isArray(providers) ? providers : [];
|
||||
for (var i = 0; i < providers.length && output.results.length < remaining; i++) {
|
||||
var provider = providers[i];
|
||||
if (!provider || !provider.handler) continue;
|
||||
if (provider.pluginId === 'verstak.search' || provider.handler === SEARCH_COMMAND_ID) continue;
|
||||
try {
|
||||
var response = await api.commands.executeFor(provider.pluginId, provider.handler, {
|
||||
query: query,
|
||||
workspaceRootPath: rootPath,
|
||||
limit: remaining - output.results.length
|
||||
});
|
||||
var normalized = normalizeProviderResults(provider, response && response.result);
|
||||
output.results = output.results.concat(normalized.slice(0, remaining - output.results.length));
|
||||
} catch (err) {
|
||||
output.errors.push(err && err.message ? err.message : String(err));
|
||||
}
|
||||
}
|
||||
return output;
|
||||
}
|
||||
|
||||
var SearchView = {
|
||||
mount: function (containerEl, props, api) {
|
||||
injectStyles();
|
||||
|
|
@ -177,6 +284,9 @@
|
|||
var state = { query: '', searching: false, results: [], status: 'Enter at least 2 characters.', error: '' };
|
||||
var searchTimer = null;
|
||||
var searchSeq = 0;
|
||||
var index = null;
|
||||
var cleanupFns = [];
|
||||
var indexRefresh = Promise.resolve();
|
||||
|
||||
function render() {
|
||||
containerEl.innerHTML = '';
|
||||
|
|
@ -219,7 +329,9 @@
|
|||
el('div', {}, [
|
||||
el('div', { className: 'search-path' }, [result.path]),
|
||||
el('div', { className: 'search-snippet' }, [result.snippet]),
|
||||
el('div', { className: 'search-meta' }, [result.matchType + (result.line ? ' - Line ' + result.line : '')])
|
||||
el('div', { className: 'search-meta' }, [
|
||||
(result.sourceLabel ? result.sourceLabel + ' - ' : '') + result.matchType + (result.line ? ' - Line ' + result.line : '')
|
||||
])
|
||||
]),
|
||||
result.openable ? el('button', {
|
||||
className: 'search-btn',
|
||||
|
|
@ -275,10 +387,15 @@
|
|||
searchSeq = seq;
|
||||
render();
|
||||
try {
|
||||
var results = await runSearch(api, rootPath, state.query);
|
||||
await indexRefresh;
|
||||
var results = await searchVaultText({ query: state.query, limit: MAX_RESULTS });
|
||||
var external = await runExternalProviders(api, rootPath, state.query, MAX_RESULTS - results.length);
|
||||
if (seq !== searchSeq) return;
|
||||
state.results = results;
|
||||
state.results = results.concat(external.results);
|
||||
state.status = state.results.length + ' result' + (state.results.length === 1 ? '' : 's');
|
||||
if (external.errors.length) {
|
||||
state.status += ' - ' + external.errors.join('; ');
|
||||
}
|
||||
} catch (err) {
|
||||
if (seq !== searchSeq) return;
|
||||
state.results = [];
|
||||
|
|
@ -290,10 +407,58 @@
|
|||
}
|
||||
}
|
||||
|
||||
async function searchVaultText(args) {
|
||||
var query = args && args.query != null ? args.query : state.query;
|
||||
var limit = args && args.limit ? Number(args.limit) : MAX_RESULTS;
|
||||
index = await loadOrBuildIndex(api, rootPath, index);
|
||||
return runLocalSearch(index, query).slice(0, limit > 0 ? limit : MAX_RESULTS);
|
||||
}
|
||||
|
||||
async function refreshIndex() {
|
||||
index = await buildIndex(api, rootPath);
|
||||
await writeStoredIndex(api, index);
|
||||
return index;
|
||||
}
|
||||
|
||||
function handleFileChanged(event) {
|
||||
var payload = (event && event.payload) || event || {};
|
||||
var changedPath = cleanPath(payload.relativePath || payload.path || '');
|
||||
if (changedPath && rootPath && changedPath !== rootPath && changedPath.indexOf(rootPath + '/') !== 0) return;
|
||||
indexRefresh = refreshIndex().catch(function (err) {
|
||||
console.error('[search] refresh index:', err);
|
||||
});
|
||||
return indexRefresh;
|
||||
}
|
||||
|
||||
function setupIntegrations() {
|
||||
if (api.commands && typeof api.commands.register === 'function') {
|
||||
api.commands.register(SEARCH_COMMAND_ID, searchVaultText).then(function (unregister) {
|
||||
if (typeof unregister === 'function') cleanupFns.push(unregister);
|
||||
}).catch(function (err) {
|
||||
console.error('[search] register command:', err);
|
||||
});
|
||||
}
|
||||
if (api.events && typeof api.events.subscribe === 'function') {
|
||||
api.events.subscribe('file.changed', handleFileChanged).then(function (unsubscribe) {
|
||||
if (typeof unsubscribe === 'function') cleanupFns.push(unsubscribe);
|
||||
}).catch(function (err) {
|
||||
console.error('[search] subscribe file.changed:', err);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
setupIntegrations();
|
||||
render();
|
||||
containerEl.__verstakSearchCleanup = function () {
|
||||
if (searchTimer) clearTimeout(searchTimer);
|
||||
searchSeq += 1;
|
||||
while (cleanupFns.length) {
|
||||
try {
|
||||
cleanupFns.pop()();
|
||||
} catch (err) {
|
||||
console.error('[search] cleanup:', err);
|
||||
}
|
||||
}
|
||||
};
|
||||
},
|
||||
unmount: function (containerEl) {
|
||||
|
|
|
|||
|
|
@ -18,7 +18,10 @@
|
|||
"permissions": [
|
||||
"files.read",
|
||||
"workbench.open",
|
||||
"ui.register"
|
||||
"ui.register",
|
||||
"storage.namespace",
|
||||
"events.subscribe",
|
||||
"commands.register"
|
||||
],
|
||||
"frontend": {
|
||||
"entry": "frontend/src/index.js"
|
||||
|
|
@ -32,11 +35,18 @@
|
|||
"component": "SearchView"
|
||||
}
|
||||
],
|
||||
"commands": [
|
||||
{
|
||||
"id": "verstak.search.searchVaultText",
|
||||
"title": "Search Vault Text",
|
||||
"handler": "verstak.search.searchVaultText"
|
||||
}
|
||||
],
|
||||
"searchProviders": [
|
||||
{
|
||||
"id": "verstak.search.vault-text",
|
||||
"label": "Vault Text Search",
|
||||
"handler": "searchVaultText"
|
||||
"handler": "verstak.search.searchVaultText"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,7 +5,9 @@ const vm = require('vm');
|
|||
|
||||
const root = path.resolve(__dirname, '..');
|
||||
const sourcePath = path.join(root, 'plugins', 'search', 'frontend', 'src', 'index.js');
|
||||
const manifestPath = path.join(root, 'plugins', 'search', 'plugin.json');
|
||||
const source = fs.readFileSync(sourcePath, 'utf8');
|
||||
const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8'));
|
||||
|
||||
class FakeNode {
|
||||
constructor(tagName) {
|
||||
|
|
@ -129,7 +131,69 @@ async function wait(ms) {
|
|||
const document = makeDocument();
|
||||
const component = loadComponent(document);
|
||||
const opened = [];
|
||||
const fileContents = {
|
||||
'Project/Docs/case.md': '# Case\nTarget phrase is here.\n',
|
||||
'Project/Docs/notes.txt': 'No match here.\n',
|
||||
};
|
||||
const pluginData = {};
|
||||
const commandHandlers = new Map();
|
||||
const eventHandlers = {};
|
||||
const providerCalls = [];
|
||||
const api = {
|
||||
storage: {
|
||||
data: {
|
||||
read: async (name) => pluginData[name] || {},
|
||||
write: async (name, data) => {
|
||||
pluginData[name] = JSON.parse(JSON.stringify(data || {}));
|
||||
},
|
||||
},
|
||||
},
|
||||
commands: {
|
||||
register: async (commandId, handler) => {
|
||||
commandHandlers.set(commandId, handler);
|
||||
return () => commandHandlers.delete(commandId);
|
||||
},
|
||||
executeFor: async (pluginId, commandId, args) => {
|
||||
providerCalls.push({ pluginId, commandId, args });
|
||||
if (pluginId === 'external.notes' && commandId === 'external.notes.search') {
|
||||
return {
|
||||
status: 'handled',
|
||||
pluginId,
|
||||
commandId,
|
||||
result: [{
|
||||
path: 'Project/External/target.note',
|
||||
type: 'note',
|
||||
matchType: 'External note',
|
||||
snippet: 'External provider target result',
|
||||
openable: false,
|
||||
}],
|
||||
};
|
||||
}
|
||||
if (pluginId === 'broken.provider') {
|
||||
throw new Error('provider unavailable');
|
||||
}
|
||||
throw new Error(`unexpected provider call ${pluginId}:${commandId}`);
|
||||
},
|
||||
},
|
||||
contributions: {
|
||||
list: async (point) => {
|
||||
if (point !== 'searchProviders') return [];
|
||||
return [
|
||||
{ pluginId: 'verstak.search', id: 'verstak.search.vault-text', label: 'Vault Text Search', handler: 'verstak.search.searchVaultText' },
|
||||
{ pluginId: 'external.notes', id: 'external.notes.search', label: 'External Notes', handler: 'external.notes.search' },
|
||||
{ pluginId: 'broken.provider', id: 'broken.provider.search', label: 'Broken Provider', handler: 'broken.provider.search' },
|
||||
];
|
||||
},
|
||||
},
|
||||
events: {
|
||||
subscribe: async (eventName, handler) => {
|
||||
eventHandlers[eventName] = eventHandlers[eventName] || [];
|
||||
eventHandlers[eventName].push(handler);
|
||||
return () => {
|
||||
eventHandlers[eventName] = (eventHandlers[eventName] || []).filter((candidate) => candidate !== handler);
|
||||
};
|
||||
},
|
||||
},
|
||||
files: {
|
||||
list: async (relativeDir) => {
|
||||
if (relativeDir === 'Project') {
|
||||
|
|
@ -148,8 +212,7 @@ async function wait(ms) {
|
|||
return [];
|
||||
},
|
||||
readText: async (relativePath) => {
|
||||
if (relativePath === 'Project/Docs/case.md') return '# Case\nTarget phrase is here.\n';
|
||||
if (relativePath === 'Project/Docs/notes.txt') return 'No match here.\n';
|
||||
if (Object.prototype.hasOwnProperty.call(fileContents, relativePath)) return fileContents[relativePath];
|
||||
throw new Error('unexpected readText path ' + relativePath);
|
||||
},
|
||||
},
|
||||
|
|
@ -164,6 +227,16 @@ async function wait(ms) {
|
|||
component.mount(container, { workspaceRootPath: 'Project' }, api);
|
||||
await flush();
|
||||
|
||||
if (!commandHandlers.has('verstak.search.searchVaultText')) throw new Error('search provider command was not registered');
|
||||
if (!eventHandlers['file.changed'] || eventHandlers['file.changed'].length !== 1) throw new Error('file.changed subscription was not registered');
|
||||
if (!manifest.permissions.includes('storage.namespace')) throw new Error('search manifest must request storage.namespace');
|
||||
if (!manifest.permissions.includes('events.subscribe')) throw new Error('search manifest must request events.subscribe');
|
||||
if (!manifest.permissions.includes('commands.register')) throw new Error('search manifest must request commands.register');
|
||||
const command = (manifest.contributes.commands || []).find((item) => item.id === 'verstak.search.searchVaultText');
|
||||
if (!command || command.handler !== 'verstak.search.searchVaultText') throw new Error('search command contribution is missing');
|
||||
const provider = (manifest.contributes.searchProviders || []).find((item) => item.id === 'verstak.search.vault-text');
|
||||
if (!provider || provider.handler !== 'verstak.search.searchVaultText') throw new Error('search provider must point at the command handler');
|
||||
|
||||
function queryInput() {
|
||||
const input = walk(container, (node) => node.getAttribute && node.getAttribute('data-search-input') === 'query');
|
||||
if (!input) throw new Error('query input not found');
|
||||
|
|
@ -178,8 +251,13 @@ async function wait(ms) {
|
|||
if (!container.textContent.includes('Project/Docs/case.md')) throw new Error('typing should search file contents');
|
||||
if (!container.textContent.includes('Target phrase is here')) throw new Error('typing should render content snippet');
|
||||
if (!container.textContent.includes('Project/Target Assets')) throw new Error('typing should search folder paths');
|
||||
if (!container.textContent.includes('Project/External/target.note')) throw new Error('external provider result should be rendered');
|
||||
if (!container.textContent.includes('External Notes')) throw new Error('external provider label should be rendered');
|
||||
if (!container.textContent.includes('provider unavailable')) throw new Error('provider failure should be reported without failing search');
|
||||
if (!container.textContent.includes('Content match')) throw new Error('content result type was not rendered');
|
||||
if (!container.textContent.includes('Folder name')) throw new Error('folder result type was not rendered');
|
||||
if (!pluginData['search-index'] || !Array.isArray(pluginData['search-index'].files)) throw new Error('search index was not written to plugin data storage');
|
||||
if (providerCalls.some((call) => call.pluginId === 'verstak.search')) throw new Error('search must not call itself as an external provider');
|
||||
|
||||
input = queryInput();
|
||||
input.value = 'image';
|
||||
|
|
@ -201,6 +279,27 @@ async function wait(ms) {
|
|||
if (!container.textContent.includes('Target phrase is here')) throw new Error('matching snippet was not rendered');
|
||||
if (container.textContent.includes('image.png')) throw new Error('binary image file should not be rendered as a result');
|
||||
|
||||
fileContents['Project/Docs/notes.txt'] = 'Edited target appears after file change.\n';
|
||||
eventHandlers['file.changed'].forEach((handler) => handler({
|
||||
type: 'file.changed',
|
||||
payload: { relativePath: 'Project/Docs/notes.txt', changeType: 'write' },
|
||||
}));
|
||||
await flush();
|
||||
|
||||
input = queryInput();
|
||||
input.value = 'edited target';
|
||||
input.dispatchEvent('input');
|
||||
button.click();
|
||||
await flush();
|
||||
|
||||
if (!container.textContent.includes('Project/Docs/notes.txt')) throw new Error('file.changed should refresh the persisted search index');
|
||||
|
||||
input = queryInput();
|
||||
input.value = 'target';
|
||||
input.dispatchEvent('input');
|
||||
button.click();
|
||||
await flush();
|
||||
|
||||
const openButton = walk(container, (node) => node.getAttribute && node.getAttribute('data-search-open') === 'Project/Docs/case.md');
|
||||
if (!openButton) throw new Error('result open button not found');
|
||||
openButton.click();
|
||||
|
|
|
|||
Loading…
Reference in New Issue