feat: Define and export SnippetId type

packages/cli/src/cmds/search/search.ts

@@ -16,7 +16,7 @@ import {
 } from '../../fulltext/FindEvents';
 import { openInBrowser } from '../open/openers';
 import { buildAppMapIndex, search } from '../../fulltext/appmap-index';
-import buildIndex from '../../rpc/explain/buildIndex';
+import buildIndexInTempDir from '../../rpc/explain/build-index-in-temp-dir';
 
 export const command = 'search <query>';
 export const describe =
@@ -183,7 +183,7 @@ export const handler = async (argv: ArgumentTypes) => {
       maxResults,
     };
 
-    const index = await buildIndex('appmaps', async (indexFile) => {
+    const index = await buildIndexInTempDir('appmaps', async (indexFile) => {
       const db = new sqlite3(indexFile);
       const fileIndex = new FileIndex(db);
       await buildAppMapIndex(fileIndex, [process.cwd()]);

packages/cli/src/fulltext/FindEvents.ts

@@ -7,7 +7,7 @@ import assert from 'assert';
 
 import { verbose } from '../utils';
 import { collectParameters } from './collectParameters';
-import { fileNameMatchesFilterPatterns } from './fileNameMatchesFilterPatterns';
+import { fileNameMatchesFilterPatterns } from './filter-patterns';
 
 type IndexItem = {
   fqid: string;

packages/cli/src/rpc/explain/EventCollector.ts

@@ -8,7 +8,7 @@ import FindEvents, {
   SearchOptions,
 } from '../../fulltext/FindEvents';
 import buildContext from './buildContext';
-import { textSearchResultToRpcSearchResult } from './collectContext';
+import { textSearchResultToRpcSearchResult } from './textSearchResultToRpcSearchResult';
 
 export default class EventCollector {
   appmapIndexes = new Map<string, FindEvents>();

packages/cli/src/rpc/explain/SearchContextCollector.ts

@@ -11,7 +11,7 @@ import EventCollector from './EventCollector';
 import indexFiles from './index-files';
 import indexSnippets from './index-snippets';
 import collectSnippets from './collect-snippets';
-import buildIndex from './buildIndex';
+import buildIndexInTempDir from './build-index-in-temp-dir';
 import { buildAppMapIndex, search } from '../../fulltext/appmap-index';
 
 export default class SearchContextCollector {
@@ -57,7 +57,7 @@ export default class SearchContextCollector {
         numResults: this.appmaps.length,
       };
     } else {
-      const appmapIndex = await buildIndex('appmaps', async (indexFile) => {
+      const appmapIndex = await buildIndexInTempDir('appmaps', async (indexFile) => {
         const db = new sqlite3(indexFile);
         const fileIndex = new FileIndex(db);
         await buildAppMapIndex(fileIndex, this.appmapDirectories);
@@ -80,7 +80,7 @@ export default class SearchContextCollector {
       log(`[search-context] Matched ${selectedAppMaps.results.length} AppMaps.`);
     }
 
-    const fileIndex = await buildIndex('files', async (indexFile) => {
+    const fileIndex = await buildIndexInTempDir('files', async (indexFile) => {
       const db = new sqlite3(indexFile);
       return await indexFiles(
         db,
@@ -96,7 +96,7 @@ export default class SearchContextCollector {
       fileIndex.close();
     }
 
-    const snippetIndex = await buildIndex('snippets', async (indexFile) => {
+    const snippetIndex = await buildIndexInTempDir('snippets', async (indexFile) => {
       const db = new sqlite3(indexFile);
       return await indexSnippets(db, fileSearchResults);
     });

packages/cli/src/rpc/explain/appmap-location.ts

@@ -0,0 +1,8 @@
+import { SearchRpc } from '@appland/rpc';
+
+export default function appmapLocation(appmap: string, event?: SearchRpc.EventMatch): string {
+  const appmapFile = [appmap, 'appmap.json'].join('.');
+  const tokens = [appmapFile];
+  if (event?.eventIds.length) tokens.push(String(event.eventIds[0]));
+  return tokens.join(':');
+}

packages/cli/src/rpc/explain/buildIndex.ts → packages/cli/src/rpc/explain/build-index-in-temp-dir.ts

@@ -15,7 +15,7 @@ type CloseableIndex<T> = {
   close: () => void;
 };
 
-export default async function buildIndex<T extends Closeable>(
+export default async function buildIndexInTempDir<T extends Closeable>(
   indexName: string,
   builder: (indexFile: string) => Promise<T>
 ): Promise<CloseableIndex<T>> {

packages/cli/src/rpc/explain/build-sequence-diagram.ts

@@ -0,0 +1,31 @@
+import { AppMapFilter, serializeFilter } from '@appland/models';
+import { SearchRpc } from '@appland/rpc';
+import assert from 'assert';
+
+import { handler as sequenceDiagramHandler } from '../appmap/sequenceDiagram';
+import { ContextV2 } from '@appland/navie';
+import appmapLocation from './appmap-location';
+
+export default async function buildSequenceDiagram(
+  result: SearchRpc.SearchResult
+): Promise<ContextV2.FileContextItem> {
+  const codeObjects = result.events.map((event) => event.fqid);
+  const appmapFilter = new AppMapFilter();
+  appmapFilter.declutter.context.on = true;
+  appmapFilter.declutter.context.names = codeObjects;
+  const filterState = serializeFilter(appmapFilter);
+
+  const plantUML = await sequenceDiagramHandler(result.appmap, {
+    filter: filterState,
+    format: 'plantuml',
+    formatOptions: { disableMarkup: true },
+  });
+  assert(typeof plantUML === 'string');
+  return {
+    directory: result.directory,
+    location: appmapLocation(result.appmap),
+    type: ContextV2.ContextItemType.SequenceDiagram,
+    content: plantUML,
+    score: result.score,
+  };
+}

packages/cli/src/rpc/explain/buildContext.ts

@@ -1,11 +1,9 @@
 import { SearchRpc } from '@appland/rpc';
-import { AppMapFilter, serializeFilter } from '@appland/models';
-import assert from 'assert';
 
-import { handler as sequenceDiagramHandler } from '../appmap/sequenceDiagram';
 import lookupSourceCode from './lookupSourceCode';
 import { warn } from 'console';
 import { ContextV2 } from '@appland/navie';
+import buildSequenceDiagram from './build-sequence-diagram';
 
 /**
  * Processes search results to build sequence diagrams, code snippets, and code object sets. This is the format
@@ -40,32 +38,11 @@ export default async function buildContext(
     return tokens.join(':');
   };
 
-  const buildSequenceDiagram = async (result: SearchRpc.SearchResult) => {
-    const codeObjects = result.events.map((event) => event.fqid);
-    const appmapFilter = new AppMapFilter();
-    appmapFilter.declutter.context.on = true;
-    appmapFilter.declutter.context.names = codeObjects;
-    const filterState = serializeFilter(appmapFilter);
-
-    const plantUML = await sequenceDiagramHandler(result.appmap, {
-      filter: filterState,
-      format: 'plantuml',
-      formatOptions: { disableMarkup: true },
-    });
-    assert(typeof plantUML === 'string');
-    sequenceDiagrams.push({
-      directory: result.directory,
-      location: appmapLocation(result.appmap),
-      type: ContextV2.ContextItemType.SequenceDiagram,
-      content: plantUML,
-      score: result.score,
-    });
-  };
-
   const examinedLocations = new Set<string>();
   for (const result of searchResults) {
     try {
-      await buildSequenceDiagram(result);
+      const diagram = await buildSequenceDiagram(result);
+      sequenceDiagrams.push(diagram);
     } catch (e) {
       warn(`Failed to build sequence diagram for ${result.appmap}`);
       warn(e);

packages/cli/src/rpc/explain/collect-snippets.ts

@@ -1,5 +1,5 @@
 import { ContextV2 } from '@appland/navie';
-import { SnippetIndex, SnippetSearchResult } from '@appland/search';
+import { parseFileChunkSnippetId, SnippetIndex, SnippetSearchResult } from '@appland/search';
 import { CHARS_PER_SNIPPET } from './collectContext';
 
 export default function collectSnippets(
@@ -10,7 +10,9 @@ export default function collectSnippets(
   const snippets = snippetIndex.searchSnippets(query, Math.round(charLimit / CHARS_PER_SNIPPET));
 
   const buildLocation = (result: SnippetSearchResult) => {
-    return `${result.filePath}:${result.startLine}-${result.endLine}`;
+    const snippetId = parseFileChunkSnippetId(result.snippetId);
+    const { filePath, startLine } = snippetId;
+    return [filePath, startLine].filter(Boolean).join(':');
   };
 
   return snippets.map((snippet) => ({

packages/cli/src/rpc/explain/collectContext.ts

@@ -2,7 +2,6 @@ import { ContextV2 } from '@appland/navie';
 import { SearchRpc } from '@appland/rpc';
 import { queryKeywords } from '@appland/search';
 
-import { SearchResult as EventSearchResult } from '../../fulltext/FindEvents';
 import Location from './location';
 import SearchContextCollector from './SearchContextCollector';
 import LocationContextCollector from './LocationContextCollector';
@@ -15,19 +14,6 @@ export const buildExclusionPattern = (dirName: string): RegExp => {
 
 const EXCLUDE_DIRS = ['.appmap', '.navie', '.yarn', 'venv', '.venv', 'node_modules', 'vendor'];
 
-export function textSearchResultToRpcSearchResult(
-  eventResult: EventSearchResult
-): SearchRpc.EventMatch {
-  const result: SearchRpc.EventMatch = {
-    fqid: eventResult.fqid,
-    score: eventResult.score,
-    eventIds: eventResult.eventIds,
-  };
-  if (eventResult.location) result.location = eventResult.location;
-  if (eventResult.elapsed) result.elapsed = eventResult.elapsed;
-  return result;
-}
-
 export const CHARS_PER_SNIPPET = 50;
 
 export class ContextCollector {

packages/cli/src/rpc/explain/fileFilter.ts

@@ -0,0 +1,29 @@
+import { FilterFn, isBinaryFile, isDataFile, isLargeFile } from '@appland/search';
+import makeDebug from 'debug';
+import { fileNameMatchesFilterPatterns } from '../../fulltext/filter-patterns';
+
+const debug = makeDebug('appmap:rpc:explain:file-filter');
+
+export default function fileFilter(
+  includePatterns: RegExp[] | undefined,
+  excludePatterns: RegExp[] | undefined
+): FilterFn {
+  return async (path: string) => {
+    debug('Filtering: %s', path);
+    if (isBinaryFile(path)) {
+      debug('Skipping binary file: %s', path);
+      return false;
+    }
+
+    const includeFile = fileNameMatchesFilterPatterns(path, includePatterns, excludePatterns);
+    if (!includeFile) return false;
+
+    const isData = isDataFile(path);
+    if (isData && (await isLargeFile(path))) {
+      debug('Skipping large data file: %s', path);
+      return false;
+    }
+
+    return true;
+  };
+}

packages/cli/src/rpc/explain/index-files.ts

@@ -5,41 +5,13 @@ import {
   buildFileIndex,
   FileIndex,
   fileTokens,
-  FilterFn,
-  isBinaryFile,
-  isDataFile,
-  isLargeFile,
   listProjectFiles,
   readFileSafe,
 } from '@appland/search';
-import { fileNameMatchesFilterPatterns } from '../../fulltext/fileNameMatchesFilterPatterns';
+import fileFilter from './fileFilter';
 
 const debug = makeDebug('appmap:rpc:explain:index-files');
 
-function fileFilter(
-  includePatterns: RegExp[] | undefined,
-  excludePatterns: RegExp[] | undefined
-): FilterFn {
-  return async (path: string) => {
-    debug('Filtering: %s', path);
-    if (isBinaryFile(path)) {
-      debug('Skipping binary file: %s', path);
-      return false;
-    }
-
-    const includeFile = fileNameMatchesFilterPatterns(path, includePatterns, excludePatterns);
-    if (!includeFile) return false;
-
-    const isData = isDataFile(path);
-    if (isData && (await isLargeFile(path))) {
-      debug('Skipping large data file: %s', path);
-      return false;
-    }
-
-    return true;
-  };
-}
-
 export default async function indexFiles(
   db: sqlite3.Database,
   directories: string[],

packages/cli/src/rpc/explain/textSearchResultToRpcSearchResult.ts

@@ -0,0 +1,16 @@
+import { SearchRpc } from '@appland/rpc';
+import { SearchResult as EventSearchResult } from '../../fulltext/FindEvents';
+
+export function textSearchResultToRpcSearchResult(
+  eventResult: EventSearchResult
+): SearchRpc.EventMatch {
+  const result: SearchRpc.EventMatch = {
+    fqid: eventResult.fqid,
+    score: eventResult.score,
+    eventIds: eventResult.eventIds,
+  };
+  if (eventResult.location) result.location = eventResult.location;
+  if (eventResult.elapsed) result.elapsed = eventResult.elapsed;
+  return result;
+}
+

packages/cli/src/rpc/search/search.ts

@@ -8,7 +8,7 @@ import { SearchResponse } from '../../fulltext/appmap-match';
 import { search as searchAppMaps } from '../../fulltext/appmap-index';
 import searchSingleAppMap from '../../cmds/search/searchSingleAppMap';
 import configuration, { AppMapDirectory } from '../configuration';
-import buildIndex from '../explain/buildIndex';
+import buildIndexInTempDir from '../explain/build-index-in-temp-dir';
 import { buildAppMapIndex } from '../../fulltext/appmap-index';
 
 export const DEFAULT_MAX_DIAGRAMS = 10;
@@ -59,7 +59,7 @@ export async function handler(
   } else {
     // Search across all AppMaps, creating a map from AppMap id to AppMapSearchResult
     const maxResults = options.maxDiagrams || options.maxResults || DEFAULT_MAX_DIAGRAMS;
-    const index = await buildIndex('appmaps', async (indexFile) => {
+    const index = await buildIndexInTempDir('appmaps', async (indexFile) => {
       const db = new sqlite3(indexFile);
       const fileIndex = new FileIndex(db);
       await buildAppMapIndex(

packages/cli/tests/unit/rpc/explain/EventCollector.spec.ts

@@ -1,7 +1,7 @@
 import { SearchRpc } from '@appland/rpc';
 import { join } from 'path';
 
-import { textSearchResultToRpcSearchResult } from '../../../../src/rpc/explain/collectContext';
+import { textSearchResultToRpcSearchResult } from '../../../../src/rpc/explain/textSearchResultToRpcSearchResult';
 import buildContext from '../../../../src/rpc/explain/buildContext';
 import { SearchResponse as AppMapSearchResponse } from '../../../../src/fulltext/appmap-match';
 import FindEvents, {

packages/search/src/build-snippet-index.ts

@@ -1,7 +1,7 @@
 import { isAbsolute, join } from 'path';
 import { Tokenizer } from './build-file-index';
 import { ContentReader } from './ioutil';
-import SnippetIndex from './snippet-index';
+import SnippetIndex, { fileChunkSnippetId } from './snippet-index';
 import { Splitter } from './splitter';
 
 export type File = {
@@ -25,15 +25,12 @@ async function indexFile(context: Context, file: File) {
   const extension = file.filePath.split('.').pop() || '';
   const chunks = await context.splitter(fileContent, extension);
 
-  chunks.forEach((chunk, index) => {
-    const snippetId = `${filePath}:${index}`;
-    const { content, startLine, endLine } = chunk;
+  chunks.forEach((chunk) => {
+    const { content, startLine } = chunk;
+    const snippetId = fileChunkSnippetId(filePath, startLine);
     context.snippetIndex.indexSnippet(
       snippetId,
       file.directory,
-      file.filePath,
-      startLine,
-      endLine,
       context.tokenizer(content, file.filePath).symbols.join(' '),
       context.tokenizer(content, file.filePath).words.join(' '),
       content

packages/search/src/cli.ts

@@ -80,15 +80,15 @@ const cli = yargs(hideBin(process.argv))
         return `.../${parts.slice(-3).join('/')}`;
       };
 
-      const printResult = (filePath: string, score: number) =>
-        console.log('%s   %s', filePathAtMostThreeEntries(filePath), score.toPrecision(3));
+      const printResult = (type: string, id: string, score: number) =>
+        console.log('%s %s   %s', type, filePathAtMostThreeEntries(id), score.toPrecision(3));
 
       console.log('File search results');
       console.log('-------------------');
       const fileSearchResults = fileIndex.search(query as string);
       for (const result of fileSearchResults) {
         const { filePath, score } = result;
-        printResult(filePath, score);
+        printResult('file', filePath, score);
       }
 
       const splitter = langchainSplitter;
@@ -104,8 +104,11 @@ const cli = yargs(hideBin(process.argv))
 
       const snippetSearchResults = snippetIndex.searchSnippets(query as string);
       for (const result of snippetSearchResults) {
-        const { snippetId, filePath, startLine, endLine, score } = result;
-        printResult(snippetId, score);
+        const { snippetId, score } = result;
+        printResult(snippetId.type, snippetId.id, score);
+
+        const [filePath, range] = snippetId.id.split(':');
+        const [startLine, endLine] = range.split('-').map((n) => parseInt(n, 10));
 
         if (isNullOrUndefined(startLine) || isNullOrUndefined(endLine)) continue;