feat: Include content patterns and content types

packages/cli/src/cmds/navie/help.ts

@@ -1,6 +1,6 @@
 import lunr from 'lunr';
 import { Help } from '@appland/navie';
-import { processFiles, processNamedFiles, verbose } from '../../utils';
+import { processFiles, verbose } from '../../utils';
 import { log, warn } from 'console';
 import { readFile } from 'fs/promises';
 import { MarkdownTextSplitter } from 'langchain/text_splitter';

packages/cli/src/fulltext/AppMapIndex.ts

@@ -287,7 +287,7 @@ export function reportMatches(
 export default class AppMapIndex {
   constructor(public directories: string[], private idx: lunr.Index) {}
 
-  async search(search: string, options: SearchOptions = {}): Promise<SearchResponse> {
+  async search(search: string, options: SearchOptions): Promise<SearchResponse> {
     let matches = this.idx.search(queryKeywords(search).join(' '));
     matches = await removeNonExistentMatches(matches);
     const numResults = matches.length;
@@ -313,7 +313,7 @@ export default class AppMapIndex {
   static async search(
     appmapDirectories: string[],
     search: string,
-    options: SearchOptions = {}
+    options: SearchOptions
   ): Promise<SearchResponse> {
     const index = await buildIndex(appmapDirectories);
     return await index.search(search, options);

packages/cli/src/fulltext/FileIndex.ts

@@ -57,6 +57,7 @@ export class FileIndex {
   async indexDirectories(
     directories: string[],
     excludePatterns: RegExp[] | undefined,
+    includePatterns: RegExp[] | undefined,
     batchSize = 100
   ) {
     for (const directory of directories) {
@@ -68,7 +69,12 @@ export class FileIndex {
             ? await listGitProjectFiles(directory)
             : await listProjectFiles(directory);
 
-        const filteredFileNames = await filterFiles(directory, fileNames, excludePatterns);
+        const filteredFileNames = await filterFiles(
+          directory,
+          fileNames,
+          excludePatterns,
+          includePatterns
+        );
 
         const options = {
           allowGenericParsing: fileNames.length < 15_000,
@@ -153,12 +159,13 @@ export function restoreFileIndex(indexFileName: string): FileIndex {
 export async function buildFileIndex(
   directories: string[],
   indexFileName: string,
-  excludePatterns?: RegExp[]
+  excludePatterns?: RegExp[],
+  includePatterns?: RegExp[]
 ): Promise<FileIndex> {
   assert(!existsSync(indexFileName), `Index file ${indexFileName} already exists`);
   const database = new sqlite3(indexFileName);
   const fileIndex = new FileIndex(database);
-  await fileIndex.indexDirectories(directories, excludePatterns);
+  await fileIndex.indexDirectories(directories, excludePatterns, includePatterns);
   console.log(`Wrote file index to ${indexFileName}`);
   return fileIndex;
 }
@@ -231,13 +238,16 @@ const BINARY_FILE_EXTENSIONS: string[] = [
 export async function filterFiles(
   directory: string,
   fileNames: string[],
-  excludePatterns?: RegExp[]
+  excludePatterns?: RegExp[],
+  includePatterns?: RegExp[]
 ): Promise<string[]> {
   const result: string[] = [];
   for (const fileName of fileNames) {
     const fileExtension = path.extname(fileName).toLowerCase();
     if (BINARY_FILE_EXTENSIONS.some((ext) => ext === fileExtension)) continue;
 
+    if (includePatterns && !includePatterns.some((pattern) => pattern.test(fileName))) continue;
+
     if (excludePatterns?.some((pattern) => pattern.test(fileName))) continue;
 
     try {

packages/cli/src/fulltext/FindEvents.ts

@@ -18,6 +18,8 @@ type IndexItem = {
 
 export type SearchOptions = {
   maxResults?: number;
+  includePatterns?: RegExp[];
+  excludePatterns?: RegExp[];
 };
 
 export type SearchResult = {
@@ -135,19 +137,33 @@ export default class FindEvents {
         );
       matches = matches.slice(0, options.maxResults);
     }
-    const searchResults = matches.map((match) => {
-      const indexItem = this.indexItemsByFqid.get(match.ref);
-      assert(indexItem);
-      const result: SearchResult = {
-        appmap: this.appmapId,
-        fqid: match.ref,
-        score: match.score,
-        elapsed: indexItem?.elapsed,
-        eventIds: indexItem?.eventIds ?? [],
-      };
-      if (indexItem?.location) result.location = indexItem.location;
-      return result;
-    });
+    const searchResults = matches
+      .map((match) => {
+        const indexItem = this.indexItemsByFqid.get(match.ref);
+        assert(indexItem);
+        const result: SearchResult = {
+          appmap: this.appmapId,
+          fqid: match.ref,
+          score: match.score,
+          elapsed: indexItem?.elapsed,
+          eventIds: indexItem?.eventIds ?? [],
+        };
+        if (indexItem?.location) {
+          const path = indexItem.location.split(':');
+
+          if (
+            options.includePatterns &&
+            !options.includePatterns.some((pattern) => pattern.test(path[0]))
+          )
+            return;
+
+          if (options.excludePatterns?.some((pattern) => pattern.test(path[0]))) return;
+
+          result.location = indexItem.location;
+        }
+        return result;
+      })
+      .filter(Boolean) as SearchResult[];
     return { type: 'event', results: searchResults, numResults };
   }
 }

packages/cli/src/rpc/explain/collectContext.ts

@@ -1,8 +1,12 @@
 import { SearchRpc } from '@appland/rpc';
-import AppMapIndex, { SearchResponse as AppMapSearchResponse } from '../../fulltext/AppMapIndex';
+import AppMapIndex, {
+  SearchResponse as AppMapSearchResponse,
+  SearchOptions as AppMapSearchOptions,
+} from '../../fulltext/AppMapIndex';
 import FindEvents, {
   SearchResponse as EventSearchResponse,
   SearchResult as EventSearchResult,
+  SearchOptions as EventsSearchOptions,
 } from '../../fulltext/FindEvents';
 import { DEFAULT_MAX_DIAGRAMS, DEFAULT_MAX_FILES } from '../search/search';
 import buildContext from './buildContext';
@@ -31,7 +35,12 @@ export class EventCollector {
 
   constructor(private query: string, private appmapSearchResponse: AppMapSearchResponse) {}
 
-  async collectEvents(maxEvents: number): Promise<{
+  async collectEvents(
+    maxEvents: number,
+    excludePatterns?: RegExp[],
+    includePatterns?: RegExp[],
+    includeTypes?: ContextV2.ContextItemType[]
+  ): Promise<{
     results: SearchRpc.SearchResult[];
     context: ContextV2.ContextResponse;
     contextSize: number;
@@ -42,15 +51,28 @@ export class EventCollector {
       let { appmap } = result;
       if (!isAbsolute(appmap)) appmap = join(result.directory, appmap);
 
-      const eventsSearchResponse = await this.findEvents(appmap, maxEvents);
+      const options: EventsSearchOptions = {
+        maxResults: maxEvents,
+      };
+      if (includePatterns) options.includePatterns = includePatterns;
+      if (excludePatterns) options.excludePatterns = excludePatterns;
+
+      const eventsSearchResponse = await this.findEvents(appmap, options);
       results.push({
         appmap: appmap,
         directory: result.directory,
         events: eventsSearchResponse.results.map(textSearchResultToRpcSearchResult),
         score: result.score,
       });
     }
-    const context = await buildContext(results);
+
+    const isIncludedType = (item: ContextV2.ContextItem) => {
+      if (includeTypes && !includeTypes.some((type) => type === item.type)) return false;
+
+      return true;
+    };
+
+    const context = (await buildContext(results)).filter(isIncludedType);
 
     const contextSize = context.reduce((acc, item) => acc + item.content.length, 0);
 
@@ -67,11 +89,11 @@ export class EventCollector {
     return index;
   }
 
-  async findEvents(appmap: string, maxResults: number): Promise<EventSearchResponse> {
+  async findEvents(appmap: string, options: AppMapSearchOptions): Promise<EventSearchResponse> {
     if (appmap.endsWith('.appmap.json')) appmap = appmap.slice(0, -'.appmap.json'.length);
 
     const index = await this.appmapIndex(appmap);
-    return index.search(this.query, { maxResults });
+    return index.search(this.query, options);
   }
 }
 
@@ -103,6 +125,8 @@ export class SourceCollector {
 export class ContextCollector {
   public appmaps: string[] | undefined;
   public excludePatterns: RegExp[] | undefined;
+  public includePatterns: RegExp[] | undefined;
+  public includeTypes: ContextV2.ContextItemType[] | undefined;
 
   query: string;
 
@@ -148,7 +172,7 @@ export class ContextCollector {
       };
     } else {
       // Search across all AppMaps, creating a map from AppMap id to AppMapSearchResult
-      const searchOptions = {
+      const searchOptions: AppMapSearchOptions = {
         maxResults: DEFAULT_MAX_DIAGRAMS,
       };
       appmapSearchResponse = await AppMapIndex.search(this.appmapDirectories, query, searchOptions);
@@ -157,7 +181,12 @@ export class ContextCollector {
     const fileSearchResponse = await withIndex(
       'files',
       (indexFileName: string) =>
-        buildFileIndex(this.sourceDirectories, indexFileName, this.excludePatterns),
+        buildFileIndex(
+          this.sourceDirectories,
+          indexFileName,
+          this.excludePatterns,
+          this.includePatterns
+        ),
       (index) => index.search(this.vectorTerms, DEFAULT_MAX_FILES)
     );
 
@@ -175,7 +204,12 @@ export class ContextCollector {
     for (;;) {
       log(`Collecting context with ${maxEventsPerDiagram} events per diagram.`);
 
-      contextCandidate = await eventsCollector.collectEvents(maxEventsPerDiagram);
+      contextCandidate = await eventsCollector.collectEvents(
+        maxEventsPerDiagram,
+        this.excludePatterns,
+        this.includePatterns,
+        this.includeTypes
+      );
 
       const codeSnippetCount = contextCandidate.context.filter(
         (item) => item.type === ContextV2.ContextItemType.CodeSnippet
@@ -218,7 +252,7 @@ export default async function collectContext(
   appmaps: string[] | undefined,
   vectorTerms: string[],
   charLimit: number,
-  exclude?: string[]
+  filters: ContextV2.ContextFilters
 ): Promise<{
   searchResponse: SearchRpc.SearchResponse;
   context: ContextV2.ContextResponse;
@@ -230,6 +264,12 @@ export default async function collectContext(
     charLimit
   );
   if (appmaps) contextCollector.appmaps = appmaps;
-  if (exclude) contextCollector.excludePatterns = exclude.map((pattern) => new RegExp(pattern));
+
+  if (filters?.exclude)
+    contextCollector.excludePatterns = filters.exclude.map((pattern) => new RegExp(pattern));
+  if (filters?.include)
+    contextCollector.includePatterns = filters.include.map((pattern) => new RegExp(pattern));
+  if (filters?.itemTypes) contextCollector.includeTypes = filters.itemTypes.map((type) => type);
+
   return await contextCollector.collectContext();
 }

packages/cli/src/rpc/explain/explain.ts

@@ -113,7 +113,7 @@ export class Explain extends EventEmitter {
     this.status.vectorTerms = vectorTerms;
 
     if (data.labels) this.status.labels = data.labels;
-    const labels = data.labels || [];
+    const labels = data.labels ?? [];
 
     if (!tokenCount) {
       warn(chalk.bold(`Warning: Token limit not set, defaulting to ${DEFAULT_TOKEN_LIMIT}`));
@@ -151,7 +151,7 @@ export class Explain extends EventEmitter {
       this.appmaps,
       keywords,
       charLimit,
-      data.exclude
+      data
     );
 
     this.status.searchResponse = searchResult.searchResponse;

packages/navie/src/lib/parse-options.ts

@@ -1,3 +1,4 @@
+import { warn } from 'console';
 import { ContextV2 } from '../context';
 
 /* eslint-disable no-continue */
@@ -48,6 +49,10 @@ export class UserOptions {
 
     const include = this.stringValue('include');
     if (include) filters.include = [include];
+
+    const types = this.stringValue('itemtype');
+    if (types)
+      filters.itemTypes = types.split(',').map((type) => type as ContextV2.ContextItemType);
   }
 }
 
@@ -97,6 +102,7 @@ export default function parseOptions(question: string): {
       }
     }
 
+    warn(`User option ${key}=${value.toString()}`);
     options.set(key.toLowerCase(), value);
   }
 

packages/navie/src/services/apply-context-service.ts

@@ -77,6 +77,8 @@ export default class ApplyContextService {
   }
 
   addSystemPrompts(context: ContextV2.ContextResponse, help: HelpResponse) {
+    if (!context || context.length === 0) return;
+
     const hasSequenceDiagram = context.some(
       (item) => item.type === ContextV2.ContextItemType.SequenceDiagram
     );