Add filter options along source and signal fields (#596)

Add more options in the search menu to make filtering/sorting easier: - Filter signal fields by EXISTS (e.g. emails, ip addresses, etc) - Sorting both source and signals fields https://github.com/lilacai/lilac/assets/2294279/b14ef225-2c50-4c52-88cf-4eabcbb03f32
databricks · Aug 25, 2023 · be25f41 · be25f41
1 parent 910e9e2
commit be25f41
Show file tree

Hide file tree

Showing 14 changed files with 204 additions and 64 deletions.
diff --git a/web/blueprint/src/lib/components/commands/CommandCreateConcept.svelte b/web/blueprint/src/lib/components/commands/CommandCreateConcept.svelte
@@ -56,7 +56,7 @@
   $: schema = schemaQuery && $schemaQuery?.data;
 
   $: stringFields = schema
-    ? childFields(schema).filter(f => f.dtype === 'string' && !isSignalField(f, schema!))
+    ? childFields(schema).filter(f => f.dtype === 'string' && !isSignalField(f))
     : [];
   $: fields = stringFields.sort((a, b) => {
     const aIsIndexed = childFields(a).some(
@@ -80,7 +80,7 @@
   $: {
     // Auto-select the first path.
     if (fields && fields.length > 0 && path === undefined) {
-      path = deserializePath(fields[0].path);
+      path = fields[0].path;
     }
   }
 

diff --git a/web/blueprint/src/lib/components/commands/selectors/FieldSelect.svelte b/web/blueprint/src/lib/components/commands/selectors/FieldSelect.svelte
@@ -28,7 +28,7 @@
         .filter(field => (filter ? filter(field) : true))
     : null;
 
-  $: sourceFields = fields?.filter(f => $schema.data && !isSignalField(f, $schema.data));
+  $: sourceFields = fields?.filter(f => $schema.data && !isSignalField(f));
 
   function formatField(field: LilacField): string {
     return `${field.path.join('.')} (${field.dtype})`;

diff --git a/web/blueprint/src/lib/components/concepts/DatasetFieldEmbeddingSelector.svelte b/web/blueprint/src/lib/components/concepts/DatasetFieldEmbeddingSelector.svelte
@@ -37,7 +37,7 @@
   $: schema = $schemaQuery.data;
   $: pathId = path ? serializePath(path) : undefined;
   $: sourceFields = schema
-    ? childFields(schema).filter(f => !isSignalField(f, schema!) && f.dtype != null)
+    ? childFields(schema).filter(f => !isSignalField(f) && f.dtype != null)
     : [];
   $: indexedFields = sourceFields.filter(f =>
     childFields(f).some(f => f.signal != null && childFields(f).some(f => f.dtype === 'embedding'))

diff --git a/web/blueprint/src/lib/components/contextMenu/SchemaFieldMenu.svelte b/web/blueprint/src/lib/components/contextMenu/SchemaFieldMenu.svelte
@@ -10,16 +10,14 @@
     isSignalRootField,
     isSortableField,
     serializePath,
-    type LilacField,
-    type LilacSchema
+    type LilacField
   } from '$lilac';
   import {Modal, OverflowMenu, OverflowMenuItem} from 'carbon-components-svelte';
   import {InProgress} from 'carbon-icons-svelte';
   import {Command, triggerCommand} from '../commands/Commands.svelte';
   import {hoverTooltip} from '../common/HoverTooltip';
 
   export let field: LilacField;
-  export let schema: LilacSchema;
 
   let deleteSignalOpen = false;
 
@@ -31,7 +29,7 @@
   const datasetStore = getDatasetContext();
   const deleteSignal = deleteSignalMutation();
 
-  $: isSignal = isSignalField(field, schema);
+  $: isSignal = isSignalField(field);
   $: isSignalRoot = isSignalRootField(field);
 
   $: isPreview = isPreviewSignal($datasetStore.selectRowsSchema?.data || null, field.path);

diff --git a/web/blueprint/src/lib/components/datasetView/DatasetSettingsModal.svelte b/web/blueprint/src/lib/components/datasetView/DatasetSettingsModal.svelte
@@ -52,7 +52,7 @@
   let preferredEmbedding: string | undefined = $appSettings.embedding;
 
   $: mediaFields = petals(schema).filter(
-    f => f.dtype === 'string' && !pathIsEqual(f.path, [ROWID]) && !isSignalField(f, schema)
+    f => f.dtype === 'string' && !pathIsEqual(f.path, [ROWID]) && !isSignalField(f)
   );
 
   $: {

diff --git a/web/blueprint/src/lib/components/datasetView/DownloadFieldList.svelte b/web/blueprint/src/lib/components/datasetView/DownloadFieldList.svelte
@@ -12,7 +12,7 @@
       checkedFields.push(field);
       checkedFields = checkedFields;
     } else {
-      checkedFields = checkedFields.filter(f => f !== field);
+      checkedFields = checkedFields.filter(f => !pathIsEqual(f.path, field.path));
     }
   }
 </script>

diff --git a/web/blueprint/src/lib/components/datasetView/DownloadModal.svelte b/web/blueprint/src/lib/components/datasetView/DownloadModal.svelte
@@ -48,7 +48,7 @@
       return {sourceFields: null, enrichedFields: null};
     }
     const petalFields = petals(schema).filter(field => ['embedding'].indexOf(field.dtype!) === -1);
-    const sourceFields = petalFields.filter(f => !isSignalField(f, schema));
+    const sourceFields = petalFields.filter(f => !isSignalField(f));
     const enrichedFields = childFields(schema)
       .filter(f => isSignalRootField(f))
       .filter(f => !childFields(f).some(f => f.dtype === 'embedding'));

diff --git a/web/blueprint/src/lib/components/datasetView/FilterPanel.svelte b/web/blueprint/src/lib/components/datasetView/FilterPanel.svelte
@@ -2,7 +2,7 @@
   import {queryConcept} from '$lib/queries/conceptQueries';
   import {getDatasetContext} from '$lib/stores/datasetStore';
   import {getDatasetViewContext} from '$lib/stores/datasetViewStore';
-  import {getSearches, getSort} from '$lib/view_utils';
+  import {displayPath, getSearches, getSort} from '$lib/view_utils';
   import {
     deserializePath,
     formatValue,
@@ -86,7 +86,7 @@
             .map(field => {
               return {
                 path: field.path,
-                text: serializePath(field.path.slice(1)),
+                text: displayPath(field.path.slice(1)),
                 disabled: false
               };
             })

diff --git a/web/blueprint/src/lib/components/datasetView/FilterPill.svelte b/web/blueprint/src/lib/components/datasetView/FilterPill.svelte
@@ -1,9 +1,9 @@
 <script lang="ts">
   import {getDatasetViewContext} from '$lib/stores/datasetViewStore';
+  import {displayPath, shortFieldName} from '$lib/view_utils';
   import {
     deserializePath,
     formatValue,
-    serializePath,
     type BinaryFilter,
     type ListFilter,
     type Op,
@@ -31,8 +31,11 @@
 
   $: formattedValue = formatValue(filter.value || 'false');
   $: path = deserializePath(filter.path);
-  $: tooltip = `${serializePath(filter.path)} ${FILTER_SHORTHANDS[filter.op]} ${formattedValue}`;
-  $: shortenPath = path.at(-1);
+  $: tooltip =
+    filter.op === 'exists'
+      ? `has ${displayPath(path)}`
+      : `${displayPath(path)} ${FILTER_SHORTHANDS[filter.op]} ${formattedValue}`;
+  $: shortenPath = shortFieldName(path);
 </script>
 
 <div class="filter-pill items-center" use:hoverTooltip={{text: tooltip}}>
@@ -48,9 +51,13 @@
       })}
     on:remove={() => datasetViewStore.removeFilter(filter)}
   >
-    <span class="font-mono">{hidePath ? '' : shortenPath}</span>
-    {FILTER_SHORTHANDS[filter.op]}
-    {formattedValue}
+    {#if filter.op === 'exists'}
+      has <span class="font-mono">{hidePath ? '' : shortenPath}</span>
+    {:else}
+      <span class="font-mono">{hidePath ? '' : shortenPath}</span>
+      {FILTER_SHORTHANDS[filter.op]}
+      {formattedValue}
+    {/if}
   </RemovableTag>
 </div>
 
@@ -62,4 +69,7 @@
   :global(.filter-pill .bx--tooltip__content) {
     @apply flex flex-col items-center;
   }
+  :global(.search-container .bx--list-box__menu) {
+    max-height: 26rem !important;
+  }
 </style>