Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Add support for multi-partition parquet:kv tables #1580

Merged
merged 7 commits into from
Nov 2, 2023
Merged

feat: Add support for multi-partition parquet:kv tables #1580

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
259f5e3
Add dropdowns for multi-partition parquet tables
georgecwan Oct 18, 2023
c8b5b1f
Merge branch 'main' of https://github.com/deephaven/web-client-ui int…
georgecwan Oct 23, 2023
a6d31a1
Add legacy format for dehydrated state
georgecwan Oct 25, 2023
a848dd6
Add test to IrisGridPartitionSelector
georgecwan Oct 25, 2023
9c9d099
Remove append command feature
georgecwan Oct 26, 2023
4875616
Add unit tests for hydration cases
georgecwan Oct 27, 2023
c80cd42
Add no partition column case
georgecwan Nov 1, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion packages/iris-grid/src/IrisGridTestUtils.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ class IrisGridTestUtils {
makeTable({
columns = this.makeColumns(),
size = 1000000000,
sort = [],
sort = [] as Sort[],
layoutHints = {} as LayoutHints,
} = {}): Table {
georgecwan marked this conversation as resolved.
Show resolved Hide resolved
// eslint-disable-next-line @typescript-eslint/no-explicit-any
Expand Down
185 changes: 150 additions & 35 deletions packages/iris-grid/src/IrisGridUtils.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,35 +10,12 @@ import IrisGridTestUtils from './IrisGridTestUtils';
import IrisGridUtils, {
DehydratedSort,
LegacyDehydratedSort,
isPanelStateV1,
} from './IrisGridUtils';

const irisGridUtils = new IrisGridUtils(dh);
const irisGridTestUtils = new IrisGridTestUtils(dh);

function makeFilter() {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
return new (dh as any).FilterCondition();
}

function makeColumns(count = 30) {
const columns: Column[] = [];

for (let i = 0; i < count; i += 1) {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const column = new (dh as any).Column({ index: i, name: `name_${i}` });
columns.push(column);
}

return columns;
}

function makeTable({
columns = makeColumns(),
sort = [] as Sort[],
} = {}): Table {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
return new (dh as any).Table({ columns, sort });
}
function makeColumn(index: number): Column {
return irisGridTestUtils.makeColumn(
`${index}`,
Expand All @@ -47,6 +24,16 @@ function makeColumn(index: number): Column {
);
}

function makeTable({
columns = irisGridTestUtils.makeColumns(10, 'name_'),
sort = [] as Sort[],
} = {}): Table {
return irisGridTestUtils.makeTable({
columns,
sort,
});
}

describe('quickfilters tests', () => {
it('exports/imports empty list', () => {
const table = irisGridTestUtils.makeTable();
Expand All @@ -62,10 +49,10 @@ describe('quickfilters tests', () => {
});

it('exports/imports quickFilters', () => {
const table = makeTable();
const table = irisGridTestUtils.makeTable();
const column = 9;
const text = '>1000';
const filter = makeFilter();
const filter = irisGridTestUtils.makeFilter();
const quickFilters = new Map([[column, { text, filter }]]);

const exportedFilters = IrisGridUtils.dehydrateQuickFilters(quickFilters);
Expand Down Expand Up @@ -93,7 +80,7 @@ describe('quickfilters tests', () => {

describe('advanced filter tests', () => {
it('exports/imports empty list', () => {
const table = makeTable();
const table = irisGridTestUtils.makeTable();
const filters = new Map();
const exportedFilters = irisGridUtils.dehydrateAdvancedFilters(
table.columns,
Expand All @@ -112,7 +99,7 @@ describe('advanced filter tests', () => {
it('exports advanced filters', () => {
const table = makeTable();
const column = 7;
const filter = makeFilter();
const filter = irisGridTestUtils.makeFilter();
const options = {
filterItems: [{ selectedType: '', value: '', key: 0 }],
filterOperators: [],
Expand Down Expand Up @@ -151,7 +138,7 @@ describe('advanced filter tests', () => {
describe('sort exporting/importing', () => {
it('exports/imports empty sort', () => {
const sort = [];
const table = makeTable({ sort });
const table = irisGridTestUtils.makeTable({ sort });
const exportedSort = IrisGridUtils.dehydrateSort(sort);
expect(exportedSort).toEqual([]);

Expand All @@ -160,7 +147,7 @@ describe('sort exporting/importing', () => {
});

it('should export (dehydrate) sorts', () => {
const columns = makeColumns();
const columns = irisGridTestUtils.makeColumns(10, 'name_');
const sort = [columns[3].sort(), columns[7].sort().abs().desc()];
const dehydratedSorts = IrisGridUtils.dehydrateSort(sort);

Expand All @@ -171,9 +158,9 @@ describe('sort exporting/importing', () => {
});

describe('should import (hydrate) sorts', () => {
const columns = makeColumns();
const columns = irisGridTestUtils.makeColumns(10, 'name_');
const sort = [columns[3].sort(), columns[7].sort().abs().desc()];
const table = makeTable({ columns, sort });
const table = irisGridTestUtils.makeTable({ columns, sort });

const dehydratedSorts = IrisGridUtils.dehydrateSort(sort);

Expand Down Expand Up @@ -210,7 +197,7 @@ describe('sort exporting/importing', () => {
describe('pendingDataMap hydration/dehydration', () => {
it('dehydrates/hydrates empty map', () => {
const pendingDataMap = new Map();
const columns = makeColumns();
const columns = irisGridTestUtils.makeColumns(10, 'name_');
const dehydratedMap = irisGridUtils.dehydratePendingDataMap(
columns,
pendingDataMap
Expand Down Expand Up @@ -242,7 +229,7 @@ describe('pendingDataMap hydration/dehydration', () => {
},
],
]);
const columns = makeColumns();
const columns = irisGridTestUtils.makeColumns(10, 'name_');
const dehydratedMap = irisGridUtils.dehydratePendingDataMap(
columns,
pendingDataMap
Expand Down Expand Up @@ -488,7 +475,7 @@ describe('validate copy ranges', () => {

describe('changeFilterColumnNamesToIndexes', () => {
const DEFAULT_FILTER = {};
const columns = makeColumns(10);
const columns = irisGridTestUtils.makeColumns(10, 'name_');
it('Replaces column names with indexes', () => {
const filters = [
{ name: 'name_1', filter: DEFAULT_FILTER },
Expand Down Expand Up @@ -680,3 +667,131 @@ describe('dehydration methods', () => {
).toBe(true);
});
});

describe('hydration methods', () => {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should have cases for no partition column as well.

it.each([
[
'hydrateIrisGridPanelStateV1',
{
isSelectingPartition: false,
partition: null,
partitionColumn: 'INVALID',
advancedSettings: [],
},
],
[
'hydrateIrisGridPanelStateV2',
{
isSelectingPartition: false,
partitions: [null],
partitionColumns: ['INVALID'],
advancedSettings: [],
},
],
])('%s invalid column error', (_label, panelState) => {
expect(() =>
IrisGridUtils.hydrateIrisGridPanelState(
irisGridTestUtils.makeModel(
irisGridTestUtils.makeTable({
columns: irisGridTestUtils.makeColumns(5, 'name_'),
})
),
georgecwan marked this conversation as resolved.
Show resolved Hide resolved
panelState
)
).toThrow('Invalid partition column INVALID');
});

it.each([
[
'hydrateIrisGridPanelStateV1 null partition',
{
isSelectingPartition: false,
partition: null,
partitionColumn: 'name_0',
advancedSettings: [],
},
],
[
'hydrateIrisGridPanelStateV1 unselected partition',
{
isSelectingPartition: false,
partition: '',
partitionColumn: 'name_0',
advancedSettings: [],
},
],
[
'hydrateIrisGridPanelStateV1 one selected partition',
{
isSelectingPartition: true,
partition: '',
partitionColumn: 'name_0',
advancedSettings: [],
},
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These tests don't seem right. A selected partition should have a value for partition,. and the "unselected partition" test - are we testing the empty string? Or isSelectingPartition?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I believe that the first test is testing for isSelectingPartition being false (in which case the value of partition) shouldn't matter and the second test is just testing for selecting a partition normally. partition should be null to signify a partition without a value so the second test is just testing for the empty string. I can use a non-empty string to make the test clearer.

],
[
'hydrateIrisGridPanelStateV2 no partition columns',
{
isSelectingPartition: false,
partitions: [],
partitionColumns: [],
advancedSettings: [],
},
],
[
'hydrateIrisGridPanelStateV2 two unselected columns',
{
isSelectingPartition: true,
partitions: [null, null],
partitionColumns: ['name_0', 'name_1'],
advancedSettings: [],
},
],
[
'hydrateIrisGridPanelStateV2 two selected columns',
{
isSelectingPartition: true,
partitions: ['a', 'b'],
partitionColumns: ['name_0', 'name_1'],
advancedSettings: [],
},
],
[
'hydrateIrisGridPanelStateV2 mixed selection columns',
{
isSelectingPartition: true,
partitions: [null, 'b', null],
partitionColumns: ['name_0', 'name_1', 'name_2'],
advancedSettings: [],
},
],
[
'hydrateIrisGridPanelStateV2 mixed selection columns',
{
isSelectingPartition: true,
partitions: ['a', null, 'b'],
partitionColumns: ['name_0', 'name_1', 'name_2'],
advancedSettings: [],
},
],
])('%s partitions and columns match', (_label, panelState) => {
const result = IrisGridUtils.hydrateIrisGridPanelState(
irisGridTestUtils.makeModel(
irisGridTestUtils.makeTable({
columns: irisGridTestUtils.makeColumns(5, 'name_'),
})
),
panelState
);
expect(result.isSelectingPartition).toBe(panelState.isSelectingPartition);
if (isPanelStateV1(panelState)) {
expect(result.partitions).toEqual([panelState.partition]);
expect(result.partitionColumns[0].name).toBe(panelState.partitionColumn);
} else {
expect(result.partitions).toEqual(panelState.partitions);
panelState.partitionColumns.forEach((partition, index) => {
expect(result.partitionColumns[index].name === partition).toBeTruthy();
});
}
});
});
Loading