chore(parquet): Make parquet WASM loader into the primary loader

modules/loader-utils/src/format-types.ts

@@ -1,5 +1,9 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
 /**
- * A worker loader definition that can be used with `@loaders.gl/core` functions
+ * An object that describes a format
  */
 export type Format = {
   /** Human readable name */
@@ -18,7 +22,6 @@ export type Format = {
   binary?: boolean;
   /** Is this a text format */
   text?: boolean;
-
   /** Test some initial bytes of content to see if this loader might be a match */
   tests?: (((ArrayBuffer: ArrayBuffer) => boolean) | ArrayBuffer | string)[];
 };

modules/parquet/src/index.ts

@@ -6,22 +6,29 @@ export {Buffer} from './polyfills/buffer/install-buffer-polyfill';
 
 // import {ArrowTable, ArrowTableBatch} from '@loaders.gl/arrow';
 
+export {ParquetFormat} from './parquet-format';
+
 export {
-  ParquetWorkerLoader,
-  ParquetLoader,
+  ParquetJSONWorkerLoader,
+  ParquetJSONLoader,
   GeoParquetWorkerLoader,
   GeoParquetLoader,
-  ParquetColumnarWorkerLoader,
-  ParquetColumnarLoader
-} from './parquet-loader';
+  // deprecated
+  ParquetJSONWorkerLoader as ParquetWorkerLoader,
+  ParquetJSONLoader as ParquetLoader
+} from './parquet-json-loader';
 
-export {ParquetWriter as _ParquetWriter} from './parquet-writer';
+export {
+  ParquetJSONWriter as _ParquetJSONWriter,
+  // deprecated
+  ParquetJSONWriter as ParquetWriter
+} from './parquet-json-writer';
 
 // EXPERIMENTAL - expose Parquet WASM loaders/writer
 
-export type {ParquetWasmLoaderOptions} from './parquet-wasm-loader';
-export {ParquetWasmLoader, ParquetWasmWorkerLoader} from './parquet-wasm-loader';
-export {ParquetWasmWriter} from './parquet-wasm-writer';
+export type {ParquetArrowLoaderOptions} from './parquet-arrow-loader';
+export {ParquetArrowLoader, ParquetArrowWorkerLoader} from './parquet-arrow-loader';
+export {ParquetArrowWriter} from './parquet-arrow-writer';
 
 // EXPERIMENTAL - expose the internal parquetjs API
 

modules/parquet/src/lib/encoders/encode-parquet-wasm.ts → modules/parquet/src/lib/encoders/encode-arrow-to-parquet.ts

@@ -7,14 +7,14 @@ import type {ArrowTable} from '@loaders.gl/schema';
 import * as arrow from 'apache-arrow';
 import {loadWasm} from '../utils/load-wasm';
 
-import type {ParquetWriterOptions} from '../../parquet-wasm-writer';
+import type {ParquetArrowWriterOptions} from '../../parquet-arrow-writer';
 
 /**
  * Encode Arrow arrow.Table to Parquet buffer
  */
-export async function encode(
+export async function encodeArrowToParquet(
   table: ArrowTable,
-  options: ParquetWriterOptions
+  options: ParquetArrowWriterOptions
 ): Promise<ArrayBuffer> {
   const wasmUrl = options.parquet?.wasmUrl!;
   const wasm = await loadWasm(wasmUrl);

modules/parquet/src/lib/parsers/get-parquet-schema.ts

@@ -4,9 +4,9 @@
 
 // loaders.gl
 import {Schema} from '@loaders.gl/schema';
+import {unpackGeoMetadata, unpackJSONStringMetadata} from '@loaders.gl/gis';
 import {ParquetReader} from '../../parquetjs/parser/parquet-reader';
 import {convertParquetSchema} from '../arrow/convert-schema-from-parquet';
-import {unpackGeoMetadata, unpackJSONStringMetadata} from '@loaders.gl/gis';
 
 export async function getSchemaFromParquetReader(reader: ParquetReader): Promise<Schema> {
   const parquetSchema = await reader.getSchema();

modules/parquet/src/lib/parsers/parse-geoparquet-to-geojson.ts

@@ -0,0 +1,36 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {ReadableFile} from '@loaders.gl/loader-utils';
+import type {
+  GeoJSONTable,
+  GeoJSONTableBatch,
+  ObjectRowTable,
+  ObjectRowTableBatch
+} from '@loaders.gl/schema';
+// import {convertGeoArrowToTable} from '@loaders.gl/gis';
+
+import type {ParquetJSONLoaderOptions} from '../../parquet-json-loader';
+
+import {parseParquetFile, parseParquetFileInBatches} from './parse-parquet-to-json';
+
+export async function parseGeoParquetFile(
+  file: ReadableFile,
+  options?: ParquetJSONLoaderOptions
+): Promise<ObjectRowTable | GeoJSONTable> {
+  const table = await parseParquetFile(file, {...options, shape: 'object-row-table'});
+  // return convertGeoArrowToTable(table, 'geojson-table');
+  return table;
+}
+
+export async function* parseGeoParquetFileInBatches(
+  file: ReadableFile,
+  options?: ParquetJSONLoaderOptions
+): AsyncIterable<ObjectRowTableBatch | GeoJSONTableBatch> {
+  const tableBatches = parseParquetFileInBatches(file, {...options, shape: 'object-row-table'});
+
+  for await (const batch of tableBatches) {
+    yield batch; // convertGeoArrowToTable(batch, 'geojson-table');
+  }
+}

modules/parquet/src/lib/parsers/parse-parquet-wasm.ts → modules/parquet/src/lib/parsers/parse-parquet-to-arrow.ts

@@ -13,7 +13,7 @@ import * as arrow from 'apache-arrow';
 import {loadWasm} from '../utils/load-wasm';
 import {makeStreamIterator} from '../utils/make-stream-iterator';
 
-export async function parseParquetFileWasm(
+export async function parseParquetFileToArrow(
   file: ReadableFile,
   options?: parquetWasm.ReaderOptions & {wasmUrl: string}
 ): Promise<ArrowTable> {
@@ -40,7 +40,7 @@ export async function parseParquetFileWasm(
   };
 }
 
-export async function* parseParquetFileInBatchesWasm(
+export async function* parseParquetFileToArrowInBatches(
   file: ReadableFile,
   options: parquetWasm.ReaderOptions & {wasmUrl: string}
 ): AsyncIterable<ArrowTableBatch> {

modules/parquet/src/lib/parsers/parse-parquet-to-columns.ts

@@ -4,7 +4,7 @@
 
 import type {ColumnarTable, ColumnarTableBatch, Schema} from '@loaders.gl/schema';
 import type {ReadableFile} from '@loaders.gl/loader-utils';
-import type {ParquetLoaderOptions} from '../../parquet-loader';
+import type {ParquetJSONLoaderOptions} from '../../parquet-json-loader';
 import {ParquetReader} from '../../parquetjs/parser/parquet-reader';
 import {ParquetRowGroup} from '../../parquetjs/schema/declare';
 import {ParquetSchema} from '../../parquetjs/schema/schema';
@@ -14,11 +14,11 @@ import {installBufferPolyfill} from '../../polyfills/buffer/index';
 import {preloadCompressions} from '../../parquetjs/compression';
 
 /**
- * @deprecated
+ * @deprecated - Use parseParquetToArrow
  */
 export async function parseParquetFileInColumns(
   file: ReadableFile,
-  options?: ParquetLoaderOptions
+  options?: ParquetJSONLoaderOptions
 ): Promise<ColumnarTable> {
   installBufferPolyfill();
   await preloadCompressions(options);
@@ -38,7 +38,7 @@ export async function parseParquetFileInColumns(
  */
 export async function* parseParquetFileInColumnarBatches(
   file: ReadableFile,
-  options?: ParquetLoaderOptions
+  options?: ParquetJSONLoaderOptions
 ): AsyncIterable<ColumnarTableBatch> {
   installBufferPolyfill();
   await preloadCompressions(options);

modules/parquet/src/lib/parsers/parse-parquet.ts → modules/parquet/src/lib/parsers/parse-parquet-to-json.ts

@@ -6,7 +6,7 @@ import {default as log} from '@probe.gl/log';
 import type {ReadableFile} from '@loaders.gl/loader-utils';
 import type {ObjectRowTable, ObjectRowTableBatch} from '@loaders.gl/schema';
 
-import type {ParquetLoaderOptions} from '../../parquet-loader';
+import type {ParquetJSONLoaderOptions} from '../../parquet-json-loader';
 import type {ParquetRow} from '../../parquetjs/schema/declare';
 import {ParquetReader} from '../../parquetjs/parser/parquet-reader';
 import {getSchemaFromParquetReader} from './get-parquet-schema';
@@ -21,7 +21,7 @@ import {preloadCompressions} from '../../parquetjs/compression';
  */
 export async function parseParquetFile(
   file: ReadableFile,
-  options?: ParquetLoaderOptions
+  options?: ParquetJSONLoaderOptions
 ): Promise<ObjectRowTable> {
   installBufferPolyfill();
   await preloadCompressions(options);
@@ -56,8 +56,7 @@ export async function parseParquetFile(
     data: rows
   };
 
-  const shape = options?.parquet?.shape;
-  return convertTable(objectRowTable, shape);
+  return objectRowTable;
 }
 
 /**
@@ -67,7 +66,7 @@ export async function parseParquetFile(
  */
 export async function* parseParquetFileInBatches(
   file: ReadableFile,
-  options?: ParquetLoaderOptions
+  options?: ParquetJSONLoaderOptions
 ): AsyncIterable<ObjectRowTableBatch> {
   installBufferPolyfill();
   await preloadCompressions(options);
@@ -84,31 +83,12 @@ export async function* parseParquetFileInBatches(
       schema,
       data: rows
     };
-    const shape = options?.parquet?.shape;
-    const table = convertTable(objectRowTable, shape);
 
     yield {
       batchType: 'data',
       schema,
-      ...table,
+      ...objectRowTable,
       length: rows.length
     };
   }
 }
-
-function convertTable(
-  objectRowTable: ObjectRowTable,
-  shape?: 'object-row-table' | 'geojson-table'
-): ObjectRowTable {
-  switch (shape) {
-    case 'object-row-table':
-      return objectRowTable;
-
-    // Hack until geoparquet fixes up forwarded shape
-    case 'geojson-table':
-      return objectRowTable;
-
-    default:
-      throw new Error(shape);
-  }
-}