forked from deephaven/deephaven-core
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Unit tests for S3 + parquet (deephaven#5441)
- Loading branch information
1 parent
af861b7
commit fb70418
Showing
16 changed files
with
507 additions
and
277 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1 +1,5 @@ | ||
| io.deephaven.project.ProjectType=JAVA_PUBLIC | ||
|
|
||
| # TODO(deephaven-core#5115): EPIC: Dependency management | ||
| testcontainers.localstack.image=localstack/localstack:3.1.0 | ||
| testcontainers.minio.image=minio/minio:RELEASE.2024-02-04T22-36-13Z |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
29 changes: 29 additions & 0 deletions
29
...sions/parquet/table/src/test/java/io/deephaven/parquet/table/S3ParquetLocalStackTest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,29 @@ | ||
| // | ||
| // Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending | ||
| // | ||
| package io.deephaven.parquet.table; | ||
|
|
||
| import io.deephaven.extensions.s3.S3Instructions.Builder; | ||
| import io.deephaven.extensions.s3.testlib.SingletonContainers.LocalStack; | ||
| import io.deephaven.extensions.s3.testlib.SingletonContainers; | ||
| import org.junit.BeforeClass; | ||
| import software.amazon.awssdk.services.s3.S3AsyncClient; | ||
|
|
||
| public class S3ParquetLocalStackTest extends S3ParquetTestBase { | ||
|
|
||
| @BeforeClass | ||
| public static void initContainer() { | ||
| // ensure container is started so container startup time isn't associated with a specific test | ||
| LocalStack.init(); | ||
| } | ||
|
|
||
| @Override | ||
| public Builder s3Instructions(Builder builder) { | ||
| return LocalStack.s3Instructions(builder); | ||
| } | ||
|
|
||
| @Override | ||
| public S3AsyncClient s3AsyncClient() { | ||
| return SingletonContainers.LocalStack.s3AsyncClient(); | ||
| } | ||
| } |
33 changes: 33 additions & 0 deletions
33
extensions/parquet/table/src/test/java/io/deephaven/parquet/table/S3ParquetMinIOTest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,33 @@ | ||
| // | ||
| // Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending | ||
| // | ||
| package io.deephaven.parquet.table; | ||
|
|
||
| import io.deephaven.extensions.s3.S3Instructions.Builder; | ||
| import io.deephaven.extensions.s3.testlib.SingletonContainers.MinIO; | ||
| import io.deephaven.extensions.s3.testlib.SingletonContainers; | ||
| import io.deephaven.stats.util.OSUtil; | ||
| import org.junit.Assume; | ||
| import org.junit.BeforeClass; | ||
| import software.amazon.awssdk.services.s3.S3AsyncClient; | ||
|
|
||
| public class S3ParquetMinIOTest extends S3ParquetTestBase { | ||
|
|
||
| @BeforeClass | ||
| public static void initContainer() { | ||
| // TODO(deephaven-core#5116): MinIO testcontainers does not work on OS X | ||
| Assume.assumeFalse("OSUtil.runningMacOS()", OSUtil.runningMacOS()); | ||
| // ensure container is started so container startup time isn't associated with a specific test | ||
| MinIO.init(); | ||
| } | ||
|
|
||
| @Override | ||
| public Builder s3Instructions(final Builder builder) { | ||
| return MinIO.s3Instructions(builder); | ||
| } | ||
|
|
||
| @Override | ||
| public S3AsyncClient s3AsyncClient() { | ||
| return SingletonContainers.MinIO.s3AsyncClient(); | ||
| } | ||
| } |
89 changes: 89 additions & 0 deletions
89
extensions/parquet/table/src/test/java/io/deephaven/parquet/table/S3ParquetRemoteTest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,89 @@ | ||
| // | ||
| // Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending | ||
| // | ||
| package io.deephaven.parquet.table; | ||
|
|
||
| import io.deephaven.engine.table.ColumnDefinition; | ||
| import io.deephaven.engine.table.Table; | ||
| import io.deephaven.engine.table.TableDefinition; | ||
| import io.deephaven.engine.testutil.junit4.EngineCleanup; | ||
| import io.deephaven.extensions.s3.Credentials; | ||
| import io.deephaven.extensions.s3.S3Instructions; | ||
| import io.deephaven.test.types.OutOfBandTest; | ||
| import org.junit.Assume; | ||
| import org.junit.Rule; | ||
| import org.junit.Test; | ||
| import org.junit.experimental.categories.Category; | ||
|
|
||
| import java.time.Duration; | ||
|
|
||
| import static org.junit.Assert.assertEquals; | ||
|
|
||
| /** | ||
| * These tests verify the behavior of Parquet implementation when reading against remote S3 servers. | ||
| **/ | ||
| @Category(OutOfBandTest.class) | ||
| public class S3ParquetRemoteTest { | ||
|
|
||
| // The following tests are disabled by default, and should be run manually. | ||
| private static final boolean ENABLE_REMOTE_S3_TESTING = false; | ||
|
|
||
| @Rule | ||
| public final EngineCleanup framework = new EngineCleanup(); | ||
|
|
||
| @Test | ||
| public void readSampleParquetFilesFromPublicS3() { | ||
| Assume.assumeTrue("Skipping test because s3 testing disabled.", ENABLE_REMOTE_S3_TESTING); | ||
| final S3Instructions s3Instructions = S3Instructions.builder() | ||
| .regionName("us-east-2") | ||
| .readTimeout(Duration.ofSeconds(60)) | ||
| .credentials(Credentials.anonymous()) | ||
| .build(); | ||
| final TableDefinition tableDefinition = TableDefinition.of( | ||
| ColumnDefinition.ofString("hash"), | ||
| ColumnDefinition.ofLong("version"), | ||
| ColumnDefinition.ofLong("size"), | ||
| ColumnDefinition.ofString("block_hash"), | ||
| ColumnDefinition.ofLong("block_number"), | ||
| ColumnDefinition.ofLong("index"), | ||
| ColumnDefinition.ofLong("virtual_size"), | ||
| ColumnDefinition.ofLong("lock_time"), | ||
| ColumnDefinition.ofLong("input_count"), | ||
| ColumnDefinition.ofLong("output_count"), | ||
| ColumnDefinition.ofBoolean("isCoinbase"), | ||
| ColumnDefinition.ofDouble("output_value"), | ||
| ColumnDefinition.ofTime("last_modified"), | ||
| ColumnDefinition.ofDouble("input_value")); | ||
| final ParquetInstructions readInstructions = new ParquetInstructions.Builder() | ||
| .setSpecialInstructions(s3Instructions) | ||
| .setTableDefinition(tableDefinition) | ||
| .build(); | ||
| ParquetTools.readTable( | ||
| "s3://aws-public-blockchain/v1.0/btc/transactions/date=2009-01-03/part-00000-bdd84ab2-82e9-4a79-8212-7accd76815e8-c000.snappy.parquet", | ||
| readInstructions).head(10).select(); | ||
|
|
||
| ParquetTools.readTable( | ||
| "s3://aws-public-blockchain/v1.0/btc/transactions/date=2023-11-13/part-00000-da3a3c27-700d-496d-9c41-81281388eca8-c000.snappy.parquet", | ||
| readInstructions).head(10).select(); | ||
| } | ||
|
|
||
| @Test | ||
| public void readKeyValuePartitionedParquetFromPublicS3() { | ||
| Assume.assumeTrue("Skipping test because s3 testing disabled.", ENABLE_REMOTE_S3_TESTING); | ||
| final S3Instructions s3Instructions = S3Instructions.builder() | ||
| .regionName("us-east-1") | ||
| .readTimeout(Duration.ofSeconds(60)) | ||
| .credentials(Credentials.anonymous()) | ||
| .build(); | ||
| final TableDefinition ookla_table_definition = TableDefinition.of( | ||
| ColumnDefinition.ofInt("quarter").withPartitioning(), | ||
| ColumnDefinition.ofString("quadkey")); | ||
| final ParquetInstructions readInstructions = new ParquetInstructions.Builder() | ||
| .setSpecialInstructions(s3Instructions) | ||
| .setTableDefinition(ookla_table_definition) | ||
| .build(); | ||
| final Table table = ParquetTools.readTable("s3://ookla-open-data/parquet/performance/type=mobile/year=2023", | ||
| readInstructions).head(10).select(); | ||
| assertEquals(2, table.numColumns()); | ||
| } | ||
| } |
Oops, something went wrong.