Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Test: hadoopfs tests remove the use of @Value.Immutable #8572

Open
wants to merge 6 commits into
base: master
Choose a base branch
from
Open

Test: hadoopfs tests remove the use of @Value.Immutable #8572

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
3658399
test: Add ImmutablePagination to remove the use of Value.Immutable
nopcoder Jan 29, 2025
987294e
Remove immutable package and unused import
nopcoder Jan 29, 2025
939d812
Reduce dep switch from Google's Options to Java's Options
nopcoder Feb 1, 2025
c958ede
Use the pagination class directly and drop the interface
nopcoder Feb 1, 2025
f113246
remove dup dependency
nopcoder Feb 1, 2025
fcaab56
fix prefix value in pagination
nopcoder Feb 2, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 0 additions & 12 deletions clients/hadoopfs/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -313,12 +313,6 @@ To export to S3:
<version>${hadoop.version}</version>
<scope>provided</scope>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 -->
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.12.0</version>
</dependency>
<dependency>
<groupId>javax.xml.bind</groupId>
<artifactId>jaxb-api</artifactId>
Expand Down Expand Up @@ -391,12 +385,6 @@ To export to S3:
<version>5.15.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.immutables</groupId>
<artifactId>value</artifactId>
<version>2.9.3</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
Expand Down
42 changes: 9 additions & 33 deletions clients/hadoopfs/src/test/java/io/lakefs/FSTestBase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,15 +1,6 @@
package io.lakefs;

import com.amazonaws.ClientConfiguration;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.S3ClientOptions;
import com.amazonaws.services.s3.model.*;
import com.aventrix.jnanoid.jnanoid.NanoIdUtils;
import com.google.common.base.Optional;
import com.google.common.collect.ImmutableMap;
import com.google.gson.FieldNamingPolicy;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
Expand All @@ -20,27 +11,19 @@
import io.lakefs.utils.ObjectLocation;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.http.HttpStatus;

import org.immutables.value.Value;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TestName;

import org.mockserver.client.MockServerClient;
import org.mockserver.junit.MockServerRule;
import org.mockserver.matchers.MatchType;
import org.mockserver.matchers.TimeToLive;
import org.mockserver.matchers.Times;
import org.mockserver.model.Cookie;
import org.mockserver.model.HttpRequest;
import org.mockserver.model.HttpResponse;
import org.mockserver.model.Parameter;

import static org.apache.commons.lang3.StringUtils.removeStart;

Expand All @@ -51,7 +34,6 @@
import java.net.URI;
import java.net.URISyntaxException;
import java.util.Arrays;
import java.util.List;

/**
* Base for all LakeFSFilesystem tests. Helps set common components up but
Expand Down Expand Up @@ -85,12 +67,6 @@ public abstract class FSTestBase {
.setFieldNamingPolicy(FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES)
.create();

@Value.Immutable static public interface Pagination {
@Value.Parameter Optional<Integer> amount();
@Value.Parameter Optional<String> after();
@Value.Parameter Optional<String> prefix();
}

@Rule
public MockServerRule mockServerRule = new MockServerRule(this);
protected MockServerClient mockServerClient;
Expand Down Expand Up @@ -245,7 +221,7 @@ protected void mockFilesInDir(String repo, String main, String dir, String... fi

// Directory can be listed!
mockListing("repo", "main",
ImmutablePagination.builder().prefix(dir + Constants.SEPARATOR).build(),
Pagination.builder().prefix(dir + Constants.SEPARATOR).build(),
allStats);
}

Expand Down Expand Up @@ -317,23 +293,23 @@ protected ObjectStats mockDirectoryMarker(ObjectLocation objectLoc) {
}

// Mock this listing and return these stats.
protected void mockListing(String repo, String ref, ImmutablePagination pagination, ObjectStats... stats) {
protected void mockListing(String repo, String ref, Pagination pagination, ObjectStats... stats) {
mockListingWithHasMore(repo, ref, pagination, false, stats);
}

protected void mockListingWithHasMore(String repo, String ref, ImmutablePagination pagination, boolean hasMore, ObjectStats... stats) {
protected void mockListingWithHasMore(String repo, String ref, Pagination pagination, boolean hasMore, ObjectStats... stats) {
HttpRequest req = request()
.withMethod("GET")
.withPath(String.format("/repositories/%s/refs/%s/objects/ls", repo, ref));
// Validate elements of pagination only if present.
if (pagination.after().isPresent()) {
req = req.withQueryStringParameter("after", pagination.after().or(""));
if (pagination.after() != null) {
req = req.withQueryStringParameter("after", pagination.after());
}
if (pagination.amount().isPresent()) {
req = req.withQueryStringParameter("amount", pagination.amount().get().toString());
if (pagination.amount() != null) {
req = req.withQueryStringParameter("amount", pagination.amount().toString());
}
if (pagination.prefix().isPresent()) {
req = req.withQueryStringParameter("prefix", pagination.prefix().or(""));
if (pagination.prefix() != null) {
req = req.withQueryStringParameter("prefix", pagination.prefix());
}
ObjectStatsList resp = new ObjectStatsList()
.results(Arrays.asList(stats))
Expand Down
2 changes: 1 addition & 1 deletion clients/hadoopfs/src/test/java/io/lakefs/LakeFSFileSystemServerS3Test.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -172,7 +172,7 @@ public void testMkdirs() throws IOException {
for (Path p = new Path(path.toString()); p != null && !p.isRoot(); p = p.getParent()) {
mockStatObjectNotFound("repo", "main", p.toString());
mockStatObjectNotFound("repo", "main", p+"/");
mockListing("repo", "main", ImmutablePagination.builder().prefix(p+"/").build());
mockListing("repo", "main", Pagination.builder().prefix(p+"/").build());
}

// physical address to directory marker object
Expand Down
Loading
Loading