Skip to content

Commit

Permalink
./gradlew spotlessApply
Browse files Browse the repository at this point in the history 
The actual command needs to be executed with JDK 8:
./gradlew spotlessApply -DflinkVersions=1.13,1.14,1.15 -DsparkVersions=2.4,3.0,3.1,3.2,3.3 -DhiveVersions=2,3 --no-build-cache
  • Loading branch information
@nastra @rdblue
nastra authored and rdblue committed Jul 27, 2022
1 parent 99b41eb commit c07f2aa
Show file tree
Hide file tree
Showing 3,078 changed files with 158,816 additions and 123,637 deletions.
The diff you're trying to view is too large. We only load the first 3000 changed files.
45 changes: 28 additions & 17 deletions aliyun/src/main/java/org/apache/iceberg/aliyun/AliyunClientFactories.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.iceberg.aliyun;

import com.aliyun.oss.OSS;
Expand All @@ -28,43 +27,52 @@

public class AliyunClientFactories {

private static final AliyunClientFactory ALIYUN_CLIENT_FACTORY_DEFAULT = new DefaultAliyunClientFactory();
private static final AliyunClientFactory ALIYUN_CLIENT_FACTORY_DEFAULT =
new DefaultAliyunClientFactory();

private AliyunClientFactories() {
}
private AliyunClientFactories() {}

public static AliyunClientFactory defaultFactory() {
return ALIYUN_CLIENT_FACTORY_DEFAULT;
}

public static AliyunClientFactory from(Map<String, String> properties) {
String factoryImpl = PropertyUtil.propertyAsString(
properties, AliyunProperties.CLIENT_FACTORY, DefaultAliyunClientFactory.class.getName());
String factoryImpl =
PropertyUtil.propertyAsString(
properties,
AliyunProperties.CLIENT_FACTORY,
DefaultAliyunClientFactory.class.getName());
return loadClientFactory(factoryImpl, properties);
}

/**
* Load an implemented {@link AliyunClientFactory} based on the class name, and initialize it.
*
* @param impl the class name.
* @param impl the class name.
* @param properties to initialize the factory.
* @return an initialized {@link AliyunClientFactory}.
*/
private static AliyunClientFactory loadClientFactory(String impl, Map<String, String> properties) {
private static AliyunClientFactory loadClientFactory(
String impl, Map<String, String> properties) {
DynConstructors.Ctor<AliyunClientFactory> ctor;
try {
ctor = DynConstructors.builder(AliyunClientFactory.class).hiddenImpl(impl).buildChecked();
} catch (NoSuchMethodException e) {
throw new IllegalArgumentException(String.format(
"Cannot initialize AliyunClientFactory, missing no-arg constructor: %s", impl), e);
throw new IllegalArgumentException(
String.format(
"Cannot initialize AliyunClientFactory, missing no-arg constructor: %s", impl),
e);
}

AliyunClientFactory factory;
try {
factory = ctor.newInstance();
} catch (ClassCastException e) {
throw new IllegalArgumentException(
String.format("Cannot initialize AliyunClientFactory, %s does not implement AliyunClientFactory.", impl), e);
String.format(
"Cannot initialize AliyunClientFactory, %s does not implement AliyunClientFactory.",
impl),
e);
}

factory.initialize(properties);
Expand All @@ -74,16 +82,19 @@ private static AliyunClientFactory loadClientFactory(String impl, Map<String, St
static class DefaultAliyunClientFactory implements AliyunClientFactory {
private AliyunProperties aliyunProperties;

DefaultAliyunClientFactory() {
}
DefaultAliyunClientFactory() {}

@Override
public OSS newOSSClient() {
Preconditions.checkNotNull(
aliyunProperties, "Cannot create aliyun oss client before initializing the AliyunClientFactory.");

return new OSSClientBuilder().build(
aliyunProperties.ossEndpoint(), aliyunProperties.accessKeyId(), aliyunProperties.accessKeySecret());
aliyunProperties,
"Cannot create aliyun oss client before initializing the AliyunClientFactory.");

return new OSSClientBuilder()
.build(
aliyunProperties.ossEndpoint(),
aliyunProperties.accessKeyId(),
aliyunProperties.accessKeySecret());
}

@Override
Expand Down
5 changes: 1 addition & 4 deletions aliyun/src/main/java/org/apache/iceberg/aliyun/AliyunClientFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.iceberg.aliyun;

import com.aliyun.oss.OSS;
Expand All @@ -39,8 +38,6 @@ public interface AliyunClientFactory extends Serializable {
*/
void initialize(Map<String, String> properties);

/**
* Returns an initialized {@link AliyunProperties}
*/
/** Returns an initialized {@link AliyunProperties} */
AliyunProperties aliyunProperties();
}
40 changes: 21 additions & 19 deletions aliyun/src/main/java/org/apache/iceberg/aliyun/AliyunProperties.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.iceberg.aliyun;

import java.io.Serializable;
Expand All @@ -26,41 +25,43 @@

public class AliyunProperties implements Serializable {
/**
* The domain name used to access OSS. OSS uses HTTP Restful APIs to provide services. Different regions are accessed
* by using different endpoints. For the same region, access over the internal network or over the Internet also uses
* different endpoints. For more information, see:
* The domain name used to access OSS. OSS uses HTTP Restful APIs to provide services. Different
* regions are accessed by using different endpoints. For the same region, access over the
* internal network or over the Internet also uses different endpoints. For more information, see:
* https://www.alibabacloud.com/help/doc-detail/31837.htm
*/
public static final String OSS_ENDPOINT = "oss.endpoint";

/**
* Aliyun uses an AccessKey pair, which includes an AccessKey ID and an AccessKey secret to implement symmetric
* encryption and verify the identity of a requester. The AccessKey ID is used to identify a user.
* <p>
* For more information about how to obtain an AccessKey pair, see:
* Aliyun uses an AccessKey pair, which includes an AccessKey ID and an AccessKey secret to
* implement symmetric encryption and verify the identity of a requester. The AccessKey ID is used
* to identify a user.
*
* <p>For more information about how to obtain an AccessKey pair, see:
* https://www.alibabacloud.com/help/doc-detail/53045.htm
*/
public static final String CLIENT_ACCESS_KEY_ID = "client.access-key-id";

/**
* Aliyun uses an AccessKey pair, which includes an AccessKey ID and an AccessKey secret to implement symmetric
* encryption and verify the identity of a requester. The AccessKey secret is used to encrypt and verify the
* signature string.
* <p>
* For more information about how to obtain an AccessKey pair, see:
* Aliyun uses an AccessKey pair, which includes an AccessKey ID and an AccessKey secret to
* implement symmetric encryption and verify the identity of a requester. The AccessKey secret is
* used to encrypt and verify the signature string.
*
* <p>For more information about how to obtain an AccessKey pair, see:
* https://www.alibabacloud.com/help/doc-detail/53045.htm
*/
public static final String CLIENT_ACCESS_KEY_SECRET = "client.access-key-secret";

/**
* The implementation class of {@link AliyunClientFactory} to customize Aliyun client configurations.
* If set, all Aliyun clients will be initialized by the specified factory.
* If not set, {@link AliyunClientFactories#defaultFactory()} is used as default factory.
* The implementation class of {@link AliyunClientFactory} to customize Aliyun client
* configurations. If set, all Aliyun clients will be initialized by the specified factory. If not
* set, {@link AliyunClientFactories#defaultFactory()} is used as default factory.
*/
public static final String CLIENT_FACTORY = "client.factory-impl";

/**
* Location to put staging files for uploading to OSS, defaults to the directory value of java.io.tmpdir.
* Location to put staging files for uploading to OSS, defaults to the directory value of
* java.io.tmpdir.
*/
public static final String OSS_STAGING_DIRECTORY = "oss.staging-dir";

Expand All @@ -79,8 +80,9 @@ public AliyunProperties(Map<String, String> properties) {
this.accessKeyId = properties.get(CLIENT_ACCESS_KEY_ID);
this.accessKeySecret = properties.get(CLIENT_ACCESS_KEY_SECRET);

this.ossStagingDirectory = PropertyUtil.propertyAsString(properties, OSS_STAGING_DIRECTORY,
System.getProperty("java.io.tmpdir"));
this.ossStagingDirectory =
PropertyUtil.propertyAsString(
properties, OSS_STAGING_DIRECTORY, System.getProperty("java.io.tmpdir"));
}

public String ossEndpoint() {
Expand Down
9 changes: 3 additions & 6 deletions aliyun/src/main/java/org/apache/iceberg/aliyun/oss/BaseOSSFile.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.iceberg.aliyun.oss;

import com.aliyun.oss.OSS;
Expand Down Expand Up @@ -62,8 +61,8 @@ public boolean exists() {
return objectMetadata() != null;
} catch (OSSException e) {

if (e.getErrorCode().equals(OSSErrorCode.NO_SUCH_BUCKET) ||
e.getErrorCode().equals(OSSErrorCode.NO_SUCH_KEY)) {
if (e.getErrorCode().equals(OSSErrorCode.NO_SUCH_BUCKET)
|| e.getErrorCode().equals(OSSErrorCode.NO_SUCH_KEY)) {
return false;
}

Expand All @@ -85,8 +84,6 @@ protected MetricsContext metrics() {

@Override
public String toString() {
return MoreObjects.toStringHelper(this)
.add("file", uri)
.toString();
return MoreObjects.toStringHelper(this).add("file", uri).toString();
}
}
33 changes: 19 additions & 14 deletions aliyun/src/main/java/org/apache/iceberg/aliyun/oss/OSSFileIO.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.iceberg.aliyun.oss;

import com.aliyun.oss.OSS;
Expand All @@ -36,14 +35,15 @@

/**
* FileIO implementation backed by OSS.
* <p>
* Locations used must follow the conventions for OSS URIs (e.g. oss://bucket/path...).
* URIs with scheme https are also treated as oss file paths.
* Using this FileIO with other schemes with result in {@link org.apache.iceberg.exceptions.ValidationException}
*
* <p>Locations used must follow the conventions for OSS URIs (e.g. oss://bucket/path...). URIs with
* scheme https are also treated as oss file paths. Using this FileIO with other schemes with result
* in {@link org.apache.iceberg.exceptions.ValidationException}
*/
public class OSSFileIO implements FileIO {
private static final Logger LOG = LoggerFactory.getLogger(OSSFileIO.class);
private static final String DEFAULT_METRICS_IMPL = "org.apache.iceberg.hadoop.HadoopMetricsContext";
private static final String DEFAULT_METRICS_IMPL =
"org.apache.iceberg.hadoop.HadoopMetricsContext";

private SerializableSupplier<OSS> oss;
private AliyunProperties aliyunProperties;
Expand All @@ -53,16 +53,16 @@ public class OSSFileIO implements FileIO {

/**
* No-arg constructor to load the FileIO dynamically.
* <p>
* All fields are initialized by calling {@link OSSFileIO#initialize(Map)} later.
*
* <p>All fields are initialized by calling {@link OSSFileIO#initialize(Map)} later.
*/
public OSSFileIO() {
}
public OSSFileIO() {}

/**
* Constructor with custom oss supplier and default aliyun properties.
* <p>
* Calling {@link OSSFileIO#initialize(Map)} will overwrite information set in this constructor.
*
* <p>Calling {@link OSSFileIO#initialize(Map)} will overwrite information set in this
* constructor.
*
* @param oss oss supplier
*/
Expand Down Expand Up @@ -107,12 +107,17 @@ public void initialize(Map<String, String> properties) {
// Report Hadoop metrics if Hadoop is available
try {
DynConstructors.Ctor<MetricsContext> ctor =
DynConstructors.builder(MetricsContext.class).hiddenImpl(DEFAULT_METRICS_IMPL, String.class).buildChecked();
DynConstructors.builder(MetricsContext.class)
.hiddenImpl(DEFAULT_METRICS_IMPL, String.class)
.buildChecked();
MetricsContext context = ctor.newInstance("oss");
context.initialize(properties);
this.metrics = context;
} catch (NoClassDefFoundError | NoSuchMethodException | ClassCastException e) {
LOG.warn("Unable to load metrics class: '{}', falling back to null metrics", DEFAULT_METRICS_IMPL, e);
LOG.warn(
"Unable to load metrics class: '{}', falling back to null metrics",
DEFAULT_METRICS_IMPL,
e);
}
}

Expand Down
12 changes: 7 additions & 5 deletions aliyun/src/main/java/org/apache/iceberg/aliyun/oss/OSSInputFile.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.iceberg.aliyun.oss;

import com.aliyun.oss.OSS;
Expand All @@ -26,9 +25,7 @@
import org.apache.iceberg.io.SeekableInputStream;
import org.apache.iceberg.metrics.MetricsContext;

/**
* @deprecated moving to package-private in 0.15.0; use OSSFileIO to create InputFile instances
*/
/** @deprecated moving to package-private in 0.15.0; use OSSFileIO to create InputFile instances */
@Deprecated
public class OSSInputFile extends BaseOSSFile implements InputFile {

Expand All @@ -38,7 +35,12 @@ public class OSSInputFile extends BaseOSSFile implements InputFile {
super(client, uri, aliyunProperties, metrics);
}

OSSInputFile(OSS client, OSSURI uri, AliyunProperties aliyunProperties, long length, MetricsContext metrics) {
OSSInputFile(
OSS client,
OSSURI uri,
AliyunProperties aliyunProperties,
long length,
MetricsContext metrics) {
super(client, uri, aliyunProperties, metrics);
ValidationException.check(length >= 0, "Invalid file length: %s", length);
this.length = length;
Expand Down
12 changes: 4 additions & 8 deletions aliyun/src/main/java/org/apache/iceberg/aliyun/oss/OSSInputStream.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.iceberg.aliyun.oss;

import com.aliyun.oss.OSS;
Expand All @@ -35,9 +34,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
* @deprecated moving to package-private in 0.15.0
*/
/** @deprecated moving to package-private in 0.15.0 */
@Deprecated
public class OSSInputStream extends SeekableInputStream {
private static final Logger LOG = LoggerFactory.getLogger(OSSInputStream.class);
Expand All @@ -55,9 +52,7 @@ public class OSSInputStream extends SeekableInputStream {
private final Counter<Long> readBytes;
private final Counter<Integer> readOperations;

/**
* @deprecated moving to package-private in 0.15.0
*/
/** @deprecated moving to package-private in 0.15.0 */
@Deprecated
public OSSInputStream(OSS client, OSSURI uri) {
this(client, uri, MetricsContext.nullMetrics());
Expand All @@ -69,7 +64,8 @@ public OSSInputStream(OSS client, OSSURI uri) {
this.createStack = Thread.currentThread().getStackTrace();

this.readBytes = metrics.counter(FileIOMetricsContext.READ_BYTES, Long.class, Unit.BYTES);
this.readOperations = metrics.counter(FileIOMetricsContext.READ_OPERATIONS, Integer.class, Unit.COUNT);
this.readOperations =
metrics.counter(FileIOMetricsContext.READ_OPERATIONS, Integer.class, Unit.COUNT);
}

@Override
Expand Down
7 changes: 4 additions & 3 deletions aliyun/src/main/java/org/apache/iceberg/aliyun/oss/OSSOutputFile.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.iceberg.aliyun.oss;

import com.aliyun.oss.OSS;
Expand All @@ -33,8 +32,10 @@ class OSSOutputFile extends BaseOSSFile implements OutputFile {
super(client, uri, aliyunProperties, metrics);
}

static OSSOutputFile fromLocation(OSS client, String location, AliyunProperties aliyunProperties) {
return new OSSOutputFile(client, new OSSURI(location), aliyunProperties, MetricsContext.nullMetrics());
static OSSOutputFile fromLocation(
OSS client, String location, AliyunProperties aliyunProperties) {
return new OSSOutputFile(
client, new OSSURI(location), aliyunProperties, MetricsContext.nullMetrics());
}

@Override
Expand Down
Loading

0 comments on commit c07f2aa

Please sign in to comment.