Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Write a lot of small files in s3 when there are two versions schema data in topic #242

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Write a lot of small files in s3 when there are two versions schema data in topic #242

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 21 additions & 0 deletions core/src/main/java/io/confluent/connect/storage/StorageSinkConnectorConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,15 @@ public class StorageSinkConnectorConfig extends AbstractConfig implements Compos
public static final long ROTATE_SCHEDULE_INTERVAL_MS_DEFAULT = -1L;
public static final String ROTATE_SCHEDULE_INTERVAL_MS_DISPLAY = "Rotate Schedule Interval (ms)";

public static final String ROTATE_MULTIPLE_SCHEMA_CONFIG = "rotate.multiple.schema";
public static final boolean ROTATE_MULTIPLE_SCHEMA_DEFAULT = true;
public static final String ROTATE_MULTIPLE_SCHEMA_DOC =
"The strategy to invoke file commits. This configuration ensures that file commits whether "
+ "are invoked at multiple schemas from the time interval. The value ``true`` means that "
+ "rotate file, and the value ``false`` means that don't rotate file until reach the time "
+ "interval or flush size.";
public static final String ROTATE_MULTIPLE_SCHEMA_DISPLAY = "Rotate Multiple Schema (boolean)";

public static final String RETRY_BACKOFF_CONFIG = "retry.backoff.ms";
public static final String
RETRY_BACKOFF_DOC =
Expand Down Expand Up @@ -207,6 +216,18 @@ public static ConfigDef newConfigDef(
ROTATE_SCHEDULE_INTERVAL_MS_DISPLAY
);

configDef.define(
ROTATE_MULTIPLE_SCHEMA_CONFIG,
Type.BOOLEAN,
ROTATE_MULTIPLE_SCHEMA_DEFAULT,
Importance.LOW,
ROTATE_MULTIPLE_SCHEMA_DOC,
group,
++orderInGroup,
Width.SHORT,
ROTATE_MULTIPLE_SCHEMA_DISPLAY
);

configDef.define(
SCHEMA_CACHE_SIZE_CONFIG,
Type.INT,
Expand Down