This repository has been archived by the owner on Jun 7, 2024. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 292
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #1587 from zalando/datalake-annotations
Support making DataLake annotations as mandatory upon creation and update of event type
- Loading branch information
Showing
11 changed files
with
291 additions
and
219 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
103 changes: 103 additions & 0 deletions
103
...re/src/main/java/org/zalando/nakadi/service/validation/EventTypeAnnotationsValidator.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,103 @@ | ||
| package org.zalando.nakadi.service.validation; | ||
|
|
||
| import org.springframework.beans.factory.annotation.Autowired; | ||
| import org.springframework.beans.factory.annotation.Value; | ||
| import org.springframework.stereotype.Component; | ||
| import org.zalando.nakadi.domain.Feature; | ||
| import org.zalando.nakadi.exceptions.runtime.InvalidEventTypeException; | ||
| import org.zalando.nakadi.plugin.api.authz.AuthorizationService; | ||
| import org.zalando.nakadi.plugin.api.authz.Subject; | ||
| import org.zalando.nakadi.service.FeatureToggleService; | ||
|
|
||
| import javax.validation.constraints.NotNull; | ||
| import java.util.Collections; | ||
| import java.util.List; | ||
| import java.util.Map; | ||
| import java.util.Optional; | ||
| import java.util.regex.Pattern; | ||
|
|
||
| @Component | ||
| public class EventTypeAnnotationsValidator { | ||
| private static final Pattern ANNOTATIONS_PERIOD_PATTERN = Pattern.compile( | ||
| "^(unlimited|(([7-9]|[1-9]\\d{1,2}|[1-2]\\d{3}|3[0-5]\\d{2}|36[0-4]\\d|3650)((\\sdays?)|(d)))" + | ||
| "|(([1-9][0-9]?|[1-4][0-9]{2}|5([0-1][0-9]|2[0-1]))((\\sweeks?)|(w)))|" + | ||
| "(([1-9]|[1-9]\\d|[1][01]\\d|120)((\\smonths?)|(m)))|(([1-9]|(10))((\\syears?)|(y))))$"); | ||
| static final String RETENTION_PERIOD_ANNOTATION = "datalake.zalando.org/retention-period"; | ||
| static final String RETENTION_REASON_ANNOTATION = "datalake.zalando.org/retention-period-reason"; | ||
| static final String MATERIALIZE_EVENTS_ANNOTATION = "datalake.zalando.org/materialize-events"; | ||
|
|
||
| private final FeatureToggleService featureToggleService; | ||
| private final AuthorizationService authorizationService; | ||
| private final List<String> enforcedAuthSubjects; | ||
|
|
||
| @Autowired | ||
| public EventTypeAnnotationsValidator( | ||
| final FeatureToggleService featureToggleService, | ||
| final AuthorizationService authorizationService, | ||
| @Value("${nakadi.data_lake.annotations.enforced_auth_subjects:}") final List<String> enforcedAuthSubjects | ||
| ) { | ||
| this.featureToggleService = featureToggleService; | ||
| this.authorizationService = authorizationService; | ||
| this.enforcedAuthSubjects = enforcedAuthSubjects; | ||
| } | ||
|
|
||
| public void validateAnnotations(final Map<String, String> annotations) throws InvalidEventTypeException { | ||
| validateDataLakeAnnotations(Optional.ofNullable(annotations).orElseGet(Collections::emptyMap)); | ||
| } | ||
|
|
||
| private void validateDataLakeAnnotations(@NotNull final Map<String, String> annotations) { | ||
| final var materializeEvents = annotations.get(MATERIALIZE_EVENTS_ANNOTATION); | ||
| final var retentionPeriod = annotations.get(RETENTION_PERIOD_ANNOTATION); | ||
|
|
||
| if (materializeEvents != null) { | ||
| if (!materializeEvents.equals("off") && !materializeEvents.equals("on")) { | ||
| throw new InvalidEventTypeException( | ||
| "Annotation " + MATERIALIZE_EVENTS_ANNOTATION | ||
| + " is not valid. Provided value: \"" | ||
| + materializeEvents | ||
| + "\". Possible values are: \"on\" or \"off\"."); | ||
| } | ||
| if (materializeEvents.equals("on")) { | ||
| if (retentionPeriod == null) { | ||
| throw new InvalidEventTypeException("Annotation " + RETENTION_PERIOD_ANNOTATION | ||
| + " is required, when " + MATERIALIZE_EVENTS_ANNOTATION + " is \"on\"."); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| if (retentionPeriod != null) { | ||
| final var retentionReason = annotations.get(RETENTION_REASON_ANNOTATION); | ||
| if (retentionReason == null || retentionReason.isEmpty()) { | ||
| throw new InvalidEventTypeException( | ||
| "Annotation " + RETENTION_REASON_ANNOTATION + " is required, when " | ||
| + RETENTION_PERIOD_ANNOTATION + " is specified."); | ||
| } | ||
|
|
||
| if (!ANNOTATIONS_PERIOD_PATTERN.matcher(retentionPeriod).find()) { | ||
| throw new InvalidEventTypeException( | ||
| "Annotation " + RETENTION_PERIOD_ANNOTATION | ||
| + " does not comply with regex. See documentation " | ||
| + "(https://docs.google.com/document/d/1-SwwpwUqauc_pXu-743YA1gO8l5_R_Gf4nbYml1ySiI" | ||
| + "/edit#heading=h.kmvigbxbn1dj) for more details."); | ||
| } | ||
| } | ||
|
|
||
| if (areDataLakeAnnotationsMandatory()) { | ||
| if (materializeEvents == null) { | ||
| throw new InvalidEventTypeException("Annotation " + MATERIALIZE_EVENTS_ANNOTATION + " is required"); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| private boolean areDataLakeAnnotationsMandatory() { | ||
| if (!featureToggleService.isFeatureEnabled(Feature.FORCE_DATA_LAKE_ANNOTATIONS)) { | ||
| return false; | ||
| } | ||
| if (enforcedAuthSubjects.contains("*")) { | ||
| return true; | ||
| } | ||
|
|
||
| final var subject = authorizationService.getSubject().map(Subject::getName).orElse(""); | ||
| return enforcedAuthSubjects.contains(subject); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
166 changes: 166 additions & 0 deletions
166
...rc/test/java/org/zalando/nakadi/service/validation/EventTypeAnnotationsValidatorTest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,166 @@ | ||
| package org.zalando.nakadi.service.validation; | ||
|
|
||
| import org.junit.Assert; | ||
| import org.junit.Before; | ||
| import org.junit.Test; | ||
|
|
||
| import java.util.Collections; | ||
| import java.util.Map; | ||
| import java.util.Optional; | ||
|
|
||
| import org.zalando.nakadi.domain.Feature; | ||
| import org.zalando.nakadi.exceptions.runtime.InvalidEventTypeException; | ||
| import org.zalando.nakadi.plugin.api.authz.AuthorizationService; | ||
| import org.zalando.nakadi.service.FeatureToggleService; | ||
|
|
||
| import static org.mockito.Mockito.mock; | ||
| import static org.mockito.Mockito.when; | ||
|
|
||
| public class EventTypeAnnotationsValidatorTest { | ||
| private static final String A_TEST_APPLICATION = "baz"; | ||
|
|
||
| private FeatureToggleService featureToggleService; | ||
| private AuthorizationService authorizationService; | ||
| private EventTypeAnnotationsValidator validator; | ||
|
|
||
| @Before | ||
| public void setUp() { | ||
| featureToggleService = mock(FeatureToggleService.class); | ||
| authorizationService = mock(AuthorizationService.class); | ||
| validator = new EventTypeAnnotationsValidator( | ||
| featureToggleService, authorizationService, Collections.singletonList(A_TEST_APPLICATION)); | ||
| } | ||
|
|
||
| @Test | ||
| public void whenMaterializationEventFormatIsWrongThenFail() { | ||
| final var annotations = Map.of( | ||
| EventTypeAnnotationsValidator.MATERIALIZE_EVENTS_ANNOTATION, "1 day" | ||
| ); | ||
| try { | ||
| validator.validateAnnotations(annotations); | ||
| Assert.fail("not reachable"); | ||
| } catch (InvalidEventTypeException e) { | ||
| Assert.assertTrue( | ||
| "When the format of the Materialize Event annotation is wrong, the name of the annotation " + | ||
| "should be present", | ||
| e.getMessage().contains(EventTypeAnnotationsValidator.MATERIALIZE_EVENTS_ANNOTATION)); | ||
| } | ||
| } | ||
|
|
||
| @Test | ||
| public void whenRetentionPeriodThenRetentionReasonRequired() { | ||
| final var annotations = Map.of( | ||
| EventTypeAnnotationsValidator.RETENTION_PERIOD_ANNOTATION, "1 day" | ||
| ); | ||
| try { | ||
| validator.validateAnnotations(annotations); | ||
| Assert.fail("not reachable"); | ||
| } catch (InvalidEventTypeException e) { | ||
| Assert.assertTrue( | ||
| "When the retention period is specified but the retention reason is not," + | ||
| " the error message should include the retention reason annotation name", | ||
| e.getMessage().contains(EventTypeAnnotationsValidator.RETENTION_REASON_ANNOTATION)); | ||
| Assert.assertTrue( | ||
| "When the retention period is specified but the retention reason is not," + | ||
| " the error message should include the retention period annotation name", | ||
| e.getMessage().contains(EventTypeAnnotationsValidator.RETENTION_PERIOD_ANNOTATION)); | ||
| } | ||
| } | ||
|
|
||
| @Test | ||
| public void whenRetentionPeriodFormatIsWrongThenFail() { | ||
| final var annotations = Map.of( | ||
| EventTypeAnnotationsValidator.RETENTION_PERIOD_ANNOTATION, "1 airplane", | ||
| EventTypeAnnotationsValidator.RETENTION_REASON_ANNOTATION, "I need my data" | ||
| ); | ||
| try { | ||
| validator.validateAnnotations(annotations); | ||
| Assert.fail("not reachable"); | ||
| } catch (InvalidEventTypeException e) { | ||
| Assert.assertTrue( | ||
| "When retention period format is wrong, the message should contain a the annotation name", | ||
| e.getMessage().contains(EventTypeAnnotationsValidator.RETENTION_PERIOD_ANNOTATION)); | ||
| Assert.assertTrue( | ||
| "When retention period format is wrong, the message should contain a link to the documentation", | ||
| e.getMessage().contains( | ||
| "https://docs.google.com/document/d/1-SwwpwUqauc_pXu-743YA1gO8l5_R_Gf4nbYml1ySiI")); | ||
| } | ||
| } | ||
|
|
||
| @Test | ||
| public void whenRetentionPeriodAndReasonThenOk() { | ||
| final String[] validRetentionPeriodValues = { | ||
| "unlimited", | ||
| "12 days", | ||
| "3650 days", | ||
| "120 months", | ||
| "1 month", | ||
| "10 years", | ||
| "25d", | ||
| "1m", | ||
| "2y", | ||
| "1 year" | ||
| }; | ||
|
|
||
| for (final String validRetentionPeriod : validRetentionPeriodValues) { | ||
| final var annotations = Map.of( | ||
| EventTypeAnnotationsValidator.RETENTION_PERIOD_ANNOTATION, validRetentionPeriod, | ||
| EventTypeAnnotationsValidator.RETENTION_REASON_ANNOTATION, "I need my data" | ||
| ); | ||
|
|
||
| validator.validateAnnotations(annotations); | ||
| } | ||
| } | ||
|
|
||
| @Test | ||
| public void whenMaterializationEventsThenOk() { | ||
| final String[] validMaterializationEventsValues = {"off", "on"}; | ||
|
|
||
| for (final var materializationEventValue : validMaterializationEventsValues) { | ||
| final var annotations = Map.of( | ||
| EventTypeAnnotationsValidator.MATERIALIZE_EVENTS_ANNOTATION, materializationEventValue, | ||
| EventTypeAnnotationsValidator.RETENTION_PERIOD_ANNOTATION, "1m", | ||
| EventTypeAnnotationsValidator.RETENTION_REASON_ANNOTATION, "for testing" | ||
| ); | ||
|
|
||
| validator.validateAnnotations(annotations); | ||
| } | ||
| } | ||
|
|
||
| @Test | ||
| public void whenDataLakeAnnotationsEnforcedThenMaterializationIsRequired() { | ||
| when(featureToggleService.isFeatureEnabled(Feature.FORCE_DATA_LAKE_ANNOTATIONS)).thenReturn(true); | ||
| when(authorizationService.getSubject()).thenReturn(Optional.of(() -> A_TEST_APPLICATION)); | ||
|
|
||
| try { | ||
| validator.validateAnnotations(Collections.emptyMap()); | ||
| Assert.fail("not reachable"); | ||
| } catch (InvalidEventTypeException e) { | ||
| Assert.assertTrue( | ||
| e.getMessage().contains(EventTypeAnnotationsValidator.MATERIALIZE_EVENTS_ANNOTATION)); | ||
| } | ||
| } | ||
|
|
||
| @Test | ||
| public void whenMaterializationIsOnThenRetentionPeriodIsRequired() { | ||
| try { | ||
| validator.validateAnnotations(Collections.singletonMap( | ||
| EventTypeAnnotationsValidator.MATERIALIZE_EVENTS_ANNOTATION, "on")); | ||
| Assert.fail("not reachable"); | ||
| } catch (InvalidEventTypeException e) { | ||
| Assert.assertTrue( | ||
| e.getMessage().contains(EventTypeAnnotationsValidator.RETENTION_PERIOD_ANNOTATION)); | ||
| } | ||
| } | ||
|
|
||
| @Test | ||
| public void itWorksWithOtherAnnotations() { | ||
| final var annotations = Map.of("some-annotation", "some-value"); | ||
| validator.validateAnnotations(annotations); | ||
| } | ||
|
|
||
| @Test | ||
| public void itWorksWithoutAnnotations() { | ||
| validator.validateAnnotations(null); | ||
| } | ||
| } |
3 changes: 1 addition & 2 deletions
3
...dation/EventTypeOptionsValidatorTest.java → ...dation/EventTypeOptionsValidatorTest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.