Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Operator Runtime Config Rotating Store Verticle #1405

Open
wants to merge 8 commits into
base: main
Choose a base branch
from
Open

Operator Runtime Config Rotating Store Verticle #1405

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
f3a34de
Add RotatingRuntimeConfigStore
BehnamMozafari Feb 13, 2025
a9b3726
Add RotatingRuntimeConfigStoreTest
BehnamMozafari Feb 13, 2025
be5b6f2
Update Main to create RotatingRuntimeConfigStore
BehnamMozafari Feb 13, 2025
66a0661
Update ConfigServiceTest and ConfigRetrieverFactory to work with new …
BehnamMozafari Feb 13, 2025
83ad547
Update config files with new runtime_config_path and runtime_config_r…
BehnamMozafari Feb 13, 2025
43a0a09
Merge main into bmz-UID2-4873-operator-rotating-store-verticle
BehnamMozafari Feb 13, 2025
0d6eae2
Add RuntimeConfigPathProp Const
BehnamMozafari Feb 13, 2025
d5e1d4b
Remove unnecessary exception throw
BehnamMozafari Feb 13, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ COPY ./target/${JAR_NAME}-${JAR_VERSION}-sources.jar /app
COPY ./target/${JAR_NAME}-${JAR_VERSION}-static.tar.gz /app/static.tar.gz
COPY ./conf/default-config.json ${EXTRA_CONFIG} /app/conf/
COPY ./conf/*.xml /app/conf/
COPY ./conf/runtime-config-defaults.json /app/conf/
COPY ./conf/feat-flag/feat-flag.json /app/conf/feat-flag/

RUN tar xzvf /app/static.tar.gz --no-same-owner --no-same-permissions && rm -f /app/static.tar.gz
Expand Down
4 changes: 3 additions & 1 deletion conf/default-config.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,5 +35,7 @@
"enclave_platform": null,
"failure_shutdown_wait_hours": 120,
"sharing_token_expiry_seconds": 2592000,
"operator_type": "public"
"operator_type": "public",
"runtime_config_path": "runtime_config/metadata.json",
"runtime_config_refresh_ms": 300000
}
10 changes: 2 additions & 8 deletions conf/docker-config.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,12 +38,6 @@
"failure_shutdown_wait_hours": 120,
"salts_expired_shutdown_hours": 12,
"operator_type": "public",
"runtime_config_store": {
"type": "file",
"config" : {
"path": "conf/runtime-config-defaults.json",
"format": "json"
},
"config_scan_period_ms": 5000
}
"runtime_config_path": "/com.uid2.core/test/runtime_config/metadata.json",
"runtime_config_refresh_ms": 300000
}
9 changes: 2 additions & 7 deletions conf/integ-config.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,6 @@
"optout_api_uri": "http://localhost:8081/optout/replicate",
"salts_expired_shutdown_hours": 12,
"operator_type": "public",
"runtime_config_store": {
"type": "http",
"config" : {
"url": "http://localhost:8088/operator/config"
},
"config_scan_period_ms": 300000
}
"runtime_config_path": "http://localhost:8088/operator/config",
"runtime_config_refresh_ms": 300000
}
10 changes: 2 additions & 8 deletions conf/local-config.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,12 +37,6 @@
"client_side_token_generate_log_invalid_http_origins": true,
"salts_expired_shutdown_hours": 12,
"operator_type": "public",
"runtime_config_store": {
"type": "file",
"config" : {
"path": "conf/runtime-config-defaults.json",
"format": "json"
},
"config_scan_period_ms": 5000
}
"runtime_config_path": "/com.uid2.core/test/runtime_config/metadata.json",
"runtime_config_refresh_ms": 300000
}
9 changes: 2 additions & 7 deletions conf/local-e2e-docker-private-config.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,11 +28,6 @@
"cloud_refresh_interval": 30,
"salts_expired_shutdown_hours": 12,
"operator_type": "private",
"runtime_config_store": {
"type": "http",
"config" : {
"url": "http://core:8088/operator/config"
},
"config_scan_period_ms": 300000
}
"runtime_config_path": "http://core:8088/operator/config",
"runtime_config_refresh_ms": 300000
}
9 changes: 2 additions & 7 deletions conf/local-e2e-docker-public-config.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,11 +34,6 @@
"cloud_refresh_interval": 30,
"salts_expired_shutdown_hours": 12,
"operator_type": "public",
"runtime_config_store": {
"type": "http",
"config" : {
"url": "http://core:8088/operator/config"
},
"config_scan_period_ms": 300000
}
"runtime_config_path": "http://core:8088/operator/config",
"runtime_config_refresh_ms": 300000
}
9 changes: 2 additions & 7 deletions conf/local-e2e-private-config.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,11 +39,6 @@
"client_side_token_generate_log_invalid_http_origins": true,
"salts_expired_shutdown_hours": 12,
"operator_type": "private",
"runtime_config_store": {
"type": "http",
"config" : {
"url": "http://localhost:8088/operator/config"
},
"config_scan_period_ms": 300000
}
"runtime_config_path": "http://localhost:8088/operator/config",
"runtime_config_refresh_ms": 300000
}
9 changes: 2 additions & 7 deletions conf/local-e2e-public-config.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,11 +40,6 @@
"client_side_token_generate_log_invalid_http_origins": true,
"salts_expired_shutdown_hours": 12,
"operator_type": "public",
"runtime_config_store": {
"type": "http",
"config" : {
"url": "http://localhost:8088/operator/config"
},
"config_scan_period_ms": 300000
}
"runtime_config_path": "http://localhost:8088/operator/config",
"runtime_config_refresh_ms": 300000
}
9 changes: 2 additions & 7 deletions conf/validator-latest-e2e-docker-public-config.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,11 +33,6 @@
"optout_delta_rotate_interval": 60,
"cloud_refresh_interval": 30,
"operator_type": "public",
"runtime_config_store": {
"type": "http",
"config" : {
"url": "http://core:8080/operator/config"
},
"config_scan_period_ms": 300000
}
"runtime_config_path": "http://core:8088/operator/config",
"runtime_config_refresh_ms": 300000
}
9 changes: 2 additions & 7 deletions scripts/aws/conf/euid-integ-config.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,6 @@
"optout_api_uri": "https://optout.integ.euid.eu/optout/replicate",
"optout_s3_folder": "optout/",
"allow_legacy_api": false,
"runtime_config_store": {
"type": "http",
"config" : {
"url": "https://core.integ.euid.eu/operator/config"
},
"config_scan_period_ms": 300000
}
"runtime_config_path": "https://core.integ.euid.eu/operator/config",
"runtime_config_refresh_ms": 300000
}
9 changes: 2 additions & 7 deletions scripts/aws/conf/euid-prod-config.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,11 +28,6 @@
"enable_phone_support": true,
"enable_v1_phone_support": false,
"enable_v2_encryption": true,
"runtime_config_store": {
"type": "http",
"config" : {
"url": "https://core.prod.euid.eu/operator/config"
},
"config_scan_period_ms": 300000
}
"runtime_config_path": "https://core.prod.euid.eu/operator/config",
"runtime_config_refresh_ms": 300000
}
9 changes: 2 additions & 7 deletions scripts/aws/conf/uid2-integ-config.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,6 @@
"optout_api_uri": "https://optout-integ.uidapi.com/optout/replicate",
"optout_s3_folder": "uid-optout-integ/",
"allow_legacy_api": false,
"runtime_config_store": {
"type": "http",
"config" : {
"url": "https://core-integ.uidapi.com/operator/config"
},
"config_scan_period_ms": 300000
}
"runtime_config_path": "https://core-integ.uidapi.com/operator/config",
"runtime_config_refresh_ms": 300000
}
9 changes: 2 additions & 7 deletions scripts/aws/conf/uid2-prod-config.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,6 @@
"refresh_token_expires_after_seconds": 2592000,
"refresh_identity_token_after_seconds": 3600,
"allow_legacy_api": false,
"runtime_config_store": {
"type": "http",
"config" : {
"url": "https://core-prod.uidapi.com/operator/config"
},
"config_scan_period_ms": 300000
}
"runtime_config_path": "https://core-prod.uidapi.com/operator/config",
"runtime_config_refresh_ms": 300000
}
9 changes: 2 additions & 7 deletions scripts/azure-cc/conf/integ-uid2-config.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,6 @@
"core_attest_url": "https://core-integ.uidapi.com/attest",
"optout_api_uri": "https://optout-integ.uidapi.com/optout/replicate",
"optout_s3_folder": "uid-optout-integ/",
"runtime_config_store": {
"type": "http",
"config" : {
"url": "https://core-integ.uidapi.com/operator/config"
},
"config_scan_period_ms": 300000
}
"runtime_config_path": "https://core-integ.uidapi.com/operator/config",
"runtime_config_refresh_ms": 300000
}
9 changes: 2 additions & 7 deletions scripts/azure-cc/conf/prod-uid2-config.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,6 @@
"optout_api_uri": "https://optout-prod.uidapi.com/optout/replicate",
"optout_s3_folder": "optout-v2/",
"identity_token_expires_after_seconds": 259200,
"runtime_config_store": {
"type": "http",
"config" : {
"url": "https://core-prod.uidapi.com/operator/config"
},
"config_scan_period_ms": 300000
}
"runtime_config_path": "https://core-prod.uidapi.com/operator/config",
"runtime_config_refresh_ms": 300000
}
9 changes: 2 additions & 7 deletions scripts/gcp-oidc/conf/integ-config.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,6 @@
"core_attest_url": "https://core.uidapi.com/attest",
"optout_api_uri": "https://optout.uidapi.com/optout/replicate",
"optout_s3_folder": "uid-optout-integ/",
"runtime_config_store": {
"type": "http",
"config" : {
"url": "https://core.uidapi.com/operator/config"
},
"config_scan_period_ms": 300000
}
"runtime_config_path": "https://core.uidapi.com/operator/config",
"runtime_config_refresh_ms": 300000
}
9 changes: 2 additions & 7 deletions scripts/gcp-oidc/conf/prod-config.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,6 @@
"optout_api_uri": "https://optout.uidapi.com/optout/replicate",
"optout_s3_folder": "optout-v2/",
"identity_token_expires_after_seconds": 259200,
"runtime_config_store": {
"type": "http",
"config" : {
"url": "https://core.uidapi.com/operator/config"
},
"config_scan_period_ms": 300000
}
"runtime_config_path": "https://core.uidapi.com/operator/config",
"runtime_config_refresh_ms": 300000
}
2 changes: 2 additions & 0 deletions src/main/java/com/uid2/operator/Const.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,5 +32,7 @@ public class Config extends com.uid2.shared.Const.Config {

public static final String ConfigScanPeriodMsProp = "config_scan_period_ms";
public static final String IdentityV3Prop = "identity_v3";
public static final String OperatorRuntimeConfigEventBus = "operator.runtime.config";
public static String RuntimeConfigPathProp = "runtime_config_path";
}
}
32 changes: 17 additions & 15 deletions src/main/java/com/uid2/operator/Main.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@
import java.util.function.Supplier;

import static com.uid2.operator.Const.Config.ConfigScanPeriodMsProp;
import static com.uid2.operator.Const.Config.OperatorRuntimeConfigEventBus;
import static io.micrometer.core.instrument.Metrics.globalRegistry;

public class Main {
Expand All @@ -77,6 +78,7 @@ public class Main {
private final RotatingClientSideKeypairStore clientSideKeypairProvider;
private final RotatingSaltProvider saltProvider;
private final CloudSyncOptOutStore optOutStore;
private final RotatingRuntimeConfigStore runtimeConfigStore;
private OperatorShutdownHandler shutdownHandler = null;
private final OperatorMetrics metrics;
private final boolean clientSideTokenGenerate;
Expand Down Expand Up @@ -146,6 +148,8 @@ public Main(Vertx vertx, JsonObject config) throws Exception {
this.keysetProvider = new RotatingKeysetProvider(fsStores, new GlobalScope(new CloudPath(keysetMdPath)));
String saltsMdPath = this.config.getString(Const.Config.SaltsMetadataPathProp);
this.saltProvider = new RotatingSaltProvider(fsStores, saltsMdPath);
String runtimeConfigMdPath = this.config.getString(Const.Config.RuntimeConfigPathProp);
this.runtimeConfigStore = new RotatingRuntimeConfigStore(vertx, fsStores, runtimeConfigMdPath);
this.optOutStore = new CloudSyncOptOutStore(vertx, fsLocal, this.config, operatorKey, Clock.systemUTC());

if (this.validateServiceLinks) {
Expand Down Expand Up @@ -269,23 +273,17 @@ private ICloudStorage wrapCloudStorageForOptOut(ICloudStorage cloudStorage) {
}
}

private Future<IConfigService> initialiseConfigService() throws Exception {
private Future<IConfigService> initialiseConfigService(ConfigRetriever dynamicConfigRetriever) {
Promise<IConfigService> promise = Promise.promise();

ConfigRetriever dynamicConfigRetriever = ConfigRetrieverFactory.create(
vertx,
config.getJsonObject("runtime_config_store"),
this.createOperatorKeyRetriever().retrieve()
);
Future<ConfigService> dynamicConfigFuture = ConfigService.create(dynamicConfigRetriever);

ConfigRetriever staticConfigRetriever = ConfigRetrieverFactory.create(
vertx,
new JsonObject()
.put("type", "json")
.put("config", config)
.put(ConfigScanPeriodMsProp, -1),
""
.put(ConfigScanPeriodMsProp, -1)
);

Future<ConfigService> staticConfigFuture = ConfigService.create(staticConfigRetriever);
Expand Down Expand Up @@ -337,9 +335,17 @@ private void run() throws Exception {
this.createVertxInstancesMetric();
this.createVertxEventLoopsMetric();

this.initialiseConfigService()
.compose(configService -> {
ConfigRetriever dynamicConfigRetriever = ConfigRetrieverFactory.create(
vertx,
new JsonObject()
.put("type", "event-bus")
.put("config", new JsonObject()
.put("address", OperatorRuntimeConfigEventBus))
);

this.createStoreVerticles()
.compose(v -> this.initialiseConfigService(dynamicConfigRetriever))
.compose(configService -> {
Comment on lines +338 to +348
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Reason for this change ->createStoreVerticles method needs to be before initialiseConfigService so that the runtime config is retrieved from core and published to event-bus. dynamicConfigRetriever needs to listen to runtime config address on event bus before publishing config otherwise the message will be discarded.

Supplier<Verticle> operatorVerticleSupplier = () -> {
UIDOperatorVerticle verticle = new UIDOperatorVerticle(configService, config, this.clientSideTokenGenerate, siteProvider, clientKeyProvider, clientSideKeypairProvider, getKeyManager(), saltProvider, optOutStore, Clock.systemUTC(), _statsCollectorQueue, new SecureLinkValidatorService(this.serviceLinkProvider, this.serviceProvider), this.shutdownHandler::handleSaltRetrievalResponse);
return verticle;
Expand All @@ -352,11 +358,6 @@ private void run() throws Exception {
Promise<Void> compositePromise = Promise.promise();
List<Future> fs = new ArrayList<>();
fs.add(createAndDeployStatsCollector());
try {
fs.add(createStoreVerticles());
} catch (Exception e) {
throw new RuntimeException(e);
}

CompositeFuture.all(fs).onComplete(ar -> {
if (ar.failed()) compositePromise.fail(new Exception(ar.cause()));
Expand Down Expand Up @@ -424,6 +425,7 @@ private Future<Void> createStoreVerticles() throws Exception {
fs.add(createAndDeployRotatingStoreVerticle("keyset", keysetProvider, "keyset_refresh_ms"));
fs.add(createAndDeployRotatingStoreVerticle("keysetkey", keysetKeyStore, "keysetkey_refresh_ms"));
fs.add(createAndDeployRotatingStoreVerticle("salt", saltProvider, "salt_refresh_ms"));
fs.add(createAndDeployRotatingStoreVerticle("runtime_config", runtimeConfigStore, "runtime_config_refresh_ms"));
fs.add(createAndDeployCloudSyncStoreVerticle("optout", fsOptOut, optOutCloudSync));
CompositeFuture.all(fs).onComplete(ar -> {
if (ar.failed()) promise.fail(new Exception(ar.cause()));
Expand Down
20 changes: 2 additions & 18 deletions src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,35 +6,19 @@
import io.vertx.core.Vertx;
import io.vertx.core.json.JsonObject;

import java.net.URI;

import static com.uid2.operator.Const.Config.ConfigScanPeriodMsProp;

public class ConfigRetrieverFactory {
public static ConfigRetriever create(Vertx vertx, JsonObject bootstrapConfig, String operatorKey) {
public static ConfigRetriever create(Vertx vertx, JsonObject bootstrapConfig) {
String type = bootstrapConfig.getString("type");
JsonObject storeConfig = bootstrapConfig.getJsonObject("config");
if (type.equals("http")) {
URI uri = URI.create(storeConfig.getString("url"));
storeConfig.remove("url");
storeConfig.put("host", uri.getHost());
int port = uri.getPort();
if (port == -1) {
port = uri.getScheme().equals("https") ? 443 : 80;
}
storeConfig.put("port", port);
storeConfig.put("path", uri.getPath());
storeConfig.put("ssl", uri.getScheme().equals("https"));
storeConfig.put("headers", new JsonObject()
.put("Authorization", "Bearer " + operatorKey));
}

ConfigStoreOptions storeOptions = new ConfigStoreOptions()
.setType(type)
.setConfig(storeConfig);

ConfigRetrieverOptions retrieverOptions = new ConfigRetrieverOptions()
.setScanPeriod(bootstrapConfig.getLong(ConfigScanPeriodMsProp))
.setScanPeriod(bootstrapConfig.getLong(ConfigScanPeriodMsProp, 5000L))
.addStore(storeOptions);

return ConfigRetriever.create(vertx, retrieverOptions);
Expand Down
Loading