Adding more work on salt encryption

pom.xml

@@ -16,7 +16,7 @@
         <!-- check micrometer.version vertx-micrometer-metrics consumes before bumping up -->
         <micrometer.version>1.12.2</micrometer.version>
         <junit-jupiter.version>5.11.2</junit-jupiter.version>
-        <uid2-shared.version>8.0.6</uid2-shared.version>
+        <uid2-shared.version>8.0.11-alpha-173-SNAPSHOT</uid2-shared.version>
         <okta-jwt.version>0.5.10</okta-jwt.version>
         <image.version>${project.version}</image.version>
     </properties>

src/main/java/com/uid2/admin/job/EncryptionJob/ClientKeyEncryptionJob.java

@@ -33,7 +33,7 @@ public String getId() {
     public void execute() throws Exception {
         PrivateSiteDataMap<LegacyClientKey> desiredPrivateState = PrivateSiteUtil.getClientKeys(globalOperators, globalClientKeys);
         multiScopeStoreWriter.uploadPrivateWithEncryption(desiredPrivateState, null);
-        PrivateSiteDataMap<LegacyClientKey> desiredPublicState = PublicSiteUtil.getPublicClients(globalClientKeys,globalOperators);
+        PrivateSiteDataMap<LegacyClientKey> desiredPublicState = PublicSiteUtil.getPublicClients(globalClientKeys, globalOperators);
         multiScopeStoreWriter.uploadPublicWithEncryption(desiredPublicState, null);
     }
 }

src/main/java/com/uid2/admin/job/EncryptionJob/ClientSideKeypairEncryptionJob.java

@@ -0,0 +1,36 @@
+package com.uid2.admin.job.EncryptionJob;
+
+import com.uid2.admin.job.model.Job;
+import com.uid2.admin.model.PrivateSiteDataMap;
+import com.uid2.admin.store.MultiScopeStoreWriter;
+import com.uid2.admin.util.PublicSiteUtil;
+import com.uid2.shared.auth.OperatorKey;
+import com.uid2.shared.model.ClientSideKeypair;
+
+import java.util.Collection;
+
+public class ClientSideKeypairEncryptionJob extends Job {
+    private final Collection<OperatorKey> globalOperators;
+    private final Collection<ClientSideKeypair> globalClientSideKeypairs;
+
+    private final MultiScopeStoreWriter<Collection<ClientSideKeypair>> multiScopeStoreWriter;
+
+    public ClientSideKeypairEncryptionJob(Collection<OperatorKey> globalOperators, Collection<ClientSideKeypair> globalClientSideKeypairs,
+                                          MultiScopeStoreWriter<Collection<ClientSideKeypair>> multiScopeStoreWriter) {
+        this.globalOperators = globalOperators;
+        this.globalClientSideKeypairs = globalClientSideKeypairs;
+        this.multiScopeStoreWriter = multiScopeStoreWriter;
+    }
+
+    @Override
+    public String getId() {
+        return "cloud-encryption-sync-clientside-keypair";
+    }
+
+    @Override
+    public void execute() throws Exception {
+        // Only public operators support clientside keypair
+        PrivateSiteDataMap<ClientSideKeypair> desiredPublicState = PublicSiteUtil.getPublicClientKeypairs(globalClientSideKeypairs, globalOperators);
+        multiScopeStoreWriter.uploadPublicWithEncryption(desiredPublicState, null);
+    }
+}

src/main/java/com/uid2/admin/job/EncryptionJob/SaltEncryptionJob.java

@@ -0,0 +1,40 @@
+package com.uid2.admin.job.EncryptionJob;
+
+import com.uid2.admin.job.model.Job;
+import com.uid2.admin.model.PrivateSiteDataMap;
+import com.uid2.admin.store.MultiScopeStoreWriter;
+import com.uid2.admin.util.PrivateSiteUtil;
+import com.uid2.admin.util.PublicSiteUtil;
+import com.uid2.shared.auth.OperatorKey;
+import com.uid2.shared.model.SaltEntry;
+import com.uid2.shared.store.RotatingSaltProvider;
+
+import java.util.Collection;
+
+public class SaltEncryptionJob extends Job {
+    private final Collection<OperatorKey> globalOperators;
+    private final Collection<RotatingSaltProvider.SaltSnapshot> saltEntries;
+    private final MultiScopeStoreWriter<Collection<RotatingSaltProvider.SaltSnapshot>> multiScopeStoreWriter;
+
+    public SaltEncryptionJob(Collection<OperatorKey> globalOperators,
+                             Collection<RotatingSaltProvider.SaltSnapshot> saltEntries,
+                             MultiScopeStoreWriter<Collection<RotatingSaltProvider.SaltSnapshot>> multiScopeStoreWriter) {
+        this.globalOperators = globalOperators;
+        this.saltEntries = saltEntries;
+        this.multiScopeStoreWriter = multiScopeStoreWriter;
+    }
+
+
+    @Override
+    public String getId() {
+        return "cloud-encryption-sync-salts";
+    }
+
+    @Override
+    public void execute() throws Exception {
+        PrivateSiteDataMap<RotatingSaltProvider.SaltSnapshot> desiredPrivateState = PrivateSiteUtil.getPrivateSaltEntries(saltEntries, globalOperators);
+        multiScopeStoreWriter.uploadPrivateWithEncryption(desiredPrivateState, null);
+        PrivateSiteDataMap<RotatingSaltProvider.SaltSnapshot> desiredPublicState = PublicSiteUtil.getPublicSaltEntries(saltEntries, globalOperators);
+        multiScopeStoreWriter.uploadPublicWithEncryption(desiredPublicState, null);
+    }
+}

src/main/java/com/uid2/admin/job/jobsync/EncryptedFilesSyncJob.java

@@ -2,7 +2,6 @@
 
 import com.fasterxml.jackson.databind.ObjectWriter;
 import com.uid2.admin.job.EncryptionJob.*;
-import com.uid2.admin.job.EncryptionJob.ClientKeyEncryptionJob;
 import com.uid2.admin.job.model.Job;
 import com.uid2.admin.store.*;
 import com.uid2.admin.store.factory.*;
@@ -17,12 +16,15 @@
 import com.uid2.shared.auth.RotatingOperatorKeyProvider;
 import com.uid2.shared.cloud.CloudUtils;
 import com.uid2.shared.cloud.ICloudStorage;
+import com.uid2.shared.cloud.TaggableCloudStorage;
+import com.uid2.shared.model.ClientSideKeypair;
 import com.uid2.shared.model.EncryptionKey;
 import com.uid2.shared.model.KeysetKey;
 import com.uid2.shared.model.Site;
 import com.uid2.shared.store.CloudPath;
 import com.uid2.admin.legacy.LegacyClientKey;
-import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider;
+import com.uid2.shared.store.EncryptedRotatingSaltProvider;
+import com.uid2.shared.store.RotatingSaltProvider;
 import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider;
 import com.uid2.shared.store.scope.GlobalScope;
 import io.vertx.core.json.JsonObject;
@@ -34,12 +36,12 @@
 public class EncryptedFilesSyncJob extends Job {
     private final JsonObject config;
     private final WriteLock writeLock;
-    private final RotatingCloudEncryptionKeyProvider RotatingCloudEncryptionKeyProvider;
+    private final RotatingCloudEncryptionKeyProvider rotatingCloudEncryptionKeyProvider;
 
     public EncryptedFilesSyncJob(JsonObject config, WriteLock writeLock, RotatingCloudEncryptionKeyProvider RotatingCloudEncryptionKeyProvider) {
         this.config = config;
         this.writeLock = writeLock;
-        this.RotatingCloudEncryptionKeyProvider = RotatingCloudEncryptionKeyProvider;
+        this.rotatingCloudEncryptionKeyProvider = RotatingCloudEncryptionKeyProvider;
     }
 
     @Override
@@ -49,20 +51,22 @@ public String getId() {
 
     @Override
     public void execute() throws Exception {
-        ICloudStorage cloudStorage = CloudUtils.createStorage(config.getString(Const.Config.CoreS3BucketProp), config);
+        TaggableCloudStorage cloudStorage = CloudUtils.createStorage(config.getString(Const.Config.CoreS3BucketProp), config);
         FileStorage fileStorage = new TmpFileStorage();
         ObjectWriter jsonWriter = JsonUtil.createJsonWriter();
         Clock clock = new InstantClock();
         VersionGenerator versionGenerator = new EpochVersionGenerator(clock);
         FileManager fileManager = new FileManager(cloudStorage, fileStorage);
 
+        RotatingSaltProvider saltProvider = new RotatingSaltProvider(cloudStorage, config.getString(Const.Config.SaltsMetadataPathProp));
+
         SiteStoreFactory siteStoreFactory = new SiteStoreFactory(
                 cloudStorage,
                 new CloudPath(config.getString(Const.Config.SitesMetadataPathProp)),
                 jsonWriter,
                 versionGenerator,
                 clock,
-                RotatingCloudEncryptionKeyProvider,
+                rotatingCloudEncryptionKeyProvider,
                 fileManager);
 
         ClientKeyStoreFactory clientKeyStoreFactory = new ClientKeyStoreFactory(
@@ -71,15 +75,15 @@ public void execute() throws Exception {
                 jsonWriter,
                 versionGenerator,
                 clock,
-                RotatingCloudEncryptionKeyProvider,
+                rotatingCloudEncryptionKeyProvider,
                 fileManager);
 
         EncryptionKeyStoreFactory encryptionKeyStoreFactory = new EncryptionKeyStoreFactory(
                 cloudStorage,
                 new CloudPath(config.getString(Const.Config.KeysMetadataPathProp)),
                 versionGenerator,
                 clock,
-                RotatingCloudEncryptionKeyProvider,
+                rotatingCloudEncryptionKeyProvider,
                 fileManager);
 
         KeyAclStoreFactory keyAclStoreFactory = new KeyAclStoreFactory(
@@ -88,7 +92,7 @@ public void execute() throws Exception {
                 jsonWriter,
                 versionGenerator,
                 clock,
-                RotatingCloudEncryptionKeyProvider,
+                rotatingCloudEncryptionKeyProvider,
                 fileManager);
 
         KeysetStoreFactory keysetStoreFactory = new KeysetStoreFactory(
@@ -98,7 +102,7 @@ public void execute() throws Exception {
                 versionGenerator,
                 clock,
                 fileManager,
-                RotatingCloudEncryptionKeyProvider,
+                rotatingCloudEncryptionKeyProvider,
                 config.getBoolean(enableKeysetConfigProp));
 
         KeysetKeyStoreFactory keysetKeyStoreFactory = new KeysetKeyStoreFactory(
@@ -107,15 +111,33 @@ public void execute() throws Exception {
                 versionGenerator,
                 clock,
                 fileManager,
-                RotatingCloudEncryptionKeyProvider,
+                rotatingCloudEncryptionKeyProvider,
                 config.getBoolean(enableKeysetConfigProp));
 
+        SaltStoreFactory saltStoreFactory = new SaltStoreFactory(
+                config,
+                new CloudPath(config.getString(Const.Config.SaltsMetadataPathProp)),
+                fileManager,
+                cloudStorage,
+                versionGenerator,
+                rotatingCloudEncryptionKeyProvider
+        );
+
+        ClientSideKeypairStoreFactory clientSideKeypairStoreFactory = new ClientSideKeypairStoreFactory(
+                cloudStorage,
+                new CloudPath(config.getString(Const.Config.ClientSideKeypairsMetadataPathProp)),
+                versionGenerator,
+                clock,
+                rotatingCloudEncryptionKeyProvider,
+                fileManager
+        );
+
         CloudPath operatorMetadataPath = new CloudPath(config.getString(Const.Config.OperatorsMetadataPathProp));
         GlobalScope operatorScope = new GlobalScope(operatorMetadataPath);
         RotatingOperatorKeyProvider operatorKeyProvider = new RotatingOperatorKeyProvider(cloudStorage, cloudStorage, operatorScope);
 
         synchronized (writeLock) {
-            RotatingCloudEncryptionKeyProvider.loadContent();
+            rotatingCloudEncryptionKeyProvider.loadContent();
             operatorKeyProvider.loadContent(operatorKeyProvider.getMetadata());
             siteStoreFactory.getGlobalReader().loadContent(siteStoreFactory.getGlobalReader().getMetadata());
             clientKeyStoreFactory.getGlobalReader().loadContent();
@@ -125,13 +147,18 @@ public void execute() throws Exception {
                 keysetStoreFactory.getGlobalReader().loadContent();
                 keysetKeyStoreFactory.getGlobalReader().loadContent();
             }
+            saltProvider.loadContent();
+            clientSideKeypairStoreFactory.getGlobalReader().loadContent();
         }
+
         Collection<OperatorKey> globalOperators = operatorKeyProvider.getAll();
         Collection<Site> globalSites = siteStoreFactory.getGlobalReader().getAllSites();
         Collection<LegacyClientKey> globalClients = clientKeyStoreFactory.getGlobalReader().getAll();
         Collection<EncryptionKey> globalEncryptionKeys = encryptionKeyStoreFactory.getGlobalReader().getSnapshot().getActiveKeySet();
         Integer globalMaxKeyId = encryptionKeyStoreFactory.getGlobalReader().getMetadata().getInteger("max_key_id");
         Map<Integer, EncryptionKeyAcl> globalKeyAcls = keyAclStoreFactory.getGlobalReader().getSnapshot().getAllAcls();
+        Collection<ClientSideKeypair> globalClientSideKeypair = clientSideKeypairStoreFactory.getGlobalReader().getAll();
+
         MultiScopeStoreWriter<Collection<Site>> siteWriter = new MultiScopeStoreWriter<>(
                 fileManager,
                 siteStoreFactory,
@@ -148,6 +175,14 @@ public void execute() throws Exception {
                 fileManager,
                 keyAclStoreFactory,
                 MultiScopeStoreWriter::areMapsEqual);
+        MultiScopeStoreWriter<Collection<RotatingSaltProvider.SaltSnapshot>> saltWriter = new MultiScopeStoreWriter<>(
+                fileManager,
+                saltStoreFactory,
+                MultiScopeStoreWriter::areCollectionsEqual);
+        MultiScopeStoreWriter<Collection<ClientSideKeypair>> clientSideKeypairWriter = new MultiScopeStoreWriter<>(
+                fileManager,
+                clientSideKeypairStoreFactory,
+                MultiScopeStoreWriter::areCollectionsEqual);
 
         SiteEncryptionJob siteEncryptionSyncJob = new SiteEncryptionJob(siteWriter, globalSites, globalOperators);
         ClientKeyEncryptionJob clientEncryptionSyncJob = new ClientKeyEncryptionJob(clientWriter, globalClients, globalOperators);
@@ -160,10 +195,15 @@ public void execute() throws Exception {
                 encryptionKeyWriter
         );
         KeyAclEncryptionJob keyAclEncryptionSyncJob = new KeyAclEncryptionJob(keyAclWriter, globalOperators, globalKeyAcls);
+        SaltEncryptionJob saltEncryptionJob = new SaltEncryptionJob(globalOperators, saltProvider.getSnapshots(), saltWriter);
+        ClientSideKeypairEncryptionJob clientSideKeypairEncryptionJob = new ClientSideKeypairEncryptionJob(globalOperators, globalClientSideKeypair, clientSideKeypairWriter);
+
         siteEncryptionSyncJob.execute();
         clientEncryptionSyncJob.execute();
         encryptionKeyEncryptionSyncJob.execute();
         keyAclEncryptionSyncJob.execute();
+        saltEncryptionJob.execute();
+        clientSideKeypairEncryptionJob.execute();
 
         if(config.getBoolean(enableKeysetConfigProp)) {
             Map<Integer, Keyset> globalKeysets = keysetStoreFactory.getGlobalReader().getSnapshot().getAllKeysets();

src/main/java/com/uid2/admin/store/factory/ClientSideKeypairStoreFactory.java

@@ -0,0 +1,79 @@
+package com.uid2.admin.store.factory;
+
+import com.uid2.admin.store.writer.ClientSideKeypairStoreWriter;
+import com.uid2.admin.store.writer.StoreWriter;
+import com.uid2.shared.model.ClientSideKeypair;
+import com.uid2.shared.store.reader.RotatingClientSideKeypairStore;
+import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider;
+import com.uid2.shared.store.reader.StoreReader;
+import com.fasterxml.jackson.databind.ObjectWriter;
+import com.uid2.admin.store.Clock;
+import com.uid2.admin.store.FileManager;
+import com.uid2.admin.store.version.VersionGenerator;
+import com.uid2.shared.cloud.ICloudStorage;
+import com.uid2.shared.store.CloudPath;
+import com.uid2.shared.store.scope.EncryptedScope;
+import com.uid2.shared.store.scope.GlobalScope;
+import com.uid2.shared.store.scope.SiteScope;
+
+import java.util.Collection;
+
+public class ClientSideKeypairStoreFactory implements EncryptedStoreFactory<Collection<ClientSideKeypair>> {
+    private final ICloudStorage fileStreamProvider;
+    private final CloudPath rootMetadataPath;
+    private final VersionGenerator versionGenerator;
+    private final Clock clock;
+    private final FileManager fileManager;
+    private final RotatingClientSideKeypairStore globalReader;
+    private final RotatingCloudEncryptionKeyProvider cloudEncryptionKeyProvider;
+
+    public ClientSideKeypairStoreFactory(
+            ICloudStorage fileStreamProvider,
+            CloudPath rootMetadataPath,
+            VersionGenerator versionGenerator,
+            Clock clock,
+            RotatingCloudEncryptionKeyProvider cloudEncryptionKeyProvider,
+            FileManager fileManager)  {
+        this.fileStreamProvider = fileStreamProvider;
+        this.rootMetadataPath = rootMetadataPath;
+        this.versionGenerator = versionGenerator;
+        this.clock = clock;
+        this.cloudEncryptionKeyProvider = cloudEncryptionKeyProvider;
+        this.fileManager = fileManager;
+        GlobalScope globalScope = new GlobalScope(rootMetadataPath);
+        globalReader = new RotatingClientSideKeypairStore(fileStreamProvider, globalScope);
+    }
+
+    public RotatingClientSideKeypairStore getGlobalReader() {
+        return globalReader;
+    }
+
+    @Override
+    public StoreWriter<Collection<ClientSideKeypair>> getEncryptedWriter(Integer siteId, boolean isPublic) {
+        return new ClientSideKeypairStoreWriter(getEncryptedReader(siteId, isPublic),
+                fileManager,
+                versionGenerator,
+                clock,
+                new EncryptedScope(rootMetadataPath, siteId, isPublic));
+    }
+
+    @Override
+    public StoreReader<Collection<ClientSideKeypair>> getEncryptedReader(Integer siteId, boolean isPublic) {
+        return new RotatingClientSideKeypairStore(fileStreamProvider, new EncryptedScope(rootMetadataPath, siteId, isPublic));
+    }
+
+    @Override
+    public RotatingCloudEncryptionKeyProvider getCloudEncryptionProvider() {
+        return cloudEncryptionKeyProvider;
+    }
+
+    @Override
+    public StoreReader<Collection<ClientSideKeypair>> getReader(Integer siteId) {
+        return new RotatingClientSideKeypairStore(fileStreamProvider, new SiteScope(rootMetadataPath, siteId));
+    }
+
+    @Override
+    public StoreWriter<Collection<ClientSideKeypair>> getWriter(Integer siteId) {
+        return new ClientSideKeypairStoreWriter(getReader(siteId), fileManager, versionGenerator, clock, new SiteScope(rootMetadataPath, siteId));
+    }
+}