diff --git a/it/server/src/test/java/com/linecorp/centraldogma/it/CacheTest.java b/it/server/src/test/java/com/linecorp/centraldogma/it/CacheTest.java deleted file mode 100644 index fe7b70ee96..0000000000 --- a/it/server/src/test/java/com/linecorp/centraldogma/it/CacheTest.java +++ /dev/null @@ -1,209 +0,0 @@ -/* - * Copyright 2020 LINE Corporation - * - * LINE Corporation licenses this file to you under the Apache License, - * version 2.0 (the "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at: - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ -package com.linecorp.centraldogma.it; - -import static com.linecorp.centraldogma.common.Revision.HEAD; -import static com.linecorp.centraldogma.common.Revision.INIT; -import static com.linecorp.centraldogma.testing.internal.TestUtil.normalizedDisplayName; -import static org.assertj.core.api.Assertions.assertThat; - -import java.util.List; -import java.util.Map; -import java.util.function.Supplier; - -import org.junit.jupiter.api.TestInfo; -import org.junit.jupiter.api.extension.RegisterExtension; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.EnumSource; - -import com.linecorp.armeria.common.metric.MoreMeters; -import com.linecorp.centraldogma.client.CentralDogma; -import com.linecorp.centraldogma.common.Change; -import com.linecorp.centraldogma.common.Commit; -import com.linecorp.centraldogma.common.Entry; -import com.linecorp.centraldogma.common.PathPattern; -import com.linecorp.centraldogma.common.PushResult; -import com.linecorp.centraldogma.common.Query; -import com.linecorp.centraldogma.common.Revision; -import com.linecorp.centraldogma.testing.junit.CentralDogmaExtension; - -class CacheTest { - - private static final String REPO_FOO = "foo"; - - @RegisterExtension - static final CentralDogmaExtension dogma = new CentralDogmaExtension(); - - private static final Supplier> metersSupplier = - () -> MoreMeters.measureAll(dogma.dogma().meterRegistry().get()); - - @ParameterizedTest(name = "getFile [{index}: {0}]") - @EnumSource(ClientType.class) - void getFile(ClientType clientType, TestInfo testInfo) { - final String project = normalizedDisplayName(testInfo); - final CentralDogma client = clientType.client(dogma); - client.createProject(project).join(); - client.createRepository(project, REPO_FOO).join(); - - final Map meters1 = metersSupplier.get(); - final PushResult res = client.forRepo(project, REPO_FOO) - .commit("Add a file", Change.ofTextUpsert("/foo.txt", "bar")) - .push() - .join(); - - final Map meters2 = metersSupplier.get(); - // Metadata needs to access to check a write quota (one cache miss). - if (clientType == ClientType.LEGACY) { - // NB: A push operation involves a history() operation to retrieve the last commit when pushing. - // Therefore we should observe five cache miss. (Thrift only) - assertThat(missCount(meters2)).isEqualTo( - missCount(meters1) + 1 + - 1 + // (CacheableHistoryCall) - 4); // (CacheableObjectLoaderCall: 2 for revision2 and 2 for revision1) - } else { - assertThat(missCount(meters2)).isEqualTo(missCount(meters1) + 1); - } - - // First getFile() should miss. - final Query query = Query.ofText("/foo.txt"); - final Entry entry = client.getFile(project, REPO_FOO, HEAD, query).join(); - final Map meters3 = metersSupplier.get(); - - assertThat(missCount(meters3)).isEqualTo(missCount(meters2) + 1); - - // Subsequent getFile() should never miss. - for (int i = 0; i < 3; i++) { - final Map meters4 = metersSupplier.get(); - - // Use the relative revision as well as the absolute revision. - final Entry cachedEntry1 = client.getFile(project, REPO_FOO, res.revision(), query).join(); - final Entry cachedEntry2 = client.getFile(project, REPO_FOO, HEAD, query).join(); - - // They should return the same result. - assertThat(cachedEntry1).isEqualTo(entry); - assertThat(cachedEntry1).isEqualTo(cachedEntry2); - - // .. and should hit the cache. - final Map meters5 = metersSupplier.get(); - assertThat(hitCount(meters5)).isEqualTo(hitCount(meters4) + 2); - assertThat(missCount(meters5)).isEqualTo(missCount(meters4)); - } - } - - @ParameterizedTest(name = "history [{index}: {0}]") - @EnumSource(ClientType.class) - void history(ClientType clientType, TestInfo testInfo) { - final String project = normalizedDisplayName(testInfo); - final CentralDogma client = clientType.client(dogma); - client.createProject(project).join(); - client.createRepository(project, REPO_FOO).join(); - - double prevMissCount = missCount(metersSupplier.get()); - final PushResult res1 = client.forRepo(project, REPO_FOO) - .commit("Add a file", Change.ofTextUpsert("/foo.txt", "bar")) - .push() - .join(); - double currentMissCount = missCount(metersSupplier.get()); - // Metadata needs to access to check a write quota (one cache miss). - if (clientType == ClientType.LEGACY) { - // NB: A push operation involves a history() operation to retrieve the last commit when pushing. - // Therefore we should observe five cache miss. (Thrift only) - assertThat(currentMissCount).isEqualTo( - prevMissCount + 1 + - 1 + // (CacheableHistoryCall) - 4); // (CacheableObjectLoaderCall: 2 for revision2 and 2 for revision1) - } else { - assertThat(currentMissCount).isEqualTo(prevMissCount + 1); - } - prevMissCount = currentMissCount; - - // Get the history in various combination of from/to revisions. - final List history1 = - client.getHistory(project, REPO_FOO, HEAD, new Revision(-2), PathPattern.all(), 0).join(); - - currentMissCount = missCount(metersSupplier.get()); - if (clientType == ClientType.LEGACY) { - assertThat(currentMissCount).isEqualTo(prevMissCount + 1); // (CacheableHistoryCall) - } else { - assertThat(currentMissCount).isEqualTo( - prevMissCount + - 1 + // (CacheableHistoryCall) - 4); // (CacheableObjectLoaderCall: 2 for revision2 and 2 for revision1) - } - prevMissCount = currentMissCount; - - final List history2 = - client.getHistory(project, REPO_FOO, HEAD, INIT, PathPattern.all(), 0).join(); - final List history3 = - client.getHistory(project, REPO_FOO, res1.revision(), new Revision(-2), PathPattern.all(), 0) - .join(); - final List history4 = - client.getHistory(project, REPO_FOO, res1.revision(), INIT, PathPattern.all(), 0).join(); - - // and they should all same. - assertThat(history1).isEqualTo(history2); - assertThat(history1).isEqualTo(history3); - assertThat(history1).isEqualTo(history4); - currentMissCount = missCount(metersSupplier.get()); - // All cached. - assertThat(currentMissCount).isEqualTo(prevMissCount); - } - - @ParameterizedTest(name = "getDiffs [{index}: {0}]") - @EnumSource(ClientType.class) - void getDiffs(ClientType clientType, TestInfo testInfo) { - final String project = normalizedDisplayName(testInfo); - final CentralDogma client = clientType.client(dogma); - client.createProject(project).join(); - client.createRepository(project, REPO_FOO).join(); - - final PushResult res1 = client.forRepo(project, REPO_FOO) - .commit("Add a file", Change.ofTextUpsert("/foo.txt", "bar")) - .push() - .join(); - - final Map meters1 = metersSupplier.get(); - - // Get the diffs in various combination of from/to revisions. - final List> diff1 = - client.getDiff(project, REPO_FOO, HEAD, new Revision(-2), PathPattern.all()).join(); - final List> diff2 = - client.getDiff(project, REPO_FOO, HEAD, INIT, PathPattern.all()).join(); - final List> diff3 = - client.getDiff(project, REPO_FOO, res1.revision(), new Revision(-2), PathPattern.all()).join(); - final List> diff4 = - client.getDiff(project, REPO_FOO, res1.revision(), INIT, PathPattern.all()).join(); - - // and they should all same. - assertThat(diff1).isEqualTo(diff2); - assertThat(diff1).isEqualTo(diff3); - assertThat(diff1).isEqualTo(diff4); - - final Map meters2 = metersSupplier.get(); - - // Should miss once and hit 3 times. - assertThat(missCount(meters2)).isEqualTo(missCount(meters1) + 1); - assertThat(hitCount(meters2)).isEqualTo(hitCount(meters1) + 3); - } - - private static Double hitCount(Map meters) { - return meters.get("cache.gets#count{cache=repository,result=hit}"); - } - - private static Double missCount(Map meters) { - return meters.get("cache.gets#count{cache=repository,result=miss}"); - } -} diff --git a/it/server/src/test/java/com/linecorp/centraldogma/it/WriteQuotaTestBase.java b/it/server/src/test/java/com/linecorp/centraldogma/it/WriteQuotaTestBase.java index a99a685ecf..d869bb1d5f 100644 --- a/it/server/src/test/java/com/linecorp/centraldogma/it/WriteQuotaTestBase.java +++ b/it/server/src/test/java/com/linecorp/centraldogma/it/WriteQuotaTestBase.java @@ -18,6 +18,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.awaitility.Awaitility.await; import java.util.List; import java.util.concurrent.CompletableFuture; @@ -64,8 +65,8 @@ void updateWriteQuota() throws Exception { CompletableFutures.allAsList(futures).join(); /// update write quota to 2qps - QuotaConfig writeQuota = new QuotaConfig(2, 1); - QuotaConfig updated = updateWriteQuota(webClient(), repositoryName, writeQuota); + final QuotaConfig writeQuota = new QuotaConfig(2, 1); + final QuotaConfig updated = updateWriteQuota(webClient(), repositoryName, writeQuota); assertThat(updated).isEqualTo(writeQuota); // Wait for releasing previously acquired locks @@ -86,9 +87,9 @@ void updateWriteQuota() throws Exception { Thread.sleep(1000); // Increase write quota - writeQuota = new QuotaConfig(5, 1); - updated = updateWriteQuota(webClient(), repositoryName, writeQuota); - assertThat(updated).isEqualTo(writeQuota); + final QuotaConfig writeQuota1 = new QuotaConfig(5, 1); + await().untilAsserted(() -> assertThat(updateWriteQuota(webClient(), repositoryName, writeQuota1)) + .isEqualTo(writeQuota1)); final List> futures4 = parallelPush(dogmaClient(), repositoryName, 4); assertThat(CompletableFutures.allAsList(futures4).join()).hasSize(8); diff --git a/server/src/main/java/com/linecorp/centraldogma/server/CentralDogma.java b/server/src/main/java/com/linecorp/centraldogma/server/CentralDogma.java index dd488f63c2..7b1cc97876 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/CentralDogma.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/CentralDogma.java @@ -668,7 +668,8 @@ private Server startServer(ProjectManager pm, CommandExecutor executor, cfg.gracefulShutdownTimeout().ifPresent( t -> sb.gracefulShutdownTimeoutMillis(t.quietPeriodMillis(), t.timeoutMillis())); - final MetadataService mds = new MetadataService(pm, executor); + final MetadataService mds = new MetadataService(pm, executor, projectInitializer); + executor.setRepositoryMetadataSupplier(mds::getRepo); final WatchService watchService = new WatchService(meterRegistry); final AuthProvider authProvider = createAuthProvider(executor, sessionManager, mds); final ProjectApiManager projectApiManager = new ProjectApiManager(pm, executor, mds); @@ -796,7 +797,7 @@ private CommandExecutor newZooKeeperCommandExecutor( new StandaloneCommandExecutor(pm, repositoryWorker, serverStatusManager, sessionManager, /* onTakeLeadership */ null, /* onReleaseLeadership */ null, /* onTakeZoneLeadership */ null, /* onReleaseZoneLeadership */ null), - meterRegistry, pm, config().writeQuotaPerRepository(), zone, + meterRegistry, config().writeQuotaPerRepository(), zone, onTakeLeadership, onReleaseLeadership, onTakeZoneLeadership, onReleaseZoneLeadership); } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/command/AbstractCommandExecutor.java b/server/src/main/java/com/linecorp/centraldogma/server/command/AbstractCommandExecutor.java index 1ac55ae08c..bd955bb078 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/command/AbstractCommandExecutor.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/command/AbstractCommandExecutor.java @@ -22,6 +22,7 @@ import java.util.concurrent.CompletionStage; import java.util.concurrent.ForkJoinPool; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BiFunction; import java.util.function.Consumer; import javax.annotation.Nullable; @@ -34,6 +35,7 @@ import com.linecorp.armeria.common.util.Exceptions; import com.linecorp.armeria.common.util.StartStopSupport; import com.linecorp.centraldogma.common.ReadOnlyException; +import com.linecorp.centraldogma.server.metadata.RepositoryMetadata; /** * Helps to implement a concrete {@link CommandExecutor}. @@ -57,6 +59,9 @@ public abstract class AbstractCommandExecutor implements CommandExecutor { private final AtomicInteger numPendingStopRequests = new AtomicInteger(); private final CommandExecutorStatusManager statusManager; + @Nullable + private BiFunction> repositoryMetadataSupplier; + /** * Creates a new instance. * @@ -120,6 +125,23 @@ public final void setWritable(boolean writable) { this.writable = writable; } + @Override + public void setRepositoryMetadataSupplier( + BiFunction> supplier) { + repositoryMetadataSupplier = requireNonNull(supplier, "supplier"); + } + + /** + * Returns the {@link RepositoryMetadata} of the specified repository. + */ + @Nullable + protected CompletableFuture repositoryMetadata(String projectName, String repoName) { + if (repositoryMetadataSupplier == null) { + return null; + } + return repositoryMetadataSupplier.apply(projectName, repoName); + } + @Override public final CompletableFuture execute(Command command) { requireNonNull(command, "command"); diff --git a/server/src/main/java/com/linecorp/centraldogma/server/command/CommandExecutor.java b/server/src/main/java/com/linecorp/centraldogma/server/command/CommandExecutor.java index 8c9307c03e..c3e86ddc82 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/command/CommandExecutor.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/command/CommandExecutor.java @@ -17,10 +17,12 @@ package com.linecorp.centraldogma.server.command; import java.util.concurrent.CompletableFuture; +import java.util.function.BiFunction; import javax.annotation.Nullable; import com.linecorp.centraldogma.server.QuotaConfig; +import com.linecorp.centraldogma.server.metadata.RepositoryMetadata; /** * An executor interface which executes {@link Command}s. @@ -66,6 +68,12 @@ public interface CommandExecutor { */ void setWriteQuota(String projectName, String repoName, @Nullable QuotaConfig writeQuota); + /** + * Sets the {@link RepositoryMetadata} supplier. + */ + void setRepositoryMetadataSupplier( + BiFunction> supplier); + /** * Executes the specified {@link Command}. * diff --git a/server/src/main/java/com/linecorp/centraldogma/server/command/ForwardingCommandExecutor.java b/server/src/main/java/com/linecorp/centraldogma/server/command/ForwardingCommandExecutor.java index 7330bb6140..f1bd99f061 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/command/ForwardingCommandExecutor.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/command/ForwardingCommandExecutor.java @@ -19,9 +19,11 @@ import static java.util.Objects.requireNonNull; import java.util.concurrent.CompletableFuture; +import java.util.function.BiFunction; import com.linecorp.centraldogma.internal.Util; import com.linecorp.centraldogma.server.QuotaConfig; +import com.linecorp.centraldogma.server.metadata.RepositoryMetadata; /** * A {@link CommandExecutor} which forwards all its method calls to another {@link CommandExecutor}. @@ -73,6 +75,12 @@ public void setWriteQuota(String projectName, String repoName, QuotaConfig write delegate().setWriteQuota(projectName, repoName, writeQuota); } + @Override + public void setRepositoryMetadataSupplier( + BiFunction> supplier) { + delegate().setRepositoryMetadataSupplier(supplier); + } + @Override public CompletableFuture execute(Command command) { return delegate().execute(command); diff --git a/server/src/main/java/com/linecorp/centraldogma/server/command/StandaloneCommandExecutor.java b/server/src/main/java/com/linecorp/centraldogma/server/command/StandaloneCommandExecutor.java index b1cd02f3ec..91f348ed9b 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/command/StandaloneCommandExecutor.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/command/StandaloneCommandExecutor.java @@ -33,12 +33,13 @@ import com.google.common.util.concurrent.RateLimiter; import com.spotify.futures.CompletableFutures; +import com.linecorp.armeria.common.util.UnmodifiableFuture; import com.linecorp.centraldogma.common.TooManyRequestsException; import com.linecorp.centraldogma.server.QuotaConfig; import com.linecorp.centraldogma.server.auth.Session; import com.linecorp.centraldogma.server.auth.SessionManager; import com.linecorp.centraldogma.server.management.ServerStatusManager; -import com.linecorp.centraldogma.server.metadata.MetadataService; +import com.linecorp.centraldogma.server.metadata.RepositoryMetadata; import com.linecorp.centraldogma.server.storage.project.Project; import com.linecorp.centraldogma.server.storage.project.ProjectManager; import com.linecorp.centraldogma.server.storage.repository.Repository; @@ -58,7 +59,6 @@ public class StandaloneCommandExecutor extends AbstractCommandExecutor { private final SessionManager sessionManager; // if permitsPerSecond is -1, a quota is checked by ZooKeeperCommandExecutor. private final double permitsPerSecond; - private final MetadataService metadataService; private final ServerStatusManager serverStatusManager; @VisibleForTesting @@ -130,7 +130,6 @@ private StandaloneCommandExecutor(ProjectManager projectManager, this.sessionManager = sessionManager; this.permitsPerSecond = permitsPerSecond; writeRateLimiters = new ConcurrentHashMap<>(); - metadataService = new MetadataService(projectManager, this); } @Override @@ -334,7 +333,12 @@ private CompletableFuture push0(RepositoryCommand c, boolean no } private CompletableFuture getRateLimiter(String projectName, String repoName) { - return metadataService.getRepo(projectName, repoName).thenApply(meta -> { + final CompletableFuture future = repositoryMetadata(projectName, repoName); + if (future == null) { + // metadata is not available yet. + return UnmodifiableFuture.completedFuture(RateLimiter.create(permitsPerSecond)); + } + return future.thenApply(meta -> { setWriteQuota(projectName, repoName, meta.writeQuota()); return writeRateLimiters.get(rateLimiterKey(projectName, repoName)); }); diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/MetadataApiService.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/MetadataApiService.java index 680a42c5da..8c64e2ce1f 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/MetadataApiService.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/MetadataApiService.java @@ -129,8 +129,8 @@ public CompletableFuture removeMember(@Param String projectName, public CompletableFuture addToken(@Param String projectName, IdAndProjectRole request, Author author) { - return mds.findTokenByAppId(request.id()) - .thenCompose(token -> mds.addToken(author, projectName, token.appId(), request.role())); + final Token token = mds.findTokenByAppId(request.id()); + return mds.addToken(author, projectName, token.appId(), request.role()); } /** @@ -147,8 +147,8 @@ public CompletableFuture updateTokenRole(@Param String projectName, Author author) { final ReplaceOperation operation = ensureSingleReplaceOperation(jsonPatch, "/role"); final ProjectRole role = ProjectRole.of(operation.value()); - return mds.findTokenByAppId(appId) - .thenCompose(token -> mds.updateTokenRole(author, projectName, token, role)); + final Token token = mds.findTokenByAppId(appId); + return mds.updateTokenRole(author, projectName, token, role); } /** @@ -160,8 +160,8 @@ public CompletableFuture updateTokenRole(@Param String projectName, public CompletableFuture removeToken(@Param String projectName, @Param String appId, Author author) { - return mds.findTokenByAppId(appId) - .thenCompose(token -> mds.removeToken(author, projectName, token.appId())); + final Token token = mds.findTokenByAppId(appId); + return mds.removeToken(author, projectName, token.appId()); } /** @@ -256,9 +256,8 @@ public CompletableFuture removeTokenRepositoryRole(@Param String proje @Param String repoName, @Param String appId, Author author) { - return mds.findTokenByAppId(appId) - .thenCompose(token -> mds.removeTokenRepositoryRole(author, - projectName, repoName, appId)); + final Token token = mds.findTokenByAppId(appId); + return mds.removeTokenRepositoryRole(author, projectName, repoName, appId); } /** diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/auth/ApplicationTokenAuthorizer.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/auth/ApplicationTokenAuthorizer.java index c15ba7f677..e00e8e8491 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/auth/ApplicationTokenAuthorizer.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/auth/ApplicationTokenAuthorizer.java @@ -16,13 +16,11 @@ package com.linecorp.centraldogma.server.internal.api.auth; -import static com.linecorp.armeria.common.util.Functions.voidFunction; import static java.util.Objects.requireNonNull; import static java.util.concurrent.CompletableFuture.completedFuture; import java.net.InetSocketAddress; import java.net.SocketAddress; -import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; import java.util.function.Function; @@ -33,6 +31,7 @@ import com.linecorp.armeria.common.auth.OAuth2Token; import com.linecorp.armeria.common.logging.LogLevel; import com.linecorp.armeria.common.util.Exceptions; +import com.linecorp.armeria.common.util.UnmodifiableFuture; import com.linecorp.armeria.server.ServiceRequestContext; import com.linecorp.armeria.server.auth.AuthTokenExtractors; import com.linecorp.armeria.server.auth.Authorizer; @@ -51,9 +50,9 @@ public class ApplicationTokenAuthorizer implements Authorizer { private static final Logger logger = LoggerFactory.getLogger( ApplicationTokenAuthorizer.class); - private final Function> tokenLookupFunc; + private final Function tokenLookupFunc; - public ApplicationTokenAuthorizer(Function> tokenLookupFunc) { + public ApplicationTokenAuthorizer(Function tokenLookupFunc) { this.tokenLookupFunc = requireNonNull(tokenLookupFunc, "tokenLookupFunc"); } @@ -64,40 +63,34 @@ public CompletionStage authorize(ServiceRequestContext ctx, HttpRequest return completedFuture(false); } - final CompletableFuture res = new CompletableFuture<>(); - tokenLookupFunc.apply(token.accessToken()) - .thenAccept(appToken -> { - if (appToken != null && appToken.isActive()) { - final String appId = appToken.appId(); - final StringBuilder login = new StringBuilder(appId); - final SocketAddress ra = ctx.remoteAddress(); - if (ra instanceof InetSocketAddress) { - login.append('@').append(((InetSocketAddress) ra).getHostString()); - } - ctx.logBuilder().authenticatedUser("app/" + appId); - final UserWithToken user = new UserWithToken(login.toString(), appToken); - AuthUtil.setCurrentUser(ctx, user); - HttpApiUtil.setVerboseResponses(ctx, user); - res.complete(true); - } else { - res.complete(false); - } - }) - // Should be authorized by the next authorizer. - .exceptionally(voidFunction(cause -> { - cause = Exceptions.peel(cause); - final LogLevel level; - if (cause instanceof IllegalArgumentException || - cause instanceof TokenNotFoundException) { - level = LogLevel.DEBUG; - } else { - level = LogLevel.WARN; - } - level.log(logger, "Failed to authorize an application token: token={}, addr={}", - token.accessToken(), ctx.clientAddress(), cause); - res.complete(false); - })); - - return res; + try { + final Token appToken = tokenLookupFunc.apply(token.accessToken()); + if (appToken != null && appToken.isActive()) { + final String appId = appToken.appId(); + final StringBuilder login = new StringBuilder(appId); + final SocketAddress ra = ctx.remoteAddress(); + if (ra instanceof InetSocketAddress) { + login.append('@').append(((InetSocketAddress) ra).getHostString()); + } + ctx.logBuilder().authenticatedUser("app/" + appId); + final UserWithToken user = new UserWithToken(login.toString(), appToken); + AuthUtil.setCurrentUser(ctx, user); + HttpApiUtil.setVerboseResponses(ctx, user); + return UnmodifiableFuture.completedFuture(true); + } + return UnmodifiableFuture.completedFuture(false); + } catch (Throwable cause) { + cause = Exceptions.peel(cause); + final LogLevel level; + if (cause instanceof IllegalArgumentException || + cause instanceof TokenNotFoundException) { + level = LogLevel.DEBUG; + } else { + level = LogLevel.WARN; + } + level.log(logger, "Failed to authorize an application token: token={}, addr={}", + token.accessToken(), ctx.clientAddress(), cause); + return UnmodifiableFuture.completedFuture(false); + } } } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/sysadmin/TokenService.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/sysadmin/TokenService.java index 9287847224..8149fc9d73 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/api/sysadmin/TokenService.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/api/sysadmin/TokenService.java @@ -30,6 +30,7 @@ import com.linecorp.armeria.common.HttpHeaderNames; import com.linecorp.armeria.common.HttpStatus; +import com.linecorp.armeria.common.ResponseEntity; import com.linecorp.armeria.common.ResponseHeaders; import com.linecorp.armeria.server.HttpStatusException; import com.linecorp.armeria.server.ServiceRequestContext; @@ -37,7 +38,6 @@ import com.linecorp.armeria.server.annotation.Default; import com.linecorp.armeria.server.annotation.Delete; import com.linecorp.armeria.server.annotation.Get; -import com.linecorp.armeria.server.annotation.HttpResult; import com.linecorp.armeria.server.annotation.Param; import com.linecorp.armeria.server.annotation.Patch; import com.linecorp.armeria.server.annotation.Post; @@ -54,7 +54,6 @@ import com.linecorp.centraldogma.server.internal.api.converter.CreateApiResponseConverter; import com.linecorp.centraldogma.server.metadata.MetadataService; import com.linecorp.centraldogma.server.metadata.Token; -import com.linecorp.centraldogma.server.metadata.Tokens; import com.linecorp.centraldogma.server.metadata.User; /** @@ -87,14 +86,11 @@ public TokenService(CommandExecutor executor, MetadataService mds) { *

Returns the list of the tokens generated before. */ @Get("/tokens") - public CompletableFuture> listTokens(User loginUser) { + public Collection listTokens(User loginUser) { if (loginUser.isSystemAdmin()) { - return mds.getTokens() - .thenApply(tokens -> tokens.appIds().values()); + return mds.getTokens().appIds().values(); } else { - return mds.getTokens() - .thenApply(Tokens::withoutSecret) - .thenApply(tokens -> tokens.appIds().values()); + return mds.getTokens().withoutSecret().appIds().values(); } } @@ -106,7 +102,7 @@ public CompletableFuture> listTokens(User loginUser) { @Post("/tokens") @StatusCode(201) @ResponseConverter(CreateApiResponseConverter.class) - public CompletableFuture> createToken(@Param String appId, + public CompletableFuture> createToken(@Param String appId, // TODO(minwoox): Remove isAdmin field. @Param @Default("false") boolean isAdmin, @Param @Default("false") boolean isSystemAdmin, @@ -127,12 +123,12 @@ public CompletableFuture> createToken(@Param String appId, tokenFuture = mds.createToken(author, appId, isSystemAdminToken); } return tokenFuture - .thenCompose(unused -> mds.findTokenByAppId(appId)) + .thenCompose(unused -> fetchTokensByAppId(appId)) .thenApply(token -> { final ResponseHeaders headers = ResponseHeaders.of(HttpStatus.CREATED, HttpHeaderNames.LOCATION, "/tokens/" + appId); - return HttpResult.of(headers, token); + return ResponseEntity.of(headers, token); }); } @@ -234,13 +230,17 @@ public CompletableFuture updateTokenLevel(ServiceRequestContext ctx, break; } return mds.updateTokenLevel(author, appId, toBeSystemAdmin).thenCompose( - unused -> mds.findTokenByAppId(appId).thenApply(Token::withoutSecret)); + unused -> fetchTokensByAppId(appId).thenApply(Token::withoutSecret)); }); } + private CompletableFuture fetchTokensByAppId(String appId) { + return mds.fetchTokens().thenApply(tokens -> tokens.get(appId)); + } + private CompletableFuture getTokenOrRespondForbidden(ServiceRequestContext ctx, String appId, User loginUser) { - return mds.findTokenByAppId(appId).thenApply(token -> { + return fetchTokensByAppId(appId).thenApply(token -> { // Give permission to the system administrators. if (!loginUser.isSystemAdmin() && !token.creation().user().equals(loginUser.id())) { diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/replication/ZooKeeperCommandExecutor.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/replication/ZooKeeperCommandExecutor.java index b3e6742218..6456693809 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/replication/ZooKeeperCommandExecutor.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/replication/ZooKeeperCommandExecutor.java @@ -42,7 +42,6 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ForkJoinPool; @@ -92,7 +91,6 @@ import com.google.common.util.concurrent.Uninterruptibles; import com.linecorp.armeria.common.util.SafeCloseable; -import com.linecorp.centraldogma.common.CentralDogmaException; import com.linecorp.centraldogma.common.Revision; import com.linecorp.centraldogma.common.TooManyRequestsException; import com.linecorp.centraldogma.internal.Jackson; @@ -109,10 +107,8 @@ import com.linecorp.centraldogma.server.command.NormalizingPushCommand; import com.linecorp.centraldogma.server.command.RemoveRepositoryCommand; import com.linecorp.centraldogma.server.command.UpdateServerStatusCommand; -import com.linecorp.centraldogma.server.metadata.MetadataService; import com.linecorp.centraldogma.server.metadata.RepositoryMetadata; import com.linecorp.centraldogma.server.storage.project.Project; -import com.linecorp.centraldogma.server.storage.project.ProjectManager; import io.micrometer.core.instrument.Gauge; import io.micrometer.core.instrument.MeterRegistry; @@ -174,8 +170,6 @@ public final class ZooKeeperCommandExecutor @Nullable private final String zone; - private MetadataService metadataService; - // Failing to acquire a lock is a critical problem, so we wait as much as we can. private long lockTimeoutNanos = TimeUnit.MINUTES.toNanos(1); @@ -408,7 +402,6 @@ private static final class ListenerInfo { public ZooKeeperCommandExecutor(ZooKeeperReplicationConfig cfg, File dataDir, CommandExecutor delegate, MeterRegistry meterRegistry, - ProjectManager projectManager, @Nullable QuotaConfig writeQuota, @Nullable String zone, @Nullable Consumer onTakeLeadership, @@ -430,7 +423,6 @@ public ZooKeeperCommandExecutor(ZooKeeperReplicationConfig cfg, this.delegate = requireNonNull(delegate, "delegate"); this.meterRegistry = requireNonNull(meterRegistry, "meterRegistry"); this.writeQuota = writeQuota; - metadataService = new MetadataService(projectManager, this); this.zone = zone; // Register the metrics which are accessible even before started. @@ -1066,15 +1058,13 @@ private WriteLock acquireWriteLock(NormalizingPushCommand command) throws Except if (writeQuota == null) { // Cache miss, load a write quota - final RepositoryMetadata meta; - try { - meta = metadataService.getRepo(command.projectName(), command.repositoryName()).get(); - } catch (InterruptedException | ExecutionException e) { - throw new CentralDogmaException("Unexpected exception caught while retrieving " + - RepositoryMetadata.class.getSimpleName(), e); + final CompletableFuture future = repositoryMetadata(command.projectName(), + command.repositoryName()); + if (future == null) { + // MetadataService is not available yet. + return null; } - - writeQuota = meta.writeQuota(); + writeQuota = future.join().writeQuota(); } if (writeQuota == null) { @@ -1392,14 +1382,9 @@ private void createZkPathIfMissing(String zkPath) throws Exception { } } - @VisibleForTesting - void setMetadataService(MetadataService metadataService) { - this.metadataService = metadataService; - } - @VisibleForTesting void setLockTimeoutMillis(long lockTimeoutMillis) { - this.lockTimeoutNanos = TimeUnit.MILLISECONDS.toNanos(lockTimeoutMillis); + lockTimeoutNanos = TimeUnit.MILLISECONDS.toNanos(lockTimeoutMillis); } private static final class WriteLock { diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/PurgeSchedulingService.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/PurgeSchedulingService.java index 84a133246e..606846dc5e 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/PurgeSchedulingService.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/PurgeSchedulingService.java @@ -150,7 +150,7 @@ private void purgeRepository(CommandExecutor commandExecutor, } private static void purgeToken(MetadataService metadataService) { - final Tokens tokens = metadataService.getTokens().join(); + final Tokens tokens = metadataService.getTokens(); final List purging = tokens.appIds().values() .stream() .filter(Token::isDeleted) diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/PurgeSchedulingServicePlugin.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/PurgeSchedulingServicePlugin.java index 4f38430225..f84a2f16d8 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/PurgeSchedulingServicePlugin.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/PurgeSchedulingServicePlugin.java @@ -53,7 +53,8 @@ public synchronized CompletionStage start(PluginContext context) { this.purgeSchedulingService = purgeSchedulingService; } final MetadataService metadataService = new MetadataService(context.projectManager(), - context.commandExecutor()); + context.commandExecutor(), + context.internalProjectInitializer()); purgeSchedulingService.start(context.commandExecutor(), metadataService); return CompletableFuture.completedFuture(null); } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/RepositoryCache.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/RepositoryCache.java index 6d9c821e1d..8377eef5c7 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/RepositoryCache.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/RepositoryCache.java @@ -19,27 +19,28 @@ import static java.util.Objects.requireNonNull; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.locks.Lock; -import java.util.function.Supplier; import javax.annotation.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.github.benmanes.caffeine.cache.AsyncLoadingCache; +import com.github.benmanes.caffeine.cache.AsyncCache; import com.github.benmanes.caffeine.cache.Caffeine; import com.github.benmanes.caffeine.cache.CaffeineSpec; import com.github.benmanes.caffeine.cache.Weigher; -import com.github.benmanes.caffeine.cache.stats.CacheStats; import com.google.common.base.MoreObjects; +import com.linecorp.armeria.common.util.SafeCloseable; +import com.linecorp.armeria.internal.common.RequestContextUtil; +import com.linecorp.centraldogma.server.storage.repository.CacheableCall; + import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.binder.cache.CaffeineCacheMetrics; -public class RepositoryCache { +public final class RepositoryCache { - private static final Logger logger = LoggerFactory.getLogger(RepositoryCache.class); + public static final Logger logger = LoggerFactory.getLogger(RepositoryCache.class); @Nullable public static String validateCacheSpec(@Nullable String cacheSpec) { @@ -56,7 +57,7 @@ public static String validateCacheSpec(@Nullable String cacheSpec) { } @SuppressWarnings("rawtypes") - private final AsyncLoadingCache cache; + private final AsyncCache cache; private final String cacheSpec; @SuppressWarnings({ "rawtypes", "unchecked" }) @@ -69,71 +70,32 @@ public RepositoryCache(String cacheSpec, MeterRegistry meterRegistry) { builder.weigher((Weigher) CacheableCall::weigh); } cache = builder.recordStats() - .buildAsync((key, executor) -> { - logger.debug("Cache miss: {}", key); - return key.execute(); - }); + .buildAsync(); CaffeineCacheMetrics.monitor(meterRegistry, cache, "repository"); } public CompletableFuture get(CacheableCall call) { requireNonNull(call, "call"); - @SuppressWarnings("unchecked") - final CompletableFuture f = (CompletableFuture) cache.get(call); - return f; - } - - @Nullable - public CompletableFuture getIfPresent(CacheableCall call) { - requireNonNull(call, "call"); - @SuppressWarnings("unchecked") - final CompletableFuture f = (CompletableFuture) cache.getIfPresent(call); - return f; - } - - public void put(CacheableCall call, T value) { - requireNonNull(call, "call"); - requireNonNull(value, "value"); - cache.put(call, CompletableFuture.completedFuture(value)); - } - - public T load(CacheableCall key, Supplier supplier, boolean logIfMiss) { - CompletableFuture existingFuture = getIfPresent(key); - if (existingFuture != null) { - final T existingValue = existingFuture.getNow(null); - if (existingValue != null) { - // Cached already. - return existingValue; - } + final CompletableFuture future = new CompletableFuture<>(); + //noinspection unchecked + final CompletableFuture prior = + (CompletableFuture) cache.asMap().putIfAbsent(call, (CompletableFuture) future); + if (prior != null) { + return prior; } - // Not cached yet. - final Lock lock = key.coarseGrainedLock(); - lock.lock(); - try { - existingFuture = getIfPresent(key); - if (existingFuture != null) { - final T existingValue = existingFuture.getNow(null); - if (existingValue != null) { - // Other thread already put the entries to the cache before we acquire the lock. - return existingValue; + call.execute().handle((result, cause) -> { + try (SafeCloseable ignored = RequestContextUtil.pop()) { + if (cause != null) { + future.completeExceptionally(cause); + } else { + future.complete(result); } } - - final T value = supplier.get(); - put(key, value); - if (logIfMiss) { - logger.debug("Cache miss: {}", key); - } - return value; - } finally { - lock.unlock(); - } - } - - public CacheStats stats() { - return cache.synchronous().stats(); + return null; + }); + return future; } public void clear() { @@ -144,7 +106,7 @@ public void clear() { public String toString() { return MoreObjects.toStringHelper(this) .add("spec", cacheSpec) - .add("stats", stats()) + .add("stats", cache.synchronous().stats()) .toString(); } } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/RepositoryWrapper.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/RepositoryWrapper.java index 7ae8102f3a..d7ea9296f6 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/RepositoryWrapper.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/RepositoryWrapper.java @@ -36,6 +36,7 @@ import com.linecorp.centraldogma.server.command.CommitResult; import com.linecorp.centraldogma.server.command.ContentTransformer; import com.linecorp.centraldogma.server.storage.project.Project; +import com.linecorp.centraldogma.server.storage.repository.CacheableCall; import com.linecorp.centraldogma.server.storage.repository.DiffResultType; import com.linecorp.centraldogma.server.storage.repository.FindOption; import com.linecorp.centraldogma.server.storage.repository.Repository; @@ -215,6 +216,11 @@ public CompletableFuture> mergeFiles(Revision revision, Merge return unwrap().mergeFiles(revision, query); } + @Override + public CompletableFuture execute(CacheableCall cacheableCall) { + return unwrap().execute(cacheableCall); + } + @Override public void addListener(RepositoryListener listener) { unwrap().addListener(listener); diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CacheableFindCall.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CacheableFindCall.java index 8634945762..00d954037c 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CacheableFindCall.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CacheableFindCall.java @@ -16,6 +16,7 @@ package com.linecorp.centraldogma.server.internal.storage.repository.cache; +import static com.linecorp.centraldogma.server.internal.storage.repository.RepositoryCache.logger; import static java.util.Objects.requireNonNull; import java.util.Map; @@ -26,11 +27,11 @@ import com.linecorp.centraldogma.common.Entry; import com.linecorp.centraldogma.common.Revision; -import com.linecorp.centraldogma.server.internal.storage.repository.CacheableCall; +import com.linecorp.centraldogma.server.storage.repository.AbstractCacheableCall; import com.linecorp.centraldogma.server.storage.repository.FindOption; import com.linecorp.centraldogma.server.storage.repository.Repository; -final class CacheableFindCall extends CacheableCall>> { +final class CacheableFindCall extends AbstractCacheableCall>> { final Revision revision; final String pathPattern; @@ -50,7 +51,7 @@ final class CacheableFindCall extends CacheableCall>> { } @Override - protected int weigh(Map> value) { + public int weigh(Map> value) { int weight = 0; weight += pathPattern.length(); weight += options.size(); @@ -65,6 +66,7 @@ protected int weigh(Map> value) { @Override public CompletableFuture>> execute() { + logger.debug("Cache miss: {}", this); return repo().find(revision, pathPattern, options); } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CacheableFindLatestRevCall.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CacheableFindLatestRevCall.java index 9b8fdbd530..d67ae9cfbd 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CacheableFindLatestRevCall.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CacheableFindLatestRevCall.java @@ -17,6 +17,7 @@ package com.linecorp.centraldogma.server.internal.storage.repository.cache; import static com.google.common.base.MoreObjects.firstNonNull; +import static com.linecorp.centraldogma.server.internal.storage.repository.RepositoryCache.logger; import static java.util.Objects.requireNonNull; import java.util.Objects; @@ -27,10 +28,10 @@ import com.linecorp.armeria.common.util.Exceptions; import com.linecorp.centraldogma.common.EntryNotFoundException; import com.linecorp.centraldogma.common.Revision; -import com.linecorp.centraldogma.server.internal.storage.repository.CacheableCall; +import com.linecorp.centraldogma.server.storage.repository.AbstractCacheableCall; import com.linecorp.centraldogma.server.storage.repository.Repository; -final class CacheableFindLatestRevCall extends CacheableCall { +final class CacheableFindLatestRevCall extends AbstractCacheableCall { static final Revision EMPTY = new Revision(Integer.MIN_VALUE); static final Revision ENTRY_NOT_FOUND = new Revision(Integer.MIN_VALUE); @@ -57,12 +58,13 @@ final class CacheableFindLatestRevCall extends CacheableCall { } @Override - protected int weigh(Revision value) { + public int weigh(Revision value) { return pathPattern.length(); } @Override public CompletableFuture execute() { + logger.debug("Cache miss: {}", this); return repo().findLatestRevision(lastKnownRevision, pathPattern, errorOnEntryNotFound) .handle((revision, cause) -> { if (cause != null) { diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CacheableHistoryCall.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CacheableHistoryCall.java index 2bf791d3a8..65dc0222be 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CacheableHistoryCall.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CacheableHistoryCall.java @@ -16,6 +16,7 @@ package com.linecorp.centraldogma.server.internal.storage.repository.cache; +import static com.linecorp.centraldogma.server.internal.storage.repository.RepositoryCache.logger; import static java.util.Objects.requireNonNull; import java.util.List; @@ -26,10 +27,10 @@ import com.linecorp.centraldogma.common.Commit; import com.linecorp.centraldogma.common.Revision; -import com.linecorp.centraldogma.server.internal.storage.repository.CacheableCall; +import com.linecorp.centraldogma.server.storage.repository.AbstractCacheableCall; import com.linecorp.centraldogma.server.storage.repository.Repository; -final class CacheableHistoryCall extends CacheableCall> { +final class CacheableHistoryCall extends AbstractCacheableCall> { final Revision from; final Revision to; @@ -52,7 +53,7 @@ final class CacheableHistoryCall extends CacheableCall> { } @Override - protected int weigh(List value) { + public int weigh(List value) { int weight = 0; weight += pathPattern.length(); for (Commit c : value) { @@ -64,6 +65,7 @@ protected int weigh(List value) { @Override public CompletableFuture> execute() { + logger.debug("Cache miss: {}", this); return repo().history(from, to, pathPattern, maxCommits); } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CacheableMergeQueryCall.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CacheableMergeQueryCall.java index 3d61a9f86c..96e507d4dd 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CacheableMergeQueryCall.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CacheableMergeQueryCall.java @@ -16,35 +16,30 @@ package com.linecorp.centraldogma.server.internal.storage.repository.cache; -import static com.google.common.base.Preconditions.checkState; import static com.linecorp.centraldogma.internal.Util.validateJsonFilePath; +import static com.linecorp.centraldogma.server.internal.storage.repository.RepositoryCache.logger; import static java.util.Objects.requireNonNull; import java.util.List; import java.util.Objects; import java.util.concurrent.CompletableFuture; -import javax.annotation.Nullable; - import com.google.common.base.MoreObjects.ToStringHelper; import com.linecorp.centraldogma.common.MergeQuery; import com.linecorp.centraldogma.common.MergeSource; import com.linecorp.centraldogma.common.MergedEntry; import com.linecorp.centraldogma.common.Revision; -import com.linecorp.centraldogma.server.internal.storage.repository.CacheableCall; +import com.linecorp.centraldogma.server.storage.repository.AbstractCacheableCall; import com.linecorp.centraldogma.server.storage.repository.Repository; -final class CacheableMergeQueryCall extends CacheableCall> { +final class CacheableMergeQueryCall extends AbstractCacheableCall> { private final Revision revision; - private final MergeQuery query; + private final MergeQuery query; private final int hashCode; - @Nullable - MergedEntry computedValue; - - CacheableMergeQueryCall(Repository repo, Revision revision, MergeQuery query) { + CacheableMergeQueryCall(Repository repo, Revision revision, MergeQuery query) { super(repo); this.revision = requireNonNull(revision, "revision"); this.query = requireNonNull(query, "query"); @@ -57,7 +52,7 @@ final class CacheableMergeQueryCall extends CacheableCall> { } @Override - protected int weigh(MergedEntry value) { + public int weigh(MergedEntry value) { int weight = 0; final List mergeSources = query.mergeSources(); weight += mergeSources.size(); @@ -76,14 +71,9 @@ protected int weigh(MergedEntry value) { } @Override - public CompletableFuture> execute() { - checkState(computedValue != null, "computedValue is not set yet."); - return CompletableFuture.completedFuture(computedValue); - } - - void computedValue(MergedEntry computedValue) { - checkState(this.computedValue == null, "computedValue is already set."); - this.computedValue = requireNonNull(computedValue, "computedValue"); + public CompletableFuture> execute() { + logger.debug("Cache miss: {}", this); + return repo().mergeFiles(revision, query); } @Override @@ -97,7 +87,7 @@ public boolean equals(Object o) { return false; } - final CacheableMergeQueryCall that = (CacheableMergeQueryCall) o; + final CacheableMergeQueryCall that = (CacheableMergeQueryCall) o; return revision.equals(that.revision) && query.equals(that.query); } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CacheableMultiDiffCall.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CacheableMultiDiffCall.java index 96863489e8..c730a601c4 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CacheableMultiDiffCall.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CacheableMultiDiffCall.java @@ -16,6 +16,7 @@ package com.linecorp.centraldogma.server.internal.storage.repository.cache; +import static com.linecorp.centraldogma.server.internal.storage.repository.RepositoryCache.logger; import static java.util.Objects.requireNonNull; import java.util.Map; @@ -26,11 +27,11 @@ import com.linecorp.centraldogma.common.Change; import com.linecorp.centraldogma.common.Revision; -import com.linecorp.centraldogma.server.internal.storage.repository.CacheableCall; +import com.linecorp.centraldogma.server.storage.repository.AbstractCacheableCall; import com.linecorp.centraldogma.server.storage.repository.DiffResultType; import com.linecorp.centraldogma.server.storage.repository.Repository; -final class CacheableMultiDiffCall extends CacheableCall>> { +final class CacheableMultiDiffCall extends AbstractCacheableCall>> { private final Revision from; private final Revision to; @@ -54,7 +55,7 @@ final class CacheableMultiDiffCall extends CacheableCall>> } @Override - protected int weigh(Map> value) { + public int weigh(Map> value) { int weight = 0; weight += pathPattern.length(); for (Change e : value.values()) { @@ -69,6 +70,7 @@ protected int weigh(Map> value) { @Override public CompletableFuture>> execute() { + logger.debug("Cache miss: {}", this); return repo().diff(from, to, pathPattern, diffResultType); } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CacheableQueryCall.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CacheableQueryCall.java index ed8a1b2942..5ca784817a 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CacheableQueryCall.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CacheableQueryCall.java @@ -17,6 +17,7 @@ package com.linecorp.centraldogma.server.internal.storage.repository.cache; import static com.google.common.base.MoreObjects.firstNonNull; +import static com.linecorp.centraldogma.server.internal.storage.repository.RepositoryCache.logger; import static java.util.Objects.requireNonNull; import java.util.Objects; @@ -27,10 +28,10 @@ import com.linecorp.centraldogma.common.Entry; import com.linecorp.centraldogma.common.Query; import com.linecorp.centraldogma.common.Revision; -import com.linecorp.centraldogma.server.internal.storage.repository.CacheableCall; +import com.linecorp.centraldogma.server.storage.repository.AbstractCacheableCall; import com.linecorp.centraldogma.server.storage.repository.Repository; -final class CacheableQueryCall extends CacheableCall> { +final class CacheableQueryCall extends AbstractCacheableCall> { static final Entry EMPTY = Entry.ofDirectory(new Revision(Integer.MAX_VALUE), "/"); @@ -49,7 +50,7 @@ final class CacheableQueryCall extends CacheableCall> { } @Override - protected int weigh(Entry value) { + public int weigh(Entry value) { int weight = 0; weight += query.path().length(); for (String e : query.expressions()) { @@ -63,6 +64,7 @@ protected int weigh(Entry value) { @Override public CompletableFuture> execute() { + logger.debug("Cache miss: {}", this); return repo().getOrNull(revision, query).thenApply(e -> firstNonNull(e, EMPTY)); } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CacheableSingleDiffCall.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CacheableSingleDiffCall.java index 4d33cfa2dc..d653043ab4 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CacheableSingleDiffCall.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CacheableSingleDiffCall.java @@ -16,6 +16,7 @@ package com.linecorp.centraldogma.server.internal.storage.repository.cache; +import static com.linecorp.centraldogma.server.internal.storage.repository.RepositoryCache.logger; import static java.util.Objects.requireNonNull; import java.util.Objects; @@ -26,10 +27,10 @@ import com.linecorp.centraldogma.common.Change; import com.linecorp.centraldogma.common.Query; import com.linecorp.centraldogma.common.Revision; -import com.linecorp.centraldogma.server.internal.storage.repository.CacheableCall; +import com.linecorp.centraldogma.server.storage.repository.AbstractCacheableCall; import com.linecorp.centraldogma.server.storage.repository.Repository; -final class CacheableSingleDiffCall extends CacheableCall> { +final class CacheableSingleDiffCall extends AbstractCacheableCall> { final Revision from; final Revision to; @@ -50,7 +51,7 @@ final class CacheableSingleDiffCall extends CacheableCall> { } @Override - protected int weigh(Change value) { + public int weigh(Change value) { int weight = 0; weight += query.path().length(); for (String e : query.expressions()) { @@ -66,6 +67,7 @@ protected int weigh(Change value) { @Override public CompletableFuture> execute() { + logger.debug("Cache miss: {}", this); return repo().diff(from, to, query); } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CachingRepository.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CachingRepository.java index ca3732ebcf..c0db212d5b 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CachingRepository.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CachingRepository.java @@ -47,6 +47,7 @@ import com.linecorp.centraldogma.server.command.ContentTransformer; import com.linecorp.centraldogma.server.internal.storage.repository.RepositoryCache; import com.linecorp.centraldogma.server.storage.project.Project; +import com.linecorp.centraldogma.server.storage.repository.CacheableCall; import com.linecorp.centraldogma.server.storage.repository.DiffResultType; import com.linecorp.centraldogma.server.storage.repository.FindOption; import com.linecorp.centraldogma.server.storage.repository.Repository; @@ -113,11 +114,7 @@ public CompletableFuture>> find(Revision revision, String p requireNonNull(options, "options"); final Revision normalizedRevision = normalizeNow(revision); - return cache.get(new CacheableFindCall(repo, normalizedRevision, pathPattern, options)) - .handleAsync((unused, cause) -> { - throwUnsafelyIfNonNull(cause); - return unused; - }, executor()); + return execute(new CacheableFindCall(repo, normalizedRevision, pathPattern, options)); } @Override @@ -136,12 +133,7 @@ public CompletableFuture> history(Revision from, Revision to, // e.g. when from = 2 and to = 4, the same result should be yielded when maxCommits >= 3. final int actualMaxCommits = Math.min( maxCommits, Math.abs(range.from().major() - range.to().major()) + 1); - return cache.get(new CacheableHistoryCall(repo, range.from(), range.to(), - pathPattern, actualMaxCommits)) - .handleAsync((unused, cause) -> { - throwUnsafelyIfNonNull(cause); - return unused; - }, executor()); + return execute(new CacheableHistoryCall(repo, range.from(), range.to(), pathPattern, actualMaxCommits)); } @Override @@ -151,11 +143,7 @@ public CompletableFuture> diff(Revision from, Revision to, Query qu requireNonNull(query, "query"); final RevisionRange range = normalizeNow(from, to).toAscending(); - return cache.get(new CacheableSingleDiffCall(repo, range.from(), range.to(), query)) - .handleAsync((unused, cause) -> { - throwUnsafelyIfNonNull(cause); - return unused; - }, executor()); + return execute(new CacheableSingleDiffCall(repo, range.from(), range.to(), query)); } @Override @@ -167,12 +155,7 @@ public CompletableFuture>> diff(Revision from, Revision to requireNonNull(diffResultType, "diffResultType"); final RevisionRange range = normalizeNow(from, to).toAscending(); - return cache.get(new CacheableMultiDiffCall(repo, range.from(), range.to(), - pathPattern, diffResultType)) - .handleAsync((unused, cause) -> { - throwUnsafelyIfNonNull(cause); - return unused; - }, executor()); + return execute(new CacheableMultiDiffCall(repo, range.from(), range.to(), pathPattern, diffResultType)); } @Override @@ -260,20 +243,15 @@ public CompletableFuture> mergeFiles(Revision revision, Merge requireNonNull(query, "query"); final Revision normalizedRevision = normalizeNow(revision); - final CacheableMergeQueryCall key = new CacheableMergeQueryCall(repo, normalizedRevision, query); - final CompletableFuture> value = cache.getIfPresent(key); - if (value != null) { - return unsafeCast(value.handleAsync((unused, cause) -> { - throwUnsafelyIfNonNull(cause); - return unused; - }, executor())); - } + return execute(new CacheableMergeQueryCall<>(repo, normalizedRevision, query)); + } - return Repository.super.mergeFiles(normalizedRevision, query).thenApply(mergedEntry -> { - key.computedValue(mergedEntry); - cache.get(key); - return mergedEntry; - }); + @Override + public CompletableFuture execute(CacheableCall cacheableCall) { + return unsafeCast(cache.get(cacheableCall).handleAsync((result, cause) -> { + throwUnsafelyIfNonNull(cause); + return result; + }, executor())); } @Override diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CacheableCompareTreesCall.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CacheableCompareTreesCall.java index 3e068c28f8..636078cae7 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CacheableCompareTreesCall.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CacheableCompareTreesCall.java @@ -15,6 +15,8 @@ */ package com.linecorp.centraldogma.server.internal.storage.repository.git; +import static com.linecorp.centraldogma.server.internal.storage.repository.RepositoryCache.logger; + import java.util.List; import java.util.Objects; import java.util.concurrent.CompletableFuture; @@ -24,24 +26,26 @@ import org.eclipse.jgit.attributes.Attribute; import org.eclipse.jgit.diff.DiffEntry; import org.eclipse.jgit.revwalk.RevTree; +import org.eclipse.jgit.treewalk.filter.TreeFilter; import com.google.common.base.MoreObjects.ToStringHelper; -import com.linecorp.centraldogma.server.internal.storage.repository.CacheableCall; -import com.linecorp.centraldogma.server.storage.repository.Repository; +import com.linecorp.centraldogma.server.storage.repository.AbstractCacheableCall; -final class CacheableCompareTreesCall extends CacheableCall> { +final class CacheableCompareTreesCall extends AbstractCacheableCall> { private static final int SHA1_LEN = 20; + private final GitRepository repo; @Nullable private final RevTree treeA; @Nullable private final RevTree treeB; private final int hashCode; - CacheableCompareTreesCall(Repository repo, @Nullable RevTree treeA, @Nullable RevTree treeB) { + CacheableCompareTreesCall(GitRepository repo, @Nullable RevTree treeA, @Nullable RevTree treeB) { super(repo); + this.repo = repo; this.treeA = treeA; this.treeB = treeB; @@ -49,7 +53,7 @@ final class CacheableCompareTreesCall extends CacheableCall> { } @Override - protected int weigh(List value) { + public int weigh(List value) { int weight = SHA1_LEN * 2; for (DiffEntry e : value) { if (e.getOldId() != null) { @@ -82,7 +86,9 @@ protected int weigh(List value) { */ @Override public CompletableFuture> execute() { - throw new IllegalStateException(); + logger.debug("Cache miss: {}", this); + final List diffEntries = repo.blockingCompareTreesUncached(treeA, treeB, TreeFilter.ALL); + return CompletableFuture.completedFuture(diffEntries); } @Override diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CacheableObjectLoaderCall.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CacheableObjectLoaderCall.java index 83188594fd..7b860b2d9a 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CacheableObjectLoaderCall.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CacheableObjectLoaderCall.java @@ -15,39 +15,48 @@ */ package com.linecorp.centraldogma.server.internal.storage.repository.git; +import static org.eclipse.jgit.lib.Constants.OBJ_TREE; + +import java.io.IOException; +import java.io.UncheckedIOException; import java.util.concurrent.CompletableFuture; import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.ObjectLoader; +import org.eclipse.jgit.lib.ObjectReader; import com.google.common.base.MoreObjects.ToStringHelper; import com.google.common.primitives.Ints; -import com.linecorp.centraldogma.server.internal.storage.repository.CacheableCall; +import com.linecorp.centraldogma.server.storage.repository.AbstractCacheableCall; import com.linecorp.centraldogma.server.storage.repository.Repository; -final class CacheableObjectLoaderCall extends CacheableCall { +final class CacheableObjectLoaderCall extends AbstractCacheableCall { + private final ObjectReader delegate; private final AnyObjectId objectId; private final int hashCode; - CacheableObjectLoaderCall(Repository repo, AnyObjectId objectId) { + CacheableObjectLoaderCall(Repository repo, ObjectReader delegate, AnyObjectId objectId) { super(repo); + this.delegate = delegate; this.objectId = objectId; hashCode = objectId.hashCode() * 31 + System.identityHashCode(repo); } @Override - protected int weigh(ObjectLoader value) { + public int weigh(ObjectLoader value) { return Ints.saturatedCast(value.getSize()); } - /** - * Never invoked because {@link GitRepository} produces the value of this call. - */ @Override public CompletableFuture execute() { - throw new IllegalStateException(); + try { + // Do not leave a dubug log here because it will be called very frequently. + return CompletableFuture.completedFuture(delegate.open(objectId, OBJ_TREE)); + } catch (IOException e) { + throw new UncheckedIOException("failed to open an object: " + objectId, e); + } } @Override diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CachingTreeObjectReader.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CachingTreeObjectReader.java index e53c2e8069..bf83e8f21e 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CachingTreeObjectReader.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/CachingTreeObjectReader.java @@ -19,7 +19,6 @@ import static org.eclipse.jgit.lib.Constants.OBJ_TREE; import java.io.IOException; -import java.io.UncheckedIOException; import javax.annotation.Nullable; @@ -58,17 +57,9 @@ public ObjectLoader open(AnyObjectId objectId, int typeHint) if (OBJ_TREE != typeHint || cache == null) { return delegate.open(objectId, typeHint); } - - // Need to convert to objectId from MutableObjectId - final AnyObjectId objectId0 = objectId.toObjectId(); - - final CacheableObjectLoaderCall key = new CacheableObjectLoaderCall(repository, objectId0); - return cache.load(key, () -> { - try { - return delegate.open(objectId0, typeHint); - } catch (IOException e) { - throw new UncheckedIOException("failed to open an object: " + objectId0, e); - } - }, false); + final CacheableObjectLoaderCall key = new CacheableObjectLoaderCall( + // Need to convert to objectId from MutableObjectId + repository, delegate, objectId.toObjectId()); + return cache.get(key).join(); } } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepository.java b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepository.java index c592ad9dec..c6f68e842a 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepository.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/git/GitRepository.java @@ -109,6 +109,7 @@ import com.linecorp.centraldogma.server.internal.storage.repository.RepositoryCache; import com.linecorp.centraldogma.server.storage.StorageException; import com.linecorp.centraldogma.server.storage.project.Project; +import com.linecorp.centraldogma.server.storage.repository.CacheableCall; import com.linecorp.centraldogma.server.storage.repository.DiffResultType; import com.linecorp.centraldogma.server.storage.repository.FindOption; import com.linecorp.centraldogma.server.storage.repository.FindOptions; @@ -1034,12 +1035,12 @@ private List blockingCompareTrees(RevTree treeA, RevTree treeB) { } final CacheableCompareTreesCall key = new CacheableCompareTreesCall(this, treeA, treeB); - return cache.load(key, () -> blockingCompareTreesUncached(treeA, treeB, TreeFilter.ALL), true); + return cache.get(key).join(); } - private List blockingCompareTreesUncached(@Nullable RevTree treeA, - @Nullable RevTree treeB, - TreeFilter filter) { + List blockingCompareTreesUncached(@Nullable RevTree treeA, + @Nullable RevTree treeB, + TreeFilter filter) { readLock(); try (DiffFormatter diffFormatter = new DiffFormatter(null)) { diffFormatter.setRepository(jGitRepository); @@ -1085,6 +1086,18 @@ public CompletableFuture watch(Revision lastKnownRevision, String path return future; } + @Override + public CompletableFuture execute(CacheableCall cacheableCall) { + // This is executed only when the CachingRepository is not enabled. + requireNonNull(cacheableCall, "cacheableCall"); + final ServiceRequestContext ctx = context(); + + return CompletableFuture.supplyAsync(() -> { + failFastIfTimedOut(this, logger, ctx, "execute", cacheableCall); + return cacheableCall.execute(); + }, repositoryWorker).thenCompose(Function.identity()); + } + @Override public void addListener(RepositoryListener listener) { listeners.add(listener); diff --git a/server/src/main/java/com/linecorp/centraldogma/server/metadata/Member.java b/server/src/main/java/com/linecorp/centraldogma/server/metadata/Member.java index 1056dbc20c..ba41258898 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/metadata/Member.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/metadata/Member.java @@ -26,11 +26,12 @@ import com.linecorp.centraldogma.common.ProjectRole; import com.linecorp.centraldogma.internal.Util; import com.linecorp.centraldogma.server.storage.project.Project; +import com.linecorp.centraldogma.server.storage.repository.HasWeight; /** * Specifies details of a member who belongs to the {@link Project}. */ -public class Member implements Identifiable { +public class Member implements Identifiable, HasWeight { /** * A login name of a member. @@ -94,6 +95,11 @@ public ProjectRole role() { return role; } + @Override + public int weight() { + return login.length() + role.name().length(); + } + /** * Returns who added this member to a project when. */ diff --git a/server/src/main/java/com/linecorp/centraldogma/server/metadata/MetadataService.java b/server/src/main/java/com/linecorp/centraldogma/server/metadata/MetadataService.java index a801056b2c..196e4aef66 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/metadata/MetadataService.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/metadata/MetadataService.java @@ -22,7 +22,6 @@ import static com.linecorp.centraldogma.internal.jsonpatch.JsonPatchUtil.encodeSegment; import static com.linecorp.centraldogma.server.internal.storage.project.ProjectApiManager.listProjectsWithoutInternal; import static com.linecorp.centraldogma.server.metadata.RepositoryMetadata.DEFAULT_PROJECT_ROLES; -import static com.linecorp.centraldogma.server.metadata.RepositorySupport.convertWithJackson; import static com.linecorp.centraldogma.server.metadata.Tokens.SECRET_PREFIX; import static com.linecorp.centraldogma.server.metadata.Tokens.validateSecret; import static com.linecorp.centraldogma.server.storage.project.InternalProjectInitializer.INTERNAL_PROJECT_DOGMA; @@ -50,6 +49,7 @@ import com.linecorp.centraldogma.common.Author; import com.linecorp.centraldogma.common.Change; import com.linecorp.centraldogma.common.ChangeConflictException; +import com.linecorp.centraldogma.common.EntryNotFoundException; import com.linecorp.centraldogma.common.ProjectRole; import com.linecorp.centraldogma.common.RedundantChangeException; import com.linecorp.centraldogma.common.RepositoryExistsException; @@ -62,6 +62,7 @@ import com.linecorp.centraldogma.internal.jsonpatch.TestAbsenceOperation; import com.linecorp.centraldogma.server.QuotaConfig; import com.linecorp.centraldogma.server.command.CommandExecutor; +import com.linecorp.centraldogma.server.storage.project.InternalProjectInitializer; import com.linecorp.centraldogma.server.storage.project.Project; import com.linecorp.centraldogma.server.storage.project.ProjectManager; @@ -91,19 +92,20 @@ public class MetadataService { private final RepositorySupport metadataRepo; private final RepositorySupport tokenRepo; private final CommandExecutor executor; + private final InternalProjectInitializer projectInitializer; private final Map> reposInAddingMetadata = new ConcurrentHashMap<>(); /** * Creates a new instance. */ - public MetadataService(ProjectManager projectManager, CommandExecutor executor) { + public MetadataService(ProjectManager projectManager, CommandExecutor executor, + InternalProjectInitializer projectInitializer) { this.projectManager = requireNonNull(projectManager, "projectManager"); this.executor = requireNonNull(executor, "executor"); - metadataRepo = new RepositorySupport<>(projectManager, executor, - entry -> convertWithJackson(entry, ProjectMetadata.class)); - tokenRepo = new RepositorySupport<>(projectManager, executor, - entry -> convertWithJackson(entry, Tokens.class)); + this.projectInitializer = requireNonNull(projectInitializer, "projectInitializer"); + metadataRepo = new RepositorySupport<>(projectManager, executor, ProjectMetadata.class); + tokenRepo = new RepositorySupport<>(projectManager, executor, Tokens.class); } /** @@ -111,72 +113,81 @@ public MetadataService(ProjectManager projectManager, CommandExecutor executor) */ public CompletableFuture getProject(String projectName) { requireNonNull(projectName, "projectName"); - return fetchMetadata(projectName).thenApply(HolderWithRevision::object); + return getOrFetchMetadata(projectName); } - private CompletableFuture> fetchMetadata(String projectName) { - return fetchMetadata0(projectName).thenCompose(holder -> { - final Set repos = projectManager.get(projectName).repos().list().keySet(); - final Set reposWithMetadata = holder.object().repos().keySet(); - - // Make sure all repositories have metadata. If not, create missing metadata. - // A repository can have missing metadata when a dev forgot to call `addRepo()` - // after creating a new repository. - final ImmutableList.Builder> builder = ImmutableList.builder(); - for (String repo : repos) { - if (reposWithMetadata.contains(repo) || - repo.equals(Project.REPO_DOGMA)) { - continue; - } - - final String projectAndRepositoryName = projectName + '/' + repo; - final CompletableFuture future = new CompletableFuture<>(); - final CompletableFuture futureInMap = - reposInAddingMetadata.computeIfAbsent(projectAndRepositoryName, key -> future); - if (futureInMap != future) { // The metadata is already in adding. - builder.add(futureInMap); - continue; - } - - logger.warn("Adding missing repository metadata: {}/{}", projectName, repo); - final Author author = projectManager.get(projectName).repos().get(repo).author(); - final CompletableFuture addRepoFuture = addRepo(author, projectName, repo); - addRepoFuture.handle((revision, cause) -> { - if (cause != null) { - future.completeExceptionally(cause); - } else { - future.complete(revision); - } - reposInAddingMetadata.remove(projectAndRepositoryName); - return null; - }); - builder.add(future); + private CompletableFuture getOrFetchMetadata(String projectName) { + final ProjectMetadata metadata = getMetadata(projectName); + final Set reposWithMetadata = metadata.repos().keySet(); + final Set repos = projectManager.get(projectName).repos().list().keySet(); + + // Make sure all repositories have metadata. If not, create missing metadata. + // A repository can have missing metadata when a dev forgot to call `addRepo()` + // after creating a new repository. + final ImmutableList.Builder> builder = ImmutableList.builder(); + for (String repo : repos) { + if (reposWithMetadata.contains(repo) || + repo.equals(Project.REPO_DOGMA)) { + continue; } - final ImmutableList> futures = builder.build(); - if (futures.isEmpty()) { - // All repositories have metadata. - return CompletableFuture.completedFuture(holder); + final String projectAndRepositoryName = projectName + '/' + repo; + final CompletableFuture future = new CompletableFuture<>(); + final CompletableFuture futureInMap = + reposInAddingMetadata.computeIfAbsent(projectAndRepositoryName, key -> future); + if (futureInMap != future) { // The metadata is already in adding. + builder.add(futureInMap); + continue; } - // Some repository did not have metadata and thus will add the missing ones. - return CompletableFutures.successfulAsList(futures, cause -> { - final Throwable peeled = Exceptions.peel(cause); - // The metadata of the repository is added by another worker, so we can ignore the exception. - if (peeled instanceof RepositoryExistsException) { - return null; + logger.warn("Adding missing repository metadata: {}/{}", projectName, repo); + final Author author = projectManager.get(projectName).repos().get(repo).author(); + final CompletableFuture addRepoFuture = addRepo(author, projectName, repo); + addRepoFuture.handle((revision, cause) -> { + if (cause != null) { + future.completeExceptionally(cause); + } else { + future.complete(revision); } - return Exceptions.throwUnsafely(cause); - }).thenCompose(unused -> { - logger.info("Fetching {}/{}{} again", - projectName, Project.REPO_DOGMA, METADATA_JSON); - return fetchMetadata0(projectName); + reposInAddingMetadata.remove(projectAndRepositoryName); + return null; }); + builder.add(future); + } + + final ImmutableList> futures = builder.build(); + if (futures.isEmpty()) { + // All repositories have metadata. + return CompletableFuture.completedFuture(metadata); + } + + // Some repository did not have metadata and thus will add the missing ones. + return CompletableFutures.successfulAsList(futures, cause -> { + final Throwable peeled = Exceptions.peel(cause); + // The metadata of the repository is added by another worker, so we can ignore the exception. + if (peeled instanceof RepositoryExistsException) { + return null; + } + return Exceptions.throwUnsafely(cause); + }).thenCompose(unused -> { + logger.info("Fetching {}/{}{} again", + projectName, Project.REPO_DOGMA, METADATA_JSON); + return fetchMetadata(projectName); }); } - private CompletableFuture> fetchMetadata0(String projectName) { - return metadataRepo.fetch(projectName, Project.REPO_DOGMA, METADATA_JSON); + private ProjectMetadata getMetadata(String projectName) { + final Project project = projectManager.get(projectName); + final ProjectMetadata metadata = project.metadata(); + if (metadata == null) { + throw new EntryNotFoundException("project metadata not found: " + projectName); + } + return metadata; + } + + private CompletableFuture fetchMetadata(String projectName) { + return metadataRepo.fetch(projectName, Project.REPO_DOGMA, METADATA_JSON) + .thenApply(HolderWithRevision::object); } /** @@ -477,20 +488,17 @@ public CompletableFuture addToken(Author author, String projectName, requireNonNull(appId, "appId"); requireNonNull(role, "role"); - return getTokens().thenCompose(tokens -> { - tokens.get(appId); // Will raise an exception if not found. - - final TokenRegistration registration = new TokenRegistration(appId, role, - UserAndTimestamp.of(author)); - final JsonPointer path = JsonPointer.compile("/tokens" + encodeSegment(registration.id())); - final Change change = - Change.ofJsonPatch(METADATA_JSON, - asJsonArray(new TestAbsenceOperation(path), - new AddOperation(path, Jackson.valueToTree(registration)))); - final String commitSummary = "Add a token '" + registration.id() + - "' to the project '" + projectName + "' with a role '" + role + '\''; - return metadataRepo.push(projectName, Project.REPO_DOGMA, author, commitSummary, change); - }); + getTokens().get(appId); // Will raise an exception if not found. + final TokenRegistration registration = new TokenRegistration(appId, role, + UserAndTimestamp.of(author)); + final JsonPointer path = JsonPointer.compile("/tokens" + encodeSegment(registration.id())); + final Change change = + Change.ofJsonPatch(METADATA_JSON, + asJsonArray(new TestAbsenceOperation(path), + new AddOperation(path, Jackson.valueToTree(registration)))); + final String commitSummary = "Add a token '" + registration.id() + + "' to the project '" + projectName + "' with a role '" + role + '\''; + return metadataRepo.push(projectName, Project.REPO_DOGMA, author, commitSummary, change); } /** @@ -603,6 +611,7 @@ public CompletableFuture addUserRepositoryRole(Author author, String p requireNonNull(role, "role"); return getProject(projectName).thenCompose(project -> { + project.repo(repoName); // Raises an exception if the repository does not exist. ensureProjectMember(project, member); final String commitSummary = "Add repository role of '" + member.id() + "' as '" + role + "' to '" + projectName + '/' + repoName + '\n'; @@ -716,6 +725,7 @@ public CompletableFuture addTokenRepositoryRole(Author author, String requireNonNull(role, "role"); return getProject(projectName).thenCompose(project -> { + project.repo(repoName); // Raises an exception if the repository does not exist. ensureProjectToken(project, appId); final String commitSummary = "Add repository role of the token '" + appId + "' as '" + role + "' to '" + projectName + '/' + repoName + "'\n"; @@ -948,13 +958,20 @@ public CompletableFuture findProjectRole(String projectName, User u } /** - * Returns a {@link Tokens}. + * Fetches the {@link Tokens} from the repository. */ - public CompletableFuture getTokens() { + public CompletableFuture fetchTokens() { return tokenRepo.fetch(INTERNAL_PROJECT_DOGMA, Project.REPO_DOGMA, TOKEN_JSON) .thenApply(HolderWithRevision::object); } + /** + * Returns a {@link Tokens}. + */ + public Tokens getTokens() { + return projectInitializer.tokens(); + } + /** * Creates a new user-level {@link Token} with the specified {@code appId}. A secret for the {@code appId} * will be automatically generated. @@ -1044,7 +1061,8 @@ public Revision purgeToken(Author author, String appId) { User.SYSTEM_ADMIN).values(); // Remove the token from projects that only have the token. for (Project project : projects) { - final ProjectMetadata projectMetadata = fetchMetadata(project.name()).join().object(); + // Fetch the metadata to get the latest information. + final ProjectMetadata projectMetadata = fetchMetadata(project.name()).join(); final boolean containsTargetTokenInTheProject = projectMetadata.tokens().values() .stream() @@ -1145,20 +1163,18 @@ public CompletableFuture updateTokenLevel(Author author, String appId, /** * Returns a {@link Token} which has the specified {@code appId}. */ - public CompletableFuture findTokenByAppId(String appId) { + public Token findTokenByAppId(String appId) { requireNonNull(appId, "appId"); - return tokenRepo.fetch(INTERNAL_PROJECT_DOGMA, Project.REPO_DOGMA, TOKEN_JSON) - .thenApply(tokens -> tokens.object().get(appId)); + return getTokens().get(appId); } /** * Returns a {@link Token} which has the specified {@code secret}. */ - public CompletableFuture findTokenBySecret(String secret) { + public Token findTokenBySecret(String secret) { requireNonNull(secret, "secret"); validateSecret(secret); - return tokenRepo.fetch(INTERNAL_PROJECT_DOGMA, Project.REPO_DOGMA, TOKEN_JSON) - .thenApply(tokens -> tokens.object().findBySecret(secret)); + return getTokens().findBySecret(secret); } /** diff --git a/server/src/main/java/com/linecorp/centraldogma/server/metadata/ProjectMetadata.java b/server/src/main/java/com/linecorp/centraldogma/server/metadata/ProjectMetadata.java index 2a1fec8f60..e954b41f51 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/metadata/ProjectMetadata.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/metadata/ProjectMetadata.java @@ -31,13 +31,14 @@ import com.linecorp.centraldogma.common.RepositoryNotFoundException; import com.linecorp.centraldogma.server.storage.project.Project; +import com.linecorp.centraldogma.server.storage.repository.HasWeight; /** * Specifies details of a {@link Project}. */ @JsonIgnoreProperties(ignoreUnknown = true) @JsonInclude(Include.NON_NULL) -public class ProjectMetadata implements Identifiable { +public class ProjectMetadata implements Identifiable, HasWeight { /** * A project name. @@ -178,6 +179,22 @@ public Member memberOrDefault(String memberId, @Nullable Member defaultMember) { return defaultMember; } + @Override + public int weight() { + int weight = name().length(); + for (RepositoryMetadata repo : repos.values()) { + weight += repo.weight(); + } + for (Member member : members.values()) { + weight += member.weight(); + } + for (TokenRegistration token : tokens.values()) { + weight += token.weight(); + } + + return weight; + } + @Override public String toString() { return MoreObjects.toStringHelper(this) diff --git a/server/src/main/java/com/linecorp/centraldogma/server/metadata/RepositoryMetadata.java b/server/src/main/java/com/linecorp/centraldogma/server/metadata/RepositoryMetadata.java index ae2d5d06cb..2addbf05de 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/metadata/RepositoryMetadata.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/metadata/RepositoryMetadata.java @@ -32,6 +32,7 @@ import com.linecorp.centraldogma.common.RepositoryRole; import com.linecorp.centraldogma.server.QuotaConfig; +import com.linecorp.centraldogma.server.storage.repository.HasWeight; import com.linecorp.centraldogma.server.storage.repository.Repository; /** @@ -40,7 +41,7 @@ @JsonIgnoreProperties(ignoreUnknown = true) @JsonInclude(Include.NON_NULL) // These are used when serializing. @JsonDeserialize(using = RepositoryMetadataDeserializer.class) -public final class RepositoryMetadata implements Identifiable { +public final class RepositoryMetadata implements Identifiable, HasWeight { public static final ProjectRoles DEFAULT_PROJECT_ROLES = ProjectRoles.of(RepositoryRole.WRITE, null); @@ -159,6 +160,14 @@ public QuotaConfig writeQuota() { return writeQuota; } + @Override + public int weight() { + int weight = 0; + weight += name.length(); + weight += roles.weight(); + return weight; + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/server/src/main/java/com/linecorp/centraldogma/server/metadata/RepositorySupport.java b/server/src/main/java/com/linecorp/centraldogma/server/metadata/RepositorySupport.java index 0b565b057a..3052173bf6 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/metadata/RepositorySupport.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/metadata/RepositorySupport.java @@ -16,12 +16,14 @@ package com.linecorp.centraldogma.server.metadata; +import static com.linecorp.centraldogma.server.internal.storage.repository.RepositoryCache.logger; import static java.util.Objects.requireNonNull; +import java.util.Objects; import java.util.concurrent.CompletableFuture; -import java.util.function.Function; import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.base.MoreObjects.ToStringHelper; import com.google.common.collect.ImmutableList; import com.linecorp.armeria.common.util.Exceptions; @@ -37,19 +39,21 @@ import com.linecorp.centraldogma.server.command.CommitResult; import com.linecorp.centraldogma.server.command.ContentTransformer; import com.linecorp.centraldogma.server.storage.project.ProjectManager; +import com.linecorp.centraldogma.server.storage.repository.AbstractCacheableCall; +import com.linecorp.centraldogma.server.storage.repository.HasWeight; import com.linecorp.centraldogma.server.storage.repository.Repository; final class RepositorySupport { private final ProjectManager projectManager; private final CommandExecutor executor; - private final Function, T> entryConverter; + private final Class entryClass; RepositorySupport(ProjectManager projectManager, CommandExecutor executor, - Function, T> entryConverter) { + Class entryClass) { this.projectManager = requireNonNull(projectManager, "projectManager"); this.executor = requireNonNull(executor, "executor"); - this.entryConverter = requireNonNull(entryConverter, "entryConverter"); + this.entryClass = requireNonNull(entryClass, "entryClass"); } public ProjectManager projectManager() { @@ -62,14 +66,6 @@ CompletableFuture> fetch(String projectName, String repoNa return fetch(projectManager().get(projectName).repos().get(repoName), path); } - CompletableFuture> fetch(String projectName, String repoName, String path, - Revision revision) { - requireNonNull(projectName, "projectName"); - requireNonNull(repoName, "repoName"); - requireNonNull(revision, "revision"); - return fetch(projectManager().get(projectName).repos().get(repoName), path, revision); - } - private CompletableFuture> fetch(Repository repository, String path) { requireNonNull(path, "path"); final Revision revision = normalize(repository); @@ -81,9 +77,9 @@ private CompletableFuture> fetch(Repository repository, St requireNonNull(repository, "repository"); requireNonNull(path, "path"); requireNonNull(revision, "revision"); - return repository.get(revision, path) - .thenApply(entryConverter) - .thenApply((T obj) -> HolderWithRevision.of(obj, revision)); + final CacheableFetchCall cacheableFetchCall = new CacheableFetchCall<>(repository, revision, path, + entryClass); + return repository.execute(cacheableFetchCall); } CompletableFuture push(String projectName, String repoName, @@ -134,7 +130,7 @@ CompletableFuture push(String projectName, String repoName, }); } - Revision normalize(Repository repository) { + static Revision normalize(Repository repository) { requireNonNull(repository, "repository"); try { return repository.normalizeNow(Revision.HEAD); @@ -143,14 +139,74 @@ Revision normalize(Repository repository) { } } - @SuppressWarnings("unchecked") - static T convertWithJackson(Entry entry, Class clazz) { - requireNonNull(entry, "entry"); - requireNonNull(clazz, "clazz"); - try { - return Jackson.treeToValue(((Entry) entry).content(), clazz); - } catch (Throwable cause) { - return Exceptions.throwUnsafely(cause); + // TODO(minwoox): Consider generalizing this class. + private static class CacheableFetchCall extends AbstractCacheableCall> { + + private final Revision revision; + private final String path; + private final Class entryClass; + private final int hashCode; + + CacheableFetchCall(Repository repo, Revision revision, String path, Class entryClass) { + super(repo); + this.revision = revision; + this.path = path; + this.entryClass = entryClass; + + hashCode = Objects.hash(revision, path, entryClass) * 31 + System.identityHashCode(repo); + assert !revision.isRelative(); + } + + @Override + public int weigh(HolderWithRevision value) { + int weight = path.length(); + final U object = value.object(); + if (object instanceof HasWeight) { + weight += ((HasWeight) object).weight(); + } + return weight; + } + + @Override + public CompletableFuture> execute() { + logger.debug("Cache miss: {}", this); + return repo().get(revision, path) + .thenApply(this::convertWithJackson) + .thenApply((U obj) -> HolderWithRevision.of(obj, revision)); + } + + @SuppressWarnings("unchecked") + U convertWithJackson(Entry entry) { + requireNonNull(entry, "entry"); + try { + return Jackson.treeToValue(((Entry) entry).content(), entryClass); + } catch (Throwable cause) { + return Exceptions.throwUnsafely(cause); + } + } + + @Override + public int hashCode() { + return hashCode; + } + + @Override + public boolean equals(Object o) { + if (!super.equals(o)) { + return false; + } + + final CacheableFetchCall that = (CacheableFetchCall) o; + return revision.equals(that.revision) && + path.equals(that.path) && + entryClass == that.entryClass; + } + + @Override + protected void toString(ToStringHelper helper) { + helper.add("revision", revision) + .add("path", path) + .add("entryClass", entryClass); } } } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/metadata/Roles.java b/server/src/main/java/com/linecorp/centraldogma/server/metadata/Roles.java index 1e57c3932b..fd77e23683 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/metadata/Roles.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/metadata/Roles.java @@ -18,6 +18,7 @@ import static java.util.Objects.requireNonNull; import java.util.Map; +import java.util.Map.Entry; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; @@ -25,11 +26,12 @@ import com.google.common.base.Objects; import com.linecorp.centraldogma.common.RepositoryRole; +import com.linecorp.centraldogma.server.storage.repository.HasWeight; /** * Role metadata for a repository. */ -public final class Roles { +public final class Roles implements HasWeight { private final ProjectRoles projectRoles; @@ -73,6 +75,24 @@ public Map tokens() { return tokens; } + @Override + public int weight() { + int weight = 0; + final RepositoryRole member = projectRoles.member(); + if (member != null) { + weight += member.name().length(); + } + for (Entry entry : users.entrySet()) { + weight += entry.getKey().length(); + weight += entry.getValue().name().length(); + } + for (Entry entry : tokens.entrySet()) { + weight += entry.getKey().length(); + weight += entry.getValue().name().length(); + } + return weight; + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/server/src/main/java/com/linecorp/centraldogma/server/metadata/TokenRegistration.java b/server/src/main/java/com/linecorp/centraldogma/server/metadata/TokenRegistration.java index ebc95bc236..691b650181 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/metadata/TokenRegistration.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/metadata/TokenRegistration.java @@ -25,12 +25,13 @@ import com.google.common.base.MoreObjects; import com.linecorp.centraldogma.common.ProjectRole; +import com.linecorp.centraldogma.server.storage.repository.HasWeight; /** * Specifies a registration of a {@link Token}. */ @JsonInclude(Include.NON_NULL) -public class TokenRegistration implements Identifiable { +public class TokenRegistration implements Identifiable, HasWeight { /** * An application identifier which belongs to a {@link Token}. @@ -88,6 +89,11 @@ public UserAndTimestamp creation() { return creation; } + @Override + public int weight() { + return appId.length() + role.name().length(); + } + @Override public String toString() { return MoreObjects.toStringHelper(this) diff --git a/server/src/main/java/com/linecorp/centraldogma/server/metadata/Tokens.java b/server/src/main/java/com/linecorp/centraldogma/server/metadata/Tokens.java index f5d2f4527f..63ab2cb869 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/metadata/Tokens.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/metadata/Tokens.java @@ -20,6 +20,7 @@ import static java.util.Objects.requireNonNull; import java.util.Map; +import java.util.Map.Entry; import java.util.function.Function; import java.util.stream.Collectors; @@ -33,12 +34,14 @@ import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableMap; +import com.linecorp.centraldogma.server.storage.repository.HasWeight; + /** * Holds a token map and a secret map for fast lookup. */ @JsonIgnoreProperties(ignoreUnknown = true) @JsonInclude(Include.NON_NULL) -public final class Tokens { +public final class Tokens implements HasWeight { static final String SECRET_PREFIX = "appToken-"; @@ -149,6 +152,17 @@ public Tokens withoutSecret() { return new Tokens(appIds, ImmutableMap.of()); } + @Override + public int weight() { + int weight = 0; + weight += secrets.size(); + for (Entry entry : secrets.entrySet()) { + weight += entry.getKey().length(); + weight += entry.getValue().length(); + } + return weight; + } + @Override public String toString() { return MoreObjects.toStringHelper(this) diff --git a/server/src/main/java/com/linecorp/centraldogma/server/storage/project/InternalProjectInitializer.java b/server/src/main/java/com/linecorp/centraldogma/server/storage/project/InternalProjectInitializer.java index 84a9866efc..5d1c4ec8b7 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/storage/project/InternalProjectInitializer.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/storage/project/InternalProjectInitializer.java @@ -17,14 +17,23 @@ package com.linecorp.centraldogma.server.storage.project; import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; import static com.linecorp.centraldogma.server.command.Command.createProject; import static com.linecorp.centraldogma.server.command.Command.createRepository; import static com.linecorp.centraldogma.server.command.Command.push; +import static com.linecorp.centraldogma.server.metadata.MetadataService.TOKEN_JSON; import static java.util.Objects.requireNonNull; import java.util.List; import java.util.concurrent.CompletableFuture; +import javax.annotation.Nullable; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.core.JsonParseException; +import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableList; @@ -42,20 +51,28 @@ import com.linecorp.centraldogma.internal.Jackson; import com.linecorp.centraldogma.server.command.Command; import com.linecorp.centraldogma.server.command.CommandExecutor; -import com.linecorp.centraldogma.server.metadata.MetadataService; import com.linecorp.centraldogma.server.metadata.Tokens; +import com.linecorp.centraldogma.server.storage.repository.Repository; +import com.linecorp.centraldogma.server.storage.repository.RepositoryListener; /** * Initializes the internal project and repositories. */ public final class InternalProjectInitializer { + private static final Logger logger = LoggerFactory.getLogger(InternalProjectInitializer.class); + public static final String INTERNAL_PROJECT_DOGMA = "dogma"; private final CommandExecutor executor; private final ProjectManager projectManager; private final CompletableFuture initialFuture = new CompletableFuture<>(); + @Nullable + private volatile Revision lastTokensRevision; + @Nullable + private volatile Tokens tokens; + /** * Creates a new instance. */ @@ -111,22 +128,24 @@ private void initialize0(String projectName) { } private void initializeTokens() { - final Entry entry = - projectManager.get(INTERNAL_PROJECT_DOGMA).repos().get(Project.REPO_DOGMA) - .getOrNull(Revision.HEAD, - Query.ofJson(MetadataService.TOKEN_JSON)).join(); + final Repository dogmaRepo = projectManager.get(INTERNAL_PROJECT_DOGMA).repos().get(Project.REPO_DOGMA); + final Entry entry = dogmaRepo.getOrNull(Revision.HEAD, Query.ofJson(TOKEN_JSON)).join(); if (entry != null && entry.hasContent()) { + setTokens(entry, dogmaRepo); return; } try { - final Change change = Change.ofJsonPatch(MetadataService.TOKEN_JSON, + final Change change = Change.ofJsonPatch(TOKEN_JSON, null, Jackson.valueToTree(new Tokens())); final String commitSummary = "Initialize the token list file: /" + INTERNAL_PROJECT_DOGMA + '/' + - Project.REPO_DOGMA + MetadataService.TOKEN_JSON; + Project.REPO_DOGMA + TOKEN_JSON; executor.execute(Command.forcePush(push(Author.SYSTEM, INTERNAL_PROJECT_DOGMA, Project.REPO_DOGMA, Revision.HEAD, commitSummary, "", Markup.PLAINTEXT, ImmutableList.of(change)))) .get(); + final Entry entry1 = dogmaRepo.getOrNull(Revision.HEAD, Query.ofJson(TOKEN_JSON)).join(); + assert entry1 != null; + setTokens(entry1, dogmaRepo); } catch (Throwable cause) { final Throwable peeled = Exceptions.peel(cause); if (peeled instanceof ChangeConflictException) { @@ -136,6 +155,44 @@ private void initializeTokens() { } } + private void setTokens(Entry entry, Repository dogmaRepo) { + try { + final Tokens tokens = Jackson.treeToValue(entry.content(), Tokens.class); + lastTokensRevision = entry.revision(); + this.tokens = tokens; + attachTokensListener(dogmaRepo); + } catch (JsonParseException | JsonMappingException e) { + throw new RuntimeException(String.format("failed to parse %s/%s/%s", INTERNAL_PROJECT_DOGMA, + Project.REPO_DOGMA, TOKEN_JSON), e); + } + } + + private void attachTokensListener(Repository dogmaRepo) { + dogmaRepo.addListener(RepositoryListener.of(Query.ofJson(TOKEN_JSON), entry -> { + if (entry == null) { + logger.warn("{} file is missing in {}/{}", TOKEN_JSON, INTERNAL_PROJECT_DOGMA, + Project.REPO_DOGMA); + return; + } + + final Revision lastRevision = entry.revision(); + final Revision lastTokensRevision = this.lastTokensRevision; + if (lastTokensRevision != null && lastRevision.compareTo(lastTokensRevision) <= 0) { + // An old data. + return; + } + + try { + final Tokens tokens = Jackson.treeToValue(entry.content(), Tokens.class); + this.lastTokensRevision = lastRevision; + this.tokens = tokens; + } catch (JsonParseException | JsonMappingException e) { + logger.warn("Invalid {} file in {}/{}", TOKEN_JSON, INTERNAL_PROJECT_DOGMA, + Project.REPO_DOGMA, e); + } + })); + } + /** * Returns a {@link CompletableFuture} which is completed when the internal project and repositories are * ready. @@ -144,6 +201,15 @@ public CompletableFuture whenInitialized() { return initialFuture; } + /** + * Returns the {@link Tokens}. + */ + public Tokens tokens() { + final Tokens tokens = this.tokens; + checkState(tokens != null, "tokens have not been loaded yet"); + return tokens; + } + /** * Creates the specified internal repositories in the internal project. */ diff --git a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/CacheableCall.java b/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/AbstractCacheableCall.java similarity index 61% rename from server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/CacheableCall.java rename to server/src/main/java/com/linecorp/centraldogma/server/storage/repository/AbstractCacheableCall.java index 2e7c9aa1a2..3f3e7e78b8 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/internal/storage/repository/CacheableCall.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/AbstractCacheableCall.java @@ -14,48 +14,33 @@ * under the License. */ -package com.linecorp.centraldogma.server.internal.storage.repository; +package com.linecorp.centraldogma.server.storage.repository; import static java.util.Objects.requireNonNull; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; - import com.google.common.base.MoreObjects; -import com.linecorp.centraldogma.server.storage.repository.Repository; - -// XXX(trustin): Consider using reflection or AOP so that it takes less effort to add more call types. -public abstract class CacheableCall { - - private static final Lock[] locks; - - static { - locks = new Lock[8192]; - for (int i = 0; i < locks.length; i++) { - locks[i] = new ReentrantLock(); - } - } +/** + * A skeletal implementation of {@link CacheableCall}. + */ +public abstract class AbstractCacheableCall implements CacheableCall { - final Repository repo; + private final Repository repo; - protected CacheableCall(Repository repo) { + /** + * Creates a new instance. + */ + protected AbstractCacheableCall(Repository repo) { this.repo = requireNonNull(repo, "repo"); } - public final Repository repo() { + /** + * Returns the {@link Repository} which this call is associated with. + */ + protected final Repository repo() { return repo; } - public final Lock coarseGrainedLock() { - return locks[Math.abs(hashCode() % locks.length)]; - } - - protected abstract int weigh(T value); - - public abstract CompletableFuture execute(); - @Override public int hashCode() { return System.identityHashCode(repo); @@ -75,7 +60,7 @@ public boolean equals(Object obj) { return false; } - final CacheableCall that = (CacheableCall) obj; + final AbstractCacheableCall that = (AbstractCacheableCall) obj; return repo == that.repo; } @@ -88,5 +73,8 @@ public final String toString() { return helper.toString(); } + /** + * Overrides this method to add more information to the {@link #toString()} result. + */ protected abstract void toString(MoreObjects.ToStringHelper helper); } diff --git a/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/CacheableCall.java b/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/CacheableCall.java new file mode 100644 index 0000000000..3f58c1fd7a --- /dev/null +++ b/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/CacheableCall.java @@ -0,0 +1,34 @@ +/* + * Copyright 2025 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package com.linecorp.centraldogma.server.storage.repository; + +import java.util.concurrent.CompletableFuture; + +/** + * A cacheable call which is used to retrieve a value. + */ +public interface CacheableCall { + + /** + * Returns the weight of the specified value. The weight is used for size-based eviction. + */ + int weigh(T value); + + /** + * Executes this call. + */ + CompletableFuture execute(); +} diff --git a/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/HasWeight.java b/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/HasWeight.java new file mode 100644 index 0000000000..69fd409167 --- /dev/null +++ b/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/HasWeight.java @@ -0,0 +1,28 @@ +/* + * Copyright 2025 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package com.linecorp.centraldogma.server.storage.repository; + +/** + * An object which has a weight. + */ +@FunctionalInterface +public interface HasWeight { + + /** + * Returns the weight of this object. + */ + int weight(); +} diff --git a/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/Repository.java b/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/Repository.java index b374ca42dd..8a5435e44e 100644 --- a/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/Repository.java +++ b/server/src/main/java/com/linecorp/centraldogma/server/storage/repository/Repository.java @@ -563,6 +563,11 @@ default CompletableFuture> mergeFiles(Revision revision, Merg return future; } + /** + * Executes the specified {@link CacheableCall} in this {@link Repository}. + */ + CompletableFuture execute(CacheableCall cacheableCall); + /** * Adds the {@link RepositoryListener} that gets notified whenever changes matching with * {@link RepositoryListener#pathPattern()} are pushed to this {@link Repository}. diff --git a/server/src/test/java/com/linecorp/centraldogma/server/command/StandaloneCommandExecutorTest.java b/server/src/test/java/com/linecorp/centraldogma/server/command/StandaloneCommandExecutorTest.java index 52fe27dfc5..f600976dd8 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/command/StandaloneCommandExecutorTest.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/command/StandaloneCommandExecutorTest.java @@ -63,7 +63,8 @@ static void setUp() { executor.execute(Command.createRepository(Author.SYSTEM, TEST_PRJ, TEST_REPO2)).join(); executor.execute(Command.createRepository(Author.SYSTEM, TEST_PRJ, TEST_REPO3)).join(); - final MetadataService mds = new MetadataService(extension.projectManager(), executor); + final MetadataService mds = new MetadataService(extension.projectManager(), executor, + extension.internalProjectInitializer()); // Metadata should be created before entering read-only mode. mds.addRepo(Author.SYSTEM, TEST_PRJ, TEST_REPO).join(); mds.addRepo(Author.SYSTEM, TEST_PRJ, TEST_REPO2).join(); @@ -73,7 +74,8 @@ static void setUp() { @Test void setWriteQuota() { final StandaloneCommandExecutor executor = (StandaloneCommandExecutor) extension.executor(); - final MetadataService mds = new MetadataService(extension.projectManager(), executor); + final MetadataService mds = new MetadataService(extension.projectManager(), executor, + extension.internalProjectInitializer()); final RateLimiter rateLimiter1 = executor.writeRateLimiters.get("test_prj/test_repo"); assertThat(rateLimiter1).isNull(); @@ -160,7 +162,8 @@ void createInternalProject() { final CommandExecutor executor = extension.executor(); final String internalProjectName = "@project"; executor.execute(Command.createProject(Author.SYSTEM, internalProjectName)).join(); - final MetadataService mds = new MetadataService(extension.projectManager(), executor); + final MetadataService mds = new MetadataService(extension.projectManager(), executor, + extension.internalProjectInitializer()); mds.addRepo(Author.SYSTEM, internalProjectName, TEST_REPO).join(); // Can create an internal project that starts with an underscore. executor.execute(Command.createRepository(Author.SYSTEM, internalProjectName, TEST_REPO)) diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/api/ProjectServiceV1ListProjectTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/api/ProjectServiceV1ListProjectTest.java index e3b93c20ce..ce4a34412e 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/internal/api/ProjectServiceV1ListProjectTest.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/api/ProjectServiceV1ListProjectTest.java @@ -341,14 +341,12 @@ void userRoleWithToken() { }); } - private Map getProjects(BlockingWebClient client) { - final ResponseEntity> response = - client.prepare() - .get(PROJECTS_PREFIX) - .asJson(new TypeReference>() {}) - .execute(); - assertThat(response.headers().status()).isEqualTo(HttpStatus.OK); - return response.content().stream() - .collect(toImmutableMap(ProjectDto::name, Function.identity())); + private static Map getProjects(BlockingWebClient client) { + await().until(() -> client.get(PROJECTS_PREFIX).status() == HttpStatus.OK); + return client.prepare() + .get(PROJECTS_PREFIX) + .asJson(new TypeReference>() {}) + .execute().content().stream() + .collect(toImmutableMap(ProjectDto::name, Function.identity())); } } diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/api/TokenServiceTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/api/TokenServiceTest.java index 12a7f0489b..1b04916ebe 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/internal/api/TokenServiceTest.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/api/TokenServiceTest.java @@ -18,6 +18,7 @@ import static com.linecorp.centraldogma.internal.api.v1.HttpApiV1Constants.API_V1_PATH_PREFIX; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.awaitility.Awaitility.await; import java.net.URI; import java.util.Collection; @@ -119,13 +120,14 @@ static void setUp() throws JsonMappingException, JsonParseException { TestAuthMessageUtil.USERNAME, TestAuthMessageUtil.PASSWORD))) .build(); - metadataService = new MetadataService(manager.projectManager(), manager.executor()); + metadataService = new MetadataService(manager.projectManager(), manager.executor(), + manager.internalProjectInitializer()); tokenService = new TokenService(manager.executor(), metadataService); } @AfterEach public void tearDown() { - final Tokens tokens = metadataService.getTokens().join(); + final Tokens tokens = metadataService.fetchTokens().join(); tokens.appIds().forEach((appId, token) -> { if (!token.isDeleted()) { metadataService.destroyToken(systemAdminAuthor, appId); @@ -147,25 +149,25 @@ void systemAdminToken() { final StandaloneCommandExecutor executor = (StandaloneCommandExecutor) manager.executor(); executor.execute(Command.createProject(Author.SYSTEM, "myPro")).join(); metadataService.addToken(Author.SYSTEM, "myPro", "forAdmin1", ProjectRole.OWNER).join(); - assertThat(metadataService.getProject("myPro").join().tokens().containsKey("forAdmin1")).isTrue(); + await().untilAsserted(() -> assertThat(metadataService.getProject("myPro").join().tokens() + .containsKey("forAdmin1")).isTrue()); - final Collection tokens = tokenService.listTokens(systemAdmin).join(); + final Collection tokens = tokenService.listTokens(systemAdmin); assertThat(tokens.stream().filter(t -> !StringUtil.isNullOrEmpty(t.secret()))).hasSize(1); assertThatThrownBy(() -> tokenService.deleteToken(ctx, "forAdmin1", guestAuthor, guest) .join()) .hasCauseInstanceOf(HttpResponseException.class); - assertThat(tokenService.deleteToken(ctx, "forAdmin1", systemAdminAuthor, systemAdmin) - .thenCompose(unused -> tokenService.purgeToken( - ctx, "forAdmin1", systemAdminAuthor, systemAdmin)).join()) + tokenService.deleteToken(ctx, "forAdmin1", systemAdminAuthor, systemAdmin).join(); + assertThat(tokenService.purgeToken(ctx, "forAdmin1", systemAdminAuthor, systemAdmin).join()) .satisfies(t -> { assertThat(t.appId()).isEqualTo(token.appId()); assertThat(t.isSystemAdmin()).isEqualTo(token.isSystemAdmin()); assertThat(t.creation()).isEqualTo(token.creation()); assertThat(t.isDeleted()).isTrue(); }); - assertThat(tokenService.listTokens(systemAdmin).join().size()).isEqualTo(0); + await().untilAsserted(() -> assertThat(tokenService.listTokens(systemAdmin).size()).isEqualTo(0)); assertThat(metadataService.getProject("myPro").join().tokens().size()).isEqualTo(0); } @@ -179,7 +181,7 @@ void userToken() { assertThat(userToken1.isActive()).isTrue(); assertThat(userToken2.isActive()).isTrue(); - final Collection tokens = tokenService.listTokens(guest).join(); + final Collection tokens = tokenService.listTokens(guest); assertThat(tokens.stream().filter(token -> !StringUtil.isNullOrEmpty(token.secret())).count()) .isEqualTo(0); @@ -205,13 +207,15 @@ void userToken() { @Test void nonRandomToken() { + final Collection tokens1 = tokenService.listTokens(systemAdmin); + assertThat(tokens1.stream().filter(t -> !StringUtil.isNullOrEmpty(t.secret()))).hasSize(0); final Token token = tokenService.createToken("forAdmin1", true, true, "appToken-secret", systemAdminAuthor, systemAdmin) .join().content(); assertThat(token.isActive()).isTrue(); - final Collection tokens = tokenService.listTokens(systemAdmin).join(); + final Collection tokens = tokenService.listTokens(systemAdmin); assertThat(tokens.stream().filter(t -> !StringUtil.isNullOrEmpty(t.secret()))).hasSize(1); assertThatThrownBy(() -> tokenService.createToken("forUser1", true, true, @@ -221,9 +225,8 @@ void nonRandomToken() { final ServiceRequestContext ctx = ServiceRequestContext.of( HttpRequest.of(HttpMethod.DELETE, "/tokens/{appId}/removed")); - - tokenService.deleteToken(this.ctx, "forAdmin1", systemAdminAuthor, systemAdmin).thenCompose( - unused -> tokenService.purgeToken(ctx, "forAdmin1", systemAdminAuthor, systemAdmin)).join(); + tokenService.deleteToken(ctx, "forAdmin1", systemAdminAuthor, systemAdmin).join(); + tokenService.purgeToken(ctx, "forAdmin1", systemAdminAuthor, systemAdmin).join(); } @Test @@ -234,12 +237,12 @@ public void updateToken() { assertThat(token.isActive()).isTrue(); tokenService.updateToken(ctx, "forUpdate", deactivation, systemAdminAuthor, systemAdmin).join(); - final Token deactivatedToken = metadataService.findTokenByAppId("forUpdate").join(); - assertThat(deactivatedToken.isActive()).isFalse(); + await().untilAsserted(() -> assertThat(metadataService.findTokenByAppId("forUpdate").isActive()) + .isFalse()); tokenService.updateToken(ctx, "forUpdate", activation, systemAdminAuthor, systemAdmin).join(); - final Token activatedToken = metadataService.findTokenByAppId("forUpdate").join(); - assertThat(activatedToken.isActive()).isTrue(); + await().untilAsserted(() -> assertThat(metadataService.findTokenByAppId("forUpdate").isActive()) + .isTrue()); assertThatThrownBy( () -> tokenService.updateToken(ctx, "forUpdate", Jackson.valueToTree( @@ -248,8 +251,8 @@ public void updateToken() { .hasCauseInstanceOf(IllegalArgumentException.class); tokenService.deleteToken(ctx, "forUpdate", systemAdminAuthor, systemAdmin).join(); - final Token deletedToken = metadataService.findTokenByAppId("forUpdate").join(); - assertThat(deletedToken.isDeleted()).isTrue(); + await().untilAsserted(() -> assertThat(metadataService.findTokenByAppId("forUpdate").isDeleted()) + .isTrue()); assertThatThrownBy( () -> tokenService.updateToken(ctx, "forUpdate", activation, systemAdminAuthor, systemAdmin).join()) diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/api/auth/RequiresRoleTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/api/auth/RequiresRoleTest.java index 9580554ff6..383c383dc8 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/internal/api/auth/RequiresRoleTest.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/api/auth/RequiresRoleTest.java @@ -17,6 +17,7 @@ package com.linecorp.centraldogma.server.internal.api.auth; import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; import java.io.File; import java.util.concurrent.ForkJoinPool; @@ -90,11 +91,13 @@ protected void configure(ServerBuilder sb) throws Exception { final CommandExecutor executor = new StandaloneCommandExecutor( pm, ForkJoinPool.commonPool(), statusManager, null, null, null, null, null); executor.start().join(); - new InternalProjectInitializer(executor, pm).initialize(); + final InternalProjectInitializer projectInitializer = new InternalProjectInitializer( + executor, pm); + projectInitializer.initialize(); executor.execute(Command.createProject(AUTHOR, "project1")).join(); - final MetadataService mds = new MetadataService(pm, executor); + final MetadataService mds = new MetadataService(pm, executor, projectInitializer); mds.createToken(AUTHOR, APP_ID_1, SECRET_1).toCompletableFuture().join(); mds.createToken(AUTHOR, APP_ID_2, SECRET_2).toCompletableFuture().join(); @@ -106,12 +109,14 @@ protected void configure(ServerBuilder sb) throws Exception { // app-1 is an owner and it has read/write permission. mds.addToken(AUTHOR, "project1", APP_ID_1, ProjectRole.OWNER) .toCompletableFuture().join(); + await().until(() -> mds.findTokenByAppId(APP_ID_1) != null); mds.addTokenRepositoryRole(AUTHOR, "project1", "repo1", APP_ID_1, RepositoryRole.WRITE) .toCompletableFuture().join(); // app-2 is a member and it has read-only permission. mds.addToken(AUTHOR, "project1", APP_ID_2, ProjectRole.MEMBER) .toCompletableFuture().join(); + await().until(() -> mds.findTokenByAppId(APP_ID_2) != null); sb.dependencyInjector( DependencyInjector.ofSingletons(new RequiresRepositoryRoleDecoratorFactory(mds), new RequiresProjectRoleDecoratorFactory(mds)), diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/command/CommandExecutorStatusManagerTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/command/CommandExecutorStatusManagerTest.java index ab8a4bfd0f..42902f9987 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/internal/command/CommandExecutorStatusManagerTest.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/command/CommandExecutorStatusManagerTest.java @@ -19,6 +19,7 @@ import static org.assertj.core.api.Assertions.assertThat; import java.util.concurrent.CompletableFuture; +import java.util.function.BiFunction; import javax.annotation.Nullable; @@ -31,6 +32,7 @@ import com.linecorp.centraldogma.server.command.CommandExecutorStatusManager; import com.linecorp.centraldogma.server.command.UpdateServerStatusCommand; import com.linecorp.centraldogma.server.management.ServerStatus; +import com.linecorp.centraldogma.server.metadata.RepositoryMetadata; class CommandExecutorStatusManagerTest { @@ -114,6 +116,10 @@ public void setWritable(boolean writable) { public void setWriteQuota(String projectName, String repoName, @Nullable QuotaConfig writeQuota) { } + @Override + public void setRepositoryMetadataSupplier( + BiFunction> supplier) {} + @Override public CompletableFuture execute(Command command) { return CompletableFuture.completedFuture(null); diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/replication/Replica.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/replication/Replica.java index e5c2db9aa5..bf1316edd2 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/internal/replication/Replica.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/replication/Replica.java @@ -29,6 +29,7 @@ import java.util.Map; import java.util.Objects; import java.util.concurrent.CompletableFuture; +import java.util.function.BiFunction; import java.util.function.Function; import javax.annotation.Nullable; @@ -42,10 +43,8 @@ import com.linecorp.centraldogma.server.ZooKeeperServerConfig; import com.linecorp.centraldogma.server.command.AbstractCommandExecutor; import com.linecorp.centraldogma.server.command.Command; -import com.linecorp.centraldogma.server.metadata.MetadataService; import com.linecorp.centraldogma.server.metadata.RepositoryMetadata; import com.linecorp.centraldogma.server.metadata.UserAndTimestamp; -import com.linecorp.centraldogma.server.storage.project.ProjectManager; import io.micrometer.core.instrument.MeterRegistry; @@ -93,8 +92,8 @@ protected void doStop(@Nullable Runnable onReleaseLeadership, protected CompletableFuture doExecute(Command command) { return (CompletableFuture) delegate.apply(command); } - }, meterRegistry, mock(ProjectManager.class), writeQuota, null, null, null, null, null); - commandExecutor.setMetadataService(mockMetaService()); + }, meterRegistry, writeQuota, null, null, null, null, null); + commandExecutor.setRepositoryMetadataSupplier(mockMetaService()); commandExecutor.setLockTimeoutMillis(10000); startFuture = start ? commandExecutor.start() : null; @@ -114,12 +113,13 @@ long localRevision() { }, Objects::nonNull); } - private static MetadataService mockMetaService() { - final MetadataService mds = mock(MetadataService.class); + private static BiFunction> mockMetaService() { + //noinspection unchecked + final BiFunction> mock = mock(BiFunction.class); final RepositoryMetadata repoMeta = RepositoryMetadata.of("", UserAndTimestamp.of(Author.SYSTEM)); - lenient().when(mds.getRepo(anyString(), anyString())) + lenient().when(mock.apply(anyString(), anyString())) .thenReturn(CompletableFuture.completedFuture(repoMeta)); - return mds; + return mock; } boolean existsLocalRevision() { diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/PurgeSchedulingServiceTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/PurgeSchedulingServiceTest.java index 76618ebb9a..cb0d148b22 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/PurgeSchedulingServiceTest.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/PurgeSchedulingServiceTest.java @@ -62,7 +62,8 @@ protected CommandExecutor newCommandExecutor(ProjectManager projectManager, Exec @Override protected void afterExecutorStarted() { - metadataService = new MetadataService(projectManager(), executor()); + metadataService = new MetadataService(projectManager(), executor(), + manager.internalProjectInitializer()); executor().execute(Command.createProject(AUTHOR, PROJA_ACTIVE)).join(); executor().execute(Command.createRepository(AUTHOR, PROJA_ACTIVE, REPOA_REMOVED)).join(); diff --git a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CachingRepositoryTest.java b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CachingRepositoryTest.java index 8fa426ec85..45f4951ffd 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CachingRepositoryTest.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/internal/storage/repository/cache/CachingRepositoryTest.java @@ -45,9 +45,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.linecorp.armeria.common.metric.MoreMeters; import com.linecorp.armeria.common.metric.NoopMeterRegistry; -import com.linecorp.armeria.common.prometheus.PrometheusMeterRegistries; import com.linecorp.centraldogma.common.Change; import com.linecorp.centraldogma.common.Commit; import com.linecorp.centraldogma.common.Entry; @@ -64,8 +62,6 @@ import com.linecorp.centraldogma.server.storage.repository.DiffResultType; import com.linecorp.centraldogma.server.storage.repository.Repository; -import io.micrometer.core.instrument.MeterRegistry; - class CachingRepositoryTest { @Mock @@ -73,7 +69,7 @@ class CachingRepositoryTest { @Test void identityQuery() { - final Repository repo = setMockNames(newCachingRepo()); + final CachingRepository repo = setMockNames(newCachingRepo()); final Query query = Query.ofText("/baz.txt"); final Entry result = Entry.ofText(new Revision(10), "/baz.txt", "qux"); @@ -99,7 +95,7 @@ void identityQuery() { @Test @SuppressWarnings("unchecked") void jsonPathQuery() throws JsonParseException { - final Repository repo = setMockNames(newCachingRepo()); + final CachingRepository repo = setMockNames(newCachingRepo()); final Query query = Query.ofJsonPath("/baz.json", "$.a"); final Entry queryResult = Entry.ofJson(new Revision(10), query.path(), "{\"a\": \"b\"}"); @@ -122,7 +118,7 @@ void jsonPathQuery() throws JsonParseException { @Test void mergeQuery() throws JsonParseException { - final Repository repo = setMockNames(newCachingRepo()); + final CachingRepository repo = setMockNames(newCachingRepo()); final MergeQuery query = MergeQuery.ofJson(MergeSource.ofRequired("/foo.json"), MergeSource.ofRequired("/bar.json")); final MergedEntry queryResult = MergedEntry.of(new Revision(10), JSON, @@ -133,15 +129,9 @@ void mergeQuery() throws JsonParseException { doReturn(new Revision(10)).when(delegateRepo).normalizeNow(HEAD); // Uncached - when(delegateRepo.find(any(), any(), any())) - .thenReturn(completedFuture(ImmutableMap.of("/foo.json", Entry.ofJson( - new Revision(10), "/foo.json", "{\"a\": \"foo\"}")))) - .thenReturn(completedFuture(ImmutableMap.of("/bar.json", Entry.ofJson( - new Revision(10), "/bar.json", "{\"a\": \"bar\"}")))); - + when(delegateRepo.mergeFiles(new Revision(10), query)).thenReturn(completedFuture(queryResult)); assertThat(repo.mergeFiles(HEAD, query).join()).isEqualTo(queryResult); - verify(delegateRepo).find(new Revision(10), "/foo.json", FIND_ONE_WITH_CONTENT); - verify(delegateRepo).find(new Revision(10), "/bar.json", FIND_ONE_WITH_CONTENT); + verify(delegateRepo).mergeFiles(new Revision(10), query); verifyNoMoreInteractions(delegateRepo); // Cached @@ -154,7 +144,7 @@ void mergeQuery() throws JsonParseException { @Test void identityQueryMissingEntry() { - final Repository repo = setMockNames(newCachingRepo()); + final CachingRepository repo = setMockNames(newCachingRepo()); final Query query = Query.ofText("/baz.txt"); doReturn(new Revision(10)).when(delegateRepo).normalizeNow(new Revision(10)); @@ -177,7 +167,7 @@ void identityQueryMissingEntry() { @Test @SuppressWarnings("unchecked") void jsonPathQueryMissingEntry() { - final Repository repo = setMockNames(newCachingRepo()); + final CachingRepository repo = setMockNames(newCachingRepo()); final Query query = Query.ofJsonPath("/baz.json", "$.a"); doReturn(new Revision(10)).when(delegateRepo).normalizeNow(new Revision(10)); @@ -199,7 +189,7 @@ void jsonPathQueryMissingEntry() { @Test void find() { - final Repository repo = setMockNames(newCachingRepo()); + final CachingRepository repo = setMockNames(newCachingRepo()); final Map> entries = ImmutableMap.of("/baz.txt", Entry.ofText(new Revision(10), "/baz.txt", "qux")); @@ -222,7 +212,7 @@ void find() { @Test void history() { - final Repository repo = setMockNames(newCachingRepo()); + final CachingRepository repo = setMockNames(newCachingRepo()); final List commits = ImmutableList.of( new Commit(new Revision(3), SYSTEM, "third", "", Markup.MARKDOWN), new Commit(new Revision(3), SYSTEM, "second", "", Markup.MARKDOWN), @@ -251,7 +241,7 @@ void history() { @Test void singleDiff() { - final Repository repo = setMockNames(newCachingRepo()); + final CachingRepository repo = setMockNames(newCachingRepo()); final Query query = Query.ofText("/foo.txt"); final Change change = Change.ofTextUpsert(query.path(), "bar"); @@ -278,7 +268,7 @@ void singleDiff() { @Test void multiDiff() { - final Repository repo = setMockNames(newCachingRepo()); + final CachingRepository repo = setMockNames(newCachingRepo()); final Map> changes = ImmutableMap.of( "/foo.txt", Change.ofTextUpsert("/foo.txt", "bar")); @@ -305,7 +295,7 @@ void multiDiff() { @Test void findLatestRevision() { - final Repository repo = setMockNames(newCachingRepo()); + final CachingRepository repo = setMockNames(newCachingRepo()); doReturn(new RevisionRange(INIT, new Revision(2))).when(delegateRepo).normalizeNow(INIT, HEAD); // Uncached @@ -324,7 +314,7 @@ void findLatestRevision() { @Test void findLatestRevisionNull() { - final Repository repo = setMockNames(newCachingRepo()); + final CachingRepository repo = setMockNames(newCachingRepo()); doReturn(new RevisionRange(INIT, new Revision(2))).when(delegateRepo).normalizeNow(INIT, HEAD); // Uncached @@ -354,7 +344,7 @@ void finaLatestRevisionHead() { @Test void watchFastPath() { - final Repository repo = setMockNames(newCachingRepo()); + final CachingRepository repo = setMockNames(newCachingRepo()); doReturn(new RevisionRange(INIT, new Revision(2))).when(delegateRepo).normalizeNow(INIT, HEAD); // Uncached @@ -375,7 +365,7 @@ void watchFastPath() { @Test void watchSlowPath() { - final Repository repo = setMockNames(newCachingRepo()); + final CachingRepository repo = setMockNames(newCachingRepo()); doReturn(new RevisionRange(INIT, new Revision(2))).when(delegateRepo).normalizeNow(INIT, HEAD); final CompletableFuture delegateWatchFuture = new CompletableFuture<>(); @@ -394,24 +384,9 @@ void watchSlowPath() { verifyNoMoreInteractions(delegateRepo); } - @Test - void metrics() { - final MeterRegistry meterRegistry = PrometheusMeterRegistries.newRegistry(); - final Repository repo = newCachingRepo(meterRegistry); - final Map meters = MoreMeters.measureAll(meterRegistry); - assertThat(meters).containsKeys("cache.load#count{cache=repository,result=success}"); - - // Do something with 'repo' so that it is not garbage-collected even before the meters are measured. - assertThat(repo.normalizeNow(HEAD)).isNotEqualTo(""); - } - - private Repository newCachingRepo() { - return newCachingRepo(NoopMeterRegistry.get()); - } - - private Repository newCachingRepo(MeterRegistry meterRegistry) { - final Repository cachingRepo = new CachingRepository( - delegateRepo, new RepositoryCache("maximumSize=1000", meterRegistry)); + private CachingRepository newCachingRepo() { + final CachingRepository cachingRepo = new CachingRepository( + delegateRepo, new RepositoryCache("maximumSize=1000", NoopMeterRegistry.get())); verifyNoMoreInteractions(delegateRepo); clearInvocations(delegateRepo); @@ -419,7 +394,7 @@ private Repository newCachingRepo(MeterRegistry meterRegistry) { return cachingRepo; } - private static Repository setMockNames(Repository mockRepo) { + private static CachingRepository setMockNames(CachingRepository mockRepo) { final Project project = mock(Project.class); when(mockRepo.parent()).thenReturn(project); when(project.name()).thenReturn("mock_proj"); diff --git a/server/src/test/java/com/linecorp/centraldogma/server/metadata/MetadataApiServiceTest.java b/server/src/test/java/com/linecorp/centraldogma/server/metadata/MetadataApiServiceTest.java index 701835da48..eabb4a4016 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/metadata/MetadataApiServiceTest.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/metadata/MetadataApiServiceTest.java @@ -19,6 +19,7 @@ import static com.linecorp.centraldogma.testing.internal.auth.TestAuthMessageUtil.getAccessToken; import static net.javacrumbs.jsonunit.fluent.JsonFluentAssert.assertThatJson; import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -266,8 +267,9 @@ void grantRoleToMemberForMetaRepository() throws Exception { assertThat(systemAdminClient.execute(request).status()).isSameAs(HttpStatus.OK); // Now the member cannot access the meta repository. - res = memberClient.get("/api/v1/projects/" + PROJECT_NAME + "/repos/meta/list"); - assertThat(res.status()).isSameAs(HttpStatus.FORBIDDEN); + await().untilAsserted(() -> assertThat(memberClient.get( + "/api/v1/projects/" + PROJECT_NAME + "/repos/meta/list").status()) + .isSameAs(HttpStatus.FORBIDDEN)); } @Test diff --git a/server/src/test/java/com/linecorp/centraldogma/server/metadata/MetadataServiceTest.java b/server/src/test/java/com/linecorp/centraldogma/server/metadata/MetadataServiceTest.java index 9f1fbdbdfc..df7dca6f77 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/metadata/MetadataServiceTest.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/metadata/MetadataServiceTest.java @@ -21,6 +21,7 @@ import static com.linecorp.centraldogma.server.storage.project.Project.REPO_META; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.awaitility.Awaitility.await; import java.util.concurrent.CompletableFuture; @@ -89,16 +90,16 @@ void project() { // Remove a project and check whether the project is removed. mds.removeProject(author, project1).join(); + await().untilAsserted(() -> assertThat(mds.getProject(project1).join().removal()).isNotNull()); metadata = mds.getProject(project1).join(); assertThat(metadata.name()).isEqualTo(project1); assertThat(metadata.creation().user()).isEqualTo(author.email()); - assertThat(metadata.removal()).isNotNull(); assertThat(metadata.removal().user()).isEqualTo(author.email()); // Restore the removed project. mds.restoreProject(author, project1).join(); - assertThat(mds.getProject(project1).join().removal()).isNull(); + await().untilAsserted(() -> assertThat(mds.getProject(project1).join().removal()).isNull()); } @Test @@ -112,7 +113,8 @@ void repository() { mds.addRepo(author, project1, repo1, ProjectRoles.of(RepositoryRole.WRITE, RepositoryRole.WRITE)) .join(); - assertThat(getProject(mds, project1).repos().get(repo1).name()).isEqualTo(repo1); + await().untilAsserted(() -> assertThat(getProject(mds, project1).repos().get(repo1).name()) + .isEqualTo(repo1)); // Fail due to duplicated addition. assertThatThrownBy(() -> mds.addRepo(author, project1, repo1).join()) @@ -131,20 +133,22 @@ void repository() { // Restore the removed repository. mds.restoreRepo(author, project1, repo1).join(); + await().untilAsserted(() -> assertThat(getRepo1(mds).removal()).isNull()); repositoryMetadata = getRepo1(mds); assertThat(repositoryMetadata.name()).isEqualTo(repo1); assertThat(repositoryMetadata.creation().user()).isEqualTo(author.email()); - assertThat(repositoryMetadata.removal()).isNull(); // Purge a repository. mds.removeRepo(author, project1, repo1).join(); mds.purgeRepo(author, project1, repo1).join(); - assertThatThrownBy(() -> getRepo1(mds)).isInstanceOf(RepositoryNotFoundException.class); + await().untilAsserted(() -> assertThatThrownBy(() -> getRepo1(mds)) + .isInstanceOf(RepositoryNotFoundException.class)); // Recreate the purged repository. mds.addRepo(author, project1, repo1, ProjectRoles.of(RepositoryRole.WRITE, RepositoryRole.WRITE)) .join(); - assertThat(getProject(mds, project1).repos().get(repo1).name()).isEqualTo(repo1); + await().untilAsserted(() -> assertThat(getProject(mds, project1).repos().get(repo1).name()) + .isEqualTo(repo1)); } @Test @@ -169,14 +173,13 @@ void repositoryProjectRoles() { final MetadataService mds = newMetadataService(manager); final ProjectMetadata metadata; - RepositoryMetadata repositoryMetadata; metadata = mds.getProject(project1).join(); assertThat(metadata).isNotNull(); mds.addRepo(author, project1, repo1, ProjectRoles.of(RepositoryRole.WRITE, RepositoryRole.WRITE)) .join(); - - repositoryMetadata = getRepo1(mds); + await().until(() -> getRepo1(mds) != null); + final RepositoryMetadata repositoryMetadata = getRepo1(mds); assertThat(repositoryMetadata.roles().projectRoles().member()).isSameAs(RepositoryRole.WRITE); // WRITE permission is not allowed for GUEST so it is automatically lowered to READ. assertThat(repositoryMetadata.roles().projectRoles().guest()).isEqualTo(RepositoryRole.READ); @@ -184,9 +187,8 @@ void repositoryProjectRoles() { final Revision revision = mds.updateRepositoryProjectRoles(author, project1, repo1, DEFAULT_PROJECT_ROLES).join(); - repositoryMetadata = getRepo1(mds); - assertThat(repositoryMetadata.roles().projectRoles().member()).isSameAs(RepositoryRole.WRITE); - assertThat(repositoryMetadata.roles().projectRoles().guest()).isNull(); + await().untilAsserted(() -> assertThat(getRepo1(mds).roles().projectRoles().guest()).isNull()); + assertThat(getRepo1(mds).roles().projectRoles().member()).isSameAs(RepositoryRole.WRITE); assertThat(mds.findRepositoryRole(project1, repo1, owner).join()).isSameAs(RepositoryRole.ADMIN); assertThat(mds.findRepositoryRole(project1, repo1, guest).join()).isNull(); @@ -213,6 +215,7 @@ void userRepositoryRole() { final MetadataService mds = newMetadataService(manager); mds.addRepo(author, project1, repo1, ProjectRoles.of(null, null)).join(); + await().until(() -> getRepo1(mds) != null); // Not a member yet. assertThatThrownBy(() -> mds.addUserRepositoryRole(author, project1, repo1, user1, RepositoryRole.READ) @@ -237,19 +240,20 @@ void userRepositoryRole() { // Add 'user1' to user repository role. final Revision revision = mds.addUserRepositoryRole(author, project1, repo1, user1, RepositoryRole.READ) .join(); + await().untilAsserted(() -> assertThat(mds.findRepositoryRole(project1, repo1, user1).join()) + .isSameAs(RepositoryRole.READ)); // Fail due to duplicated addition. assertThatThrownBy(() -> mds.addUserRepositoryRole(author, project1, repo1, user1, RepositoryRole.READ) .join()) .hasCauseInstanceOf(ChangeConflictException.class); - assertThat(mds.findRepositoryRole(project1, repo1, user1).join()).isSameAs(RepositoryRole.READ); - assertThat(mds.updateUserRepositoryRole(author, project1, repo1, user1, RepositoryRole.WRITE) .join().major()) .isEqualTo(revision.major() + 1); - assertThat(mds.findRepositoryRole(project1, repo1, user1).join()).isSameAs(RepositoryRole.WRITE); + await().untilAsserted(() -> assertThat(mds.findRepositoryRole(project1, repo1, user1).join()) + .isSameAs(RepositoryRole.WRITE)); // Updating the same operation will return the same revision. assertThat(mds.updateUserRepositoryRole(author, project1, repo1, user1, RepositoryRole.WRITE) @@ -279,12 +283,11 @@ void tokenRepositoryRole() { assertThatThrownBy(() -> mds.createToken(author, app1).join()) .hasCauseInstanceOf(ChangeConflictException.class); - final Token token = mds.findTokenByAppId(app1).join(); - assertThat(token).isNotNull(); + await().untilAsserted(() -> assertThat(mds.findTokenByAppId(app1)).isNotNull()); // Token 'app2' is not created yet. assertThatThrownBy(() -> mds.addToken(author, project1, app2, ProjectRole.MEMBER).join()) - .hasCauseInstanceOf(TokenNotFoundException.class); + .isInstanceOf(TokenNotFoundException.class); // Token is not registered to the project yet. assertThatThrownBy(() -> mds.addTokenRepositoryRole(author, project1, repo1, app1, RepositoryRole.READ) @@ -295,9 +298,11 @@ void tokenRepositoryRole() { // Be a token of the project. mds.addToken(author, project1, app1, ProjectRole.MEMBER).join(); + await().until(() -> mds.findTokenByAppId(app1) != null); mds.addTokenRepositoryRole(author, project1, repo1, app1, RepositoryRole.READ).join(); - assertThat(mds.findRepositoryRole(project1, repo1, app1).join()).isSameAs(RepositoryRole.READ); + await().untilAsserted(() -> assertThat(mds.findRepositoryRole(project1, repo1, app1).join()) + .isSameAs(RepositoryRole.READ)); // Try once more assertThatThrownBy(() -> mds.addToken(author, project1, app1, ProjectRole.MEMBER).join()) @@ -308,7 +313,8 @@ void tokenRepositoryRole() { final Revision revision = mds.updateTokenRepositoryRole(author, project1, repo1, app1, RepositoryRole.WRITE).join(); - assertThat(mds.findRepositoryRole(project1, repo1, app1).join()).isSameAs(RepositoryRole.WRITE); + await().untilAsserted(() -> assertThat(mds.findRepositoryRole(project1, repo1, app1).join()) + .isSameAs(RepositoryRole.WRITE)); // Update invalid token assertThatThrownBy(() -> mds.updateTokenRepositoryRole(author, project1, repo1, app2, @@ -334,6 +340,7 @@ void removeMember() { mds.addMember(author, project1, user1, ProjectRole.MEMBER).join(); mds.addMember(author, project1, user2, ProjectRole.MEMBER).join(); + await().untilAsserted(() -> assertThat(mds.getMember(project1, user2).join()).isNotNull()); mds.addUserRepositoryRole(author, project1, repo1, user1, RepositoryRole.READ).join(); mds.addUserRepositoryRole(author, project1, repo1, user2, RepositoryRole.READ).join(); @@ -345,7 +352,7 @@ void removeMember() { // Remove 'user1' from the project. mds.removeMember(author, project1, user1).join(); // Remove user repository role of 'user1', too. - assertThat(mds.findRepositoryRole(project1, repo1, user1).join()).isNull(); + await().untilAsserted(() -> assertThat(mds.findRepositoryRole(project1, repo1, user1).join()).isNull()); assertThat(mds.findRepositoryRole(project1, repo1, user2).join()).isSameAs(RepositoryRole.READ); @@ -364,6 +371,7 @@ void removeToken() { mds.addToken(author, project1, app1, ProjectRole.MEMBER).join(); mds.addToken(author, project1, app2, ProjectRole.MEMBER).join(); + await().until(() -> mds.findTokenByAppId(app2) != null); mds.addTokenRepositoryRole(author, project1, repo1, app1, RepositoryRole.READ).join(); mds.addTokenRepositoryRole(author, project1, repo1, app2, RepositoryRole.READ).join(); @@ -372,7 +380,7 @@ void removeToken() { // Remove 'app1' from the project. mds.removeToken(author, project1, app1).join(); // Remove token repository role of 'app1', too. - assertThat(mds.findRepositoryRole(project1, repo1, app1).join()).isNull(); + await().untilAsserted(() -> assertThat(mds.findRepositoryRole(project1, repo1, app1).join()).isNull()); assertThat(mds.findRepositoryRole(project1, repo1, app2).join()).isSameAs(RepositoryRole.READ); @@ -392,16 +400,18 @@ void destroyToken() { mds.addToken(author, project1, app1, ProjectRole.MEMBER).join(); mds.addToken(author, project1, app2, ProjectRole.MEMBER).join(); + await().until(() -> mds.findTokenByAppId(app2) != null); mds.addTokenRepositoryRole(author, project1, repo1, app1, RepositoryRole.READ).join(); mds.addTokenRepositoryRole(author, project1, repo1, app2, RepositoryRole.READ).join(); - assertThat(mds.findRepositoryRole(project1, repo1, app1).join()).isSameAs(RepositoryRole.READ); + await().untilAsserted(() -> assertThat(mds.findRepositoryRole(project1, repo1, app1).join()) + .isSameAs(RepositoryRole.READ)); // Remove 'app1' from the system completely. mds.destroyToken(author, app1).join(); mds.purgeToken(author, app1); - assertThat(mds.findRepositoryRole(project1, repo1, app1).join()).isNull(); + await().untilAsserted(() -> assertThat(mds.findRepositoryRole(project1, repo1, app1).join()).isNull()); assertThat(mds.findRepositoryRole(project1, repo1, app2).join()).isSameAs(RepositoryRole.READ); @@ -414,15 +424,13 @@ void destroyToken() { void tokenActivationAndDeactivation() { final MetadataService mds = newMetadataService(manager); - Token token; mds.createToken(author, app1).join(); - token = mds.getTokens().join().get(app1); - assertThat(token).isNotNull(); - assertThat(token.creation().user()).isEqualTo(owner.id()); + await().untilAsserted(() -> assertThat(mds.getTokens().get(app1)).isNotNull()); + assertThat(mds.getTokens().get(app1).creation().user()).isEqualTo(owner.id()); final Revision revision = mds.deactivateToken(author, app1).join(); - token = mds.getTokens().join().get(app1); - assertThat(token.isActive()).isFalse(); + await().untilAsserted(() -> assertThat(mds.getTokens().get(app1).isActive()).isFalse()); + final Token token = mds.getTokens().get(app1); assertThat(token.deactivation()).isNotNull(); assertThat(token.deactivation().user()).isEqualTo(owner.id()); @@ -430,7 +438,7 @@ void tokenActivationAndDeactivation() { assertThat(mds.deactivateToken(author, app1).join()).isEqualTo(revision); assertThat(mds.activateToken(author, app1).join().major()).isEqualTo(revision.major() + 1); - assertThat(mds.getTokens().join().get(app1).isActive()).isTrue(); + await().untilAsserted(() -> assertThat(mds.getTokens().get(app1).isActive()).isTrue()); // Executing the same operation will return the same revision. assertThat(mds.activateToken(author, app1).join().major()).isEqualTo(revision.major() + 1); @@ -441,38 +449,39 @@ void updateWriteQuota() { final MetadataService mds = newMetadataService(manager); mds.addRepo(author, project1, repo1, ProjectRoles.of(RepositoryRole.WRITE, RepositoryRole.WRITE)) .join(); - RepositoryMetadata repoMeta = mds.getRepo(project1, repo1).join(); - assertThat(repoMeta.writeQuota()).isNull(); + await().until(() -> getRepo1(mds) != null); + assertThat(mds.getRepo(project1, repo1).join().writeQuota()).isNull(); final QuotaConfig writeQuota1 = new QuotaConfig(5, 2); mds.updateWriteQuota(Author.SYSTEM, project1, repo1, writeQuota1).join(); - repoMeta = mds.getRepo(project1, repo1).join(); - assertThat(repoMeta.writeQuota()).isEqualTo(writeQuota1); + await().untilAsserted(() -> assertThat(getRepo1(mds).writeQuota()).isEqualTo(writeQuota1)); final QuotaConfig writeQuota2 = new QuotaConfig(3, 1); mds.updateWriteQuota(Author.SYSTEM, project1, repo1, writeQuota2).join(); - repoMeta = mds.getRepo(project1, repo1).join(); - assertThat(repoMeta.writeQuota()).isEqualTo(writeQuota2); + await().untilAsserted(() -> assertThat(getRepo1(mds).writeQuota()).isEqualTo(writeQuota2)); } @Test void updateUser() { final MetadataService mds = newMetadataService(manager); - Token token; mds.createToken(author, app1).join(); - token = mds.getTokens().join().get(app1); - assertThat(token).isNotNull(); - assertThat(token.isSystemAdmin()).isFalse(); + await().untilAsserted(() -> assertThat(mds.getTokens().get(app1)).isNotNull()); + assertThat(mds.getTokens().get(app1).isSystemAdmin()).isFalse(); + + final Tokens tokens = mds.getTokens(); + // tokens are cached so the same instance is returned. + assertThat(tokens).isSameAs(mds.getTokens()); final Revision revision = mds.updateTokenLevel(author, app1, true).join(); - token = mds.getTokens().join().get(app1); - assertThat(token.isSystemAdmin()).isTrue(); + await().untilAsserted(() -> assertThat(mds.getTokens().get(app1).isSystemAdmin()).isTrue()); + // Now the reference is different. + assertThat(mds.getTokens()).isNotSameAs(tokens); + assertThat(mds.updateTokenLevel(author, app1, true).join()).isEqualTo(revision); assertThat(mds.updateTokenLevel(author, app1, false).join()).isEqualTo(revision.forward(1)); - token = mds.getTokens().join().get(app1); - assertThat(token.isSystemAdmin()).isFalse(); + await().untilAsserted(() -> assertThat(mds.getTokens().get(app1).isSystemAdmin()).isFalse()); assertThat(mds.updateTokenLevel(author, app1, false).join()).isEqualTo(revision.forward(1)); } @@ -482,7 +491,8 @@ private static RepositoryMetadata getRepo1(MetadataService mds) { } private static MetadataService newMetadataService(ProjectManagerExtension extension) { - return new MetadataService(extension.projectManager(), extension.executor()); + return new MetadataService(extension.projectManager(), extension.executor(), + extension.internalProjectInitializer()); } private static ProjectMetadata getProject(MetadataService mds, String projectName) { diff --git a/server/src/test/java/com/linecorp/centraldogma/server/metadata/MissingRepositoryMetadataTest.java b/server/src/test/java/com/linecorp/centraldogma/server/metadata/MissingRepositoryMetadataTest.java index 77103afe48..85651179d4 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/metadata/MissingRepositoryMetadataTest.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/metadata/MissingRepositoryMetadataTest.java @@ -27,6 +27,7 @@ import com.linecorp.centraldogma.common.RepositoryRole; import com.linecorp.centraldogma.server.command.Command; import com.linecorp.centraldogma.server.command.CommandExecutor; +import com.linecorp.centraldogma.server.storage.project.InternalProjectInitializer; import com.linecorp.centraldogma.server.storage.project.ProjectManager; import com.linecorp.centraldogma.server.storage.repository.RepositoryManager; import com.linecorp.centraldogma.testing.internal.ProjectManagerExtension; @@ -59,13 +60,14 @@ void missingRepositoryMetadata() { final ProjectManager pm = manager.projectManager(); final RepositoryManager rm = pm.get(PROJ).repos(); final CommandExecutor executor = manager.executor(); + final InternalProjectInitializer projectInitializer = new InternalProjectInitializer(executor, pm); // Create a new repository without adding metadata. rm.create("repo", AUTHOR); assertThat(rm.get("repo")).isNotNull(); // Try to access the repository metadata, which will trigger its auto-generation. - final MetadataService mds = new MetadataService(pm, executor); + final MetadataService mds = new MetadataService(pm, executor, projectInitializer); final RepositoryMetadata metadata = mds.getRepo(PROJ, "repo").join(); assertThat(metadata.id()).isEqualTo("repo"); assertThat(metadata.name()).isEqualTo("repo"); diff --git a/server/src/test/java/com/linecorp/centraldogma/server/metadata/TokenTest.java b/server/src/test/java/com/linecorp/centraldogma/server/metadata/TokenTest.java index b4fed9864b..f0c8e95fd3 100644 --- a/server/src/test/java/com/linecorp/centraldogma/server/metadata/TokenTest.java +++ b/server/src/test/java/com/linecorp/centraldogma/server/metadata/TokenTest.java @@ -18,6 +18,7 @@ import static com.linecorp.centraldogma.internal.jsonpatch.JsonPatchOperation.asJsonArray; import static com.linecorp.centraldogma.server.metadata.MetadataService.TOKEN_JSON; import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; import java.util.Collection; @@ -66,7 +67,8 @@ class TokenTest { @BeforeAll static void setUp() throws JsonParseException { - metadataService = new MetadataService(manager.projectManager(), manager.executor()); + metadataService = new MetadataService(manager.projectManager(), manager.executor(), + manager.internalProjectInitializer()); tokenService = new TokenService(manager.executor(), metadataService); // Put the legacy token. @@ -111,7 +113,7 @@ private static String tokenJson(boolean legacy) { @Test void updateToken() throws JsonParseException { - final Collection tokens = tokenService.listTokens(USER).join(); + final Collection tokens = tokenService.listTokens(USER); assertThat(tokens.size()).isOne(); final Token token = Iterables.getFirst(tokens, null); assertThat(token.appId()).isEqualTo(APP_ID); @@ -125,10 +127,10 @@ void updateToken() throws JsonParseException { "value", "inactive"))); tokenService.updateToken(CTX, APP_ID, deactivation, AUTHOR, USER).join(); - Token updated = metadataService.findTokenByAppId(APP_ID).join(); + await().untilAsserted(() -> assertThat(metadataService.findTokenByAppId(APP_ID).isActive()).isFalse()); + Token updated = metadataService.findTokenByAppId(APP_ID); assertThat(updated.appId()).isEqualTo(APP_ID); assertThat(updated.isSystemAdmin()).isTrue(); - assertThat(updated.isActive()).isFalse(); final JsonNode activation = Jackson.valueToTree( ImmutableList.of( @@ -137,10 +139,10 @@ void updateToken() throws JsonParseException { "value", "active"))); tokenService.updateToken(CTX, APP_ID, activation, AUTHOR, USER).join(); - updated = metadataService.findTokenByAppId(APP_ID).join(); + await().untilAsserted(() -> assertThat(metadataService.findTokenByAppId(APP_ID).isActive()).isTrue()); + updated = metadataService.findTokenByAppId(APP_ID); assertThat(updated.appId()).isEqualTo(APP_ID); assertThat(updated.isSystemAdmin()).isTrue(); - assertThat(updated.isActive()).isTrue(); } @Test diff --git a/testing-internal/src/main/java/com/linecorp/centraldogma/testing/internal/ProjectManagerExtension.java b/testing-internal/src/main/java/com/linecorp/centraldogma/testing/internal/ProjectManagerExtension.java index 8c352a16ec..e502d9c5c0 100644 --- a/testing-internal/src/main/java/com/linecorp/centraldogma/testing/internal/ProjectManagerExtension.java +++ b/testing-internal/src/main/java/com/linecorp/centraldogma/testing/internal/ProjectManagerExtension.java @@ -27,10 +27,12 @@ import com.linecorp.armeria.common.metric.NoopMeterRegistry; import com.linecorp.centraldogma.common.ShuttingDownException; +import com.linecorp.centraldogma.server.CentralDogmaBuilder; import com.linecorp.centraldogma.server.command.CommandExecutor; import com.linecorp.centraldogma.server.command.StandaloneCommandExecutor; import com.linecorp.centraldogma.server.internal.storage.project.DefaultProjectManager; import com.linecorp.centraldogma.server.management.ServerStatusManager; +import com.linecorp.centraldogma.server.metadata.MetadataService; import com.linecorp.centraldogma.server.storage.project.InternalProjectInitializer; import com.linecorp.centraldogma.server.storage.project.ProjectManager; import com.linecorp.centraldogma.testing.junit.AbstractAllOrEachExtension; @@ -80,6 +82,8 @@ public void before(ExtensionContext context) throws Exception { executor.start().get(); internalProjectInitializer = new InternalProjectInitializer(executor, projectManager); internalProjectInitializer.initialize(); + final MetadataService mds = new MetadataService(projectManager, executor, internalProjectInitializer); + executor.setRepositoryMetadataSupplier(mds::getRepo); afterExecutorStarted(); } @@ -116,6 +120,13 @@ public ScheduledExecutorService purgeWorker() { return purgeWorker; } + /** + * Returns an {@link InternalProjectInitializer}. + */ + public InternalProjectInitializer internalProjectInitializer() { + return internalProjectInitializer; + } + /** * Override this method to configure a project after the executor started. */ @@ -134,7 +145,8 @@ protected Executor newWorker() { protected ProjectManager newProjectManager(Executor repositoryWorker, Executor purgeWorker) { try { return new DefaultProjectManager(dataDir, repositoryWorker, - purgeWorker, NoopMeterRegistry.get(), null); + purgeWorker, NoopMeterRegistry.get(), + CentralDogmaBuilder.DEFAULT_REPOSITORY_CACHE_SPEC); } catch (Exception e) { // Should not reach here. throw new Error(e); diff --git a/xds/src/main/java/com/linecorp/centraldogma/xds/group/v1/XdsGroupService.java b/xds/src/main/java/com/linecorp/centraldogma/xds/group/v1/XdsGroupService.java index 49191bc808..28b1c79d6e 100644 --- a/xds/src/main/java/com/linecorp/centraldogma/xds/group/v1/XdsGroupService.java +++ b/xds/src/main/java/com/linecorp/centraldogma/xds/group/v1/XdsGroupService.java @@ -28,6 +28,7 @@ import com.linecorp.centraldogma.common.RepositoryExistsException; import com.linecorp.centraldogma.server.command.CommandExecutor; import com.linecorp.centraldogma.server.metadata.MetadataService; +import com.linecorp.centraldogma.server.storage.project.InternalProjectInitializer; import com.linecorp.centraldogma.server.storage.project.Project; import com.linecorp.centraldogma.server.storage.project.ProjectManager; import com.linecorp.centraldogma.xds.group.v1.XdsGroupServiceGrpc.XdsGroupServiceImplBase; @@ -47,10 +48,11 @@ public final class XdsGroupService extends XdsGroupServiceImplBase { /** * Creates a new instance. */ - public XdsGroupService(ProjectManager projectManager, CommandExecutor commandExecutor) { + public XdsGroupService(ProjectManager projectManager, CommandExecutor commandExecutor, + InternalProjectInitializer internalProjectInitializer) { this.projectManager = projectManager; this.commandExecutor = commandExecutor; - mds = new MetadataService(projectManager, commandExecutor); + mds = new MetadataService(projectManager, commandExecutor, internalProjectInitializer); } @Override diff --git a/xds/src/main/java/com/linecorp/centraldogma/xds/internal/ControlPlaneService.java b/xds/src/main/java/com/linecorp/centraldogma/xds/internal/ControlPlaneService.java index 5fe2f98ca6..bf6aa71a35 100644 --- a/xds/src/main/java/com/linecorp/centraldogma/xds/internal/ControlPlaneService.java +++ b/xds/src/main/java/com/linecorp/centraldogma/xds/internal/ControlPlaneService.java @@ -111,7 +111,8 @@ void start(PluginInitContext pluginInitContext) { final GrpcService xdsApplicationService = GrpcService.builder() .addService(new XdsGroupService(pluginInitContext.projectManager(), - commandExecutor)) + commandExecutor, + pluginInitContext.internalProjectInitializer())) .addService(new XdsListenerService(xdsResourceManager)) .addService(new XdsRouteService(xdsResourceManager)) .addService(new XdsClusterService(xdsResourceManager))