From 38caff7875132edc5c81e269f73fcd39670a84ba Mon Sep 17 00:00:00 2001 From: stefan-egli Date: Tue, 28 Nov 2023 15:33:26 +0100 Subject: [PATCH] OAK-10382 : added flatfile command to the project (#1223) * OAK-10382 : added flatfile command to the project * OAK-10382 : adding a unit test for FlatFileCommand * OAK-10382 : fix mongo url --- .../jackrabbit/oak/run/AvailableModes.java | 1 + .../jackrabbit/oak/run/FlatFileCommand.java | 138 ++++++++++++++++++ .../oak/run/FlatFileCommandTest.java | 98 +++++++++++++ 3 files changed, 237 insertions(+) create mode 100644 oak-run/src/main/java/org/apache/jackrabbit/oak/run/FlatFileCommand.java create mode 100644 oak-run/src/test/java/org/apache/jackrabbit/oak/run/FlatFileCommandTest.java diff --git a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/AvailableModes.java b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/AvailableModes.java index ecfd16d4b26..6c723cb93f1 100644 --- a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/AvailableModes.java +++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/AvailableModes.java @@ -45,6 +45,7 @@ public final class AvailableModes { .put(DocumentStoreCheckCommand.NAME, new DocumentStoreCheckCommand()) .put("explore", new ExploreCommand()) .put(NodeStateExportCommand.NAME, new NodeStateExportCommand()) + .put(FlatFileCommand.NAME, new FlatFileCommand()) .put(FrozenNodeRefsByScanningCommand.NAME, new FrozenNodeRefsByScanningCommand()) .put(FrozenNodeRefsUsingIndexCommand.NAME, new FrozenNodeRefsUsingIndexCommand()) .put("garbage", new GarbageCommand()) diff --git a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/FlatFileCommand.java b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/FlatFileCommand.java new file mode 100644 index 00000000000..0f14b955ef9 --- /dev/null +++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/FlatFileCommand.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.run; + +import java.io.File; +import java.io.IOException; +import java.util.HashSet; +import java.util.Set; + +import org.apache.commons.io.FileUtils; +import org.apache.jackrabbit.guava.common.collect.ImmutableSet; +import org.apache.jackrabbit.oak.index.indexer.document.flatfile.SimpleFlatFileUtil; +import org.apache.jackrabbit.oak.run.cli.BlobStoreOptions; +import org.apache.jackrabbit.oak.run.cli.BlobStoreOptions.Type; +import org.apache.jackrabbit.oak.run.cli.NodeStoreFixture; +import org.apache.jackrabbit.oak.run.cli.NodeStoreFixtureProvider; +import org.apache.jackrabbit.oak.run.cli.Options; +import org.apache.jackrabbit.oak.run.cli.OptionsBean; +import org.apache.jackrabbit.oak.run.commons.Command; +import org.apache.jackrabbit.oak.spi.state.NodeState; +import org.apache.jackrabbit.oak.spi.state.NodeStore; + +import joptsimple.OptionParser; +import joptsimple.OptionSet; +import joptsimple.OptionSpec; + +/** + * The flatfile command is an extract of the ability to create a filefile from + * the index command. Other than the origin, the flatfile command doesn't + * require any checkpoint nor index name. + */ +@SuppressWarnings({ "rawtypes", "unchecked" }) +public class FlatFileCommand implements Command { + + public static final String NAME = "flatfile"; + private final String summary = "Provides flatfile operations"; + + public static class FlatFileOptions implements OptionsBean { + + private final OptionSpec outFileOpt; + private final OptionSpec flatfile; + protected OptionSet options; + protected final Set actionOpts; + private final Set operationNames; + + public FlatFileOptions(OptionParser parser) { + flatfile = parser.accepts("flatfile", + "Create a flatfile based on head of a repository"); + outFileOpt = parser.accepts("out", "Name of the flatfile to create") + .withRequiredArg().ofType(File.class).defaultsTo(new File("temp")); + // Set of options which define action + actionOpts = ImmutableSet.of(flatfile); + operationNames = collectionOperationNames(actionOpts); + } + + @Override + public void configure(OptionSet options) { + this.options = options; + } + + @Override + public String title() { + return ""; + } + + @Override + public String description() { + return "The flatfile command supports creation of a flatfile for current head of a repository."; + } + + @Override + public int order() { + return 50; + } + + @Override + public Set operationNames() { + return operationNames; + } + + public File getOutFile() throws IOException { + File outFile = outFileOpt.value(options); + FileUtils.forceMkdir(outFile.getParentFile()); + return outFile; + } + + private static Set collectionOperationNames(Set actionOpts) { + Set result = new HashSet<>(); + for (OptionSpec spec : actionOpts) { + result.addAll(spec.options()); + } + return result; + } + } + + @Override + public void execute(String... args) throws Exception { + OptionParser parser = new OptionParser(); + Options opts = new Options(); + opts.setCommandName(NAME); + opts.setSummary(summary); + opts.registerOptionsFactory(FlatFileOptions::new); + opts.registerOptionsFactory(BlobStoreOptions::new); + opts.parseAndConfigure(parser, args); + BlobStoreOptions bsOpts = opts.getOptionBean(BlobStoreOptions.class); + Type bsType = bsOpts.getBlobStoreType(); + if (bsType == Type.NONE) { + System.err.println("BlobStore args missing."); + System.err.println("Hint: consider use fake one via: --fake-ds-path=."); + System.exit(1); + } + FlatFileOptions ffOpts = opts.getOptionBean(FlatFileOptions.class); + File out = ffOpts.getOutFile(); + + try (NodeStoreFixture fixture = NodeStoreFixtureProvider.create(opts, true)) { + NodeStore store = fixture.getStore(); + NodeState root = store.getRoot(); + SimpleFlatFileUtil.createFlatFileFor(root, out); + } + } + +} \ No newline at end of file diff --git a/oak-run/src/test/java/org/apache/jackrabbit/oak/run/FlatFileCommandTest.java b/oak-run/src/test/java/org/apache/jackrabbit/oak/run/FlatFileCommandTest.java new file mode 100644 index 00000000000..93ae1dc813b --- /dev/null +++ b/oak-run/src/test/java/org/apache/jackrabbit/oak/run/FlatFileCommandTest.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.run; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeTrue; + +import java.io.File; + +import org.apache.jackrabbit.oak.plugins.document.DocumentMKBuilderProvider; +import org.apache.jackrabbit.oak.plugins.document.DocumentNodeStore; +import org.apache.jackrabbit.oak.plugins.document.MongoConnectionFactory; +import org.apache.jackrabbit.oak.plugins.document.MongoUtils; +import org.apache.jackrabbit.oak.plugins.document.util.MongoConnection; +import org.apache.jackrabbit.oak.spi.blob.MemoryBlobStore; +import org.junit.After; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; + +public class FlatFileCommandTest { + + @Rule + public MongoConnectionFactory connectionFactory = new MongoConnectionFactory(); + + @Rule + public DocumentMKBuilderProvider builderProvider = new DocumentMKBuilderProvider(); + + private DocumentNodeStore ns; + + private File tmpFlatfileOut; + + @BeforeClass + public static void assumeMongoDB() { + assumeTrue(MongoUtils.isAvailable()); + } + + private DocumentNodeStore createDocumentNodeStore() { + MongoConnection c = connectionFactory.getConnection(); + assertNotNull(c); + MongoUtils.dropCollections(c.getDatabase()); + return builderProvider.newBuilder().setBlobStore(new MemoryBlobStore()) + .setMongoDB(c.getMongoClient(), c.getDBName()).getNodeStore(); + } + + @Before + public void before() { + ns = createDocumentNodeStore(); + tmpFlatfileOut = new File("tmp.flatfile.out"); + if (tmpFlatfileOut.exists()) { + tmpFlatfileOut.delete(); + } + assertFalse(tmpFlatfileOut.exists()); + } + + @After + public void after() { + if (tmpFlatfileOut != null && tmpFlatfileOut.exists()) { + assertTrue("File should not now be deleted but is not : " + tmpFlatfileOut, + tmpFlatfileOut.delete()); + } + tmpFlatfileOut = null; + if (ns != null) { + ns.dispose(); + ns = null; + } + } + + @Test + public void flatfile() throws Exception { + FlatFileCommand cmd = new FlatFileCommand(); + assertFalse("File should not exist but does : " + tmpFlatfileOut, + tmpFlatfileOut.exists()); + cmd.execute(MongoUtils.URL, "--out", + tmpFlatfileOut.getAbsolutePath(), "--fake-ds-path=."); + assertTrue("File should exist but does not : " + tmpFlatfileOut, + tmpFlatfileOut.exists()); + assertTrue(tmpFlatfileOut.exists()); + } + +} \ No newline at end of file