Add :create command

Summary:
Occasionally it's useful to just create an empty DB from inside the
shell, e.g. for playing with schema changes.

Reviewed By: josefs

Differential Revision: D65269320

fbshipit-source-id: eace7fce4b0faf210e5d7e64a65286f9d621187a

glean/client/hs/Glean.hs

@@ -160,6 +160,8 @@ module Glean
 
   -- * Writing
   , fillDatabase
+  , create
+  , finish
   , finalize
   , completePredicates
   , CompletePredicates(..)

glean/hs/Glean/Backend/Types.hs

@@ -19,6 +19,8 @@ module Glean.Backend.Types
   , loadPredicates
   , databases
   , localDatabases
+  , create
+  , finish
   , fillDatabase
   , finalize
   , completePredicates
@@ -45,10 +47,10 @@ import Control.Exception
 import Control.Monad
 import Data.Bits
 import Data.Default
-import Data.Either
 import Data.Hashable
 import qualified Data.HashMap.Strict as HashMap
 import Data.Map (Map)
+import Data.Maybe
 import Data.Monoid
 import Data.Text (Text)
 import qualified Data.Text as Text
@@ -249,31 +251,56 @@ fillDatabase
     -- ^ Caller-supplied action to write data into the DB.
   -> IO b
 fillDatabase env repo handle maybeDeps ifexists action =
-  tryBracket create finish (const action)
-  where
-  create = do
-    r <- kickOffDatabase env def
-      { kickOff_repo = repo
-      , kickOff_fill = Just $ KickOffFill_writeHandle handle
-      , kickOff_properties = HashMap.fromList $
-          [ ("glean.schema_id", id)
-          | Just (SchemaId id) <- [schemaId env]
-          ]
-      , kickOff_dependencies = maybeDeps
-      }
-    when (kickOffResponse_alreadyExists r) ifexists
-
-  finish _ e = do
-    workFinished env WorkFinished
-      { workFinished_work = def
-          { work_repo = repo
-          , work_handle = handle
-          }
-      , workFinished_outcome = case e of
-          Left ex -> Outcome_failure (Failure (showt ex))
-          Right _ -> Outcome_success def
-      }
-    when (isRight e) $ finalize env repo
+  tryBracket
+    (do
+      exists <- create env repo handle maybeDeps
+      when exists ifexists
+    )
+    (\_ ex -> do
+      let failure = case ex of Left e -> Just (showt e); _ -> Nothing
+      finish env repo handle failure
+    )
+    (const action)
+
+-- | Create a database. The schema ID is set from the Backend.
+create
+  :: Backend a
+  => a
+  -> Repo
+  -> Text -- ^ A handle, can be anything
+  -> Maybe Dependencies
+  -> IO Bool  -- ^ Returns 'True' if the DB already existed
+create backend repo handle maybeDeps = do
+  r <- kickOffDatabase backend def
+    { kickOff_repo = repo
+    , kickOff_fill = Just $ KickOffFill_writeHandle handle
+    , kickOff_properties = HashMap.fromList $
+        [ ("glean.schema_id", id)
+        | Just (SchemaId id) <- [schemaId backend]
+        ]
+    , kickOff_dependencies = maybeDeps
+    }
+  return (kickOffResponse_alreadyExists r)
+
+-- | Finish a DB created with 'create'
+finish
+  :: Backend a
+  => a
+  -> Repo
+  -> Text -- ^ The handle passed to 'create'
+  -> Maybe Text  -- ^ @Just e@ => failed with message e
+  -> IO ()
+finish backend repo handle failure = do
+  workFinished backend WorkFinished
+    { workFinished_work = def
+        { work_repo = repo
+        , work_handle = handle
+        }
+    , workFinished_outcome = case failure of
+        Just ex -> Outcome_failure (Failure ex)
+        Nothing -> Outcome_success def
+    }
+  when (isNothing failure) $ finalize backend repo
 
 -- | Wait for a database to finish finalizing and enter the "complete"
 -- state after all writing has finished. Before the database is

glean/shell/Glean/Shell.hs

@@ -454,8 +454,10 @@ helptext = vcat
             <> "Set the EDITOR environment variable to choose an editor")
       , ("limit <n>",
             "Set limit on the number of query results")
-      , ("load (<file> | <db>/<hash> <file> ...)",
+      , ("load [<file> | <db>/<hash> <file> ...]",
             "Create a DB from file(s) of JSON facts")
+      , ("create [<db>/<hash>]",
+            "Create an empty DB")
       , ("timeout off|<n>",
             "Set the query time budget")
       , ("expand off|on|<predicate>...",
@@ -598,6 +600,7 @@ commands =
   , Cmd "list-all" completeDatabaseName $ const . displayDatabases True False
   , Cmd "dump" Haskeline.noCompletion $ \str _ -> dumpCmd str
   , Cmd "load" Haskeline.completeFilename $ \str _ -> loadCmd str
+  , Cmd "create" Haskeline.noCompletion $ \str _ -> createCmd str
   , Cmd "more" Haskeline.noCompletion $ const $ const moreCmd
   , Cmd "database" completeDatabases $ const . dbCmd
   , Cmd "db" completeDatabaseName $ const . dbCmd
@@ -703,12 +706,34 @@ loadCmd str
     load repo [file]
     setRepo repo
 
+  -- no files: behave like :new
+  | [] <- args = createCmd str
+
   | otherwise
   = liftIO $ throwIO $ ErrorCall
       "syntax:  :load (<file> | <db>/<hash> <file> ...)"
   where
     args = Data.List.words str
 
+createCmd :: String -> Eval ()
+createCmd str
+  | [db] <- args
+  , Just repo <- Glean.parseRepo db = do
+    create repo
+    setRepo repo
+
+  | [] <- args = do
+    let name = "tmp"
+    hash <- pickHash name
+    let repo = Thrift.Repo name hash
+    create repo
+    setRepo repo
+
+  | otherwise =
+     liftIO $ throwIO $ ErrorCall "syntax:  :create [<db>/<hash>]"
+  where
+    args = Data.List.words str
+
 dumpCmd :: String -> Eval ()
 dumpCmd str =
   withBackend $ \backend ->

glean/shell/Glean/Shell/Index.hs

@@ -10,6 +10,7 @@ module Glean.Shell.Index
   ( indexCmd
   , pickHash
   , load
+  , create
   ) where
 
 import Options.Applicative as OptParse
@@ -30,7 +31,7 @@ import System.FilePath
 import System.IO.Temp
 import TextShow
 
-import Glean
+import qualified Glean
 import Glean.Shell.Types
 import Glean.Util.Some
 import Glean.Write
@@ -59,7 +60,7 @@ indexCmd str = do
     let repo = Glean.Repo name hash
     withBackend $ \backend -> do
       let exists = throwIO (ErrorCall (show repo <> ": already exists"))
-      liftIO $ fillDatabase backend repo "" Nothing exists $
+      liftIO $ Glean.fillDatabase backend repo "" Nothing exists $
         runIndexer (Some backend) repo
           IndexerParams {
             indexerRoot = root,
@@ -76,7 +77,7 @@ withSystemTempDirectory' str action = Eval $ do
 
 pickHash :: Text -> Eval Text
 pickHash name = withBackend $ \be -> do
-  r <- liftIO $ listDatabases be def
+  r <- liftIO $ Glean.listDatabases be def
   let
     hashes =
       [ repo_hash
@@ -90,10 +91,16 @@ pickHash name = withBackend $ \be -> do
 load :: Glean.Repo -> [FilePath] -> Eval ()
 load repo files = withBackend $ \be ->  liftIO $ do
   let onExisting  = throwIO $ ErrorCall "database already exists"
-  void $ fillDatabase be repo "" Nothing onExisting $ forM_ files $ \file -> do
-    r <- Foreign.CPP.Dynamic.parseJSON =<< B.readFile file
-    val <- either (throwIO  . ErrorCall . Text.unpack) return r
-    batches <- case Aeson.parse parseJsonFactBatches val of
-      Error str -> throwIO $ ErrorCall str
-      Aeson.Success x -> return x
-    Glean.sendJsonBatch be repo batches Nothing
+  void $ Glean.fillDatabase be repo "" Nothing onExisting $
+    forM_ files $ \file -> do
+      r <- Foreign.CPP.Dynamic.parseJSON =<< B.readFile file
+      val <- either (throwIO  . ErrorCall . Text.unpack) return r
+      batches <- case Aeson.parse parseJsonFactBatches val of
+        Error str -> throwIO $ ErrorCall str
+        Aeson.Success x -> return x
+      Glean.sendJsonBatch be repo batches Nothing
+
+create :: Glean.Repo -> Eval ()
+create repo = withBackend $ \be ->  liftIO $ do
+  exists <- Glean.create be repo "" Nothing
+  when exists $ throwIO $ ErrorCall "database already exists"

glean/shell/tests/integration/test-shell.t

@@ -21,7 +21,8 @@
     :schema [predicate|type]                    Show schema for the given predicate or type
     :edit                                       Edit a query in an external editor. Set the EDITOR environment variable to choose an editor
     :limit <n>                                  Set limit on the number of query results
-    :load (<file> | <db>/<hash> <file> ...)     Create a DB from file(s) of JSON facts
+    :load [<file> | <db>/<hash> <file> ...]     Create a DB from file(s) of JSON facts
+    :create [<db>/<hash>]                       Create an empty DB
     :timeout off|<n>                            Set the query time budget
     :expand off|on|<predicate>...               Recursively expand nested facts in the response
     :pager off|on                               Enable/disable result paging

glean/shell/tests/shell_test.py

@@ -236,6 +236,25 @@ def test(self):
         self.assertIn("expr/1", output)
 
 
+class GleanShellCreate(GleanShellTest):
+    def test(self):
+        prompt = "tmp>"
+        self.shellCommand(":create", prompt=prompt)
+        output = self.shellCommand(":db", prompt=prompt)
+        self.assertIn("tmp", output)
+
+        output = self.shellCommand(":stat", prompt=prompt)
+        self.assertIsNotNone(re.search("0 facts", output))
+
+        prompt = "test>"
+        self.shellCommand(":create test/0", prompt=prompt)
+        output = self.shellCommand(":db", prompt=prompt)
+        self.assertIn("test/0", output)
+
+        output = self.shellCommand(":stat", prompt=prompt)
+        self.assertIsNotNone(re.search("0 facts", output))
+
+
 class GleanShellLoadBroken(GleanShellTest):
     def test(self):
         self.shellCommand(":load glean/shell/tests/error.glean")

glean/website/docs/shell.md

@@ -108,7 +108,13 @@ variable to choose an editor.
 Set limit on the number of query results. If there are more results
 than the limit, then you can type `:more` to fetch the next `N`
 results.
-* `:timeout off|MILLISECONDS`<br />
+* `:load [<file> | <db>/<hash> <file> ...]` <br />
+Create a new DB, loading facts from the given file(s).
+  * `<file>` is a file of facts in JSON format
+  * `<db>/<hash>` optionally the name of the DB to create
+* `:create [<db>/<hash>]` <br />
+Create a new empty DB.
+  * `<db>/<hash>` optionally the name of the DB to create
 Set the query time budget. If the time limit expires, the results so
 far are returned, and you can type `:more` to see more results.
 * `:expand off|on`<br />