Use Haskell to configure mithril in e2e tests

.buildkite/pipeline.yml

@@ -588,11 +588,6 @@ steps:
       agents:
         system: ${windows}
       env:
-        NODE_DB_DIR: "test\\e2e\\state\\node\\db"
-        NODE_DIR: "test\\e2e\\state\\node"
-        WALLET_DB_DIR: "test\\e2e\\state\\wallet"
-        AGGREGATOR_ENDPOINT: "${PREPROD_AGGREGATOR_ENDPOINT}"
-        GENESIS_VERIFICATION_KEY: "${PREPROD_GENESIS_VERIFICATION_KEY}"
       concurrency: 1
       concurrency_group: 'windows-tests'
 

lib/integration/exe/e2e.hs

@@ -19,6 +19,10 @@ import qualified Data.ByteString.Lazy as BL
 import qualified Data.Text as T
 import qualified Test.Integration.Scenario.Preprod as Preprod
 
+import Cardano.Launcher.Mithril
+    ( downloadLatestSnapshot
+    , downloadMithril
+    )
 import Cardano.Launcher.Node
     ( CardanoNodeConfig (..)
     , isWindows
@@ -71,11 +75,14 @@ import Network.URI
     ( parseURI
     )
 import System.Directory
-    ( createDirectoryIfMissing
-    , getCurrentDirectory
+    ( copyFile
+    , createDirectoryIfMissing
+    , doesDirectoryExist
+    , listDirectory
     )
 import System.Environment
     ( lookupEnv
+    , setEnv
     )
 import System.FilePath
     ( takeDirectory
@@ -105,28 +112,15 @@ import Test.Utils.Paths
 
 -- ENV configuration -----------------------------------------------------------
 
-data E2EConfig = E2EConfig
-    { walletDbDir :: FilePath
-    , nodeDir :: FilePath
-    , nodeDbDir :: FilePath
-    , preprodMnemonics :: [SomeMnemonic]
+newtype E2EConfig = E2EConfig
+    { preprodMnemonics :: [SomeMnemonic]
     }
 
 getConfig :: IO E2EConfig
 getConfig = do
-    walletDbDir <- fromMaybe defaultWalletDbDir <$> lookupEnv "WALLET_DB_DIR"
-    nodeDbDir <- fromMaybe defaultNodeDbDir <$> lookupEnv "NODE_DB_DIR"
-    nodeDir <- fromMaybe defaultNodeDir <$> lookupEnv "NODE_DIR"
     preprodMnemonics <- getPreprodMnemonics
     pure $ E2EConfig {..}
   where
-    -- Probably too fine grained control with all these settings, but works for now
-    defaultNodeDbDir   = repoRoot </> "test" </> "e2e" </> "state" </> "node" </> "db"
-    defaultNodeDir   = repoRoot </> "test" </> "e2e" </> "state" </> "node"
-    defaultWalletDbDir = repoRoot </> "test" </> "e2e" </> "state" </> "wallet"
-
-    repoRoot = ".." </> ".." -- works when run with 'cabal test'
-
     getPreprodMnemonics :: IO [SomeMnemonic]
     getPreprodMnemonics
         = map (SomeMnemonic . unsafeMkMnemonic @15 . T.words)
@@ -160,19 +154,44 @@ main = withUtf8 $ do
 
 -- setup -----------------------------------------------------------------------
 
+-- | Recursively copy a directory from 'src' to 'dst'
+copyDir :: FilePath -> FilePath -> IO ()
+copyDir src dst = do
+  -- Create the destination directory (and parents) if needed.
+  createDirectoryIfMissing True dst
+  -- List all items in the source directory.
+  entries <- listDirectory src
+  -- Process each entry.
+  forM_ entries $ \entry -> do
+    let srcPath = src </> entry
+        dstPath = dst </> entry
+    isDir <- doesDirectoryExist srcPath
+    if isDir
+      then copyDir srcPath dstPath  -- recursively copy subdirectories
+      else copyFile srcPath dstPath   -- copy individual files
+
+-- Usage:
+--
+
 configureContext :: E2EConfig -> (Context -> IO ()) -> IO ()
-configureContext E2EConfig{walletDbDir,nodeDbDir,nodeDir,preprodMnemonics} action = do
-    cwd <- getCurrentDirectory
-    let nodeSocket = if isWindows
-            then "\\\\.\\pipe\\socket"
-            else cwd </> nodeDbDir </> "node.socket"
+configureContext E2EConfig{preprodMnemonics} action = do
+    -- TODO only set if unset
+    setEnv "GENESIS_VERIFICATION_KEY" "5b3132372c37332c3132342c3136312c362c3133372c3133312c3231332c3230372c3131372c3139382c38352c3137362c3139392c3136322c3234312c36382c3132332c3131392c3134352c31332c3233322c3234332c34392c3232392c322c3234392c3230352c3230352c33392c3233352c34345d"
+    setEnv "AGGREGATOR_ENDPOINT" "https://aggregator.release-preprod.api.mithril.network/aggregator"
+
     withConfigDir $ \configDir -> do
+        downloadLatestSnapshot configDir =<< downloadMithril
+        --copyDir ("db" </> "db") (configDir </> "db")
+
+        let nodeSocket = if isWindows
+                then "\\\\.\\pipe\\socket"
+                else "node.socket"
         let nodeConfig =
                 CardanoNodeConfig
-                    { nodeDir = cwd </> nodeDir
-                    , nodeConfigFile = cwd </> configDir </> "config.json"
-                    , nodeTopologyFile = cwd </> configDir </> "topology.json"
-                    , nodeDatabaseDir = cwd </> nodeDbDir
+                    { nodeDir = configDir
+                    , nodeConfigFile = configDir </> "config.json"
+                    , nodeTopologyFile = configDir </> "topology.json"
+                    , nodeDatabaseDir = configDir </> "db"
                     , nodeDlgCertFile = Nothing
                     , nodeSignKeyFile = Nothing
                     , nodeOpCertFile = Nothing
@@ -189,11 +208,10 @@ configureContext E2EConfig{walletDbDir,nodeDbDir,nodeDir,preprodMnemonics} actio
             let walletConfig =
                     CardanoWalletConfig
                         { walletPort = 8090
-                        , walletDatabaseDir = walletDbDir
-                        , walletNetwork = Launcher.Testnet $
-                            configDir </> "byron-genesis.json"
+                        , walletDatabaseDir = "wallet-db"
+                        , walletNetwork = Launcher.Testnet "byron-genesis.json"
                         , executable = Nothing
-                        , workingDir = Nothing
+                        , workingDir = Just configDir
                         , extraArgs = []
                         }
             withCardanoWallet nullTracer node walletConfig $ \wallet -> do
@@ -247,7 +265,7 @@ configureContext E2EConfig{walletDbDir,nodeDbDir,nodeDir,preprodMnemonics} actio
 -- compile-time, and tweaked for less verbose logs.
 withConfigDir :: (FilePath -> IO a) -> IO a
 withConfigDir action = liftIO $
-  withSystemTempDirectory "e2e-node-configs" $ \tmpDir -> do
+  withSystemTempDirectory "e2e" $ \tmpDir -> do
     -- Write the embedded config dir to the temporary dir
     forM_ embeddedConfigs $ \(relPath, content) -> do
       let dest = tmpDir </> relPath

lib/launcher/cardano-wallet-launcher.cabal

@@ -32,10 +32,12 @@ library
       base
     , bytestring
     , contra-tracer
+    , directory
     , either
     , extra
     , filepath
     , fmt
+    , http-conduit
     , iohk-monitoring
     , network
     , process
@@ -49,6 +51,7 @@ library
     Cardano.Launcher
     Cardano.Launcher.Node
     Cardano.Launcher.Wallet
+    Cardano.Launcher.Mithril
     Cardano.Startup
     Data.MaybeK
   if os(windows)

lib/launcher/src/Cardano/Launcher/Mithril.hs

@@ -0,0 +1,83 @@
+{-# LANGUAGE OverloadedStrings #-}
+
+module Cardano.Launcher.Mithril where
+
+import qualified Data.ByteString as BS
+
+import Control.Monad.Extra
+    ( unlessM
+    )
+import Network.HTTP.Simple
+    ( getResponseBody
+    , httpBS
+    , parseRequest
+    )
+import Prelude
+import System.Directory
+    ( doesFileExist
+    )
+import System.Info
+    ( arch
+    , os
+    )
+import System.Process
+    ( callProcess
+    )
+
+-- | Download the latest snapshot node db into /db relative to the supplied dir.
+downloadLatestSnapshot :: FilePath -> FilePath -> IO ()
+downloadLatestSnapshot outputParentDir mithril = do
+    callProcess mithril ["cdb", "download", "latest", "--download-dir", outputParentDir]
+
+-- | Downloads the latest Mithril release package,
+-- extracts it, and returns the path to the mithril client executable.
+downloadMithril :: IO FilePath
+downloadMithril = do
+    putStrLn $ "Downloading " <> mithrilPackage <> " from " <> downloadUrl
+    req <- parseRequest downloadUrl
+    response <- httpBS req
+    BS.writeFile mithrilPackage (getResponseBody response)
+    putStrLn $ "Downloaded " <> mithrilPackage
+
+    -- Extract the gz archive using 7z.
+    putStrLn $ "Extracting " <> mithrilPackage <> " using 7z..."
+    callProcess "7z" ["x", mithrilPackage]
+
+    -- Extract the tar archive.
+    putStrLn $ "Extracting " <> mithrilTar <> " using tar..."
+    callProcess "tar" ["xf", mithrilTar] -- TODO: skip interactive overwite check
+
+    let clientPath = "mithril-client" <> if isWindows then ".exe" else ""
+    unlessM (doesFileExist clientPath) $
+        fail $ unwords
+            [ "downloadLatest: didn't find"
+            , clientPath
+            , "in mithril archive"
+            ]
+
+    putStrLn $ "Mithril client available at: " <> clientPath
+    return clientPath
+
+  where
+    isWindows = os == "mingw32"
+
+    -- Define the platform and version.
+    platform = osTag <> "-" <> osArch
+      where
+
+        osTag :: String
+        osTag = case os of
+          "darwin" -> "macos"
+          "mingw32" -> "windows"
+          other -> other
+
+        osArch :: String
+        osArch = case arch of
+            "x86_64" -> "x64"
+            other -> other
+
+    version       = "2450.0"
+    mithrilTar    = "mithril-" <> version <> "-" <> platform <> ".tar"
+    mithrilPackage = mithrilTar <> ".gz"
+    downloadUrl   = "https://github.com/input-output-hk/mithril/releases/download/"
+                   <> version <> "/" <> mithrilPackage

scripts/buildkite/main/windows-e2e.bat

@@ -1,34 +1,7 @@
 set PATH=%PATH%;C:\Users\hal\AppData\Local\Microsoft\WinGet\Links
 
-REM ------------- mithril -------------
-
-SET mithril-tar=mithril-2450.0-windows-x64.tar
-echo %mithril-tar%
-SET mithril-package=%mithril-tar%.gz
-echo %mithril-package%
-wget https://github.com/input-output-hk/mithril/releases/download/2450.0/%mithril-package%
-7z x .\%mithril-package%
-tar xf .\%mithril-tar%
-.\mithril-client.exe --version
-for /f "delims=" %%i in ('.\mithril-client.exe cdb snapshot list --json ^| jq -r .[0].digest') do set digest=%%i
-echo %digest%
-.\mithril-client.exe cdb download --download-dir . %digest%
-REM delete the old db folder
-REM this is stupid but will do for now
-mkdir %NODE_DB_DIR%
-rd /q /s %NODE_DB_DIR%
-REM move the new db folder entirely into the old db folder
-move .\db %NODE_DB_DIR%
-ls %NODE_DB_DIR%
-
-REM ------------- ruby tests -------------
-
 cd test\e2e
 call bundle install
 call bundle exec rake get_latest_windows_tests[%BUILDKITE_BRANCH%,bins,any,latest]
-cd ..\..
-
-echo %NODE_DB_DIR%
-echo %WALLET_DB_DIR%
-test\e2e\bins\cardano-wallet-integration-test-e2e.exe
+bins\cardano-wallet-integration-test-e2e.exe
 exit /b %ERRORLEVEL%