Use Haskell to configure mithril in e2e tests

.buildkite/pipeline.yml

@@ -588,11 +588,6 @@ steps:
       agents:
         system: ${windows}
       env:
-        NODE_DB_DIR: "test\\e2e\\state\\node\\db"
-        NODE_DIR: "test\\e2e\\state\\node"
-        WALLET_DB_DIR: "test\\e2e\\state\\wallet"
-        AGGREGATOR_ENDPOINT: "${PREPROD_AGGREGATOR_ENDPOINT}"
-        GENESIS_VERIFICATION_KEY: "${PREPROD_GENESIS_VERIFICATION_KEY}"
       concurrency: 1
       concurrency_group: 'windows-tests'
 

justfile

@@ -74,11 +74,7 @@ unit-tests-cabal:
 
 # run cardano-wallet-integration:e2e suite against the preprod network
 e2e:
-  # ugly env workaround as the default values work with cabal but not nix
-  WALLET_DB_DIR=test/e2e/state/wallet \
-  NODE_DIR=test/e2e/state/node \
-  NODE_DB_DIR=test/e2e/state/node/db \
-  nix shell '.#cardano-node' '.#cardano-wallet' '.#e2e' -c e2e
+  nix shell '.#cardano-node' '.#cardano-wallet' '.#e2e' nixpkgs#gnutar nixpkgs#p7zip -c e2e
 
 add_missing_json_goldens:
     CREATE_MISSING_GOLDEN=1 just unit-tests-cabal-match "JSON"

lib/integration/cardano-wallet-integration.cabal

@@ -239,6 +239,7 @@ test-suite e2e
     , file-embed
     , filepath
     , hspec
+    , network
     , network-uri
     , temporary
     , text

lib/integration/exe/e2e.hs

@@ -19,6 +19,10 @@ import qualified Data.ByteString.Lazy as BL
 import qualified Data.Text as T
 import qualified Test.Integration.Scenario.Preprod as Preprod
 
+import Cardano.Launcher.Mithril
+    ( downloadLatestSnapshot
+    , downloadMithril
+    )
 import Cardano.Launcher.Node
     ( CardanoNodeConfig (..)
     , isWindows
@@ -46,6 +50,7 @@ import Cardano.Wallet.Unsafe
     )
 import Control.Monad
     ( forM_
+    , when
     )
 import Control.Monad.IO.Class
     ( liftIO
@@ -60,22 +65,26 @@ import Data.FileEmbed
     )
 import Data.Maybe
     ( fromMaybe
+    , isNothing
     )
 import Data.MaybeK
     ( MaybeK (..)
     )
 import Main.Utf8
     ( withUtf8
     )
+import Network.Socket
+    ( PortNumber
+    )
 import Network.URI
     ( parseURI
     )
 import System.Directory
     ( createDirectoryIfMissing
-    , getCurrentDirectory
     )
 import System.Environment
     ( lookupEnv
+    , setEnv
     )
 import System.FilePath
     ( takeDirectory
@@ -102,31 +111,28 @@ import Test.Integration.Framework.Setup
 import Test.Utils.Paths
     ( getTestDataPath
     )
+import Text.Read
+    ( readMaybe
+    )
 
 -- ENV configuration -----------------------------------------------------------
 
 data E2EConfig = E2EConfig
-    { walletDbDir :: FilePath
-    , nodeDir :: FilePath
-    , nodeDbDir :: FilePath
-    , preprodMnemonics :: [SomeMnemonic]
+    { preprodMnemonics :: [SomeMnemonic]
+    , alreadyRunningWallet :: Maybe PortNumber
     }
 
 getConfig :: IO E2EConfig
 getConfig = do
-    walletDbDir <- fromMaybe defaultWalletDbDir <$> lookupEnv "WALLET_DB_DIR"
-    nodeDbDir <- fromMaybe defaultNodeDbDir <$> lookupEnv "NODE_DB_DIR"
-    nodeDir <- fromMaybe defaultNodeDir <$> lookupEnv "NODE_DIR"
     preprodMnemonics <- getPreprodMnemonics
-    pure $ E2EConfig {..}
-  where
-    -- Probably too fine grained control with all these settings, but works for now
-    defaultNodeDbDir   = repoRoot </> "test" </> "e2e" </> "state" </> "node" </> "db"
-    defaultNodeDir   = repoRoot </> "test" </> "e2e" </> "state" </> "node"
-    defaultWalletDbDir = repoRoot </> "test" </> "e2e" </> "state" </> "wallet"
+    alreadyRunningWallet <- (readMaybe =<<) <$> lookupEnv "HAL_E2E_ALREADY_RUNNING_WALLET_PORT"
 
-    repoRoot = ".." </> ".." -- works when run with 'cabal test'
+    -- Needed for mithril-client
+    setEnvIfMissing "GENESIS_VERIFICATION_KEY" mithrilPreprodGenesisVerificationKey
+    setEnvIfMissing "AGGREGATOR_ENDPOINT" mithrilPreprodAggregatorEndpoint
 
+    pure $ E2EConfig {..}
+  where
     getPreprodMnemonics :: IO [SomeMnemonic]
     getPreprodMnemonics
         = map (SomeMnemonic . unsafeMkMnemonic @15 . T.words)
@@ -146,6 +152,11 @@ getConfig = do
             , "buffalo buffalo buffalo buffalo buffalo buffalo buffalo buffalo buffalo buffalo buffalo buffalo buffalo buffalo buffalo"
             ]
 
+    setEnvIfMissing :: String -> String -> IO ()
+    setEnvIfMissing var value = do
+        isUnset <- isNothing <$> lookupEnv var
+        when isUnset $ setEnv var value
+
 -- main ------------------------------------------------------------------------
 
 main :: IO ()
@@ -161,18 +172,24 @@ main = withUtf8 $ do
 -- setup -----------------------------------------------------------------------
 
 configureContext :: E2EConfig -> (Context -> IO ()) -> IO ()
-configureContext E2EConfig{walletDbDir,nodeDbDir,nodeDir,preprodMnemonics} action = do
-    cwd <- getCurrentDirectory
-    let nodeSocket = if isWindows
-            then "\\\\.\\pipe\\socket"
-            else cwd </> nodeDbDir </> "node.socket"
-    withConfigDir $ \configDir -> do
+configureContext (E2EConfig preprodMnemonics alreadyRunningWallet) action =
+    case alreadyRunningWallet of
+        Just port -> action =<< contextFromWalletPort port
+        Nothing -> launchNodeAndWalletViaMithril
+  where
+    launchNodeAndWalletViaMithril :: IO ()
+    launchNodeAndWalletViaMithril = withConfigDir $ \dir -> do
+        putStrLn "~~~ Downloading latest mithril snapshot"
+        downloadLatestSnapshot dir =<< downloadMithril dir
+        let nodeSocket = if isWindows
+                then "\\\\.\\pipe\\socket"
+                else "node.socket"
         let nodeConfig =
                 CardanoNodeConfig
-                    { nodeDir = cwd </> nodeDir
-                    , nodeConfigFile = cwd </> configDir </> "config.json"
-                    , nodeTopologyFile = cwd </> configDir </> "topology.json"
-                    , nodeDatabaseDir = cwd </> nodeDbDir
+                    { nodeDir = dir
+                    , nodeConfigFile = dir </> "config.json"
+                    , nodeTopologyFile = dir </> "topology.json"
+                    , nodeDatabaseDir = dir </> "db"
                     , nodeDlgCertFile = Nothing
                     , nodeSignKeyFile = Nothing
                     , nodeOpCertFile = Nothing
@@ -189,28 +206,28 @@ configureContext E2EConfig{walletDbDir,nodeDbDir,nodeDir,preprodMnemonics} actio
             let walletConfig =
                     CardanoWalletConfig
                         { walletPort = 8090
-                        , walletDatabaseDir = walletDbDir
-                        , walletNetwork = Launcher.Testnet $
-                            configDir </> "byron-genesis.json"
+                        , walletDatabaseDir = "wallet-db"
+                        , walletNetwork = Launcher.Testnet "byron-genesis.json"
                         , executable = Nothing
-                        , workingDir = Nothing
+                        , workingDir = Just dir
                         , extraArgs = []
                         }
-            withCardanoWallet nullTracer node walletConfig $ \wallet -> do
-                action =<< contextFromNetwork wallet
-  where
-    contextFromNetwork :: CardanoWalletConn -> IO Context
-    contextFromNetwork (CardanoWalletConn port _) = do
+            withCardanoWallet nullTracer node walletConfig
+                $ \(CardanoWalletConn walletPort _)  -> do
+                    action =<< contextFromWalletPort walletPort
+
+    contextFromWalletPort :: PortNumber -> IO Context
+    contextFromWalletPort walletPort = do
         manager <- httpManager
-        let mUri = parseURI $ "http://localhost:" <> show port <> "/"
+        let mUri = parseURI ("http://localhost:" <> show walletPort <> "/")
         let baseUri = case mUri of
               Just uri -> uri
               Nothing  -> error "Invalid URI"
 
         let ctx = Context
                 { _manager = (baseUri, manager)
                 , _walletPort =
-                    Port $ fromIntegral port
+                    Port $ fromIntegral walletPort
                 , _mainEra =
                     ApiConway
                 , _faucet =
@@ -237,7 +254,7 @@ configureContext E2EConfig{walletDbDir,nodeDbDir,nodeDir,preprodMnemonics} actio
     setupAsBuildkiteSections WaitingForNodeConnection = "--- Waiting for node connection"
     setupAsBuildkiteSections CreatingWallets          = "--- Creating wallets"
     setupAsBuildkiteSections WaitingForWalletsToSync  = "--- Syncing wallets"
-    setupAsBuildkiteSections PreprodSetupReady        = "--- Running tests"
+    setupAsBuildkiteSections PreprodSetupReady        = "+++ Running tests"
 
 -- node configs ----------------------------------------------------------------
 
@@ -247,7 +264,7 @@ configureContext E2EConfig{walletDbDir,nodeDbDir,nodeDir,preprodMnemonics} actio
 -- compile-time, and tweaked for less verbose logs.
 withConfigDir :: (FilePath -> IO a) -> IO a
 withConfigDir action = liftIO $
-  withSystemTempDirectory "e2e-node-configs" $ \tmpDir -> do
+  withSystemTempDirectory "e2e" $ \tmpDir -> do
     -- Write the embedded config dir to the temporary dir
     forM_ embeddedConfigs $ \(relPath, content) -> do
       let dest = tmpDir </> relPath
@@ -288,3 +305,9 @@ withConfigDir action = liftIO $
 embeddedConfigs :: [(FilePath, BS.ByteString)]
 embeddedConfigs = $(embedDir
     $(getTestDataPath ("configs" </> "cardano" </> "preprod")))
+
+mithrilPreprodGenesisVerificationKey :: String
+mithrilPreprodGenesisVerificationKey = "5b3132372c37332c3132342c3136312c362c3133372c3133312c3231332c3230372c3131372c3139382c38352c3137362c3139392c3136322c3234312c36382c3132332c3131392c3134352c31332c3233322c3234332c34392c3232392c322c3234392c3230352c3230352c33392c3233352c34345d"
+
+mithrilPreprodAggregatorEndpoint :: String
+mithrilPreprodAggregatorEndpoint = "https://aggregator.release-preprod.api.mithril.network/aggregator"

lib/launcher/cardano-wallet-launcher.cabal

@@ -32,10 +32,12 @@ library
       base
     , bytestring
     , contra-tracer
+    , directory
     , either
     , extra
     , filepath
     , fmt
+    , http-conduit
     , iohk-monitoring
     , network
     , process
@@ -49,6 +51,7 @@ library
     Cardano.Launcher
     Cardano.Launcher.Node
     Cardano.Launcher.Wallet
+    Cardano.Launcher.Mithril
     Cardano.Startup
     Data.MaybeK
   if os(windows)

lib/launcher/src/Cardano/Launcher/Mithril.hs

@@ -0,0 +1,96 @@
+{-# LANGUAGE OverloadedStrings #-}
+
+module Cardano.Launcher.Mithril
+    ( downloadLatestSnapshot
+    , downloadMithril
+    , MithrilExePath (..)
+    )
+    where
+
+import Prelude
+
+import qualified Data.ByteString as BS
+
+import Control.Monad.Extra
+    ( unlessM
+    )
+import Network.HTTP.Simple
+    ( getResponseBody
+    , httpBS
+    , parseRequest
+    )
+import System.Directory
+    ( doesFileExist
+    , withCurrentDirectory
+    )
+import System.FilePath
+    ( (</>)
+    )
+import System.Info
+    ( arch
+    , os
+    )
+import System.Process
+    ( callProcess
+    )
+
+newtype MithrilExePath = MithrilExePath { mithrilExePath :: FilePath }
+
+-- | Download the latest snapshot node db into /db relative to the supplied dir.
+downloadLatestSnapshot :: FilePath -> MithrilExePath -> IO ()
+downloadLatestSnapshot outputParentDir (MithrilExePath mithril) = do
+    callProcess mithril ["cdb", "download", "latest", "--download-dir", outputParentDir]
+
+-- | Downloads the latest Mithril release package,
+-- extracts it, and returns the path to the mithril client executable.
+--
+-- May interactively ask how to handle conflicts if items in the supplied
+-- working directory conflict with items in the mithril release archive.
+downloadMithril :: FilePath -> IO MithrilExePath
+downloadMithril workingDir = withCurrentDirectory workingDir $ do
+    putStrLn $ "Downloading " <> mithrilPackage <> " from " <> downloadUrl
+    req <- parseRequest downloadUrl
+    response <- httpBS req
+    BS.writeFile mithrilPackage (getResponseBody response)
+    putStrLn $ "Downloaded " <> mithrilPackage
+
+    -- Extract the gz archive using 7z.
+    putStrLn $ "Extracting " <> mithrilPackage <> " using 7z..."
+    callProcess "7z" ["x", mithrilPackage]
+
+    -- Extract the tar archive.
+    putStrLn $ "Extracting " <> mithrilTar <> " using tar..."
+    callProcess "tar" ["xf", mithrilTar]
+
+    let clientPath = workingDir </> ("mithril-client" <> if isWindows then ".exe" else "")
+    unlessM (doesFileExist clientPath) $
+        fail $ unwords
+            [ "downloadLatest: didn't find"
+            , clientPath
+            , "in mithril archive"
+            ]
+
+    putStrLn $ "Mithril client available at: " <> clientPath
+    return $ MithrilExePath clientPath
+  where
+    isWindows = os == "mingw32"
+
+    -- Define the platform and version.
+    platform = osTag <> "-" <> osArch
+      where
+        osTag :: String
+        osTag = case os of
+          "darwin" -> "macos"
+          "mingw32" -> "windows"
+          other -> other
+
+        osArch :: String
+        osArch = case arch of
+            "x86_64" -> "x64"
+            other -> other
+
+    version       = "2450.0"
+    mithrilTar    = "mithril-" <> version <> "-" <> platform <> ".tar"
+    mithrilPackage = mithrilTar <> ".gz"
+    downloadUrl   = "https://github.com/input-output-hk/mithril/releases/download/"
+                   <> version <> "/" <> mithrilPackage

scripts/buildkite/main/windows-e2e.bat

@@ -1,34 +1,7 @@
 set PATH=%PATH%;C:\Users\hal\AppData\Local\Microsoft\WinGet\Links
 
-REM ------------- mithril -------------
-
-SET mithril-tar=mithril-2450.0-windows-x64.tar
-echo %mithril-tar%
-SET mithril-package=%mithril-tar%.gz
-echo %mithril-package%
-wget https://github.com/input-output-hk/mithril/releases/download/2450.0/%mithril-package%
-7z x .\%mithril-package%
-tar xf .\%mithril-tar%
-.\mithril-client.exe --version
-for /f "delims=" %%i in ('.\mithril-client.exe cdb snapshot list --json ^| jq -r .[0].digest') do set digest=%%i
-echo %digest%
-.\mithril-client.exe cdb download --download-dir . %digest%
-REM delete the old db folder
-REM this is stupid but will do for now
-mkdir %NODE_DB_DIR%
-rd /q /s %NODE_DB_DIR%
-REM move the new db folder entirely into the old db folder
-move .\db %NODE_DB_DIR%
-ls %NODE_DB_DIR%
-
-REM ------------- ruby tests -------------
-
 cd test\e2e
 call bundle install
 call bundle exec rake get_latest_windows_tests[%BUILDKITE_BRANCH%,bins,any,latest]
-cd ..\..
-
-echo %NODE_DB_DIR%
-echo %WALLET_DB_DIR%
-test\e2e\bins\cardano-wallet-integration-test-e2e.exe
+bins\cardano-wallet-integration-test-e2e.exe
 exit /b %ERRORLEVEL%