From 64491e09878e88aafea11946dba5df9d85c2021f Mon Sep 17 00:00:00 2001 From: Michael MacDonald Date: Tue, 17 Dec 2024 22:39:16 +0000 Subject: [PATCH] DAOS-10028 client: Add Go bindings for libdaos (Pool) Start the work of converting the raw cgo in the daos tool into proper Go bindings for libdaos. This patch covers pool functionality and adds some new infrastructure common to both pools and containers. Features: daos_cmd Required-githooks: true Signed-off-by: Michael MacDonald --- src/control/cmd/daos/attribute.go | 195 +-- src/control/cmd/daos/container.go | 40 +- src/control/cmd/daos/health.go | 35 +- src/control/cmd/daos/health_test.go | 110 +- src/control/cmd/daos/pool.go | 521 +------- src/control/cmd/daos/pool_test.go | 476 ++++++++ src/control/cmd/daos/pretty/pool.go | 35 +- src/control/cmd/daos/stubbed.go | 3 + src/control/cmd/daos/util.go | 9 + src/control/cmd/daos/util_test.go | 60 + src/control/cmd/dmg/pool.go | 16 +- src/control/common/test/utils.go | 9 + src/control/lib/daos/api/api.go | 5 +- src/control/lib/daos/api/attribute.go | 268 +++++ src/control/lib/daos/api/errors.go | 52 +- src/control/lib/daos/api/handle.go | 92 ++ src/control/lib/daos/api/libdaos.go | 55 + .../lib/daos/api/libdaos_attr_stubs.go | 175 +++ .../lib/daos/api/libdaos_pool_stubs.go | 450 +++++++ src/control/lib/daos/api/libdaos_stubs.go | 26 + src/control/lib/daos/api/pool.go | 675 +++++++++++ src/control/lib/daos/api/pool_test.go | 1057 +++++++++++++++++ src/control/lib/daos/api/util.go | 117 ++ src/control/lib/daos/api/util.h | 101 ++ src/control/lib/daos/attribute.go | 39 + src/control/lib/daos/pool.go | 10 +- src/control/lib/daos/util.go | 18 + src/control/lib/ranklist/ranklist.go | 3 +- 28 files changed, 4039 insertions(+), 613 deletions(-) create mode 100644 src/control/cmd/daos/pool_test.go create mode 100644 src/control/cmd/daos/util_test.go create mode 100644 src/control/lib/daos/api/attribute.go create mode 100644 src/control/lib/daos/api/handle.go create mode 100644 src/control/lib/daos/api/libdaos_attr_stubs.go create mode 100644 src/control/lib/daos/api/libdaos_pool_stubs.go create mode 100644 src/control/lib/daos/api/pool.go create mode 100644 src/control/lib/daos/api/pool_test.go create mode 100644 src/control/lib/daos/api/util.go create mode 100644 src/control/lib/daos/api/util.h create mode 100644 src/control/lib/daos/attribute.go create mode 100644 src/control/lib/daos/util.go diff --git a/src/control/cmd/daos/attribute.go b/src/control/cmd/daos/attribute.go index 9514dad70f04..677d2beaebe8 100644 --- a/src/control/cmd/daos/attribute.go +++ b/src/control/cmd/daos/attribute.go @@ -1,5 +1,6 @@ // -// (C) Copyright 2018-2021 Intel Corporation. +// (C) Copyright 2018-2024 Intel Corporation. +// (C) Copyright 2025 Google LLC // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -7,13 +8,17 @@ package main import ( + "context" "fmt" - "io" + "strings" "unsafe" "github.com/pkg/errors" - "github.com/daos-stack/daos/src/control/lib/txtfmt" + "github.com/daos-stack/daos/src/control/cmd/daos/pretty" + "github.com/daos-stack/daos/src/control/common/cmdutil" + "github.com/daos-stack/daos/src/control/lib/daos" + "github.com/daos-stack/daos/src/control/logging" ) /* @@ -21,75 +26,137 @@ import ( */ import "C" +type attrType int + +const ( + poolAttr attrType = iota + contAttr +) + +func (at attrType) String() string { + switch at { + case poolAttr: + return "pool" + case contAttr: + return "container" + default: + return "unknown" + } +} + type ( - attribute struct { - Name string `json:"name"` - Value []byte `json:"value,omitempty"` + attrCmd interface { + MustLogCtx() context.Context + cmdutil.JSONOutputter + logging.Logger } - attrList []*attribute + attrListerGetter interface { + ListAttributes(context.Context) ([]string, error) + GetAttributes(context.Context, ...string) (daos.AttributeList, error) + } + + attrSetter interface { + SetAttributes(context.Context, ...*daos.Attribute) error + } + + attrDeleter interface { + DeleteAttributes(context.Context, ...string) error + } ) -func (al attrList) asMap() map[string][]byte { - m := make(map[string][]byte) - for _, a := range al { - m[a.Name] = a.Value +func listAttributes(cmd attrCmd, alg attrListerGetter, at attrType, id string, verbose bool) error { + var attrs daos.AttributeList + if !verbose { + attrNames, err := alg.ListAttributes(cmd.MustLogCtx()) + if err != nil { + return errors.Wrapf(err, "failed to list attributes for %s %s", at, id) + } + attrs = attrListFromNames(attrNames) + } else { + var err error + attrs, err = alg.GetAttributes(cmd.MustLogCtx()) + if err != nil { + return errors.Wrapf(err, "failed to get attributes for %s %s", at, id) + } } - return m -} -func (al attrList) asList() []string { - names := make([]string, len(al)) - for i, a := range al { - names[i] = a.Name + if cmd.JSONOutputEnabled() { + if verbose { + return cmd.OutputJSON(attrs.AsMap(), nil) + } + return cmd.OutputJSON(attrs.AsList(), nil) } - return names + + var bld strings.Builder + title := fmt.Sprintf("Attributes for %s %s:", at, id) + pretty.PrintAttributes(&bld, title, attrs...) + + cmd.Info(bld.String()) + + return nil } -func printAttributes(out io.Writer, header string, attrs ...*attribute) { - fmt.Fprintf(out, "%s\n", header) +func getAttributes(cmd attrCmd, alg attrListerGetter, at attrType, id string, names ...string) error { + attrs, err := alg.GetAttributes(cmd.MustLogCtx(), names...) + if err != nil { + return errors.Wrapf(err, "failed to get attributes for %s %s", at, id) + } - if len(attrs) == 0 { - fmt.Fprintln(out, " No attributes found.") - return - } - - nameTitle := "Name" - valueTitle := "Value" - titles := []string{nameTitle} - - table := []txtfmt.TableRow{} - for _, attr := range attrs { - row := txtfmt.TableRow{} - row[nameTitle] = attr.Name - if len(attr.Value) != 0 { - row[valueTitle] = string(attr.Value) - if len(titles) == 1 { - titles = append(titles, valueTitle) - } + if cmd.JSONOutputEnabled() { + // Maintain compatibility with older behavior. + if len(names) == 1 && len(attrs) == 1 { + return cmd.OutputJSON(attrs[0], nil) } - table = append(table, row) + return cmd.OutputJSON(attrs, nil) } - tf := txtfmt.NewTableFormatter(titles...) - tf.InitWriter(out) - tf.Format(table) + var bld strings.Builder + title := fmt.Sprintf("Attributes for %s %s:", at, id) + pretty.PrintAttributes(&bld, title, attrs...) + + cmd.Info(bld.String()) + + return nil } -type attrType int +func setAttributes(cmd attrCmd, as attrSetter, at attrType, id string, attrMap map[string]string) error { + if len(attrMap) == 0 { + return errors.New("attribute name and value are required") + } -const ( - poolAttr attrType = iota - contAttr -) + attrs := make(daos.AttributeList, 0, len(attrMap)) + for key, val := range attrMap { + attrs = append(attrs, &daos.Attribute{ + Name: key, + Value: []byte(val), + }) + } -func listDaosAttributes(hdl C.daos_handle_t, at attrType, verbose bool) (attrList, error) { + if err := as.SetAttributes(cmd.MustLogCtx(), attrs...); err != nil { + return errors.Wrapf(err, "failed to set attributes on %s %s", at, id) + } + cmd.Infof("Attributes successfully set on %s %q", at, id) + + return nil +} + +func delAttributes(cmd attrCmd, ad attrDeleter, at attrType, id string, names ...string) error { + attrsString := strings.Join(names, ",") + if err := ad.DeleteAttributes(cmd.MustLogCtx(), names...); err != nil { + return errors.Wrapf(err, "failed to delete attributes %s on %s %s", attrsString, at, id) + } + cmd.Infof("Attribute(s) %s successfully deleted on %s %q", attrsString, at, id) + + return nil +} + +// NB: These will be removed in the next patch, which adds the container APIs. +func listDaosAttributes(hdl C.daos_handle_t, at attrType, verbose bool) (daos.AttributeList, error) { var rc C.int expectedSize, totalSize := C.size_t(0), C.size_t(0) switch at { - case poolAttr: - rc = C.daos_pool_list_attr(hdl, nil, &totalSize, nil) case contAttr: rc = C.daos_cont_list_attr(hdl, nil, &totalSize, nil) default: @@ -109,8 +176,6 @@ func listDaosAttributes(hdl C.daos_handle_t, at attrType, verbose bool) (attrLis defer C.free(buf) switch at { - case poolAttr: - rc = C.daos_pool_list_attr(hdl, (*C.char)(buf), &totalSize, nil) case contAttr: rc = C.daos_cont_list_attr(hdl, (*C.char)(buf), &totalSize, nil) default: @@ -130,9 +195,9 @@ func listDaosAttributes(hdl C.daos_handle_t, at attrType, verbose bool) (attrLis return getDaosAttributes(hdl, at, attrNames) } - attrs := make([]*attribute, len(attrNames)) + attrs := make(daos.AttributeList, len(attrNames)) for i, name := range attrNames { - attrs[i] = &attribute{Name: name} + attrs[i] = &daos.Attribute{Name: name} } return attrs, nil @@ -141,7 +206,7 @@ func listDaosAttributes(hdl C.daos_handle_t, at attrType, verbose bool) (attrLis // getDaosAttributes fetches the values for the given list of attribute names. // Uses the bulk attribute fetch API to minimize roundtrips. -func getDaosAttributes(hdl C.daos_handle_t, at attrType, names []string) (attrList, error) { +func getDaosAttributes(hdl C.daos_handle_t, at attrType, names []string) (daos.AttributeList, error) { if len(names) == 0 { attrList, err := listDaosAttributes(hdl, at, false) if err != nil { @@ -171,8 +236,6 @@ func getDaosAttributes(hdl C.daos_handle_t, at attrType, names []string) (attrLi attrSizes := make([]C.size_t, numAttr) var rc C.int switch at { - case poolAttr: - rc = C.daos_pool_get_attr(hdl, C.int(numAttr), &attrNames[0], nil, &attrSizes[0], nil) case contAttr: rc = C.daos_cont_get_attr(hdl, C.int(numAttr), &attrNames[0], nil, &attrSizes[0], nil) default: @@ -199,8 +262,6 @@ func getDaosAttributes(hdl C.daos_handle_t, at attrType, names []string) (attrLi // Do the actual fetch of all values in one go. switch at { - case poolAttr: - rc = C.daos_pool_get_attr(hdl, C.int(numAttr), &attrNames[0], &attrValues[0], &attrSizes[0], nil) case contAttr: rc = C.daos_cont_get_attr(hdl, C.int(numAttr), &attrNames[0], &attrValues[0], &attrSizes[0], nil) default: @@ -214,9 +275,9 @@ func getDaosAttributes(hdl C.daos_handle_t, at attrType, names []string) (attrLi // Note that we are copying the values into Go-managed byte slices // for safety and simplicity so that we can free the C memory as soon // as this function exits. - attrs := make([]*attribute, numAttr) + attrs := make(daos.AttributeList, numAttr) for i, name := range names { - attrs[i] = &attribute{ + attrs[i] = &daos.Attribute{ Name: name, Value: C.GoBytes(attrValues[i], C.int(attrSizes[i])), } @@ -228,7 +289,7 @@ func getDaosAttributes(hdl C.daos_handle_t, at attrType, names []string) (attrLi // getDaosAttribute fetches the value for the given attribute name. // NB: For operations involving multiple attributes, the getDaosAttributes() // function is preferred for efficiency. -func getDaosAttribute(hdl C.daos_handle_t, at attrType, name string) (*attribute, error) { +func getDaosAttribute(hdl C.daos_handle_t, at attrType, name string) (*daos.Attribute, error) { attrs, err := getDaosAttributes(hdl, at, []string{name}) if err != nil { return nil, err @@ -241,7 +302,7 @@ func getDaosAttribute(hdl C.daos_handle_t, at attrType, name string) (*attribute // setDaosAttributes sets the values for the given list of attribute names. // Uses the bulk attribute set API to minimize roundtrips. -func setDaosAttributes(hdl C.daos_handle_t, at attrType, attrs attrList) error { +func setDaosAttributes(hdl C.daos_handle_t, at attrType, attrs daos.AttributeList) error { if len(attrs) == 0 { return nil } @@ -277,8 +338,6 @@ func setDaosAttributes(hdl C.daos_handle_t, at attrType, attrs attrList) error { attrCount := C.int(len(attrs)) var rc C.int switch at { - case poolAttr: - rc = C.daos_pool_set_attr(hdl, attrCount, &attrNames[0], &valBufs[0], &valSizes[0], nil) case contAttr: rc = C.daos_cont_set_attr(hdl, attrCount, &attrNames[0], &valBufs[0], &valSizes[0], nil) default: @@ -291,12 +350,12 @@ func setDaosAttributes(hdl C.daos_handle_t, at attrType, attrs attrList) error { // setDaosAttribute sets the value for the given attribute name. // NB: For operations involving multiple attributes, the setDaosAttributes() // function is preferred for efficiency. -func setDaosAttribute(hdl C.daos_handle_t, at attrType, attr *attribute) error { +func setDaosAttribute(hdl C.daos_handle_t, at attrType, attr *daos.Attribute) error { if attr == nil { return errors.Errorf("nil %T", attr) } - return setDaosAttributes(hdl, at, attrList{attr}) + return setDaosAttributes(hdl, at, daos.AttributeList{attr}) } func delDaosAttribute(hdl C.daos_handle_t, at attrType, name string) error { @@ -305,8 +364,6 @@ func delDaosAttribute(hdl C.daos_handle_t, at attrType, name string) error { var rc C.int switch at { - case poolAttr: - rc = C.daos_pool_del_attr(hdl, 1, &attrName, nil) case contAttr: rc = C.daos_cont_del_attr(hdl, 1, &attrName, nil) default: diff --git a/src/control/cmd/daos/container.go b/src/control/cmd/daos/container.go index e69c78a370ee..8d8980e8002e 100644 --- a/src/control/cmd/daos/container.go +++ b/src/control/cmd/daos/container.go @@ -1,5 +1,6 @@ // // (C) Copyright 2021-2024 Intel Corporation. +// (C) Copyright 2025 Google LLC // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -19,6 +20,7 @@ import ( "github.com/jessevdk/go-flags" "github.com/pkg/errors" + "github.com/daos-stack/daos/src/control/cmd/daos/pretty" "github.com/daos-stack/daos/src/control/lib/daos" "github.com/daos-stack/daos/src/control/lib/txtfmt" "github.com/daos-stack/daos/src/control/lib/ui" @@ -221,14 +223,14 @@ func queryContainer(poolUUID, contUUID uuid.UUID, poolHandle, contHandle C.daos_ return ci, nil } -func (cmd *containerBaseCmd) connectPool(flags C.uint, ap *C.struct_cmd_args_s) (func(), error) { +func (cmd *containerBaseCmd) connectPool(flags daos.PoolConnectFlag, ap *C.struct_cmd_args_s) (func(), error) { if err := cmd.poolBaseCmd.connectPool(flags); err != nil { return nil, err } if ap != nil { ap.pool = cmd.cPoolHandle - if err := copyUUID(&ap.p_uuid, cmd.poolUUID); err != nil { + if err := copyUUID(&ap.p_uuid, cmd.pool.UUID()); err != nil { cmd.disconnectPool() return nil, err } @@ -299,7 +301,7 @@ func (cmd *containerCreateCmd) Execute(_ []string) (err error) { cmd.poolBaseCmd.Args.Pool.UUID = pu } - disconnectPool, err := cmd.connectPool(C.DAOS_PC_RW, ap) + disconnectPool, err := cmd.connectPool(daos.PoolConnectFlagReadWrite, ap) if err != nil { return err } @@ -317,7 +319,7 @@ func (cmd *containerCreateCmd) Execute(_ []string) (err error) { defer cmd.closeContainer() var ci *daos.ContainerInfo - ci, err = queryContainer(cmd.poolUUID, cmd.contUUID, cmd.cPoolHandle, cmd.cContHandle) + ci, err = queryContainer(cmd.pool.UUID(), cmd.contUUID, cmd.cPoolHandle, cmd.cContHandle) if err != nil { if errors.Cause(err) != daos.NoPermission { return errors.Wrapf(err, "failed to query new container %s", contID) @@ -327,7 +329,7 @@ func (cmd *containerCreateCmd) Execute(_ []string) (err error) { cmd.Errorf("container %s was created, but query failed", contID) ci = new(daos.ContainerInfo) - ci.PoolUUID = cmd.poolUUID + ci.PoolUUID = cmd.pool.UUID() ci.Type = cmd.Type.String() ci.ContainerUUID = cmd.contUUID ci.ContainerLabel = cmd.Args.Label @@ -406,9 +408,9 @@ func (cmd *containerCreateCmd) contCreate() (string, error) { } if len(cmd.Attrs.ParsedProps) != 0 { - attrs := make(attrList, 0, len(cmd.Attrs.ParsedProps)) + attrs := make(daos.AttributeList, 0, len(cmd.Attrs.ParsedProps)) for key, val := range cmd.Attrs.ParsedProps { - attrs = append(attrs, &attribute{ + attrs = append(attrs, &daos.Attribute{ Name: key, Value: []byte(val), }) @@ -627,7 +629,7 @@ func (cmd *existingContainerCmd) resolveAndConnect(contFlags C.uint, ap *C.struc } var cleanupPool func() - cleanupPool, err = cmd.connectPool(C.DAOS_PC_RO, ap) + cleanupPool, err = cmd.connectPool(daos.PoolConnectFlagReadOnly, ap) if err != nil { return } @@ -651,7 +653,7 @@ func (cmd *existingContainerCmd) resolveAndConnect(contFlags C.uint, ap *C.struc }, nil } -func (cmd *existingContainerCmd) getAttr(name string) (*attribute, error) { +func (cmd *existingContainerCmd) getAttr(name string) (*daos.Attribute, error) { return getDaosAttribute(cmd.cContHandle, contAttr, name) } @@ -731,7 +733,7 @@ func printContainers(out io.Writer, contIDs []*ContainerID) { } func (cmd *containerListCmd) Execute(_ []string) error { - cleanup, err := cmd.resolveAndConnect(C.DAOS_PC_RO, nil) + cleanup, err := cmd.resolveAndConnect(daos.PoolConnectFlagReadOnly, nil) if err != nil { return err } @@ -990,7 +992,7 @@ func (cmd *containerQueryCmd) Execute(_ []string) error { } defer cleanup() - ci, err := queryContainer(cmd.poolUUID, cmd.contUUID, cmd.cPoolHandle, cmd.cContHandle) + ci, err := queryContainer(cmd.pool.UUID(), cmd.contUUID, cmd.cPoolHandle, cmd.cContHandle) if err != nil { return errors.Wrapf(err, "failed to query container %s", @@ -1123,14 +1125,14 @@ func (cmd *containerListAttrsCmd) Execute(args []string) error { if cmd.JSONOutputEnabled() { if cmd.Verbose { - return cmd.OutputJSON(attrs.asMap(), nil) + return cmd.OutputJSON(attrs.AsMap(), nil) } - return cmd.OutputJSON(attrs.asList(), nil) + return cmd.OutputJSON(attrs.AsList(), nil) } var bld strings.Builder title := fmt.Sprintf("Attributes for container %s:", cmd.ContainerID()) - printAttributes(&bld, title, attrs...) + pretty.PrintAttributes(&bld, title, attrs...) cmd.Info(bld.String()) @@ -1208,7 +1210,7 @@ func (cmd *containerGetAttrCmd) Execute(args []string) error { } defer cleanup() - var attrs attrList + var attrs daos.AttributeList if len(cmd.Args.Attrs.ParsedProps) == 0 { attrs, err = listDaosAttributes(cmd.cContHandle, contAttr, true) } else { @@ -1229,7 +1231,7 @@ func (cmd *containerGetAttrCmd) Execute(args []string) error { var bld strings.Builder title := fmt.Sprintf("Attributes for container %s:", cmd.ContainerID()) - printAttributes(&bld, title, attrs...) + pretty.PrintAttributes(&bld, title, attrs...) cmd.Info(bld.String()) @@ -1278,9 +1280,9 @@ func (cmd *containerSetAttrCmd) Execute(args []string) error { } defer cleanup() - attrs := make(attrList, 0, len(cmd.Args.Attrs.ParsedProps)) + attrs := make(daos.AttributeList, 0, len(cmd.Args.Attrs.ParsedProps)) for key, val := range cmd.Args.Attrs.ParsedProps { - attrs = append(attrs, &attribute{ + attrs = append(attrs, &daos.Attribute{ Name: key, Value: []byte(val), }) @@ -1472,7 +1474,7 @@ func (f *ContainerID) Complete(match string) (comps []flags.Completion) { } defer fini() - cleanup, err := pf.resolveAndConnect(C.DAOS_PC_RO, nil) + cleanup, err := pf.resolveAndConnect(daos.PoolConnectFlagReadOnly, nil) if err != nil { return } diff --git a/src/control/cmd/daos/health.go b/src/control/cmd/daos/health.go index 61f1d1df1425..fa71a5b9a08a 100644 --- a/src/control/cmd/daos/health.go +++ b/src/control/cmd/daos/health.go @@ -1,5 +1,6 @@ // // (C) Copyright 2024 Intel Corporation. +// (C) Copyright 2025 Google LLC // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -9,6 +10,7 @@ package main import ( "fmt" "strings" + "unsafe" "github.com/google/uuid" @@ -16,11 +18,17 @@ import ( "github.com/daos-stack/daos/src/control/cmd/daos/pretty" "github.com/daos-stack/daos/src/control/common/cmdutil" "github.com/daos-stack/daos/src/control/lib/daos" + "github.com/daos-stack/daos/src/control/lib/daos/api" "github.com/daos-stack/daos/src/control/lib/ranklist" "github.com/daos-stack/daos/src/control/lib/ui" "github.com/daos-stack/daos/src/control/logging" ) +/* +#include "util.h" +*/ +import "C" + type healthCmds struct { Check healthCheckCmd `command:"check" description:"Perform DAOS system health checks"` NetTest netTestCmd `command:"net-test" description:"Perform non-destructive DAOS networking tests"` @@ -62,6 +70,8 @@ func collectBuildInfo(log logging.Logger, shi *daos.SystemHealthInfo) error { } func (cmd *healthCheckCmd) Execute([]string) error { + ctx := cmd.MustLogCtx() + // TODO (DAOS-10028): Move this logic into the daos package once the API is available. systemHealth := &daos.SystemHealthInfo{ ComponentBuildInfo: make(map[string]daos.ComponentBuild), @@ -72,7 +82,7 @@ func (cmd *healthCheckCmd) Execute([]string) error { return err } - sysInfo, err := cmd.apiProvider.GetSystemInfo(cmd.MustLogCtx()) + sysInfo, err := cmd.apiProvider.GetSystemInfo(ctx) if err != nil { cmd.Errorf("failed to query system information: %v", err) } @@ -80,7 +90,10 @@ func (cmd *healthCheckCmd) Execute([]string) error { cmd.Infof("Checking DAOS system: %s", systemHealth.SystemInfo.Name) - pools, err := getPoolList(cmd.Logger, cmd.SysName, true) + pools, err := api.GetPoolList(ctx, api.GetPoolListReq{ + SysName: cmd.SysName, + Query: true, + }) if err != nil { cmd.Errorf("failed to get pool list: %v", err) } @@ -88,13 +101,18 @@ func (cmd *healthCheckCmd) Execute([]string) error { for _, pool := range pools { systemHealth.Pools[pool.UUID] = pool - poolHdl, _, err := poolConnect(pool.UUID.String(), cmd.SysName, daos.PoolConnectFlagReadOnly, false) + pcResp, err := api.PoolConnect(ctx, api.PoolConnectReq{ + SysName: cmd.SysName, + ID: pool.UUID.String(), + Flags: daos.PoolConnectFlagReadOnly, + Query: false, + }) if err != nil { cmd.Errorf("failed to connect to pool %s: %v", pool.Label, err) continue } defer func() { - if err := poolDisconnectAPI(poolHdl); err != nil { + if err := pcResp.Connection.Disconnect(ctx); err != nil { cmd.Errorf("failed to disconnect from pool %s: %v", pool.Label, err) } }() @@ -104,7 +122,7 @@ func (cmd *healthCheckCmd) Execute([]string) error { if pool.DisabledTargets > 0 { queryMask.SetOptions(daos.PoolQueryOptionDisabledEngines) } - tpi, err := queryPool(poolHdl, queryMask) + tpi, err := pcResp.Connection.Query(ctx, queryMask) if err != nil { cmd.Errorf("failed to query pool %s: %v", pool.Label, err) continue @@ -113,6 +131,13 @@ func (cmd *healthCheckCmd) Execute([]string) error { pool.DisabledRanks = tpi.DisabledRanks pool.DeadRanks = tpi.DeadRanks + /* temporary, until we get the container API bindings */ + var poolHdl C.daos_handle_t + if err := pcResp.Connection.FillHandle(unsafe.Pointer(&poolHdl)); err != nil { + cmd.Errorf("failed to fill handle for pool %s: %v", pool.Label, err) + continue + } + poolConts, err := listContainers(poolHdl) if err != nil { cmd.Errorf("failed to list containers on pool %s: %v", pool.Label, err) diff --git a/src/control/cmd/daos/health_test.go b/src/control/cmd/daos/health_test.go index 95565b36bb2d..a3a3f2410186 100644 --- a/src/control/cmd/daos/health_test.go +++ b/src/control/cmd/daos/health_test.go @@ -1,5 +1,6 @@ // // (C) Copyright 2024 Intel Corporation. +// (C) Copyright 2025 Google LLC // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -11,54 +12,87 @@ import ( "testing" "github.com/dustin/go-humanize" - "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" + "github.com/pkg/errors" - "github.com/daos-stack/daos/src/control/common/cmdutil" "github.com/daos-stack/daos/src/control/common/test" "github.com/daos-stack/daos/src/control/lib/daos" "github.com/daos-stack/daos/src/control/lib/ranklist" - "github.com/daos-stack/daos/src/control/lib/ui" - "github.com/daos-stack/daos/src/control/logging" +) + +var ( + runSelfTestResult []*daos.SelfTestResult + runSelfTestErr error ) func RunSelfTest(ctx context.Context, cfg *daos.SelfTestConfig) ([]*daos.SelfTestResult, error) { - return []*daos.SelfTestResult{}, nil + return runSelfTestResult, runSelfTestErr } func TestDaos_netTestCmdExecute(t *testing.T) { - // Quickie smoke test for the UI -- will flesh out later. - var opts cliOptions - log, buf := logging.NewTestLogger(t.Name()) - defer test.ShowBufferOnFailure(t, buf) - args := []string{ - "health", "net-test", - "--ranks", "0-3", - "--tags", "4-9", - "--size", "20 MiB", - "--rep-count", "2222", - "--bytes", "--verbose", - } - expArgs := netTestCmd{} - expArgs.Ranks.Replace(ranklist.MustCreateRankSet("0-3")) - expArgs.Tags.Replace(ranklist.MustCreateRankSet("4-9")) - expArgs.XferSize.Bytes = 20 * humanize.MiByte - expArgs.RepCount = 2222 - expArgs.Verbose = true - expArgs.TpsBytes = true + baseArgs := test.JoinArgs(nil, "health", "net-test") - if err := parseOpts(args, &opts, log); err != nil { - t.Fatal(err) - } - cmpOpts := cmp.Options{ - cmpopts.IgnoreUnexported(netTestCmd{}), - cmp.Comparer(func(a, b ranklist.RankSet) bool { - return a.String() == b.String() - }), - cmp.Comparer(func(a, b ui.ByteSizeFlag) bool { - return a.String() == b.String() - }), - cmpopts.IgnoreTypes(cmdutil.LogCmd{}, cmdutil.JSONOutputCmd{}), + for name, tc := range map[string]struct { + args []string + expArgs netTestCmd + expErr error + }{ + "all set (long)": { + args: test.JoinArgs(baseArgs, + "--ranks", "0-3", + "--tags", "4-9", + "--size", "20 MiB", + "--rep-count", "2222", + "--max-inflight", "1234", + "--bytes", "--verbose", + ), + expArgs: func() netTestCmd { + cmd := netTestCmd{} + cmd.Ranks.Replace(ranklist.MustCreateRankSet("0-3")) + cmd.Tags.Replace(ranklist.MustCreateRankSet("4-9")) + cmd.XferSize.Bytes = 20 * humanize.MiByte + cmd.RepCount = 2222 + cmd.MaxInflight = 1234 + cmd.Verbose = true + cmd.TpsBytes = true + return cmd + }(), + }, + "all set (short)": { + args: test.JoinArgs(baseArgs, + "-r", "0-3", + "-t", "4-9", + "-s", "20 MiB", + "-c", "2222", + "-m", "1234", + "-y", "-v", + ), + expArgs: func() netTestCmd { + cmd := netTestCmd{} + cmd.Ranks.Replace(ranklist.MustCreateRankSet("0-3")) + cmd.Tags.Replace(ranklist.MustCreateRankSet("4-9")) + cmd.XferSize.Bytes = 20 * humanize.MiByte + cmd.RepCount = 2222 + cmd.MaxInflight = 1234 + cmd.Verbose = true + cmd.TpsBytes = true + return cmd + }(), + }, + "selftest fails": { + args: []string{"health", "net-test"}, + expErr: errors.New("whoops"), + }, + } { + t.Run(name, func(t *testing.T) { + if tc.expErr != nil { + prevErr := runSelfTestErr + t.Cleanup(func() { + runSelfTestErr = prevErr + }) + runSelfTestErr = tc.expErr + } + + runCmdTest(t, tc.args, tc.expArgs, tc.expErr, "Health.NetTest") + }) } - test.CmpAny(t, "health net-test args", expArgs, opts.Health.NetTest, cmpOpts...) } diff --git a/src/control/cmd/daos/pool.go b/src/control/cmd/daos/pool.go index 831a775db1a1..c82cfc7fc9e9 100644 --- a/src/control/cmd/daos/pool.go +++ b/src/control/cmd/daos/pool.go @@ -1,5 +1,6 @@ // // (C) Copyright 2021-2024 Intel Corporation. +// (C) Copyright 2025 Google LLC // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -7,7 +8,6 @@ package main import ( - "fmt" "os" "strings" "unsafe" @@ -16,10 +16,10 @@ import ( "github.com/pkg/errors" "github.com/daos-stack/daos/src/control/cmd/daos/pretty" - "github.com/daos-stack/daos/src/control/common" "github.com/daos-stack/daos/src/control/lib/daos" + "github.com/daos-stack/daos/src/control/lib/daos/api" + "github.com/daos-stack/daos/src/control/lib/ranklist" "github.com/daos-stack/daos/src/control/lib/ui" - "github.com/daos-stack/daos/src/control/logging" ) /* @@ -67,8 +67,10 @@ type PoolID struct { type poolBaseCmd struct { daosCmd - poolUUID uuid.UUID + pool *api.PoolHandle + // deprecated params -- gradually remove in favor of PoolHandle + poolUUID uuid.UUID cPoolHandle C.daos_handle_t Args struct { @@ -76,138 +78,66 @@ type poolBaseCmd struct { } `positional-args:"yes"` } -func (cmd *poolBaseCmd) poolUUIDPtr() *C.uchar { - if cmd.poolUUID == uuid.Nil { - cmd.Errorf("poolUUIDPtr(): nil UUID") - return nil - } - return (*C.uchar)(unsafe.Pointer(&cmd.poolUUID[0])) -} - func (cmd *poolBaseCmd) PoolID() ui.LabelOrUUIDFlag { return cmd.Args.Pool.LabelOrUUIDFlag } -// poolConnect is a convenience wrapper around poolConnectAPI. -func poolConnect(poolID, sysName string, flags uint, query bool) (C.daos_handle_t, *C.daos_pool_info_t, error) { - var cSysName *C.char - if sysName != "" { - cSysName = C.CString(sysName) - defer freeString(cSysName) - } - - cPoolID := C.CString(poolID) - defer freeString(cPoolID) - - var hdl C.daos_handle_t - var infoPtr *C.daos_pool_info_t - if query { - infoPtr = &C.daos_pool_info_t{ - pi_bits: C.ulong(daos.DefaultPoolQueryMask), - } - } - - return hdl, infoPtr, poolConnectAPI(cPoolID, cSysName, C.uint(flags), &hdl, infoPtr) -} - -// poolConnectAPI is a lower-level wrapper around daos_pool_connect(). -func poolConnectAPI(poolID, sysName *C.char, flags C.uint, hdl *C.daos_handle_t, info *C.daos_pool_info_t) error { - return daosError(C.daos_pool_connect(poolID, sysName, flags, hdl, info, nil)) -} - -// poolDisconnectAPI is a convenience wrapper around daos_pool_disconnect(). -func poolDisconnectAPI(hdl C.daos_handle_t) error { - // Hack for NLT fault injection testing: If the rc - // is -DER_NOMEM, retry once in order to actually - // shut down and release resources. - rc := C.daos_pool_disconnect(hdl, nil) - if rc == -C.DER_NOMEM { - rc = C.daos_pool_disconnect(hdl, nil) - // DAOS-8866, daos_pool_disconnect() might have failed, but worked anyway. - if rc == -C.DER_NO_HDL { - rc = -C.DER_SUCCESS - } - } - - return daosError(rc) -} - -func (cmd *poolBaseCmd) connectPool(flags C.uint) error { - sysName := cmd.SysName - var cSysName *C.char - if sysName != "" { - cSysName := C.CString(sysName) - defer freeString(cSysName) +func (cmd *poolBaseCmd) connectPool(flags daos.PoolConnectFlag) error { + req := api.PoolConnectReq{ + SysName: cmd.SysName, + Flags: flags, + Query: true, } switch { case cmd.PoolID().HasLabel(): - var poolInfo C.daos_pool_info_t - cLabel := C.CString(cmd.PoolID().Label) - defer freeString(cLabel) - - cmd.Debugf("connecting to pool: %s", cmd.PoolID().Label) - if err := poolConnectAPI(cLabel, cSysName, flags, &cmd.cPoolHandle, &poolInfo); err != nil { - return err - } - var err error - cmd.poolUUID, err = uuidFromC(poolInfo.pi_uuid) - if err != nil { - cmd.disconnectPool() - return err - } + req.ID = cmd.PoolID().Label case cmd.PoolID().HasUUID(): - cmd.poolUUID = cmd.PoolID().UUID - cmd.Debugf("connecting to pool: %s", cmd.poolUUID) - cUUIDstr := C.CString(cmd.poolUUID.String()) - defer freeString(cUUIDstr) - if err := poolConnectAPI(cUUIDstr, cSysName, flags, &cmd.cPoolHandle, nil); err != nil { - return err - } + req.ID = cmd.PoolID().UUID.String() default: return errors.New("no pool UUID or label supplied") } + resp, err := PoolConnect(cmd.MustLogCtx(), req) + if err != nil { + return err + } + cmd.pool = resp.Connection + + if err := cmd.pool.FillHandle(unsafe.Pointer(&cmd.cPoolHandle)); err != nil { + if pdErr := cmd.pool.Disconnect(cmd.MustLogCtx()); pdErr != nil { + cmd.Errorf("pool disconnect failed in cleanup: %v", pdErr) + } + return err + } + return nil } func (cmd *poolBaseCmd) disconnectPool() { - cmd.Debugf("disconnecting pool %s", cmd.PoolID()) - if err := poolDisconnectAPI(cmd.cPoolHandle); err != nil { + if err := cmd.pool.Disconnect(cmd.MustLogCtx()); err != nil { cmd.Errorf("pool disconnect failed: %v", err) } } -func (cmd *poolBaseCmd) resolveAndConnect(flags C.uint, ap *C.struct_cmd_args_s) (func(), error) { +func (cmd *poolBaseCmd) resolveAndConnect(flags daos.PoolConnectFlag, ap *C.struct_cmd_args_s) (func(), error) { if err := cmd.connectPool(flags); err != nil { return nil, errors.Wrapf(err, "failed to connect to pool %s", cmd.PoolID()) } if ap != nil { - if err := copyUUID(&ap.p_uuid, cmd.poolUUID); err != nil { + if err := copyUUID(&ap.p_uuid, cmd.pool.UUID()); err != nil { return nil, err } ap.pool = cmd.cPoolHandle - switch { - case cmd.PoolID().HasLabel(): - pLabel := C.CString(cmd.PoolID().Label) - defer freeString(pLabel) - C.strncpy(&ap.pool_str[0], pLabel, C.DAOS_PROP_LABEL_MAX_LEN) - case cmd.PoolID().HasUUID(): - pUUIDstr := C.CString(cmd.poolUUID.String()) - defer freeString(pUUIDstr) - C.strncpy(&ap.pool_str[0], pUUIDstr, C.DAOS_PROP_LABEL_MAX_LEN) - } - } - return func() { - cmd.disconnectPool() - }, nil -} + pLabel := C.CString(cmd.pool.Label) + defer freeString(pLabel) + C.strncpy(&ap.pool_str[0], pLabel, C.DAOS_PROP_LABEL_MAX_LEN) + } -func (cmd *poolBaseCmd) getAttr(name string) (*attribute, error) { - return getDaosAttribute(cmd.cPoolHandle, poolAttr, name) + return cmd.disconnectPool, nil } type poolCmd struct { @@ -228,130 +158,6 @@ type poolQueryCmd struct { HealthOnly bool `short:"t" long:"health-only" description:"Only perform pool health related queries"` } -func convertPoolSpaceInfo(in *C.struct_daos_pool_space, mt C.uint) *daos.StorageUsageStats { - if in == nil { - return nil - } - - return &daos.StorageUsageStats{ - Total: uint64(in.ps_space.s_total[mt]), - Free: uint64(in.ps_space.s_free[mt]), - Min: uint64(in.ps_free_min[mt]), - Max: uint64(in.ps_free_max[mt]), - Mean: uint64(in.ps_free_mean[mt]), - MediaType: daos.StorageMediaType(mt), - } -} - -func convertPoolRebuildStatus(in *C.struct_daos_rebuild_status) *daos.PoolRebuildStatus { - if in == nil { - return nil - } - - out := &daos.PoolRebuildStatus{ - Status: int32(in.rs_errno), - } - if out.Status == 0 { - out.TotalObjects = uint64(in.rs_toberb_obj_nr) - out.Objects = uint64(in.rs_obj_nr) - out.Records = uint64(in.rs_rec_nr) - switch { - case in.rs_version == 0: - out.State = daos.PoolRebuildStateIdle - case C.get_rebuild_state(in) == C.DRS_COMPLETED: - out.State = daos.PoolRebuildStateDone - default: - out.State = daos.PoolRebuildStateBusy - } - } - - return out -} - -func convertPoolInfo(pinfo *C.daos_pool_info_t) (*daos.PoolInfo, error) { - poolInfo := new(daos.PoolInfo) - - poolInfo.QueryMask = daos.PoolQueryMask(pinfo.pi_bits) - poolInfo.UUID = uuid.Must(uuidFromC(pinfo.pi_uuid)) - poolInfo.TotalTargets = uint32(pinfo.pi_ntargets) - poolInfo.DisabledTargets = uint32(pinfo.pi_ndisabled) - poolInfo.ActiveTargets = uint32(pinfo.pi_space.ps_ntargets) - poolInfo.TotalEngines = uint32(pinfo.pi_nnodes) - poolInfo.ServiceLeader = uint32(pinfo.pi_leader) - poolInfo.Version = uint32(pinfo.pi_map_ver) - poolInfo.State = daos.PoolServiceStateReady - if poolInfo.DisabledTargets > 0 { - poolInfo.State = daos.PoolServiceStateDegraded - } - - poolInfo.Rebuild = convertPoolRebuildStatus(&pinfo.pi_rebuild_st) - if poolInfo.QueryMask.HasOption(daos.PoolQueryOptionSpace) { - poolInfo.TierStats = []*daos.StorageUsageStats{ - convertPoolSpaceInfo(&pinfo.pi_space, C.DAOS_MEDIA_SCM), - convertPoolSpaceInfo(&pinfo.pi_space, C.DAOS_MEDIA_NVME), - } - } - - return poolInfo, nil -} - -func queryPool(poolHdl C.daos_handle_t, queryMask daos.PoolQueryMask) (*daos.PoolInfo, error) { - var enabledRanks *C.d_rank_list_t - var disabledRanks *C.d_rank_list_t - defer func() { - C.d_rank_list_free(enabledRanks) - C.d_rank_list_free(disabledRanks) - }() - - var rc C.int - cPoolInfo := C.daos_pool_info_t{ - pi_bits: C.uint64_t(queryMask), - } - if queryMask.HasOption(daos.PoolQueryOptionEnabledEngines) && queryMask.HasOption(daos.PoolQueryOptionDisabledEngines) { - enaQm := queryMask - enaQm.ClearOptions(daos.PoolQueryOptionDisabledEngines) - cPoolInfo.pi_bits = C.uint64_t(enaQm) - rc = C.daos_pool_query(poolHdl, &enabledRanks, &cPoolInfo, nil, nil) - if err := daosError(rc); err != nil { - return nil, err - } - - /* second query to just get disabled ranks */ - rc = C.daos_pool_query(poolHdl, &disabledRanks, nil, nil, nil) - } else if queryMask.HasOption(daos.PoolQueryOptionEnabledEngines) { - rc = C.daos_pool_query(poolHdl, &enabledRanks, &cPoolInfo, nil, nil) - } else if queryMask.HasOption(daos.PoolQueryOptionDisabledEngines) { - rc = C.daos_pool_query(poolHdl, &disabledRanks, &cPoolInfo, nil, nil) - } else { - rc = C.daos_pool_query(poolHdl, nil, &cPoolInfo, nil, nil) - } - - if err := daosError(rc); err != nil { - return nil, err - } - - poolInfo, err := convertPoolInfo(&cPoolInfo) - if err != nil { - return nil, err - } - poolInfo.QueryMask = queryMask - - if enabledRanks != nil { - poolInfo.EnabledRanks, err = rankSetFromC(enabledRanks) - if err != nil { - return nil, err - } - } - if disabledRanks != nil { - poolInfo.DisabledRanks, err = rankSetFromC(disabledRanks) - if err != nil { - return nil, err - } - } - - return poolInfo, nil -} - func (cmd *poolQueryCmd) Execute(_ []string) error { queryMask := daos.DefaultPoolQueryMask if cmd.HealthOnly { @@ -360,15 +166,14 @@ func (cmd *poolQueryCmd) Execute(_ []string) error { if cmd.ShowEnabledRanks { queryMask.SetOptions(daos.PoolQueryOptionEnabledEngines) } - queryMask.SetOptions(daos.PoolQueryOptionDisabledEngines) - cleanup, err := cmd.resolveAndConnect(C.DAOS_PC_RO, nil) + cleanup, err := cmd.resolveAndConnect(daos.PoolConnectFlagReadOnly, nil) if err != nil { return err } defer cleanup() - poolInfo, err := queryPool(cmd.cPoolHandle, queryMask) + poolInfo, err := cmd.pool.Query(cmd.MustLogCtx(), queryMask) if err != nil { return errors.Wrapf(err, "failed to query pool %q", cmd.PoolID()) } @@ -391,73 +196,20 @@ func (cmd *poolQueryCmd) Execute(_ []string) error { type poolQueryTargetsCmd struct { poolBaseCmd - Rank uint32 `long:"rank" required:"1" description:"Engine rank of the targets to be queried"` - Targets string `long:"target-idx" description:"Comma-separated list of target idx(s) to be queried"` -} - -// For using the pretty printer that dmg uses for this target info. -func convertPoolTargetInfo(ptinfo *C.daos_target_info_t) (*daos.PoolQueryTargetInfo, error) { - pqti := new(daos.PoolQueryTargetInfo) - pqti.Type = daos.PoolQueryTargetType(ptinfo.ta_type) - pqti.State = daos.PoolQueryTargetState(ptinfo.ta_state) - pqti.Space = []*daos.StorageUsageStats{ - { - Total: uint64(ptinfo.ta_space.s_total[C.DAOS_MEDIA_SCM]), - Free: uint64(ptinfo.ta_space.s_free[C.DAOS_MEDIA_SCM]), - MediaType: C.DAOS_MEDIA_SCM, - }, - { - Total: uint64(ptinfo.ta_space.s_total[C.DAOS_MEDIA_NVME]), - Free: uint64(ptinfo.ta_space.s_free[C.DAOS_MEDIA_NVME]), - MediaType: C.DAOS_MEDIA_NVME, - }, - } - - return pqti, nil + Rank uint32 `long:"rank" required:"1" description:"Engine rank of the target(s) to be queried"` + Targets ui.RankSetFlag `long:"target-idx" description:"Comma-separated list of target index(es) to be queried (default: all)"` } func (cmd *poolQueryTargetsCmd) Execute(_ []string) error { - cleanup, err := cmd.resolveAndConnect(C.DAOS_PC_RO, nil) + cleanup, err := cmd.resolveAndConnect(daos.PoolConnectFlagReadOnly, nil) if err != nil { return err } defer cleanup() - var idxList []uint32 - if err = common.ParseNumberList(cmd.Targets, &idxList); err != nil { - return errors.WithMessage(err, "parsing target list") - } - - if len(idxList) == 0 { - pi, err := queryPool(cmd.cPoolHandle, daos.HealthOnlyPoolQueryMask) - if err != nil || (pi.TotalTargets == 0 || pi.TotalEngines == 0) { - if err != nil { - return errors.Wrap(err, "pool query failed") - } - return errors.New("failed to derive target count from pool query") - } - tgtCount := pi.TotalTargets / pi.TotalEngines - for i := uint32(0); i < tgtCount; i++ { - idxList = append(idxList, i) - } - } - - ptInfo := new(C.daos_target_info_t) - var rc C.int - - infos := make([]*daos.PoolQueryTargetInfo, 0, len(idxList)) - for tgt := 0; tgt < len(idxList); tgt++ { - rc = C.daos_pool_query_target(cmd.cPoolHandle, C.uint32_t(idxList[tgt]), C.uint32_t(cmd.Rank), ptInfo, nil) - if err := daosError(rc); err != nil { - return errors.Wrapf(err, - "failed to query pool %s rank:target %d:%d", cmd.poolUUID, cmd.Rank, idxList[tgt]) - } - - tgtInfo, err := convertPoolTargetInfo(ptInfo) - if err != nil { - return err - } - infos = append(infos, tgtInfo) + infos, err := cmd.pool.QueryTargets(cmd.MustLogCtx(), ranklist.Rank(cmd.Rank), &cmd.Targets.RankSet) + if err != nil { + return errors.Wrapf(err, "failed to query targets for pool %s", cmd.PoolID()) } if cmd.JSONOutputEnabled() { @@ -483,32 +235,13 @@ type poolListAttrsCmd struct { } func (cmd *poolListAttrsCmd) Execute(_ []string) error { - cleanup, err := cmd.resolveAndConnect(C.DAOS_PC_RO, nil) + cleanup, err := cmd.resolveAndConnect(daos.PoolConnectFlagReadOnly, nil) if err != nil { return err } defer cleanup() - attrs, err := listDaosAttributes(cmd.cPoolHandle, poolAttr, cmd.Verbose) - if err != nil { - return errors.Wrapf(err, - "failed to list attributes for pool %s", cmd.poolUUID) - } - - if cmd.JSONOutputEnabled() { - if cmd.Verbose { - return cmd.OutputJSON(attrs.asMap(), nil) - } - return cmd.OutputJSON(attrs.asList(), nil) - } - - var bld strings.Builder - title := fmt.Sprintf("Attributes for pool %s:", cmd.poolUUID) - printAttributes(&bld, title, attrs...) - - cmd.Info(bld.String()) - - return nil + return listAttributes(cmd, cmd.pool, poolAttr, cmd.pool.ID(), cmd.Verbose) } type poolGetAttrCmd struct { @@ -520,37 +253,13 @@ type poolGetAttrCmd struct { } func (cmd *poolGetAttrCmd) Execute(_ []string) error { - cleanup, err := cmd.resolveAndConnect(C.DAOS_PC_RO, nil) + cleanup, err := cmd.resolveAndConnect(daos.PoolConnectFlagReadOnly, nil) if err != nil { return err } defer cleanup() - var attrs attrList - if len(cmd.Args.Attrs.ParsedProps) == 0 { - attrs, err = listDaosAttributes(cmd.cPoolHandle, poolAttr, true) - } else { - attrs, err = getDaosAttributes(cmd.cPoolHandle, poolAttr, cmd.Args.Attrs.ParsedProps.ToSlice()) - } - if err != nil { - return errors.Wrapf(err, "failed to get attributes for pool %s", cmd.PoolID()) - } - - if cmd.JSONOutputEnabled() { - // Maintain compatibility with older behavior. - if len(cmd.Args.Attrs.ParsedProps) == 1 && len(attrs) == 1 { - return cmd.OutputJSON(attrs[0], nil) - } - return cmd.OutputJSON(attrs, nil) - } - - var bld strings.Builder - title := fmt.Sprintf("Attributes for pool %s:", cmd.PoolID()) - printAttributes(&bld, title, attrs...) - - cmd.Info(bld.String()) - - return nil + return getAttributes(cmd, cmd.pool, poolAttr, cmd.pool.ID(), cmd.Args.Attrs.ParsedProps.ToSlice()...) } type poolSetAttrCmd struct { @@ -562,53 +271,31 @@ type poolSetAttrCmd struct { } func (cmd *poolSetAttrCmd) Execute(_ []string) error { - cleanup, err := cmd.resolveAndConnect(C.DAOS_PC_RW, nil) + cleanup, err := cmd.resolveAndConnect(daos.PoolConnectFlagReadWrite, nil) if err != nil { return err } defer cleanup() - if len(cmd.Args.Attrs.ParsedProps) == 0 { - return errors.New("attribute name and value are required") - } - - attrs := make(attrList, 0, len(cmd.Args.Attrs.ParsedProps)) - for key, val := range cmd.Args.Attrs.ParsedProps { - attrs = append(attrs, &attribute{ - Name: key, - Value: []byte(val), - }) - } - - if err := setDaosAttributes(cmd.cPoolHandle, poolAttr, attrs); err != nil { - return errors.Wrapf(err, "failed to set attributes on pool %s", cmd.PoolID()) - } - - return nil + return setAttributes(cmd, cmd.pool, poolAttr, cmd.pool.ID(), cmd.Args.Attrs.ParsedProps) } type poolDelAttrCmd struct { poolBaseCmd Args struct { - Name string `positional-arg-name:"" required:"1"` + Attrs ui.GetPropertiesFlag `positional-arg-name:"key[,key...]" required:"1"` } `positional-args:"yes"` } func (cmd *poolDelAttrCmd) Execute(_ []string) error { - cleanup, err := cmd.resolveAndConnect(C.DAOS_PC_RW, nil) + cleanup, err := cmd.resolveAndConnect(daos.PoolConnectFlagReadWrite, nil) if err != nil { return err } defer cleanup() - if err := delDaosAttribute(cmd.cPoolHandle, poolAttr, cmd.Args.Name); err != nil { - return errors.Wrapf(err, - "failed to delete attribute %q on pool %s", - cmd.Args.Name, cmd.poolUUID) - } - - return nil + return delAttributes(cmd, cmd.pool, poolAttr, cmd.pool.ID(), cmd.Args.Attrs.ParsedProps.ToSlice()...) } type poolAutoTestCmd struct { @@ -625,14 +312,14 @@ func (cmd *poolAutoTestCmd) Execute(_ []string) error { } defer deallocCmdArgs() - cleanup, err := cmd.resolveAndConnect(C.DAOS_PC_RW, nil) + cleanup, err := cmd.resolveAndConnect(daos.PoolConnectFlagReadWrite, nil) if err != nil { return err } defer cleanup() ap.pool = cmd.cPoolHandle - if err := copyUUID(&ap.p_uuid, cmd.poolUUID); err != nil { + if err := copyUUID(&ap.p_uuid, cmd.pool.UUID()); err != nil { return err } ap.p_op = C.POOL_AUTOTEST @@ -649,108 +336,12 @@ func (cmd *poolAutoTestCmd) Execute(_ []string) error { rc := C.pool_autotest_hdlr(ap) if err := daosError(rc); err != nil { - return errors.Wrapf(err, "failed to run autotest for pool %s", - cmd.poolUUID) + return errors.Wrapf(err, "failed to run autotest for pool %s", cmd.PoolID()) } return nil } -func getPoolList(log logging.Logger, sysName string, queryEnabled bool) ([]*daos.PoolInfo, error) { - var cSysName *C.char - if sysName != "" { - cSysName := C.CString(sysName) - defer freeString(cSysName) - } - - var cPools []C.daos_mgmt_pool_info_t - for { - var rc C.int - var poolCount C.size_t - - // First, fetch the total number of pools in the system. - // We may not have access to all of them, so this is an upper bound. - rc = C.daos_mgmt_list_pools(cSysName, &poolCount, nil, nil) - if err := daosError(rc); err != nil { - return nil, err - } - log.Debugf("pools in system: %d", poolCount) - - if poolCount < 1 { - return nil, nil - } - - // Now, we actually fetch the pools into the buffer that we've created. - cPools = make([]C.daos_mgmt_pool_info_t, poolCount) - rc = C.daos_mgmt_list_pools(cSysName, &poolCount, &cPools[0], nil) - err := daosError(rc) - if err == nil { - cPools = cPools[:poolCount] // adjust the slice to the number of pools retrieved - log.Debugf("fetched %d pools", len(cPools)) - break - } - if err == daos.StructTooSmall { - log.Notice("server-side pool list changed; re-fetching") - continue - } - log.Errorf("failed to fetch pool list: %s", err) - return nil, err - } - - pools := make([]*daos.PoolInfo, 0, len(cPools)) - for i := 0; i < len(cPools); i++ { - cPool := &cPools[i] - - svcRanks, err := rankSetFromC(cPool.mgpi_svc) - if err != nil { - return nil, err - } - poolUUID, err := uuidFromC(cPool.mgpi_uuid) - if err != nil { - return nil, err - } - poolLabel := C.GoString(cPool.mgpi_label) - - var pool *daos.PoolInfo - if queryEnabled { - poolHandle, poolInfo, err := poolConnect(poolUUID.String(), sysName, daos.PoolConnectFlagReadOnly, true) - if err != nil { - log.Errorf("failed to connect to pool %q: %s", poolLabel, err) - continue - } - - var qErr error - pool, qErr = convertPoolInfo(poolInfo) - if qErr != nil { - log.Errorf("failed to query pool %q: %s", poolLabel, qErr) - } - if err := poolDisconnectAPI(poolHandle); err != nil { - log.Errorf("failed to disconnect from pool %q: %s", poolLabel, err) - } - if qErr != nil { - continue - } - - // Add a few missing pieces that the query doesn't fill in. - pool.Label = poolLabel - pool.ServiceReplicas = svcRanks.Ranks() - } else { - // Just populate the basic info. - pool = &daos.PoolInfo{ - UUID: poolUUID, - Label: poolLabel, - ServiceReplicas: svcRanks.Ranks(), - State: daos.PoolServiceStateReady, - } - } - - pools = append(pools, pool) - } - - log.Debugf("fetched %d/%d pools", len(pools), len(cPools)) - return pools, nil -} - type poolListCmd struct { daosCmd Verbose bool `short:"v" long:"verbose" description:"Add pool UUIDs and service replica lists to display"` @@ -758,7 +349,9 @@ type poolListCmd struct { } func (cmd *poolListCmd) Execute(_ []string) error { - pools, err := getPoolList(cmd.Logger, cmd.SysName, !cmd.NoQuery) + pools, err := GetPoolList(cmd.MustLogCtx(), api.GetPoolListReq{ + Query: !cmd.NoQuery, + }) if err != nil { return err } diff --git a/src/control/cmd/daos/pool_test.go b/src/control/cmd/daos/pool_test.go new file mode 100644 index 000000000000..c54c21c1f6ce --- /dev/null +++ b/src/control/cmd/daos/pool_test.go @@ -0,0 +1,476 @@ +// +// (C) Copyright 2024 Intel Corporation. +// (C) Copyright 2025 Google LLC +// +// SPDX-License-Identifier: BSD-2-Clause-Patent +// + +package main + +import ( + "context" + "strings" + "testing" + + "github.com/dustin/go-humanize" + "github.com/pkg/errors" + + "github.com/daos-stack/daos/src/control/common/test" + "github.com/daos-stack/daos/src/control/lib/daos" + "github.com/daos-stack/daos/src/control/lib/daos/api" + "github.com/daos-stack/daos/src/control/lib/ranklist" + "github.com/daos-stack/daos/src/control/lib/ui" +) + +var ( + defaultPoolInfo *daos.PoolInfo = &daos.PoolInfo{ + QueryMask: daos.DefaultPoolQueryMask, + State: daos.PoolServiceStateReady, + UUID: test.MockPoolUUID(1), + Label: "test-pool", + TotalTargets: 48, + TotalEngines: 3, + ActiveTargets: 48, + DisabledTargets: 0, + Version: 1, + ServiceLeader: 2, + ServiceReplicas: []ranklist.Rank{0, 1, 2}, + TierStats: []*daos.StorageUsageStats{ + { + MediaType: daos.StorageMediaTypeScm, + Total: 64 * humanize.TByte, + Free: 16 * humanize.TByte, + }, + { + MediaType: daos.StorageMediaTypeNvme, + Total: 1 * humanize.PByte, + Free: 512 * humanize.TByte, + }, + }, + } +) + +var ( + defaultGetPoolListResult = []*daos.PoolInfo{ + defaultPoolInfo, + } + + getPoolListResult []*daos.PoolInfo = defaultGetPoolListResult + getPoolListErr error +) + +func GetPoolList(ctx context.Context, req api.GetPoolListReq) ([]*daos.PoolInfo, error) { + return getPoolListResult, getPoolListErr +} + +func TestDaos_poolListCmd(t *testing.T) { + baseArgs := test.JoinArgs(nil, "pool", "list") + + for name, tc := range map[string]struct { + args []string + expErr error + expArgs poolListCmd + }{ + "all set (long)": { + args: test.JoinArgs(baseArgs, "--verbose", "--no-query"), + expArgs: poolListCmd{ + NoQuery: true, + Verbose: true, + }, + }, + "all set (short)": { + args: test.JoinArgs(baseArgs, "-v", "-n"), + expArgs: poolListCmd{ + NoQuery: true, + Verbose: true, + }, + }, + "query fails": { + args: []string{"pool", "list"}, + expErr: errors.New("whoops"), + }, + } { + t.Run(name, func(t *testing.T) { + t.Cleanup(api.ResetTestStubs) + if tc.expErr != nil { + prevErr := getPoolListErr + t.Cleanup(func() { + getPoolListErr = prevErr + }) + getPoolListErr = tc.expErr + } + + runCmdTest(t, tc.args, tc.expArgs, tc.expErr, "Pool.List") + }) + } +} + +var ( + defaultPoolConnectResp *api.PoolConnectResp = &api.PoolConnectResp{ + Connection: &api.PoolHandle{}, + Info: defaultPoolInfo, + } + + poolConnectResp *api.PoolConnectResp = defaultPoolConnectResp + poolConnectErr error +) + +func PoolConnect(ctx context.Context, req api.PoolConnectReq) (*api.PoolConnectResp, error) { + return poolConnectResp, poolConnectErr +} + +func TestDaos_poolQueryCmd(t *testing.T) { + baseArgs := test.JoinArgs(nil, "pool", "query", defaultPoolInfo.Label) + + for name, tc := range map[string]struct { + args []string + expErr error + expArgs poolQueryCmd + setup func(t *testing.T) + }{ + "invalid flag": { + args: test.JoinArgs(baseArgs, "--bad"), + expErr: errors.New("unknown flag"), + }, + "missing pool ID": { + args: baseArgs[:len(baseArgs)-1], + expErr: errors.New("no pool UUID or label supplied"), + }, + "connect fails": { + args: baseArgs, + expErr: errors.New("whoops"), + setup: func(t *testing.T) { + prevErr := poolConnectErr + t.Cleanup(func() { + poolConnectErr = prevErr + }) + poolConnectErr = errors.New("whoops") + }, + }, + "all set (long)": { + args: test.JoinArgs(baseArgs, "--show-enabled", "--health-only"), + expArgs: poolQueryCmd{ + ShowEnabledRanks: true, + HealthOnly: true, + }, + }, + "all set (short)": { + args: test.JoinArgs(baseArgs, "-e", "-t"), + expArgs: poolQueryCmd{ + ShowEnabledRanks: true, + HealthOnly: true, + }, + }, + } { + t.Run(name, func(t *testing.T) { + t.Cleanup(api.ResetTestStubs) + if tc.setup != nil { + tc.setup(t) + } + + runCmdTest(t, tc.args, tc.expArgs, tc.expErr, "Pool.Query") + }) + } +} + +func TestDaos_poolQueryTargetsCmd(t *testing.T) { + baseArgs := test.JoinArgs(nil, "pool", "query-targets", defaultPoolInfo.Label) + + for name, tc := range map[string]struct { + args []string + expErr error + expArgs poolQueryTargetsCmd + setup func(t *testing.T) + }{ + "invalid flag": { + args: test.JoinArgs(baseArgs, "--rank=2", "--bad"), + expErr: errors.New("unknown flag"), + }, + "missing pool ID": { + args: test.JoinArgs(baseArgs[:len(baseArgs)-1], "--rank=2"), + expErr: errors.New("no pool UUID or label supplied"), + }, + "missing rank argument": { + args: baseArgs, + expErr: errors.New("required flag"), + }, + "connect fails": { + args: test.JoinArgs(baseArgs, "--rank=2"), + expErr: errors.New("whoops"), + setup: func(t *testing.T) { + prevErr := poolConnectErr + t.Cleanup(func() { + poolConnectErr = prevErr + }) + poolConnectErr = errors.New("whoops") + }, + }, + "success (rank only)": { + args: test.JoinArgs(baseArgs, "--rank=2"), + expArgs: poolQueryTargetsCmd{ + Rank: 2, + }, + }, + "success (rank and target)": { + args: test.JoinArgs(baseArgs, "--rank=2", "--target-idx=1,2"), + expArgs: poolQueryTargetsCmd{ + Rank: 2, + Targets: ui.RankSetFlag{ + RankSet: *ranklist.MustCreateRankSet("1,2"), + }, + }, + }, + } { + t.Run(name, func(t *testing.T) { + t.Cleanup(api.ResetTestStubs) + if tc.setup != nil { + tc.setup(t) + } + + runCmdTest(t, tc.args, tc.expArgs, tc.expErr, "Pool.QueryTargets") + }) + } +} + +func TestDaos_poolSetAttrCmd(t *testing.T) { + baseArgs := test.JoinArgs(nil, "pool", "set-attr", defaultPoolInfo.Label) + keysOnlyArg := "key1,key2" + keyValArg := "key1:val1,key2:val2" + + for name, tc := range map[string]struct { + args []string + expErr error + expArgs poolSetAttrCmd + setup func(t *testing.T) + }{ + "invalid flag": { + args: test.JoinArgs(baseArgs, "--bad", keyValArg), + expErr: errors.New("unknown flag"), + }, + "connect fails": { + args: test.JoinArgs(baseArgs, keyValArg), + expErr: errors.New("whoops"), + setup: func(t *testing.T) { + prevErr := poolConnectErr + t.Cleanup(func() { + poolConnectErr = prevErr + }) + poolConnectErr = errors.New("whoops") + }, + }, + "missing required arguments": { + args: baseArgs, + expErr: errors.New("required argument"), + }, + "malformed required arguments": { + args: test.JoinArgs(baseArgs, keysOnlyArg), + expErr: errors.New("invalid property"), + }, + "success": { + args: test.JoinArgs(baseArgs, keyValArg), + expArgs: poolSetAttrCmd{ + Args: struct { + Attrs ui.SetPropertiesFlag `positional-arg-name:"key:val[,key:val...]" required:"1"` + }{ + Attrs: ui.SetPropertiesFlag{ + ParsedProps: map[string]string{ + "key1": "val1", + "key2": "val2", + }, + }, + }, + }, + }, + } { + t.Run(name, func(t *testing.T) { + t.Cleanup(api.ResetTestStubs) + if tc.setup != nil { + tc.setup(t) + } + + runCmdTest(t, tc.args, tc.expArgs, tc.expErr, "Pool.SetAttr") + }) + } +} + +func TestDaos_poolGetAttrCmd(t *testing.T) { + baseArgs := test.JoinArgs(nil, "pool", "get-attr", defaultPoolInfo.Label) + keysOnlyArg := "key1,key2" + + for name, tc := range map[string]struct { + args []string + expErr error + expArgs poolGetAttrCmd + setup func(t *testing.T) + }{ + "invalid flag": { + args: test.JoinArgs(baseArgs, "--bad"), + expErr: errors.New("unknown flag"), + }, + "missing pool ID": { + args: baseArgs[:len(baseArgs)-1], + expErr: errors.New("no pool UUID or label supplied"), + }, + "connect fails": { + args: baseArgs, + expErr: errors.New("whoops"), + setup: func(t *testing.T) { + prevErr := poolConnectErr + t.Cleanup(func() { + poolConnectErr = prevErr + }) + poolConnectErr = errors.New("whoops") + }, + }, + "malformed arguments": { + args: test.JoinArgs(baseArgs, strings.ReplaceAll(keysOnlyArg, ",", ":")), + expErr: errors.New("key cannot contain"), + }, + "unknown key(s)": { + args: test.JoinArgs(baseArgs, keysOnlyArg), + expErr: daos.Nonexistent, + }, + "success (one key)": { + args: test.JoinArgs(baseArgs, "one"), + expArgs: poolGetAttrCmd{ + Args: struct { + Attrs ui.GetPropertiesFlag `positional-arg-name:"key[,key...]"` + }{ + Attrs: ui.GetPropertiesFlag{ + ParsedProps: map[string]struct{}{ + "one": {}, + }, + }, + }, + }, + }, + "success (all keys)": { + args: baseArgs, + expArgs: poolGetAttrCmd{}, + }, + } { + t.Run(name, func(t *testing.T) { + t.Cleanup(api.ResetTestStubs) + if tc.setup != nil { + tc.setup(t) + } + + runCmdTest(t, tc.args, tc.expArgs, tc.expErr, "Pool.GetAttr") + }) + } +} + +func TestDaos_poolDelAttrCmd(t *testing.T) { + baseArgs := test.JoinArgs(nil, "pool", "del-attr", defaultPoolInfo.Label) + keysOnlyArg := "key1,key2" + + for name, tc := range map[string]struct { + args []string + expErr error + expArgs poolDelAttrCmd + setup func(t *testing.T) + }{ + "invalid flag": { + args: test.JoinArgs(baseArgs, "--bad"), + expErr: errors.New("unknown flag"), + }, + "missing required arguments": { + args: baseArgs, + expErr: errors.New("required argument"), + }, + "connect fails": { + args: test.JoinArgs(baseArgs, keysOnlyArg), + expErr: errors.New("whoops"), + setup: func(t *testing.T) { + prevErr := poolConnectErr + t.Cleanup(func() { + poolConnectErr = prevErr + }) + poolConnectErr = errors.New("whoops") + }, + }, + "malformed arguments": { + args: test.JoinArgs(baseArgs, strings.ReplaceAll(keysOnlyArg, ",", ":")), + expErr: errors.New("key cannot contain"), + }, + "success (one key)": { + args: test.JoinArgs(baseArgs, "one"), + expArgs: poolDelAttrCmd{ + Args: struct { + Attrs ui.GetPropertiesFlag `positional-arg-name:"key[,key...]" required:"1"` + }{ + Attrs: ui.GetPropertiesFlag{ + ParsedProps: map[string]struct{}{ + "one": {}, + }, + }, + }, + }, + }, + } { + t.Run(name, func(t *testing.T) { + t.Cleanup(api.ResetTestStubs) + if tc.setup != nil { + tc.setup(t) + } + + runCmdTest(t, tc.args, tc.expArgs, tc.expErr, "Pool.DelAttr") + }) + } +} + +func TestDaos_poolListAttrCmd(t *testing.T) { + baseArgs := test.JoinArgs(nil, "pool", "list-attr", defaultPoolInfo.Label) + + for name, tc := range map[string]struct { + args []string + expErr error + expArgs poolListAttrsCmd + setup func(t *testing.T) + }{ + "invalid flag": { + args: test.JoinArgs(baseArgs, "--bad"), + expErr: errors.New("unknown flag"), + }, + "missing pool ID": { + args: baseArgs[:len(baseArgs)-1], + expErr: errors.New("no pool UUID or label supplied"), + }, + "connect fails": { + args: baseArgs, + expErr: errors.New("whoops"), + setup: func(t *testing.T) { + prevErr := poolConnectErr + t.Cleanup(func() { + poolConnectErr = prevErr + }) + poolConnectErr = errors.New("whoops") + }, + }, + "success": { + args: baseArgs, + expArgs: poolListAttrsCmd{}, + }, + "success (verbose, short)": { + args: test.JoinArgs(baseArgs, "-V"), + expArgs: poolListAttrsCmd{ + Verbose: true, + }, + }, + "success (verbose, long)": { + args: test.JoinArgs(baseArgs, "--verbose"), + expArgs: poolListAttrsCmd{ + Verbose: true, + }, + }, + } { + t.Run(name, func(t *testing.T) { + t.Cleanup(api.ResetTestStubs) + if tc.setup != nil { + tc.setup(t) + } + + runCmdTest(t, tc.args, tc.expArgs, tc.expErr, "Pool.ListAttrs") + }) + } +} diff --git a/src/control/cmd/daos/pretty/pool.go b/src/control/cmd/daos/pretty/pool.go index 75e7d9d13e9a..6f46b1b77903 100644 --- a/src/control/cmd/daos/pretty/pool.go +++ b/src/control/cmd/daos/pretty/pool.go @@ -1,5 +1,6 @@ // // (C) Copyright 2020-2024 Intel Corporation. +// (C) Copyright 2025 Google LLC // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -54,7 +55,7 @@ func printPoolTiers(memFileBytes uint64, suss []*daos.StorageUsageStats, w *txtf } // PrintPoolInfo generates a human-readable representation of the supplied -// PoolQueryResp struct and writes it to the supplied io.Writer. +// PoolInfo struct and writes it to the supplied io.Writer. func PrintPoolInfo(pi *daos.PoolInfo, out io.Writer) error { if pi == nil { return errors.Errorf("nil %T", pi) @@ -338,3 +339,35 @@ func PrintPoolList(pools []*daos.PoolInfo, out io.Writer, verbose bool) error { return printPoolList(pools, out) } + +// PrintAttributes generates a human-readable representation of the supplied +// list of daos.Attributes and writes it to the supplied io.Writer. +func PrintAttributes(out io.Writer, header string, attrs ...*daos.Attribute) { + fmt.Fprintf(out, "%s\n", header) + + if len(attrs) == 0 { + fmt.Fprintln(out, " No attributes found.") + return + } + + nameTitle := "Name" + valueTitle := "Value" + titles := []string{nameTitle} + + table := []txtfmt.TableRow{} + for _, attr := range attrs { + row := txtfmt.TableRow{} + row[nameTitle] = attr.Name + if len(attr.Value) != 0 { + row[valueTitle] = string(attr.Value) + if len(titles) == 1 { + titles = append(titles, valueTitle) + } + } + table = append(table, row) + } + + tf := txtfmt.NewTableFormatter(titles...) + tf.InitWriter(out) + tf.Format(table) +} diff --git a/src/control/cmd/daos/stubbed.go b/src/control/cmd/daos/stubbed.go index 4a08ad77255d..000e8be5a208 100644 --- a/src/control/cmd/daos/stubbed.go +++ b/src/control/cmd/daos/stubbed.go @@ -1,5 +1,6 @@ // // (C) Copyright 2024 Intel Corporation. +// (C) Copyright 2025 Google LLC // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -12,4 +13,6 @@ import "github.com/daos-stack/daos/src/control/lib/daos/api" var ( RunSelfTest = api.RunSelfTest + GetPoolList = api.GetPoolList + PoolConnect = api.PoolConnect ) diff --git a/src/control/cmd/daos/util.go b/src/control/cmd/daos/util.go index d5b128bf9a45..7d1f5b15ea89 100644 --- a/src/control/cmd/daos/util.go +++ b/src/control/cmd/daos/util.go @@ -1,5 +1,6 @@ // // (C) Copyright 2021-2024 Intel Corporation. +// (C) Copyright 2025 Google LLC // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -336,3 +337,11 @@ func _writeDunsPath(path, ct string, poolUUID uuid.UUID, contUUID uuid.UUID) err return nil } + +func attrListFromNames(names []string) daos.AttributeList { + attrs := make(daos.AttributeList, len(names)) + for i, name := range names { + attrs[i] = &daos.Attribute{Name: name} + } + return attrs +} diff --git a/src/control/cmd/daos/util_test.go b/src/control/cmd/daos/util_test.go new file mode 100644 index 000000000000..dfb53e5c62cf --- /dev/null +++ b/src/control/cmd/daos/util_test.go @@ -0,0 +1,60 @@ +// +// (C) Copyright 2024 Intel Corporation. +// (C) Copyright 2025 Google LLC +// +// SPDX-License-Identifier: BSD-2-Clause-Patent +// + +package main + +import ( + "fmt" + "reflect" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/daos-stack/daos/src/control/common/cmdutil" + "github.com/daos-stack/daos/src/control/common/test" + "github.com/daos-stack/daos/src/control/lib/ranklist" + "github.com/daos-stack/daos/src/control/lib/ui" + "github.com/daos-stack/daos/src/control/logging" +) + +func runCmdTest(t *testing.T, args []string, expCmd any, expErr error, cmdPath string, cmpOpts ...cmp.Option) { + t.Helper() + + var opts cliOptions + log, buf := logging.NewTestLogger(t.Name()) + defer test.ShowBufferOnFailure(t, buf) + + if err := parseOpts(args, &opts, log); err != nil { + test.CmpErr(t, expErr, err) + if expErr != nil { + return + } + } + + testCmd := reflect.ValueOf(opts) + for _, subCmd := range strings.Split(cmdPath, ".") { + testCmd = testCmd.FieldByName(subCmd) + if !testCmd.IsValid() || testCmd.IsZero() { + t.Fatalf("failed to select subcommand struct using %q", cmdPath) + } + } + + cmpOpts = append(cmpOpts, []cmp.Option{ + cmpopts.IgnoreUnexported(ui.GetPropertiesFlag{}, ui.SetPropertiesFlag{}, ui.PropertiesFlag{}), + cmpopts.IgnoreUnexported(testCmd.Interface()), + cmpopts.IgnoreTypes(cmdutil.LogCmd{}, cmdutil.JSONOutputCmd{}), + cmp.Comparer(func(a, b ranklist.RankSet) bool { + return a.String() == b.String() + }), + cmp.Comparer(func(a, b ui.ByteSizeFlag) bool { + return a.String() == b.String() + }), + }...) + test.CmpAny(t, fmt.Sprintf("%s args", cmdPath), expCmd, testCmd.Interface(), cmpOpts...) +} diff --git a/src/control/cmd/dmg/pool.go b/src/control/cmd/dmg/pool.go index 0b74128990bb..f0d72cfa20cf 100644 --- a/src/control/cmd/dmg/pool.go +++ b/src/control/cmd/dmg/pool.go @@ -1,5 +1,6 @@ // // (C) Copyright 2019-2024 Intel Corporation. +// (C) Copyright 2025 Google LLC // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -698,8 +699,8 @@ func (cmd *poolQueryCmd) Execute(args []string) error { type poolQueryTargetsCmd struct { poolCmd - Rank uint32 `long:"rank" required:"1" description:"Engine rank of the targets to be queried"` - Targets string `long:"target-idx" description:"Comma-separated list of target idx(s) to be queried"` + Rank uint32 `long:"rank" required:"1" description:"Engine rank of the target(s) to be queried"` + Targets ui.RankSetFlag `long:"target-idx" description:"Comma-separated list of target index(es) to be queried (default: all)"` } // Execute is run when PoolQueryTargetsCmd subcommand is activated @@ -707,11 +708,7 @@ func (cmd *poolQueryTargetsCmd) Execute(args []string) error { ctx := cmd.MustLogCtx() var tgtsList []uint32 - if len(cmd.Targets) > 0 { - if err := common.ParseNumberList(cmd.Targets, &tgtsList); err != nil { - return errors.WithMessage(err, "parsing target list") - } - } else { + if cmd.Targets.RankSet.Count() == 0 { pi, err := control.PoolQuery(ctx, cmd.ctlInvoker, &control.PoolQueryReq{ ID: cmd.PoolID().String(), QueryMask: daos.DefaultPoolQueryMask, @@ -726,6 +723,11 @@ func (cmd *poolQueryTargetsCmd) Execute(args []string) error { for i := uint32(0); i < tgtCount; i++ { tgtsList = append(tgtsList, i) } + } else { + tgtsList = make([]uint32, cmd.Targets.RankSet.Count()) + for i, rank := range cmd.Targets.RankSet.Ranks() { + tgtsList[i] = uint32(rank) + } } req := &control.PoolQueryTargetReq{ diff --git a/src/control/common/test/utils.go b/src/control/common/test/utils.go index ee685bf568a9..aabcccbc04e8 100644 --- a/src/control/common/test/utils.go +++ b/src/control/common/test/utils.go @@ -1,5 +1,6 @@ // // (C) Copyright 2018-2024 Intel Corporation. +// (C) Copyright 2025 Google LLC // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -433,3 +434,11 @@ func MustLogContext(t *testing.T, log logging.Logger) context.Context { } return ctx } + +// JoinArgs creates a new string slice from a base string and optional +// additional string arguments. Does not modify the base string. +func JoinArgs(base []string, args ...string) []string { + joined := make([]string, len(base)) + copy(joined, base) + return append(joined, args...) +} diff --git a/src/control/lib/daos/api/api.go b/src/control/lib/daos/api/api.go index 51ef20e669a5..add21b6da57f 100644 --- a/src/control/lib/daos/api/api.go +++ b/src/control/lib/daos/api/api.go @@ -1,5 +1,6 @@ // // (C) Copyright 2024 Intel Corporation. +// (C) Copyright 2025 Google LLC // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -26,10 +27,6 @@ type ( } ) -func daosError(rc C.int) error { - return daos.ErrorFromRC(int(rc)) -} - func (api *api) isInitialized() bool { api.RLock() defer api.RUnlock() diff --git a/src/control/lib/daos/api/attribute.go b/src/control/lib/daos/api/attribute.go new file mode 100644 index 000000000000..79e2630069e2 --- /dev/null +++ b/src/control/lib/daos/api/attribute.go @@ -0,0 +1,268 @@ +// +// (C) Copyright 2018-2024 Intel Corporation. +// (C) Copyright 2025 Google LLC +// +// SPDX-License-Identifier: BSD-2-Clause-Patent +// + +package api + +import ( + "unsafe" + + "github.com/pkg/errors" + + "github.com/daos-stack/daos/src/control/lib/daos" +) + +/* +#include +#include + +#include +*/ +import "C" + +type attrType int + +const ( + poolAttr attrType = iota + contAttr +) + +func (at attrType) String() string { + switch at { + case poolAttr: + return "pool" + case contAttr: + return "container" + default: + return "unknown" + } +} + +func listDaosAttributes(hdl C.daos_handle_t, at attrType) ([]string, error) { + var rc C.int + expectedSize, totalSize := C.size_t(0), C.size_t(0) + + switch at { + case poolAttr: + rc = daos_pool_list_attr(hdl, nil, &totalSize, nil) + /*case contAttr: + rc = daos_cont_list_attr(hdl, nil, &totalSize, nil)*/ + default: + return nil, errors.Wrapf(daos.InvalidInput, "unknown attr type %d", at) + } + if err := daosError(rc); err != nil { + return nil, errors.Wrapf(err, "failed to list %s attributes", at) + } + + if totalSize < 1 { + return nil, nil + } + + attrNames := []string{} + expectedSize = totalSize + cNamesBuf := C.malloc(totalSize) + defer C.free(cNamesBuf) + + switch at { + case poolAttr: + rc = daos_pool_list_attr(hdl, (*C.char)(cNamesBuf), &totalSize, nil) + /*case contAttr: + rc = daos_cont_list_attr(hdl, (*C.char)(buf), &totalSize, nil)*/ + default: + return nil, errors.Wrapf(daos.InvalidInput, "unknown attr type %d", at) + } + if err := daosError(rc); err != nil { + return nil, errors.Wrapf(err, "failed to list %s attributes", at) + } + + if err := iterStringsBuf(cNamesBuf, expectedSize, func(name string) { + attrNames = append(attrNames, name) + }); err != nil { + return nil, err + } + + return attrNames, nil +} + +// getDaosAttributes fetches the values for the given list of attribute names. +// Uses the bulk attribute fetch API to minimize roundtrips. +func getDaosAttributes(hdl C.daos_handle_t, at attrType, reqAttrNames []string) (daos.AttributeList, error) { + if len(reqAttrNames) == 0 { + attrNameList, err := listDaosAttributes(hdl, at) + if err != nil { + return nil, errors.Wrapf(err, "failed to list %s attributes", at) + } + reqAttrNames = attrNameList + } + numAttr := len(reqAttrNames) + + if numAttr == 0 { + return nil, nil + } + + // First, build a slice of C strings for the requested attribute names. + cAttrNames := make([]*C.char, numAttr) + for i, name := range reqAttrNames { + if name == "" { + return nil, errors.Wrapf(daos.InvalidInput, "empty %s attribute name at index %d", at, i) + } + cAttrNames[i] = C.CString(name) + } + defer func(nameSlice []*C.char) { + for _, name := range nameSlice { + freeString(name) + } + }(cAttrNames) + + // Next, create a slice of C.size_t entries to hold the sizes of the values. + // We have to do this first in order to know the buffer sizes to allocate + // before fetching the actual values. + cAttrSizes := make([]C.size_t, numAttr) + var rc C.int + switch at { + case poolAttr: + rc = daos_pool_get_attr(hdl, C.int(numAttr), &cAttrNames[0], nil, &cAttrSizes[0], nil) + /*case contAttr: + rc = daos_cont_get_attr(hdl, C.int(numAttr), &attrNames[0], nil, &attrSizes[0], nil)*/ + default: + return nil, errors.Wrapf(daos.InvalidInput, "unknown attr type %d", at) + } + if err := daosError(rc); err != nil { + return nil, errors.Wrapf(err, "failed to get %s attribute sizes", at) + } + + // Now, create a slice of buffers to hold the values. + cAttrValues := make([]unsafe.Pointer, numAttr) + defer func(valueSlice []unsafe.Pointer) { + for _, value := range valueSlice { + C.free(value) + } + }(cAttrValues) + for i, size := range cAttrSizes { + if size < 1 { + return nil, errors.Wrapf(daos.MiscError, "failed to get %s attribute %s: size is %d", at, reqAttrNames[i], size) + } + + cAttrValues[i] = C.malloc(size) + } + + // Do the actual fetch of all values in one go. + switch at { + case poolAttr: + rc = daos_pool_get_attr(hdl, C.int(numAttr), &cAttrNames[0], &cAttrValues[0], &cAttrSizes[0], nil) + /*case contAttr: + rc = daos_cont_get_attr(hdl, C.int(numAttr), &attrNames[0], &attrValues[0], &attrSizes[0], nil)*/ + default: + return nil, errors.Wrapf(daos.InvalidInput, "unknown attr type %d", at) + } + if err := daosError(rc); err != nil { + return nil, errors.Wrapf(err, "failed to get %s attribute values", at) + } + + // Finally, create a slice of attribute structs to hold the results. + // Note that we are copying the values into Go-managed byte slices + // for safety and simplicity so that we can free the C memory as soon + // as this function exits. + attrs := make([]*daos.Attribute, numAttr) + for i, name := range reqAttrNames { + attrs[i] = &daos.Attribute{ + Name: name, + Value: C.GoBytes(cAttrValues[i], C.int(cAttrSizes[i])), + } + } + + return attrs, nil +} + +// setDaosAttributes sets the values for the given list of attribute names. +// Uses the bulk attribute set API to minimize roundtrips. +func setDaosAttributes(hdl C.daos_handle_t, at attrType, attrs daos.AttributeList) error { + if len(attrs) == 0 { + return errors.Wrapf(daos.InvalidInput, "no %s attributes provided", at) + } + + // First, build a slice of C strings for the attribute names. + attrNames := make([]*C.char, len(attrs)) + for i, attr := range attrs { + if attr == nil { + return errors.Wrapf(daos.InvalidInput, "nil %s attribute at index %d", at, i) + } + if attr.Name == "" { + return errors.Wrapf(daos.InvalidInput, "empty %s attribute name at index %d", at, i) + } + attrNames[i] = C.CString(attr.Name) + } + defer func(nameSlice []*C.char) { + for _, name := range nameSlice { + freeString(name) + } + }(attrNames) + + // Next, create a slice of C.size_t entries to hold the sizes of the values, + // and a slice of pointers to the actual values. + attrSizes := make([]C.size_t, len(attrs)) + attrValues := make([]unsafe.Pointer, len(attrs)) + for i, attr := range attrs { + attrSizes[i] = C.size_t(len(attr.Value)) + if attrSizes[i] == 0 { + return errors.Wrapf(daos.InvalidInput, "empty %s attribute value at index %d", at, i) + } + // NB: We are copying the values into C memory for safety and simplicity. + attrValues[i] = C.malloc(attrSizes[i]) + valSlice := unsafe.Slice((*byte)(attrValues[i]), attrSizes[i]) + copy(valSlice[:], attr.Value) + } + defer func(bufSlice []unsafe.Pointer) { + for _, buf := range bufSlice { + C.free(buf) + } + }(attrValues) + + attrCount := C.int(len(attrs)) + var rc C.int + switch at { + case poolAttr: + rc = daos_pool_set_attr(hdl, attrCount, &attrNames[0], &attrValues[0], &attrSizes[0], nil) + /*case contAttr: + rc = daos_cont_set_attr(hdl, attrCount, &attrNames[0], &valBufs[0], &valSizes[0], nil)*/ + default: + return errors.Wrapf(daos.InvalidInput, "unknown attr type %d", at) + } + + return errors.Wrapf(daosError(rc), "failed to set %s attributes", at) +} + +// delDaosAttributes deletes the given attributes. +func delDaosAttributes(hdl C.daos_handle_t, at attrType, names []string) error { + if len(names) == 0 { + return errors.Wrapf(daos.InvalidInput, "no %s attribute names provided", at) + } + + attrNames := make([]*C.char, len(names)) + for i, name := range names { + if name == "" { + return errors.Wrapf(daos.InvalidInput, "empty %s attribute name at index %d", at, i) + } + attrNames[i] = C.CString(name) + } + defer func(nameSlice []*C.char) { + for _, name := range nameSlice { + freeString(name) + } + }(attrNames) + + var rc C.int + switch at { + case poolAttr: + rc = daos_pool_del_attr(hdl, C.int(len(attrNames)), &attrNames[0], nil) + /*case contAttr: + rc = daos_cont_del_attr(hdl, 1, &attrName, nil)*/ + default: + return errors.Wrapf(daos.InvalidInput, "unknown attr type %d", at) + } + + return errors.Wrapf(daosError(rc), "failed to delete %s attributes", at) +} diff --git a/src/control/lib/daos/api/errors.go b/src/control/lib/daos/api/errors.go index 6d1b4b665e30..f82afb52a83f 100644 --- a/src/control/lib/daos/api/errors.go +++ b/src/control/lib/daos/api/errors.go @@ -1,13 +1,61 @@ // // (C) Copyright 2024 Intel Corporation. +// (C) Copyright 2025 Google LLC // // SPDX-License-Identifier: BSD-2-Clause-Patent // package api -import "github.com/pkg/errors" +import ( + "context" + + "github.com/pkg/errors" + + "github.com/daos-stack/daos/src/control/lib/daos" +) + +/* +#include +*/ +import "C" var ( - ErrNoSystemRanks = errors.New("no ranks in system") + ErrNoSystemRanks = errors.New("no ranks in system") + ErrInvalidPoolHandle = errors.New("pool handle is nil or invalid") + + errInvalidContainerHandle = errors.New("container handle is nil or invalid") + errNilCtx = errors.New("nil context") + errNoCtxHdl = errors.New("no handle in context") ) + +// dfsError converts a return code from a DFS API +// call to a Go error. +func dfsError(rc C.int) error { + if rc == 0 { + return nil + } + + strErr := C.strerror(rc) + return errors.Errorf("DFS error %d: %s", rc, C.GoString(strErr)) +} + +// daosError converts a return code from a DAOS API +// call to a Go error. +func daosError(rc C.int) error { + return daos.ErrorFromRC(int(rc)) +} + +// ctxErr recasts a context error as a DAOS error. +func ctxErr(err error) error { + switch { + case err == nil: + return nil + case errors.Is(err, context.Canceled): + return errors.Wrap(daos.Canceled, "DAOS API context canceled") + case errors.Is(err, context.DeadlineExceeded): + return errors.Wrap(daos.TimedOut, "DAOS API context deadline exceeded") + default: + return errors.Wrap(daos.MiscError, "DAOS API context error") + } +} diff --git a/src/control/lib/daos/api/handle.go b/src/control/lib/daos/api/handle.go new file mode 100644 index 000000000000..e729b7401e59 --- /dev/null +++ b/src/control/lib/daos/api/handle.go @@ -0,0 +1,92 @@ +// +// (C) Copyright 2025 Google LLC +// +// SPDX-License-Identifier: BSD-2-Clause-Patent +// + +package api + +import ( + "fmt" + "unsafe" + + "github.com/google/uuid" + "github.com/pkg/errors" + + "github.com/daos-stack/daos/src/control/logging" +) + +/* +#include + +#cgo LDFLAGS: -ldaos_common +*/ +import "C" + +const ( + MissingPoolLabel = "" + MissingContainerLabel = "" +) + +type ( + // ctxHdlKey is a type used for storing handles as context values. + ctxHdlKey string + + // connHandle is an opaque type used to represent a DAOS connection (pool or container). + connHandle struct { + UUID uuid.UUID + Label string + daosHandle C.daos_handle_t + } +) + +// invalidate clears the handle so that it cannot be reused inadvertently. +func (ch *connHandle) invalidate() { + if ch == nil { + return + } + ch.UUID = uuid.Nil + ch.Label = "" + ch.daosHandle.cookie = 0 +} + +// FillHandle copies the handle to the supplied pointer, +// which must be a reference to a C.daos_handle_t. +// NB: Caller is responsible for keeping the copy in sync with +// this handle -- use of this method should be discouraged as +// it is provided for compatibility with older code that calls +// into libdaos directly. +func (ch *connHandle) FillHandle(cHandle unsafe.Pointer) error { + if ch == nil || cHandle == nil { + return errors.New("invalid handle") + } + (*C.daos_handle_t)(cHandle).cookie = ch.daosHandle.cookie + + return nil +} + +// IsValid returns true if the pool or container handle is valid. +func (ch *connHandle) IsValid() bool { + if ch == nil { + return false + } + return bool(daos_handle_is_valid(ch.daosHandle)) +} + +// ID returns the label if available, otherwise the UUID. +func (ch *connHandle) ID() string { + id := ch.Label + if id == "" || id == MissingPoolLabel || id == MissingContainerLabel { + id = ch.UUID.String() + } + + return id +} + +func (ch *connHandle) String() string { + id := ch.Label + if id == "" || id == MissingPoolLabel || id == MissingContainerLabel { + id = logging.ShortUUID(ch.UUID) + } + return fmt.Sprintf("%s:%t", id, ch.IsValid()) +} diff --git a/src/control/lib/daos/api/libdaos.go b/src/control/lib/daos/api/libdaos.go index d7c6bfed82d2..426507b98adf 100644 --- a/src/control/lib/daos/api/libdaos.go +++ b/src/control/lib/daos/api/libdaos.go @@ -1,5 +1,6 @@ // // (C) Copyright 2024 Intel Corporation. +// (C) Copyright 2025 Google LLC // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -12,10 +13,12 @@ package api #include #include #include +#include #cgo LDFLAGS: -lcart -lgurt -ldaos -ldaos_common */ import "C" +import "unsafe" func daos_init() C.int { return C.daos_init() @@ -29,6 +32,10 @@ func dc_agent_fini() { C.dc_agent_fini() } +func daos_handle_is_valid(handle C.daos_handle_t) C.bool { + return C.daos_handle_is_valid(handle) +} + func daos_mgmt_get_sys_info(sys *C.char, sys_info **C.struct_daos_sys_info) C.int { return C.daos_mgmt_get_sys_info(sys, sys_info) } @@ -36,3 +43,51 @@ func daos_mgmt_get_sys_info(sys *C.char, sys_info **C.struct_daos_sys_info) C.in func daos_mgmt_put_sys_info(sys_info *C.struct_daos_sys_info) { C.daos_mgmt_put_sys_info(sys_info) } + +func daos_pool_connect(poolID *C.char, sys *C.char, flags C.uint32_t, poolHdl *C.daos_handle_t, poolInfo *C.daos_pool_info_t, ev *C.struct_daos_event) C.int { + return C.daos_pool_connect(poolID, sys, flags, poolHdl, poolInfo, ev) +} + +func daos_pool_disconnect(poolHdl C.daos_handle_t) C.int { + // Hack for NLT fault injection testing: If the rc + // is -DER_NOMEM, retry once in order to actually + // shut down and release resources. + rc := C.daos_pool_disconnect(poolHdl, nil) + if rc == -C.DER_NOMEM { + rc = C.daos_pool_disconnect(poolHdl, nil) + // DAOS-8866, daos_pool_disconnect() might have failed, but worked anyway. + if rc == -C.DER_NO_HDL { + rc = -C.DER_SUCCESS + } + } + + return rc +} + +func daos_pool_query(poolHdl C.daos_handle_t, rankList **C.d_rank_list_t, poolInfo *C.daos_pool_info_t, props *C.daos_prop_t, ev *C.struct_daos_event) C.int { + return C.daos_pool_query(poolHdl, rankList, poolInfo, props, ev) +} + +func daos_pool_query_target(poolHdl C.daos_handle_t, tgt C.uint32_t, rank C.uint32_t, info *C.daos_target_info_t, ev *C.struct_daos_event) C.int { + return C.daos_pool_query_target(poolHdl, tgt, rank, info, ev) +} + +func daos_pool_list_attr(poolHdl C.daos_handle_t, buf *C.char, size *C.size_t, ev *C.struct_daos_event) C.int { + return C.daos_pool_list_attr(poolHdl, buf, size, ev) +} + +func daos_pool_get_attr(poolHdl C.daos_handle_t, n C.int, names **C.char, values *unsafe.Pointer, sizes *C.size_t, ev *C.struct_daos_event) C.int { + return C.daos_pool_get_attr(poolHdl, n, names, values, sizes, ev) +} + +func daos_pool_set_attr(poolHdl C.daos_handle_t, n C.int, names **C.char, values *unsafe.Pointer, sizes *C.size_t, ev *C.struct_daos_event) C.int { + return C.daos_pool_set_attr(poolHdl, n, names, values, sizes, ev) +} + +func daos_pool_del_attr(poolHdl C.daos_handle_t, n C.int, name **C.char, ev *C.struct_daos_event) C.int { + return C.daos_pool_del_attr(poolHdl, n, name, ev) +} + +func daos_mgmt_list_pools(sysName *C.char, poolCount *C.daos_size_t, pools *C.daos_mgmt_pool_info_t, ev *C.struct_daos_event) C.int { + return C.daos_mgmt_list_pools(sysName, poolCount, pools, ev) +} diff --git a/src/control/lib/daos/api/libdaos_attr_stubs.go b/src/control/lib/daos/api/libdaos_attr_stubs.go new file mode 100644 index 000000000000..f3446fa56ee1 --- /dev/null +++ b/src/control/lib/daos/api/libdaos_attr_stubs.go @@ -0,0 +1,175 @@ +// +// (C) Copyright 2025 Google LLC +// +// SPDX-License-Identifier: BSD-2-Clause-Patent +// +//go:build test_stubs +// +build test_stubs + +package api + +import ( + "unsafe" + + "github.com/daos-stack/daos/src/control/lib/daos" +) + +import "C" + +var ( + daos_default_AttrList daos.AttributeList = daos.AttributeList{ + { + Name: "one", + Value: []byte("1"), + }, + { + Name: "two", + Value: []byte("2"), + }, + { + Name: "three", + Value: []byte("3"), + }, + } +) + +func daos_test_get_mappedNames(nameMap map[string]struct{}) []string { + names := make([]string, 0, len(nameMap)) + for name := range nameMap { + names = append(names, name) + } + return names +} + +func list_attrs(buf *C.char, size *C.size_t, RCList []C.int, CallCount *int, RC C.int, AttrList daos.AttributeList) C.int { + if len(RCList) > 0 { + rc := RCList[*CallCount] + *CallCount++ + if rc != 0 { + return rc + } + } + if RC != 0 { + return RC + } + + bufSize := 0 + for _, attr := range AttrList { + bufSize += len(attr.Name) + 1 + } + *size = C.size_t(bufSize) + + if buf == nil { + return RC + } + + bufSlice := unsafe.Slice((*C.char)(buf), bufSize) + bufPtr := 0 + for _, attr := range AttrList { + for i := 0; i < len(attr.Name); i++ { + bufSlice[bufPtr] = C.char(attr.Name[i]) + bufPtr++ + } + bufSlice[bufPtr] = C.char(0) + bufPtr++ + } + + return RC +} + +func get_attr(n C.int, names **C.char, values *unsafe.Pointer, sizes *C.size_t, + RCList []C.int, CallCount *int, RC C.int, AttrList daos.AttributeList, SetN *int, ReqNames *map[string]struct{}) C.int { + if len(RCList) > 0 { + rc := RCList[*CallCount] + *CallCount++ + if rc != 0 { + return rc + } + } + if RC != 0 { + return RC + } + + *SetN = int(n) + *ReqNames = make(map[string]struct{}) + cReqNames := unsafe.Slice(names, n) + for i := 0; i < int(n); i++ { + reqNames := *ReqNames + reqNames[C.GoString(cReqNames[i])] = struct{}{} + } + + if len(*ReqNames) > 0 && len(AttrList) == 0 { + return -C.int(daos.Nonexistent) + } + + attrListMap := AttrList.AsMap() + reqAttrCt := 0 + for attrName := range *ReqNames { + if _, ok := attrListMap[attrName]; !ok { + return -C.int(daos.Nonexistent) + } + reqAttrCt++ + } + + if reqAttrCt == 0 { + return RC + } + + var valuesSlice []unsafe.Pointer + if values != nil { + valuesSlice = unsafe.Slice(values, reqAttrCt) + } + sizesSlice := unsafe.Slice(sizes, reqAttrCt) + idx := 0 + for _, attr := range AttrList { + reqNames := *ReqNames + if _, ok := reqNames[attr.Name]; !ok { + continue + } + sizesSlice[idx] = C.size_t(len(attr.Value)) + if values != nil { + valSlice := unsafe.Slice((*byte)(valuesSlice[idx]), sizesSlice[idx]) + copy(valSlice[:], attr.Value) + } + idx++ + } + + return RC +} + +func set_attr(n C.int, names **C.char, values *unsafe.Pointer, sizes *C.size_t, RC C.int, AttrList *daos.AttributeList) C.int { + if RC != 0 { + return RC + } + + namesSlice := unsafe.Slice(names, n) + valuesSlice := unsafe.Slice(values, n) + sizesSlice := unsafe.Slice(sizes, n) + attrList := *AttrList + for i := 0; i < int(n); i++ { + valueSlice := unsafe.Slice((*byte)(valuesSlice[i]), sizesSlice[i]) + attrList = append(attrList, &daos.Attribute{ + Name: C.GoString(namesSlice[i]), + Value: make([]byte, sizesSlice[i]), + }) + copy(attrList[len(attrList)-1].Value, valueSlice) + } + *AttrList = attrList + + return RC +} + +func del_attr(n C.int, name **C.char, RC C.int, AttrNames *[]string) C.int { + if RC != 0 { + return RC + } + + attrNames := *AttrNames + nameSlice := unsafe.Slice(name, n) + for i := 0; i < int(n); i++ { + attrNames = append(attrNames, C.GoString(nameSlice[i])) + } + *AttrNames = attrNames + + return RC +} diff --git a/src/control/lib/daos/api/libdaos_pool_stubs.go b/src/control/lib/daos/api/libdaos_pool_stubs.go new file mode 100644 index 000000000000..ae7b57d931fc --- /dev/null +++ b/src/control/lib/daos/api/libdaos_pool_stubs.go @@ -0,0 +1,450 @@ +// +// (C) Copyright 2025 Google LLC +// +// SPDX-License-Identifier: BSD-2-Clause-Patent +// +//go:build test_stubs +// +build test_stubs + +package api + +import ( + "unsafe" + + "github.com/dustin/go-humanize" + + "github.com/daos-stack/daos/src/control/common/test" + "github.com/daos-stack/daos/src/control/lib/daos" + "github.com/daos-stack/daos/src/control/lib/ranklist" +) + +/* +#include +#include +#include + +#include "util.h" + +static inline void +set_rebuild_state(struct daos_rebuild_status *drs, int32_t state) +{ + drs->rs_state = state; +} +*/ +import "C" + +func daos_gds2cds(sus []*daos.StorageUsageStats) C.struct_daos_space { + return C.struct_daos_space{ + s_total: [2]C.uint64_t{ + C.uint64_t(sus[0].Total), + C.uint64_t(sus[1].Total), + }, + s_free: [2]C.uint64_t{ + C.uint64_t(sus[0].Free), + C.uint64_t(sus[1].Free), + }, + } +} + +func daos_gpi2cpi(gpi *daos.PoolInfo) *C.daos_pool_info_t { + cpi := &C.daos_pool_info_t{ + pi_uuid: uuidToC(gpi.UUID), + pi_ntargets: C.uint32_t(gpi.TotalTargets), + pi_nnodes: C.uint32_t(gpi.TotalEngines), + pi_ndisabled: C.uint32_t(gpi.DisabledTargets), + pi_map_ver: C.uint32_t(gpi.Version), + pi_leader: C.uint32_t(gpi.ServiceLeader), + pi_bits: C.uint64_t(gpi.QueryMask), + pi_rebuild_st: C.struct_daos_rebuild_status{ + rs_errno: C.int32_t(gpi.Rebuild.Status), + rs_obj_nr: C.uint64_t(gpi.Rebuild.Objects), + rs_rec_nr: C.uint64_t(gpi.Rebuild.Records), + }, + pi_space: C.struct_daos_pool_space{ + ps_ntargets: C.uint32_t(gpi.ActiveTargets), + ps_space: daos_gds2cds(gpi.TierStats), + ps_free_min: [2]C.uint64_t{ + C.uint64_t(gpi.TierStats[0].Min), + C.uint64_t(gpi.TierStats[1].Min), + }, + ps_free_max: [2]C.uint64_t{ + C.uint64_t(gpi.TierStats[0].Max), + C.uint64_t(gpi.TierStats[1].Max), + }, + ps_free_mean: [2]C.uint64_t{ + C.uint64_t(gpi.TierStats[0].Mean), + C.uint64_t(gpi.TierStats[1].Mean), + }, + }, + } + + // some funky mismatch between the Go/C states... fix this later. + switch gpi.Rebuild.State { + case daos.PoolRebuildStateIdle: + cpi.pi_rebuild_st.rs_version = 0 + case daos.PoolRebuildStateBusy: + cpi.pi_rebuild_st.rs_version = 1 + C.set_rebuild_state(&cpi.pi_rebuild_st, C.DRS_IN_PROGRESS) + case daos.PoolRebuildStateDone: + cpi.pi_rebuild_st.rs_version = 1 + C.set_rebuild_state(&cpi.pi_rebuild_st, C.DRS_COMPLETED) + } + return cpi +} + +// defaultPoolInfo should be used to get a copy of the default pool info. +func defaultPoolInfo() *daos.PoolInfo { + return copyPoolInfo(&daos_default_PoolInfo) +} + +func copyPoolInfo(in *daos.PoolInfo) *daos.PoolInfo { + if in == nil { + return nil + } + + out := new(daos.PoolInfo) + *out = *in + + if in.Rebuild != nil { + out.Rebuild = new(daos.PoolRebuildStatus) + *out.Rebuild = *in.Rebuild + } + if in.TierStats != nil { + out.TierStats = make([]*daos.StorageUsageStats, len(in.TierStats)) + for i, s := range in.TierStats { + out.TierStats[i] = new(daos.StorageUsageStats) + *out.TierStats[i] = *s + } + } + if in.ServiceReplicas != nil { + out.ServiceReplicas = make([]ranklist.Rank, len(in.ServiceReplicas)) + copy(out.ServiceReplicas, in.ServiceReplicas) + } + if in.EnabledRanks != nil { + out.EnabledRanks = ranklist.NewRankSet() + out.EnabledRanks.Replace(in.EnabledRanks) + } + if in.DisabledRanks != nil { + out.DisabledRanks = ranklist.NewRankSet() + out.DisabledRanks.Replace(in.DisabledRanks) + } + + return out +} + +var ( + daos_default_pool_connect_Handle C.daos_handle_t = C.daos_handle_t{cookie: 42} + + daos_default_PoolInfo daos.PoolInfo = daos.PoolInfo{ + QueryMask: daos.DefaultPoolQueryMask, + State: daos.PoolServiceStateDegraded, + UUID: test.MockPoolUUID(1), + Label: "test-pool", + TotalTargets: 48, + TotalEngines: 3, + ActiveTargets: 32, + DisabledTargets: 16, + Version: 2, + ServiceLeader: 1, + ServiceReplicas: []ranklist.Rank{0, 1, 2}, + EnabledRanks: ranklist.MustCreateRankSet("0,2"), + DisabledRanks: ranklist.MustCreateRankSet("1"), + Rebuild: &daos.PoolRebuildStatus{ + Status: 0, + Objects: 1, + Records: 2, + State: daos.PoolRebuildStateBusy, + }, + TierStats: []*daos.StorageUsageStats{ + { + MediaType: daos.StorageMediaTypeScm, + Total: 64 * humanize.TByte, + Free: 16 * humanize.TByte, + Min: 1 * humanize.TByte, + Max: 4 * humanize.TByte, + Mean: 2 * humanize.TByte, + }, + { + MediaType: daos.StorageMediaTypeNvme, + Total: 64 * humanize.PByte, + Free: 16 * humanize.PByte, + Min: 1 * humanize.PByte, + Max: 4 * humanize.PByte, + Mean: 2 * humanize.PByte, + }, + }, + } + + daos_default_PoolQueryTargetInfo daos.PoolQueryTargetInfo = daos.PoolQueryTargetInfo{ + Type: daos.PoolQueryTargetType(1), + State: daos.PoolTargetStateUp, + Space: func() []*daos.StorageUsageStats { + tiStats := make([]*daos.StorageUsageStats, len(daos_default_PoolInfo.TierStats)) + for i, tier := range daos_default_PoolInfo.TierStats { + tiStats[i] = &daos.StorageUsageStats{ + MediaType: tier.MediaType, + Total: tier.Total, + Free: tier.Free, + } + } + return tiStats + }(), + } +) + +func defaultPoolHdl() *C.daos_handle_t { + newHdl := C.daos_handle_t{cookie: daos_default_pool_connect_Handle.cookie} + return &newHdl +} + +var ( + daos_pool_connect_SetPoolID string + daos_pool_connect_SetSys string + daos_pool_connect_SetFlags daos.PoolConnectFlag + daos_pool_connect_QueryMask daos.PoolQueryMask + daos_pool_connect_Handle *C.daos_handle_t = defaultPoolHdl() + daos_pool_connect_Info *daos.PoolInfo = defaultPoolInfo() + daos_pool_connect_RC C.int = 0 +) + +func reset_daos_pool_connect() { + daos_pool_connect_SetPoolID = "" + daos_pool_connect_SetSys = "" + daos_pool_connect_SetFlags = 0 + daos_pool_connect_QueryMask = 0 + daos_pool_connect_Handle = defaultPoolHdl() + daos_pool_connect_Info = defaultPoolInfo() + daos_pool_connect_RC = 0 +} + +func daos_pool_connect(poolID *C.char, sys *C.char, flags C.uint32_t, poolHdl *C.daos_handle_t, poolInfo *C.daos_pool_info_t, ev *C.struct_daos_event) C.int { + if daos_pool_connect_RC != 0 { + return daos_pool_connect_RC + } + + // capture the parameters set by the test + daos_pool_connect_SetPoolID = C.GoString(poolID) + daos_pool_connect_SetSys = C.GoString(sys) + daos_pool_connect_SetFlags = daos.PoolConnectFlag(flags) + daos_pool_connect_QueryMask = daos.PoolQueryMask(poolInfo.pi_bits) + + // set the return values + poolHdl.cookie = daos_pool_connect_Handle.cookie + *poolInfo = *daos_gpi2cpi(daos_pool_connect_Info) + + return daos_pool_connect_RC +} + +var ( + daos_pool_disconnect_RC C.int = 0 +) + +func daos_pool_disconnect(poolHdl C.daos_handle_t) C.int { + return daos_pool_disconnect_RC +} + +var ( + daos_pool_query_PoolInfo *daos.PoolInfo = defaultPoolInfo() + daos_pool_query_RC C.int = 0 +) + +func reset_daos_pool_query() { + daos_pool_query_PoolInfo = defaultPoolInfo() + daos_pool_query_RC = 0 +} + +func daos_pool_query(poolHdl C.daos_handle_t, rankList **C.d_rank_list_t, retPoolInfo *C.daos_pool_info_t, props *C.daos_prop_t, ev *C.struct_daos_event) C.int { + if daos_pool_query_RC != 0 { + return daos_pool_query_RC + } + + if retPoolInfo == nil { + *rankList = ranklistFromGo(daos_pool_query_PoolInfo.DisabledRanks) + return daos_pool_query_RC + } + + queryBits := retPoolInfo.pi_bits + *retPoolInfo = *daos_gpi2cpi(daos_pool_query_PoolInfo) + retPoolInfo.pi_bits = queryBits + + if queryBits&C.DPI_ENGINES_ENABLED != 0 { + *rankList = ranklistFromGo(daos_pool_query_PoolInfo.EnabledRanks) + } + if queryBits&C.DPI_ENGINES_DISABLED != 0 { + *rankList = ranklistFromGo(daos_pool_query_PoolInfo.DisabledRanks) + } + + if props != nil { + propEntries := unsafe.Slice(props.dpp_entries, props.dpp_nr) + for i := range propEntries { + switch propEntries[i].dpe_type { + case C.DAOS_PROP_PO_LABEL: + C.set_dpe_str(&propEntries[i], C.CString(daos_pool_query_PoolInfo.Label)) + case C.DAOS_PROP_PO_SVC_LIST: + rlPtr := ranklistFromGo(ranklist.RankSetFromRanks(daos_pool_query_PoolInfo.ServiceReplicas)) + C.set_dpe_val_ptr(&propEntries[i], (unsafe.Pointer)(rlPtr)) + } + } + } + + return daos_pool_query_RC +} + +var ( + daos_pool_query_target_SetTgt C.uint32_t = C.uint32_t(ranklist.NilRank) + daos_pool_query_target_SetRank C.uint32_t = C.uint32_t(ranklist.NilRank) + daos_pool_query_target_Info *daos.PoolQueryTargetInfo = &daos_default_PoolQueryTargetInfo + daos_pool_query_target_RC C.int = 0 +) + +func reset_daos_pool_query_target() { + daos_pool_query_target_SetTgt = C.uint32_t(ranklist.NilRank) + daos_pool_query_target_SetRank = C.uint32_t(ranklist.NilRank) + daos_pool_query_target_Info = &daos_default_PoolQueryTargetInfo + daos_pool_query_target_RC = 0 +} + +func daos_pool_query_target(poolHdl C.daos_handle_t, tgt C.uint32_t, rank C.uint32_t, info *C.daos_target_info_t, ev *C.struct_daos_event) C.int { + if daos_pool_query_target_RC != 0 { + return daos_pool_query_target_RC + } + + daos_pool_query_target_SetTgt = tgt + daos_pool_query_target_SetRank = rank + + info.ta_type = C.daos_target_type_t(daos_pool_query_target_Info.Type) + info.ta_state = C.daos_target_state_t(daos_pool_query_target_Info.State) + info.ta_space = daos_gds2cds(daos_pool_query_target_Info.Space) + + return daos_pool_query_target_RC +} + +var ( + daos_pool_list_attr_AttrList daos.AttributeList = daos_default_AttrList + daos_pool_list_attr_CallCount int + daos_pool_list_attr_RCList []C.int + daos_pool_list_attr_RC C.int = 0 +) + +func reset_daos_pool_list_attr() { + daos_pool_list_attr_AttrList = daos_default_AttrList + daos_pool_list_attr_CallCount = 0 + daos_pool_list_attr_RCList = nil + daos_pool_list_attr_RC = 0 +} + +func daos_pool_list_attr(poolHdl C.daos_handle_t, buf *C.char, size *C.size_t, ev *C.struct_daos_event) C.int { + return list_attrs(buf, size, daos_pool_list_attr_RCList, &daos_pool_list_attr_CallCount, daos_pool_list_attr_RC, daos_pool_list_attr_AttrList) +} + +var ( + daos_pool_get_attr_SetN int + daos_pool_get_attr_ReqNames map[string]struct{} + daos_pool_get_attr_CallCount int + daos_pool_get_attr_RCList []C.int + daos_pool_get_attr_AttrList daos.AttributeList = daos_default_AttrList + daos_pool_get_attr_RC C.int = 0 +) + +func reset_daos_pool_get_attr() { + daos_pool_get_attr_SetN = 0 + daos_pool_get_attr_ReqNames = nil + daos_pool_get_attr_CallCount = 0 + daos_pool_get_attr_RCList = nil + daos_pool_get_attr_AttrList = daos_default_AttrList + daos_pool_get_attr_RC = 0 +} + +func daos_pool_get_attr(poolHdl C.daos_handle_t, n C.int, names **C.char, values *unsafe.Pointer, sizes *C.size_t, ev *C.struct_daos_event) C.int { + return get_attr(n, names, values, sizes, daos_pool_get_attr_RCList, &daos_pool_get_attr_CallCount, daos_pool_get_attr_RC, daos_pool_get_attr_AttrList, &daos_pool_get_attr_SetN, &daos_pool_get_attr_ReqNames) +} + +var ( + daos_pool_set_attr_AttrList daos.AttributeList + daos_pool_set_attr_RC C.int = 0 +) + +func reset_daos_pool_set_attr() { + daos_pool_set_attr_AttrList = nil + daos_pool_set_attr_RC = 0 +} + +func daos_pool_set_attr(poolHdl C.daos_handle_t, n C.int, names **C.char, values *unsafe.Pointer, sizes *C.size_t, ev *C.struct_daos_event) C.int { + return set_attr(n, names, values, sizes, daos_pool_set_attr_RC, &daos_pool_set_attr_AttrList) +} + +var ( + daos_pool_del_attr_AttrNames []string + daos_pool_del_attr_RC C.int = 0 +) + +func reset_daos_pool_del_attr() { + daos_pool_del_attr_AttrNames = nil + daos_pool_del_attr_RC = 0 +} + +func daos_pool_del_attr(poolHdl C.daos_handle_t, n C.int, name **C.char, ev *C.struct_daos_event) C.int { + return del_attr(n, name, daos_pool_del_attr_RC, &daos_pool_del_attr_AttrNames) +} + +var ( + daos_pool_list_cont_RC C.int = 0 +) + +func reset_daos_pool_list_cont() { + daos_pool_list_cont_RC = 0 +} + +func daos_pool_list_cont(poolHdl C.daos_handle_t, nCont *C.daos_size_t, conts *C.struct_daos_pool_cont_info, ev *C.struct_daos_event) C.int { + if daos_pool_list_cont_RC != 0 { + return daos_pool_list_cont_RC + } + + return daos_pool_list_cont_RC +} + +var ( + daos_mgmt_list_pools_SetSys string + daos_mgmt_list_pools_RetPools []*daos.PoolInfo = []*daos.PoolInfo{defaultPoolInfo()} + daos_mgmt_list_pools_CallCount int + daos_mgmt_list_pools_RCList []C.int + daos_mgmt_list_pools_RC C.int = 0 +) + +func reset_daos_mgmt_list_pools() { + daos_mgmt_list_pools_SetSys = "" + daos_mgmt_list_pools_RetPools = []*daos.PoolInfo{defaultPoolInfo()} + daos_mgmt_list_pools_CallCount = 0 + daos_mgmt_list_pools_RCList = nil + daos_mgmt_list_pools_RC = 0 +} + +func daos_mgmt_list_pools(sysName *C.char, poolCount *C.daos_size_t, pools *C.daos_mgmt_pool_info_t, ev *C.struct_daos_event) C.int { + if len(daos_mgmt_list_pools_RCList) > 0 { + rc := daos_mgmt_list_pools_RCList[daos_mgmt_list_pools_CallCount] + daos_mgmt_list_pools_CallCount++ + if rc != 0 { + return rc + } + } + if daos_mgmt_list_pools_RC != 0 { + return daos_mgmt_list_pools_RC + } + + *poolCount = C.daos_size_t(len(daos_mgmt_list_pools_RetPools)) + + daos_mgmt_list_pools_SetSys = C.GoString(sysName) + if *poolCount == 0 || pools == nil { + return daos_mgmt_list_pools_RC + } + + poolSlice := unsafe.Slice(pools, *poolCount) + for i, pool := range daos_mgmt_list_pools_RetPools { + poolSlice[i].mgpi_uuid = uuidToC(pool.UUID) + poolSlice[i].mgpi_label = C.CString(pool.Label) + poolSlice[i].mgpi_svc = ranklistFromGo(ranklist.RankSetFromRanks(pool.ServiceReplicas)) + poolSlice[i].mgpi_ldr = C.d_rank_t(pool.ServiceLeader) + } + + return daos_mgmt_list_pools_RC +} diff --git a/src/control/lib/daos/api/libdaos_stubs.go b/src/control/lib/daos/api/libdaos_stubs.go index 341b90bdd34a..e7c00a4076d1 100644 --- a/src/control/lib/daos/api/libdaos_stubs.go +++ b/src/control/lib/daos/api/libdaos_stubs.go @@ -1,5 +1,6 @@ // // (C) Copyright 2024 Intel Corporation. +// (C) Copyright 2025 Google LLC // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -18,9 +19,26 @@ import ( /* #include #include +#include + +#include "util.h" + */ import "C" +// ResetTestStubs will call the reset functions for all test stubs in order +// to reset state between tests. +func ResetTestStubs() { + reset_daos_pool_connect() + reset_daos_pool_query() + reset_daos_pool_query_target() + reset_daos_pool_list_attr() + reset_daos_pool_get_attr() + reset_daos_pool_set_attr() + reset_daos_pool_del_attr() + reset_daos_mgmt_list_pools() +} + var ( daos_init_RC C.int = 0 ) @@ -33,6 +51,14 @@ func daos_fini() {} func dc_agent_fini() {} +var ( + daos_handle_is_valid_Bool C.bool = true +) + +func daos_handle_is_valid(handle C.daos_handle_t) C.bool { + return daos_handle_is_valid_Bool +} + var ( defaultSystemInfo *daos.SystemInfo = &daos.SystemInfo{ Name: build.DefaultSystemName, diff --git a/src/control/lib/daos/api/pool.go b/src/control/lib/daos/api/pool.go new file mode 100644 index 000000000000..c4dad3d337f6 --- /dev/null +++ b/src/control/lib/daos/api/pool.go @@ -0,0 +1,675 @@ +// +// (C) Copyright 2025 Google LLC +// +// SPDX-License-Identifier: BSD-2-Clause-Patent +// + +package api + +import ( + "context" + "unsafe" + + "github.com/google/uuid" + "github.com/pkg/errors" + + "github.com/daos-stack/daos/src/control/build" + "github.com/daos-stack/daos/src/control/lib/daos" + "github.com/daos-stack/daos/src/control/lib/ranklist" + "github.com/daos-stack/daos/src/control/logging" +) + +/* +#include +#include +#include + +#include "util.h" + +static inline uint32_t +get_rebuild_state(struct daos_rebuild_status *drs) +{ + if (drs == NULL) + return 0; + + return drs->rs_state; +} +*/ +import "C" + +type ( + // PoolHandle is an opaque type used to represent a DAOS Pool connection. + PoolHandle struct { + connHandle + } +) + +const ( + poolHandleKey ctxHdlKey = "poolHandle" +) + +// phFromContext retrieves the PoolHandle from the supplied context, if available. +func phFromCtx(ctx context.Context) (*PoolHandle, error) { + if ctx == nil { + return nil, errNilCtx + } + + ph, ok := ctx.Value(poolHandleKey).(*PoolHandle) + if !ok { + return nil, errNoCtxHdl + } + + return ph, nil +} + +// toCtx returns a new context with the PoolHandle stashed in it. +// NB: Will panic if the context already has a different PoolHandle stashed. +func (ph *PoolHandle) toCtx(ctx context.Context) context.Context { + if ph == nil { + return ctx + } + + stashed, _ := phFromCtx(ctx) + if stashed != nil { + if stashed.UUID() == ph.UUID() { + return ctx + } + panic("attempt to stash different PoolHandle in context") + } + + return context.WithValue(ctx, poolHandleKey, ph) +} + +// newPoolSpaceInfo constructs a Go type from the underlying C type. +func newPoolSpaceInfo(dps *C.struct_daos_pool_space, mt C.uint) *daos.StorageUsageStats { + if dps == nil { + return nil + } + + return &daos.StorageUsageStats{ + Total: uint64(dps.ps_space.s_total[mt]), + Free: uint64(dps.ps_space.s_free[mt]), + Min: uint64(dps.ps_free_min[mt]), + Max: uint64(dps.ps_free_max[mt]), + Mean: uint64(dps.ps_free_mean[mt]), + MediaType: daos.StorageMediaType(mt), + } +} + +// newPoolRebuildStatus constructs a Go type from the underlying C type. +func newPoolRebuildStatus(drs *C.struct_daos_rebuild_status) *daos.PoolRebuildStatus { + if drs == nil { + return nil + } + + compatRebuildState := func() daos.PoolRebuildState { + switch { + case drs.rs_version == 0: + return daos.PoolRebuildStateIdle + case C.get_rebuild_state(drs) == C.DRS_COMPLETED: + return daos.PoolRebuildStateDone + default: + return daos.PoolRebuildStateBusy + } + } + + return &daos.PoolRebuildStatus{ + Status: int32(drs.rs_errno), + Objects: uint64(drs.rs_obj_nr), + Records: uint64(drs.rs_rec_nr), + State: compatRebuildState(), + } +} + +// newPoolInfo constructs a Go type from the underlying C type. +func newPoolInfo(cpi *C.daos_pool_info_t) *daos.PoolInfo { + if cpi == nil { + return nil + } + + poolInfo := new(daos.PoolInfo) + + poolInfo.QueryMask = daos.PoolQueryMask(cpi.pi_bits) + poolInfo.UUID = uuid.Must(uuidFromC(cpi.pi_uuid)) + poolInfo.TotalTargets = uint32(cpi.pi_ntargets) + poolInfo.DisabledTargets = uint32(cpi.pi_ndisabled) + poolInfo.ActiveTargets = uint32(cpi.pi_space.ps_ntargets) + poolInfo.TotalEngines = uint32(cpi.pi_nnodes) + poolInfo.ServiceLeader = uint32(cpi.pi_leader) + poolInfo.Version = uint32(cpi.pi_map_ver) + poolInfo.State = daos.PoolServiceStateReady + if poolInfo.DisabledTargets > 0 { + poolInfo.State = daos.PoolServiceStateDegraded + } + + poolInfo.Rebuild = newPoolRebuildStatus(&cpi.pi_rebuild_st) + if poolInfo.QueryMask.HasOption(daos.PoolQueryOptionSpace) { + poolInfo.TierStats = []*daos.StorageUsageStats{ + newPoolSpaceInfo(&cpi.pi_space, C.DAOS_MEDIA_SCM), + newPoolSpaceInfo(&cpi.pi_space, C.DAOS_MEDIA_NVME), + } + } + + return poolInfo +} + +func poolInfoFromProps(pi *daos.PoolInfo, propEntries []C.struct_daos_prop_entry) { + if pi == nil || len(propEntries) == 0 { + return + } + + for _, entry := range propEntries { + switch entry.dpe_type { + case C.DAOS_PROP_PO_LABEL: + pi.Label = C.GoString(C.get_dpe_str(&entry)) + case C.DAOS_PROP_PO_SVC_LIST: + rlPtr := C.get_dpe_val_ptr(&entry) + if rlPtr == nil { + return + } + rs, err := rankSetFromC((*C.d_rank_list_t)(rlPtr)) + if err != nil { + return + } + pi.ServiceReplicas = rs.Ranks() + } + } +} + +// Disconnect signals that the client no longer needs the DAOS pool +// connection and that it is safe to release resources allocated for +// the connection. +func (ph *PoolHandle) Disconnect(ctx context.Context) error { + if ph == nil { + return ErrInvalidPoolHandle + } + logging.FromContext(ctx).Debugf("PoolHandle.Disconnect(%s)", ph) + + if err := daosError(daos_pool_disconnect(ph.daosHandle)); err != nil { + return errors.Wrap(err, "failed to disconnect from pool") + } + ph.invalidate() + + return nil +} + +// UUID returns the DAOS pool's UUID. +func (ph *PoolHandle) UUID() uuid.UUID { + if ph == nil { + return uuid.Nil + } + return ph.connHandle.UUID +} + +type ( + // PoolConnectReq defines the parameters for a PoolConnect request. + PoolConnectReq struct { + SysName string + ID string + Flags daos.PoolConnectFlag + Query bool + } + + // PoolConnectResp contains the response to a PoolConnect request. + PoolConnectResp struct { + Connection *PoolHandle + Info *daos.PoolInfo + } +) + +// PoolConnect establishes a connection to the specified DAOS pool. +// NB: The caller is responsible for disconnecting from the pool when +// finished. +func PoolConnect(ctx context.Context, req PoolConnectReq) (*PoolConnectResp, error) { + if ctx == nil { + return nil, errNilCtx + } + logging.FromContext(ctx).Debugf("PoolConnect(%+v)", req) + + if req.ID == "" { + return nil, errors.Wrap(daos.InvalidInput, "no pool ID provided") + } + if req.SysName == "" { + req.SysName = build.DefaultSystemName + } + if req.Flags == 0 { + req.Flags = daos.PoolConnectFlagReadOnly + } + + var dpi C.daos_pool_info_t + if req.Query { + dpi.pi_bits = C.ulong(daos.DefaultPoolQueryMask) + } + var poolConn PoolHandle + + cPoolID := C.CString(req.ID) + defer freeString(cPoolID) + cSys := C.CString(req.SysName) + defer freeString(cSys) + + if err := daosError(daos_pool_connect(cPoolID, cSys, C.uint(req.Flags), &poolConn.daosHandle, &dpi, nil)); err != nil { + return nil, errors.Wrap(err, "failed to connect to pool") + } + + poolInfo := newPoolInfo(&dpi) + poolConn.connHandle.UUID = poolInfo.UUID + if req.ID != poolInfo.UUID.String() { + poolInfo.Label = req.ID + } else { + // If the connection was made with a UUID, then we don't know the label without + // a query. This should be a rare scenario. If the request allows it, try a query. + poolInfo.Label = MissingPoolLabel + if req.Query { + qpi, err := poolConn.Query(ctx, daos.HealthOnlyPoolQueryMask) + if err != nil { + return nil, errors.Wrap(err, "failed to query pool for label") + } + poolInfo.Label = qpi.Label + } + } + // Set the label on the connection for convenience. + poolConn.connHandle.Label = poolInfo.Label + + logging.FromContext(ctx).Debugf("Connected to Pool %s", &poolConn) + return &PoolConnectResp{ + Connection: &poolConn, + Info: poolInfo, + }, nil +} + +// getPoolConn retrieves the PoolHandle set in the context, if available, +// or tries to establish a new connection to the specified pool. +func getPoolConn(ctx context.Context, sysName, poolID string, flags daos.PoolConnectFlag) (*PoolHandle, func(), error) { + nulCleanup := func() {} + ph, err := phFromCtx(ctx) + if err == nil { + if poolID != "" { + return nil, nulCleanup, errors.Wrap(daos.InvalidInput, "PoolHandle found in context with non-empty poolID") + } + return ph, nulCleanup, nil + } + + resp, err := PoolConnect(ctx, PoolConnectReq{ + ID: poolID, + SysName: sysName, + Flags: flags, + Query: false, + }) + if err != nil { + return nil, nulCleanup, err + } + + cleanup := func() { + err := resp.Connection.Disconnect(ctx) + if err != nil { + logging.FromContext(ctx).Error(err.Error()) + } + } + return resp.Connection, cleanup, nil +} + +// Query is a convenience wrapper around the PoolQuery() function. +func (ph *PoolHandle) Query(ctx context.Context, mask daos.PoolQueryMask) (*daos.PoolInfo, error) { + if ph == nil { + return nil, ErrInvalidPoolHandle + } + return PoolQuery(ph.toCtx(ctx), "", "", mask) +} + +// PoolQuery retrieves information about the DAOS Pool, including health and rebuild status, +// storage usage, and other details. +func PoolQuery(ctx context.Context, sysName, poolID string, queryMask daos.PoolQueryMask) (*daos.PoolInfo, error) { + if queryMask == 0 { + queryMask = daos.DefaultPoolQueryMask + } + poolConn, disconnect, err := getPoolConn(ctx, sysName, poolID, daos.PoolConnectFlagReadOnly) + if err != nil { + return nil, err + } + defer disconnect() + logging.FromContext(ctx).Debugf("PoolQuery(%s:%s)", poolConn, queryMask) + + var enabledRanks *C.d_rank_list_t + var disabledRanks *C.d_rank_list_t + defer func() { + C.d_rank_list_free(enabledRanks) + C.d_rank_list_free(disabledRanks) + }() + + // Query for some additional information stored as properties. + queryProps := C.daos_prop_alloc(2) + if queryProps == nil { + return nil, errors.Wrap(daos.NoMemory, "failed to allocate property list") + } + propEntries := unsafe.Slice(queryProps.dpp_entries, queryProps.dpp_nr) + propEntries[0].dpe_type = C.DAOS_PROP_PO_LABEL + propEntries[1].dpe_type = C.DAOS_PROP_PO_SVC_LIST + defer func() { + C.daos_prop_free(queryProps) + }() + + var rc C.int + cPoolInfo := C.daos_pool_info_t{ + pi_bits: C.uint64_t(queryMask), + } + if queryMask.HasOption(daos.PoolQueryOptionEnabledEngines) && queryMask.HasOption(daos.PoolQueryOptionDisabledEngines) { + enaQm := queryMask + enaQm.ClearOptions(daos.PoolQueryOptionDisabledEngines) + cPoolInfo.pi_bits = C.uint64_t(enaQm) + rc = daos_pool_query(poolConn.daosHandle, &enabledRanks, &cPoolInfo, queryProps, nil) + if err := daosError(rc); err != nil { + return nil, errors.Wrap(err, "failed to query pool") + } + + /* second query to just get disabled ranks */ + rc = daos_pool_query(poolConn.daosHandle, &disabledRanks, nil, nil, nil) + } else if queryMask.HasOption(daos.PoolQueryOptionEnabledEngines) { + rc = daos_pool_query(poolConn.daosHandle, &enabledRanks, &cPoolInfo, queryProps, nil) + } else if queryMask.HasOption(daos.PoolQueryOptionDisabledEngines) { + rc = daos_pool_query(poolConn.daosHandle, &disabledRanks, &cPoolInfo, queryProps, nil) + } else { + rc = daos_pool_query(poolConn.daosHandle, nil, &cPoolInfo, queryProps, nil) + } + + if err := daosError(rc); err != nil { + return nil, errors.Wrap(err, "failed to query pool") + } + + poolInfo := newPoolInfo(&cPoolInfo) + poolInfo.QueryMask = queryMask + poolInfoFromProps(poolInfo, propEntries) + + if enabledRanks != nil { + poolInfo.EnabledRanks, err = rankSetFromC(enabledRanks) + if err != nil { + return nil, err + } + } + if disabledRanks != nil { + poolInfo.DisabledRanks, err = rankSetFromC(disabledRanks) + if err != nil { + return nil, err + } + } + + return poolInfo, nil +} + +func newPoolTargetInfo(ptinfo *C.daos_target_info_t) *daos.PoolQueryTargetInfo { + return &daos.PoolQueryTargetInfo{ + Type: daos.PoolQueryTargetType(ptinfo.ta_type), + State: daos.PoolQueryTargetState(ptinfo.ta_state), + Space: []*daos.StorageUsageStats{ + { + Total: uint64(ptinfo.ta_space.s_total[C.DAOS_MEDIA_SCM]), + Free: uint64(ptinfo.ta_space.s_free[C.DAOS_MEDIA_SCM]), + MediaType: C.DAOS_MEDIA_SCM, + }, + { + Total: uint64(ptinfo.ta_space.s_total[C.DAOS_MEDIA_NVME]), + Free: uint64(ptinfo.ta_space.s_free[C.DAOS_MEDIA_NVME]), + MediaType: C.DAOS_MEDIA_NVME, + }, + }, + } +} + +// QueryTargets is a convenience wrapper around the PoolQueryTargets() function. +func (ph *PoolHandle) QueryTargets(ctx context.Context, rank ranklist.Rank, targets *ranklist.RankSet) ([]*daos.PoolQueryTargetInfo, error) { + if ph == nil { + return nil, ErrInvalidPoolHandle + } + return PoolQueryTargets(ph.toCtx(ctx), "", "", rank, targets) +} + +// PoolQueryTargets retrieves information about storage targets in the DAOS Pool. +func PoolQueryTargets(ctx context.Context, sysName, poolID string, rank ranklist.Rank, reqTargets *ranklist.RankSet) ([]*daos.PoolQueryTargetInfo, error) { + targets := ranklist.NewRankSet() + targets.Replace(reqTargets) + + if targets.Count() == 0 { + pi, err := PoolQuery(ctx, sysName, poolID, daos.HealthOnlyPoolQueryMask) + if err != nil || (pi.TotalTargets == 0 || pi.TotalEngines == 0) { + if err != nil { + return nil, errors.Wrap(err, "pool query failed") + } + return nil, errors.New("failed to derive target count from pool query") + } + tgtCount := pi.TotalTargets / pi.TotalEngines + for i := uint32(0); i < tgtCount; i++ { + targets.Add(ranklist.Rank(i)) + } + } + poolConn, disconnect, err := getPoolConn(ctx, sysName, poolID, daos.PoolConnectFlagReadOnly) + if err != nil { + return nil, err + } + defer disconnect() + logging.FromContext(ctx).Debugf("PoolQueryTargets(%s:%d:[%s])", poolConn, rank, targets) + + ptInfo := C.daos_target_info_t{} + var rc C.int + + infos := make([]*daos.PoolQueryTargetInfo, 0, targets.Count()) + for _, tgt := range targets.Ranks() { + rc = daos_pool_query_target(poolConn.daosHandle, C.uint32_t(tgt), C.uint32_t(rank), &ptInfo, nil) + if err := daosError(rc); err != nil { + return nil, errors.Wrapf(err, "failed to query pool %s rank:target %d:%d", poolID, rank, tgt) + } + + infos = append(infos, newPoolTargetInfo(&ptInfo)) + } + + return infos, nil +} + +// ListAttributes is a convenience wrapper around the PoolListAttributes() function. +func (ph *PoolHandle) ListAttributes(ctx context.Context) ([]string, error) { + if ph == nil { + return nil, ErrInvalidPoolHandle + } + return PoolListAttributes(ph.toCtx(ctx), "", "") +} + +// PoolListAttributes returns a list of user-definable pool attribute names. +func PoolListAttributes(ctx context.Context, sysName, poolID string) ([]string, error) { + poolConn, disconnect, err := getPoolConn(ctx, sysName, poolID, daos.PoolConnectFlagReadOnly) + if err != nil { + return nil, err + } + defer disconnect() + logging.FromContext(ctx).Debugf("PoolListAttributes(%s)", poolConn) + + if err := ctx.Err(); err != nil { + return nil, ctxErr(err) + } + + return listDaosAttributes(poolConn.daosHandle, poolAttr) +} + +// GetAttributes is a convenience wrapper around the PoolGetAttributes() function. +func (ph *PoolHandle) GetAttributes(ctx context.Context, attrNames ...string) (daos.AttributeList, error) { + if ph == nil { + return nil, ErrInvalidPoolHandle + } + return PoolGetAttributes(ph.toCtx(ctx), "", "", attrNames...) +} + +// PoolGetAttributes fetches the specified pool attributes. If no +// attribute names are provided, all attributes are fetched. +func PoolGetAttributes(ctx context.Context, sysName, poolID string, names ...string) (daos.AttributeList, error) { + poolConn, disconnect, err := getPoolConn(ctx, sysName, poolID, daos.PoolConnectFlagReadOnly) + if err != nil { + return nil, err + } + defer disconnect() + logging.FromContext(ctx).Debugf("PoolGetAttributes(%s:%v)", poolConn, names) + + if err := ctx.Err(); err != nil { + return nil, ctxErr(err) + } + + return getDaosAttributes(poolConn.daosHandle, poolAttr, names) +} + +// SetAttributes is a convenience wrapper around the PoolSetAttributes() function. +func (ph *PoolHandle) SetAttributes(ctx context.Context, attrs ...*daos.Attribute) error { + if ph == nil { + return ErrInvalidPoolHandle + } + return PoolSetAttributes(ph.toCtx(ctx), "", "", attrs...) +} + +// PoolSetAttributes sets the specified pool attributes. +func PoolSetAttributes(ctx context.Context, sysName, poolID string, attrs ...*daos.Attribute) error { + poolConn, disconnect, err := getPoolConn(ctx, sysName, poolID, daos.PoolConnectFlagReadOnly) + if err != nil { + return err + } + defer disconnect() + logging.FromContext(ctx).Debugf("PoolSetAttributes(%s:%v)", poolConn, attrs) + + if err := ctx.Err(); err != nil { + return ctxErr(err) + } + + return setDaosAttributes(poolConn.daosHandle, poolAttr, attrs) +} + +// DeleteAttributes is a convenience wrapper around the PoolDeleteAttributes() function. +func (ph *PoolHandle) DeleteAttributes(ctx context.Context, attrNames ...string) error { + if ph == nil { + return ErrInvalidPoolHandle + } + return PoolDeleteAttributes(ph.toCtx(ctx), "", "", attrNames...) +} + +// PoolDeleteAttributes deletes the specified pool attributes. +func PoolDeleteAttributes(ctx context.Context, sysName, poolID string, attrNames ...string) error { + poolConn, disconnect, err := getPoolConn(ctx, sysName, poolID, daos.PoolConnectFlagReadOnly) + if err != nil { + return err + } + defer disconnect() + logging.FromContext(ctx).Debugf("PoolDeleteAttributes(%s:%+v)", poolConn, attrNames) + + if err := ctx.Err(); err != nil { + return ctxErr(err) + } + + return delDaosAttributes(poolConn.daosHandle, poolAttr, attrNames) +} + +type ( + // GetPoolListReq defines the parameters for a GetPoolList request. + GetPoolListReq struct { + SysName string + Query bool + } +) + +// GetPoolList returns a list of DAOS pools in the system. +func GetPoolList(ctx context.Context, req GetPoolListReq) ([]*daos.PoolInfo, error) { + if ctx == nil { + return nil, errNilCtx + } + + log := logging.FromContext(ctx) + log.Debugf("GetPoolList(%+v)", req) + + if req.SysName == "" { + req.SysName = build.DefaultSystemName + } + cSysName := C.CString(req.SysName) + defer freeString(cSysName) + + var cPools []C.daos_mgmt_pool_info_t + for { + var rc C.int + var poolCount C.size_t + + // First, fetch the total number of pools in the system. + // We may not have access to all of them, so this is an upper bound. + rc = daos_mgmt_list_pools(cSysName, &poolCount, nil, nil) + if err := daosError(rc); err != nil { + return nil, errors.Wrap(err, "failed to list pools") + } + log.Debugf("pools in system: %d", poolCount) + + if poolCount < 1 { + return nil, nil + } + + // Now, fetch the pools into a buffer sized for the number of pools found. + cPools = make([]C.daos_mgmt_pool_info_t, poolCount) + rc = daos_mgmt_list_pools(cSysName, &poolCount, &cPools[0], nil) + err := daosError(rc) + if err == nil { + cPools = cPools[:poolCount] // adjust the slice to the number of pools retrieved + log.Debugf("fetched %d pools", len(cPools)) + break + } + if err == daos.StructTooSmall { + log.Notice("server-side pool list changed; re-fetching") + continue + } + log.Errorf("failed to fetch pool list: %s", err) + return nil, errors.Wrap(err, "failed to list pools") + } + + pools := make([]*daos.PoolInfo, 0, len(cPools)) + for i := 0; i < len(cPools); i++ { + cPool := &cPools[i] + + svcLdr := uint32(cPool.mgpi_ldr) + svcRanks, err := rankSetFromC(cPool.mgpi_svc) + if err != nil { + return nil, err + } + defer func() { + C.d_rank_list_free(cPool.mgpi_svc) + }() + poolUUID, err := uuidFromC(cPool.mgpi_uuid) + if err != nil { + return nil, err + } + poolLabel := C.GoString(cPool.mgpi_label) + + var pool *daos.PoolInfo + if req.Query { + pcResp, err := PoolConnect(ctx, PoolConnectReq{ + ID: poolUUID.String(), + SysName: req.SysName, + Flags: daos.PoolConnectFlagReadOnly, + Query: true, + }) + if err != nil { + log.Errorf("failed to connect to pool %q: %s", poolLabel, err) + continue + } + if err := pcResp.Connection.Disconnect(ctx); err != nil { + log.Errorf("failed to disconnect from pool %q: %s", poolLabel, err) + } + pool = pcResp.Info + + // Add a few missing pieces that the query doesn't fill in. + pool.Label = poolLabel + pool.ServiceLeader = svcLdr + pool.ServiceReplicas = svcRanks.Ranks() + } else { + // Just populate the basic info. + pool = &daos.PoolInfo{ + UUID: poolUUID, + Label: poolLabel, + ServiceLeader: svcLdr, + ServiceReplicas: svcRanks.Ranks(), + State: daos.PoolServiceStateReady, + } + } + + pools = append(pools, pool) + } + + log.Debugf("fetched %d/%d pools", len(pools), len(cPools)) + return pools, nil +} diff --git a/src/control/lib/daos/api/pool_test.go b/src/control/lib/daos/api/pool_test.go new file mode 100644 index 000000000000..93bf48a30d35 --- /dev/null +++ b/src/control/lib/daos/api/pool_test.go @@ -0,0 +1,1057 @@ +// +// (C) Copyright 2025 Google LLC +// +// SPDX-License-Identifier: BSD-2-Clause-Patent +// + +package api + +import ( + "context" + "fmt" + "reflect" + "sort" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/pkg/errors" + + "github.com/daos-stack/daos/src/control/build" + "github.com/daos-stack/daos/src/control/common/test" + "github.com/daos-stack/daos/src/control/lib/daos" + "github.com/daos-stack/daos/src/control/lib/ranklist" + "github.com/daos-stack/daos/src/control/logging" +) + +var ( + testPoolName = "test-pool" +) + +func TestAPI_PoolConnect(t *testing.T) { + defaultReq := PoolConnectReq{ + ID: daos_default_PoolInfo.Label, + SysName: build.DefaultSystemName, + Flags: daos.PoolConnectFlagReadWrite, + Query: true, + } + + for name, tc := range map[string]struct { + setup func(t *testing.T) + ctx context.Context + connReq PoolConnectReq + checkParams func(t *testing.T) + expResp *PoolConnectResp + expErr error + }{ + "nil context": { + connReq: defaultReq, + expErr: errNilCtx, + }, + "no poolID in req": { + ctx: test.Context(t), + connReq: PoolConnectReq{ + SysName: defaultReq.SysName, + Flags: defaultReq.Flags, + Query: defaultReq.Query, + }, + expErr: errors.Wrap(daos.InvalidInput, "no pool ID provided"), + }, + "daos_pool_connect() fails": { + setup: func(t *testing.T) { + daos_pool_connect_RC = -_Ctype_int(daos.IOError) + }, + ctx: test.Context(t), + connReq: defaultReq, + expErr: errors.Wrap(daos.IOError, "failed to connect to pool"), + }, + "daos_pool_connect() succeeds": { + ctx: test.Context(t), + connReq: defaultReq, + checkParams: func(t *testing.T) { + test.CmpAny(t, "poolID", defaultReq.ID, daos_pool_connect_SetPoolID) + test.CmpAny(t, "sysName", defaultReq.SysName, daos_pool_connect_SetSys) + test.CmpAny(t, "flags", defaultReq.Flags, daos_pool_connect_SetFlags) + test.CmpAny(t, "query", daos.DefaultPoolQueryMask, daos_pool_connect_QueryMask) + }, + expResp: &PoolConnectResp{ + Connection: &PoolHandle{ + connHandle: connHandle{ + Label: daos_default_PoolInfo.Label, + UUID: daos_default_PoolInfo.UUID, + daosHandle: daos_default_pool_connect_Handle, + }, + }, + Info: defaultPoolInfo(), + }, + }, + "Connect with UUID and query enabled": { + ctx: test.Context(t), + connReq: PoolConnectReq{ + ID: daos_default_PoolInfo.UUID.String(), + SysName: defaultReq.SysName, + Flags: defaultReq.Flags, + Query: true, + }, + expResp: &PoolConnectResp{ + Connection: &PoolHandle{ + connHandle: connHandle{ + Label: daos_default_PoolInfo.Label, + UUID: daos_default_PoolInfo.UUID, + daosHandle: daos_default_pool_connect_Handle, + }, + }, + Info: defaultPoolInfo(), + }, + }, + "Connect with UUID and query enabled -- query fails": { + setup: func(t *testing.T) { + daos_pool_query_RC = -_Ctype_int(daos.IOError) + }, + ctx: test.Context(t), + connReq: PoolConnectReq{ + ID: daos_default_PoolInfo.UUID.String(), + SysName: defaultReq.SysName, + Flags: defaultReq.Flags, + Query: true, + }, + expErr: daos.IOError, + }, + "Connect with UUID and query disabled": { + ctx: test.Context(t), + connReq: PoolConnectReq{ + ID: daos_default_PoolInfo.UUID.String(), + SysName: defaultReq.SysName, + Flags: defaultReq.Flags, + Query: false, + }, + expResp: &PoolConnectResp{ + Connection: &PoolHandle{ + connHandle: connHandle{ + Label: MissingPoolLabel, + UUID: daos_default_PoolInfo.UUID, + daosHandle: daos_default_pool_connect_Handle, + }, + }, + Info: func() *daos.PoolInfo { + out := defaultPoolInfo() + out.Label = MissingPoolLabel + return out + }(), + }, + }, + } { + t.Run(name, func(t *testing.T) { + t.Cleanup(ResetTestStubs) + if tc.setup != nil { + tc.setup(t) + } + log, buf := logging.NewTestLogger(name) + defer test.ShowBufferOnFailure(t, buf) + + gotResp, gotErr := PoolConnect(mustLogCtx(tc.ctx, log), tc.connReq) + test.CmpErr(t, tc.expErr, gotErr) + if tc.expErr != nil { + return + } + + if tc.checkParams != nil { + tc.checkParams(t) + } + + cmpOpts := cmp.Options{ + cmp.Comparer(func(a, b *PoolHandle) bool { + return a != nil && b != nil && a.String() == b.String() + }), + // These fields aren't populated in the PoolConnect() query. + cmpopts.IgnoreFields(daos.PoolInfo{}, + "EnabledRanks", "DisabledRanks", "DeadRanks", "ServiceReplicas", + ), + } + test.CmpAny(t, "PoolConnectResp", tc.expResp, gotResp, cmpOpts...) + }) + } +} + +var ( + testCtxPoolHandle = &PoolHandle{ + connHandle: connHandle{ + UUID: test.MockPoolUUID(43), + Label: "test-ctx-pool", + }, + } + + testConnPoolHandle = &PoolHandle{ + connHandle: connHandle{ + daosHandle: daos_default_pool_connect_Handle, + UUID: daos_default_PoolInfo.UUID, + Label: daos_default_PoolInfo.Label, + }, + } +) + +func TestAPI_getPoolConn(t *testing.T) { + for name, tc := range map[string]struct { + setup func(t *testing.T) + ctx context.Context + poolID string + flags daos.PoolConnectFlag + checkParams func(t *testing.T) + expHdl *PoolHandle + expErr error + }{ + "pool handle in context with non-empty ID": { + ctx: testCtxPoolHandle.toCtx(test.Context(t)), + poolID: testPoolName, + expErr: errors.New("PoolHandle found in context with non-empty poolID"), + }, + "pool handle in context": { + ctx: testCtxPoolHandle.toCtx(test.Context(t)), + expHdl: testCtxPoolHandle, + }, + "pool handle not in context, no poolID": { + ctx: test.Context(t), + expErr: errors.Wrap(daos.InvalidInput, "no pool ID provided"), + }, + "pool handle from Connect()": { + ctx: test.Context(t), + poolID: daos_default_PoolInfo.Label, + checkParams: func(t *testing.T) { + test.CmpAny(t, "poolID", daos_default_PoolInfo.Label, daos_pool_connect_SetPoolID) + test.CmpAny(t, "sysName", build.DefaultSystemName, daos_pool_connect_SetSys) + test.CmpAny(t, "flags", daos.PoolConnectFlagReadOnly, daos_pool_connect_SetFlags) + test.CmpAny(t, "query", daos.PoolQueryMask(0), daos_pool_connect_QueryMask) + }, + expHdl: testConnPoolHandle, + }, + } { + t.Run(name, func(t *testing.T) { + t.Cleanup(ResetTestStubs) + if tc.setup != nil { + tc.setup(t) + } + log, buf := logging.NewTestLogger(name) + defer test.ShowBufferOnFailure(t, buf) + + ctx := tc.ctx + if ctx == nil { + ctx = test.Context(t) + } + + ph, cleanup, gotErr := getPoolConn(mustLogCtx(ctx, log), "", tc.poolID, tc.flags) + test.CmpErr(t, tc.expErr, gotErr) + if tc.expErr != nil { + return + } + t.Cleanup(cleanup) + + if tc.checkParams != nil { + tc.checkParams(t) + } + + cmpOpts := cmp.Options{ + cmp.Comparer(func(a, b *PoolHandle) bool { + return a != nil && b != nil && a.String() == b.String() + }), + } + test.CmpAny(t, "PoolHandle", tc.expHdl, ph, cmpOpts...) + }) + } +} + +func TestAPI_PoolQuery(t *testing.T) { + for name, tc := range map[string]struct { + setup func(t *testing.T) + ctx context.Context + poolID string + queryMask daos.PoolQueryMask + checkParams func(t *testing.T) + expResp *daos.PoolInfo + expErr error + }{ + "nil context": { + expErr: errNilCtx, + }, + "pool handle in context with non-empty ID": { + ctx: testCtxPoolHandle.toCtx(test.Context(t)), + poolID: testPoolName, + expErr: errors.New("PoolHandle found in context with non-empty poolID"), + }, + "pool handle not in context, no poolID": { + ctx: test.Context(t), + expErr: errors.Wrap(daos.InvalidInput, "no pool ID provided"), + }, + "daos_pool_query() fails": { + setup: func(t *testing.T) { + daos_pool_query_RC = -_Ctype_int(daos.IOError) + }, + ctx: test.Context(t), + poolID: daos_default_PoolInfo.Label, + expErr: errors.Wrap(daos.IOError, "failed to query pool"), + }, + "daos_pool_query() fails on enabled ranks": { + setup: func(t *testing.T) { + daos_pool_query_RC = -_Ctype_int(daos.IOError) + }, + ctx: test.Context(t), + poolID: daos_default_PoolInfo.Label, + queryMask: daos.MustNewPoolQueryMask(daos.PoolQueryOptionEnabledEngines, daos.PoolQueryOptionDisabledEngines), + expErr: errors.Wrap(daos.IOError, "failed to query pool"), + }, + "unspecified query mask": { + ctx: testCtxPoolHandle.toCtx(test.Context(t)), + expResp: func() *daos.PoolInfo { + out := defaultPoolInfo() + out.QueryMask = daos.DefaultPoolQueryMask + out.EnabledRanks = nil + return out + }(), + }, + "default query mask": { + ctx: testCtxPoolHandle.toCtx(test.Context(t)), + queryMask: daos.DefaultPoolQueryMask, + expResp: func() *daos.PoolInfo { + out := defaultPoolInfo() + out.QueryMask = daos.DefaultPoolQueryMask + out.EnabledRanks = nil + return out + }(), + }, + "health-only query mask": { + ctx: testCtxPoolHandle.toCtx(test.Context(t)), + queryMask: daos.HealthOnlyPoolQueryMask, + expResp: func() *daos.PoolInfo { + out := defaultPoolInfo() + out.QueryMask = daos.HealthOnlyPoolQueryMask + out.EnabledRanks = nil + out.TierStats = nil + return out + }(), + }, + "enabled ranks": { + ctx: testCtxPoolHandle.toCtx(test.Context(t)), + queryMask: daos.MustNewPoolQueryMask(daos.PoolQueryOptionEnabledEngines), + expResp: func() *daos.PoolInfo { + out := defaultPoolInfo() + out.QueryMask = daos.MustNewPoolQueryMask(daos.PoolQueryOptionEnabledEngines) + out.DisabledRanks = nil + out.TierStats = nil + return out + }(), + }, + "enabled & disabled ranks": { + ctx: testCtxPoolHandle.toCtx(test.Context(t)), + queryMask: daos.MustNewPoolQueryMask(daos.PoolQueryOptionEnabledEngines, daos.PoolQueryOptionDisabledEngines), + expResp: func() *daos.PoolInfo { + out := defaultPoolInfo() + out.QueryMask = daos.MustNewPoolQueryMask(daos.PoolQueryOptionEnabledEngines, daos.PoolQueryOptionDisabledEngines) + out.TierStats = nil + return out + }(), + }, + "space-only": { + ctx: testCtxPoolHandle.toCtx(test.Context(t)), + queryMask: daos.MustNewPoolQueryMask(daos.PoolQueryOptionSpace), + expResp: func() *daos.PoolInfo { + out := defaultPoolInfo() + out.QueryMask = daos.MustNewPoolQueryMask(daos.PoolQueryOptionSpace) + out.EnabledRanks = nil + out.DisabledRanks = nil + return out + }(), + }, + } { + t.Run(name, func(t *testing.T) { + t.Cleanup(ResetTestStubs) + if tc.setup != nil { + tc.setup(t) + } + log, buf := logging.NewTestLogger(name) + defer test.ShowBufferOnFailure(t, buf) + + gotResp, err := PoolQuery(mustLogCtx(tc.ctx, log), "", tc.poolID, tc.queryMask) + test.CmpErr(t, tc.expErr, err) + if tc.expErr != nil { + return + } + + if tc.checkParams != nil { + tc.checkParams(t) + } + + cmpOpts := cmp.Options{ + cmp.Comparer(func(a, b ranklist.RankSet) bool { + return a.String() == b.String() + }), + } + test.CmpAny(t, "PoolQuery() PoolInfo", tc.expResp, gotResp, cmpOpts...) + }) + } +} + +func TestAPI_PoolQueryTargets(t *testing.T) { + allTgtCt := daos_default_PoolInfo.TotalTargets / daos_default_PoolInfo.TotalEngines + + for name, tc := range map[string]struct { + setup func(t *testing.T) + ctx context.Context + poolID string + rank ranklist.Rank + targets *ranklist.RankSet + checkParams func(t *testing.T) + expResp []*daos.PoolQueryTargetInfo + expErr error + }{ + "nil context": { + expErr: errNilCtx, + }, + "pool handle in context with non-empty ID": { + ctx: testCtxPoolHandle.toCtx(test.Context(t)), + poolID: testPoolName, + expErr: errors.New("PoolHandle found in context with non-empty poolID"), + }, + "pool handle not in context, no poolID": { + ctx: test.Context(t), + expErr: errors.Wrap(daos.InvalidInput, "no pool ID provided"), + }, + "daos_pool_query() fails": { + setup: func(t *testing.T) { + daos_pool_query_RC = -_Ctype_int(daos.IOError) + }, + ctx: test.Context(t), + poolID: daos_default_PoolInfo.Label, + expErr: errors.Wrap(daos.IOError, "failed to query pool"), + }, + "daos_pool_query_target() fails": { + setup: func(t *testing.T) { + daos_pool_query_target_RC = -_Ctype_int(daos.IOError) + }, + ctx: test.Context(t), + poolID: daos_default_PoolInfo.Label, + expErr: daos.IOError, + }, + "pool query returns zero targets": { + setup: func(t *testing.T) { + daos_pool_query_PoolInfo = defaultPoolInfo() + daos_pool_query_PoolInfo.TotalTargets = 0 + }, + ctx: test.Context(t), + poolID: daos_default_PoolInfo.Label, + expErr: errors.New("failed to derive target count"), + }, + "pool query returns zero engines": { + setup: func(t *testing.T) { + daos_pool_query_PoolInfo = defaultPoolInfo() + daos_pool_query_PoolInfo.TotalEngines = 0 + }, + ctx: test.Context(t), + poolID: daos_default_PoolInfo.Label, + expErr: errors.New("failed to derive target count"), + }, + "nil target set gets all": { + ctx: test.Context(t), + poolID: daos_default_PoolInfo.Label, + rank: 1, + targets: nil, + checkParams: func(t *testing.T) { + test.CmpAny(t, "rank", _Ctype_uint32_t(1), daos_pool_query_target_SetRank) + test.CmpAny(t, "last target", _Ctype_uint32_t(allTgtCt-1), daos_pool_query_target_SetTgt) + }, + expResp: func() []*daos.PoolQueryTargetInfo { + infos := make([]*daos.PoolQueryTargetInfo, allTgtCt) + for i := range infos { + infos[i] = &daos_default_PoolQueryTargetInfo + } + return infos + }(), + }, + "empty target set gets all": { + ctx: test.Context(t), + poolID: daos_default_PoolInfo.Label, + rank: 1, + targets: ranklist.NewRankSet(), + checkParams: func(t *testing.T) { + test.CmpAny(t, "rank", _Ctype_uint32_t(1), daos_pool_query_target_SetRank) + test.CmpAny(t, "last target", _Ctype_uint32_t(allTgtCt-1), daos_pool_query_target_SetTgt) + }, + expResp: func() []*daos.PoolQueryTargetInfo { + infos := make([]*daos.PoolQueryTargetInfo, allTgtCt) + for i := range infos { + infos[i] = &daos_default_PoolQueryTargetInfo + } + return infos + }(), + }, + "specified target should not query pool for target list": { + setup: func(t *testing.T) { + daos_pool_query_RC = -_Ctype_int(daos.IOError) // fail if the pool is queried + }, + ctx: test.Context(t), + poolID: daos_default_PoolInfo.Label, + rank: 1, + targets: ranklist.MustCreateRankSet("1"), + checkParams: func(t *testing.T) { + test.CmpAny(t, "rank", _Ctype_uint32_t(1), daos_pool_query_target_SetRank) + test.CmpAny(t, "last target", _Ctype_uint32_t(1), daos_pool_query_target_SetTgt) + }, + expResp: func() []*daos.PoolQueryTargetInfo { + infos := make([]*daos.PoolQueryTargetInfo, 1) + for i := range infos { + infos[i] = &daos_default_PoolQueryTargetInfo + } + return infos + }(), + }, + } { + t.Run(name, func(t *testing.T) { + t.Cleanup(ResetTestStubs) + if tc.setup != nil { + tc.setup(t) + } + log, buf := logging.NewTestLogger(name) + defer test.ShowBufferOnFailure(t, buf) + + gotResp, err := PoolQueryTargets(mustLogCtx(tc.ctx, log), "", tc.poolID, tc.rank, tc.targets) + test.CmpErr(t, tc.expErr, err) + if tc.expErr != nil { + return + } + + if tc.checkParams != nil { + tc.checkParams(t) + } + + cmpOpts := cmp.Options{ + cmp.Comparer(func(a, b ranklist.RankSet) bool { + return a.String() == b.String() + }), + } + test.CmpAny(t, "PoolQueryTargets() response", tc.expResp, gotResp, cmpOpts...) + }) + } +} + +func TestAPI_PoolListAttributes(t *testing.T) { + for name, tc := range map[string]struct { + setup func(t *testing.T) + ctx context.Context + poolID string + expNames []string + expErr error + }{ + "nil context": { + expErr: errNilCtx, + }, + "pool handle in context with non-empty ID": { + ctx: testCtxPoolHandle.toCtx(test.Context(t)), + poolID: testPoolName, + expErr: errors.New("PoolHandle found in context with non-empty poolID"), + }, + "pool handle not in context, no poolID": { + ctx: test.Context(t), + expErr: errors.Wrap(daos.InvalidInput, "no pool ID provided"), + }, + "daos_pool_list_attr() fails (get buf size)": { + setup: func(t *testing.T) { + daos_pool_list_attr_RC = -_Ctype_int(daos.IOError) + }, + ctx: testCtxPoolHandle.toCtx(test.Context(t)), + expErr: errors.Wrap(daos.IOError, "failed to list pool attributes"), + }, + "daos_pool_list_attr() fails (fetch names)": { + setup: func(t *testing.T) { + daos_pool_list_attr_RCList = []_Ctype_int{ + 0, + -_Ctype_int(daos.IOError), + } + }, + ctx: testCtxPoolHandle.toCtx(test.Context(t)), + expErr: errors.Wrap(daos.IOError, "failed to list pool attributes"), + }, + "no attributes set": { + setup: func(t *testing.T) { + daos_pool_list_attr_AttrList = nil + }, + ctx: testCtxPoolHandle.toCtx(test.Context(t)), + }, + "success": { + ctx: testCtxPoolHandle.toCtx(test.Context(t)), + expNames: []string{ + daos_default_AttrList[0].Name, + daos_default_AttrList[1].Name, + daos_default_AttrList[2].Name, + }, + }, + } { + t.Run(name, func(t *testing.T) { + t.Cleanup(ResetTestStubs) + if tc.setup != nil { + tc.setup(t) + } + log, buf := logging.NewTestLogger(name) + defer test.ShowBufferOnFailure(t, buf) + + gotNames, err := PoolListAttributes(mustLogCtx(tc.ctx, log), "", tc.poolID) + test.CmpErr(t, tc.expErr, err) + if tc.expErr != nil { + return + } + + test.CmpAny(t, "PoolListAttributes()", tc.expNames, gotNames) + }) + } +} + +func TestAPI_PoolGetAttributes(t *testing.T) { + for name, tc := range map[string]struct { + setup func(t *testing.T) + ctx context.Context + poolID string + attrNames []string + checkParams func(t *testing.T) + expAttrs daos.AttributeList + expErr error + }{ + "nil context": { + expErr: errNilCtx, + }, + "pool handle in context with non-empty ID": { + ctx: testCtxPoolHandle.toCtx(test.Context(t)), + poolID: testPoolName, + expErr: errors.New("PoolHandle found in context with non-empty poolID"), + }, + "pool handle not in context, no poolID": { + ctx: test.Context(t), + expErr: errors.Wrap(daos.InvalidInput, "no pool ID provided"), + }, + "daos_pool_list_attr() fails": { + setup: func(t *testing.T) { + daos_pool_list_attr_RC = -_Ctype_int(daos.IOError) + }, + ctx: testCtxPoolHandle.toCtx(test.Context(t)), + expErr: errors.Wrap(daos.IOError, "failed to list pool attributes"), + }, + "daos_pool_get_attr() fails (sizes)": { + setup: func(t *testing.T) { + daos_pool_get_attr_RC = -_Ctype_int(daos.IOError) + }, + ctx: testCtxPoolHandle.toCtx(test.Context(t)), + expErr: errors.Wrap(daos.IOError, "failed to get pool attribute sizes"), + }, + "daos_pool_get_attr() fails (values)": { + setup: func(t *testing.T) { + daos_pool_get_attr_RCList = []_Ctype_int{ + 0, + -_Ctype_int(daos.IOError), + } + }, + ctx: testCtxPoolHandle.toCtx(test.Context(t)), + expErr: errors.Wrap(daos.IOError, "failed to get pool attribute values"), + }, + "empty requested attribute name": { + ctx: testCtxPoolHandle.toCtx(test.Context(t)), + attrNames: test.JoinArgs(nil, "a", ""), + expErr: errors.Errorf("empty pool attribute name at index 1"), + }, + "no attributes set; attributes requested": { + setup: func(t *testing.T) { + daos_pool_get_attr_AttrList = nil + }, + ctx: testCtxPoolHandle.toCtx(test.Context(t)), + attrNames: test.JoinArgs(nil, "foo"), + checkParams: func(t *testing.T) { + test.CmpAny(t, "req attr names", test.JoinArgs(nil, "foo"), daos_pool_get_attr_ReqNames) + }, + expErr: errors.Wrap(daos.Nonexistent, "failed to get pool attribute sizes"), + }, + "unknown attribute requested": { + ctx: testCtxPoolHandle.toCtx(test.Context(t)), + attrNames: test.JoinArgs(nil, "foo"), + checkParams: func(t *testing.T) { + test.CmpAny(t, "req attr names", test.JoinArgs(nil, "foo"), daos_pool_get_attr_ReqNames) + }, + expErr: errors.Wrap(daos.Nonexistent, "failed to get pool attribute sizes"), + }, + "no attributes set; no attributes requested": { + setup: func(t *testing.T) { + daos_pool_list_attr_AttrList = nil + }, + ctx: testCtxPoolHandle.toCtx(test.Context(t)), + }, + "success; all attributes": { + ctx: testCtxPoolHandle.toCtx(test.Context(t)), + expAttrs: daos_default_AttrList, + }, + "success; requested attributes": { + ctx: testCtxPoolHandle.toCtx(test.Context(t)), + attrNames: test.JoinArgs(nil, daos_default_AttrList[0].Name, daos_default_AttrList[2].Name), + checkParams: func(t *testing.T) { + reqNames := test.JoinArgs(nil, daos_default_AttrList[0].Name, daos_default_AttrList[2].Name) + sort.Strings(reqNames) + gotNames := daos_test_get_mappedNames(daos_pool_get_attr_ReqNames) + sort.Strings(gotNames) + test.CmpAny(t, "req attr names", reqNames, gotNames) + }, + expAttrs: daos.AttributeList{ + daos_default_AttrList[0], + daos_default_AttrList[2], + }, + }, + } { + t.Run(name, func(t *testing.T) { + t.Cleanup(ResetTestStubs) + if tc.setup != nil { + tc.setup(t) + } + log, buf := logging.NewTestLogger(name) + defer test.ShowBufferOnFailure(t, buf) + + gotAttrs, err := PoolGetAttributes(mustLogCtx(tc.ctx, log), "", tc.poolID, tc.attrNames...) + test.CmpErr(t, tc.expErr, err) + if tc.expErr != nil { + return + } + + if tc.checkParams != nil { + tc.checkParams(t) + } + + test.CmpAny(t, "PoolGetAttributes() daos.AttributeList", tc.expAttrs, gotAttrs) + }) + } +} + +func TestAPI_PoolSetAttributes(t *testing.T) { + for name, tc := range map[string]struct { + setup func(t *testing.T) + ctx context.Context + poolID string + toSet daos.AttributeList + expErr error + }{ + "nil context": { + expErr: errNilCtx, + }, + "pool handle in context with non-empty ID": { + ctx: testCtxPoolHandle.toCtx(test.Context(t)), + poolID: testPoolName, + expErr: errors.New("PoolHandle found in context with non-empty poolID"), + }, + "pool handle not in context, no poolID": { + ctx: test.Context(t), + expErr: errors.Wrap(daos.InvalidInput, "no pool ID provided"), + }, + "no attributes to set": { + ctx: testCtxPoolHandle.toCtx(test.Context(t)), + expErr: errors.Wrap(daos.InvalidInput, "no pool attributes provided"), + }, + "nil toSet attribute": { + ctx: testCtxPoolHandle.toCtx(test.Context(t)), + toSet: append(daos_default_AttrList, nil), + expErr: errors.Wrap(daos.InvalidInput, "nil pool attribute at index 3"), + }, + "toSet attribute with empty name": { + ctx: testCtxPoolHandle.toCtx(test.Context(t)), + toSet: append(daos_default_AttrList, &daos.Attribute{Name: ""}), + expErr: errors.Wrap(daos.InvalidInput, "empty pool attribute name at index 3"), + }, + "toSet attribute with empty value": { + ctx: testCtxPoolHandle.toCtx(test.Context(t)), + toSet: append(daos_default_AttrList, &daos.Attribute{Name: "empty"}), + expErr: errors.Wrap(daos.InvalidInput, "empty pool attribute value at index 3"), + }, + "daos_pool_set_attr() fails": { + setup: func(t *testing.T) { + daos_pool_set_attr_RC = -_Ctype_int(daos.IOError) + }, + ctx: testCtxPoolHandle.toCtx(test.Context(t)), + toSet: daos_default_AttrList, + expErr: errors.Wrap(daos.IOError, "failed to set pool attributes"), + }, + "success": { + ctx: testCtxPoolHandle.toCtx(test.Context(t)), + toSet: daos_default_AttrList, + }, + } { + t.Run(name, func(t *testing.T) { + t.Cleanup(ResetTestStubs) + if tc.setup != nil { + tc.setup(t) + } + log, buf := logging.NewTestLogger(name) + defer test.ShowBufferOnFailure(t, buf) + + err := PoolSetAttributes(mustLogCtx(tc.ctx, log), "", tc.poolID, tc.toSet...) + test.CmpErr(t, tc.expErr, err) + if tc.expErr != nil { + return + } + + test.CmpAny(t, "PoolSetAttributes() daos.AttributeList", tc.toSet, daos_pool_set_attr_AttrList) + }) + } +} + +func TestAPI_PoolDeleteAttributes(t *testing.T) { + for name, tc := range map[string]struct { + setup func(t *testing.T) + ctx context.Context + poolID string + toDelete []string + expErr error + }{ + "nil context": { + expErr: errNilCtx, + }, + "pool handle in context with non-empty ID": { + ctx: testCtxPoolHandle.toCtx(test.Context(t)), + poolID: testPoolName, + expErr: errors.New("PoolHandle found in context with non-empty poolID"), + }, + "pool handle not in context, no poolID": { + ctx: test.Context(t), + expErr: errors.Wrap(daos.InvalidInput, "no pool ID provided"), + }, + "no attributes to delete": { + ctx: testCtxPoolHandle.toCtx(test.Context(t)), + expErr: errors.Wrap(daos.InvalidInput, "no pool attribute names provided"), + }, + "empty name in toDelete list": { + ctx: testCtxPoolHandle.toCtx(test.Context(t)), + toDelete: test.JoinArgs(nil, "foo", "", "bar"), + expErr: errors.Wrap(daos.InvalidInput, "empty pool attribute name at index 1"), + }, + "daos_pool_det_attr() fails": { + setup: func(t *testing.T) { + daos_pool_del_attr_RC = -_Ctype_int(daos.IOError) + }, + ctx: testCtxPoolHandle.toCtx(test.Context(t)), + toDelete: test.JoinArgs(nil, daos_default_AttrList[0].Name), + expErr: errors.Wrap(daos.IOError, "failed to delete pool attributes"), + }, + "success": { + ctx: testCtxPoolHandle.toCtx(test.Context(t)), + toDelete: test.JoinArgs(nil, daos_default_AttrList[0].Name), + }, + } { + t.Run(name, func(t *testing.T) { + t.Cleanup(ResetTestStubs) + if tc.setup != nil { + tc.setup(t) + } + log, buf := logging.NewTestLogger(name) + defer test.ShowBufferOnFailure(t, buf) + + err := PoolDeleteAttributes(mustLogCtx(tc.ctx, log), "", tc.poolID, tc.toDelete...) + test.CmpErr(t, tc.expErr, err) + if tc.expErr != nil { + return + } + + test.CmpAny(t, "PoolDeleteAttributes() AttrNames", tc.toDelete, daos_pool_del_attr_AttrNames) + }) + } +} + +func TestAPI_PoolHandleMethods(t *testing.T) { + testHandle := &PoolHandle{} + + thType := reflect.TypeOf(testHandle) + for i := 0; i < thType.NumMethod(); i++ { + method := thType.Method(i) + methArgs := make([]reflect.Value, 0) + var expResults int + + switch method.Name { + case "Disconnect": + expResults = 1 + case "Query": + methArgs = append(methArgs, reflect.ValueOf(daos.DefaultPoolQueryMask)) + expResults = 2 + case "QueryTargets": + methArgs = append(methArgs, reflect.ValueOf(ranklist.Rank(1)), reflect.ValueOf((*ranklist.RankSet)(nil))) + expResults = 2 + case "ListAttributes": + expResults = 2 + case "GetAttributes": + methArgs = append(methArgs, reflect.ValueOf(daos_default_AttrList[0].Name)) + expResults = 2 + case "SetAttributes": + methArgs = append(methArgs, reflect.ValueOf(daos_default_AttrList[0])) + expResults = 1 + case "DeleteAttributes": + methArgs = append(methArgs, reflect.ValueOf(daos_default_AttrList[0].Name)) + expResults = 1 + case "FillHandle", "IsValid", "String", "UUID", "ID": + // No tests for these. The main point of this suite is to ensure that the + // convenience wrappers handle inputs as expected. + continue + default: + // If you're here, you need to add a case to test your new method. + t.Fatalf("unhandled method %q", method.Name) + } + + // Not intended to be exhaustive; just verify that they accept the parameters + // we expect and return something sensible for errors. + for name, tc := range map[string]struct { + setup func(t *testing.T) + th *PoolHandle + expErr error + }{ + fmt.Sprintf("%s: nil handle", method.Name): { + th: nil, + expErr: ErrInvalidPoolHandle, + }, + fmt.Sprintf("%s: success", method.Name): { + th: testHandle, + }, + } { + t.Run(name, func(t *testing.T) { + thArg := reflect.ValueOf(tc.th) + if tc.th == nil { + thArg = reflect.New(thType).Elem() + } + ctxArg := reflect.ValueOf(test.Context(t)) + testArgs := append([]reflect.Value{thArg, ctxArg}, methArgs...) + t.Logf("\nargs: %+v", testArgs) + + retVals := method.Func.Call(testArgs) + if len(retVals) != expResults { + t.Fatalf("expected %d return values, got %d", expResults, len(retVals)) + } + + if err, ok := retVals[len(retVals)-1].Interface().(error); ok { + test.CmpErr(t, tc.expErr, err) + } else { + test.CmpErr(t, tc.expErr, nil) + } + }) + } + } +} + +func TestAPI_GetPoolList(t *testing.T) { + defaultReq := GetPoolListReq{ + SysName: "non-default", + Query: true, + } + defaultPoolInfoResp := []*daos.PoolInfo{ + { + State: daos.PoolServiceStateReady, + UUID: daos_default_PoolInfo.UUID, + Label: daos_default_PoolInfo.Label, + ServiceReplicas: daos_default_PoolInfo.ServiceReplicas, + ServiceLeader: daos_default_PoolInfo.ServiceLeader, + }, + } + + for name, tc := range map[string]struct { + setup func(t *testing.T) + ctx context.Context + req GetPoolListReq + checkParams func(t *testing.T) + expPools []*daos.PoolInfo + expErr error + }{ + "nil context": { + expErr: errNilCtx, + }, + "daos_mgmt_list_pools fails (sizes)": { + setup: func(t *testing.T) { + daos_mgmt_list_pools_RC = -_Ctype_int(daos.IOError) + }, + ctx: test.Context(t), + expErr: errors.Wrap(daos.IOError, "failed to list pools"), + }, + "daos_mgmt_list_pools fetch fails (not retryable)": { + setup: func(t *testing.T) { + daos_mgmt_list_pools_RCList = []_Ctype_int{ + 0, + -_Ctype_int(daos.NoMemory), + } + }, + ctx: test.Context(t), + expErr: errors.Wrap(daos.NoMemory, "failed to list pools"), + }, + "daos_pool_connect fails": { + setup: func(t *testing.T) { + daos_pool_connect_RC = -_Ctype_int(daos.IOError) + }, + ctx: test.Context(t), + req: defaultReq, + expPools: []*daos.PoolInfo{}, + }, + "daos_mgmt_list_pools fetch fails (retryable)": { + setup: func(t *testing.T) { + daos_mgmt_list_pools_RCList = []_Ctype_int{ + 0, + -_Ctype_int(daos.StructTooSmall), + 0, + 0, + } + }, + ctx: test.Context(t), + expPools: defaultPoolInfoResp, + }, + "default system name supplied": { + ctx: test.Context(t), + req: GetPoolListReq{}, + checkParams: func(t *testing.T) { + test.CmpAny(t, "sysName", build.DefaultSystemName, daos_mgmt_list_pools_SetSys) + }, + expPools: defaultPoolInfoResp, + }, + "success (no pools)": { + setup: func(t *testing.T) { + daos_mgmt_list_pools_RetPools = nil + }, + ctx: test.Context(t), + req: defaultReq, + }, + "success (no query)": { + ctx: test.Context(t), + req: GetPoolListReq{ + SysName: defaultReq.SysName, + }, + checkParams: func(t *testing.T) { + test.CmpAny(t, "sysName", defaultReq.SysName, daos_mgmt_list_pools_SetSys) + }, + expPools: defaultPoolInfoResp, + }, + "success (query)": { + ctx: test.Context(t), + req: defaultReq, + checkParams: func(t *testing.T) { + test.CmpAny(t, "sysName", defaultReq.SysName, daos_mgmt_list_pools_SetSys) + }, + expPools: func() []*daos.PoolInfo { + pi := copyPoolInfo(&daos_default_PoolInfo) + pi.EnabledRanks = nil + pi.DisabledRanks = nil + + return []*daos.PoolInfo{pi} + }(), + }, + } { + t.Run(name, func(t *testing.T) { + t.Cleanup(ResetTestStubs) + if tc.setup != nil { + tc.setup(t) + } + log, buf := logging.NewTestLogger(name) + defer test.ShowBufferOnFailure(t, buf) + + gotPools, err := GetPoolList(mustLogCtx(tc.ctx, log), tc.req) + test.CmpErr(t, tc.expErr, err) + if tc.expErr != nil { + return + } + + if tc.checkParams != nil { + tc.checkParams(t) + } + + test.CmpAny(t, "GetPoolList() PoolList", tc.expPools, gotPools) + }) + } +} diff --git a/src/control/lib/daos/api/util.go b/src/control/lib/daos/api/util.go new file mode 100644 index 000000000000..9c461cd5c16f --- /dev/null +++ b/src/control/lib/daos/api/util.go @@ -0,0 +1,117 @@ +package api + +import ( + "context" + "unsafe" + + "github.com/google/uuid" + "github.com/pkg/errors" + + "github.com/daos-stack/daos/src/control/lib/daos" + "github.com/daos-stack/daos/src/control/lib/ranklist" + "github.com/daos-stack/daos/src/control/logging" +) + +/* +#include +#include + +#include + +#include "util.h" +*/ +import "C" + +func goBool2int(in bool) (out C.int) { + if in { + out = 1 + } + return +} + +func copyUUID(dst *C.uuid_t, src uuid.UUID) error { + if dst == nil { + return errors.Wrap(daos.InvalidInput, "nil dest uuid_t") + } + + for i, v := range src { + dst[i] = C.uchar(v) + } + + return nil +} + +func uuidToC(in uuid.UUID) (out C.uuid_t) { + for i, v := range in { + out[i] = C.uchar(v) + } + + return +} + +func uuidFromC(cUUID C.uuid_t) (uuid.UUID, error) { + return uuid.FromBytes(C.GoBytes(unsafe.Pointer(&cUUID[0]), C.int(len(cUUID)))) +} + +func freeString(s *C.char) { + C.free(unsafe.Pointer(s)) +} + +func iterStringsBuf(cBuf unsafe.Pointer, expected C.size_t, cb func(string)) error { + var curLen C.size_t + + // Create a Go slice for easy iteration (no pointer arithmetic in Go). + bufSlice := unsafe.Slice((*C.char)(cBuf), expected) + for total := C.size_t(0); total < expected; total += curLen + 1 { + chunk := bufSlice[total:] + curLen = C.strnlen(&chunk[0], expected-total) + + if curLen >= expected-total { + return errors.Wrap(daos.NoMemory, "corrupt buffer") + } + + chunk = bufSlice[total : total+curLen] + cb(C.GoString(&chunk[0])) + } + + return nil +} + +func rankSetFromC(cRankList *C.d_rank_list_t) (*ranklist.RankSet, error) { + if cRankList == nil { + return nil, errors.Wrap(daos.InvalidInput, "nil ranklist") + } + + cRankSlice := unsafe.Slice(cRankList.rl_ranks, cRankList.rl_nr) + rs := ranklist.NewRankSet() + for _, cRank := range cRankSlice { + rs.Add(ranklist.Rank(cRank)) + } + + return rs, nil +} + +func ranklistFromGo(rs *ranklist.RankSet) *C.d_rank_list_t { + if rs == nil { + return nil + } + + rl := C.d_rank_list_alloc(C.uint32_t(rs.Count())) + cRanks := unsafe.Slice(rl.rl_ranks, rs.Count()) + for i, r := range rs.Ranks() { + cRanks[i] = C.d_rank_t(r) + } + + return rl +} + +func mustLogCtx(parent context.Context, log logging.Logger) context.Context { + if parent == nil { + return nil + } + ctx, err := logging.ToContext(parent, log) + if err != nil { + panic(err) + } + return ctx +} diff --git a/src/control/lib/daos/api/util.h b/src/control/lib/daos/api/util.h new file mode 100644 index 000000000000..d0ddb067ca23 --- /dev/null +++ b/src/control/lib/daos/api/util.h @@ -0,0 +1,101 @@ +#ifndef __DAOS_API_UTIL_H__ +#define __DAOS_API_UTIL_H__ + +// #define D_LOGFAC DD_FAC(client) +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif + +#include + +/* cgo is unable to work directly with preprocessor macros + * so we have to provide these glue helpers. + */ +static inline uint64_t +daos_prop_co_status_val(uint32_t status, uint32_t flag, uint32_t ver) +{ + return DAOS_PROP_CO_STATUS_VAL(status, flag, ver); +} + +static void +daos_free(void *ptr) +{ + D_FREE(ptr); +} + +/* cgo is unable to work directly with unions, so we have + * to provide these glue helpers. + */ +static inline char * +get_dpe_str(struct daos_prop_entry *dpe) +{ + if (dpe == NULL) + return NULL; + + return dpe->dpe_str; +} + +static inline uint64_t +get_dpe_val(struct daos_prop_entry *dpe) +{ + if (dpe == NULL) + return 0; + + return dpe->dpe_val; +} + +static inline void * +get_dpe_val_ptr(struct daos_prop_entry *dpe) +{ + if (dpe == NULL) + return NULL; + + return dpe->dpe_val_ptr; +} + +static inline bool +dpe_is_negative(struct daos_prop_entry *dpe) +{ + if (dpe == NULL) + return 0; + + return dpe->dpe_flags & DAOS_PROP_ENTRY_NOT_SET; +} + +static inline void +set_dpe_str(struct daos_prop_entry *dpe, d_string_t str) +{ + if (dpe == NULL) + return; + + dpe->dpe_str = str; +} + +static inline void +set_dpe_val(struct daos_prop_entry *dpe, uint64_t val) +{ + if (dpe == NULL) + return; + + dpe->dpe_val = val; +} + +static inline void +set_dpe_val_ptr(struct daos_prop_entry *dpe, void *val_ptr) +{ + if (dpe == NULL) + return; + + dpe->dpe_val_ptr = val_ptr; +} + +/*static inline uint32_t +get_rebuild_state(struct daos_rebuild_status *drs) +{ + if (drs == NULL) + return 0; + + return drs->rs_state; +}*/ + +#endif /* __DAOS_API_UTIL_H__ */ \ No newline at end of file diff --git a/src/control/lib/daos/attribute.go b/src/control/lib/daos/attribute.go new file mode 100644 index 000000000000..e27968c216e1 --- /dev/null +++ b/src/control/lib/daos/attribute.go @@ -0,0 +1,39 @@ +// +// (C) Copyright 2025 Google LLC +// +// SPDX-License-Identifier: BSD-2-Clause-Patent +// + +package daos + +import "sort" + +type ( + // Attribute is a pool or container attribute. + Attribute struct { + Name string `json:"name"` + Value []byte `json:"value,omitempty"` + } + + // AttributeList is a list of attributes. + AttributeList []*Attribute +) + +// AsMap returns the attributes list as a map. +func (al AttributeList) AsMap() map[string][]byte { + m := make(map[string][]byte) + for _, a := range al { + m[a.Name] = a.Value + } + return m +} + +// AsList returns the attributes list as a sorted list of attribute names. +func (al AttributeList) AsList() []string { + names := make([]string, len(al)) + for i, a := range al { + names[i] = a.Name + } + sort.Strings(names) + return names +} diff --git a/src/control/lib/daos/pool.go b/src/control/lib/daos/pool.go index 6555a8cf7210..a4a8e10740f7 100644 --- a/src/control/lib/daos/pool.go +++ b/src/control/lib/daos/pool.go @@ -1,5 +1,6 @@ // // (C) Copyright 2020-2024 Intel Corporation. +// (C) Copyright 2025 Google LLC // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -106,6 +107,9 @@ type ( // PoolQueryMask implements a bitmask for pool query options. PoolQueryMask C.uint64_t + + // PoolConnectFlag represents DAOS pool connect options. + PoolConnectFlag uint ) const ( @@ -126,11 +130,11 @@ const ( PoolQueryOptionDeadEngines PoolQueryOption = "dead_engines" // PoolConnectFlagReadOnly indicates that the connection is read-only. - PoolConnectFlagReadOnly = C.DAOS_PC_RO + PoolConnectFlagReadOnly PoolConnectFlag = C.DAOS_PC_RO // PoolConnectFlagReadWrite indicates that the connection is read-write. - PoolConnectFlagReadWrite = C.DAOS_PC_RW + PoolConnectFlagReadWrite PoolConnectFlag = C.DAOS_PC_RW // PoolConnectFlagExclusive indicates that the connection is exclusive. - PoolConnectFlagExclusive = C.DAOS_PC_EX + PoolConnectFlagExclusive PoolConnectFlag = C.DAOS_PC_EX ) func (pqo PoolQueryOption) String() string { diff --git a/src/control/lib/daos/util.go b/src/control/lib/daos/util.go new file mode 100644 index 000000000000..39776f956061 --- /dev/null +++ b/src/control/lib/daos/util.go @@ -0,0 +1,18 @@ +// +// (C) Copyright 2025 Google LLC +// +// SPDX-License-Identifier: BSD-2-Clause-Patent +// + +package daos + +import "unsafe" + +/* +#include +*/ +import "C" + +func freeString(s *C.char) { + C.free(unsafe.Pointer(s)) +} diff --git a/src/control/lib/ranklist/ranklist.go b/src/control/lib/ranklist/ranklist.go index c65d3e7259f3..0b6861e005ce 100644 --- a/src/control/lib/ranklist/ranklist.go +++ b/src/control/lib/ranklist/ranklist.go @@ -1,5 +1,6 @@ // // (C) Copyright 2020-2024 Intel Corporation. +// (C) Copyright 2025 Google LLC // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -98,7 +99,7 @@ func (rs *RankSet) Merge(other *RankSet) { // Replace replaces the contents of the receiver with the supplied RankSet. func (rs *RankSet) Replace(other *RankSet) { - if rs == nil || other == nil { + if rs == nil || other == nil || other.ns == nil { return }