diff --git a/CHANGELOG.md b/CHANGELOG.md index f32cf14f..1e672427 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,16 @@ [Releases](https://github.com/Huawei/eSDK_K8S_Plugin/releases) +## Changes since v4.1.0 + +- Support OpenShift 4.13 +- Support Centos 8.4 X86_64 +- Support Rocky Linux 8.6 X86_64 +- Support EulerOS V2R11 X86_64 +- Support k8s 1.16 and 1.27 +- Support configuring the timeout for executing commands +- Support create volume snapshot for Hyper-Metro + ## Changes since v4.0.0 **Enhancements** diff --git a/Makefile b/Makefile index 07c636cd..f7e17eab 100644 --- a/Makefile +++ b/Makefile @@ -54,6 +54,10 @@ COPY_FILE: mkdir -p ./${PACKAGE}/helm/ cp -r ./helm/* ./${PACKAGE}/helm/ + mkdir -p ./${PACKAGE}/manual/ + cp -r ./manual/* ./${PACKAGE}/manual/ + cp -r ./helm/esdk/crds ./${PACKAGE}/manual/esdk/crds + mkdir -p ./${PACKAGE}/tools cp -r ./tools/imageUpload/* ./${PACKAGE}/tools diff --git a/RELEASE.md b/RELEASE.md index d7ab401c..e0f5facd 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,3 +1,36 @@ +# 4.2.0 + +## Supported Huawei storage products + +| Storage Product | Version | +| ------------------- |-------------------------------------------------| +| OceanStor Dorado V6 | 6.0.0, 6.0.1, 6.1.0, 6.1.2, 6.1.3, 6.1.5, 6.1.6 | +| OceanStor Dorado V3 | V300R002 | +| OceanStor V6 | 6.1.3, 6.1.5, 6.1.6 | +| OceanStor V5/F V5 | V500R007, V500R007 Kunpeng | +| OceanStor V3/F V3 | V300R006 | +| FusionStorage | V100R006C30 | +| FusionStorage Block | 8.0.0, 8.0.1 | +| OceanStor Pacific | 8.1.0, 8.1.1, 8.1.2, 8.1.3, 8.1.5 | + +## Supported container platforms and operating systems (OSs) + +| Container platform/OS | Version | +|-----------------------|----------------------------------------------------------------------------------------------------| +| Kubernetes | 1.16, 1.18 - 1.27 | +| Red Hat OpenShift | 4.6 EUS, 4.7, 4.8, 4.9, 4.10, 4.11, 4.12, 4.13 | +| Tanzu Kubernetes | TKGI 1.14.1, TKGI 1.15, TKGI 1.16 | +| CCE Agile | 22.3.2 | +| CentOS | 7.6 x86_64, 7.7 x86_64, 7.9 x86_64, 8.2 x86_64, 8.4 x86_64 | +| Rocky Linux | 8.6 x86_64 | +| SUSE | 15 SP2 x86_64, 15 SP3 x86_64, | +| Red Hat CoreOS | 4.6 x86_64, 4.7 x86_64, 4.8 x86_64, 4.9 x86_64, 4.10 x86_64, 4.11 x86_64, 4.12 x86_64, 4.13 x86_64 | +| Ubuntu | 18.04 x86_64, 20.04 x86_64, 22.04 x86_64 | +| Kylin | V10 SP1 Arm/x86_64, V10 SP2 Arm/x86_64, 7.6 x86_64 | +| Debian | 11 x86_64, 9 x86_64 | +| EulerOS | v2R9 x86_64, V2R10 Arm/x86_64, V2R11 x86_64 | + + # 4.1.0 ## Supported Huawei storage products @@ -18,7 +51,7 @@ | Container platform/OS | Version | |-----------------------|--------------------------------------------------------------------------| | Kubernetes | 1.18 - 1.26 | -| Red Hat OpenShift | 4.6 EUS, 4.7, 4.8, 4.9, 4.10, 4.11, 1.12 | +| Red Hat OpenShift | 4.6 EUS, 4.7, 4.8, 4.9, 4.10, 4.11, 4.12 | | Tanzu Kubernetes | TKGI 1.14.1 | | CCE Agile | 22.3.2 | | CentOS | 7.6 x86_64, 7.7 x86_64, 7.9 x86_64, 8.2 x86_64 | diff --git a/cli/client/client_helper.go b/cli/client/client_helper.go index 8ce792da..fc18acb4 100644 --- a/cli/client/client_helper.go +++ b/cli/client/client_helper.go @@ -17,6 +17,7 @@ package client import ( + "context" "encoding/json" "errors" "fmt" @@ -155,3 +156,34 @@ func safeToArray[T any](t T) []T { } return []T{t} } + +func getObjectType(object interface{}) ObjectType { + switch object.(type) { + case *corev1.Namespace, *corev1.NamespaceList: + return Namespace + case *corev1.Node, *corev1.NodeList: + return Node + case *corev1.Pod, *corev1.PodList: + return Pod + default: + return Unknown + } +} + +func (r *CommonCallHandler[T]) CheckObjectExist(ctx context.Context, namespace, nodeName, + objectName string) (bool, error) { + var object, empty T + err := r.client.GetObject(ctx, getObjectType(&object), namespace, nodeName, JSON, &object, objectName) + if err != nil { + return false, err + } + + return !reflect.DeepEqual(object, empty), nil +} + +func (r *CommonCallHandler[T]) GetObject(ctx context.Context, namespace, nodeName string, + objectName ...string) (T, error) { + var object T + err := r.client.GetObject(ctx, getObjectType(&object), namespace, nodeName, JSON, &object, objectName...) + return object, err +} diff --git a/cli/client/client_helper_test.go b/cli/client/client_helper_test.go new file mode 100644 index 00000000..07f408b7 --- /dev/null +++ b/cli/client/client_helper_test.go @@ -0,0 +1,175 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package client + +import ( + "context" + "errors" + "reflect" + "testing" + + "github.com/agiledragon/gomonkey/v2" + "github.com/smartystreets/goconvey/convey" + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestGetObjectType_get_namespace_type(t *testing.T) { + // arrange + var mockObject corev1.Namespace + var except = Namespace + convey.Convey("test get_namespace_type", t, func() { + // action + objectType := getObjectType(&mockObject) + // assert + convey.So(objectType, convey.ShouldResemble, except) + }) +} + +func TestGetObjectType_get_node_type(t *testing.T) { + // arrange + var mockObject corev1.NodeList + var except = Node + convey.Convey("test get_node_type", t, func() { + // action + objectType := getObjectType(&mockObject) + // assert + convey.So(objectType, convey.ShouldResemble, except) + }) +} + +func TestGetObjectType_get_pod_type(t *testing.T) { + // arrange + var mockObject corev1.Pod + var except = Pod + convey.Convey("test get_pod_type", t, func() { + // action + objectType := getObjectType(&mockObject) + // assert + convey.So(objectType, convey.ShouldResemble, except) + }) +} + +func TestGetObjectType_get_unknown_type(t *testing.T) { + // arrange + var mockObject interface{} + var except = Unknown + convey.Convey("test get_unknown_type", t, func() { + // action + objectType := getObjectType(&mockObject) + // assert + convey.So(objectType, convey.ShouldResemble, except) + }) +} + +func TestCommonCallHandler_CheckObjectExist_check_exist_namespace(t *testing.T) { + // arrange + var mockNamespace, mockNodeName, mockObjectName = IgnoreNamespace, IgnoreNode, "namespace" + var mockCli = NewCommonCallHandler[corev1.Namespace](&KubernetesCLI{}) + var except = true + // mock + patches := gomonkey.ApplyMethod(reflect.TypeOf(&KubernetesCLI{}), "GetObject", + func(cli *KubernetesCLI, ctx context.Context, objectType ObjectType, namespace string, nodeName string, + outputType OutputType, data interface{}, objectName ...string) error { + if objectType == Namespace && namespace == mockNamespace && nodeName == mockNodeName && + outputType == JSON && objectName[0] == mockObjectName { + if ns, ok := data.(*corev1.Namespace); ok { + *ns = corev1.Namespace{} + (*ns).Name = mockObjectName + } + return nil + } + return errors.New("") + }) + defer patches.Reset() + + convey.Convey("test check_exist_namespace", t, func() { + // action + exist, err := mockCli.CheckObjectExist(context.Background(), mockNamespace, mockNodeName, mockObjectName) + // assert + convey.So(exist, convey.ShouldResemble, except) + convey.So(err, convey.ShouldBeNil) + }) +} + +func TestCommonCallHandler_CheckObjectExist_check_not_exist_node(t *testing.T) { + // arrange + var mockNamespace, mockNodeName, mockObjectName = IgnoreNamespace, IgnoreNode, "node" + var mockCli = NewCommonCallHandler[corev1.Node](&KubernetesCLI{}) + var except = false + // mock + patches := gomonkey.ApplyMethod(reflect.TypeOf(&KubernetesCLI{}), "GetObject", + func(cli *KubernetesCLI, ctx context.Context, objectType ObjectType, namespace string, nodeName string, + outputType OutputType, data interface{}, objectName ...string) error { + if objectType == Node && namespace == mockNamespace && nodeName == mockNodeName && + outputType == JSON && objectName[0] == mockObjectName { + return nil + } + return errors.New("") + }) + defer patches.Reset() + + convey.Convey("test check_not_exist_node", t, func() { + // action + exist, err := mockCli.CheckObjectExist(context.Background(), mockNamespace, mockNodeName, mockObjectName) + // assert + convey.So(exist, convey.ShouldResemble, except) + convey.So(err, convey.ShouldBeNil) + }) +} + +func TestCommonCallHandler_GetObject_get_podList(t *testing.T) { + // arrange + var mockNamespace, mockNodeName = "namespace", IgnoreNode + var mockCli = NewCommonCallHandler[corev1.PodList](&KubernetesCLI{}) + var except = corev1.PodList{ + Items: []corev1.Pod{ + { + ObjectMeta: v1.ObjectMeta{ + Name: "pod1", + }, + }, + { + ObjectMeta: v1.ObjectMeta{ + Name: "pod2", + }, + }, + }, + } + // mock + patches := gomonkey.ApplyMethod(reflect.TypeOf(&KubernetesCLI{}), "GetObject", + func(cli *KubernetesCLI, ctx context.Context, objectType ObjectType, namespace string, nodeName string, + outputType OutputType, data interface{}, objectName ...string) error { + if objectType == Pod && namespace == mockNamespace && nodeName == mockNodeName && + outputType == JSON && objectName == nil { + if podList, ok := data.(*corev1.PodList); ok { + *podList = except + } + return nil + } + return errors.New("") + }) + defer patches.Reset() + + convey.Convey("test get pod list", t, func() { + // action + object, err := mockCli.GetObject(context.Background(), mockNamespace, mockNodeName) + // assert + convey.So(object, convey.ShouldResemble, except) + convey.So(err, convey.ShouldBeNil) + }) +} diff --git a/cli/client/kube_client.go b/cli/client/kube_client.go index 685ab2b2..1de7cfec 100644 --- a/cli/client/kube_client.go +++ b/cli/client/kube_client.go @@ -17,6 +17,7 @@ package client import ( + "context" "encoding/json" "fmt" @@ -125,3 +126,45 @@ func (k *KubernetesCLI) GetNameSpace() (string, error) { namespace := serviceAccount.ObjectMeta.Namespace return namespace, nil } + +// GetObject used to get the specified format data of the object with specified conditions and unmarshal to the data. +func (k *KubernetesCLI) GetObject(ctx context.Context, objectType ObjectType, namespace, nodeName string, + outputType OutputType, data interface{}, objectName ...string) error { + return NewKubernetesCLIArgs(k.CLI()). + SelectObject(objectType, objectName...). + WithSpecifiedNamespace(namespace). + WithSpecifiedNode(nodeName). + WithOutPutFormat(outputType). + Get(ctx, &data) +} + +// ExecCmdInSpecifiedContainer used to executes the specified command in the container with specified conditions. +func (k *KubernetesCLI) ExecCmdInSpecifiedContainer(ctx context.Context, namespace, containerName, cmd string, + podName ...string) ([]byte, error) { + return NewKubernetesCLIArgs(k.CLI()). + SelectObject(Pod, podName...). + WithSpecifiedNamespace(namespace). + WithSpecifiedContainer(containerName). + Exec(ctx, cmd) +} + +// CopyContainerFileToLocal used to copying a Local File to a Container with Specified Conditions +func (k *KubernetesCLI) CopyContainerFileToLocal(ctx context.Context, namespace, containerName, src, dst string, + podName ...string) ([]byte, error) { + return NewKubernetesCLIArgs(k.CLI()). + SelectObject(Pod, podName...). + WithSpecifiedNamespace(namespace). + WithSpecifiedContainer(containerName). + Copy(ctx, src, dst, ContainerToLocal) +} + +// GetConsoleLogs used to get the console logs of a specified container. +func (k *KubernetesCLI) GetConsoleLogs(ctx context.Context, namespace, containerName string, isHistoryLogs bool, + podName ...string) ([]byte, error) { + return NewKubernetesCLIArgs(k.CLI()). + SelectObject(Pod, podName...). + WithSpecifiedNamespace(namespace). + WithSpecifiedContainer(containerName). + WithHistoryLogs(isHistoryLogs). + Logs(ctx) +} diff --git a/cli/client/kube_client_test.go b/cli/client/kube_client_test.go new file mode 100644 index 00000000..8eec0443 --- /dev/null +++ b/cli/client/kube_client_test.go @@ -0,0 +1,137 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package client + +import ( + "context" + "encoding/json" + "testing" + + "github.com/smartystreets/goconvey/convey" + coreV1 "k8s.io/api/core/v1" +) + +func TestKubernetesCLI_GetObject_success(t *testing.T) { + // arrange + var mockCli *KubernetesCLI = &KubernetesCLI{cli: CLIKubernetes} + var mockNamespace, mockNodeName, mockObjectName string = "huawei-csi", IgnoreNode, "huawei-csi-node-9lxhm" + var mockObjectType ObjectType = Pod + var mockOutputType OutputType = JSON + var mockData, except coreV1.Pod = coreV1.Pod{}, coreV1.Pod{} + json.Unmarshal([]byte(returnStr), &except) + ctx := context.Background() + // mock + patches := mockExecReturnStdOut("kubectl get pod huawei-csi-node-9lxhm -n huawei-csi -o=json") + defer patches.Reset() + + convey.Convey("test get object success", t, func() { + // action + err := mockCli.GetObject(ctx, mockObjectType, mockNamespace, mockNodeName, mockOutputType, &mockData, + mockObjectName) + + //assert + convey.So(mockData, convey.ShouldResemble, except) + convey.So(err, convey.ShouldBeNil) + }) +} + +func TestKubernetesCLI_GetObject_failed_without_objectType(t *testing.T) { + // arrange + var mockCli *KubernetesCLI = &KubernetesCLI{cli: CLIKubernetes} + var mockNamespace, mockNodeName, mockObjectName string = "huawei-csi", IgnoreNode, "huawei-csi-node-9lxhm" + var mockObjectType ObjectType = "" + var mockOutputType OutputType = JSON + var mockData, except coreV1.Pod = coreV1.Pod{}, coreV1.Pod{} + ctx := context.Background() + // mock + patches := mockExecReturnStdOut("kubectl get pod huawei-csi-node-9lxhm -n huawei-csi -o=json") + defer patches.Reset() + + convey.Convey("test get object success", t, func() { + // action + err := mockCli.GetObject(ctx, mockObjectType, mockNamespace, mockNodeName, mockOutputType, &mockData, + mockObjectName) + + //assert + convey.So(mockData, convey.ShouldResemble, except) + convey.So(err, convey.ShouldBeError) + }) +} + +func TestKubernetesCLI_CopyContainerFileToLocal_success(t *testing.T) { + // arrange + var mockCli *KubernetesCLI = &KubernetesCLI{cli: CLIKubernetes} + var mockNamespace, mockContainerName, mockObjectName, mockSrc, mockDst = "huawei-csi", "huawei-csi-driver", + "huawei-csi-node-9lxhm", "tmp/a.tar", "/tmp/slave1/a.tar" + mockCtx := context.Background() + //mock + patches := mockExecReturnStdOut("kubectl cp huawei-csi/huawei-csi-node-9lxhm:tmp/a.tar " + + "/tmp/slave1/a.tar -c huawei-csi-driver") + defer patches.Reset() + + convey.Convey("test copy file from container to local", t, func() { + // action + out, err := mockCli.CopyContainerFileToLocal(mockCtx, mockNamespace, mockContainerName, mockSrc, mockDst, + mockObjectName) + + //assert + convey.So(out, convey.ShouldResemble, []byte(returnStr)) + convey.So(err, convey.ShouldBeNil) + }) +} + +func TestKubernetesCLI_GetConsoleLogs_success(t *testing.T) { + // arrange + var mockCli *KubernetesCLI = &KubernetesCLI{cli: CLIKubernetes} + var mockNamespace, mockContainerName, mockObjectName string = "huawei-csi", "huawei-csi-driver", + "huawei-csi-node-9lxhm" + mockCtx := context.Background() + // mock + patches := mockExecReturnStdOut("kubectl logs huawei-csi-node-9lxhm -c huawei-csi-driver -n huawei-csi") + defer patches.Reset() + + convey.Convey("test get container console logs", t, func() { + // action + out, err := mockCli.GetConsoleLogs(mockCtx, mockNamespace, mockContainerName, false, mockObjectName) + + //assert + convey.So(out, convey.ShouldResemble, []byte(returnStr)) + convey.So(err, convey.ShouldBeNil) + }) +} + +func TestKubernetesCLI_ExecCmdInSpecifiedContainer_success(t *testing.T) { + // arrange + var mockCli *KubernetesCLI = &KubernetesCLI{cli: CLIKubernetes} + var mockNamespace, mockContainerName, mockCmd, mockObjectName = "huawei-csi", "huawei-csi-driver", "collect.sh", + "huawei-csi-node-9lxhm" + mockCtx := context.Background() + // mock + patches := mockExecReturnStdOut("kubectl exec huawei-csi-node-9lxhm -c huawei-csi-driver " + + "-n huawei-csi -- collect.sh") + defer patches.Reset() + + convey.Convey("test exec script in container", t, func() { + // action + out, err := mockCli.ExecCmdInSpecifiedContainer(mockCtx, mockNamespace, mockContainerName, mockCmd, + mockObjectName) + + //assert + convey.So(out, convey.ShouldResemble, []byte(returnStr)) + convey.So(err, convey.ShouldBeNil) + }) +} diff --git a/cli/client/kubectl_options.go b/cli/client/kubectl_options.go new file mode 100644 index 00000000..4a6bb018 --- /dev/null +++ b/cli/client/kubectl_options.go @@ -0,0 +1,304 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package client + +import ( + "context" + "encoding/json" + "errors" + "fmt" + + "github.com/ghodss/yaml" + + "huawei-csi-driver/cli/helper" +) + +type Filter struct { + node string + namespace string + container string + allowNotFound bool + isHistoryLog bool +} + +type KubernetesCLIArgs struct { + client string + + // Collection of objects to manipulate + objectName []string + + // Specified condition for objects + objectType ObjectType + outputFormat OutputType + selector Filter +} + +// ObjectType Define the operation object type. +type ObjectType string + +// OutputType Defines the output format type. +type OutputType string + +// CopyType Defines the file copy type. +type CopyType byte + +const ( + Pod ObjectType = "pod" // Operate pod objects. + Node ObjectType = "node" // Operate node objects. + Namespace ObjectType = "namespace" // Operate namespace objects. + Unknown ObjectType = "" // Unknown object + + JSON OutputType = "-o=json" // Obtains data in JSON format. + YAML OutputType = "-o=yaml" // Obtains data in YAML format. + + LocalToContainer CopyType = 0 // Copy files from the local host to the container. + ContainerToLocal CopyType = 1 // Copy files from the container to the local host. + + IgnoreNode = "" // used to ignore the specified condition of the node when invoking an interface. + IgnoreContainer = "" // used to ignore the specified condition of the container when invoking an interface. + IgnoreNamespace = "" // used to ignore the specified condition of the namespace when invoking an interface. + + getStr = "get" + execStr = "exec" + logsStr = "logs" + copyStr = "cp" + + ignoreNotFound = "--ignore-not-found" +) + +var execReturnStdOut = helper.BashExecReturnStdOut + +// NewKubernetesCLIArgs get a *KubernetesCLIArgs instance +func NewKubernetesCLIArgs(client string) *KubernetesCLIArgs { + return &KubernetesCLIArgs{ + client: client, + objectName: make([]string, 0), + } +} + +// WithSpecifiedNamespace specifies the namespace of the object to be manipulated. +func (k *KubernetesCLIArgs) WithSpecifiedNamespace(namespace string) *KubernetesCLIArgs { + k.selector.namespace = namespace + return k +} + +// WithSpecifiedNode specifies the node of the object to be manipulated. +func (k *KubernetesCLIArgs) WithSpecifiedNode(nodeName string) *KubernetesCLIArgs { + k.selector.node = nodeName + return k +} + +// WithSpecifiedContainer specifies the container of the object to be manipulated. +func (k *KubernetesCLIArgs) WithSpecifiedContainer(containerName string) *KubernetesCLIArgs { + k.selector.container = containerName + return k +} + +// WithOutPutFormat specifies the output format of the object to be manipulated. +func (k *KubernetesCLIArgs) WithOutPutFormat(outputType OutputType) *KubernetesCLIArgs { + k.outputFormat = outputType + return k +} + +// WithIgnoreNotFound adds the --ignore-not-found option. +func (k *KubernetesCLIArgs) WithIgnoreNotFound() *KubernetesCLIArgs { + k.selector.allowNotFound = true + return k +} + +// WithHistoryLogs adds the -p option. +func (k *KubernetesCLIArgs) WithHistoryLogs(isHistoryLog bool) *KubernetesCLIArgs { + k.selector.isHistoryLog = isHistoryLog + return k +} + +// SelectObject specifies the type and name of the object to be operated. +func (k *KubernetesCLIArgs) SelectObject(objectType ObjectType, objectName ...string) *KubernetesCLIArgs { + k.objectType = objectType + k.setObject(objectName) + return k +} + +func (k *KubernetesCLIArgs) setObject(objectName []string) { + if len(objectName) != 0 { + k.objectName = append(k.objectName, objectName...) + } +} + +func (k *KubernetesCLIArgs) getObject() ([]string, error) { + switch k.objectType { + case Node, Pod, Namespace: + return k.objectName, nil + default: + return nil, errors.New("unknown object type") + } +} + +// Get obtains object data based on the configured parameters and unmarshal to the data. +func (k *KubernetesCLIArgs) Get(ctx context.Context, data interface{}) error { + object, err := k.getObject() + if err != nil { + return err + } + + args := []string{getStr} + args = append(args, string(k.objectType)) + args = append(args, object...) + args = append(args, k.selector.getAllFilter()...) + args = append(args, string(k.outputFormat)) + + out, err := execReturnStdOut(ctx, k.client, args) + if err != nil { + return err + } + + switch k.outputFormat { + case JSON: + err = json.Unmarshal(out, data) + case YAML: + err = yaml.Unmarshal(out, data) + default: + return errors.New("outputFormat not valid") + } + return err +} + +// Exec run the command in the specified container based on the configured parameters. +func (k *KubernetesCLIArgs) Exec(ctx context.Context, cmd string) ([]byte, error) { + objects, err := k.getObject() + if err != nil { + return nil, err + } + + res := make([]byte, 0) + for _, object := range objects { + args := []string{execStr, object} + args = append(args, k.selector.getAllFilter()...) + args = append(args, []string{"--", cmd}...) + out, err := execReturnStdOut(ctx, k.client, args) + if err != nil { + return nil, err + } + + res = append(res, out...) + } + + return res, nil +} + +// Copy local files and specified container files based on the configured parameters. +func (k *KubernetesCLIArgs) Copy(ctx context.Context, containerPath, localPath string, cpType CopyType) ([]byte, error) { + objects, err := k.getObject() + if err != nil { + return nil, err + } + + var isContainerToLocal bool + switch cpType { + case ContainerToLocal: + isContainerToLocal = true + case LocalToContainer: + default: + return nil, errors.New("copyType not valid ") + } + + res := make([]byte, 0) + for _, object := range objects { + args := []string{copyStr} + containerPath = fmt.Sprintf("%s/%s:%s", k.selector.namespace, object, containerPath) + if isContainerToLocal { + args = append(args, []string{containerPath, localPath}...) + } else { + args = append(args, []string{localPath, containerPath}...) + } + + args = append(args, k.selector.getContainerFilter()...) + out, err := execReturnStdOut(ctx, k.client, args) + if err != nil { + return nil, err + } + + res = append(res, out...) + } + + return res, nil +} + +// Logs obtains the console logs of a specified container. +func (k *KubernetesCLIArgs) Logs(ctx context.Context) ([]byte, error) { + objects, err := k.getObject() + if err != nil { + return nil, err + } + + res := make([]byte, 0) + for _, object := range objects { + args := []string{logsStr, object} + args = append(args, k.selector.getAllFilter()...) + out, err := execReturnStdOut(ctx, k.client, args) + if err != nil { + return nil, err + } + + res = append(res, out...) + } + + return res, nil +} + +func (f *Filter) getNamespaceFilter() []string { + if f.namespace != IgnoreNamespace { + return []string{"-n", f.namespace} + } + return []string{} +} + +func (f *Filter) getNodeFilter() []string { + if f.node != IgnoreNode { + return []string{fmt.Sprintf("--field-selector spec.nodeName=%s", f.node)} + } + return []string{} +} + +func (f *Filter) getContainerFilter() []string { + if f.container != IgnoreContainer { + return []string{"-c", f.container} + } + return []string{} +} + +func (f *Filter) getIgnoreFilter() []string { + if f.allowNotFound { + return []string{ignoreNotFound} + } + return []string{} +} + +func (f *Filter) getHistoryFilter() []string { + if f.isHistoryLog { + return []string{"-p"} + } + return []string{} +} + +func (f *Filter) getAllFilter() []string { + args := append(f.getNodeFilter(), f.getContainerFilter()...) + args = append(args, f.getNamespaceFilter()...) + args = append(f.getIgnoreFilter(), args...) + args = append(f.getHistoryFilter(), args...) + return args +} diff --git a/cli/client/kubectl_options_test.go b/cli/client/kubectl_options_test.go new file mode 100644 index 00000000..e8c6b4d1 --- /dev/null +++ b/cli/client/kubectl_options_test.go @@ -0,0 +1,174 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package client + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "testing" + + "github.com/agiledragon/gomonkey/v2" + "github.com/smartystreets/goconvey/convey" + coreV1 "k8s.io/api/core/v1" +) + +var returnStr = `{ + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": "2023-07-28T03:22:21Z", + "generateName": "huawei-csi-node-", + "labels": { + "app": "huawei-csi-node", + "controller-revision-hash": "5d99df786c", + "pod-template-generation": "1", + "provisioner": "csi.huawei.com" + }, + "name": "huawei-csi-node-9lxhm", + "namespace": "huawei-csi", + "resourceVersion": "3047239", + "selfLink": "/api/v1/namespaces/huawei-csi/pods/huawei-csi-node-9lxhm", + "uid": "3227c5e1-3cce-49ee-8d08-9e12e3ca3877" + } +}` + +func mockExecReturnStdOut(exceptCMD string) *gomonkey.Patches { + return gomonkey.ApplyFunc(execReturnStdOut, func(ctx context.Context, cli string, args []string) ([]byte, error) { + cmd := fmt.Sprintf("%s %s", cli, strings.Join(args, " ")) + fmt.Println(cmd) + if cmd != exceptCMD { + return nil, errors.New("error") + } + return []byte(returnStr), nil + }) +} + +func TestKubernetesCLIArgs_Get_failed_without_options(t *testing.T) { + // arrange + var mockArgs = NewKubernetesCLIArgs("kubectl") + var mockData, except = coreV1.Pod{}, coreV1.Pod{} + mockCtx := context.Background() + // mock + patches := mockExecReturnStdOut("kubectl get pod huawei-csi-node-9lxhm -n huawei-csi -o=json") + defer patches.Reset() + + convey.Convey("test get object failed without options", t, func() { + // action + err := mockArgs.Get(mockCtx, &mockData) + + //assert + convey.So(mockData, convey.ShouldResemble, except) + convey.So(err, convey.ShouldBeError) + }) +} + +func TestKubernetesCLIArgs_Get_success(t *testing.T) { + // arrange + var mockArgs = NewKubernetesCLIArgs("kubectl").SelectObject(Pod, "huawei-csi-node-9lxhm"). + WithSpecifiedNamespace("huawei-csi"). + WithSpecifiedNode(IgnoreNode). + WithOutPutFormat(JSON) + var mockData, except = coreV1.Pod{}, coreV1.Pod{} + json.Unmarshal([]byte(returnStr), &except) + mockCtx := context.Background() + // mock + patches := mockExecReturnStdOut("kubectl get pod huawei-csi-node-9lxhm -n huawei-csi -o=json") + defer patches.Reset() + + convey.Convey("test get object success", t, func() { + // action + err := mockArgs.Get(mockCtx, &mockData) + + //assert + convey.So(mockData, convey.ShouldResemble, except) + convey.So(err, convey.ShouldBeNil) + }) +} + +func TestKubernetesCLIArgs_Exec_success(t *testing.T) { + // arrange + var mockArgs = NewKubernetesCLIArgs("kubectl").SelectObject(Pod, "huawei-csi-node-9lxhm"). + WithSpecifiedNamespace("huawei-csi"). + WithSpecifiedContainer("huawei-csi-driver"). + WithSpecifiedNode(IgnoreNode) + var mockCmd = "collect.sh" + var except = []byte(returnStr) + mockCtx := context.Background() + // mock + patches := mockExecReturnStdOut("kubectl exec huawei-csi-node-9lxhm -c huawei-csi-driver " + + "-n huawei-csi -- collect.sh") + defer patches.Reset() + + convey.Convey("test exec cmd in container success", t, func() { + // action + out, err := mockArgs.Exec(mockCtx, mockCmd) + + // assert + convey.So(out, convey.ShouldResemble, except) + convey.So(err, convey.ShouldBeNil) + }) +} + +func TestKubernetesCLIArgs_Copy_success(t *testing.T) { + // arrange + var mockArgs = NewKubernetesCLIArgs("kubectl").SelectObject(Pod, "huawei-csi-node-9lxhm"). + WithSpecifiedNamespace("huawei-csi"). + WithSpecifiedContainer("huawei-csi-driver"). + WithSpecifiedNode(IgnoreNode) + var mockContainerPath, mockLocalPath = "tmp/a.tar", "/tmp/slave1/a.tar" + var mockCopyType = ContainerToLocal + var except = []byte(returnStr) + mockCtx := context.Background() + // mock + patches := mockExecReturnStdOut("kubectl cp huawei-csi/huawei-csi-node-9lxhm:tmp/a.tar " + + "/tmp/slave1/a.tar -c huawei-csi-driver") + defer patches.Reset() + + convey.Convey("test copy file from container to local success", t, func() { + // action + out, err := mockArgs.Copy(mockCtx, mockContainerPath, mockLocalPath, mockCopyType) + + // assert + convey.So(out, convey.ShouldResemble, except) + convey.So(err, convey.ShouldBeNil) + }) +} + +func TestKubernetesCLIArgs_Logs(t *testing.T) { + // arrange + var mockArgs = NewKubernetesCLIArgs("kubectl").SelectObject(Pod, "huawei-csi-node-9lxhm"). + WithSpecifiedNamespace("huawei-csi"). + WithSpecifiedContainer("huawei-csi-driver"). + WithSpecifiedNode(IgnoreNode) + var except = []byte(returnStr) + mockCtx := context.Background() + // mock + patches := mockExecReturnStdOut("kubectl logs huawei-csi-node-9lxhm -c huawei-csi-driver -n huawei-csi") + defer patches.Reset() + + convey.Convey("test get container console logs success", t, func() { + // action + out, err := mockArgs.Logs(mockCtx) + + // assert + convey.So(out, convey.ShouldResemble, except) + convey.So(err, convey.ShouldBeNil) + }) +} diff --git a/cli/client/types.go b/cli/client/types.go index a26c887d..8b1571ea 100644 --- a/cli/client/types.go +++ b/cli/client/types.go @@ -16,6 +16,8 @@ package client +import "context" + // ResourceType Defines the resource type, e.g. secret, configmap... type ResourceType string @@ -27,4 +29,13 @@ type KubernetesClient interface { DeleteResourceByQualifiedNames(qualifiedNames []string, namespace string) (string, error) GetResource(name []string, namespace, outputType string, resourceType ResourceType) ([]byte, error) CheckResourceExist(name, namespace string, resourceType ResourceType) (bool, error) + + GetObject(ctx context.Context, objectType ObjectType, namespace, nodeName string, outputType OutputType, + data interface{}, objectName ...string) error + ExecCmdInSpecifiedContainer(ctx context.Context, namespace, containerName, cmd string, + podName ...string) ([]byte, error) + CopyContainerFileToLocal(ctx context.Context, namespace, containerName, src, dst string, + podName ...string) ([]byte, error) + GetConsoleLogs(ctx context.Context, namespace, containerName string, isHistoryLogs bool, + podName ...string) ([]byte, error) } diff --git a/cli/cmd/collect.go b/cli/cmd/collect.go new file mode 100644 index 00000000..0b92264b --- /dev/null +++ b/cli/cmd/collect.go @@ -0,0 +1,32 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package command + +import ( + "github.com/spf13/cobra" + + "huawei-csi-driver/cli/cmd/options" +) + +func init() { + options.NewFlagsOptions(collectCmd).WithParent(RootCmd) +} + +var collectCmd = &cobra.Command{ + Use: "collect", + Short: "collect messages in Kubernetes", +} diff --git a/cli/cmd/collect_logs.go b/cli/cmd/collect_logs.go new file mode 100644 index 00000000..3df023bf --- /dev/null +++ b/cli/cmd/collect_logs.go @@ -0,0 +1,68 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package command + +import ( + "github.com/spf13/cobra" + + "huawei-csi-driver/cli/cmd/options" + "huawei-csi-driver/cli/config" + "huawei-csi-driver/cli/helper" + "huawei-csi-driver/cli/resources" +) + +func init() { + options.NewFlagsOptions(collectLogsCmd). + WithNameSpace(true). + WithAllNodes(). + WithNodeName(). + WithParent(collectCmd) +} + +var ( + collectLogsExample = helper.Examples(` + # Collect logs of all nodes in specified namespace + oceanctl collect logs -n + + # Collect logs of specified node in specified namespace + oceanctl collect logs -n -N + + # Collect logs of all nodes in specified namespace + oceanctl collect logs -n -a + + # Collect logs of specified node in specified namespace + oceanctl collect logs -n -N -a`) +) + +var collectLogsCmd = &cobra.Command{ + Use: "logs", + Short: "Collect logs of one or more nodes in specified namespace in Kubernetes", + Example: collectLogsExample, + RunE: func(cmd *cobra.Command, args []string) error { + return runCollectLogs() + }, +} + +func runCollectLogs() error { + res := resources.NewResourceBuilder(). + AllNodes(config.IsAllNodes). + NodeName(config.NodeName). + NamespaceParam(config.Namespace). + Build() + + return resources.NewLogs(res).Collect() +} diff --git a/cli/cmd/create_cert.go b/cli/cmd/create_cert.go new file mode 100644 index 00000000..f6e5b432 --- /dev/null +++ b/cli/cmd/create_cert.go @@ -0,0 +1,75 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package command + +import ( + "github.com/spf13/cobra" + + "huawei-csi-driver/cli/client" + "huawei-csi-driver/cli/cmd/options" + "huawei-csi-driver/cli/config" + "huawei-csi-driver/cli/helper" + "huawei-csi-driver/cli/resources" +) + +func init() { + options.NewFlagsOptions(createSecretCmd). + WithNameSpace(false). + WithFilename(true). + WithBackend(true). + WithParent(CreateCmd) +} + +var ( + createSecretExample = helper.Examples(` + # Create certificate in default(huawei-csi) namespace based on cert.crt file + oceanctl create cert -f /path/to/cert.crt -b + + # Create certificate in specified namespace based on cert.crt file + oceanctl create cert -f /path/to/cert.crt -n -b + + # Create certificate in specified namespace based on cert.pem file + oceanctl create cert -f /path/to/cert.pem -n -b + `) +) + +var createSecretCmd = &cobra.Command{ + Use: "cert ", + Short: "Create a certificate for specified backend in Kubernetes", + Example: createSecretExample, + RunE: func(cmd *cobra.Command, args []string) error { + return runCreateCert(args) + }, +} + +func runCreateCert(SecretNames []string) error { + res := resources.NewResourceBuilder(). + ResourceNames(string(client.Secret), SecretNames...). + ResourceTypes(string(client.Secret)). + NamespaceParam(config.Namespace). + DefaultNamespace(). + FileName(config.FileName). + BoundBackend(config.Backend). + Build() + + validator := resources.NewValidatorBuilder(res).ValidateNameIsExist().ValidateNameIsSingle().Build() + if err := validator.Validate(); err != nil { + return helper.PrintlnError(err) + } + + return resources.NewCert(res).Create() +} diff --git a/cli/cmd/delete_cert.go b/cli/cmd/delete_cert.go new file mode 100644 index 00000000..9452b1f7 --- /dev/null +++ b/cli/cmd/delete_cert.go @@ -0,0 +1,67 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package command + +import ( + "github.com/spf13/cobra" + + "huawei-csi-driver/cli/cmd/options" + "huawei-csi-driver/cli/config" + "huawei-csi-driver/cli/helper" + "huawei-csi-driver/cli/resources" +) + +func init() { + options.NewFlagsOptions(deleteSecretCmd). + WithNameSpace(false). + WithBackend(true). + WithParent(deleteCmd) +} + +var ( + deleteSecretExample = helper.Examples(` + # Delete certificate of specified backend in default(huawei-csi) namespace + oceanctl delete cert -b + + # Delete certificate of specified backend in specified namespace + oceanctl delete cert -n -b + `) +) + +var deleteSecretCmd = &cobra.Command{ + Use: "cert", + Short: "Delete the certificate of specified backend in Kubernetes", + Example: deleteSecretExample, + RunE: func(cmd *cobra.Command, args []string) error { + return runDeleteCert() + }, +} + +func runDeleteCert() error { + res := resources.NewResourceBuilder(). + NamespaceParam(config.Namespace). + DefaultNamespace(). + BoundBackend(config.Backend). + Build() + + validator := resources.NewValidatorBuilder(res).ValidateBackend().Build() + if err := validator.Validate(); err != nil { + return helper.PrintlnError(err) + } + + return resources.NewCert(res).Delete() +} diff --git a/cli/cmd/get_cert.go b/cli/cmd/get_cert.go new file mode 100644 index 00000000..88823a3a --- /dev/null +++ b/cli/cmd/get_cert.go @@ -0,0 +1,67 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package command + +import ( + "github.com/spf13/cobra" + + "huawei-csi-driver/cli/cmd/options" + "huawei-csi-driver/cli/config" + "huawei-csi-driver/cli/helper" + "huawei-csi-driver/cli/resources" +) + +func init() { + options.NewFlagsOptions(getSecretCmd). + WithNameSpace(false). + WithBackend(true). + WithParent(getCmd) +} + +var ( + getSecretExample = helper.Examples(` + # Get certificate of specified backend in default(huawei-csi) namespace + oceanctl get cert -b + + # Get certificate of specified backend in specified namespace + oceanctl get cert -n -b + `) +) + +var getSecretCmd = &cobra.Command{ + Use: "cert", + Short: "Get the certificate of specified backend in Kubernetes", + Example: getSecretExample, + RunE: func(cmd *cobra.Command, args []string) error { + return runGetCert() + }, +} + +func runGetCert() error { + res := resources.NewResourceBuilder(). + NamespaceParam(config.Namespace). + DefaultNamespace(). + BoundBackend(config.Backend). + Build() + + validator := resources.NewValidatorBuilder(res).ValidateBackend().Build() + if err := validator.Validate(); err != nil { + return helper.PrintlnError(err) + } + + return resources.NewCert(res).Get() +} diff --git a/cli/cmd/options/flag_options.go b/cli/cmd/options/flag_options.go index 37971d89..7307d119 100644 --- a/cli/cmd/options/flag_options.go +++ b/cli/cmd/options/flag_options.go @@ -50,7 +50,7 @@ func (b *FlagsOptions) WithNameSpace(required bool) *FlagsOptions { // WithFilename This function will add a filename flag // If required is true, filename flag must be set func (b *FlagsOptions) WithFilename(required bool) *FlagsOptions { - b.cmd.PersistentFlags().StringVarP(&config.FileName, "filename", "f", "", "path to yaml file") + b.cmd.PersistentFlags().StringVarP(&config.FileName, "filename", "f", "", "path to file") if required { b.markPersistentFlagRequired("filename") } @@ -108,3 +108,26 @@ func (b *FlagsOptions) markPersistentFlagRequired(name string) { log.Errorf("MarkPersistentFlagRequired failed, error: %v", err) } } + +// WithBackend This function will add a backend flag +// If required is true, filename flag must be set +func (b *FlagsOptions) WithBackend(required bool) *FlagsOptions { + b.cmd.PersistentFlags().StringVarP(&config.Backend, "backend", "b", "", "bound to backend") + if required { + b.markPersistentFlagRequired("backend") + } + return b +} + +// WithAllNodes This function will add isAllNodes +func (b *FlagsOptions) WithAllNodes() *FlagsOptions { + b.cmd.PersistentFlags().BoolVarP(&config.IsAllNodes, "all", "a", false, "Collect all nodes messages") + return b +} + +// WithNodeName This function will add nodeName +func (b *FlagsOptions) WithNodeName() *FlagsOptions { + b.cmd.PersistentFlags().StringVarP(&config.NodeName, "nodename", "N", "", "Specify the node "+ + "for which information is to be collected.") + return b +} diff --git a/cli/cmd/update_cert.go b/cli/cmd/update_cert.go new file mode 100644 index 00000000..297c9eb2 --- /dev/null +++ b/cli/cmd/update_cert.go @@ -0,0 +1,72 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package command + +import ( + "github.com/spf13/cobra" + + "huawei-csi-driver/cli/cmd/options" + "huawei-csi-driver/cli/config" + "huawei-csi-driver/cli/helper" + "huawei-csi-driver/cli/resources" +) + +func init() { + options.NewFlagsOptions(updateSecretCmd). + WithNameSpace(false). + WithFilename(true). + WithBackend(true). + WithParent(updateCmd) +} + +var ( + updateSecretExample = helper.Examples(` + # Update certificate of specified backend in default(huawei-csi) namespace based on cert.crt file + oceanctl update cert -b -f /path/to/cert.crt + + # Update certificate of specified backend in specified namespace based on cert.pem file + oceanctl update cert -b -n namespace -f /path/to/cert.pem + + # Update certificate of specified backend in specified namespace based on cert.crt file + oceanctl update cert -b -n namespace -f /path/to/cert.crt + `) +) + +var updateSecretCmd = &cobra.Command{ + Use: "cert", + Short: "Update the certificate of specified backend in Kubernetes", + Example: updateSecretExample, + RunE: func(cmd *cobra.Command, args []string) error { + return runUpdateCert() + }, +} + +func runUpdateCert() error { + res := resources.NewResourceBuilder(). + NamespaceParam(config.Namespace). + DefaultNamespace(). + FileName(config.FileName). + BoundBackend(config.Backend). + Build() + + validator := resources.NewValidatorBuilder(res).ValidateBackend().Build() + if err := validator.Validate(); err != nil { + return helper.PrintlnError(err) + } + + return resources.NewCert(res).Update() +} diff --git a/cli/config/config.go b/cli/config/config.go index 47c94523..2e1d9177 100644 --- a/cli/config/config.go +++ b/cli/config/config.go @@ -21,8 +21,8 @@ import ( ) const ( - //CliVersion oceanctl version - CliVersion = "v4.1.0" + // CliVersion oceanctl version + CliVersion = "v4.2.0" // DefaultMaxClientThreads default max client threads DefaultMaxClientThreads = "30" @@ -58,7 +58,7 @@ var ( // FileType the value of input format flag, set by options.WithInputFileType(). FileType string - //DeleteAll the value of all flag, set by options.DeleteAll(). + // DeleteAll the value of all flag, set by options.DeleteAll(). DeleteAll bool // ChangePassword the value of password flag, set by options.WithPassword(). @@ -70,6 +70,15 @@ var ( // NotValidateName the value of validate flag, set by options.WithNotValidateName NotValidateName bool + // Backend the value of backend flag, set by options.WithBackend(). + Backend string + // Client when the discoverOperating() function executes successfully, this field will be set. Client client.KubernetesClient + + // IsAllNodes the value of allNodes flag, set by options.WithAllNodes(). + IsAllNodes bool + + // NodeName the value of nodeName flag, set by options.WithNodeName() + NodeName string ) diff --git a/cli/helper/async_task.go b/cli/helper/async_task.go new file mode 100644 index 00000000..b52e88c4 --- /dev/null +++ b/cli/helper/async_task.go @@ -0,0 +1,82 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package helper + +import ( + "sync" + + "huawei-csi-driver/utils/log" +) + +// Task Define interfaces required by asynchronous tasks. +type Task interface { + Do() +} + +// TaskHandler Define asynchronous task processing configuration +type TaskHandler struct { + task chan Task + maxHandlerLimit int + wg *sync.WaitGroup +} + +// NewTransmitter initialize a TaskHandler instance +func NewTransmitter(maxHandlerNum, taskCapacity int) *TaskHandler { + return &TaskHandler{ + task: make(chan Task, taskCapacity), + maxHandlerLimit: maxHandlerNum, + wg: &sync.WaitGroup{}, + } +} + +// AddTask add an asynchronous Task +func (t *TaskHandler) AddTask(task Task) { + t.task <- task +} + +// Start Create maxHandlerLimit goroutines to execute the task. +func (t *TaskHandler) Start() { + t.wg.Add(t.maxHandlerLimit) + for i := 0; i < t.maxHandlerLimit; i++ { + go t.handle() + } +} + +func (t *TaskHandler) handle() { + defer func() { + if e := recover(); e != nil { + log.Errorf("an error occurred when executing the async task, error: %v", e) + go t.handle() + } else { + t.wg.Done() + } + }() + for task := range t.task { + task.Do() + } +} + +// End the adding task. +func (t *TaskHandler) End() { + close(t.task) +} + +// Wait for all tasks to complete +func (t *TaskHandler) Wait() { + t.End() + t.wg.Wait() +} diff --git a/cli/helper/command_helper.go b/cli/helper/command_helper.go index 4cff1c77..cb705c56 100644 --- a/cli/helper/command_helper.go +++ b/cli/helper/command_helper.go @@ -17,6 +17,7 @@ package helper import ( + "context" "errors" "fmt" "os" @@ -39,7 +40,11 @@ func ExecWithStdin(cli string, data []byte, args []string) error { go func() { defer stdin.Close() - _, _ = stdin.Write(data) + + _, err = stdin.Write(data) + if err != nil { + log.Warningf("ExecWithStdin Stdin.write failed, err: %v", err) + } }() out, err := cmd.CombinedOutput() @@ -80,14 +85,13 @@ func StartStdInput() (string, string, error) { func getInputString(tips string, isVisible bool) (string, error) { fmt.Print(tips) - var sh string + var cmd *exec.Cmd if isVisible { - sh = "stty erase '^H' -isig -ixon && read -r str && echo $str" + cmd = exec.Command("/bin/bash", "-c", "stty erase '^H' -isig -ixon && read -r str && echo $str") } else { - sh = "stty erase '^H' -isig -ixon && read -sr pwd && echo $pwd" + cmd = exec.Command("/bin/bash", "-c", "stty erase '^H' -isig -ixon && read -sr pwd && echo $pwd") } - cmd := exec.Command("/bin/bash", "-c", sh) cmd.Stdin = os.Stdin bs, err := cmd.CombinedOutput() if err != nil { @@ -121,3 +125,15 @@ func GetSelectedNumber(tips string, maxValue int) (int, error) { fmt.Printf("Input invalid. The valid backend number is [1-%d].\n", maxValue) return GetSelectedNumber(tips, maxValue) } + +// BashExecReturnStdOut used to exec command, and return stdout. +func BashExecReturnStdOut(ctx context.Context, cli string, args []string) ([]byte, error) { + command := fmt.Sprintf("%s %s", cli, strings.Join(args, " ")) + LogInfof(ctx, "bash exec command: %v", command) + cmd := exec.Command("/bin/bash", "-c", command) + stdout, err := cmd.CombinedOutput() + if err != nil { + return []byte{}, errors.New(string(stdout)) + } + return stdout, nil +} diff --git a/cli/helper/goroutine_limit.go b/cli/helper/goroutine_limit.go new file mode 100644 index 00000000..9405bdbc --- /dev/null +++ b/cli/helper/goroutine_limit.go @@ -0,0 +1,95 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package helper + +import ( + "sync" + "sync/atomic" + + "huawei-csi-driver/utils/log" +) + +// GlobalGoroutineLimit Configure the total goroutine limit +type GlobalGoroutineLimit struct { + localNum int32 + maxGoroutineNum int32 + limit int32 + allCond []*sync.Cond +} + +// LocalGoroutineLimit Record local goroutine limit under total limit +type LocalGoroutineLimit struct { + cond *sync.Cond + currentGoroutineNum int32 + limit *int32 +} + +// NewGlobalGoroutineLimit initialize a GlobalGoroutineLimit instance +func NewGlobalGoroutineLimit(maxGoroutineNum int32) *GlobalGoroutineLimit { + res := &GlobalGoroutineLimit{ + allCond: make([]*sync.Cond, maxGoroutineNum), + maxGoroutineNum: maxGoroutineNum, + limit: maxGoroutineNum, + } + return res +} + +// NewLocalGoroutineLimit initialize a LocalGoroutineLimit instance +func NewLocalGoroutineLimit(global *GlobalGoroutineLimit) *LocalGoroutineLimit { + res := &LocalGoroutineLimit{ + limit: &global.limit, + cond: sync.NewCond(&sync.Mutex{}), + } + atomic.AddInt32(&global.localNum, 1) + atomic.StoreInt32(&global.limit, global.maxGoroutineNum/global.localNum) + global.allCond = append(global.allCond, res.cond) + return res +} + +// Do create a Goroutine to Execution function +func (l *LocalGoroutineLimit) Do(f func()) { + l.cond.L.Lock() + for atomic.LoadInt32(&l.currentGoroutineNum) >= atomic.LoadInt32(l.limit) { + l.cond.Wait() + } + l.cond.L.Unlock() + atomic.AddInt32(&l.currentGoroutineNum, 1) + go func() { + defer func() { + if e := recover(); e != nil { + log.Errorf("an error occurred when executing the sub-goroutine, error: %v", e) + } + atomic.AddInt32(&l.currentGoroutineNum, -1) + l.cond.Signal() + }() + f() + }() +} + +// Update LocalGoroutine number to Decrease the limit per LocalGoroutine +func (g *GlobalGoroutineLimit) Update() { + atomic.AddInt32(&g.localNum, -1) + if atomic.LoadInt32(&g.localNum) == 0 { + return + } + atomic.StoreInt32(&g.limit, g.maxGoroutineNum/atomic.LoadInt32(&g.localNum)) + for _, cond := range g.allCond { + if cond != nil { + cond.Signal() + } + } +} diff --git a/cli/helper/once.go b/cli/helper/once.go new file mode 100644 index 00000000..084a3e38 --- /dev/null +++ b/cli/helper/once.go @@ -0,0 +1,45 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package helper + +import ( + "sync" + "sync/atomic" +) + +// Once same as sync.Once, but the Do() method is overridden. +type Once struct { + done uint32 + m sync.Mutex +} + +// Do same as sync.Once.Do(), but you can determine whether the execution is successful by returning error. +func (o *Once) Do(f func() error) { + if atomic.LoadUint32(&o.done) == 0 { + o.doSlow(f) + } +} + +func (o *Once) doSlow(f func() error) { + o.m.Lock() + defer o.m.Unlock() + if o.done == 0 { + if f() == nil { + atomic.StoreUint32(&o.done, 1) + } + } +} diff --git a/cli/helper/utils.go b/cli/helper/utils.go index 74539950..c436309f 100644 --- a/cli/helper/utils.go +++ b/cli/helper/utils.go @@ -17,6 +17,7 @@ package helper import ( + "context" "crypto/sha256" "encoding/json" "fmt" @@ -122,6 +123,12 @@ func PrintBackend[T any](t []T, notFound []string, printFunc func(t []T)) { PrintNotFoundBackend(notFound...) } +// PrintSecret print secret +func PrintSecret[T any](t []T, notFound []string, printFunc func(t []T)) { + printFunc(t) + PrintNotFoundSecret(notFound...) +} + // PrintNotFoundBackend print not found backend func PrintNotFoundBackend(names ...string) { for _, name := range names { @@ -129,11 +136,23 @@ func PrintNotFoundBackend(names ...string) { } } +// PrintNotFoundSecret print not found secret +func PrintNotFoundSecret(names ...string) { + for _, name := range names { + fmt.Printf("Error from server (NotFound): cert \"%s\" not found\n", name) + } +} + // PrintNoResourceBackend print not found backend func PrintNoResourceBackend(namespace string) { fmt.Printf("No backends found in %s namespace\n", namespace) } +// PrintNoResourceCert print no cert found +func PrintNoResourceCert(backend, namespace string) { + fmt.Printf("Error from server (NotFound): no cert found on backend %s in %s namespace\n", backend, namespace) +} + // GetPrintFunc get print function by format type func GetPrintFunc[T any](format string) func(t []T) { switch format { @@ -236,3 +255,46 @@ func GetBackendName(name string) string { } return BuildBackendName(name) } + +// LogInfof write message +func LogInfof(ctx context.Context, format string, args ...interface{}) { + msg := fmt.Sprintf(format, args...) + tag := ctx.Value("tag") + if tag == nil { + log.Infof("%s", msg) + return + } + log.Infof("<<%s>> %s", tag, msg) +} + +// LogWarningf write error log and return the error +func LogWarningf(ctx context.Context, format string, err error) error { + errMessage := fmt.Sprintf(format, err) + tag := ctx.Value("tag") + if tag == nil { + log.Warningf("%s", errMessage) + return err + } + log.Warningf("<<%s>> %s", tag, errMessage) + return err +} + +// ConvertInterface convert interface +func ConvertInterface(i interface{}) interface{} { + switch x := i.(type) { + case map[interface{}]interface{}: + m2 := map[string]interface{}{} + for k, v := range x { + if key, ok := k.(string); ok { + m2[key] = ConvertInterface(v) + } + } + return m2 + case []interface{}: + for i, v := range x { + x[i] = ConvertInterface(v) + } + default: + } + return i +} diff --git a/cli/resources/backend.go b/cli/resources/backend.go index b0d3946d..e5ba15d5 100644 --- a/cli/resources/backend.go +++ b/cli/resources/backend.go @@ -249,13 +249,18 @@ func buildBackendShow(claims []xuanwuV1.StorageBackendClaim, contentList []xuanw func deleteSbcReferenceResources(claim xuanwuV1.StorageBackendClaim) error { _, secretName := k8string.SplitQualifiedName(claim.Spec.SecretMeta) _, configmapName := k8string.SplitQualifiedName(claim.Spec.ConfigMapMeta) - + _, certSecretName := k8string.SplitQualifiedName(claim.Spec.CertSecret) referenceResources := []string{ k8string.JoinQualifiedName(string(client.Secret), secretName), k8string.JoinQualifiedName(string(client.ConfigMap), configmapName), k8string.JoinQualifiedName(string(client.Storagebackendclaim), claim.Name), } + if certSecretName != "" { + referenceResources = append(referenceResources, + k8string.JoinQualifiedName(string(client.Secret), certSecretName)) + } + _, err := config.Client.DeleteResourceByQualifiedNames(referenceResources, claim.Namespace) return err } diff --git a/cli/resources/backend_helper.go b/cli/resources/backend_helper.go index 47c52194..87df97c7 100644 --- a/cli/resources/backend_helper.go +++ b/cli/resources/backend_helper.go @@ -59,10 +59,10 @@ type BackendConfiguration struct { Configured bool `json:"-" yaml:"configured"` Provisioner string `json:"provisioner,omitempty" yaml:"provisioner"` Parameters struct { - Protocol string `json:"protocol,omitempty" yaml:"protocol"` - ParentName string `json:"parentname" yaml:"parentname"` - Portals []string `json:"portals,omitempty" yaml:"portals"` - Alua []map[string][]map[string]interface{} `json:"ALUA,omitempty" yaml:"ALUA"` + Protocol string `json:"protocol,omitempty" yaml:"protocol"` + ParentName string `json:"parentname,omitempty" yaml:"parentname"` + Portals interface{} `json:"portals,omitempty" yaml:"portals"` + Alua map[string]map[string]interface{} `json:"ALUA,omitempty" yaml:"ALUA"` } `json:"parameters,omitempty" yaml:"parameters"` } @@ -187,6 +187,8 @@ func (b *BackendConfiguration) ToConfigMapConfig() (ConfigMapConfig, error) { Backends BackendConfiguration `json:"backends"` }{*b} + config.Backends.Parameters.Portals = helper.ConvertInterface(config.Backends.Parameters.Portals) + output, err := json.MarshalIndent(&config, "", " ") if err != nil { return ConfigMapConfig{}, helper.LogErrorf(" json.MarshalIndent failed: %v", err) @@ -313,7 +315,7 @@ func LoadBackendsFromConfigMap(configmap corev1.ConfigMap) (map[string]*BackendC return result, nil } -//AnalyseBackendExist analyse backend,an error is returned if backends not exist +// AnalyseBackendExist analyse backend,an error is returned if backends not exist func AnalyseBackendExist(jsonStr string) (interface{}, error) { var config map[string]interface{} if err := json.Unmarshal([]byte(jsonStr), &config); err != nil { diff --git a/cli/resources/cert.go b/cli/resources/cert.go new file mode 100644 index 00000000..7d8d3dfc --- /dev/null +++ b/cli/resources/cert.go @@ -0,0 +1,234 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package resources + +import ( + "fmt" + "os" + + corev1 "k8s.io/api/core/v1" + k8string "k8s.io/utils/strings" + + "huawei-csi-driver/cli/client" + "huawei-csi-driver/cli/config" + "huawei-csi-driver/cli/helper" + xuanwuV1 "huawei-csi-driver/client/apis/xuanwu/v1" + "huawei-csi-driver/utils/log" +) + +type Cert struct { + // resource of request + resource *Resource +} + +// NewCert initialize a Cert instance +func NewCert(resource *Resource) *Cert { + return &Cert{resource: resource} +} + +// Get query Cert resources +func (c *Cert) Get() error { + storageBackendClaimClient := client.NewCommonCallHandler[xuanwuV1.StorageBackendClaim](config.Client) + claim, err := storageBackendClaimClient.QueryByName(c.resource.namespace, c.resource.backend) + if err != nil { + return err + } + + if claim.Name == "" { + helper.PrintNotFoundBackend(c.resource.backend) + return nil + } + + _, certSecretName := k8string.SplitQualifiedName(claim.Spec.CertSecret) + + if certSecretName == "" { + helper.PrintNoResourceCert(claim.Name, claim.Namespace) + return nil + } + + secretClient := client.NewCommonCallHandler[corev1.Secret](config.Client) + secret, err := secretClient.QueryByName(c.resource.namespace, certSecretName) + if err != nil { + return err + } + + shows := fetchCertShows([]corev1.Secret{secret}, c.resource.namespace, claim.Name) + helper.PrintSecret(shows, []string{}, helper.PrintWithTable[CertShow]) + return nil +} + +func (c *Cert) Delete() error { + storageBackendClaimClient := client.NewCommonCallHandler[xuanwuV1.StorageBackendClaim](config.Client) + oldClaim, err := storageBackendClaimClient.QueryByName(c.resource.namespace, c.resource.backend) + if err != nil { + return err + } + + if oldClaim.Name == "" { + helper.PrintNotFoundBackend(c.resource.backend) + return nil + } + + _, certSecretName := k8string.SplitQualifiedName(oldClaim.Spec.CertSecret) + + if certSecretName == "" { + helper.PrintNoResourceCert(oldClaim.Name, oldClaim.Namespace) + return nil + } + + newClaim := oldClaim.DeepCopy() + newClaim.Spec.UseCert = false + newClaim.Spec.CertSecret = "" + if err = storageBackendClaimClient.Update(*newClaim); err != nil { + return err + } + + secretClient := client.NewCommonCallHandler[corev1.Secret](config.Client) + secret, err := secretClient.QueryByName(c.resource.namespace, certSecretName) + if err != nil { + return err + } + + if err := deleteSecretResources(secret); err != nil { + return helper.LogErrorf("delete cert reference resource failed, error: %v", err) + } + + helper.PrintOperateResult("cert", "deleted", secret.Name) + return nil +} + +// Update update Cert +func (c *Cert) Update() error { + storageBackendClaimClient := client.NewCommonCallHandler[xuanwuV1.StorageBackendClaim](config.Client) + claim, err := storageBackendClaimClient.QueryByName(c.resource.namespace, c.resource.backend) + if err != nil { + return err + } + + if claim.Name == "" { + helper.PrintNotFoundBackend(c.resource.backend) + return nil + } + + _, certSecretName := k8string.SplitQualifiedName(claim.Spec.CertSecret) + + if certSecretName == "" { + helper.PrintNoResourceCert(claim.Name, claim.Namespace) + return nil + } + + certConfig, err := c.LoadCertFile() + if err != nil { + return helper.LogErrorf("load cert failed: error: %v", err) + } + + secretClient := client.NewCommonCallHandler[corev1.Secret](config.Client) + oldSecret, err := secretClient.QueryByName(c.resource.namespace, certSecretName) + if err != nil { + return err + } + + newSecret := oldSecret.DeepCopy() + newSecret.Data = map[string][]byte{ + "tls.crt": certConfig.Cert, + } + + if err = secretClient.Update(*newSecret); err != nil { + return helper.LogErrorf("apply cert failed, error: %v", err) + } + helper.PrintOperateResult("cert", "updated", newSecret.Name) + return nil +} + +// Create create Cert +func (c *Cert) Create() error { + certConfig, err := c.LoadCertFile() + if err != nil { + return helper.LogErrorf("load cert failed: error: %v", err) + } + certConfig.Name = c.resource.names[0] + + storageBackendClaimClient := client.NewCommonCallHandler[xuanwuV1.StorageBackendClaim](config.Client) + oldClaim, err := storageBackendClaimClient.QueryByName(c.resource.namespace, c.resource.backend) + if err != nil { + return err + } + + if oldClaim.Name == "" { + helper.PrintNotFoundBackend(c.resource.backend) + return nil + } + + if oldClaim.Spec.CertSecret != "" { + return fmt.Errorf("a cert already exists on the backend [%s]", oldClaim.Name) + } + + // create secret resource + secretClient := client.NewCommonCallHandler[corev1.Secret](config.Client) + if err = secretClient.Create(certConfig.ToCertSecret()); err != nil { + return err + } + + newClaim := oldClaim.DeepCopy() + newClaim.Spec.UseCert = true + newClaim.Spec.CertSecret = k8string.JoinQualifiedName(newClaim.Namespace, certConfig.Name) + + if err = storageBackendClaimClient.Update(*newClaim); err != nil { + if err := secretClient.DeleteByNames(newClaim.Namespace, certConfig.Name); err != nil { + log.Errorf("delete new created cert failed, error: %v", err) + } + return err + } + + // print create result + helper.PrintOperateResult("cert", "created", certConfig.Name) + return nil +} + +func (c *Cert) LoadCertFile() (*CertConfig, error) { + certData, err := os.ReadFile(c.resource.fileName) + if err != nil { + return nil, err + } + + return c.LoadCertsFromDate(certData) +} + +// LoadCertsFromDate load cert from bytes +func (c *Cert) LoadCertsFromDate(Data []byte) (*CertConfig, error) { + return &CertConfig{ + Namespace: c.resource.namespace, + Cert: Data, + }, nil +} + +func deleteSecretResources(secret corev1.Secret) error { + referenceResources := []string{ + k8string.JoinQualifiedName(string(client.Secret), secret.Name), + } + + _, err := config.Client.DeleteResourceByQualifiedNames(referenceResources, secret.Namespace) + return err +} + +func fetchCertShows(secrets []corev1.Secret, namespace, backend string) []CertShow { + result := make([]CertShow, 0) + for _, secret := range secrets { + result = append(result, CertShow{Name: secret.Name, Namespace: namespace, BoundBackend: backend}) + } + return result +} diff --git a/cli/resources/cert_helper.go b/cli/resources/cert_helper.go new file mode 100644 index 00000000..5eb81628 --- /dev/null +++ b/cli/resources/cert_helper.go @@ -0,0 +1,54 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package resources + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// CertConfig used to create a cert object +type CertConfig struct { + Name string + Namespace string + Cert []byte +} + +// CertShow the content echoed by executing the oceanctl get cert +type CertShow struct { + Namespace string `show:"NAMESPACE"` + Name string `show:"NAME"` + BoundBackend string `show:"BOUNDBACKEND"` +} + +// ToCertSecret convert CertSecretConfig to Secret resource +func (c *CertConfig) ToCertSecret() corev1.Secret { + return corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + APIVersion: ApiVersion, + Kind: KindSecret, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: c.Name, + Namespace: c.Namespace, + }, + Data: map[string][]byte{ + "tls.crt": c.Cert, + }, + Type: "Opaque", + } +} diff --git a/cli/resources/file_logs_factory.go b/cli/resources/file_logs_factory.go new file mode 100644 index 00000000..721853ae --- /dev/null +++ b/cli/resources/file_logs_factory.go @@ -0,0 +1,184 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package resources + +import ( + "context" + "errors" + "fmt" + "path" + "strconv" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + + "huawei-csi-driver/cli/config" + "huawei-csi-driver/cli/helper" +) + +type FileLogsCollect interface { + GetFileLogs(namespace, podName string, container *corev1.Container) (err error) + GetHostInformation(namespace, containerName, nodeName, podName string) error + CopyToLocal(namespace, nodeName, podName, containerName string) error +} + +type BaseFileLogsCollect struct{} + +type FileLogsCollector struct { + BaseFileLogsCollect + fileLogPath string +} + +type PodType byte + +const ( + CSI PodType = 0 + CSM PodType = 1 + Xuanwu PodType = 2 + UnKnow PodType = 3 + + temporaryDirectoryNamePrefix = "huawei" + + scriptPath = "/tmp/collect.sh" + + hostInformationFilePath = "/tmp/host_information" + + fileLogsDirectory = "file" +) + +var ( + fileLogCollectSet = map[PodType]FileLogsCollect{} + + containerLogsPath = fmt.Sprintf("/tmp/%s-%s", temporaryDirectoryNamePrefix, + strconv.Itoa(int(time.Now().UnixNano()))) + localLogsPrefixPath = fmt.Sprintf("/tmp/%s-%s", temporaryDirectoryNamePrefix, + strconv.Itoa(int(time.Now().UnixNano()))) +) + +func init() { + RegisterCollector(CSI, &FileLogsCollector{}) + RegisterCollector(CSM, &FileLogsCollector{}) + RegisterCollector(Xuanwu, &FileLogsCollector{}) +} + +func (b *BaseFileLogsCollect) getContainerFileLogs(namespace, podName, containerName string, + fileLogsPaths ...string) error { + ctx := context.WithValue(context.Background(), "tag", podName) + cmd := fmt.Sprintf("%s %s", "mkdir", containerLogsPath) + _, err := config.Client.ExecCmdInSpecifiedContainer(ctx, namespace, containerName, cmd, podName) + if err != nil { + return helper.LogWarningf(ctx, "create container file logs path failed, error: %v", err) + } + + cmd = fmt.Sprintf("cp -a %s %s 2>/dev/null || :", strings.Join(fileLogsPaths, " "), containerLogsPath) + _, err = config.Client.ExecCmdInSpecifiedContainer(ctx, namespace, containerName, cmd, podName) + if err != nil { + return helper.LogWarningf(ctx, "get container file logs failed, error: %v", err) + } + + return nil +} + +func (b *BaseFileLogsCollect) deleteFileLogsInContainer(namespace, podName, containerName string, + filePaths ...string) error { + ctx := context.WithValue(context.Background(), "tag", podName) + cmd := "rm -rf " + strings.Join(filePaths, " ") + _, err := config.Client.ExecCmdInSpecifiedContainer(ctx, namespace, containerName, cmd, podName) + if err != nil { + return helper.LogWarningf(ctx, "delete file logs in container failed, error: %v", err) + } + + return nil +} + +func (b *BaseFileLogsCollect) compressLogsInContainer(namespace, podName, containerName string) error { + ctx := context.WithValue(context.Background(), "tag", podName) + compressedLogsName := fmt.Sprintf("%s-%s.tar", namespace, podName) + cmd := fmt.Sprintf("%s %s -C %s .", "tar -czvf", path.Join(containerLogsPath, compressedLogsName), + containerLogsPath) + _, err := config.Client.ExecCmdInSpecifiedContainer(ctx, namespace, containerName, cmd, podName) + if err != nil { + return helper.LogWarningf(ctx, "compress logs in container failed, error: %v", err) + } + + return nil +} + +// CopyToLocal copy the compressed log file to the local host. +func (b *BaseFileLogsCollect) CopyToLocal(namespace, nodeName, podName, containerName string) error { + defer b.deleteFileLogsInContainer(namespace, podName, containerName, containerLogsPath) + ctx := context.WithValue(context.Background(), "tag", podName) + compressedLogsName := fmt.Sprintf("%s-%s.tar", namespace, podName) + _, err := config.Client.CopyContainerFileToLocal(ctx, namespace, containerName, + path.Join(containerLogsPath[1:], compressedLogsName), + path.Join(localLogsPrefixPath, nodeName, fileLogsDirectory, compressedLogsName), podName) + if err != nil { + return helper.LogWarningf(ctx, "copy container compressed logs to local failed, error: %v", err) + } + + return nil +} + +// GetHostInformation get the host information of a specified node. +func (b *BaseFileLogsCollect) GetHostInformation(namespace, containerName, nodeName, podName string) error { + ctx := context.WithValue(context.Background(), "tag", podName) + _, err := config.Client.ExecCmdInSpecifiedContainer(ctx, namespace, containerName, scriptPath, podName) + if err != nil { + return helper.LogWarningf(ctx, "get node host information failed, error: %v", err) + } + defer b.deleteFileLogsInContainer(namespace, podName, containerName, hostInformationFilePath) + + _, fileName := path.Split(hostInformationFilePath) + _, err = config.Client.CopyContainerFileToLocal(ctx, namespace, containerName, + hostInformationFilePath, + path.Join(localLogsPrefixPath, nodeName, fileName), podName) + if err != nil { + return helper.LogWarningf(ctx, "copy node host information to local failed, error: %v", err) + } + return nil +} + +// GetFileLogs get the file log of a specified node. +func (c *FileLogsCollector) GetFileLogs(namespace, podName string, container *corev1.Container) (err error) { + c.fileLogPath, err = getContainerFileLogPaths(container) + if err != nil { + return + } + + if err = c.getContainerFileLogs(namespace, podName, container.Name, c.fileLogPath); err != nil { + return + } + + err = c.compressLogsInContainer(namespace, podName, container.Name) + + return +} + +// RegisterCollector used to register a collector into the collectorSet +func RegisterCollector(name PodType, collector FileLogsCollect) { + fileLogCollectSet[name] = collector +} + +// LoadSupportedCollector used to load supported collector. Return a collector of type FileLogsCollect and nil error +// if a client with the specified testName exists. If not exists, return an error with not supported. +func LoadSupportedCollector(name PodType) (FileLogsCollect, error) { + if client, ok := fileLogCollectSet[name]; ok { + return client, nil + } + return nil, errors.New("not valid collector") +} diff --git a/cli/resources/logs.go b/cli/resources/logs.go new file mode 100644 index 00000000..37bf7c3a --- /dev/null +++ b/cli/resources/logs.go @@ -0,0 +1,311 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package resources defines the command execution logic. +package resources + +import ( + "archive/zip" + "context" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path" + "path/filepath" + "strings" + "sync" + "time" + + coreV1 "k8s.io/api/core/v1" + + "huawei-csi-driver/cli/client" + "huawei-csi-driver/cli/config" + "huawei-csi-driver/cli/helper" + "huawei-csi-driver/utils/log" +) + +const ( + localCompressedLogsPrefixPath = "/tmp" + localOceanctlLogPath = "/var/log/huawei/oceanctl-log" + + logsSeparator = "-" + logsSeparatorLength = 35 + + maxTransmissionTaskWait = 100 + maxTransmissionsNum = 10 + + maxCollectGoroutineNum = 1000 +) + +var ( + checkNamespaceExistFun func(ctx context.Context, ns string, node string, objectName string) (bool, error) + checkNodeExistFun func(ctx context.Context, ns string, node string, objectName string) (bool, error) + getPodListFun func(ctx context.Context, ns string, node string, + objectName ...string) (coreV1.PodList, error) + + logsSeparatorStr = strings.Repeat(logsSeparator, logsSeparatorLength) +) + +// Logs Records specified conditions and provides a method for collecting logs. +type Logs struct { + resource *Resource + + nodePodList map[string][]coreV1.Pod +} + +// NewLogs initialize a Logs instance +func NewLogs(resource *Resource) *Logs { + return &Logs{ + resource: resource, + nodePodList: make(map[string][]coreV1.Pod), + } +} + +func initFun() { + checkNamespaceExistFun = client.NewCommonCallHandler[coreV1.Namespace](config.Client).CheckObjectExist + checkNodeExistFun = client.NewCommonCallHandler[coreV1.Node](config.Client).CheckObjectExist + getPodListFun = client.NewCommonCallHandler[coreV1.PodList](config.Client).GetObject +} + +func (lg *Logs) initLogsType() error { + if !lg.resource.isAllNodes && lg.resource.nodeName != "" { + _, err := checkNodeExistFun(context.Background(), client.IgnoreNamespace, client.IgnoreNode, + lg.resource.nodeName) + if err != nil { + return helper.LogErrorf("check node exist failed, error: %v", err) + } + return nil + } + lg.resource.nodeName = client.IgnoreNode + return nil +} + +func (lg *Logs) initNodePodList() error { + podList, err := getPodListFun(context.Background(), lg.resource.namespace, lg.resource.nodeName) + if err != nil { + return helper.LogErrorf("get pod list failed, error: %v", err) + } + + if lg.resource.nodeName != client.IgnoreNode && len(podList.Items) != 0 { + lg.nodePodList[lg.resource.nodeName] = podList.Items + } else if lg.resource.nodeName == client.IgnoreNode { + for _, pod := range podList.Items { + if _, ok := lg.nodePodList[pod.Spec.NodeName]; !ok { + lg.nodePodList[pod.Spec.NodeName] = []coreV1.Pod{pod} + continue + } + lg.nodePodList[pod.Spec.NodeName] = append(lg.nodePodList[pod.Spec.NodeName], pod) + } + } + + return nil +} + +func (lg *Logs) checkNamespaceExist() error { + _, err := checkNamespaceExistFun(context.Background(), client.IgnoreNamespace, client.IgnoreNode, + lg.resource.namespace) + if err != nil { + return helper.LogErrorf("check namespace exist failed, error: %v", err) + } + return nil +} + +func (lg *Logs) initialize() error { + if lg.resource == nil { + return helper.LogErrorf("collect failed, error: %v", errors.New("resource is nil")) + } + initFun() + err := lg.checkNamespaceExist() + if err != nil { + return err + } + + err = lg.initLogsType() + if err != nil { + return err + } + + err = lg.initNodePodList() + return err +} + +// Collect logs based on specified conditions +func (lg *Logs) Collect() error { + log.Infof("%s Start Recording And Collecting Log Information: Namespace: %s Node: %s %s", + logsSeparatorStr, lg.resource.namespace, lg.getNodeName(), logsSeparatorStr) + defer log.Infof("%s Stop Recording And Collecting Log Information: Namespace: %s Node: %s %s", + logsSeparatorStr, lg.resource.namespace, lg.getNodeName(), logsSeparatorStr) + err := lg.initialize() + if err != nil { + return err + } + + err = createNodeLogsPath(lg.nodePodList) + if err != nil { + return err + } + defer deleteLocalLogsFile() + + wg := sync.WaitGroup{} + wg.Add(len(lg.nodePodList)) + display := NewDisplay() + globalGoroutineLimit := helper.NewGlobalGoroutineLimit(maxCollectGoroutineNum) + transmitter := helper.NewTransmitter(maxTransmissionsNum, maxTransmissionTaskWait) + transmitter.Start() + for nodeName, pods := range lg.nodePodList { + localGoroutineLimit := helper.NewLocalGoroutineLimit(globalGoroutineLimit) + nodeLogsCollector := NewNodeLogsCollector(pods, localGoroutineLimit, transmitter, display) + display.Add(fmt.Sprintf("node[%s] ", nodeName), nodeLogsCollector.completionStatus.Display) + go func() { + nodeLogsCollector.Collect() + globalGoroutineLimit.Update() + wg.Done() + }() + } + ctx, cancel := context.WithCancel(context.Background()) + go display.Show(ctx) + wg.Wait() + cancel() + transmitter.Wait() + + err = compressLocalLogs(lg.nodePodList, lg.getLocalCompressedLogsFileName()) + return err +} + +func createNodeLogsPath(nodeList map[string][]coreV1.Pod) error { + for node := range nodeList { + err := os.MkdirAll(path.Join(localLogsPrefixPath, node, fileLogsDirectory), os.ModePerm) + if err != nil { + return helper.LogErrorf("create node logs directory failed, error: %v", err) + } + + err = os.MkdirAll(path.Join(localLogsPrefixPath, node, consoleLogDirectory), os.ModePerm) + if err != nil { + return helper.LogErrorf("create node logs directory failed, error: %v", err) + } + } + return nil +} + +func (lg *Logs) getNodeName() string { + if lg.resource.nodeName == client.IgnoreNode { + return "all" + } + return lg.resource.nodeName +} + +func (lg *Logs) getLocalCompressedLogsFileName() string { + nowTime := time.Now().Format("2006-01-02 15:04:05") + return fmt.Sprintf("%s-%s-%s.zip", lg.resource.namespace, + strings.Join(strings.Split(nowTime, " "), "-"), + lg.getNodeName()) +} + +func deleteLocalLogsFile() error { + err := os.RemoveAll(localLogsPrefixPath) + if err != nil { + return helper.LogErrorf("delete local logs file failed,error: %v", err) + } + return nil +} + +func compressLocalLogs(nodeList map[string][]coreV1.Pod, fileName string) error { + nodeLogsDirList := make([]string, 0) + for node := range nodeList { + nodeLogsDirList = append(nodeLogsDirList, path.Join(localLogsPrefixPath, node)) + } + nodeLogsDirList = append(nodeLogsDirList, localOceanctlLogPath) + + return zipMultiFiles(path.Join(localCompressedLogsPrefixPath, fileName), nodeLogsDirList...) +} + +func zipMultiFiles(zipPath string, filePaths ...string) error { + // Create zip file and it's parent dir. + if err := os.MkdirAll(filepath.Dir(zipPath), os.ModePerm); err != nil { + return helper.LogErrorf("create compressed logs directory failed, error: %v", err) + } + archive, err := os.Create(zipPath) + if err != nil { + return helper.LogErrorf("create compressed logs file failed, error: %v", err) + } + defer archive.Close() + + // New zip writer. + zipWriter := zip.NewWriter(archive) + defer zipWriter.Close() + + // Traverse the file or directory. + for _, rootPath := range filePaths { + // Remove the trailing path separator if path is a directory. + rootPath = strings.TrimSuffix(rootPath, string(os.PathSeparator)) + + // Visit all the files or directories in the tree. + err = filepath.Walk(rootPath, walkFunc(rootPath, zipWriter)) + if err != nil { + return err + } + } + return nil +} + +func walkFunc(rootPath string, zipWriter *zip.Writer) filepath.WalkFunc { + return func(path string, info fs.FileInfo, err error) error { + // If a file is a symbolic link it will be skipped. + if info.Mode()&os.ModeSymlink != 0 { + return nil + } + + // Create a local file header. + header, err := zip.FileInfoHeader(info) + if err != nil { + return helper.LogErrorf("get compressed file info header failed, error: %v", err) + } + + // Set compression method. + header.Method = zip.Deflate + + // Set relative path of a file as the header name. + header.Name, err = filepath.Rel(filepath.Dir(rootPath), path) + if err != nil { + return helper.LogErrorf("get relative directory failed, error: %v", err) + } + if info.IsDir() { + header.Name += string(os.PathSeparator) + } + + // Create writer for the file header and save content of the file. + headerWriter, err := zipWriter.CreateHeader(header) + if err != nil { + return helper.LogErrorf("create writer for writing file to compressed file failed, error: %v", err) + } + if info.IsDir() { + return nil + } + f, err := os.Open(path) + if err != nil { + return helper.LogErrorf("open log file failed, error:%v", err) + } + defer f.Close() + _, err = io.Copy(headerWriter, f) + if err != nil { + return helper.LogErrorf("write file to compress file failed, error: %v", err) + } + + return nil + } +} diff --git a/cli/resources/logs_helper.go b/cli/resources/logs_helper.go new file mode 100644 index 00000000..1fe6e486 --- /dev/null +++ b/cli/resources/logs_helper.go @@ -0,0 +1,354 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package resources + +import ( + "context" + "errors" + "fmt" + "os" + "path" + "strings" + "sync" + "sync/atomic" + "time" + + coreV1 "k8s.io/api/core/v1" + + "huawei-csi-driver/cli/config" + "huawei-csi-driver/cli/helper" +) + +const ( + csiFlagContainer = "huawei-csi-driver" + csmFlagContainer = "cmi-controller" + + progressBarLength = 100 + + consoleLogDirectory = "console" +) + +var ( + identifyPodTypesFunc = make(map[PodType]func(pod *coreV1.Pod) bool) + xuanwuPodPrefixNameList = []string{"xuanwu-backup-mngt", "xuanwu-backup-service", "xuanwu-base-mngt", + "xuanwu-disaster-service", "xuanwu-disaster-mngt", "xuanwu-metadata-service", "xuanwu-volume-service"} +) + +// NodeLogCollector Collecting Node Logs +type NodeLogCollector struct { + podList []coreV1.Pod + wg sync.WaitGroup + fileLogsOnce []helper.Once + hostInformationOnce helper.Once + + // collect completion status + completionStatus Status + + local *helper.LocalGoroutineLimit + transmitter *helper.TaskHandler + + display *Display +} + +// Status Collection status display +type Status struct { + completed int32 + total int +} + +// Display Collects the progress display function of all nodes to display the progress. +type Display struct { + displayFunc []func() + prefixDesc []string + totalLines int +} + +// TransmitTask Configuring a Pod Log File Transfer Task +type TransmitTask struct { + namespace string + nodeName string + podName string + containerName string + FileLogsCollect +} + +func init() { + RegisterIdentifyPodTypeFunc(CSI, checkCSIPod) + RegisterIdentifyPodTypeFunc(CSM, checkCSMPod) + RegisterIdentifyPodTypeFunc(Xuanwu, checkXuanwuPod) +} + +// Do copy the compressed log file to the local host. +func (t *TransmitTask) Do() { + _ = t.CopyToLocal(t.namespace, t.nodeName, t.podName, t.containerName) +} + +func newTransmitTask(namespace, nodeName, podName, containerName string, collect FileLogsCollect) *TransmitTask { + return &TransmitTask{ + namespace: namespace, + nodeName: nodeName, + podName: podName, + containerName: containerName, + FileLogsCollect: collect, + } +} + +// NewDisplay initialize a Display instance +func NewDisplay() *Display { + return &Display{ + displayFunc: make([]func(), 0), + prefixDesc: make([]string, 0), + } +} + +// Add the progress function to be displayed. +func (d *Display) Add(prefixDesc string, f func()) { + d.displayFunc = append(d.displayFunc, f) + d.prefixDesc = append(d.prefixDesc, prefixDesc) + d.totalLines++ +} + +func (d *Display) show() { + for idx, display := range d.displayFunc { + fmt.Printf("%s", d.prefixDesc[idx]) + display() + } +} + +func (d *Display) hideCursor() { + fmt.Printf("\033[?25l") +} + +func (d *Display) displayCursor() { + fmt.Printf("\033[?25h") +} + +func (d *Display) resetCursor() { + fmt.Printf("\033[%dA\r", d.totalLines) +} + +// Show all progresses +func (d *Display) Show(ctx context.Context) { + d.hideCursor() + defer d.displayCursor() + d.show() + for { + select { + case <-ctx.Done(): + d.resetCursor() + d.show() + return + default: + d.resetCursor() + d.show() + time.Sleep(10 * time.Millisecond) + } + } +} + +func (n *Status) getPercent() int { + return int(atomic.LoadInt32(&n.completed)) * 100 / n.total +} + +func (n *Status) getCompletedStr() string { + return fmt.Sprintf("%c[1;40;32m%s%c[0m", 0x1B, strings.Repeat("+", n.getPercent()*progressBarLength/100), 0x1B) +} + +func (n *Status) getRemainedStr() string { + return fmt.Sprintf("%s", strings.Repeat("-", progressBarLength-n.getPercent()*progressBarLength/100)) +} + +// Display displays the pod log collection status of the current node. +func (n *Status) Display() { + fmt.Printf("Collection Progress：[%s%s] %d/%d Pods\n", + n.getCompletedStr(), n.getRemainedStr(), atomic.LoadInt32(&n.completed), n.total) +} + +func checkCSIPod(pod *coreV1.Pod) bool { + for _, container := range pod.Spec.Containers { + if container.Name == csiFlagContainer { + return true + } + } + return false +} + +func checkCSMPod(pod *coreV1.Pod) bool { + for _, container := range pod.Spec.Containers { + if container.Name == csmFlagContainer { + return true + } + } + return false +} + +func checkXuanwuPod(pod *coreV1.Pod) bool { + for _, prefix := range xuanwuPodPrefixNameList { + if strings.HasPrefix(pod.Name, prefix) { + return true + } + } + return false +} + +// RegisterIdentifyPodTypeFunc used to register a func into the identifyPodTypeFuncSet +func RegisterIdentifyPodTypeFunc(name PodType, f func(pod *coreV1.Pod) bool) { + identifyPodTypesFunc[name] = f +} + +// NewNodeLogsCollector initialize a NodeLogsCollector instance +func NewNodeLogsCollector(podList []coreV1.Pod, local *helper.LocalGoroutineLimit, + transmitter *helper.TaskHandler, display *Display) *NodeLogCollector { + return &NodeLogCollector{ + podList: podList, + completionStatus: Status{ + total: len(podList), + }, + local: local, + transmitter: transmitter, + fileLogsOnce: make([]helper.Once, len(podList)), + display: display, + } +} + +// Collect collects container logs of specified conditions on the node. +func (n *NodeLogCollector) Collect() { + n.wg.Add(n.completionStatus.total) + for idx := range n.podList { + pod := &n.podList[idx] + localIdx := idx + n.local.Do(func() { + n.collectPodLogs(pod, localIdx) + atomic.AddInt32(&n.completionStatus.completed, 1) + n.wg.Done() + }) + } + n.wg.Wait() +} + +func (n *NodeLogCollector) collectPodLogs(pod *coreV1.Pod, onceIdx int) { + ctx := context.WithValue(context.Background(), "tag", pod.Name) + var isRunning = pod.Status.Phase == coreV1.PodRunning + fileLogCollector, err := LoadSupportedCollector(getPodType(pod)) + if err != nil { + _ = helper.LogWarningf(ctx, "unknown pod types, error: %v", err) + return + } + + if !isRunning { + logPath, err := getPodFileLogPaths(pod) + if err != nil { + return + } + + msg := fmt.Sprintf("Failed to collect [%s] file logs on node [%s], please collect logs manually,"+ + " file logs path is [%s]", pod.Name, pod.Spec.NodeName, logPath) + n.display.Add("", func() { + fmt.Printf("%c[1;40;31m%s%c[0m\n", 0x1B, msg, 0x1B) + }) + _ = helper.LogWarningf(ctx, "error: %v", errors.New(msg)) + } + + for idx := range pod.Spec.Containers { + container := &pod.Spec.Containers[idx] + getConsoleLogs(ctx, pod.Namespace, container.Name, pod.Name, pod.Spec.NodeName, false) + getConsoleLogs(ctx, pod.Namespace, container.Name, pod.Name, pod.Spec.NodeName, true) + if isRunning { + n.fileLogsOnce[onceIdx].Do(func() error { + err = fileLogCollector.GetFileLogs(pod.Namespace, pod.Name, container) + if err == nil { + n.transmitter.AddTask(newTransmitTask(pod.Namespace, pod.Spec.NodeName, pod.Name, container.Name, + fileLogCollector)) + } + return err + }) + + n.hostInformationOnce.Do(func() error { + return fileLogCollector.GetHostInformation(pod.Namespace, container.Name, pod.Spec.NodeName, pod.Name) + }) + } + } +} + +func getConsoleLogs(ctx context.Context, namespace, containerName, podName, nodeName string, isHistoryLogs bool) { + logs, err := config.Client.GetConsoleLogs(ctx, namespace, containerName, isHistoryLogs, podName) + if err != nil { + _ = helper.LogWarningf(ctx, "get container console logs failed, error: %v", err) + } else { + _ = saveConsoleLog(logs, namespace, podName, containerName, nodeName, isHistoryLogs) + } +} + +func getPodType(pod *coreV1.Pod) PodType { + for podType, f := range identifyPodTypesFunc { + if f(pod) { + return podType + } + } + return UnKnow +} + +func saveConsoleLog(logs []byte, namespace, podName, containerName, nodeName string, isHistoryLogs bool) error { + ctx := context.WithValue(context.Background(), "tag", podName) + fileName := fmt.Sprintf("%s-%s-%s.log", namespace, podName, containerName) + if isHistoryLogs { + fileName = "last-" + fileName + } + file, err := os.Create(path.Join(localLogsPrefixPath, nodeName, consoleLogDirectory, fileName)) + if err != nil { + return helper.LogWarningf(ctx, "create container console log file failed, error: %v", err) + } + defer file.Close() + + _, err = file.Write(logs) + if err != nil { + return helper.LogWarningf(ctx, "write container console log to file failed, error: %v", err) + } + + return nil +} + +func getPodFileLogPaths(pod *coreV1.Pod) (string, error) { + for idx := range pod.Spec.Containers { + logPath, err := getContainerFileLogPaths(&pod.Spec.Containers[idx]) + if err == nil { + return logPath, err + } + } + return "", helper.LogWarningf(context.Background(), "get pod file log paths failed, error: %v", + errors.New("not found a available file log directory")) +} + +func getContainerFileLogPaths(container *coreV1.Container) (string, error) { + if container.Args == nil { + return "", helper.LogWarningf(context.Background(), "get container file log paths failed, error: %v", + errors.New("args is nil")) + } + for _, arg := range container.Args { + if strings.HasPrefix(arg, "--log-file-dir=") { + logPath := strings.Split(arg, "=") + if len(logPath) != 2 { + return "", helper.LogWarningf(context.Background(), "get container file log paths failed, error: %v", + errors.New("log-file-dir is not set correctly")) + } + return logPath[1], nil + } + } + + return "", helper.LogWarningf(context.Background(), "get container file log paths failed, error: %v", + errors.New("log-file-dir is not set")) +} diff --git a/cli/resources/resource.go b/cli/resources/resource.go index ee72dd47..634c7097 100644 --- a/cli/resources/resource.go +++ b/cli/resources/resource.go @@ -50,6 +50,11 @@ type ResourceBuilder struct { output string notValidateName bool + + backend string + + isAllNodes bool + nodeName string } // NewResourceBuilder initialize a ResourceBuilder instance @@ -154,3 +159,21 @@ func (b *ResourceBuilder) FileType(fileType string) *ResourceBuilder { b.fileType = fileType return b } + +// BoundBackend instructs the builder to request backend name. +func (b *ResourceBuilder) BoundBackend(backend string) *ResourceBuilder { + b.backend = backend + return b +} + +// AllNodes instructs the builder to request isAllNodes options. +func (b *ResourceBuilder) AllNodes(isAllNodes bool) *ResourceBuilder { + b.isAllNodes = isAllNodes + return b +} + +// NodeName instructs the builder to request node name. +func (b *ResourceBuilder) NodeName(nodeName string) *ResourceBuilder { + b.nodeName = nodeName + return b +} diff --git a/cli/resources/validate.go b/cli/resources/validate.go index 093699ee..99a41dcc 100644 --- a/cli/resources/validate.go +++ b/cli/resources/validate.go @@ -17,6 +17,7 @@ package resources import ( + "errors" "fmt" "strings" @@ -103,3 +104,11 @@ func (b *ValidatorBuilder) ValidateOutputFormat() *ValidatorBuilder { return b } + +// ValidateBackend used to validate backend +func (b *ValidatorBuilder) ValidateBackend() *ValidatorBuilder { + if b.resource.backend == "" { + b.errs = append(b.errs, errors.New("backend name must be provided")) + } + return b +} diff --git a/client/apis/xuanwu/v1/register.go b/client/apis/xuanwu/v1/register.go index 25581a2b..281c41da 100644 --- a/client/apis/xuanwu/v1/register.go +++ b/client/apis/xuanwu/v1/register.go @@ -51,6 +51,8 @@ func addKnownTypes(scheme *runtime.Scheme) error { &StorageBackendClaimList{}, &StorageBackendContent{}, &StorageBackendContentList{}, + &ResourceTopology{}, + &ResourceTopologyList{}, ) v1.AddToGroupVersion(scheme, SchemeGroupVersion) return nil diff --git a/client/apis/xuanwu/v1/resourcetopology.go b/client/apis/xuanwu/v1/resourcetopology.go new file mode 100644 index 00000000..a86832ba --- /dev/null +++ b/client/apis/xuanwu/v1/resourcetopology.go @@ -0,0 +1,92 @@ +/* +Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" + +type ResourceTopologyStatusPhase string + +const ( + ResourceTopologyStatusNormal ResourceTopologyStatusPhase = "Normal" + ResourceTopologyStatusPending ResourceTopologyStatusPhase = "Pending" + ResourceTopologyStatusDeleting ResourceTopologyStatusPhase = "Deleting" +) + +type ResourceTopologySpec struct { + // Provisioner is the volume provisioner name + // +kubebuilder:validation:Required + Provisioner string `json:"provisioner" protobuf:"bytes,2,name=provisioner"` + + // VolumeHandle is the backend name and identity of the volume, format as . + // +kubebuilder:validation:Required + VolumeHandle string `json:"volumeHandle" protobuf:"bytes,2,name=volumeHandle"` + + // Tags defines pv and other relationships and ownership + // +kubebuilder:validation:Required + Tags []Tag `json:"tags" protobuf:"bytes,2,name=tags"` +} + +type ResourceTopologyStatus struct { + // Status is the status of the ResourceTopology + Status ResourceTopologyStatusPhase `json:"status,omitempty" protobuf:"bytes,2,opt,name=status"` + + // Tags defines pv and other relationships and ownership + Tags []Tag `json:"tags,omitempty" protobuf:"bytes,3,opt,name=tags"` +} + +// Tag defines pv and other relationships and ownership +type Tag struct { + ResourceInfo `json:",inline"` + + // Owner defines who does the resource belongs to + // +kubebuilder:validation:Optional + Owner ResourceInfo `json:"owner" protobuf:"bytes,2,name=owner"` +} + +// ResourceInfo define resource information +type ResourceInfo struct { + metaV1.TypeMeta `json:",inline"` + // NameSpace is the namespace of the resource + Namespace string `json:"namespace,omitempty" protobuf:"bytes,2,opt,name=target"` + // Name is the name of the resource + Name string `json:"name,omitempty" protobuf:"bytes,2,opt,name=name"` +} + +// ResourceTopology is the Schema for the ResourceTopologys API +// +genclient +// +genclient:nonNamespaced +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:object:root=true +// +kubebuilder:resource:scope=Cluster,shortName="rt" +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Provisioner",type=string,JSONPath=`.spec.provisioner` +// +kubebuilder:printcolumn:name="VolumeHandle",type=string,JSONPath=`.spec.volumeHandle` +// +kubebuilder:printcolumn:name="Status",type=string,JSONPath=`.status.status` +// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` +type ResourceTopology struct { + metaV1.TypeMeta `json:",inline"` + metaV1.ObjectMeta `json:"metadata,omitempty"` + Spec ResourceTopologySpec `json:"spec,omitempty"` + Status ResourceTopologyStatus `json:"status,omitempty"` +} + +// ResourceTopologyList contains a list of ResourceTopology +// +kubebuilder:object:root=true +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +type ResourceTopologyList struct { + metaV1.TypeMeta `json:",inline"` + metaV1.ListMeta `json:"metadata,omitempty"` + Items []ResourceTopology `json:"items"` +} diff --git a/client/apis/xuanwu/v1/storagebackendclaim.go b/client/apis/xuanwu/v1/storagebackendclaim.go index 5e4d232d..d5c0a6b8 100644 --- a/client/apis/xuanwu/v1/storagebackendclaim.go +++ b/client/apis/xuanwu/v1/storagebackendclaim.go @@ -38,6 +38,15 @@ type StorageBackendClaimSpec struct { // User defined parameter for extension // +optional Parameters map[string]string `json:"parameters,omitempty" protobuf:"bytes,8,opt,name=parameters"` + + // UseCert is used to decide whether to use the certificate + // +kubebuilder:default=false + // +optional + UseCert bool `json:"useCert,omitempty" protobuf:"bytes,9,opt,name=useCert"` + + // CertSecret is the name of the secret that holds the certificate + // +optional + CertSecret string `json:"certSecret,omitempty" protobuf:"bytes,9,opt,name=certSecret"` } // StorageBackendClaimStatus defines the observed state of StorageBackend @@ -72,6 +81,12 @@ type StorageBackendClaimStatus struct { // MetroBackend is the backend that form hyperMetro MetroBackend string `json:"metroBackend,omitempty" protobuf:"bytes,2,opt,name=metroBackend"` + + // UseCert is used to decide whether to use the certificate + UseCert bool `json:"useCert,omitempty" protobuf:"bytes,9,opt,name=useCert"` + + // CertSecret is the name of the secret that holds the certificate + CertSecret string `json:"certSecret,omitempty" protobuf:"bytes,9,opt,name=certSecret"` } // StorageBackendPhase defines the phase of StorageBackend diff --git a/client/apis/xuanwu/v1/storagebackendcontent.go b/client/apis/xuanwu/v1/storagebackendcontent.go index 71acdb12..743a1ec3 100644 --- a/client/apis/xuanwu/v1/storagebackendcontent.go +++ b/client/apis/xuanwu/v1/storagebackendcontent.go @@ -44,6 +44,15 @@ type StorageBackendContentSpec struct { // User defined parameter for extension // +optional Parameters map[string]string `json:"parameters,omitempty" protobuf:"bytes,8,opt,name=parameters"` + + // UseCert is used to decide whether to use the certificate + // +kubebuilder:default=false + // +optional + UseCert bool `json:"useCert,omitempty" protobuf:"bytes,9,opt,name=useCert"` + + // CertSecret is the name of the secret that holds the certificate + // +optional + CertSecret string `json:"certSecret,omitempty" protobuf:"bytes,9,opt,name=certSecret"` } // StorageBackendContentStatus defines the observed state of StorageBackendContent @@ -80,6 +89,12 @@ type StorageBackendContentStatus struct { // SN is the unique identifier of a storage device. SN string `json:"sn,omitempty" protobuf:"bytes,1,opt,name=sn"` + + // UseCert is used to decide whether to use the certificate + UseCert bool `json:"useCert,omitempty" protobuf:"bytes,9,opt,name=useCert"` + + // CertSecret is the name of the secret that holds the certificate + CertSecret string `json:"certSecret,omitempty" protobuf:"bytes,9,opt,name=certSecret"` } // CapacityType means the capacity types diff --git a/client/apis/xuanwu/v1/zz_generated.deepcopy.go b/client/apis/xuanwu/v1/zz_generated.deepcopy.go index 1baab91d..279f1ec2 100644 --- a/client/apis/xuanwu/v1/zz_generated.deepcopy.go +++ b/client/apis/xuanwu/v1/zz_generated.deepcopy.go @@ -22,6 +22,126 @@ import ( runtime "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ResourceInfo) DeepCopyInto(out *ResourceInfo) { + *out = *in + out.TypeMeta = in.TypeMeta + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResourceInfo. +func (in *ResourceInfo) DeepCopy() *ResourceInfo { + if in == nil { + return nil + } + out := new(ResourceInfo) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ResourceTopology) DeepCopyInto(out *ResourceTopology) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResourceTopology. +func (in *ResourceTopology) DeepCopy() *ResourceTopology { + if in == nil { + return nil + } + out := new(ResourceTopology) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ResourceTopology) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ResourceTopologyList) DeepCopyInto(out *ResourceTopologyList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]ResourceTopology, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResourceTopologyList. +func (in *ResourceTopologyList) DeepCopy() *ResourceTopologyList { + if in == nil { + return nil + } + out := new(ResourceTopologyList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ResourceTopologyList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ResourceTopologySpec) DeepCopyInto(out *ResourceTopologySpec) { + *out = *in + if in.Tags != nil { + in, out := &in.Tags, &out.Tags + *out = make([]Tag, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResourceTopologySpec. +func (in *ResourceTopologySpec) DeepCopy() *ResourceTopologySpec { + if in == nil { + return nil + } + out := new(ResourceTopologySpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ResourceTopologyStatus) DeepCopyInto(out *ResourceTopologyStatus) { + *out = *in + if in.Tags != nil { + in, out := &in.Tags, &out.Tags + *out = make([]Tag, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResourceTopologyStatus. +func (in *ResourceTopologyStatus) DeepCopy() *ResourceTopologyStatus { + if in == nil { + return nil + } + out := new(ResourceTopologyStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *StorageBackendClaim) DeepCopyInto(out *StorageBackendClaim) { *out = *in @@ -250,3 +370,21 @@ func (in *StorageBackendContentStatus) DeepCopy() *StorageBackendContentStatus { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Tag) DeepCopyInto(out *Tag) { + *out = *in + out.ResourceInfo = in.ResourceInfo + out.Owner = in.Owner + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Tag. +func (in *Tag) DeepCopy() *Tag { + if in == nil { + return nil + } + out := new(Tag) + in.DeepCopyInto(out) + return out +} diff --git a/cmd/storage-backend-controller/main.go b/cmd/storage-backend-controller/main.go index d227a829..d672eeb2 100644 --- a/cmd/storage-backend-controller/main.go +++ b/cmd/storage-backend-controller/main.go @@ -28,8 +28,6 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" coreV1 "k8s.io/client-go/kubernetes/typed/core/v1" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/tools/record" "huawei-csi-driver/csi/app" @@ -65,7 +63,7 @@ func main() { } ctx := context.Background() - k8sClient, storageBackendClient, err := getKubernetesClient(ctx) + k8sClient, storageBackendClient, err := utils.GetK8SAndSBCClient(ctx) if err != nil { return } @@ -176,36 +174,6 @@ func ensureCRDExist(ctx context.Context, client *clientSet.Clientset) error { return nil } -func getKubernetesClient(ctx context.Context) (*kubernetes.Clientset, *clientSet.Clientset, error) { - var config *rest.Config - var err error - if app.GetGlobalConfig().KubeConfig != "" { - config, err = clientcmd.BuildConfigFromFlags("", app.GetGlobalConfig().KubeConfig) - } else { - config, err = rest.InClusterConfig() - } - - if err != nil { - log.AddContext(ctx).Errorf("Error getting cluster config, kube config: %s, %v", - app.GetGlobalConfig().KubeConfig, err) - return nil, nil, err - } - - k8sClient, err := kubernetes.NewForConfig(config) - if err != nil { - log.AddContext(ctx).Errorf("Error getting kubernetes client, %v", err) - return nil, nil, err - } - - storageBackendClient, err := clientSet.NewForConfig(config) - if err != nil { - log.AddContext(ctx).Errorf("Error getting storage backend client, %v", err) - return nil, nil, err - } - - return k8sClient, storageBackendClient, nil -} - func startWithLeaderElectionOnCondition(ctx context.Context, k8sClient *kubernetes.Clientset, storageBackendClient *clientSet.Clientset, recorder record.EventRecorder, ch chan os.Signal) { if !app.GetGlobalConfig().EnableLeaderElection { diff --git a/cmd/storage-backend-sidecar/main.go b/cmd/storage-backend-sidecar/main.go index 8fa1da50..85b6de04 100644 --- a/cmd/storage-backend-sidecar/main.go +++ b/cmd/storage-backend-sidecar/main.go @@ -30,8 +30,6 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" coreV1 "k8s.io/client-go/kubernetes/typed/core/v1" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/tools/record" "huawei-csi-driver/csi/app" @@ -77,7 +75,7 @@ func main() { } ctx := context.Background() - k8sClient, storageBackendClient, err := getKubernetesClient(ctx) + k8sClient, storageBackendClient, err := utils.GetK8SAndSBCClient(ctx) if err != nil { log.AddContext(ctx).Errorf("GetKubernetesClient failed, error: %v", err) return @@ -203,33 +201,3 @@ func ensureCRDExist(ctx context.Context, client *clientSet.Clientset) error { return nil } - -func getKubernetesClient(ctx context.Context) (*kubernetes.Clientset, *clientSet.Clientset, error) { - var config *rest.Config - var err error - if app.GetGlobalConfig().KubeConfig != "" { - config, err = clientcmd.BuildConfigFromFlags("", app.GetGlobalConfig().KubeConfig) - } else { - config, err = rest.InClusterConfig() - } - - if err != nil { - log.AddContext(ctx).Errorf("Error getting cluster config, kube config: %s, %v", - app.GetGlobalConfig().KubeConfig, err) - return nil, nil, err - } - - k8sClient, err := kubernetes.NewForConfig(config) - if err != nil { - log.AddContext(ctx).Errorf("Error getting kubernetes client, %v", err) - return nil, nil, err - } - - storageBackendClient, err := clientSet.NewForConfig(config) - if err != nil { - log.AddContext(ctx).Errorf("Error getting storage backend client, %v", err) - return nil, nil, err - } - - return k8sClient, storageBackendClient, nil -} diff --git a/connector/connector.go b/connector/connector.go index 7f71e297..bc38373b 100644 --- a/connector/connector.go +++ b/connector/connector.go @@ -19,8 +19,6 @@ package connector import ( "context" "fmt" - "time" - "huawei-csi-driver/utils/log" ) @@ -62,8 +60,7 @@ const ( ) var ( - connectors = map[string]Connector{} - ScanVolumeTimeout = 3 * time.Second + connectors = map[string]Connector{} ) type Connector interface { diff --git a/connector/connector_utils.go b/connector/connector_utils.go index c6e93c99..025fb8df 100644 --- a/connector/connector_utils.go +++ b/connector/connector_utils.go @@ -252,19 +252,27 @@ var GetDevicesByGUID = func(ctx context.Context, tgtLunGUID string) ([]string, e } func reScanNVMe(ctx context.Context, device string) error { - if match, _ := regexp.MatchString(`nvme[0-9]+n[0-9]+`, device); match { + var err error + if match, err := regexp.MatchString(`nvme[0-9]+n[0-9]+`, device); err == nil && match { output, err := utils.ExecShellCmd(ctx, "echo 1 > /sys/block/%s/device/rescan_controller", device) if err != nil { log.AddContext(ctx).Warningf("rescan nvme path error: %s", output) return err } - } else if match, _ := regexp.MatchString(`nvme[0-9]+$`, device); match { + } else if match, err = regexp.MatchString(`nvme[0-9]+$`, device); err == nil && match { output, err := utils.ExecShellCmd(ctx, "nvme ns-rescan /dev/%s", device) if err != nil { log.AddContext(ctx).Warningf("rescan nvme path error: %s", output) return err } } + + if err != nil { + log.AddContext(ctx).Warningf("pattern compile failed, err: %v", err) + return err + } + + log.AddContext(ctx).Warningf("device %s match failed, err: %v", device, err) return nil } @@ -434,7 +442,7 @@ func getSessionIdByDevice(devPath string) (string, error) { // WatchDMDevice is an aggregate drive letter monitor. func WatchDMDevice(ctx context.Context, lunWWN string, expectPathNumber int) (DMDeviceInfo, error) { log.AddContext(ctx).Infof("Watch DM Disk Generation. lunWWN: %s,expectPathNumber: %d", lunWWN, expectPathNumber) - var timeout = time.After(ScanVolumeTimeout) + var timeout = time.After(time.Second * time.Duration(app.GetGlobalConfig().ScanVolumeTimeout)) var dm DMDeviceInfo var err = errors.New(VolumeNotFound) for { @@ -710,8 +718,13 @@ func removeMultiPathDevice(ctx context.Context, multiPathName string, devices [] func RemoveDevice(ctx context.Context, device string) (string, error) { var multiPathName string var err error + if strings.HasPrefix(device, "dm") { - devices, _ := getDeviceFromDM(device) + devices, err := getDeviceFromDM(device) + if err != nil { + log.AddContext(ctx).Errorf("RemoveDevice.getDeviceFromDM failed, device: %s, err: %v", device, err) + return "", err + } multiPathName, err = removeMultiPathDevice(ctx, device, devices) } else if strings.HasPrefix(device, "sd") { err = removeSCSIDevice(ctx, device) @@ -1089,9 +1102,9 @@ var ResizeMountPath = func(ctx context.Context, volumePath string) error { return extResize(ctx, devicePath) case "xfs": return xfsResize(ctx, volumePath) + default: + return fmt.Errorf("resize of format %s is not supported for device %s", fsType, devicePath) } - - return fmt.Errorf("resize of format %s is not supported for device %s", fsType, devicePath) } func extResize(ctx context.Context, devicePath string) error { @@ -1306,7 +1319,8 @@ func VerifyDeviceAvailableOfDM(ctx context.Context, tgtLunWWN string, expectPath start := time.Now() dm, err := WatchDMDevice(ctx, tgtLunWWN, expectPathNumber) - log.AddContext(ctx).Infof("WatchDMDevice-%s:%-36s%-8d%-20s%v", ScanVolumeTimeout, + log.AddContext(ctx).Infof("WatchDMDevice-%s:%-36s%-8d%-20s%v", + time.Second*time.Duration(app.GetGlobalConfig().ScanVolumeTimeout), tgtLunWWN, expectPathNumber, time.Now().Sub(start), err) if err == nil { var dev string @@ -1841,7 +1855,7 @@ func getDevicePathRefs(device string, mountMap map[string]string) []string { } // ReadMountPoints read mount file -// mountMap[mountPath] = devicePath +// mountMap: key means mountPath; value means devicePath. func ReadMountPoints(ctx context.Context) (map[string]string, error) { data, err := ConsistentRead(procMountsPath, maxListTries) if err != nil { diff --git a/connector/connector_utils_test.go b/connector/connector_utils_test.go index 461edaa5..f716818a 100644 --- a/connector/connector_utils_test.go +++ b/connector/connector_utils_test.go @@ -455,12 +455,12 @@ func TestWatchDMDevice(t *testing.T) { 3, []string{"sdb", "sdc", "sdd"}, 100 * time.Millisecond, - 200 * time.Millisecond, + 10000 * time.Millisecond, errors.New(VolumeNotFound), }, } - stubs := gostub.Stub(&ScanVolumeTimeout, 10*time.Millisecond) + stubs := gostub.New() defer stubs.Reset() for _, c := range cases { @@ -690,17 +690,30 @@ func helperFuncForTestGetDeviceFromSymLink(t *testing.T, mockTargetNameWithoutLi t.Errorf("create mock target without link file failed, error: %v", err) return "", true } - if err := withoutLinkPath.Close(); err != nil { - t.Errorf("close file %s failed, error: %v", withoutLinkPath.Name(), err) + defer func() { + if err = withoutLinkPath.Close(); err != nil { + t.Errorf("close file %s failed, error: %v", withoutLinkPath.Name(), err) + } + }() + err = withoutLinkPath.Chmod(0600) + if err != nil { + t.Errorf("file withoutLinkPath chmod to 0600 failed, error: %v", err) + return "", true } - deviceFile, err := os.Create(path.Join(tempDir, mockDeviceName)) if err != nil { t.Errorf("create mock device file failed, error: %v", err) return "", true } - if err := deviceFile.Close(); err != nil { - t.Errorf("close file %s failed, error: %v", deviceFile.Name(), err) + defer func() { + if err := deviceFile.Close(); err != nil { + t.Errorf("close file %s failed, error: %v", deviceFile.Name(), err) + } + }() + err = deviceFile.Chmod(0600) + if err != nil { + t.Errorf("file deviceFile chmod to 0600 failed, error: %v", err) + return "", true } if err := os.Symlink(path.Join(tempDir, mockDeviceName), path.Join(tempDir, mockTargetName)); err != nil { diff --git a/connector/fibrechannel/fc_helper.go b/connector/fibrechannel/fc_helper.go index 4b5dde90..162c9f8c 100644 --- a/connector/fibrechannel/fc_helper.go +++ b/connector/fibrechannel/fc_helper.go @@ -68,20 +68,6 @@ const ( var expectPathCount sync.Map -func getDevicePathNumber(lunWWN string) int { - v, ok := expectPathCount.Load(lunWWN) - if !ok { - return 1 - } - - number, ok := v.(int) - if !ok { - return 1 - } - - return number -} - func parseFCInfo(ctx context.Context, connectionProperties map[string]interface{}) (*connectorInfo, error) { var info = new(connectorInfo) var exist bool @@ -407,7 +393,11 @@ func getPciNumber(hba map[string]string) (string, string) { } func formatLunId(lunId string) string { - intLunId, _ := strconv.Atoi(lunId) + intLunId, err := strconv.Atoi(lunId) + if err != nil { + log.Warningf("formatLunId failed, lunId: %v, err: %v", lunId, err) + } + if intLunId < 256 { return lunId } else { @@ -548,13 +538,13 @@ func rescanHosts(ctx context.Context, hbas []map[string]string, conn *connectorI return } - hba := pro[0].(map[string]string) + hba, ok := pro[0].(map[string]string) if !ok { log.AddContext(ctx).Errorf("the %v is not map[string]string", pro[0]) return } - ctls := pro[1].([][]string) + ctls, ok := pro[1].([][]string) if !ok { log.AddContext(ctx).Errorf("the %v is not [][]string", pro[1]) return diff --git a/connector/nfs/nfs_helper.go b/connector/nfs/nfs_helper.go index 5a16f10e..921e3112 100644 --- a/connector/nfs/nfs_helper.go +++ b/connector/nfs/nfs_helper.go @@ -304,7 +304,7 @@ func getFSType(ctx context.Context, sourcePath string) (string, error) { func formatDisk(ctx context.Context, sourcePath, fsType, diskSizeType string) error { var cmd string - if "xfs" == fsType { + if fsType == "xfs" { cmd = fmt.Sprintf("mkfs -t %s -f %s", fsType, sourcePath) } else { // Handle ext types @@ -319,8 +319,11 @@ func formatDisk(ctx context.Context, sourcePath, fsType, diskSizeType string) er cmd = fmt.Sprintf("mkfs -t %s -T largefile -F %s", fsType, sourcePath) case "veryLarge": cmd = fmt.Sprintf("mkfs -t %s -T largefile4 -F %s", fsType, sourcePath) + default: + return fmt.Errorf("%v:%v not found", "diskSizeType", diskSizeType) } } + output, err := utils.ExecShellCmd(ctx, cmd) if err != nil { if strings.Contains(output, "in use by the system") { diff --git a/csi/app/config/config.go b/csi/app/config/config.go index dae38173..a24a0276 100644 --- a/csi/app/config/config.go +++ b/csi/app/config/config.go @@ -37,6 +37,7 @@ type loggingConfig struct { type serviceConfig struct { Controller bool EnableLeaderElection bool + EnableLabel bool Endpoint string DrEndpoint string @@ -66,6 +67,7 @@ type connectorConfig struct { ScanVolumeTimeout int ConnectorThreads int AllPathOnline bool + ExecCommandTimeout int } type k8sConfig struct { diff --git a/csi/app/config/config_mock.go b/csi/app/config/config_mock.go index 32fc542c..100f99f6 100644 --- a/csi/app/config/config_mock.go +++ b/csi/app/config/config_mock.go @@ -50,6 +50,7 @@ func mockServiceConfig() serviceConfig { return serviceConfig{ Controller: false, EnableLeaderElection: false, + EnableLabel: false, Endpoint: "", DrEndpoint: "", diff --git a/csi/app/options/connector.go b/csi/app/options/connector.go index b3015de4..df376722 100644 --- a/csi/app/options/connector.go +++ b/csi/app/options/connector.go @@ -45,6 +45,7 @@ type connectorOptions struct { scanVolumeTimeout int connectorThreads int allPathOnline bool + execCommandTimeout int } // NewConnectorOptions returns connector configurations @@ -83,6 +84,9 @@ func (opt *connectorOptions) AddFlags(ff *flag.FlagSet) { ff.BoolVar(&opt.allPathOnline, "all-path-online", false, "Whether to check the number of online paths for DM-multipath aggregation, default false") + ff.IntVar(&opt.execCommandTimeout, "exec-command-timeout", + 30, + "The timeout for running command on host") } // ApplyFlags assign the connector flags @@ -94,6 +98,7 @@ func (opt *connectorOptions) ApplyFlags(cfg *config.Config) { cfg.ScanVolumeTimeout = opt.scanVolumeTimeout cfg.ConnectorThreads = opt.connectorThreads cfg.AllPathOnline = opt.allPathOnline + cfg.ExecCommandTimeout = opt.execCommandTimeout } // ValidateFlags validate the connector flags @@ -150,3 +155,10 @@ func (opt *connectorOptions) validateConnectorThreads() error { } return nil } +func (opt *connectorOptions) validateExecCommandTimeout() error { + if opt.execCommandTimeout < 1 || opt.execCommandTimeout > 600 { + return fmt.Errorf("the value of execCommandTimeout ranges from 1 to 600, current is: %d", + opt.scanVolumeTimeout) + } + return nil +} diff --git a/csi/app/options/service.go b/csi/app/options/service.go index db22c226..d0203e20 100644 --- a/csi/app/options/service.go +++ b/csi/app/options/service.go @@ -23,17 +23,14 @@ import ( "time" "huawei-csi-driver/csi/app/config" -) - -const ( - defaultDriverName = "csi.huawei.com" - nodeNameEnv = "CSI_NODENAME" + "huawei-csi-driver/pkg/constants" ) // serviceOptions include service's configuration type serviceOptions struct { controller bool enableLeaderElection bool + enableLabel bool driverName string endpoint string @@ -72,7 +69,7 @@ func (opt *serviceOptions) AddFlags(ff *flag.FlagSet) { false, "Run as a controller service") ff.StringVar(&opt.driverName, "driver-name", - defaultDriverName, + constants.DefaultDriverName, "CSI driver name") ff.IntVar(&opt.backendUpdateInterval, "backend-update-interval", 60, @@ -81,7 +78,7 @@ func (opt *serviceOptions) AddFlags(ff *flag.FlagSet) { "", "absolute path to the kubeconfig file") ff.StringVar(&opt.nodeName, "nodename", - os.Getenv(nodeNameEnv), + os.Getenv(constants.NodeNameEnv), "node name in kubernetes cluster") ff.StringVar(&opt.kubeletRootDir, "kubeletRootDir", "/var/lib", @@ -94,6 +91,9 @@ func (opt *serviceOptions) AddFlags(ff *flag.FlagSet) { ff.IntVar(&opt.webHookPort, "web-hook-port", 0, "The number of volumes that controller can publish to the node") + ff.BoolVar(&opt.enableLabel, "enable-label", + false, + "csi enable label") ff.BoolVar(&opt.enableLeaderElection, "enable-leader-election", false, "backend enable leader election") @@ -121,6 +121,7 @@ func (opt *serviceOptions) AddFlags(ff *flag.FlagSet) { func (opt *serviceOptions) ApplyFlags(cfg *config.Config) { cfg.Endpoint = opt.endpoint cfg.DrEndpoint = opt.drEndpoint + cfg.EnableLabel = opt.enableLabel cfg.Controller = opt.controller cfg.DriverName = opt.driverName cfg.BackendUpdateInterval = opt.backendUpdateInterval diff --git a/csi/backend/backend.go b/csi/backend/backend.go index bb7be364..55fb4076 100644 --- a/csi/backend/backend.go +++ b/csi/backend/backend.go @@ -136,8 +136,8 @@ func analyzePools(backend *Backend, config map[string]interface{}) error { configPools, _ := config["pools"].([]interface{}) for _, i := range configPools { - name := i.(string) - if name == "" { + name, ok := i.(string) + if !ok || name == "" { continue } @@ -512,8 +512,7 @@ func filterByStoragePool(ctx context.Context, poolName string, candidatePools [] return filterPools, nil } -func filterByVolumeType(ctx context.Context, volumeType string, candidatePools []*StoragePool) ([]*StoragePool, - error) { +func filterByVolumeType(ctx context.Context, volumeType string, candidatePools []*StoragePool) ([]*StoragePool, error) { var filterPools []*StoragePool for _, pool := range candidatePools { @@ -545,9 +544,15 @@ func filterByAllocType(ctx context.Context, allocType string, candidatePools []* valid = true } else if allocType == "thin" || allocType == "" { supportThin, exist := pool.Capabilities["SupportThin"].(bool) + if !exist { + log.AddContext(ctx).Warningf("convert supportThin to bool failed, data: %v", pool.Capabilities["SupportThin"]) + } valid = exist && supportThin } else if allocType == "thick" { supportThick, exist := pool.Capabilities["SupportThick"].(bool) + if !exist { + log.AddContext(ctx).Warningf("convert supportThick to bool failed, data: %v", pool.Capabilities["SupportThick"]) + } valid = exist && supportThick } @@ -889,7 +894,13 @@ func filterByCapacity(requestSize int64, allocType string, candidatePools []*Sto var filterPools []*StoragePool for _, pool := range candidatePools { supportThin, thinExist := pool.Capabilities["SupportThin"].(bool) + if !thinExist { + log.Warningf("convert supportThin to bool failed, data: %v", pool.Capabilities["SupportThin"]) + } supportThick, thickExist := pool.Capabilities["SupportThick"].(bool) + if !thickExist { + log.Warningf("convert supportThick to bool failed, data: %v", pool.Capabilities["SupportThick"]) + } if (allocType == "thin" || allocType == "") && thinExist && supportThin { filterPools = append(filterPools, pool) } else if allocType == "thick" && thickExist && supportThick { @@ -995,15 +1006,19 @@ func validateBackendName(ctx context.Context, backendName string, selectBackend } func validateVolumeType(ctx context.Context, volumeType string, selectBackend *Backend) error { - if filterPools, _ := filterByVolumeType(ctx, volumeType, selectBackend.Pools); len(filterPools) == 0 { + if filterPools, err := filterByVolumeType(ctx, volumeType, selectBackend.Pools); len(filterPools) == 0 { + if err != nil { + return err + } return utils.Errorf(ctx, "the volumeType between StorageClass(%s) and PVC annotation(%s) "+ - "is different", volumeType, selectBackend.Name) + "is different, err: filterPools is empty", volumeType, selectBackend.Name) } return nil } // RegisterOneBackend used to register a backend to plugin -func RegisterOneBackend(ctx context.Context, backendID, configmapMeta, secretMeta string) (string, error) { +func RegisterOneBackend(ctx context.Context, backendID, configmapMeta, secretMeta, certSecret string, + useCert bool) (string, error) { mutex.Lock() defer mutex.Unlock() @@ -1017,7 +1032,7 @@ func RegisterOneBackend(ctx context.Context, backendID, configmapMeta, secretMet return backendName, nil } - storageInfo, err := GetStorageBackendInfo(ctx, backendID, configmapMeta, secretMeta) + storageInfo, err := GetStorageBackendInfo(ctx, backendID, configmapMeta, secretMeta, certSecret, useCert) if err != nil { return "", err } @@ -1074,6 +1089,10 @@ func RegisterAllBackend(ctx context.Context) error { log.AddContext(ctx).Warningf("Get storageBackendContent failed, error: [%v]", err) continue } + if content == nil || content.Status == nil { + log.AddContext(ctx).Warningf("Get empty storageBackendContent, content: %+v", content) + continue + } if !content.Status.Online { log.AddContext(ctx).Warningf("StorageBackendContent: [%s] is offline(online: false), "+ "will not register.", content.Name) @@ -1090,7 +1109,15 @@ func RegisterAllBackend(ctx context.Context) error { "[%s], error: [%v]", content.Spec.BackendClaim, content.Name, err) continue } - _, err = RegisterOneBackend(ctx, content.Spec.BackendClaim, configmapMeta, secretMeta) + + useCert, certSecret, err := pkgUtils.GetCertMeta(ctx, content.Spec.BackendClaim) + if err != nil { + log.AddContext(ctx).Warningf("Get storageBackendClaim: [%s] CertMeta failed, storageBackendContent: "+ + "[%s], error: [%v]", content.Spec.BackendClaim, content.Name, err) + continue + } + + _, err = RegisterOneBackend(ctx, content.Spec.BackendClaim, configmapMeta, secretMeta, certSecret, useCert) if err != nil { log.AddContext(ctx).Warningf("RegisterOneBackend failed, meta: [%s %s %s], error: %v", content.Spec.BackendClaim, configmapMeta, secretMeta, err) diff --git a/csi/backend/backend_get.go b/csi/backend/backend_get.go index 8d14e0fd..ebdfe29f 100644 --- a/csi/backend/backend_get.go +++ b/csi/backend/backend_get.go @@ -96,7 +96,14 @@ var GetBackendWithFresh = func(ctx context.Context, backendName string, update b log.AddContext(ctx).Errorf("GetConfigMeta %s failed, error %v", backendMeta, err) return nil } - _, err = RegisterOneBackend(ctx, backendMeta, configmapMeta, secretMeta) + + useCert, certSecret, err := pkgUtils.GetCertMeta(ctx, backendMeta) + if err != nil { + log.AddContext(ctx).Errorf("GetCertMeta %s failed, error %v", backendMeta, err) + return nil + } + + _, err = RegisterOneBackend(ctx, backendMeta, configmapMeta, secretMeta, certSecret, useCert) if err != nil { msg := fmt.Sprintf("RegisterBackend %s failed, error %v", backendMeta, err) log.AddContext(ctx).Errorln(msg) @@ -183,7 +190,8 @@ func addSecretInfo(secret *coreV1.Secret, storageConfig map[string]interface{}) } // GetStorageBackendInfo used to get storage config info -func GetStorageBackendInfo(ctx context.Context, backendID, configmapMeta, secretMeta string) (map[string]interface{}, error) { +func GetStorageBackendInfo(ctx context.Context, backendID, configmapMeta, secretMeta, certSecret string, + useCert bool) (map[string]interface{}, error) { log.AddContext(ctx).Infof("start GetStorageBackendInfo: %s.", backendID) backendMapData, err := GetBackendConfigmapMap(ctx, configmapMeta) if err != nil { @@ -208,5 +216,8 @@ func GetStorageBackendInfo(ctx context.Context, backendID, configmapMeta, secret backendMapData["backendID"] = backendID + backendMapData["useCert"] = useCert + backendMapData["certSecret"] = certSecret + return backendMapData, nil } diff --git a/csi/backend/backend_test.go b/csi/backend/backend_test.go index f283aece..9b15f5b6 100644 --- a/csi/backend/backend_test.go +++ b/csi/backend/backend_test.go @@ -288,7 +288,10 @@ func TestFilterByVolumeType(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, _ := filterByVolumeType(ctx, tt.volumeType, tt.candidatePools) + got, err := filterByVolumeType(ctx, tt.volumeType, tt.candidatePools) + if err != nil { + t.Errorf("test filterByVolumeType faild. err: %v", err) + } if !reflect.DeepEqual(got, tt.expect) { t.Errorf("test filterByVolumeType faild. got: %v, expect: %v", got, tt.expect) } diff --git a/csi/backend/plugin/fusionstorage-nas.go b/csi/backend/plugin/fusionstorage-nas.go index 99c77041..15b49bcc 100644 --- a/csi/backend/plugin/fusionstorage-nas.go +++ b/csi/backend/plugin/fusionstorage-nas.go @@ -53,7 +53,10 @@ func (p *FusionStorageNasPlugin) Init(config, parameters map[string]interface{}, if !exist || len(portals) != 1 { return errors.New("portals must be provided for fusionstorage-nas nfs backend and just support one portal") } - p.portal = portals[0].(string) + p.portal, exist = portals[0].(string) + if !exist { + return errors.New(fmt.Sprintf("portals: %v must be string", portals[0])) + } } err := p.init(config, keepLogin) @@ -123,13 +126,13 @@ func (p *FusionStorageNasPlugin) UpdateBackendCapabilities() (map[string]interfa "SupportQoS": true, "SupportQuota": true, "SupportClone": false, + "SupportLabel": false, } err := p.updateNFS4Capability(capabilities) if err != nil { return nil, nil, err } - return capabilities, nil, nil } @@ -190,8 +193,7 @@ func (p *FusionStorageNasPlugin) Validate(ctx context.Context, param map[string] } // Login verification - cli := client.NewClient(clientConfig.Url, clientConfig.User, clientConfig.SecretName, - clientConfig.SecretNamespace, clientConfig.ParallelNum, clientConfig.BackendID, clientConfig.AccountName) + cli := client.NewClient(clientConfig) err = cli.ValidateLogin(ctx) if err != nil { return err diff --git a/csi/backend/plugin/fusionstorage-san.go b/csi/backend/plugin/fusionstorage-san.go index e4aaf270..8f27189d 100644 --- a/csi/backend/plugin/fusionstorage-san.go +++ b/csi/backend/plugin/fusionstorage-san.go @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + // Package plugin provide storage function package plugin @@ -68,9 +69,15 @@ func (p *FusionStorageSanPlugin) Init(config, parameters map[string]interface{}, } if strings.ToLower(protocol) == "scsi" { - scsi := portals[0].(map[string]interface{}) + scsi, ok := portals[0].(map[string]interface{}) + if !ok { + return errors.New("scsi portals convert to map[string]interface{} failed") + } for k, v := range scsi { - manageIP := v.(string) + manageIP, ok := v.(string) + if !ok { + continue + } ip := net.ParseIP(manageIP) if ip == nil { return fmt.Errorf("Manage IP %s of host %s is invalid", manageIP, k) @@ -204,26 +211,6 @@ func (p *FusionStorageSanPlugin) DetachVolume(ctx context.Context, return nil } -func (p *FusionStorageSanPlugin) mutexGetClient(ctx context.Context) (*client.Client, error) { - p.clientMutex.Lock() - var err error - if !p.storageOnline || p.clientCount == 0 { - err = p.cli.Login(ctx) - p.storageOnline = err == nil - if err == nil { - p.clientCount++ - } - } else { - p.clientCount++ - } - p.clientMutex.Unlock() - return p.cli, err -} - -func (p *FusionStorageSanPlugin) getClient(ctx context.Context) (*client.Client, error) { - return p.mutexGetClient(ctx) -} - func (p *FusionStorageSanPlugin) mutexReleaseClient(ctx context.Context, plugin *FusionStorageSanPlugin, cli *client.Client) { @@ -248,8 +235,8 @@ func (p *FusionStorageSanPlugin) UpdateBackendCapabilities() (map[string]interfa "SupportThick": false, "SupportQoS": true, "SupportClone": true, + "SupportLabel": false, } - return capabilities, nil, nil } @@ -297,8 +284,7 @@ func (p *FusionStorageSanPlugin) Validate(ctx context.Context, param map[string] } // Login verification - cli := client.NewClient(clientConfig.Url, clientConfig.User, clientConfig.SecretName, - clientConfig.SecretNamespace, clientConfig.ParallelNum, clientConfig.BackendID, clientConfig.AccountName) + cli := client.NewClient(clientConfig) err = cli.ValidateLogin(ctx) if err != nil { return err diff --git a/csi/backend/plugin/fusionstorage.go b/csi/backend/plugin/fusionstorage.go index 48067d98..b047cb86 100644 --- a/csi/backend/plugin/fusionstorage.go +++ b/csi/backend/plugin/fusionstorage.go @@ -44,40 +44,21 @@ type FusionStoragePlugin struct { } func (p *FusionStoragePlugin) init(config map[string]interface{}, keepLogin bool) error { - configUrls, exist := config["urls"].([]interface{}) - if !exist || len(configUrls) <= 0 { - return errors.New("urls must be provided") - } - - url := configUrls[0].(string) - - user, exist := config["user"].(string) - if !exist { - return errors.New("user must be provided") - } - - secretName, exist := config["secretName"].(string) - if !exist { - return errors.New("SecretName must be provided") - } - secretNamespace, exist := config["secretNamespace"].(string) - if !exist { - return errors.New("SecretNamespace must be provided") + clientConfig, err := p.getNewClientConfig(context.Background(), config) + if err != nil { + return err } - backendID, exist := config["backendID"].(string) - if !exist { - return errors.New("backendID must be provided") + cli := client.NewClient(clientConfig) + err = cli.Login(context.Background()) + if err != nil { + return err } - accountName, _ := config["accountName"].(string) - parallelNum, _ := config["maxClientThreads"].(string) - - cli := client.NewClient(url, user, secretName, secretNamespace, parallelNum, backendID, accountName) - err := cli.Login(context.Background()) + err = cli.SetAccountId(context.TODO()) if err != nil { - return err + return pkgUtils.Errorln(context.TODO(), fmt.Sprintf("setAccountId failed, error: %v", err)) } if !keepLogin { @@ -152,7 +133,10 @@ func (p *FusionStoragePlugin) updatePoolCapabilities(poolNames []string, storage for _, name := range poolNames { if i, exist := pools[name]; exist { - pool := i.(map[string]interface{}) + pool, ok := i.(map[string]interface{}) + if !ok { + continue + } totalCapacity := int64(pool["totalCapacity"].(float64)) usedCapacity := int64(pool["usedCapacity"].(float64)) @@ -230,5 +214,8 @@ func (p *FusionStoragePlugin) getNewClientConfig(ctx context.Context, config map newClientConfig.AccountName, _ = config["accountName"].(string) newClientConfig.ParallelNum, _ = config["maxClientThreads"].(string) + newClientConfig.UseCert, _ = config["useCert"].(bool) + newClientConfig.CertSecretMeta, _ = config["certSecret"].(string) + return newClientConfig, nil } diff --git a/csi/backend/plugin/oceanstor-dtree.go b/csi/backend/plugin/oceanstor-dtree.go index e1aa554b..ba9ae7b7 100644 --- a/csi/backend/plugin/oceanstor-dtree.go +++ b/csi/backend/plugin/oceanstor-dtree.go @@ -21,6 +21,7 @@ import ( "errors" "fmt" + "huawei-csi-driver/pkg/constants" "huawei-csi-driver/storage/oceanstor/client" "huawei-csi-driver/storage/oceanstor/volume" "huawei-csi-driver/utils" @@ -175,7 +176,11 @@ func (p *OceanstorDTreePlugin) Validate(ctx context.Context, param map[string]in } // Login verification - cli := client.NewClient(clientConfig) + cli, err := client.NewClient(clientConfig) + if err != nil { + return err + } + err = cli.ValidateLogin(ctx) if err != nil { return err @@ -252,6 +257,9 @@ func (p *OceanstorDTreePlugin) UpdateBackendCapabilities() (map[string]interface return nil, nil, err } + // close dTree pvc label switch + capabilities[string(constants.SupportLabel)] = false + err = p.updateNFS4Capability(capabilities) if err != nil { return nil, nil, err diff --git a/csi/backend/plugin/oceanstor-nas.go b/csi/backend/plugin/oceanstor-nas.go index 40486ac5..6cc99bca 100644 --- a/csi/backend/plugin/oceanstor-nas.go +++ b/csi/backend/plugin/oceanstor-nas.go @@ -28,10 +28,10 @@ import ( ) const ( - HYPER_METRO_VSTORE_PAIR_ACTIVE = "0" - HYPER_METRO_VSTORE_PAIR_LINK_STATUS_CONNECTED = "1" - HYPER_METRO_DOMAIN_ACTIVE = "1" - HYPER_METRO_DOMAIN_RUNNING_STATUS_NORMAL = "0" + HyperMetroVstorePairActive = "0" + HyperMetroVstorePairLinkStatusConnected = "1" + HyperMetroDomainActive = "1" + HyperMetroDomainRunningStatusNormal = "0" ConsistentSnapshotsSpecification = "128" ) @@ -73,7 +73,11 @@ func (p *OceanstorNasPlugin) Init(config, parameters map[string]interface{}, kee return err } - p.portal = portals[0].(string) + p.portal, exist = portals[0].(string) + if !exist { + return errors.New("portals must be string") + } + p.vStorePairID, exist = config["metrovStorePairID"].(string) if exist { log.Infof("The metro vStorePair ID is %s", p.vStorePairID) @@ -152,12 +156,12 @@ func (p *OceanstorNasPlugin) UpdatePoolCapabilities(poolNames []string) (map[str return p.updatePoolCapabilities(poolNames, "2") } -func (p *OceanstorNasPlugin) UpdateReplicaRemotePlugin(remote Plugin) { - p.replicaRemotePlugin = remote.(*OceanstorNasPlugin) -} - func (p *OceanstorNasPlugin) UpdateMetroRemotePlugin(remote Plugin) { - p.metroRemotePlugin = remote.(*OceanstorNasPlugin) + var ok bool + p.metroRemotePlugin, ok = remote.(*OceanstorNasPlugin) + if !ok { + log.Warningf("convert metroRemotePlugin to OceanstorNasPlugin failed, data: %v", remote) + } } func (p *OceanstorNasPlugin) CreateSnapshot(ctx context.Context, @@ -235,6 +239,7 @@ func (p *OceanstorNasPlugin) updateHyperMetroCapability(capabilities map[string] return err } + var ok bool if p.product == "DoradoV6" && vStorePair != nil { fsHyperMetroDomain, err := p.cli.GetFSHyperMetroDomain(context.Background(), vStorePair["DOMAINNAME"].(string)) @@ -243,21 +248,24 @@ func (p *OceanstorNasPlugin) updateHyperMetroCapability(capabilities map[string] } if fsHyperMetroDomain == nil || - fsHyperMetroDomain["RUNNINGSTATUS"] != HYPER_METRO_DOMAIN_RUNNING_STATUS_NORMAL { + fsHyperMetroDomain["RUNNINGSTATUS"] != HyperMetroDomainRunningStatusNormal { capabilities["SupportMetro"] = false return nil } p.nasHyperMetro = volume.NASHyperMetro{ - FsHyperMetroActiveSite: fsHyperMetroDomain["CONFIGROLE"] == HYPER_METRO_DOMAIN_ACTIVE, + FsHyperMetroActiveSite: fsHyperMetroDomain["CONFIGROLE"] == HyperMetroDomainActive, LocVStoreID: vStorePair["LOCALVSTOREID"].(string), RmtVStoreID: vStorePair["REMOTEVSTOREID"].(string), } - p.metroDomainID = vStorePair["DOMAINID"].(string) + p.metroDomainID, ok = vStorePair["DOMAINID"].(string) + if !ok { + return fmt.Errorf("convert DOMAINID: %v to string failed", vStorePair["DOMAINID"]) + } } else { if vStorePair == nil || - vStorePair["ACTIVEORPASSIVE"] != HYPER_METRO_VSTORE_PAIR_ACTIVE || - vStorePair["LINKSTATUS"] != HYPER_METRO_VSTORE_PAIR_LINK_STATUS_CONNECTED || + vStorePair["ACTIVEORPASSIVE"] != HyperMetroVstorePairActive || + vStorePair["LINKSTATUS"] != HyperMetroVstorePairLinkStatusConnected || vStorePair["LOCALVSTORENAME"] != p.cli.GetvStoreName() { capabilities["SupportMetro"] = false } @@ -374,7 +382,11 @@ func (p *OceanstorNasPlugin) Validate(ctx context.Context, param map[string]inte } // Login verification - cli := client.NewClient(clientConfig) + cli, err := client.NewClient(clientConfig) + if err != nil { + return err + } + err = cli.ValidateLogin(ctx) if err != nil { return err diff --git a/csi/backend/plugin/oceanstor-san.go b/csi/backend/plugin/oceanstor-san.go index 824559ef..6f8eb070 100644 --- a/csi/backend/plugin/oceanstor-san.go +++ b/csi/backend/plugin/oceanstor-san.go @@ -184,7 +184,10 @@ func (p *OceanstorSanPlugin) isHyperMetro(ctx context.Context, lun map[string]in } func (p *OceanstorSanPlugin) metroHandler(ctx context.Context, req handlerRequest) ([]reflect.Value, error) { - localLunID := req.lun["ID"].(string) + localLunID, ok := req.lun["ID"].(string) + if !ok { + log.AddContext(ctx).Warningf("req.lun[\"ID\"] is not string") + } pair, err := req.localCli.GetHyperMetroPairByLocalObjID(ctx, localLunID) if err != nil { return nil, err @@ -210,7 +213,10 @@ func (p *OceanstorSanPlugin) metroHandler(ctx context.Context, req handlerReques "csi", p.metroRemotePlugin.portals, p.metroRemotePlugin.alua) metroAttacher := attacher.NewMetroAttacher(localAttacher, remoteAttacher, p.protocol) - lunName := req.lun["NAME"].(string) + lunName, ok := req.lun["NAME"].(string) + if !ok { + log.AddContext(ctx).Warningf("req.lun[\"NAME\"] is not string") + } out := utils.ReflectCall(metroAttacher, req.method, ctx, lunName, req.parameters) return out, nil @@ -363,12 +369,12 @@ func (p *OceanstorSanPlugin) UpdatePoolCapabilities(poolNames []string) (map[str return p.updatePoolCapabilities(poolNames, "1") } -func (p *OceanstorSanPlugin) UpdateReplicaRemotePlugin(remote Plugin) { - p.replicaRemotePlugin = remote.(*OceanstorSanPlugin) -} - func (p *OceanstorSanPlugin) UpdateMetroRemotePlugin(remote Plugin) { - p.metroRemotePlugin = remote.(*OceanstorSanPlugin) + var ok bool + p.metroRemotePlugin, ok = remote.(*OceanstorSanPlugin) + if !ok { + log.Warningf("convert metroRemotePlugin to OceanstorSanPlugin failed, data: %v", remote) + } } func (p *OceanstorSanPlugin) CreateSnapshot(ctx context.Context, @@ -491,7 +497,11 @@ func (p *OceanstorSanPlugin) Validate(ctx context.Context, param map[string]inte } // Login verification - cli := client.NewClient(clientConfig) + cli, err := client.NewClient(clientConfig) + if err != nil { + return err + } + err = cli.ValidateLogin(ctx) if err != nil { return err diff --git a/csi/backend/plugin/oceanstor.go b/csi/backend/plugin/oceanstor.go index 07029379..4e0d9bcf 100644 --- a/csi/backend/plugin/oceanstor.go +++ b/csi/backend/plugin/oceanstor.go @@ -23,6 +23,8 @@ import ( "strconv" "strings" + "huawei-csi-driver/csi/app" + "huawei-csi-driver/pkg/constants" pkgUtils "huawei-csi-driver/pkg/utils" "huawei-csi-driver/storage/oceanstor/client" "huawei-csi-driver/storage/oceanstor/clientv6" @@ -51,9 +53,12 @@ func (p *OceanstorPlugin) init(config map[string]interface{}, keepLogin bool) er return err } - cli := client.NewClient(backendClientConfig) - err = cli.Login(context.Background()) + cli, err := client.NewClient(backendClientConfig) if err != nil { + return err + } + + if err = cli.Login(context.Background()); err != nil { log.Errorf("plugin init login failed, err: %v", err) return err } @@ -73,10 +78,14 @@ func (p *OceanstorPlugin) init(config map[string]interface{}, keepLogin bool) er cli.Logout(context.Background()) } - if p.product == utils.OceanStorDoradoV6 { - clientV6 := clientv6.NewClientV6(backendClientConfig) + if p.product == constants.OceanStorDoradoV6 { + clientV6, err := clientv6.NewClientV6(backendClientConfig) + if err != nil { + return err + } + cli.Logout(context.Background()) - err := p.switchClient(clientV6) + err = p.switchClient(clientV6) if err != nil { return err } @@ -120,6 +129,10 @@ func (p *OceanstorPlugin) formatInitParam(config map[string]interface{}) (res *c } res.VstoreName, _ = config["vstoreName"].(string) res.ParallelNum, _ = config["maxClientThreads"].(string) + + res.UseCert, _ = config["useCert"].(bool) + res.CertSecretMeta, _ = config["certSecret"].(string) + return } @@ -140,6 +153,9 @@ func (p *OceanstorPlugin) updateBackendCapabilities() (map[string]interface{}, e supportReplication := utils.IsSupportFeature(features, "HyperReplication") supportClone := utils.IsSupportFeature(features, "HyperClone") || utils.IsSupportFeature(features, "HyperCopy") supportApplicationType := p.product == "DoradoV6" + supportLabel := app.GetGlobalConfig().EnableLabel && p.cli.GetStorageVersion() >= constants.MinVersionSupportLabel + + log.Debugf("enableLabel: %v, storageVersion: %v", app.GetGlobalConfig().EnableLabel, p.cli.GetStorageVersion()) capabilities := map[string]interface{}{ "SupportThin": supportThin, @@ -150,6 +166,7 @@ func (p *OceanstorPlugin) updateBackendCapabilities() (map[string]interface{}, e "SupportApplicationType": supportApplicationType, "SupportClone": supportClone, "SupportMetroNAS": supportMetroNAS, + "SupportLabel": supportLabel, } return capabilities, nil @@ -164,7 +181,10 @@ func (p *OceanstorPlugin) getRemoteDevices() (string, error) { var devicesSN []string for _, dev := range devices { - deviceSN := dev["SN"].(string) + deviceSN, ok := dev["SN"].(string) + if !ok { + continue + } devicesSN = append(devicesSN, deviceSN) } return strings.Join(devicesSN, ";"), nil @@ -187,14 +207,15 @@ func (p *OceanstorPlugin) updateBackendSpecifications() (map[string]interface{}, func (p *OceanstorPlugin) UpdateBackendCapabilities() (map[string]interface{}, map[string]interface{}, error) { capabilities, err := p.updateBackendCapabilities() if err != nil { + log.Errorf("updateBackendCapabilities failed, err: %v", err) return nil, nil, err } specifications, err := p.updateBackendSpecifications() if err != nil { + log.Errorf("updateBackendSpecifications failed, err: %v", err) return nil, nil, err } - p.capabilities = capabilities return capabilities, specifications, nil } @@ -290,8 +311,14 @@ func (p *OceanstorPlugin) analyzePoolsCapacity(pools []map[string]interface{}) m capabilities := make(map[string]interface{}) for _, pool := range pools { - name := pool["NAME"].(string) - freeCapacity, _ := strconv.ParseInt(pool["USERFREECAPACITY"].(string), 10, 64) + name, ok := pool["NAME"].(string) + if !ok { + continue + } + freeCapacity, err := strconv.ParseInt(pool["USERFREECAPACITY"].(string), 10, 64) + if err != nil { + log.Warningf("analysisPoolsCapacity parseInt failed, data: %v, err: %v", pool["USERFREECAPACITY"], err) + } capabilities[name] = map[string]interface{}{ "FreeCapacity": freeCapacity * 512, @@ -341,15 +368,13 @@ func (p *OceanstorPlugin) getNewClientConfig(ctx context.Context, param map[stri configUrls, exist := param["urls"].([]interface{}) if !exist || len(configUrls) <= 0 { msg := fmt.Sprintf("Verify urls: [%v] failed. urls must be provided.", param["urls"]) - log.AddContext(ctx).Errorln(msg) - return data, errors.New(msg) + return data, pkgUtils.Errorln(ctx, msg) } for _, configUrl := range configUrls { url, ok := configUrl.(string) if !ok { msg := fmt.Sprintf("Verify url: [%v] failed. url convert to string failed.", configUrl) - log.AddContext(ctx).Errorln(msg) - return data, errors.New(msg) + return data, pkgUtils.Errorln(ctx, msg) } data.Urls = append(data.Urls, url) } @@ -362,23 +387,20 @@ func (p *OceanstorPlugin) getNewClientConfig(ctx context.Context, param map[stri data.User, exist = param["user"].(string) if !exist { msg := fmt.Sprintf("Verify user: [%v] failed. user must be provided.", data.User) - log.AddContext(ctx).Errorln(msg) - return data, errors.New(msg) + return data, pkgUtils.Errorln(ctx, msg) } data.SecretName, exist = param["secretName"].(string) if !exist { msg := fmt.Sprintf("Verify SecretName: [%v] failed. SecretName must be provided.", data.SecretName) - log.AddContext(ctx).Errorln(msg) - return data, errors.New(msg) + return data, pkgUtils.Errorln(ctx, msg) } data.SecretNamespace, exist = param["secretNamespace"].(string) if !exist { msg := fmt.Sprintf("Verify SecretNamespace: [%v] failed. SecretNamespace must be provided.", data.SecretNamespace) - log.AddContext(ctx).Errorln(msg) - return data, errors.New(msg) + return data, pkgUtils.Errorln(ctx, msg) } data.BackendID, exist = param["backendID"].(string) @@ -391,5 +413,8 @@ func (p *OceanstorPlugin) getNewClientConfig(ctx context.Context, param map[stri data.VstoreName, _ = param["vstoreName"].(string) data.ParallelNum, _ = param["maxClientThreads"].(string) + data.UseCert, _ = param["useCert"].(bool) + data.CertSecretMeta, _ = param["certSecret"].(string) + return data, nil } diff --git a/csi/backend/plugin/plugin.go b/csi/backend/plugin/plugin.go index 5bd5414e..0cccf878 100644 --- a/csi/backend/plugin/plugin.go +++ b/csi/backend/plugin/plugin.go @@ -36,7 +36,6 @@ type Plugin interface { UpdateBackendCapabilities() (map[string]interface{}, map[string]interface{}, error) UpdatePoolCapabilities([]string) (map[string]interface{}, error) UpdateMetroRemotePlugin(Plugin) - UpdateReplicaRemotePlugin(Plugin) CreateSnapshot(context.Context, string, string) (map[string]interface{}, error) DeleteSnapshot(context.Context, string, string) error SmartXQoSQuery @@ -88,6 +87,3 @@ func (p *basePlugin) DetachVolume(context.Context, string, map[string]interface{ func (p *basePlugin) UpdateMetroRemotePlugin(Plugin) { } - -func (p *basePlugin) UpdateReplicaRemotePlugin(Plugin) { -} diff --git a/csi/driver/controller.go b/csi/driver/controller.go index 6fe8f7ad..71fb5015 100644 --- a/csi/driver/controller.go +++ b/csi/driver/controller.go @@ -30,6 +30,7 @@ import ( "huawei-csi-driver/csi/app" "huawei-csi-driver/csi/backend" "huawei-csi-driver/csi/backend/plugin" + pkgUtils "huawei-csi-driver/pkg/utils" "huawei-csi-driver/utils" "huawei-csi-driver/utils/log" ) @@ -72,28 +73,35 @@ func (d *Driver) DeleteVolume(ctx context.Context, req *csi.DeleteVolumeRequest) log.AddContext(ctx).Infof("Start to delete volume %s", volumeId) backendName, volName := utils.SplitVolumeId(volumeId) - backend := backend.GetBackendWithFresh(ctx, backendName, true) - if backend == nil { + + b := backend.GetBackendWithFresh(ctx, backendName, true) + if b == nil { log.AddContext(ctx).Warningf("Backend %s doesn't exist. Ignore this request and return success. "+ "CAUTION: volume need to manually delete from array.", backendName) return &csi.DeleteVolumeResponse{}, nil } var err error - if backend.Storage == plugin.DTreeStorage { - err = backend.Plugin.DeleteDTreeVolume(ctx, map[string]interface{}{ - "parentname": backend.Parameters["parentname"], + if b.Storage == plugin.DTreeStorage { + err = b.Plugin.DeleteDTreeVolume(ctx, map[string]interface{}{ + "parentname": b.Parameters["parentname"], "name": volName, }) } else { - err = backend.Plugin.DeleteVolume(ctx, volName) + err = b.Plugin.DeleteVolume(ctx, volName) } + if err != nil { log.AddContext(ctx).Errorf("Delete volume %s error: %v", volumeId, err) return nil, status.Error(codes.Internal, err.Error()) } log.AddContext(ctx).Infof("Volume %s is deleted", volumeId) + + // Delete the topology after the volume is successfully deleted. + // This prevents the DeleteLabel function from being repeatedly invoked when the volume fails to be deleted. + go pkgUtils.DeletePVLabel(volumeId) + return &csi.DeleteVolumeResponse{}, nil } diff --git a/csi/driver/controller_helper.go b/csi/driver/controller_helper.go index 5d62f521..81a29963 100644 --- a/csi/driver/controller_helper.go +++ b/csi/driver/controller_helper.go @@ -31,6 +31,7 @@ import ( "huawei-csi-driver/cli/helper" "huawei-csi-driver/csi/backend" "huawei-csi-driver/pkg/constants" + pkgUtils "huawei-csi-driver/pkg/utils" "huawei-csi-driver/utils" "huawei-csi-driver/utils/log" ) @@ -215,7 +216,8 @@ func processVolumeContentSource(ctx context.Context, req *csi.CreateVolumeReques parameters["sourceSnapshotName"] = sourceSnapshotName parameters["snapshotParentId"] = snapshotParentId parameters["backend"] = sourceBackendName - log.AddContext(ctx).Infof("Start to create volume from snapshot %s", sourceSnapshotName) + log.AddContext(ctx).Infof("Start to create volume from snapshot %s, param: %+v", + sourceSnapshotName, parameters) } else if contentVolume := contentSource.GetVolume(); contentVolume != nil { sourceVolumeId := contentVolume.GetVolumeId() sourceBackendName, sourceVolumeName := utils.SplitVolumeId(sourceVolumeId) @@ -434,6 +436,7 @@ func processCreateVolumeParametersAfterSelect(parameters map[string]interface{}, parameters["accountName"] = backend.GetAccountName(localPool.Parent) } +// createVolume used to create a lun/filesystem in huawei storage func (d *Driver) createVolume(ctx context.Context, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) { parameters, err := processCreateVolumeParameters(ctx, req) if err != nil { @@ -455,9 +458,14 @@ func (d *Driver) createVolume(ctx context.Context, req *csi.CreateVolumeRequest) } log.AddContext(ctx).Infof("Volume %s is created", req.GetName()) - return &csi.CreateVolumeResponse{ + res := &csi.CreateVolumeResponse{ Volume: makeCreateVolumeResponse(ctx, req, vol, localPool), - }, nil + } + + // The topology creation result does not affect current task. + go pkgUtils.CreatePVLabel(req.GetName(), res.GetVolume().GetVolumeId()) + + return res, nil } // In the volume import scenario, only the fields in the annotation are obtained. @@ -465,9 +473,10 @@ func (d *Driver) createVolume(ctx context.Context, req *csi.CreateVolumeRequest) func (d *Driver) manageVolume(ctx context.Context, req *csi.CreateVolumeRequest, volumeName, backendName string) ( *csi.CreateVolumeResponse, error) { log.AddContext(ctx).Infof("Start to manage Volume %s for backend %s.", volumeName, backendName) - selectBackend := backend.GetBackendWithFresh(ctx, backendName, true) + selectBackend := backend.GetBackendWithFresh(ctx, helper.GetBackendName(backendName), true) if selectBackend == nil { - log.AddContext(ctx).Errorf("Backend %s doesn't exist. Manage Volume %s failed.", backendName, volumeName) + log.AddContext(ctx).Errorf("Backend %s doesn't exist. Manage Volume %s failed.", + helper.GetBackendName(backendName), volumeName) return &csi.CreateVolumeResponse{}, fmt.Errorf("backend %s doesn't exist. Manage Volume %s failed", backendName, volumeName) } @@ -504,10 +513,16 @@ func (d *Driver) manageVolume(ctx context.Context, req *csi.CreateVolumeRequest, attributes := getAttributes(req, vol, backendName) log.AddContext(ctx).Infof("Volume %s is created by manage", req.GetName()) - return &csi.CreateVolumeResponse{ + + res := &csi.CreateVolumeResponse{ Volume: getVolumeResponse(accessibleTopologies, attributes, backendName+"."+volumeName, req.GetCapacityRange().GetRequiredBytes()), - }, nil + } + + // The topology creation result does not affect current task. + go pkgUtils.CreatePVLabel(req.GetName(), res.GetVolume().GetVolumeId()) + + return res, nil } func validateCapacity(ctx context.Context, req *csi.CreateVolumeRequest, vol utils.Volume) error { diff --git a/csi/driver/controller_helper_test.go b/csi/driver/controller_helper_test.go index 817699a4..e998b4c6 100644 --- a/csi/driver/controller_helper_test.go +++ b/csi/driver/controller_helper_test.go @@ -107,9 +107,23 @@ func initDriver() *Driver { app.GetGlobalConfig().NodeName) } +func initPool(poolName string) *backend.StoragePool { + return &backend.StoragePool{ + Name: poolName, + Storage: "oceanstor-nas", + Parent: "fake-bakcend", + Capabilities: map[string]interface{}{}, + Plugin: plugin.GetPlugin("oceanstor-nas"), + } +} + func TestCreateVolumeWithoutBackend(t *testing.T) { driver := initDriver() req := mockCreateRequest() + + s := gostub.StubFunc(&pkgUtils.CreatePVLabel) + defer s.Reset() + m := gomonkey.ApplyFunc(pkgUtils.ListClaim, func(ctx context.Context, client clientSet.Interface, namespace string) ( *xuanwuv1.StorageBackendClaimList, error) { @@ -123,26 +137,20 @@ func TestCreateVolumeWithoutBackend(t *testing.T) { } } -func initPool(poolName string) *backend.StoragePool { - return &backend.StoragePool{ - Name: poolName, - Storage: "oceanstor-nas", - Parent: "fake-bakcend", - Capabilities: map[string]interface{}{}, - Plugin: plugin.GetPlugin("oceanstor-nas"), - } -} - func TestCreateVolume(t *testing.T) { localPool := initPool("local-pool") - gostub.StubFunc(&backend.SelectStoragePool, localPool, nil, nil) + + s := gostub.StubFunc(&backend.SelectStoragePool, localPool, nil, nil) + defer s.Reset() + + s.StubFunc(&pkgUtils.CreatePVLabel) plg := plugin.GetPlugin("oceanstor-nas") - createPatch := gomonkey.ApplyMethod(reflect.TypeOf(plg), "CreateVolume", + m := gomonkey.ApplyMethod(reflect.TypeOf(plg), "CreateVolume", func(*plugin.OceanstorNasPlugin, context.Context, string, map[string]interface{}) (utils.Volume, error) { return utils.NewVolume("fake-nfs"), nil }) - defer createPatch.Reset() + defer m.Reset() driver := initDriver() req := mockCreateRequest() @@ -152,13 +160,15 @@ func TestCreateVolume(t *testing.T) { } } -func TestImportVolumeWithOutBackend(t *testing.T) { +func TestImportVolumeWithoutBackend(t *testing.T) { driver := initDriver() req := mockCreateRequest() s := gostub.StubFunc(&backend.GetBackendWithFresh, nil) defer s.Reset() + s.StubFunc(&pkgUtils.CreatePVLabel) + _, err := driver.manageVolume(context.TODO(), req, "fake-nfs", "fake-backend") if err == nil { t.Error("test import without backend failed") @@ -168,19 +178,23 @@ func TestImportVolumeWithOutBackend(t *testing.T) { func TestImportVolume(t *testing.T) { plg := plugin.GetPlugin("oceanstor-nas") localPool := initPool("local-pool") - gostub.StubFunc(&backend.GetBackendWithFresh, &backend.Backend{ + + s := gostub.StubFunc(&backend.GetBackendWithFresh, &backend.Backend{ Name: "fake-backend", Plugin: plg, Pools: []*backend.StoragePool{localPool}, }) + defer s.Reset() - queryPatch := gomonkey.ApplyMethod(reflect.TypeOf(plg), "QueryVolume", + s.StubFunc(&pkgUtils.CreatePVLabel) + + m := gomonkey.ApplyMethod(reflect.TypeOf(plg), "QueryVolume", func(*plugin.OceanstorNasPlugin, context.Context, string, map[string]interface{}) (utils.Volume, error) { vol := utils.NewVolume("fake-nfs") vol.SetSize(1024 * 1024 * 1024) return vol, nil }) - defer queryPatch.Reset() + defer m.Reset() driver := initDriver() req := mockCreateRequest() diff --git a/csi/driver/node.go b/csi/driver/node.go index 4d0b16ec..fdeff87e 100644 --- a/csi/driver/node.go +++ b/csi/driver/node.go @@ -23,6 +23,10 @@ import ( "strings" "time" + "github.com/container-storage-interface/spec/lib/go/csi" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "huawei-csi-driver/connector" _ "huawei-csi-driver/connector/nfs" // init the nfs connector "huawei-csi-driver/csi/app" @@ -30,10 +34,6 @@ import ( "huawei-csi-driver/pkg/constants" "huawei-csi-driver/utils" "huawei-csi-driver/utils/log" - - "github.com/container-storage-interface/spec/lib/go/csi" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" ) func (d *Driver) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRequest) (*csi.NodeStageVolumeResponse, error) { @@ -102,12 +102,16 @@ func (d *Driver) NodePublishVolume(ctx context.Context, log.AddContext(ctx).Errorf("publish block volume fail, volume: %s, error: %v", volumeId, err) return nil, status.Error(codes.Internal, err.Error()) } - return &csi.NodePublishVolumeResponse{}, nil - } - if err := manage.PublishFilesystem(ctx, req); err != nil { - log.AddContext(ctx).Errorf("publish filesystem volume fail, volume: %s, error: %v", volumeId, err) - return nil, status.Error(codes.Internal, err.Error()) + } else { + if err := manage.PublishFilesystem(ctx, req); err != nil { + log.AddContext(ctx).Errorf("publish filesystem volume fail, volume: %s, error: %v", volumeId, err) + return nil, status.Error(codes.Internal, err.Error()) + } } + + go nodeAddLabel(utils.NewContext(), volumeId, targetPath) + + log.AddContext(ctx).Infof("Volume %s is node published from %s", volumeId, targetPath) return &csi.NodePublishVolumeResponse{}, nil } @@ -146,6 +150,9 @@ func (d *Driver) NodeUnpublishVolume(ctx context.Context, } } } + + go nodeDeleteLabel(utils.NewContext(), volumeId, targetPath) + log.AddContext(ctx).Infof("Volume %s is node unpublished from %s", volumeId, targetPath) return &csi.NodeUnpublishVolumeResponse{}, nil } diff --git a/csi/driver/node_helper.go b/csi/driver/node_helper.go new file mode 100644 index 00000000..aed6f24c --- /dev/null +++ b/csi/driver/node_helper.go @@ -0,0 +1,296 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package driver + +import ( + "context" + "errors" + k8sError "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "strings" + + xuanwuv1 "huawei-csi-driver/client/apis/xuanwu/v1" + "huawei-csi-driver/csi/app" + "huawei-csi-driver/pkg/constants" + pkgUtils "huawei-csi-driver/pkg/utils" + "huawei-csi-driver/utils" + "huawei-csi-driver/utils/log" +) + +func nodeAddLabel(ctx context.Context, volumeID, targetPath string) { + backendName, _ := utils.SplitVolumeId(volumeID) + backendName = pkgUtils.MakeMetaWithNamespace(app.GetGlobalConfig().Namespace, backendName) + + supportLabel, err := pkgUtils.IsBackendCapabilitySupport(ctx, backendName, constants.SupportLabel) + if err != nil { + log.AddContext(ctx).Errorf("IsBackendCapabilitySupport failed, backendName: %v, label: %v, err: %v", + backendName, supportLabel, err) + } + if supportLabel { + if err := addLabel(ctx, volumeID, targetPath); err != nil { + log.AddContext(ctx).Errorf("nodeAddLabel failed, err: %v", err) + } + } +} + +func nodeDeleteLabel(ctx context.Context, volumeID, targetPath string) { + backendName, _ := utils.SplitVolumeId(volumeID) + backendName = pkgUtils.MakeMetaWithNamespace(app.GetGlobalConfig().Namespace, backendName) + + supportLabel, err := pkgUtils.IsBackendCapabilitySupport(ctx, backendName, constants.SupportLabel) + if err != nil { + log.AddContext(ctx).Errorf("IsBackendCapabilitySupport failed, backendName: %v, label: %v, err: %v", + backendName, supportLabel, err) + } + if supportLabel { + if err := deleteLabel(ctx, volumeID, targetPath); err != nil { + log.AddContext(ctx).Errorf("nodeDeleteLabel failed, err: %v", err) + } + } +} + +func addLabel(ctx context.Context, volumeID, targetPath string) error { + _, volumeName := utils.SplitVolumeId(volumeID) + topoName := pkgUtils.GetTopoName(volumeName) + podName, namespace, sc, pvName, err := getTargetPathPodRelateInfo(ctx, targetPath) + if err != nil { + log.AddContext(ctx).Errorf("get podName failed, pvName: %v targetPath: %v err: %v", + pvName, targetPath, err) + return err + } + if podName == "" { + log.AddContext(ctx).Errorf("get podName failed, target pod not exist, targetPath: %v err: %v", + targetPath, err) + return err + } + if sc == "" { + log.AddContext(ctx).Infof("addLabel static pv, volumeID: %v, targetPath: %v", volumeID, targetPath) + return nil + } + + topo, err := app.GetGlobalConfig().BackendUtils.XuanwuV1().ResourceTopologies().Get(ctx, + topoName, metav1.GetOptions{}) + log.AddContext(ctx).Debugf("get topo info, topo: %+v, err: %v, notFound: %v", topo, + err, k8sError.IsNotFound(err)) + if k8sError.IsNotFound(err) { + _, err = app.GetGlobalConfig().BackendUtils.XuanwuV1().ResourceTopologies().Create(ctx, + formatTopologies(volumeID, namespace, podName, pvName, topoName), + metav1.CreateOptions{}) + if err != nil { + log.AddContext(ctx).Errorf("create label failed, data: %+v volumeName: %v targetPath: %v", + topo, volumeName, targetPath) + return err + } + log.AddContext(ctx).Infof("node create label success, data: %+v volumeName: %v targetPath: %v", + topo, volumeName, targetPath) + return nil + } + if err != nil { + log.AddContext(ctx).Errorf("get topo failed, topoName: %v err: %v", topoName, err) + return err + } + + // if pvc label not exist then add + // add new topo item + addPodTopoItem(&topo.Spec.Tags, podName, namespace, volumeName) + + // update topo + _, err = app.GetGlobalConfig().BackendUtils.XuanwuV1().ResourceTopologies().Update(ctx, + topo, metav1.UpdateOptions{}) + if err != nil { + log.AddContext(ctx).Errorf("node add label failed, data:%+v, volumeName: %v targetPath: %v err: %v", + topo, volumeName, targetPath, err) + return err + } + + log.AddContext(ctx).Infof("node add label success, data: %+v volumeName: %v targetPath: %v", + topo, volumeName, targetPath) + return nil +} + +func formatTopologies(volumeID, namespace, podName, pvName, topoName string) *xuanwuv1.ResourceTopology { + topologySpec := xuanwuv1.ResourceTopologySpec{ + Provisioner: constants.DefaultTopoDriverName, + VolumeHandle: volumeID, + Tags: []xuanwuv1.Tag{ + { + ResourceInfo: xuanwuv1.ResourceInfo{ + TypeMeta: metav1.TypeMeta{Kind: constants.PVKind, APIVersion: constants.KubernetesV1}, + Name: pvName, + }, + }, + { + ResourceInfo: xuanwuv1.ResourceInfo{ + TypeMeta: metav1.TypeMeta{Kind: constants.PodKind, APIVersion: constants.KubernetesV1}, + Name: podName, + Namespace: namespace, + }, + }, + }, + } + return &xuanwuv1.ResourceTopology{ + TypeMeta: metav1.TypeMeta{Kind: constants.TopologyKind, APIVersion: constants.XuanwuV1}, + ObjectMeta: metav1.ObjectMeta{Name: topoName}, + Spec: topologySpec, + } +} + +func addPodTopoItem(tags *[]xuanwuv1.Tag, podName, namespace, volumeName string) { + // if pvc label not exist then add + var existPvLabel bool + for _, tag := range *tags { + if tag.Kind == constants.PVKind { + existPvLabel = true + break + } + } + if !existPvLabel { + *tags = append(*tags, xuanwuv1.Tag{ + ResourceInfo: xuanwuv1.ResourceInfo{ + TypeMeta: metav1.TypeMeta{Kind: constants.PVKind, APIVersion: constants.KubernetesV1}, + Name: volumeName, + }, + }) + } + + // add pod label + *tags = append(*tags, xuanwuv1.Tag{ + ResourceInfo: xuanwuv1.ResourceInfo{ + TypeMeta: metav1.TypeMeta{Kind: constants.PodKind, APIVersion: constants.KubernetesV1}, + Name: podName, + Namespace: namespace, + }, + }) +} + +func deleteLabel(ctx context.Context, volumeID, targetPath string) error { + _, volumeName := utils.SplitVolumeId(volumeID) + topoName := pkgUtils.GetTopoName(volumeName) + podName, namespace, sc, pvName, err := getTargetPathPodRelateInfo(ctx, targetPath) + if err != nil { + log.AddContext(ctx).Errorf("get targetPath pvRelateInfo failed, targetPath: %v, err: %v", targetPath, err) + return err + } + + if sc == "" { + log.AddContext(ctx).Infof("deleteLabel static pv, volumeID: %v, targetPath: %v", volumeID, targetPath) + return nil + } + + // get topo label + topo, err := app.GetGlobalConfig().BackendUtils.XuanwuV1().ResourceTopologies().Get(ctx, topoName, + metav1.GetOptions{}) + if k8sError.IsNotFound(err) { + log.AddContext(ctx).Infof("node delete label success, topo not found, "+ + "data: %+v pvName: %v targetPath: %v", topo, pvName, targetPath) + return nil + } + if err != nil { + log.AddContext(ctx).Errorf("get topo failed, topoName: %v, err: %v", topoName, err) + return err + } + if topo == nil { + log.AddContext(ctx).Errorf("get nil topo, topoName: %v", topoName) + return errors.New("topo is nil") + } + + // filter pod + topo.Spec.Tags = filterDeleteTopoTag(ctx, topo.Spec.Tags, podName, namespace) + + // update topo + _, err = app.GetGlobalConfig().BackendUtils.XuanwuV1().ResourceTopologies().Update(ctx, + topo, metav1.UpdateOptions{}) + if err != nil { + log.AddContext(ctx).Errorf("node add label failed, data:%+v, volumeName: %v targetPath: %v err: %v", + topo, volumeName, targetPath, err) + return err + } + + log.AddContext(ctx).Infof("node delete label success, data: %+v volumeName: %v targetPath: %v", + topo, volumeName, targetPath) + return nil +} + +func filterDeleteTopoTag(ctx context.Context, tags []xuanwuv1.Tag, currentPodName, namespace string) []xuanwuv1.Tag { + var newTags []xuanwuv1.Tag + for _, tag := range tags { + if tag.Kind != constants.PodKind { + newTags = append(newTags, tag) + continue + } + + if tag.Kind == constants.PodKind && tag.Name == currentPodName && tag.Namespace == namespace { + continue + } + + _, err := app.GetGlobalConfig().K8sUtils.GetPod(ctx, tag.Namespace, tag.Name) + if k8sError.IsNotFound(err) { + continue + } + if err != nil { + log.AddContext(ctx).Errorf("get pod failed, podInfo: %v, err: %v", tag, err) + } + + newTags = append(newTags, tag) + } + return newTags +} + +// getTargetPathPodRelateInfo get podName nameSpace sc volumeName +func getTargetPathPodRelateInfo(ctx context.Context, targetPath string) (string, string, string, string, error) { + k8sAPI := app.GetGlobalConfig().K8sUtils + + targetPathArr := strings.Split(targetPath, "/") + if len(targetPathArr) < 2 { + return "", "", "", "", pkgUtils.Errorf(ctx, "targetPath: %s is invalid", targetPath) + } + volumeName := targetPathArr[len(targetPathArr)-2] + + log.AddContext(ctx).Debugf("targetPath: %v, volumeName: %v", targetPath, volumeName) + + // get pv info + pv, err := k8sAPI.GetPVByName(ctx, volumeName) + if err != nil { + log.AddContext(ctx).Errorf("get pv failed, pvName: %v, err: %v", volumeName, err) + return "", "", "", volumeName, err + } + if pv == nil { + log.AddContext(ctx).Errorf("get nil pv, pvName: %v", volumeName) + return "", "", "", "", errors.New("pv is nil") + } + if pv.Spec.ClaimRef == nil { + log.AddContext(ctx).Errorf("get nil pv.Spec.ClaimRef, pvName: %v", volumeName) + return "", "", "", "", errors.New("pv.Spec.ClaimRef is nil") + } + + // get all pod in namespace + pods, err := k8sAPI.ListPods(ctx, pv.Spec.ClaimRef.Namespace) + if err != nil { + log.AddContext(ctx).Errorf("list pods failed, namespace: %v, err: %v", pv.Spec.ClaimRef.Namespace, err) + return "", "", "", "", err + } + + // get target pod name + log.AddContext(ctx).Debugf("getPodInfo podList: %+v, targetPath: %v", pods.Items, targetPath) + for _, pod := range pods.Items { + if strings.Contains(targetPath, string(pod.UID)) { + return pod.Name, pv.Spec.ClaimRef.Namespace, pv.Spec.StorageClassName, volumeName, nil + } + } + + return "", pv.Spec.ClaimRef.Namespace, pv.Spec.StorageClassName, volumeName, nil +} diff --git a/csi/main.go b/csi/main.go index cb6e04cc..f70abf46 100644 --- a/csi/main.go +++ b/csi/main.go @@ -18,9 +18,6 @@ package main import ( "context" - "encoding/json" - "fmt" - "io/ioutil" "net" "os" "os/signal" @@ -32,7 +29,6 @@ import ( "github.com/sirupsen/logrus" "google.golang.org/grpc" - "huawei-csi-driver/connector" "huawei-csi-driver/connector/host" connutils "huawei-csi-driver/connector/utils" "huawei-csi-driver/connector/utils/lock" @@ -48,13 +44,11 @@ import ( ) const ( - configFile = "/etc/huawei/csi.json" - secretFile = "/etc/huawei/secret/secret.json" versionFile = "/csi/version" controllerLogFile = "huawei-csi-controller" nodeLogFile = "huawei-csi-node" - csiVersion = "4.1.0" + csiVersion = "4.2.0" endpointDirPerm = 0755 ) @@ -71,73 +65,6 @@ type CSISecret struct { Secrets map[string]interface{} `json:"secrets"` } -func parseConfig() { - data, err := ioutil.ReadFile(configFile) - if err != nil { - notify.Stop("Read config file %s error: %v", configFile, err) - } - - err = json.Unmarshal(data, &config) - if err != nil { - notify.Stop("Unmarshal config file %s error: %v", configFile, err) - } - - if len(config.Backends) <= 0 { - notify.Stop("Must configure at least one backend") - } - - secretData, err := ioutil.ReadFile(secretFile) - if err != nil { - notify.Stop("Read config file %s error: %v", secretFile, err) - } - - err = json.Unmarshal(secretData, &secret) - if err != nil { - notify.Stop("Unmarshal config file %s error: %v", secretFile, err) - } - - err = mergeData(config, secret) - if err != nil { - notify.Stop("Merge configs error: %v", err) - } - - // nodeName flag is only considered for node plugin - if "" == app.GetGlobalConfig().NodeName && !app.GetGlobalConfig().Controller { - log.Warningln("Node name is empty. Topology aware volume provisioning feature may not behave normal") - } - - connector.ScanVolumeTimeout = time.Second * time.Duration(app.GetGlobalConfig().ScanVolumeTimeout) -} - -func getSecret(backendSecret, backendConfig map[string]interface{}, secretKey string) { - if secretValue, exist := backendSecret[secretKey].(string); exist { - backendConfig[secretKey] = secretValue - } else { - log.Fatalln(fmt.Sprintf("The key %s is not in secret %v.", secretKey, backendSecret)) - } -} - -func mergeData(config CSIConfig, secret CSISecret) error { - for _, backendConfig := range config.Backends { - backendName, exist := backendConfig["name"].(string) - if !exist { - return fmt.Errorf("the key name does not exist in backend") - } - Secret, exist := secret.Secrets[backendName] - if !exist { - return fmt.Errorf("the key %s is not in secret", backendName) - } - - backendSecret := Secret.(map[string]interface{}) - getSecret(backendSecret, backendConfig, "user") - getSecret(backendSecret, backendConfig, "password") - // 兼容之前的后端注册，后续删除所有相关代码 - backendConfig["secretName"] = backendName - backendConfig["secretNamespace"] = "huawei-csi" - } - return nil -} - func updateBackendCapabilities(ctx context.Context) { err := backend.RegisterAllBackend(ctx) if err != nil { diff --git a/csi/manage/san_manager.go b/csi/manage/san_manager.go index c71cd44e..167cc921 100644 --- a/csi/manage/san_manager.go +++ b/csi/manage/san_manager.go @@ -108,9 +108,9 @@ func (m *SanManager) UnStageVolume(ctx context.Context, req *csi.NodeUnstageVolu } // ExpandVolume return nil error if specified volume expand success -// If getting device wwn failed, return an error with call getDeviceWwn(). -// If the device expand failed according to the specified wwn, return an error with call connector.ResizeBlock(). -// If the volume capability is mount, will need to call connector.ResizeMountPath() +// If getting device wwn failed, return an error with call getDeviceWwn. +// If the device expand failed according to the specified wwn, return an error with call connector.ResizeBlock. +// If the volume capability is mount, will need to call connector.ResizeMountPath. func (m *SanManager) ExpandVolume(ctx context.Context, req *csi.NodeExpandVolumeRequest) error { capacityRange := req.GetCapacityRange() if capacityRange == nil || capacityRange.RequiredBytes <= 0 { @@ -182,7 +182,7 @@ func getDeviceWwn(ctx context.Context, volumeId, targetPath string, // information to the disk. if err = utils.WriteWWNFileIfNotExist(ctx, wwn, volumeId); err != nil { // If write wwn filed, there is nothing we can do and a retry is unlikely to help, because the mapping - // information doesn't exist in /proc/mount file, so the error with call utils.WriteWWNFileIfNotExist() + // information doesn't exist in /proc/mount file, so the error with call utils.WriteWWNFileIfNotExist // will not return log.AddContext(ctx).Warningf("write wwn file failed, wwn: %s, volumeId: %s error: %v", wwn, volumeId, err) diff --git a/csi/manage/san_manager_test.go b/csi/manage/san_manager_test.go index 3871ece3..12d3b330 100644 --- a/csi/manage/san_manager_test.go +++ b/csi/manage/san_manager_test.go @@ -316,7 +316,7 @@ func checkTargetMapContainsSourceMap(source, target map[string]interface{}) bool func clearWwnFileGeneratedByTest() { err := os.RemoveAll("/csi/disks/test_backend.pvc-san-xxx.wwn") if err != nil { - log.Errorf("clear wwn file generated by test failed, error: %v", err) + log.Errorln("clear wwn file generated by test failed") return } } diff --git a/csi/provider/backend.go b/csi/provider/backend.go index 623022ef..1512554a 100644 --- a/csi/provider/backend.go +++ b/csi/provider/backend.go @@ -37,8 +37,14 @@ func (p *Provider) AddStorageBackend(ctx context.Context, req *drcsi.AddStorageB log.AddContext(ctx).Infof("Start to add storage backend %s.", req.Name) defer log.AddContext(ctx).Infof("Finished to add storage backend %s.", req.Name) + useCert, certSecret, err := pkgUtils.GetCertMeta(ctx, req.Name) + if err != nil { + msg := fmt.Sprintf("GetCertMeta %s failed. error: %v", req.Name, err) + return nil, pkgUtils.Errorln(ctx, msg) + } + // backendId: / eg:huawei-csi/nfs-180 - backendId, err := backend.RegisterOneBackend(ctx, req.Name, req.ConfigmapMeta, req.SecretMeta) + backendId, err := backend.RegisterOneBackend(ctx, req.Name, req.ConfigmapMeta, req.SecretMeta, certSecret, useCert) if err != nil { msg := fmt.Sprintf("RegisterBackend %s failed, error %v", req.Name, err) return nil, pkgUtils.Errorln(ctx, msg) @@ -95,8 +101,14 @@ func (p *Provider) UpdateStorageBackend(ctx context.Context, req *drcsi.UpdateSt return &drcsi.UpdateStorageBackendResponse{}, pkgUtils.Errorln(ctx, msg) } + useCert, certSecret, err := pkgUtils.GetCertMeta(ctx, req.BackendId) + if err != nil { + msg := fmt.Sprintf("GetCertMeta [%s] failed. error: %v", req.BackendId, err) + return &drcsi.UpdateStorageBackendResponse{}, pkgUtils.Errorln(ctx, msg) + } + // backendId: / eg:huawei-csi/nfs-180 - _, err = backend.RegisterOneBackend(ctx, req.BackendId, req.ConfigmapMeta, req.SecretMeta) + _, err = backend.RegisterOneBackend(ctx, req.BackendId, req.ConfigmapMeta, req.SecretMeta, certSecret, useCert) if err != nil { msg := fmt.Sprintf("RegisterBackend %s failed, error %v", req.Name, err) return nil, pkgUtils.Errorln(ctx, msg) diff --git a/docs/eSDK Huawei Storage Kubernetes CSI Plugins V4.1.0 User Guide 02.pdf b/docs/eSDK Huawei Storage Kubernetes CSI Plugins V4.1.0 User Guide 02.pdf deleted file mode 100644 index a979bc34..00000000 Binary files a/docs/eSDK Huawei Storage Kubernetes CSI Plugins V4.1.0 User Guide 02.pdf and /dev/null differ diff --git "a/docs/eSDK Huawei Storage Kubernetes CSI Plugins V4.1.0 \347\224\250\346\210\267\346\214\207\345\215\227 02.pdf" "b/docs/eSDK Huawei Storage Kubernetes CSI Plugins V4.1.0 \347\224\250\346\210\267\346\214\207\345\215\227 02.pdf" deleted file mode 100644 index 2d8e0075..00000000 Binary files "a/docs/eSDK Huawei Storage Kubernetes CSI Plugins V4.1.0 \347\224\250\346\210\267\346\214\207\345\215\227 02.pdf" and /dev/null differ diff --git a/docs/eSDK Huawei Storage Kubernetes CSI Plugins V4.2.0 User Guide 01.pdf b/docs/eSDK Huawei Storage Kubernetes CSI Plugins V4.2.0 User Guide 01.pdf new file mode 100644 index 00000000..6c70ab89 Binary files /dev/null and b/docs/eSDK Huawei Storage Kubernetes CSI Plugins V4.2.0 User Guide 01.pdf differ diff --git "a/docs/eSDK Huawei Storage Kubernetes CSI Plugins V4.2.0 \347\224\250\346\210\267\346\214\207\345\215\227 01.pdf" "b/docs/eSDK Huawei Storage Kubernetes CSI Plugins V4.2.0 \347\224\250\346\210\267\346\214\207\345\215\227 01.pdf" new file mode 100644 index 00000000..4f512012 Binary files /dev/null and "b/docs/eSDK Huawei Storage Kubernetes CSI Plugins V4.2.0 \347\224\250\346\210\267\346\214\207\345\215\227 01.pdf" differ diff --git a/helm/esdk/Chart.yaml b/helm/esdk/Chart.yaml index 1e965fb1..3a042f6c 100644 --- a/helm/esdk/Chart.yaml +++ b/helm/esdk/Chart.yaml @@ -15,14 +15,14 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 4.1.0 +version: 4.2.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. # It is strongly recommended not to modify this parameter -appVersion: "4.1.0" +appVersion: "4.2.0" home: https://github.com/Huawei/eSDK_K8S_Plugin sources: diff --git a/helm/esdk/crds/backend/xuanwu.huawei.io_storagebackendclaims.yaml b/helm/esdk/crds/backend/xuanwu.huawei.io_storagebackendclaims.yaml index 46f9db17..c73e7889 100644 --- a/helm/esdk/crds/backend/xuanwu.huawei.io_storagebackendclaims.yaml +++ b/helm/esdk/crds/backend/xuanwu.huawei.io_storagebackendclaims.yaml @@ -66,6 +66,9 @@ spec: spec: description: StorageBackendClaimSpec defines the desired state of StorageBackend properties: + certSecret: + description: CertSecret is the name of the secret that holds the certificate + type: string configmapMeta: description: ConfigMapMeta used to config the storage management info, the format is /. @@ -87,6 +90,10 @@ spec: description: SecretMeta used to config the storage sensitive info, the format is /. type: string + useCert: + default: false + description: UseCert is used to decide whether to use the certificate + type: boolean required: - provider type: object @@ -96,6 +103,9 @@ spec: boundContentName: description: BoundContentName is the binding reference type: string + certSecret: + description: CertSecret is the name of the secret that holds the certificate + type: string configmapMeta: description: ConfigmapMeta is current storage configmap namespace and name, format is /, such as xuanwu/backup-instance-configmap @@ -124,6 +134,9 @@ spec: storageType: description: StorageType is storage type type: string + useCert: + description: UseCert is used to decide whether to use the certificate + type: boolean required: - configmapMeta - secretMeta diff --git a/helm/esdk/crds/backend/xuanwu.huawei.io_storagebackendcontents.yaml b/helm/esdk/crds/backend/xuanwu.huawei.io_storagebackendcontents.yaml index a7c934f0..ad33b3f5 100644 --- a/helm/esdk/crds/backend/xuanwu.huawei.io_storagebackendcontents.yaml +++ b/helm/esdk/crds/backend/xuanwu.huawei.io_storagebackendcontents.yaml @@ -61,6 +61,9 @@ spec: description: BackendClaim is the bound StorageBackendClaim namespace and name, format is /. type: string + certSecret: + description: CertSecret is the name of the secret that holds the certificate + type: string configmapMeta: description: ConfigmapMeta is current storage configmap namespace and name, format is /. such as xuanwu/backup-instance-configmap @@ -82,6 +85,10 @@ spec: description: SecretMeta is current storage secret namespace and name, format is /. such as xuanwu/backup-instance-secret type: string + useCert: + default: false + description: UseCert is used to decide whether to use the certificate + type: boolean required: - provider type: object @@ -101,6 +108,9 @@ spec: description: Capacity get the storage total capacity, used capacity and free capacity. type: object + certSecret: + description: CertSecret is the name of the secret that holds the certificate + type: string configmapMeta: description: ConfigmapMeta is current storage configmap namespace and name, format is /. @@ -132,6 +142,9 @@ spec: description: Specification get the storage total specification of used capacity and free capacity. type: object + useCert: + description: UseCert is used to decide whether to use the certificate + type: boolean vendorName: description: VendorName means the flag of the storage vendor, such as EMC/IBM/NetApp/Huawei diff --git a/helm/esdk/crds/resourcetopologies/xuanwu.huawei.io_resourcetopologies.yaml b/helm/esdk/crds/resourcetopologies/xuanwu.huawei.io_resourcetopologies.yaml new file mode 100644 index 00000000..74387e6b --- /dev/null +++ b/helm/esdk/crds/resourcetopologies/xuanwu.huawei.io_resourcetopologies.yaml @@ -0,0 +1,174 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.8.0 + creationTimestamp: null + name: resourcetopologies.xuanwu.huawei.io +spec: + group: xuanwu.huawei.io + names: + kind: ResourceTopology + listKind: ResourceTopologyList + plural: resourcetopologies + shortNames: + - rt + singular: resourcetopology + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .spec.provisioner + name: Provisioner + type: string + - jsonPath: .spec.volumeHandle + name: VolumeHandle + type: string + - jsonPath: .status.status + name: Status + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: ResourceTopology is the Schema for the ResourceTopologys API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + provisioner: + description: Provisioner is the volume provisioner name + type: string + tags: + description: Tags defines pv and other relationships and ownership + items: + description: Tag defines pv and other relationships and ownership + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this + representation of an object. Servers should convert recognized + schemas to the latest internal value, and may reject unrecognized + values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. Servers may infer this from the endpoint + the client submits requests to. Cannot be updated. In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: Name is the name of the resource + type: string + namespace: + description: NameSpace is the namespace of the resource + type: string + owner: + description: Owner defines who does the resource belongs to + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of + this representation of an object. Servers should convert + recognized schemas to the latest internal value, and may + reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST + resource this object represents. Servers may infer this + from the endpoint the client submits requests to. Cannot + be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: Name is the name of the resource + type: string + namespace: + description: NameSpace is the namespace of the resource + type: string + type: object + type: object + type: array + volumeHandle: + description: VolumeHandle is the backend name and identity of the + volume, format as . + type: string + required: + - provisioner + - tags + - volumeHandle + type: object + status: + properties: + status: + description: Status is the status of the ResourceTopology + type: string + tags: + description: Tags defines pv and other relationships and ownership + items: + description: Tag defines pv and other relationships and ownership + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this + representation of an object. Servers should convert recognized + schemas to the latest internal value, and may reject unrecognized + values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. Servers may infer this from the endpoint + the client submits requests to. Cannot be updated. In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: Name is the name of the resource + type: string + namespace: + description: NameSpace is the namespace of the resource + type: string + owner: + description: Owner defines who does the resource belongs to + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of + this representation of an object. Servers should convert + recognized schemas to the latest internal value, and may + reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST + resource this object represents. Servers may infer this + from the endpoint the client submits requests to. Cannot + be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: Name is the name of the resource + type: string + namespace: + description: NameSpace is the namespace of the resource + type: string + type: object + type: object + type: array + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/helm/esdk/templates/csidriver.yaml b/helm/esdk/templates/csidriver.yaml index 7d359c9b..3da41000 100644 --- a/helm/esdk/templates/csidriver.yaml +++ b/helm/esdk/templates/csidriver.yaml @@ -1,7 +1,11 @@ +{{ if .Values.CSIDriverObject.isCreate }} apiVersion: storage.k8s.io/v1 kind: CSIDriver metadata: name: {{ .Values.csiDriver.driverName }} spec: - attachRequired: {{ .Values.attachRequired }} - fsGroupPolicy: {{ .Values.fsGroupPolicy }} + attachRequired: {{ .Values.CSIDriverObject.attachRequired }} + {{ if ne .Values.CSIDriverObject.fsGroupPolicy "null" }} + fsGroupPolicy: {{ .Values.CSIDriverObject.fsGroupPolicy }} + {{ end }} +{{ end }} diff --git a/helm/esdk/templates/huawei-csi-controller.yaml b/helm/esdk/templates/huawei-csi-controller.yaml index d6fbaee4..9be16e96 100644 --- a/helm/esdk/templates/huawei-csi-controller.yaml +++ b/helm/esdk/templates/huawei-csi-controller.yaml @@ -629,6 +629,9 @@ rules: - secrets verbs: - get + - apiGroups: [ "xuanwu.huawei.io" ] + resources: [ "resourcetopologies" ] + verbs: [ "create", "get", "update", "delete" ] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -703,7 +706,8 @@ spec: volumeMounts: - mountPath: /csi name: socket-dir - {{ if .Values.attachRequired }} + # When 'isCreate' is true and 'attachRequired' is false, csi-attacher sidecar will not be deployed + {{ if or (not .Values.CSIDriverObject.isCreate) (.Values.CSIDriverObject.attachRequired)}} - name: csi-attacher args: - "--csi-address=$(ADDRESS)" @@ -846,6 +850,7 @@ spec: - "--logging-module={{ .Values.csiDriver.controllerLogging.module }}" - "--log-level={{ .Values.csiDriver.controllerLogging.level }}" - "--volume-name-prefix={{ default "pvc" (.Values.controller).volumeNamePrefix }}" + - "--enable-label={{ .Values.csiDriver.enableLabel }}" {{ if eq .Values.csiDriver.controllerLogging.module "file" }} - "--log-file-dir={{ .Values.csiDriver.controllerLogging.fileDir }}" - "--log-file-size={{ .Values.csiDriver.controllerLogging.fileSize }}" @@ -878,6 +883,13 @@ spec: name: socket-dir - mountPath: /var/log name: log + resources: + limits: + cpu: 300m + memory: 1Gi + requests: + cpu: 50m + memory: 100Mi volumes: - emptyDir: null name: socket-dir diff --git a/helm/esdk/templates/huawei-csi-node.yaml b/helm/esdk/templates/huawei-csi-node.yaml index 8af0e7e3..cca959e0 100644 --- a/helm/esdk/templates/huawei-csi-node.yaml +++ b/helm/esdk/templates/huawei-csi-node.yaml @@ -1,4 +1,3 @@ ---- apiVersion: v1 kind: ServiceAccount metadata: @@ -85,7 +84,18 @@ rules: - "xuanwu.huawei.io" resources: - storagebackendclaims + - storagebackendcontents + verbs: + - get + - apiGroups: [ "xuanwu.huawei.io" ] + resources: [ "resourcetopologies" ] + verbs: [ "create", "get", "update", "delete" ] + - apiGroups: + - "" + resources: + - pods verbs: + - list - get --- apiVersion: rbac.authorization.k8s.io/v1 @@ -164,6 +174,7 @@ spec: - "--nvme-multipath-type={{ .Values.csiDriver.nvmeMultipathType }}" {{ end }} - "--scan-volume-timeout={{ .Values.csiDriver.scanVolumeTimeout }}" + - "--exec-command-timeout={{ int (.Values.csiDriver).execCommandTimeout | default 30 }}" - "--logging-module={{ .Values.csiDriver.nodeLogging.module }}" - "--log-level={{ .Values.csiDriver.nodeLogging.level }}" {{ if eq .Values.csiDriver.nodeLogging.module "file" }} @@ -216,8 +227,6 @@ spec: - mountPath: {{ .Values.kubeletConfigDir }} mountPropagation: Bidirectional name: pods-dir - - mountPath: /etc - name: etc-dir - mountPath: /var/log name: log-dir - mountPath: /dev @@ -225,6 +234,17 @@ spec: name: dev-dir - mountPath: /var/lib/iscsi name: iscsi-dir + - mountPath: /etc/iscsi + name: iscsi-config-dir + - mountPath: /etc/nvme + name: nvme-config-dir + resources: + limits: + cpu: 300m + memory: 500Mi + requests: + cpu: 50m + memory: 100Mi hostNetwork: true hostPID: true serviceAccountName: huawei-csi-node @@ -241,10 +261,6 @@ spec: path: {{ .Values.kubeletConfigDir }} type: Directory name: pods-dir - - hostPath: - path: /etc - type: Directory - name: etc-dir - hostPath: path: /dev type: Directory @@ -255,4 +271,12 @@ spec: - hostPath: path: /var/log/ type: Directory - name: log-dir \ No newline at end of file + name: log-dir + - hostPath: + path: /etc/iscsi + type: DirectoryOrCreate + name: iscsi-config-dir + - hostPath: + path: /etc/nvme + type: DirectoryOrCreate + name: nvme-config-dir \ No newline at end of file diff --git a/helm/esdk/values.yaml b/helm/esdk/values.yaml index f3b44bcc..09d71daf 100644 --- a/helm/esdk/values.yaml +++ b/helm/esdk/values.yaml @@ -1,8 +1,8 @@ images: # Images provided by Huawei - huaweiCSIService: huawei-csi:4.1.0 - storageBackendSidecar: storage-backend-sidecar:4.1.0 - storageBackendController: storage-backend-controller:4.1.0 + huaweiCSIService: huawei-csi:4.2.0 + storageBackendSidecar: storage-backend-sidecar:4.2.0 + storageBackendController: storage-backend-controller:4.2.0 # CSI-related sidecar images provided by the Kubernetes community. # These must match the appropriate Kubernetes version. @@ -32,22 +32,33 @@ kubernetes: # CCE is usually /mnt/paas/kubernetes/kubelet kubeletConfigDir: /var/lib/kubelet -# fsGroupPolicy: Defines if the underlying volume supports changing ownership and permission of the volume before being mounted. -# Allowed values: -# ReadWriteOnceWithFSType: supports volume ownership and permissions change only if the fsType is defined -# and the volume's accessModes contains ReadWriteOnce. -# File: kubernetes may use fsGroup to change permissions and ownership of the volume -# to match user requested fsGroup in the pod's security policy regardless of fstype or access mode. -# None: volumes will be mounted with no modifications. -# Default value: ReadWriteOnceWithFSType -fsGroupPolicy: ReadWriteOnceWithFSType - -# attachRequired: Whether to skip any attach operation altogether. -# Allowed values: -# true: attach will be called. -# false: attach will be skipped. -# Default value: true -attachRequired: true +CSIDriverObject: + # isCreate: create CSIDriver Object + # If the Kubernetes version is lower than 1.18, set this parameter to false. + # Allowed values: + # true: will create CSIDriver object during installation. + # false: will not create CSIDriver object during installation. + # Default value: false + isCreate: false + # If the Kubernetes version is lower than 1.20, set this parameter to null. + # fsGroupPolicy: Defines if the underlying volume supports changing ownership and permission of the volume before being mounted. + # 'fsGroupPolicy' is only valid when 'isCreate' is true + # Allowed values: + # ReadWriteOnceWithFSType: supports volume ownership and permissions change only if the fsType is defined + # and the volume's accessModes contains ReadWriteOnce. + # File: kubernetes may use fsGroup to change permissions and ownership of the volume + # to match user requested fsGroup in the pod's security policy regardless of fstype or access mode. + # None: volumes will be mounted with no modifications. + # Default value: null + fsGroupPolicy: null + # If the Kubernetes version is lower than 1.18, set this parameter to true. + # attachRequired: Whether to skip any attach operation altogether. + # When 'isCreate' is true and 'attachRequired' is false, csi-attacher sidecar will not be deployed + # Allowed values: + # true: attach will be called. + # false: attach will be skipped. + # Default value: true + attachRequired: true controller: # controllerCount: Define the number of huawei-csi controller @@ -167,6 +178,8 @@ csiDriver: nvmeMultipathType: HW-UltraPath-NVMe # Timeout interval for waiting for multipath aggregation when DM-multipath is used on the host. support 1~600 scanVolumeTimeout: 3 + # Timeout interval for running command on the host. support 1~600 + execCommandTimeout: 30 # check the number of paths for multipath aggregation # Allowed values: # true: the number of paths aggregated by DM-multipath is equal to the number of online paths @@ -175,6 +188,8 @@ csiDriver: allPathOnline: false # Interval for updating backend capabilities. support 60~600 backendUpdateInterval: 60 + # label enable + enableLabel: false # Huawei-csi-controller log configuration controllerLogging: # Log record type, support [file, console] diff --git a/lib/drcsi/rpc/common.go b/lib/drcsi/rpc/common.go index d778e1ba..dd64f37b 100644 --- a/lib/drcsi/rpc/common.go +++ b/lib/drcsi/rpc/common.go @@ -2,7 +2,7 @@ package rpc import ( "context" - "fmt" + "errors" "google.golang.org/grpc" @@ -20,7 +20,7 @@ func GetProviderName(ctx context.Context, conn *grpc.ClientConn) (string, error) } name := rsp.GetProvider() if name == "" { - return "", fmt.Errorf("drcsi name is empty") + return "", errors.New("drcsi name is empty") } return name, nil } diff --git a/manual/esdk/deploy/csidriver.yaml b/manual/esdk/deploy/csidriver.yaml new file mode 100644 index 00000000..7828c54d --- /dev/null +++ b/manual/esdk/deploy/csidriver.yaml @@ -0,0 +1,6 @@ +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + name: csi.huawei.com +spec: + attachRequired: true diff --git a/manual/esdk/deploy/huawei-csi-controller.yaml b/manual/esdk/deploy/huawei-csi-controller.yaml new file mode 100644 index 00000000..c1c24d1e --- /dev/null +++ b/manual/esdk/deploy/huawei-csi-controller.yaml @@ -0,0 +1,867 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: storage-backend-controller-role + labels: + provisioner: csi.huawei.com +rules: + - apiGroups: [ "admissionregistration.k8s.io" ] + resources: [ "validatingwebhookconfigurations" ] + verbs: [ "create", "get", "update", "delete" ] + - apiGroups: [ "" ] + resources: [ "configmaps", "secrets", "events" ] + verbs: [ "create", "get", "update", "delete" ] + - apiGroups: [ "coordination.k8s.io" ] + resources: [ "leases" ] + verbs: [ "create", "get", "update", "delete" ] + - apiGroups: [ "xuanwu.huawei.io" ] + resources: [ "storagebackendclaims", "storagebackendclaims/status", "storagebackendcontents", + "storagebackendcontents/status" ] + verbs: [ "create", "get", "list", "watch", "update", "delete" ] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: storage-backend-sidecar-role + labels: + provisioner: csi.huawei.com +rules: + - apiGroups: [ "" ] + resources: [ "events", "configmaps" ] + verbs: [ "create", "get", "update", "delete" ] + - apiGroups: [ "coordination.k8s.io" ] + resources: [ "leases" ] + verbs: [ "create", "get", "update", "delete" ] + - apiGroups: [ "xuanwu.huawei.io" ] + resources: [ "storagebackendcontents", "storagebackendcontents/status" ] + verbs: [ "get", "list", "watch", "update" ] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: storage-backend-controller-binding + labels: + app: xuanwu-volume-service + provisioner: csi.huawei.com +subjects: + - kind: ServiceAccount + name: huawei-csi-controller + namespace: huawei-csi +roleRef: + kind: ClusterRole + name: storage-backend-controller-role + apiGroup: rbac.authorization.k8s.io + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: storage-backend-sidecar-binding + labels: + app: xuanwu-volume-service + provisioner: csi.huawei.com +subjects: + - kind: ServiceAccount + name: huawei-csi-controller + namespace: huawei-csi +roleRef: + kind: ClusterRole + name: storage-backend-sidecar-role + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + provisioner: csi.huawei.com + name: huawei-csi-controller + namespace: huawei-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + provisioner: csi.huawei.com + name: huawei-csi-attacher-role-cfg + namespace: huawei-csi +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: huawei-csi-attacher-cfg +subjects: + - kind: ServiceAccount + name: huawei-csi-controller + namespace: huawei-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + provisioner: csi.huawei.com + name: huawei-csi-attacher-cfg + namespace: huawei-csi +rules: + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - watch + - list + - delete + - update + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + provisioner: csi.huawei.com + name: huawei-csi-provisioner-role-cfg + namespace: huawei-csi +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: huawei-csi-provisioner-cfg +subjects: + - kind: ServiceAccount + name: huawei-csi-controller + namespace: huawei-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + provisioner: csi.huawei.com + name: huawei-csi-provisioner-cfg + namespace: huawei-csi +rules: + - apiGroups: + - "" + resources: + - endpoints + verbs: + - get + - watch + - list + - delete + - update + - create + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - watch + - list + - delete + - update + - create + - apiGroups: + - storage.k8s.io + resources: + - csistoragecapacities + verbs: + - get + - list + - watch + - create + - update + - patch + - delete + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - apiGroups: + - apps + resources: + - replicasets + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + provisioner: csi.huawei.com + name: huawei-csi-provisioner-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: huawei-csi-provisioner-runner +subjects: + - kind: ServiceAccount + name: huawei-csi-controller + namespace: huawei-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + provisioner: csi.huawei.com + name: huawei-csi-provisioner-runner +rules: + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - create + - delete + - update + - apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - update + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshots + verbs: + - get + - list + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - get + - list + - apiGroups: + - storage.k8s.io + resources: + - csinodes + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + - update +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + provisioner: csi.huawei.com + name: huawei-csi-attacher-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: huawei-csi-attacher-runner +subjects: + - kind: ServiceAccount + name: huawei-csi-controller + namespace: huawei-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + provisioner: csi.huawei.com + name: huawei-csi-attacher-runner +rules: + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - patch + - update + - apiGroups: + - storage.k8s.io + resources: + - csinodes + verbs: + - get + - list + - watch + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + - patch + - update + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments/status + verbs: + - patch + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + provisioner: csi.huawei.com + name: huawei-csi-csi-resizer-role-cfg + namespace: huawei-csi +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: huawei-csi-resizer-cfg +subjects: + - kind: ServiceAccount + name: huawei-csi-controller + namespace: huawei-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + provisioner: csi.huawei.com + name: huawei-csi-resizer-cfg + namespace: huawei-csi +rules: + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - watch + - list + - delete + - update + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + provisioner: csi.huawei.com + name: huawei-csi-csi-resizer-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: huawei-csi-resizer-runner +subjects: + - kind: ServiceAccount + name: huawei-csi-controller + namespace: huawei-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + provisioner: csi.huawei.com + name: huawei-csi-resizer-runner +rules: + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - patch + - update + - apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - persistentvolumeclaims/status + verbs: + - patch + - update + - apiGroups: + - "" + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + provisioner: csi.huawei.com + name: huawei-csi-csi-snapshotter-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: huawei-csi-snapshotter-runner +subjects: + - kind: ServiceAccount + name: huawei-csi-controller + namespace: huawei-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + provisioner: csi.huawei.com + name: huawei-csi-snapshotter-runner +rules: + - apiGroups: + - "" + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotclasses + verbs: + - get + - list + - watch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - create + - get + - list + - watch + - update + - delete + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents/status + verbs: + - update + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + provisioner: csi.huawei.com + name: huawei-csi-snapshot-controller-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: huawei-csi-snapshot-controller-runner +subjects: + - kind: ServiceAccount + name: huawei-csi-controller + namespace: huawei-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + provisioner: csi.huawei.com + name: huawei-csi-snapshot-controller-runner +rules: + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - update + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotclasses + verbs: + - get + - list + - watch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - create + - get + - list + - watch + - update + - delete + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents/status + verbs: + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshots + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshots/status + verbs: + - update + - patch + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + provisioner: csi.huawei.com + name: huawei-csi-controller-runner +rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - apiGroups: [ "xuanwu.huawei.io" ] + resources: [ "resourcetopologies" ] + verbs: [ "create", "get", "update", "delete" ] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + provisioner: csi.huawei.com + name: huawei-csi-controller-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: huawei-csi-controller-runner +subjects: + - kind: ServiceAccount + name: huawei-csi-controller + namespace: huawei-csi + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + provisioner: csi.huawei.com + name: huawei-csi-controller + namespace: huawei-csi +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: huawei-csi-controller + provisioner: csi.huawei.com + template: + metadata: + labels: + app: huawei-csi-controller + provisioner: csi.huawei.com + spec: + hostNetwork: true + serviceAccount: huawei-csi-controller + containers: + - name: liveness-probe + args: + - "--csi-address=/csi/csi.sock" + - "--health-port=9808" + image: k8s.gcr.io/sig-storage/livenessprobe:v2.5.0 + imagePullPolicy: "IfNotPresent" + volumeMounts: + - mountPath: /csi + name: socket-dir + - name: csi-provisioner + args: + - "--csi-address=$(ADDRESS)" + - "--timeout=6h" + - "--volume-name-prefix=pvc" + env: + - name: ADDRESS + value: /csi/csi.sock + image: k8s.gcr.io/sig-storage/csi-provisioner:v3.0.0 + imagePullPolicy: "IfNotPresent" + volumeMounts: + - mountPath: /csi + name: socket-dir + # When 'isCreate' is true and 'attachRequired' is false, csi-attacher sidecar will not be deployed + - name: csi-attacher + args: + - "--csi-address=$(ADDRESS)" + env: + - name: ADDRESS + value: /csi/csi.sock + image: k8s.gcr.io/sig-storage/csi-attacher:v3.4.0 + imagePullPolicy: "IfNotPresent" + volumeMounts: + - mountPath: /csi + name: socket-dir + - name: csi-resizer + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + - "--handle-volume-inuse-error=false" + env: + - name: ADDRESS + value: /csi/csi.sock + image: k8s.gcr.io/sig-storage/csi-resizer:v1.4.0 + imagePullPolicy: IfNotPresent + volumeMounts: + - mountPath: /csi + name: socket-dir + - name: csi-snapshotter + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + env: + - name: ADDRESS + value: /csi/csi.sock + image: k8s.gcr.io/sig-storage/csi-snapshotter:v4.2.1 + imagePullPolicy: "IfNotPresent" + volumeMounts: + - mountPath: /csi + name: socket-dir + - name: snapshot-controller + args: + - "--v=5" + image: k8s.gcr.io/sig-storage/snapshot-controller:v4.2.1 + imagePullPolicy: "IfNotPresent" + - name: storage-backend-controller + image: storage-backend-controller:4.2.0 + imagePullPolicy: "IfNotPresent" + env: + - name: CSI_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + args: + - "--logging-module=file" + - "--log-level=info" + - "--log-file-dir=/var/log/huawei" + - "--log-file-size=20M" + - "--max-backups=9" + - "--web-hook-port=4433" + - "--enable-leader-election=false" + - "--leader-lease-duration=8s" + - "--leader-renew-deadline=6s" + - "--leader-retry-period=2s" + ports: + - containerPort: 4433 + volumeMounts: + - mountPath: /var/log + name: log + - name: storage-backend-sidecar + image: storage-backend-sidecar:4.2.0 + imagePullPolicy: "IfNotPresent" + env: + - name: DRCSI_ENDPOINT + value: /csi/dr-csi.sock + - name: CSI_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + args: + - "--logging-module=file" + - "--log-level=info" + - "--log-file-dir=/var/log/huawei" + - "--log-file-size=20M" + - "--max-backups=9" + - "--dr-endpoint=$(DRCSI_ENDPOINT)" + - "--enable-leader-election=false" + - "--leader-lease-duration=8s" + - "--leader-renew-deadline=6s" + - "--leader-retry-period=2s" + volumeMounts: + - mountPath: /csi + name: socket-dir + - mountPath: /var/log + name: log + - name: huawei-csi-driver + image: huawei-csi:4.2.0 + imagePullPolicy: "IfNotPresent" + args: + - "--endpoint=$(CSI_ENDPOINT)" + - "--dr-endpoint=$(DRCSI_ENDPOINT)" + - "--controller" + - "--backend-update-interval=60" + - "--driver-name=csi.huawei.com" + - "--logging-module=file" + - "--log-level=info" + - "--volume-name-prefix=pvc" + - "--enable-label=false" + - "--log-file-dir=/var/log/huawei" + - "--log-file-size=20M" + - "--max-backups=9" + env: + - name: CSI_ENDPOINT + value: /csi/csi.sock + - name: DRCSI_ENDPOINT + value: /csi/dr-csi.sock + - name: CSI_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 60 + timeoutSeconds: 3 + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + volumeMounts: + - mountPath: /csi + name: socket-dir + - mountPath: /var/log + name: log + resources: + limits: + cpu: 300m + memory: 1Gi + requests: + cpu: 50m + memory: 100Mi + volumes: + - emptyDir: null + name: socket-dir + - hostPath: + path: /var/log/ + type: Directory + name: log + +--- +apiVersion: v1 +kind: Service +metadata: + name: huawei-csi-controller + namespace: huawei-csi + labels: + app: huawei-csi-controller + provisioner: csi.huawei.com +spec: + selector: + app: huawei-csi-controller + ports: + - name: storage-backend-controller + protocol: TCP + port: 4433 + targetPort: 4433 \ No newline at end of file diff --git a/manual/esdk/deploy/huawei-csi-node.yaml b/manual/esdk/deploy/huawei-csi-node.yaml new file mode 100644 index 00000000..951c23bc --- /dev/null +++ b/manual/esdk/deploy/huawei-csi-node.yaml @@ -0,0 +1,267 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + provisioner: csi.huawei.com + name: huawei-csi-node + namespace: huawei-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + provisioner: csi.huawei.com + name: huawei-csi-driver-registrar-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: huawei-csi-driver-registrar-runner +subjects: + - kind: ServiceAccount + name: huawei-csi-node + namespace: huawei-csi +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + provisioner: csi.huawei.com + name: huawei-csi-driver-registrar-runner +rules: + - apiGroups: + - "" + resources: + - events + verbs: + - get + - list + - watch + - create + - update + - patch + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - apiGroups: + - "" + resources: + - pods + verbs: + - list + - apiGroups: + - "" + resources: + - persistentvolumes + - persistentvolumeclaims + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + provisioner: csi.huawei.com + name: huawei-csi-node-runner +rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - update + - create + - apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - apiGroups: + - "xuanwu.huawei.io" + resources: + - storagebackendclaims + - storagebackendcontents + verbs: + - get + - apiGroups: [ "xuanwu.huawei.io" ] + resources: [ "resourcetopologies" ] + verbs: [ "create", "get", "update", "delete" ] + - apiGroups: + - "" + resources: + - pods + verbs: + - list + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + provisioner: csi.huawei.com + name: huawei-csi-node-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: huawei-csi-node-runner +subjects: + - kind: ServiceAccount + name: huawei-csi-node + namespace: huawei-csi +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + provisioner: csi.huawei.com + name: huawei-csi-node + namespace: huawei-csi +spec: + selector: + matchLabels: + app: huawei-csi-node + provisioner: csi.huawei.com + template: + metadata: + labels: + app: huawei-csi-node + provisioner: csi.huawei.com + spec: + containers: + - name: liveness-probe + args: + - "--csi-address=/csi/csi.sock" + - "--health-port=9800" + image: k8s.gcr.io/sig-storage/livenessprobe:v2.5.0 + imagePullPolicy: "IfNotPresent" + volumeMounts: + - mountPath: /csi + name: socket-dir + - name: csi-node-driver-registrar + args: + - "--csi-address=/csi/csi.sock" + - "--kubelet-registration-path=/var/lib/kubelet/plugins/csi.huawei.com/csi.sock" + image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.3.0 + imagePullPolicy: "IfNotPresent" + volumeMounts: + - mountPath: /csi + name: socket-dir + - mountPath: /registration + name: registration-dir + - name: huawei-csi-driver + image: huawei-csi:4.2.0 + imagePullPolicy: "IfNotPresent" + args: + - "--endpoint=/csi/csi.sock" + - "--driver-name=csi.huawei.com" + - "--connector-threads=4" + - "--volume-use-multipath=true" + - "--all-path-online=false" + - "--scsi-multipath-type=DM-multipath" + - "--nvme-multipath-type=HW-UltraPath-NVMe" + - "--scan-volume-timeout=3" + - "--exec-command-timeout=30" + - "--logging-module=file" + - "--log-level=info" + - "--log-file-dir=/var/log/huawei" + - "--log-file-size=20M" + - "--max-backups=9" + env: + - name: CSI_NODENAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CSI_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + lifecycle: + preStop: + exec: + command: + - /bin/sh + - -c + - rm -f /csi/csi.sock + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 60 + timeoutSeconds: 3 + ports: + - containerPort: 9800 + name: healthz + protocol: TCP + securityContext: + allowPrivilegeEscalation: true + capabilities: + add: + - SYS_ADMIN + privileged: true + volumeMounts: + - mountPath: /csi + name: socket-dir + - mountPath: /var/lib/kubelet + mountPropagation: Bidirectional + name: pods-dir + - mountPath: /var/log + name: log-dir + - mountPath: /dev + mountPropagation: HostToContainer + name: dev-dir + - mountPath: /var/lib/iscsi + name: iscsi-dir + - mountPath: /etc/iscsi + name: iscsi-config-dir + - mountPath: /etc/nvme + name: nvme-config-dir + resources: + limits: + cpu: 300m + memory: 500Mi + requests: + cpu: 50m + memory: 100Mi + hostNetwork: true + hostPID: true + serviceAccountName: huawei-csi-node + volumes: + - hostPath: + path: /var/lib/kubelet/plugins/csi.huawei.com + type: DirectoryOrCreate + name: socket-dir + - hostPath: + path: /var/lib/kubelet/plugins_registry + type: Directory + name: registration-dir + - hostPath: + path: /var/lib/kubelet + type: Directory + name: pods-dir + - hostPath: + path: /dev + type: Directory + name: dev-dir + - hostPath: + path: /var/lib/iscsi + name: iscsi-dir + - hostPath: + path: /var/log/ + type: Directory + name: log-dir + - hostPath: + path: /etc/iscsi + type: DirectoryOrCreate + name: iscsi-config-dir + - hostPath: + path: /etc/nvme + type: DirectoryOrCreate + name: nvme-config-dir diff --git a/pkg/client/clientset/versioned/clientset.go b/pkg/client/clientset/versioned/clientset.go index b6754567..b1028c71 100644 --- a/pkg/client/clientset/versioned/clientset.go +++ b/pkg/client/clientset/versioned/clientset.go @@ -17,10 +17,11 @@ package versioned import ( "fmt" + "k8s.io/client-go/tools/clientcmd" + "net/http" discovery "k8s.io/client-go/discovery" rest "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" flowcontrol "k8s.io/client-go/util/flowcontrol" xuanwuv1 "huawei-csi-driver/pkg/client/clientset/versioned/typed/xuanwu/v1" @@ -31,8 +32,7 @@ type Interface interface { XuanwuV1() xuanwuv1.XuanwuV1Interface } -// Clientset contains the clients for groups. Each group has exactly one -// version included in a Clientset. +// Clientset contains the clients for groups. type Clientset struct { *discovery.DiscoveryClient xuanwuV1 *xuanwuv1.XuanwuV1Client @@ -54,22 +54,45 @@ func (c *Clientset) Discovery() discovery.DiscoveryInterface { // NewForConfig creates a new Clientset for the given config. // If config's RateLimiter is not set and QPS and Burst are acceptable, // NewForConfig will generate a rate-limiter in configShallowCopy. +// NewForConfig is equivalent to NewForConfigAndClient(c, httpClient), +// where httpClient was generated with rest.HTTPClientFor(c). func NewForConfig(c *rest.Config) (*Clientset, error) { configShallowCopy := *c + + if configShallowCopy.UserAgent == "" { + configShallowCopy.UserAgent = rest.DefaultKubernetesUserAgent() + } + + // share the transport between all clients + httpClient, err := rest.HTTPClientFor(&configShallowCopy) + if err != nil { + return nil, err + } + + return NewForConfigAndClient(&configShallowCopy, httpClient) +} + +// NewForConfigAndClient creates a new Clientset for the given config and http client. +// Note the http client provided takes precedence over the configured transport values. +// If config's RateLimiter is not set and QPS and Burst are acceptable, +// NewForConfigAndClient will generate a rate-limiter in configShallowCopy. +func NewForConfigAndClient(c *rest.Config, httpClient *http.Client) (*Clientset, error) { + configShallowCopy := *c if configShallowCopy.RateLimiter == nil && configShallowCopy.QPS > 0 { if configShallowCopy.Burst <= 0 { return nil, fmt.Errorf("burst is required to be greater than 0 when RateLimiter is not set and QPS is set to greater than 0") } configShallowCopy.RateLimiter = flowcontrol.NewTokenBucketRateLimiter(configShallowCopy.QPS, configShallowCopy.Burst) } + var cs Clientset var err error - cs.xuanwuV1, err = xuanwuv1.NewForConfig(&configShallowCopy) + cs.xuanwuV1, err = xuanwuv1.NewForConfigAndClient(&configShallowCopy, httpClient) if err != nil { return nil, err } - cs.DiscoveryClient, err = discovery.NewDiscoveryClientForConfig(&configShallowCopy) + cs.DiscoveryClient, err = discovery.NewDiscoveryClientForConfigAndClient(&configShallowCopy, httpClient) if err != nil { return nil, err } @@ -79,11 +102,11 @@ func NewForConfig(c *rest.Config) (*Clientset, error) { // NewForConfigOrDie creates a new Clientset for the given config and // panics if there is an error in the config. func NewForConfigOrDie(c *rest.Config) *Clientset { - var cs Clientset - cs.xuanwuV1 = xuanwuv1.NewForConfigOrDie(c) - - cs.DiscoveryClient = discovery.NewDiscoveryClientForConfigOrDie(c) - return &cs + cs, err := NewForConfig(c) + if err != nil { + panic(err) + } + return cs } // New creates a new Clientset for the given RESTClient. diff --git a/pkg/client/clientset/versioned/fake/clientset_generated.go b/pkg/client/clientset/versioned/fake/clientset_generated.go index 2a6565af..97b869ba 100644 --- a/pkg/client/clientset/versioned/fake/clientset_generated.go +++ b/pkg/client/clientset/versioned/fake/clientset_generated.go @@ -19,6 +19,7 @@ import ( clientset "huawei-csi-driver/pkg/client/clientset/versioned" xuanwuv1 "huawei-csi-driver/pkg/client/clientset/versioned/typed/xuanwu/v1" fakexuanwuv1 "huawei-csi-driver/pkg/client/clientset/versioned/typed/xuanwu/v1/fake" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/discovery" diff --git a/pkg/client/clientset/versioned/fake/register.go b/pkg/client/clientset/versioned/fake/register.go index 77ff908c..8ec64565 100644 --- a/pkg/client/clientset/versioned/fake/register.go +++ b/pkg/client/clientset/versioned/fake/register.go @@ -17,6 +17,7 @@ package fake import ( xuanwuv1 "huawei-csi-driver/client/apis/xuanwu/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" schema "k8s.io/apimachinery/pkg/runtime/schema" diff --git a/pkg/client/clientset/versioned/scheme/register.go b/pkg/client/clientset/versioned/scheme/register.go index 1543d67c..97cab162 100644 --- a/pkg/client/clientset/versioned/scheme/register.go +++ b/pkg/client/clientset/versioned/scheme/register.go @@ -17,6 +17,7 @@ package scheme import ( xuanwuv1 "huawei-csi-driver/client/apis/xuanwu/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" schema "k8s.io/apimachinery/pkg/runtime/schema" diff --git a/pkg/client/clientset/versioned/typed/xuanwu/v1/fake/fake_resourcetopology.go b/pkg/client/clientset/versioned/typed/xuanwu/v1/fake/fake_resourcetopology.go new file mode 100644 index 00000000..7e4cda83 --- /dev/null +++ b/pkg/client/clientset/versioned/typed/xuanwu/v1/fake/fake_resourcetopology.go @@ -0,0 +1,130 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2022-2023. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + "context" + xuanwuv1 "huawei-csi-driver/client/apis/xuanwu/v1" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + labels "k8s.io/apimachinery/pkg/labels" + schema "k8s.io/apimachinery/pkg/runtime/schema" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + testing "k8s.io/client-go/testing" +) + +// FakeResourceTopologies implements ResourceTopologyInterface +type FakeResourceTopologies struct { + Fake *FakeXuanwuV1 +} + +var resourcetopologiesResource = schema.GroupVersionResource{Group: "xuanwu.huawei.io", Version: "v1", Resource: "resourcetopologies"} + +var resourcetopologiesKind = schema.GroupVersionKind{Group: "xuanwu.huawei.io", Version: "v1", Kind: "ResourceTopology"} + +// Get takes name of the resourceTopology, and returns the corresponding resourceTopology object, and an error if there is any. +func (c *FakeResourceTopologies) Get(ctx context.Context, name string, options v1.GetOptions) (result *xuanwuv1.ResourceTopology, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootGetAction(resourcetopologiesResource, name), &xuanwuv1.ResourceTopology{}) + if obj == nil { + return nil, err + } + return obj.(*xuanwuv1.ResourceTopology), err +} + +// List takes label and field selectors, and returns the list of ResourceTopologies that match those selectors. +func (c *FakeResourceTopologies) List(ctx context.Context, opts v1.ListOptions) (result *xuanwuv1.ResourceTopologyList, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootListAction(resourcetopologiesResource, resourcetopologiesKind, opts), &xuanwuv1.ResourceTopologyList{}) + if obj == nil { + return nil, err + } + + label, _, _ := testing.ExtractFromListOptions(opts) + if label == nil { + label = labels.Everything() + } + list := &xuanwuv1.ResourceTopologyList{ListMeta: obj.(*xuanwuv1.ResourceTopologyList).ListMeta} + for _, item := range obj.(*xuanwuv1.ResourceTopologyList).Items { + if label.Matches(labels.Set(item.Labels)) { + list.Items = append(list.Items, item) + } + } + return list, err +} + +// Watch returns a watch.Interface that watches the requested resourceTopologies. +func (c *FakeResourceTopologies) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + return c.Fake. + InvokesWatch(testing.NewRootWatchAction(resourcetopologiesResource, opts)) +} + +// Create takes the representation of a resourceTopology and creates it. Returns the server's representation of the resourceTopology, and an error, if there is any. +func (c *FakeResourceTopologies) Create(ctx context.Context, resourceTopology *xuanwuv1.ResourceTopology, opts v1.CreateOptions) (result *xuanwuv1.ResourceTopology, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootCreateAction(resourcetopologiesResource, resourceTopology), &xuanwuv1.ResourceTopology{}) + if obj == nil { + return nil, err + } + return obj.(*xuanwuv1.ResourceTopology), err +} + +// Update takes the representation of a resourceTopology and updates it. Returns the server's representation of the resourceTopology, and an error, if there is any. +func (c *FakeResourceTopologies) Update(ctx context.Context, resourceTopology *xuanwuv1.ResourceTopology, opts v1.UpdateOptions) (result *xuanwuv1.ResourceTopology, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootUpdateAction(resourcetopologiesResource, resourceTopology), &xuanwuv1.ResourceTopology{}) + if obj == nil { + return nil, err + } + return obj.(*xuanwuv1.ResourceTopology), err +} + +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). +func (c *FakeResourceTopologies) UpdateStatus(ctx context.Context, resourceTopology *xuanwuv1.ResourceTopology, opts v1.UpdateOptions) (*xuanwuv1.ResourceTopology, error) { + obj, err := c.Fake. + Invokes(testing.NewRootUpdateSubresourceAction(resourcetopologiesResource, "status", resourceTopology), &xuanwuv1.ResourceTopology{}) + if obj == nil { + return nil, err + } + return obj.(*xuanwuv1.ResourceTopology), err +} + +// Delete takes name of the resourceTopology and deletes it. Returns an error if one occurs. +func (c *FakeResourceTopologies) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { + _, err := c.Fake. + Invokes(testing.NewRootDeleteActionWithOptions(resourcetopologiesResource, name, opts), &xuanwuv1.ResourceTopology{}) + return err +} + +// DeleteCollection deletes a collection of objects. +func (c *FakeResourceTopologies) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { + action := testing.NewRootDeleteCollectionAction(resourcetopologiesResource, listOpts) + + _, err := c.Fake.Invokes(action, &xuanwuv1.ResourceTopologyList{}) + return err +} + +// Patch applies the patch and returns the patched resourceTopology. +func (c *FakeResourceTopologies) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *xuanwuv1.ResourceTopology, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootPatchSubresourceAction(resourcetopologiesResource, name, pt, data, subresources...), &xuanwuv1.ResourceTopology{}) + if obj == nil { + return nil, err + } + return obj.(*xuanwuv1.ResourceTopology), err +} diff --git a/pkg/client/clientset/versioned/typed/xuanwu/v1/fake/fake_storagebackendclaim.go b/pkg/client/clientset/versioned/typed/xuanwu/v1/fake/fake_storagebackendclaim.go index d3636b90..c48b0b8b 100644 --- a/pkg/client/clientset/versioned/typed/xuanwu/v1/fake/fake_storagebackendclaim.go +++ b/pkg/client/clientset/versioned/typed/xuanwu/v1/fake/fake_storagebackendclaim.go @@ -17,8 +17,8 @@ package fake import ( "context" - xuanwuv1 "huawei-csi-driver/client/apis/xuanwu/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" labels "k8s.io/apimachinery/pkg/labels" schema "k8s.io/apimachinery/pkg/runtime/schema" @@ -114,7 +114,7 @@ func (c *FakeStorageBackendClaims) UpdateStatus(ctx context.Context, storageBack // Delete takes name of the storageBackendClaim and deletes it. Returns an error if one occurs. func (c *FakeStorageBackendClaims) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { _, err := c.Fake. - Invokes(testing.NewDeleteAction(storagebackendclaimsResource, c.ns, name), &xuanwuv1.StorageBackendClaim{}) + Invokes(testing.NewDeleteActionWithOptions(storagebackendclaimsResource, c.ns, name, opts), &xuanwuv1.StorageBackendClaim{}) return err } diff --git a/pkg/client/clientset/versioned/typed/xuanwu/v1/fake/fake_storagebackendcontent.go b/pkg/client/clientset/versioned/typed/xuanwu/v1/fake/fake_storagebackendcontent.go index 47897112..e8f253c4 100644 --- a/pkg/client/clientset/versioned/typed/xuanwu/v1/fake/fake_storagebackendcontent.go +++ b/pkg/client/clientset/versioned/typed/xuanwu/v1/fake/fake_storagebackendcontent.go @@ -17,8 +17,8 @@ package fake import ( "context" - xuanwuv1 "huawei-csi-driver/client/apis/xuanwu/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" labels "k8s.io/apimachinery/pkg/labels" schema "k8s.io/apimachinery/pkg/runtime/schema" @@ -107,7 +107,7 @@ func (c *FakeStorageBackendContents) UpdateStatus(ctx context.Context, storageBa // Delete takes name of the storageBackendContent and deletes it. Returns an error if one occurs. func (c *FakeStorageBackendContents) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { _, err := c.Fake. - Invokes(testing.NewRootDeleteAction(storagebackendcontentsResource, name), &xuanwuv1.StorageBackendContent{}) + Invokes(testing.NewRootDeleteActionWithOptions(storagebackendcontentsResource, name, opts), &xuanwuv1.StorageBackendContent{}) return err } diff --git a/pkg/client/clientset/versioned/typed/xuanwu/v1/fake/fake_xuanwu_client.go b/pkg/client/clientset/versioned/typed/xuanwu/v1/fake/fake_xuanwu_client.go index c96bdedf..c2b99427 100644 --- a/pkg/client/clientset/versioned/typed/xuanwu/v1/fake/fake_xuanwu_client.go +++ b/pkg/client/clientset/versioned/typed/xuanwu/v1/fake/fake_xuanwu_client.go @@ -17,6 +17,7 @@ package fake import ( v1 "huawei-csi-driver/pkg/client/clientset/versioned/typed/xuanwu/v1" + rest "k8s.io/client-go/rest" testing "k8s.io/client-go/testing" ) @@ -25,6 +26,10 @@ type FakeXuanwuV1 struct { *testing.Fake } +func (c *FakeXuanwuV1) ResourceTopologies() v1.ResourceTopologyInterface { + return &FakeResourceTopologies{c} +} + func (c *FakeXuanwuV1) StorageBackendClaims(namespace string) v1.StorageBackendClaimInterface { return &FakeStorageBackendClaims{c, namespace} } diff --git a/pkg/client/clientset/versioned/typed/xuanwu/v1/generated_expansion.go b/pkg/client/clientset/versioned/typed/xuanwu/v1/generated_expansion.go index 5e619668..c935c9c5 100644 --- a/pkg/client/clientset/versioned/typed/xuanwu/v1/generated_expansion.go +++ b/pkg/client/clientset/versioned/typed/xuanwu/v1/generated_expansion.go @@ -15,6 +15,8 @@ package v1 +type ResourceTopologyExpansion interface{} + type StorageBackendClaimExpansion interface{} type StorageBackendContentExpansion interface{} diff --git a/pkg/client/clientset/versioned/typed/xuanwu/v1/resourcetopology.go b/pkg/client/clientset/versioned/typed/xuanwu/v1/resourcetopology.go new file mode 100644 index 00000000..47e65408 --- /dev/null +++ b/pkg/client/clientset/versioned/typed/xuanwu/v1/resourcetopology.go @@ -0,0 +1,181 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2022-2023. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package v1 + +import ( + "context" + v1 "huawei-csi-driver/client/apis/xuanwu/v1" + scheme "huawei-csi-driver/pkg/client/clientset/versioned/scheme" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + rest "k8s.io/client-go/rest" +) + +// ResourceTopologiesGetter has a method to return a ResourceTopologyInterface. +// A group's client should implement this interface. +type ResourceTopologiesGetter interface { + ResourceTopologies() ResourceTopologyInterface +} + +// ResourceTopologyInterface has methods to work with ResourceTopology resources. +type ResourceTopologyInterface interface { + Create(ctx context.Context, resourceTopology *v1.ResourceTopology, opts metav1.CreateOptions) (*v1.ResourceTopology, error) + Update(ctx context.Context, resourceTopology *v1.ResourceTopology, opts metav1.UpdateOptions) (*v1.ResourceTopology, error) + UpdateStatus(ctx context.Context, resourceTopology *v1.ResourceTopology, opts metav1.UpdateOptions) (*v1.ResourceTopology, error) + Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error + Get(ctx context.Context, name string, opts metav1.GetOptions) (*v1.ResourceTopology, error) + List(ctx context.Context, opts metav1.ListOptions) (*v1.ResourceTopologyList, error) + Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *v1.ResourceTopology, err error) + ResourceTopologyExpansion +} + +// resourceTopologies implements ResourceTopologyInterface +type resourceTopologies struct { + client rest.Interface +} + +// newResourceTopologies returns a ResourceTopologies +func newResourceTopologies(c *XuanwuV1Client) *resourceTopologies { + return &resourceTopologies{ + client: c.RESTClient(), + } +} + +// Get takes name of the resourceTopology, and returns the corresponding resourceTopology object, and an error if there is any. +func (c *resourceTopologies) Get(ctx context.Context, name string, options metav1.GetOptions) (result *v1.ResourceTopology, err error) { + result = &v1.ResourceTopology{} + err = c.client.Get(). + Resource("resourcetopologies"). + Name(name). + VersionedParams(&options, scheme.ParameterCodec). + Do(ctx). + Into(result) + return +} + +// List takes label and field selectors, and returns the list of ResourceTopologies that match those selectors. +func (c *resourceTopologies) List(ctx context.Context, opts metav1.ListOptions) (result *v1.ResourceTopologyList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } + result = &v1.ResourceTopologyList{} + err = c.client.Get(). + Resource("resourcetopologies"). + VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). + Do(ctx). + Into(result) + return +} + +// Watch returns a watch.Interface that watches the requested resourceTopologies. +func (c *resourceTopologies) Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } + opts.Watch = true + return c.client.Get(). + Resource("resourcetopologies"). + VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). + Watch(ctx) +} + +// Create takes the representation of a resourceTopology and creates it. Returns the server's representation of the resourceTopology, and an error, if there is any. +func (c *resourceTopologies) Create(ctx context.Context, resourceTopology *v1.ResourceTopology, opts metav1.CreateOptions) (result *v1.ResourceTopology, err error) { + result = &v1.ResourceTopology{} + err = c.client.Post(). + Resource("resourcetopologies"). + VersionedParams(&opts, scheme.ParameterCodec). + Body(resourceTopology). + Do(ctx). + Into(result) + return +} + +// Update takes the representation of a resourceTopology and updates it. Returns the server's representation of the resourceTopology, and an error, if there is any. +func (c *resourceTopologies) Update(ctx context.Context, resourceTopology *v1.ResourceTopology, opts metav1.UpdateOptions) (result *v1.ResourceTopology, err error) { + result = &v1.ResourceTopology{} + err = c.client.Put(). + Resource("resourcetopologies"). + Name(resourceTopology.Name). + VersionedParams(&opts, scheme.ParameterCodec). + Body(resourceTopology). + Do(ctx). + Into(result) + return +} + +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). +func (c *resourceTopologies) UpdateStatus(ctx context.Context, resourceTopology *v1.ResourceTopology, opts metav1.UpdateOptions) (result *v1.ResourceTopology, err error) { + result = &v1.ResourceTopology{} + err = c.client.Put(). + Resource("resourcetopologies"). + Name(resourceTopology.Name). + SubResource("status"). + VersionedParams(&opts, scheme.ParameterCodec). + Body(resourceTopology). + Do(ctx). + Into(result) + return +} + +// Delete takes name of the resourceTopology and deletes it. Returns an error if one occurs. +func (c *resourceTopologies) Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error { + return c.client.Delete(). + Resource("resourcetopologies"). + Name(name). + Body(&opts). + Do(ctx). + Error() +} + +// DeleteCollection deletes a collection of objects. +func (c *resourceTopologies) DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error { + var timeout time.Duration + if listOpts.TimeoutSeconds != nil { + timeout = time.Duration(*listOpts.TimeoutSeconds) * time.Second + } + return c.client.Delete(). + Resource("resourcetopologies"). + VersionedParams(&listOpts, scheme.ParameterCodec). + Timeout(timeout). + Body(&opts). + Do(ctx). + Error() +} + +// Patch applies the patch and returns the patched resourceTopology. +func (c *resourceTopologies) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *v1.ResourceTopology, err error) { + result = &v1.ResourceTopology{} + err = c.client.Patch(pt). + Resource("resourcetopologies"). + Name(name). + SubResource(subresources...). + VersionedParams(&opts, scheme.ParameterCodec). + Body(data). + Do(ctx). + Into(result) + return +} diff --git a/pkg/client/clientset/versioned/typed/xuanwu/v1/storagebackendclaim.go b/pkg/client/clientset/versioned/typed/xuanwu/v1/storagebackendclaim.go index af8cea88..c87eacf8 100644 --- a/pkg/client/clientset/versioned/typed/xuanwu/v1/storagebackendclaim.go +++ b/pkg/client/clientset/versioned/typed/xuanwu/v1/storagebackendclaim.go @@ -17,10 +17,10 @@ package v1 import ( "context" - "time" - v1 "huawei-csi-driver/client/apis/xuanwu/v1" scheme "huawei-csi-driver/pkg/client/clientset/versioned/scheme" + "time" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" diff --git a/pkg/client/clientset/versioned/typed/xuanwu/v1/storagebackendcontent.go b/pkg/client/clientset/versioned/typed/xuanwu/v1/storagebackendcontent.go index f02833f1..a1fcf13c 100644 --- a/pkg/client/clientset/versioned/typed/xuanwu/v1/storagebackendcontent.go +++ b/pkg/client/clientset/versioned/typed/xuanwu/v1/storagebackendcontent.go @@ -17,10 +17,10 @@ package v1 import ( "context" - "time" - v1 "huawei-csi-driver/client/apis/xuanwu/v1" scheme "huawei-csi-driver/pkg/client/clientset/versioned/scheme" + "time" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" diff --git a/pkg/client/clientset/versioned/typed/xuanwu/v1/xuanwu_client.go b/pkg/client/clientset/versioned/typed/xuanwu/v1/xuanwu_client.go index 14b7a162..2a121a21 100644 --- a/pkg/client/clientset/versioned/typed/xuanwu/v1/xuanwu_client.go +++ b/pkg/client/clientset/versioned/typed/xuanwu/v1/xuanwu_client.go @@ -18,11 +18,14 @@ package v1 import ( v1 "huawei-csi-driver/client/apis/xuanwu/v1" "huawei-csi-driver/pkg/client/clientset/versioned/scheme" + "net/http" + rest "k8s.io/client-go/rest" ) type XuanwuV1Interface interface { RESTClient() rest.Interface + ResourceTopologiesGetter StorageBackendClaimsGetter StorageBackendContentsGetter } @@ -32,6 +35,10 @@ type XuanwuV1Client struct { restClient rest.Interface } +func (c *XuanwuV1Client) ResourceTopologies() ResourceTopologyInterface { + return newResourceTopologies(c) +} + func (c *XuanwuV1Client) StorageBackendClaims(namespace string) StorageBackendClaimInterface { return newStorageBackendClaims(c, namespace) } @@ -41,12 +48,28 @@ func (c *XuanwuV1Client) StorageBackendContents() StorageBackendContentInterface } // NewForConfig creates a new XuanwuV1Client for the given config. +// NewForConfig is equivalent to NewForConfigAndClient(c, httpClient), +// where httpClient was generated with rest.HTTPClientFor(c). func NewForConfig(c *rest.Config) (*XuanwuV1Client, error) { config := *c if err := setConfigDefaults(&config); err != nil { return nil, err } - client, err := rest.RESTClientFor(&config) + httpClient, err := rest.HTTPClientFor(&config) + if err != nil { + return nil, err + } + return NewForConfigAndClient(&config, httpClient) +} + +// NewForConfigAndClient creates a new XuanwuV1Client for the given config and http client. +// Note the http client provided takes precedence over the configured transport values. +func NewForConfigAndClient(c *rest.Config, h *http.Client) (*XuanwuV1Client, error) { + config := *c + if err := setConfigDefaults(&config); err != nil { + return nil, err + } + client, err := rest.RESTClientForConfigAndClient(&config, h) if err != nil { return nil, err } diff --git a/pkg/client/informers/externalversions/factory.go b/pkg/client/informers/externalversions/factory.go index 698512eb..e54931fd 100644 --- a/pkg/client/informers/externalversions/factory.go +++ b/pkg/client/informers/externalversions/factory.go @@ -16,13 +16,13 @@ package externalversions import ( + versioned "huawei-csi-driver/pkg/client/clientset/versioned" + internalinterfaces "huawei-csi-driver/pkg/client/informers/externalversions/internalinterfaces" + xuanwu "huawei-csi-driver/pkg/client/informers/externalversions/xuanwu" reflect "reflect" sync "sync" time "time" - versioned "huawei-csi-driver/pkg/client/clientset/versioned" - internalinterfaces "huawei-csi-driver/pkg/client/informers/externalversions/internalinterfaces" - xuanwu "huawei-csi-driver/pkg/client/informers/externalversions/xuanwu" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" schema "k8s.io/apimachinery/pkg/runtime/schema" @@ -44,6 +44,11 @@ type sharedInformerFactory struct { // startedInformers is used for tracking which informers have been started. // This allows Start() to be called multiple times safely. startedInformers map[reflect.Type]bool + // wg tracks how many goroutines were started. + wg sync.WaitGroup + // shuttingDown is true when Shutdown has been called. It may still be running + // because it needs to wait for goroutines. + shuttingDown bool } // WithCustomResyncConfig sets a custom resync period for the specified informer types. @@ -104,20 +109,39 @@ func NewSharedInformerFactoryWithOptions(client versioned.Interface, defaultResy return factory } -// Start initializes all requested informers. func (f *sharedInformerFactory) Start(stopCh <-chan struct{}) { f.lock.Lock() defer f.lock.Unlock() + if f.shuttingDown { + return + } + for informerType, informer := range f.informers { if !f.startedInformers[informerType] { - go informer.Run(stopCh) + f.wg.Add(1) + // We need a new variable in each loop iteration, + // otherwise the goroutine would use the loop variable + // and that keeps changing. + informer := informer + go func() { + defer f.wg.Done() + informer.Run(stopCh) + }() f.startedInformers[informerType] = true } } } -// WaitForCacheSync waits for all started informers' cache were synced. +func (f *sharedInformerFactory) Shutdown() { + f.lock.Lock() + f.shuttingDown = true + f.lock.Unlock() + + // Will return immediately if there is nothing to wait for. + f.wg.Wait() +} + func (f *sharedInformerFactory) WaitForCacheSync(stopCh <-chan struct{}) map[reflect.Type]bool { informers := func() map[reflect.Type]cache.SharedIndexInformer { f.lock.Lock() @@ -164,11 +188,58 @@ func (f *sharedInformerFactory) InformerFor(obj runtime.Object, newFunc internal // SharedInformerFactory provides shared informers for resources in all known // API group versions. +// +// It is typically used like this: +// +// ctx, cancel := context.Background() +// defer cancel() +// factory := NewSharedInformerFactory(client, resyncPeriod) +// defer factory.WaitForStop() // Returns immediately if nothing was started. +// genericInformer := factory.ForResource(resource) +// typedInformer := factory.SomeAPIGroup().V1().SomeType() +// factory.Start(ctx.Done()) // Start processing these informers. +// synced := factory.WaitForCacheSync(ctx.Done()) +// for v, ok := range synced { +// if !ok { +// fmt.Fprintf(os.Stderr, "caches failed to sync: %v", v) +// return +// } +// } +// +// // Creating informers can also be created after Start, but then +// // Start must be called again: +// anotherGenericInformer := factory.ForResource(resource) +// factory.Start(ctx.Done()) type SharedInformerFactory interface { internalinterfaces.SharedInformerFactory - ForResource(resource schema.GroupVersionResource) (GenericInformer, error) + + // Start initializes all requested informers. They are handled in goroutines + // which run until the stop channel gets closed. + Start(stopCh <-chan struct{}) + + // Shutdown marks a factory as shutting down. At that point no new + // informers can be started anymore and Start will return without + // doing anything. + // + // In addition, Shutdown blocks until all goroutines have terminated. For that + // to happen, the close channel(s) that they were started with must be closed, + // either before Shutdown gets called or while it is waiting. + // + // Shutdown may be called multiple times, even concurrently. All such calls will + // block until all goroutines have terminated. + Shutdown() + + // WaitForCacheSync blocks until all started informers' caches were synced + // or the stop channel gets closed. WaitForCacheSync(stopCh <-chan struct{}) map[reflect.Type]bool + // ForResource gives generic access to a shared informer of the matching type. + ForResource(resource schema.GroupVersionResource) (GenericInformer, error) + + // InternalInformerFor returns the SharedIndexInformer for obj using an internal + // client. + InformerFor(obj runtime.Object, newFunc internalinterfaces.NewInformerFunc) cache.SharedIndexInformer + Xuanwu() xuanwu.Interface } diff --git a/pkg/client/informers/externalversions/generic.go b/pkg/client/informers/externalversions/generic.go index 9c343daf..24e84c9a 100644 --- a/pkg/client/informers/externalversions/generic.go +++ b/pkg/client/informers/externalversions/generic.go @@ -17,8 +17,8 @@ package externalversions import ( "fmt" - v1 "huawei-csi-driver/client/apis/xuanwu/v1" + schema "k8s.io/apimachinery/pkg/runtime/schema" cache "k8s.io/client-go/tools/cache" ) @@ -50,6 +50,8 @@ func (f *genericInformer) Lister() cache.GenericLister { func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource) (GenericInformer, error) { switch resource { // Group=xuanwu.huawei.io, Version=v1 + case v1.SchemeGroupVersion.WithResource("resourcetopologies"): + return &genericInformer{resource: resource.GroupResource(), informer: f.Xuanwu().V1().ResourceTopologies().Informer()}, nil case v1.SchemeGroupVersion.WithResource("storagebackendclaims"): return &genericInformer{resource: resource.GroupResource(), informer: f.Xuanwu().V1().StorageBackendClaims().Informer()}, nil case v1.SchemeGroupVersion.WithResource("storagebackendcontents"): diff --git a/pkg/client/informers/externalversions/internalinterfaces/factory_interfaces.go b/pkg/client/informers/externalversions/internalinterfaces/factory_interfaces.go index 71c61189..dc3faef7 100644 --- a/pkg/client/informers/externalversions/internalinterfaces/factory_interfaces.go +++ b/pkg/client/informers/externalversions/internalinterfaces/factory_interfaces.go @@ -16,9 +16,9 @@ package internalinterfaces import ( + versioned "huawei-csi-driver/pkg/client/clientset/versioned" time "time" - versioned "huawei-csi-driver/pkg/client/clientset/versioned" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" cache "k8s.io/client-go/tools/cache" diff --git a/pkg/client/informers/externalversions/xuanwu/v1/interface.go b/pkg/client/informers/externalversions/xuanwu/v1/interface.go index 8db00946..c54db252 100644 --- a/pkg/client/informers/externalversions/xuanwu/v1/interface.go +++ b/pkg/client/informers/externalversions/xuanwu/v1/interface.go @@ -21,6 +21,8 @@ import ( // Interface provides access to all the informers in this group version. type Interface interface { + // ResourceTopologies returns a ResourceTopologyInformer. + ResourceTopologies() ResourceTopologyInformer // StorageBackendClaims returns a StorageBackendClaimInformer. StorageBackendClaims() StorageBackendClaimInformer // StorageBackendContents returns a StorageBackendContentInformer. @@ -38,6 +40,11 @@ func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakList return &version{factory: f, namespace: namespace, tweakListOptions: tweakListOptions} } +// ResourceTopologies returns a ResourceTopologyInformer. +func (v *version) ResourceTopologies() ResourceTopologyInformer { + return &resourceTopologyInformer{factory: v.factory, tweakListOptions: v.tweakListOptions} +} + // StorageBackendClaims returns a StorageBackendClaimInformer. func (v *version) StorageBackendClaims() StorageBackendClaimInformer { return &storageBackendClaimInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} diff --git a/pkg/client/informers/externalversions/xuanwu/v1/resourcetopology.go b/pkg/client/informers/externalversions/xuanwu/v1/resourcetopology.go new file mode 100644 index 00000000..3d004a1f --- /dev/null +++ b/pkg/client/informers/externalversions/xuanwu/v1/resourcetopology.go @@ -0,0 +1,86 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2022-2023. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +// Code generated by informer-gen. DO NOT EDIT. + +package v1 + +import ( + "context" + xuanwuv1 "huawei-csi-driver/client/apis/xuanwu/v1" + versioned "huawei-csi-driver/pkg/client/clientset/versioned" + internalinterfaces "huawei-csi-driver/pkg/client/informers/externalversions/internalinterfaces" + v1 "huawei-csi-driver/pkg/client/listers/xuanwu/v1" + time "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + watch "k8s.io/apimachinery/pkg/watch" + cache "k8s.io/client-go/tools/cache" +) + +// ResourceTopologyInformer provides access to a shared informer and lister for +// ResourceTopologies. +type ResourceTopologyInformer interface { + Informer() cache.SharedIndexInformer + Lister() v1.ResourceTopologyLister +} + +type resourceTopologyInformer struct { + factory internalinterfaces.SharedInformerFactory + tweakListOptions internalinterfaces.TweakListOptionsFunc +} + +// NewResourceTopologyInformer constructs a new informer for ResourceTopology type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewResourceTopologyInformer(client versioned.Interface, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { + return NewFilteredResourceTopologyInformer(client, resyncPeriod, indexers, nil) +} + +// NewFilteredResourceTopologyInformer constructs a new informer for ResourceTopology type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewFilteredResourceTopologyInformer(client versioned.Interface, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { + return cache.NewSharedIndexInformer( + &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.XuanwuV1().ResourceTopologies().List(context.TODO(), options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.XuanwuV1().ResourceTopologies().Watch(context.TODO(), options) + }, + }, + &xuanwuv1.ResourceTopology{}, + resyncPeriod, + indexers, + ) +} + +func (f *resourceTopologyInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { + return NewFilteredResourceTopologyInformer(client, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) +} + +func (f *resourceTopologyInformer) Informer() cache.SharedIndexInformer { + return f.factory.InformerFor(&xuanwuv1.ResourceTopology{}, f.defaultInformer) +} + +func (f *resourceTopologyInformer) Lister() v1.ResourceTopologyLister { + return v1.NewResourceTopologyLister(f.Informer().GetIndexer()) +} diff --git a/pkg/client/informers/externalversions/xuanwu/v1/storagebackendclaim.go b/pkg/client/informers/externalversions/xuanwu/v1/storagebackendclaim.go index 77947e9d..609c4676 100644 --- a/pkg/client/informers/externalversions/xuanwu/v1/storagebackendclaim.go +++ b/pkg/client/informers/externalversions/xuanwu/v1/storagebackendclaim.go @@ -17,12 +17,12 @@ package v1 import ( "context" - time "time" - xuanwuv1 "huawei-csi-driver/client/apis/xuanwu/v1" versioned "huawei-csi-driver/pkg/client/clientset/versioned" internalinterfaces "huawei-csi-driver/pkg/client/informers/externalversions/internalinterfaces" v1 "huawei-csi-driver/pkg/client/listers/xuanwu/v1" + time "time" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" watch "k8s.io/apimachinery/pkg/watch" diff --git a/pkg/client/informers/externalversions/xuanwu/v1/storagebackendcontent.go b/pkg/client/informers/externalversions/xuanwu/v1/storagebackendcontent.go index 2ce8e97a..8c99f2ff 100644 --- a/pkg/client/informers/externalversions/xuanwu/v1/storagebackendcontent.go +++ b/pkg/client/informers/externalversions/xuanwu/v1/storagebackendcontent.go @@ -17,12 +17,12 @@ package v1 import ( "context" - time "time" - xuanwuv1 "huawei-csi-driver/client/apis/xuanwu/v1" versioned "huawei-csi-driver/pkg/client/clientset/versioned" internalinterfaces "huawei-csi-driver/pkg/client/informers/externalversions/internalinterfaces" v1 "huawei-csi-driver/pkg/client/listers/xuanwu/v1" + time "time" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" watch "k8s.io/apimachinery/pkg/watch" diff --git a/pkg/client/listers/xuanwu/v1/expansion_generated.go b/pkg/client/listers/xuanwu/v1/expansion_generated.go index 5710574a..bee6ff6b 100644 --- a/pkg/client/listers/xuanwu/v1/expansion_generated.go +++ b/pkg/client/listers/xuanwu/v1/expansion_generated.go @@ -15,6 +15,10 @@ package v1 +// ResourceTopologyListerExpansion allows custom methods to be added to +// ResourceTopologyLister. +type ResourceTopologyListerExpansion interface{} + // StorageBackendClaimListerExpansion allows custom methods to be added to // StorageBackendClaimLister. type StorageBackendClaimListerExpansion interface{} diff --git a/pkg/client/listers/xuanwu/v1/resourcetopology.go b/pkg/client/listers/xuanwu/v1/resourcetopology.go new file mode 100644 index 00000000..933e349a --- /dev/null +++ b/pkg/client/listers/xuanwu/v1/resourcetopology.go @@ -0,0 +1,66 @@ +/* + Copyright (c) Huawei Technologies Co., Ltd. 2022-2023. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +// Code generated by lister-gen. DO NOT EDIT. + +package v1 + +import ( + v1 "huawei-csi-driver/client/apis/xuanwu/v1" + + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/tools/cache" +) + +// ResourceTopologyLister helps list ResourceTopologies. +// All objects returned here must be treated as read-only. +type ResourceTopologyLister interface { + // List lists all ResourceTopologies in the indexer. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*v1.ResourceTopology, err error) + // Get retrieves the ResourceTopology from the index for a given name. + // Objects returned here must be treated as read-only. + Get(name string) (*v1.ResourceTopology, error) + ResourceTopologyListerExpansion +} + +// resourceTopologyLister implements the ResourceTopologyLister interface. +type resourceTopologyLister struct { + indexer cache.Indexer +} + +// NewResourceTopologyLister returns a new ResourceTopologyLister. +func NewResourceTopologyLister(indexer cache.Indexer) ResourceTopologyLister { + return &resourceTopologyLister{indexer: indexer} +} + +// List lists all ResourceTopologies in the indexer. +func (s *resourceTopologyLister) List(selector labels.Selector) (ret []*v1.ResourceTopology, err error) { + err = cache.ListAll(s.indexer, selector, func(m interface{}) { + ret = append(ret, m.(*v1.ResourceTopology)) + }) + return ret, err +} + +// Get retrieves the ResourceTopology from the index for a given name. +func (s *resourceTopologyLister) Get(name string) (*v1.ResourceTopology, error) { + obj, exists, err := s.indexer.GetByKey(name) + if err != nil { + return nil, err + } + if !exists { + return nil, errors.NewNotFound(v1.Resource("resourcetopology"), name) + } + return obj.(*v1.ResourceTopology), nil +} diff --git a/pkg/client/listers/xuanwu/v1/storagebackendclaim.go b/pkg/client/listers/xuanwu/v1/storagebackendclaim.go index 4cf23212..53af1581 100644 --- a/pkg/client/listers/xuanwu/v1/storagebackendclaim.go +++ b/pkg/client/listers/xuanwu/v1/storagebackendclaim.go @@ -17,6 +17,7 @@ package v1 import ( v1 "huawei-csi-driver/client/apis/xuanwu/v1" + "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/tools/cache" diff --git a/pkg/client/listers/xuanwu/v1/storagebackendcontent.go b/pkg/client/listers/xuanwu/v1/storagebackendcontent.go index 2554444f..9b70c02b 100644 --- a/pkg/client/listers/xuanwu/v1/storagebackendcontent.go +++ b/pkg/client/listers/xuanwu/v1/storagebackendcontent.go @@ -17,6 +17,7 @@ package v1 import ( v1 "huawei-csi-driver/client/apis/xuanwu/v1" + "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/tools/cache" diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index 060e3663..343324ea 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -15,10 +15,12 @@ // Package constants is related with provider constants package constants +import "errors" + type FileType string const ( - ProviderVersion = "4.1.0" + ProviderVersion = "4.2.0" ProviderVendorName = "Huawei" EndpointDirPermission = 0755 @@ -40,6 +42,32 @@ const ( Ext3 FileType = "ext3" Ext4 FileType = "ext4" Xfs FileType = "xfs" + + // NodeNameEnv is defined in helm file + NodeNameEnv = "CSI_NODENAME" + + // DefaultDriverName is default huawei csi driver name + DefaultDriverName = "csi.huawei.com" + // DefaultTopoDriverName is default topo driver name + DefaultTopoDriverName = "cmi.huawei.com" + + // PVKind is defined by k8s + PVKind = "PersistentVolume" + // PodKind is defined by k8s + PodKind = "Pod" + // TopologyKind is topology resource kind + TopologyKind = "ResourceTopology" + + // KubernetesV1 is kubernetes v1 api version + KubernetesV1 = "v1" + // XuanwuV1 is xuanwu v1 api version + XuanwuV1 = "xuanwu.huawei.io/v1" + + NotMountStr = "not mounted" +) + +var ( + ErrTimeout = errors.New("timeout") ) // DRCSIConfig contains storage normal configuration diff --git a/pkg/constants/storage.go b/pkg/constants/storage.go new file mode 100644 index 00000000..c7f19821 --- /dev/null +++ b/pkg/constants/storage.go @@ -0,0 +1,46 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package constants is storage-related constants +package constants + +const ( + // OceanStorDoradoV6 Dorado V6 and OceanStor V6 are exactly the same + OceanStorDoradoV6 = "DoradoV6" + // OceanStorDoradoV3 is dorado v3 + OceanStorDoradoV3 = "DoradoV3" + // OceanStorV3 is oceanstor v3 + OceanStorV3 = "OceanStorV3" + // OceanStorV5 is oceanstor v5 + OceanStorV5 = "OceanStorV5" + + // MinVersionSupportLabel version gte 6.1.7 support label function + MinVersionSupportLabel = "6.1.7" +) + +// BackendCapability backend capability +type BackendCapability string + +var SupportThin BackendCapability = "SupportThin" +var SupportThick BackendCapability = "SupportThick" +var SupportQoS BackendCapability = "SupportQoS" +var SupportQuota BackendCapability = "SupportQuota" +var SupportClone BackendCapability = "SupportClone" +var SupportMetro BackendCapability = "SupportMetro" +var SupportReplication BackendCapability = "SupportReplication" +var SupportApplicationType BackendCapability = "SupportApplicationType" +var SupportMetroNAS BackendCapability = "SupportMetroNAS" +var SupportLabel BackendCapability = "SupportLabel" diff --git a/pkg/sidecar/controller/content_sync.go b/pkg/sidecar/controller/content_sync.go index e23ec7cc..72f8db05 100644 --- a/pkg/sidecar/controller/content_sync.go +++ b/pkg/sidecar/controller/content_sync.go @@ -111,6 +111,16 @@ func (ctrl *backendController) shouldUpdateContent(ctx context.Context, content needUpdate = true } + if content.Status.UseCert != content.Spec.UseCert { + content.Status.UseCert = content.Spec.UseCert + needUpdate = true + } + + if content.Status.CertSecret != content.Spec.CertSecret { + content.Status.CertSecret = content.Spec.CertSecret + needUpdate = true + } + if content.Status.MaxClientThreads != content.Spec.MaxClientThreads { content.Status.MaxClientThreads = content.Spec.MaxClientThreads needUpdate = true diff --git a/pkg/sidecar/controller/sidecar_controller.go b/pkg/sidecar/controller/sidecar_controller.go index 0ca38ec0..2b6b7d8d 100644 --- a/pkg/sidecar/controller/sidecar_controller.go +++ b/pkg/sidecar/controller/sidecar_controller.go @@ -423,10 +423,8 @@ func (ctrl *backendController) updateContentTask(ctx context.Context, return nil, errors.New(msg) } - // start to update the secret info, only secret or maxClientThreads changed, we will update - if content.Status == nil || (content.Spec.SecretMeta == content.Status.SecretMeta && - content.Spec.MaxClientThreads == content.Status.MaxClientThreads && - content.Status.SN != "") { + // start to update the secret info, only secret, useCert, certSecret or maxClientThreads changed, we will update + if !needUpdate(content) { return nil, nil } @@ -470,3 +468,31 @@ func (ctrl *backendController) getContentTask(ctx context.Context, "storageBackendContent": newContent, }, nil } + +func needUpdate(content *xuanwuv1.StorageBackendContent) bool { + if content.Status == nil { + return false + } + + if content.Spec.SecretMeta != content.Status.SecretMeta { + return true + } + + if content.Spec.MaxClientThreads != content.Status.MaxClientThreads { + return true + } + + if content.Spec.UseCert != content.Status.UseCert { + return true + } + + if content.Spec.CertSecret != content.Status.CertSecret { + return true + } + + if content.Status.SN == "" { + return true + } + + return false +} diff --git a/pkg/storage-backend/controller/claim_sync.go b/pkg/storage-backend/controller/claim_sync.go index 77cbefdc..4bf3da25 100644 --- a/pkg/storage-backend/controller/claim_sync.go +++ b/pkg/storage-backend/controller/claim_sync.go @@ -542,6 +542,8 @@ func (ctrl *BackendController) updateStorageBackendClaim(ctx context.Context, cl *xuanwuv1.StorageBackendClaim, error) { claim.Status.MaxClientThreads = claim.Spec.MaxClientThreads claim.Status.SecretMeta = claim.Spec.SecretMeta + claim.Status.UseCert = claim.Spec.UseCert + claim.Status.CertSecret = claim.Spec.CertSecret newClaim, err := ctrl.updateClaimStatusWithEvent(ctx, claim, "UpdateClaim", "Successful update claim for storageBackendClaim") if err != nil { @@ -559,6 +561,8 @@ func (ctrl *BackendController) updateStorageBackendClaim(ctx context.Context, cl content.Spec.MaxClientThreads = claim.Spec.MaxClientThreads content.Spec.SecretMeta = claim.Spec.SecretMeta + content.Spec.UseCert = claim.Spec.UseCert + content.Spec.CertSecret = claim.Spec.CertSecret _, err = utils.UpdateContent(ctx, ctrl.clientSet, content) if err != nil { log.AddContext(ctx).Errorf("updateStorageBackendClaim: update storageBackendContent %s failed, "+ diff --git a/pkg/utils/k8s.go b/pkg/utils/k8s.go index 54aad5a0..a3d3c461 100644 --- a/pkg/utils/k8s.go +++ b/pkg/utils/k8s.go @@ -19,14 +19,24 @@ package utils import ( "context" + "crypto/x509" + "encoding/pem" "errors" "fmt" coreV1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/scheme" + clientV1 "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/tools/record" xuanwuv1 "huawei-csi-driver/client/apis/xuanwu/v1" "huawei-csi-driver/csi/app" + clientSet "huawei-csi-driver/pkg/client/clientset/versioned" + "huawei-csi-driver/pkg/constants" "huawei-csi-driver/utils" "huawei-csi-driver/utils/log" ) @@ -65,6 +75,48 @@ func GetPasswordFromBackendID(ctx context.Context, backendID string) (string, er return utils.GetPasswordFromSecret(ctx, secretName, namespace) } +// GetCertSecretFromBackendID get cert secret meta from backend +func GetCertSecretFromBackendID(ctx context.Context, backendID string) (bool, string, error) { + useCert, secret, err := GetCertMeta(ctx, backendID) + if err != nil { + return false, "", err + } + return useCert, secret, nil +} + +// GetCertPool get cert pool +func GetCertPool(ctx context.Context, useCert bool, secretMeta string) (bool, *x509.CertPool, error) { + if !useCert { + log.AddContext(ctx).Infoln("useCert is false, skip get cert pool") + return false, nil, nil + } + log.AddContext(ctx).Infoln("Start get cert pool") + + namespace, secretName, err := SplitMetaNamespaceKey(secretMeta) + if err != nil { + return false, nil, fmt.Errorf("split secret secretMeta %s namespace failed, error: %v", secretMeta, err) + } + + certMeta, err := utils.GetCertFromSecret(ctx, secretName, namespace) + if err != nil { + return false, nil, fmt.Errorf("get cert from secret %s failed, error: %v", secretName, err) + } + + certBlock, _ := pem.Decode(certMeta) + if certBlock == nil { + return false, nil, fmt.Errorf("certificate data decode failed") + } + + cert, err := x509.ParseCertificate(certBlock.Bytes) + if err != nil { + return false, nil, fmt.Errorf("error parse certificate: %v", err) + } + + certPool := x509.NewCertPool() + certPool.AddCert(cert) + return true, certPool, nil +} + func GetBackendConfigmapByClaimName(ctx context.Context, claimNameMeta string) (*coreV1.ConfigMap, error) { log.AddContext(ctx).Infof("Get configmap meta data by claim meta: [%s]", claimNameMeta) configmapMeta, _, err := GetConfigMeta(ctx, claimNameMeta) @@ -108,7 +160,6 @@ func GetClaimByMeta(ctx context.Context, claimNameMeta string) (*xuanwuv1.Storag msg := fmt.Sprintf("StorageBackendClaim: [%s] is nil, get claim failed.", claimName) return nil, Errorln(ctx, msg) } - return claim, nil } @@ -128,6 +179,23 @@ func GetConfigMeta(ctx context.Context, claimNameMeta string) (string, string, e return claim.Spec.ConfigMapMeta, claim.Spec.SecretMeta, nil } +// GetCertMeta get cert meta from backend claim +func GetCertMeta(ctx context.Context, claimNameMeta string) (bool, string, error) { + log.AddContext(ctx).Infof("Get claim: [%s] config meta.", claimNameMeta) + + claim, err := GetClaimByMeta(ctx, claimNameMeta) + if err != nil { + return false, "", err + } + + if claim == nil { + msg := fmt.Sprintf("Get claim failed, claim: [%s] is nil", claimNameMeta) + return false, "", Errorln(ctx, msg) + } + + return claim.Spec.UseCert, claim.Spec.CertSecret, nil +} + func GetContentByClaimMeta(ctx context.Context, claimNameMeta string) (*xuanwuv1.StorageBackendContent, error) { log.AddContext(ctx).Debugf("Start to get storageBackendContent by claimMeta: [%s].", claimNameMeta) @@ -171,7 +239,6 @@ func GetSBCTOnlineStatusByClaim(ctx context.Context, backendID string) (bool, er content.Name) return false, Errorln(ctx, msg) } - if content.Status == nil { msg := fmt.Sprintf("StorageBackendContent: [%s] content.status is nil, GetSBCTOnlineStatusByClaim failed.", content.Name) @@ -181,6 +248,39 @@ func GetSBCTOnlineStatusByClaim(ctx context.Context, backendID string) (bool, er return content.Status.Online, nil } +// GetSBCTCapabilitiesByClaim get backend capabilities +func GetSBCTCapabilitiesByClaim(ctx context.Context, backendID string) (map[string]bool, error) { + content, err := GetContentByClaimMeta(ctx, backendID) + if err != nil { + msg := fmt.Sprintf("GetContentByClaimMeta: [%s] failed, err: [%v]", backendID, err) + return nil, Errorln(ctx, msg) + } + + if content == nil { + msg := fmt.Sprintf("StorageBackendContent: [%s] content is nil, GetSBCTOnlineStatusByClaim failed.", + content.Name) + return nil, Errorln(ctx, msg) + } + if content.Status == nil { + msg := fmt.Sprintf("StorageBackendContent: [%s] content.status is nil, "+ + "GetSBCTOnlineStatusByClaim failed.", content.Name) + return nil, Errorln(ctx, msg) + } + + return content.Status.Capabilities, nil +} + +// IsBackendCapabilitySupport valid backend capability +func IsBackendCapabilitySupport(ctx context.Context, backendID string, + capability constants.BackendCapability) (bool, error) { + capabilities, err := GetSBCTCapabilitiesByClaim(ctx, backendID) + if err != nil { + log.AddContext(ctx).Errorf("GetSBCTCapabilitiesByClaim failed, backendID: %v, err: %v", backendID, err) + return false, err + } + return capabilities[string(capability)], nil +} + func SetSBCTOnlineStatus(ctx context.Context, content *xuanwuv1.StorageBackendContent, status bool) error { content.Status.Online = status @@ -216,3 +316,51 @@ func SetStorageBackendContentOnlineStatus(ctx context.Context, backendID string, backendID, online) return nil } + +// GetK8SAndSBCClient return k8sClient, storageBackendClient +func GetK8SAndSBCClient(ctx context.Context) (*kubernetes.Clientset, *clientSet.Clientset, error) { + var config *rest.Config + var err error + if app.GetGlobalConfig().KubeConfig != "" { + config, err = clientcmd.BuildConfigFromFlags("", app.GetGlobalConfig().KubeConfig) + } else { + config, err = rest.InClusterConfig() + } + + if err != nil { + log.AddContext(ctx).Errorf("Error getting cluster config, kube config: %s, %v", + app.GetGlobalConfig().KubeConfig, err) + return nil, nil, err + } + + k8sClient, err := kubernetes.NewForConfig(config) + if err != nil { + log.AddContext(ctx).Errorf("Error getting kubernetes client, %v", err) + return nil, nil, err + } + + storageBackendClient, err := clientSet.NewForConfig(config) + if err != nil { + log.AddContext(ctx).Errorf("Error getting storage backend client, %v", err) + return nil, nil, err + } + + return k8sClient, storageBackendClient, nil +} + +// InitRecorder used to init event recorder +func InitRecorder(client kubernetes.Interface, componentName string) record.EventRecorder { + eventBroadcaster := record.NewBroadcaster() + eventBroadcaster.StartRecordingToSink(&clientV1.EventSinkImpl{Interface: client.CoreV1().Events(v1.NamespaceAll)}) + return eventBroadcaster.NewRecorder(scheme.Scheme, coreV1.EventSource{Component: fmt.Sprintf(componentName)}) +} + +// GetEventRecorder used to get event recorder +func GetEventRecorder(ctx context.Context) record.EventRecorder { + c, _, err := GetK8SAndSBCClient(ctx) + if err != nil { + log.AddContext(ctx).Errorf("GetK8SAndSBCClient failed.") + } + + return InitRecorder(c, "huawei-csi") +} diff --git a/pkg/utils/label.go b/pkg/utils/label.go new file mode 100644 index 00000000..9be69e08 --- /dev/null +++ b/pkg/utils/label.go @@ -0,0 +1,114 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package utils is label-related function and method. +package utils + +import ( + "context" + "fmt" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "strings" + + xuanwuv1 "huawei-csi-driver/client/apis/xuanwu/v1" + "huawei-csi-driver/csi/app" + "huawei-csi-driver/pkg/constants" + "huawei-csi-driver/utils" + "huawei-csi-driver/utils/log" +) + +// GetTopoName get topo name +func GetTopoName(pvName string) string { + return fmt.Sprintf("topo-%s", strings.ReplaceAll(pvName, "_", "-")) +} + +// CreatePVLabel create label when create pvc +var CreatePVLabel = func(pvName, volumeId string) { + ctx := utils.NewContext() + backendName, _ := utils.SplitVolumeId(volumeId) + backendName = MakeMetaWithNamespace(app.GetGlobalConfig().Namespace, backendName) + + supportLabel, err := IsBackendCapabilitySupport(ctx, backendName, constants.SupportLabel) + if err != nil { + log.AddContext(ctx).Errorf("CreatePVLabel get backend capability support failed,"+ + " backendName: %v, label: %v, err: %v", backendName, supportLabel, err) + } + if supportLabel { + CreateLabel(pvName, volumeId) + } +} + +// CreateLabel used to create ResourceTopology resource +var CreateLabel = func(pvName, volumeId string) { + ctx := utils.NewContext() + var err error + _, volumeName := utils.SplitVolumeId(volumeId) + topologyName := GetTopoName(volumeName) + + topologySpec := xuanwuv1.ResourceTopologySpec{ + Provisioner: constants.DefaultTopoDriverName, + VolumeHandle: volumeId, + Tags: []xuanwuv1.Tag{ + { + ResourceInfo: xuanwuv1.ResourceInfo{ + TypeMeta: metav1.TypeMeta{Kind: constants.PVKind, APIVersion: constants.KubernetesV1}, + Name: pvName, + }, + }, + }, + } + + _, err = app.GetGlobalConfig().BackendUtils.XuanwuV1().ResourceTopologies().Create(ctx, + &xuanwuv1.ResourceTopology{ + TypeMeta: metav1.TypeMeta{Kind: constants.TopologyKind, APIVersion: constants.XuanwuV1}, + ObjectMeta: metav1.ObjectMeta{Name: topologyName}, + Spec: topologySpec, + }, + metav1.CreateOptions{}) + + if err != nil { + log.AddContext(ctx).Errorf("Create topologies for pv: %s failed. error: %v", volumeName, err) + } else { + log.AddContext(ctx).Infof("Create topologies: %s success.", topologyName) + } +} + +// DeletePVLabel delete label when delete pvc +var DeletePVLabel = func(volumeId string) { + ctx := utils.NewContext() + backendName, volName := utils.SplitVolumeId(volumeId) + + backendName = MakeMetaWithNamespace(app.GetGlobalConfig().Namespace, backendName) + supportLabel, err := IsBackendCapabilitySupport(ctx, backendName, constants.SupportLabel) + if err != nil { + log.AddContext(ctx).Errorf("DeletePVLabel get backend capability support failed,"+ + " backendName: %v, label: %v, err: %v", backendName, supportLabel, err) + } + if supportLabel { + DeleteLabel(volName) + } +} + +// DeleteLabel used to delete label resource +var DeleteLabel = func(volName string) { + err := app.GetGlobalConfig().BackendUtils.XuanwuV1().ResourceTopologies().Delete(context.TODO(), + GetTopoName(volName), metav1.DeleteOptions{}) + if err != nil { + log.Errorf("Delete topologies: %s failed. error: %v", GetTopoName(volName), err) + } else { + log.Infof("Delete topologies: %s success.", GetTopoName(volName)) + } +} diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index a63e6975..03de3328 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -230,9 +230,31 @@ func Errorf(ctx context.Context, format string, a ...interface{}) error { // NeedChangeContent returns need update content or not func NeedChangeContent(storageBackend *xuanwuv1.StorageBackendClaim) bool { - return storageBackend.Status != nil && storageBackend.Status.BoundContentName != "" && - (storageBackend.Status.SecretMeta != storageBackend.Spec.SecretMeta || - storageBackend.Status.MaxClientThreads != storageBackend.Spec.MaxClientThreads) + if storageBackend.Status == nil { + return false + } + + if storageBackend.Status.BoundContentName == "" { + return false + } + + if storageBackend.Status.SecretMeta != storageBackend.Spec.SecretMeta { + return true + } + + if storageBackend.Status.MaxClientThreads != storageBackend.Spec.MaxClientThreads { + return true + } + + if storageBackend.Status.UseCert != storageBackend.Spec.UseCert { + return true + } + + if storageBackend.Status.CertSecret != storageBackend.Spec.CertSecret { + return true + } + + return false } // GetNameSpaceFromEnv get the namespace from the env diff --git a/pkg/webhook/convert.go b/pkg/webhook/convert.go new file mode 100644 index 00000000..77cd06d9 --- /dev/null +++ b/pkg/webhook/convert.go @@ -0,0 +1,61 @@ +/* +Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package webhook validate the request +package webhook + +import ( + admissionV1 "k8s.io/api/admission/v1" + admissionV1beta1 "k8s.io/api/admission/v1beta1" +) + +func transformV1beta1AdmitFuncToV1AdmitFunc(f admitV1Func) admitV1beta1Func { + return func(review admissionV1beta1.AdmissionReview) *admissionV1beta1.AdmissionResponse { + v1Review := admissionV1.AdmissionReview{ + Request: &admissionV1.AdmissionRequest{ + Kind: review.Request.Kind, + Namespace: review.Request.Namespace, + Name: review.Request.Name, + Object: review.Request.Object, + Resource: review.Request.Resource, + Operation: admissionV1.Operation(review.Request.Operation), + UID: review.Request.UID, + DryRun: review.Request.DryRun, + OldObject: review.Request.OldObject, + Options: review.Request.Options, + RequestKind: review.Request.RequestKind, + RequestResource: review.Request.RequestResource, + RequestSubResource: review.Request.RequestSubResource, + SubResource: review.Request.SubResource, + UserInfo: review.Request.UserInfo, + }} + v1Response := f(v1Review) + + var pt *admissionV1beta1.PatchType + if v1Response.PatchType != nil { + t := admissionV1beta1.PatchType(*v1Response.PatchType) + pt = &t + } + + return &admissionV1beta1.AdmissionResponse{ + UID: v1Response.UID, + Allowed: v1Response.Allowed, + AuditAnnotations: v1Response.AuditAnnotations, + Patch: v1Response.Patch, + PatchType: pt, + Result: v1Response.Result, + Warnings: v1Response.Warnings, + } + } +} diff --git a/pkg/webhook/convert_test.go b/pkg/webhook/convert_test.go new file mode 100644 index 00000000..96754e20 --- /dev/null +++ b/pkg/webhook/convert_test.go @@ -0,0 +1,42 @@ +/* +Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package webhook validate the request +package webhook + +import ( + "testing" + + admissionV1beta1 "k8s.io/api/admission/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +func TestTransformV1beta1AdmitFuncToV1AdmitFunc(t *testing.T) { + v1beta1Func := transformV1beta1AdmitFuncToV1AdmitFunc(admitStorageBackendClaim) + req := admissionV1beta1.AdmissionReview{ + TypeMeta: metav1.TypeMeta{}, + Request: &admissionV1beta1.AdmissionRequest{ + Operation: admissionV1beta1.Create, + Object: runtime.RawExtension{}, + OldObject: runtime.RawExtension{}, + }, + Response: nil, + } + + res := v1beta1Func(req) + if res.Allowed { + t.Errorf("res [%v] error, should be not allowed", res) + } +} diff --git a/pkg/webhook/validate_webhook.go b/pkg/webhook/validate_webhook.go index 374773e2..658d67e6 100644 --- a/pkg/webhook/validate_webhook.go +++ b/pkg/webhook/validate_webhook.go @@ -68,7 +68,7 @@ func CreateValidateWebhook(ctx context.Context, admissionWebhook AdmissionWebHoo }}, SideEffects: &sideEffect, FailurePolicy: &failurePolicy, - AdmissionReviewVersions: []string{"v1"}, + AdmissionReviewVersions: []string{"v1", "v1beta1"}, MatchPolicy: &matchPolicy, } diff --git a/pkg/webhook/webhook.go b/pkg/webhook/webhook.go index fbb60caf..5e72a030 100644 --- a/pkg/webhook/webhook.go +++ b/pkg/webhook/webhook.go @@ -28,6 +28,7 @@ import ( "sync" admissionV1 "k8s.io/api/admission/v1" + admissionV1beta1 "k8s.io/api/admission/v1beta1" apisErrors "k8s.io/apimachinery/pkg/api/errors" metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -87,14 +88,18 @@ var AdmitFunc func(admissionV1.AdmissionReview) *admissionV1.AdmissionResponse // admitV1Func handles a v1 admission type admitV1Func func(admissionV1.AdmissionReview) *admissionV1.AdmissionResponse +type admitV1beta1Func func(admissionV1beta1.AdmissionReview) *admissionV1beta1.AdmissionResponse + // admitHandler is a handler, for both validators and mutators, that supports multiple admission review versions type admitHandler struct { - admitV1 admitV1Func + admitV1 admitV1Func + admitV1Beta1 admitV1beta1Func } func newDelegateToV1AdmitHandler(f admitV1Func) admitHandler { return admitHandler{ - admitV1: f, + admitV1: f, + admitV1Beta1: transformV1beta1AdmitFuncToV1AdmitFunc(f), } } @@ -115,6 +120,23 @@ func (c *Controller) getV1AdmissionReview(ctx context.Context, obj runtime.Objec return responseAdmissionReview, nil } +func (c *Controller) getV1Beta1AdmissionReview(ctx context.Context, obj runtime.Object, + gvk *schema.GroupVersionKind, admit admitHandler) ( + runtime.Object, error) { + requestedAdmissionReview, ok := obj.(*admissionV1beta1.AdmissionReview) + if !ok { + msg := fmt.Sprintf("Expected v1beta1.AdmissionReview but got: %T", obj) + log.AddContext(ctx).Errorln(msg) + return nil, errors.New(msg) + } + + responseAdmissionReview := &admissionV1beta1.AdmissionReview{} + responseAdmissionReview.SetGroupVersionKind(*gvk) + responseAdmissionReview.Response = admit.admitV1Beta1(*requestedAdmissionReview) + responseAdmissionReview.Response.UID = requestedAdmissionReview.Request.UID + return responseAdmissionReview, nil +} + func (c *Controller) getRequestBody(ctx context.Context, w http.ResponseWriter, r *http.Request) ([]byte, error) { if r.Body == nil { msg := "expected request body to be non-empty" @@ -168,6 +190,9 @@ func (c *Controller) serve(w http.ResponseWriter, r *http.Request, admit admitHa case admissionV1.SchemeGroupVersion.WithKind("AdmissionReview"): responseObj, err = c.getV1AdmissionReview(ctx, obj, gvk, admit) admissionVersion = "v1.AdmissionReview" + case admissionV1beta1.SchemeGroupVersion.WithKind("AdmissionReview"): + responseObj, err = c.getV1Beta1AdmissionReview(ctx, obj, gvk, admit) + admissionVersion = "v1beta1.AdmissionReview" default: err = errors.New("unsupported group version") admissionVersion = fmt.Sprintf("%v", gvk) @@ -419,8 +444,7 @@ func validateCommon(ctx context.Context, claim *xuanwuv1.StorageBackendClaim) er log.AddContext(ctx).Infof("claim name: %s", claim.Name) storageInfo, err := backend.GetStorageBackendInfo(ctx, utils.MakeMetaWithNamespace(app.GetGlobalConfig().Namespace, claim.Name), - claim.Spec.ConfigMapMeta, - claim.Spec.SecretMeta) + claim.Spec.ConfigMapMeta, claim.Spec.SecretMeta, claim.Spec.CertSecret, claim.Spec.UseCert) if err != nil { return err } diff --git a/proto/proto.go b/proto/proto.go index 2f79e29f..e2fd1422 100644 --- a/proto/proto.go +++ b/proto/proto.go @@ -79,7 +79,11 @@ func VerifyIscsiPortals(portals []interface{}) ([]string, error) { var verifiedPortals []string for _, i := range portals { - portal := i.(string) + portal, ok := i.(string) + if !ok { + log.Warningf("VerifyIscsiPortals, convert portal to string failed, data: %v", i) + continue + } ip := net.ParseIP(portal) if ip == nil { return nil, fmt.Errorf("%s of portals is invalid", portal) diff --git a/storage/fusionstorage/client/client.go b/storage/fusionstorage/client/client.go index 959c61c9..bf6b222c 100644 --- a/storage/fusionstorage/client/client.go +++ b/storage/fusionstorage/client/client.go @@ -40,7 +40,6 @@ import ( const ( noAuthenticated int64 = 10000003 offLineCodeInt int64 = 1077949069 - offLineCode = "1077949069" defaultParallelCount int = 50 maxParallelCount int = 1000 @@ -50,6 +49,8 @@ const ( loginFailedWithArg = 1077987870 userPasswordInvalid = 1073754390 IPLock = 1077949071 + + unconnectedError = "unconnected" ) var ( @@ -81,6 +82,8 @@ type Client struct { secretNamespace string secretName string backendID string + useCert bool + certSecretMeta string accountName string accountId int @@ -100,14 +103,16 @@ type NewClientConfig struct { ParallelNum string BackendID string AccountName string + UseCert bool + CertSecretMeta string } -func NewClient(url, user, secretName, secretNamespace, parallelNum, backendID, accountName string) *Client { +func NewClient(clientConfig *NewClientConfig) *Client { var err error var parallelCount int - if len(parallelNum) > 0 { - parallelCount, err = strconv.Atoi(parallelNum) + if len(clientConfig.ParallelNum) > 0 { + parallelCount, err = strconv.Atoi(clientConfig.ParallelNum) if err != nil || parallelCount > maxParallelCount || parallelCount < minParallelCount { log.Warningf("The config parallelNum %d is invalid, set it to the default value %d", parallelCount, defaultParallelCount) parallelCount = defaultParallelCount @@ -119,12 +124,14 @@ func NewClient(url, user, secretName, secretNamespace, parallelNum, backendID, a log.Infof("Init parallel count is %d", parallelCount) clientSemaphore = utils.NewSemaphore(parallelCount) return &Client{ - url: url, - user: user, - secretName: secretName, - secretNamespace: secretNamespace, - backendID: backendID, - accountName: accountName, + url: clientConfig.Url, + user: clientConfig.User, + secretName: clientConfig.SecretName, + secretNamespace: clientConfig.SecretNamespace, + backendID: clientConfig.BackendID, + accountName: clientConfig.AccountName, + useCert: clientConfig.UseCert, + certSecretMeta: clientConfig.CertSecretMeta, } } @@ -136,10 +143,20 @@ func (cli *Client) DuplicateClient() *Client { } func (cli *Client) ValidateLogin(ctx context.Context) error { - jar, _ := cookiejar.New(nil) + jar, err := cookiejar.New(nil) + if err != nil { + log.AddContext(ctx).Errorf("create jar failed, error: %v", err) + return err + } + + useCert, certPool, err := pkgUtils.GetCertPool(ctx, cli.useCert, cli.certSecretMeta) + if err != nil { + return err + } + cli.client = &http.Client{ Transport: &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + TLSClientConfig: &tls.Config{InsecureSkipVerify: !useCert, RootCAs: certPool}, }, Jar: jar, Timeout: 60 * time.Second, @@ -172,13 +189,11 @@ func (cli *Client) ValidateLogin(ctx context.Context) error { } func (cli *Client) Login(ctx context.Context) error { - jar, _ := cookiejar.New(nil) - cli.client = &http.Client{ - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - }, - Jar: jar, - Timeout: 60 * time.Second, + var err error + cli.client, err = newHTTPClientByBackendID(ctx, cli.backendID) + if err != nil { + log.AddContext(ctx).Errorf("get http client by backend %s failed, err is %v", cli.backendID, err) + return err } log.AddContext(ctx).Infof("Try to login %s.", cli.url) @@ -225,16 +240,12 @@ func (cli *Client) Login(ctx context.Context) error { cli.authToken = respHeader["X-Auth-Token"][0] - err = cli.setAccountId(ctx) - if err != nil { - return pkgUtils.Errorln(ctx, fmt.Sprintf("setAccountId failed, error: %v", err)) - } - log.AddContext(ctx).Infof("Login %s success", cli.url) return nil } -func (cli *Client) setAccountId(ctx context.Context) error { +func (cli *Client) SetAccountId(ctx context.Context) error { + log.AddContext(ctx).Debugf("setAccountId start. account name: %s", cli.accountName) if cli.accountName == "" { cli.accountName = types.DefaultAccountName cli.accountId = types.DefaultAccountId @@ -287,9 +298,21 @@ func (cli *Client) KeepAlive(ctx context.Context) { } } -func (cli *Client) doCall(ctx context.Context, - method string, url string, - data map[string]interface{}) (http.Header, []byte, error) { +func (cli *Client) reLoginLock(ctx context.Context) { + log.AddContext(ctx).Debugln("Try to reLoginLock.") + cli.reloginMutex.Lock() + log.AddContext(ctx).Debugln("ReLoginLock success.") +} + +func (cli *Client) reLoginUnlock(ctx context.Context) { + log.AddContext(ctx).Debugln("Try to reLoginUnlock.") + cli.reloginMutex.Unlock() + log.AddContext(ctx).Debugln("ReLoginUnlock success.") +} + +func (cli *Client) doCall(ctx context.Context, method string, url string, data map[string]any) ( + http.Header, []byte, error) { + var err error var reqUrl string var reqBody io.Reader @@ -315,12 +338,14 @@ func (cli *Client) doCall(ctx context.Context, req.Header.Set("Referer", cli.url) req.Header.Set("Content-Type", "application/json") + // When the non-login/logout interface is invoked, if a thread is relogin, the new token is used after the relogin + // is complete. This prevents the relogin interface from being invoked for multiple times. if url != "/dsware/service/v1.3/sec/login" && url != "/dsware/service/v1.3/sec/logout" { - cli.reloginMutex.Lock() + cli.reLoginLock(ctx) if cli.authToken != "" { req.Header.Set("X-Auth-Token", cli.authToken) } - cli.reloginMutex.Unlock() + cli.reLoginUnlock(ctx) } else { if cli.authToken != "" { req.Header.Set("X-Auth-Token", cli.authToken) @@ -328,15 +353,15 @@ func (cli *Client) doCall(ctx context.Context, } log.FilteredLog(ctx, isFilterLog(method, url), utils.IsDebugLog(method, url, debugLog), - fmt.Sprintf("Request method: %s, url: %s, body: %v", method, reqUrl, data)) + fmt.Sprintf("Request method: %s, url: %s, body: %v", method, req.URL, data)) clientSemaphore.Acquire() defer clientSemaphore.Release() resp, err := cli.client.Do(req) if err != nil { - log.AddContext(ctx).Errorf("Send request method: %s, url: %s, error: %v", method, reqUrl, err) - return nil, nil, errors.New("unconnected") + log.AddContext(ctx).Errorf("Send request method: %s, url: %s, error: %v", method, req.URL, err) + return nil, nil, errors.New(unconnectedError) } defer resp.Body.Close() @@ -348,14 +373,14 @@ func (cli *Client) doCall(ctx context.Context, } log.FilteredLog(ctx, isFilterLog(method, url), utils.IsDebugLog(method, url, debugLog), - fmt.Sprintf("Response method: %s, url: %s, body: %s", method, reqUrl, respBody)) + fmt.Sprintf("Response method: %s, url: %s, body: %s", method, req.URL, respBody)) return resp.Header, respBody, nil } func (cli *Client) baseCall(ctx context.Context, method string, url string, data map[string]interface{}) (http.Header, - map[string]interface{}, error) { - var body map[string]interface{} + map[string]any, error) { + var body map[string]any respHeader, respBody, err := cli.doCall(ctx, method, url, data) if err != nil { return nil, nil, err @@ -368,54 +393,43 @@ func (cli *Client) baseCall(ctx context.Context, method string, url string, data return respHeader, body, nil } -func (cli *Client) call(ctx context.Context, - method string, url string, - data map[string]interface{}) (http.Header, map[string]interface{}, error) { - var body map[string]interface{} +func (cli *Client) retryCall(ctx context.Context, method string, url string, data map[string]any) ( + http.Header, map[string]any, error) { - respHeader, respBody, err := cli.doCall(ctx, method, url, data) + log.AddContext(ctx).Debugf("retry call: method: %s, url: %s, data: %v.", method, url, data) + var err error + var respHeader http.Header + var respBody []byte + err = cli.reLogin(ctx) if err != nil { - if err.Error() == "unconnected" { - goto RETRY - } + return nil, nil, err + } + respHeader, respBody, err = cli.doCall(ctx, method, url, data) + if err != nil { return nil, nil, err } + var body map[string]any err = json.Unmarshal(respBody, &body) if err != nil { log.AddContext(ctx).Errorf("Unmarshal response body %s error: %v", respBody, err) return nil, nil, err } - if errorCodeInterface, exist := body["errorCode"]; exist { - if errorCode, ok := errorCodeInterface.(string); ok && errorCode == offLineCode { - log.AddContext(ctx).Warningf("User offline, try to relogin %s", cli.url) - goto RETRY - } - - // Compatible with int error code 1077949069 - if errorCode, ok := errorCodeInterface.(float64); ok && int64(errorCode) == offLineCodeInt { - log.AddContext(ctx).Warningf("User offline, try to relogin %s", cli.url) - goto RETRY - } - - // Compatible with FusionStorage 6.3 - if errorCode, ok := errorCodeInterface.(float64); ok && int64(errorCode) == noAuthenticated { - log.AddContext(ctx).Warningf("User offline, try to relogin %s", cli.url) - goto RETRY - } - } return respHeader, body, nil +} -RETRY: - err = cli.reLogin(ctx) - if err == nil { - respHeader, respBody, err = cli.doCall(ctx, method, url, data) - } +func (cli *Client) call(ctx context.Context, method string, url string, data map[string]any) ( + http.Header, map[string]any, error) { + var body map[string]any + respHeader, respBody, err := cli.doCall(ctx, method, url, data) if err != nil { + if err.Error() == unconnectedError { + return cli.retryCall(ctx, method, url, data) + } return nil, nil, err } @@ -425,14 +439,25 @@ RETRY: return nil, nil, err } + errCode, err := getErrorCode(body) + if err != nil { + log.AddContext(ctx).Errorf("Get error code failed. error: %v", err) + return nil, nil, err + } + + if int64(errCode) == offLineCodeInt || int64(errCode) == noAuthenticated { + log.AddContext(ctx).Warningf("User offline, try to relogin %s", cli.url) + return cli.retryCall(ctx, method, url, data) + } + return respHeader, body, nil } func (cli *Client) reLogin(ctx context.Context) error { - oldToken := cli.authToken + cli.reLoginLock(ctx) + defer cli.reLoginUnlock(ctx) - cli.reloginMutex.Lock() - defer cli.reloginMutex.Unlock() + oldToken := cli.authToken if cli.authToken != "" && oldToken != cli.authToken { // Coming here indicates other thread had already done relogin, so no need to relogin again return nil @@ -502,3 +527,65 @@ func (cli *Client) checkErrorCode(ctx context.Context, resp map[string]interface return true } + +func getErrorCode(body map[string]any) (int, error) { + // demo 1: + // | "name" | "type" | + // | result | int32 | + // | errorCode | int32 | + // | suggestion | string | + if errorCodeInterface, exist := body["errorCode"]; exist { + if errorCodeFloat, ok := errorCodeInterface.(float64); ok { + return int(errorCodeFloat), nil + } + + if errorCodeString, ok := errorCodeInterface.(string); ok { + return strconv.Atoi(errorCodeString) + } + } + // demo 2: + // | "name" | "type" | + // | data | array | + // | - xx | - xx | + // | result | json | + // | - code | int32 | + if result, exist := body["result"].(map[string]any); exist { + if errorCodeFloat, ok := result["code"].(float64); ok { + return int(errorCodeFloat), nil + } + + if errorCodeString, ok := result["code"].(string); ok { + return strconv.Atoi(errorCodeString) + } + } + + return 0, nil +} + +func newHTTPClientByBackendID(ctx context.Context, backendID string) (*http.Client, error) { + jar, err := cookiejar.New(nil) + if err != nil { + log.AddContext(ctx).Errorf("create jar failed, error: %v", err) + return nil, err + } + + useCert, certMeta, err := pkgUtils.GetCertSecretFromBackendID(ctx, backendID) + if err != nil { + log.AddContext(ctx).Errorf("get cert secret from backend [%v] failed, error: %v", backendID, err) + return nil, err + } + + useCert, certPool, err := pkgUtils.GetCertPool(ctx, useCert, certMeta) + if err != nil { + log.AddContext(ctx).Errorf("get cert pool failed, error: %v", err) + return nil, err + } + + return &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: !useCert, RootCAs: certPool}, + }, + Jar: jar, + Timeout: 60 * time.Second, + }, nil +} diff --git a/storage/fusionstorage/client/client_namespace_test.go b/storage/fusionstorage/client/client_namespace_test.go index c164f684..655e32b2 100644 --- a/storage/fusionstorage/client/client_namespace_test.go +++ b/storage/fusionstorage/client/client_namespace_test.go @@ -23,26 +23,12 @@ import ( "bou.ke/monkey" . "github.com/smartystreets/goconvey/convey" - - "huawei-csi-driver/utils/log" ) const ( logName = "clientNamespaceTest.log" ) -var testClient *Client - -func TestMain(m *testing.M) { - log.MockInitLogging(logName) - defer log.MockStopLogging(logName) - - testClient = NewClient("https://192.168.125.*:8088", "dev-account", "mock-sec-name", - "mock-sec-namespace", "50", "mock-id", "mock-accountName") - - m.Run() -} - func TestAllowNfsShareAccess(t *testing.T) { Convey("Normal", t, func() { guard := monkey.PatchInstanceMethod(reflect.TypeOf(testClient), "Post", diff --git a/storage/fusionstorage/client/client_system.go b/storage/fusionstorage/client/client_system.go index 44cf2719..575ce486 100644 --- a/storage/fusionstorage/client/client_system.go +++ b/storage/fusionstorage/client/client_system.go @@ -22,6 +22,7 @@ import ( "fmt" "strings" + pkgUtils "huawei-csi-driver/pkg/utils" "huawei-csi-driver/utils" "huawei-csi-driver/utils/log" ) @@ -133,7 +134,11 @@ func (cli *Client) GetAllAccounts(ctx context.Context) ([]string, error) { var accounts []string for _, d := range respData { - data := d.(map[string]interface{}) + data, ok := d.(map[string]interface{}) + if !ok { + log.AddContext(ctx).Warningf("convert responseData to map failed, data: %v", d) + continue + } accountName, exist := data["name"].(string) if !exist || accountName == "" { continue @@ -169,7 +174,10 @@ func (cli *Client) GetAllPools(ctx context.Context) (map[string]interface{}, err log.AddContext(ctx).Errorln(msg) return nil, errors.New(msg) } - name := pool["poolName"].(string) + name, ok := pool["poolName"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert poolName to string failed, data: %v", pool["poolName"]) + } pools[name] = pool } @@ -236,7 +244,10 @@ func (cli *Client) GetNFSServiceSetting(ctx context.Context) (map[string]bool, e for k, v := range data { if k == "nfsv41_status" { - setting["SupportNFS41"] = v.(bool) + setting["SupportNFS41"], ok = v.(bool) + if !ok { + log.AddContext(ctx).Warningf("convert map[SupportNFS41] to bool failed, data: %v", v) + } break } } diff --git a/storage/fusionstorage/client/client_test.go b/storage/fusionstorage/client/client_test.go new file mode 100644 index 00000000..4d8737ba --- /dev/null +++ b/storage/fusionstorage/client/client_test.go @@ -0,0 +1,103 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2020-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package client + +import ( + "testing" + + "github.com/prashantv/gostub" + . "github.com/smartystreets/goconvey/convey" + + "huawei-csi-driver/csi/app" + cfg "huawei-csi-driver/csi/app/config" + "huawei-csi-driver/utils/log" +) + +var ( + testClient *Client +) + +func TestMain(m *testing.M) { + log.MockInitLogging(logName) + defer log.MockStopLogging(logName) + + getGlobalConfig := gostub.StubFunc(&app.GetGlobalConfig, cfg.MockCompletedConfig()) + defer getGlobalConfig.Reset() + + clientConfig := &NewClientConfig{ + Url: "https://127.0.0.1:8088", + User: "dev-account", + SecretName: "mock-sec-name", + SecretNamespace: "mock-sec-namespace", + ParallelNum: "", + BackendID: "mock-backend-id", + AccountName: "dev-account", + UseCert: false, + CertSecretMeta: "", + } + + testClient = NewClient(clientConfig) + + m.Run() +} + +func TestGetErrorCode(t *testing.T) { + Convey("Normal case", t, func() { + errCode, err := getErrorCode(map[string]any{ + "name": "mock-name", + "errorCode": 12345.0, + "suggestion": "mock-suggestion", + }) + So(errCode, ShouldEqual, 12345) + So(err, ShouldBeNil) + }) + + Convey("Error code is string ", t, func() { + errCode, err := getErrorCode(map[string]any{ + "name": "mock-name", + "errorCode": "12345", + "suggestion": "mock-suggestion", + }) + So(errCode, ShouldEqual, 12345) + So(err, ShouldBeNil) + }) + + Convey("Error code in result", t, func() { + errCode, err := getErrorCode(map[string]any{ + "result": map[string]any{ + "code": 12345.0, + }, + "data": map[string]any{ + "name": "mock-name", + }, + }) + So(errCode, ShouldEqual, 12345) + So(err, ShouldBeNil) + }) + + Convey("Can not convert to int", t, func() { + _, err := getErrorCode(map[string]any{ + "result": map[string]any{ + "code": "a12345", + }, + "data": map[string]any{ + "name": "mock-name", + }, + }) + So(err, ShouldBeError) + }) +} diff --git a/storage/fusionstorage/utils/utils.go b/storage/fusionstorage/utils/utils.go index 4483a335..b69f824d 100644 --- a/storage/fusionstorage/utils/utils.go +++ b/storage/fusionstorage/utils/utils.go @@ -86,17 +86,14 @@ func ExtractStorageQuotaParameters(ctx context.Context, storageQuotaConfig strin } // CheckErrorCode used to check Response +// Response data struct +// Response +// data +// ... +// result: +// code: 0 +// description: "" func CheckErrorCode(response map[string]interface{}) error { - // Response example - // Response: { - // "data": { - // "id": 14 - // }, - // "result": { - // "code": 0, - // "description": "" - // } - // } result, ok := response["result"].(map[string]interface{}) if !ok { diff --git a/storage/fusionstorage/volume/nas.go b/storage/fusionstorage/volume/nas.go index 329f8b1b..e078e1e0 100644 --- a/storage/fusionstorage/volume/nas.go +++ b/storage/fusionstorage/volume/nas.go @@ -130,7 +130,7 @@ func (p *NAS) preProcessQuota(ctx context.Context, params map[string]interface{} return pkgUtils.Errorln(ctx, fmt.Sprintf("extract storageQuota %s failed", v)) } - params["spaceQuota"] = quotaParams["spaceQuota"].(string) + params["spaceQuota"] = quotaParams["spaceQuota"] if v, exist := quotaParams["gracePeriod"]; exist { gracePeriod, err := utils.TransToIntStrict(ctx, v) if err != nil { @@ -614,9 +614,14 @@ func (p *NAS) waitFilesystemCreated(ctx context.Context, fsName string) error { func (p *NAS) revertShare(ctx context.Context, taskResult map[string]interface{}) error { shareID, exist := taskResult["shareID"].(string) if !exist { + log.AddContext(ctx).Warningf("convert shareID to string failed, data: %v", taskResult["shareID"]) return nil } accountId, exist := taskResult["accountId"].(string) + if !exist { + log.AddContext(ctx).Warningf("convert accountID to string failed, data: %v", taskResult["accountId"]) + return nil + } return p.deleteShare(ctx, shareID, accountId) } @@ -726,8 +731,10 @@ func (p *NAS) Delete(ctx context.Context, fsName string) error { } fsID := strconv.FormatInt(int64(fs["id"].(float64)), 10) - accountId := fs["account_id"].(string) - + accountId, ok := fs["account_id"].(string) + if !ok { + return pkgUtils.Errorf(ctx, "convert accountID to string failed, data: %v", fs["account_id"]) + } fsIdInShare, err := p.DeleteNfsShare(ctx, fsName, accountId) if err != nil { return pkgUtils.Errorln(ctx, fmt.Sprintf("DeleteNfsShare failed, err: %v", err)) diff --git a/storage/fusionstorage/volume/san.go b/storage/fusionstorage/volume/san.go index eb55ea47..001bbdf6 100644 --- a/storage/fusionstorage/volume/san.go +++ b/storage/fusionstorage/volume/san.go @@ -22,6 +22,7 @@ import ( "fmt" "strconv" + pkgUtils "huawei-csi-driver/pkg/utils" "huawei-csi-driver/storage/fusionstorage/client" "huawei-csi-driver/storage/fusionstorage/smartx" "huawei-csi-driver/utils" @@ -58,7 +59,10 @@ func (p *SAN) getQoS(ctx context.Context, params map[string]interface{}) error { } func (p *SAN) preCreate(ctx context.Context, params map[string]interface{}) error { - name := params["name"].(string) + name, ok := params["name"].(string) + if !ok { + return pkgUtils.Errorf(ctx, "convert sanName to string failed, data: %v", name) + } params["name"] = utils.GetFusionStorageLunName(name) if v, exist := params["storagepool"].(string); exist { @@ -123,8 +127,10 @@ func (p *SAN) prepareVolObj(ctx context.Context, params, res map[string]interfac func (p *SAN) createLun(ctx context.Context, params, taskResult map[string]interface{}) (map[string]interface{}, error) { - name := params["name"].(string) - + name, ok := params["name"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert sanName to string failed, data: %v", params["name"]) + } vol, err := p.cli.GetVolumeByName(ctx, name) if err != nil { log.AddContext(ctx).Errorf("Get LUN %s error: %v", name, err) @@ -152,8 +158,10 @@ func (p *SAN) createLun(ctx context.Context, } func (p *SAN) clone(ctx context.Context, params map[string]interface{}) error { - cloneFrom := params["clonefrom"].(string) - + cloneFrom, ok := params["clonefrom"].(string) + if !ok { + return pkgUtils.Errorf(ctx, "convert cloneFrom to string failed, data: %v", params["clonefrom"]) + } srcVol, err := p.cli.GetVolumeByName(ctx, cloneFrom) if err != nil { log.AddContext(ctx).Errorf("Get clone src vol %s error: %v", cloneFrom, err) @@ -165,7 +173,10 @@ func (p *SAN) clone(ctx context.Context, params map[string]interface{}) error { return errors.New(msg) } - volCapacity := params["capacity"].(int64) + volCapacity, ok := params["capacity"].(int64) + if !ok { + log.AddContext(ctx).Warningf("convert volCapacity to int64 failed, data: %v", params["capacity"]) + } if volCapacity < int64(srcVol["volSize"].(float64)) { msg := fmt.Sprintf("Clone vol capacity must be >= src %s", cloneFrom) log.AddContext(ctx).Errorln(msg) @@ -184,8 +195,10 @@ func (p *SAN) clone(ctx context.Context, params map[string]interface{}) error { p.cli.DeleteSnapshot(ctx, snapshotName) }() - volName := params["name"].(string) - + volName, ok := params["name"].(string) + if !ok { + return pkgUtils.Errorf(ctx, "convert volName to string failed, data: %v", params["name"]) + } err = p.cli.CreateVolumeFromSnapshot(ctx, volName, volCapacity, snapshotName) if err != nil { log.AddContext(ctx).Errorf("Create volume %s from %s error: %v", volName, snapshotName, err) @@ -196,8 +209,10 @@ func (p *SAN) clone(ctx context.Context, params map[string]interface{}) error { } func (p *SAN) createFromSnapshot(ctx context.Context, params map[string]interface{}) error { - srcSnapshotName := params["fromSnapshot"].(string) - + srcSnapshotName, ok := params["fromSnapshot"].(string) + if !ok { + return pkgUtils.Errorf(ctx, "convert srcSnapshotName to string failed, data: %v", params["fromSnapshot"]) + } srcSnapshot, err := p.cli.GetSnapshotByName(ctx, srcSnapshotName) if err != nil { log.AddContext(ctx).Errorf("Get clone src snapshot %s error: %v", srcSnapshotName, err) @@ -209,15 +224,20 @@ func (p *SAN) createFromSnapshot(ctx context.Context, params map[string]interfac return errors.New(msg) } - volCapacity := params["capacity"].(int64) + volCapacity, ok := params["capacity"].(int64) + if !ok { + log.AddContext(ctx).Warningf("convert volCapacity to int64 failed, data: %v", params["capacity"]) + } if volCapacity < int64(srcSnapshot["snapshotSize"].(float64)) { msg := fmt.Sprintf("Clone vol capacity must be >= src snapshot %s", srcSnapshotName) log.AddContext(ctx).Errorln(msg) return errors.New(msg) } - volName := params["name"].(string) - + volName, ok := params["name"].(string) + if !ok { + return pkgUtils.Errorf(ctx, "convert volName to string failed, data: %v", params["name"]) + } err = p.cli.CreateVolumeFromSnapshot(ctx, volName, volCapacity, srcSnapshotName) if err != nil { log.AddContext(ctx).Errorf("Clone snapshot %s to %s error: %v", srcSnapshotName, volName, err) @@ -244,7 +264,10 @@ func (p *SAN) createQoS(ctx context.Context, params, taskResult map[string]inter return nil, nil } - volName := taskResult["volumeName"].(string) + volName, ok := taskResult["volumeName"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert volName to string failed, data: %v", taskResult["volumeName"]) + } qosName, err := p.cli.GetQoSNameByVolume(ctx, volName) if err != nil { return nil, err @@ -344,7 +367,10 @@ func (p *SAN) Expand(ctx context.Context, name string, newSize int64) (bool, err func (p *SAN) preExpandCheckCapacity(ctx context.Context, params, taskResult map[string]interface{}) (map[string]interface{}, error) { // check the local pool - localParentId := params["localParentId"].(int64) + localParentId, ok := params["localParentId"].(int64) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert localParentId to string failed, data: %v", params["localParentId"]) + } pool, err := p.cli.GetPoolById(ctx, localParentId) if err != nil || pool == nil { log.AddContext(ctx).Errorf("Get storage pool %s info error: %v", localParentId, err) @@ -356,8 +382,14 @@ func (p *SAN) preExpandCheckCapacity(ctx context.Context, func (p *SAN) expandLocalLun(ctx context.Context, params, taskResult map[string]interface{}) (map[string]interface{}, error) { - lunName := params["lunName"].(string) - newSize := params["size"].(int64) + lunName, ok := params["lunName"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert lunName to string failed, data: %v", params["lunName"]) + } + newSize, ok := params["size"].(int64) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert newSize to int64 failed, data: %v", params["size"]) + } err := p.cli.ExtendVolume(ctx, lunName, newSize) if err != nil { @@ -393,10 +425,9 @@ func (p *SAN) CreateSnapshot(ctx context.Context, log.AddContext(ctx).Errorln(msg) return nil, errors.New(msg) } else { - snapshotCreated, _ := strconv.ParseInt(snapshot["createTime"].(string), 10, 64) snapshotSize := int64(snapshot["snapshotSize"].(float64)) * 1024 * 1024 return map[string]interface{}{ - "CreationTime": snapshotCreated, + "CreationTime": utils.ParseIntWithDefault(snapshot["createTime"].(string), 10, 64, 0), "SizeBytes": snapshotSize, "ParentID": strconv.FormatInt(int64(lun["volId"].(float64)), 10), }, nil @@ -421,7 +452,7 @@ func (p *SAN) CreateSnapshot(ctx context.Context, return nil, err } - snapshotCreated, _ := strconv.ParseInt(snapshot["createTime"].(string), 10, 64) + snapshotCreated := utils.ParseIntWithDefault(snapshot["createTime"].(string), 10, 64, 0) snapshotSize := int64(snapshot["snapshotSize"].(float64)) * 1024 * 1024 return map[string]interface{}{ "CreationTime": snapshotCreated, @@ -453,9 +484,14 @@ func (p *SAN) DeleteSnapshot(ctx context.Context, snapshotName string) error { func (p *SAN) createSnapshot(ctx context.Context, params, taskResult map[string]interface{}) (map[string]interface{}, error) { - lunName := params["lunName"].(string) - snapshotName := params["snapshotName"].(string) - + lunName, ok := params["lunName"].(string) + if !ok { + log.AddContext(ctx).Warningf("convert lunName to string failed, data: %v", params["lunName"]) + } + snapshotName, ok := params["snapshotName"].(string) + if !ok { + log.AddContext(ctx).Warningf("convert snapshotName to string failed, data: %v", params["snapshotName"]) + } err := p.cli.CreateSnapshot(ctx, snapshotName, lunName) if err != nil { log.AddContext(ctx).Errorf("Create snapshot %s for lun %s error: %v", snapshotName, lunName, err) diff --git a/storage/oceanstor/attacher/attacher.go b/storage/oceanstor/attacher/attacher.go index bf389219..ef45126c 100644 --- a/storage/oceanstor/attacher/attacher.go +++ b/storage/oceanstor/attacher/attacher.go @@ -24,6 +24,7 @@ import ( "strings" "huawei-csi-driver/connector/nvme" + pkgUtils "huawei-csi-driver/pkg/utils" "huawei-csi-driver/storage/oceanstor/client" "huawei-csi-driver/utils" "huawei-csi-driver/utils/log" @@ -153,9 +154,17 @@ func (p *Attacher) createHostGroup(ctx context.Context, hostID, mappingID string hostGroupName := p.getHostGroupName(hostID) for _, i := range hostGroupsByHostID { - group := i.(map[string]interface{}) + group, ok := i.(map[string]interface{}) + if !ok { + log.AddContext(ctx).Warningf("convert group to map failed, data: %v", i) + continue + } if group["NAME"].(string) == hostGroupName { - hostGroupID = group["ID"].(string) + hostGroupID, ok = group["ID"].(string) + if !ok { + log.AddContext(ctx).Warningf("convert hostGroupID to string failed, data: %v", group["ID"]) + continue + } return p.addToHostGroupMapping(ctx, hostGroupName, hostGroupID, mappingID) } } @@ -218,7 +227,6 @@ func (p *Attacher) addToHostGroupMapping(ctx context.Context, groupName, groupID func (p *Attacher) createLunGroup(ctx context.Context, lunID, hostID, mappingID string) error { var err error var lunGroup map[string]interface{} - var lunGroupID string lunGroupsByLunID, err := p.cli.QueryAssociateLunGroup(ctx, 11, lunID) if err != nil { @@ -228,9 +236,16 @@ func (p *Attacher) createLunGroup(ctx context.Context, lunID, hostID, mappingID lunGroupName := p.getLunGroupName(hostID) for _, i := range lunGroupsByLunID { - group := i.(map[string]interface{}) + group, ok := i.(map[string]interface{}) + if !ok { + log.AddContext(ctx).Warningf("convert group to map failed, data: %v", i) + continue + } if group["NAME"].(string) == lunGroupName { - lunGroupID = group["ID"].(string) + lunGroupID, ok := group["ID"].(string) + if !ok { + return errors.New("convert group[\"ID\"] to string failed") + } return p.addToLUNGroupMapping(ctx, lunGroupName, lunGroupID, mappingID) } } @@ -419,8 +434,16 @@ func (p *Attacher) getTargetISCSIProperties(ctx context.Context) ([]string, []st validIPs := map[string]bool{} validIQNs := map[string]string{} for _, i := range ports { - port := i.(map[string]interface{}) - portID := port["ID"].(string) + port, ok := i.(map[string]interface{}) + if !ok { + log.AddContext(ctx).Warningf("convert port to map failed, data: %v", i) + continue + } + portID, ok := port["ID"].(string) + if !ok { + log.AddContext(ctx).Warningf("convert portID to string failed, data: %v", port["ID"]) + continue + } portIqn := strings.Split(strings.Split(portID, ",")[0], "+")[1] splitIqn := strings.Split(portIqn, ":") @@ -577,8 +600,13 @@ func (p *Attacher) attachISCSI(ctx context.Context, hostID string, parameters ma } isFree, freeExist := initiator["ISFREE"].(string) + if !freeExist { + log.AddContext(ctx).Warningf("convert isFree to string failed, data: %v", initiator["ISFREE"]) + } parent, parentExist := initiator["PARENTID"].(string) - + if !parentExist { + log.AddContext(ctx).Warningf("convert parentID to string failed, data: %v", initiator["PARENTID"]) + } if freeExist && isFree == "true" { err := p.cli.AddIscsiInitiatorToHost(ctx, name, hostID) if err != nil { @@ -622,7 +650,13 @@ func (p *Attacher) attachFC(ctx context.Context, hostID string, parameters map[s } isFree, freeExist := initiator["ISFREE"].(string) + if !freeExist { + log.AddContext(ctx).Warningf("convert isFree to string failed, data: %v", initiator["ISFREE"]) + } parent, parentExist := initiator["PARENTID"].(string) + if !parentExist { + log.AddContext(ctx).Warningf("convert parentID to string failed, data: %v", initiator["PARENTID"]) + } if freeExist && isFree == "true" { addWWNs = append(addWWNs, wwn) @@ -668,8 +702,13 @@ func (p *Attacher) attachRoCE(ctx context.Context, hostID string, parameters map } isFree, freeExist := initiator["ISFREE"].(string) + if !freeExist { + log.AddContext(ctx).Warningf("convert isFree to string failed, data: %v", initiator["ISFREE"]) + } parent, parentExist := initiator["PARENTID"].(string) - + if !parentExist { + log.AddContext(ctx).Warningf("convert parentID to string failed, data: %v", initiator["PARENTID"]) + } if freeExist && isFree == "true" { err := p.cli.AddRoCEInitiatorToHost(ctx, name, hostID) if err != nil { @@ -697,8 +736,10 @@ func (p *Attacher) doMapping(ctx context.Context, hostID, lunName string) (strin return "", "", errors.New(msg) } - lunID := lun["ID"].(string) - + lunID, ok := lun["ID"].(string) + if !ok { + return "", "", pkgUtils.Errorf(ctx, "convert lunID to string failed, data: %v", lun["ID"]) + } mappingID, err := p.createMapping(ctx, hostID) if err != nil { log.AddContext(ctx).Errorf("Create mapping for host %s error: %v", hostID, err) @@ -740,9 +781,10 @@ func (p *Attacher) doUnmapping(ctx context.Context, hostID, lunName string) (str log.AddContext(ctx).Infof("LUN %s doesn't exist while detaching", lunName) return "", nil } - - lunID := lun["ID"].(string) - + lunID, ok := lun["ID"].(string) + if !ok { + return "", pkgUtils.Errorf(ctx, "convert lunID to string failed, data: %v", lun["ID"]) + } lunGroupsByLunID, err := p.cli.QueryAssociateLunGroup(ctx, 11, lunID) if err != nil { log.AddContext(ctx).Errorf("Query associated lungroups of lun %s error: %v", lunID, err) @@ -750,11 +792,17 @@ func (p *Attacher) doUnmapping(ctx context.Context, hostID, lunName string) (str } lunGroupName := p.getLunGroupName(hostID) - for _, i := range lunGroupsByLunID { - group := i.(map[string]interface{}) + group, ok := i.(map[string]interface{}) + if !ok { + log.AddContext(ctx).Warningf("convert group to map failed, data: %v", i) + continue + } if group["NAME"].(string) == lunGroupName { - lunGroupID := group["ID"].(string) + lunGroupID, ok := group["ID"].(string) + if !ok { + return "", pkgUtils.Errorf(ctx, "convert lunGroupID to string failed, data: %v", group["ID"]) + } err = p.cli.RemoveLunFromGroup(ctx, lunID, lunGroupID) if err != nil { log.AddContext(ctx).Errorf("Remove lun %s from group %s error: %v", @@ -784,7 +832,10 @@ func (p *Attacher) ControllerDetach(ctx context.Context, return "", nil } - hostID := host["ID"].(string) + hostID, ok := host["ID"].(string) + if !ok { + return "", pkgUtils.Errorf(ctx, "convert hostID to string failed, data: %v", host["ID"]) + } wwn, err := p.doUnmapping(ctx, hostID, lunName) if err != nil { log.AddContext(ctx).Errorf("Unmapping LUN %s from host %s error: %v", lunName, hostID, err) diff --git a/storage/oceanstor/attacher/dorado_v6_attacher.go b/storage/oceanstor/attacher/dorado_v6_attacher.go index c5fcc85a..7007c364 100644 --- a/storage/oceanstor/attacher/dorado_v6_attacher.go +++ b/storage/oceanstor/attacher/dorado_v6_attacher.go @@ -18,6 +18,7 @@ package attacher import ( "context" + "errors" "huawei-csi-driver/storage/oceanstor/client" "huawei-csi-driver/utils" @@ -77,7 +78,10 @@ func (p *DoradoV6Attacher) ControllerAttach(ctx context.Context, return nil, err } - hostID := host["ID"].(string) + hostID, ok := host["ID"].(string) + if !ok { + return nil, errors.New("convert host[\"ID\"] to string failed") + } hostAlua := utils.GetAlua(ctx, p.alua, host["NAME"].(string)) if hostAlua != nil && p.needUpdateHost(host, hostAlua) { diff --git a/storage/oceanstor/attacher/oceanstor_attacher.go b/storage/oceanstor/attacher/oceanstor_attacher.go index d601cd80..7ec6a0b2 100644 --- a/storage/oceanstor/attacher/oceanstor_attacher.go +++ b/storage/oceanstor/attacher/oceanstor_attacher.go @@ -18,6 +18,7 @@ package attacher import ( "context" + "errors" "huawei-csi-driver/storage/oceanstor/client" "huawei-csi-driver/utils" @@ -134,8 +135,14 @@ func (p *OceanStorAttacher) ControllerAttach(ctx context.Context, return nil, err } - hostID := host["ID"].(string) - hostName := host["NAME"].(string) + hostID, ok := host["ID"].(string) + if !ok { + return nil, errors.New("convert host[\"ID\"] to string failed") + } + hostName, ok := host["NAME"].(string) + if !ok { + return nil, errors.New("convert host[\"NAME\"] to string failed") + } if p.protocol == "iscsi" { err = p.attachISCSI(ctx, hostID, hostName, parameters) diff --git a/storage/oceanstor/client/client.go b/storage/oceanstor/client/client.go index 397ca38c..213b8f90 100644 --- a/storage/oceanstor/client/client.go +++ b/storage/oceanstor/client/client.go @@ -51,6 +51,8 @@ const ( IPLockErrorCode = 1077949071 WrongPasswordErrorCode = 1077987870 + UserOffline = 1077949069 + UserUnauthorized = -401 ) type BaseClientInterface interface { @@ -154,15 +156,53 @@ type HTTP interface { Do(req *http.Request) (*http.Response, error) } -var newHTTPClient = func() HTTP { - jar, _ := cookiejar.New(nil) +func newHTTPClientByBackendID(backendID string) (HTTP, error) { + jar, err := cookiejar.New(nil) + if err != nil { + log.Errorf("create jar failed, error: %v", err) + return nil, err + } + + useCert, certMeta, err := pkgUtils.GetCertSecretFromBackendID(context.Background(), backendID) + if err != nil { + log.Errorf("get cert secret from backend [%v] failed, error: %v", backendID, err) + return nil, err + } + + useCert, certPool, err := pkgUtils.GetCertPool(context.Background(), useCert, certMeta) + if err != nil { + log.Errorf("get cert pool failed, error: %v", err) + return nil, err + } + return &http.Client{ Transport: &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + TLSClientConfig: &tls.Config{InsecureSkipVerify: !useCert, RootCAs: certPool}, }, Jar: jar, Timeout: 60 * time.Second, + }, nil +} + +func newHTTPClientByCertMeta(useCert bool, certMeta string) (HTTP, error) { + jar, err := cookiejar.New(nil) + if err != nil { + log.Errorf("create jar failed, error: %v", err) + return nil, err } + + useCert, certPool, err := pkgUtils.GetCertPool(context.Background(), useCert, certMeta) + if err != nil { + return nil, err + } + + return &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: !useCert, RootCAs: certPool}, + }, + Jar: jar, + Timeout: 60 * time.Second, + }, nil } type Response struct { @@ -179,9 +219,11 @@ type NewClientConfig struct { VstoreName string ParallelNum string BackendID string + UseCert bool + CertSecretMeta string } -func NewClient(param *NewClientConfig) *BaseClient { +func NewClient(param *NewClientConfig) (*BaseClient, error) { var err error var parallelCount int @@ -198,15 +240,22 @@ func NewClient(param *NewClientConfig) *BaseClient { log.Infof("Init parallel count is %d", parallelCount) ClientSemaphore = utils.NewSemaphore(parallelCount) + + httpClient, err := newHTTPClientByCertMeta(param.UseCert, param.CertSecretMeta) + if err != nil { + log.Errorf("new http client by cert meta failed, err is %v", err) + return nil, err + } + return &BaseClient{ Urls: param.Urls, User: param.User, SecretName: param.SecretName, SecretNamespace: param.SecretNamespace, VStoreName: param.VstoreName, - Client: newHTTPClient(), + Client: httpClient, BackendID: param.BackendID, - } + }, nil } func (cli *BaseClient) Call(ctx context.Context, @@ -216,8 +265,7 @@ func (cli *BaseClient) Call(ctx context.Context, var err error r, err = cli.BaseCall(ctx, method, url, data) - if (err != nil && err.Error() == "unconnected") || - (r.Error != nil && int64(r.Error["code"].(float64)) == -401) { + if needReLogin(r, err) { // Current connection fails, try to relogin to other Urls if exist, // if relogin success, resend the request again. log.AddContext(ctx).Infof("Try to relogin and resend request method: %s, Url: %s", method, url) @@ -231,6 +279,22 @@ func (cli *BaseClient) Call(ctx context.Context, return r, err } +// needReLogin determine if it is necessary to log in to the storage again +func needReLogin(r Response, err error) bool { + var unconnected, unauthorized, offline bool + if err != nil && err.Error() == "unconnected" { + unconnected = true + } + + if r.Error != nil { + if code, ok := r.Error["code"].(float64); ok { + unauthorized = int64(code) == UserUnauthorized + offline = int64(code) == UserOffline + } + } + return unconnected || unauthorized || offline +} + func (cli *BaseClient) GetRequest(ctx context.Context, method string, url string, data map[string]interface{}) (*http.Request, error) { @@ -294,14 +358,14 @@ func (cli *BaseClient) BaseCall(ctx context.Context, } log.FilteredLog(ctx, isFilterLog(method, url), utils.IsDebugLog(method, url, debugLog), - fmt.Sprintf("Request method: %s, Url: %s, body: %v", method, reqUrl, data)) + fmt.Sprintf("Request method: %s, Url: %s, body: %v", method, req.URL, data)) ClientSemaphore.Acquire() defer ClientSemaphore.Release() resp, err := cli.Client.Do(req) if err != nil { - log.AddContext(ctx).Errorf("Send request method: %s, Url: %s, error: %v", method, reqUrl, err) + log.AddContext(ctx).Errorf("Send request method: %s, Url: %s, error: %v", method, req.URL, err) return r, errors.New("unconnected") } @@ -314,7 +378,7 @@ func (cli *BaseClient) BaseCall(ctx context.Context, } log.FilteredLog(ctx, isFilterLog(method, url), utils.IsDebugLog(method, url, debugLog), - fmt.Sprintf("Response method: %s, Url: %s, body: %s", method, reqUrl, body)) + fmt.Sprintf("Response method: %s, Url: %s, body: %s", method, req.URL, body)) err = json.Unmarshal(body, &r) if err != nil { @@ -401,27 +465,42 @@ func (cli *BaseClient) ValidateLogin(ctx context.Context) error { return fmt.Errorf("validate login %s error: %+v", cli.Url, resp) } + cli.setDeviceIdFromRespData(ctx, resp) + log.AddContext(ctx).Infof("Validate login %s success", cli.Url) return nil } +func (cli *BaseClient) setDeviceIdFromRespData(ctx context.Context, resp Response) { + respData, ok := resp.Data.(map[string]interface{}) + if !ok { + log.AddContext(ctx).Warningf("convert response data: %v to map[string]interface{} failed", resp.Data) + } + + cli.DeviceId, ok = respData["deviceid"].(string) + if !ok { + log.AddContext(ctx).Warningf("not found deviceId, response data is: %v", resp.Data) + } + + cli.Token, ok = respData["iBaseToken"].(string) + if !ok { + log.AddContext(ctx).Warningf("not found iBaseToken, response data is: %v", resp.Data) + } +} + func (cli *BaseClient) Login(ctx context.Context) error { var resp Response var err error - password, err := pkgUtils.GetPasswordFromBackendID(ctx, cli.BackendID) + cli.Client, err = newHTTPClientByBackendID(cli.BackendID) if err != nil { + log.AddContext(ctx).Errorf("new http client by backend %s failed, err is %v", cli.BackendID, err) return err } - data := map[string]interface{}{ - "username": cli.User, - "password": password, - "scope": "0", - } - - if len(cli.VStoreName) > 0 && cli.VStoreName != defaultVStore { - data["vstorename"] = cli.VStoreName + data, err := cli.getRequestParams(ctx, cli.BackendID) + if err != nil { + return err } cli.DeviceId = "" @@ -449,8 +528,8 @@ func (cli *BaseClient) Login(ctx context.Context) error { return err } - code := int64(resp.Error["code"].(float64)) - if code != 0 { + errCode, _ := resp.Error["code"].(float64) + if code := int64(errCode); code != 0 { msg := fmt.Sprintf("Login %s error: %+v", cli.Url, resp) if code == WrongPasswordErrorCode || code == IPLockErrorCode { err := pkgUtils.SetStorageBackendContentOnlineStatus(ctx, cli.BackendID, false) @@ -462,8 +541,7 @@ func (cli *BaseClient) Login(ctx context.Context) error { return errors.New(msg) } - err = cli.setDataFromRespData(ctx, resp) - if err != nil { + if err = cli.setDataFromRespData(ctx, resp); err != nil { setErr := pkgUtils.SetStorageBackendContentOnlineStatus(ctx, cli.BackendID, false) if setErr != nil { log.AddContext(ctx).Errorf("SetStorageBackendContentOffline [%s] failed. error: %v", cli.BackendID, setErr) @@ -492,15 +570,15 @@ func (cli *BaseClient) setDataFromRespData(ctx context.Context, resp Response) e } vStoreName, exist := respData["vstoreName"].(string) - if !exist { + vStoreID, idExist := respData["vstoreId"].(string) + if !exist && !idExist { log.AddContext(ctx).Infof("storage client login response vstoreName is empty, set it to default %s", defaultVStore) cli.VStoreName = defaultVStore - } else { + } else if exist { cli.VStoreName = vStoreName } - vStoreID, idExist := respData["vstoreId"].(string) if !idExist { log.AddContext(ctx).Infof("storage client login response vstoreID is empty, set it to default %s", defaultVStoreID) @@ -658,9 +736,16 @@ func (cli *BaseClient) getObj(ctx context.Context, url string, start, end int, f } var objList []map[string]interface{} - respData := resp.Data.([]interface{}) + respData, ok := resp.Data.([]interface{}) + if !ok { + return nil, errors.New("convert resp.Data to []interface{} failed") + } for _, i := range respData { - device := i.(map[string]interface{}) + device, ok := i.(map[string]interface{}) + if !ok { + log.AddContext(ctx).Warningf("convert resp.Data to map[string]interface{} failed") + continue + } objList = append(objList, device) } return objList, nil @@ -688,3 +773,22 @@ func (cli *BaseClient) getBatchObjs(ctx context.Context, url string, filterLog b } return objList, nil } + +func (cli *BaseClient) getRequestParams(ctx context.Context, backendID string) (map[string]interface{}, error) { + password, err := pkgUtils.GetPasswordFromBackendID(ctx, backendID) + if err != nil { + return nil, err + } + + data := map[string]interface{}{ + "username": cli.User, + "password": password, + "scope": "0", + } + + if len(cli.VStoreName) > 0 && cli.VStoreName != defaultVStore { + data["vstorename"] = cli.VStoreName + } + + return data, err +} diff --git a/storage/oceanstor/client/client_applicationtype.go b/storage/oceanstor/client/client_applicationtype.go index e360e111..b0766b44 100644 --- a/storage/oceanstor/client/client_applicationtype.go +++ b/storage/oceanstor/client/client_applicationtype.go @@ -61,7 +61,7 @@ func (cli *BaseClient) GetApplicationTypeByName(ctx context.Context, appType str // This will be used for param to create LUN val, ok := applicationTypes["ID"].(string) if !ok { - return result, fmt.Errorf("application type is not valid") + return result, errors.New("application type is not valid") } result = val } diff --git a/storage/oceanstor/client/client_clone.go b/storage/oceanstor/client/client_clone.go index 3b547ae4..cf276161 100644 --- a/storage/oceanstor/client/client_clone.go +++ b/storage/oceanstor/client/client_clone.go @@ -18,6 +18,7 @@ package client import ( "context" + "errors" "fmt" "huawei-csi-driver/utils/log" @@ -88,13 +89,19 @@ func (cli *BaseClient) GetClonePairInfo(ctx context.Context, clonePairID string) return nil, nil } - respData := resp.Data.([]interface{}) + respData, ok := resp.Data.([]interface{}) + if !ok { + return nil, errors.New("convert resp.Data to []interface{} failed") + } if len(respData) <= 0 { log.AddContext(ctx).Infof("clonePair %s does not exist", clonePairID) return nil, nil } - clonePair := respData[0].(map[string]interface{}) + clonePair, ok := respData[0].(map[string]interface{}) + if !ok { + return nil, errors.New("convert respData[0] to map[string]interface{} failed") + } return clonePair, nil } @@ -119,7 +126,10 @@ func (cli *BaseClient) CreateClonePair(ctx context.Context, return nil, fmt.Errorf("Create ClonePair from %s to %s, error: %d", srcLunID, dstLunID, code) } - respData := resp.Data.(map[string]interface{}) + respData, ok := resp.Data.(map[string]interface{}) + if !ok { + return nil, errors.New("convert resp.Data to map[string]interface{} failed") + } return respData, nil } @@ -213,6 +223,9 @@ func (cli *BaseClient) CloneFileSystem(ctx context.Context, name string, allocTy return nil, fmt.Errorf("Clone FS from %s error: %d", parentID, code) } - respData := resp.Data.(map[string]interface{}) + respData, ok := resp.Data.(map[string]interface{}) + if !ok { + return nil, errors.New("convert resp.Data to map[string]interface{} failed") + } return respData, nil } diff --git a/storage/oceanstor/client/client_dtree.go b/storage/oceanstor/client/client_dtree.go index 74108a54..fe86a17e 100644 --- a/storage/oceanstor/client/client_dtree.go +++ b/storage/oceanstor/client/client_dtree.go @@ -28,9 +28,6 @@ const ( ParentTypeFS int = 40 ParentTypeDTree int = 16445 - QuotaSwitchOpen bool = true - QuotaSwitchClose bool = false - SecurityStyleUnix int = 3 ) diff --git a/storage/oceanstor/client/client_fc.go b/storage/oceanstor/client/client_fc.go index be49538f..4057465e 100644 --- a/storage/oceanstor/client/client_fc.go +++ b/storage/oceanstor/client/client_fc.go @@ -59,7 +59,10 @@ func (cli *BaseClient) QueryFCInitiatorByHost(ctx context.Context, hostID string return nil, nil } - initiators := resp.Data.([]interface{}) + initiators, ok := resp.Data.([]interface{}) + if !ok { + return nil, errors.New("convert resp.Data to []interface{} failed") + } return initiators, nil } @@ -81,8 +84,14 @@ func (cli *BaseClient) GetFCInitiator(ctx context.Context, wwn string) (map[stri return nil, nil } - respData := resp.Data.([]interface{}) - initiator := respData[0].(map[string]interface{}) + respData, ok := resp.Data.([]interface{}) + if !ok { + return nil, errors.New("convert resp.Data to []interface{} failed") + } + initiator, ok := respData[0].(map[string]interface{}) + if !ok { + return nil, errors.New("convert respData[0] to map[string]interface{} failed") + } return initiator, nil } @@ -100,7 +109,10 @@ func (cli *BaseClient) GetFCInitiatorByID(ctx context.Context, wwn string) (map[ return nil, errors.New(msg) } - respData := resp.Data.(map[string]interface{}) + respData, ok := resp.Data.(map[string]interface{}) + if !ok { + return nil, errors.New("convert resp.Data to map[string]interface{} failed") + } return respData, nil } @@ -178,9 +190,15 @@ func (cli *BaseClient) GetFCTargetWWNs(ctx context.Context, initiatorWWN string) } var tgtWWNs []string - respData := resp.Data.([]interface{}) + respData, ok := resp.Data.([]interface{}) + if !ok { + return nil, errors.New("convert resp.Data to []interface{} failed") + } for _, tgt := range respData { - tgtPort := tgt.(map[string]interface{}) + tgtPort, ok := tgt.(map[string]interface{}) + if !ok { + return nil, errors.New("convert tgtPort to map[string]interface{} failed") + } tgtWWNs = append(tgtWWNs, tgtPort["TARGET_PORT_WWN"].(string)) } @@ -205,6 +223,9 @@ func (cli *BaseClient) GetFCHostLink(ctx context.Context, hostID string) ([]inte return nil, nil } - respData := resp.Data.([]interface{}) + respData, ok := resp.Data.([]interface{}) + if !ok { + return nil, errors.New("convert resp.Data to []interface{} failed") + } return respData, nil } diff --git a/storage/oceanstor/client/client_filesystem.go b/storage/oceanstor/client/client_filesystem.go index 8b231d39..92806e11 100644 --- a/storage/oceanstor/client/client_filesystem.go +++ b/storage/oceanstor/client/client_filesystem.go @@ -113,7 +113,10 @@ func (cli *BaseClient) GetFileSystemByName(ctx context.Context, name string) (ma return nil, nil } - respData := resp.Data.([]interface{}) + respData, ok := resp.Data.([]interface{}) + if !ok { + return nil, errors.New("convert resp.Data to []interface{} failed") + } return cli.getObjByvStoreName(respData), nil } @@ -131,7 +134,10 @@ func (cli *BaseClient) GetFileSystemByID(ctx context.Context, id string) (map[st return nil, errors.New(msg) } - fs := resp.Data.(map[string]interface{}) + fs, ok := resp.Data.(map[string]interface{}) + if !ok { + return nil, errors.New("convert resp.Data to map[string]interface{} failed") + } return fs, nil } @@ -162,13 +168,19 @@ func (cli *BaseClient) GetNfsShareByPath(ctx context.Context, path, vStoreID str return nil, nil } - respData := resp.Data.([]interface{}) + respData, ok := resp.Data.([]interface{}) + if !ok { + return nil, errors.New("convert resp.Data to []interface{} failed") + } if len(respData) == 0 { log.AddContext(ctx).Infof("Nfs share of path %s does not exist", path) return nil, nil } - share := respData[0].(map[string]interface{}) + share, ok := respData[0].(map[string]interface{}) + if !ok { + return nil, errors.New("convert respData[0] to map[string]interface{} failed") + } return share, nil } @@ -192,7 +204,11 @@ func (cli *BaseClient) GetNfsShareAccess(ctx context.Context, } for _, ac := range clients { - access := ac.(map[string]interface{}) + access, ok := ac.(map[string]interface{}) + if !ok { + log.AddContext(ctx).Warningf("convert ac: %v to map[string]interface{} failed.", ac) + continue + } if access["NAME"].(string) == name { return access, nil } @@ -219,10 +235,15 @@ func (cli *BaseClient) GetNfsShareAccessCount(ctx context.Context, parentID, vSt return 0, fmt.Errorf("Get nfs share access count of %s error: %d", parentID, code) } - respData := resp.Data.(map[string]interface{}) - countStr := respData["COUNT"].(string) - count, _ := strconv.ParseInt(countStr, 10, 64) - + respData, ok := resp.Data.(map[string]interface{}) + if !ok { + return 0, errors.New("convert resp.Data to map[string]interface{} failed") + } + countStr, ok := respData["COUNT"].(string) + if !ok { + return 0, errors.New("convert respData[\"COUNT\"] to string failed") + } + count := utils.ParseIntWithDefault(countStr, 10, 64, 0) return count, nil } @@ -249,7 +270,10 @@ func (cli *BaseClient) GetNfsShareAccessRange(ctx context.Context, parentID, vSt return nil, nil } - respData := resp.Data.([]interface{}) + respData, ok := resp.Data.([]interface{}) + if !ok { + return nil, errors.New("convert resp.Data to []interface{} failed") + } return respData, nil } @@ -348,7 +372,10 @@ func (cli *BaseClient) CreateNfsShare(ctx context.Context, code := int64(resp.Error["code"].(float64)) if code == shareAlreadyExist || code == sharePathAlreadyExist { - sharePath := params["sharepath"].(string) + sharePath, ok := params["sharepath"].(string) + if !ok { + return nil, errors.New("convert sharepath to string failed") + } log.AddContext(ctx).Infof("Nfs share %s already exists while creating", sharePath) share, err := cli.GetNfsShareByPath(ctx, sharePath, vStoreID) @@ -372,7 +399,10 @@ func (cli *BaseClient) CreateNfsShare(ctx context.Context, return nil, fmt.Errorf("create nfs share %v error: %d", data, code) } - respData := resp.Data.(map[string]interface{}) + respData, ok := resp.Data.(map[string]interface{}) + if !ok { + return nil, errors.New("convert resp.Data to map[string]interface{} failed") + } return respData, nil } @@ -446,7 +476,10 @@ func (cli *BaseClient) GetNFSServiceSetting(ctx context.Context) (map[string]boo "SupportNFS4": false, "SupportNFS41": false, } - respData := resp.Data.(map[string]interface{}) + respData, ok := resp.Data.(map[string]interface{}) + if !ok { + return nil, errors.New("convert resp.Data to map[string]interface{} failed") + } for k, v := range respData { var err error if k == "SUPPORTV3" { diff --git a/storage/oceanstor/client/client_fssnapshot.go b/storage/oceanstor/client/client_fssnapshot.go index b15318a2..642e9da0 100644 --- a/storage/oceanstor/client/client_fssnapshot.go +++ b/storage/oceanstor/client/client_fssnapshot.go @@ -20,8 +20,8 @@ import ( "context" "errors" "fmt" - "strconv" + "huawei-csi-driver/utils" "huawei-csi-driver/utils/log" ) @@ -88,12 +88,18 @@ func (cli *BaseClient) GetFSSnapshotByName(ctx context.Context, parentID, snapsh return nil, nil } - respData := resp.Data.([]interface{}) + respData, ok := resp.Data.([]interface{}) + if !ok { + return nil, errors.New("convert resp.Data to []interface{} failed") + } if len(respData) <= 0 { return nil, nil } - snapshot := respData[0].(map[string]interface{}) + snapshot, ok := respData[0].(map[string]interface{}) + if !ok { + return nil, errors.New("convert respData[0] to map[string]interface{} failed") + } return snapshot, nil } @@ -111,9 +117,15 @@ func (cli *BaseClient) GetFSSnapshotCountByParentId(ctx context.Context, ParentI return 0, errors.New(msg) } - respData := resp.Data.(map[string]interface{}) - countStr := respData["COUNT"].(string) - count, _ := strconv.Atoi(countStr) + respData, ok := resp.Data.(map[string]interface{}) + if !ok { + return 0, errors.New("convert resp.Data to map[string]interface{} failed") + } + countStr, ok := respData["COUNT"].(string) + if !ok { + return 0, errors.New("convert respData[\"COUNT\"] to string failed") + } + count := utils.AtoiWithDefault(countStr, 0) return count, nil } @@ -136,6 +148,9 @@ func (cli *BaseClient) CreateFSSnapshot(ctx context.Context, name, parentID stri return nil, fmt.Errorf("Create snapshot %s for FS %s error: %d", name, parentID, code) } - respData := resp.Data.(map[string]interface{}) + respData, ok := resp.Data.(map[string]interface{}) + if !ok { + return nil, errors.New("convert resp.Data to map[string]interface{} failed") + } return respData, nil } diff --git a/storage/oceanstor/client/client_host.go b/storage/oceanstor/client/client_host.go index 0b7ca379..0b14c8ca 100644 --- a/storage/oceanstor/client/client_host.go +++ b/storage/oceanstor/client/client_host.go @@ -123,7 +123,10 @@ func (cli *BaseClient) QueryAssociateHostGroup(ctx context.Context, objType int, return nil, nil } - respData := resp.Data.([]interface{}) + respData, ok := resp.Data.([]interface{}) + if !ok { + return nil, errors.New("convert resp.Data to []interface{} failed") + } return respData, nil } @@ -149,7 +152,10 @@ func (cli *BaseClient) CreateHost(ctx context.Context, name string) (map[string] return nil, errors.New(msg) } - host := resp.Data.(map[string]interface{}) + host, ok := resp.Data.(map[string]interface{}) + if !ok { + return nil, errors.New("convert resp.Data to map[string]interface{} failed") + } return host, nil } @@ -198,13 +204,19 @@ func (cli *BaseClient) GetHostByName(ctx context.Context, name string) (map[stri return nil, nil } - respData := resp.Data.([]interface{}) + respData, ok := resp.Data.([]interface{}) + if !ok { + return nil, errors.New("convert resp.Data to []interface{} failed") + } if len(respData) <= 0 { log.AddContext(ctx).Infof("Host %s does not exist", name) return nil, nil } - host := respData[0].(map[string]interface{}) + host, ok := respData[0].(map[string]interface{}) + if !ok { + return nil, errors.New("convert respData[0] to map[string]interface{} failed") + } return host, nil } @@ -249,7 +261,10 @@ func (cli *BaseClient) CreateHostGroup(ctx context.Context, name string) (map[st return nil, errors.New(msg) } - hostGroup := resp.Data.(map[string]interface{}) + hostGroup, ok := resp.Data.(map[string]interface{}) + if !ok { + return nil, errors.New("convert resp.Data to map[string]interface{} failed") + } return hostGroup, nil } @@ -272,13 +287,19 @@ func (cli *BaseClient) GetHostGroupByName(ctx context.Context, name string) (map return nil, nil } - respData := resp.Data.([]interface{}) + respData, ok := resp.Data.([]interface{}) + if !ok { + return nil, errors.New("convert resp.Data to []interface{} failed") + } if len(respData) <= 0 { log.AddContext(ctx).Infof("Hostgroup %s does not exist", name) return nil, nil } - hostGroup := respData[0].(map[string]interface{}) + hostGroup, ok := respData[0].(map[string]interface{}) + if !ok { + return nil, errors.New("convert respData[0] to map[string]interface{} failed") + } return hostGroup, nil } diff --git a/storage/oceanstor/client/client_hypermetro.go b/storage/oceanstor/client/client_hypermetro.go index ce509bd4..43dae0c4 100644 --- a/storage/oceanstor/client/client_hypermetro.go +++ b/storage/oceanstor/client/client_hypermetro.go @@ -20,6 +20,7 @@ import ( "context" "fmt" + pkgUtils "huawei-csi-driver/pkg/utils" "huawei-csi-driver/utils/log" ) @@ -64,9 +65,16 @@ func (cli *BaseClient) GetHyperMetroDomainByName(ctx context.Context, name strin return nil, nil } - respData := resp.Data.([]interface{}) + respData, ok := resp.Data.([]interface{}) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert respData to arr failed, data: %v", resp.Data) + } for _, i := range respData { - domain := i.(map[string]interface{}) + domain, ok := i.(map[string]interface{}) + if !ok { + log.AddContext(ctx).Warningf("convert domain to map failed, data: %v", i) + continue + } if domain["NAME"].(string) == name { return domain, nil } @@ -88,7 +96,10 @@ func (cli *BaseClient) GetHyperMetroDomain(ctx context.Context, domainID string) return nil, fmt.Errorf("Get HyperMetroDomain %s error: %d", domainID, code) } - respData := resp.Data.(map[string]interface{}) + respData, ok := resp.Data.(map[string]interface{}) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert respData to arr failed, data: %v", resp.Data) + } return respData, nil } @@ -109,9 +120,16 @@ func (cli *BaseClient) GetFSHyperMetroDomain(ctx context.Context, domainName str return nil, nil } - respData := resp.Data.([]interface{}) + respData, ok := resp.Data.([]interface{}) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert respData to arr failed, data: %v", resp.Data) + } for _, d := range respData { - domain := d.(map[string]interface{}) + domain, ok := d.(map[string]interface{}) + if !ok { + log.AddContext(ctx).Warningf("convert domain to map failed, data: %v", d) + continue + } if domain["NAME"].(string) == domainName { return domain, nil } @@ -140,13 +158,19 @@ func (cli *BaseClient) GetHyperMetroPair(ctx context.Context, pairID string) (ma return nil, nil } - respData := resp.Data.([]interface{}) + respData, ok := resp.Data.([]interface{}) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert respData to arr failed, data: %v", resp.Data) + } if len(respData) <= 0 { log.AddContext(ctx).Infof("Hypermetro %s does not exist", pairID) return nil, nil } - pair := respData[0].(map[string]interface{}) + pair, ok := respData[0].(map[string]interface{}) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert pair to map failed, data: %v", respData[0]) + } return pair, nil } @@ -169,9 +193,16 @@ func (cli *BaseClient) GetHyperMetroPairByLocalObjID(ctx context.Context, objID return nil, nil } - respData := resp.Data.([]interface{}) + respData, ok := resp.Data.([]interface{}) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert respData to arr failed, data: %v", resp.Data) + } for _, i := range respData { - pair := i.(map[string]interface{}) + pair, ok := i.(map[string]interface{}) + if !ok { + log.AddContext(ctx).Warningf("convert pair to map failed, data: %v", i) + continue + } if pair["LOCALOBJID"] == objID { return pair, nil } @@ -195,7 +226,10 @@ func (cli *BaseClient) CreateHyperMetroPair(ctx context.Context, data map[string return nil, fmt.Errorf("Create hypermetro %v error: %d", data, code) } - respData := resp.Data.(map[string]interface{}) + respData, ok := resp.Data.(map[string]interface{}) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert respData to arr failed, data: %v", resp.Data) + } return respData, nil } diff --git a/storage/oceanstor/client/client_iscsi.go b/storage/oceanstor/client/client_iscsi.go index 09c675d4..97a877d3 100644 --- a/storage/oceanstor/client/client_iscsi.go +++ b/storage/oceanstor/client/client_iscsi.go @@ -22,6 +22,7 @@ import ( "fmt" "strings" + pkgUtils "huawei-csi-driver/pkg/utils" "huawei-csi-driver/utils/log" ) @@ -60,7 +61,10 @@ func (cli *BaseClient) GetISCSIHostLink(ctx context.Context, hostID string) ([]i return nil, nil } - respData := resp.Data.([]interface{}) + respData, ok := resp.Data.([]interface{}) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert respData to arr failed, data: %v", resp.Data) + } return respData, nil } @@ -84,8 +88,14 @@ func (cli *BaseClient) GetIscsiInitiator(ctx context.Context, initiator string) return nil, nil } - respData := resp.Data.([]interface{}) - ini := respData[0].(map[string]interface{}) + respData, ok := resp.Data.([]interface{}) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert respData to arr failed, data: %v", resp.Data) + } + ini, ok := respData[0].(map[string]interface{}) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert ini to map failed, data: %v", respData[0]) + } return ini, nil } @@ -104,7 +114,10 @@ func (cli *BaseClient) GetIscsiInitiatorByID(ctx context.Context, initiator stri return nil, errors.New(msg) } - respData := resp.Data.(map[string]interface{}) + respData, ok := resp.Data.(map[string]interface{}) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert respData to map failed, data: %v", resp.Data) + } return respData, nil } @@ -129,7 +142,10 @@ func (cli *BaseClient) AddIscsiInitiator(ctx context.Context, initiator string) return nil, errors.New(msg) } - respData := resp.Data.(map[string]interface{}) + respData, ok := resp.Data.(map[string]interface{}) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert respData to map failed, data: %v", resp.Data) + } return respData, nil } @@ -205,6 +221,9 @@ func (cli *BaseClient) GetIscsiTgtPort(ctx context.Context) ([]interface{}, erro return nil, nil } - respData := resp.Data.([]interface{}) + respData, ok := resp.Data.([]interface{}) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert respData to string failed, data: %v", resp.Data) + } return respData, nil } diff --git a/storage/oceanstor/client/client_lun.go b/storage/oceanstor/client/client_lun.go index 297e8a8c..06d1ed76 100644 --- a/storage/oceanstor/client/client_lun.go +++ b/storage/oceanstor/client/client_lun.go @@ -23,6 +23,8 @@ import ( "fmt" "strconv" + pkgUtils "huawei-csi-driver/pkg/utils" + "huawei-csi-driver/utils" "huawei-csi-driver/utils/log" ) @@ -87,7 +89,10 @@ func (cli *BaseClient) QueryAssociateLunGroup(ctx context.Context, objType int, return nil, nil } - respData := resp.Data.([]interface{}) + respData, ok := resp.Data.([]interface{}) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert respData to arr failed, data: %v", resp.Data) + } return respData, nil } @@ -110,7 +115,10 @@ func (cli *BaseClient) GetLunByName(ctx context.Context, name string) (map[strin return nil, nil } - respData := resp.Data.([]interface{}) + respData, ok := resp.Data.([]interface{}) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert respData to arr failed, data: %v", resp.Data) + } if len(respData) <= 0 { log.AddContext(ctx).Infof("Lun %s does not exist", name) return nil, nil @@ -141,7 +149,11 @@ func (cli *BaseClient) GetLunByID(ctx context.Context, id string) (map[string]in return nil, errors.New(msg) } - lun := resp.Data.(map[string]interface{}) + lun, ok := resp.Data.(map[string]interface{}) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert lun to map failed, data: %v", resp.Data) + } + return lun, nil } @@ -216,13 +228,20 @@ func (cli *BaseClient) GetLunGroupByName(ctx context.Context, name string) (map[ return nil, nil } - respData := resp.Data.([]interface{}) + respData, ok := resp.Data.([]interface{}) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert respData to arr failed, data: %v", resp.Data) + } if len(respData) <= 0 { log.AddContext(ctx).Infof("Lungroup %s does not exist", name) return nil, nil } - group := respData[0].(map[string]interface{}) + group, ok := respData[0].(map[string]interface{}) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert group to arr failed, data: %v", respData[0]) + } + return group, nil } @@ -247,7 +266,10 @@ func (cli *BaseClient) CreateLunGroup(ctx context.Context, name string) (map[str return nil, errors.New(msg) } - lunGroup := resp.Data.(map[string]interface{}) + lunGroup, ok := resp.Data.(map[string]interface{}) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert lunGroup to map failed, data: %v", resp.Data) + } return lunGroup, nil } @@ -300,7 +322,10 @@ func (cli *BaseClient) CreateLun(ctx context.Context, params map[string]interfac return nil, fmt.Errorf("create volume %v error: %d", data, code) } - respData := resp.Data.(map[string]interface{}) + respData, ok := resp.Data.(map[string]interface{}) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert respData to map failed, data: %v", resp.Data) + } return respData, nil } @@ -359,9 +384,16 @@ func (cli *BaseClient) GetLunCountOfMapping(ctx context.Context, mappingID strin return 0, errors.New(msg) } - respData := resp.Data.(map[string]interface{}) - countStr := respData["COUNT"].(string) - count, _ := strconv.ParseInt(countStr, 10, 64) + respData, ok := resp.Data.(map[string]interface{}) + if !ok { + return 0, pkgUtils.Errorf(ctx, "convert respData to map failed, data: %v", resp.Data) + } + countStr, ok := respData["COUNT"].(string) + if !ok { + return 0, pkgUtils.Errorf(ctx, "convert countStr to string failed, data: %v", respData["COUNT"]) + } + + count := utils.ParseIntWithDefault(countStr, 10, 64, 0) return count, nil } @@ -379,9 +411,16 @@ func (cli *BaseClient) GetLunCountOfHost(ctx context.Context, hostID string) (in return 0, errors.New(msg) } - respData := resp.Data.(map[string]interface{}) - countStr := respData["COUNT"].(string) - count, _ := strconv.ParseInt(countStr, 10, 64) + respData, ok := resp.Data.(map[string]interface{}) + if !ok { + return 0, pkgUtils.Errorf(ctx, "convert respData to map failed, data: %v", resp.Data) + } + + countStr, ok := respData["COUNT"].(string) + if !ok { + return 0, pkgUtils.Errorf(ctx, "convert countStr to string failed, data: %v", respData["COUNT"]) + } + count := utils.ParseIntWithDefault(countStr, 10, 64, 0) return count, nil } @@ -399,9 +438,18 @@ func (cli *BaseClient) GetHostLunId(ctx context.Context, hostID, lunID string) ( return "", fmt.Errorf("Get hostLunId of host %s, lun %s error: %d", hostID, lunID, code) } - respData := resp.Data.([]interface{}) + respData, ok := resp.Data.([]interface{}) + if !ok { + return "", pkgUtils.Errorf(ctx, "convert respData to arr failed, data: %v", resp.Data) + } + for _, i := range respData { - hostLunInfo := i.(map[string]interface{}) + hostLunInfo, ok := i.(map[string]interface{}) + if !ok { + log.AddContext(ctx).Warningf(fmt.Sprintf("convert hostLunInfo to map failed, data: %v", i)) + continue + } + if hostLunInfo["ID"].(string) == lunID { var associateData map[string]interface{} associateDataBytes := []byte(hostLunInfo["ASSOCIATEMETADATA"].(string)) diff --git a/storage/oceanstor/client/client_luncopy.go b/storage/oceanstor/client/client_luncopy.go index 642642d9..3f325043 100644 --- a/storage/oceanstor/client/client_luncopy.go +++ b/storage/oceanstor/client/client_luncopy.go @@ -20,6 +20,7 @@ import ( "context" "fmt" + pkgUtils "huawei-csi-driver/pkg/utils" "huawei-csi-driver/utils/log" ) @@ -63,7 +64,10 @@ func (cli *BaseClient) CreateLunCopy(ctx context.Context, name, srcLunID, dstLun return nil, fmt.Errorf("Create luncopy from %s to %s error: %d", srcLunID, dstLunID, code) } - respData := resp.Data.(map[string]interface{}) + respData, ok := resp.Data.(map[string]interface{}) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert respData to map failed, data: %v", resp.Data) + } return respData, nil } @@ -81,7 +85,10 @@ func (cli *BaseClient) GetLunCopyByID(ctx context.Context, lunCopyID string) (ma return nil, fmt.Errorf("Get luncopy %s error: %d", lunCopyID, code) } - respData := resp.Data.(map[string]interface{}) + respData, ok := resp.Data.(map[string]interface{}) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert respData to map failed, data: %v", resp.Data) + } return respData, nil } @@ -104,14 +111,20 @@ func (cli *BaseClient) GetLunCopyByName(ctx context.Context, name string) (map[s return nil, nil } - respData := resp.Data.([]interface{}) + respData, ok := resp.Data.([]interface{}) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert respData to arr failed, data: %v", resp.Data) + } if len(respData) <= 0 { log.AddContext(ctx).Infof("Luncopy %s does not exist", name) return nil, nil } - luncopy := respData[0].(map[string]interface{}) - return luncopy, nil + lunCopy, ok := respData[0].(map[string]interface{}) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert lunCopy to map failed, data: %v", respData[0]) + } + return lunCopy, nil } // StartLunCopy used for start lun copy diff --git a/storage/oceanstor/client/client_lunsnapshot.go b/storage/oceanstor/client/client_lunsnapshot.go index 7ef68e04..d906dfad 100644 --- a/storage/oceanstor/client/client_lunsnapshot.go +++ b/storage/oceanstor/client/client_lunsnapshot.go @@ -20,6 +20,7 @@ import ( "context" "fmt" + pkgUtils "huawei-csi-driver/pkg/utils" "huawei-csi-driver/utils/log" ) @@ -59,7 +60,10 @@ func (cli *BaseClient) CreateLunSnapshot(ctx context.Context, name, lunID string return nil, fmt.Errorf("Create snapshot %s for lun %s error: %d", name, lunID, code) } - respData := resp.Data.(map[string]interface{}) + respData, ok := resp.Data.(map[string]interface{}) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert respData to map failed, data: %v", resp.Data) + } return respData, nil } @@ -82,12 +86,18 @@ func (cli *BaseClient) GetLunSnapshotByName(ctx context.Context, name string) (m return nil, nil } - respData := resp.Data.([]interface{}) + respData, ok := resp.Data.([]interface{}) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert respData to arr failed, data: %v", resp.Data) + } if len(respData) <= 0 { return nil, nil } - snapshot := respData[0].(map[string]interface{}) + snapshot, ok := respData[0].(map[string]interface{}) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert snapshot to map failed, data: %v", respData[0]) + } return snapshot, nil } diff --git a/storage/oceanstor/client/client_mapping.go b/storage/oceanstor/client/client_mapping.go index 5c1a4ebc..b4540134 100644 --- a/storage/oceanstor/client/client_mapping.go +++ b/storage/oceanstor/client/client_mapping.go @@ -21,6 +21,7 @@ import ( "errors" "fmt" + pkgUtils "huawei-csi-driver/pkg/utils" "huawei-csi-driver/utils/log" ) @@ -65,7 +66,10 @@ func (cli *BaseClient) CreateMapping(ctx context.Context, name string) (map[stri return nil, errors.New(msg) } - mapping := resp.Data.(map[string]interface{}) + mapping, ok := resp.Data.(map[string]interface{}) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert mapping to map failed, data: %v", resp.Data) + } return mapping, nil } @@ -88,13 +92,19 @@ func (cli *BaseClient) GetMappingByName(ctx context.Context, name string) (map[s return nil, nil } - respData := resp.Data.([]interface{}) + respData, ok := resp.Data.([]interface{}) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert respData to arr failed, data: %v", resp.Data) + } if len(respData) <= 0 { log.AddContext(ctx).Infof("Mapping %s does not exist", name) return nil, nil } - mapping := respData[0].(map[string]interface{}) + mapping, ok := respData[0].(map[string]interface{}) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert mapping to map failed, data: %v", respData[0]) + } return mapping, nil } diff --git a/storage/oceanstor/client/client_qos.go b/storage/oceanstor/client/client_qos.go index e2582b99..c5993a1b 100644 --- a/storage/oceanstor/client/client_qos.go +++ b/storage/oceanstor/client/client_qos.go @@ -21,6 +21,7 @@ import ( "fmt" "time" + pkgUtils "huawei-csi-driver/pkg/utils" "huawei-csi-driver/utils/log" ) @@ -95,7 +96,10 @@ func (cli *BaseClient) CreateQos(ctx context.Context, name, objID, objType, vSto return nil, fmt.Errorf("Create qos %v error: %d", data, code) } - respData := resp.Data.(map[string]interface{}) + respData, ok := resp.Data.(map[string]interface{}) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert respData to map failed, data: %v", resp.Data) + } return respData, nil } @@ -190,12 +194,18 @@ func (cli *BaseClient) GetQosByName(ctx context.Context, name, vStoreID string) return nil, nil } - respData := resp.Data.([]interface{}) + respData, ok := resp.Data.([]interface{}) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert respData to arr failed, data: %v", resp.Data) + } if len(respData) <= 0 { return nil, nil } - qos := respData[0].(map[string]interface{}) + qos, ok := respData[0].(map[string]interface{}) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert qos to map failed, data: %v", respData[0]) + } return qos, nil } @@ -216,7 +226,10 @@ func (cli *BaseClient) GetQosByID(ctx context.Context, qosID, vStoreID string) ( return nil, fmt.Errorf("Get qos by ID %s error: %d", qosID, code) } - qos := resp.Data.(map[string]interface{}) + qos, ok := resp.Data.(map[string]interface{}) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert qos to map failed, data: %v", resp.Data) + } return qos, nil } diff --git a/storage/oceanstor/client/client_replication.go b/storage/oceanstor/client/client_replication.go index 571771e2..0ce9209e 100644 --- a/storage/oceanstor/client/client_replication.go +++ b/storage/oceanstor/client/client_replication.go @@ -19,8 +19,8 @@ package client import ( "context" "fmt" - "strconv" + pkgUtils "huawei-csi-driver/pkg/utils" "huawei-csi-driver/utils/log" ) @@ -33,10 +33,6 @@ type Replication interface { GetReplicationPairByResID(ctx context.Context, resID string, resType int) ([]map[string]interface{}, error) // GetReplicationPairByID used for get replication pair by pair id GetReplicationPairByID(ctx context.Context, pairID string) (map[string]interface{}, error) - // GetReplicationvStorePairCount used for get replication vstore pair count - GetReplicationvStorePairCount(ctx context.Context) (int64, error) - // GetReplicationvStorePairRange used for get replication vstore pair range - GetReplicationvStorePairRange(ctx context.Context, startRange, endRange int64) ([]interface{}, error) // GetReplicationvStorePairByvStore used for get replication vstore pair by vstore id GetReplicationvStorePairByvStore(ctx context.Context, vStoreID string) (map[string]interface{}, error) // DeleteReplicationPair used for delete replication pair by pair id @@ -63,7 +59,10 @@ func (cli *BaseClient) CreateReplicationPair(ctx context.Context, data map[strin return nil, fmt.Errorf("Create replication %v error: %d", data, code) } - respData := resp.Data.(map[string]interface{}) + respData, ok := resp.Data.(map[string]interface{}) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert respData to map failed, data: %v", resp.Data) + } return respData, nil } @@ -147,7 +146,10 @@ func (cli *BaseClient) GetReplicationPairByResID(ctx context.Context, resID stri var pairs []map[string]interface{} - respData := resp.Data.([]interface{}) + respData, ok := resp.Data.([]interface{}) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert respData to arr failed, data: %v", resp.Data) + } for _, i := range respData { pairs = append(pairs, i.(map[string]interface{})) } @@ -168,49 +170,10 @@ func (cli *BaseClient) GetReplicationPairByID(ctx context.Context, pairID string return nil, fmt.Errorf("Get replication pair %s error: %d", pairID, code) } - respData := resp.Data.(map[string]interface{}) - return respData, nil -} - -// GetReplicationvStorePairCount used for get replication vstore pair count -func (cli *BaseClient) GetReplicationvStorePairCount(ctx context.Context) (int64, error) { - resp, err := cli.Get(ctx, "/replication_vstorepair/count", nil) - if err != nil { - return 0, err - } - - code := int64(resp.Error["code"].(float64)) - if code != 0 { - return 0, fmt.Errorf("Get replication vstore pair count error: %d", code) - } - - respData := resp.Data.(map[string]interface{}) - countStr := respData["COUNT"].(string) - count, _ := strconv.ParseInt(countStr, 10, 64) - - return count, nil -} - -// GetReplicationvStorePairRange used for get replication vstore pair range -func (cli *BaseClient) GetReplicationvStorePairRange(ctx context.Context, startRange, endRange int64) ( - []interface{}, error) { - - url := fmt.Sprintf("/replication_vstorepair?range=[%d-%d]", startRange, endRange) - resp, err := cli.Get(ctx, url, nil) - if err != nil { - return nil, err - } - - code := int64(resp.Error["code"].(float64)) - if code != 0 { - return nil, fmt.Errorf("Get replication vstore pairs error: %d", code) + respData, ok := resp.Data.(map[string]interface{}) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert respData to map failed, data: %v", resp.Data) } - - if resp.Data == nil { - return nil, nil - } - - respData := resp.Data.([]interface{}) return respData, nil } @@ -232,12 +195,18 @@ func (cli *BaseClient) GetReplicationvStorePairByvStore(ctx context.Context, return nil, nil } - respData := resp.Data.([]interface{}) + respData, ok := resp.Data.([]interface{}) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert respData to arr failed, data: %v", resp.Data) + } if len(respData) == 0 { log.AddContext(ctx).Infof("Replication vstore pair of vstore %s does not exist", vStoreID) return nil, nil } - pair := respData[0].(map[string]interface{}) + pair, ok := respData[0].(map[string]interface{}) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert pair to map failed, data: %v", respData[0]) + } return pair, nil } diff --git a/storage/oceanstor/client/client_roce.go b/storage/oceanstor/client/client_roce.go index 686a9c4b..206b961f 100644 --- a/storage/oceanstor/client/client_roce.go +++ b/storage/oceanstor/client/client_roce.go @@ -22,6 +22,7 @@ import ( URL "net/url" "strings" + pkgUtils "huawei-csi-driver/pkg/utils" "huawei-csi-driver/utils/log" ) @@ -57,11 +58,17 @@ func (cli *BaseClient) GetRoCEInitiator(ctx context.Context, initiator string) ( return nil, nil } - respData := resp.Data.([]interface{}) + respData, ok := resp.Data.([]interface{}) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert respData to arr failed, data: %v", resp.Data) + } if len(respData) == 0 { return nil, nil } - ini := respData[0].(map[string]interface{}) + ini, ok := respData[0].(map[string]interface{}) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert ini to map failed, data: %v", respData[0]) + } return ini, nil } @@ -79,7 +86,10 @@ func (cli *BaseClient) GetRoCEInitiatorByID(ctx context.Context, initiator strin return nil, fmt.Errorf("Get RoCE initiator by ID %s error: %d", initiator, code) } - respData := resp.Data.(map[string]interface{}) + respData, ok := resp.Data.(map[string]interface{}) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert respData to map failed, data: %v", resp.Data) + } return respData, nil } @@ -103,7 +113,10 @@ func (cli *BaseClient) AddRoCEInitiator(ctx context.Context, initiator string) ( return nil, fmt.Errorf("add RoCE initiator %s error: %d", initiator, code) } - respData := resp.Data.(map[string]interface{}) + respData, ok := resp.Data.(map[string]interface{}) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert respData to map failed, data: %v", resp.Data) + } return respData, nil } @@ -144,12 +157,18 @@ func (cli *BaseClient) GetRoCEPortalByIP(ctx context.Context, tgtPortal string) return nil, nil } - respData := resp.Data.([]interface{}) + respData, ok := resp.Data.([]interface{}) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert respData to arr failed, data: %v", resp.Data) + } if len(respData) == 0 { log.AddContext(ctx).Infof("RoCE portal %s does not exist", tgtPortal) return nil, nil } - portal := respData[0].(map[string]interface{}) + portal, ok := respData[0].(map[string]interface{}) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert portal to string failed, data: %v", respData[0]) + } return portal, nil } diff --git a/storage/oceanstor/client/client_system.go b/storage/oceanstor/client/client_system.go index a16c1295..8ca2985b 100644 --- a/storage/oceanstor/client/client_system.go +++ b/storage/oceanstor/client/client_system.go @@ -21,6 +21,7 @@ import ( "errors" "fmt" + pkgUtils "huawei-csi-driver/pkg/utils" "huawei-csi-driver/utils/log" ) @@ -62,13 +63,19 @@ func (cli *BaseClient) GetPoolByName(ctx context.Context, name string) (map[stri return nil, nil } - respData := resp.Data.([]interface{}) + respData, ok := resp.Data.([]interface{}) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert resp.Data to arr failed, data: %v", resp.Data) + } if len(respData) <= 0 { log.AddContext(ctx).Infof("Pool %s does not exist", name) return nil, nil } - pool := respData[0].(map[string]interface{}) + pool, ok := respData[0].(map[string]interface{}) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert respData[0] to map failed, data: %v", respData[0]) + } return pool, nil } @@ -92,10 +99,21 @@ func (cli *BaseClient) GetAllPools(ctx context.Context) (map[string]interface{}, pools := make(map[string]interface{}) - respData := resp.Data.([]interface{}) + respData, ok := resp.Data.([]interface{}) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert resp.Data to arr failed, data: %v", resp.Data) + } for _, p := range respData { - pool := p.(map[string]interface{}) - name := pool["NAME"].(string) + pool, ok := p.(map[string]interface{}) + if !ok { + log.AddContext(ctx).Warningf(fmt.Sprintf("convert pool to map failed, data: %v", p)) + continue + } + name, ok := pool["NAME"].(string) + if !ok { + log.AddContext(ctx).Warningf(fmt.Sprintf("convert name to map failed, data: %v", pool["NAME"])) + continue + } pools[name] = pool } @@ -121,9 +139,16 @@ func (cli *BaseClient) GetLicenseFeature(ctx context.Context) (map[string]int, e return result, nil } - respData := resp.Data.([]interface{}) + respData, ok := resp.Data.([]interface{}) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert resp.Data to arr failed, data: %v", resp.Data) + } for _, i := range respData { - feature := i.(map[string]interface{}) + feature, ok := i.(map[string]interface{}) + if !ok { + log.AddContext(ctx).Warningf(fmt.Sprintf("convert feature to map failed, data: %v", i)) + continue + } for k, v := range feature { result[k] = int(v.(float64)) } @@ -144,7 +169,10 @@ func (cli *BaseClient) GetSystem(ctx context.Context) (map[string]interface{}, e return nil, errors.New(msg) } - respData := resp.Data.(map[string]interface{}) + respData, ok := resp.Data.(map[string]interface{}) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert respData to map failed, data: %v", resp.Data) + } cli.setStorageVersion(respData) return respData, nil } @@ -166,9 +194,16 @@ func (cli *BaseClient) GetRemoteDeviceBySN(ctx context.Context, sn string) (map[ return nil, nil } - respData := resp.Data.([]interface{}) + respData, ok := resp.Data.([]interface{}) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert resp.Data to arr failed, data: %v", resp.Data) + } for _, i := range respData { - device := i.(map[string]interface{}) + device, ok := i.(map[string]interface{}) + if !ok { + log.AddContext(ctx).Warningf("convert device to map failed, data: %v", i) + continue + } if device["SN"] == sn { return device, nil } diff --git a/storage/oceanstor/client/client_test.go b/storage/oceanstor/client/client_test.go index c6ca9379..53df756a 100644 --- a/storage/oceanstor/client/client_test.go +++ b/storage/oceanstor/client/client_test.go @@ -62,30 +62,21 @@ func TestLogin(t *testing.T) { "\"iBaseToken\":\"508C457614FEA5413316AC0945ED0EE047765A96DD6524462C93EA5BE834B440\"," + "\"lastloginip\":\"192.168.125.25\",\"lastlogintime\":1645117156,\"pwdchangetime\":1643562159," + "\"roleId\":\"1\",\"usergroup\":\"\",\"userid\":\"admin\",\"username\":\"dev-account\",\"userscope\":\"0\"}," + - "\"error\":{\"code\":0,\"description\":\"0\"}}", - false, + "\"error\":{\"code\":0,\"description\":\"0\"}}", false, }, { "The user name or password is incorrect", "{\"data\":{},\"error\":{\"code\":1077949061,\"description\":\"The User name or PassWord is incorrect.\"," + - "\"errorParam\":\"\",\"suggestion\":\"Check the User name and PassWord, and try again.\"}}", - true, + "\"errorParam\":\"\",\"suggestion\":\"Check the User name and PassWord, and try again.\"}}", true, }, { "The IP address has been locked", "{\"data\":{},\"error\":{\"code\":1077949071,\"description\":\"The IP address has been locked.\"," + - "\"errorParam\":\"\",\"suggestion\":\"Contact the administrator.\"}}", - true, + "\"errorParam\":\"\",\"suggestion\":\"Contact the administrator.\"}}", true, }, } - m := gomonkey.ApplyFunc(pkgUtils.GetPasswordFromBackendID, - func(ctx context.Context, backendID string) (string, error) { - return "mock", nil - }) - m.ApplyFunc(pkgUtils.SetStorageBackendContentOnlineStatus, func(ctx context.Context, backendID string, online bool) error { - return nil - }) + m := getTestLoginPatches() defer m.Reset() for _, s := range cases { @@ -103,6 +94,21 @@ func TestLogin(t *testing.T) { } } +func getTestLoginPatches() *gomonkey.Patches { + m := gomonkey.ApplyFunc(pkgUtils.GetPasswordFromBackendID, + func(ctx context.Context, backendID string) (string, error) { + return "mock", nil + }) + m.ApplyFunc(pkgUtils.GetCertSecretFromBackendID, + func(ctx context.Context, backendID string) (bool, string, error) { + return false, "", nil + }) + m.ApplyFunc(pkgUtils.SetStorageBackendContentOnlineStatus, func(ctx context.Context, backendID string, online bool) error { + return nil + }) + return m +} + func TestLogout(t *testing.T) { var cases = []struct { Name string @@ -158,8 +164,7 @@ func TestReLogin(t *testing.T) { "\"iBaseToken\":\"508C457614FEA5413316AC0945ED0EE047765A96DD6524462C93EA5BE834B440\"," + "\"lastloginip\":\"192.168.125.25\",\"lastlogintime\":1645117156,\"pwdchangetime\":1643562159," + "\"roleId\":\"1\",\"usergroup\":\"\",\"userid\":\"admin\",\"username\":\"dev-account\", " + - "\"userscope\":\"0\"},\"error\":{\"code\":0,\"description\":\"0\"}}", - false, + "\"userscope\":\"0\"},\"error\":{\"code\":0,\"description\":\"0\"}}", false, }, { "The User name or PassWord is incorrect", @@ -170,19 +175,11 @@ func TestReLogin(t *testing.T) { { "The IP address has been locked", "{\"data\":{},\"error\":{\"code\":1077949071,\"description\":\"The IP address has been " + - "locked.\",\"errorParam\":\"\",\"suggestion\":\"Contact the administrator.\"}}", - true, + "locked.\",\"errorParam\":\"\",\"suggestion\":\"Contact the administrator.\"}}", true, }, } - m := gomonkey.ApplyFunc(pkgUtils.GetPasswordFromBackendID, - func(ctx context.Context, backendID string) (string, error) { - return "mock", nil - }) - m.ApplyFunc(pkgUtils.SetStorageBackendContentOnlineStatus, func(ctx context.Context, backendID string, online bool) error { - return nil - }) - + m := getTestLoginPatches() defer m.Reset() for _, s := range cases { @@ -1767,7 +1764,7 @@ func TestMain(m *testing.M) { getGlobalConfig := gostub.StubFunc(&app.GetGlobalConfig, cfg.MockCompletedConfig()) defer getGlobalConfig.Reset() - testClient = NewClient(&NewClientConfig{ + testClient, _ = NewClient(&NewClientConfig{ Urls: []string{"https://127.0.0.1:8088"}, User: "dev-account", SecretName: "mock-sec-name", diff --git a/storage/oceanstor/client/client_vstore.go b/storage/oceanstor/client/client_vstore.go index a37aed45..6b5a4009 100644 --- a/storage/oceanstor/client/client_vstore.go +++ b/storage/oceanstor/client/client_vstore.go @@ -20,6 +20,7 @@ import ( "context" "fmt" + pkgUtils "huawei-csi-driver/pkg/utils" "huawei-csi-driver/utils/log" ) @@ -54,13 +55,19 @@ func (cli *BaseClient) GetvStoreByName(ctx context.Context, name string) (map[st return nil, nil } - respData := resp.Data.([]interface{}) + respData, ok := resp.Data.([]interface{}) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert respData to arr failed, data: %v", resp.Data) + } if len(respData) == 0 { log.AddContext(ctx).Infof("vstore %s does not exist", name) return nil, nil } - vstore := respData[0].(map[string]interface{}) + vstore, ok := respData[0].(map[string]interface{}) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert respData[0] to map failed, data: %v", respData[0]) + } return vstore, nil } @@ -81,12 +88,18 @@ func (cli *BaseClient) GetvStorePairByID(ctx context.Context, pairID string) (ma return nil, nil } - respData := resp.Data.([]interface{}) + respData, ok := resp.Data.([]interface{}) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert respData to arr failed, data: %v", resp.Data) + } if len(respData) == 0 { log.AddContext(ctx).Infof("vstore pair %s does not exist", pairID) return nil, nil } - pair := respData[0].(map[string]interface{}) + pair, ok := respData[0].(map[string]interface{}) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert respData[0] to map failed, data: %v", respData[0]) + } return pair, nil } diff --git a/storage/oceanstor/clientv6/clientv6.go b/storage/oceanstor/clientv6/clientv6.go index 7da1a8bf..62796a0d 100644 --- a/storage/oceanstor/clientv6/clientv6.go +++ b/storage/oceanstor/clientv6/clientv6.go @@ -30,7 +30,7 @@ type ClientV6 struct { client.BaseClient } -func NewClientV6(param *client.NewClientConfig) *ClientV6 { +func NewClientV6(param *client.NewClientConfig) (*ClientV6, error) { var err error var parallelCount int @@ -48,9 +48,14 @@ func NewClientV6(param *client.NewClientConfig) *ClientV6 { log.Infof("Init parallel count is %d", parallelCount) client.ClientSemaphore = utils.NewSemaphore(parallelCount) - return &ClientV6{ - *client.NewClient(param), + cli, err := client.NewClient(param) + if err != nil { + return nil, err } + + return &ClientV6{ + *cli, + }, nil } // SplitCloneFS used to split clone for dorado or oceantor v6 diff --git a/storage/oceanstor/clientv6/clientv6_test.go b/storage/oceanstor/clientv6/clientv6_test.go index acd2e732..c74e14bb 100644 --- a/storage/oceanstor/clientv6/clientv6_test.go +++ b/storage/oceanstor/clientv6/clientv6_test.go @@ -81,7 +81,7 @@ func TestMain(m *testing.M) { log.MockInitLogging(logName) defer log.MockStopLogging(logName) - testClient = NewClientV6(&client.NewClientConfig{ + testClient, _ = NewClientV6(&client.NewClientConfig{ Urls: []string{"https://192.168.125.*:8088"}, User: "dev-account", SecretName: "mock-sec-name", diff --git a/storage/oceanstor/smartx/smartx.go b/storage/oceanstor/smartx/smartx.go index 24b04d2f..1f20c36f 100644 --- a/storage/oceanstor/smartx/smartx.go +++ b/storage/oceanstor/smartx/smartx.go @@ -25,6 +25,7 @@ import ( "strings" "time" + "huawei-csi-driver/pkg/constants" "huawei-csi-driver/storage/oceanstor/client" "huawei-csi-driver/utils" "huawei-csi-driver/utils/log" @@ -35,10 +36,10 @@ type qosParameterList map[string]struct{} var ( oceanStorQosValidators = map[string]qosParameterValidators{ - utils.OceanStorDoradoV6: doradoV6ParameterValidators, - utils.OceanStorDoradoV3: doradoParameterValidators, - utils.OceanStorV3: oceanStorV3V5ParameterValidators, - utils.OceanStorV5: oceanStorV3V5ParameterValidators, + constants.OceanStorDoradoV6: doradoV6ParameterValidators, + constants.OceanStorDoradoV3: doradoParameterValidators, + constants.OceanStorV3: oceanStorV3V5ParameterValidators, + constants.OceanStorV5: oceanStorV3V5ParameterValidators, } doradoParameterValidators = map[string]func(int) bool{ @@ -111,13 +112,13 @@ var ( // one of parameter is mandatory for respective products oceanStorQoSMandatoryParameters = map[string]qosParameterList{ - utils.OceanStorDoradoV6: oceanStorCommonParameters, - utils.OceanStorDoradoV3: { + constants.OceanStorDoradoV6: oceanStorCommonParameters, + constants.OceanStorDoradoV3: { "MAXBANDWIDTH": struct{}{}, "MAXIOPS": struct{}{}, }, - utils.OceanStorV3: oceanStorCommonParameters, - utils.OceanStorV5: oceanStorCommonParameters, + constants.OceanStorV3: oceanStorCommonParameters, + constants.OceanStorV5: oceanStorCommonParameters, } ) @@ -163,7 +164,7 @@ func validateQoSParametersSupport(ctx context.Context, product string, qosParam } } - if product != utils.OceanStorDoradoV6 && lowerLimit && upperLimit { + if product != constants.OceanStorDoradoV6 && lowerLimit && upperLimit { return utils.Errorf(ctx, "Cannot specify both lower and upper limits in qos for OceanStor %s", product) } @@ -190,7 +191,7 @@ func ExtractQoSParameters(ctx context.Context, product string, qosConfig string) key, val, product) } - if product == utils.OceanStorDoradoV6 && key == "LATENCY" { + if product == constants.OceanStorDoradoV6 && key == "LATENCY" { // convert OceanStoreDoradoV6 Latency from millisecond to microsecond params[key] = value * 1000 continue diff --git a/storage/oceanstor/volume/base.go b/storage/oceanstor/volume/base.go index fd8e786d..ce6db51d 100644 --- a/storage/oceanstor/volume/base.go +++ b/storage/oceanstor/volume/base.go @@ -22,6 +22,7 @@ import ( "fmt" "strconv" + pkgUtils "huawei-csi-driver/pkg/utils" "huawei-csi-driver/storage/oceanstor/client" "huawei-csi-driver/storage/oceanstor/smartx" "huawei-csi-driver/utils" @@ -99,8 +100,7 @@ func (p *Base) getPoolID(ctx context.Context, params map[string]interface{}) err return fmt.Errorf("storage pool %s doesn't exist", poolName) } - params["poolID"] = pool["ID"].(string) - + params["poolID"] = pool["ID"] return nil } @@ -145,7 +145,11 @@ func (p *Base) getRemotePoolID(ctx context.Context, func (p *Base) preExpandCheckCapacity(ctx context.Context, params, taskResult map[string]interface{}) (map[string]interface{}, error) { // check the local pool - localParentName := params["localParentName"].(string) + localParentName, ok := params["localParentName"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert localParentName to string failed, data: %v", params["localParentName"]) + } + pool, err := p.cli.GetPoolByName(ctx, localParentName) if err != nil || pool == nil { msg := fmt.Sprintf("Get storage pool %s info error: %v", localParentName, err) @@ -157,7 +161,7 @@ func (p *Base) preExpandCheckCapacity(ctx context.Context, } func (p *Base) getSnapshotReturnInfo(snapshot map[string]interface{}, snapshotSize int64) map[string]interface{} { - snapshotCreated, _ := strconv.ParseInt(snapshot["TIMESTAMP"].(string), 10, 64) + snapshotCreated := utils.ParseIntWithDefault(snapshot["TIMESTAMP"].(string), 10, 64, 0) snapshotSizeBytes := snapshotSize * 512 return map[string]interface{}{ "CreationTime": snapshotCreated, @@ -168,18 +172,35 @@ func (p *Base) getSnapshotReturnInfo(snapshot map[string]interface{}, snapshotSi func (p *Base) createReplicationPair(ctx context.Context, params, taskResult map[string]interface{}) (map[string]interface{}, error) { - resType := taskResult["resType"].(int) - remoteDeviceID := taskResult["remoteDeviceID"].(string) + resType, ok := taskResult["resType"].(int) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert resType to int failed, data: %v", taskResult["resType"]) + } + remoteDeviceID, ok := taskResult["remoteDeviceID"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert remoteDeviceID to string failed, data: %v", taskResult["remoteDeviceID"]) + } var localID string var remoteID string - if resType == 11 { - localID = taskResult["localLunID"].(string) - remoteID = taskResult["remoteLunID"].(string) + localID, ok = taskResult["localLunID"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert localID to string failed, data: %v", taskResult["localLunID"]) + } + remoteID, ok = taskResult["remoteLunID"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert remoteID to string failed, data: %v", taskResult["remoteLunID"]) + } } else { - localID = taskResult["localFSID"].(string) - remoteID = taskResult["remoteFSID"].(string) + localID, ok = taskResult["localFSID"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert localID to string failed, data: %v", taskResult["localFSID"]) + } + remoteID, ok = taskResult["remoteFSID"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert remoteID to string failed, data: %v", taskResult["remoteFSID"]) + } } data := map[string]interface{}{ @@ -208,7 +229,10 @@ func (p *Base) createReplicationPair(ctx context.Context, return nil, err } - pairID := pair["ID"].(string) + pairID, ok := pair["ID"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert pairID to string failed, data: %v", pair["ID"]) + } err = p.cli.SyncReplicationPair(ctx, pairID) if err != nil { log.AddContext(ctx).Errorf("Sync replication pair %s error: %v", pairID, err) diff --git a/storage/oceanstor/volume/dtree.go b/storage/oceanstor/volume/dtree.go index 7dcbcd7c..922fb7cf 100644 --- a/storage/oceanstor/volume/dtree.go +++ b/storage/oceanstor/volume/dtree.go @@ -203,7 +203,6 @@ func (p *DTree) createDtree(ctx context.Context, data["vstoreId"] = params["vstoreid"] } data["PARENTTYPE"] = client.ParentTypeFS - data["QUOTASWITCH"] = client.QuotaSwitchOpen data["securityStyle"] = client.SecurityStyleUnix res, err := p.cli.CreateDTree(ctx, data) @@ -342,7 +341,11 @@ func (p *DTree) allowShareAccess(ctx context.Context, params, taskResult map[str // Remove all other extra access for _, i := range accesses { - access := i.(map[string]interface{}) + access, ok := i.(map[string]interface{}) + if !ok { + log.AddContext(ctx).Warningf("allowShareAccess convert access to map failed, data: %v", i) + continue + } accessID, _ := utils.ToStringWithFlag(access["ID"]) err = p.cli.DeleteNfsShareAccess(ctx, accessID, vStoreID) @@ -406,7 +409,11 @@ func (p *DTree) revertShareAccess(ctx context.Context, taskResult map[string]int if _, exist := accesses[i]; !exist { continue } - access := accesses[i].(map[string]interface{}) + access, ok := accesses[i].(map[string]interface{}) + if !ok { + log.AddContext(ctx).Warningf("revertShareAccess convert access to map failed, data: %v", accesses[i]) + continue + } accessID, _ := utils.ToStringWithFlag(access["ID"]) err := p.cli.DeleteNfsShareAccess(ctx, accessID, vStoreID) if err != nil { @@ -454,7 +461,11 @@ func (p *DTree) deleteShareAccess(ctx context.Context, params, if _, exist := accesses[i]; !exist { continue } - access := accesses[i].(map[string]interface{}) + access, ok := accesses[i].(map[string]interface{}) + if !ok { + log.AddContext(ctx).Warningf("deleteShareAccess convert access to map failed, data: %v", accesses[i]) + continue + } accessID, _ := utils.ToStringWithFlag(access["ID"]) err := p.cli.DeleteNfsShareAccess(ctx, accessID, vStoreID) if err != nil { diff --git a/storage/oceanstor/volume/nas.go b/storage/oceanstor/volume/nas.go index bc7ccc5f..d6f65f30 100644 --- a/storage/oceanstor/volume/nas.go +++ b/storage/oceanstor/volume/nas.go @@ -25,6 +25,8 @@ import ( "strings" "time" + "huawei-csi-driver/pkg/constants" + pkgUtils "huawei-csi-driver/pkg/utils" "huawei-csi-driver/storage/oceanstor/client" "huawei-csi-driver/storage/oceanstor/smartx" "huawei-csi-driver/utils" @@ -81,7 +83,10 @@ func (p *NAS) preCreate(ctx context.Context, params map[string]interface{}) erro return err } - name := params["name"].(string) + name, ok := params["name"].(string) + if !ok { + return pkgUtils.Errorf(ctx, "convert fsName to string failed, data: %v", params["name"]) + } params["name"] = utils.GetFileSystemName(name) if v, exist := params["sourcevolumename"].(string); exist { @@ -173,7 +178,6 @@ func (p *NAS) Create(ctx context.Context, params map[string]interface{}) (utils. } taskflow.AddTask("Create-Local-FS", p.createLocalFS, p.revertLocalFS) - if replicationOK && replication { taskflow.AddTask("Create-Remote-FS", p.createRemoteFS, p.revertRemoteFS) taskflow.AddTask("Create-Remote-QoS", p.createRemoteQoS, p.revertRemoteQoS) @@ -235,7 +239,10 @@ func (p *NAS) validateManageWorkLoadType(ctx context.Context, params, fs map[str func (p *NAS) createLocalFS(ctx context.Context, params, taskResult map[string]interface{}) ( map[string]interface{}, error) { - fsName := params["name"].(string) + fsName, ok := params["name"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert fsName to string failed, data: %v", params["name"]) + } fs, err := p.cli.GetFileSystemByName(ctx, fsName) if err != nil { log.AddContext(ctx).Errorf("Get filesystem %s error: %v", fsName, err) @@ -244,21 +251,33 @@ func (p *NAS) createLocalFS(ctx context.Context, params, taskResult map[string]i var isClone bool if fs == nil { - params["parentid"] = params["poolID"].(string) - params["vstoreId"] = params["localVStoreID"].(string) + params["parentid"] = params["poolID"] + params["vstoreId"] = params["localVStoreID"] if _, exist := params["clonefrom"]; exist { fs, err = p.clone(ctx, params) + if err != nil { + log.AddContext(ctx).Warningf("p.clone() failed, param:%+v", params) + } isClone = true } else if _, exist := params["fromSnapshot"]; exist { fs, err = p.createFromSnapshot(ctx, params) + if err != nil { + log.AddContext(ctx).Warningf("p.createFromSnapshot() failed, param:%+v", params) + } isClone = true } else { fs, err = p.cli.CreateFileSystem(ctx, params) + if err != nil { + log.AddContext(ctx).Warningf("CreateFileSystem() failed, param:%+v", params) + } } } else { if fs["ISCLONEFS"].(string) != "false" { - fsID := fs["ID"].(string) + fsID, ok := fs["ID"].(string) + if !ok { + log.AddContext(ctx).Warningf("convert fsID to string failed, data: %v", fs["ID"]) + } err = p.waitFSSplitDone(ctx, fsID) } } @@ -268,7 +287,10 @@ func (p *NAS) createLocalFS(ctx context.Context, params, taskResult map[string]i return nil, err } - localFSID := fs["ID"].(string) + localFSID, ok := fs["ID"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert localFSID to string failed, data: %v", fs["ID"]) + } if err = p.updateFileSystem(ctx, isClone, localFSID, params); err != nil { log.AddContext(ctx).Errorf("Update filesystem %s error: %v", fsName, err) return nil, err @@ -315,7 +337,10 @@ func (p *NAS) updateFileSystem(ctx context.Context, isClone bool, objID string, } func (p *NAS) clone(ctx context.Context, params map[string]interface{}) (map[string]interface{}, error) { - clonefrom := params["clonefrom"].(string) + clonefrom, ok := params["clonefrom"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert clonefrom to string failed, data: %v", params["clonefrom"]) + } cloneFromFS, err := p.cli.GetFileSystemByName(ctx, clonefrom) if err != nil { log.AddContext(ctx).Errorf("Get clone src filesystem %s error: %v", clonefrom, err) @@ -332,7 +357,10 @@ func (p *NAS) clone(ctx context.Context, params map[string]interface{}) (map[str return nil, err } - cloneFSCapacity := params["capacity"].(int64) + cloneFSCapacity, ok := params["capacity"].(int64) + if !ok { + log.AddContext(ctx).Warningf("convert cloneFSCapacity to int64 failed, data: %v", params["capacity"]) + } if cloneFSCapacity < srcFSCapacity { msg := fmt.Sprintf("Clone filesystem capacity must be >= src %s", clonefrom) log.AddContext(ctx).Errorln(msg) @@ -361,8 +389,14 @@ func (p *NAS) clone(ctx context.Context, params map[string]interface{}) (map[str } func (p *NAS) createFromSnapshot(ctx context.Context, params map[string]interface{}) (map[string]interface{}, error) { - srcSnapshotName := params["fromSnapshot"].(string) - snapshotParentId := params["snapshotparentid"].(string) + srcSnapshotName, ok := params["fromSnapshot"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert srcSnapshotName to string failed, data: %v", params["fromSnapshot"]) + } + snapshotParentId, ok := params["snapshotparentid"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert snapshotParentId to string failed, data: %v", params["snapshotparentid"]) + } srcSnapshot, err := p.cli.GetFSSnapshotByName(ctx, snapshotParentId, srcSnapshotName) if err != nil { log.AddContext(ctx).Errorf("Get src filesystem snapshot %s error: %v", srcSnapshotName, err) @@ -374,7 +408,10 @@ func (p *NAS) createFromSnapshot(ctx context.Context, params map[string]interfac return nil, msg } - parentName := srcSnapshot["PARENTNAME"].(string) + parentName, ok := srcSnapshot["PARENTNAME"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert parentName to string failed, data: %v", srcSnapshot["PARENTNAME"]) + } parentFS, err := p.cli.GetFileSystemByName(ctx, parentName) if err != nil { log.AddContext(ctx).Errorf("Get clone src filesystem %s error: %v", parentName, err) @@ -420,7 +457,10 @@ func (p *NAS) cloneFilesystem(ctx context.Context, req *CloneFilesystemRequest) return nil, err } - cloneFSID := cloneFS["ID"].(string) + cloneFSID, ok := cloneFS["ID"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert cloneFSID to string failed, data: %v", cloneFS["ID"]) + } if req.CloneFsCapacity > req.SrcCapacity { err := p.cli.ExtendFileSystem(ctx, cloneFSID, req.CloneFsCapacity) if err != nil { @@ -473,7 +513,10 @@ func (p *NAS) waitFSSplitDone(ctx context.Context, fsID string) error { return false, fmt.Errorf("filesystem %s has the bad healthStatus code %s", fs["NAME"], fs["HEALTHSTATUS"].(string)) } - splitStatus := fs["SPLITSTATUS"].(string) + splitStatus, ok := fs["SPLITSTATUS"].(string) + if !ok { + return false, pkgUtils.Errorf(ctx, "convert splitStatus to string failed, data: %v", fs["SPLITSTATUS"]) + } if splitStatus == filesystemSplitStatusQueuing || splitStatus == filesystemSplitStatusSplitting || splitStatus == filesystemSplitStatusNotStart { @@ -548,8 +591,14 @@ func (p *NAS) createRemoteQoS(ctx context.Context, return nil, nil } - fsID := taskResult["remoteFSID"].(string) - remoteCli := taskResult["remoteCli"].(client.BaseClientInterface) + fsID, ok := taskResult["remoteFSID"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert fsID to string failed, data: %v", taskResult["remoteFSID"]) + } + remoteCli, ok := taskResult["remoteCli"].(client.BaseClientInterface) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert remoteCli to BaseClientInterface failed, data: %v", taskResult["remoteCli"]) + } smartX := smartx.NewSmartX(remoteCli) qosID, err := smartX.CreateQos(ctx, fsID, "fs", "", qos) @@ -573,14 +622,20 @@ func (p *NAS) revertRemoteQoS(ctx context.Context, taskResult map[string]interfa if !fsIDExist || !qosIDExist { return nil } - remoteCli := taskResult["remoteCli"].(client.BaseClientInterface) + remoteCli, ok := taskResult["remoteCli"].(client.BaseClientInterface) + if !ok { + return pkgUtils.Errorf(ctx, "convert remoteCli to client.BaseClientInterface failed, data: %v", taskResult["remoteCli"]) + } smartX := smartx.NewSmartX(remoteCli) return smartX.DeleteQos(ctx, qosID, fsID, "fs", "") } func (p *NAS) createShare(ctx context.Context, params, taskResult map[string]interface{}) (map[string]interface{}, error) { - fsName := params["name"].(string) + fsName, ok := params["name"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert fsName to string failed, data: %v", params["name"]) + } sharePath := utils.GetSharePath(fsName) activeClient := p.getActiveClient(taskResult) vStoreID := p.getVStoreID(taskResult) @@ -640,8 +695,16 @@ func (p *NAS) getCurrentShareAccess(ctx context.Context, shareID, vStoreID strin } for _, c := range clients { - client := c.(map[string]interface{}) - name := client["NAME"].(string) + client, ok := c.(map[string]interface{}) + if !ok { + log.AddContext(ctx).Warningf("convert client to map failed, data: %v", c) + continue + } + name, ok := client["NAME"].(string) + if !ok { + log.AddContext(ctx).Warningf("convert client name to string failed, data: %v", client["NAME"]) + continue + } accesses[name] = c } } @@ -651,8 +714,14 @@ func (p *NAS) getCurrentShareAccess(ctx context.Context, shareID, vStoreID strin func (p *NAS) allowShareAccess(ctx context.Context, params, taskResult map[string]interface{}) (map[string]interface{}, error) { - shareID := taskResult["shareID"].(string) - authClient := params["authclient"].(string) + shareID, ok := taskResult["shareID"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert shareID to string failed, data: %v", taskResult["shareID"]) + } + authClient, ok := params["authclient"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert authClient to string failed, data: %v", params["authclient"]) + } activeClient := p.getActiveClient(taskResult) vStoreID := p.getVStoreID(taskResult) accesses, err := p.getCurrentShareAccess(ctx, shareID, vStoreID, activeClient) @@ -687,9 +756,16 @@ func (p *NAS) allowShareAccess(ctx context.Context, // Remove all other extra access for _, i := range accesses { - access := i.(map[string]interface{}) - accessID := access["ID"].(string) - + access, ok := i.(map[string]interface{}) + if !ok { + log.AddContext(ctx).Warningf("convert access to map failed, data: %v", i) + continue + } + accessID, ok := access["ID"].(string) + if !ok { + log.AddContext(ctx).Warningf("convert accessID to string failed, data: %v", access["ID"]) + continue + } err := activeClient.DeleteNfsShareAccess(ctx, accessID, vStoreID) if err != nil { log.AddContext(ctx).Warningf("Delete extra nfs share access %s error: %v", accessID, err) @@ -702,7 +778,10 @@ func (p *NAS) allowShareAccess(ctx context.Context, } func (p *NAS) revertShareAccess(ctx context.Context, taskResult map[string]interface{}) error { - shareID := taskResult["shareID"].(string) + shareID, ok := taskResult["shareID"].(string) + if !ok { + return pkgUtils.Errorf(ctx, "convert shareID to string failed, data: %v", taskResult["shareID"]) + } authClient, exist := taskResult["authClient"].(string) if !exist { return nil @@ -720,8 +799,16 @@ func (p *NAS) revertShareAccess(ctx context.Context, taskResult map[string]inter if _, exist := accesses[i]; !exist { continue } - access := accesses[i].(map[string]interface{}) - accessID := access["ID"].(string) + access, ok := accesses[i].(map[string]interface{}) + if !ok { + log.AddContext(ctx).Warningf("convert access to map failed, data: %v", accesses[i]) + continue + } + accessID, ok := access["ID"].(string) + if !ok { + log.AddContext(ctx).Warningf("convert accessID to string failed, data: %v", access["ID"]) + continue + } err := p.cli.DeleteNfsShareAccess(ctx, accessID, vStoreID) if err != nil { log.AddContext(ctx).Warningf("Delete extra nfs share access %s error: %v", accessID, err) @@ -765,7 +852,10 @@ func (p *NAS) Delete(ctx context.Context, fsName string) error { return nil } - fsID := fs["ID"].(string) + fsID, ok := fs["ID"].(string) + if !ok { + log.AddContext(ctx).Warningf("convert fsID to string failed, data: %v", fs["ID"]) + } fsSnapshotNum, err := p.cli.GetFSSnapshotCountByParentId(ctx, fsID) if err != nil { log.AddContext(ctx).Errorf("Failed to get the snapshot count of filesystem %s error: %v", fsID, err) @@ -774,14 +864,17 @@ func (p *NAS) Delete(ctx context.Context, fsName string) error { var replicationIDs []string replicationIDBytes := []byte(fs["REMOTEREPLICATIONIDS"].(string)) - json.Unmarshal(replicationIDBytes, &replicationIDs) - + err = json.Unmarshal(replicationIDBytes, &replicationIDs) + if err != nil { + return pkgUtils.Errorf(ctx, "Unmarshal replicationIDBytes failed, err: %v", err) + } var hypermetroIDs []string hypermetroIDBytes := []byte(fs["HYPERMETROPAIRIDS"].(string)) - json.Unmarshal(hypermetroIDBytes, &hypermetroIDs) - + err = json.Unmarshal(hypermetroIDBytes, &hypermetroIDs) + if err != nil { + return pkgUtils.Errorf(ctx, "Unmarshal hypermetroIDBytes failed, error: %v", err) + } taskflow := taskflow.NewTaskFlow(ctx, "Delete-FileSystem-Volume") - if len(replicationIDs) > 0 { if p.replicaRemoteCli == nil { msg := "remote client for replication is nil" @@ -858,7 +951,7 @@ func (p *NAS) Expand(ctx context.Context, fsName string, newSize int64) error { return errors.New(msg) } - curSize, _ := strconv.ParseInt(fs["CAPACITY"].(string), 10, 64) + curSize := utils.ParseIntWithDefault(fs["CAPACITY"].(string), 10, 64, 0) if newSize <= curSize { msg := fmt.Sprintf("Filesystem %s newSize %d must be greater than curSize %d", fsName, newSize, curSize) log.AddContext(ctx).Errorln(msg) @@ -867,12 +960,16 @@ func (p *NAS) Expand(ctx context.Context, fsName string, newSize int64) error { var replicationIDs []string replicationIDBytes := []byte(fs["REMOTEREPLICATIONIDS"].(string)) - _ = json.Unmarshal(replicationIDBytes, &replicationIDs) - + err = json.Unmarshal(replicationIDBytes, &replicationIDs) + if err != nil { + return pkgUtils.Errorf(ctx, "Unmarshal replicationIDBytes failed, error: %v", err) + } var hyperMetroIDs []string hyperMetroIDBytes := []byte(fs["HYPERMETROPAIRIDS"].(string)) - _ = json.Unmarshal(hyperMetroIDBytes, &hyperMetroIDs) - + err = json.Unmarshal(hyperMetroIDBytes, &hyperMetroIDs) + if err != nil { + return pkgUtils.Errorf(ctx, "Unmarshal hyperMetroIDBytes failed, error: %v", err) + } expandTask := taskflow.NewTaskFlow(ctx, "Expand-FileSystem-Volume") expandTask.AddTask("Expand-PreCheck-Capacity", p.preExpandCheckCapacity, nil) @@ -940,7 +1037,10 @@ func (p *NAS) getvStorePair(ctx context.Context) (map[string]interface{}, error) return nil, errors.New(msg) } - vStoreID := vStore["ID"].(string) + vStoreID, ok := vStore["ID"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert vStoreID to string failed, data: %v", vStore["ID"]) + } vStorePair, err := p.cli.GetReplicationvStorePairByvStore(ctx, vStoreID) if err != nil { @@ -963,7 +1063,10 @@ func (p *NAS) getvStorePair(ctx context.Context) (map[string]interface{}, error) return nil, errors.New(msg) } - remotevStore := vStorePair["REMOTEVSTORENAME"].(string) + remotevStore, ok := vStorePair["REMOTEVSTORENAME"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert remotevStore to string failed, data: %v", vStorePair["REMOTEVSTORENAME"]) + } if remotevStore != p.replicaRemoteCli.GetvStoreName() { msg := fmt.Sprintf("Remote vstore %s does not correspond with configuration", remotevStore) log.AddContext(ctx).Errorln(msg) @@ -995,10 +1098,20 @@ func (p *NAS) getReplicationParams(ctx context.Context, return nil, err } + var ok bool if vStorePair != nil { - vStorePairID = vStorePair["ID"].(string) - remoteDeviceID = vStorePair["REMOTEDEVICEID"].(string) - remoteDeviceSN = vStorePair["REMOTEDEVICESN"].(string) + vStorePairID, ok = vStorePair["ID"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert vStorePairID to string failed, data: %v", vStorePair["ID"]) + } + remoteDeviceID, ok = vStorePair["REMOTEDEVICEID"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert remoteDeviceID to string failed, data: %v", vStorePair["REMOTEDEVICEID"]) + } + remoteDeviceSN, ok = vStorePair["REMOTEDEVICESN"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert remoteDeviceSN to string failed, data: %v", vStorePair["REMOTEDEVICESN"]) + } } remoteSystem, err := p.replicaRemoteCli.GetSystem(ctx) @@ -1008,7 +1121,10 @@ func (p *NAS) getReplicationParams(ctx context.Context, } if remoteDeviceID == "" { - sn := remoteSystem["ID"].(string) + sn, ok := remoteSystem["ID"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert sn to string failed, data: %v", remoteSystem["ID"]) + } remoteDeviceID, err = p.getRemoteDeviceID(ctx, sn) if err != nil { return nil, err @@ -1036,8 +1152,14 @@ func (p *NAS) getReplicationParams(ctx context.Context, func (p *NAS) createRemoteFS(ctx context.Context, params, taskResult map[string]interface{}) (map[string]interface{}, error) { - fsName := params["name"].(string) - remoteCli := taskResult["remoteCli"].(client.BaseClientInterface) + fsName, ok := params["name"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert fsName to string failed, data: %v", params["name"]) + } + remoteCli, ok := taskResult["remoteCli"].(client.BaseClientInterface) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert remoteCli to client.BaseClientInterface failed, data: %v", taskResult["remoteCli"]) + } fs, err := remoteCli.GetFileSystemByName(ctx, fsName) if err != nil { @@ -1051,8 +1173,8 @@ func (p *NAS) createRemoteFS(ctx context.Context, return nil, err } - params["parentid"] = taskResult["remotePoolID"].(string) - params["vstoreId"] = params["remoteVStoreID"].(string) + params["parentid"] = taskResult["remotePoolID"] + params["vstoreId"] = params["remoteVStoreID"] fs, err = remoteCli.CreateFileSystem(ctx, params) if err != nil { log.AddContext(ctx).Errorf("Create remote filesystem %s error: %v", fsName, err) @@ -1070,7 +1192,10 @@ func (p *NAS) revertRemoteFS(ctx context.Context, taskResult map[string]interfac if !exist || fsID == "" { return nil } - remoteCli := taskResult["remoteCli"].(client.BaseClientInterface) + remoteCli, ok := taskResult["remoteCli"].(client.BaseClientInterface) + if !ok { + return pkgUtils.Errorf(ctx, "convert remoteCli to client.BaseClientInterface failed, data: %v", taskResult["remoteCli"]) + } deleteParams := map[string]interface{}{ "ID": fsID, } @@ -1082,15 +1207,20 @@ func (p *NAS) revertRemoteFS(ctx context.Context, taskResult map[string]interfac func (p *NAS) deleteReplicationPair(ctx context.Context, params, taskResult map[string]interface{}) (map[string]interface{}, error) { - replicationIDs := params["replicationIDs"].([]string) - + replicationIDs, ok := params["replicationIDs"].([]string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert replicationIDs to []string failed, data: %v", params["replicationIDs"]) + } for _, pairID := range replicationIDs { pair, err := p.cli.GetReplicationPairByID(ctx, pairID) if err != nil { return nil, err } - runningStatus := pair["RUNNINGSTATUS"].(string) + runningStatus, ok := pair["RUNNINGSTATUS"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert runningStatus to string failed, data: %v", pair["RUNNINGSTATUS"]) + } if runningStatus == replicationPairRunningStatusNormal || runningStatus == replicationPairRunningStatusSync { p.cli.SplitReplicationPair(ctx, pairID) @@ -1129,7 +1259,10 @@ func (p *NAS) setActiveClient(ctx context.Context, func (p *NAS) deleteHyperMetroShare(ctx context.Context, params, taskResult map[string]interface{}) (map[string]interface{}, error) { - name := params["name"].(string) + name, ok := params["name"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert fsName to string failed, data: %v", params["name"]) + } activeClient := p.getActiveClient(taskResult) vStoreID := p.getVStoreID(taskResult) err := p.deleteShare(ctx, name, vStoreID, activeClient) @@ -1146,7 +1279,10 @@ func (p *NAS) deleteShare(ctx context.Context, name, vStoreID string, cli client } if share != nil { - shareID := share["ID"].(string) + shareID, ok := share["ID"].(string) + if !ok { + return pkgUtils.Errorf(ctx, "convert shareID to string failed, data: %v", share["ID"]) + } err := cli.DeleteNfsShare(ctx, shareID, vStoreID) if err != nil { log.AddContext(ctx).Errorf("Delete share %s error: %v", shareID, err) @@ -1169,7 +1305,10 @@ func (p *NAS) deleteFS(ctx context.Context, fsName string, cli client.BaseClient return nil } - fsID := fs["ID"].(string) + fsID, ok := fs["ID"].(string) + if !ok { + log.AddContext(ctx).Warningf("convert fsID to string failed, data: %v", fs["ID"]) + } vStoreID, _ := fs["vstoreId"].(string) qosID, ok := fs["IOCLASSID"].(string) if ok && qosID != "" { @@ -1194,7 +1333,10 @@ func (p *NAS) deleteFS(ctx context.Context, fsName string, cli client.BaseClient func (p *NAS) deleteLocalFS(ctx context.Context, params, taskResult map[string]interface{}) (map[string]interface{}, error) { - name := params["name"].(string) + name, ok := params["name"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert fsName to string failed, data: %v", params["name"]) + } vStoreID := p.getVStoreID(taskResult) err := p.deleteShare(ctx, name, vStoreID, p.cli) if err != nil { @@ -1206,7 +1348,10 @@ func (p *NAS) deleteLocalFS(ctx context.Context, func (p *NAS) deleteReplicationRemoteFS(ctx context.Context, params, taskResult map[string]interface{}) (map[string]interface{}, error) { - name := params["name"].(string) + name, ok := params["name"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert fsName to string failed, data: %v", params["name"]) + } vStoreID, _ := taskResult["remoteVStoreID"].(string) err := p.deleteShare(ctx, name, vStoreID, p.replicaRemoteCli) if err != nil { @@ -1218,13 +1363,19 @@ func (p *NAS) deleteReplicationRemoteFS(ctx context.Context, func (p *NAS) deleteHyperMetroLocalFS(ctx context.Context, params, taskResult map[string]interface{}) (map[string]interface{}, error) { - name := params["name"].(string) + name, ok := params["name"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert fsName to string failed, data: %v", params["name"]) + } return nil, p.deleteFS(ctx, name, p.cli) } func (p *NAS) deleteHyperMetroRemoteFS(ctx context.Context, params, taskResult map[string]interface{}) (map[string]interface{}, error) { - name := params["name"].(string) + name, ok := params["name"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert fsName to string failed, data: %v", params["name"]) + } err := p.deleteFS(ctx, name, p.metroRemoteCli) return nil, err @@ -1251,14 +1402,29 @@ func (p *NAS) getHyperMetroParams(ctx context.Context, func (p *NAS) createHyperMetro(ctx context.Context, params, taskResult map[string]interface{}) (map[string]interface{}, error) { - vStorePairID := params["vStorePairID"].(string) - - localFSID := taskResult["localFSID"].(string) - remoteFSID := taskResult["remoteFSID"].(string) + vStorePairID, ok := params["vStorePairID"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert vStorePairID to string failed, data: %v", params["vStorePairID"]) + } + localFSID, ok := taskResult["localFSID"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert localFSID to string failed, data: %v", taskResult["localFSID"]) + } + remoteFSID, ok := taskResult["remoteFSID"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert remoteFSID to string failed, data: %v", taskResult["remoteFSID"]) + } activeClient := p.getActiveClient(taskResult) if activeClient != p.cli { - localFSID = taskResult["remoteFSID"].(string) - remoteFSID = taskResult["localFSID"].(string) + localFSID, ok = taskResult["remoteFSID"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert localFSID to string failed, data: %v", taskResult["remoteFSID"]) + } + + remoteFSID, ok = taskResult["localFSID"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert remoteFSID to string failed, data: %v", taskResult["localFSID"]) + } } data := map[string]interface{}{ @@ -1280,9 +1446,12 @@ func (p *NAS) createHyperMetro(ctx context.Context, return nil, err } - pairID := pair["ID"].(string) + pairID, ok := pair["ID"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert pairID to string failed, data: %v", pair["ID"]) + } // There is no need to synchronize when use NAS Dorado V6 or OceanStor V6 HyperMetro Volume - if p.product != utils.OceanStorDoradoV6 { + if p.product != constants.OceanStorDoradoV6 { err = activeClient.SyncHyperMetroPair(ctx, pairID) if err != nil { log.AddContext(ctx).Errorf("Sync nas hypermetro pair %s error: %v", pairID, err) @@ -1315,7 +1484,10 @@ func (p *NAS) revertHyperMetro(ctx context.Context, taskResult map[string]interf return nil } - status := pair["RUNNINGSTATUS"].(string) + status, ok := pair["RUNNINGSTATUS"].(string) + if !ok { + log.AddContext(ctx).Warningf("convert RUNNINGSTATUS to string failed, data: %v", pair["RUNNINGSTATUS"]) + } if status == hyperMetroPairRunningStatusNormal || status == hyperMetroPairRunningStatusToSync || status == hyperMetroPairRunningStatusSyncing { @@ -1332,7 +1504,10 @@ func (p *NAS) revertHyperMetro(ctx context.Context, taskResult map[string]interf func (p *NAS) deleteHyperMetro(ctx context.Context, params, taskResult map[string]interface{}) (map[string]interface{}, error) { - hypermetroIDs := params["hypermetroIDs"].([]string) + hypermetroIDs, ok := params["hypermetroIDs"].([]string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert hypermetroIDs to []string failed, data: %v", params["hypermetroIDs"]) + } activeClient := p.getActiveClient(taskResult) for _, pairID := range hypermetroIDs { pair, err := activeClient.GetHyperMetroPair(ctx, pairID) @@ -1344,7 +1519,11 @@ func (p *NAS) deleteHyperMetro(ctx context.Context, continue } - status := pair["RUNNINGSTATUS"].(string) + status, ok := pair["RUNNINGSTATUS"].(string) + if !ok { + log.AddContext(ctx).Warningf("convert RUNNINGSTATUS to string failed, data: %v", pair["RUNNINGSTATUS"]) + } + if status == hyperMetroPairRunningStatusNormal || status == hyperMetroPairRunningStatusToSync || status == hyperMetroPairRunningStatusSyncing { @@ -1409,7 +1588,10 @@ func (p *NAS) preExpandCheckRemoteCapacity(ctx context.Context, } // check the remote pool - remoteFsName := params["name"].(string) + remoteFsName, ok := params["name"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert remoteFsName to string failed, data: %v", params["name"]) + } remoteFs, err := cli.GetFileSystemByName(ctx, remoteFsName) if err != nil { log.AddContext(ctx).Errorf("Get filesystem %s error: %v", remoteFsName, err) @@ -1422,7 +1604,10 @@ func (p *NAS) preExpandCheckRemoteCapacity(ctx context.Context, return nil, errors.New(msg) } - newSize := params["size"].(int64) + newSize, ok := params["size"].(int64) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert newSize to int64 failed, data: %v", params["size"]) + } curSize, err := strconv.ParseInt(remoteFs["CAPACITY"].(string), 10, 64) if err != nil { return nil, err @@ -1454,8 +1639,14 @@ func (p *NAS) expandFS(ctx context.Context, objID string, newSize int64, cli cli func (p *NAS) expandReplicationRemoteFS(ctx context.Context, params, taskResult map[string]interface{}) (map[string]interface{}, error) { - fsID := taskResult["remoteFSID"].(string) - newSize := params["size"].(int64) + fsID, ok := taskResult["remoteFSID"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert remoteFSID to string failed, data: %v", taskResult["remoteFSID"]) + } + newSize, ok := params["size"].(int64) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert newSize to int64 failed, data: %v", params["size"]) + } err := p.expandFS(ctx, fsID, newSize, p.replicaRemoteCli) if err != nil { log.AddContext(ctx).Errorf("Expand replica filesystem %s error: %v", fsID, err) @@ -1471,8 +1662,14 @@ func (p *NAS) expandHyperMetroRemoteFS(ctx context.Context, return nil, nil } - fsID := taskResult["remoteFSID"].(string) - newSize := params["size"].(int64) + fsID, ok := taskResult["remoteFSID"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert fsID to string failed, data: %v", taskResult["remoteFSID"]) + } + newSize, ok := params["size"].(int64) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert newSize to int64 failed, data: %v", params["size"]) + } err := p.expandFS(ctx, fsID, newSize, p.metroRemoteCli) if err != nil { log.AddContext(ctx).Errorf("Expand hyperMetro filesystem %s error: %v", fsID, err) @@ -1484,7 +1681,10 @@ func (p *NAS) expandHyperMetroRemoteFS(ctx context.Context, func (p *NAS) expandLocalFS(ctx context.Context, params, taskResult map[string]interface{}) (map[string]interface{}, error) { - newSize := params["size"].(int64) + newSize, ok := params["size"].(int64) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert newSize to int64 failed, data: %v", params["size"]) + } activeClient := p.getActiveClient(taskResult) fsID := p.getActiveFsID(taskResult) err := p.expandFS(ctx, fsID, newSize, activeClient) @@ -1507,14 +1707,21 @@ func (p *NAS) CreateSnapshot(ctx context.Context, fsName, snapshotName string) ( return nil, errors.New(msg) } - fsId := fs["ID"].(string) + fsId, ok := fs["ID"].(string) + if !ok { + log.AddContext(ctx).Warningf("convert fsID to string failed, data: %v", fs["ID"]) + } snapshot, err := p.cli.GetFSSnapshotByName(ctx, fsId, snapshotName) if err != nil { log.AddContext(ctx).Errorf("Get filesystem snapshot by name %s error: %v", snapshotName, err) return nil, err } - snapshotSize, _ := strconv.ParseInt(fs["CAPACITY"].(string), 10, 64) + snapshotSize, err := strconv.ParseInt(fs["CAPACITY"].(string), 10, 64) + if err != nil { + log.AddContext(ctx).Errorf("parse filesystem failed. err:%v, CAPACITY: %v", err, fs["CAPACITY"]) + return nil, err + } if snapshot != nil { log.AddContext(ctx).Infof("The snapshot %s is already exist.", snapshotName) return p.getSnapshotReturnInfo(snapshot, snapshotSize), nil @@ -1542,7 +1749,10 @@ func (p *NAS) DeleteSnapshot(ctx context.Context, snapshotParentId, snapshotName return nil } - snapshotId := snapshot["ID"].(string) + snapshotId, ok := snapshot["ID"].(string) + if !ok { + return pkgUtils.Errorf(ctx, "convert snapshotId to string failed, data: %v", snapshot["ID"]) + } err = p.cli.DeleteFSSnapshot(ctx, snapshotId) if err != nil { log.AddContext(ctx).Errorf("Delete filesystem snapshot %s error: %v", snapshotId, err) diff --git a/storage/oceanstor/volume/san.go b/storage/oceanstor/volume/san.go index d5d2a411..6c6877ce 100644 --- a/storage/oceanstor/volume/san.go +++ b/storage/oceanstor/volume/san.go @@ -24,6 +24,7 @@ import ( "strconv" "time" + pkgUtils "huawei-csi-driver/pkg/utils" "huawei-csi-driver/storage/oceanstor/client" "huawei-csi-driver/storage/oceanstor/smartx" "huawei-csi-driver/utils" @@ -52,7 +53,10 @@ func (p *SAN) preCreate(ctx context.Context, params map[string]interface{}) erro return err } - name := params["name"].(string) + name, ok := params["name"].(string) + if !ok { + return pkgUtils.Errorf(ctx, "format name to string failed, data: %v", params["name"]) + } params["name"] = p.cli.MakeLunName(name) if v, exist := params["sourcevolumename"].(string); exist { @@ -149,13 +153,16 @@ func (p *SAN) Delete(ctx context.Context, name string) error { return nil } - rssStr := lun["HASRSSOBJECT"].(string) - + rssStr, ok := lun["HASRSSOBJECT"].(string) + if !ok { + return pkgUtils.Errorf(ctx, "convert rssStr to string failed, data: %v", lun["HASRSSOBJECT"]) + } var rss map[string]string - json.Unmarshal([]byte(rssStr), &rss) - + err = json.Unmarshal([]byte(rssStr), &rss) + if err != nil { + return pkgUtils.Errorf(ctx, "Unmarshal san HASRSSOBJECT failed, data: %v, err: %v", rssStr, err) + } taskflow := taskflow.NewTaskFlow(ctx, "Delete-LUN-Volume") - if hyperMetro, ok := rss["HyperMetro"]; ok && hyperMetro == "TRUE" { taskflow.AddTask("Delete-HyperMetro", p.deleteHyperMetro, nil) taskflow.AddTask("Delete-HyperMetro-Remote-LUN", p.deleteHyperMetroRemoteLun, nil) @@ -199,7 +206,7 @@ func (p *SAN) Expand(ctx context.Context, name string, newSize int64) (bool, err } isAttached := lun["EXPOSEDTOINITIATOR"] == "true" - curSize, _ := strconv.ParseInt(lun["CAPACITY"].(string), 10, 64) + curSize := utils.ParseIntWithDefault(lun["CAPACITY"].(string), 10, 64, 0) if newSize <= curSize { msg := fmt.Sprintf("Lun %s newSize %d must be greater than curSize %d", lunName, newSize, curSize) log.AddContext(ctx).Errorln(msg) @@ -207,8 +214,10 @@ func (p *SAN) Expand(ctx context.Context, name string, newSize int64) (bool, err } var rss map[string]string - json.Unmarshal([]byte(lun["HASRSSOBJECT"].(string)), &rss) - + err = json.Unmarshal([]byte(lun["HASRSSOBJECT"].(string)), &rss) + if err != nil { + return false, pkgUtils.Errorf(ctx, "Unmarshal HASHSSOBJECT failed, error: %v", err) + } expandTask := taskflow.NewTaskFlow(ctx, "Expand-LUN-Volume") expandTask.AddTask("Expand-PreCheck-Capacity", p.preExpandCheckCapacity, nil) @@ -249,8 +258,10 @@ func (p *SAN) Expand(ctx context.Context, name string, newSize int64) (bool, err func (p *SAN) createLocalLun(ctx context.Context, params, taskResult map[string]interface{}) (map[string]interface{}, error) { - lunName := params["name"].(string) - + lunName, ok := params["name"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "parse lun name to string failed, data: %v", params["name"]) + } lun, err := p.cli.GetLunByName(ctx, lunName) if err != nil { log.AddContext(ctx).Errorf("Get LUN %s error: %v", lunName, err) @@ -258,7 +269,7 @@ func (p *SAN) createLocalLun(ctx context.Context, } if lun == nil { - params["parentid"] = params["poolID"].(string) + params["parentid"] = params["poolID"] if _, exist := params["clonefrom"]; exist { lun, err = p.clone(ctx, params, taskResult) @@ -287,7 +298,10 @@ func (p *SAN) createLocalLun(ctx context.Context, } func (p *SAN) clonePair(ctx context.Context, params map[string]interface{}) (map[string]interface{}, error) { - cloneFrom := params["clonefrom"].(string) + cloneFrom, ok := params["clonefrom"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "parse clonefrom to string failed, data: %v", params["clonefrom"]) + } srcLun, err := p.cli.GetLunByName(ctx, cloneFrom) if err != nil { log.AddContext(ctx).Errorf("Get clone src LUN %s error: %v", cloneFrom, err) @@ -303,7 +317,10 @@ func (p *SAN) clonePair(ctx context.Context, params map[string]interface{}) (map if err != nil { return nil, err } - cloneLunCapacity := params["capacity"].(int64) + cloneLunCapacity, ok := params["capacity"].(int64) + if !ok { + log.AddContext(ctx).Warningf("parse cloneLunCapacity to int64 failed, data: %v", params["capacity"]) + } if cloneLunCapacity < srcLunCapacity { msg := fmt.Sprintf("Clone LUN capacity must be >= src %s", cloneFrom) log.AddContext(ctx).Errorln(msg) @@ -323,11 +340,21 @@ func (p *SAN) clonePair(ctx context.Context, params map[string]interface{}) (map return nil, err } } - srcLunID := srcLun["ID"].(string) - dstLunID := dstLun["ID"].(string) + srcLunID, ok := srcLun["ID"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert srcLunID to string failed, data: %v", srcLun["ID"]) + } + dstLunID, ok := dstLun["ID"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert dstLunID to string failed, data: %v", dstLun["ID"]) + } + cloneSpeed, ok := params["clonespeed"].(int) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert clonespeed to int failed, data: %v", params["clonespeed"]) + } - cloneSpeed := params["clonespeed"].(int) - err = p.createClonePair(ctx, clonePairRequest{srcLunID: srcLunID, + err = p.createClonePair(ctx, clonePairRequest{ + srcLunID: srcLunID, dstLunID: dstLunID, cloneLunCapacity: cloneLunCapacity, srcLunCapacity: srcLunCapacity, @@ -344,7 +371,10 @@ func (p *SAN) clonePair(ctx context.Context, params map[string]interface{}) (map func (p *SAN) fromSnapshotByClonePair(ctx context.Context, params map[string]interface{}) (map[string]interface{}, error) { - srcSnapshotName := params["fromSnapshot"].(string) + srcSnapshotName, ok := params["fromSnapshot"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "format srcSnapshotName to string failed, data: %v", params["fromSnapshot"]) + } srcSnapshot, err := p.cli.GetLunSnapshotByName(ctx, srcSnapshotName) if err != nil { return nil, err @@ -360,7 +390,10 @@ func (p *SAN) fromSnapshotByClonePair(ctx context.Context, return nil, err } - cloneLunCapacity := params["capacity"].(int64) + cloneLunCapacity, ok := params["capacity"].(int64) + if !ok { + return nil, pkgUtils.Errorf(ctx, "parse capacity to int64 failed, data: %v", params["capacity"]) + } if cloneLunCapacity < srcSnapshotCapacity { msg := fmt.Sprintf("Clone target LUN capacity must be >= src snapshot %s", srcSnapshotName) log.AddContext(ctx).Errorln(msg) @@ -381,9 +414,19 @@ func (p *SAN) fromSnapshotByClonePair(ctx context.Context, } } - srcSnapshotID := srcSnapshot["ID"].(string) - dstLunID := dstLun["ID"].(string) - cloneSpeed := params["clonespeed"].(int) + srcSnapshotID, ok := srcSnapshot["ID"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert srcSnapshotID to string failed,data: %v", srcSnapshot["ID"]) + } + dstLunID, ok := dstLun["ID"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "parse dstLunID to string failed, data: %v", dstLun["ID"]) + } + + cloneSpeed, ok := params["clonespeed"].(int) + if !ok { + return nil, pkgUtils.Errorf(ctx, "parse clonespeed to int failed, data: %v", params["clonespeed"]) + } err = p.createClonePair(ctx, clonePairRequest{srcLunID: srcSnapshotID, dstLunID: dstLunID, cloneLunCapacity: cloneLunCapacity, @@ -392,7 +435,6 @@ func (p *SAN) fromSnapshotByClonePair(ctx context.Context, if err != nil { log.AddContext(ctx).Errorf("Clone snapshot by clone pair, source snapshot ID %s,"+ " target lun ID %s error: %s", srcSnapshotID, dstLunID, err) - p.cli.DeleteLun(ctx, dstLunID) return nil, err } @@ -418,7 +460,10 @@ func (p *SAN) createClonePair(ctx context.Context, return err } - clonePairID := clonePair["ID"].(string) + clonePairID, ok := clonePair["ID"].(string) + if !ok { + return pkgUtils.Errorf(ctx, "clonePairID convert to string failed, data: %v", clonePair["ID"]) + } if clonePairReq.srcLunCapacity < clonePairReq.cloneLunCapacity { err = p.cli.ExtendLun(ctx, clonePairReq.dstLunID, clonePairReq.cloneLunCapacity) if err != nil { @@ -445,7 +490,10 @@ func (p *SAN) createClonePair(ctx context.Context, } func (p *SAN) lunCopy(ctx context.Context, params map[string]interface{}) (map[string]interface{}, error) { - clonefrom := params["clonefrom"].(string) + clonefrom, ok := params["clonefrom"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "clonefrom convert to string failed, data: %v", params["clonefrom"]) + } srcLun, err := p.cli.GetLunByName(ctx, clonefrom) if err != nil { log.AddContext(ctx).Errorf("Get clone src LUN %s error: %v", clonefrom, err) @@ -460,8 +508,10 @@ func (p *SAN) lunCopy(ctx context.Context, params map[string]interface{}) (map[s if err != nil { return nil, err } - - cloneLunCapacity := params["capacity"].(int64) + cloneLunCapacity, ok := params["capacity"].(int64) + if !ok { + return nil, pkgUtils.Errorf(ctx, "parse copyLunCapacity to int64 failed, data: %v", params["capacity"]) + } if cloneLunCapacity < srcLunCapacity { msg := fmt.Sprintf("Clone LUN capacity must be >= src %s", clonefrom) log.AddContext(ctx).Errorln(msg) @@ -478,10 +528,16 @@ func (p *SAN) lunCopy(ctx context.Context, params map[string]interface{}) (map[s } } - srcLunID := srcLun["ID"].(string) - dstLunID := dstLun["ID"].(string) - snapshotName := fmt.Sprintf("k8s_lun_%s_to_%s_snap", srcLunID, dstLunID) + srcLunID, ok := srcLun["ID"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "srcLunID convert to string failed, data: %v", srcLun["ID"]) + } + dstLunID, ok := dstLun["ID"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "dstLunID convert to string failed, data: %v", dstLun["ID"]) + } + snapshotName := fmt.Sprintf("k8s_lun_%s_to_%s_snap", srcLunID, dstLunID) smartX := smartx.NewSmartX(p.cli) snapshot, err := p.cli.GetLunSnapshotByName(ctx, snapshotName) if err != nil { @@ -529,7 +585,11 @@ func (p *SAN) ensureLUNCopy(ctx context.Context, snapshotID, dstLunID string, cl func (p *SAN) fromSnapshotByLunCopy(ctx context.Context, params map[string]interface{}) (map[string]interface{}, error) { - srcSnapshotName := params["fromSnapshot"].(string) + srcSnapshotName, ok := params["fromSnapshot"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "srcSnapshotName convert to string failed, data: %v", params["fromSnapshot"]) + } + srcSnapshot, err := p.cli.GetLunSnapshotByName(ctx, srcSnapshotName) if err != nil { return nil, err @@ -562,7 +622,10 @@ func (p *SAN) fromSnapshotByLunCopy(ctx context.Context, } } - dstLunID := dstLun["ID"].(string) + dstLunID, ok := dstLun["ID"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "dstLunID convert to string failed, data: %v", dstLun["ID"]) + } lunCopyName, err := p.createLunCopy(ctx, srcSnapshot["ID"].(string), dstLunID, params["clonespeed"].(int), false) if err != nil { @@ -604,7 +667,10 @@ func (p *SAN) createLunCopy(ctx context.Context, } } - lunCopyID := lunCopy["ID"].(string) + lunCopyID, ok := lunCopy["ID"].(string) + if !ok { + return "", pkgUtils.Errorf(ctx, "lunCopyID convert to string failed, data: %v", lunCopy["ID"]) + } err = p.cli.StartLunCopy(ctx, lunCopyID) if err != nil { @@ -650,7 +716,10 @@ func (p *SAN) createLocalQoS(ctx context.Context, return nil, nil } - lunID := taskResult["localLunID"].(string) + lunID, ok := taskResult["localLunID"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "lunID convert to string failed, data: %v", taskResult["localLunID"]) + } lun, err := p.cli.GetLunByID(ctx, lunID) if err != nil { return nil, err @@ -695,7 +764,10 @@ func (p *SAN) getLunCopyOfLunID(ctx context.Context, lunID string) (string, erro var lunCopyIDs []string - json.Unmarshal([]byte(lunCopyIDStr), &lunCopyIDs) + err = json.Unmarshal([]byte(lunCopyIDStr), &lunCopyIDs) + if err != nil { + return "", pkgUtils.Errorf(ctx, "Unmarshal lunCopyIDStr failed, error: %v", err) + } if len(lunCopyIDs) <= 0 { return "", nil } @@ -718,8 +790,16 @@ func (p *SAN) deleteLunCopy(ctx context.Context, lunCopyName string, isDeleteSna return nil } - lunCopyID := lunCopy["ID"].(string) - runningStatus := lunCopy["RUNNINGSTATUS"].(string) + lunCopyID, ok := lunCopy["ID"].(string) + if !ok { + return pkgUtils.Errorf(ctx, "lunCopyID convert to string failed, data: %v", lunCopy["ID"]) + } + + runningStatus, ok := lunCopy["RUNNINGSTATUS"].(string) + if !ok { + return pkgUtils.Errorf(ctx, "runningStatus convert to string failed, data: %v", lunCopy["RUNNINGSTATUS"]) + } + if runningStatus == lunCopyRunningStatusQueuing || runningStatus == lunCopyRunningStatusCopying { p.cli.StopLunCopy(ctx, lunCopyID) @@ -730,10 +810,17 @@ func (p *SAN) deleteLunCopy(ctx context.Context, lunCopyName string, isDeleteSna return err } - snapshotName := lunCopy["SOURCELUNNAME"].(string) + snapshotName, ok := lunCopy["SOURCELUNNAME"].(string) + if !ok { + return pkgUtils.Errorf(ctx, "snapshotName convert to string failed, data: %v", lunCopy["SOURCELUNNAME"]) + } + snapshot, err := p.cli.GetLunSnapshotByName(ctx, snapshotName) if err == nil && snapshot != nil && isDeleteSnapshot { - snapshotID := snapshot["ID"].(string) + snapshotID, ok := snapshot["ID"].(string) + if !ok { + return pkgUtils.Errorf(ctx, "snapshotID convert to string failed, data: %v", snapshot["ID"]) + } smartX := smartx.NewSmartX(p.cli) smartX.DeleteLunSnapshot(ctx, snapshotID) } @@ -751,12 +838,18 @@ func (p *SAN) waitLunCopyFinish(ctx context.Context, lunCopyName string) error { return true, nil } - healthStatus := lunCopy["HEALTHSTATUS"].(string) + healthStatus, ok := lunCopy["HEALTHSTATUS"].(string) + if !ok { + return false, pkgUtils.Errorf(ctx, "healthStatus convert to string failed, data: %v", lunCopy["HEALTHSTATUS"]) + } if healthStatus == lunCopyHealthStatusFault { - return false, fmt.Errorf("Luncopy %s is at fault status", lunCopyName) + return false, fmt.Errorf("luncopy %s is at fault status", lunCopyName) } - runningStatus := lunCopy["RUNNINGSTATUS"].(string) + runningStatus, ok := lunCopy["RUNNINGSTATUS"].(string) + if !ok { + return false, pkgUtils.Errorf(ctx, "runningStatus convert to string failed, data: %v", lunCopy["RUNNINGSTATUS"]) + } if runningStatus == lunCopyRunningStatusQueuing || runningStatus == lunCopyRunningStatusCopying { return false, nil @@ -785,12 +878,18 @@ func (p *SAN) waitClonePairFinish(ctx context.Context, clonePairID string) error return true, nil } - healthStatus := clonePair["copyStatus"].(string) + healthStatus, ok := clonePair["copyStatus"].(string) + if !ok { + return false, pkgUtils.Errorf(ctx, "healthStatus convert to string failed, data: %v", clonePair["copyStatus"]) + } if healthStatus == clonePairHealthStatusFault { return false, fmt.Errorf("ClonePair %s is at fault status", clonePairID) } - runningStatus := clonePair["syncStatus"].(string) + runningStatus, ok := clonePair["syncStatus"].(string) + if !ok { + return false, pkgUtils.Errorf(ctx, "runningStatus convert to string failed, data: %v", clonePair["syncStatus"]) + } if runningStatus == clonePairRunningStatusNormal { return true, nil } else if runningStatus == clonePairRunningStatusSyncing || @@ -812,7 +911,10 @@ func (p *SAN) waitClonePairFinish(ctx context.Context, clonePairID string) error func (p *SAN) waitCloneFinish(ctx context.Context, lun map[string]interface{}, taskResult map[string]interface{}) error { - lunID := lun["ID"].(string) + lunID, ok := lun["ID"].(string) + if !ok { + return pkgUtils.Errorf(ctx, "lunID convert to string failed, data: %v", lun["ID"]) + } if p.product == "DoradoV6" { // ID of clone pair is the same as destination LUN ID err := p.waitClonePairFinish(ctx, lunID) @@ -838,9 +940,15 @@ func (p *SAN) waitCloneFinish(ctx context.Context, func (p *SAN) createRemoteLun(ctx context.Context, params, taskResult map[string]interface{}) (map[string]interface{}, error) { - lunName := params["name"].(string) - remoteCli := taskResult["remoteCli"].(client.BaseClientInterface) + lunName, ok := params["name"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "lunName convert to string failed, data: %v", params["name"]) + } + remoteCli, ok := taskResult["remoteCli"].(client.BaseClientInterface) + if !ok { + return nil, pkgUtils.Errorf(ctx, "remoteCli convert to client.BaseClientInterface failed, data: %v", taskResult["remoteCli"]) + } lun, err := remoteCli.GetLunByName(ctx, lunName) if err != nil { log.AddContext(ctx).Errorf("Get remote LUN %s error: %v", lunName, err) @@ -853,7 +961,7 @@ func (p *SAN) createRemoteLun(ctx context.Context, return nil, err } - params["parentid"] = taskResult["remotePoolID"].(string) + params["parentid"] = taskResult["remotePoolID"] lun, err = remoteCli.CreateLun(ctx, params) if err != nil { log.AddContext(ctx).Errorf("Create remote LUN %s error: %v", lunName, err) @@ -871,7 +979,10 @@ func (p *SAN) revertRemoteLun(ctx context.Context, taskResult map[string]interfa if !exist { return nil } - remoteCli := taskResult["remoteCli"].(client.BaseClientInterface) + remoteCli, ok := taskResult["remoteCli"].(client.BaseClientInterface) + if !ok { + return pkgUtils.Errorf(ctx, "remoteCli convert to client.BaseClientInterface failed, data: %v", taskResult["remoteCli"]) + } return remoteCli.DeleteLun(ctx, lunID) } @@ -882,9 +993,15 @@ func (p *SAN) createRemoteQoS(ctx context.Context, return nil, nil } - lunID := taskResult["remoteLunID"].(string) - remoteCli := taskResult["remoteCli"].(client.BaseClientInterface) + lunID, ok := taskResult["remoteLunID"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "lunID convert to string failed, data: %v", taskResult["remoteLunID"]) + } + remoteCli, ok := taskResult["remoteCli"].(client.BaseClientInterface) + if !ok { + return nil, pkgUtils.Errorf(ctx, "remoteCli convert to client.BaseClientInterface failed, data: %v", taskResult["remoteCli"]) + } lun, err := remoteCli.GetLunByID(ctx, lunID) if err != nil { return nil, err @@ -911,16 +1028,29 @@ func (p *SAN) revertRemoteQoS(ctx context.Context, taskResult map[string]interfa if !lunIDExist || !qosIDExist { return nil } - remoteCli := taskResult["remoteCli"].(client.BaseClientInterface) + remoteCli, ok := taskResult["remoteCli"].(client.BaseClientInterface) + if !ok { + return pkgUtils.Errorf(ctx, "remoteCli convert to client.BaseClientInterface failed, data: %v", taskResult["remoteCli"]) + } smartX := smartx.NewSmartX(remoteCli) return smartX.DeleteQos(ctx, qosID, lunID, "lun", "") } func (p *SAN) createHyperMetro(ctx context.Context, params, taskResult map[string]interface{}) (map[string]interface{}, error) { - domainID := taskResult["metroDomainID"].(string) - localLunID := taskResult["localLunID"].(string) - remoteLunID := taskResult["remoteLunID"].(string) + domainID, ok := taskResult["metroDomainID"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "domainID convert to string failed, data: %v", taskResult["metroDomainID"]) + } + + localLunID, ok := taskResult["localLunID"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "localLunID convert to string failed, data: %v", taskResult["localLunID"]) + } + remoteLunID, ok := taskResult["remoteLunID"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "remoteLunID convert to string failed, data: %v", taskResult["remoteLunID"]) + } pair, err := p.cli.GetHyperMetroPairByLocalObjID(ctx, localLunID) if err != nil { @@ -949,7 +1079,10 @@ func (p *SAN) createHyperMetro(ctx context.Context, return nil, err } - pairID = pair["ID"].(string) + pairID, ok = pair["ID"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "pairID convert to string failed, data: %v", pair["ID"]) + } if needFirstSync { err := p.cli.SyncHyperMetroPair(ctx, pairID) if err != nil { @@ -959,7 +1092,10 @@ func (p *SAN) createHyperMetro(ctx context.Context, } } } else { - pairID = pair["ID"].(string) + pairID, ok = pair["ID"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "pairID convert to string failed, data: %v", pair["ID"]) + } } err = p.waitHyperMetroSyncFinish(ctx, pairID) @@ -986,12 +1122,18 @@ func (p *SAN) waitHyperMetroSyncFinish(ctx context.Context, pairID string) error return false, errors.New(msg) } - healthStatus := pair["HEALTHSTATUS"].(string) + healthStatus, ok := pair["HEALTHSTATUS"].(string) + if !ok { + return false, pkgUtils.Errorf(ctx, "healthStatus convert to string failed, data: %v", pair["HEALTHSTATUS"]) + } if healthStatus == hyperMetroPairHealthStatusFault { return false, fmt.Errorf("Hypermetro pair %s is fault", pairID) } - runningStatus := pair["RUNNINGSTATUS"].(string) + runningStatus, ok := pair["RUNNINGSTATUS"].(string) + if !ok { + return false, pkgUtils.Errorf(ctx, "runningStatus convert to string failed, data: %v", pair["RUNNINGSTATUS"]) + } if runningStatus == hyperMetroPairRunningStatusToSync || runningStatus == hyperMetroPairRunningStatusSyncing { return false, nil @@ -1051,7 +1193,12 @@ func (p *SAN) getHyperMetroParams(ctx context.Context, log.AddContext(ctx).Errorln(msg) return nil, errors.New(msg) } - if status := domain["RUNNINGSTATUS"].(string); status != hyperMetroDomainRunningStatusNormal { + + status, ok := domain["RUNNINGSTATUS"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "status convert to string failed, data: %v", domain["RUNNINGSTATUS"]) + } + if status != hyperMetroDomainRunningStatusNormal { msg := fmt.Sprintf("Hypermetro domain %s status is not normal", metroDomain) log.AddContext(ctx).Errorln(msg) return nil, errors.New(msg) @@ -1066,7 +1213,10 @@ func (p *SAN) getHyperMetroParams(ctx context.Context, func (p *SAN) deleteLocalLunCopy(ctx context.Context, params, taskResult map[string]interface{}) (map[string]interface{}, error) { - lunID := params["lunID"].(string) + lunID, ok := params["lunID"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "format lunID to string failed, data: %v", params["lunID"]) + } lunCopyName, err := p.getLunCopyOfLunID(ctx, lunID) if err != nil { @@ -1087,8 +1237,10 @@ func (p *SAN) deleteLocalLunCopy(ctx context.Context, func (p *SAN) deleteLocalHyperCopy(ctx context.Context, params, taskResult map[string]interface{}) (map[string]interface{}, error) { - lunID := params["lunID"].(string) - + lunID, ok := params["lunID"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "format lunID to string failed, data: %v", params["lunID"]) + } // ID of clone pair is the same as destination LUN ID clonePair, err := p.cli.GetClonePairInfo(ctx, lunID) if err != nil { @@ -1099,7 +1251,10 @@ func (p *SAN) deleteLocalHyperCopy(ctx context.Context, return nil, nil } - clonePairID := clonePair["ID"].(string) + clonePairID, ok := clonePair["ID"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "format clonePairID to string failed, data: %v", clonePair["ID"]) + } err = p.cli.DeleteClonePair(ctx, clonePairID) if err != nil { log.AddContext(ctx).Errorf("Delete clone pair %s error: %v", clonePairID, err) @@ -1111,7 +1266,11 @@ func (p *SAN) deleteLocalHyperCopy(ctx context.Context, func (p *SAN) deleteLocalLun(ctx context.Context, params, taskResult map[string]interface{}) (map[string]interface{}, error) { - lunName := params["lunName"].(string) + + lunName, ok := params["lunName"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "format lunName to string failed, data: %v", params["lunName"]) + } err := p.deleteLun(ctx, lunName, p.cli) return nil, err } @@ -1123,14 +1282,21 @@ func (p *SAN) deleteHyperMetroRemoteLun(ctx context.Context, return nil, nil } - lunName := params["lunName"].(string) + lunName, ok := params["lunName"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "format lunName to string failed, data: %v", params["lunName"]) + } err := p.deleteLun(ctx, lunName, p.metroRemoteCli) return nil, err } func (p *SAN) deleteHyperMetro(ctx context.Context, params, taskResult map[string]interface{}) (map[string]interface{}, error) { - lunID := params["lunID"].(string) + + lunID, ok := params["lunID"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "format lunID to string failed, data: %v", params["lunID"]) + } pair, err := p.cli.GetHyperMetroPairByLocalObjID(ctx, lunID) if err != nil { @@ -1141,9 +1307,14 @@ func (p *SAN) deleteHyperMetro(ctx context.Context, return nil, nil } - pairID := pair["ID"].(string) - status := pair["RUNNINGSTATUS"].(string) - + pairID, ok := pair["ID"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "parse pairID to string failed, data: %v", pair["ID"]) + } + status, ok := pair["RUNNINGSTATUS"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "parse running status to string failed, data: %v", pair["RUNNINGSTATUS"]) + } if status == hyperMetroPairRunningStatusNormal || status == hyperMetroPairRunningStatusToSync || status == hyperMetroPairRunningStatusSyncing { @@ -1162,7 +1333,10 @@ func (p *SAN) deleteHyperMetro(ctx context.Context, func (p *SAN) preExpandCheckRemoteCapacity(ctx context.Context, params map[string]interface{}, cli client.BaseClientInterface) (string, error) { // check the remote pool - name := params["name"].(string) + name, ok := params["name"].(string) + if !ok { + return "", pkgUtils.Errorf(ctx, "format name to string failed, data: %v", params["name"]) + } remoteLunName := p.cli.MakeLunName(name) remoteLun, err := cli.GetLunByName(ctx, remoteLunName) if err != nil { @@ -1175,7 +1349,11 @@ func (p *SAN) preExpandCheckRemoteCapacity(ctx context.Context, return "", errors.New(msg) } - newSize := params["size"].(int64) + newSize, ok := params["size"].(int64) + if !ok { + return "", pkgUtils.Errorf(ctx, "format newSize to int64 failed, data: %v", params["size"]) + } + curSize, err := strconv.ParseInt(remoteLun["CAPACITY"].(string), 10, 64) if err != nil { return "", err @@ -1217,8 +1395,10 @@ func (p *SAN) preExpandReplicationCheckRemoteCapacity(ctx context.Context, func (p *SAN) suspendHyperMetro(ctx context.Context, params, taskResult map[string]interface{}) (map[string]interface{}, error) { - lunID := params["lunID"].(string) - + lunID, ok := params["lunID"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "format lunID to string failed, data: %v", params["lunID"]) + } pair, err := p.cli.GetHyperMetroPairByLocalObjID(ctx, lunID) if err != nil { log.AddContext(ctx).Errorf("Get hypermetro pair by local obj ID %s error: %v", lunID, err) @@ -1228,9 +1408,15 @@ func (p *SAN) suspendHyperMetro(ctx context.Context, return nil, nil } - pairID := pair["ID"].(string) - status := pair["RUNNINGSTATUS"].(string) + pairID, ok := pair["ID"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "format pairID to string failed, data: %v", pair["ID"]) + } + status, ok := pair["RUNNINGSTATUS"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "format running status to string failed, data: %v", pair["RUNNINGSTATUS"]) + } if status == hyperMetroPairRunningStatusNormal || status == hyperMetroPairRunningStatusToSync || status == hyperMetroPairRunningStatusSyncing { @@ -1247,9 +1433,14 @@ func (p *SAN) suspendHyperMetro(ctx context.Context, func (p *SAN) expandHyperMetroRemoteLun(ctx context.Context, params, taskResult map[string]interface{}) (map[string]interface{}, error) { - remoteLunID := taskResult["remoteLunID"].(string) - newSize := params["size"].(int64) - + remoteLunID, ok := taskResult["remoteLunID"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert remoteLunID to string failed, data: %v", taskResult["remoteLunID"]) + } + newSize, ok := params["size"].(int64) + if !ok { + return nil, pkgUtils.Errorf(ctx, "format newSize to int64 failed, data: %v", params["size"]) + } err := p.metroRemoteCli.ExtendLun(ctx, remoteLunID, newSize) if err != nil { log.AddContext(ctx).Errorf("Extend hypermetro remote lun %s error: %v", remoteLunID, err) @@ -1261,7 +1452,11 @@ func (p *SAN) expandHyperMetroRemoteLun(ctx context.Context, func (p *SAN) syncHyperMetro(ctx context.Context, params, taskResult map[string]interface{}) (map[string]interface{}, error) { - pairID := taskResult["hyperMetroPairID"].(string) + + pairID, ok := taskResult["hyperMetroPairID"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "format pairID to string failed, data: %v", taskResult["hyperMetroPairID"]) + } if pairID == "" { return nil, nil } @@ -1277,9 +1472,16 @@ func (p *SAN) syncHyperMetro(ctx context.Context, func (p *SAN) expandLocalLun(ctx context.Context, params, taskResult map[string]interface{}) (map[string]interface{}, error) { - lunID := params["lunID"].(string) - newSize := params["size"].(int64) + lunID, ok := params["lunID"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "format lunID to string failed, data: %v", params["lunID"]) + } + + newSize, ok := params["size"].(int64) + if !ok { + return nil, pkgUtils.Errorf(ctx, "format newSize to int64 failed, data: %v", params["size"]) + } err := p.cli.ExtendLun(ctx, lunID, newSize) if err != nil { log.AddContext(ctx).Errorf("Expand lun %s error: %v", lunID, err) @@ -1301,8 +1503,10 @@ func (p *SAN) CreateSnapshot(ctx context.Context, log.AddContext(ctx).Errorln(msg) return nil, errors.New(msg) } - - lunId := lun["ID"].(string) + lunId, ok := lun["ID"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "parse lunID to string failed, data: %v", lun["ID"]) + } snapshot, err := p.cli.GetLunSnapshotByName(ctx, snapshotName) if err != nil { log.AddContext(ctx).Errorf("Get lun snapshot by name %s error: %v", snapshotName, err) @@ -1310,13 +1514,16 @@ func (p *SAN) CreateSnapshot(ctx context.Context, } if snapshot != nil { - snapshotParentId := snapshot["PARENTID"].(string) + snapshotParentId, ok := snapshot["PARENTID"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "parse snapshotParentId to string failed, data: %v", snapshot["PARENTID"]) + } if snapshotParentId != lunId { msg := fmt.Sprintf("Snapshot %s is already exist, but the parent LUN %s is incompatible", snapshotName, lunName) log.AddContext(ctx).Errorln(msg) return nil, errors.New(msg) } else { - snapshotSize, _ := strconv.ParseInt(snapshot["USERCAPACITY"].(string), 10, 64) + snapshotSize := utils.ParseIntWithDefault(snapshot["USERCAPACITY"].(string), 10, 64, 0) return p.getSnapshotReturnInfo(snapshot, snapshotSize), nil } } @@ -1342,7 +1549,7 @@ func (p *SAN) CreateSnapshot(ctx context.Context, return nil, err } - snapshotSize, _ := strconv.ParseInt(result["snapshotSize"].(string), 10, 64) + snapshotSize := utils.ParseIntWithDefault(result["snapshotSize"].(string), 10, 64, 0) return p.getSnapshotReturnInfo(snapshot, snapshotSize), nil } @@ -1372,8 +1579,16 @@ func (p *SAN) DeleteSnapshot(ctx context.Context, snapshotName string) error { func (p *SAN) createSnapshot(ctx context.Context, params, taskResult map[string]interface{}) (map[string]interface{}, error) { - lunID := params["lunID"].(string) - snapshotName := params["snapshotName"].(string) + + lunID, ok := params["lunID"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "format lunID to string failed, data: %v", params["lunID"]) + } + + snapshotName, ok := params["snapshotName"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "format snapshotName to string failed, data: %v", params["snapshotName"]) + } snapshot, err := p.cli.CreateLunSnapshot(ctx, snapshotName, lunID) if err != nil { @@ -1405,7 +1620,10 @@ func (p *SAN) waitSnapshotReady(ctx context.Context, snapshotName string) error return false, errors.New(msg) } - runningStatus := snapshot["RUNNINGSTATUS"].(string) + runningStatus, ok := snapshot["RUNNINGSTATUS"].(string) + if !ok { + return false, pkgUtils.Errorf(ctx, "format runningStatus to string failed, data: %v", snapshot["RUNNINGSTATUS"]) + } if err != nil { return false, err } @@ -1425,7 +1643,10 @@ func (p *SAN) waitSnapshotReady(ctx context.Context, snapshotName string) error } func (p *SAN) revertSnapshot(ctx context.Context, taskResult map[string]interface{}) error { - snapshotID := taskResult["snapshotId"].(string) + snapshotID, ok := taskResult["snapshotId"].(string) + if !ok { + return pkgUtils.Errorf(ctx, "format snapshotID to string failed, data: %v", taskResult["snapshotId"]) + } err := p.cli.DeleteLunSnapshot(ctx, snapshotID) if err != nil { log.AddContext(ctx).Errorf("Delete snapshot %s error: %v", snapshotID, err) @@ -1436,8 +1657,10 @@ func (p *SAN) revertSnapshot(ctx context.Context, taskResult map[string]interfac func (p *SAN) activateSnapshot(ctx context.Context, params, taskResult map[string]interface{}) (map[string]interface{}, error) { - snapshotID := taskResult["snapshotId"].(string) - + snapshotID, ok := taskResult["snapshotId"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "format snapshotID to string failed, data: %v", taskResult["snapshotId"]) + } err := p.cli.ActivateLunSnapshot(ctx, snapshotID) if err != nil { log.AddContext(ctx).Errorf("Activate snapshot %s error: %v", snapshotID, err) @@ -1448,8 +1671,11 @@ func (p *SAN) activateSnapshot(ctx context.Context, func (p *SAN) deleteSnapshot(ctx context.Context, params, taskResult map[string]interface{}) (map[string]interface{}, error) { - snapshotID := params["snapshotId"].(string) + snapshotID, ok := params["snapshotId"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "format snapshotID to string failed, data: %v", params["snapshotId"]) + } err := p.cli.DeleteLunSnapshot(ctx, snapshotID) if err != nil { log.AddContext(ctx).Errorf("Delete snapshot %s error: %v", snapshotID, err) @@ -1461,8 +1687,10 @@ func (p *SAN) deleteSnapshot(ctx context.Context, func (p *SAN) deactivateSnapshot(ctx context.Context, params, taskResult map[string]interface{}) (map[string]interface{}, error) { - snapshotID := params["snapshotId"].(string) - + snapshotID, ok := params["snapshotId"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "format snapshotID to string failed, data: %v", params["snapshotId"]) + } err := p.cli.DeactivateLunSnapshot(ctx, snapshotID) if err != nil { log.AddContext(ctx).Errorf("Deactivate snapshot %s error: %v", snapshotID, err) @@ -1490,7 +1718,10 @@ func (p *SAN) getReplicationParams(ctx context.Context, return nil, err } - sn := remoteSystem["ID"].(string) + sn, ok := remoteSystem["ID"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "parse remoteDeviceID to string failed, data: %v", remoteSystem["ID"]) + } remoteDeviceID, err := p.getRemoteDeviceID(ctx, sn) if err != nil { return nil, err @@ -1508,7 +1739,10 @@ func (p *SAN) getReplicationParams(ctx context.Context, func (p *SAN) deleteReplicationPair(ctx context.Context, params, taskResult map[string]interface{}) (map[string]interface{}, error) { - lunID := params["lunID"].(string) + lunID, ok := params["lunID"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "format lunID to string failed, data: %v", params["lunID"]) + } pairs, err := p.cli.GetReplicationPairByResID(ctx, lunID, 11) if err != nil { @@ -1520,9 +1754,14 @@ func (p *SAN) deleteReplicationPair(ctx context.Context, } for _, pair := range pairs { - pairID := pair["ID"].(string) - - runningStatus := pair["RUNNINGSTATUS"].(string) + pairID, ok := pair["ID"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "format pairID to string failed, data: %v", pair["ID"]) + } + runningStatus, ok := pair["RUNNINGSTATUS"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "format runningStatus to string failed, data: %v", pair["RUNNINGSTATUS"]) + } if runningStatus == replicationPairRunningStatusNormal || runningStatus == replicationPairRunningStatusSync { p.cli.SplitReplicationPair(ctx, pairID) @@ -1545,7 +1784,10 @@ func (p *SAN) deleteReplicationRemoteLun(ctx context.Context, return nil, nil } - lunName := params["lunName"].(string) + lunName, ok := params["lunName"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "format lunName to string failed, data: %v", params["lunName"]) + } err := p.deleteLun(ctx, lunName, p.replicaRemoteCli) return nil, err } @@ -1561,8 +1803,10 @@ func (p *SAN) deleteLun(ctx context.Context, name string, cli client.BaseClientI return nil } - lunID := lun["ID"].(string) - + lunID, ok := lun["ID"].(string) + if !ok { + return pkgUtils.Errorf(ctx, "format lunID to string failed, data: %v", lun["ID"]) + } qosID, exist := lun["IOCLASSID"].(string) if exist && qosID != "" { smartX := smartx.NewSmartX(cli) @@ -1584,8 +1828,10 @@ func (p *SAN) deleteLun(ctx context.Context, name string, cli client.BaseClientI func (p *SAN) splitReplication(ctx context.Context, params, taskResult map[string]interface{}) (map[string]interface{}, error) { - lunID := params["lunID"].(string) - + lunID, ok := params["lunID"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "format lunID to string failed, data: %v", params["lunID"]) + } pairs, err := p.cli.GetReplicationPairByResID(ctx, lunID, 11) if err != nil { return nil, err @@ -1598,9 +1844,15 @@ func (p *SAN) splitReplication(ctx context.Context, replicationPairIDs := []string{} for _, pair := range pairs { - pairID := pair["ID"].(string) + pairID, ok := pair["ID"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "format pairID to string failed, data: %v", pair["ID"]) + } + runningStatus, ok := pair["RUNNINGSTATUS"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "format runningStatus to string failed, data: %v", pair["RUNNINGSTATUS"]) + } - runningStatus := pair["RUNNINGSTATUS"].(string) if runningStatus != replicationPairRunningStatusNormal && runningStatus != replicationPairRunningStatusSync { continue @@ -1621,9 +1873,15 @@ func (p *SAN) splitReplication(ctx context.Context, func (p *SAN) expandReplicationRemoteLun(ctx context.Context, params, taskResult map[string]interface{}) (map[string]interface{}, error) { - remoteLunID := taskResult["remoteLunID"].(string) - newSize := params["size"].(int64) + remoteLunID, ok := taskResult["remoteLunID"].(string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "format remoteLunID to string failed, data: %v", taskResult["remoteLunID"]) + } + newSize, ok := params["size"].(int64) + if !ok { + return nil, pkgUtils.Errorf(ctx, "format newSize to int64 failed, data: %v", params["size"]) + } err := p.replicaRemoteCli.ExtendLun(ctx, remoteLunID, newSize) if err != nil { log.AddContext(ctx).Errorf("Extend replication remote lun %s error: %v", remoteLunID, err) @@ -1635,7 +1893,10 @@ func (p *SAN) expandReplicationRemoteLun(ctx context.Context, func (p *SAN) syncReplication(ctx context.Context, params, taskResult map[string]interface{}) (map[string]interface{}, error) { - replicationPairIDs := taskResult["replicationPairIDs"].([]string) + replicationPairIDs, ok := taskResult["replicationPairIDs"].([]string) + if !ok { + return nil, pkgUtils.Errorf(ctx, "format replicationPairIDs to []string failed, data: %v", taskResult["replicationPairIDs"]) + } for _, pairID := range replicationPairIDs { err := p.cli.SyncReplicationPair(ctx, pairID) diff --git a/utils/host_test.go b/utils/host_test.go index 5dbe6b98..d9491587 100644 --- a/utils/host_test.go +++ b/utils/host_test.go @@ -27,6 +27,8 @@ import ( "huawei-csi-driver/utils/log" ) +const FilePermission0777 = os.FileMode(0777) + func TestChmodFsPermission(t *testing.T) { currentDir, err := os.Getwd() if err != nil { @@ -34,10 +36,19 @@ func TestChmodFsPermission(t *testing.T) { } targetPath := path.Join(currentDir, "fsPermissionTest.txt") - _, err = os.Create(targetPath) + targetFile, err := os.Create(targetPath) if err != nil { log.Errorf("Create file/directory [%s] failed.", targetPath) } + defer func() { + if err := targetFile.Close(); err != nil { + t.Errorf("close file %s failed, error: %v", targetFile.Name(), err) + } + }() + err = targetFile.Chmod(FilePermission0777) + if err != nil { + log.Errorf("file targetFile chmod to 0600 failed, error: %v", err) + } defer func() { err := os.Remove(targetPath) diff --git a/utils/k8sutils/k8s_utils.go b/utils/k8sutils/k8s_utils.go index 6a125d51..34b83b02 100644 --- a/utils/k8sutils/k8s_utils.go +++ b/utils/k8sutils/k8s_utils.go @@ -53,6 +53,15 @@ type Interface interface { // GetVolume returns volumes on the node at K8S side GetVolume(ctx context.Context, nodeName string, driverName string) (map[string]struct{}, error) + // GetPVByName get all pv info + GetPVByName(ctx context.Context, name string) (*corev1.PersistentVolume, error) + + // ListPods get pods by namespace + ListPods(ctx context.Context, namespace string) (*corev1.PodList, error) + + // GetPod get pod by name and namespace + GetPod(ctx context.Context, namespace, podName string) (*corev1.Pod, error) + // GetVolumeAttributes returns volume attributes of PV GetVolumeAttributes(ctx context.Context, pvName string) (map[string]string, error) @@ -60,6 +69,7 @@ type Interface interface { Activate() // Deactivate the k8s helpers when stop the service Deactivate() + secretOps ConfigmapOps persistentVolumeClaimOps @@ -234,15 +244,25 @@ func (k *KubeClient) getPVByPVCName(ctx context.Context, namespace string, return pv, nil } -func (k *KubeClient) getPVByName(ctx context.Context, name string) (*corev1.PersistentVolume, error) { +func (k *KubeClient) GetPVByName(ctx context.Context, name string) (*corev1.PersistentVolume, error) { return k.clientSet.CoreV1(). PersistentVolumes(). Get(ctx, name, metav1.GetOptions{}) } +func (k *KubeClient) ListPods(ctx context.Context, namespace string) (*corev1.PodList, error) { + return k.clientSet.CoreV1(). + Pods(namespace).List(ctx, metav1.ListOptions{}) +} + +func (k *KubeClient) GetPod(ctx context.Context, namespace, podName string) (*corev1.Pod, error) { + return k.clientSet.CoreV1(). + Pods(namespace).Get(ctx, podName, metav1.GetOptions{}) +} + // GetVolumeAttributes returns volume attributes of PV func (k *KubeClient) GetVolumeAttributes(ctx context.Context, pvName string) (map[string]string, error) { - pv, err := k.getPVByName(ctx, pvName) + pv, err := k.GetPVByName(ctx, pvName) if err != nil { return nil, err } diff --git a/utils/utils.go b/utils/utils.go index aebf9594..7fc303a6 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -34,28 +34,22 @@ import ( "syscall" "time" + "github.com/container-storage-interface/spec/lib/go/csi" + "golang.org/x/sys/unix" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/util/uuid" + "huawei-csi-driver/cli/helper" "huawei-csi-driver/csi/app" "huawei-csi-driver/pkg/constants" "huawei-csi-driver/utils/log" - - "github.com/container-storage-interface/spec/lib/go/csi" - "golang.org/x/sys/unix" - "k8s.io/apimachinery/pkg/api/resource" ) const ( DoradoV6Prefix = "V600" OceanStorV5Prefix = "V500" - // OceanStorDoradoV6 Dorado V6 and OceanStor V6 are exactly the same - OceanStorDoradoV6 = "DoradoV6" - OceanStorDoradoV3 = "DoradoV3" - OceanStorV3 = "OceanStorV3" - OceanStorV5 = "OceanStorV5" - - defaultTimeout = 30 - longTimeout = 60 + longTimeout = 60 ) var ( @@ -135,12 +129,13 @@ func execShellCmdTimeout(ctx context.Context, var timeOut bool done := make(chan string) timeCh := make(chan bool) - defer func() { - close(done) - close(timeCh) - }() go func() { + defer func() { + close(done) + close(timeCh) + }() + output, timeOut, err = fun(ctx, format, logFilter, args...) if !timeOut { done <- "run command done" @@ -156,7 +151,7 @@ func execShellCmdTimeout(ctx context.Context, case <-done: return output, err case <-timeCh: - return "", constants.TimeoutError + return "", constants.ErrTimeout } } @@ -180,7 +175,7 @@ func execShellCmd(ctx context.Context, format string, logFilter bool, args ...in killProcess := true var killProcessAndSubprocess bool - timeoutDuration := defaultTimeout * time.Second + timeoutDuration := time.Duration(app.GetGlobalConfig().ExecCommandTimeout) * time.Second // Processes are not killed when formatting or capacity expansion commands time out. if strings.Contains(cmd, "mkfs") || strings.Contains(cmd, "resize2fs") || strings.Contains(cmd, "xfs_growfs") { @@ -306,9 +301,9 @@ func SplitVolumeId(volumeId string) (string, string) { func SplitSnapshotId(snapshotId string) (string, string, string) { splits := strings.SplitN(snapshotId, ".", 3) if len(splits) == 3 { - return splits[0], splits[1], splits[2] + return helper.GetBackendName(splits[0]), splits[1], splits[2] } - return splits[0], "", "" + return helper.GetBackendName(splits[0]), "", "" } func MergeMap(args ...map[string]interface{}) map[string]interface{} { @@ -409,9 +404,9 @@ func GetProductVersion(systemInfo map[string]interface{}) (string, error) { } if strings.HasPrefix(productVersion, DoradoV6Prefix) { - return OceanStorDoradoV6, nil + return constants.OceanStorDoradoV6, nil } else if strings.HasPrefix(productVersion, OceanStorV5Prefix) { - return OceanStorV5, nil + return constants.OceanStorV5, nil } productMode, ok := systemInfo["PRODUCTMODE"].(string) @@ -419,22 +414,20 @@ func GetProductVersion(systemInfo map[string]interface{}) (string, error) { log.Warningln("There is no PRODUCTMODE field in system info") } - if match, _ := regexp.MatchString(`8[0-9][0-9]`, productMode); match { - return OceanStorDoradoV3, nil + if match, err := regexp.MatchString(`8[0-9][0-9]`, productMode); err == nil && match { + return constants.OceanStorDoradoV3, nil } - return OceanStorV3, nil + return constants.OceanStorV3, nil } func IsSupportFeature(features map[string]int, feature string) bool { - var support bool - status, exist := features[feature] if exist { - support = status == 1 || status == 2 + return status == 1 || status == 2 } - return support + return false } func TransVolumeCapacity(size int64, unit int64) int64 { @@ -569,6 +562,10 @@ func IgnoreExistCode(err error, checkExitCode []string) error { // IsCapacityAvailable indicates whether the volume size is an integer multiple of 512. func IsCapacityAvailable(volumeSizeBytes int64, allocationUnitBytes int64) bool { + if allocationUnitBytes == 0 { + log.Warningf("IsCapacityAvailable.allocationUnitBytes is invalid, can't be zero") + return false + } return volumeSizeBytes%allocationUnitBytes == 0 } @@ -887,6 +884,35 @@ func GetPasswordFromSecret(ctx context.Context, SecretName, SecretNamespace stri return string(password), nil } +// GetCertFromSecret used to get cert from secret +func GetCertFromSecret(ctx context.Context, SecretName, SecretNamespace string) ([]byte, error) { + log.AddContext(ctx).Infof("Get cert from secret: %s, ns: %s.", SecretName, SecretNamespace) + secret, err := app.GetGlobalConfig().K8sUtils.GetSecret(ctx, SecretName, SecretNamespace) + if err != nil { + msg := fmt.Sprintf("Get secret with name [%s] and namespace [%s] failed, error: [%v]", + SecretName, SecretNamespace, err) + log.AddContext(ctx).Errorln(msg) + return nil, errors.New(msg) + } + + if secret == nil || secret.Data == nil { + msg := fmt.Sprintf("Get secret with name [%s] and namespace [%s], but "+ + "secret is nil or the data not exist in secret", SecretName, SecretNamespace) + log.AddContext(ctx).Errorln(msg) + return nil, errors.New(msg) + } + + cert, exist := secret.Data["tls.crt"] + if !exist { + msg := fmt.Sprintf("Get secret with name [%s] and namespace [%s], but "+ + "cert field not exist in secret data", SecretName, SecretNamespace) + log.AddContext(ctx).Errorln(msg) + return nil, errors.New(msg) + } + + return cert, nil +} + // StringContain return the string prefix whether in the target string list func StringContain(strPrefix string, stringList []string) bool { for _, s := range stringList { @@ -930,3 +956,28 @@ func ToStringSafe(i interface{}) string { r, _ := ToStringWithFlag(i) return r } + +// ParseIntWithDefault parseInt without error +func ParseIntWithDefault(s string, base int, bitSize int, defaultResult int64) int64 { + result, err := strconv.ParseInt(s, base, bitSize) + if err != nil { + log.Warningf("ParseInt failed, data: %s, err: %v", s, err) + return defaultResult + } + return result +} + +// AtoiWithDefault Atoi without error +func AtoiWithDefault(s string, defaultResult int) int { + result, err := strconv.Atoi(s) + if err != nil { + log.Warningf("Atoi failed, data: %s, err: %v", s, err) + return defaultResult + } + return result +} + +// NewContext new a context +func NewContext() context.Context { + return context.WithValue(context.Background(), "requestId", string(uuid.NewUUID())) +} diff --git a/utils/utils_test.go b/utils/utils_test.go index 4d2e3d1e..20756cc2 100644 --- a/utils/utils_test.go +++ b/utils/utils_test.go @@ -212,6 +212,45 @@ func TestGetPasswordFromSecret(t *testing.T) { }) } +func TestGetCertFromSecretFailed(t *testing.T) { + Convey("TestGetCertFromSecret secret is nil case", t, func() { + m := mockGetSecret(nil, nil) + defer m.Reset() + + _, err := GetCertFromSecret(context.TODO(), "sec-name", "sec-namespace") + So(err, ShouldBeError) + }) + + Convey("TestGetCertFromSecret get secret error case", t, func() { + m := mockGetSecret(nil, errors.New("mock error")) + defer m.Reset() + + _, err := GetCertFromSecret(context.TODO(), "sec-name", "sec-namespace") + So(err, ShouldBeError) + }) + + Convey("GetCertFromSecret secret data dose not have cert case", t, func() { + m := mockGetSecret(map[string][]byte{}, nil) + defer m.Reset() + + _, err := GetCertFromSecret(context.TODO(), "sec-name", "sec-namespace") + So(err, ShouldBeError) + }) +} + +func TestGetCertFromSecretSuccess(t *testing.T) { + Convey("GetCertFromSecret normal case", t, func() { + m := mockGetSecret(map[string][]byte{ + "tls.crt": []byte("mock-cert"), + }, nil) + defer m.Reset() + + pw, err := GetCertFromSecret(context.TODO(), "sec-name", "sec-namespace") + So(err, ShouldBeNil) + So(pw, ShouldEqual, []byte("mock-cert")) + }) +} + func TestIsContainFileType(t *testing.T) { type IsContainParam struct { target constants.FileType diff --git a/utils/version/version.go b/utils/version/version.go index 4858447e..baab1b8d 100644 --- a/utils/version/version.go +++ b/utils/version/version.go @@ -56,7 +56,7 @@ func InitVersion(versionFile, version string) error { log.Infof("Open version file %s is exist.", versionFile) return nil } - log.Errorf("Open version file %s failed %v", versionFile, err) + log.Errorln("Open version file %s failed", versionFile) return err } defer func(file *os.File) { diff --git a/utils/wwn_helper_test.go b/utils/wwn_helper_test.go index 77a9ef7f..c1df5240 100644 --- a/utils/wwn_helper_test.go +++ b/utils/wwn_helper_test.go @@ -171,7 +171,7 @@ func mockWriteWwnFile() error { func cleanMockFile() { err := os.RemoveAll(defaultWwnFileDir) if err != nil { - log.Errorf("clean mock file failed, error: %v", err) + log.Errorln("clean mock file failed") return } }