diff --git a/go.mod b/go.mod index 3bb4ba813a..48dabb98bd 100644 --- a/go.mod +++ b/go.mod @@ -349,7 +349,7 @@ require ( require ( github.com/DataDog/ebpf-manager v0.2.16 - github.com/GuanceCloud/cliutils v1.1.21-0.20240821062850-3079bb7ed98d + github.com/GuanceCloud/cliutils v1.1.21-0.20240904042137-2a87297900d6 github.com/andrewkroh/sys v0.0.0-20151128191922-287798fe3e43 github.com/brianvoe/gofakeit/v6 v6.28.0 github.com/cilium/ebpf v0.11.0 diff --git a/go.sum b/go.sum index 0d45fb5ffb..c9fd3801a8 100644 --- a/go.sum +++ b/go.sum @@ -143,8 +143,8 @@ github.com/DataDog/sketches-go v1.4.1 h1:j5G6as+9FASM2qC36lvpvQAj9qsv/jUs3FtO8Cw github.com/DataDog/sketches-go v1.4.1/go.mod h1:xJIXldczJyyjnbDop7ZZcLxJdV3+7Kra7H1KMgpgkLk= github.com/GuanceCloud/client_model v0.0.0-20230418154757-93bd4e878a5e h1:i34dA4kiRTfG+KdvkIXCLPDduarVeFlQhGDD3TefgS4= github.com/GuanceCloud/client_model v0.0.0-20230418154757-93bd4e878a5e/go.mod h1:PMnE48aPzuRu83FmWZugC0O3d54ZupJd/MmiaYxz8sM= -github.com/GuanceCloud/cliutils v1.1.21-0.20240821062850-3079bb7ed98d h1:dM5YEBXuFuwiZ5roAzxfA8RK2VwE0TnHAl5zAsHSKFg= -github.com/GuanceCloud/cliutils v1.1.21-0.20240821062850-3079bb7ed98d/go.mod h1:Qbeedf/Ji3immd8Ka01NDQG6SP6j8JBnbZwsHTtxyqs= +github.com/GuanceCloud/cliutils v1.1.21-0.20240904042137-2a87297900d6 h1:hXvV/9i3aWkVVnpnHLngbpjZwTU+ut36YxOZOS2J4MM= +github.com/GuanceCloud/cliutils v1.1.21-0.20240904042137-2a87297900d6/go.mod h1:Qbeedf/Ji3immd8Ka01NDQG6SP6j8JBnbZwsHTtxyqs= github.com/GuanceCloud/confd v0.1.101 h1:yjHgfl6YzAlTbFOFMTE4ERpFJzIyovOW7ZFc2/ZssL0= github.com/GuanceCloud/confd v0.1.101/go.mod h1:o0opIwOX+yNwV9nh56x5ymFMJ+YBD8JuPxBJ7a1mEmo= github.com/GuanceCloud/dockertest/v3 v3.9.4 h1:ScSNhfA2HSNLfrYoNd1KSRxkrymlKiBE60g4f6eUoOk= diff --git a/internal/cmds/pldbg.go b/internal/cmds/pldbg.go index 20e11f7f94..1a0adc9f5d 100644 --- a/internal/cmds/pldbg.go +++ b/internal/cmds/pldbg.go @@ -215,28 +215,24 @@ func plScriptTmpStore(category point.Category) (*manager.ScriptStore, map[string errs := map[string]map[string]error{} { // default - ns := manager.DefaultScriptNS + ns := manager.NSDefault plPath := filepath.Join(datakit.InstallDir, "pipeline") - scripts, scriptsPath := manager.ReadPlScriptFromPlStructPath(plPath) - errs[ns] = store.UpdateScriptsWithNS(ns, scripts[category], scriptsPath[category], nil) + scripts, _ := manager.ReadWorkspaceScripts(plPath) + errs[ns] = store.UpdateScriptsWithNS(ns, scripts[category], nil) } { // gitrepo - ns := manager.GitRepoScriptNS + ns := manager.NSGitRepo plPath := filepath.Join(datakit.GitReposRepoFullPath, "pipeline") - scripts, scriptsPath := manager.ReadPlScriptFromPlStructPath(plPath) - errs[ns] = store.UpdateScriptsWithNS(ns, scripts[category], scriptsPath[category], nil) + scripts, _ := manager.ReadWorkspaceScripts(plPath) + errs[ns] = store.UpdateScriptsWithNS(ns, scripts[category], nil) } { // remote - ns := manager.RemoteScriptNS + ns := manager.NSRemote plPath := filepath.Join(datakit.PipelineRemoteDir, plremote.GetConentFileName()) if tarMap, err := targzutil.ReadTarToMap(plPath); err == nil { allCategory := plremote.ConvertContentMapToThreeMap(tarMap) scripts := allCategory[category.String()] - scriptsPath := map[string]string{} - for k := range scripts { - scriptsPath[k] = filepath.Join(plPath, category.String(), k) - } - errs[ns] = store.UpdateScriptsWithNS(ns, scripts, scriptsPath, nil) + errs[ns] = store.UpdateScriptsWithNS(ns, scripts, nil) } } diff --git a/internal/confd/confd.go b/internal/confd/confd.go index 59174cb99a..3218058210 100644 --- a/internal/confd/confd.go +++ b/internal/confd/confd.go @@ -497,9 +497,9 @@ func getPipelineData() { // Update pipeline script. l.Debug("before set pipelines from confd ") - if managerwkr, ok := plval.GetManager(); ok && managerwkr != nil { - manager.LoadScripts2StoreFromPlStructPath( - managerwkr, manager.ConfdScriptNS, datakit.ConfdPipelineDir, nil) + if m, ok := plval.GetManager(); ok && m != nil { + m.LoadScriptsFromWorkspace( + manager.NSConfd, datakit.ConfdPipelineDir, nil) } } diff --git a/internal/config/load.go b/internal/config/load.go index 792b95f848..c347712976 100644 --- a/internal/config/load.go +++ b/internal/config/load.go @@ -234,7 +234,7 @@ func GetPipelinePath(category point.Category, pipeLineName string) (string, erro } { - files := plmanager.SearchPlFilePathFromPlStructPath(datakit.GitReposRepoFullPath) + files := plmanager.SearchWorkspaceScripts(datakit.GitReposRepoFullPath) if f, ok := files[category]; ok { if plPath, ok := f[pipeLineName]; ok { if _, err := os.Stat(plPath); err != nil { @@ -246,7 +246,7 @@ func GetPipelinePath(category point.Category, pipeLineName string) (string, erro } } - files := plmanager.SearchPlFilePathFromPlStructPath(datakit.PipelineDir) + files := plmanager.SearchWorkspaceScripts(datakit.PipelineDir) if f, ok := files[category]; ok { if plPath, ok := f[pipeLineName]; ok { if _, err := os.Stat(plPath); err != nil { diff --git a/internal/httpapi/api_dca.go b/internal/httpapi/api_dca.go index 82eade76cf..1b10a69654 100644 --- a/internal/httpapi/api_dca.go +++ b/internal/httpapi/api_dca.go @@ -211,9 +211,8 @@ func ReloadDataKit(ctx context.Context) error { case 3: l.Info("before set pipelines") - if managerwkr, ok := plval.GetManager(); ok && managerwkr != nil { - manager.LoadScripts2StoreFromPlStructPath(managerwkr, - manager.GitRepoScriptNS, + if m, ok := plval.GetManager(); ok && m != nil { + m.LoadScriptsFromWorkspace(manager.NSGitRepo, filepath.Join(datakit.GitReposRepoFullPath, "pipeline"), nil) } @@ -895,7 +894,7 @@ func dcaTestPipelines(c *gin.Context) { category := point.CatString(body.Category) - pls, errs := pipeline.NewPipelineMulti(category, body.Pipeline[body.Category], nil, nil) + pls, errs := pipeline.NewPipelineMulti(category, body.Pipeline[body.Category], nil) if err, ok := errs[body.ScriptName]; ok && err != nil { context.fail(dcaError{ErrorCode: "400", ErrorMsg: fmt.Sprintf("pipeline parse error: %s", err.Error())}) return @@ -935,6 +934,7 @@ func dcaTestPipelines(c *gin.Context) { point.RUM, point.Security, point.Tracing, + point.DialTesting, point.UnknownCategory: arr, err := dec.Decode([]byte(data)) diff --git a/internal/httpapi/api_pipeline.go b/internal/httpapi/api_pipeline.go index 159f027f43..9ed3459145 100644 --- a/internal/httpapi/api_pipeline.go +++ b/internal/httpapi/api_pipeline.go @@ -244,7 +244,7 @@ func apiPipelineDebugHandler(w http.ResponseWriter, req *http.Request, whatever func parsePipeline(category point.Category, scriptName string, scripts map[string]string, buks *plmap.AggBuckets, ) (*manager.PlScript, error) { - success, faild := pipeline.NewPipelineMulti(category, scripts, nil, buks) + success, faild := pipeline.NewPipelineMulti(category, scripts, buks) if err, ok := faild[scriptName]; ok && err != nil { return nil, err } diff --git a/internal/io/dataway/ptgroup.go b/internal/io/dataway/ptgroup.go index 0914788797..e584ad0bb4 100644 --- a/internal/io/dataway/ptgroup.go +++ b/internal/io/dataway/ptgroup.go @@ -56,6 +56,7 @@ func (ptg *ptGrouper) setExtKVs() { switch ptg.cat { case point.Logging, + point.DialTesting, point.Network, point.KeyEvent, point.RUM: diff --git a/internal/io/feed_test.go b/internal/io/feed_test.go index 59bcd41cc1..18ca8e39ff 100644 --- a/internal/io/feed_test.go +++ b/internal/io/feed_test.go @@ -98,11 +98,10 @@ func TestRunpl(t *T.T) { if !ok { t.Error("!ok") } - manager.LoadScripts(m, manager.RemoteScriptNS, + m.LoadScripts(manager.NSRemote, map[point.Category]map[string]string{ point.Logging: {"a.p": "add_key('a', 1)"}, - }, - nil, nil) + }, nil) if _, ok := m.QueryScript(point.Logging, "a.p"); !ok { t.Error("!ok") } diff --git a/internal/io/filter/inputdata.go b/internal/io/filter/inputdata.go index 89d37c0174..a7427d0294 100644 --- a/internal/io/filter/inputdata.go +++ b/internal/io/filter/inputdata.go @@ -26,6 +26,7 @@ func (d *KVs) Setup(category point.Category, pt *point.Point) { switch category { case point.Logging, + point.DialTesting, point.Network, point.KeyEvent, point.RUM: diff --git a/internal/io/pipeline_pull.go b/internal/io/pipeline_pull.go index 0a6561a997..29db4b5011 100644 --- a/internal/io/pipeline_pull.go +++ b/internal/io/pipeline_pull.go @@ -8,11 +8,13 @@ package io import ( "encoding/base64" "fmt" + + "github.com/GuanceCloud/cliutils/point" ) // PullPipeline returns name/text, updateTime, err. -func PullPipeline(ts, relaTS int64) (mFiles, plRelation map[string]map[string]string, - defaultPl map[string]string, updateTime int64, relationTS int64, err error, +func PullPipeline(ts, relaTS int64) (mFiles, plRelation map[point.Category]map[string]string, + defaultPl map[point.Category]string, updateTime int64, relationTS int64, err error, ) { defer func() { if err := recover(); err != nil { @@ -35,34 +37,36 @@ func PullPipeline(ts, relaTS int64) (mFiles, plRelation map[string]map[string]st } func parsePipelinePullStruct(pulledStruct *pullPipelineReturn) ( - map[string]map[string]string, map[string]map[string]string, - map[string]string, int64, int64, error, + map[point.Category]map[string]string, map[point.Category]map[string]string, + map[point.Category]string, int64, int64, error, ) { - mFiles := make(map[string]map[string]string) - defaultPl := make(map[string]string) + mFiles := make(map[point.Category]map[string]string) + defaultPl := make(map[point.Category]string) for _, v := range pulledStruct.Pipelines { bys, err := base64.StdEncoding.DecodeString(v.Base64Text) if err != nil { return nil, nil, nil, 0, 0, err } + cat := point.CatString(v.Category) if v.AsDefault { - defaultPl[v.Category] = v.Name + defaultPl[cat] = v.Name } - if val, ok := mFiles[v.Category]; ok { + if val, ok := mFiles[cat]; ok { val[v.Name] = string(bys) } else { mf := make(map[string]string) mf[v.Name] = string(bys) - mFiles[v.Category] = mf + mFiles[cat] = mf } } - plRelation := make(map[string]map[string]string) + plRelation := make(map[point.Category]map[string]string) for _, v := range pulledStruct.Relation { - if m, ok := plRelation[v.Category]; !ok { - plRelation[v.Category] = map[string]string{ + cat := point.CatString(v.Category) + if m, ok := plRelation[cat]; !ok { + plRelation[cat] = map[string]string{ v.Source: v.Name, } } else { diff --git a/internal/io/pipeline_pull_test.go b/internal/io/pipeline_pull_test.go index 61d2a5b2cf..7615fd82fb 100644 --- a/internal/io/pipeline_pull_test.go +++ b/internal/io/pipeline_pull_test.go @@ -9,6 +9,7 @@ import ( "encoding/base64" "testing" + "github.com/GuanceCloud/cliutils/point" "github.com/stretchr/testify/assert" ) @@ -31,8 +32,8 @@ func TestPullPipeline(t *testing.T) { RelationTS int64 Pipelines *pullPipelineReturn Expect *struct { - mFiles, plRelation map[string]map[string]string - defaultPl map[string]string + mFiles, plRelation map[point.Category]map[string]string + defaultPl map[point.Category]string updateTime int64 relationUpdateTime int64 } @@ -44,10 +45,12 @@ func TestPullPipeline(t *testing.T) { UpdateTime: 1641796675, Pipelines: []*pipelineUnit{ { + Category: point.SLogging, Name: "123.p", Base64Text: base64.StdEncoding.EncodeToString([]byte("text1")), }, { + Category: point.SLogging, Name: "456.p", Base64Text: base64.StdEncoding.EncodeToString([]byte("text2")), AsDefault: true, @@ -55,35 +58,37 @@ func TestPullPipeline(t *testing.T) { }, Relation: []*pipelineRelation{ { - Source: "a1", - Name: "abc", + Category: point.SLogging, + Source: "a1", + Name: "abc", }, { - Source: "a2", - Name: "abc", + Category: point.SLogging, + Source: "a2", + Name: "abc", }, }, }, Expect: &struct { - mFiles, plRelation map[string]map[string]string - defaultPl map[string]string + mFiles, plRelation map[point.Category]map[string]string + defaultPl map[point.Category]string updateTime int64 relationUpdateTime int64 }{ - mFiles: map[string]map[string]string{ - "": { + mFiles: map[point.Category]map[string]string{ + point.Logging: { "123.p": "text1", "456.p": "text2", }, }, - plRelation: map[string]map[string]string{ - "": { + plRelation: map[point.Category]map[string]string{ + point.Logging: { "a1": "abc", "a2": "abc", }, }, - defaultPl: map[string]string{ - "": "456.p", + defaultPl: map[point.Category]string{ + point.Logging: "456.p", }, updateTime: 1641796675, }, @@ -95,14 +100,14 @@ func TestPullPipeline(t *testing.T) { UpdateTime: -1, }, Expect: &struct { - mFiles, plRelation map[string]map[string]string - defaultPl map[string]string + mFiles, plRelation map[point.Category]map[string]string + defaultPl map[point.Category]string updateTime int64 relationUpdateTime int64 }{ - mFiles: map[string]map[string]string{}, - plRelation: map[string]map[string]string{}, - defaultPl: map[string]string{}, + mFiles: map[point.Category]map[string]string{}, + plRelation: map[point.Category]map[string]string{}, + defaultPl: map[point.Category]string{}, updateTime: -1, }, }, @@ -132,8 +137,8 @@ func TestParsePipelinePullStruct(t *testing.T) { name string pipelines *pullPipelineReturn expect *struct { - mfiles, relation map[string]map[string]string - defaultPl map[string]string + mfiles, relation map[point.Category]map[string]string + defaultPl map[point.Category]string updateTime int64 relationUpdateTime int64 err error @@ -163,39 +168,41 @@ func TestParsePipelinePullStruct(t *testing.T) { }, Relation: []*pipelineRelation{ { - Source: "a1", - Name: "abc", + Category: point.SLogging, + Source: "a1", + Name: "abc", }, { - Source: "a2", - Name: "abc", + Category: point.SLogging, + Source: "a2", + Name: "abc", }, }, }, expect: &struct { - mfiles, relation map[string]map[string]string - defaultPl map[string]string + mfiles, relation map[point.Category]map[string]string + defaultPl map[point.Category]string updateTime int64 relationUpdateTime int64 err error }{ - mfiles: map[string]map[string]string{ - "logging": { + mfiles: map[point.Category]map[string]string{ + point.Logging: { "123.p": "text123", "1234.p": "text1234", }, - "metric": { + point.Metric: { "456.p": "text456", }, }, - relation: map[string]map[string]string{ - "": { + relation: map[point.Category]map[string]string{ + point.Logging: { "a1": "abc", "a2": "abc", }, }, - defaultPl: map[string]string{ - "logging": "1234.p", + defaultPl: map[point.Category]string{ + point.Logging: "1234.p", }, updateTime: 1653020819, }, @@ -223,22 +230,22 @@ func TestParsePipelinePullStruct(t *testing.T) { }, }, expect: &struct { - mfiles, relation map[string]map[string]string - defaultPl map[string]string + mfiles, relation map[point.Category]map[string]string + defaultPl map[point.Category]string updateTime int64 relationUpdateTime int64 err error }{ - mfiles: map[string]map[string]string{ - "logging": { + mfiles: map[point.Category]map[string]string{ + point.Logging: { "123.p": "text1234", }, - "metric": { + point.Metric: { "456.p": "text456", }, }, - relation: map[string]map[string]string{}, - defaultPl: map[string]string{}, + relation: map[point.Category]map[string]string{}, + defaultPl: map[point.Category]string{}, updateTime: 1653020819, }, }, diff --git a/internal/pipeline/pipeline.go b/internal/pipeline/pipeline.go index 734227a213..a0a99b558e 100644 --- a/internal/pipeline/pipeline.go +++ b/internal/pipeline/pipeline.go @@ -46,7 +46,7 @@ func InitPipeline(cfg *plval.PipelineCfg, upFn plmap.UploadFunc, gTags map[strin } func NewPlScriptSampleFromFile(category point.Category, path string, buks ...*plmap.AggBuckets) (*plmanager.PlScript, error) { - name, script, err := plmanager.ReadPlScriptFromFile(path) + name, script, err := plmanager.ReadScript(path) if err != nil { return nil, err } @@ -55,7 +55,7 @@ func NewPlScriptSampleFromFile(category point.Category, path string, buks ...*pl } func NewPlScriptSimple(category point.Category, name, script string, buks ...*plmap.AggBuckets) (*plmanager.PlScript, error) { - scs, errs := plmanager.NewScripts(map[string]string{name: script}, map[string]string{}, nil, "", category, buks...) + scs, errs := plmanager.NewScripts(map[string]string{name: script}, nil, "", category, buks...) if v, ok := errs[name]; ok { return nil, v @@ -68,10 +68,9 @@ func NewPlScriptSimple(category point.Category, name, script string, buks ...*pl return nil, fmt.Errorf("unknown error") } -func NewPipelineMulti(category point.Category, scripts map[string]string, - scriptPath map[string]string, buks *plmap.AggBuckets, +func NewPipelineMulti(category point.Category, scripts map[string]string, buks *plmap.AggBuckets, ) (map[string]*plmanager.PlScript, map[string]error) { - return plmanager.NewScripts(scripts, scriptPath, nil, "", category, buks) + return plmanager.NewScripts(scripts, nil, "", category, buks) } type Pipeline struct { diff --git a/internal/pipeline/pl.go b/internal/pipeline/pl.go index 8430dce45a..76c29df5f9 100644 --- a/internal/pipeline/pl.go +++ b/internal/pipeline/pl.go @@ -10,7 +10,6 @@ import ( "time" plmanager "github.com/GuanceCloud/cliutils/pipeline/manager" - "github.com/GuanceCloud/cliutils/pipeline/manager/relation" "github.com/GuanceCloud/cliutils/pipeline/ptinput" "github.com/GuanceCloud/cliutils/point" "github.com/GuanceCloud/platypus/pkg/ast" @@ -86,7 +85,7 @@ func RunPl(category point.Category, pts []*point.Point, } if v, ok := plval.GetOffload(); ok && v != nil && - script.NS() == plmanager.RemoteScriptNS && + script.NS() == plmanager.NSRemote && category == point.Logging { ptsOffload = append(ptsOffload, pt) continue @@ -169,7 +168,7 @@ func searchScript(cat point.Category, } relat := center.GetScriptRelation() - scriptName, ok := relation.ScriptName(relat, cat, pt, scriptMap) + scriptName, ok := plmanager.ScriptName(relat, cat, pt, scriptMap) if !ok { return nil, false } diff --git a/internal/pipeline/remote/remote.go b/internal/pipeline/remote/remote.go index d0d47831e6..5530011f02 100644 --- a/internal/pipeline/remote/remote.go +++ b/internal/pipeline/remote/remote.go @@ -10,7 +10,6 @@ import ( "context" "encoding/json" "io/fs" - "io/ioutil" "os" "path/filepath" "strings" @@ -54,11 +53,15 @@ type pipelineRemoteConfig struct { func StartPipelineRemote(urls []string) { runPipelineRemote.Do(func() { l = logger.SLogger(pipelineRemoteName) - g := datakit.G("pipeline_remote") - - g.Go(func(ctx context.Context) error { - return pullMain(urls, &pipelineRemoteImpl{}) - }) + if runner, err := pullMain(urls, &pipelineRemoteImpl{}); err != nil { + l.Error(err) + return + } else if runner != nil { + g := datakit.G("pipeline_remote") + g.Go(func(ctx context.Context) error { + return runner() + }) + } }) } @@ -75,8 +78,8 @@ type IPipelineRemote interface { ReadFile(filename string) ([]byte, error) WriteFile(filename string, data []byte, perm fs.FileMode) error ReadDir(dirname string) ([]fs.FileInfo, error) - PullPipeline(int64, int64) (mFiles map[string]map[string]string, plRelation map[string]map[string]string, - defaultPl map[string]string, updateTime int64, relationTS int64, err error) + PullPipeline(int64, int64) (mFiles map[point.Category]map[string]string, plRelation map[point.Category]map[string]string, + defaultPl map[point.Category]string, updateTime int64, relationTS int64, err error) GetTickerDurationAndBreak() (time.Duration, bool) Remove(name string) error FeedLastError(inputName string, err string) @@ -126,8 +129,8 @@ func (*pipelineRemoteImpl) ReadDir(dirname string) ([]fs.FileInfo, error) { } func (*pipelineRemoteImpl) PullPipeline(ts, relationTS int64) ( - map[string]map[string]string, map[string]map[string]string, - map[string]string, int64, int64, error, + map[point.Category]map[string]string, map[point.Category]map[string]string, + map[point.Category]string, int64, int64, error, ) { return io.PullPipeline(ts, relationTS) } @@ -159,11 +162,11 @@ func (*pipelineRemoteImpl) WriteTarFromMap(data map[string]string, dest string) //------------------------------------------------------------------------------ -func pullMain(urls []string, ipr IPipelineRemote) error { +func pullMain(urls []string, ipr IPipelineRemote) (func() error, error) { l.Info("start") if len(urls) == 0 { - return nil + return nil, nil } pathConfig := filepath.Join(datakit.PipelineRemoteDir, pipelineRemoteConfigFile) @@ -173,30 +176,40 @@ func pullMain(urls []string, ipr IPipelineRemote) error { td, isReturn := ipr.GetTickerDurationAndBreak() l.Infof("duration: %s", td.String()) - tick := time.NewTicker(td) - defer tick.Stop() - + // first run var err error + err = doPull(pathConfig, pathRelation, urls[0], ipr) + if err != nil { + l.Warnf("doPull: %s, ignored", err.Error()) + } + if isReturn { + return nil, nil + } - for { - err = doPull(pathConfig, pathRelation, urls[0], ipr) - if err != nil { - l.Warnf("doPull: %s, ignored", err.Error()) - } + return func() error { + tick := time.NewTicker(td) + defer tick.Stop() - if isReturn { - return nil - } + for { + err = doPull(pathConfig, pathRelation, urls[0], ipr) + if err != nil { + l.Warnf("doPull: %s, ignored", err.Error()) + } + + if isReturn { + return nil + } - select { - case <-datakit.Exit.Wait(): - l.Info("exit") - return nil + select { + case <-datakit.Exit.Wait(): + l.Info("exit") + return nil - case <-tick.C: - l.Debug("triggered") + case <-tick.C: + l.Debug("triggered") + } } - } + }, nil } func doPull(pathConfig, pathRelation, siteURL string, ipr IPipelineRemote) error { @@ -225,12 +238,17 @@ func doPull(pathConfig, pathRelation, siteURL string, ipr IPipelineRemote) error } if localTS == updateTime || updateTime <= 0 { - l.Debugf("already up to date: %d", updateTime) + l.Debugf("pipeline already up to date: %d", updateTime) } else { if updateTime == deleteAll { l.Debug("deleteAll") - spRelation.UpdateDefaultPl(nil) + // cleanup default pipeline + managerWkr.UpdateDefaultScript(nil) + if m, ok := plval.GetManager(); ok && m != nil { + // cleanup all remote scripts + m.LoadScripts(plmanager.NSRemote, nil, nil) + } // remove lcoal files if err := removeLocalRemote(ipr); err != nil { @@ -248,7 +266,7 @@ func doPull(pathConfig, pathRelation, siteURL string, ipr IPipelineRemote) error l.Debug("dumpFiles succeeded") loadContentPipeline(mFiles) - spRelation.UpdateDefaultPl(defaultPl) + managerWkr.UpdateDefaultScript(defaultPl) err = updatePipelineRemoteConfig(pathConfig, siteURL, updateTime, ipr) if err != nil { @@ -292,17 +310,12 @@ func removeLocalRemote(ipr IPipelineRemote) error { } } } - managerWkr, ok := plval.GetManager() - if !ok || managerWkr == nil { - return nil - } - - // cleanup all scripts - plmanager.LoadScripts(managerWkr, plmanager.RemoteScriptNS, nil, nil, nil) return nil } -func dumpFiles(mFiles map[string]map[string]string, defaultPl map[string]string, ipr IPipelineRemote) error { +func dumpFiles(mFiles map[point.Category]map[string]string, + defaultPl map[point.Category]string, ipr IPipelineRemote, +) error { l.Debugf("dumpFiles: %#v", mFiles) // remove lcoal files if err := removeLocalRemote(ipr); err != nil { @@ -320,19 +333,39 @@ type relationInfo struct { Relation map[string]map[string]string `json:"relation"` } -func dumpRelation(path string, relation map[string]map[string]string) error { +func dumpRelation(path string, relation map[point.Category]map[string]string) error { + rl := map[string]map[string]string{} + for c, r := range relation { + rl[c.String()] = r + } + if body, err := json.Marshal(&relationInfo{ - Relation: relation, + Relation: rl, }); err != nil { return err } else { - if err := ioutil.WriteFile(path, body, os.ModePerm); err != nil { + if err := os.WriteFile(path, body, os.ModePerm); err != nil { return err } } return nil } +func updatePipelineRemoteConfig(pathConfig, siteURL string, latestTime int64, ipr IPipelineRemote) error { + cf := pipelineRemoteConfig{ + SiteURL: siteURL, + UpdateTime: latestTime, + } + bys, err := ipr.Marshal(cf) + if err != nil { + return err + } + if err := ipr.WriteFile(pathConfig, bys, os.ModePerm); err != nil { + return err + } + return nil +} + func getPipelineRemoteConfig(pathConfig, siteURL string, ipr IPipelineRemote) (int64, error) { if !ipr.FileExist(pathConfig) { return 0, nil //nolint:nilerr @@ -363,21 +396,6 @@ func getPipelineRemoteConfig(pathConfig, siteURL string, ipr IPipelineRemote) (i return cf.UpdateTime, nil } -func updatePipelineRemoteConfig(pathConfig, siteURL string, latestTime int64, ipr IPipelineRemote) error { - cf := pipelineRemoteConfig{ - SiteURL: siteURL, - UpdateTime: latestTime, - } - bys, err := ipr.Marshal(cf) - if err != nil { - return err - } - if err := ipr.WriteFile(pathConfig, bys, os.ModePerm); err != nil { - return err - } - return nil -} - // ConvertContentMapToThreeMap more info see test case. func ConvertContentMapToThreeMap(in map[string]string) map[string]map[string]string { out := make(map[string]map[string]string) @@ -400,16 +418,22 @@ func ConvertContentMapToThreeMap(in map[string]string) map[string]map[string]str } // more info see test case. -func convertThreeMapToContentMap(in map[string]map[string]string, defaultPl map[string]string) map[string]string { +func convertThreeMapToContentMap(in map[point.Category]map[string]string, + defaultPl map[point.Category]string, +) map[string]string { out := make(map[string]string) for category, mVal := range in { for name, content := range mVal { - out[filepath.Join(category, name)] = content + out[filepath.Join(category.String(), name)] = content } } if defaultPl != nil { - if v, err := json.Marshal(defaultPl); err != nil { + df := map[string]string{} + for cat, name := range defaultPl { + df[cat.String()] = name + } + if v, err := json.Marshal(df); err != nil { l.Error(err) } else { out[pipelineRemoteDefaultScriptFile] = string(v) @@ -419,7 +443,7 @@ func convertThreeMapToContentMap(in map[string]map[string]string, defaultPl map[ return out } -func loadContentPipeline(in map[string]map[string]string) { +func loadContentPipeline(in map[point.Category]map[string]string) { managerWkr, ok := plval.GetManager() if !ok || managerWkr == nil { return @@ -427,9 +451,8 @@ func loadContentPipeline(in map[string]map[string]string) { inS := map[point.Category]map[string]string{} - for categoryShort, val := range in { - cat := point.CatString(categoryShort) + for cat, val := range in { inS[cat] = val } - plmanager.LoadScripts(managerWkr, plmanager.RemoteScriptNS, inS, nil, nil) + managerWkr.LoadScripts(plmanager.NSRemote, inS, nil) } diff --git a/internal/pipeline/remote/remote_test.go b/internal/pipeline/remote/remote_test.go index aa14359730..7e1529b460 100644 --- a/internal/pipeline/remote/remote_test.go +++ b/internal/pipeline/remote/remote_test.go @@ -12,6 +12,7 @@ import ( "testing" "time" + "github.com/GuanceCloud/cliutils/point" "github.com/stretchr/testify/assert" "gitlab.jiagouyun.com/cloudcare-tools/datakit/internal/pipeline/plval" ) @@ -147,25 +148,25 @@ func (mock *pipelineRemoteMockerTest) ReadDir(dirname string) ([]fs.FileInfo, er return mock.readDirResult, nil } -func (mock *pipelineRemoteMockerTest) PullPipeline(ts, relationTS int64) (mFiles, plRelation map[string]map[string]string, - defaultPl map[string]string, updateTime int64, relationUpdateAt int64, err error, +func (mock *pipelineRemoteMockerTest) PullPipeline(ts, relationTS int64) (mFiles, plRelation map[point.Category]map[string]string, + defaultPl map[point.Category]string, updateTime int64, relationUpdateAt int64, err error, ) { if mock.errPullPipeline != nil { return nil, nil, nil, 0, 0, mock.errPullPipeline } - return map[string]map[string]string{ - "logging": { + return map[point.Category]map[string]string{ + point.Logging: { "123.p": "text123", "456.p": "text456", }, - }, map[string]map[string]string{ - "logging": { + }, map[point.Category]map[string]string{ + point.Logging: { "123": "123.p", "234": "123.p", }, - }, map[string]string{ - "logging": "123.p", + }, map[point.Category]string{ + point.Logging: "123.p", }, mock.pullPipelineUpdateTime, relationUpdateAt, nil } @@ -233,8 +234,11 @@ func TestPullMain(t *testing.T) { mock.isFileExist = tc.fileExist mock.errReadFile = tc.failedReadFile - err := pullMain(tc.urls, mock) + fn, err := pullMain(tc.urls, mock) assert.Equal(t, tc.expectError, err, "pullMain found error: %v", err) + if fn != nil { + _ = fn() + } }) } } @@ -372,7 +376,7 @@ func TestDoPull(t *testing.T) { func TestDumpFiles(t *testing.T) { cases := []struct { name string - files map[string]map[string]string + files map[point.Category]map[string]string readDir []fs.FileInfo failedReadDir error failedWriteTarFromMap error @@ -380,8 +384,8 @@ func TestDumpFiles(t *testing.T) { }{ { name: "normal", - files: map[string]map[string]string{ - "logging": { + files: map[point.Category]map[string]string{ + point.Logging: { "123.p": "text123", "456.p": "text456", }, @@ -395,8 +399,8 @@ func TestDumpFiles(t *testing.T) { { name: "WriteTarFromMap_fail", failedWriteTarFromMap: errGeneral, - files: map[string]map[string]string{ - "logging": { + files: map[point.Category]map[string]string{ + point.Logging: { "123.p": "text123", "456.p": "text456", }, @@ -640,23 +644,23 @@ func TestConvertContentMapToThreeMap(t *testing.T) { func TestConvertThreeMapToContentMap(t *testing.T) { cases := []struct { name string - in map[string]map[string]string - inDefault map[string]string + in map[point.Category]map[string]string + inDefault map[point.Category]string expect map[string]string }{ { name: "normal", - in: map[string]map[string]string{ - "logging": { + in: map[point.Category]map[string]string{ + point.Logging: { "123.p": "text123", "1234.p": "text1234", }, - "metric": { + point.Metric: { "456.p": "text456", }, }, - inDefault: map[string]string{ - "logging": "123.p", + inDefault: map[point.Category]string{ + point.Logging: "123.p", }, expect: map[string]string{ "logging/123.p": "text123", @@ -667,12 +671,12 @@ func TestConvertThreeMapToContentMap(t *testing.T) { }, { name: "normal1", - in: map[string]map[string]string{ - "logging": { + in: map[point.Category]map[string]string{ + point.Logging: { "123.p": "text123", "1234.p": "text1234", }, - "metric": { + point.Metric: { "456.p": "text456", }, }, @@ -684,16 +688,16 @@ func TestConvertThreeMapToContentMap(t *testing.T) { }, { name: "normal2", - in: map[string]map[string]string{ - "logging": { + in: map[point.Category]map[string]string{ + point.Logging: { "123.p": "text123", "1234.p": "text1234", }, - "metric": { + point.Metric: { "456.p": "text456", }, }, - inDefault: map[string]string{}, + inDefault: map[point.Category]string{}, expect: map[string]string{ "logging/123.p": "text123", "logging/1234.p": "text1234", diff --git a/internal/plugins/inputs/confd_handle.go b/internal/plugins/inputs/confd_handle.go index a174181da2..8fe634bfdd 100644 --- a/internal/plugins/inputs/confd_handle.go +++ b/internal/plugins/inputs/confd_handle.go @@ -8,6 +8,7 @@ package inputs import ( "context" + "errors" "fmt" "math/rand" "time" @@ -24,7 +25,7 @@ func handleInput(confdInputs map[string][]*ConfdInfo, handleList []handle, ctx c case <-ctx.Done(): tip := "confd handleInput timeout" l.Error(tip) - errs = append(errs, fmt.Errorf(tip)) + errs = append(errs, errors.New(tip)) return errs default: @@ -48,9 +49,9 @@ func handleInput(confdInputs map[string][]*ConfdInfo, handleList []handle, ctx c case 3: l.Info("before set pipelines") - if managerWkr, ok := plval.GetManager(); ok && managerWkr != nil { - plmanager.LoadScripts2StoreFromPlStructPath(managerWkr, - plmanager.ConfdScriptNS, datakit.ConfdPipelineDir, nil) + if m, ok := plval.GetManager(); ok && m != nil { + m.LoadScriptsFromWorkspace(plmanager.NSConfd, + datakit.ConfdPipelineDir, nil) } } } diff --git a/internal/plugins/inputs/sqlserver/input_test.go b/internal/plugins/inputs/sqlserver/input_test.go index 0168c0580d..6c4097b02e 100644 --- a/internal/plugins/inputs/sqlserver/input_test.go +++ b/internal/plugins/inputs/sqlserver/input_test.go @@ -192,7 +192,7 @@ func TestPipeline(t *testing.T) { pl, errs := manager.NewScripts(map[string]string{ "test.p": pScrpit, - }, nil, nil, "", point.Logging) + }, nil, "", point.Logging) if len(errs) > 0 { t.Fatal(errs) diff --git a/vendor/github.com/GuanceCloud/cliutils/filter/eval.go b/vendor/github.com/GuanceCloud/cliutils/filter/eval.go index 6a7b7d6590..c403de7c82 100644 --- a/vendor/github.com/GuanceCloud/cliutils/filter/eval.go +++ b/vendor/github.com/GuanceCloud/cliutils/filter/eval.go @@ -144,12 +144,18 @@ func toInt64(i interface{}) int64 { } func binEval(op ItemType, lhs, rhs interface{}) bool { - if _, ok := rhs.(*Regex); !ok { + if _, ok := rhs.(*Regex); ok { + if _, isStr := lhs.(string); !isStr { + log.Warnf("non-string(type %s) can not match with regexp", reflect.TypeOf(lhs)) + return false + } + } else { // rhs are all literals tl := reflect.TypeOf(lhs).String() tr := reflect.TypeOf(rhs).String() switch op { case GTE, GT, LT, LTE, EQ, NEQ: // type conflict detecting on comparison expr - if tl != tr { + if _, ok := rhs.(*NilLiteral); !ok && // any type can compare to nil/null + tl != tr { log.Warnf("type conflict %+#v(%s) <> %+#v(%s)", lhs, reflect.TypeOf(lhs), rhs, reflect.TypeOf(rhs)) return false } @@ -170,15 +176,25 @@ func binEval(op ItemType, lhs, rhs interface{}) bool { return almostEqual(lv, f) } + case *NilLiteral: + if _, ok := rhs.(*NilLiteral); !ok { // nil compared to non-nil always false + log.Warnf("rhs %v not nil", rhs) + return false + } + + return lv.String() == Nil + default: // NOTE: interface{} EQ/NEQ, see: https://stackoverflow.com/a/34246225/342348 - switch reg := rhs.(type) { + switch rv := rhs.(type) { case *Regex: - ok, err := regexp.MatchString(reg.Regex, lhs.(string)) + log.Debugf("lhs: %v, rhs: %v", lhs, rhs) + ok, err := regexp.MatchString(rv.Regex, lhs.(string)) if err != nil { log.Error(err) } return ok + default: return lhs == rhs } @@ -191,6 +207,12 @@ func binEval(op ItemType, lhs, rhs interface{}) bool { return !rhs.(*Regex).Re.MatchString(lhs.(string)) case NEQ: + _, lok := lhs.(*NilLiteral) + _, rok := rhs.(*NilLiteral) + if lok && rok { + return false + } + return lhs != rhs case GTE, GT, LT, LTE: // rhs/lhs should be number or string @@ -306,6 +328,10 @@ func (e *BinaryExpr) singleEval(data KVs) bool { } case *Regex: arr = append(arr, x) + case *NilLiteral: + arr = append(arr, x) + case *BoolLiteral: + arr = append(arr, x.Val) default: log.Warnf("unsupported node list with type `%s'", reflect.TypeOf(elem).String()) } @@ -314,64 +340,104 @@ func (e *BinaryExpr) singleEval(data KVs) bool { case *Regex: lit = rhs + case *NilLiteral: + lit = rhs + + case *BoolLiteral: + lit = rhs.Val + default: + log.Errorf("invalid RHS, got type `%s'", reflect.TypeOf(e.RHS).String()) return false } - // first: fetch left-handle-symbol and OP on right-handle-symbol - switch lhs := e.LHS.(type) { + var lhs *Identifier + switch left := e.LHS.(type) { // Left part can be string/bool/number/nil literal and identifier + case *NilLiteral: + return binEval(e.Op, nilVal, lit) + + case *NumberLiteral: + if left.IsInt { + return binEval(e.Op, left.Int, lit) + } else { + return binEval(e.Op, left.Float, lit) + } + + case *BoolLiteral: + return binEval(e.Op, left.Val, lit) + + case *StringLiteral: + return binEval(e.Op, left.Val, lit) + case *Identifier: - name := lhs.Name - - switch e.Op { - case MATCH, NOT_MATCH: - for _, item := range e.RHS.(NodeList) { - if v, ok := data.Get(name); ok { - switch x := v.(type) { - case string: - if binEval(e.Op, x, item) { - return true - } - default: - continue - } - } - } - return false + lhs = left // we get detailed lhs value later... - case IN: - for _, item := range arr { - if v, ok := data.Get(name); ok { - if binEval(EQ, v, item) { + default: + log.Errorf("unknown LHS type, expect Identifier, got `%s'", reflect.TypeOf(e.LHS).String()) + return false + } + + name := lhs.Name + + switch e.Op { + case MATCH, NOT_MATCH: + for _, item := range e.RHS.(NodeList) { + if v, ok := data.Get(name); ok { + switch x := v.(type) { + case string: + if binEval(e.Op, x, item) { return true } + default: + continue } } - return false + } + return false - case NOT_IN: - for _, item := range arr { - if v, ok := data.Get(name); ok { - if binEval(EQ, v, item) { - return false - } + case IN: + for _, item := range arr { + if v, ok := data.Get(name); ok { + if binEval(EQ, v, item) { + return true } + } else { + return binEval(EQ, item, nilVal) } + } + return false - return true - - case GTE, GT, LT, LTE, NEQ, EQ: - + case NOT_IN: + for _, item := range arr { if v, ok := data.Get(name); ok { - if binEval(e.Op, v, lit) { - return true + if binEval(EQ, v, item) { + return false + } + } else { + if binEval(EQ, item, nilVal) { + return false } } } + return true + + case GTE, GT, LT, LTE, NEQ, EQ: + if v, ok := data.Get(name); ok { + if binEval(e.Op, v, lit) { + return true + } + } else { // not exist in data + return binEval(e.Op, lit, nilVal) + } default: - log.Errorf("unknown LHS type, expect Identifier, got `%s'", reflect.TypeOf(e.LHS).String()) + log.Warnf("unsupported operation %s on single-eval expr", e.Op) } + return false } + +var ( + nilVal = &NilLiteral{} +) diff --git a/vendor/github.com/GuanceCloud/cliutils/filter/lex.go b/vendor/github.com/GuanceCloud/cliutils/filter/lex.go index 5f3de7f710..f6d1ed9e2a 100644 --- a/vendor/github.com/GuanceCloud/cliutils/filter/lex.go +++ b/vendor/github.com/GuanceCloud/cliutils/filter/lex.go @@ -77,8 +77,10 @@ var ( "filter": FILTER, "identifier": IDENTIFIER, - "in": IN, - "notin": NOT_IN, + "in": IN, + + "notin": NOT_IN, // deprecated + "not_in": NOT_IN, // same as notin "limit": LIMIT, "link": LINK, @@ -480,8 +482,6 @@ func lexEscape(l *Lexer) stateFn { return lexString } - log.Debugf("n: %d, base: %d, max: %d", n, base, max) - var x uint32 for n > 0 { d := uint32(digitVal(ch)) @@ -502,7 +502,6 @@ func lexEscape(l *Lexer) stateFn { l.errorf("escape sequence is an invalid Unicode code point") } - log.Debugf("get number %d", x) return lexString } @@ -546,8 +545,6 @@ func (l *Lexer) peek() rune { func (l *Lexer) emit(t ItemType) { *l.itemp = Item{t, l.start, l.input[l.start:l.pos]} - // log.Debugf("emit: %+#v", l.itemp) - l.start = l.pos l.scannedItem = true } diff --git a/vendor/github.com/GuanceCloud/cliutils/filter/parse.go b/vendor/github.com/GuanceCloud/cliutils/filter/parse.go index e188c67dfd..f122a81b58 100644 --- a/vendor/github.com/GuanceCloud/cliutils/filter/parse.go +++ b/vendor/github.com/GuanceCloud/cliutils/filter/parse.go @@ -44,8 +44,6 @@ type parser struct { } func GetConds(input string) (WhereConditions, error) { - log.Debugf("parse %s", input) - var err error p := newParser(input) defer parserPool.Put(p) diff --git a/vendor/github.com/GuanceCloud/cliutils/pipeline/manager/manager.go b/vendor/github.com/GuanceCloud/cliutils/pipeline/manager/manager.go index 65fc943fa3..0619c26663 100644 --- a/vendor/github.com/GuanceCloud/cliutils/pipeline/manager/manager.go +++ b/vendor/github.com/GuanceCloud/cliutils/pipeline/manager/manager.go @@ -10,14 +10,13 @@ import ( "os" "path/filepath" - "github.com/GuanceCloud/cliutils/pipeline/manager/relation" "github.com/GuanceCloud/cliutils/pipeline/ptinput/plmap" "github.com/GuanceCloud/cliutils/point" ) type Manager struct { storeMap map[point.Category]*ScriptStore - relation *relation.ScriptRelation + relation *ScriptRelation } type ManagerCfg struct { @@ -35,7 +34,7 @@ func NewManagerCfg(upFn plmap.UploadFunc, gTags [][2]string) ManagerCfg { func NewManager(cfg ManagerCfg) *Manager { center := &Manager{ storeMap: map[point.Category]*ScriptStore{}, - relation: relation.NewPipelineRelation(), + relation: NewPipelineRelation(), } for _, cat := range point.AllCategories() { center.storeMap[cat] = NewScriptStore(cat, cfg) @@ -43,93 +42,85 @@ func NewManager(cfg ManagerCfg) *Manager { return center } -func (c *Manager) whichStore(category point.Category) (*ScriptStore, bool) { +func (m *Manager) whichStore(category point.Category) (*ScriptStore, bool) { if category == point.MetricDeprecated { category = point.Metric } - if v, ok := c.storeMap[category]; ok && v != nil { + if v, ok := m.storeMap[category]; ok && v != nil { return v, ok } return nil, false } -func (c *Manager) GetScriptRelation() *relation.ScriptRelation { - return c.relation +func (m *Manager) UpdateDefaultScript(mp map[point.Category]string) { + for _, cat := range point.AllCategories() { + if store, ok := m.whichStore(cat); ok { + if v, ok := mp[cat]; ok && v != "" { + store.SetDefaultScript(v) + } else { + store.SetDefaultScript("") + } + } + } } -func (c *Manager) QueryScript(category point.Category, name string) (*PlScript, bool) { - if v, ok := c.whichStore(category); ok { - return v.IndexGet(name) +func (m *Manager) GetScriptRelation() *ScriptRelation { + return m.relation +} + +func (m *Manager) QueryScript(category point.Category, name string, + DisableDefaultP ...struct{}) (*PlScript, bool) { + + if v, ok := m.whichStore(category); ok { + if ss, ok := v.IndexGet(name); ok { + return ss, ok + } + + if len(DisableDefaultP) == 0 { + return v.IndexDefault() + } } return nil, false } -func (c *Manager) ScriptCount(category point.Category) int { - if v, ok := c.whichStore(category); ok { +func (m *Manager) ScriptCount(category point.Category) int { + if v, ok := m.whichStore(category); ok { return v.Count() } return 0 } -func ReadPlScriptFromFile(fp string) (string, string, error) { - fp = filepath.Clean(fp) - if v, err := os.ReadFile(filepath.Clean(fp)); err == nil { - _, sName := filepath.Split(fp) - return sName, string(v), nil - } else { - return "", "", err +func (m *Manager) LoadScriptsFromWorkspace(ns, plPath string, tags map[string]string) { + if plPath == "" { + return } + + scripts, _ := ReadWorkspaceScripts(plPath) + + m.LoadScripts(ns, scripts, tags) } -func SearchPlFilePathFormDir(dirPath string) map[string]string { - ret := map[string]string{} - dirPath = filepath.Clean(dirPath) - if dirEntry, err := os.ReadDir(dirPath); err != nil { - l.Warn(err) - } else { - for _, v := range dirEntry { - if v.IsDir() { - continue - } - sName := v.Name() - if filepath.Ext(sName) != ".p" { - continue - } - ret[sName] = filepath.Join(dirPath, sName) +// LoadScripts is used to load and clean the script, parameter scripts example: +// {point.Logging: {ScriptName: ScriptContent},... }. +func (m *Manager) LoadScripts(ns string, scripts map[point.Category](map[string]string), + tags map[string]string, +) { + for _, cat := range point.AllCategories() { + if ss, ok := scripts[cat]; ok { + m.LoadScriptWithCat(cat, ns, ss, tags) + } else { + // cleanup the store for this category + m.LoadScriptWithCat(cat, ns, map[string]string{}, tags) } } - return ret } -func ReadPlScriptFromDir(dirPath string) (map[string]string, map[string]string) { - if dirPath == "" { - return nil, nil - } - - ret := map[string]string{} - retPath := map[string]string{} - dirPath = filepath.Clean(dirPath) - if dirEntry, err := os.ReadDir(dirPath); err != nil { - l.Warn(err) - } else { - for _, v := range dirEntry { - if v.IsDir() { - continue - } - sName := v.Name() - if filepath.Ext(sName) != ".p" { - continue - } - sPath := filepath.Join(dirPath, sName) - if name, script, err := ReadPlScriptFromFile(sPath); err == nil { - ret[name] = script - retPath[name] = sPath - } else { - l.Error(err) - } - } +func (m *Manager) LoadScriptWithCat(category point.Category, ns string, + scripts, tags map[string]string, +) { + if v, ok := m.whichStore(category); ok { + v.UpdateScriptsWithNS(ns, scripts, tags) } - return ret, retPath } func CategoryDirName() map[point.Category]string { @@ -144,16 +135,24 @@ func CategoryDirName() map[point.Category]string { point.RUM: "rum", point.Security: "security", point.Profiling: "profiling", + point.DialTesting: "dialtesting", } } -func SearchPlFilePathFromPlStructPath(basePath string) map[point.Category](map[string]string) { +func SearchWorkspaceScripts(basePath string) map[point.Category](map[string]string) { files := map[point.Category](map[string]string){} - files[point.Logging] = SearchPlFilePathFormDir(basePath) + var err error + files[point.Logging], err = SearchScripts(basePath) + if err != nil { + log.Warn(err) + } for category, dirName := range CategoryDirName() { - s := SearchPlFilePathFormDir(filepath.Join(basePath, dirName)) + s, err := SearchScripts(filepath.Join(basePath, dirName)) + if err != nil { + log.Warn(err) + } if _, ok := files[category]; !ok { files[category] = map[string]string{} } @@ -164,85 +163,75 @@ func SearchPlFilePathFromPlStructPath(basePath string) map[point.Category](map[s return files } -func ReadPlScriptFromPlStructPath(basePath string) ( +func ReadWorkspaceScripts(basePath string) ( map[point.Category](map[string]string), map[point.Category](map[string]string), ) { - if basePath == "" { - return nil, nil - } + scriptsPath := SearchWorkspaceScripts(basePath) scripts := map[point.Category](map[string]string){} - scriptsPath := map[point.Category](map[string]string){} - - scripts[point.Logging], scriptsPath[point.Logging] = ReadPlScriptFromDir(basePath) - - for category, dirName := range CategoryDirName() { - s, p := ReadPlScriptFromDir(filepath.Join(basePath, dirName)) - if _, ok := scripts[category]; !ok { - scripts[category] = map[string]string{} - } - if _, ok := scriptsPath[category]; !ok { - scriptsPath[category] = map[string]string{} + for cat, ssPath := range scriptsPath { + if _, ok := scripts[cat]; !ok { + scripts[cat] = map[string]string{} } - - for k, v := range s { - scripts[category][k] = v - } - for k, v := range p { - scriptsPath[category][k] = v + for _, path := range ssPath { + if name, script, err := ReadScript(path); err == nil { + scripts[cat][name] = script + } else { + log.Error(err) + } } } + return scripts, scriptsPath } -func LoadDefaultScripts2Store(center *Manager, rootDir string, tags map[string]string) { - if rootDir == "" { - return - } - - plPath := filepath.Join(rootDir, "pipeline") - LoadScripts2StoreFromPlStructPath(center, DefaultScriptNS, plPath, tags) -} +func SearchScripts(dirPath string) (map[string]string, error) { + ret := map[string]string{} + dirPath = filepath.Clean(dirPath) -func LoadScripts2StoreFromPlStructPath(center *Manager, ns, plPath string, tags map[string]string) { - if plPath == "" { - return + dirEntry, err := os.ReadDir(dirPath) + if err != nil { + return nil, err } - scripts, path := ReadPlScriptFromPlStructPath(plPath) - - LoadScripts(center, ns, scripts, path, tags) + for _, v := range dirEntry { + if v.IsDir() { + // todo: support sub dir + continue + } + if sName := v.Name(); filepath.Ext(sName) == ".p" { + ret[sName] = filepath.Join(dirPath, sName) + } + } + return ret, nil } -// LoadScripts is used to load and clean the script, parameter scripts example: {datakit.Logging: {ScriptName: ScriptContent},... }. -func LoadScripts(center *Manager, ns string, scripts, scriptPath map[point.Category](map[string]string), - tags map[string]string, -) { - allCategoryScript := FillScriptCategoryMap(scripts) - for category, m := range allCategoryScript { - LoadScript(center, category, ns, m, scriptPath[category], tags) +func ReadScripts(dirPath string) (map[string]string, map[string]string) { + ret := map[string]string{} + + scriptsPath, err := SearchScripts(dirPath) + if err != nil { + log.Warn(err) + return nil, nil } -} -func LoadScript(centor *Manager, category point.Category, ns string, - scripts, path, tags map[string]string, -) { - if v, ok := centor.whichStore(category); ok { - v.UpdateScriptsWithNS(ns, scripts, path, tags) + for _, path := range scriptsPath { + if name, script, err := ReadScript(path); err == nil { + ret[name] = script + } else { + log.Error(err) + } } + + return ret, scriptsPath } -func FillScriptCategoryMap(scripts map[point.Category](map[string]string)) map[point.Category](map[string]string) { - allCategoryScript := map[point.Category](map[string]string){} - for _, cat := range point.AllCategories() { - allCategoryScript[cat] = map[string]string{} - } - for k, v := range scripts { - for name, script := range v { - if v, ok := allCategoryScript[k]; ok { - v[name] = script - } - } +func ReadScript(fp string) (string, string, error) { + fp = filepath.Clean(fp) + if v, err := os.ReadFile(filepath.Clean(fp)); err == nil { + _, sName := filepath.Split(fp) + return sName, string(v), nil + } else { + return "", "", err } - return allCategoryScript } diff --git a/vendor/github.com/GuanceCloud/cliutils/pipeline/manager/name_rule.go b/vendor/github.com/GuanceCloud/cliutils/pipeline/manager/name_rule.go new file mode 100644 index 0000000000..d7834a53c4 --- /dev/null +++ b/vendor/github.com/GuanceCloud/cliutils/pipeline/manager/name_rule.go @@ -0,0 +1,87 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +// Package manager for managing pipeline scripts +package manager + +import ( + "github.com/GuanceCloud/cliutils/point" +) + +func _rumSName(pt *point.Point) string { + if id := pt.Get("app_id"); id != nil { + if rumID, ok := id.(string); ok { + return rumID + "_" + pt.Name() + } + } + return "" +} + +func _securitySName(pt *point.Point) string { + if scheckCat := pt.Get("category"); scheckCat != nil { + if cat, ok := scheckCat.(string); ok { + return cat + } + } + return "" +} + +func _apmSName(pt *point.Point) string { + if apmSvc := pt.Get("service"); apmSvc != nil { + if svc, ok := apmSvc.(string); ok { + return svc + } + } + return "" +} + +func _defaultCatSName(pt *point.Point) string { + return pt.Name() +} + +func ScriptName(relation *ScriptRelation, cat point.Category, pt *point.Point, scriptMap map[string]string) (string, bool) { + if pt == nil { + return "", false + } + + var scriptName string + + // built-in rules last + switch cat { //nolint:exhaustive + case point.RUM: + scriptName = _rumSName(pt) + case point.Security: + scriptName = _securitySName(pt) + case point.Tracing, point.Profiling: + scriptName = _apmSName(pt) + default: + scriptName = _defaultCatSName(pt) + } + + if scriptName == "" { + return "", false + } + + // remote relation first + if relation != nil { + if sName, ok := relation.Query(cat, scriptName); ok { + return sName, true + } + } + + // config rules second + if sName, ok := scriptMap[scriptName]; ok { + switch sName { + case "-": + return "", false + case "": + default: + return sName, true + } + } + + // built-in rule last + return scriptName + ".p", true +} diff --git a/vendor/github.com/GuanceCloud/cliutils/pipeline/manager/relation.go b/vendor/github.com/GuanceCloud/cliutils/pipeline/manager/relation.go new file mode 100644 index 0000000000..486cce1c23 --- /dev/null +++ b/vendor/github.com/GuanceCloud/cliutils/pipeline/manager/relation.go @@ -0,0 +1,66 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +// Package manager for managing pipeline scripts +package manager + +import ( + "sync" + + "github.com/GuanceCloud/cliutils/point" +) + +// var remoteRelation = &PipelineRelation{} + +type ScriptRelation struct { + // map[]: map[]: + relation map[point.Category]map[string]string + + updateAt int64 + + rwMutex sync.RWMutex +} + +func NewPipelineRelation() *ScriptRelation { + return &ScriptRelation{} +} + +func (relation *ScriptRelation) UpdateAt() int64 { + relation.rwMutex.RLock() + defer relation.rwMutex.RUnlock() + + return relation.updateAt +} + +func (relation *ScriptRelation) UpdateRelation(updateAt int64, rel map[point.Category]map[string]string) { + relation.rwMutex.Lock() + defer relation.rwMutex.Unlock() + + relation.updateAt = updateAt + + // reset relation + relation.relation = map[point.Category]map[string]string{} + + for cat, relat := range rel { + m := map[string]string{} + relation.relation[cat] = m + for source, name := range relat { + m[source] = name + } + } +} + +func (relation *ScriptRelation) Query(cat point.Category, source string) (string, bool) { + relation.rwMutex.RLock() + defer relation.rwMutex.RUnlock() + + if v, ok := relation.relation[cat]; ok { + if name, ok := v[source]; ok { + return name, true + } + } + + return "", false +} diff --git a/vendor/github.com/GuanceCloud/cliutils/pipeline/manager/relation/name_rule.go b/vendor/github.com/GuanceCloud/cliutils/pipeline/manager/relation/name_rule.go deleted file mode 100644 index 80f24115a6..0000000000 --- a/vendor/github.com/GuanceCloud/cliutils/pipeline/manager/relation/name_rule.go +++ /dev/null @@ -1,41 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the MIT License. -// This product includes software developed at Guance Cloud (https://www.guance.com/). -// Copyright 2021-present Guance, Inc. - -package relation - -import ( - "github.com/GuanceCloud/cliutils/point" -) - -func _rumSName(pt *point.Point) string { - if id := pt.Get("app_id"); id != nil { - if rumID, ok := id.(string); ok { - return rumID + "_" + pt.Name() - } - } - return "" -} - -func _securitySName(pt *point.Point) string { - if scheckCat := pt.Get("category"); scheckCat != nil { - if cat, ok := scheckCat.(string); ok { - return cat - } - } - return "" -} - -func _apmSName(pt *point.Point) string { - if apmSvc := pt.Get("service"); apmSvc != nil { - if svc, ok := apmSvc.(string); ok { - return svc - } - } - return "" -} - -func _defaultCatSName(pt *point.Point) string { - return pt.Name() -} diff --git a/vendor/github.com/GuanceCloud/cliutils/pipeline/manager/relation/relation.go b/vendor/github.com/GuanceCloud/cliutils/pipeline/manager/relation/relation.go deleted file mode 100644 index 10aa90b070..0000000000 --- a/vendor/github.com/GuanceCloud/cliutils/pipeline/manager/relation/relation.go +++ /dev/null @@ -1,166 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the MIT License. -// This product includes software developed at Guance Cloud (https://www.guance.com/). -// Copyright 2021-present Guance, Inc. - -// Package relation record source-name relation -package relation - -import ( - "sync" - - "github.com/GuanceCloud/cliutils/logger" - "github.com/GuanceCloud/cliutils/point" -) - -var l = logger.DefaultSLogger("pl-relation") - -func InitLog() { - l = logger.SLogger("pl-relation") -} - -// var remoteRelation = &PipelineRelation{} - -type ScriptRelation struct { - // map[]: map[]: - relation map[point.Category]map[string]string - - // map[]: - defaultScript map[point.Category]string - - updateAt int64 - - rwMutex sync.RWMutex -} - -func NewPipelineRelation() *ScriptRelation { - return &ScriptRelation{} -} - -func (relation *ScriptRelation) UpdateAt() int64 { - relation.rwMutex.RLock() - defer relation.rwMutex.RUnlock() - - return relation.updateAt -} - -func (relation *ScriptRelation) UpdateDefaultPl(defaultPl map[string]string) { - relation.rwMutex.Lock() - defer relation.rwMutex.Unlock() - - if len(relation.defaultScript) > 0 || relation.defaultScript == nil { - relation.defaultScript = map[point.Category]string{} - } - - for categoryShort, name := range defaultPl { - category := point.CatString(categoryShort) - if category == point.UnknownCategory { - l.Warnf("unknown category: %s", categoryShort) - } - relation.defaultScript[category] = name - } -} - -func (relation *ScriptRelation) UpdateRelation(updateAt int64, rel map[string]map[string]string) { - relation.rwMutex.Lock() - defer relation.rwMutex.Unlock() - - relation.updateAt = updateAt - if len(relation.relation) > 0 || relation.relation == nil { - relation.relation = map[point.Category]map[string]string{} - } - - for categoryShort, relat := range rel { - category := point.CatString(categoryShort) - if category == point.UnknownCategory { - l.Warnf("unknown category: %s", categoryShort) - } - for source, name := range relat { - if v, ok := relation.relation[category]; !ok { - relation.relation[category] = map[string]string{ - source: name, - } - } else { - v[source] = name - } - } - } -} - -func (relation *ScriptRelation) Query(category point.Category, source string) (string, bool) { - relation.rwMutex.RLock() - defer relation.rwMutex.RUnlock() - - if v, ok := relation.relation[category]; ok { - if name, ok := v[source]; ok { - return name, true - } - } - - // defaultPl - if v, ok := relation.defaultScript[category]; ok { - return v, true - } - - return "", false -} - -func ScriptName(relation *ScriptRelation, cat point.Category, pt *point.Point, scriptMap map[string]string) (string, bool) { - if pt == nil { - return "", false - } - - var scriptName string - - // built-in rules last - switch cat { //nolint:exhaustive - case point.RUM: - scriptName = _rumSName(pt) - case point.Security: - scriptName = _securitySName(pt) - case point.Tracing, point.Profiling: - scriptName = _apmSName(pt) - default: - scriptName = _defaultCatSName(pt) - } - - if scriptName == "" { - return "", false - } - - // configuration first - if sName, ok := scriptMap[scriptName]; ok { - switch sName { - case "-": - return "", false - case "": - default: - return sName, true - } - } - - if relation != nil { - // remote relation sencond - if sName, ok := relation.Query(cat, scriptName); ok { - return sName, true - } - } - - return scriptName + ".p", true -} - -// func QueryRemoteRelation(category point.Category, source string) (string, bool) { -// return remoteRelation.query(category, source) -// } - -// func RelationRemoteUpdateAt() int64 { -// return remoteRelation.UpdateAt() -// } - -// func UpdateRemoteDefaultPl(defaultPl map[string]string) { -// remoteRelation.UpdateDefaultPl(defaultPl) -// } - -// func UpdateRemoteRelation(updateAt int64, rel map[string]map[string]string) { -// remoteRelation.UpdateRelation(updateAt, rel) -// } diff --git a/vendor/github.com/GuanceCloud/cliutils/pipeline/manager/script.go b/vendor/github.com/GuanceCloud/cliutils/pipeline/manager/script.go index 5f17a149c7..17fe6795c8 100644 --- a/vendor/github.com/GuanceCloud/cliutils/pipeline/manager/script.go +++ b/vendor/github.com/GuanceCloud/cliutils/pipeline/manager/script.go @@ -30,26 +30,22 @@ type Option struct { } type PlScript struct { - name string // script name - filePath string - script string // script content - - ns string // script 所属 namespace - category point.Category - - proc *plruntime.Script - - plBuks *plmap.AggBuckets + name string // script name + script string // script content + ns string // script 所属 namespace + proc *plruntime.Script + plBuks *plmap.AggBuckets ptWindow *ptwindow.WindowPool + cache *plcache.Cache + tags map[string]string updateTS int64 - tags map[string]string - cache *plcache.Cache + category point.Category } -func NewScripts(scripts, scriptPath, scriptTags map[string]string, ns string, cat point.Category, +func NewScripts(scripts, scriptTags map[string]string, ns string, cat point.Category, buks ...*plmap.AggBuckets, ) (map[string]*PlScript, map[string]error) { var plbuks *plmap.AggBuckets @@ -57,19 +53,9 @@ func NewScripts(scripts, scriptPath, scriptTags map[string]string, ns string, ca plbuks = buks[0] } - switch cat { - case point.Metric: + switch cat { //nolint:exhaustive case point.MetricDeprecated: cat = point.Metric - case point.Network: - case point.KeyEvent: - case point.Object: - case point.CustomObject: - case point.Tracing: - case point.RUM: - case point.Security: - case point.Logging: - case point.Profiling: case point.UnknownCategory, point.DynamicDWCategory: retErr := map[string]error{} for k := range scripts { @@ -77,15 +63,12 @@ func NewScripts(scripts, scriptPath, scriptTags map[string]string, ns string, ca } return nil, retErr } + ret, retErr := plengine.ParseScript(scripts, funcs.FuncsMap, funcs.FuncsCheckMap) retScipt := map[string]*PlScript{} for name, ng := range ret { - var sPath string - if len(scriptPath) > 0 { - sPath = scriptPath[name] - } cache, _ := plcache.NewCache(time.Second, 100) ptWin := ptwindow.NewManager() @@ -106,7 +89,6 @@ func NewScripts(scripts, scriptPath, scriptTags map[string]string, ns string, ca retScipt[name] = &PlScript{ script: scripts[name], name: name, - filePath: sPath, ns: ns, category: cat, proc: ng, @@ -157,7 +139,6 @@ func (script *PlScript) Run(plpt ptinput.PlInputPt, signal plruntime.Signal, opt if opt != nil { disable = opt.DisableAddStatusField ignore = opt.IgnoreStatus - // spiltLen = opt.MaxFieldValLen } ProcLoggingStatus(plpt, disable, ignore) @@ -182,10 +163,6 @@ func (script *PlScript) Name() string { return script.name } -func (script PlScript) FilePath() string { - return script.filePath -} - func (script *PlScript) Category() point.Category { return script.category } diff --git a/vendor/github.com/GuanceCloud/cliutils/pipeline/manager/scriptstore.go b/vendor/github.com/GuanceCloud/cliutils/pipeline/manager/scriptstore.go index 5cc2b73c8f..0b9956aa84 100644 --- a/vendor/github.com/GuanceCloud/cliutils/pipeline/manager/scriptstore.go +++ b/vendor/github.com/GuanceCloud/cliutils/pipeline/manager/scriptstore.go @@ -6,6 +6,7 @@ package manager import ( + "path/filepath" "sync" "time" @@ -15,40 +16,42 @@ import ( "github.com/GuanceCloud/cliutils/point" ) -var l = logger.DefaultSLogger("pl-script") +var log = logger.DefaultSLogger("pl-script") const ( - DefaultScriptNS = "default" // 内置 pl script， 优先级最低 - GitRepoScriptNS = "gitrepo" // git 管理的 pl script - ConfdScriptNS = "confd" // confd 管理的 pl script - RemoteScriptNS = "remote" // remote pl script，优先级最高 + NSDefault = "default" // 内置 pl script， 优先级最低 + NSGitRepo = "gitrepo" // git 管理的 pl script + NSConfd = "confd" // confd 管理的 pl script + NSRemote = "remote" // remote pl script，优先级最高 ) -var plScriptNSSearchOrder = [4]string{ - RemoteScriptNS, // 优先级最高的 ns - ConfdScriptNS, - GitRepoScriptNS, - DefaultScriptNS, +var nsSearchOrder = [4]string{ + NSRemote, // 优先级最高的 ns + NSConfd, + NSGitRepo, + NSDefault, } func InitLog() { - l = logger.SLogger("pl-script") + log = logger.SLogger("pl-script") } -func InitStore(center *Manager, installDir string, tags map[string]string) { +func InitStore(manager *Manager, installDir string, tags map[string]string) { stats.InitLog() - LoadDefaultScripts2Store(center, installDir, tags) + + plPath := filepath.Join(installDir, "pipeline") + manager.LoadScriptsFromWorkspace(NSDefault, plPath, tags) } func NSFindPriority(ns string) int { switch ns { - case DefaultScriptNS: + case NSDefault: return 0 // lowest priority - case GitRepoScriptNS: + case NSGitRepo: return 1 - case ConfdScriptNS: + case NSConfd: return 2 - case RemoteScriptNS: + case NSRemote: return 3 default: return -1 @@ -60,6 +63,8 @@ type ScriptStore struct { storage scriptStorage + defultScript string + index map[string]*PlScript indexLock sync.RWMutex @@ -76,10 +81,10 @@ func NewScriptStore(category point.Category, cfg ManagerCfg) *ScriptStore { category: category, storage: scriptStorage{ scripts: map[string]map[string]*PlScript{ - RemoteScriptNS: {}, - ConfdScriptNS: {}, - GitRepoScriptNS: {}, - DefaultScriptNS: {}, + NSRemote: {}, + NSConfd: {}, + NSGitRepo: {}, + NSDefault: {}, }, }, index: map[string]*PlScript{}, @@ -87,6 +92,18 @@ func NewScriptStore(category point.Category, cfg ManagerCfg) *ScriptStore { } } +func (store *ScriptStore) SetDefaultScript(name string) { + store.indexLock.Lock() + defer store.indexLock.Unlock() + store.defultScript = name +} + +func (store *ScriptStore) GetDefaultScript() string { + store.indexLock.RLock() + defer store.indexLock.RUnlock() + return store.defultScript +} + func (store *ScriptStore) IndexGet(name string) (*PlScript, bool) { store.indexLock.RLock() defer store.indexLock.RUnlock() @@ -96,14 +113,24 @@ func (store *ScriptStore) IndexGet(name string) (*PlScript, bool) { return nil, false } +func (store *ScriptStore) IndexDefault() (*PlScript, bool) { + store.indexLock.RLock() + defer store.indexLock.RUnlock() + + if v, ok := store.index[store.defultScript]; ok { + return v, ok + } + return nil, false +} + func (store *ScriptStore) Count() int { store.storage.RLock() defer store.storage.RUnlock() - return len(store.storage.scripts[RemoteScriptNS]) + - len(store.storage.scripts[ConfdScriptNS]) + - len(store.storage.scripts[GitRepoScriptNS]) + - len(store.storage.scripts[DefaultScriptNS]) + return len(store.storage.scripts[NSRemote]) + + len(store.storage.scripts[NSConfd]) + + len(store.storage.scripts[NSGitRepo]) + + len(store.storage.scripts[NSDefault]) } func (store *ScriptStore) GetWithNs(name, ns string) (*PlScript, bool) { @@ -184,7 +211,7 @@ func (store *ScriptStore) indexDeleteAndBack(name, ns string, scripts4back map[s return } - if nsCur > len(plScriptNSSearchOrder) { + if nsCur > len(nsSearchOrder) { return } @@ -202,7 +229,7 @@ func (store *ScriptStore) indexDeleteAndBack(name, ns string, scripts4back map[s return } - for _, v := range plScriptNSSearchOrder[len(plScriptNSSearchOrder)-nsCur:] { + for _, v := range nsSearchOrder[len(nsSearchOrder)-nsCur:] { if v, ok := scripts4back[v]; ok { if s, ok := v[name]; ok { store.indexStore(s) @@ -235,7 +262,7 @@ func (store *ScriptStore) indexDeleteAndBack(name, ns string, scripts4back map[s } func (store *ScriptStore) UpdateScriptsWithNS(ns string, - namedScript, scriptPath, scriptTags map[string]string, + namedScript, scriptTags map[string]string, ) map[string]error { store.storage.Lock() defer store.storage.Unlock() @@ -245,7 +272,7 @@ func (store *ScriptStore) UpdateScriptsWithNS(ns string, } aggBuk := plmap.NewAggBuks(store.cfg.upFn, store.cfg.gTags) - retScripts, retErr := NewScripts(namedScript, scriptPath, scriptTags, ns, store.category, + retScripts, retErr := NewScripts(namedScript, scriptTags, ns, store.category, aggBuk) for name, err := range retErr { @@ -332,28 +359,3 @@ func (store *ScriptStore) UpdateScriptsWithNS(ns string, } return nil } - -func (store *ScriptStore) LoadDotPScript2Store(ns, dirPath string, scriptTags map[string]string, filePath []string) { - if len(filePath) > 0 { - namedScript := map[string]string{} - scriptPath := map[string]string{} - for _, fp := range filePath { - if name, script, err := ReadPlScriptFromFile(fp); err != nil { - l.Error(err) - } else { - scriptPath[name] = fp - namedScript[name] = script - } - } - if err := store.UpdateScriptsWithNS(ns, namedScript, scriptTags, scriptPath); err != nil { - l.Error(err) - } - } - - if dirPath != "" { - namedScript, filePath := ReadPlScriptFromDir(dirPath) - if err := store.UpdateScriptsWithNS(ns, namedScript, scriptTags, filePath); err != nil { - l.Error(err) - } - } -} diff --git a/vendor/github.com/GuanceCloud/cliutils/pipeline/pipeline.go b/vendor/github.com/GuanceCloud/cliutils/pipeline/pipeline.go index 088deb0c1f..e93d383989 100644 --- a/vendor/github.com/GuanceCloud/cliutils/pipeline/pipeline.go +++ b/vendor/github.com/GuanceCloud/cliutils/pipeline/pipeline.go @@ -8,7 +8,6 @@ package pipeline import ( "github.com/GuanceCloud/cliutils/pipeline/manager" - "github.com/GuanceCloud/cliutils/pipeline/manager/relation" "github.com/GuanceCloud/cliutils/pipeline/offload" "github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs" "github.com/GuanceCloud/cliutils/pipeline/ptinput/ipdb/geoip" @@ -21,8 +20,6 @@ import ( func InitLog() { // pipeline scripts manager manager.InitLog() - // scripts relation - relation.InitLog() // pipeline offload offload.InitLog() diff --git a/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_addkey.go b/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_addkey.go index 14ef38f70e..bcb60ce72f 100644 --- a/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_addkey.go +++ b/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_addkey.go @@ -55,10 +55,7 @@ func AddKey(ctx *runtime.Task, funcExpr *ast.CallExpr) *errchain.PlError { if errRun != nil { return errRun.ChainAppend(ctx.Name(), funcExpr.NamePos) } - if err := addKey2PtWithVal(ctx.InData(), key, val, dtype, ptinput.KindPtDefault); err != nil { - l.Debug(err) - return nil - } + _ = addKey2PtWithVal(ctx.InData(), key, val, dtype, ptinput.KindPtDefault) return nil } diff --git a/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_adjust_timezone.go b/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_adjust_timezone.go index 0de0cd8a08..4088b41231 100644 --- a/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_adjust_timezone.go +++ b/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_adjust_timezone.go @@ -67,10 +67,7 @@ func AdjustTimezone(ctx *runtime.Task, funcExpr *ast.CallExpr) *errchain.PlError switch cont := logTS.Value.(type) { case int64: cont = detectTimezone(cont, time.Now().UnixNano(), minuteAllow) - if err := addKey2PtWithVal(ctx.InData(), key, cont, ast.Int, ptinput.KindPtDefault); err != nil { - l.Debug(err) - return nil - } + _ = addKey2PtWithVal(ctx.InData(), key, cont, ast.Int, ptinput.KindPtDefault) default: return runtime.NewRunError(ctx, fmt.Sprintf( "param value expect int64, got `%s`", reflect.TypeOf(cont)), diff --git a/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_b64dec.go b/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_b64dec.go index 2c3c378bae..e368ba607b 100644 --- a/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_b64dec.go +++ b/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_b64dec.go @@ -61,10 +61,6 @@ func B64dec(ctx *runtime.Task, funcExpr *ast.CallExpr) *errchain.PlError { if err != nil { return runtime.NewRunError(ctx, err.Error(), funcExpr.NamePos) } - if err := addKey2PtWithVal(ctx.InData(), key, string(res), ast.String, - ptinput.KindPtDefault); err != nil { - l.Debug(err) - return nil - } + _ = addKey2PtWithVal(ctx.InData(), key, string(res), ast.String, ptinput.KindPtDefault) return nil } diff --git a/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_b64enc.go b/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_b64enc.go index cb4f82a058..7f01da6779 100644 --- a/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_b64enc.go +++ b/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_b64enc.go @@ -58,10 +58,6 @@ func B64enc(ctx *runtime.Task, funcExpr *ast.CallExpr) *errchain.PlError { return nil } res := base64.StdEncoding.EncodeToString([]byte(cont)) - if err := addKey2PtWithVal(ctx.InData(), key, res, ast.String, - ptinput.KindPtDefault); err != nil { - l.Debug(err) - return nil - } + _ = addKey2PtWithVal(ctx.InData(), key, res, ast.String, ptinput.KindPtDefault) return nil } diff --git a/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_cast.go b/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_cast.go index c5cfaf959f..552108665d 100644 --- a/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_cast.go +++ b/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_cast.go @@ -66,11 +66,6 @@ func Cast(ctx *runtime.Task, funcExpr *ast.CallExpr) *errchain.PlError { } val, dtype := doCast(v.Value, castType) - if err = addKey2PtWithVal(ctx.InData(), key, val, dtype, - ptinput.KindPtDefault); err != nil { - l.Debug(err) - return nil - } - + _ = addKey2PtWithVal(ctx.InData(), key, val, dtype, ptinput.KindPtDefault) return nil } diff --git a/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_conv_traceid_hex_to_dec.go b/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_conv_traceid_hex_to_dec.go index abd4d3027c..60810b52b4 100644 --- a/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_conv_traceid_hex_to_dec.go +++ b/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_conv_traceid_hex_to_dec.go @@ -54,13 +54,8 @@ func ConvTraceIDW3C2DD(ctx *runtime.Task, funcExpr *ast.CallExpr) *errchain.PlEr if ddTraceID, err := convTraceW3CToDD(w3cTraceID); err != nil { l.Debug(err) - return nil } else { - if err := addKey2PtWithVal(ctx.InData(), key, ddTraceID, ast.String, - ptinput.KindPtDefault); err != nil { - l.Debug(err) - return nil - } + addKey2PtWithVal(ctx.InData(), key, ddTraceID, ast.String, ptinput.KindPtDefault) } return nil } diff --git a/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_cover.go b/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_cover.go index b02462d0ce..5d97eb5323 100644 --- a/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_cover.go +++ b/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_cover.go @@ -143,11 +143,8 @@ func Cover(ctx *runtime.Task, funcExpr *ast.CallExpr) *errchain.PlError { } } - if err := addKey2PtWithVal(ctx.InData(), key, string(arrCont), ast.String, - ptinput.KindPtDefault); err != nil { - l.Debug(err) - return nil - } + _ = addKey2PtWithVal(ctx.InData(), key, string(arrCont), + ast.String, ptinput.KindPtDefault) return nil } diff --git a/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_create_point.go b/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_create_point.go index b6affcd6c9..180b39bd40 100644 --- a/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_create_point.go +++ b/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_create_point.go @@ -186,6 +186,8 @@ func ptCategory(cat string) point.Category { return point.CustomObject case point.SSecurity, point.CS: return point.Security + case point.SDialTesting, point.CDT: + return point.DialTesting } return point.UnknownCategory } diff --git a/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_datetime.go b/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_datetime.go index e5c38cf157..77d0005517 100644 --- a/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_datetime.go +++ b/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_datetime.go @@ -129,17 +129,14 @@ func DateTime(ctx *runtime.Task, funcExpr *ast.CallExpr) *errchain.PlError { } if datetimeInnerFormat(fmts) { - if v, err := DateFormatHandle(&t, fmts); err != nil { + v, err := DateFormatHandle(&t, fmts) + if err != nil { return runtime.NewRunError(ctx, err.Error(), funcExpr.NamePos) - } else if err := addKey2PtWithVal(ctx.InData(), key, v, ast.String, - ptinput.KindPtDefault); err != nil { - l.Debug(err) - return nil } - } else if err := addKey2PtWithVal(ctx.InData(), key, timefmt.Format(t, fmts), ast.String, - ptinput.KindPtDefault); err != nil { - l.Debug(err) - return nil + _ = addKey2PtWithVal(ctx.InData(), key, v, ast.String, ptinput.KindPtDefault) + } else { + _ = addKey2PtWithVal(ctx.InData(), key, timefmt.Format(t, fmts), + ast.String, ptinput.KindPtDefault) } return nil diff --git a/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_decoder.go b/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_decoder.go index 98e85f853a..2cf81f411b 100644 --- a/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_decoder.go +++ b/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_decoder.go @@ -90,12 +90,8 @@ func Decode(ctx *runtime.Task, funcExpr *ast.CallExpr) *errchain.PlError { return nil } - if err := addKey2PtWithVal(ctx.InData(), key, newcont, ast.String, - ptinput.KindPtDefault); err != nil { - l.Debug(err) - return nil - } - + _ = addKey2PtWithVal(ctx.InData(), key, newcont, + ast.String, ptinput.KindPtDefault) return nil } diff --git a/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_default_time.go b/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_default_time.go index abe438c184..44a1f551eb 100644 --- a/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_default_time.go +++ b/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_default_time.go @@ -68,11 +68,8 @@ func DefaultTime(ctx *runtime.Task, funcExpr *ast.CallExpr) *errchain.PlError { if nanots, err := TimestampHandle(cont, tz); err != nil { usePointTime(ctx, key, err) - return nil - } else if err := addKey2PtWithVal(ctx.InData(), key, nanots, - ast.Int, ptinput.KindPtDefault); err != nil { - l.Debug(err) - return nil + } else { + addKey2PtWithVal(ctx.InData(), key, nanots, ast.Int, ptinput.KindPtDefault) } return nil diff --git a/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_default_time_with_fmt.go b/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_default_time_with_fmt.go index 18cdc19c21..167beb9a12 100644 --- a/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_default_time_with_fmt.go +++ b/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_default_time_with_fmt.go @@ -106,10 +106,7 @@ func DefaultTimeWithFmt(ctx *runtime.Task, funcExpr *ast.CallExpr) *errchain.PlE timeStr, goTimeFmt, tz, err) return nil } else { - if err := addKey2PtWithVal(ctx.InData(), key, t.UnixNano(), ast.Int, ptinput.KindPtDefault); err != nil { - l.Debug(err) - return nil - } + _ = addKey2PtWithVal(ctx.InData(), key, t.UnixNano(), ast.Int, ptinput.KindPtDefault) return nil } } diff --git a/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_duration_precision.go b/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_duration_precision.go index 9c5add7a93..dddb045376 100644 --- a/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_duration_precision.go +++ b/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_duration_precision.go @@ -71,10 +71,8 @@ func DurationPrecision(ctx *runtime.Task, funcExpr *ast.CallExpr) *errchain.PlEr tValue *= 10 } } - if err := addKey2PtWithVal(ctx.InData(), key, tValue, ast.Int, - ptinput.KindPtDefault); err != nil { - return runtime.NewRunError(ctx, err.Error(), funcExpr.NamePos) - } + _ = addKey2PtWithVal(ctx.InData(), key, tValue, ast.Int, + ptinput.KindPtDefault) return nil } diff --git a/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_geo_ip.go b/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_geo_ip.go index aa9bb6e856..db0713e775 100644 --- a/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_geo_ip.go +++ b/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_geo_ip.go @@ -69,10 +69,7 @@ func GeoIP(ctx *runtime.Task, funcExpr *ast.CallExpr) *errchain.PlError { return nil } else { for k, v := range dic { - if err := addKey2PtWithVal(ctx.InData(), k, v, ast.String, ptinput.KindPtDefault); err != nil { - l.Debug(err) - return nil - } + _ = addKey2PtWithVal(ctx.InData(), k, v, ast.String, ptinput.KindPtDefault) } } diff --git a/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_getkey.go b/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_getkey.go index 62388b54bd..f4bb4f863e 100644 --- a/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_getkey.go +++ b/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_getkey.go @@ -27,9 +27,6 @@ func GetkeyChecking(ctx *runtime.Task, funcExpr *ast.CallExpr) *errchain.PlError } func Getkey(ctx *runtime.Task, funcExpr *ast.CallExpr) *errchain.PlError { - if funcExpr == nil { - return runtime.NewRunError(ctx, "unreachable", funcExpr.NamePos) - } if len(funcExpr.Param) != 1 { return runtime.NewRunError(ctx, fmt.Sprintf("func %s expected 1 args", funcExpr.Name), funcExpr.NamePos) diff --git a/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_gjson.go b/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_gjson.go index 9ed5a8fe6f..180354f04c 100644 --- a/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_gjson.go +++ b/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_gjson.go @@ -28,10 +28,6 @@ func GJSONChecking(ctx *runtime.Task, funcExpr *ast.CallExpr) *errchain.PlError switch funcExpr.Param[1].NodeType { //nolint:exhaustive case ast.TypeIdentifier, ast.TypeStringLiteral: - var err error - if err != nil { - return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[1].StartPos()) - } default: return runtime.NewRunError(ctx, fmt.Sprintf("expect AttrExpr, IndexExpr or Identifier, got %s", funcExpr.Param[1].NodeType), funcExpr.Param[1].StartPos()) @@ -103,10 +99,6 @@ func GJSON(ctx *runtime.Task, funcExpr *ast.CallExpr) *errchain.PlError { return nil } - if err := addKey2PtWithVal(ctx.InData(), targetKey, v, dtype, ptinput.KindPtDefault); err != nil { - l.Debug(err) - return nil - } - + _ = addKey2PtWithVal(ctx.InData(), targetKey, v, dtype, ptinput.KindPtDefault) return nil } diff --git a/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_grok.go b/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_grok.go index 836c789943..2aa82ba42d 100644 --- a/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_grok.go +++ b/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_grok.go @@ -113,8 +113,7 @@ func Grok(ctx *runtime.Task, funcExpr *ast.CallExpr) *errchain.PlError { continue } } - if err := addKey2PtWithVal(ctx.InData(), name, result[i], dtype, ptinput.KindPtDefault); err != nil { - l.Debug(err) + if ok := addKey2PtWithVal(ctx.InData(), name, result[i], dtype, ptinput.KindPtDefault); !ok { ctx.Regs.ReturnAppend(false, ast.Bool) return nil } @@ -126,8 +125,7 @@ func Grok(ctx *runtime.Task, funcExpr *ast.CallExpr) *errchain.PlError { return nil } for i, name := range grokRe.MatchNames() { - if err := addKey2PtWithVal(ctx.InData(), name, result[i], ast.String, ptinput.KindPtDefault); err != nil { - l.Debug(err) + if ok := addKey2PtWithVal(ctx.InData(), name, result[i], ast.String, ptinput.KindPtDefault); !ok { ctx.Regs.ReturnAppend(false, ast.Bool) return nil } diff --git a/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_json.go b/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_json.go index 326809a275..97a2b7d7f7 100644 --- a/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_json.go +++ b/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_json.go @@ -158,17 +158,12 @@ func JSON(ctx *runtime.Task, funcExpr *ast.CallExpr) *errchain.PlError { default: return nil } - if err := addKey2PtWithVal(ctx.InData(), targetKey, v, dtype, ptinput.KindPtDefault); err != nil { - l.Debug(err) + if ok := addKey2PtWithVal(ctx.InData(), targetKey, v, dtype, ptinput.KindPtDefault); !ok { return nil } if deleteAfterExtract { - if err := addKey2PtWithVal(ctx.InData(), srcKey, dstS, ast.String, - ptinput.KindPtDefault); err != nil { - l.Debug(err) - return nil - } + _ = addKey2PtWithVal(ctx.InData(), srcKey, dstS, ast.String, ptinput.KindPtDefault) } return nil diff --git a/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_kv.go b/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_kv.go index d2109c2959..a54516936e 100644 --- a/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_kv.go +++ b/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_kv.go @@ -260,9 +260,7 @@ func KVSplit(ctx *runtime.Task, funcExpr *ast.CallExpr) *errchain.PlError { } for k, v := range result { - if err := addKey2PtWithVal(ctx.InData(), k, v, ast.String, ptinput.KindPtDefault); err != nil { - l.Debug(err) - } + _ = addKey2PtWithVal(ctx.InData(), k, v, ast.String, ptinput.KindPtDefault) } ctx.Regs.ReturnAppend(true, ast.Bool) diff --git a/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_lowercase.go b/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_lowercase.go index 3f64db2b16..2156f02cf9 100644 --- a/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_lowercase.go +++ b/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_lowercase.go @@ -46,11 +46,7 @@ func Lowercase(ctx *runtime.Task, funcExpr *ast.CallExpr) *errchain.PlError { } v := strings.ToLower(cont) - if err = addKey2PtWithVal(ctx.InData(), key, v, ast.String, - ptinput.KindPtDefault); err != nil { - l.Debug(err) - return nil - } + _ = addKey2PtWithVal(ctx.InData(), key, v, ast.String, ptinput.KindPtDefault) return nil } diff --git a/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_parse_duration.go b/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_parse_duration.go index a423aa59f8..48d2f80b81 100644 --- a/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_parse_duration.go +++ b/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_parse_duration.go @@ -60,10 +60,6 @@ func ParseDuration(ctx *runtime.Task, funcExpr *ast.CallExpr) *errchain.PlError return nil } - if err := addKey2PtWithVal(ctx.InData(), key, int64(du), ast.Int, - ptinput.KindPtDefault); err != nil { - l.Debug(err) - return nil - } + _ = addKey2PtWithVal(ctx.InData(), key, int64(du), ast.Int, ptinput.KindPtDefault) return nil } diff --git a/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_pt_kvs.go b/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_pt_kvs.go index 3a39813a84..c551221a32 100644 --- a/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_pt_kvs.go +++ b/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_pt_kvs.go @@ -185,12 +185,12 @@ func ptKvsSet(ctx *runtime.Task, funcExpr *ast.CallExpr, vals ...any) *errchain. } if asTag { - if err := pt.SetTag(name, val, getValDtype(val)); err != nil { + if ok := pt.SetTag(name, val, getValDtype(val)); !ok { ctx.Regs.ReturnAppend(false, ast.Bool) return nil } } else { - if err := pt.Set(name, val, getValDtype(val)); err != nil { + if ok := pt.Set(name, val, getValDtype(val)); !ok { ctx.Regs.ReturnAppend(false, ast.Bool) return nil } diff --git a/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_replace.go b/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_replace.go index c5e4e14fbb..76d09c27ea 100644 --- a/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_replace.go +++ b/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_replace.go @@ -87,11 +87,7 @@ func Replace(ctx *runtime.Task, funcExpr *ast.CallExpr) *errchain.PlError { } newCont := reg.ReplaceAllString(cont, dz) - if err := addKey2PtWithVal(ctx.InData(), key, newCont, ast.String, - ptinput.KindPtDefault); err != nil { - l.Debug(err) - return nil - } + _ = addKey2PtWithVal(ctx.InData(), key, newCont, ast.String, ptinput.KindPtDefault) return nil } diff --git a/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_sql_cover.go b/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_sql_cover.go index f31e7c3474..ed78bf10cb 100644 --- a/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_sql_cover.go +++ b/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_sql_cover.go @@ -51,11 +51,7 @@ func SQLCover(ctx *runtime.Task, funcExpr *ast.CallExpr) *errchain.PlError { l.Debug(err) return nil } - if err := addKey2PtWithVal(ctx.InData(), key, v, ast.String, - ptinput.KindPtDefault); err != nil { - l.Debug(err) - return nil - } + _ = addKey2PtWithVal(ctx.InData(), key, v, ast.String, ptinput.KindPtDefault) return nil } diff --git a/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_strfmt.go b/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_strfmt.go index c460414771..1cc071cca8 100644 --- a/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_strfmt.go +++ b/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_strfmt.go @@ -61,11 +61,7 @@ func Strfmt(ctx *runtime.Task, funcExpr *ast.CallExpr) *errchain.PlError { } strfmt := fmt.Sprintf(fmts, outdata...) - if err := addKey2PtWithVal(ctx.InData(), key, strfmt, ast.String, - ptinput.KindPtDefault); err != nil { - l.Debug(err) - return nil - } + _ = addKey2PtWithVal(ctx.InData(), key, strfmt, ast.String, ptinput.KindPtDefault) return nil } diff --git a/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_trim.go b/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_trim.go index 72084bb8a7..8f31fbc045 100644 --- a/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_trim.go +++ b/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_trim.go @@ -64,11 +64,7 @@ func Trim(ctx *runtime.Task, funcExpr *ast.CallExpr) *errchain.PlError { val = strings.Trim(cont, cutset) } - if err = addKey2PtWithVal(ctx.InData(), key, val, ast.String, - ptinput.KindPtDefault); err != nil { - l.Debug(err) - return nil - } + _ = addKey2PtWithVal(ctx.InData(), key, val, ast.String, ptinput.KindPtDefault) return nil } diff --git a/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_uppercase.go b/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_uppercase.go index 969ec47eb0..64064678d4 100644 --- a/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_uppercase.go +++ b/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_uppercase.go @@ -45,11 +45,7 @@ func Uppercase(ctx *runtime.Task, funcExpr *ast.CallExpr) *errchain.PlError { v := strings.ToUpper(cont) - if err := addKey2PtWithVal(ctx.InData(), key, v, ast.String, - ptinput.KindPtDefault); err != nil { - l.Debug(err) - return nil - } + _ = addKey2PtWithVal(ctx.InData(), key, v, ast.String, ptinput.KindPtDefault) return nil } diff --git a/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_urldecode.go b/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_urldecode.go index 3ab44c55ba..81841e55bb 100644 --- a/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_urldecode.go +++ b/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_urldecode.go @@ -44,9 +44,8 @@ func URLDecode(ctx *runtime.Task, funcExpr *ast.CallExpr) *errchain.PlError { if v, err := UrldecodeHandle(cont); err != nil { return runtime.NewRunError(ctx, err.Error(), funcExpr.NamePos) - } else if err := addKey2PtWithVal(ctx.InData(), key, v, ast.String, ptinput.KindPtDefault); err != nil { - l.Debug(err) - return nil + } else { + addKey2PtWithVal(ctx.InData(), key, v, ast.String, ptinput.KindPtDefault) } return nil diff --git a/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_xml.go b/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_xml.go index c0ad876c46..309aeb0ca4 100644 --- a/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_xml.go +++ b/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/fn_xml.go @@ -104,11 +104,8 @@ func XML(ctx *runtime.Task, funcExpr *ast.CallExpr) *errchain.PlError { return nil } - if err := addKey2PtWithVal(ctx.InData(), fieldName, dest.InnerText(), ast.String, - ptinput.KindPtDefault); err != nil { - l.Debug(err) - return nil - } + _ = addKey2PtWithVal(ctx.InData(), fieldName, dest.InnerText(), + ast.String, ptinput.KindPtDefault) return nil } diff --git a/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/utils.go b/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/utils.go index e6a6808e73..b88ee93589 100644 --- a/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/utils.go +++ b/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs/utils.go @@ -229,10 +229,10 @@ func pointTime(in any) int64 { } } -func addKey2PtWithVal(in any, key string, value any, dtype ast.DType, kind ptinput.KeyKind) error { +func addKey2PtWithVal(in any, key string, value any, dtype ast.DType, kind ptinput.KeyKind) bool { pt, err := getPoint(in) if err != nil { - return err + return false } if key == "_" { diff --git a/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/point.go b/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/point.go index d34f34b342..cfff2617da 100644 --- a/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/point.go +++ b/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/point.go @@ -7,6 +7,7 @@ package ptinput import ( + "errors" "fmt" "time" @@ -35,12 +36,12 @@ type PlInputPt interface { SetPtName(m string) Get(key string) (any, ast.DType, error) - GetWithIsTag(key string) (any, bool, bool) - Set(key string, value any, dtype ast.DType) error + Set(key string, value any, dtype ast.DType) bool + Delete(key string) RenameKey(from, to string) error - SetTag(key string, value any, dtype ast.DType) error + SetTag(key string, value any, dtype ast.DType) bool PtTime() time.Time @@ -196,6 +197,8 @@ func (pt *PlPoint) Category() point.Category { return pt.category } +var ErrKeyNotExist = errors.New("key not exist") + func (pt *PlPoint) Get(key string) (any, ast.DType, error) { if v, ok := pt.tags[key]; ok { return v, ast.String, nil @@ -205,7 +208,7 @@ func (pt *PlPoint) Get(key string) (any, ast.DType, error) { v, dtype := valueDtype(v) return v, dtype, nil } - return nil, ast.Invalid, fmt.Errorf("unsupported pt key type") + return nil, ast.Nil, ErrKeyNotExist } func (pt *PlPoint) GetWithIsTag(key string) (any, bool, bool) { @@ -220,35 +223,35 @@ func (pt *PlPoint) GetWithIsTag(key string) (any, bool, bool) { return nil, false, false } -func (pt *PlPoint) Set(key string, value any, dtype ast.DType) error { +func (pt *PlPoint) Set(key string, value any, dtype ast.DType) bool { if _, ok := pt.tags[key]; ok { // is tag if dtype == ast.Void || dtype == ast.Invalid { delete(pt.tags, key) - return nil + return true } if v, err := plruntime.Conv2String(value, dtype); err == nil { pt.tags[key] = v - return nil + return true } else { - return err + return false } } else { // is field switch dtype { //nolint:exhaustive case ast.Nil, ast.Void, ast.Invalid: pt.fields[key] = nil - return nil + return true case ast.List, ast.Map: if v, err := plruntime.Conv2String(value, dtype); err == nil { pt.fields[key] = v } else { pt.fields[key] = nil - return nil + return true } default: pt.fields[key] = value } } - return nil + return true } func (pt *PlPoint) Delete(key string) { @@ -272,15 +275,15 @@ func (pt *PlPoint) RenameKey(from, to string) error { return nil } -func (pt *PlPoint) SetTag(key string, value any, dtype ast.DType) error { +func (pt *PlPoint) SetTag(key string, value any, dtype ast.DType) bool { delete(pt.fields, key) if str, err := plruntime.Conv2String(value, dtype); err == nil { pt.tags[key] = str - return nil + return true } else { pt.tags[key] = "" - return err + return false } } diff --git a/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/utils/utils.go b/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/utils/utils.go index c20b541df7..b3cd5448c8 100644 --- a/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/utils/utils.go +++ b/vendor/github.com/GuanceCloud/cliutils/pipeline/ptinput/utils/utils.go @@ -15,7 +15,7 @@ import ( func PtCatOption(cat point.Category) []point.Option { var opt []point.Option switch cat { - case point.Logging: + case point.Logging, point.DialTesting: opt = point.DefaultLoggingOptions() case point.Tracing, point.Network, @@ -33,6 +33,7 @@ func PtCatOption(cat point.Category) []point.Option { case point.DynamicDWCategory, point.MetricDeprecated, point.UnknownCategory: // pass + default: } return opt diff --git a/vendor/github.com/GuanceCloud/cliutils/point/category.go b/vendor/github.com/GuanceCloud/cliutils/point/category.go index e9a647537d..91daa904ad 100644 --- a/vendor/github.com/GuanceCloud/cliutils/point/category.go +++ b/vendor/github.com/GuanceCloud/cliutils/point/category.go @@ -71,6 +71,7 @@ func AllCategories() []Category { Security, Profiling, DynamicDWCategory, + DialTesting, } } @@ -89,6 +90,7 @@ const ( RUM Security Profiling + DialTesting SUnknownCategory = "unknown" SDynamicDWCategory = "dynamic_dw" // NOTE: not used @@ -103,6 +105,7 @@ const ( SRUM = "rum" SSecurity = "security" SProfiling = "profiling" + SDialTesting = "dialtesting" URLUnknownCategory = "/v1/write/unknown" URLDynamicDWCategory = "/v1/write/dynamic_dw" // NOTE: not used @@ -117,6 +120,7 @@ const ( URLRUM = "/v1/write/rum" URLSecurity = "/v1/write/security" URLProfiling = "/v1/write/profiling" + URLDialTesting = "/v1/write/dialtesting" // NOTE: not used CUnknown = "UNKNOWN" CDynamicDW = "DYNAMIC_DW" @@ -130,6 +134,7 @@ const ( CR = "R" CS = "S" CP = "P" + CDT = "DT" ) var ( @@ -147,6 +152,8 @@ var ( Security: URLSecurity, Profiling: URLProfiling, + DialTesting: URLDialTesting, + DynamicDWCategory: URLDynamicDWCategory, UnknownCategory: URLUnknownCategory, @@ -163,6 +170,7 @@ var ( RUM: CR, Security: CS, Profiling: CP, + DialTesting: CDT, UnknownCategory: CUnknown, DynamicDWCategory: CDynamicDW, } @@ -179,6 +187,7 @@ var ( RUM: SRUM, Security: SSecurity, Profiling: SProfiling, + DialTesting: SDialTesting, UnknownCategory: SUnknownCategory, DynamicDWCategory: SDynamicDWCategory, } diff --git a/vendor/modules.txt b/vendor/modules.txt index 66ed98fe3c..f8e22ed651 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -37,7 +37,7 @@ github.com/DataDog/sketches-go/ddsketch/mapping github.com/DataDog/sketches-go/ddsketch/pb/sketchpb github.com/DataDog/sketches-go/ddsketch/stat github.com/DataDog/sketches-go/ddsketch/store -# github.com/GuanceCloud/cliutils v1.1.21-0.20240821062850-3079bb7ed98d +# github.com/GuanceCloud/cliutils v1.1.21-0.20240904042137-2a87297900d6 ## explicit; go 1.19 github.com/GuanceCloud/cliutils github.com/GuanceCloud/cliutils/dialtesting @@ -50,7 +50,6 @@ github.com/GuanceCloud/cliutils/network/http github.com/GuanceCloud/cliutils/network/ws github.com/GuanceCloud/cliutils/pipeline github.com/GuanceCloud/cliutils/pipeline/manager -github.com/GuanceCloud/cliutils/pipeline/manager/relation github.com/GuanceCloud/cliutils/pipeline/offload github.com/GuanceCloud/cliutils/pipeline/ptinput github.com/GuanceCloud/cliutils/pipeline/ptinput/funcs