Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Chore pipeline #110

Merged
merged 7 commits into from
Sep 4, 2024
Merged

Chore pipeline #110

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
9a8d986
add plpt package
vircoys Aug 27, 2024
0775a89
add category: dialtesting
vircoys Aug 27, 2024
0ba6e95
update utils.go
vircoys Aug 27, 2024
b2c1d5b
update pipeline
vircoys Aug 27, 2024
a0e1e5e
refact
vircoys Aug 27, 2024
8469192
refact script store
vircoys Sep 2, 2024
1445aa2
refact
vircoys Sep 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
241 changes: 115 additions & 126 deletions pipeline/manager/manager.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,13 @@
"os"
"path/filepath"

"github.com/GuanceCloud/cliutils/pipeline/manager/relation"
"github.com/GuanceCloud/cliutils/pipeline/ptinput/plmap"
"github.com/GuanceCloud/cliutils/point"
)

type Manager struct {
storeMap map[point.Category]*ScriptStore
relation *relation.ScriptRelation
relation *ScriptRelation
}

type ManagerCfg struct {
Expand All @@ -35,101 +34,93 @@
func NewManager(cfg ManagerCfg) *Manager {
center := &Manager{
storeMap: map[point.Category]*ScriptStore{},
relation: relation.NewPipelineRelation(),
relation: NewPipelineRelation(),
}
for _, cat := range point.AllCategories() {
center.storeMap[cat] = NewScriptStore(cat, cfg)
}
return center
}

func (c *Manager) whichStore(category point.Category) (*ScriptStore, bool) {
func (m *Manager) whichStore(category point.Category) (*ScriptStore, bool) {
if category == point.MetricDeprecated {
category = point.Metric
}
if v, ok := c.storeMap[category]; ok && v != nil {
if v, ok := m.storeMap[category]; ok && v != nil {
return v, ok
}
return nil, false
}

func (c *Manager) GetScriptRelation() *relation.ScriptRelation {
return c.relation
func (m *Manager) UpdateDefaultScript(mp map[point.Category]string) {
for _, cat := range point.AllCategories() {
if store, ok := m.whichStore(cat); ok {
if v, ok := mp[cat]; ok && v != "" {
store.SetDefaultScript(v)
} else {
store.SetDefaultScript("")
}
}
}
}

func (c *Manager) QueryScript(category point.Category, name string) (*PlScript, bool) {
if v, ok := c.whichStore(category); ok {
return v.IndexGet(name)
func (m *Manager) GetScriptRelation() *ScriptRelation {
return m.relation
}

func (m *Manager) QueryScript(category point.Category, name string,
DisableDefaultP ...struct{}) (*PlScript, bool) {

Check failure on line 72 in pipeline/manager/manager.go

View workflow job for this annotation

GitHub Actions / build 

captLocal: `DisableDefaultP' should not be capitalized (gocritic)

if v, ok := m.whichStore(category); ok {
if ss, ok := v.IndexGet(name); ok {
return ss, ok
}

if len(DisableDefaultP) == 0 {
return v.IndexDefault()
}
}
return nil, false
}

func (c *Manager) ScriptCount(category point.Category) int {
if v, ok := c.whichStore(category); ok {
func (m *Manager) ScriptCount(category point.Category) int {
if v, ok := m.whichStore(category); ok {
return v.Count()
}
return 0
}

func ReadPlScriptFromFile(fp string) (string, string, error) {
fp = filepath.Clean(fp)
if v, err := os.ReadFile(filepath.Clean(fp)); err == nil {
_, sName := filepath.Split(fp)
return sName, string(v), nil
} else {
return "", "", err
func (m *Manager) LoadScriptsFromWorkspace(ns, plPath string, tags map[string]string) {
if plPath == "" {
return
}

scripts, _ := ReadWorkspaceScripts(plPath)

m.LoadScripts(ns, scripts, tags)
}

func SearchPlFilePathFormDir(dirPath string) map[string]string {
ret := map[string]string{}
dirPath = filepath.Clean(dirPath)
if dirEntry, err := os.ReadDir(dirPath); err != nil {
l.Warn(err)
} else {
for _, v := range dirEntry {
if v.IsDir() {
continue
}
sName := v.Name()
if filepath.Ext(sName) != ".p" {
continue
}
ret[sName] = filepath.Join(dirPath, sName)
// LoadScripts is used to load and clean the script, parameter scripts example:
// {point.Logging: {ScriptName: ScriptContent},... }.
func (m *Manager) LoadScripts(ns string, scripts map[point.Category](map[string]string),
tags map[string]string,
) {
for _, cat := range point.AllCategories() {
if ss, ok := scripts[cat]; ok {
m.LoadScriptWithCat(cat, ns, ss, tags)
} else {
// cleanup the store for this category
m.LoadScriptWithCat(cat, ns, map[string]string{}, tags)
}
}
return ret
}

func ReadPlScriptFromDir(dirPath string) (map[string]string, map[string]string) {
if dirPath == "" {
return nil, nil
}

ret := map[string]string{}
retPath := map[string]string{}
dirPath = filepath.Clean(dirPath)
if dirEntry, err := os.ReadDir(dirPath); err != nil {
l.Warn(err)
} else {
for _, v := range dirEntry {
if v.IsDir() {
continue
}
sName := v.Name()
if filepath.Ext(sName) != ".p" {
continue
}
sPath := filepath.Join(dirPath, sName)
if name, script, err := ReadPlScriptFromFile(sPath); err == nil {
ret[name] = script
retPath[name] = sPath
} else {
l.Error(err)
}
}
func (m *Manager) LoadScriptWithCat(category point.Category, ns string,
scripts, tags map[string]string,
) {
if v, ok := m.whichStore(category); ok {
v.UpdateScriptsWithNS(ns, scripts, tags)
}
return ret, retPath
}

func CategoryDirName() map[point.Category]string {
Expand All @@ -144,16 +135,24 @@
point.RUM: "rum",
point.Security: "security",
point.Profiling: "profiling",
point.DialTesting: "dialtesting",
}
}

func SearchPlFilePathFromPlStructPath(basePath string) map[point.Category](map[string]string) {
func SearchWorkspaceScripts(basePath string) map[point.Category](map[string]string) {
files := map[point.Category](map[string]string){}

files[point.Logging] = SearchPlFilePathFormDir(basePath)
var err error
files[point.Logging], err = SearchScripts(basePath)
if err != nil {
log.Warn(err)
}

for category, dirName := range CategoryDirName() {
s := SearchPlFilePathFormDir(filepath.Join(basePath, dirName))
s, err := SearchScripts(filepath.Join(basePath, dirName))
if err != nil {
log.Warn(err)
}
if _, ok := files[category]; !ok {
files[category] = map[string]string{}
}
Expand All @@ -164,85 +163,75 @@
return files
}

func ReadPlScriptFromPlStructPath(basePath string) (
func ReadWorkspaceScripts(basePath string) (
map[point.Category](map[string]string), map[point.Category](map[string]string),
) {
if basePath == "" {
return nil, nil
}
scriptsPath := SearchWorkspaceScripts(basePath)

scripts := map[point.Category](map[string]string){}
scriptsPath := map[point.Category](map[string]string){}

scripts[point.Logging], scriptsPath[point.Logging] = ReadPlScriptFromDir(basePath)

for category, dirName := range CategoryDirName() {
s, p := ReadPlScriptFromDir(filepath.Join(basePath, dirName))
if _, ok := scripts[category]; !ok {
scripts[category] = map[string]string{}
}
if _, ok := scriptsPath[category]; !ok {
scriptsPath[category] = map[string]string{}
for cat, ssPath := range scriptsPath {
if _, ok := scripts[cat]; !ok {
scripts[cat] = map[string]string{}
}

for k, v := range s {
scripts[category][k] = v
}
for k, v := range p {
scriptsPath[category][k] = v
for _, path := range ssPath {
if name, script, err := ReadScript(path); err == nil {
scripts[cat][name] = script
} else {
log.Error(err)
}
}
}

return scripts, scriptsPath
}

func LoadDefaultScripts2Store(center *Manager, rootDir string, tags map[string]string) {
if rootDir == "" {
return
}

plPath := filepath.Join(rootDir, "pipeline")
LoadScripts2StoreFromPlStructPath(center, DefaultScriptNS, plPath, tags)
}
func SearchScripts(dirPath string) (map[string]string, error) {
ret := map[string]string{}
dirPath = filepath.Clean(dirPath)

func LoadScripts2StoreFromPlStructPath(center *Manager, ns, plPath string, tags map[string]string) {
if plPath == "" {
return
dirEntry, err := os.ReadDir(dirPath)
if err != nil {
return nil, err
}

scripts, path := ReadPlScriptFromPlStructPath(plPath)

LoadScripts(center, ns, scripts, path, tags)
for _, v := range dirEntry {
if v.IsDir() {
// todo: support sub dir
continue
}
if sName := v.Name(); filepath.Ext(sName) == ".p" {
ret[sName] = filepath.Join(dirPath, sName)
}
}
return ret, nil
}

// LoadScripts is used to load and clean the script, parameter scripts example: {datakit.Logging: {ScriptName: ScriptContent},... }.
func LoadScripts(center *Manager, ns string, scripts, scriptPath map[point.Category](map[string]string),
tags map[string]string,
) {
allCategoryScript := FillScriptCategoryMap(scripts)
for category, m := range allCategoryScript {
LoadScript(center, category, ns, m, scriptPath[category], tags)
func ReadScripts(dirPath string) (map[string]string, map[string]string) {
ret := map[string]string{}

scriptsPath, err := SearchScripts(dirPath)
if err != nil {
log.Warn(err)
return nil, nil
}
}

func LoadScript(centor *Manager, category point.Category, ns string,
scripts, path, tags map[string]string,
) {
if v, ok := centor.whichStore(category); ok {
v.UpdateScriptsWithNS(ns, scripts, path, tags)
for _, path := range scriptsPath {
if name, script, err := ReadScript(path); err == nil {
ret[name] = script
} else {
log.Error(err)
}
}

return ret, scriptsPath
}

func FillScriptCategoryMap(scripts map[point.Category](map[string]string)) map[point.Category](map[string]string) {
allCategoryScript := map[point.Category](map[string]string){}
for _, cat := range point.AllCategories() {
allCategoryScript[cat] = map[string]string{}
}
for k, v := range scripts {
for name, script := range v {
if v, ok := allCategoryScript[k]; ok {
v[name] = script
}
}
func ReadScript(fp string) (string, string, error) {
fp = filepath.Clean(fp)
if v, err := os.ReadFile(filepath.Clean(fp)); err == nil {
_, sName := filepath.Split(fp)
return sName, string(v), nil
} else {
return "", "", err
}
return allCategoryScript
}
Loading
Loading