diff --git a/docs/types/flowfile.md b/docs/types/flowfile.md index 65d4b52..e354cdc 100644 --- a/docs/types/flowfile.md +++ b/docs/types/flowfile.md @@ -14,7 +14,8 @@ A workspace can have multiple flow files located anywhere in the workspace direc | Field | Description | Type | Default | Required | | ----- | ----------- | ---- | ------- | -------- | -| `description` | A description of the executables defined within the flow file. This description will be set as the executables' description if not defined at the executable level. | `string` | | [] | +| `description` | A description of the executables defined within the flow file. This description will used as a shared description for all executables in the flow file. | `string` | | [] | +| `descriptionFile` | A path to a markdown file that contains the description of the executables defined within the flow file. | `string` | | [] | | `executables` | | `array` ([Executable](#Executable)) | [] | [] | | `fromFile` | | [FromFile](#FromFile) | [] | [] | | `namespace` | The namespace to be given to all executables in the flow file. If not set, the executables in the file will be grouped into the root (*) namespace. Namespaces can be reused across multiple flow files. Namespaces are used to reference executables in the CLI using the format `workspace:namespace/name`. | `string` | | [] | diff --git a/docs/types/workspace.md b/docs/types/workspace.md index 1d88072..962f0fb 100644 --- a/docs/types/workspace.md +++ b/docs/types/workspace.md @@ -15,6 +15,7 @@ Every workspace has a workspace config file named `flow.yaml` in the root of the | Field | Description | Type | Default | Required | | ----- | ----------- | ---- | ------- | -------- | | `description` | A description of the workspace. This description is rendered as markdown in the interactive UI. | `string` | | [] | +| `descriptionFile` | A path to a markdown file that contains the description of the workspace. | `string` | | [] | | `displayName` | The display name of the workspace. This is used in the interactive UI. | `string` | | [] | | `executables` | | [ExecutableFilter](#ExecutableFilter) | | [] | | `tags` | | [CommonTags](#CommonTags) | [] | [] | diff --git a/flow.yaml b/flow.yaml index 20a72b3..e5deb77 100644 --- a/flow.yaml +++ b/flow.yaml @@ -1,4 +1,3 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/jahvon/flow/HEAD/schemas/workspace_schema.json -displayName: flow -git: - enabled: false +displayName: flow repository +descriptionFile: README.md diff --git a/internal/cache/executable_generator.go b/internal/cache/executable_generator.go index 6cbb72d..cebe641 100644 --- a/internal/cache/executable_generator.go +++ b/internal/cache/executable_generator.go @@ -16,11 +16,14 @@ import ( const generatedTag = "generated" func generatedExecutables( - logger io.Logger, - wsName, wsPath, flowFileNs, flowFilePath string, - files []string, + logger io.Logger, wsName string, flowFile *executable.FlowFile, ) (executable.ExecutableList, error) { executables := make(executable.ExecutableList, 0) + wsPath := flowFile.WorkspacePath() + flowFilePath := flowFile.ConfigPath() + flowFileNs := flowFile.Namespace + files := flowFile.FromFile + for _, file := range files { expandedFile := utils.ExpandDirectory(logger, file, wsPath, flowFilePath, nil) exec, err := executablesFromFile(logger, file, expandedFile) @@ -28,6 +31,7 @@ func generatedExecutables( return nil, err } exec.SetContext(wsName, wsPath, flowFileNs, flowFilePath) + exec.SetInheritedFields(flowFile) executables = append(executables, exec) } diff --git a/internal/cache/executables_cache.go b/internal/cache/executables_cache.go index 1ad2174..2612e02 100644 --- a/internal/cache/executables_cache.go +++ b/internal/cache/executables_cache.go @@ -69,14 +69,7 @@ func (c *ExecutableCacheImpl) Update(logger io.Logger) error { //nolint:gocognit } for _, flowFile := range flowFiles { if len(flowFile.FromFile) > 0 { - generated, err := generatedExecutables( - logger, - name, - wsCfg.Location(), - flowFile.Namespace, - flowFile.ConfigPath(), - flowFile.FromFile, - ) + generated, err := generatedExecutables(logger, name, flowFile) if err != nil { logger.Errorx( "failed to generate executables from files", @@ -160,14 +153,7 @@ func (c *ExecutableCacheImpl) GetExecutableByRef(logger io.Logger, ref executabl cfg.SetDefaults() cfg.SetContext(wsInfo.WorkspaceName, wsInfo.WorkspacePath, cfgPath) - generated, err := generatedExecutables( - logger, - wsInfo.WorkspaceName, - wsInfo.WorkspacePath, - cfg.Namespace, - cfg.ConfigPath(), - cfg.FromFile, - ) + generated, err := generatedExecutables(logger, wsInfo.WorkspaceName, cfg) if err != nil { logger.Warnx( "failed to generate executables from files", @@ -213,14 +199,7 @@ func (c *ExecutableCacheImpl) GetExecutableList(logger io.Logger) (executable.Ex cfg.SetDefaults() cfg.SetContext(wsInfo.WorkspaceName, wsInfo.WorkspacePath, cfgPath) - generated, err := generatedExecutables( - logger, - wsInfo.WorkspaceName, - wsInfo.WorkspacePath, - cfg.Namespace, - cfg.ConfigPath(), - cfg.FromFile, - ) + generated, err := generatedExecutables(logger, wsInfo.WorkspaceName, cfg) if err != nil { logger.Warnx( "failed to generate executables from files", diff --git a/schemas/flowfile_schema.json b/schemas/flowfile_schema.json index 763eceb..c4dcdfa 100644 --- a/schemas/flowfile_schema.json +++ b/schemas/flowfile_schema.json @@ -478,7 +478,12 @@ }, "properties": { "description": { - "description": "A description of the executables defined within the flow file. This description will be set as the executables'\ndescription if not defined at the executable level.\n", + "description": "A description of the executables defined within the flow file. This description will used as a shared description\nfor all executables in the flow file.\n", + "type": "string", + "default": "" + }, + "descriptionFile": { + "description": "A path to a markdown file that contains the description of the executables defined within the flow file.", "type": "string", "default": "" }, diff --git a/schemas/workspace_schema.json b/schemas/workspace_schema.json index 1955d23..c39a3c6 100644 --- a/schemas/workspace_schema.json +++ b/schemas/workspace_schema.json @@ -39,6 +39,11 @@ "type": "string", "default": "" }, + "descriptionFile": { + "description": "A path to a markdown file that contains the description of the workspace.", + "type": "string", + "default": "" + }, "displayName": { "description": "The display name of the workspace. This is used in the interactive UI.", "type": "string", diff --git a/types/executable/executable.gen.go b/types/executable/executable.gen.go index bf8aedb..230256b 100644 --- a/types/executable/executable.gen.go +++ b/types/executable/executable.gen.go @@ -108,6 +108,10 @@ type Executable struct { // flowFilePath corresponds to the JSON schema field "flowFilePath". flowFilePath string `json:"flowFilePath,omitempty" yaml:"flowFilePath,omitempty" mapstructure:"flowFilePath,omitempty"` + // inheritedDescription corresponds to the JSON schema field + // "inheritedDescription". + inheritedDescription string `json:"inheritedDescription,omitempty" yaml:"inheritedDescription,omitempty" mapstructure:"inheritedDescription,omitempty"` + // Launch corresponds to the JSON schema field "launch". Launch *LaunchExecutableType `json:"launch,omitempty" yaml:"launch,omitempty" mapstructure:"launch,omitempty"` diff --git a/types/executable/executable.go b/types/executable/executable.go index 6124bc8..cb1c3cd 100644 --- a/types/executable/executable.go +++ b/types/executable/executable.go @@ -75,6 +75,24 @@ func (e *Executable) SetContext(workspaceName, workspacePath, namespace, flowFil e.flowFilePath = flowFilePath } +func (e *Executable) SetInheritedFields(flowFile *FlowFile) { + e.MergeTags(flowFile.Tags) + if e.Visibility == nil && flowFile.Visibility != nil { + v := ExecutableVisibility(*flowFile.Visibility) + e.Visibility = &v + } + var descFromFIle string + if flowFile.DescriptionFile != "" { + mdBytes, err := os.ReadFile(flowFile.DescriptionFile) + if err != nil { + descFromFIle += fmt.Sprintf("**error rendering description file**: %s", err) + } else { + descFromFIle += string(mdBytes) + } + } + e.inheritedDescription = strings.Join([]string{flowFile.Description, descFromFIle}, "\n") +} + func (e *Executable) YAML() (string, error) { enriched := &enrichedExecutable{ ID: e.ID(), diff --git a/types/executable/executable_md.go b/types/executable/executable_md.go index ed2b9be..0e925dd 100644 --- a/types/executable/executable_md.go +++ b/types/executable/executable_md.go @@ -8,14 +8,7 @@ import ( func execMarkdown(e *Executable) string { var mkdwn string mkdwn += fmt.Sprintf("# [Executable] %s\n", e.Ref()) - if e.Description != "" { - mkdwn += "| \n" - lines := strings.Split(e.Description, "\n") - for _, line := range lines { - mkdwn += fmt.Sprintf("| %s\n", line) - } - mkdwn += "| \n\n" - } + mkdwn += execDescriptionMarkdown(e) if e.Visibility != nil { mkdwn += fmt.Sprintf("**Visibility:** %s\n", *e.Visibility) } @@ -43,6 +36,27 @@ func execMarkdown(e *Executable) string { return mkdwn } +func execDescriptionMarkdown(e *Executable) string { + var mkdwn string + const descSpacer = "| \n" + if e.Description != "" { + mkdwn += descSpacer + lines := strings.Split(e.Description, "\n") + for _, line := range lines { + mkdwn += fmt.Sprintf("| %s\n", line) + } + mkdwn += descSpacer + } + if e.inheritedDescription != "" { + for _, line := range strings.Split(e.inheritedDescription, "\n") { + mkdwn += fmt.Sprintf("| %s\n", line) + } + mkdwn += descSpacer + } + mkdwn += "\n" + return mkdwn +} + func execTypeMarkdown(spec *Executable) string { var mkdwn string switch { diff --git a/types/executable/executable_schema.yaml b/types/executable/executable_schema.yaml index 3ccbf01..d7ddf18 100644 --- a/types/executable/executable_schema.yaml +++ b/types/executable/executable_schema.yaml @@ -389,6 +389,11 @@ properties: default: "" goJSONSchema: identifier: flowFilePath + inheritedDescription: + type: string + default: "" + goJSONSchema: + identifier: inheritedDescription #### Executable runner type fields #### go-jsonschema does not support oneOf, so we need to define the types separately and validate them in go. exec: diff --git a/types/executable/flowfile.gen.go b/types/executable/flowfile.gen.go index c4391b1..43a6507 100644 --- a/types/executable/flowfile.gen.go +++ b/types/executable/flowfile.gen.go @@ -15,11 +15,15 @@ type FlowFile struct { configPath string `json:"configPath,omitempty" yaml:"configPath,omitempty" mapstructure:"configPath,omitempty"` // A description of the executables defined within the flow file. This description - // will be set as the executables' - // description if not defined at the executable level. + // will used as a shared description + // for all executables in the flow file. // Description string `json:"description,omitempty" yaml:"description,omitempty" mapstructure:"description,omitempty"` + // A path to a markdown file that contains the description of the executables + // defined within the flow file. + DescriptionFile string `json:"descriptionFile,omitempty" yaml:"descriptionFile,omitempty" mapstructure:"descriptionFile,omitempty"` + // Executables corresponds to the JSON schema field "executables". Executables ExecutableList `json:"executables,omitempty" yaml:"executables,omitempty" mapstructure:"executables,omitempty"` diff --git a/types/executable/flowfile.go b/types/executable/flowfile.go index 473773e..fc6f6e2 100644 --- a/types/executable/flowfile.go +++ b/types/executable/flowfile.go @@ -22,7 +22,7 @@ func (f *FlowFile) SetContext(workspaceName, workspacePath, configPath string) { exec.Visibility = &v } exec.SetDefaults() - exec.MergeTags(f.Tags) + exec.SetInheritedFields(f) } } diff --git a/types/executable/flowfile_schema.yaml b/types/executable/flowfile_schema.yaml index baebdf7..41d3aad 100644 --- a/types/executable/flowfile_schema.yaml +++ b/types/executable/flowfile_schema.yaml @@ -50,8 +50,12 @@ properties: description: type: string description: | - A description of the executables defined within the flow file. This description will be set as the executables' - description if not defined at the executable level. + A description of the executables defined within the flow file. This description will used as a shared description + for all executables in the flow file. + default: "" + descriptionFile: + type: string + description: A path to a markdown file that contains the description of the executables defined within the flow file. default: "" #### Executable config context fields workspaceName: diff --git a/types/workspace/schema.yaml b/types/workspace/schema.yaml index 691b376..1a7594b 100644 --- a/types/workspace/schema.yaml +++ b/types/workspace/schema.yaml @@ -42,6 +42,10 @@ properties: type: string description: A description of the workspace. This description is rendered as markdown in the interactive UI. default: "" + descriptionFile: + type: string + description: A path to a markdown file that contains the description of the workspace. + default: "" assignedName: type: string goJSONSchema: diff --git a/types/workspace/workspace.gen.go b/types/workspace/workspace.gen.go index 6c79bce..c05f18a 100644 --- a/types/workspace/workspace.gen.go +++ b/types/workspace/workspace.gen.go @@ -24,6 +24,9 @@ type Workspace struct { // interactive UI. Description string `json:"description,omitempty" yaml:"description,omitempty" mapstructure:"description,omitempty"` + // A path to a markdown file that contains the description of the workspace. + DescriptionFile string `json:"descriptionFile,omitempty" yaml:"descriptionFile,omitempty" mapstructure:"descriptionFile,omitempty"` + // The display name of the workspace. This is used in the interactive UI. DisplayName string `json:"displayName,omitempty" yaml:"displayName,omitempty" mapstructure:"displayName,omitempty"` diff --git a/types/workspace/workspace.go b/types/workspace/workspace.go index 00aaead..a815253 100644 --- a/types/workspace/workspace.go +++ b/types/workspace/workspace.go @@ -49,32 +49,7 @@ func (w *Workspace) JSON() (string, error) { } func (w *Workspace) Markdown() string { - var mkdwn string - if w.DisplayName != "" { - mkdwn = fmt.Sprintf("# [Workspace] %s\n", w.DisplayName) - } else { - mkdwn = fmt.Sprintf("# [Workspace] %s\n", w.AssignedName()) - } - - mkdwn += fmt.Sprintf("## Location\n%s\n", w.Location()) - if w.Description != "" { - mkdwn += fmt.Sprintf("## Description\n%s\n", w.Description) - } - if w.Tags != nil && len(w.Tags) > 0 { - mkdwn += "## Tags\n" - for _, tag := range w.Tags { - mkdwn += fmt.Sprintf("- %s\n", tag) - } - } - if w.Executables != nil { - execs, err := yaml.Marshal(w.Executables) - if err != nil { - mkdwn += "## Executables\nerror\n" - } else { - mkdwn += fmt.Sprintf("## Executables\n```yaml\n%s```\n", string(execs)) - } - } - return mkdwn + return workspaceMarkdown(w) } func DefaultWorkspaceConfig(name string) *Workspace { diff --git a/types/workspace/workspace_md.go b/types/workspace/workspace_md.go new file mode 100644 index 0000000..696f1a3 --- /dev/null +++ b/types/workspace/workspace_md.go @@ -0,0 +1,68 @@ +package workspace + +import ( + "fmt" + "os" + "path/filepath" + "strings" +) + +func workspaceMarkdown(w *Workspace) string { + var mkdwn string + if w.DisplayName != "" { + mkdwn = fmt.Sprintf("# [Workspace] %s\n", w.DisplayName) + } else { + mkdwn = fmt.Sprintf("# [Workspace] %s\n", w.AssignedName()) + } + mkdwn += workspaceDescription(w) + if len(w.Tags) > 0 { + mkdwn += "**Tags**\n" + for _, tag := range w.Tags { + mkdwn += fmt.Sprintf("- %s\n", tag) + } + } + if w.Executables != nil { + mkdwn += "**Executable Filter**\n" + if len(w.Executables.Included) > 0 { + mkdwn += "Included\n" + for _, line := range w.Executables.Included { + mkdwn += fmt.Sprintf(" %s\n", line) + } + } + if len(w.Executables.Excluded) > 0 { + mkdwn += "Excluded\n" + for _, line := range w.Executables.Excluded { + mkdwn += fmt.Sprintf(" %s\n", line) + } + } + } + mkdwn += fmt.Sprintf("\n\n_Workspace can be found in_ [%s](%s)\n", w.Location(), w.Location()) + return mkdwn +} + +func workspaceDescription(w *Workspace) string { + var mkdwn string + const descSpacer = "| \n" + if w.Description != "" { + mkdwn += descSpacer + lines := strings.Split(w.Description, "\n") + for _, line := range lines { + mkdwn += fmt.Sprintf("| %s\n", line) + } + mkdwn += descSpacer + } + if w.DescriptionFile != "" { + mdBytes, err := os.ReadFile(filepath.Clean(w.DescriptionFile)) + if err != nil { + mkdwn += fmt.Sprintf("| **error rendering description file**: %s\n", err) + } else { + lines := strings.Split(string(mdBytes), "\n") + for _, line := range lines { + mkdwn += fmt.Sprintf("| %s\n", line) + } + } + mkdwn += descSpacer + } + mkdwn += "\n" + return mkdwn +}