(feat) read flowfile and ws description from file (#140)

docs/types/flowfile.md

@@ -14,7 +14,8 @@ A workspace can have multiple flow files located anywhere in the workspace direc
 
 | Field | Description | Type | Default | Required |
 | ----- | ----------- | ---- | ------- | -------- |
-| `description` | A description of the executables defined within the flow file. This description will be set as the executables' description if not defined at the executable level.  | `string` |  | [] |
+| `description` | A description of the executables defined within the flow file. This description will used as a shared description for all executables in the flow file.  | `string` |  | [] |
+| `descriptionFile` | A path to a markdown file that contains the description of the executables defined within the flow file. | `string` |  | [] |
 | `executables` |  | `array` ([Executable](#Executable)) | [] | [] |
 | `fromFile` |  | [FromFile](#FromFile) | [] | [] |
 | `namespace` | The namespace to be given to all executables in the flow file. If not set, the executables in the file will be grouped into the root (*) namespace.  Namespaces can be reused across multiple flow files.  Namespaces are used to reference executables in the CLI using the format `workspace:namespace/name`.  | `string` |  | [] |

docs/types/workspace.md

@@ -15,6 +15,7 @@ Every workspace has a workspace config file named `flow.yaml` in the root of the
 | Field | Description | Type | Default | Required |
 | ----- | ----------- | ---- | ------- | -------- |
 | `description` | A description of the workspace. This description is rendered as markdown in the interactive UI. | `string` |  | [] |
+| `descriptionFile` | A path to a markdown file that contains the description of the workspace. | `string` |  | [] |
 | `displayName` | The display name of the workspace. This is used in the interactive UI. | `string` |  | [] |
 | `executables` |  | [ExecutableFilter](#ExecutableFilter) | <no value> | [] |
 | `tags` |  | [CommonTags](#CommonTags) | [] | [] |

flow.yaml

@@ -1,4 +1,3 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/jahvon/flow/HEAD/schemas/workspace_schema.json
-displayName: flow
-git:
-    enabled: false
+displayName: flow repository
+descriptionFile: README.md

internal/cache/executable_generator.go

@@ -16,18 +16,22 @@ import (
 const generatedTag = "generated"
 
 func generatedExecutables(
-	logger io.Logger,
-	wsName, wsPath, flowFileNs, flowFilePath string,
-	files []string,
+	logger io.Logger, wsName string, flowFile *executable.FlowFile,
 ) (executable.ExecutableList, error) {
 	executables := make(executable.ExecutableList, 0)
+	wsPath := flowFile.WorkspacePath()
+	flowFilePath := flowFile.ConfigPath()
+	flowFileNs := flowFile.Namespace
+	files := flowFile.FromFile
+
 	for _, file := range files {
 		expandedFile := utils.ExpandDirectory(logger, file, wsPath, flowFilePath, nil)
 		exec, err := executablesFromFile(logger, file, expandedFile)
 		if err != nil {
 			return nil, err
 		}
 		exec.SetContext(wsName, wsPath, flowFileNs, flowFilePath)
+		exec.SetInheritedFields(flowFile)
 		executables = append(executables, exec)
 	}
 

internal/cache/executables_cache.go

@@ -69,14 +69,7 @@ func (c *ExecutableCacheImpl) Update(logger io.Logger) error { //nolint:gocognit
 		}
 		for _, flowFile := range flowFiles {
 			if len(flowFile.FromFile) > 0 {
-				generated, err := generatedExecutables(
-					logger,
-					name,
-					wsCfg.Location(),
-					flowFile.Namespace,
-					flowFile.ConfigPath(),
-					flowFile.FromFile,
-				)
+				generated, err := generatedExecutables(logger, name, flowFile)
 				if err != nil {
 					logger.Errorx(
 						"failed to generate executables from files",
@@ -160,14 +153,7 @@ func (c *ExecutableCacheImpl) GetExecutableByRef(logger io.Logger, ref executabl
 	cfg.SetDefaults()
 	cfg.SetContext(wsInfo.WorkspaceName, wsInfo.WorkspacePath, cfgPath)
 
-	generated, err := generatedExecutables(
-		logger,
-		wsInfo.WorkspaceName,
-		wsInfo.WorkspacePath,
-		cfg.Namespace,
-		cfg.ConfigPath(),
-		cfg.FromFile,
-	)
+	generated, err := generatedExecutables(logger, wsInfo.WorkspaceName, cfg)
 	if err != nil {
 		logger.Warnx(
 			"failed to generate executables from files",
@@ -213,14 +199,7 @@ func (c *ExecutableCacheImpl) GetExecutableList(logger io.Logger) (executable.Ex
 		cfg.SetDefaults()
 		cfg.SetContext(wsInfo.WorkspaceName, wsInfo.WorkspacePath, cfgPath)
 
-		generated, err := generatedExecutables(
-			logger,
-			wsInfo.WorkspaceName,
-			wsInfo.WorkspacePath,
-			cfg.Namespace,
-			cfg.ConfigPath(),
-			cfg.FromFile,
-		)
+		generated, err := generatedExecutables(logger, wsInfo.WorkspaceName, cfg)
 		if err != nil {
 			logger.Warnx(
 				"failed to generate executables from files",

schemas/flowfile_schema.json

@@ -478,7 +478,12 @@
   },
   "properties": {
     "description": {
-      "description": "A description of the executables defined within the flow file. This description will be set as the executables'\ndescription if not defined at the executable level.\n",
+      "description": "A description of the executables defined within the flow file. This description will used as a shared description\nfor all executables in the flow file.\n",
+      "type": "string",
+      "default": ""
+    },
+    "descriptionFile": {
+      "description": "A path to a markdown file that contains the description of the executables defined within the flow file.",
       "type": "string",
       "default": ""
     },

schemas/workspace_schema.json

@@ -39,6 +39,11 @@
       "type": "string",
       "default": ""
     },
+    "descriptionFile": {
+      "description": "A path to a markdown file that contains the description of the workspace.",
+      "type": "string",
+      "default": ""
+    },
     "displayName": {
       "description": "The display name of the workspace. This is used in the interactive UI.",
       "type": "string",

types/executable/executable.go

@@ -75,6 +75,24 @@ func (e *Executable) SetContext(workspaceName, workspacePath, namespace, flowFil
 	e.flowFilePath = flowFilePath
 }
 
+func (e *Executable) SetInheritedFields(flowFile *FlowFile) {
+	e.MergeTags(flowFile.Tags)
+	if e.Visibility == nil && flowFile.Visibility != nil {
+		v := ExecutableVisibility(*flowFile.Visibility)
+		e.Visibility = &v
+	}
+	var descFromFIle string
+	if flowFile.DescriptionFile != "" {
+		mdBytes, err := os.ReadFile(flowFile.DescriptionFile)
+		if err != nil {
+			descFromFIle += fmt.Sprintf("**error rendering description file**: %s", err)
+		} else {
+			descFromFIle += string(mdBytes)
+		}
+	}
+	e.inheritedDescription = strings.Join([]string{flowFile.Description, descFromFIle}, "\n")
+}
+
 func (e *Executable) YAML() (string, error) {
 	enriched := &enrichedExecutable{
 		ID:   e.ID(),

types/executable/executable_md.go

@@ -8,14 +8,7 @@ import (
 func execMarkdown(e *Executable) string {
 	var mkdwn string
 	mkdwn += fmt.Sprintf("# [Executable] %s\n", e.Ref())
-	if e.Description != "" {
-		mkdwn += "| \n"
-		lines := strings.Split(e.Description, "\n")
-		for _, line := range lines {
-			mkdwn += fmt.Sprintf("| %s\n", line)
-		}
-		mkdwn += "| \n\n"
-	}
+	mkdwn += execDescriptionMarkdown(e)
 	if e.Visibility != nil {
 		mkdwn += fmt.Sprintf("**Visibility:** %s\n", *e.Visibility)
 	}
@@ -43,6 +36,27 @@ func execMarkdown(e *Executable) string {
 	return mkdwn
 }
 
+func execDescriptionMarkdown(e *Executable) string {
+	var mkdwn string
+	const descSpacer = "| \n"
+	if e.Description != "" {
+		mkdwn += descSpacer
+		lines := strings.Split(e.Description, "\n")
+		for _, line := range lines {
+			mkdwn += fmt.Sprintf("| %s\n", line)
+		}
+		mkdwn += descSpacer
+	}
+	if e.inheritedDescription != "" {
+		for _, line := range strings.Split(e.inheritedDescription, "\n") {
+			mkdwn += fmt.Sprintf("| %s\n", line)
+		}
+		mkdwn += descSpacer
+	}
+	mkdwn += "\n"
+	return mkdwn
+}
+
 func execTypeMarkdown(spec *Executable) string {
 	var mkdwn string
 	switch {

types/executable/executable_schema.yaml

@@ -389,6 +389,11 @@ properties:
     default: ""
     goJSONSchema:
       identifier: flowFilePath
+  inheritedDescription:
+    type: string
+    default: ""
+    goJSONSchema:
+      identifier: inheritedDescription
   #### Executable runner type fields
   #### go-jsonschema does not support oneOf, so we need to define the types separately and validate them in go.
   exec:

types/executable/flowfile.go

@@ -22,7 +22,7 @@ func (f *FlowFile) SetContext(workspaceName, workspacePath, configPath string) {
 			exec.Visibility = &v
 		}
 		exec.SetDefaults()
-		exec.MergeTags(f.Tags)
+		exec.SetInheritedFields(f)
 	}
 }
 

types/executable/flowfile_schema.yaml

@@ -50,8 +50,12 @@ properties:
   description:
     type: string
     description: |
-      A description of the executables defined within the flow file. This description will be set as the executables'
-      description if not defined at the executable level.
+      A description of the executables defined within the flow file. This description will used as a shared description
+      for all executables in the flow file.
+    default: ""
+  descriptionFile:
+    type: string
+    description: A path to a markdown file that contains the description of the executables defined within the flow file.
     default: ""
   #### Executable config context fields
   workspaceName:

types/workspace/schema.yaml

@@ -42,6 +42,10 @@ properties:
     type: string
     description: A description of the workspace. This description is rendered as markdown in the interactive UI.
     default: ""
+  descriptionFile:
+    type: string
+    description: A path to a markdown file that contains the description of the workspace.
+    default: ""
   assignedName:
     type: string
     goJSONSchema:

types/workspace/workspace.go

@@ -49,32 +49,7 @@ func (w *Workspace) JSON() (string, error) {
 }
 
 func (w *Workspace) Markdown() string {
-	var mkdwn string
-	if w.DisplayName != "" {
-		mkdwn = fmt.Sprintf("# [Workspace] %s\n", w.DisplayName)
-	} else {
-		mkdwn = fmt.Sprintf("# [Workspace] %s\n", w.AssignedName())
-	}
-
-	mkdwn += fmt.Sprintf("## Location\n%s\n", w.Location())
-	if w.Description != "" {
-		mkdwn += fmt.Sprintf("## Description\n%s\n", w.Description)
-	}
-	if w.Tags != nil && len(w.Tags) > 0 {
-		mkdwn += "## Tags\n"
-		for _, tag := range w.Tags {
-			mkdwn += fmt.Sprintf("- %s\n", tag)
-		}
-	}
-	if w.Executables != nil {
-		execs, err := yaml.Marshal(w.Executables)
-		if err != nil {
-			mkdwn += "## Executables\nerror\n"
-		} else {
-			mkdwn += fmt.Sprintf("## Executables\n```yaml\n%s```\n", string(execs))
-		}
-	}
-	return mkdwn
+	return workspaceMarkdown(w)
 }
 
 func DefaultWorkspaceConfig(name string) *Workspace {

types/workspace/workspace_md.go

@@ -0,0 +1,68 @@
+package workspace
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+func workspaceMarkdown(w *Workspace) string {
+	var mkdwn string
+	if w.DisplayName != "" {
+		mkdwn = fmt.Sprintf("# [Workspace] %s\n", w.DisplayName)
+	} else {
+		mkdwn = fmt.Sprintf("# [Workspace] %s\n", w.AssignedName())
+	}
+	mkdwn += workspaceDescription(w)
+	if len(w.Tags) > 0 {
+		mkdwn += "**Tags**\n"
+		for _, tag := range w.Tags {
+			mkdwn += fmt.Sprintf("- %s\n", tag)
+		}
+	}
+	if w.Executables != nil {
+		mkdwn += "**Executable Filter**\n"
+		if len(w.Executables.Included) > 0 {
+			mkdwn += "Included\n"
+			for _, line := range w.Executables.Included {
+				mkdwn += fmt.Sprintf("  %s\n", line)
+			}
+		}
+		if len(w.Executables.Excluded) > 0 {
+			mkdwn += "Excluded\n"
+			for _, line := range w.Executables.Excluded {
+				mkdwn += fmt.Sprintf("  %s\n", line)
+			}
+		}
+	}
+	mkdwn += fmt.Sprintf("\n\n_Workspace can be found in_ [%s](%s)\n", w.Location(), w.Location())
+	return mkdwn
+}
+
+func workspaceDescription(w *Workspace) string {
+	var mkdwn string
+	const descSpacer = "| \n"
+	if w.Description != "" {
+		mkdwn += descSpacer
+		lines := strings.Split(w.Description, "\n")
+		for _, line := range lines {
+			mkdwn += fmt.Sprintf("| %s\n", line)
+		}
+		mkdwn += descSpacer
+	}
+	if w.DescriptionFile != "" {
+		mdBytes, err := os.ReadFile(filepath.Clean(w.DescriptionFile))
+		if err != nil {
+			mkdwn += fmt.Sprintf("| **error rendering description file**: %s\n", err)
+		} else {
+			lines := strings.Split(string(mdBytes), "\n")
+			for _, line := range lines {
+				mkdwn += fmt.Sprintf("| %s\n", line)
+			}
+		}
+		mkdwn += descSpacer
+	}
+	mkdwn += "\n"
+	return mkdwn
+}