(merge) Recursively copy content/ dir and add goldmark extensions

cmd/anna/anna.go

@@ -37,11 +37,10 @@ func (cmd *Cmd) VanillaRender() {
 	e.DeepDataMerge.LinkStore = make(map[template.URL][]*parser.Note, 10)
 
 	helper := helpers.Helper{
-		ErrorLogger:  e.ErrorLogger,
-		SiteDataPath: helpers.SiteDataPath,
+		ErrorLogger: e.ErrorLogger,
 	}
 
-	helper.CreateRenderedDir(helper.SiteDataPath)
+	helper.CreateRenderedDir(helpers.SiteDataPath)
 
 	p.ParseConfig(helpers.SiteDataPath + "layout/config.yml")
 	p.ParseRobots(helpers.SiteDataPath+"layout/robots.txt", helpers.SiteDataPath+"rendered/robots.txt")
@@ -72,7 +71,7 @@ func (cmd *Cmd) VanillaRender() {
 	e.GenerateJSONIndex(helpers.SiteDataPath)
 
 	e.GenerateLinkStore()
-	e.GenerateNoteJSONIdex(helper.SiteDataPath)
+	e.GenerateNoteJSONIdex(helpers.SiteDataPath)
 
 	templ, err := template.ParseGlob(helpers.SiteDataPath + "layout/*.html")
 	if err != nil {

go.mod

@@ -5,8 +5,12 @@ go 1.22.2
 require (
 	github.com/PuerkitoBio/goquery v1.9.1
 	github.com/cheggaaa/pb/v3 v3.1.5
+	github.com/mangoumbrella/goldmark-figure v1.0.0
 	github.com/spf13/cobra v1.8.0
 	github.com/yuin/goldmark v1.7.1
+	go.abhg.dev/goldmark/anchor v0.1.1
+	go.abhg.dev/goldmark/mermaid v0.5.0
+	go.abhg.dev/goldmark/toc v0.10.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 

go.sum

@@ -6,19 +6,39 @@ github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsVi
 github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
 github.com/cheggaaa/pb/v3 v3.1.5 h1:QuuUzeM2WsAqG2gMqtzaWithDJv0i+i6UlnwSCI4QLk=
 github.com/cheggaaa/pb/v3 v3.1.5/go.mod h1:CrxkeghYTXi1lQBEI7jSn+3svI3cuc19haAj6jM60XI=
+github.com/chromedp/cdproto v0.0.0-20230220211738-2b1ec77315c9 h1:wMSvdj3BswqfQOXp2R1bJOAE7xIQLt2dlMQDMf836VY=
+github.com/chromedp/cdproto v0.0.0-20230220211738-2b1ec77315c9/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs=
+github.com/chromedp/chromedp v0.9.1 h1:CC7cC5p1BeLiiS2gfNNPwp3OaUxtRMBjfiw3E3k6dFA=
+github.com/chromedp/chromedp v0.9.1/go.mod h1:DUgZWRvYoEfgi66CgZ/9Yv+psgi+Sksy5DTScENWjaQ=
+github.com/chromedp/sysutil v1.0.0 h1:+ZxhTpfpZlmchB58ih/LBHX52ky7w2VhQVKQMucy3Ic=
+github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww=
 github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
 github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
+github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU=
+github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM=
+github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og=
+github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
+github.com/gobwas/ws v1.1.0 h1:7RFti/xnNkMJnrK7D1yQ/iCIB5OrrY/54/H930kIbHA=
+github.com/gobwas/ws v1.1.0/go.mod h1:nzvNcVha5eUziGrbxFCo6qFIojQHjJV5cLYIbezhfL0=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
+github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/mangoumbrella/goldmark-figure v1.0.0 h1:L+ebP73dET0a2n68PV7m5oFJmwwH5GzRIJLgszR3fwo=
+github.com/mangoumbrella/goldmark-figure v1.0.0/go.mod h1:iIL+fhdmCQDpE0l/TKtGhokWzIbo5lo/Y2OIAcx6usI=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
@@ -27,6 +47,8 @@ github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D
 github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
 github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
 github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
@@ -38,9 +60,17 @@ github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
 github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
+github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 github.com/yuin/goldmark v1.7.1 h1:3bajkSilaCbjdKVsKdZjZCLBNPL9pYzrCakKaf4U49U=
 github.com/yuin/goldmark v1.7.1/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E=
+go.abhg.dev/goldmark/anchor v0.1.1 h1:NUH3hAzhfeymRqZKOkSoFReZlEAmfXBZlbXEzpD2Qgc=
+go.abhg.dev/goldmark/anchor v0.1.1/go.mod h1:zYKiaHXTdugwVJRZqInVdmNGQRM3ZRJ6AGBC7xP7its=
+go.abhg.dev/goldmark/mermaid v0.5.0 h1:mDkykpSPJ+5wCQ8bSXgzJ2KQskjXkI5Ndxz7JYDHW38=
+go.abhg.dev/goldmark/mermaid v0.5.0/go.mod h1:OCyk2o85TX2drWHH+HRy6bih2yZlUwbbv/R1MMh1YLs=
+go.abhg.dev/goldmark/toc v0.10.0 h1:de3LrIimwtGhBMKh7aEl1c6n4XWwOdukIO5wOAMYZzg=
+go.abhg.dev/goldmark/toc v0.10.0/go.mod h1:OpH0qqRP9v/eosCV28ZeqGI78jZ8rri3C7Jh8fzEo2M=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
@@ -85,3 +115,5 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+pgregory.net/rapid v1.1.0 h1:CMa0sjHSru3puNx+J0MIAuiiEV4N0qj8/cMWGBBCsjw=
+pgregory.net/rapid v1.1.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04=

pkg/helpers/helper.go

@@ -8,6 +8,7 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
+	"strings"
 
 	"github.com/cheggaaa/pb/v3"
 )
@@ -17,8 +18,7 @@ const SiteDataPath string = "site/"
 var version string = "2.0.0" // use variable
 
 type Helper struct {
-	ErrorLogger  *log.Logger
-	SiteDataPath string
+	ErrorLogger *log.Logger
 }
 
 // Copies the contents of the dirPath directory to outDirPath
@@ -50,6 +50,19 @@ func (h *Helper) CopyFiles(srcPath string, destPath string) {
 	}
 	defer source.Close()
 
+	// Creating subdirectories if the filepath contains '/'
+	if strings.Contains(string(destPath), "/") {
+		// Extracting the directory path from the page path
+		splitPaths := strings.Split(string(destPath), "/")
+		filename := splitPaths[len(splitPaths)-1]
+		pagePathWithoutFilename, _ := strings.CutSuffix(string(destPath), filename)
+
+		err := os.MkdirAll(pagePathWithoutFilename, 0750)
+		if err != nil {
+			h.ErrorLogger.Fatal(err)
+		}
+	}
+
 	destination, err := os.Create(destPath)
 	if err != nil {
 		h.ErrorLogger.Fatal(err)

pkg/parser/parser.go

@@ -12,9 +12,14 @@ import (
 	"time"
 
 	"github.com/acmpesuecc/anna/pkg/helpers"
+	figure "github.com/mangoumbrella/goldmark-figure"
 	"github.com/yuin/goldmark"
 	"github.com/yuin/goldmark/extension"
+	"github.com/yuin/goldmark/parser"
 	"github.com/yuin/goldmark/renderer/html"
+	"go.abhg.dev/goldmark/anchor"
+	"go.abhg.dev/goldmark/mermaid"
+	"go.abhg.dev/goldmark/toc"
 	"gopkg.in/yaml.v3"
 )
 
@@ -88,21 +93,17 @@ type Parser struct {
 }
 
 func (p *Parser) ParseMDDir(baseDirPath string, baseDirFS fs.FS) {
+	helper := helpers.Helper{
+		ErrorLogger: p.ErrorLogger,
+	}
 	fs.WalkDir(baseDirFS, ".", func(path string, dir fs.DirEntry, err error) error {
 		if path != "." && path != ".obsidian" {
 			if dir.IsDir() {
 				subDir := os.DirFS(path)
 				p.ParseMDDir(path, subDir)
 			} else {
+				fileName := strings.TrimPrefix(path, baseDirPath)
 				if filepath.Ext(path) == ".md" {
-					// OLD IMPL
-					// fileName := filepath.Base(path)
-					//
-					// NEW IMPL
-					// /contents/notes/2134321.md ==> notes/2134321.md
-					fileName := strings.TrimPrefix(path, baseDirPath)
-					// fmt.Println(fileNameWithPath, fileName)
-
 					content, err := os.ReadFile(baseDirPath + path)
 					if err != nil {
 						p.ErrorLogger.Fatal(err)
@@ -118,7 +119,8 @@ func (p *Parser) ParseMDDir(baseDirPath string, baseDirFS fs.FS) {
 							p.AddFile(baseDirPath, fileName, fronmatter, markdownContent, body)
 						}
 					}
-
+				} else {
+					helper.CopyFiles(helpers.SiteDataPath+"content/"+fileName, helpers.SiteDataPath+"rendered/"+fileName)
 				}
 			}
 		}
@@ -128,7 +130,6 @@ func (p *Parser) ParseMDDir(baseDirPath string, baseDirFS fs.FS) {
 
 func (p *Parser) AddFile(baseDirPath string, dirEntryPath string, frontmatter Frontmatter, markdownContent string, body string) {
 	p.MdFilesName = append(p.MdFilesName, dirEntryPath)
-	// fmt.Println(baseDirPath, dirEntryPath)
 	filepath := baseDirPath + dirEntryPath
 	p.MdFilesPath = append(p.MdFilesPath, filepath)
 
@@ -155,7 +156,6 @@ func (p *Parser) AddFile(baseDirPath string, dirEntryPath string, frontmatter Fr
 
 		// Adding the page to the merged map storing all site pages
 		if frontmatter.Type == "post" {
-			// url = "posts/" + url
 			p.Posts = append(p.Posts, page)
 		}
 
@@ -167,12 +167,9 @@ func (p *Parser) AddFile(baseDirPath string, dirEntryPath string, frontmatter Fr
 			p.TagsMap[template.URL(tagsMapKey)] = append(p.TagsMap[template.URL(tagsMapKey)], page)
 
 		}
-
 	}
 
 	if frontmatter.Type == "note" {
-		// url = "notes/" + url
-
 		markdownContent = strings.TrimFunc(markdownContent, func(r rune) bool {
 			return r == '\n' || r == '\t'
 		})
@@ -220,6 +217,9 @@ func (p *Parser) ParseMarkdownContent(filecontent string) (Frontmatter, string,
 		return Frontmatter{}, "", "", false
 	}
 
+	// If the first section of the page contains a title field, continue parsing
+	// Else, prevent parsing of the current file
+	// TODO: Add this to documentation
 	regex := regexp.MustCompile(`title(.*): (.*)`)
 	match := regex.FindStringSubmatch(splitContents[1])
 
@@ -237,13 +237,44 @@ func (p *Parser) ParseMarkdownContent(filecontent string) (Frontmatter, string,
 
 	// Parsing markdown to HTML
 	var parsedMarkdown bytes.Buffer
+	var md goldmark.Markdown
+
+	if parsedFrontmatter.Type == "post" {
+		md = goldmark.New(
+			goldmark.WithParserOptions(parser.WithAutoHeadingID()),
+			goldmark.WithExtensions(
+				extension.TaskList,
+				figure.Figure,
+				&toc.Extender{
+					Compact: true,
+				},
+				&mermaid.Extender{
+					RenderMode: mermaid.RenderModeClient, // or RenderModeClient
+				},
+				&anchor.Extender{
+					Texter: anchor.Text("#"),
+				},
+			),
+			goldmark.WithRendererOptions(
+				html.WithUnsafe(),
+			),
+		)
+	} else {
+		md = goldmark.New(
+			goldmark.WithParserOptions(parser.WithAutoHeadingID()),
+			goldmark.WithExtensions(
+				extension.TaskList,
+				figure.Figure,
+				&mermaid.Extender{
+					RenderMode: mermaid.RenderModeClient, // or RenderModeClient
+				},
+			),
+			goldmark.WithRendererOptions(
+				html.WithUnsafe(),
+			),
+		)
 
-	md := goldmark.New(
-		goldmark.WithExtensions(extension.TaskList),
-		goldmark.WithRendererOptions(
-			html.WithUnsafe(),
-		),
-	)
+	}
 
 	if err := md.Convert([]byte(markdown), &parsedMarkdown); err != nil {
 		p.ErrorLogger.Fatal(err)

site/content/docs-update.md

@@ -0,0 +1,16 @@
+---
+date: 2024-04-22
+title: Anna Documentation Updates
+type: page
+---
+
+- we are common mark compliant
+- page and posts field to frontmatter
+- recursively copies contents/ dir, so images and static content can be added in sub-dirs
+- goldmark extension details: mermaid, toc, figure(Add caption beloe link, etc)
+
+## developer guide
+
+- profiling:
+  - live pprof during live reload
+  - creating .pprof files