From 488ed275db283f20bd7af12cee158d7f27e053ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Thu, 27 Jun 2024 16:22:35 +0200 Subject: [PATCH] Add js.Batch Fixes #12626 Closes #7499 Closes #12874 --- common/herrors/errors.go | 15 + common/maps/scratch.go | 14 + debug.log | 0 hugolib/integrationtest_builder.go | 16 +- hugolib/pages_capture.go | 10 +- hugolib/rebuild_test.go | 12 + hugolib/site.go | 6 +- lazy/init.go | 2 +- lazy/once.go | 10 +- media/mediaType.go | 12 +- resources/page/page.go | 3 +- resources/resource.go | 6 + resources/resource/resources.go | 157 ++- resources/resource/resources_test.go | 125 ++ resources/resource/resourcetypes.go | 15 + resources/resource_transformers/js/build.go | 188 +-- resources/resource_transformers/js/options.go | 172 ++- .../resource_transformers/js/options_test.go | 102 +- .../resource_transformers/js/transform.go | 67 + resources/transform.go | 6 + .../go_templates/texttemplate/exec.go | 5 +- .../texttemplate/hugo_template.go | 34 +- tpl/js/batch-esm-callback.gotmpl | 17 + tpl/js/batch.go | 1105 +++++++++++++++++ tpl/js/batch_integration_test.go | 432 +++++++ tpl/js/js.go | 23 +- 26 files changed, 2352 insertions(+), 202 deletions(-) create mode 100644 debug.log create mode 100644 resources/resource/resources_test.go create mode 100644 resources/resource_transformers/js/transform.go create mode 100644 tpl/js/batch-esm-callback.gotmpl create mode 100644 tpl/js/batch.go create mode 100644 tpl/js/batch_integration_test.go diff --git a/common/herrors/errors.go b/common/herrors/errors.go index e7f91462e31..67ca1d1d4da 100644 --- a/common/herrors/errors.go +++ b/common/herrors/errors.go @@ -133,6 +133,21 @@ func IsNotExist(err error) bool { return false } +// IsExist returns true if the error is a file exists error. +// Unlike os.IsExist, this also considers wrapped errors. +func IsExist(err error) bool { + if os.IsExist(err) { + return true + } + + // os.IsExist does not consider wrapped errors. + if os.IsExist(errors.Unwrap(err)) { + return true + } + + return false +} + var nilPointerErrRe = regexp.MustCompile(`at <(.*)>: error calling (.*?): runtime error: invalid memory address or nil pointer dereference`) const deferredPrefix = "__hdeferred/" diff --git a/common/maps/scratch.go b/common/maps/scratch.go index e9f412540b2..3bb160ae037 100644 --- a/common/maps/scratch.go +++ b/common/maps/scratch.go @@ -107,6 +107,20 @@ func (c *Scratch) Get(key string) any { return val } +// GetOrCreate returns the value for the given key if it exists, or creates it +// using the given func and stores that value in the map. +// For internal use. +func (c *Scratch) GetOrCreate(key string, create func() any) any { + c.mu.Lock() + defer c.mu.Unlock() + if val, found := c.values[key]; found { + return val + } + val := create() + c.values[key] = val + return val +} + // Values returns the raw backing map. Note that you should just use // this method on the locally scoped Scratch instances you obtain via newScratch, not // .Page.Scratch etc., as that will lead to concurrency issues. diff --git a/debug.log b/debug.log new file mode 100644 index 00000000000..e69de29bb2d diff --git a/hugolib/integrationtest_builder.go b/hugolib/integrationtest_builder.go index 551b807dba0..a1d9c385ce7 100644 --- a/hugolib/integrationtest_builder.go +++ b/hugolib/integrationtest_builder.go @@ -81,6 +81,13 @@ func TestOptWithNFDOnDarwin() TestOpt { } } +// TestOptWithOSFs enables the real file system. +func TestOptWithOSFs() TestOpt { + return func(c *IntegrationTestConfig) { + c.NeedsOsFS = true + } +} + // TestOptWithWorkingDir allows setting any config optiona as a function al option. func TestOptWithConfig(fn func(c *IntegrationTestConfig)) TestOpt { return func(c *IntegrationTestConfig) { @@ -280,8 +287,9 @@ func (s *IntegrationTestBuilder) negate(match string) (string, bool) { func (s *IntegrationTestBuilder) AssertFileContent(filename string, matches ...string) { s.Helper() content := strings.TrimSpace(s.FileContent(filename)) + for _, m := range matches { - cm := qt.Commentf("File: %s Match %s", filename, m) + cm := qt.Commentf("File: %s Match %s\nContent:\n%s", filename, m, content) lines := strings.Split(m, "\n") for _, match := range lines { match = strings.TrimSpace(match) @@ -291,7 +299,8 @@ func (s *IntegrationTestBuilder) AssertFileContent(filename string, matches ...s var negate bool match, negate = s.negate(match) if negate { - s.Assert(content, qt.Not(qt.Contains), match, cm) + if !s.Assert(content, qt.Not(qt.Contains), match, cm) { + } continue } s.Assert(content, qt.Contains, match, cm) @@ -303,7 +312,8 @@ func (s *IntegrationTestBuilder) AssertFileContentExact(filename string, matches s.Helper() content := s.FileContent(filename) for _, m := range matches { - s.Assert(content, qt.Contains, m, qt.Commentf(m)) + cm := qt.Commentf("File: %s Match %s\nContent:\n%s", filename, m, content) + s.Assert(content, qt.Contains, m, cm) } } diff --git a/hugolib/pages_capture.go b/hugolib/pages_capture.go index 96c2c0f96e2..3993e7ff33d 100644 --- a/hugolib/pages_capture.go +++ b/hugolib/pages_capture.go @@ -149,7 +149,15 @@ func (c *pagesCollector) Collect() (collectErr error) { id.p, false, func(fim hugofs.FileMetaInfo) bool { - return true + if id.isStructuralChange() { + return true + } + fimp := fim.Meta().PathInfo + if fimp == nil { + return true + } + + return fimp.Path() == id.p.Path() }, ) } else if id.p.IsBranchBundle() { diff --git a/hugolib/rebuild_test.go b/hugolib/rebuild_test.go index 2219fe81226..e3d7c79525b 100644 --- a/hugolib/rebuild_test.go +++ b/hugolib/rebuild_test.go @@ -71,6 +71,18 @@ Foo. ` +func TestRebuildEditLeafBundleHeaderOnly(t *testing.T) { + b := TestRunning(t, rebuildFilesSimple) + b.AssertFileContent("public/mysection/mysectionbundle/index.html", + "My Section Bundle Content Content.") + + b.EditFileReplaceAll("content/mysection/mysectionbundle/index.md", "My Section Bundle Content.", "My Section Bundle Content Edited.").Build() + b.AssertFileContent("public/mysection/mysectionbundle/index.html", + "My Section Bundle Content Edited.") + b.AssertRenderCountPage(1) + b.AssertRenderCountContent(1) +} + func TestRebuildEditTextFileInLeafBundle(t *testing.T) { b := TestRunning(t, rebuildFilesSimple) b.AssertFileContent("public/mysection/mysectionbundle/index.html", diff --git a/hugolib/site.go b/hugolib/site.go index 08031390ba1..242a2b9812c 100644 --- a/hugolib/site.go +++ b/hugolib/site.go @@ -1522,7 +1522,11 @@ func (s *Site) renderForTemplate(ctx context.Context, name, outputFormat string, } if err = s.Tmpl().ExecuteWithContext(ctx, templ, w, d); err != nil { - return fmt.Errorf("render of %q failed: %w", name, err) + filename := name + if p, ok := d.(*pageState); ok { + filename = p.pathOrTitle() + } + return fmt.Errorf("render of %q failed: %w", filename, err) } return } diff --git a/lazy/init.go b/lazy/init.go index 7b88a53518b..bef3867a9a0 100644 --- a/lazy/init.go +++ b/lazy/init.go @@ -36,7 +36,7 @@ type Init struct { prev *Init children []*Init - init onceMore + init OnceMore out any err error f func(context.Context) (any, error) diff --git a/lazy/once.go b/lazy/once.go index c6abcd88493..0ebdb335aa0 100644 --- a/lazy/once.go +++ b/lazy/once.go @@ -24,13 +24,13 @@ import ( // * it can be reset, so the action can be repeated if needed // * it has methods to check if it's done or in progress -type onceMore struct { +type OnceMore struct { mu sync.Mutex lock uint32 done uint32 } -func (t *onceMore) Do(f func()) { +func (t *OnceMore) Do(f func()) { if atomic.LoadUint32(&t.done) == 1 { return } @@ -53,15 +53,15 @@ func (t *onceMore) Do(f func()) { f() } -func (t *onceMore) InProgress() bool { +func (t *OnceMore) InProgress() bool { return atomic.LoadUint32(&t.lock) == 1 } -func (t *onceMore) Done() bool { +func (t *OnceMore) Done() bool { return atomic.LoadUint32(&t.done) == 1 } -func (t *onceMore) ResetWithLock() *sync.Mutex { +func (t *OnceMore) ResetWithLock() *sync.Mutex { t.mu.Lock() defer atomic.StoreUint32(&t.done, 0) return &t.mu diff --git a/media/mediaType.go b/media/mediaType.go index a7ba1309a7d..97b10879c07 100644 --- a/media/mediaType.go +++ b/media/mediaType.go @@ -273,9 +273,13 @@ func (t Types) GetByType(tp string) (Type, bool) { return Type{}, false } +func (t Types) normalizeSuffix(s string) string { + return strings.ToLower(strings.TrimPrefix(s, ".")) +} + // BySuffix will return all media types matching a suffix. func (t Types) BySuffix(suffix string) []Type { - suffix = strings.ToLower(suffix) + suffix = t.normalizeSuffix(suffix) var types []Type for _, tt := range t { if tt.hasSuffix(suffix) { @@ -287,7 +291,7 @@ func (t Types) BySuffix(suffix string) []Type { // GetFirstBySuffix will return the first type matching the given suffix. func (t Types) GetFirstBySuffix(suffix string) (Type, SuffixInfo, bool) { - suffix = strings.ToLower(suffix) + suffix = t.normalizeSuffix(suffix) for _, tt := range t { if tt.hasSuffix(suffix) { return tt, SuffixInfo{ @@ -304,7 +308,7 @@ func (t Types) GetFirstBySuffix(suffix string) (Type, SuffixInfo, bool) { // is ambiguous. // The lookup is case insensitive. func (t Types) GetBySuffix(suffix string) (tp Type, si SuffixInfo, found bool) { - suffix = strings.ToLower(suffix) + suffix = t.normalizeSuffix(suffix) for _, tt := range t { if tt.hasSuffix(suffix) { if found { @@ -324,7 +328,7 @@ func (t Types) GetBySuffix(suffix string) (tp Type, si SuffixInfo, found bool) { } func (t Types) IsTextSuffix(suffix string) bool { - suffix = strings.ToLower(suffix) + suffix = t.normalizeSuffix(suffix) for _, tt := range t { if tt.hasSuffix(suffix) { return tt.IsText() diff --git a/resources/page/page.go b/resources/page/page.go index 4cda8d31fce..40bd0e43e80 100644 --- a/resources/page/page.go +++ b/resources/page/page.go @@ -70,8 +70,7 @@ type ChildCareProvider interface { // section. RegularPagesRecursive() Pages - // Resources returns a list of all resources. - Resources() resource.Resources + resource.ResourcesProvider } type MarkupProvider interface { diff --git a/resources/resource.go b/resources/resource.go index cc7008e5a88..b32cb0baed1 100644 --- a/resources/resource.go +++ b/resources/resource.go @@ -47,6 +47,7 @@ var ( _ resource.Cloner = (*genericResource)(nil) _ resource.ResourcesLanguageMerger = (*resource.Resources)(nil) _ resource.Identifier = (*genericResource)(nil) + _ resource.PathProvider = (*genericResource)(nil) _ identity.IdentityGroupProvider = (*genericResource)(nil) _ identity.DependencyManagerProvider = (*genericResource)(nil) _ identity.Identity = (*genericResource)(nil) @@ -463,6 +464,11 @@ func (l *genericResource) Key() string { return key } +// TODO1 test and document this. Consider adding it to the Resource interface. +func (l *genericResource) Path() string { + return l.paths.TargetPath() +} + func (l *genericResource) MediaType() media.Type { return l.sd.MediaType } diff --git a/resources/resource/resources.go b/resources/resource/resources.go index 32bcdbb088a..bf0b1168df0 100644 --- a/resources/resource/resources.go +++ b/resources/resource/resources.go @@ -16,6 +16,7 @@ package resource import ( "fmt" + "path" "strings" "github.com/gohugoio/hugo/common/paths" @@ -29,6 +30,53 @@ var _ ResourceFinder = (*Resources)(nil) // I.e. both pages and images etc. type Resources []Resource +// TODO1 +// TODO1 move to a func + template func. Maybe. +func (r Resources) Mount(base, target string) ResourceGetter { + return resourceGetterFunc(func(namev any) Resource { + name1, err := cast.ToStringE(namev) + if err != nil { + panic(err) + } + + isTargetAbs := strings.HasPrefix(target, "/") + + if target != "" { + name1 = strings.TrimPrefix(name1, target) + if !isTargetAbs { + name1 = paths.TrimLeading(name1) + } + } + + if base != "" && isTargetAbs { + name1 = path.Join(base, name1) + } + + for _, res := range r { + name2 := res.Name() + + if base != "" && !isTargetAbs { + name2 = paths.TrimLeading(strings.TrimPrefix(name2, base)) + } + + // TODO1 remove. + // fmt.Println("name1", name1, "name2", name2, "base", base) + + if strings.EqualFold(name1, name2) { + return res + } + + } + + return nil + }) +} + +type ResourcesProvider interface { + // Resources returns a list of all resources. + Resources() Resources +} + // var _ resource.ResourceFinder = (*Namespace)(nil) // ResourcesConverter converts a given slice of Resource objects to Resources. type ResourcesConverter interface { @@ -63,13 +111,25 @@ func (r Resources) Get(name any) Resource { panic(err) } - namestr = paths.AddLeadingSlash(namestr) + isDotCurrent := strings.HasPrefix(namestr, "./") + if isDotCurrent { + namestr = strings.TrimPrefix(namestr, "./") + } else { + namestr = paths.AddLeadingSlash(namestr) + } + + check := func(name string) bool { + if !isDotCurrent { + name = paths.AddLeadingSlash(name) + } + return strings.EqualFold(namestr, name) + } // First check the Name. // Note that this can be modified by the user in the front matter, // also, it does not contain any language code. for _, resource := range r { - if strings.EqualFold(namestr, paths.AddLeadingSlash(resource.Name())) { + if check(resource.Name()) { return resource } } @@ -77,7 +137,7 @@ func (r Resources) Get(name any) Resource { // Finally, check the normalized name. for _, resource := range r { if nop, ok := resource.(NameNormalizedProvider); ok { - if strings.EqualFold(namestr, paths.AddLeadingSlash(nop.NameNormalized())) { + if check(nop.NameNormalized()) { return resource } } @@ -197,14 +257,31 @@ type Source interface { Publish() error } -// ResourceFinder provides methods to find Resources. -// Note that GetRemote (as found in resources.GetRemote) is -// not covered by this interface, as this is only available as a global template function. -type ResourceFinder interface { +type ResourceGetter interface { // Get locates the Resource with the given name in the current context (e.g. in .Page.Resources). // // It returns nil if no Resource could found, panics if name is invalid. Get(name any) Resource +} + +// StaleInfoResourceGetter is a ResourceGetter that also provides information about +// whether the underlying resources are stale. +type StaleInfoResourceGetter interface { + StaleInfo + ResourceGetter +} + +type resourceGetterFunc func(name any) Resource + +func (f resourceGetterFunc) Get(name any) Resource { + return f(name) +} + +// ResourceFinder provides methods to find Resources. +// Note that GetRemote (as found in resources.GetRemote) is +// not covered by this interface, as this is only available as a global template function. +type ResourceFinder interface { + ResourceGetter // GetMatch finds the first Resource matching the given pattern, or nil if none found. // @@ -235,3 +312,69 @@ type ResourceFinder interface { // It returns nil if no Resources could found, panics if typ is invalid. ByType(typ any) Resources } + +// NewResourceGetter creates a new ResourceGetter from the given objects. +// If multiple objects are provided, they are merged into one where +// the first match wins. +func NewResourceGetter(os ...any) StaleInfoResourceGetter { + var getters multiResourceGetter + for _, o := range os { + if g, ok := unwrapResourceGetter(o); ok { + getters = append(getters, g) + } + } + + return struct { + StaleInfo + ResourceGetter + }{ + StaleInfoFunc( + func() uint32 { + for _, g := range getters { + if s, ok := g.(StaleInfo); ok { + if i := s.StaleVersion(); i > 0 { + return i + } + } + } + return 0 + }, + ), + getters, + } +} + +type multiResourceGetter []ResourceGetter + +func (m multiResourceGetter) Get(name any) Resource { + for _, g := range m { + if res := g.Get(name); res != nil { + return res + } + } + return nil +} + +func unwrapResourceGetter(v any) (ResourceGetter, bool) { + if v == nil { + return nil, false + } + switch vv := v.(type) { + case ResourceGetter: + return vv, true + case ResourcesProvider: + return vv.Resources(), true + case func(name any) Resource: + return resourceGetterFunc(vv), true + case []any: + var getters multiResourceGetter + for _, vv := range vv { + if g, ok := unwrapResourceGetter(vv); ok { + getters = append(getters, g) + } + } + return getters, len(getters) > 0 + } + + return nil, false +} diff --git a/resources/resource/resources_test.go b/resources/resource/resources_test.go new file mode 100644 index 00000000000..645bd0e4b8f --- /dev/null +++ b/resources/resource/resources_test.go @@ -0,0 +1,125 @@ +// Copyright 2024 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package resource + +import ( + "testing" + + qt "github.com/frankban/quicktest" +) + +func TestResourcesMount(t *testing.T) { + c := qt.New(t) + c.Assert(true, qt.IsTrue) + + var m ResourceGetter + var r Resources + + check := func(in, expect string) { + c.Helper() + r := m.Get(in) + c.Assert(r, qt.Not(qt.IsNil)) + c.Assert(r.Name(), qt.Equals, expect) + } + + checkNil := func(in string) { + c.Helper() + r := m.Get(in) + c.Assert(r, qt.IsNil) + } + + // Misc tests. + r = Resources{ + testResource{name: "/foo/theme.css"}, + } + + m = r.Mount("/foo", ".") + check("./theme.css", "/foo/theme.css") + + // Relative target. + r = Resources{ + testResource{name: "/a/b/c/d.txt"}, + testResource{name: "/a/b/c/e/f.txt"}, + testResource{name: "/a/b/d.txt"}, + testResource{name: "/a/b/e.txt"}, + } + + // namvev ./theme.css base /js/hugoheadlessui/components target . name /js/hugoheadlessui/components/theme.css name1 theme.css name2 . + + m = r.Mount("/a/b/c", "z") + check("z/d.txt", "/a/b/c/d.txt") + // check("./z/d.txt", "/a/b/c/d.txt") + check("z/e/f.txt", "/a/b/c/e/f.txt") + + m = r.Mount("/a/b", "") + check("d.txt", "/a/b/d.txt") + m = r.Mount("/a/b", ".") + check("d.txt", "/a/b/d.txt") + m = r.Mount("/a/b", "./") + check("d.txt", "/a/b/d.txt") + check("./d.txt", "/a/b/d.txt") + + m = r.Mount("/a/b", ".") + check("./d.txt", "/a/b/d.txt") + + // Absolute target. + m = r.Mount("/a/b/c", "/z") + check("/z/d.txt", "/a/b/c/d.txt") + check("/z/e/f.txt", "/a/b/c/e/f.txt") + checkNil("/z/f.txt") + + m = r.Mount("/a/b", "/z") + check("/z/c/d.txt", "/a/b/c/d.txt") + check("/z/c/e/f.txt", "/a/b/c/e/f.txt") + check("/z/d.txt", "/a/b/d.txt") + checkNil("/z/f.txt") + + m = r.Mount("", "") + check("/a/b/c/d.txt", "/a/b/c/d.txt") + check("/a/b/c/e/f.txt", "/a/b/c/e/f.txt") + check("/a/b/d.txt", "/a/b/d.txt") + checkNil("/a/b/f.txt") + + m = r.Mount("/a/b", "/a/b") + check("/a/b/c/d.txt", "/a/b/c/d.txt") + check("/a/b/c/e/f.txt", "/a/b/c/e/f.txt") + check("/a/b/d.txt", "/a/b/d.txt") + checkNil("/a/b/f.txt") + + // Resources with relative paths. + r = Resources{ + testResource{name: "a/b/c/d.txt"}, + testResource{name: "a/b/c/e/f.txt"}, + testResource{name: "a/b/d.txt"}, + testResource{name: "a/b/e.txt"}, + testResource{name: "n.txt"}, + } + + m = r.Mount("a/b", "z") + check("z/d.txt", "a/b/d.txt") + checkNil("/z/d.txt") +} + +type testResource struct { + Resource + name string +} + +func (r testResource) Name() string { + return r.name +} + +func (r testResource) NameNormalized() string { + return r.name +} diff --git a/resources/resource/resourcetypes.go b/resources/resource/resourcetypes.go index 0fb87f37137..7684960b4be 100644 --- a/resources/resource/resourcetypes.go +++ b/resources/resource/resourcetypes.go @@ -108,6 +108,14 @@ type MediaTypeProvider interface { MediaType() media.Type } +// TODO1 consider removing.s +type PathProvider interface { + // Path is the relative path to this resource. + // In most cases this will be the same as the RelPermalink(), + // but it will not trigger any lazy publishing. + Path() string +} + type ResourceLinksProvider interface { // Permalink represents the absolute link to this resource. Permalink() string @@ -244,6 +252,13 @@ type StaleInfo interface { StaleVersion() uint32 } +// StaleInfoFunc is a function that returns the StaleVersion for one or more resources. +type StaleInfoFunc func() uint32 + +func (f StaleInfoFunc) StaleVersion() uint32 { + return f() +} + // StaleVersion returns the StaleVersion for the given os, // or 0 if not set. func StaleVersion(os any) uint32 { diff --git a/resources/resource_transformers/js/build.go b/resources/resource_transformers/js/build.go index cc68d225335..ccbc8e30567 100644 --- a/resources/resource_transformers/js/build.go +++ b/resources/resource_transformers/js/build.go @@ -1,4 +1,4 @@ -// Copyright 2020 The Hugo Authors. All rights reserved. +// Copyright 2024 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -16,25 +16,20 @@ package js import ( "errors" "fmt" - "io" "os" "path" "path/filepath" "regexp" "strings" - "github.com/spf13/afero" - - "github.com/gohugoio/hugo/hugofs" - "github.com/gohugoio/hugo/media" - + "github.com/evanw/esbuild/pkg/api" "github.com/gohugoio/hugo/common/herrors" + "github.com/gohugoio/hugo/common/hugio" "github.com/gohugoio/hugo/common/text" - + "github.com/gohugoio/hugo/hugofs" "github.com/gohugoio/hugo/hugolib/filesystems" - "github.com/gohugoio/hugo/resources/internal" + "github.com/gohugoio/hugo/identity" - "github.com/evanw/esbuild/pkg/api" "github.com/gohugoio/hugo/resources" "github.com/gohugoio/hugo/resources/resource" ) @@ -53,54 +48,48 @@ func New(fs *filesystems.SourceFilesystem, rs *resources.Spec) *Client { } } -type buildTransformation struct { - optsm map[string]any - c *Client +// Process processes a resource with the user provided options. +func (c *Client) Process(res resources.ResourceTransformer, opts map[string]any) (resource.Resource, error) { + return res.Transform( + &buildTransformation{c: c, optsm: opts}, + ) } -func (t *buildTransformation) Key() internal.ResourceTransformationKey { - return internal.NewResourceTransformationKey("jsbuild", t.optsm) +func (c *Client) BuildBundle(opts Options) (api.BuildResult, error) { + return c.build(opts, nil) } -func (t *buildTransformation) Transform(ctx *resources.ResourceTransformationCtx) error { - ctx.OutMediaType = media.Builtin.JavascriptType - - opts, err := decodeOptions(t.optsm) - if err != nil { - return err +// Note that transformCtx may be nil. +func (c *Client) build(opts Options, transformCtx *resources.ResourceTransformationCtx) (api.BuildResult, error) { + dependencyManager := opts.DependencyManager + if transformCtx != nil { + dependencyManager = transformCtx.DependencyManager // TODO1 } - - if opts.TargetPath != "" { - ctx.OutPath = opts.TargetPath - } else { - ctx.ReplaceOutPathExtension(".js") + if dependencyManager == nil { + dependencyManager = identity.NopManager } - src, err := io.ReadAll(ctx.From) - if err != nil { - return err - } + opts.ResolveDir = c.rs.Cfg.BaseConfig().WorkingDir // where node_modules gets resolved + opts.TsConfig = c.rs.ResolveJSConfigFile("tsconfig.json") - opts.sourceDir = filepath.FromSlash(path.Dir(ctx.SourcePath)) - opts.resolveDir = t.c.rs.Cfg.BaseConfig().WorkingDir // where node_modules gets resolved - opts.contents = string(src) - opts.mediaType = ctx.InMediaType - opts.tsConfig = t.c.rs.ResolveJSConfigFile("tsconfig.json") + if err := opts.validate(); err != nil { + return api.BuildResult{}, err + } buildOptions, err := toBuildOptions(opts) if err != nil { - return err + return api.BuildResult{}, err } - buildOptions.Plugins, err = createBuildPlugins(ctx.DependencyManager, t.c, opts) + buildOptions.Plugins, err = createBuildPlugins(c, dependencyManager, opts) if err != nil { - return err + return api.BuildResult{}, err } if buildOptions.Sourcemap == api.SourceMapExternal && buildOptions.Outdir == "" { buildOptions.Outdir, err = os.MkdirTemp(os.TempDir(), "compileOutput") if err != nil { - return err + return api.BuildResult{}, err } defer os.Remove(buildOptions.Outdir) } @@ -110,13 +99,13 @@ func (t *buildTransformation) Transform(ctx *resources.ResourceTransformationCtx for i, ext := range opts.Inject { impPath := filepath.FromSlash(ext) if filepath.IsAbs(impPath) { - return fmt.Errorf("inject: absolute paths not supported, must be relative to /assets") + return api.BuildResult{}, fmt.Errorf("inject: absolute paths not supported, must be relative to /assets") } - m := resolveComponentInAssets(t.c.rs.Assets.Fs, impPath) + m := resolveComponentInAssets(c.rs.Assets.Fs, impPath) if m == nil { - return fmt.Errorf("inject: file %q not found", ext) + return api.BuildResult{}, fmt.Errorf("inject: file %q not found", ext) } opts.Inject[i] = m.Filename @@ -130,46 +119,60 @@ func (t *buildTransformation) Transform(ctx *resources.ResourceTransformationCtx result := api.Build(buildOptions) if len(result.Errors) > 0 { - createErr := func(msg api.Message) error { - loc := msg.Location - if loc == nil { + if msg.Location == nil { return errors.New(msg.Text) } - path := loc.File - if path == stdinImporter { - path = ctx.SourcePath - } - - errorMessage := msg.Text - errorMessage = strings.ReplaceAll(errorMessage, nsImportHugo+":", "") - var ( - f afero.File - err error + contentr hugio.ReadSeekCloser + errorMessage string + loc = msg.Location + errorPath = loc.File + err error ) - if strings.HasPrefix(path, nsImportHugo) { - path = strings.TrimPrefix(path, nsImportHugo+":") - f, err = hugofs.Os.Open(path) - } else { - var fi os.FileInfo - fi, err = t.c.sfs.Fs.Stat(path) - if err == nil { - m := fi.(hugofs.FileMetaInfo).Meta() - path = m.Filename - f, err = m.Open() + var resolvedError *ErrorMessageResolved + + if opts.ErrorMessageResolveFunc != nil { + resolvedError = opts.ErrorMessageResolveFunc(msg) + } + + if resolvedError == nil { + if errorPath == stdinImporter { + errorPath = transformCtx.SourcePath } + errorMessage = msg.Text + errorMessage = strings.ReplaceAll(errorMessage, nsImportHugo+":", "") + + if strings.HasPrefix(errorPath, nsImportHugo) { + errorPath = strings.TrimPrefix(errorPath, nsImportHugo+":") + contentr, err = hugofs.Os.Open(errorPath) + } else { + var fi os.FileInfo + fi, err = c.sfs.Fs.Stat(errorPath) + if err == nil { + m := fi.(hugofs.FileMetaInfo).Meta() + errorPath = m.Filename + contentr, err = m.Open() + } + } + } else { + contentr = resolvedError.Content + errorPath = resolvedError.Path + errorMessage = resolvedError.Message + } + + if contentr != nil { + defer contentr.Close() } if err == nil { fe := herrors. - NewFileErrorFromName(errors.New(errorMessage), path). + NewFileErrorFromName(errors.New(errorMessage), errorPath). UpdatePosition(text.Position{Offset: -1, LineNumber: loc.Line, ColumnNumber: loc.Column}). - UpdateContent(f, nil) + UpdateContent(contentr, nil) - f.Close() return fe } @@ -185,38 +188,37 @@ func (t *buildTransformation) Transform(ctx *resources.ResourceTransformationCtx // Return 1, log the rest. for i, err := range errors { if i > 0 { - t.c.rs.Logger.Errorf("js.Build failed: %s", err) + c.rs.Logger.Errorf("js.Build failed: %s", err) } } - return errors[0] + return result, errors[0] } - if buildOptions.Sourcemap == api.SourceMapExternal { - content := string(result.OutputFiles[1].Contents) - symPath := path.Base(ctx.OutPath) + ".map" - re := regexp.MustCompile(`//# sourceMappingURL=.*\n?`) - content = re.ReplaceAllString(content, "//# sourceMappingURL="+symPath+"\n") + // TODO1 option etc. fmt.Printf("%s", api.AnalyzeMetafile(result.Metafile, api.AnalyzeMetafileOptions{})) + + if transformCtx != nil { + if buildOptions.Sourcemap == api.SourceMapExternal { + content := string(result.OutputFiles[1].Contents) + symPath := path.Base(transformCtx.OutPath) + ".map" + re := regexp.MustCompile(`//# sourceMappingURL=.*\n?`) + content = re.ReplaceAllString(content, "//# sourceMappingURL="+symPath+"\n") + + if err = transformCtx.PublishSourceMap(string(result.OutputFiles[0].Contents)); err != nil { + return result, err + } + _, err := transformCtx.To.Write([]byte(content)) + if err != nil { + return result, err + } + } else { + _, err := transformCtx.To.Write(result.OutputFiles[0].Contents) + if err != nil { + return result, err + } - if err = ctx.PublishSourceMap(string(result.OutputFiles[0].Contents)); err != nil { - return err - } - _, err := ctx.To.Write([]byte(content)) - if err != nil { - return err - } - } else { - _, err := ctx.To.Write(result.OutputFiles[0].Contents) - if err != nil { - return err } } - return nil -} -// Process process esbuild transform -func (c *Client) Process(res resources.ResourceTransformer, opts map[string]any) (resource.Resource, error) { - return res.Transform( - &buildTransformation{c: c, optsm: opts}, - ) + return result, nil } diff --git a/resources/resource_transformers/js/options.go b/resources/resource_transformers/js/options.go index 8c271d032d7..921071f5851 100644 --- a/resources/resource_transformers/js/options.go +++ b/resources/resource_transformers/js/options.go @@ -20,6 +20,7 @@ import ( "path/filepath" "strings" + "github.com/gohugoio/hugo/common/hugio" "github.com/gohugoio/hugo/common/maps" "github.com/gohugoio/hugo/common/paths" "github.com/gohugoio/hugo/identity" @@ -39,8 +40,52 @@ const ( stdinImporter = "" ) -// Options esbuild configuration type Options struct { + ExternalOptions + InternalOptions +} + +func (opts *Options) validate() error { + if opts.ImportOnResolveFunc != nil && opts.ImportOnLoadFunc == nil { + return fmt.Errorf("ImportOnLoadFunc must be set if ImportOnResolveFunc is set") + } + if opts.ImportOnResolveFunc == nil && opts.ImportOnLoadFunc != nil { + return fmt.Errorf("ImportOnResolveFunc must be set if ImportOnLoadFunc is set") + } + return nil +} + +// InternalOptions holds internal options for the js.Build template function. +type InternalOptions struct { + MediaType media.Type + OutDir string + Contents string + SourceDir string + ResolveDir string + + DependencyManager identity.Manager + + // TODO1 + Write bool // Set to false to write to memory. + AllowOverwrite bool + Splitting bool + TsConfig string + EntryPoints []string + ImportOnResolveFunc func(string, api.OnResolveArgs) string + ImportOnLoadFunc func(api.OnLoadArgs) string + ImportParamsOnLoadFunc func(args api.OnLoadArgs) json.RawMessage + ErrorMessageResolveFunc func(api.Message) *ErrorMessageResolved + Stdin bool +} + +type ErrorMessageResolved struct { + Path string + Message string + Content hugio.ReadSeekCloser +} + +// ExternalOptions holds user facing options for the js.Build template function. +type ExternalOptions struct { // If not set, the source path will be used as the base target path. // Note that the target path's extension may change if the target MIME type // is different, e.g. when the source is TypeScript. @@ -105,17 +150,10 @@ type Options struct { // Deprecated: This no longer have any effect and will be removed. // TODO(bep) remove. See https://github.com/evanw/esbuild/commit/869e8117b499ca1dbfc5b3021938a53ffe934dba AvoidTDZ bool - - mediaType media.Type - outDir string - contents string - sourceDir string - resolveDir string - tsConfig string } -func decodeOptions(m map[string]any) (Options, error) { - var opts Options +func DecodeExternalOptions(m map[string]any) (ExternalOptions, error) { + var opts ExternalOptions if err := mapstructure.WeakDecode(m, &opts); err != nil { return opts, err @@ -212,31 +250,52 @@ func resolveComponentInAssets(fs afero.Fs, impPath string) *hugofs.FileMeta { return m } -func createBuildPlugins(depsManager identity.Manager, c *Client, opts Options) ([]api.Plugin, error) { +func createBuildPlugins(c *Client, depsManager identity.Manager, opts Options) ([]api.Plugin, error) { fs := c.rs.Assets resolveImport := func(args api.OnResolveArgs) (api.OnResolveResult, error) { impPath := args.Path + shimmed := false if opts.Shims != nil { override, found := opts.Shims[impPath] if found { impPath = override + shimmed = true } } - isStdin := args.Importer == stdinImporter + + if opts.ImportOnResolveFunc != nil { + if s := opts.ImportOnResolveFunc(impPath, args); s != "" { + return api.OnResolveResult{Path: s, Namespace: nsImportHugo}, nil + } + } + + importer := args.Importer + + isStdin := importer == stdinImporter var relDir string if !isStdin { - rel, found := fs.MakePathRelative(args.Importer, true) - if !found { - // Not in any of the /assets folders. - // This is an import from a node_modules, let - // ESBuild resolve this. - return api.OnResolveResult{}, nil + if strings.HasPrefix(importer, "@hugo-virtual") { + // TODO1 constants. + relDir = filepath.Dir(strings.TrimPrefix(importer, "@hugo-virtual")) + } else { + rel, found := fs.MakePathRelative(importer, true) + + if !found { + if shimmed { + relDir = opts.SourceDir + } else { + // Not in any of the /assets folders. + // This is an import from a node_modules, let + // ESBuild resolve this. + return api.OnResolveResult{}, nil + } + } else { + relDir = filepath.Dir(rel) + } } - - relDir = filepath.Dir(rel) } else { - relDir = opts.sourceDir + relDir = opts.SourceDir } // Imports not starting with a "." is assumed to live relative to /assets. @@ -272,16 +331,26 @@ func createBuildPlugins(depsManager identity.Manager, c *Client, opts Options) ( }) build.OnLoad(api.OnLoadOptions{Filter: `.*`, Namespace: nsImportHugo}, func(args api.OnLoadArgs) (api.OnLoadResult, error) { - b, err := os.ReadFile(args.Path) - if err != nil { - return api.OnLoadResult{}, fmt.Errorf("failed to read %q: %w", args.Path, err) + var c string + if opts.ImportOnLoadFunc != nil { + if s := opts.ImportOnLoadFunc(args); s != "" { + c = s + } } - c := string(b) + + if c == "" { + b, err := os.ReadFile(args.Path) + if err != nil { + return api.OnLoadResult{}, fmt.Errorf("failed to read %q: %w", args.Path, err) + } + c = string(b) + } + return api.OnLoadResult{ // See https://github.com/evanw/esbuild/issues/502 // This allows all modules to resolve dependencies // in the main project's node_modules. - ResolveDir: opts.resolveDir, + ResolveDir: opts.ResolveDir, Contents: &c, Loader: loaderFromFilename(args.Path), }, nil @@ -299,21 +368,30 @@ func createBuildPlugins(depsManager identity.Manager, c *Client, opts Options) ( if err != nil { return nil, fmt.Errorf("failed to marshal params: %w", err) } - bs := string(b) paramsPlugin := api.Plugin{ Name: "hugo-params-plugin", Setup: func(build api.PluginBuild) { build.OnResolve(api.OnResolveOptions{Filter: `^@params$`}, func(args api.OnResolveArgs) (api.OnResolveResult, error) { return api.OnResolveResult{ - Path: args.Path, + Path: args.Importer, Namespace: nsParams, }, nil }) build.OnLoad(api.OnLoadOptions{Filter: `.*`, Namespace: nsParams}, func(args api.OnLoadArgs) (api.OnLoadResult, error) { + bb := b + + if opts.ImportParamsOnLoadFunc != nil { + if bbb := opts.ImportParamsOnLoadFunc(args); bbb != nil { + bb = bbb + } + } + + s := string(bb) + return api.OnLoadResult{ - Contents: &bs, + Contents: &s, Loader: api.LoaderJSON, }, nil }) @@ -353,7 +431,7 @@ func toBuildOptions(opts Options) (buildOptions api.BuildOptions, err error) { return } - mediaType := opts.mediaType + mediaType := opts.MediaType if mediaType.IsZero() { mediaType = media.Builtin.JavascriptType } @@ -371,7 +449,7 @@ func toBuildOptions(opts Options) (buildOptions api.BuildOptions, err error) { case media.Builtin.JSXType.SubType: loader = api.LoaderJSX default: - err = fmt.Errorf("unsupported Media Type: %q", opts.mediaType) + err = fmt.Errorf("unsupported Media Type: %q", opts.MediaType) return } @@ -408,7 +486,7 @@ func toBuildOptions(opts Options) (buildOptions api.BuildOptions, err error) { } // By default we only need to specify outDir and no outFile - outDir := opts.outDir + outDir := opts.OutDir outFile := "" var sourceMap api.SourceMap switch opts.SourceMap { @@ -424,8 +502,9 @@ func toBuildOptions(opts Options) (buildOptions api.BuildOptions, err error) { } buildOptions = api.BuildOptions{ - Outfile: outFile, - Bundle: true, + Outfile: outFile, + Bundle: true, + Metafile: true, Target: target, Format: format, @@ -435,9 +514,12 @@ func toBuildOptions(opts Options) (buildOptions api.BuildOptions, err error) { MinifyIdentifiers: opts.Minify, MinifySyntax: opts.Minify, - Outdir: outDir, - Define: defines, + Outdir: outDir, + Write: opts.Write, + AllowOverwrite: opts.AllowOverwrite, + Splitting: opts.Splitting, + Define: defines, External: opts.Externals, JSXFactory: opts.JSXFactory, @@ -446,16 +528,18 @@ func toBuildOptions(opts Options) (buildOptions api.BuildOptions, err error) { JSX: jsx, JSXImportSource: opts.JSXImportSource, - Tsconfig: opts.tsConfig, + Tsconfig: opts.TsConfig, - // Note: We're not passing Sourcefile to ESBuild. - // This makes ESBuild pass `stdin` as the Importer to the import - // resolver, which is what we need/expect. - Stdin: &api.StdinOptions{ - Contents: opts.contents, - ResolveDir: opts.resolveDir, + EntryPoints: opts.EntryPoints, + } + + if opts.Stdin { + // This makes ESBuild pass `stdin` as the Importer to the import. + buildOptions.Stdin = &api.StdinOptions{ + Contents: opts.Contents, + ResolveDir: opts.ResolveDir, Loader: loader, - }, + } } return } diff --git a/resources/resource_transformers/js/options_test.go b/resources/resource_transformers/js/options_test.go index 53aa9b6bbff..18a208d1b31 100644 --- a/resources/resource_transformers/js/options_test.go +++ b/resources/resource_transformers/js/options_test.go @@ -50,7 +50,11 @@ func TestOptionKey(t *testing.T) { func TestToBuildOptions(t *testing.T) { c := qt.New(t) - opts, err := toBuildOptions(Options{mediaType: media.Builtin.JavascriptType}) + opts, err := toBuildOptions(Options{ + InternalOptions: InternalOptions{ + MediaType: media.Builtin.JavascriptType, + }, + }) c.Assert(err, qt.IsNil) c.Assert(opts, qt.DeepEquals, api.BuildOptions{ @@ -62,13 +66,19 @@ func TestToBuildOptions(t *testing.T) { }, }) - opts, err = toBuildOptions(Options{ - Target: "es2018", - Format: "cjs", - Minify: true, - mediaType: media.Builtin.JavascriptType, - AvoidTDZ: true, - }) + opts, err = toBuildOptions( + Options{ + ExternalOptions: ExternalOptions{ + Target: "es2018", + Format: "cjs", + Minify: true, + AvoidTDZ: true, + }, + InternalOptions: InternalOptions{ + MediaType: media.Builtin.JavascriptType, + }, + }, + ) c.Assert(err, qt.IsNil) c.Assert(opts, qt.DeepEquals, api.BuildOptions{ Bundle: true, @@ -82,10 +92,17 @@ func TestToBuildOptions(t *testing.T) { }, }) - opts, err = toBuildOptions(Options{ - Target: "es2018", Format: "cjs", Minify: true, mediaType: media.Builtin.JavascriptType, - SourceMap: "inline", - }) + opts, err = toBuildOptions( + Options{ + ExternalOptions: ExternalOptions{ + Target: "es2018", Format: "cjs", Minify: true, + SourceMap: "inline", + }, + InternalOptions: InternalOptions{ + MediaType: media.Builtin.JavascriptType, + }, + }, + ) c.Assert(err, qt.IsNil) c.Assert(opts, qt.DeepEquals, api.BuildOptions{ Bundle: true, @@ -100,10 +117,17 @@ func TestToBuildOptions(t *testing.T) { }, }) - opts, err = toBuildOptions(Options{ - Target: "es2018", Format: "cjs", Minify: true, mediaType: media.Builtin.JavascriptType, - SourceMap: "inline", - }) + opts, err = toBuildOptions( + Options{ + ExternalOptions: ExternalOptions{ + Target: "es2018", Format: "cjs", Minify: true, + SourceMap: "inline", + }, + InternalOptions: InternalOptions{ + MediaType: media.Builtin.JavascriptType, + }, + }, + ) c.Assert(err, qt.IsNil) c.Assert(opts, qt.DeepEquals, api.BuildOptions{ Bundle: true, @@ -118,10 +142,18 @@ func TestToBuildOptions(t *testing.T) { }, }) - opts, err = toBuildOptions(Options{ - Target: "es2018", Format: "cjs", Minify: true, mediaType: media.Builtin.JavascriptType, - SourceMap: "external", - }) + opts, err = toBuildOptions( + Options{ + ExternalOptions: ExternalOptions{ + Target: "es2018", Format: "cjs", Minify: true, + SourceMap: "external", + }, + InternalOptions: InternalOptions{ + MediaType: media.Builtin.JavascriptType, + }, + }, + ) + c.Assert(err, qt.IsNil) c.Assert(opts, qt.DeepEquals, api.BuildOptions{ Bundle: true, @@ -136,10 +168,17 @@ func TestToBuildOptions(t *testing.T) { }, }) - opts, err = toBuildOptions(Options{ - mediaType: media.Builtin.JavascriptType, - JSX: "automatic", JSXImportSource: "preact", - }) + opts, err = toBuildOptions( + Options{ + ExternalOptions: ExternalOptions{ + JSX: "automatic", JSXImportSource: "preact", + }, + InternalOptions: InternalOptions{ + MediaType: media.Builtin.JavascriptType, + }, + }, + ) + c.Assert(err, qt.IsNil) c.Assert(opts, qt.DeepEquals, api.BuildOptions{ Bundle: true, @@ -173,10 +212,17 @@ func TestToBuildOptionsTarget(t *testing.T) { {"esnext", api.ESNext}, } { c.Run(test.target, func(c *qt.C) { - opts, err := toBuildOptions(Options{ - Target: test.target, - mediaType: media.Builtin.JavascriptType, - }) + opts, err := toBuildOptions( + Options{ + ExternalOptions: ExternalOptions{ + Target: test.target, + }, + InternalOptions: InternalOptions{ + MediaType: media.Builtin.JavascriptType, + }, + }, + ) + c.Assert(err, qt.IsNil) c.Assert(opts.Target, qt.Equals, test.expect) }) diff --git a/resources/resource_transformers/js/transform.go b/resources/resource_transformers/js/transform.go new file mode 100644 index 00000000000..d6cfcf54563 --- /dev/null +++ b/resources/resource_transformers/js/transform.go @@ -0,0 +1,67 @@ +// Copyright 2024 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package js + +import ( + "io" + "path" + "path/filepath" + + "github.com/gohugoio/hugo/media" + "github.com/gohugoio/hugo/resources" + "github.com/gohugoio/hugo/resources/internal" +) + +type buildTransformation struct { + optsm map[string]any + c *Client +} + +func (t *buildTransformation) Key() internal.ResourceTransformationKey { + return internal.NewResourceTransformationKey("jsbuild", t.optsm) +} + +func (t *buildTransformation) Transform(ctx *resources.ResourceTransformationCtx) error { + ctx.OutMediaType = media.Builtin.JavascriptType + + var opts Options + + if t.optsm != nil { + optsExt, err := DecodeExternalOptions(t.optsm) + if err != nil { + return err + } + opts.ExternalOptions = optsExt + } + + if opts.TargetPath != "" { + ctx.OutPath = opts.TargetPath + } else { + ctx.ReplaceOutPathExtension(".js") + } + + src, err := io.ReadAll(ctx.From) + if err != nil { + return err + } + + opts.SourceDir = filepath.FromSlash(path.Dir(ctx.SourcePath)) + opts.Contents = string(src) + opts.MediaType = ctx.InMediaType + opts.Stdin = true + + _, err = t.c.build(opts, ctx) + + return err +} diff --git a/resources/transform.go b/resources/transform.go index 336495e6d07..9781ea6c31c 100644 --- a/resources/transform.go +++ b/resources/transform.go @@ -52,6 +52,7 @@ var ( _ identity.IdentityGroupProvider = (*resourceAdapterInner)(nil) _ resource.Source = (*resourceAdapter)(nil) _ resource.Identifier = (*resourceAdapter)(nil) + _ resource.PathProvider = (*resourceAdapter)(nil) _ resource.ResourceNameTitleProvider = (*resourceAdapter)(nil) _ resource.WithResourceMetaProvider = (*resourceAdapter)(nil) _ identity.DependencyManagerProvider = (*resourceAdapter)(nil) @@ -277,6 +278,11 @@ func (r *resourceAdapter) Key() string { return r.target.(resource.Identifier).Key() } +func (r *resourceAdapter) Path() string { + r.init(false, false) + return r.target.(resource.PathProvider).Path() +} + func (r *resourceAdapter) MediaType() media.Type { r.init(false, false) return r.target.MediaType() diff --git a/tpl/internal/go_templates/texttemplate/exec.go b/tpl/internal/go_templates/texttemplate/exec.go index bd8c82bd705..f7bffad7aae 100644 --- a/tpl/internal/go_templates/texttemplate/exec.go +++ b/tpl/internal/go_templates/texttemplate/exec.go @@ -305,7 +305,10 @@ func (s *state) walkIfOrWith(typ parse.NodeType, dot reflect.Value, pipe *parse. } if truth { if typ == parse.NodeWith { - s.walk(val, list) + func() { + defer s.pushWithValue(val)() + s.walk(val, list) + }() } else { s.walk(dot, list) } diff --git a/tpl/internal/go_templates/texttemplate/hugo_template.go b/tpl/internal/go_templates/texttemplate/hugo_template.go index 12dbe041217..2be75eadda0 100644 --- a/tpl/internal/go_templates/texttemplate/hugo_template.go +++ b/tpl/internal/go_templates/texttemplate/hugo_template.go @@ -19,6 +19,7 @@ import ( "reflect" "github.com/gohugoio/hugo/common/hreflect" + "github.com/gohugoio/hugo/common/types" "github.com/gohugoio/hugo/tpl/internal/go_templates/texttemplate/parse" ) @@ -110,14 +111,31 @@ func (t *Template) executeWithState(state *state, value reflect.Value) (err erro // template so that multiple executions of the same template // can execute in parallel. type state struct { - tmpl *Template - ctx context.Context // Added for Hugo. The original data context. - prep Preparer // Added for Hugo. - helper ExecHelper // Added for Hugo. - wr io.Writer - node parse.Node // current node, for errors - vars []variable // push-down stack of variable values. - depth int // the height of the stack of executing templates. + tmpl *Template + ctx context.Context // Added for Hugo. The original data context. + prep Preparer // Added for Hugo. + helper ExecHelper // Added for Hugo. + withValues []reflect.Value // Added for Hugo. Push-down stack of values. + + wr io.Writer + node parse.Node // current node, for errors + vars []variable // push-down stack of variable values. + depth int // the height of the stack of executing templates. +} + +func (s *state) pushWithValue(value reflect.Value) func() { + s.withValues = append(s.withValues, value) + return func() { + // TODO1 integrate with GO 1.23. + v, _ := indirect(s.withValues[len(s.withValues)-1]) + if hreflect.IsValid(v) { + if closer, ok := v.Interface().(types.Closer); ok { + closer.Close() + } + } + + s.withValues = s.withValues[:len(s.withValues)-1] + } } func (s *state) evalFunction(dot reflect.Value, node *parse.IdentifierNode, cmd parse.Node, args []parse.Node, final reflect.Value) reflect.Value { diff --git a/tpl/js/batch-esm-callback.gotmpl b/tpl/js/batch-esm-callback.gotmpl new file mode 100644 index 00000000000..389e9f6ed83 --- /dev/null +++ b/tpl/js/batch-esm-callback.gotmpl @@ -0,0 +1,17 @@ +{{ range $i, $e := .Modules -}} + import { default as {{ printf "Mod%d" $i }} } from "{{ .ImportPath }}"; +{{ end -}} +{{ range $i, $e := .Callbacks }} + {{ $id := printf "Callback%d" $i }} + import { default as {{ $id }} } from "{{ . }}"; +{{ end }} +{{/* */}} +let mods = []; +{{ range $i, $e := .Modules -}} + mods.push({{ .CallbackJSON $i }}); +{{ end -}} +{{/* */}} +{{ range $i, $e := .Callbacks }} + {{ $id := printf "Callback%d" $i }} + {{ $id }}(mods); +{{ end }} diff --git a/tpl/js/batch.go b/tpl/js/batch.go new file mode 100644 index 00000000000..1ed00edc09b --- /dev/null +++ b/tpl/js/batch.go @@ -0,0 +1,1105 @@ +// Copyright 2024 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package js provides functions for building JavaScript resources +package js + +import ( + "bytes" + "context" + _ "embed" + "encoding/json" + "fmt" + "os" + "path" + "path/filepath" + "reflect" + "sort" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/evanw/esbuild/pkg/api" + "github.com/gohugoio/hugo/cache/dynacache" + "github.com/gohugoio/hugo/common/herrors" + "github.com/gohugoio/hugo/common/hugio" + "github.com/gohugoio/hugo/common/maps" + "github.com/gohugoio/hugo/common/paths" + "github.com/gohugoio/hugo/hugofs" + "github.com/gohugoio/hugo/identity" + "github.com/gohugoio/hugo/lazy" + "github.com/gohugoio/hugo/media" + "github.com/gohugoio/hugo/resources" + "github.com/gohugoio/hugo/resources/resource" + "github.com/gohugoio/hugo/resources/resource_transformers/js" + template "github.com/gohugoio/hugo/tpl/internal/go_templates/texttemplate" + "github.com/mitchellh/mapstructure" + "github.com/spf13/cast" +) + +// TODO1 move/consolidate. +const ( + hugoVirtualNS = "@hugo-virtual" + paramsNS = "@params" +) + +type Batcher interface { + Config() OptionsSetter + Group(id string) BatcherGroup + Build() (*Package, error) +} + +type BatcherGroup interface { + Callback(id string) OptionsSetter + Script(id string) OptionsSetter + Instance(sid, iid string) OptionsSetter +} + +type ScriptOptions struct { + // The script to build. + // TODO1 handle stale. + Resource resource.Resource + + // The import context to use. + // Note that we will always fall back to the resource's own import context. + ImportContext resource.ResourceGetter + + // Params marshaled to JSON. + Params json.RawMessage +} + +func (o ScriptOptions) Compile(m map[string]any) (*ScriptOptions, error) { + var s optionsGetSet // TODO1 type. + if err := mapstructure.WeakDecode(m, &s); err != nil { + return nil, err + } + + paramsJSON, err := json.Marshal(s.Params) + if err != nil { + panic(err) + } + + return &ScriptOptions{ + Resource: s.Resource, + ImportContext: resource.NewResourceGetter(s.ImportContext), + Params: paramsJSON, + }, nil +} + +func (o *ScriptOptions) Dir() string { + return path.Dir(o.Resource.(resource.PathProvider).Path()) +} + +type ParamsOptions struct { + Params json.RawMessage +} + +type ScriptOptionsGetSetter interface { + GetOptions() *ScriptOptions + SetOptions(map[string]any) string +} + +type OptionsSetter interface { + SetOptions(map[string]any) string +} + +func (ns *Namespace) Batch(id string, store *maps.Scratch) (Batcher, error) { + key := path.Join(nsBundle, id) + b := store.GetOrCreate(key, func() any { + return &batcher{ + id: id, + scriptGroups: make(map[string]*scriptGroup), + client: ns, + configOptions: newOptions(), + } + }) + return b.(*batcher), nil +} + +func (b *batcher) reset() { + b.mu.Lock() + defer b.mu.Unlock() + for _, v := range b.scriptGroups { + // TODO1 check if this is complete. + v.Reset() + } +} + +func (b *batcher) Group(id string) BatcherGroup { + b.mu.Lock() + defer b.mu.Unlock() + + group, found := b.scriptGroups[id] + if !found { + group = &scriptGroup{ + id: id, client: b.client, + scriptsOptions: make(map[string]*options), + instancesOptions: make(map[instanceID]*options), + callbacksOptions: make(map[string]*options), + } + b.scriptGroups[id] = group + } + + return group +} + +var _ Batcher = (*batcher)(nil) + +type batchTemplateContext struct { + keyPath string + ID string + Callbacks []string + Modules []batchTemplateExecutionsContext +} + +type batchTemplateExecutionsContext struct { + ID string `json:"id"` + ImportPath string `json:"-"` + Instances []batchTemplateExecution `json:"instances"` + + r resource.Resource +} + +func (b batchTemplateExecutionsContext) CallbackJSON(i int) string { + mod := fmt.Sprintf("Mod%d", i) + + v := struct { + Mod string `json:"mod"` + batchTemplateExecutionsContext + }{ + mod, + b, + } + + bb, err := json.Marshal(v) + if err != nil { + panic(err) + } + s := string(bb) + + s = strings.ReplaceAll(s, fmt.Sprintf("%q", mod), mod) + + return s +} + +type batchTemplateExecution struct { + ID string `json:"id"` + Params json.RawMessage `json:"params"` +} + +type scriptGroup struct { + mu sync.Mutex + + id string + + client *Namespace + + scriptsOptions map[string]*options + instancesOptions map[instanceID]*options + callbacksOptions map[string]*options + + // Compiled. + scripts scriptMap + instances instanceMap + callbacks scriptMap +} + +func (g *scriptGroup) Reset() { + for _, v := range g.scriptsOptions { + v.Reset() + } + for _, v := range g.instancesOptions { + v.Reset() + } + for _, v := range g.callbacksOptions { + v.Reset() + } +} + +type script struct { + ID string + *ScriptOptions +} + +type instance struct { + instanceID + *ParamsOptions +} + +type ( + instanceMap map[instanceID]*ParamsOptions + instances []*instance +) + +func (i instances) ByScriptID(id string) instances { + var a instances + for _, v := range i { + if v.instanceID.scriptID == id { + a = append(a, v) + } + } + return a +} + +func (p instanceMap) Sorted() instances { + var a []*instance + for k, v := range p { + a = append(a, &instance{instanceID: k, ParamsOptions: v}) + } + sort.Slice(a, func(i, j int) bool { + ai := a[i] + aj := a[j] + if ai.instanceID.scriptID != aj.instanceID.scriptID { + return ai.instanceID.scriptID < aj.instanceID.scriptID + } + return ai.instanceID.instanceID < aj.instanceID.instanceID + }) + return a +} + +type scriptMap map[string]*ScriptOptions + +func (s scriptMap) Sorted() []*script { + var a []*script + for k, v := range s { + a = append(a, &script{ID: k, ScriptOptions: v}) + } + sort.Slice(a, func(i, j int) bool { + return a[i].ID < a[j].ID + }) + return a +} + +func (s *scriptGroup) compile() error { + // TODO1 lock? + s.scripts = make(map[string]*ScriptOptions) + s.instances = make(map[instanceID]*ParamsOptions) + s.callbacks = make(map[string]*ScriptOptions) + + for k, v := range s.scriptsOptions { + compiled, err := compileScriptOptions(v) + if err != nil { + return err + } + s.scripts[k] = compiled + } + + for k, v := range s.instancesOptions { + compiled, err := compileParamsOptions(v) + if err != nil { + return err + } + s.instances[k] = compiled + } + + for k, v := range s.callbacksOptions { + compiled, err := compileScriptOptions(v) + if err != nil { + return err + } + s.callbacks[k] = compiled + } + + return nil +} + +type instanceID struct { + scriptID string + instanceID string +} + +func (s *scriptGroup) Script(id string) OptionsSetter { + s.mu.Lock() + defer s.mu.Unlock() + if v, found := s.scriptsOptions[id]; found { + return v.Get() + } + s.scriptsOptions[id] = newOptions() + return s.scriptsOptions[id].Get() +} + +func (s *scriptGroup) Instance(sid, iid string) OptionsSetter { + s.mu.Lock() + defer s.mu.Unlock() + id := instanceID{scriptID: sid, instanceID: iid} + if v, found := s.instancesOptions[id]; found { + return v.Get() + } + s.instancesOptions[id] = newOptions() + return s.instancesOptions[id].Get() +} + +func (s *scriptGroup) Callback(id string) OptionsSetter { + s.mu.Lock() + defer s.mu.Unlock() + if v, found := s.callbacksOptions[id]; found { + return v.Get() + } + s.callbacksOptions[id] = newOptions() + return s.callbacksOptions[id].Get() +} + +type scriptGroups map[string]*scriptGroup + +func (s scriptGroups) Sorted() []*scriptGroup { + var a []*scriptGroup + for _, v := range s { + a = append(a, v) + } + sort.Slice(a, func(i, j int) bool { + return a[i].id < a[j].id + }) + return a +} + +type batcher struct { + mu sync.Mutex + id string + scriptGroups scriptGroups + + client *Namespace + + configOptions *options + + // The last successfully built package. + // If this is non-nil and not stale, we can reuse it (e.g. on server rebuilds) + prevBuild *Package + + // Compiled. + config js.ExternalOptions +} + +func (b *batcher) compile() error { + var err error + b.config, err = js.DecodeExternalOptions(b.configOptions.commit().opts) + if err != nil { + return err + } + + for _, v := range b.scriptGroups { + if err := v.compile(); err != nil { + return err + } + } + return nil +} + +func (b *batcher) Config() OptionsSetter { + return b.configOptions.Get() +} + +var _ resource.StaleInfo = (*options)(nil) + +type options struct { + getOnce[*optionsSetter] +} + +func (o *options) Reset() { + mu := o.once.ResetWithLock() + o.v.staleVersion.Store(0) + mu.Unlock() +} + +func (o *options) StaleVersion() uint32 { + return o.v.staleVersion.Load() +} + +// func (o ScriptOptions) Compile(m map[string]any) (*ScriptOptions, error) { + +func newOptions() *options { + return &options{getOnce[*optionsSetter]{ + v: &optionsSetter{}, + }} +} + +type optionsSetter struct { + staleVersion atomic.Uint32 + opts map[string]any +} + +// TODO1 try to avoid stale page resources when changing the head. +func (o *optionsSetter) SetOptions(m map[string]any) string { + if o.opts != nil { + if reflect.DeepEqual(o.opts, m) { + return "" + } + var isStale bool + for k, v := range m { + vv, found := o.opts[k] + if !found { + isStale = true + } else { + if si, ok := vv.(resource.StaleInfo); ok { + isStale = si.StaleVersion() > 0 + } else { + isStale = !reflect.DeepEqual(v, vv) + } + } + + if isStale { + break + } + } + + if !isStale { + return "" + } + + o.staleVersion.Add(1) + } + + o.opts = m + + return "" +} + +type getOnce[T any] struct { + v T + once lazy.OnceMore +} + +func (g *getOnce[T]) Get() T { + var v T + g.once.Do(func() { + v = g.v + }) + return v +} + +func (g *getOnce[T]) commit() T { + g.once.Do(func() {}) + return g.v +} + +func compileParamsOptions(o *options) (*ParamsOptions, error) { + v := struct { + Params map[string]any + }{} + + m := o.commit().opts + + if err := mapstructure.WeakDecode(m, &v); err != nil { + return nil, err + } + + paramsJSON, err := json.Marshal(v.Params) + if err != nil { + return nil, err + } + + return &ParamsOptions{ + Params: paramsJSON, + }, nil +} + +func compileScriptOptions(o *options) (*ScriptOptions, error) { + v := struct { + Resource resource.Resource + ImportContext any + Params map[string]any + }{} + + m := o.commit().opts + + if err := mapstructure.WeakDecode(m, &v); err != nil { + panic(err) + } + + var paramsJSON []byte + if v.Params != nil { + var err error + paramsJSON, err = json.Marshal(v.Params) + if err != nil { + panic(err) + } + } + + compiled := &ScriptOptions{ + Resource: v.Resource, + ImportContext: resource.NewResourceGetter(v.ImportContext), + Params: paramsJSON, + } + + return compiled, nil +} + +type optionsGetSet struct { + Resource resource.Resource + ImportContext any + Params map[string]any + + // Compiled values + compiled *ScriptOptions +} + +func (s *optionsGetSet) GetOptions() *ScriptOptions { + return s.compiled +} + +func (s *optionsGetSet) SetOptions(m map[string]any) string { + if err := mapstructure.WeakDecode(m, &s); err != nil { + panic(err) + } + + paramsJSON, err := json.Marshal(s.Params) + if err != nil { + panic(err) + } + + s.compiled = &ScriptOptions{ + Resource: s.Resource, + ImportContext: resource.NewResourceGetter(s.ImportContext), + Params: paramsJSON, + } + + return "" +} + +var ( + _ resource.StaleInfo = (*Package)(nil) + _ identity.IsProbablyDependencyProvider = (*Package)(nil) + _ identity.Identity = (*Package)(nil) +) + +// TODO1 names. +type Package struct { + origin *batcher + outDir string + id string + staleVersion uint32 + b *batcher + Groups map[string]resource.Resources +} + +// TODO1 add a group.Subscribe (identity) method? + +func (p *Package) IdentifierBase() string { + return p.id +} + +// TODO1 need this? We cannot cache this value for long. +func (p *Package) StaleVersion() uint32 { + p.b.mu.Lock() + defer p.b.mu.Unlock() + if p.staleVersion == 0 { + p.staleVersion = p.calculateStaleVersion() + } + return p.staleVersion +} + +func (p *Package) MarkStale() { + p.origin.reset() +} + +func (p *Package) IsProbablyDependency(other identity.Identity) bool { + depsFinder := identity.NewFinder(identity.FinderConfig{}) + var b bool + p.forEeachStaleInfo(func(rr resource.StaleInfo) bool { + identity.WalkIdentitiesShallow(other, func(level int, left identity.Identity) bool { + identity.WalkIdentitiesShallow(rr, func(level int, right identity.Identity) bool { + if i := depsFinder.Contains(left, right, -1); i > 0 { + b = true + } + return b + }) + return b + }) + return b + }) + + // TODO1 why is this called twice on change? + + return b +} + +// You should not depend on the invocation order when calling this. +// TODO1 check that this does not get called on first build. +func (p *Package) forEeachStaleInfo(f func(si resource.StaleInfo) bool) { + check := func(v any) bool { + if si, ok := v.(resource.StaleInfo); ok { + return f(si) + } + return false + } + for _, v := range p.b.scriptGroups { + if b := func() bool { + v.mu.Lock() + defer v.mu.Unlock() + + for _, vv := range v.instancesOptions { + if check(vv) { + return true + } + } + + for _, vv := range v.scriptsOptions { + if check(vv) { + return true + } + } + + for _, vv := range v.callbacksOptions { + if check(vv) { + return true + } + } + + return false + }(); b { + return + } + } +} + +func (p *Package) calculateStaleVersion() uint32 { + // Return the first non-zero value found. + var i uint32 + p.forEeachStaleInfo(func(si resource.StaleInfo) bool { + if i = si.StaleVersion(); i > 0 { + return true + } + return false + }) + + return i +} + +func logTime(name string, start time.Time) { + elapsed := time.Since(start) + fmt.Printf("%s in %s\n", name, elapsed) +} + +func (b *batcher) Build() (*Package, error) { + key := dynacache.CleanKey(b.id + ".js") + + p, err := b.client.bundlesCache.GetOrCreate(key, func(string) (*Package, error) { + return b.build() + }) + if err != nil { + return nil, err + } + + if p.b != b { + panic("bundler mismatch") + } + + return p, nil +} + +// TODO1 remove. +func deb(what string, v ...any) { + fmt.Println(what, v) +} + +func (b *batcher) build() (*Package, error) { + b.mu.Lock() + defer b.mu.Unlock() + + // Use the unexported calculateStaleVersion + if b.prevBuild != nil && b.prevBuild.calculateStaleVersion() == 0 { + return b.prevBuild, nil + } + + p, err := b.doBuild() + if err != nil { + return nil, err + } + b.prevBuild = p + return p, nil +} + +func (b *batcher) doBuild() (*Package, error) { + start := time.Now() + defer logTime("build", start) // TODO1 remove. + + keyPath := b.id + + type importContext struct { + name string + resourceGetter resource.ResourceGetter + scriptOptions *ScriptOptions // TODO1 remove resourceGetter? + } + + state := struct { + importResource *maps.Cache[string, resource.Resource] + resultResource *maps.Cache[string, resource.Resource] + importerImportContext *maps.Cache[string, importContext] + pathGroup *maps.Cache[string, string] + }{ + importResource: maps.NewCache[string, resource.Resource](), + resultResource: maps.NewCache[string, resource.Resource](), + importerImportContext: maps.NewCache[string, importContext](), + pathGroup: maps.NewCache[string, string](), + } + + var entryPoints []string + addResource := func(group, pth string, r resource.Resource, isResult bool) { + state.pathGroup.Set(pth, group) + state.importResource.Set(pth, r) + if isResult { + state.resultResource.Set(pth, r) + } + entryPoints = append(entryPoints, pth) + } + + if err := b.compile(); err != nil { + return nil, err + } + + for k, v := range b.scriptGroups { + keyPath := keyPath + "_" + k + + var callbacks []string + for _, vv := range v.callbacks.Sorted() { + callbackKeyPath := keyPath + "_" + vv.ID + callbackImpPath := paths.AddLeadingSlash(callbackKeyPath + "_callback" + vv.Resource.MediaType().FirstSuffix.FullSuffix) + callbacks = append(callbacks, callbackImpPath) + addResource(k, callbackImpPath, vv.Resource, false) + } + + t := &batchTemplateContext{ + keyPath: keyPath, + ID: v.id, + Callbacks: callbacks, + } + + instances := v.instances.Sorted() + + for _, vv := range v.scripts.Sorted() { + if vv.Resource == nil { + // TODO1 others, init. + return nil, fmt.Errorf("resource not set for %q", vv.ID) + } + keyPath := keyPath + "_" + vv.ID + opts := vv.ScriptOptions + impPath := path.Join(hugoVirtualNS, opts.Dir(), keyPath+opts.Resource.MediaType().FirstSuffix.FullSuffix) + impCtx := opts.ImportContext + + state.importerImportContext.Set(impPath, importContext{ + name: keyPath, + resourceGetter: impCtx, + scriptOptions: opts, + }) + + bt := batchTemplateExecutionsContext{ + ID: vv.ID, + r: vv.Resource, + ImportPath: impPath, + } + state.importResource.Set(bt.ImportPath, vv.Resource) + for _, vvv := range instances.ByScriptID(vv.ID) { + bt.Instances = append(bt.Instances, batchTemplateExecution{ID: vvv.instanceID.instanceID, Params: vvv.Params}) + sort.Slice(bt.Instances, func(i, j int) bool { + return bt.Instances[i].ID < bt.Instances[j].ID + }) + } + t.Modules = append(t.Modules, bt) + } + + sort.Slice(t.Modules, func(i, j int) bool { + return t.Modules[i].ID < t.Modules[j].ID + }) + + r, s, err := b.client.buildBatch(t) + if err != nil { + return nil, err + } + + state.importerImportContext.Set(s, importContext{ + name: s, + resourceGetter: nil, + scriptOptions: nil, + }) + + addResource(v.id, s, r, true) + } + + absPublishDir := b.client.d.AbsPublishDir + mediaTypes := b.client.d.ResourceSpec.MediaTypes() + cssMt, _, _ := mediaTypes.GetFirstBySuffix("css") + + cacheDir := filepath.Join(b.client.d.SourceSpec.Cfg.Dirs().CacheDir, "_jsbatch") + if err := os.Mkdir(cacheDir, 0o777); err != nil && !herrors.IsExist(err) { + return nil, err + } + outDir, err := os.MkdirTemp(cacheDir, "jsbatch") + if err != nil { + return nil, err + } + + var importResulveMu sync.Mutex + + externalOptions := b.config + externalOptions.Format = "esm" // Maybe allow other formats for simple 1 script setups. Also consider splitting below. + + jopts := js.Options{ + ExternalOptions: externalOptions, + InternalOptions: js.InternalOptions{ + OutDir: outDir, + Write: true, + AllowOverwrite: true, + Splitting: true, + ImportOnResolveFunc: func(imp string, args api.OnResolveArgs) string { + importResulveMu.Lock() + defer importResulveMu.Unlock() + + if _, found := state.importResource.Get(imp); found { + return imp + } + + var importContextPath string + if args.Kind == api.ResolveEntryPoint { + importContextPath = args.Path + } else { + importContextPath = args.Importer + } + importContext, _ := state.importerImportContext.Get(importContextPath) + + if importContext.resourceGetter != nil { + resolved := importContext.resourceGetter.Get(imp) + + if resolved != nil { + imp := hugoVirtualNS + resolved.(resource.PathProvider).Path() + state.importResource.Set(imp, resolved) + state.importerImportContext.Set(imp, importContext) // TODO1 test case (see headlessui) + return imp + + } + } + return "" + }, + ImportOnLoadFunc: func(args api.OnLoadArgs) string { + importResulveMu.Lock() + defer importResulveMu.Unlock() + + imp := args.Path + + if r, found := state.importResource.Get(imp); found { + content, err := r.(resource.ContentProvider).Content(context.Background()) // TODO1 + if err != nil { + panic(err) + } + return cast.ToString(content) + } + return "" + }, + ImportParamsOnLoadFunc: func(args api.OnLoadArgs) json.RawMessage { + if importContext, found := state.importerImportContext.Get(args.Path); found { + if importContext.scriptOptions != nil { + return importContext.scriptOptions.Params + } + } + return nil + }, + ErrorMessageResolveFunc: func(args api.Message) *js.ErrorMessageResolved { + if loc := args.Location; loc != nil { + path := strings.TrimPrefix(loc.File, "ns-hugo:") // TODO1 + if r, found := state.importResource.Get(path); found { + path = strings.TrimPrefix(path, hugoVirtualNS) + var contentr hugio.ReadSeekCloser + if cp, ok := r.(hugio.ReadSeekCloserProvider); ok { + contentr, _ = cp.ReadSeekCloser() + } + return &js.ErrorMessageResolved{ + Content: contentr, + Path: path, + Message: args.Text, + } + + } + + } + return nil + }, + EntryPoints: entryPoints, + }, + } + + result, err := b.client.client.BuildBundle(jopts) + if err != nil { + return nil, err + } + + cwd, err := os.Getwd() // TODO1 + if err != nil { + return nil, err + } + m := fromJSONToMeta(cwd, result.Metafile) + + groups := make(map[string]resource.Resources) + + // TODO1 + addFoo := func(filename, targetPath, group string, mt media.Type) error { + rd := resources.ResourceSourceDescriptor{ + LazyPublish: true, + OpenReadSeekCloser: func() (hugio.ReadSeekCloser, error) { + return os.Open(filename) + }, + MediaType: mt, + TargetPath: targetPath, + // DependencyManager: idm, TODO1 + } + r, err := b.client.d.ResourceSpec.NewResource(rd) + if err != nil { + return err + } + + groups[group] = append(groups[group], r) + + return nil + } + + createAndAddResource3 := func(o esBuildResultMetaOutput) (bool, error) { + p := filepath.ToSlash(strings.TrimPrefix(o.filename, outDir)) + ext := path.Ext(p) + mt, _, found := mediaTypes.GetBySuffix(ext) + if !found { + return false, nil + } + groupPath := p + group, found := state.pathGroup.Get(groupPath) + + if !found { + return false, nil + } + + if err := addFoo(o.filename, p, group, mt); err != nil { + // TODO1 + return false, err + } + + if o.CSSBundle != "" { + p := filepath.ToSlash(strings.TrimPrefix(o.CSSBundle, outDir)) + if err := addFoo(o.CSSBundle, p, group, cssMt); err != nil { + return false, err + } + } + + return true, nil + } + + for _, o := range m.Outputs { + handled, err := createAndAddResource3(o) + if err != nil { + return nil, err + } + if !handled { + // Copy to destination. + p := strings.TrimPrefix(o.filename, outDir) + if err := hugio.CopyFile(hugofs.Os, o.filename, filepath.Join(absPublishDir, p)); err != nil { + return nil, fmt.Errorf("failed to copy %q to %q: %w", o.filename, absPublishDir, err) + } + } + } + + return &Package{ + origin: b, + outDir: outDir, + b: b, + id: path.Join(nsBundle, b.id), + Groups: groups, + }, nil +} + +const nsBundle = "__hugo-js-bundle" + +func (ns *Namespace) buildBatch(t *batchTemplateContext) (resource.Resource, string, error) { + var buf bytes.Buffer + if err := batchEsmCallbackTemplate.Execute(&buf, t); err != nil { + return nil, "", err + } + + s := paths.AddLeadingSlash(t.keyPath + ".js") + r, err := ns.createClient.FromString(s, buf.String()) + if err != nil { + return nil, "", err + } + + return r, s, nil +} + +//go:embed batch-esm-callback.gotmpl +var batchEsmCallbackTemplateString string +var batchEsmCallbackTemplate *template.Template + +func init() { + batchEsmCallbackTemplate = template.Must(template.New("batch-esm-callback").Parse(batchEsmCallbackTemplateString)) +} + +func fromJSONToMeta(cwd, s string) esBuildResultMeta { + var m esBuildResultMeta + if err := json.Unmarshal([]byte(s), &m); err != nil { + panic(err) + } + if err := m.Compile(cwd); err != nil { + panic(err) + } + + return m +} + +type esBuildResultMeta struct { + Outputs map[string]esBuildResultMetaOutput + + // Compiled values. + cssBundleEntryPoint map[string]esBuildResultMetaOutput +} + +func (e *esBuildResultMeta) Compile(cwd string) error { + // Rewrite the paths to be absolute. + // See https://github.com/evanw/esbuild/issues/338 + outputs := make(map[string]esBuildResultMetaOutput) + for k, v := range e.Outputs { + filename := filepath.Join(cwd, k) + if err := v.Compile(filename); err != nil { + return err + } + if v.CSSBundle != "" { + v.CSSBundle = filepath.Join(cwd, v.CSSBundle) + } + outputs[filename] = v + } + e.Outputs = outputs + + e.cssBundleEntryPoint = make(map[string]esBuildResultMetaOutput) + for _, v := range e.Outputs { + if v.CSSBundle != "" { + e.cssBundleEntryPoint[v.CSSBundle] = v + } + } + return nil +} + +type esBuildResultMetaOutput struct { + Bytes int64 + Exports []string + Imports []esBuildResultMetaOutputImport + EntryPoint string + CSSBundle string + + // compiled values. + filename string +} + +func (e *esBuildResultMetaOutput) Compile(filename string) error { + e.filename = filename + return nil +} + +type esBuildResultMetaOutputImport struct { + Path string + Kind string +} + +// TODO1 remove the now superflous Close harness in the template package. diff --git a/tpl/js/batch_integration_test.go b/tpl/js/batch_integration_test.go new file mode 100644 index 00000000000..c4b3795ce6f --- /dev/null +++ b/tpl/js/batch_integration_test.go @@ -0,0 +1,432 @@ +// Copyright 2024 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package js provides functions for building JavaScript resources +package js_test + +import ( + "fmt" + "testing" + + "github.com/bep/logg" + "github.com/gohugoio/hugo/hugolib" +) + +// TODO1 fix shims vs headlessui. + +func TestBatch(t *testing.T) { + files := ` +-- hugo.toml -- +disableKinds = ["taxonomy", "term"] +disableLiveReload = true +baseURL = "https://example.com" + +# TOOD1 +[build] + [[build.cachebusters]] + source = '.*' + target = '.*' + +-- package.json -- +{ + "devDependencies": { + "react": "^18.3.1", + "react-dom": "^18.3.1" + } +} +-- assets/js/shims/react.js -- +-- assets/js/shims/react-dom.js -- +module.exports = window.ReactDOM; +module.exports = window.React; +-- content/mybundle/index.md -- +--- +title: "My Bundle" +--- +-- content/mybundle/bundlestyles.css -- +import './foo.css' +import './bar.css' +.bundlestyles { + background-color: blue; +} +-- content/mybundle/bundlereact.jsx -- +import * as React from "react"; +import './foo.css' +import './bundlestyles.css' +window.React1 = React; + +let text = 'Click me, too!' + +export default function MyBundleButton() { + return ( + + ) +} + +-- assets/js/reactcallback.js -- +import * as ReactDOM from 'react-dom/client'; +import * as React from 'react'; + +export default function Callback(modules) { + for (const module of modules) { + for (const instance of module.instances) { + /* This is a convention in this project. */ + let elId = §§${module.id}-${instance.id}§§; + let el = document.getElementById(elId); + if (!el) { + console.warn(§§Element with id ${elId} not found§§); + continue; + } + const root = ReactDOM.createRoot(el); + const reactEl = React.createElement(module.mod, instance.params); + root.render(reactEl); + } + } +} +-- assets/other/foo.css -- +@import './bar.css'; + +.foo { + background-color: blue; +} +-- assets/other/bar.css -- +.bar { + background-color: red; +} +-- assets/js/button.css -- +button { + background-color: red; +} +-- assets/js/bar.css -- +.bar-assets { + background-color: red; +} +-- assets/js/helper.js -- +import './bar.css' + +export function helper() { + console.log('helper'); +} + +-- assets/js/react1styles_nested.css -- +.react1styles_nested { + background-color: red; +} +-- assets/js/react1styles.css -- +@import './react1styles_nested.css'; +.react1styles { + background-color: red; +} +-- assets/js/react1.jsx -- +import * as React from "react"; +import './button.css' +import './foo.css' +import './bundlestyles.css' +import './react1styles.css' + +window.React1 = React; + +let text = 'Click me' + +export default function MyButton() { + return ( + + ) +} + +-- assets/js/react2.jsx -- +import * as React from "react"; +import { helper } from './helper.js' +import './foo.css' + +window.React2 = React; + +let text = 'Click me, too!' + +export default function MyOtherButton() { + return ( + + ) +} +-- assets/js/main1.js -- +import * as React from "react"; +import * as params from '@params'; + +console.log('main1.React', React) +console.log('main1.params.id', params.id) + +// TODO1 make it work without this. +export default function Main1() {}; + +-- assets/js/main2.js -- +import * as React from "react"; +import * as params from '@params'; + +console.log('main2.React', React) +console.log('main2.params.id', params.id) + +export default function Main2() {}; + +-- assets/js/main3.js -- +import * as React from "react"; +import * as params from '@params'; + +console.log('main3.params.id', params.id) + +export default function Main3() {}; + +-- layouts/_default/single.html -- +Single. +{{ $r := .Resources.GetMatch "*.jsx" }} +{{ $batch := (js.Batch "mybundle" site.Home.Store) }} +{{ $otherCSS := (resources.Match "/other/*.css").Mount "/other" "." }} + {{ with $batch.Config }} + {{ $shims := dict "react" "js/shims/react.js" "react-dom/client" "js/shims/react-dom.js" }} + {{ .SetOptions (dict + "target" "es2018" + "params" (dict "id" "config") + "shims" $shims + ) + }} +{{ end }} +{{ with $batch.Group "reactbatch" }} + {{ with .Script "r3" }} + {{ .SetOptions (dict + "resource" $r + "importContext" (slice $ $otherCSS) + "params" (dict "id" "r3") + ) + }} + {{ end }} + {{ with .Instance "r3" "r2i1" }} + {{ .SetOptions (dict "title" "r2 instance 1")}} + {{ end }} +{{ end }} +-- layouts/index.html -- +Home. +{{ with (templates.Defer (dict "key" "global")) }} +{{ $batch := (js.Batch "mybundle" site.Home.Store) }} +{{ range $k, $v := $batch.Build.Groups }} + {{ $k }}: + {{ range . }} + {{ .RelPermalink }} + {{ end }} + {{ end }} +{{ end }} +{{ $myContentBundle := site.GetPage "mybundle" }} +{{ $batch := (js.Batch "mybundle" site.Home.Store) }} +{{ $otherCSS := (resources.Match "/other/*.css").Mount "/other" "." }} +{{ with $batch.Group "mains" }} + {{ with .Script "main1" }} + {{ .SetOptions (dict + "resource" (resources.Get "js/main1.js") + "params" (dict "id" "main1") + ) + }} + {{ end }} + {{ with .Script "main2" }} + {{ .SetOptions (dict + "resource" (resources.Get "js/main2.js") + "params" (dict "id" "main2") + ) + }} + {{ end }} + {{ with .Script "main3" }} + {{ .SetOptions (dict + "resource" (resources.Get "js/main3.js") + ) + }} + {{ end }} +{{ with .Instance "main1" "m1i1" }}{{ .SetOptions (dict "title" "Main1 Instance 1")}}{{ end }} +{{ with .Instance "main1" "m1i2" }}{{ .SetOptions (dict "title" "Main1 Instance 2")}}{{ end }} +{{ end }} +{{ with $batch.Group "reactbatch" }} + {{ with .Callback "reactcallback" }} + {{ .SetOptions ( dict "resource" (resources.Get "js/reactcallback.js") )}} + {{ end }} + {{ with .Script "r1" }} + {{ .SetOptions (dict + "resource" (resources.Get "js/react1.jsx") + "importContext" (slice $myContentBundle $otherCSS) + "params" (dict "id" "r1") + ) + }} + {{ end }} + {{ with .Instance "r1" "i1" }}{{ .SetOptions (dict "title" "Instance 1")}}{{ end }} + {{ with .Instance "r1" "i2" }}{{ .SetOptions (dict "title" "Instance 2")}}{{ end }} + {{ with .Script "r2" }} + {{ .SetOptions (dict + "resource" (resources.Get "js/react2.jsx") + "importContext" $otherCSS + "params" (dict "id" "r2") + ) + }} + {{ end }} + {{ with .Instance "r2" "i1" }}{{ .SetOptions (dict "title" "Instance 2-1")}}{{ end }} +{{ end }} + +` + + b := hugolib.NewIntegrationTestBuilder( + hugolib.IntegrationTestConfig{ + T: t, + NeedsOsFS: true, + NeedsNpmInstall: true, + TxtarString: files, + Running: true, + LogLevel: logg.LevelWarn, + // PrintAndKeepTempDir: true, + }).Build() + + fmt.Println(b.LogString()) + + b.AssertFileContent("public/mybundle_reactbatch.css", + ".bar {", + ) + + // Verify params resolution. + b.AssertFileContent("public/mybundle_mains.js", + ` +var id = "main1"; +console.log("main1.params.id", id); +var id2 = "main2"; +console.log("main2.params.id", id2); + +# Params from top level config. +var id3 = "config"; +console.log("main3.params.id", id3); +`) + + b.AssertFileContent("public/mybundle_reactbatch.js", ` +"mod": MyButton, "id": "r1" +"mod": MyOtherButton, "id": "r2" +"mod": MyBundleButton, "id": "r3" + + `) + + b.EditFileReplaceAll("content/mybundle/bundlestyles.css", ".bundlestyles", ".bundlestyles-edit").Build() + b.AssertFileContent("public/mybundle_reactbatch.css", ".bundlestyles-edit {") + + b.EditFileReplaceAll("assets/other/bar.css", ".bar {", ".bar-edit {").Build() + b.AssertFileContent("public/mybundle_reactbatch.css", ".bar-edit {") + + b.EditFileReplaceAll("assets/other/bar.css", ".bar-edit {", ".bar-edit2 {").Build() + b.AssertFileContent("public/mybundle_reactbatch.css", ".bar-edit2 {") +} + +func TestEsBuildResolvePageBundle(t *testing.T) { + files := ` +-- hugo.toml -- +-- content/mybundle/index.md -- +--- +title: "My Bundle" +--- +-- content/mybundle/mystyles1.css -- +body { + background-color: blue; +} +-- content/mybundle/mystyles2.css -- +button { + background-color: red; +} +-- content/mybundle/myscript.js -- +import "./mystyles1.css"; +import "./mystyles2.css"; +console.log('Hello, world!'); + +// TODO1 make it work without this. +export default {}; +-- layouts/_default/single.html -- +Single. +TODO1 directory structure vs ID: +{{ $batch := (js.Batch "myjsbundle" .Store) }} +{{ $js := .Resources.GetMatch "*.js" }} +{{ with $batch.UseScriptGroup "g1" }} + {{ with .Script "s1" }} + {{ if not .GetImportContext }} + {{ .SetImportContext $ }} + {{ end }} + {{ if not .GetResource }} + {{ .SetResource $js }} + {{ end }} + {{ .AddInstance "i1" (dict "title" "Instance s1-1") }} + {{ end }} +{{ end }} +{{ range $batch.Build.Groups }} + {{ range $i, $e := . }} + {{ $i }}: {{ $e.RelPermalink }}| + {{ end }} +{{ end }} + +` + + // TODO1 check what happens without AddInstance. + + b := hugolib.Test(t, files, hugolib.TestOptWithOSFs()) + + b.AssertFileContent("public/myjsbundle_g1.css", "body", "button") + b.AssertFileContent("public/myjsbundle_g1.js", `Hello, world!`) +} + +// TODO1 executing "_default/single.html" at <$batch.Build.Groups>: error calling Build: Could not resolve "./mystyles.css"` error file source. + +// TODO1 move this. +func TestResourcesGet(t *testing.T) { + files := ` +-- hugo.toml -- +-- assets/text/txt1.txt -- +Text 1. +-- assets/text/txt2.txt -- +Text 2. +-- assets/text/sub/txt3.txt -- +Text 3. +-- assets/text/sub/txt4.txt -- +Text 4. +-- content/mybundle/index.md -- +--- +title: "My Bundle" +--- +-- content/mybundle/txt1.txt -- +Text 1. +-- content/mybundle/sub/txt2.txt -- +Text 1. +-- layouts/index.html -- +{{ $mybundle := site.GetPage "mybundle" }} +{{ $subResources := resources.Match "text/sub/*.*" }} + {{ $subResourcesMount := $subResources.Mount "newroot" }} +resources:text/txt1.txt:{{ with resources.Get "text/txt1.txt" }}{{ .Name }}{{ end }}| +resources:text/txt2.txt:{{ with resources.Get "text/txt2.txt" }}{{ .Name }}{{ end }}| +resources:text/sub/txt3.txt:{{ with resources.Get "text/sub/txt3.txt" }}{{ .Name }}{{ end }}| +subResources.range:{{ range $subResources }}{{ .Name }}|{{ end }}| +subResources:"text/sub/txt3.txt:{{ with $subResources.Get "text/sub/txt3.txt" }}{{ .Name }}{{ end }}| +subResourcesMount:newroot/txt3.txt:{{ with $subResourcesMount.Get "newroot/txt3.txt" }}{{ .Name }}{{ end }}| +page:txt1.txt:{{ with $mybundle.Resources.Get "txt1.txt" }}{{ .Name }}{{ end }}| +page:./txt1.txt:{{ with $mybundle.Resources.Get "./txt1.txt" }}{{ .Name }}{{ end }}| +page:sub/txt2.txt:{{ with $mybundle.Resources.Get "sub/txt2.txt" }}{{ .Name }}{{ end }}| +` + b := hugolib.Test(t, files) + + b.AssertFileContent("public/index.html", ` + asdf +resources:text/txt1.txt:/text/txt1.txt| +resources:text/txt2.txt:/text/txt2.txt| +resources:text/sub/txt3.txt:/text/sub/txt3.txt| +subResources:"text/sub/txt3.txt:/text/sub/txt3.txt| +subResourcesMount:newroot/txt3.txt:/text/sub/txt3.txt| +page:txt1.txt:txt1.txt| +page:./txt1.txt:txt1.txt| +page:sub/txt2.txt:sub/txt2.txt| +`) +} + +// TODO1 check .Name in bundles on renames. +// TODO1 https://esbuild.github.io/content-types/#local-css diff --git a/tpl/js/js.go b/tpl/js/js.go index c68e0af9272..ba4748a8331 100644 --- a/tpl/js/js.go +++ b/tpl/js/js.go @@ -17,9 +17,11 @@ package js import ( "errors" + "github.com/gohugoio/hugo/cache/dynacache" "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/resources" "github.com/gohugoio/hugo/resources/resource" + "github.com/gohugoio/hugo/resources/resource_factories/create" "github.com/gohugoio/hugo/resources/resource_transformers/babel" "github.com/gohugoio/hugo/resources/resource_transformers/js" "github.com/gohugoio/hugo/tpl/internal/resourcehelpers" @@ -31,15 +33,28 @@ func New(deps *deps.Deps) *Namespace { return &Namespace{} } return &Namespace{ - client: js.New(deps.BaseFs.Assets, deps.ResourceSpec), - babelClient: babel.New(deps.ResourceSpec), + d: deps, + bundlesCache: dynacache.GetOrCreatePartition[string, *Package]( + deps.MemCache, + "/jsb1", + // Mark it to clear on rebuild, but each package evaluate itself for changes. + dynacache.OptionsPartition{ClearWhen: dynacache.ClearOnRebuild, Weight: 10}, + ), + client: js.New(deps.BaseFs.Assets, deps.ResourceSpec), + createClient: create.New(deps.ResourceSpec), + babelClient: babel.New(deps.ResourceSpec), } } // Namespace provides template functions for the "js" namespace. type Namespace struct { - client *js.Client - babelClient *babel.Client + d *deps.Deps + + client *js.Client + createClient *create.Client + babelClient *babel.Client + + bundlesCache *dynacache.Partition[string, *Package] } // Build processes the given Resource with ESBuild.