diff --git a/.gitignore b/.gitignore
index e73e9fc..cdd6847 100644
--- a/.gitignore
+++ b/.gitignore
@@ -21,7 +21,6 @@ coverage.html
*.so
dist
/data
-/ctl/data
node_modules
package.json
package-lock.json
\ No newline at end of file
diff --git a/.goreleaser.yml b/.goreleaser.yml
index b322d0b..e84fa8b 100644
--- a/.goreleaser.yml
+++ b/.goreleaser.yml
@@ -1,11 +1,11 @@
-project_name: cloudcat
+project_name: ski
env:
- GO111MODULE=on
builds:
- - id: cloudcat
- main: ./cmd
+ - id: ski
+ main: ./ski
env:
- CGO_ENABLED=0
ldflags:
@@ -14,7 +14,7 @@ builds:
- -X main.CommitSHA={{ .ShortCommit }}
flags:
- -trimpath
- binary: cloudcat
+ binary: ski
goos:
- darwin
- linux
diff --git a/LICENSE b/LICENSE
index 0ad25db..a2bd002 100644
--- a/LICENSE
+++ b/LICENSE
@@ -1,661 +1,19 @@
- GNU AFFERO GENERAL PUBLIC LICENSE
- Version 3, 19 November 2007
-
- Copyright (C) 2007 Free Software Foundation, Inc.
- Copyright (C)
-
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU Affero General Public License as published
- by the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU Affero General Public License for more details.
-
- You should have received a copy of the GNU Affero General Public License
- along with this program. If not, see ", "(c, code)=>eval(code)", false)
- callable, err := rt.RunProgram(program)
- if err != nil {
- panic(err)
- }
- call, ok := goja.AssertFunction(callable)
- if !ok {
- panic("err init executor")
- }
- return func(c any, a string) (any, error) {
- value, err := call(goja.Undefined(), rt.ToValue(c), rt.ToValue(a))
- if err != nil {
- return nil, err
- }
- if value == nil {
- return nil, nil
- }
- return value.Export(), nil
- }
-}
-
-type ap struct {
- eval func(any, string) (any, error)
-}
-
-func (p *ap) GetString(_ *plugin.Context, c any, a string) (string, error) {
- v, err := p.eval(c, a)
- if err != nil {
- return "", err
- }
- if v == nil {
- return "", nil
- }
- if s, ok := v.(string); ok {
- return s, nil
- }
- bytes, err := json.Marshal(v)
- if err != nil {
- return "", err
- }
- return string(bytes), nil
-}
-
-func (p *ap) GetStrings(_ *plugin.Context, c any, a string) ([]string, error) {
- v, err := p.eval(c, a)
- if err != nil {
- return nil, err
- }
- slice, ok := v.([]any)
- if !ok {
- slice = []any{v}
- }
- ret := make([]string, len(slice))
- for i, v := range slice {
- if s, ok := v.(string); ok {
- ret[i] = s
- } else {
- bytes, _ := json.Marshal(v)
- ret[i] = string(bytes)
- }
- }
- return ret, nil
-}
-
-func (p *ap) GetElement(_ *plugin.Context, c any, a string) (string, error) {
- return p.GetString(nil, c, a)
-}
-
-func (p *ap) GetElements(_ *plugin.Context, c any, a string) ([]string, error) {
- return p.GetStrings(nil, c, a)
-}
-
-func TestAnalyzer(t *testing.T) {
- ctx := plugin.NewContext(plugin.ContextOptions{})
- parser.Register("ap", &ap{eval()})
- testCases := []struct {
- schema string
- want any
- }{
- {
- `
-{ ap: '"foo"' }
-`, `"foo"`,
- },
- {
- `
-- ap: "null"
-- or
-- ap: '"foo"'
-`, `"foo"`,
- },
- {
- `
-- ap: "null"
-- or
-- ap: "null"
-- or
-- ap: '"foo"'
-`, `"foo"`,
- },
- {
- `
-- ap: '"foo"'
-- and
-- ap: '"bar"'
-`, `"foobar"`,
- },
- {
- `
-- ap: '"foo"'
-- and
-- ap: '"bar"'
-- and
-- ap: '"aaa"'
-`, `"foobaraaa"`,
- },
- {
- `
-type: integer
-rule: { ap: '1' }
-`, 1,
- },
- {
- `
-type: boolean
-rule: { ap: '1' }
-`, true,
- },
- {
- `
-type: number
-rule: { ap: '2.1' }
-`, 2.1,
- },
- {
- `
-type: array
-rule:
- - ap: '1'
- - and
- - ap: '2'
-`, `["1","2"]`,
- },
- {
- `
-type: object
-properties:
- string: { ap: '"str"' }
- integer: !integer { ap: '1' }
- number: !number { ap: '1.1' }
- boolean: !boolean { ap: '1' }
- array: !array { ap: "[\"i1\", \"i2\"]" }
- object: !object { ap: "({\"foo\":\"bar\"})" }
-`, `{"array":["i1","i2"],"boolean":true,"integer":1,"number":1.1,"object":{"foo":"bar"},"string":"str"}`,
- },
- {
- `
-type: object
-format: number
-rule: { ap: '({"foo":"1.1"})' }
-`, `{"foo":1.1}`,
- },
- {
- `
-type: array
-properties:
- n: !number { ap: '12' }
-`, `[{"n":12}]`,
- },
- {
- `
-type: array
-format: number
-rule: { ap: "1" }
-`, `[1]`,
- },
- {
- `
-type: object
-properties:
- ? ap: '"k"'
- : ap: '"v"'
-`, `{"k":"v"}`,
- },
- {
- `
-type: object
-properties:
- $key: { ap: '"k"' }
- $value: { ap: '"v"' }
-`, `{"k":"v"}`,
- },
- {
- `
-type: object
-init: { ap: "[1,2,3]" }
-properties:
- ? ap: c
- : ap: c + 1
-`, `{"1":"11", "2":"21", "3":"31"}`,
- },
- {
- `
-type: object
-init: { ap: '["a","b","c",1,2,3]' }
-properties:
- $key: { ap: c }
- $value: { ap: c }
-`, `{"1":"1", "2":"2", "3":"3", "a":"a", "b":"b", "c":"c"}`,
- },
- {
- `
-type: object
-properties:
- num: !integer { ap: '2' }
- msg: { ap: '"foooo"' }
- $after: { ap: c.num = c.num + 1; c.msg = "hello" }
-`, `{"num":3,"msg":"hello"}`,
- },
- }
- for i, testCase := range testCases {
- t.Run(strconv.Itoa(i), func(t *testing.T) {
- s := new(Schema)
- err := yaml.Unmarshal([]byte(testCase.schema), s)
- if err != nil {
- t.Fatal(err)
- }
- result := Analyze(ctx, s, "")
- if want, ok := testCase.want.(string); ok {
- bytes, err := json.Marshal(result)
- assert.NoError(t, err)
- assert.JSONEq(t, want, string(bytes))
- return
- }
- assert.Equal(t, testCase.want, result)
- })
- }
-}
-
-func TestFormat(t *testing.T) {
- t.Parallel()
- formatter := new(defaultFormatHandler)
- testCases := []struct {
- data any
- typ Type
- want any
- }{
- {"", StringType, ""},
- {"1", StringType, "1"},
- {"2.1", NumberType, 2.1},
- {"", NumberType, nil},
- {"3", IntegerType, 3},
- {"1", BooleanType, true},
- {"", BooleanType, nil},
- {`{"k":"v"}`, ObjectType, map[string]any{"k": "v"}},
- {`[1,2]`, ArrayType, []any{1.0, 2.0}},
- {[]string{"1", "2"}, IntegerType, []any{1, 2}},
- {map[string]any{"k": "1"}, IntegerType, map[string]any{"k": 1}},
- }
-
- for i, testCase := range testCases {
- t.Run(fmt.Sprintf("Cases %v", i), func(t *testing.T) {
- got, err := formatter.Format(testCase.data, testCase.typ)
- assert.NoError(t, err)
- assert.Equal(t, testCase.want, got)
- })
- }
-
- errCases := []struct {
- data any
- typ Type
- want any
- }{
- {"9-", IntegerType, nil},
- {"114", BooleanType, nil},
- {[]string{"1", "?"}, IntegerType, nil},
- {map[string]any{"k": "!"}, NumberType, nil},
- }
-
- for i, testCase := range errCases {
- t.Run(fmt.Sprintf("Err cases %v", i), func(t *testing.T) {
- got, err := formatter.Format(testCase.data, testCase.typ)
- assert.Error(t, err)
- assert.Equal(t, testCase.want, got)
- })
- }
-}
diff --git a/cache.go b/cache.go
index 4717ecd..f739ed7 100644
--- a/cache.go
+++ b/cache.go
@@ -1,35 +1,30 @@
-package cloudcat
+package ski
import (
"context"
"sync"
"time"
- "github.com/shiroyk/cloudcat/plugin"
"github.com/spf13/cast"
)
// A Cache interface is used to store bytes.
type Cache interface {
- Get(ctx context.Context, key string) ([]byte, bool)
- Set(ctx context.Context, key string, value []byte)
- Del(ctx context.Context, key string)
+ Get(ctx context.Context, key string) ([]byte, error)
+ Set(ctx context.Context, key string, value []byte) error
+ Del(ctx context.Context, key string) error
}
-type cacheTimeoutKey struct{}
+var cacheTimeoutKey byte
// WithCacheTimeout returns the context with the cache timeout.
func WithCacheTimeout(ctx context.Context, timeout time.Duration) context.Context {
- if c, ok := ctx.(*plugin.Context); ok {
- c.SetValue(cacheTimeoutKey{}, timeout)
- return ctx
- }
- return context.WithValue(ctx, cacheTimeoutKey{}, timeout)
+ return WithValue(ctx, &cacheTimeoutKey, timeout)
}
-// CacheTimeout returns the context cache timeout value.
+// CacheTimeout returns the context cache timeout values.
func CacheTimeout(ctx context.Context) time.Duration {
- return cast.ToDuration(ctx.Value(cacheTimeoutKey{}))
+ return cast.ToDuration(ctx.Value(&cacheTimeoutKey))
}
// memoryCache is an implementation of Cache that stores bytes in in-memory.
@@ -40,38 +35,40 @@ type memoryCache struct {
}
// Get returns the []byte and true, if not existing returns false.
-func (c *memoryCache) Get(_ context.Context, key string) ([]byte, bool) {
+func (c *memoryCache) Get(_ context.Context, key string) ([]byte, error) {
c.Lock()
defer c.Unlock()
if ddl, exist := c.timeout[key]; exist {
if time.Now().Unix() > ddl {
delete(c.items, key)
delete(c.timeout, key)
- return []byte{}, false
+ return []byte{}, nil
}
}
if b, ok := c.items[key]; ok {
- return b, true
+ return b, nil
}
- return []byte{}, false
+ return nil, nil
}
// Set saves []byte to the cache with key
-func (c *memoryCache) Set(ctx context.Context, key string, value []byte) {
+func (c *memoryCache) Set(ctx context.Context, key string, value []byte) error {
c.Lock()
+ defer c.Unlock()
c.items[key] = value
if timeout := CacheTimeout(ctx); timeout > 0 {
c.timeout[key] = time.Now().Add(timeout).Unix()
}
- c.Unlock()
+ return nil
}
// Del removes key from the cache
-func (c *memoryCache) Del(_ context.Context, key string) {
+func (c *memoryCache) Del(_ context.Context, key string) error {
c.Lock()
+ defer c.Unlock()
delete(c.items, key)
delete(c.timeout, key)
- c.Unlock()
+ return nil
}
// NewCache returns a new Cache that will store items in in-memory.
diff --git a/cache_test.go b/cache_test.go
index 893a1d4..b95687c 100644
--- a/cache_test.go
+++ b/cache_test.go
@@ -1,4 +1,4 @@
-package cloudcat
+package ski
import (
"context"
@@ -14,26 +14,24 @@ func TestCache(t *testing.T) {
ctx := context.Background()
key, value := "testCacheKey", "testCacheValue"
- if _, ok := c.Get(ctx, key); ok {
- t.Fatal("retrieved value before adding it")
+ if v, _ := c.Get(ctx, key); len(v) != 0 {
+ t.Fatal("retrieved values before adding it")
}
- c.Set(ctx, key, []byte(value))
+ _ = c.Set(ctx, key, []byte(value))
v, _ := c.Get(ctx, key)
assert.Equal(t, value, string(v))
- c.Del(ctx, key)
- if _, ok := c.Get(ctx, key); ok {
- t.Fatal("delete failed")
- }
+ _ = c.Del(ctx, key)
+ v, _ = c.Get(ctx, key)
+ assert.Empty(t, v)
- c.Set(WithCacheTimeout(ctx, time.Millisecond), key, []byte(value))
+ _ = c.Set(WithCacheTimeout(ctx, time.Millisecond), key, []byte(value))
v1, _ := c.Get(ctx, key)
assert.Equal(t, value, string(v1))
time.Sleep(1 * time.Second)
- if _, ok := c.Get(ctx, key); ok {
- t.Fatalf("not expired: %v", key)
- }
+ v, _ = c.Get(ctx, key)
+ assert.Empty(t, v, "not expired: %v", key)
}
diff --git a/cmd/README.md b/cmd/README.md
deleted file mode 100644
index aa8e45b..0000000
--- a/cmd/README.md
+++ /dev/null
@@ -1,58 +0,0 @@
-# Cloudcat
-
-## Usage
-run the **Model**
-```shell
-cat << EOF | ./cloudcat -d -m -
-source:
- name: HackerNews
- http: https://news.ycombinator.com/best
-schema:
- type: array
- init:
- - gq: "#hnmain tbody -> slice(2) -> child('tr:not(.spacer,.morespace,:last-child)')"
- js: |
- content?.reduce((acc, v, i, arr) => {
- if (i % 2 === 0) {
- acc.push(arr.slice(i, i + 2).join(''));
- }
- return acc;
- }, [])
- properties:
- index: !integer
- - gq: .rank
- regex: /[^\d]/
- title: { gq: .titleline>:first-child }
- by: { gq: .hnuser }
- age: { gq: .age }
- comments: !integer
- - gq: .subline>:last-child
- regex: /[^\d]/
-EOF
-```
-run the **Script**
-```shell
-cat << EOF | ./cloudcat -d -s -
-const http = require('cloudcat/http');
-let res = http.get('https://news.ycombinator.com/best');
-let stories = cat.getElements('gq', "#hnmain tbody -> slice(2) -> child('tr:not(.spacer,.morespace,:last-child)')", res.string());
-stories?.reduce((acc, v, i, arr) => {
- if (i % 2 === 0) {
- let item = arr.slice(i, i + 2).join('');
- let index = cat.getString('gq', '.rank', item);
- let title = cat.getString('gq', '.titleline>:first-child', item);
- let by = cat.getString('gq', '.hnuser', item);
- let age = cat.getString('gq', '.age', item);
- let comments = cat.getString('gq', '.subline>:last-child', item);
- acc.push({
- index: parseInt(index?.replace(/[^\d]+/g, ''), 10),
- title: title,
- by: by,
- age: age,
- comments: parseInt(comments?.replace(/[^\d]+/g, ''), 10)
- });
- }
- return acc;
-}, []);
-EOF
-```
\ No newline at end of file
diff --git a/cmd/main.go b/cmd/main.go
deleted file mode 100644
index e2dc641..0000000
--- a/cmd/main.go
+++ /dev/null
@@ -1,239 +0,0 @@
-package main
-
-import (
- "bufio"
- "encoding/json"
- "errors"
- "flag"
- "fmt"
- "io"
- "net/http"
- urlpkg "net/url"
- "os"
- "path/filepath"
- "strings"
-
- "log/slog"
-
- "github.com/shiroyk/cloudcat"
- "github.com/shiroyk/cloudcat/js"
- _ "github.com/shiroyk/cloudcat/js/modules"
- _ "github.com/shiroyk/cloudcat/parsers"
- "github.com/shiroyk/cloudcat/plugin"
- "gopkg.in/yaml.v3"
-)
-
-// Model the model
-type Model struct {
- Source struct {
- Name string `yaml:"name"`
- HTTP string `yaml:"http"`
- Proxy string `yaml:"proxy"`
- } `yaml:"source"`
- Schema *cloudcat.Schema `yaml:"schema"`
-}
-
-var (
- scriptFlag = flag.String("s", "", "run script")
- modelFlag = flag.String("m", "", "run model")
- timeoutFlag = flag.Duration("t", plugin.DefaultTimeout, "run timeout")
- debugFlag = flag.Bool("d", false, "output the debug log")
- outputFlag = flag.String("o", "", "write to file instead of stdout")
- pluginFlag = flag.String("p", "", "plugin directory path")
- versionFlag = flag.Bool("v", false, "output version")
-)
-
-func runModel() (err error) {
- var bytes []byte
- if *modelFlag == "-" {
- bytes, err = io.ReadAll(os.Stdin)
- } else {
- bytes, err = os.ReadFile(*modelFlag) //nolint:gosec
- }
- if err != nil {
- return
- }
-
- var model Model
- err = yaml.Unmarshal(bytes, &model)
- if err != nil {
- return
- }
-
- if model.Source.HTTP == "" || model.Schema == nil {
- return errors.New("model is invalid")
- }
-
- timeout := plugin.DefaultTimeout
- if timeoutFlag != nil {
- timeout = *timeoutFlag
- }
-
- requestURI := model.Source.HTTP
- if _, urlErr := urlpkg.Parse(requestURI); urlErr == nil {
- requestURI = fmt.Sprintf("GET %s HTTP/1.1\n\n", requestURI)
- }
-
- req, err := http.ReadRequest(bufio.NewReader(strings.NewReader(requestURI)))
- if err != nil {
- return err
- }
- req.RequestURI = ""
-
- ctx := plugin.NewContext(plugin.ContextOptions{
- Timeout: timeout,
- Logger: slog.New(loggerHandler()),
- URL: req.URL.String(),
- })
- defer ctx.Cancel()
-
- fetch := cloudcat.MustResolve[cloudcat.Fetch]()
-
- if model.Source.Proxy != "" {
- url, err := urlpkg.Parse(model.Source.Proxy)
- if err != nil {
- return err
- }
- req = req.WithContext(cloudcat.WithProxyURL(ctx, url))
- } else {
- req = req.WithContext(ctx)
- }
-
- res, err := fetch.Do(req)
- if err != nil {
- return err
- }
-
- defer res.Body.Close()
- body, err := io.ReadAll(res.Body)
- if err != nil {
- return err
- }
-
- return outputJSON(cloudcat.Analyze(ctx, model.Schema, string(body)))
-}
-
-func runScript() (err error) {
- var bytes []byte
- if *scriptFlag == "-" {
- bytes, err = io.ReadAll(os.Stdin)
- } else {
- bytes, err = os.ReadFile(*scriptFlag) //nolint:gosec
- }
- if err != nil {
- return
- }
-
- timeout := plugin.DefaultTimeout
- if timeoutFlag != nil {
- timeout = *timeoutFlag
- }
-
- ctx := plugin.NewContext(plugin.ContextOptions{
- Timeout: timeout,
- Logger: slog.New(loggerHandler()),
- })
- defer ctx.Cancel()
-
- value, err := js.RunString(ctx, string(bytes))
- if err != nil {
- return err
- }
-
- return outputJSON(value)
-}
-
-func loggerHandler() slog.Handler {
- opt := new(slog.HandlerOptions)
- if *debugFlag {
- opt.Level = slog.LevelDebug
- }
- return slog.NewTextHandler(os.Stdout, opt)
-}
-
-func outputJSON(data any) (err error) {
- bytes, err := json.MarshalIndent(data, "", "\t")
- if err != nil {
- return err
- }
-
- if *outputFlag == "" {
- fmt.Println(string(bytes)) //nolint:forbidigo
- return
- }
-
- ext := filepath.Ext(*outputFlag)
- if ext == "" {
- *outputFlag += ".json"
- }
- return os.WriteFile(*outputFlag, bytes, 0o600)
-}
-
-// expandPath expands path "." or "~"
-func expandPath(path string) (string, error) {
- // expand local directory
- if strings.HasPrefix(path, ".") {
- cwd, err := os.Getwd()
- if err != nil {
- return "", err
- }
- return filepath.Join(cwd, path[1:]), nil
- }
- // expand ~ as shortcut for home directory
- if strings.HasPrefix(path, "~") {
- home, err := os.UserHomeDir()
- if err != nil {
- return "", err
- }
- return filepath.Join(home, path[1:]), nil
- }
- return path, nil
-}
-
-func main() {
- flag.Parse()
-
- if *versionFlag {
- fmt.Println(fmt.Sprintf("cloudcat %v/%v", Version, CommitSHA))
- os.Exit(0)
- return
- }
-
- cloudcat.Provide(cloudcat.NewCache())
- cloudcat.Provide(cloudcat.NewCookie())
- cloudcat.ProvideLazy[cloudcat.Fetch](func() (cloudcat.Fetch, error) {
- transport := http.DefaultTransport.(*http.Transport)
- transport.Proxy = cloudcat.ProxyFromRequest
- client := &http.Client{Transport: transport}
- return client, nil
- })
-
- if pluginFlag != nil && *pluginFlag != "" {
- pluginPath, err := expandPath(*pluginFlag)
- if err != nil {
- return
- }
- size, err := plugin.LoadPlugin(pluginPath)
- if err != nil {
- if size == 0 {
- panic(fmt.Sprintf("failed to load external modules: %v", err))
- } else {
- slog.Warn(fmt.Sprintf("failed to load some external modules :%v", err))
- }
- }
- }
-
- if scriptFlag != nil && *scriptFlag != "" {
- if err := runScript(); err != nil {
- fmt.Println(err.Error())
- os.Exit(1)
- }
- } else if modelFlag != nil && *modelFlag != "" {
- if err := runModel(); err != nil {
- fmt.Println(err.Error())
- os.Exit(1)
- }
- } else {
- flag.Usage()
- }
-}
diff --git a/context.go b/context.go
new file mode 100644
index 0000000..e22a0f9
--- /dev/null
+++ b/context.go
@@ -0,0 +1,64 @@
+package ski
+
+import (
+ "context"
+ "maps"
+ "sync"
+)
+
+// Context multiple values context
+type Context interface {
+ context.Context
+ // SetValue store key with value
+ SetValue(key, value any)
+}
+
+type valuesCtx struct {
+ context.Context
+ mu sync.RWMutex
+ values map[any]any
+}
+
+func (c *valuesCtx) Value(key any) any {
+ c.mu.RLock()
+ v, ok := c.values[key]
+ c.mu.RUnlock()
+ if ok {
+ return v
+ }
+ return c.Context.Value(key)
+}
+
+func (c *valuesCtx) SetValue(key, value any) {
+ c.mu.Lock()
+ c.values[key] = value
+ c.mu.Unlock()
+}
+
+var _ctxKey byte
+
+// NewContext returns a new can store multiple values context with values
+func NewContext(parent context.Context, values map[any]any) Context {
+ if parent == nil {
+ panic("cannot create context from nil parent")
+ }
+ var clone map[any]any
+ if values == nil {
+ clone = make(map[any]any)
+ } else {
+ clone = maps.Clone(values)
+ }
+ ctx := &valuesCtx{Context: parent, values: clone}
+ clone[&_ctxKey] = ctx
+ return ctx
+}
+
+// WithValue if parent exists multiple values Context then set the key/value.
+// or returns a copy of parent in which the value associated with key is val.
+func WithValue(ctx context.Context, key, value any) context.Context {
+ if v, ok := ctx.Value(&_ctxKey).(*valuesCtx); ok {
+ v.SetValue(key, value)
+ return ctx
+ }
+ return context.WithValue(ctx, key, value)
+}
diff --git a/context_test.go b/context_test.go
new file mode 100644
index 0000000..83beeff
--- /dev/null
+++ b/context_test.go
@@ -0,0 +1,28 @@
+package ski
+
+import (
+ "context"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestNewContext(t *testing.T) {
+ timeout, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+
+ ctx := NewContext(timeout, nil)
+ assert.Nil(t, ctx.Value("key1"))
+
+ ctx.SetValue("key1", "value1")
+ assert.Equal(t, "value1", ctx.Value("key1"))
+
+ var key2 byte
+ ctx.SetValue(&key2, "value2")
+ assert.Equal(t, "value2", ctx.Value(&key2))
+
+ WithValue(context.WithValue(ctx, "key3", "value3"), "key4", "value4")
+
+ assert.Equal(t, "value4", ctx.Value("key4"))
+}
diff --git a/cookie.go b/cookie.go
index 0873f2f..ae8b7ce 100644
--- a/cookie.go
+++ b/cookie.go
@@ -1,4 +1,4 @@
-package cloudcat
+package ski
import (
"net/http"
@@ -6,30 +6,23 @@ import (
"net/url"
)
-// Cookie manages storage and use of cookies in HTTP requests.
-// Implementations of Cookie must be safe for concurrent use by multiple
+// CookieJar manages storage and use of cookies in HTTP requests.
+// Implementations of CookieJar must be safe for concurrent use by multiple
// goroutines.
-type Cookie interface {
+type CookieJar interface {
http.CookieJar
- // CookieString returns the cookies string for the given URL.
- CookieString(u *url.URL) []string
- // DeleteCookie delete the cookies for the given URL.
- DeleteCookie(u *url.URL)
+ // RemoveCookie delete the cookies for the given URL.
+ RemoveCookie(u *url.URL)
}
-// memoryCookie is an implementation of Cookie that stores http.Cookie in in-memory.
+// memoryCookie is an implementation of CookieJar that stores http.Cookie in in-memory.
type memoryCookie struct {
*cookiejar.Jar
}
-// CookieString returns the cookies string for the given URL.
-func (c *memoryCookie) CookieString(u *url.URL) []string {
- return CookieToString(c.Cookies(u))
-}
-
-// DeleteCookie delete the cookies for the given URL.
-func (c *memoryCookie) DeleteCookie(u *url.URL) {
+// RemoveCookie remove the cookies for the given URL.
+func (c *memoryCookie) RemoveCookie(u *url.URL) {
exists := c.Cookies(u)
cookie := make([]*http.Cookie, 0, len(exists))
for _, e := range exists {
@@ -39,11 +32,8 @@ func (c *memoryCookie) DeleteCookie(u *url.URL) {
c.SetCookies(u, cookie)
}
-// NewCookie returns a new Cookie that will store cookies in in-memory.
-func NewCookie() Cookie {
- jar, err := cookiejar.New(nil)
- if err != nil {
- panic(err)
- }
+// NewCookieJar returns a new CookieJar that will store cookies in in-memory.
+func NewCookieJar() CookieJar {
+ jar, _ := cookiejar.New(nil)
return &memoryCookie{jar}
}
diff --git a/cookie_test.go b/cookie_test.go
index fd74989..024deca 100644
--- a/cookie_test.go
+++ b/cookie_test.go
@@ -1,6 +1,7 @@
-package cloudcat
+package ski
import (
+ "net/http"
"net/url"
"testing"
@@ -9,17 +10,13 @@ import (
func TestCookie(t *testing.T) {
t.Parallel()
- c := NewCookie()
+ c := NewCookieJar()
u, _ := url.Parse("https://github.com")
- if len(c.Cookies(u)) > 0 {
- t.Fatal("retrieved cookie before adding it")
- }
-
- raw := "has_recent_activity=1; path=/; secure; HttpOnly; SameSite=Lax"
- c.SetCookies(u, ParseSetCookie(raw))
- assert.Equal(t, []string{"has_recent_activity=1"}, c.CookieString(u))
- c.DeleteCookie(u)
+ cookies := []*http.Cookie{{Name: "has_recent_activity", Value: "1", Path: "/", Secure: true}}
+ c.SetCookies(u, cookies)
+ assert.NotNil(t, c.Cookies(u))
+ c.RemoveCookie(u)
assert.Nil(t, c.Cookies(u))
}
diff --git a/di.go b/di.go
deleted file mode 100644
index fa53e1a..0000000
--- a/di.go
+++ /dev/null
@@ -1,143 +0,0 @@
-package cloudcat
-
-import (
- "fmt"
- "reflect"
- "sync"
- "sync/atomic"
-)
-
-// di a simple dependencies injection
-// Inspired by https://github.com/samber/do
-
-var diServices = new(sync.Map)
-
-type lazyService[T any] struct {
- load atomic.Bool
- instance T
- initFunc func() (T, error)
-}
-
-func (s *lazyService[T]) initOrGet() (instance T, err error) {
- if s.load.Load() {
- return s.instance, nil
- }
- if !s.load.Swap(true) {
- s.instance, err = s.initFunc()
- if err != nil {
- s.load.Store(false)
- }
- }
- return s.instance, err
-}
-
-// Provide save the value and return is it saved
-func Provide[T any](value T) bool {
- return ProvideNamed(getName[T](), value)
-}
-
-// ProvideLazy save the lazy init value and return is it saved
-func ProvideLazy[T any](initFunc func() (T, error)) bool {
- return ProvideNamed(getName[T](), &lazyService[T]{initFunc: initFunc})
-}
-
-// ProvideNamed save the value for the name and return is it saved
-func ProvideNamed(name string, value any) (ok bool) {
- _, ok = diServices.LoadOrStore(name, value)
- return !ok
-}
-
-// Override save the value and return is it override
-func Override[T any](value T) bool {
- return OverrideNamed(getName[T](), value)
-}
-
-// OverrideLazy save the value for the name and return is it override
-func OverrideLazy[T any](initFunc func() (T, error)) bool {
- return OverrideNamed(getName[T](), &lazyService[T]{initFunc: initFunc})
-}
-
-// OverrideNamed save the value for the name and return is it override
-func OverrideNamed(name string, value any) (ok bool) {
- _, ok = diServices.Swap(name, value)
- return ok
-}
-
-// Resolve get the value, if not exist returns error
-func Resolve[T any]() (T, error) {
- return ResolveNamed[T](getName[T]())
-}
-
-// ResolveLazy get the value lazy once, if not exist returns error
-func ResolveLazy[T any]() func() (T, error) {
- var (
- once sync.Once
- value T
- err error
- )
- g := func() {
- value, err = Resolve[T]()
- }
- return func() (T, error) {
- once.Do(g)
- return value, err
- }
-}
-
-// ResolveNamed get the value for the name if not exist returns error
-func ResolveNamed[T any](name string) (value T, err error) {
- if v, exists := diServices.Load(name); exists {
- switch target := v.(type) {
- case *lazyService[T]:
- return target.initOrGet()
- case T:
- return target, nil
- }
- return value, fmt.Errorf("%T type assertion to %T error", v, value)
- }
-
- return value, fmt.Errorf("%s not declared", name)
-}
-
-// MustResolve get the value, if not exist create panic
-func MustResolve[T any]() T {
- value, err := Resolve[T]()
- if err != nil {
- panic(err)
- }
- return value
-}
-
-// MustResolveLazy get the value lazy once, if not exist create panic
-func MustResolveLazy[T any]() func() T {
- g := ResolveLazy[T]()
- return func() T {
- value, err := g()
- if err != nil {
- panic(err)
- }
- return value
- }
-}
-
-// MustResolveNamed get the value for the name, if not exist create panic
-func MustResolveNamed[T any](name string) T {
- value, err := ResolveNamed[T](name)
- if err != nil {
- panic(err)
- }
- return value
-}
-
-// getName returns the type name
-func getName[T any]() string {
- var v T
-
- // struct
- if t := reflect.TypeOf(v); t != nil {
- return t.String()
- }
-
- // interface
- return reflect.TypeOf(new(T)).String()
-}
diff --git a/di_test.go b/di_test.go
deleted file mode 100644
index 83cb0a0..0000000
--- a/di_test.go
+++ /dev/null
@@ -1,110 +0,0 @@
-package cloudcat
-
-import (
- "errors"
- "testing"
-
- "github.com/stretchr/testify/assert"
-)
-
-func TestProvide(t *testing.T) {
- t.Parallel()
-
- type test1 interface{}
- Provide(new(test1))
- _, err := Resolve[test1]()
- assert.NoError(t, err)
-}
-
-func TestProvideLazy(t *testing.T) {
- t.Parallel()
-
- times := 0
- type test2 interface{}
- ProvideLazy(func() (test2, error) {
- if times == 0 {
- times++
- return nil, errors.New("something")
- }
- return new(test2), nil //nolint:nilnil
- })
- v, _ := Resolve[test2]()
- assert.Nil(t, v)
- v, _ = Resolve[test2]()
- assert.NotNil(t, v)
-}
-
-func TestResolve(t *testing.T) {
- t.Parallel()
-
- type test3 struct{}
- Provide(test3{})
- _, err := Resolve[test3]()
- assert.NoError(t, err)
-}
-
-func TestResolveLazy(t *testing.T) {
- t.Parallel()
-
- type test7 struct{}
- Provide(test7{})
- f := ResolveLazy[test7]()
- _, err := f()
- assert.NoError(t, err)
-
- times := 0
- type test8 interface{}
- ProvideLazy(func() (test8, error) {
- if times == 0 {
- times++
- return nil, errors.New("something")
- }
- return new(test8), nil //nolint:nilnil
- })
- f2 := ResolveLazy[test8]()
- v, _ := f2()
- assert.Nil(t, v)
- v, _ = f2()
- assert.Nil(t, v)
-}
-
-func TestMustResolve(t *testing.T) {
- t.Parallel()
-
- type test4 interface{}
- Provide(new(test4))
- MustResolve[test4]()
-}
-
-func TestMustResolveLazy(t *testing.T) {
- t.Parallel()
- defer func() {
- r := recover()
- assert.NotNil(t, r)
- assert.ErrorContains(t, r.(error), "test10 not declared")
- }()
-
- type test9 interface{}
- Provide(new(test9))
- assert.NotNil(t, MustResolveLazy[test9]()())
- type test10 interface{}
- assert.NotNil(t, MustResolveLazy[test10]()())
-}
-
-func TestMustResolveNamed(t *testing.T) {
- t.Parallel()
-
- type test5 struct{}
- assert.True(t, ProvideNamed("named1", test5{}))
- assert.False(t, ProvideNamed("named1", test5{}))
- MustResolveNamed[test5]("named1")
-}
-
-func TestOverride(t *testing.T) {
- t.Parallel()
-
- type test6 struct{}
- assert.False(t, Override(test6{}))
- assert.True(t, Override(test6{}))
- MustResolve[test6]()
-}
diff --git a/executor.go b/executor.go
new file mode 100644
index 0000000..0defe8f
--- /dev/null
+++ b/executor.go
@@ -0,0 +1,165 @@
+package ski
+
+import (
+ "context"
+ "fmt"
+ "slices"
+ "strings"
+ "sync"
+ "unicode"
+)
+
+type (
+ // Executor accept the argument and output result.
+ // when the parameter is a slice, it needs to be wrapped with Iterator.
+ Executor interface {
+ Exec(context.Context, any) (any, error)
+ }
+ // NewExecutor to create a new Executor
+ NewExecutor func(...Executor) (Executor, error)
+)
+
+// Register registers the NewExecutor with the given name.
+// Valid name: [a-zA-Z_][a-zA-Z0-9_]* (leading and trailing underscores are allowed)
+func Register(name string, fn NewExecutor) {
+ if name == "" {
+ panic("ski: invalid pattern")
+ }
+ if fn == nil {
+ panic("ski: NewExecutor is nil")
+ }
+ if !isValidName(name) {
+ panic(fmt.Sprintf("ski: invalid name %q", name))
+ }
+
+ executors.Lock()
+ defer executors.Unlock()
+
+ name, method, _ := strings.Cut(name, ".")
+ entries := executors.registry[name]
+ executors.registry[name] = append(entries, entry{fn, method})
+}
+
+// GetExecutor returns a NewExecutor with the given name
+func GetExecutor(name string) (NewExecutor, bool) {
+ executors.RLock()
+ defer executors.RUnlock()
+
+ name, method, _ := strings.Cut(name, ".")
+ entries, ok := executors.registry[name]
+ if !ok {
+ return nil, false
+ }
+ for _, entry := range entries {
+ if entry.method == method {
+ return entry.new, true
+ }
+ }
+ return nil, false
+}
+
+// GetExecutors returns the all NewExecutor with the given name
+func GetExecutors(name string) (map[string]NewExecutor, bool) {
+ executors.RLock()
+ defer executors.RUnlock()
+
+ name, _, _ = strings.Cut(name, ".")
+ entries, ok := executors.registry[name]
+ if !ok {
+ return nil, false
+ }
+ ret := make(map[string]NewExecutor, len(entries))
+ for _, entry := range entries {
+ ret[entry.method] = entry.new
+ }
+ return ret, true
+}
+
+// RemoveExecutor removes an Executor with the given name
+func RemoveExecutor(name string) {
+ executors.Lock()
+ defer executors.Unlock()
+
+ name, method, _ := strings.Cut(name, ".")
+ entries, ok := executors.registry[name]
+ if !ok {
+ return
+ }
+
+ if method == "" {
+ delete(executors.registry, name)
+ return
+ }
+
+ newEntries := slices.DeleteFunc(entries, func(e entry) bool {
+ return e.method == method
+ })
+
+ if len(newEntries) == 0 {
+ delete(executors.registry, name)
+ } else {
+ executors.registry[name] = newEntries
+ }
+}
+
+// AllExecutors returns the all NewExecutor
+func AllExecutors() map[string]NewExecutor {
+ executors.RLock()
+ defer executors.RUnlock()
+
+ ret := make(map[string]NewExecutor)
+ for name, entries := range executors.registry {
+ for _, entry := range entries {
+ if entry.method == "" {
+ ret[name] = entry.new
+ } else {
+ ret[name+"."+entry.method] = entry.new
+ }
+ }
+ }
+ return ret
+}
+
+func isValidName(s string) bool {
+ if s == "" {
+ return false
+ }
+ if !unicode.IsLetter(rune(s[0])) && s[0] != '_' {
+ return false
+ }
+ hasDot := false
+ for i := 0; i < len(s); i++ {
+ char := rune(s[i])
+ if char == '.' {
+ if hasDot {
+ return false
+ }
+ hasDot = true
+ if i == len(s)-1 {
+ return false
+ }
+ next := s[i+1]
+ if !unicode.IsLetter(rune(next)) && next != '_' {
+ return false
+ }
+ i++
+ continue
+ }
+ if !unicode.IsLetter(char) && !unicode.IsDigit(char) && char != '_' {
+ return false
+ }
+ }
+ return true
+}
+
+type entry struct {
+ new NewExecutor
+ method string
+}
+
+var executors = struct {
+ sync.RWMutex
+ registry map[string][]entry
+}{
+ registry: make(map[string][]entry),
+}
diff --git a/executor_test.go b/executor_test.go
new file mode 100644
index 0000000..ec93363
--- /dev/null
+++ b/executor_test.go
@@ -0,0 +1,29 @@
+package ski
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestValidName(t *testing.T) {
+ testsCases := []struct {
+ name string
+ ok bool
+ }{
+ {"foo", true},
+ {"foo.bar", true},
+ {"_.bar_", true},
+ {"_foo.bar_", true},
+ {"foo123.bar123", true},
+ {"123foo.bar123", false},
+ {"foo.bar.baz", false},
+ {"foo.bar.baz.", false},
+ {"foo.bar.baz..", false},
+ {"foo.bar.baz.", false},
+ }
+
+ for _, testCase := range testsCases {
+ assert.Equal(t, testCase.ok, isValidName(testCase.name), testCase.name)
+ }
+}
diff --git a/fetch.go b/fetch.go
index 5fe0396..8ffac7a 100644
--- a/fetch.go
+++ b/fetch.go
@@ -1,11 +1,11 @@
-package cloudcat
+package ski
import (
"context"
+ "net"
"net/http"
"net/url"
-
- "github.com/shiroyk/cloudcat/plugin"
+ "time"
)
// Fetch http client interface
@@ -16,23 +16,42 @@ type Fetch interface {
Do(*http.Request) (*http.Response, error)
}
-type requestProxyKey struct{}
+// NewFetch return the http.Client implementation
+func NewFetch() Fetch {
+ return &http.Client{
+ Transport: &http.Transport{
+ Proxy: ProxyFromRequest,
+ DialContext: (&net.Dialer{
+ Timeout: 30 * time.Second,
+ KeepAlive: 30 * time.Second,
+ }).DialContext,
+ ForceAttemptHTTP2: true,
+ MaxIdleConns: 100,
+ IdleConnTimeout: 90 * time.Second,
+ TLSHandshakeTimeout: 10 * time.Second,
+ ExpectContinueTimeout: 1 * time.Second,
+ },
+ Jar: NewCookieJar(),
+ }
+}
+
+var requestProxyKey byte
// WithProxyURL returns a copy of parent context in which the proxy associated with context.
func WithProxyURL(ctx context.Context, proxy *url.URL) context.Context {
if proxy == nil {
return ctx
}
- if c, ok := ctx.(*plugin.Context); ok {
- c.SetValue(requestProxyKey{}, proxy)
- return ctx
+ if c, ok := ctx.(Context); ok {
+ c.SetValue(&requestProxyKey, proxy)
+ return c
}
- return context.WithValue(ctx, requestProxyKey{}, proxy)
+ return context.WithValue(ctx, &requestProxyKey, proxy)
}
// ProxyFromContext returns a proxy URL on context.
func ProxyFromContext(ctx context.Context) *url.URL {
- if proxy := ctx.Value(requestProxyKey{}); proxy != nil {
+ if proxy := ctx.Value(&requestProxyKey); proxy != nil {
return proxy.(*url.URL)
}
return nil
diff --git a/go.mod b/go.mod
index 272c98e..73d9a3b 100644
--- a/go.mod
+++ b/go.mod
@@ -1,30 +1,27 @@
-module github.com/shiroyk/cloudcat
+module github.com/shiroyk/ski
go 1.21
require (
- github.com/PuerkitoBio/goquery v1.8.1
- github.com/antchfx/htmlquery v1.3.0
- github.com/dlclark/regexp2 v1.10.0
- github.com/dop251/goja v0.0.0-20231027120936-b396bb4c349d
- github.com/ohler55/ojg v1.21.1
- github.com/shiroyk/cloudcat/plugin v0.4.0
+ github.com/PuerkitoBio/goquery v1.9.2
+ github.com/andybalholm/cascadia v1.3.2
+ github.com/antchfx/htmlquery v1.3.2
+ github.com/antchfx/xpath v1.3.1
+ github.com/dlclark/regexp2 v1.11.2
+ github.com/grafana/sobek v0.0.0-20240711133011-3a280d337ef4
+ github.com/ohler55/ojg v1.23.0
github.com/spf13/cast v1.6.0
- github.com/stretchr/testify v1.8.4
- golang.org/x/crypto v0.18.0
- golang.org/x/net v0.20.0
+ github.com/stretchr/testify v1.9.0
+ golang.org/x/crypto v0.25.0
+ golang.org/x/net v0.27.0
gopkg.in/yaml.v3 v3.0.1
)
require (
- github.com/andybalholm/cascadia v1.3.2 // indirect
- github.com/antchfx/xpath v1.2.5 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
- github.com/go-sourcemap/sourcemap v2.1.4-0.20211119122758-180fcef48034+incompatible // indirect
+ github.com/go-sourcemap/sourcemap v2.1.4+incompatible // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
- github.com/google/pprof v0.0.0-20231101202521-4ca4178f5c7a // indirect
+ github.com/google/pprof v0.0.0-20240711041743-f6c9dda6c6da // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
- golang.org/x/text v0.14.0 // indirect
+ golang.org/x/text v0.16.0 // indirect
)
-
-replace github.com/shiroyk/cloudcat/plugin => ./plugin
diff --git a/go.sum b/go.sum
index ebf171f..aaf8004 100644
--- a/go.sum
+++ b/go.sum
@@ -1,116 +1,85 @@
-github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM=
-github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ=
-github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
+github.com/PuerkitoBio/goquery v1.9.2 h1:4/wZksC3KgkQw7SQgkKotmKljk0M6V8TUvA8Wb4yPeE=
+github.com/PuerkitoBio/goquery v1.9.2/go.mod h1:GHPCaP0ODyyxqcNoFGYlAprUFH81NuRPd0GX3Zu2Mvk=
github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss=
github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
-github.com/antchfx/htmlquery v1.3.0 h1:5I5yNFOVI+egyia5F2s/5Do2nFWxJz41Tr3DyfKD25E=
-github.com/antchfx/htmlquery v1.3.0/go.mod h1:zKPDVTMhfOmcwxheXUsx4rKJy8KEY/PU6eXr/2SebQ8=
-github.com/antchfx/xpath v1.2.3/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=
-github.com/antchfx/xpath v1.2.5 h1:hqZ+wtQ+KIOV/S3bGZcIhpgYC26um2bZYP2KVGcR7VY=
-github.com/antchfx/xpath v1.2.5/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=
-github.com/chzyer/logex v1.2.0/go.mod h1:9+9sk7u7pGNWYMkh0hdiL++6OeibzJccyQU4p4MedaY=
-github.com/chzyer/readline v1.5.0/go.mod h1:x22KAscuvRqlLoK9CsoYsmxoXZMMFVyOl86cAH8qUic=
-github.com/chzyer/test v0.0.0-20210722231415-061457976a23/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
-github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/antchfx/htmlquery v1.3.2 h1:85YdttVkR1rAY+Oiv/nKI4FCimID+NXhDn82kz3mEvs=
+github.com/antchfx/htmlquery v1.3.2/go.mod h1:1mbkcEgEarAokJiWhTfr4hR06w/q2ZZjnYLrDt6CTUk=
+github.com/antchfx/xpath v1.3.1 h1:PNbFuUqHwWl0xRjvUPjJ95Agbmdj2uzzIwmQKgu4oCk=
+github.com/antchfx/xpath v1.3.1/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
-github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
-github.com/dlclark/regexp2 v1.10.0 h1:+/GIL799phkJqYW+3YbOd8LCcbHzT0Pbo8zl70MHsq0=
-github.com/dlclark/regexp2 v1.10.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
-github.com/dop251/goja v0.0.0-20211022113120-dc8c55024d06/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk=
-github.com/dop251/goja v0.0.0-20231027120936-b396bb4c349d h1:wi6jN5LVt/ljaBG4ue79Ekzb12QfJ52L9Q98tl8SWhw=
-github.com/dop251/goja v0.0.0-20231027120936-b396bb4c349d/go.mod h1:QMWlm50DNe14hD7t24KEqZuUdC9sOTy8W6XbCU1mlw4=
-github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y=
-github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d/go.mod h1:DngW8aVqWbuLRMHItjPUyqdj+HWPvnQe8V8y1nDpIbM=
+github.com/dlclark/regexp2 v1.11.2 h1:/u628IuisSTwri5/UKloiIsH8+qF2Pu7xEQX+yIKg68=
+github.com/dlclark/regexp2 v1.11.2/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
-github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg=
-github.com/go-sourcemap/sourcemap v2.1.4-0.20211119122758-180fcef48034+incompatible h1:bopx7t9jyUNX1ebhr0G4gtQWmUOgwQRI0QsYhdYLgkU=
-github.com/go-sourcemap/sourcemap v2.1.4-0.20211119122758-180fcef48034+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg=
+github.com/go-sourcemap/sourcemap v2.1.4+incompatible h1:a+iTbH5auLKxaNwQFg0B+TCYl6lbukKPc7b5x0n1s6Q=
+github.com/go-sourcemap/sourcemap v2.1.4+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg=
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/google/pprof v0.0.0-20230207041349-798e818bf904/go.mod h1:uglQLonpP8qtYCYyzA+8c/9qtqgA3qsXGYqCPKARAFg=
-github.com/google/pprof v0.0.0-20231101202521-4ca4178f5c7a h1:fEBsGL/sjAuJrgah5XqmmYsTLzJp/TO9Lhy39gkverk=
-github.com/google/pprof v0.0.0-20231101202521-4ca4178f5c7a/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik=
-github.com/ianlancetaylor/demangle v0.0.0-20220319035150-800ac71e25c2/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
-github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
-github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
-github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
+github.com/google/pprof v0.0.0-20240711041743-f6c9dda6c6da h1:xRmpO92tb8y+Z85iUOMOicpCfaYcv7o3Cg3wKrIpg8g=
+github.com/google/pprof v0.0.0-20240711041743-f6c9dda6c6da/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo=
+github.com/grafana/sobek v0.0.0-20240711133011-3a280d337ef4 h1:SKC348XXnCe9EIsAJ+xs5lzlZbzRsrGkqVbJ3451p3k=
+github.com/grafana/sobek v0.0.0-20240711133011-3a280d337ef4/go.mod h1:tUEHKWaMrxFGrMgjeAH85OEceCGQiSl6a/6Wckj/Vf4=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
-github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/ohler55/ojg v1.21.1 h1:b2RLUaDcy9gvn46dmhTjezu/TDauoR0/kgKTqkwIxto=
-github.com/ohler55/ojg v1.21.1/go.mod h1:gQhDVpQLqrmnd2eqGAvJtn+NfKoYJbe/A4Sj3/Vro4o=
+github.com/ohler55/ojg v1.23.0 h1:xjJasLaKf4dKkyJq0CNXQMRdL7F1172tms885aPKcS0=
+github.com/ohler55/ojg v1.23.0/go.mod h1:gQhDVpQLqrmnd2eqGAvJtn+NfKoYJbe/A4Sj3/Vro4o=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0=
github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo=
-github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
-github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc=
-golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg=
+golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30=
+golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
-golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo=
-golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY=
+golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys=
+golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
-golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
-golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
-golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
-golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
+golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
-gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
diff --git a/parsers/gq/README.md b/gq/README.md
similarity index 100%
rename from parsers/gq/README.md
rename to gq/README.md
diff --git a/parsers/gq/buildin_function.go b/gq/buildin_function.go
similarity index 62%
rename from parsers/gq/buildin_function.go
rename to gq/buildin_function.go
index 219d36c..230252b 100644
--- a/parsers/gq/buildin_function.go
+++ b/gq/buildin_function.go
@@ -1,31 +1,31 @@
package gq
import (
+ "context"
"errors"
"fmt"
"net/url"
"strings"
"github.com/PuerkitoBio/goquery"
- "github.com/shiroyk/cloudcat/plugin"
+ "github.com/shiroyk/ski"
"github.com/spf13/cast"
+ "golang.org/x/net/html"
)
type (
- // GFunc is the type of gq parse function.
- GFunc func(ctx *plugin.Context, content any, args ...string) (any, error)
+ // Func is the type of gq parse function.
+ Func func(ctx context.Context, content any, args ...string) (any, error)
// FuncMap is the type of the map defining the mapping from names to functions.
- FuncMap map[string]GFunc
+ FuncMap map[string]Func
)
func builtins() FuncMap {
return FuncMap{
- "get": Get,
- "set": Set,
+ "zip": Zip,
"attr": Attr,
"href": Href,
"html": Html,
- "join": Join,
"prev": Prev,
"text": Text,
"next": Next,
@@ -50,101 +50,37 @@ func contentToString(content any, fn func(*goquery.Selection) (string, error)) (
list[i] = result
return true
})
- if len(list) == 1 {
+ switch len(list) {
+ case 0:
+ return nil, nil
+ case 1:
return list[0], nil
+ default:
+ return ski.NewIterator(list), nil
}
- return list, nil
- case string:
- return c, nil
- case []string:
- if len(c) == 1 {
- return c[0], nil
- }
+ case *html.Node:
+ return fn(goquery.NewDocumentFromNode(c).Selection)
+ case string, ski.Iterator:
return c, nil
+ case nil:
+ return nil, nil
default:
return nil, fmt.Errorf("unexpected type %T", content)
}
}
-// Get returns the value associated with this context for key, or nil
-// if no value is associated with key.
-func Get(ctx *plugin.Context, _ any, args ...string) (any, error) {
- if len(args) == 0 {
- return nil, fmt.Errorf("get function must has one argment")
- }
-
- key, err := cast.ToStringE(args[0])
- if err != nil {
- return nil, err
- }
-
- return ctx.Value(key), nil
-}
-
-// Set value associated with key is val.
-// The first argument is the key, and the second argument is value.
-// if the value is present will store the content.
-func Set(ctx *plugin.Context, content any, args ...string) (any, error) {
- if len(args) == 0 {
- return nil, fmt.Errorf("set function must has least one argment")
- }
-
- key, err := cast.ToStringE(args[0])
- if err != nil {
- return nil, err
- }
-
- if len(args) > 1 {
- ctx.SetValue(key, args[1])
- } else {
- ctx.SetValue(key, content)
- }
-
- return content, nil
-}
-
// Text gets the combined text contents of each element in the set of matched
// elements, including their descendants.
-func Text(_ *plugin.Context, content any, _ ...string) (any, error) {
+func Text(_ context.Context, content any, _ ...string) (any, error) {
return contentToString(content, func(node *goquery.Selection) (string, error) {
return strings.TrimSpace(node.Text()), nil
})
}
-// Join the text with the separator, if not present separator uses the default separator ", ".
-func Join(ctx *plugin.Context, content any, args ...string) (any, error) {
- if str, ok := content.(string); ok {
- return str, nil
- }
-
- if node, ok := content.(*goquery.Selection); ok {
- text, err := Text(ctx, node)
- if err != nil {
- return nil, err
- }
- if str, ok := text.(string); ok {
- return str, nil
- }
- content = text
- }
-
- list, err := cast.ToStringSliceE(content)
- if err != nil {
- return nil, err
- }
-
- sep := ", "
- if len(args) > 0 {
- sep = args[0]
- }
-
- return strings.Join(list, sep), nil
-}
-
// Attr gets the specified attribute's value for the first element in the
// Selection.
// The first argument is the name of the attribute, the second is the default value
-func Attr(_ *plugin.Context, content any, args ...string) (any, error) {
+func Attr(_ context.Context, content any, args ...string) (any, error) {
if len(args) == 0 {
return "", fmt.Errorf("attr(name) must has name")
}
@@ -161,33 +97,43 @@ func Attr(_ *plugin.Context, content any, args ...string) (any, error) {
}
// Href gets the href attribute's value, if URL is not absolute returns the absolute URL.
-func Href(ctx *plugin.Context, content any, args ...string) (any, error) {
+func Href(ctx context.Context, content any, args ...string) (any, error) {
if node, ok := content.(*goquery.Selection); ok {
- var path string
href, exists := node.Attr("href")
if !exists {
return nil, errors.New("href attribute's value is not exist")
}
- if len(args) > 0 {
- path = args[0]
- if !strings.HasSuffix(path, "/") {
- path = path + "/"
- }
- href = strings.TrimPrefix(href, "/")
- }
- hrefURL, err := url.Parse(href)
- if err != nil {
- return nil, err
+ if strings.HasPrefix(href, "http://") || strings.HasPrefix(href, "https://") {
+ return href, nil
}
+ var base string
- baseURL, err := url.Parse(ctx.BaseURL())
- if err != nil {
- return nil, err
+ if v := ctx.Value("baseURL"); v != nil {
+ base = v.(string)
+ } else if len(args) > 0 {
+ base = args[0]
+ }
+ if len(base) > 0 {
+ if !strings.HasPrefix(href, ".") {
+ if !strings.HasPrefix(href, "/") {
+ href = "/" + href
+ }
+ return strings.TrimSuffix(base, "/") + href, nil
+ }
+ hrefURL, err := url.Parse(href)
+ if err != nil {
+ return nil, err
+ }
+ baseURL, err := url.Parse(base)
+ if err != nil {
+ return nil, err
+ }
+ return baseURL.ResolveReference(hrefURL).String(), nil
}
- return baseURL.JoinPath(path).ResolveReference(hrefURL).String(), nil
+ return href, nil
}
- return nil, fmt.Errorf("unexpected content type %T", content)
+ return nil, fmt.Errorf("href: unexpected content type %T", content)
}
// Html the first argument is outer.
@@ -195,7 +141,7 @@ func Href(ctx *plugin.Context, content any, args ...string) (any, error) {
// the selection - that is, the HTML including the first element's
// tag and attributes, or gets the HTML contents of the first element
// in the set of matched elements. It includes text and comment nodes;
-func Html(_ *plugin.Context, content any, args ...string) (any, error) { //nolint
+func Html(_ context.Context, content any, args ...string) (any, error) { //nolint
var err error
var outer bool
@@ -229,7 +175,7 @@ func Html(_ *plugin.Context, content any, args ...string) (any, error) { //nolin
// Selection.
// If present selector gets all preceding siblings of each element up to but not
// including the element matched by the selector.
-func Prev(_ *plugin.Context, content any, args ...string) (any, error) {
+func Prev(_ context.Context, content any, args ...string) (any, error) {
if node, ok := content.(*goquery.Selection); ok {
if len(args) > 0 {
return node.PrevUntil(args[0]), nil
@@ -237,14 +183,14 @@ func Prev(_ *plugin.Context, content any, args ...string) (any, error) {
return node.Prev(), nil
}
- return nil, fmt.Errorf("unexpected content type %T", content)
+ return nil, fmt.Errorf("prev: unexpected content type %T", content)
}
// Next gets the immediately following sibling of each element in the
// Selection.
// If present selector gets all following siblings of each element up to but not
// including the element matched by the selector.
-func Next(_ *plugin.Context, content any, args ...string) (any, error) {
+func Next(_ context.Context, content any, args ...string) (any, error) {
if node, ok := content.(*goquery.Selection); ok {
if len(args) > 0 {
return node.NextUntil(args[0]), nil
@@ -252,7 +198,7 @@ func Next(_ *plugin.Context, content any, args ...string) (any, error) {
return node.Next(), nil
}
- return nil, fmt.Errorf("unexpected type %T", content)
+ return nil, fmt.Errorf("next: unexpected type %T", content)
}
// Slice reduces the set of matched elements to a subset specified by a range
@@ -265,7 +211,7 @@ func Next(_ *plugin.Context, content any, args ...string) (any, error) {
//
// The indices may be negative, in which case they represent an offset from the
// end of the selection.
-func Slice(_ *plugin.Context, content any, args ...string) (any, error) {
+func Slice(_ context.Context, content any, args ...string) (any, error) {
if len(args) == 0 {
return nil, fmt.Errorf("slice(start, end) must have at least one int argument")
}
@@ -286,12 +232,12 @@ func Slice(_ *plugin.Context, content any, args ...string) (any, error) {
return node.Eq(start), nil
}
- return nil, fmt.Errorf("unexpected type %T", content)
+ return nil, fmt.Errorf("slice: unexpected type %T", content)
}
// Child gets the child elements of each element in the Selection.
// If present the selector will return filtered by the specified selector.
-func Child(_ *plugin.Context, content any, args ...string) (any, error) {
+func Child(_ context.Context, content any, args ...string) (any, error) {
if node, ok := content.(*goquery.Selection); ok {
if len(args) > 0 {
return node.ChildrenFiltered(args[0]), nil
@@ -299,12 +245,12 @@ func Child(_ *plugin.Context, content any, args ...string) (any, error) {
return node.Children(), nil
}
- return nil, fmt.Errorf("unexpected type %T", content)
+ return nil, fmt.Errorf("child: unexpected type %T", content)
}
// Parent gets the parent of each element in the Selection.
// if present the selector will return filtered by a selector.
-func Parent(_ *plugin.Context, content any, args ...string) (any, error) {
+func Parent(_ context.Context, content any, args ...string) (any, error) {
if node, ok := content.(*goquery.Selection); ok {
if len(args) > 0 {
return node.ParentFiltered(args[0]), nil
@@ -312,12 +258,12 @@ func Parent(_ *plugin.Context, content any, args ...string) (any, error) {
return node.Parent(), nil
}
- return nil, fmt.Errorf("unexpected type %T", content)
+ return nil, fmt.Errorf("parent: unexpected type %T", content)
}
// Parents gets the ancestors of each element in the current Selection.
// if present the selector will return filtered by a selector.
-func Parents(_ *plugin.Context, content any, args ...string) (any, error) {
+func Parents(_ context.Context, content any, args ...string) (any, error) {
if node, ok := content.(*goquery.Selection); ok { //nolint:nestif
if len(args) > 0 {
if len(args) > 1 {
@@ -334,21 +280,66 @@ func Parents(_ *plugin.Context, content any, args ...string) (any, error) {
return node.Parents(), nil
}
- return nil, fmt.Errorf("unexpected type %T", content)
+ return nil, fmt.Errorf("parents: unexpected type %T", content)
}
-func Prefix(_ *plugin.Context, content any, args ...string) (ret any, err error) {
+// Zip returns an element array of first selector element length,
+// the first of which contains the first elements of the given selector,
+// the second of which contains the second elements of the given selector, and so on.
+func Zip(_ context.Context, content any, args ...string) (any, error) {
+ sel, ok := content.(*goquery.Selection)
+ if !ok {
+ return nil, fmt.Errorf("zip: unexpected type %T", content)
+ }
+
+ if len(args) == 0 {
+ return nil, fmt.Errorf("zip(selector) must have at least one string argument")
+ }
+
+ first := sel.Find(args[0])
+ length := first.Length()
+ zip := make([]string, 0, length*len(args))
+ first.Each(func(i int, s *goquery.Selection) {
+ html, _ := goquery.OuterHtml(s)
+ zip = append(zip, html)
+ })
+
+ for _, arg := range args[1:] {
+ sel.Find(arg).Each(func(i int, s *goquery.Selection) {
+ html, _ := goquery.OuterHtml(s)
+ zip = append(zip, html)
+ })
+ }
+
+ ret := make([]string, 0, length)
+ for i := 0; i < length; i++ {
+ var s string
+ for j := 0; j < len(args); j++ {
+ s += zip[i+j*length]
+ }
+ ret = append(ret, s)
+ }
+
+ return ret, nil
+}
+
+func Prefix(_ context.Context, content any, args ...string) (ret any, err error) {
if len(args) == 0 {
return content, nil
}
switch src := content.(type) {
case string:
return args[0] + src, nil
- case []string:
- for i := range src {
- src[i] = args[0] + src[i]
+ case ski.Iterator:
+ ret := make([]string, src.Len())
+ for i := 0; i < src.Len(); i++ {
+ if s, ok := src.At(i).(string); ok {
+ ret[i] = args[0] + s
+ } else {
+ return nil, fmt.Errorf("prefix: unexpected type %T", src.At(i))
+ }
}
- return src, nil
+ return ret, nil
case *goquery.Selection:
return args[0] + src.Text(), nil
default:
@@ -356,18 +347,23 @@ func Prefix(_ *plugin.Context, content any, args ...string) (ret any, err error)
}
}
-func Suffix(_ *plugin.Context, content any, args ...string) (ret any, err error) {
+func Suffix(_ context.Context, content any, args ...string) (any, error) {
if len(args) == 0 {
return content, nil
}
switch src := content.(type) {
case string:
return src + args[0], nil
- case []string:
- for i := range src {
- src[i] = src[i] + args[0]
+ case ski.Iterator:
+ ret := make([]string, src.Len())
+ for i := 0; i < src.Len(); i++ {
+ if s, ok := src.At(i).(string); ok {
+ ret[i] = s + args[0]
+ } else {
+ return nil, fmt.Errorf("suffix: unexpected type %T", src.At(i))
+ }
}
- return src, nil
+ return ret, nil
case *goquery.Selection:
return src.Text() + args[0], nil
default:
diff --git a/gq/buildin_function_test.go b/gq/buildin_function_test.go
new file mode 100644
index 0000000..bb3e99a
--- /dev/null
+++ b/gq/buildin_function_test.go
@@ -0,0 +1,136 @@
+package gq
+
+import (
+ "testing"
+)
+
+func TestBuildInFuncText(t *testing.T) {
+ t.Parallel()
+
+ assertValue(t, `#main #n1 -> text`, "1")
+
+ assertValue(t, `#main #n1`, "1")
+}
+
+func TestBuildInFuncAttr(t *testing.T) {
+ t.Parallel()
+ assertError(t, `#main #n1 -> text -> attr`, "attr(name) must has name")
+
+ assertError(t, `#main -> attr()`, "attr(name) must has name")
+
+ assertValue(t, `#main #n1 -> attr(class)`, "one even row")
+
+ assertValue(t, `#main #n1 -> attr(empty, default)`, "default")
+}
+
+func TestBuildInFuncHref(t *testing.T) {
+ t.Parallel()
+ assertError(t, `.body ul #a4 -> text -> href`, "unexpected content type string")
+
+ assertValue(t, `.body ul #a4 a -> href(https://localhost)`, "https://localhost/home")
+
+ assertValue(t, `.body ul #a4 a -> href(https://localhost/path/)`, "https://localhost/path/home")
+}
+
+func TestBuildInFuncHtml(t *testing.T) {
+ t.Parallel()
+ assertError(t, `.body -> html(test)`, "html(outer) `outer` must bool type value: true/false")
+
+ assertValue(t, `.body ul a -> html`, []string{"Google", "Github", "Golang", "Home"})
+
+ assertValue(t, `.body ul a -> slice(0,2) -> html(true)`,
+ []string{
+ "Google ",
+ "Github "})
+}
+
+func TestBuildInFuncPrev(t *testing.T) {
+ t.Parallel()
+ assertError(t, `#foot #nf3 -> text -> prev`, "unexpected content type string")
+
+ assertValue(t, `#foot #nf3 -> prev`, "f2")
+
+ assertValue(t, `#foot #nf3 -> prev(#nf1)`, "f2")
+}
+
+func TestBuildInFuncNext(t *testing.T) {
+ t.Parallel()
+ assertError(t, `#foot #nf2 -> text -> next`, "unexpected type string")
+
+ assertValue(t, `#foot #nf2 -> next`, "f3")
+
+ assertValue(t, `#foot #nf2 -> next(#nf4)`, "f3")
+}
+
+func TestBuildInFuncSlice(t *testing.T) {
+ t.Parallel()
+ assertError(t, `#main -> slice`, "slice(start, end) must have at least one int argument")
+
+ assertError(t, `#main div -> text -> slice(0)`, "slice: unexpected type")
+
+ assertValue(t, `#main div -> slice(0)`, "1")
+
+ assertValue(t, `#main div -> slice(-1)`, "6")
+
+ assertValue(t, `#main div -> slice(0, 3)`, []string{"1", "2", "3"})
+
+ assertValue(t, `#main div -> slice(0, -2)`, []string{"1", "2", "3", "4"})
+}
+
+func TestBuildInFuncChild(t *testing.T) {
+ t.Parallel()
+ assertError(t, `.body ul -> text -> child`, "unexpected type string")
+
+ assertValue(t, `.body ul li -> child(a)`, []string{"Google", "Github", "Golang", "Home"})
+
+ assertValue(t, `.body ul li -> child`, []string{"Google", "Github", "Golang", "Home"})
+}
+
+func TestBuildInFuncParent(t *testing.T) {
+ t.Parallel()
+ assertError(t, `.body ul -> text -> parent`, "unexpected type string")
+
+ assertValue(t, `.body ul a -> parent(#a1) -> attr(id)`, "a1")
+
+ assertValue(t, `.body ul a -> parent -> attr(id)`, []string{"a1", "a2", "a3", "a4"})
+}
+
+func TestBuildInFuncParents(t *testing.T) {
+ t.Parallel()
+ assertError(t, `.body ul -> text -> parents`, "unexpected type string")
+
+ assertError(t, `.body ul .selected -> parents(div, test)`, "parents(selector, until) `until` must bool type value: true/false")
+
+ assertValue(t, `.body ul .selected -> parents(div, true) -> attr(id)`, "url")
+
+ assertValue(t, `.body ul .selected -> parents -> slice(0) -> attr(id)`, "url")
+}
+
+func TestBuildInFuncPrefix(t *testing.T) {
+ t.Parallel()
+
+ assertValue(t, `#main #n1 -> text -> prefix(A)`, "A1")
+
+ assertValue(t, `#main #n1 -> prefix(B)`, "B1")
+}
+
+func TestBuildInFuncSuffix(t *testing.T) {
+ t.Parallel()
+
+ assertValue(t, `#main #n1 -> text -> suffix(A)`, "1A")
+
+ assertValue(t, `#main #n1 -> suffix(B)`, "1B")
+}
+
+func TestBuildInZip(t *testing.T) {
+ t.Parallel()
+
+ assertElements(t, `-> zip('#main div', '#foot div')`, []string{
+ `1
f1
`,
+ `2
f2
`,
+ `3
f3
`,
+ `4
f4
`,
+ `5
f5
`,
+ `6
f6
`,
+ })
+}
diff --git a/gq/gq.go b/gq/gq.go
new file mode 100644
index 0000000..6440a8d
--- /dev/null
+++ b/gq/gq.go
@@ -0,0 +1,248 @@
+// Package gq the goquery executor
+package gq
+
+import (
+ "context"
+ "fmt"
+ "maps"
+ "strings"
+ "sync/atomic"
+
+ "github.com/PuerkitoBio/goquery"
+ "github.com/andybalholm/cascadia"
+ "github.com/shiroyk/ski"
+ "golang.org/x/net/html"
+)
+
+var buildInFuncs atomic.Value
+
+func init() {
+ buildInFuncs.Store(builtins())
+ ski.Register("gq", new_value())
+ ski.Register("gq.element", new_element())
+ ski.Register("gq.elements", new_elements())
+}
+
+// SetFuncs set external FuncMap
+func SetFuncs(m FuncMap) {
+ funcs := maps.Clone(builtins())
+ maps.Copy(funcs, m)
+ buildInFuncs.Store(funcs)
+}
+
+func new_value() ski.NewExecutor {
+ return ski.StringExecutor(func(str string) (ski.Executor, error) {
+ ret, err := compile(str)
+ if err != nil {
+ return nil, err
+ }
+ ret.calls = append(ret.calls, call{fn: value})
+ return ret, nil
+ })
+}
+
+func new_element() ski.NewExecutor {
+ return ski.StringExecutor(func(str string) (ski.Executor, error) {
+ ret, err := compile(str)
+ if err != nil {
+ return nil, err
+ }
+ ret.calls = append(ret.calls, call{fn: element})
+ return ret, nil
+ })
+}
+
+func new_elements() ski.NewExecutor {
+ return ski.StringExecutor(func(str string) (ski.Executor, error) {
+ ret, err := compile(str)
+ if err != nil {
+ return nil, err
+ }
+ ret.calls = append(ret.calls, call{fn: elements})
+ return ret, nil
+ })
+}
+
+func compile(raw string) (ret matcher, err error) {
+ funcs := strings.Split(raw, "->")
+ if len(funcs) == 1 {
+ ret.Matcher, err = cascadia.Compile(funcs[0])
+ return
+ }
+ selector := strings.TrimSpace(funcs[0])
+ if len(selector) == 0 {
+ ret.Matcher = new(emptyMatcher)
+ } else {
+ ret.Matcher, err = cascadia.Compile(selector)
+ if err != nil {
+ return
+ }
+ }
+
+ ret.calls = make([]call, 0, len(funcs)-1)
+
+ for _, function := range funcs[1:] {
+ function = strings.TrimSpace(function)
+ if function == "" {
+ continue
+ }
+ name, args, err := parseFuncArguments(function)
+ if err != nil {
+ return ret, err
+ }
+ fn, ok := buildInFuncs.Load().(FuncMap)[name]
+ if !ok {
+ return ret, fmt.Errorf("function %s not exists", name)
+ }
+ ret.calls = append(ret.calls, call{fn, args})
+ }
+
+ return
+}
+
+type call struct {
+ fn Func
+ args []string
+}
+
+type matcher struct {
+ goquery.Matcher
+ calls []call
+}
+
+func (f matcher) Exec(ctx context.Context, arg any) (any, error) {
+ nodes, err := selection(arg)
+ if err != nil {
+ return nil, err
+ }
+
+ var node any = nodes.FindMatcher(f)
+
+ for _, c := range f.calls {
+ node, err = c.fn(ctx, node, c.args...)
+ if err != nil || node == nil {
+ return nil, err
+ }
+ }
+
+ return node, nil
+}
+
+func value(ctx context.Context, node any, _ ...string) (any, error) {
+ if node == nil {
+ return nil, nil
+ }
+ v, err := Text(ctx, node)
+ if err != nil {
+ return nil, err
+ }
+ return v, nil
+}
+
+func element(_ context.Context, node any, _ ...string) (any, error) {
+ switch t := node.(type) {
+ default:
+ return nil, fmt.Errorf("unexpected type %T", node)
+ case string, []string, *html.Node, ski.Iterator, nil:
+ return t, nil
+ case *goquery.Selection:
+ if len(t.Nodes) == 0 {
+ return nil, nil
+ }
+ return t.Nodes[0], nil
+ case []*html.Node:
+ if len(t) == 0 {
+ return nil, nil
+ }
+ return t[0], nil
+ }
+}
+
+func elements(_ context.Context, node any, _ ...string) (any, error) {
+ switch t := node.(type) {
+ default:
+ return nil, fmt.Errorf("unexpected type %T", node)
+ case string, []string, *html.Node, ski.Iterator, nil:
+ return t, nil
+ case *goquery.Selection:
+ return ski.NewIterator(t.Nodes), nil
+ case []*html.Node:
+ return ski.NewIterator(t), nil
+ }
+}
+
+func cloneNode(n *html.Node) *html.Node {
+ m := &html.Node{
+ Type: n.Type,
+ DataAtom: n.DataAtom,
+ Data: n.Data,
+ Attr: make([]html.Attribute, len(n.Attr)),
+ FirstChild: n.FirstChild,
+ LastChild: n.LastChild,
+ }
+ copy(m.Attr, n.Attr)
+ return m
+}
+
+// selection converts content to goquery.Selection
+func selection(content any) (*goquery.Selection, error) {
+ switch data := content.(type) {
+ default:
+ return nil, fmt.Errorf("unexpected type %T", content)
+ case nil:
+ return new(goquery.Selection), nil
+ case *html.Node:
+ root := &html.Node{Type: html.DocumentNode}
+ root.AppendChild(cloneNode(data))
+ return goquery.NewDocumentFromNode(root).Selection, nil
+ case ski.Iterator:
+ if data.Len() == 0 {
+ return nil, nil
+ }
+ root := &html.Node{Type: html.DocumentNode}
+ doc := goquery.NewDocumentFromNode(root)
+ for i := 0; i < data.Len(); i++ {
+ switch v := data.At(i).(type) {
+ case *html.Node:
+ root.AppendChild(cloneNode(v))
+ case string:
+ nodes, err := html.ParseFragment(strings.NewReader(v), &html.Node{Type: html.ElementNode})
+ if err != nil {
+ return nil, err
+ }
+ for _, node := range nodes {
+ root.AppendChild(cloneNode(node))
+ }
+ default:
+ return nil, fmt.Errorf("unexpected type %T", v)
+ }
+ }
+ return doc.Selection, nil
+ case []string:
+ doc, err := goquery.NewDocumentFromReader(strings.NewReader(strings.Join(data, "\n")))
+ if err != nil {
+ return nil, err
+ }
+ return doc.Selection, nil
+ case fmt.Stringer:
+ doc, err := goquery.NewDocumentFromReader(strings.NewReader(data.String()))
+ if err != nil {
+ return nil, err
+ }
+ return doc.Selection, nil
+ case string:
+ doc, err := goquery.NewDocumentFromReader(strings.NewReader(data))
+ if err != nil {
+ return nil, err
+ }
+ return doc.Selection, nil
+ }
+}
+
+type emptyMatcher struct{}
+
+func (emptyMatcher) Match(*html.Node) bool { return true }
+
+func (emptyMatcher) MatchAll(node *html.Node) []*html.Node { return []*html.Node{node} }
+
+func (emptyMatcher) Filter(nodes []*html.Node) []*html.Node { return nodes }
diff --git a/gq/gq_test.go b/gq/gq_test.go
new file mode 100644
index 0000000..14fc865
--- /dev/null
+++ b/gq/gq_test.go
@@ -0,0 +1,255 @@
+package gq
+
+import (
+ "bytes"
+ "context"
+ "fmt"
+ "testing"
+
+ "log/slog"
+
+ "github.com/shiroyk/ski"
+ "github.com/stretchr/testify/assert"
+ "golang.org/x/net/html"
+)
+
+var (
+ ctx = context.Background()
+ content = `
+
+
+ Tests for siblings
+
+
+
+
1
+
2
+
3
+
4
+
5
+
6
+
Google f1
`,
+ `f2
`,
+ `f3
`,
+ })
+
+ assertElements(t, `#foot div -> slice(0, 3) -> html(true)`, []string{
+ `f1
`,
+ `f2
`,
+ `f3
`,
+ })
+
+ assertElements(t, `#foot div -> slice(0, 3) -> text`, []string{"f1", "f2", "f3"})
+}
+
+func TestNodeSelect(t *testing.T) {
+ t.Run("single", func(t *testing.T) {
+ exec, err := new_element()(ski.String(`script -> slice(0)`))
+ if !assert.NoError(t, err) {
+ return
+ }
+ v, err := exec.Exec(ctx, content)
+ if !assert.NoError(t, err) {
+ return
+ }
+ {
+ exec, err = new_value()(ski.String(`-> attr(type)`))
+ if !assert.NoError(t, err) {
+ return
+ }
+ v1, err := exec.Exec(ctx, v)
+ if assert.NoError(t, err) {
+ assert.EqualValues(t, "text/javascript", v1)
+ }
+ }
+ {
+ exec, err = new_value()(ski.String(`script -> attr(type)`))
+ if !assert.NoError(t, err) {
+ return
+ }
+ v2, err := exec.Exec(ctx, v)
+ if assert.NoError(t, err) {
+ assert.EqualValues(t, "text/javascript", v2)
+ }
+ }
+ })
+
+ t.Run("multiple", func(t *testing.T) {
+ exec, err := new_elements()(ski.String(`#foot div -> slice(0, 3)`))
+ if !assert.NoError(t, err) {
+ return
+ }
+ v, err := exec.Exec(ctx, content)
+ if !assert.NoError(t, err) {
+ return
+ }
+ {
+ exec, err = new_value()(ski.String(`-> text`))
+ if !assert.NoError(t, err) {
+ return
+ }
+ v1, err := exec.Exec(ctx, v)
+ if assert.NoError(t, err) {
+ assert.EqualValues(t, []string{"f1", "f2", "f3"}, v1)
+ }
+ }
+ {
+ exec, err = new_value()(ski.String(`div -> text`))
+ if !assert.NoError(t, err) {
+ return
+ }
+ v2, err := exec.Exec(ctx, v)
+ if assert.NoError(t, err) {
+ assert.EqualValues(t, []string{"f1", "f2", "f3"}, v2)
+ }
+ }
+ })
+}
+
+func TestExternalFunc(t *testing.T) {
+ {
+ fun := func(logger *slog.Logger) Func {
+ return func(_ context.Context, content any, args ...string) (any, error) {
+ logger.Info(fmt.Sprintf("result type was %T", content))
+ return content, nil
+ }
+ }
+ data := new(bytes.Buffer)
+ SetFuncs(FuncMap{"logger": fun(slog.New(slog.NewTextHandler(data, nil)))})
+ exec, err := new_value()(ski.String(".body ul a -> logger -> text"))
+ if assert.NoError(t, err) {
+ v, err := exec.Exec(ctx, content)
+ if assert.NoError(t, err) {
+ assert.EqualValues(t, []string{"Google", "Github", "Golang", "Home"}, v)
+ }
+ }
+ assert.Contains(t, data.String(), `result type was *goquery.Selection`)
+ }
+
+ {
+ fun := func(_ context.Context, content any, args ...string) (any, error) {
+ return nil, nil
+ }
+ SetFuncs(FuncMap{"nil": fun})
+ exec, err := new_value()(ski.String(".body ul a -> nil -> text"))
+ if assert.NoError(t, err) {
+ v, err := exec.Exec(ctx, content)
+ if assert.NoError(t, err) {
+ assert.Equal(t, nil, v)
+ }
+ }
+ }
+}
diff --git a/parsers/gq/tokenizer.go b/gq/tokenizer.go
similarity index 54%
rename from parsers/gq/tokenizer.go
rename to gq/tokenizer.go
index fc40295..03e6dd6 100644
--- a/parsers/gq/tokenizer.go
+++ b/gq/tokenizer.go
@@ -13,50 +13,19 @@ const (
doubleQuoteState
)
-type ruleFunc struct {
- name string
- args []string
-}
-
-func parseRuleFunctions(funcMap FuncMap, ruleStr string) (rule string, funcs []ruleFunc, err error) {
- ruleFuncs := strings.Split(ruleStr, "->")
- if len(ruleFuncs) == 1 {
- return ruleFuncs[0], funcs, nil
- }
- rule = strings.TrimSpace(ruleFuncs[0])
-
- for _, function := range ruleFuncs[1:] {
- function = strings.TrimSpace(function)
- if function == "" {
- continue
- }
- fn, err := parseFuncArguments(function)
- if err != nil {
- return "", nil, err
- }
- if _, ok := funcMap[fn.name]; !ok {
- return "", nil, fmt.Errorf("function %s not exists", fn.name)
- }
- funcs = append(funcs, fn)
- }
-
- return
-}
-
-func parseFuncArguments(s string) (ret ruleFunc, err error) {
+func parseFuncArguments(s string) (name string, args []string, err error) {
openBracket := strings.IndexByte(s, '(')
closeBracket := strings.LastIndexByte(s, ')')
if openBracket == -1 {
- return ruleFunc{name: s}, nil
+ return s, nil, nil
}
if closeBracket == -1 {
- return ret, fmt.Errorf("unexpected function %s not close bracket", s)
+ return name, nil, fmt.Errorf("unexpected function %s not close bracket", s)
}
- funcName := s[0:openBracket]
- args := make([]string, 0)
+ name = s[0:openBracket]
arg := strings.Builder{}
state := commonState
offset := openBracket + 1
@@ -107,7 +76,7 @@ func parseFuncArguments(s string) (ret ruleFunc, err error) {
}
if state == singleQuoteState || state == doubleQuoteState {
- return ret, fmt.Errorf("unexpected function %s argument quote not closed", s)
+ return name, nil, fmt.Errorf("unexpected function %s argument quote not closed", s)
}
if arg.Cap() > 0 {
@@ -115,8 +84,5 @@ func parseFuncArguments(s string) (ret ruleFunc, err error) {
arg.Reset()
}
- return ruleFunc{
- name: funcName,
- args: args,
- }, nil
+ return
}
diff --git a/parsers/gq/tokenizer_test.go b/gq/tokenizer_test.go
similarity index 61%
rename from parsers/gq/tokenizer_test.go
rename to gq/tokenizer_test.go
index 454c2d8..ddb5211 100644
--- a/parsers/gq/tokenizer_test.go
+++ b/gq/tokenizer_test.go
@@ -4,16 +4,15 @@ import (
"testing"
)
-func TestParseRuleFunction(t *testing.T) {
+func TestParseFuncArguments(t *testing.T) {
t.Parallel()
rules := []string{
- `-> -> unknown`, `-> text(`, `-> text(")`,
+ `-> text(`, `-> text(")`,
`-> text("')`, `-> text('")`, `-> text(' ", ")`,
`-> text("\")`, `-> text('\')`, `-> text(" ", ')`,
}
- funcs := builtins()
for _, rule := range rules {
- if _, _, err := parseRuleFunctions(funcs, rule); err == nil {
+ if _, _, err := parseFuncArguments(rule); err == nil {
t.Fatalf("Unexpected function and argument parse %s", rule)
}
}
diff --git a/parsers/json/README.md b/jq/README.md
similarity index 100%
rename from parsers/json/README.md
rename to jq/README.md
diff --git a/jq/jq.go b/jq/jq.go
new file mode 100644
index 0000000..d98994f
--- /dev/null
+++ b/jq/jq.go
@@ -0,0 +1,71 @@
+// Package jq the json path executor
+package jq
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+
+ "github.com/ohler55/ojg/jp"
+ "github.com/ohler55/ojg/oj"
+ "github.com/shiroyk/ski"
+)
+
+func init() {
+ ski.Register("jq", new_expr())
+}
+
+type expr struct {
+ jp.Expr
+ normal bool
+}
+
+func new_expr() ski.NewExecutor {
+ return ski.StringExecutor(func(str string) (ski.Executor, error) {
+ x, err := jp.ParseString(str)
+ if err != nil {
+ return nil, err
+ }
+ return expr{x, x.Normal()}, nil
+ })
+}
+
+func (e expr) Exec(_ context.Context, arg any) (any, error) {
+ obj, err := doc(arg)
+ if err != nil {
+ return nil, err
+ }
+ if e.normal {
+ return e.First(obj), nil
+ }
+ return e.Get(obj), nil
+}
+
+func doc(content any) (any, error) {
+ switch data := content.(type) {
+ default:
+ return content, nil
+ case fmt.Stringer:
+ return oj.ParseString(data.String())
+ case json.RawMessage:
+ return oj.Parse(data)
+ case ski.Iterator:
+ if data.Len() == 0 {
+ return nil, nil
+ }
+ s, ok := data.At(0).(string)
+ if !ok {
+ return data, nil
+ }
+ return oj.ParseString(s)
+ case []byte:
+ return oj.Parse(data)
+ case []string:
+ if len(data) == 0 {
+ return nil, nil
+ }
+ return oj.ParseString(data[0])
+ case string:
+ return oj.ParseString(data)
+ }
+}
diff --git a/jq/jq_test.go b/jq/jq_test.go
new file mode 100644
index 0000000..581174b
--- /dev/null
+++ b/jq/jq_test.go
@@ -0,0 +1,67 @@
+package jq
+
+import (
+ "context"
+ "testing"
+
+ "github.com/shiroyk/ski"
+ "github.com/stretchr/testify/assert"
+)
+
+var (
+ content = `
+{
+ "store": {
+ "book": [
+ {
+ "category": "reference",
+ "author": "Nigel Rees",
+ "title": "Sayings of the Century",
+ "price": 8.95
+ },
+ {
+ "category": "fiction",
+ "author": "Evelyn Waugh",
+ "title": "Sword of Honour",
+ "price": 12.99
+ },
+ {
+ "category": "fiction",
+ "author": "Herman Melville",
+ "title": "Moby Dick",
+ "isbn": "0-553-21311-3",
+ "price": 8.99
+ },
+ {
+ "category": "fiction",
+ "author": "J. R. R. Tolkien",
+ "title": "The Lord of the Rings",
+ "isbn": "0-395-19395-8",
+ "price": 22.99
+ }
+ ],
+ "bicycle": {
+ "color": "red",
+ "price": 19.95
+ }
+ },
+ "expensive": 10
+}`
+)
+
+func assertValue(t *testing.T, arg string, expected any) {
+ exec, err := new_expr()(ski.String(arg))
+ if assert.NoError(t, err) {
+ v, err := exec.Exec(context.Background(), content)
+ if assert.NoError(t, err) {
+ assert.Equal(t, expected, v)
+ }
+ }
+}
+
+func TestGetString(t *testing.T) {
+ t.Parallel()
+ assertValue(t, `$.store.book[-1].price`, 22.99)
+ assertValue(t, `$.store.book[*].author`, []any{"Nigel Rees", "Evelyn Waugh", "Herman Melville", "J. R. R. Tolkien"})
+ assertValue(t, `$.store.book[?(@.price < 10)].isbn`, []any{`0-553-21311-3`})
+}
diff --git a/js/console.go b/js/console.go
index fc5e273..ad2bf1e 100644
--- a/js/console.go
+++ b/js/console.go
@@ -1,41 +1,121 @@
package js
import (
- "context"
-
- "github.com/dop251/goja"
+ "bytes"
"log/slog"
+
+ "github.com/grafana/sobek"
+ "github.com/shiroyk/ski"
)
// console implements the js console
type console struct{}
-// EnableConsole enables the console
-func EnableConsole(vm *goja.Runtime) {
- _ = vm.Set("console", new(console))
+// EnableConsole enables the console with the slog.Logger
+func EnableConsole(rt *sobek.Runtime) {
+ _ = rt.Set("console", new(console))
+}
+
+func (c *console) log(level slog.Level, call sobek.FunctionCall, rt *sobek.Runtime) sobek.Value {
+ ski.Logger(Context(rt)).Log(Context(rt), level, Format(call, rt).String())
+ return sobek.Undefined()
+}
+
+// Log calls slog.Log.
+func (c *console) Log(call sobek.FunctionCall, rt *sobek.Runtime) sobek.Value {
+ return c.log(slog.LevelInfo, call, rt)
+}
+
+// Info calls slog.Info.
+func (c *console) Info(call sobek.FunctionCall, rt *sobek.Runtime) sobek.Value {
+ return c.log(slog.LevelInfo, call, rt)
+}
+
+// Warn calls slog.Warn.
+func (c *console) Warn(call sobek.FunctionCall, rt *sobek.Runtime) sobek.Value {
+ return c.log(slog.LevelWarn, call, rt)
}
-func (c *console) log(level slog.Level, call goja.FunctionCall, vm *goja.Runtime) goja.Value {
- slog.Log(context.Background(), level, Format(call, vm).String())
- return goja.Undefined()
+// Warn calls slog.Error.
+func (c *console) Error(call sobek.FunctionCall, rt *sobek.Runtime) sobek.Value {
+ return c.log(slog.LevelError, call, rt)
}
-// Log calls Logger.Log.
-func (c *console) Log(call goja.FunctionCall, vm *goja.Runtime) goja.Value {
- return c.log(slog.LevelInfo, call, vm)
+// Debug calls slog.Debug.
+func (c *console) Debug(call sobek.FunctionCall, rt *sobek.Runtime) sobek.Value {
+ return c.log(slog.LevelDebug, call, rt)
}
-// Info calls Logger.Info.
-func (c *console) Info(call goja.FunctionCall, vm *goja.Runtime) goja.Value {
- return c.log(slog.LevelInfo, call, vm)
+func runeFormat(rt *sobek.Runtime, f rune, val sobek.Value, w *bytes.Buffer) bool {
+ switch f {
+ case 's':
+ w.WriteString(val.String())
+ case 'd':
+ w.WriteString(val.ToNumber().String())
+ case 'j':
+ if json, ok := rt.Get("JSON").(*sobek.Object); ok {
+ if stringify, ok := sobek.AssertFunction(json.Get("stringify")); ok {
+ res, err := stringify(json, val)
+ if err != nil {
+ panic(err)
+ }
+ w.WriteString(res.String())
+ }
+ }
+ case '%':
+ w.WriteByte('%')
+ return false
+ default:
+ w.WriteByte('%')
+ w.WriteRune(f)
+ return false
+ }
+ return true
}
-// Warn calls Logger.Warn.
-func (c *console) Warn(call goja.FunctionCall, vm *goja.Runtime) goja.Value {
- return c.log(slog.LevelWarn, call, vm)
+func bufferFormat(vm *sobek.Runtime, b *bytes.Buffer, f string, args ...sobek.Value) {
+ pct := false
+ argNum := 0
+ for _, chr := range f {
+ if pct { //nolint:nestif
+ if argNum < len(args) {
+ if runeFormat(vm, chr, args[argNum], b) {
+ argNum++
+ }
+ } else {
+ b.WriteByte('%')
+ b.WriteRune(chr)
+ }
+ pct = false
+ } else {
+ if chr == '%' {
+ pct = true
+ } else {
+ b.WriteRune(chr)
+ }
+ }
+ }
+
+ for _, arg := range args[argNum:] {
+ b.WriteByte(' ')
+ b.WriteString(arg.String())
+ }
}
-// Warn calls Logger.Error.
-func (c *console) Error(call goja.FunctionCall, vm *goja.Runtime) goja.Value {
- return c.log(slog.LevelError, call, vm)
+// Format implements js format
+func Format(call sobek.FunctionCall, rt *sobek.Runtime) sobek.Value {
+ var b bytes.Buffer
+ var f string
+
+ if arg := call.Argument(0); !sobek.IsUndefined(arg) {
+ f = arg.String()
+ }
+
+ var args []sobek.Value
+ if len(call.Arguments) > 0 {
+ args = call.Arguments[1:]
+ }
+ bufferFormat(rt, &b, f, args...)
+
+ return rt.ToValue(b.String())
}
diff --git a/js/console_test.go b/js/console_test.go
index 45f7990..1f525e4 100644
--- a/js/console_test.go
+++ b/js/console_test.go
@@ -1,21 +1,36 @@
package js
import (
+ "bytes"
+ "context"
+ "log/slog"
+ "strconv"
"testing"
- "github.com/dop251/goja"
+ "github.com/shiroyk/ski"
"github.com/stretchr/testify/assert"
)
func TestConsole(t *testing.T) {
t.Parallel()
- vm := goja.New()
- vm.SetFieldNameMapper(FieldNameMapper{})
- EnableConsole(vm)
+ data := new(bytes.Buffer)
+ vm := NewVM()
+ ctx := ski.WithLogger(context.Background(), slog.New(slog.NewTextHandler(data, nil)))
- _, err := vm.RunString(`
- console.log("hello %s", "cloudcat");
- console.log("json %j", {'foo': 'bar'});
- `)
- assert.NoError(t, err)
+ for i, c := range []struct {
+ str, want string
+ }{
+ {`console.log("hello %s", "ski");`, "hello ski"},
+ {`console.log("json %j", {'foo': 'bar'});`, `json {\"foo\":\"bar\"}`},
+ } {
+ t.Run(strconv.Itoa(i), func(t *testing.T) {
+ data.Reset()
+ vm.Run(ctx, func() {
+ _, err := vm.Runtime().RunString(c.str)
+ if assert.NoError(t, err) {
+ assert.Contains(t, data.String(), c.want)
+ }
+ })
+ })
+ }
}
diff --git a/js/ctx.go b/js/ctx.go
deleted file mode 100644
index c9a8476..0000000
--- a/js/ctx.go
+++ /dev/null
@@ -1,113 +0,0 @@
-package js
-
-import (
- "fmt"
- "log/slog"
-
- "github.com/dop251/goja"
- "github.com/shiroyk/cloudcat/plugin"
- "github.com/shiroyk/cloudcat/plugin/parser"
-)
-
-var attr = slog.String("source", "js")
-
-// ctxWrapper an analyzer context
-type ctxWrapper struct {
- ctx *plugin.Context
- BaseURL string
- URL string `js:"url"`
-}
-
-// NewCtxWrapper returns a new ctxWrapper instance
-func NewCtxWrapper(vm VM, ctx *plugin.Context) goja.Value {
- return vm.Runtime().ToValue(&ctxWrapper{ctx, ctx.BaseURL(), ctx.URL()})
-}
-
-// Log print the msg to logger
-func (c *ctxWrapper) Log(call goja.FunctionCall, vm *goja.Runtime) goja.Value {
- c.ctx.Logger().Info(Format(call, vm).String(), attr)
- return goja.Undefined()
-}
-
-// GetVar returns the value associated with this context for key, or nil
-// if no value is associated with key.
-func (c *ctxWrapper) GetVar(call goja.FunctionCall, vm *goja.Runtime) goja.Value {
- return vm.ToValue(c.ctx.Value(call.Argument(0).String()))
-}
-
-// SetVar value associated with key is val.
-func (c *ctxWrapper) SetVar(key string, value goja.Value) error {
- v, err := Unwrap(value)
- if err != nil {
- return err
- }
- c.ctx.SetValue(key, v)
- return nil
-}
-
-// ClearVar clean all values
-func (c *ctxWrapper) ClearVar() {
- c.ctx.ClearValue()
-}
-
-// Cancel this context releases resources associated with it, so code should
-// call cancel as soon as the operations running in this Context complete.
-func (c *ctxWrapper) Cancel() {
- c.ctx.Cancel()
-}
-
-// GetString gets the string of the content with the given arguments
-func (c *ctxWrapper) GetString(key string, rule string, content any) (ret string, err error) {
- str, err := ToStrings(content)
- if err != nil {
- return
- }
-
- if p, ok := parser.GetParser(key); ok {
- return p.GetString(c.ctx, str, rule)
- }
-
- return ret, fmt.Errorf("parser %s not found", key)
-}
-
-// GetStrings gets the string of the content with the given arguments
-func (c *ctxWrapper) GetStrings(key string, rule string, content any) (ret []string, err error) {
- str, err := ToStrings(content)
- if err != nil {
- return
- }
-
- if p, ok := parser.GetParser(key); ok {
- return p.GetStrings(c.ctx, str, rule)
- }
-
- return ret, fmt.Errorf("parser %s not found", key)
-}
-
-// GetElement gets the string of the content with the given arguments
-func (c *ctxWrapper) GetElement(key string, rule string, content any) (ret string, err error) {
- str, err := ToStrings(content)
- if err != nil {
- return
- }
-
- if p, ok := parser.GetParser(key); ok {
- return p.GetElement(c.ctx, str, rule)
- }
-
- return ret, fmt.Errorf("parser %s not found", key)
-}
-
-// GetElements gets the string of the content with the given arguments
-func (c *ctxWrapper) GetElements(key string, rule string, content any) (ret []string, err error) {
- str, err := ToStrings(content)
- if err != nil {
- return
- }
-
- if p, ok := parser.GetParser(key); ok {
- return p.GetElements(c.ctx, str, rule)
- }
-
- return ret, fmt.Errorf("parser %s not found", key)
-}
diff --git a/js/ctx_test.go b/js/ctx_test.go
deleted file mode 100644
index 88cc23b..0000000
--- a/js/ctx_test.go
+++ /dev/null
@@ -1,66 +0,0 @@
-package js
-
-import (
- "fmt"
- "log/slog"
- "testing"
-
- "github.com/shiroyk/cloudcat/plugin"
- "github.com/shiroyk/cloudcat/plugin/parser"
-)
-
-type testParser struct{}
-
-func (t *testParser) GetString(_ *plugin.Context, content any, arg string) (string, error) {
- if str, ok := content.(string); ok {
- return str + arg, nil
- }
- return "", fmt.Errorf("type not supported")
-}
-
-func (t *testParser) GetStrings(_ *plugin.Context, content any, arg string) ([]string, error) {
- if str, ok := content.([]string); ok {
- return append(str, arg), nil
- }
- return nil, fmt.Errorf("type not supported")
-}
-
-func (t *testParser) GetElement(ctx *plugin.Context, content any, arg string) (string, error) {
- return t.GetString(ctx, content, arg)
-}
-
-func (t *testParser) GetElements(ctx *plugin.Context, content any, arg string) ([]string, error) {
- return t.GetStrings(ctx, content, arg)
-}
-
-func TestCtxWrapper(t *testing.T) {
- t.Parallel()
- parser.Register("test", new(testParser))
- ctx := plugin.NewContext(plugin.ContextOptions{
- URL: "http://localhost/home",
- Logger: slog.Default(),
- })
- vm := NewTestVM(t)
-
- testCase := []string{
- `cat.log('start test');`,
- `assert.equal(cat.baseURL, "http://localhost");`,
- `assert.equal(cat.url,"http://localhost/home");`,
- `cat.setVar('v1', 114514);`,
- `assert.equal(cat.getVar('v1'), 114514);`,
- `cat.clearVar();
- assert.equal(cat.getVar('v1'), null);`,
- `assert.equal(cat.getString('test', '1', 'foo'), 'foo1');`,
- `assert.equal(cat.getStrings('test', '2', ['foo'])[1], '2');`,
- `assert.equal(cat.getElement('test', '3', 'foo'), 'foo3');`,
- `assert.equal(cat.getElements('test', '4', ['foo'])[1], '4');`,
- }
- for i, s := range testCase {
- t.Run(fmt.Sprintf("Script%v", i), func(t *testing.T) {
- _, err := vm.RunString(ctx, s)
- if err != nil {
- t.Fatal(err)
- }
- })
- }
-}
diff --git a/js/doc.go b/js/doc.go
index 7ba7781..48e3852 100644
--- a/js/doc.go
+++ b/js/doc.go
@@ -1,2 +1,32 @@
-// Package js the JavaScript implementation
+// Package js the JavaScript implementation.
+//
+// Run ESM/CJS modules:
+//
+// func main() {
+// module, err := js.GetScheduler().Loader().CompileModule("", "module.exports = () => 'some value'")
+// if err != nil {
+// fmt.Println(err)
+// return
+// }
+//
+// ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+// defer cancel()
+//
+// value, err := js.RunModule(ctx, module)
+// if err != nil {
+// fmt.Println(err)
+// return
+// }
+//
+// fmt.Println(value.Export())
+// }
+//
+// Configure JS Scheduler:
+//
+// func init() {
+// js.SetScheduler(js.NewScheduler(js.SchedulerOptions{
+// MaxVMs: 8,
+// Loader: js.NewModuleLoader(),
+// }))
+// }
package js
diff --git a/js/eventloop.go b/js/eventloop.go
index 253d14c..bb3da6f 100644
--- a/js/eventloop.go
+++ b/js/eventloop.go
@@ -1,226 +1,161 @@
package js
import (
- "fmt"
"sync"
-
- "github.com/dop251/goja"
)
-// Copyright grafana/k6, licensed under the AGPL License.
-
-// EventLoop implements an event with
-// handling of unhandled rejected promises.
-//
-// A specific thing about this event loop is that it will wait to return
-// not only until the queue is empty but until nothing is registered that it will run in the future.
-// This is in contrast with more common behaviours where it only returns on
-// a specific event/action or when the loop is empty.
-// This is required as in k6 iterations (for which event loop will be primary used)
-// are supposed to be independent and any work started in them needs to finish,
-// but also they need to end when all the instructions are done.
-// Additionally because of this on any error while the event loop will exit it's
-// required to wait on the event loop to be empty before the execution can continue.
+// EventLoop implements an eventloop.
type EventLoop struct {
- lock sync.Mutex
- queue []func() error
- wakeupCh chan struct{} // TODO: maybe use sync.Cond ?
- registeredCallbacks int
- runtime *goja.Runtime
-
- // pendingPromiseRejections are rejected promises with no handler,
- // if there is something in this map at an end of an event loop then it will exit with an error.
- // It's similar to what Deno and Node do.
- pendingPromiseRejections map[*goja.Promise]struct{}
+ queue []func() // queue to store the job to be executed
+ doneJobs []func() // job of Done
+ enqueue uint // Count of job in the event loop
+ cond *sync.Cond // Condition variable for synchronization
}
-// NewEventLoop returns a new event loop with a few helpers attached to it:
-// - adding setTimeout javascript implementation
-// - reporting (and aborting on) unhandled promise rejections
-func NewEventLoop(runtime *goja.Runtime) *EventLoop {
- e := &EventLoop{
- wakeupCh: make(chan struct{}, 1),
- pendingPromiseRejections: make(map[*goja.Promise]struct{}),
- runtime: runtime,
+// NewEventLoop create a new EventLoop instance
+func NewEventLoop() *EventLoop {
+ return &EventLoop{
+ cond: sync.NewCond(new(sync.Mutex)),
+ doneJobs: make([]func(), 0),
}
- runtime.SetPromiseRejectionTracker(e.promiseRejectionTracker)
- _ = runtime.GlobalObject().SetSymbol(enqueueCallbackSymbol, e.RegisterCallback)
- return e
}
-func (e *EventLoop) wakeup() {
- select {
- case e.wakeupCh <- struct{}{}:
- default:
+// Start the event loop and execute the provided function
+func (e *EventLoop) Start(f func()) {
+ e.cond.L.Lock()
+ e.queue = []func(){f}
+ e.cond.L.Unlock()
+ for {
+ e.cond.L.Lock()
+
+ if len(e.queue) > 0 {
+ queue := e.queue
+ e.queue = make([]func(), 0, len(queue))
+ e.cond.L.Unlock()
+
+ for _, job := range queue {
+ job()
+ }
+ continue
+ }
+
+ if e.enqueue > 0 {
+ e.cond.Wait()
+ e.cond.L.Unlock()
+ continue
+ }
+
+ e.cond.L.Unlock()
+
+ if len(e.doneJobs) > 0 {
+ for _, job := range e.doneJobs {
+ job()
+ }
+ e.doneJobs = e.doneJobs[:0]
+ }
+
+ return
}
}
-var enqueueCallbackSymbol = goja.NewSymbol("__enqueueCallback__")
-
-type EnqueueCallback func(func() error)
+type Enqueue func(func())
-// RegisterCallback signals to the event loop that you are going to do some
-// asynchronous work off the main thread and that you may need to execute some
-// code back on the main thread when you are done. So, once you call this
-// method, the event loop will wait for you to finish and give it the callback
-// it needs to run back on the main thread before it can end the whole current
-// script iteration.
+// EnqueueJob return a function Enqueue to add a job to the job queue.
+// Usage:
//
-// RegisterCallback() *must* be called from the main runtime thread, but its
-// result enqueueCallback() is thread-safe and can be called from any goroutine.
-// enqueueCallback() ensures that its callback parameter is added to the VM
-// runtime's tasks queue, to be executed on the main runtime thread eventually,
-// when the VM is done with the other tasks before it. Unless the whole event
-// loop has been stopped, invoking enqueueCallback() will queue its argument and
-// "wake up" the loop (if it was idle, but not stopped).
+// func main() {
+// server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+// w.WriteHeader(http.StatusOK)
+// _, _ = w.Write([]byte(`{"foo":"bar"}`))
+// }))
+// defer server.Close()
//
-// Keep in mind that once you call RegisterCallback(), you *must* also call
-// enqueueCallback() exactly once, even if don't actually need to run any code
-// on the main thread. If that's the case, you can pass an empty no-op callback
-// to it, but you must call it! The event loop will wait for the
-// enqueueCallback() invocation and the k6 iteration won't finish and will be
-// stuck until the VM itself has been stopped (e.g. because the whole test or
-// scenario has ended). Any error returned by any callback on the main thread
-// will abort the current iteration and no further event loop callbacks will be
-// executed in the same iteration.
+// loop := NewEventLoop()
+// runtime := sobek.New()
//
-// A common pattern for async work is something like this:
+// _ = runtime.Set("fetch", func(call sobek.FunctionCall) sobek.Value {
+// promise, resolve, reject := runtime.NewPromise()
+// enqueue := loop.EnqueueJob()
//
-// func doAsyncWork(vm js.VM) *goja.Promise {
-// enqueueCallback := vm.Runtime().GlobalObject().GetSymbol(enqueueCallbackSymbol).Export().(func() EnqueueCallback)()
-// p, resolve, reject := vm.Runtime().NewPromise()
+// go func() {
+// res, err := http.Get(call.Argument(0).String())
+// if err != nil {
+// enqueue(func() { reject(err) })
+// return
+// }
+// loop.OnDone(func() { res.Body.Close() })
//
-// // Do the actual async work in a new independent goroutine, but make
-// // sure that the Promise resolution is done on the main thread:
-// go func() {
-// // Also make sure to abort early if the context is cancelled, so
-// // the VM is not stuck when the scenario ends or Ctrl+C is used:
-// result, err := doTheActualAsyncWork()
-// enqueueCallback(func() error {
-// if err != nil {
-// reject(err)
-// } else {
-// resolve(result)
-// }
-// return nil // do not abort the iteration
-// })
-// }()
+// data, err := io.ReadAll(res.Body)
+// if err != nil {
+// enqueue(func() { reject(err) })
+// return
+// }
//
-// return p
-// }
+// enqueue(func() { resolve(string(data)) })
+// }()
//
-// This ensures that the actual work happens asynchronously, while the Promise
-// is immediately returned and the main thread resumes execution. It also
-// ensures that the Promise resolution happens safely back on the main thread
-// once the async work is done, as required by goja and all other JS runtimes.
+// return runtime.ToValue(promise)
+// })
//
-// TODO: rename to ReservePendingCallback or something more appropriate?
-func (e *EventLoop) RegisterCallback() EnqueueCallback {
- e.lock.Lock()
- var callbackCalled bool
- e.registeredCallbacks++
- e.lock.Unlock()
-
- return func(f func() error) {
- e.lock.Lock()
- if callbackCalled { // this is protected by the lock on the event loop
- e.lock.Unlock() // let not lock up the whole event loop, somebody could recover from the panic
- panic("RegisterCallback called twice")
+// var (
+// ret sobek.Value
+// err error
+// )
+//
+// loop.Start(func() { ret, err = runtime.RunString(fmt.Sprintf(`fetch("%s")`, server.URL)) })
+//
+// if err != nil {
+// fmt.Println(err)
+// }
+// promise, ok := ret.Export().(*sobek.Promise)
+// if !ok {
+// panic("expect promise")
+// return
+// }
+//
+// switch promise.State() {
+// case sobek.PromiseStatePending:
+// panic("unexpect pending state")
+// case sobek.PromiseStateRejected:
+// fmt.Println(promise.Result().String())
+// case sobek.PromiseStateFulfilled:
+// fmt.Println(promise.Result().Export())
+// }
+// }
+func (e *EventLoop) EnqueueJob() Enqueue {
+ e.cond.L.Lock()
+ called := false
+ e.enqueue++
+ e.cond.L.Unlock()
+ return func(job func()) {
+ e.cond.L.Lock()
+ if called {
+ e.cond.L.Unlock()
+ panic("Enqueue already called")
}
- callbackCalled = true
- e.queue = append(e.queue, f)
- e.registeredCallbacks--
- e.lock.Unlock()
- e.wakeup()
+ e.queue = append(e.queue, job) // Add the job to the queue
+ called = true
+ e.enqueue--
+ e.cond.Signal() // Signal the condition variable
+ e.cond.L.Unlock()
}
}
-func (e *EventLoop) promiseRejectionTracker(p *goja.Promise, op goja.PromiseRejectionOperation) {
- // No locking necessary here as the goja runtime will call this synchronously
- // Read Notes on https://tc39.es/ecma262/#sec-host-promise-rejection-tracker
- if op == goja.PromiseRejectionReject {
- e.pendingPromiseRejections[p] = struct{}{}
- } else { // goja.PromiseRejectionHandle so a promise that was previously rejected without handler now got one
- delete(e.pendingPromiseRejections, p)
- }
+// Stop the eventloop
+func (e *EventLoop) Stop() {
+ e.cond.L.Lock()
+ defer e.cond.L.Unlock()
+ // clean the queue
+ e.queue = e.queue[:0]
+ e.enqueue = 0
+ e.doneJobs = e.doneJobs[:0]
+ e.cond.Signal()
}
-func (e *EventLoop) popAll() (queue []func() error, awaiting bool) {
- e.lock.Lock()
- queue = e.queue
- e.queue = make([]func() error, 0, len(queue))
- awaiting = e.registeredCallbacks != 0
- e.lock.Unlock()
- return
-}
-
-func (e *EventLoop) putInfront(queue []func() error) {
- e.lock.Lock()
- e.queue = append(queue, e.queue...)
- e.lock.Unlock()
-}
+// OnDone add a function to execute when done.
+func (e *EventLoop) OnDone(job func()) {
+ e.cond.L.Lock()
+ defer e.cond.L.Unlock()
-// Start will run the event loop until it's empty and there are no uninvoked registered callbacks
-// or a queued function returns an error. The provided firstCallback will be the first thing executed.
-// After Start returns the event loop can be reused as long as waitOnRegistered is called.
-func (e *EventLoop) Start(firstCallback func() error) error {
- e.pendingPromiseRejections = make(map[*goja.Promise]struct{})
- e.queue = []func() error{firstCallback}
- for {
- queue, awaiting := e.popAll()
-
- if len(queue) == 0 {
- if !awaiting {
- return nil
- }
- <-e.wakeupCh
- continue
- }
-
- for i, f := range queue {
- if err := f(); err != nil {
- e.putInfront(queue[i+1:])
- return err
- }
- }
-
- // This will get a random unhandled rejection instead of the first one, for example.
- // But that seems to be the case in other tools as well so it seems to not be that big of a problem.
- for promise := range e.pendingPromiseRejections {
- value := promise.Result()
- if !goja.IsNull(value) && !goja.IsUndefined(value) {
- if o := value.ToObject(e.runtime); o != nil {
- if stack := o.Get("stack"); stack != nil {
- value = stack
- }
- }
- }
- // this is the de facto wording in both firefox and deno at least
- return fmt.Errorf("Uncaught (in promise) %s", value) //nolint:stylecheck
- }
- }
-}
-
-// WaitOnRegistered waits on all registered callbacks so we know nothing is still doing work.
-// This does call back the callbacks and more can be queued over time.
-// A different mechanism needs to be used to tell the users that the event loop has errored out or winding down for a
-// different reason.
-func (e *EventLoop) WaitOnRegistered() {
- for {
- queue, awaiting := e.popAll()
- if len(queue) == 0 {
- if !awaiting {
- return
- }
- <-e.wakeupCh
- continue
- }
-
- for _, f := range queue {
- if err := f(); err != nil {
- // TODO figure out if we should buffer all errors happening or send them on a channel
- continue
- }
- }
- }
+ e.doneJobs = append(e.doneJobs, job)
}
diff --git a/js/eventloop_test.go b/js/eventloop_test.go
index cbbab5c..d42c860 100644
--- a/js/eventloop_test.go
+++ b/js/eventloop_test.go
@@ -1,8 +1,7 @@
package js
import (
- "errors"
- "fmt"
+ "context"
"sync/atomic"
"testing"
"time"
@@ -10,226 +9,125 @@ import (
"github.com/stretchr/testify/assert"
)
-func TestBasicEventLoop(t *testing.T) {
+func TestEventLoop(t *testing.T) {
t.Parallel()
- loop := NewEventLoop(NewTestVM(t).Runtime())
- var ran int
- f := func() error { //nolint:unparam
- ran++
- return nil
- }
- assert.NoError(t, loop.Start(f))
- assert.Equal(t, 1, ran)
- assert.NoError(t, loop.Start(f))
- assert.Equal(t, 2, ran)
- assert.Error(t, loop.Start(func() error {
- _ = f()
- loop.RegisterCallback()(f)
- return errors.New("something")
- }))
- assert.Equal(t, 3, ran)
+ loop := NewEventLoop()
+ var i int
+ f := func() { i++ }
+ loop.Start(f)
+ assert.Equal(t, 1, i)
+ loop.Start(f)
+ assert.Equal(t, 2, i)
}
-func TestEventLoopRegistered(t *testing.T) {
+func TestEventLoopEnqueue(t *testing.T) {
t.Parallel()
- loop := NewEventLoop(NewTestVM(t).Runtime())
- var ran int
- f := func() error {
- ran++
- r := loop.RegisterCallback()
+ loop := NewEventLoop()
+ sleep := time.Millisecond * 500
+ var i int
+ f := func() {
+ i++
+ r := loop.EnqueueJob()
go func() {
- time.Sleep(time.Second)
- r(func() error {
- ran++
- return nil
- })
- }()
- return nil
- }
- start := time.Now()
- assert.NoError(t, loop.Start(f))
- took := time.Since(start)
- assert.Equal(t, 2, ran)
- assert.Less(t, time.Second, took)
- assert.Greater(t, time.Second+time.Millisecond*100, took)
-}
-
-func TestEventLoopWaitOnRegistered(t *testing.T) {
- t.Parallel()
- var ran int
- loop := NewEventLoop(NewTestVM(t).Runtime())
- f := func() error {
- ran++
- r := loop.RegisterCallback()
- go func() {
- time.Sleep(time.Second)
- r(func() error {
- ran++
- return nil
- })
+ time.Sleep(sleep)
+ r(func() { i++ })
}()
- return fmt.Errorf("expected")
}
start := time.Now()
- assert.Error(t, loop.Start(f))
+ loop.Start(f)
took := time.Since(start)
- loop.WaitOnRegistered()
- took2 := time.Since(start)
- assert.Equal(t, 2, ran)
- assert.Greater(t, time.Millisecond*50, took)
- assert.Less(t, time.Second, took2)
- assert.Greater(t, time.Second+time.Millisecond*100, took2)
+ assert.Equal(t, 2, i)
+ assert.Less(t, sleep, took)
}
-func TestEventLoopAllCallbacksGetCalled(t *testing.T) {
+func TestEventLoopAllJobCalled(t *testing.T) {
t.Parallel()
sleepTime := time.Millisecond * 500
- loop := NewEventLoop(NewTestVM(t).Runtime())
+ loop := NewEventLoop()
var called int64
- f := func() error {
- for i := 0; i < 100; i++ {
- bad := i == 99
- r := loop.RegisterCallback()
+ f := func() {
+ for i := 0; i < 10; i++ {
+ bad := i == 9
+ e := loop.EnqueueJob()
go func() {
if !bad {
time.Sleep(sleepTime)
}
- r(func() error {
- if bad {
- return errors.New("something")
- }
- atomic.AddInt64(&called, 1)
- return nil
- })
+ e(func() { atomic.AddInt64(&called, 1) })
}()
}
- return fmt.Errorf("expected")
}
+ all := time.Now()
for i := 0; i < 3; i++ {
called = 0
start := time.Now()
- assert.Error(t, loop.Start(f))
+ loop.Start(f)
took := time.Since(start)
- loop.WaitOnRegistered()
took2 := time.Since(start)
- assert.Greater(t, time.Millisecond*50, took)
+ assert.Less(t, time.Millisecond*500, took)
assert.Less(t, sleepTime, took2)
assert.Greater(t, sleepTime+time.Millisecond*100, took2)
- assert.EqualValues(t, called, 99)
+ assert.EqualValues(t, 10, called)
}
+ took := time.Since(all)
+ assert.Less(t, time.Millisecond*500*3, took)
}
-func TestEventLoopPanicOnDoubleCallback(t *testing.T) {
+func TestEventLoopPanicOnDoubleEnqueue(t *testing.T) {
t.Parallel()
- loop := NewEventLoop(NewTestVM(t).Runtime())
- var ran int
- f := func() error {
- ran++
- r := loop.RegisterCallback()
+ loop := NewEventLoop()
+ var i int
+ f := func() {
+ i++
+ e := loop.EnqueueJob()
go func() {
time.Sleep(time.Second)
- r(func() error {
- ran++
- return nil
- })
+ e(func() { i++ })
- assert.Panics(t, func() { r(func() error { return nil }) })
+ assert.Panics(t, func() { e(func() {}) })
}()
- return nil
}
start := time.Now()
- assert.NoError(t, loop.Start(f))
+ loop.Start(f)
took := time.Since(start)
- assert.Equal(t, 2, ran)
+ assert.Equal(t, 2, i)
assert.Less(t, time.Second, took)
assert.Greater(t, time.Second+time.Millisecond*100, took)
}
-func TestEventLoopRejectUndefined(t *testing.T) {
+func TestEventLoopStop(t *testing.T) {
t.Parallel()
- vm := NewTestVM(t)
- loop := NewEventLoop(vm.Runtime())
- err := loop.Start(func() error {
- _, err := vm.Runtime().RunString("Promise.reject()")
- return err
- })
- loop.WaitOnRegistered()
- assert.EqualError(t, err, "Uncaught (in promise) undefined")
-}
+ loop := NewEventLoop()
-func TestEventLoopRejectString(t *testing.T) {
- t.Parallel()
- vm := NewTestVM(t)
- loop := NewEventLoop(vm.Runtime())
- err := loop.Start(func() error {
- _, err := vm.Runtime().RunString("Promise.reject('some string')")
- return err
- })
- loop.WaitOnRegistered()
- assert.EqualError(t, err, "Uncaught (in promise) some string")
-}
+ ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*500)
+ defer cancel()
-func TestEventLoopRejectSyntaxError(t *testing.T) {
- t.Parallel()
- vm := NewTestVM(t)
- loop := NewEventLoop(vm.Runtime())
- err := loop.Start(func() error {
- _, err := vm.Runtime().RunString("Promise.resolve().then(()=> {some.syntax.error})")
- return err
- })
- loop.WaitOnRegistered()
- assert.EqualError(t, err, "Uncaught (in promise) ReferenceError: some is not defined\n\tat :1:30(1)\n")
-}
+ go func() {
+ <-ctx.Done()
+ loop.Stop()
+ }()
-func TestEventLoopRejectGoError(t *testing.T) {
- t.Parallel()
- vm := NewTestVM(t)
- loop := NewEventLoop(vm.Runtime())
- rt := vm.Runtime()
- assert.NoError(t, rt.Set("f", rt.ToValue(func() error {
- return errors.New("some error")
- })))
- err := loop.Start(func() error {
- _, err := vm.Runtime().RunString("Promise.resolve().then(()=> {f()})")
- return err
- })
- loop.WaitOnRegistered()
- assert.EqualError(t, err, "Uncaught (in promise) GoError: some error\n\tat github.com/shiroyk/cloudcat/js.TestEventLoopRejectGoError.func1 (native)\n\tat :1:31(2)\n")
-}
+ start := time.Now()
+ loop.Start(func() { loop.EnqueueJob() })
+ <-ctx.Done()
-func TestEventLoopRejectThrow(t *testing.T) {
- t.Parallel()
- vm := NewTestVM(t)
- loop := NewEventLoop(vm.Runtime())
- rt := vm.Runtime()
- assert.NoError(t, rt.Set("f", rt.ToValue(func() error {
- Throw(rt, errors.New("throw error"))
- return nil
- })))
- err := loop.Start(func() error {
- _, err := vm.Runtime().RunString("Promise.resolve().then(()=> {f()})")
- return err
- })
- loop.WaitOnRegistered()
- assert.EqualError(t, err, "Uncaught (in promise) throw error")
+ took := time.Since(start)
+ assert.Less(t, time.Millisecond*500, took)
}
-func TestEventLoopAsyncAwait(t *testing.T) {
+func TestEventLoopOnDone(t *testing.T) {
t.Parallel()
- vm := NewTestVM(t)
- loop := NewEventLoop(vm.Runtime())
- err := loop.Start(func() error {
- _, err := vm.Runtime().RunString(`
- async function a() {
- some.error.here
- }
- Promise.resolve().then(async () => {
- await a();
- })
- `)
- return err
- })
- loop.WaitOnRegistered()
- assert.EqualError(t, err, "Uncaught (in promise) ReferenceError: some is not defined\n\tat a (:3:13(1))\n\tat :6:20(2)\n")
+ loop := NewEventLoop()
+ {
+ var i int
+ loop.Start(func() { loop.OnDone(func() { i++ }) })
+ assert.Equal(t, 1, i)
+ }
+ {
+
+ var i int
+ loop.Start(func() { loop.OnDone(func() { loop.OnDone(func() { i++ }) }) })
+ assert.Equal(t, 0, i)
+ }
}
diff --git a/js/format.go b/js/format.go
deleted file mode 100644
index 18ad29b..0000000
--- a/js/format.go
+++ /dev/null
@@ -1,81 +0,0 @@
-package js
-
-import (
- "bytes"
-
- "github.com/dop251/goja"
-)
-
-func runeFormat(vm *goja.Runtime, f rune, val goja.Value, w *bytes.Buffer) bool {
- switch f {
- case 's':
- w.WriteString(val.String())
- case 'd':
- w.WriteString(val.ToNumber().String())
- case 'j':
- if json, ok := vm.Get("JSON").(*goja.Object); ok {
- if stringify, ok := goja.AssertFunction(json.Get("stringify")); ok {
- res, err := stringify(json, val)
- if err != nil {
- panic(err)
- }
- w.WriteString(res.String())
- }
- }
- case '%':
- w.WriteByte('%')
- return false
- default:
- w.WriteByte('%')
- w.WriteRune(f)
- return false
- }
- return true
-}
-
-func bufferFormat(vm *goja.Runtime, b *bytes.Buffer, f string, args ...goja.Value) {
- pct := false
- argNum := 0
- for _, chr := range f {
- if pct { //nolint:nestif
- if argNum < len(args) {
- if runeFormat(vm, chr, args[argNum], b) {
- argNum++
- }
- } else {
- b.WriteByte('%')
- b.WriteRune(chr)
- }
- pct = false
- } else {
- if chr == '%' {
- pct = true
- } else {
- b.WriteRune(chr)
- }
- }
- }
-
- for _, arg := range args[argNum:] {
- b.WriteByte(' ')
- b.WriteString(arg.String())
- }
-}
-
-// Format implements js format
-func Format(call goja.FunctionCall, vm *goja.Runtime) goja.Value {
- var b bytes.Buffer
- var fmt string
-
- if arg := call.Argument(0); !goja.IsUndefined(arg) {
- fmt = arg.String()
- }
-
- var args []goja.Value
- if len(call.Arguments) > 0 {
- args = call.Arguments[1:]
- }
- bufferFormat(vm, &b, fmt, args...)
-
- return vm.ToValue(b.String())
-}
diff --git a/js/js.go b/js/js.go
deleted file mode 100644
index 63a7f4f..0000000
--- a/js/js.go
+++ /dev/null
@@ -1,154 +0,0 @@
-package js
-
-import (
- "context"
- "errors"
- "fmt"
- "runtime"
- "sync"
- "sync/atomic"
- "time"
-
- "log/slog"
-
- "github.com/dop251/goja"
- "github.com/shiroyk/cloudcat"
-)
-
-const (
- // DefaultMaxTimeToWaitGetVM default retries time
- DefaultMaxTimeToWaitGetVM = 500 * time.Millisecond
- // DefaultMaxRetriesGetVM default retries times
- DefaultMaxRetriesGetVM = 3
-)
-
-var (
- defaultScheduler atomic.Value
- once sync.Once
- // ErrSchedulerClosed the scheduler is closed error
- ErrSchedulerClosed = errors.New("scheduler is closed")
-)
-
-// SetScheduler set the default Scheduler.
-func SetScheduler(scheduler Scheduler) {
- defaultScheduler.Store(scheduler)
-}
-
-// GetScheduler returns the Scheduler.
-func GetScheduler() Scheduler {
- once.Do(func() {
- defaultScheduler.CompareAndSwap(nil, NewScheduler(Options{InitialVMs: 2, MaxVMs: runtime.GOMAXPROCS(0)}))
- })
- return defaultScheduler.Load().(Scheduler)
-}
-
-// RunString the js string
-func RunString(ctx context.Context, script string) (goja.Value, error) {
- tr, err := GetScheduler().Get()
- if err != nil {
- return nil, err
- }
- return tr.RunString(ctx, script)
-}
-
-// Run the js program
-func Run(ctx context.Context, p Program) (goja.Value, error) {
- tr, err := GetScheduler().Get()
- if err != nil {
- return nil, err
- }
- return tr.Run(ctx, p)
-}
-
-// Scheduler the VM scheduler
-type Scheduler interface {
- // Get the VM
- Get() (VM, error)
- // Release the VM
- Release(VM)
- // Close the scheduler
- Close() error
-}
-
-// Options Scheduler options
-type Options struct {
- InitialVMs int `yaml:"initial-vms"`
- MaxVMs int `yaml:"max-vms"`
- MaxRetriesGetVM int `yaml:"max-retries-get-vm"`
- MaxTimeToWaitGetVM time.Duration `yaml:"max-time-to-wait-get-vm"`
- ModulePath []string `yaml:"module-path"`
-}
-
-type schedulerImpl struct {
- mu *sync.Mutex
- vms chan VM
- initVMs, maxVMs, maxRetriesGetVM int
- unInitVMs *atomic.Int64
- closed *atomic.Bool
- maxTimeToWaitGetVM time.Duration
- modulePath []string
-}
-
-// NewScheduler returns a new Scheduler
-func NewScheduler(opt Options) Scheduler {
- scheduler := &schedulerImpl{
- mu: new(sync.Mutex),
- closed: new(atomic.Bool),
- unInitVMs: new(atomic.Int64),
- maxVMs: cloudcat.ZeroOr(opt.MaxVMs, 1),
- initVMs: cloudcat.ZeroOr(opt.InitialVMs, 1),
- maxRetriesGetVM: cloudcat.ZeroOr(opt.MaxRetriesGetVM, DefaultMaxRetriesGetVM),
- maxTimeToWaitGetVM: cloudcat.ZeroOr(opt.MaxTimeToWaitGetVM, DefaultMaxTimeToWaitGetVM),
- modulePath: opt.ModulePath,
- }
- scheduler.vms = make(chan VM, scheduler.maxVMs)
- for i := 0; i < scheduler.initVMs; i++ {
- scheduler.vms <- NewVM(scheduler.modulePath...)
- }
- scheduler.unInitVMs.Store(int64(scheduler.maxVMs - scheduler.initVMs))
- return scheduler
-}
-
-// Close the scheduler
-func (s *schedulerImpl) Close() error {
- s.closed.Store(true)
- close(s.vms)
- return nil
-}
-
-// Get the VM
-func (s *schedulerImpl) Get() (VM, error) {
- timer := time.NewTimer(s.maxTimeToWaitGetVM)
-
- defer func() {
- timer.Stop()
- }()
-
- for i := 1; i <= s.maxRetriesGetVM; i++ {
- select {
- case vm, ok := <-s.vms:
- if !ok {
- return nil, ErrSchedulerClosed
- }
- return vm, nil
- case <-timer.C:
- if s.unInitVMs.Add(-1) >= 0 {
- return NewVM(s.modulePath...), nil
- }
- s.unInitVMs.Add(1)
- slog.Warn(fmt.Sprintf("could not get VM in %v", time.Duration(i)*s.maxTimeToWaitGetVM))
- timer.Reset(s.maxTimeToWaitGetVM)
- }
- }
- return nil, fmt.Errorf("could not get VM in %v",
- time.Duration(s.maxRetriesGetVM)*s.maxTimeToWaitGetVM)
-}
-
-// Release the VM
-func (s *schedulerImpl) Release(vm VM) {
- if s.closed.Load() {
- return
- }
-
- s.vms <- vm
-}
diff --git a/js/js_test.go b/js/js_test.go
deleted file mode 100644
index 8967b0e..0000000
--- a/js/js_test.go
+++ /dev/null
@@ -1,53 +0,0 @@
-package js
-
-import (
- "context"
- "errors"
- "sync"
- "testing"
- "time"
-)
-
-func TestScheduler(t *testing.T) {
- goroutineNum := 15
- blockNum := 4
- SetScheduler(NewScheduler(Options{InitialVMs: 2, MaxVMs: 4}))
- wg := new(sync.WaitGroup)
-
- for i := 1; i <= goroutineNum; i++ {
- wg.Add(1)
- go func(i int) {
- timeout := time.Second
- script := "1"
- if i < blockNum {
- script = `while(true){}`
- timeout *= 2
- }
-
- ctx, _ := context.WithTimeout(context.Background(), timeout)
- defer func() {
- wg.Done()
- }()
-
- _, err := RunString(ctx, script)
- if err != nil && !errors.Is(err, context.DeadlineExceeded) {
- t.Errorf("%v: %v", i, err)
- }
- }(i)
- }
- wg.Wait()
-}
-
-func BenchmarkScheduler(b *testing.B) {
- b.ResetTimer()
- b.ReportAllocs()
- wg := sync.WaitGroup{}
- for n := 0; n < b.N; n++ {
- wg.Add(1)
- go func() {
- _, _ = RunString(context.Background(), `1`)
- wg.Done()
- }()
- }
- b.StopTimer()
-}
diff --git a/js/loader.go b/js/loader.go
new file mode 100644
index 0000000..0bc4818
--- /dev/null
+++ b/js/loader.go
@@ -0,0 +1,448 @@
+package js
+
+import (
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "io/fs"
+ "net/http"
+ "net/url"
+ "os"
+ "path"
+ "path/filepath"
+ "strings"
+ "sync"
+ "text/template"
+
+ "github.com/grafana/sobek"
+ "github.com/grafana/sobek/parser"
+ "github.com/shiroyk/ski"
+)
+
+var (
+ // ErrInvalidModule module is invalid
+ ErrInvalidModule = errors.New("invalid module")
+ // ErrIllegalModuleName module name is illegal
+ ErrIllegalModuleName = errors.New("illegal module name")
+ // ErrNotFoundModule not found module
+ ErrNotFoundModule = errors.New("not found module")
+)
+
+type (
+ // ModuleLoader the js module loader.
+ ModuleLoader interface {
+ // CompileModule compile module from source string (cjs/esm).
+ CompileModule(name, source string) (sobek.CyclicModuleRecord, error)
+ // ResolveModule resolve the module returns the sobek.ModuleRecord.
+ ResolveModule(any, string) (sobek.ModuleRecord, error)
+ // EnableRequire enable the global function require to the sobek.Runtime.
+ EnableRequire(*sobek.Runtime) ModuleLoader
+ // EnableImportModuleDynamically sobek runtime SetImportModuleDynamically
+ EnableImportModuleDynamically(*sobek.Runtime) ModuleLoader
+ }
+
+ // LoaderOption the default moduleLoader options.
+ LoaderOption func(*moduleLoader)
+
+ // FileLoader is a type alias for a function that returns the contents of the referenced file.
+ FileLoader func(specifier *url.URL, name string) ([]byte, error)
+
+ // emptyLoader
+ emptyLoader struct{}
+)
+
+// WithBaseLoader the base directory of module loader.
+func WithBaseLoader(base *url.URL) LoaderOption {
+ return func(o *moduleLoader) { o.base = base }
+}
+
+// WithFileLoader the file loader of module loader.
+func WithFileLoader(loader FileLoader) LoaderOption {
+ return func(o *moduleLoader) { o.fileLoader = loader }
+}
+
+// WithSourceMapLoader the source map loader of module loader.
+func WithSourceMapLoader(loader func(path string) ([]byte, error)) LoaderOption {
+ return func(o *moduleLoader) { o.sourceLoader = parser.WithSourceMapLoader(loader) }
+}
+
+// NewModuleLoader returns a new module resolver
+// if the fileLoader option not provided, uses the default DefaultFileLoader.
+func NewModuleLoader(opts ...LoaderOption) ModuleLoader {
+ ml := &moduleLoader{
+ modules: make(map[string]moduleCache),
+ goModules: make(map[string]sobek.CyclicModuleRecord),
+ parsers: make(map[string]sobek.CyclicModuleRecord),
+ reverse: make(map[sobek.ModuleRecord]*url.URL),
+ }
+
+ for _, option := range opts {
+ option(ml)
+ }
+
+ if ml.base == nil {
+ ml.base = &url.URL{Scheme: "file", Path: "."}
+ }
+ if ml.fileLoader == nil {
+ ml.fileLoader = DefaultFileLoader(ski.NewFetch())
+ }
+ if ml.sourceLoader == nil {
+ ml.sourceLoader = parser.WithDisableSourceMaps
+ }
+ return ml
+}
+
+// DefaultFileLoader the default file loader.
+// Supports file and HTTP scheme loading.
+func DefaultFileLoader(fetch ski.Fetch) FileLoader {
+ return func(specifier *url.URL, name string) ([]byte, error) {
+ switch specifier.Scheme {
+ case "http", "https":
+ req, err := http.NewRequest(http.MethodGet, specifier.String(), nil)
+ if err != nil {
+ return nil, err
+ }
+ res, err := fetch.Do(req)
+ if err != nil {
+ return nil, err
+ }
+ defer res.Body.Close()
+ body, err := io.ReadAll(res.Body)
+ return body, err
+ case "file":
+ return fs.ReadFile(os.DirFS("."), specifier.Path)
+ default:
+ return nil, fmt.Errorf("scheme not supported %s", specifier.Scheme)
+ }
+ }
+}
+
+type (
+ // moduleLoader the ModuleLoader implement.
+ // Allows loading and interop between ES module and CommonJS module.
+ moduleLoader struct {
+ sync.Mutex
+ modules map[string]moduleCache
+ goModules map[string]sobek.CyclicModuleRecord
+ parsers map[string]sobek.CyclicModuleRecord
+ reverse map[sobek.ModuleRecord]*url.URL
+
+ fileLoader FileLoader
+
+ base *url.URL
+ sourceLoader parser.Option
+ }
+
+ moduleCache struct {
+ mod sobek.CyclicModuleRecord
+ err error
+ }
+)
+
+// EnableRequire enable the global function require to the sobek.Runtime.
+func (ml *moduleLoader) EnableRequire(rt *sobek.Runtime) ModuleLoader {
+ _ = rt.Set("require", ml.require)
+ return ml
+}
+
+// require resolve the module instance.
+func (ml *moduleLoader) require(call sobek.FunctionCall, rt *sobek.Runtime) sobek.Value {
+ name := call.Argument(0).String()
+ mod, err := ml.ResolveModule(ml.getCurrentModuleRecord(rt), name)
+ if err != nil {
+ Throw(rt, err)
+ }
+ if mod, ok := mod.(*goModule); ok {
+ instance, err := mod.mod.Instantiate(rt)
+ if err != nil {
+ Throw(rt, err)
+ }
+ return instance
+ }
+
+ instance := rt.GetModuleInstance(mod)
+ if instance == nil {
+ if err = mod.Link(); err != nil {
+ Throw(rt, err)
+ }
+ cm, ok := mod.(sobek.CyclicModuleRecord)
+ if !ok {
+ Throw(rt, ErrInvalidModule)
+ }
+ promise := rt.CyclicModuleRecordEvaluate(cm, ml.ResolveModule)
+ if promise.State() == sobek.PromiseStateRejected {
+ panic(promise.Result())
+ }
+ instance = rt.GetModuleInstance(mod)
+ }
+
+ switch mod.(type) {
+ case *cjsModule:
+ return instance.(*cjsModuleInstance).GetBindingValue("default")
+ case *sobek.SourceTextModuleRecord:
+ if v := instance.GetBindingValue("default"); v != nil {
+ return v
+ }
+ }
+
+ return rt.NamespaceObjectFor(mod)
+}
+
+func (ml *moduleLoader) EnableImportModuleDynamically(rt *sobek.Runtime) ModuleLoader {
+ rt.SetImportModuleDynamically(func(referencingScriptOrModule any, specifier sobek.Value, promiseCapability any) {
+ NewPromise(rt,
+ func() (sobek.ModuleRecord, error) {
+ return ml.ResolveModule(referencingScriptOrModule, specifier.String())
+ },
+ func(module sobek.ModuleRecord, err error) (any, error) {
+ rt.FinishLoadingImportModule(referencingScriptOrModule, specifier, promiseCapability, module, err)
+ return nil, err
+ })
+ })
+ return ml
+}
+
+func (ml *moduleLoader) getCurrentModuleRecord(rt *sobek.Runtime) sobek.ModuleRecord {
+ var buf [2]sobek.StackFrame
+ frames := rt.CaptureCallStack(2, buf[:0])
+ if len(frames) == 0 {
+ return nil
+ }
+ mod, _ := ml.ResolveModule(nil, frames[1].SrcName())
+ return mod
+}
+
+// ResolveModule resolve the module returns the sobek.ModuleRecord.
+func (ml *moduleLoader) ResolveModule(referencingScriptOrModule any, name string) (sobek.ModuleRecord, error) {
+ switch {
+ case strings.HasPrefix(name, modulePrefix):
+ ml.Lock()
+ defer ml.Unlock()
+ if mod, ok := ml.goModules[name]; ok {
+ return mod, nil
+ }
+ if e, ok := GetModule(name); ok {
+ mod := &goModule{mod: e}
+ ml.goModules[name] = mod
+ return mod, nil
+ }
+ return nil, ErrNotFoundModule
+ case strings.HasPrefix(name, _js_executor_prefix):
+ ml.Lock()
+ defer ml.Unlock()
+ name = strings.TrimPrefix(name, _js_executor_prefix)
+ if mod, ok := ml.parsers[name]; ok {
+ return mod, nil
+ }
+ if e, ok := ski.GetExecutors(name); ok {
+ mod := &goModule{mod: _js_executor(e)}
+ ml.parsers[name] = mod
+ return mod, nil
+ }
+ return nil, ErrNotFoundModule
+ default:
+ return ml.resolve(ml.reversePath(referencingScriptOrModule), name)
+ }
+}
+
+func (ml *moduleLoader) resolve(base *url.URL, specifier string) (sobek.ModuleRecord, error) {
+ if specifier == "" {
+ return nil, ErrIllegalModuleName
+ }
+
+ if isBasePath(specifier) {
+ return ml.loadAsFileOrDirectory(base, specifier)
+ }
+
+ if strings.Contains(specifier, "://") {
+ uri, err := url.Parse(specifier)
+ if err != nil {
+ return nil, err
+ }
+ return ml.loadModule(uri, "")
+ }
+
+ mod, err := ml.loadNodeModules(specifier)
+ if err != nil {
+ return nil, fmt.Errorf("module %s not found with error %s", specifier, err)
+ }
+ return mod, nil
+}
+
+func (ml *moduleLoader) reversePath(referencingScriptOrModule any) *url.URL {
+ if referencingScriptOrModule == nil {
+ return ml.base
+ }
+ mod, ok := referencingScriptOrModule.(sobek.ModuleRecord)
+ if !ok {
+ return ml.base
+ }
+
+ ml.Lock()
+ p, ok := ml.reverse[mod]
+ ml.Unlock()
+
+ if !ok {
+ return ml.base
+ }
+
+ if p.String() == "file://-" {
+ return ml.base
+ }
+ return p
+}
+
+func (ml *moduleLoader) loadAsFileOrDirectory(modPath *url.URL, modName string) (sobek.ModuleRecord, error) {
+ mod, err := ml.loadAsFile(modPath, modName)
+ if err != nil {
+ return ml.loadAsDirectory(modPath.JoinPath(modName))
+ }
+ return mod, nil
+}
+
+func (ml *moduleLoader) loadAsFile(modPath *url.URL, modName string) (module sobek.ModuleRecord, err error) {
+ if module, err = ml.loadModule(modPath, modName); err == nil {
+ return
+ }
+ if module, err = ml.loadModule(modPath, modName+".js"); err == nil {
+ return
+ }
+ return ml.loadModule(modPath, modName+".json")
+}
+
+func (ml *moduleLoader) loadAsDirectory(modPath *url.URL) (module sobek.ModuleRecord, err error) {
+ buf, err := ml.fileLoader(modPath.JoinPath("package.json"), "package.json")
+ if err != nil {
+ return ml.loadModule(modPath, "index.js")
+ }
+ var pkg struct {
+ Main string `json:"main"`
+ }
+ err = json.Unmarshal(buf, &pkg)
+ if err != nil || len(pkg.Main) == 0 {
+ return ml.loadModule(modPath, "index.js")
+ }
+
+ if module, err = ml.loadAsFile(modPath, pkg.Main); module != nil || err != nil {
+ return
+ }
+
+ return ml.loadModule(modPath, "index.js")
+}
+
+func (ml *moduleLoader) loadNodeModules(modName string) (mod sobek.ModuleRecord, err error) {
+ start := ml.base.Path
+ for {
+ var p string
+ if path.Base(start) != "node_modules" {
+ p = path.Join(start, "node_modules")
+ } else {
+ p = start
+ }
+ if mod, err = ml.loadAsFileOrDirectory(ml.base.JoinPath(p), modName); mod != nil || err != nil {
+ return
+ }
+ if start == ".." { // Dir('..') is '.'
+ break
+ }
+ parent := path.Dir(start)
+ if parent == start {
+ break
+ }
+ start = parent
+ }
+
+ return nil, fmt.Errorf("not found module %s at %s", modName, ml.base)
+}
+
+func (ml *moduleLoader) loadModule(modPath *url.URL, modName string) (sobek.ModuleRecord, error) {
+ file := modPath.JoinPath(modName)
+ specifier := file.String()
+
+ ml.Lock()
+ defer ml.Unlock()
+
+ cache, exists := ml.modules[specifier]
+ if exists {
+ return cache.mod, cache.err
+ }
+
+ buf, err := ml.fileLoader(file, modName)
+ if err != nil {
+ return nil, err
+ }
+ mod, err := ml.CompileModule(specifier, string(buf))
+ if err == nil {
+ file.Path = filepath.Dir(file.Path)
+ ml.reverse[mod] = file
+ }
+ ml.modules[specifier] = moduleCache{mod: mod, err: err}
+ return mod, err
+}
+
+func (ml *moduleLoader) CompileModule(name, source string) (sobek.CyclicModuleRecord, error) {
+ if filepath.Ext(name) == ".json" {
+ source = "module.exports = JSON.parse('" + template.JSEscapeString(source) + "')"
+ return ml.compileCjsModule(name, source)
+ }
+
+ ast, err := sobek.Parse(name, source, parser.IsModule, ml.sourceLoader)
+ if err != nil {
+ return nil, err
+ }
+
+ isModule := len(ast.ExportEntries) > 0 || len(ast.ImportEntries) > 0 || ast.HasTLA
+ if !isModule {
+ return ml.compileCjsModule(name, source)
+ }
+
+ return sobek.ModuleFromAST(ast, ml.ResolveModule)
+}
+
+func (ml *moduleLoader) compileCjsModule(name, source string) (sobek.CyclicModuleRecord, error) {
+ source = "(function(exports, require, module) {" + source + "\n})"
+
+ ast, err := sobek.Parse(name, source, ml.sourceLoader)
+ if err != nil {
+ return nil, err
+ }
+
+ prg, err := sobek.CompileAST(ast, false)
+ if err != nil {
+ return nil, err
+ }
+
+ return &cjsModule{prg: prg}, nil
+}
+
+func isBasePath(path string) bool {
+ return strings.HasPrefix(path, "/") ||
+ strings.HasPrefix(path, "./") ||
+ strings.HasPrefix(path, "../") ||
+ path == "." || path == ".."
+}
+
+var errNotSupport = errors.New("js.ModuleLoader not provided, require and module not working")
+
+func (e emptyLoader) CompileModule(name string, source string) (sobek.CyclicModuleRecord, error) {
+ return sobek.ParseModule(name, source, e.ResolveModule)
+}
+func (emptyLoader) ResolveModule(any, string) (sobek.ModuleRecord, error) {
+ return nil, errNotSupport
+}
+func (e emptyLoader) EnableRequire(rt *sobek.Runtime) ModuleLoader {
+ _ = rt.Set("require", func() {
+ panic(rt.NewGoError(errNotSupport))
+ })
+ return e
+}
+func (e emptyLoader) EnableImportModuleDynamically(rt *sobek.Runtime) ModuleLoader {
+ rt.SetImportModuleDynamically(func(referencingScriptOrModule any, specifier sobek.Value, promiseCapability any) {
+ NewPromise(rt,
+ func() (sobek.ModuleRecord, error) {
+ return nil, errNotSupport
+ })
+ })
+ return e
+}
diff --git a/js/loader_test.go b/js/loader_test.go
new file mode 100644
index 0000000..24ff9a9
--- /dev/null
+++ b/js/loader_test.go
@@ -0,0 +1,313 @@
+package js
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "io/fs"
+ "net/http"
+ "net/url"
+ "strconv"
+ "strings"
+ "sync"
+ "testing"
+ "testing/fstest"
+ _ "unsafe"
+
+ "github.com/grafana/sobek"
+ "github.com/shiroyk/ski"
+ "github.com/stretchr/testify/assert"
+)
+
+type fetch struct{}
+
+func (*fetch) Do(req *http.Request) (*http.Response, error) {
+ source := `module.exports = { foo: 'bar' + require('ski/gomod1').key }`
+ if req.URL.Query().Get("type") == "esm" {
+ source = `
+import gomod1 from "ski/gomod1";
+const a = async () => 4;
+export default async () => gomod1.key + 1 + (await a())`
+ }
+ return &http.Response{Body: io.NopCloser(strings.NewReader(source))}, nil
+}
+
+type gomod1 struct{}
+
+func (gomod1) Instantiate(rt *sobek.Runtime) (sobek.Value, error) {
+ return rt.ToValue(map[string]string{"key": "gomod1"}), nil
+}
+
+type gomod2 struct{}
+
+func (gomod2) Instantiate(rt *sobek.Runtime) (sobek.Value, error) {
+ return rt.ToValue(struct {
+ Key string `js:"key"`
+ }{Key: "gomod2"}), nil
+}
+
+type gomod3 struct{}
+
+func (gomod3) Instantiate(rt *sobek.Runtime) (sobek.Value, error) {
+ return rt.ToValue(map[string]string{"key": "gomod3"}), nil
+}
+
+func (gomod3) Global() {}
+
+func TestModuleLoader(t *testing.T) {
+ t.Parallel()
+ fetch := new(fetch)
+ mfs := fstest.MapFS{
+ "node_modules/module1/index.js": &fstest.MapFile{
+ Data: []byte(`module.exports = function() { return "module1" };`),
+ },
+ "node_modules/module2/index.js": &fstest.MapFile{
+ Data: []byte(`
+ import m1 from "module1";
+ export default function() { return m1() + "/module2" };
+ `),
+ },
+ "node_modules/module3/index.js": &fstest.MapFile{
+ Data: []byte(`
+ import module2 from "module2";
+ import { module3 } from "./module3";
+ export default function() { return module2() + module3() };
+ `),
+ },
+ "node_modules/module3/module3.js": &fstest.MapFile{
+ Data: []byte(`export function module3() { return "/module3" };`),
+ },
+ "node_modules/module4/lib/module4.js": &fstest.MapFile{
+ Data: []byte(`export default () => { return "/module4" };`),
+ },
+ "node_modules/module4/package.json": &fstest.MapFile{
+ Data: []byte(`{"main": "lib/module4.js"}`),
+ },
+ "node_modules/module5/lib/index.js": &fstest.MapFile{
+ Data: []byte(`
+ import { msg as msg6 } from "module6";
+ export const msg = "/module5";
+ export default () => msg + msg6;`),
+ },
+ "node_modules/module5/package.json": &fstest.MapFile{
+ Data: []byte(`{"main": "lib/index.js"}`),
+ },
+ "node_modules/module6/lib/index.js": &fstest.MapFile{
+ Data: []byte(`
+ import { msg as msg5 } from "module5";
+ export const msg = "/module6";
+ export default () => msg + msg5;`),
+ },
+ "node_modules/module6/package.json": &fstest.MapFile{
+ Data: []byte(`{"main": "lib/index.js"}`),
+ },
+ "node_modules/module7/index.js": &fstest.MapFile{
+ Data: []byte(`export default async () => "dynamic import " + (await import('module6')).msg;`),
+ },
+ "es_script1.js": &fstest.MapFile{
+ Data: []byte(`
+ import module3 from "module3";
+ export default function() { return module3() + "/es_script1" };
+ `),
+ },
+ "es_script2.js": &fstest.MapFile{
+ Data: []byte(`export const value = () => 555;`),
+ },
+ "cjs_script1.js": &fstest.MapFile{
+ Data: []byte(`module.exports = () => { return require('module4')() + "/cjs_script1" };`),
+ },
+ "cjs_script2.js": &fstest.MapFile{
+ Data: []byte(`
+ const { value } = require('./es_script2');
+ exports.value = () => value();
+ `),
+ },
+ "json1.json": &fstest.MapFile{
+ Data: []byte(`{"key": "json1"}`),
+ },
+ }
+ loader := NewModuleLoader(WithFileLoader(func(specifier *url.URL, name string) ([]byte, error) {
+ switch specifier.Scheme {
+ case "http", "https":
+ res, err := fetch.Do(&http.Request{URL: specifier})
+ if err != nil {
+ return nil, err
+ }
+ body, err := io.ReadAll(res.Body)
+ return body, err
+ case "file":
+ return mfs.ReadFile(specifier.Path)
+ default:
+ return nil, fmt.Errorf("unexpected scheme %s", specifier.Scheme)
+ }
+ }))
+ Register("gomod1", new(gomod1))
+ Register("gomod2", new(gomod2))
+ Register("gomod3", new(gomod3))
+ vm := NewTestVM(t, WithModuleLoader(loader))
+
+ {
+ scriptCases := []struct{ name, s string }{
+ {"gomod1", `assert.equal(require("ski/gomod1").key, "gomod1")`},
+ {"gomod2", `assert.equal(require("ski/gomod2").key, "gomod2")`},
+ {"gomod3", `assert.equal(gomod3.key, "gomod3")`},
+ {"remote cjs", `assert.equal(require("http://foo.com/foo.min.js?type=cjs").foo, "bargomod1")`},
+ {"remote esm", `(async () => assert.equal(await require("http://foo.com/foo.min.js?type=esm")(), "gomod114"))()`},
+ {"module1", `assert.equal(require("module1")(), "module1")`},
+ {"module2", `assert.equal(require("module2")(), "module1/module2")`},
+ {"module3", `assert.equal(require("module3")(), "module1/module2/module3")`},
+ {"module4", `assert.equal(require("module4")(), "/module4")`},
+ {"module5", `assert.equal(require("module5")(), "/module5/module6")`},
+ {"module6", `assert.equal(require("module6")(), "/module6/module5")`},
+ {"module7", `(async () => assert.equal(await require("module7")(), "dynamic import /module6"))()`},
+ {"es_script1", `assert.equal(require("./es_script1")(), "module1/module2/module3/es_script1")`},
+ {"es_script2", `assert.equal(require("./es_script2").value(), 555)`},
+ {"cjs_script1", `assert.equal(require("./cjs_script1")(), "/module4/cjs_script1")`},
+ {"cjs_script2", `assert.equal(require("./cjs_script2").value(), 555)`},
+ {"json1", `assert.equal(require("./json1.json").key, "json1")`},
+ }
+
+ for _, script := range scriptCases {
+ t.Run(fmt.Sprintf("script %s", script.name), func(t *testing.T) {
+ vm.Run(context.Background(), func() {
+ _, err := vm.Runtime().RunString(script.s)
+ assert.NoError(t, err)
+ })
+ })
+ }
+ }
+ {
+ moduleCases := []struct{ name, s string }{
+ {"gomod1", `import gomod1 from "ski/gomod1";
+ export default () => assert.equal(gomod1.key, "gomod1")`},
+ {"gomod2", `import gomod2 from "ski/gomod2";
+ export default () => assert.equal(gomod2.key, "gomod2")`},
+ {"gomod3", `export default () => assert.equal(gomod3.key, "gomod3")`},
+ {"remote cjs", `import foo from "http://foo.com/foo.min.js?type=cjs";
+ export default () => assert.equal(foo.foo, "bargomod1")`},
+ {"remote esm", `import foo from "http://foo.com/foo.min.js?type=esm";
+ export default async () => assert.equal(await foo(), "gomod114")`},
+ {"module1", `import module1 from "module1";
+ export default () => assert.equal(module1(), "module1");`},
+ {"module2", `import m2 from "module2";
+ export default () => assert.equal(m2(), "module1/module2");`},
+ {"module3", `import module3 from "module3";
+ export default () => assert.equal(module3(), "module1/module2/module3");`},
+ {"module4", `import module4 from "module4";
+ export default () => assert.equal(module4(), "/module4");`},
+ {"module5", `import module5 from "module5";
+ export default () => assert.equal(module5(), "/module5/module6");`},
+ {"module6", `import module6 from "module6";
+ export default () => assert.equal(module6(), "/module6/module5");`},
+ {"module7", `import module7 from "module7";
+ export default async () => assert.equal(await module7(), "dynamic import /module6");`},
+ {"es_script1", `import es from "./es_script1";
+ export default () => assert.equal(es(), "module1/module2/module3/es_script1");`},
+ {"es_script2", `import { value } from "./es_script2";
+ export default () => assert.equal(value(), 555);`},
+ {"cjs_script1", `import cjs from "./cjs_script1";
+ export default () => assert.equal(cjs(), "/module4/cjs_script1");`},
+ {"cjs_script2", `import { value } from "./cjs_script2";
+ export default () => assert.equal(value(), 555);`},
+ {"json1", `import j from "./json1.json";
+ export default () => assert.equal(j.key, "json1");`},
+ }
+
+ for _, script := range moduleCases {
+ t.Run(fmt.Sprintf("module %v", script.name), func(t *testing.T) {
+ mod, err := loader.CompileModule("", script.s)
+ if assert.NoError(t, err) {
+ _, err = vm.RunModule(context.Background(), mod)
+ assert.NoError(t, err)
+ }
+ })
+ }
+ }
+}
+
+func TestConcurrentLoader(t *testing.T) {
+ t.Parallel()
+ num := 8
+
+ mfs := make(fstest.MapFS, num)
+ for i := 0; i < num; i++ {
+ mfs[fmt.Sprintf("module%d.js", i)] = &fstest.MapFile{Data: []byte(`export default () => ` + strconv.Itoa(i))}
+ }
+
+ fileLoader := WithFileLoader(func(specifier *url.URL, name string) ([]byte, error) {
+ return fs.ReadFile(mfs, specifier.Path)
+ })
+ scheduler := NewScheduler(SchedulerOptions{
+ InitialVMs: 2,
+ Loader: NewModuleLoader(fileLoader),
+ })
+
+ var wg sync.WaitGroup
+
+ for i := 0; i < num; i++ {
+ wg.Add(1)
+ go func(j int) {
+ defer wg.Done()
+
+ vm, err := scheduler.Get()
+ if assert.NoError(t, err) {
+ mod, err := scheduler.Loader().CompileModule("", fmt.Sprintf(`
+ import m from './module%d.js';
+ export default () => m()`, j))
+ if assert.NoError(t, err) {
+ v, err := vm.RunModule(context.Background(), mod)
+ if assert.NoError(t, err) {
+ assert.Equal(t, int64(j), v.ToInteger())
+ }
+ }
+ }
+ }(i)
+ }
+
+ wg.Wait()
+}
+
+type testExec struct{ v any }
+
+func new_testExec(arg ...ski.Executor) (ski.Executor, error) {
+ return testExec{ski.ExecToString(arg[0])}, nil
+}
+
+func (t testExec) Exec(context.Context, any) (any, error) { return t.v, nil }
+
+func TestJSExecutor(t *testing.T) {
+ ski.Register("loader_executor", new_testExec)
+ ski.Register("loader_executor.other", new_testExec)
+ vm := NewTestVM(t, WithModuleLoader(NewModuleLoader()))
+
+ for i, s := range []string{
+ `assert.equal(require("executor/loader_executor")('foo').exec(''), 'foo');`,
+ `assert.equal(require("executor/loader_executor").other('bar').exec(''), 'bar');`,
+ } {
+ t.Run(strconv.Itoa(i), func(t *testing.T) {
+ _, err := vm.Runtime().RunString(s)
+ assert.NoError(t, err)
+ })
+ }
+}
+
+func TestESMExecutor(t *testing.T) {
+ exec, err := new_executor()(ski.String(`export default (ctx) => ctx.get('content') + 1`))
+ if assert.NoError(t, err) {
+ v, err := exec.Exec(context.Background(), "a")
+ if assert.NoError(t, err) {
+ assert.Equal(t, "a1", v)
+ }
+ }
+}
+
+func NewTestVM(t *testing.T, opts ...Option) VM {
+ vm := NewVM(opts...)
+ p := vm.Runtime().NewObject()
+ _ = p.Set("equal", func(call sobek.FunctionCall) sobek.Value {
+ assert.Equal(t, call.Argument(1).Export(), call.Argument(0).Export(), call.Argument(2).String())
+ return sobek.Undefined()
+ })
+ _ = vm.Runtime().Set("assert", p)
+ return vm
+}
diff --git a/js/module.go b/js/module.go
new file mode 100644
index 0000000..052503e
--- /dev/null
+++ b/js/module.go
@@ -0,0 +1,276 @@
+package js
+
+import (
+ "context"
+ "maps"
+ "sync"
+
+ "github.com/grafana/sobek"
+ "github.com/shiroyk/ski"
+)
+
+// Module is what a module needs to return
+type Module interface {
+ Instantiate(*sobek.Runtime) (sobek.Value, error)
+}
+
+// Global implements the interface will load into global when the VM initialize (InitGlobalModule).
+type Global interface {
+ Module
+ Global() // is it a global module
+}
+
+// Register the given mod as an external JavaScript module that can be imported
+// by name.
+func Register(name string, mod Module) {
+ if _, ok := mod.(Global); !ok {
+ name = modulePrefix + name
+ }
+ registry.Lock()
+ registry.native[name] = mod
+ registry.Unlock()
+}
+
+// GetModule get the module
+func GetModule(name string) (Module, bool) {
+ registry.RLock()
+ defer registry.RUnlock()
+ module, ok := registry.native[name]
+ return module, ok
+}
+
+func RemoveModule(name string) {
+ registry.Lock()
+ delete(registry.native, name)
+ registry.Unlock()
+}
+
+// AllModule get all module
+func AllModule() map[string]Module {
+ registry.RLock()
+ defer registry.RUnlock()
+ return maps.Clone(registry.native)
+}
+
+const modulePrefix = "ski/"
+
+var registry = struct {
+ sync.RWMutex
+ native map[string]Module
+}{
+ native: make(map[string]Module),
+}
+
+type cjsModule struct {
+ prg *sobek.Program
+ exportedNames []string
+ o sync.Once
+}
+
+func (cm *cjsModule) Link() error { return nil }
+
+func (cm *cjsModule) InitializeEnvironment() error { return nil }
+
+func (cm *cjsModule) Instantiate(_ *sobek.Runtime) (sobek.CyclicModuleInstance, error) {
+ return &cjsModuleInstance{m: cm}, nil
+}
+
+func (cm *cjsModule) RequestedModules() []string { return nil }
+
+func (cm *cjsModule) Evaluate(_ *sobek.Runtime) *sobek.Promise { return nil }
+
+func (cm *cjsModule) GetExportedNames(callback func([]string), _ ...sobek.ModuleRecord) bool {
+ callback(cm.exportedNames)
+ return true
+}
+
+func (cm *cjsModule) ResolveExport(exportName string, _ ...sobek.ResolveSetElement) (*sobek.ResolvedBinding, bool) {
+ return &sobek.ResolvedBinding{
+ Module: cm,
+ BindingName: exportName,
+ }, false
+}
+
+type cjsModuleInstance struct {
+ m *cjsModule
+ exports *sobek.Object
+}
+
+func (cmi *cjsModuleInstance) HasTLA() bool { return false }
+
+func (cmi *cjsModuleInstance) GetBindingValue(name string) sobek.Value {
+ if name == "default" {
+ if d := cmi.exports.Get("default"); d != nil {
+ return d
+ }
+ return cmi.exports
+ }
+ return cmi.exports.Get(name)
+}
+
+func (cmi *cjsModuleInstance) ExecuteModule(rt *sobek.Runtime, _, _ func(any)) (sobek.CyclicModuleInstance, error) {
+ f, err := rt.RunProgram(cmi.m.prg)
+ if err != nil {
+ return nil, err
+ }
+
+ jsModule := rt.NewObject()
+ cmi.exports = rt.NewObject()
+ _ = jsModule.Set("exports", cmi.exports)
+ if call, ok := sobek.AssertFunction(f); ok {
+ jsRequire := rt.Get("require")
+
+ // Run the module source, with "cmi.exports" as "this",
+ // "cmi.exports" as the "exports" variable, "jsRequire"
+ // as the "require" variable and "jsModule" as the
+ // "module" variable (Nodejs capable).
+ _, err = call(cmi.exports, cmi.exports, jsRequire, jsModule)
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ exports := jsModule.Get("exports")
+ if sobek.IsNull(exports) {
+ return nil, ErrInvalidModule
+ }
+ cmi.exports = exports.ToObject(rt)
+ cmi.m.o.Do(func() {
+ cmi.m.exportedNames = cmi.exports.Keys()
+ })
+ return cmi, nil
+}
+
+type goModule struct {
+ mod Module
+ once sync.Once
+ exportedNames []string
+}
+
+func (gm *goModule) Link() error { return nil }
+
+func (gm *goModule) RequestedModules() []string { return nil }
+
+func (gm *goModule) InitializeEnvironment() error { return nil }
+
+func (gm *goModule) Instantiate(rt *sobek.Runtime) (sobek.CyclicModuleInstance, error) {
+ instance, err := gm.mod.Instantiate(rt)
+ if err != nil {
+ return nil, err
+ }
+ exports := instance.ToObject(rt)
+ gm.once.Do(func() { gm.exportedNames = exports.Keys() })
+ return &goModuleInstance{exports}, nil
+}
+
+func (gm *goModule) GetExportedNames(callback func([]string), _ ...sobek.ModuleRecord) bool {
+ callback(gm.exportedNames)
+ return true
+}
+
+func (gm *goModule) ResolveExport(exportName string, _ ...sobek.ResolveSetElement) (*sobek.ResolvedBinding, bool) {
+ return &sobek.ResolvedBinding{
+ Module: gm,
+ BindingName: exportName,
+ }, false
+}
+
+func (gm *goModule) Evaluate(_ *sobek.Runtime) *sobek.Promise { return nil }
+
+type goModuleInstance struct{ *sobek.Object }
+
+func (gmi *goModuleInstance) GetBindingValue(name string) sobek.Value {
+ if gmi.Object == nil {
+ return nil
+ }
+ if name == "default" {
+ if v := gmi.Get("default"); v != nil {
+ return v
+ }
+ return gmi.Object
+ }
+ return gmi.Get(name)
+}
+
+func (gmi *goModuleInstance) HasTLA() bool { return false }
+
+func (gmi *goModuleInstance) ExecuteModule(_ *sobek.Runtime, _, _ func(any)) (sobek.CyclicModuleInstance, error) {
+ return gmi, nil
+}
+
+const _js_executor_prefix = "executor/"
+
+type _js_executor map[string]ski.NewExecutor
+
+func (m _js_executor) Instantiate(rt *sobek.Runtime) (sobek.Value, error) {
+ var object *sobek.Object
+ main, ok := m[""]
+ if ok {
+ object = rt.ToValue(toJSExec(main)).ToObject(rt)
+ } else {
+ object = rt.NewObject()
+ }
+ proto := object.Prototype()
+ for k, v := range m {
+ if k == "" {
+ continue
+ }
+ _ = proto.Set(k, toJSExec(v))
+ }
+ return object, nil
+}
+
+type _js_exec struct {
+ e ski.Executor
+}
+
+func (e _js_exec) Exec(call sobek.FunctionCall, rt *sobek.Runtime) sobek.Value {
+ v, err := e.e.Exec(Context(rt), call.Argument(0).Export())
+ if err != nil {
+ return sobek.Null()
+ }
+ return rt.ToValue(v)
+}
+
+func toJSExec(init ski.NewExecutor) func(call sobek.FunctionCall, rt *sobek.Runtime) sobek.Value {
+ return func(call sobek.FunctionCall, rt *sobek.Runtime) sobek.Value {
+ args := make([]ski.Executor, 0, len(call.Arguments))
+ for _, arg := range call.Arguments {
+ args = append(args, ski.Raw(arg.Export()))
+ }
+ exec, err := init(args...)
+ if err != nil {
+ Throw(rt, err)
+ }
+ return rt.ToValue(_js_exec{exec})
+ }
+}
+
+// Executor the ski.Executor
+type Executor struct{ sobek.CyclicModuleRecord }
+
+func new_executor() ski.NewExecutor {
+ return ski.StringExecutor(func(str string) (ski.Executor, error) {
+ module, err := GetScheduler().Loader().CompileModule("", str)
+ if err != nil {
+ return nil, err
+ }
+ return Executor{module}, nil
+ })
+}
+
+func (p Executor) Exec(ctx context.Context, arg any) (any, error) {
+ value, err := RunModule(ski.WithValue(ctx, "content", arg), p)
+ if err != nil {
+ return nil, err
+ }
+
+ unwrap, err := Unwrap(value)
+ if err != nil {
+ return nil, err
+ }
+ if s, ok := unwrap.([]any); ok {
+ return ski.NewIterator(s), nil
+ }
+ return unwrap, nil
+}
diff --git a/js/module_test.go b/js/module_test.go
new file mode 100644
index 0000000..5f1b7d3
--- /dev/null
+++ b/js/module_test.go
@@ -0,0 +1,44 @@
+package js
+
+import (
+ "context"
+ "strconv"
+ "testing"
+
+ "github.com/shiroyk/ski"
+ "github.com/stretchr/testify/assert"
+)
+
+func TestExecutor(t *testing.T) {
+ t.Parallel()
+ vm := NewVM(WithModuleLoader(NewModuleLoader()))
+
+ cases := []struct {
+ script string
+ excepted any
+ }{
+ {`export default () => 1`, int64(1)},
+ {`export default () => [1]`, []any{int64(1)}},
+ {`export default () => ["a"]`, []any{"a"}},
+ {`export default () => [{"a": 1}]`, []any{map[string]any{"a": int64(1)}}},
+ }
+
+ for i, c := range cases {
+ t.Run(strconv.Itoa(i), func(t *testing.T) {
+ module, err := vm.Loader().CompileModule("", c.script)
+ if assert.NoError(t, err) {
+ exec := Executor{module}
+ v, err := exec.Exec(context.Background(), nil)
+ if assert.NoError(t, err) {
+ if s, ok := v.(ski.Iterator); ok {
+ for j := 0; j < s.Len(); j++ {
+ assert.Equal(t, c.excepted.([]any)[j], s.At(j), "at %d", j)
+ }
+ } else {
+ assert.Equal(t, c.excepted, v)
+ }
+ }
+ }
+ })
+ }
+}
diff --git a/js/modules/cache/cache.go b/js/modules/cache/cache.go
index 9cbfb01..dea8b2f 100644
--- a/js/modules/cache/cache.go
+++ b/js/modules/cache/cache.go
@@ -2,72 +2,78 @@
package cache
import (
+ "errors"
"time"
- "github.com/dop251/goja"
- "github.com/shiroyk/cloudcat"
- "github.com/shiroyk/cloudcat/js"
- "github.com/shiroyk/cloudcat/plugin/jsmodule"
+ "github.com/grafana/sobek"
+ "github.com/shiroyk/ski"
+ "github.com/shiroyk/ski/js"
)
-// Module js module
-type Module struct{}
-
-// Exports returns the module instance
-func (*Module) Exports() any {
- return &Cache{cloudcat.MustResolve[cloudcat.Cache]()}
-}
-
func init() {
- jsmodule.Register("cache", new(Module))
+ js.Register("cache", &Cache{ski.NewCache()})
}
// Cache interface is used to store string or bytes.
-type Cache struct {
- cache cloudcat.Cache
+type Cache struct{ ski.Cache }
+
+func (c *Cache) Instantiate(rt *sobek.Runtime) (sobek.Value, error) {
+ if c.Cache == nil {
+ return nil, errors.New("Cache can not nil")
+ }
+ return rt.ToValue(map[string]func(call sobek.FunctionCall, vm *sobek.Runtime) sobek.Value{
+ "get": c.Get,
+ "getBytes": c.GetBytes,
+ "set": c.Set,
+ "setBytes": c.SetBytes,
+ "del": c.Del,
+ }), nil
}
// Get returns string.
-func (c *Cache) Get(call goja.FunctionCall, vm *goja.Runtime) goja.Value {
- if bytes, ok := c.cache.Get(js.VMContext(vm), call.Argument(0).String()); ok {
+func (c *Cache) Get(call sobek.FunctionCall, vm *sobek.Runtime) sobek.Value {
+ if bytes, err := c.Cache.Get(js.Context(vm), call.Argument(0).String()); err == nil && bytes != nil {
return vm.ToValue(string(bytes))
}
- return goja.Undefined()
+ return sobek.Undefined()
}
// GetBytes returns ArrayBuffer.
-func (c *Cache) GetBytes(call goja.FunctionCall, vm *goja.Runtime) goja.Value {
- if bytes, ok := c.cache.Get(js.VMContext(vm), call.Argument(0).String()); ok {
+func (c *Cache) GetBytes(call sobek.FunctionCall, vm *sobek.Runtime) sobek.Value {
+ if bytes, err := c.Cache.Get(js.Context(vm), call.Argument(0).String()); err == nil && bytes != nil {
return vm.ToValue(vm.NewArrayBuffer(bytes))
}
- return goja.Undefined()
+ return sobek.Undefined()
}
// Set saves string to the cache with key.
-func (c *Cache) Set(call goja.FunctionCall, vm *goja.Runtime) goja.Value {
- ctx := js.VMContext(vm)
- if !goja.IsUndefined(call.Argument(2)) {
+func (c *Cache) Set(call sobek.FunctionCall, vm *sobek.Runtime) sobek.Value {
+ ctx := js.Context(vm)
+ if !sobek.IsUndefined(call.Argument(2)) {
timeout, err := time.ParseDuration(call.Argument(2).String())
if err != nil {
js.Throw(vm, err)
}
- ctx = cloudcat.WithCacheTimeout(ctx, timeout)
+ ctx = ski.WithCacheTimeout(ctx, timeout)
}
- c.cache.Set(ctx, call.Argument(0).String(), []byte(call.Argument(1).String()))
+ err := c.Cache.Set(ctx, call.Argument(0).String(), []byte(call.Argument(1).String()))
+ if err != nil {
+ js.Throw(vm, err)
+ }
- return goja.Undefined()
+ return sobek.Undefined()
}
// SetBytes saves ArrayBuffer to the cache with key.
-func (c *Cache) SetBytes(call goja.FunctionCall, vm *goja.Runtime) goja.Value {
- ctx := js.VMContext(vm)
- if !goja.IsUndefined(call.Argument(2)) {
+func (c *Cache) SetBytes(call sobek.FunctionCall, vm *sobek.Runtime) sobek.Value {
+ ctx := js.Context(vm)
+ if !sobek.IsUndefined(call.Argument(2)) {
timeout, err := time.ParseDuration(call.Argument(2).String())
if err != nil {
js.Throw(vm, err)
}
- ctx = cloudcat.WithCacheTimeout(ctx, timeout)
+ ctx = ski.WithCacheTimeout(ctx, timeout)
}
value, err := js.ToBytes(call.Argument(1).Export())
@@ -75,13 +81,19 @@ func (c *Cache) SetBytes(call goja.FunctionCall, vm *goja.Runtime) goja.Value {
js.Throw(vm, err)
}
- c.cache.Set(ctx, call.Argument(0).String(), value)
+ err = c.Cache.Set(ctx, call.Argument(0).String(), value)
+ if err != nil {
+ js.Throw(vm, err)
+ }
- return goja.Undefined()
+ return sobek.Undefined()
}
// Del removes key from the cache.
-func (c *Cache) Del(call goja.FunctionCall, vm *goja.Runtime) goja.Value {
- c.cache.Del(js.VMContext(vm), call.Argument(0).String())
- return goja.Undefined()
+func (c *Cache) Del(call sobek.FunctionCall, vm *sobek.Runtime) sobek.Value {
+ err := c.Cache.Del(js.Context(vm), call.Argument(0).String())
+ if err != nil {
+ js.Throw(vm, err)
+ }
+ return sobek.Undefined()
}
diff --git a/js/modules/cache/cache_test.go b/js/modules/cache/cache_test.go
index bf9014b..c78f8a7 100644
--- a/js/modules/cache/cache_test.go
+++ b/js/modules/cache/cache_test.go
@@ -1,27 +1,32 @@
package cache
import (
- "context"
"testing"
- "github.com/shiroyk/cloudcat"
- "github.com/shiroyk/cloudcat/js/modulestest"
+ "github.com/grafana/sobek"
+ "github.com/shiroyk/ski"
+ "github.com/shiroyk/ski/js"
+ "github.com/shiroyk/ski/js/modulestest"
"github.com/stretchr/testify/assert"
)
func TestCache(t *testing.T) {
t.Parallel()
- cloudcat.Provide[cloudcat.Cache](cloudcat.NewCache())
- ctx := context.Background()
- vm := modulestest.New(t)
+ vm := modulestest.New(t, js.WithInitial(func(rt *sobek.Runtime) {
+ cache := Cache{ski.NewCache()}
+ instantiate, err := cache.Instantiate(rt)
+ if err != nil {
+ t.Fatal(err)
+ }
+ _ = rt.Set("cache", instantiate)
+ }))
- _, err := vm.RunString(ctx, `
- const cache = require('cloudcat/cache');
+ _, err := vm.Runtime().RunString(`
cache.set("cache1", "1");
cache.del("cache1");
assert.true(!cache.get("cache1"), "cache should be deleted");
cache.set("cache2", "2", "1s");
- cache.get("cache2");
+ assert.equal(cache.get("not exists"), undefined);
assert.equal(cache.get("not exists"), undefined);
assert.equal(cache.get("cache2"), "2");
cache.setBytes("cache3", new Uint8Array([50]));
diff --git a/js/modules/cookie/cookie.go b/js/modules/cookie/cookie.go
deleted file mode 100644
index 61e27e3..0000000
--- a/js/modules/cookie/cookie.go
+++ /dev/null
@@ -1,68 +0,0 @@
-// Package cookie the cookie JS implementation
-package cookie
-
-import (
- "net/url"
-
- "github.com/dop251/goja"
- "github.com/shiroyk/cloudcat"
- "github.com/shiroyk/cloudcat/js"
- "github.com/shiroyk/cloudcat/plugin/jsmodule"
-)
-
-// Module js module
-type Module struct{}
-
-// Exports returns module instance
-func (*Module) Exports() any {
- return &Cookie{cloudcat.MustResolve[cloudcat.Cookie]()}
-}
-
-func init() {
- jsmodule.Register("cookie", new(Module))
-}
-
-// Cookie manages storage and use of cookies in HTTP requests.
-type Cookie struct {
- cookie cloudcat.Cookie
-}
-
-// Get returns the cookies string for the given URL.
-func (c *Cookie) Get(uri string) ([]string, error) {
- u, err := url.Parse(uri)
- if err != nil {
- return nil, err
- }
- return c.cookie.CookieString(u), nil
-}
-
-// Set handles the receipt of the cookies strung in a reply for the given URL.
-func (c *Cookie) Set(call goja.FunctionCall, vm *goja.Runtime) (ret goja.Value) {
- u, err := url.Parse(call.Argument(0).String())
- if err != nil {
- js.Throw(vm, err)
- }
-
- str, err := js.ToStrings(call.Argument(1).Export())
- if err != nil {
- js.Throw(vm, err)
- }
-
- switch cookie := str.(type) {
- case string:
- c.cookie.SetCookies(u, cloudcat.ParseCookie(cookie))
- case []string:
- c.cookie.SetCookies(u, cloudcat.ParseSetCookie(cookie...))
- }
- return
-}
-
-// Del handles the receipt of the cookies in a reply for the given URL.
-func (c *Cookie) Del(uri string) error {
- u, err := url.Parse(uri)
- if err != nil {
- return err
- }
- c.cookie.DeleteCookie(u)
- return nil
-}
diff --git a/js/modules/cookie/cookie_test.go b/js/modules/cookie/cookie_test.go
deleted file mode 100644
index 2b99880..0000000
--- a/js/modules/cookie/cookie_test.go
+++ /dev/null
@@ -1,35 +0,0 @@
-package cookie
-
-import (
- "context"
- "testing"
-
- "github.com/shiroyk/cloudcat"
- "github.com/shiroyk/cloudcat/js/modulestest"
- "github.com/stretchr/testify/assert"
-)
-
-func TestCookie(t *testing.T) {
- t.Parallel()
- cloudcat.Provide[cloudcat.Cookie](cloudcat.NewCookie())
- ctx := context.Background()
- vm := modulestest.New(t)
-
- _, _ = vm.Runtime().RunString(`const cookie = require('cloudcat/cookie')`)
-
- var err error
- errScript := []string{`cookie.set('\x0000', "");`, `cookie.get('\x0000');`, `cookie.del('\x0000');`}
- for _, s := range errScript {
- _, err = vm.RunString(ctx, s)
- assert.ErrorContains(t, err, "net/url: invalid control character in URL")
- }
-
- _, err = vm.RunString(ctx, `
- cookie.set("https://github.com", ["test=1; path=/; secure; HttpOnly;"]);
- cookie.del("https://github.com");
- assert.true(!cookie.get("https://github.com").length, "cookie should be deleted");
- cookie.set("https://github.com", ["has_recent_activity=1; path=/; secure; HttpOnly; SameSite=Lax"]);
- assert.equal("has_recent_activity=1", cookie.get("https://github.com")[0]);
- `)
- assert.NoError(t, err)
-}
diff --git a/js/modules/crypto/crypto.go b/js/modules/crypto/crypto.go
index ef93620..ac6cc7f 100644
--- a/js/modules/crypto/crypto.go
+++ b/js/modules/crypto/crypto.go
@@ -5,23 +5,26 @@ import (
"encoding/base64"
"encoding/hex"
- "github.com/dop251/goja"
- "github.com/shiroyk/cloudcat/plugin/jsmodule"
+ "github.com/grafana/sobek"
+ "github.com/shiroyk/ski/js"
)
-// Module js module
-type Module struct{}
+func init() {
+ js.Register("crypto", new(Crypto))
+}
-// Exports returns module instance
-func (*Module) Exports() any {
- return map[string]any{
+// Crypto js module
+type Crypto struct{}
+
+// Instantiate returns module instance
+func (*Crypto) Instantiate(rt *sobek.Runtime) (sobek.Value, error) {
+ return rt.ToValue(map[string]any{
"aes": Aes,
"createCipher": CreateCipher,
"createHash": CreateHash,
"createHMAC": CreateHMAC,
"des": Des,
"hmac": Hmac,
- "md4": Md4,
"md5": Md5,
"randomBytes": RandomBytes,
"ripemd160": Ripemd160,
@@ -32,44 +35,22 @@ func (*Module) Exports() any {
"sha512": Sha512,
"sha512_224": Sha512_224,
"sha512_256": Sha512_256,
- }
-}
-
-func init() {
- jsmodule.Register("crypto", new(Module))
+ }), nil
}
// Encoder the encoded
-type Encoder struct {
- data []byte
-}
+type Encoder struct{ data []byte }
// Base64 encode to base64
-func (e *Encoder) Base64() string {
- return base64.StdEncoding.EncodeToString(e.data)
-}
-
-// Base64url encode to base64url
-func (e *Encoder) Base64url() string {
- return base64.URLEncoding.EncodeToString(e.data)
-}
-
-// Base64rawurl encode to base64rawurl
-func (e *Encoder) Base64rawurl() string {
- return base64.URLEncoding.WithPadding(base64.NoPadding).EncodeToString(e.data)
-}
+func (e *Encoder) Base64() string { return base64.StdEncoding.EncodeToString(e.data) }
// Hex encode to hex
-func (e *Encoder) Hex() string {
- return hex.EncodeToString(e.data)
-}
+func (e *Encoder) Hex() string { return hex.EncodeToString(e.data) }
// String encode to string
-func (e *Encoder) String() string {
- return string(e.data)
-}
+func (e *Encoder) String() string { return string(e.data) }
// Binary encode to arraybuffer
-func (e *Encoder) Binary(_ goja.FunctionCall, vm *goja.Runtime) goja.Value {
+func (e *Encoder) Binary(_ sobek.FunctionCall, vm *sobek.Runtime) sobek.Value {
return vm.ToValue(vm.NewArrayBuffer(e.data))
}
diff --git a/js/modules/crypto/digest.go b/js/modules/crypto/digest.go
index 1426c15..8415de4 100644
--- a/js/modules/crypto/digest.go
+++ b/js/modules/crypto/digest.go
@@ -2,84 +2,60 @@ package crypto
import (
"crypto/hmac"
- "crypto/md5" //nolint:gosec
+ "crypto/md5"
"crypto/rand"
- "crypto/sha1" //nolint:gosec
+ "crypto/sha1"
"crypto/sha256"
"crypto/sha512"
"errors"
"fmt"
"hash"
- "github.com/dop251/goja"
- "github.com/shiroyk/cloudcat/js"
- "golang.org/x/crypto/md4" //nolint:staticcheck
- "golang.org/x/crypto/ripemd160" //nolint:staticcheck
+ "github.com/grafana/sobek"
+ "github.com/shiroyk/ski/js"
+ "golang.org/x/crypto/ripemd160"
)
-// Copyright grafana/k6, licensed under the AGPL License.
-
-// RandomBytes returns random data of the given size.
-func RandomBytes(call goja.FunctionCall, vm *goja.Runtime) (ret goja.Value) {
+// RandomBytes returns a random ArrayBuffer of the given size.
+func RandomBytes(call sobek.FunctionCall, rt *sobek.Runtime) sobek.Value {
size := int(call.Argument(0).ToInteger())
if size < 1 {
- js.Throw(vm, errors.New("invalid size"))
+ js.Throw(rt, errors.New("invalid size"))
}
bytes := make([]byte, size)
if _, err := rand.Read(bytes); err != nil {
- js.Throw(vm, err)
+ js.Throw(rt, err)
}
- return vm.ToValue(vm.NewArrayBuffer(bytes))
-}
-
-// Md4 returns the MD4 Hash of input in the given encoding.
-func Md4(input any) (any, error) {
- return Hash("md4", input)
+ return rt.ToValue(rt.NewArrayBuffer(bytes))
}
// Md5 returns the MD5 Hash of input in the given encoding.
-func Md5(input any) (any, error) {
- return Hash("md5", input)
-}
+func Md5(input any) (any, error) { return Hash("md5", input) }
// Sha1 returns the SHA1 Hash of input in the given encoding.
-func Sha1(input any) (any, error) {
- return Hash("sha1", input)
-}
+func Sha1(input any) (any, error) { return Hash("sha1", input) }
// Sha256 returns the SHA256 Hash of input in the given encoding.
-func Sha256(input any) (any, error) {
- return Hash("sha256", input)
-}
+func Sha256(input any) (any, error) { return Hash("sha256", input) }
// Sha384 returns the SHA384 Hash of input in the given encoding.
-func Sha384(input any) (any, error) {
- return Hash("sha384", input)
-}
+func Sha384(input any) (any, error) { return Hash("sha384", input) }
// Sha512 returns the SHA512 Hash of input in the given encoding.
-func Sha512(input any) (any, error) {
- return Hash("sha512", input)
-}
+func Sha512(input any) (any, error) { return Hash("sha512", input) }
// Sha512_224 returns the SHA512/224 Hash of input in the given encoding.
-func Sha512_224(input any) (any, error) {
- return Hash("sha512_224", input)
-}
+func Sha512_224(input any) (any, error) { return Hash("sha512_224", input) }
// Sha512_256 returns the SHA512/256 Hash of input in the given encoding.
-func Sha512_256(input any) (any, error) {
- return Hash("sha512_256", input)
-}
+func Sha512_256(input any) (any, error) { return Hash("sha512_256", input) }
// Ripemd160 returns the RIPEMD160 Hash of input in the given encoding.
-func Ripemd160(input any) (any, error) {
- return Hash("ripemd160_256", input)
-}
+func Ripemd160(input any) (any, error) { return Hash("ripemd160", input) }
// CreateHash returns a Hasher instance that uses the given algorithm.
func CreateHash(algorithm string) (*Hasher, error) {
- h := parseHashFunc(algorithm) //nolint:ifshort
+ h := hashFunc(algorithm)
if h == nil {
return nil, fmt.Errorf("invalid algorithm: %s", algorithm)
}
@@ -97,16 +73,16 @@ func Hash(algorithm string, input any) (*Encoder, error) {
// CreateHMAC returns a new HMAC Hash using the given algorithm and key.
func CreateHMAC(algorithm string, key any) (*Hasher, error) {
- h := parseHashFunc(algorithm)
+ h := hashFunc(algorithm)
if h == nil {
return nil, fmt.Errorf("invalid algorithm: %s", algorithm)
}
- kb, err := js.ToBytes(key)
+ data, err := js.ToBytes(key)
if err != nil {
return nil, err
}
- return &Hasher{hmac.New(h, kb)}, nil
+ return &Hasher{hmac.New(h, data)}, nil
}
// Hmac returns a new Encoder of input using the given algorithm and key.
@@ -118,29 +94,27 @@ func Hmac(algorithm string, key, input any) (*Encoder, error) {
return hasher.Encrypt(input)
}
-func parseHashFunc(a string) func() hash.Hash {
- var h func() hash.Hash
- switch a {
- case "md4":
- h = md4.New
+func hashFunc(name string) func() hash.Hash {
+ switch name {
case "md5":
- h = md5.New
+ return md5.New
case "sha1":
- h = sha1.New
+ return sha1.New
case "sha256":
- h = sha256.New
+ return sha256.New
case "sha384":
- h = sha512.New384
+ return sha512.New384
+ case "sha512":
+ return sha512.New
case "sha512_224":
- h = sha512.New512_224
+ return sha512.New512_224
case "sha512_256":
- h = sha512.New512_256
- case "sha512":
- h = sha512.New
- case "ripemd160_256":
- h = ripemd160.New
+ return sha512.New512_256
+ case "ripemd160":
+ return ripemd160.New
+ default:
+ return nil
}
- return h
}
// Hasher wraps a hash.Hash.
diff --git a/js/modules/crypto/digest_test.go b/js/modules/crypto/digest_test.go
index d8b470b..523736d 100644
--- a/js/modules/crypto/digest_test.go
+++ b/js/modules/crypto/digest_test.go
@@ -1,347 +1,75 @@
package crypto
import (
- "context"
- "crypto/rand"
- "errors"
"testing"
- "github.com/shiroyk/cloudcat/js/modulestest"
+ "github.com/grafana/sobek"
+ "github.com/shiroyk/ski/js"
+ "github.com/shiroyk/ski/js/modulestest"
"github.com/stretchr/testify/assert"
)
-type MockReader struct{}
-
-func (MockReader) Read(_ []byte) (n int, err error) {
- return -1, errors.New("contrived failure")
-}
-
func TestHashAlgorithms(t *testing.T) {
- if testing.Short() {
- return
- }
-
- vm := modulestest.New(t)
- _, _ = vm.Runtime().RunString(`
- const crypto = require('cloudcat/crypto');
- `)
-
- t.Run("RandomBytesSuccess", func(t *testing.T) {
- _, err := vm.RunString(context.Background(), `
- let buf = crypto.randomBytes(5);
- assert.equal(5, buf.byteLength);
- `)
-
- assert.NoError(t, err)
- })
-
- t.Run("RandomBytesInvalidSize", func(t *testing.T) {
- _, err := vm.RunString(context.Background(), `crypto.randomBytes(-1);`)
-
- assert.Error(t, err)
- })
-
- t.Run("RandomBytesFailure", func(t *testing.T) {
- SavedReader := rand.Reader
- rand.Reader = MockReader{}
- _, err := vm.RunString(context.Background(), `crypto.randomBytes(5);`)
- rand.Reader = SavedReader
-
- assert.Error(t, err)
- })
-
- t.Run("MD4", func(t *testing.T) {
- _, err := vm.RunString(context.Background(), `
- var correct = "aa010fbc1d14c795d86ef98c95479d17";
- var hash = crypto.md4("hello world").hex();
- assert.equal(correct, hash);
- `)
- assert.NoError(t, err)
- })
-
- t.Run("MD5", func(t *testing.T) {
- _, err := vm.RunString(context.Background(), `
- var correct = "5eb63bbbe01eeed093cb22bb8f5acdc3";
- var hash = crypto.md5("hello world").hex();
- assert.equal(correct, hash);
- `)
-
- assert.NoError(t, err)
- })
-
- t.Run("SHA1", func(t *testing.T) {
- _, err := vm.RunString(context.Background(), `
- var correct = "2aae6c35c94fcfb415dbe95f408b9ce91ee846ed";
- var hash = crypto.sha1("hello world").hex();
- assert.equal(correct, hash);
- `)
-
- assert.NoError(t, err)
- })
-
- t.Run("SHA256", func(t *testing.T) {
- _, err := vm.RunString(context.Background(), `
- var correct = "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9";
- var hash = crypto.sha256("hello world").hex();
- assert.equal(correct, hash);
- `)
-
- assert.NoError(t, err)
- })
-
- t.Run("SHA384", func(t *testing.T) {
- _, err := vm.RunString(context.Background(), `
- var correct = "fdbd8e75a67f29f701a4e040385e2e23986303ea10239211af907fcbb83578b3e417cb71ce646efd0819dd8c088de1bd";
- var hash = crypto.sha384("hello world").hex();
- assert.equal(correct, hash);
- `)
-
- assert.NoError(t, err)
- })
-
- t.Run("SHA512", func(t *testing.T) {
- _, err := vm.RunString(context.Background(), `
- var correct = "309ecc489c12d6eb4cc40f50c902f2b4d0ed77ee511a7c7a9bcd3ca86d4cd86f989dd35bc5ff499670da34255b45b0cfd830e81f605dcf7dc5542e93ae9cd76f";
- var hash = crypto.sha512("hello world").hex();
- assert.equal(correct, hash);
- `)
-
- assert.NoError(t, err)
- })
-
- t.Run("SHA512_224", func(t *testing.T) {
- _, err := vm.RunString(context.Background(), `
- var hash = crypto.sha512_224("hello world").hex();
- var correct = "22e0d52336f64a998085078b05a6e37b26f8120f43bf4db4c43a64ee";
- assert.equal(correct, hash);
- `)
-
- assert.NoError(t, err)
- })
-
- t.Run("SHA512_256", func(t *testing.T) {
- _, err := vm.RunString(context.Background(), `
- var hash = crypto.sha512_256("hello world").hex();
- var correct = "0ac561fac838104e3f2e4ad107b4bee3e938bf15f2b15f009ccccd61a913f017";
- assert.equal(correct, hash);
- `)
-
- assert.NoError(t, err)
- })
-
- t.Run("RIPEMD160", func(t *testing.T) {
- _, err := vm.RunString(context.Background(), `
- var hash = crypto.ripemd160("hello world").hex();
- var correct = "98c615784ccb5fe5936fbc0cbe9dfdb408d92f0f";
- assert.equal(correct, hash);
- `)
-
- assert.NoError(t, err)
- })
-}
-
-func TestStreamingApi(t *testing.T) {
- if testing.Short() {
- return
+ vm := modulestest.New(t, js.WithInitial(func(rt *sobek.Runtime) {
+ c := new(Crypto)
+ instance, _ := c.Instantiate(rt)
+ _ = rt.Set("crypto", instance)
+ }))
+
+ testCases := []struct {
+ algorithm, origin, want string
+ }{
+ {"md5", "hello md5", "741fc6b1878e208346359af502dd11c5"},
+ {"ripemd160", "hello ripemd160", "6fb0548fc1acb266457d6ddae686905295b47a2a"},
+ {"sha1", "hello sha1", "64faca92dec81be17500f67d521fbd32bb3a6968"},
+ {"sha256", "hello sha256", "433855b7d2b96c23a6f60e70c655eb4305e8806b682a9596a200642f947259b1"},
+ {"sha384", "hello sha384", "5a37b3a56f9a5ae7b267d25303801d2a610c329d799e9a61879fe35b8108ccb8a4c1154c420ea69fdb6d177fbf6db8b6"},
+ {"sha512", "hello sha512", "ae9ae8f823f9b841bd94062d0af09c2dcffc04a705a89e5415330ed1279f369ea990ca92d63adda838696efe28436c0c14d8e805cd0f04b6c6a0e25127de838c"},
+ {"sha512_224", "hello sha512_224", "60765c29a50404c4ff1797540fd5bd38383a24d1232e39030638e647"},
+ {"sha512_256", "hello sha512_256", "b5e03d2c411178f6c174370e2f420d274cd20b9635ae7a41e40120d826a4b23b"},
}
- vm := modulestest.New(t)
- _, _ = vm.Runtime().RunString(`
- const crypto = require('cloudcat/crypto');
- `)
-
- // Empty strings are still hashable
- t.Run("Empty", func(t *testing.T) {
- _, err := vm.RunString(context.Background(), `
- var correctHex = "d41d8cd98f00b204e9800998ecf8427e";
- var hasher = crypto.createHash("md5");
- assert.equal(correctHex, hasher.digest().hex());
- `)
-
- assert.NoError(t, err)
- })
-
- t.Run("UpdateOnce", func(t *testing.T) {
- _, err := vm.RunString(context.Background(), `
- var correctHex = "5eb63bbbe01eeed093cb22bb8f5acdc3";
-
- var hasher = crypto.createHash("md5");
- hasher.update("hello world");
- assert.equal(correctHex, hasher.digest().hex());
- `)
-
- assert.NoError(t, err)
- })
-
- t.Run("UpdateMultiple", func(t *testing.T) {
- _, err := vm.RunString(context.Background(), `
- var correctHex = "5eb63bbbe01eeed093cb22bb8f5acdc3";
-
- var hasher = crypto.createHash("md5");
- hasher.update("hello");
- hasher.update(" ");
- hasher.update("world");
-
- assert.equal(correctHex, hasher.digest().hex());
- `)
-
- assert.NoError(t, err)
- })
-}
-
-func TestOutputEncoding(t *testing.T) {
- if testing.Short() {
- return
+ for _, testCase := range testCases {
+ t.Run(testCase.algorithm, func(t *testing.T) {
+ _, err := vm.Runtime().RunString(`{
+ let correct = "` + testCase.want + `";
+ let hash = crypto.` + testCase.algorithm + `("` + testCase.origin + `").hex();
+ assert.equal(hash, correct);
+ }`)
+ assert.NoError(t, err)
+ })
}
-
- vm := modulestest.New(t)
- _, _ = vm.Runtime().RunString(`
- const crypto = require('cloudcat/crypto');
- `)
-
- t.Run("Valid", func(t *testing.T) {
- _, err := vm.RunString(context.Background(), `
- let correctHex = "5eb63bbbe01eeed093cb22bb8f5acdc3";
- let correctBase64 = "XrY7u+Ae7tCTyyK7j1rNww==";
- let correctBase64URL = "XrY7u-Ae7tCTyyK7j1rNww=="
- let correctBase64RawURL = "XrY7u-Ae7tCTyyK7j1rNww";
- let correctBinary = new Uint8Array([94,182,59,187,224,30,238,208,147,203,34,187,143,90,205,195]);
-
- let hasher = crypto.createHash("md5");
- let encoder = hasher.encrypt("hello world");
-
- assert.equal(correctHex, encoder.hex());
- assert.equal(correctBase64, encoder.base64());
- assert.equal(correctBase64URL, encoder.base64url());
- assert.equal(correctBase64RawURL, encoder.base64rawurl());
- assert.equal(correctBinary, encoder.binary());
- `)
-
- assert.NoError(t, err)
- })
-
- t.Run("Invalid", func(t *testing.T) {
- _, err := vm.RunString(context.Background(), `
- crypto.createHash("md5").encrypt("hello world").someInvalidEncoding();
- `)
- assert.ErrorContains(t, err, "Object has no member 'someInvalidEncoding'")
- })
}
func TestHMac(t *testing.T) {
- if testing.Short() {
- return
+ vm := modulestest.New(t, js.WithInitial(func(rt *sobek.Runtime) {
+ c := new(Crypto)
+ instance, _ := c.Instantiate(rt)
+ _ = rt.Set("crypto", instance)
+ }))
+
+ testCases := []struct {
+ algorithm, origin, want string
+ }{
+ {"md5", "hello hmac md5", "6c241e7c650d8a839aeff9a7a28db599"},
+ {"ripemd160", "hello hmac ripemd160", "dfbd49aebc8a7cc33ffd3f6e16ab922a23329c2d"},
+ {"sha1", "hello hmac sha1", "754cfe3b0dc73755f9d7cfa90ec979e2c1d42f08"},
+ {"sha256", "hello hmac sha256", "1d103c86749c67b0c5531bcf4b1125f32540a3bad4165f4efe804a1a5b4dd9f1"},
+ {"sha384", "hello hmac sha384", "bc19f1775949f93a53909fb674c65e6978d6fa80173ead68717543d5e01c229ae0d7f6c5f8901147e9998dd477c701cb"},
+ {"sha512", "hello hmac sha512", "1f893eec7580ed74a38053c88d0a380c99213f7cb727984692b25f318e49b3e4f0b9c5ae9c5ba942287738d8d812608c0223e1a599bf4b1429a2972cb2a7844a"},
+ {"sha512_224", "hello hmac sha512_224", "5f4a8c8cb6404ad3ff85ccbde756d231ff2544be3be702a4706c8a9b"},
+ {"sha512_256", "hello hmac sha512_256", "e466b90580a96d60c34a4fb164afc725840c94d30ce1bdafaa00f8f830771dd8"},
}
- vm := modulestest.New(t)
- _, _ = vm.Runtime().RunString(`
- const crypto = require('cloudcat/crypto');
- `)
-
- testData := map[string]string{
- "md4": "92d8f5c302cf04cca0144d7a9feb1596",
- "md5": "e04f2ec05c8b12e19e46936b171c9d03",
- "sha1": "c113b62711ff5d8e8100bbb17b998591af81dc24",
- "sha256": "7fd04df92f636fd450bc841c9418e5825c17f33ad9c87c518115a45971f7f77e",
- "sha384": "d331e169e2dcfc742e80a3bf4dcc76d0e6425ab3777a3ac217ac6b2552aad5529ed4d40135b06e53a495ac7425d1e462",
- "sha512_224": "bac4e6256bdbf81d029aec48af4fdd4b14001db6721f07c429a80817",
- "sha512_256": "e3d0763ba92a4f40676c3d5b234d9842b71951e6e0767082cfb3f5e14c124b22",
- "sha512": "cd3146f96a3005024108ff56b025517552435589a4c218411f165da0a368b6f47228b20a1a4bf081e4aae6f07e2790f27194fc77f0addc890e98ce1951cacc9f",
- "ripemd160_256": "00bb4ce0d6afd4c7424c9d01b8a6caa3e749b08b",
- }
- for algorithm, value := range testData {
- _ = vm.Runtime().Set("correctHex", vm.Runtime().ToValue(value))
- _ = vm.Runtime().Set("algorithm", vm.Runtime().ToValue(algorithm))
-
- t.Run(algorithm+" hasher: valid", func(t *testing.T) {
- _, err := vm.RunString(context.Background(), `
- var hasher = crypto.createHMAC(algorithm, "a secret");
- assert.equal(correctHex, hasher.encrypt("some data to hash").hex());
- `)
-
- assert.NoError(t, err)
- })
-
- t.Run(algorithm+" wrapper: valid", func(t *testing.T) {
- _, err := vm.RunString(context.Background(), `
- var resultHex = crypto.hmac(algorithm, "a secret", "some data to hash").hex();
- assert.equal(correctHex, resultHex);
- `)
-
+ for _, testCase := range testCases {
+ t.Run(testCase.algorithm, func(t *testing.T) {
+ _, err := vm.Runtime().RunString(`{
+ let correct = "` + testCase.want + `";
+ let origin = "` + testCase.origin + `";
+ let hasher = crypto.createHMAC("` + testCase.algorithm + `", "some secret");
+ assert.equal(hasher.encrypt(origin).hex(), correct);
+ }`)
assert.NoError(t, err)
})
-
- t.Run(algorithm+" ArrayBuffer: valid", func(t *testing.T) {
- _, err := vm.RunString(context.Background(), `
- var data = new Uint8Array([115,111,109,101,32,100,97,116,97,32,116,
- 111,32,104,97,115,104]);
- var resultHex = crypto.hmac(algorithm, "a secret", data).hex();
- assert.equal(correctHex, resultHex);
- `)
-
- assert.NoError(t, err)
- })
- }
-
- // Algorithms not supported or typing error
- invalidData := map[string]string{
- "md6": "e04f2ec05c8b12e19e46936b171c9d03",
- "sha526": "7fd04df92f636fd450bc841c9418e5825c17f33ad9c87c518115a45971f7f77e",
- "sha348": "d331e169e2dcfc742e80a3bf4dcc76d0e6425ab3777a3ac217ac6b2552aad5529ed4d40135b06e53a495ac7425d1e462",
}
- for algorithm, value := range invalidData {
- algorithm := algorithm
- _ = vm.Runtime().Set("correctHex", vm.Runtime().ToValue(value))
- _ = vm.Runtime().Set("algorithm", vm.Runtime().ToValue(algorithm))
- t.Run(algorithm+" hasher: invalid", func(t *testing.T) {
- _, err := vm.RunString(context.Background(), `
- var hasher = crypto.createHMAC(algorithm, "a secret");
- assert.equal(correctHex, hasher.hash("some data to hash").hex())
- `)
-
- assert.Contains(t, err.Error(), "invalid algorithm: "+algorithm)
- })
-
- t.Run(algorithm+" wrapper: invalid", func(t *testing.T) {
- _, err := vm.RunString(context.Background(), `
- var resultHex = crypto.hmac(algorithm, "a secret", "some data to hash").hex();
- assert.equal(correctHex, resultHex);
- `)
-
- assert.Contains(t, err.Error(), "invalid algorithm: "+algorithm)
- })
- }
-}
-
-func TestAWSv4(t *testing.T) {
- // example values from https://docs.aws.amazon.com/general/latest/gr/signature-v4-examples.html
- vm := modulestest.New(t)
-
- _, err := vm.Runtime().RunString(`
- const crypto = require('cloudcat/crypto');
- let hmacSHA256 = function(data, key) {
- return crypto.hmac("sha256", key, data);
- };
-
- let expectedKDate = '969fbb94feb542b71ede6f87fe4d5fa29c789342b0f407474670f0c2489e0a0d'
- let expectedKRegion = '69daa0209cd9c5ff5c8ced464a696fd4252e981430b10e3d3fd8e2f197d7a70c'
- let expectedKService = 'f72cfd46f26bc4643f06a11eabb6c0ba18780c19a8da0c31ace671265e3c87fa'
- let expectedKSigning = 'f4780e2d9f65fa895f9c67b32ce1baf0b0d8a43505a000a1a9e090d414db404d'
-
- let key = 'wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY';
- let dateStamp = '20120215';
- let regionName = 'us-east-1';
- let serviceName = 'iam';
-
- let kDate = hmacSHA256(dateStamp, "AWS4" + key);
- let kRegion = hmacSHA256(regionName, kDate.binary());
- let kService = hmacSHA256(serviceName, kRegion.binary());
- let kSigning = hmacSHA256("aws4_request", kService.binary());
-
- assert.equal(expectedKDate, kDate.hex());
- assert.equal(expectedKRegion, kRegion.hex());
- assert.equal(expectedKService, kService.hex());
- assert.equal(expectedKSigning, kSigning.hex());
- `)
- assert.NoError(t, err)
}
diff --git a/js/modules/crypto/symmetric.go b/js/modules/crypto/symmetric.go
index de38ef7..519b970 100644
--- a/js/modules/crypto/symmetric.go
+++ b/js/modules/crypto/symmetric.go
@@ -9,7 +9,7 @@ import (
"fmt"
"strings"
- "github.com/shiroyk/cloudcat/js"
+ "github.com/shiroyk/ski/js"
)
// Aes returns a new AES cipher
diff --git a/js/modules/crypto/symmetric_test.go b/js/modules/crypto/symmetric_test.go
index 3bf6176..d5b815e 100644
--- a/js/modules/crypto/symmetric_test.go
+++ b/js/modules/crypto/symmetric_test.go
@@ -1,10 +1,11 @@
package crypto
import (
- "context"
"testing"
- "github.com/shiroyk/cloudcat/js/modulestest"
+ "github.com/grafana/sobek"
+ "github.com/shiroyk/ski/js"
+ "github.com/shiroyk/ski/js/modulestest"
"github.com/stretchr/testify/assert"
)
@@ -14,10 +15,24 @@ func TestCipherAlgorithm(t *testing.T) {
return
}
- vm := modulestest.New(t)
- _, _ = vm.Runtime().RunString(`
- const crypto = require('cloudcat/crypto');
- `)
+ vm := modulestest.New(t, js.WithInitial(func(rt *sobek.Runtime) {
+ c := new(Crypto)
+ instance, _ := c.Instantiate(rt)
+ _ = rt.Set("crypto", instance)
+ }))
+
+ t.Run("Cipher", func(t *testing.T) {
+ _, err := vm.Runtime().RunString(`{
+ let key = "1111111111111111";
+ let iv = "1111111111111111";
+ let text = "hello aes";
+ let aes = crypto.createCipher("AES/ECB/ZERO", key, iv);
+ let result = aes.encrypt(text);
+ let decrypt = aes.decrypt(result.binary()).string();
+ assert.equal(text, decrypt);
+ }`)
+ assert.NoError(t, err)
+ })
t.Run("AES", func(t *testing.T) {
mode := []string{"ECB", "CBC", "CFB", "OFB", "CTR", "GCM"}
@@ -27,15 +42,15 @@ func TestCipherAlgorithm(t *testing.T) {
for _, p := range padding {
_ = vm.Runtime().Set("P", p)
t.Run(m+"/"+p, func(t *testing.T) {
- _, err := vm.RunString(context.Background(), `
- var key = "1111111111111111";
- var iv = "1111111111111111";
- var text = "hello aes";
- var aes = crypto.aes(key, iv, 'AES'+'/'+M+'/'+P);
- var result = aes.encrypt(text);
- var decrypt = aes.decrypt(result.binary()).string();
+ _, err := vm.Runtime().RunString(`{
+ let key = "1111111111111111";
+ let iv = "1111111111111111";
+ let text = "hello aes";
+ let aes = crypto.aes(key, iv, 'AES'+'/'+M+'/'+P);
+ let result = aes.encrypt(text);
+ let decrypt = aes.decrypt(result.binary()).string();
assert.equal(text, decrypt);
- `)
+ }`)
assert.NoError(t, err)
})
}
@@ -50,15 +65,15 @@ func TestCipherAlgorithm(t *testing.T) {
for _, p := range padding {
_ = vm.Runtime().Set("P", p)
t.Run(m+"/"+p, func(t *testing.T) {
- _, err := vm.RunString(context.Background(), `
- var key = "11111111";
- var iv = "11111111";
- var text = "hello des";
- var des = crypto.des(key, iv, 'DES'+'/'+M+'/'+P);
- var result = des.encrypt(text);
- var decrypt = des.decrypt(result.binary()).string();
+ _, err := vm.Runtime().RunString(`{
+ let key = "11111111";
+ let iv = "11111111";
+ let text = "hello des";
+ let des = crypto.des(key, iv, 'DES'+'/'+M+'/'+P);
+ let result = des.encrypt(text);
+ let decrypt = des.decrypt(result.binary()).string();
assert.equal(text, decrypt);
- `)
+ }`)
assert.NoError(t, err)
})
}
@@ -73,14 +88,14 @@ func TestCipherAlgorithm(t *testing.T) {
for _, p := range padding {
_ = vm.Runtime().Set("P", p)
t.Run(m+"/"+p, func(t *testing.T) {
- _, err := vm.RunString(context.Background(), `
- var key = "111111111111111111111111";
- var text = "hello des";
- var des = crypto.tripleDes(key, null, 'TripleDes'+'/'+M+'/'+P);
- var result = des.encrypt(text);
- var decrypt = des.decrypt(result.binary()).string();
+ _, err := vm.Runtime().RunString(`{
+ let key = "111111111111111111111111";
+ let text = "hello des";
+ let des = crypto.tripleDes(key, null, 'TripleDes'+'/'+M+'/'+P);
+ let result = des.encrypt(text);
+ let decrypt = des.decrypt(result.binary()).string();
assert.equal(text, decrypt);
- `)
+ }`)
assert.NoError(t, err)
})
}
diff --git a/js/modules/encoding/encoding.go b/js/modules/encoding/encoding.go
index 13dddf0..d317487 100644
--- a/js/modules/encoding/encoding.go
+++ b/js/modules/encoding/encoding.go
@@ -5,23 +5,22 @@ import (
"encoding/base64"
"strings"
- "github.com/dop251/goja"
- "github.com/shiroyk/cloudcat/js"
- "github.com/shiroyk/cloudcat/plugin/jsmodule"
+ "github.com/grafana/sobek"
+ "github.com/shiroyk/ski/js"
)
-// Module js module
-type Module struct{}
-
-// Exports returns module instance
-func (*Module) Exports() any {
- return map[string]any{
- "base64": &Base64{},
- }
+func init() {
+ js.Register("encoding", new(Encoding))
}
-func init() {
- jsmodule.Register("encoding", new(Module))
+// Encoding js module
+type Encoding struct{}
+
+// Instantiate returns Encoding module instance
+func (*Encoding) Instantiate(rt *sobek.Runtime) (sobek.Value, error) {
+ return rt.ToValue(map[string]any{
+ "base64": new(Base64),
+ }), nil
}
// Base64 encoding and decoding
@@ -49,7 +48,7 @@ func (Base64) EncodeURI(input any) (string, error) {
}
// Decode returns the string decoding of input.
-func (Base64) Decode(call goja.FunctionCall, vm *goja.Runtime) (ret goja.Value) {
+func (Base64) Decode(call sobek.FunctionCall, vm *sobek.Runtime) (ret sobek.Value) {
input := call.Argument(0).Export()
toBuffer := call.Argument(1).ToBoolean()
diff --git a/js/modules/encoding/encoding_test.go b/js/modules/encoding/encoding_test.go
index 8f6720f..8c3b2fd 100644
--- a/js/modules/encoding/encoding_test.go
+++ b/js/modules/encoding/encoding_test.go
@@ -1,25 +1,22 @@
package encoding
import (
- "context"
"fmt"
"testing"
- "github.com/shiroyk/cloudcat/js/modulestest"
+ "github.com/grafana/sobek"
+ "github.com/shiroyk/ski/js"
+ "github.com/shiroyk/ski/js/modulestest"
"github.com/stretchr/testify/assert"
)
func TestEncodingBase64(t *testing.T) {
t.Parallel()
- if testing.Short() {
- return
- }
-
- vm := modulestest.New(t)
- _, _ = vm.Runtime().RunString(`
- const encoding = require('cloudcat/encoding');
- `)
+ vm := modulestest.New(t, js.WithInitial(func(rt *sobek.Runtime) {
+ instantiate, _ := new(Encoding).Instantiate(rt)
+ _ = rt.Set("encoding", instantiate)
+ }))
buffer := vm.Runtime().NewArrayBuffer([]byte{100, 97, 110, 107, 111, 103, 97, 105})
@@ -50,7 +47,7 @@ func TestEncodingBase64(t *testing.T) {
if testCase.url {
code += "URI"
}
- _, err := vm.RunString(context.Background(), code+"(raw, padding);assert.equal(want, result);")
+ _, err := vm.Runtime().RunString(code + "(raw, padding);assert.equal(want, result);")
assert.NoError(t, err)
})
}
@@ -73,7 +70,7 @@ func TestEncodingBase64(t *testing.T) {
_ = vm.Runtime().Set("want", testCase.want)
_ = vm.Runtime().Set("toBuffer", testCase.toBuffer)
- _, err := vm.RunString(context.Background(), `
+ _, err := vm.Runtime().RunString(`
assert.equal(want, encoding.base64.decode(raw, toBuffer));
`)
assert.NoError(t, err)
diff --git a/js/modules/http/cookiejar.go b/js/modules/http/cookiejar.go
new file mode 100644
index 0000000..08e804f
--- /dev/null
+++ b/js/modules/http/cookiejar.go
@@ -0,0 +1,156 @@
+package http
+
+import (
+ "errors"
+ "net/http"
+ "net/url"
+ "time"
+
+ "github.com/grafana/sobek"
+ "github.com/shiroyk/ski"
+ "github.com/shiroyk/ski/js"
+ "github.com/spf13/cast"
+)
+
+// CookieJar manages storage and use of cookies in HTTP requests.
+type CookieJar struct{ ski.CookieJar }
+
+func (j *CookieJar) Instantiate(rt *sobek.Runtime) (sobek.Value, error) {
+ if j.CookieJar == nil {
+ return nil, errors.New("CookieJar can not nil")
+ }
+ return rt.ToValue(map[string]func(call sobek.FunctionCall, rt *sobek.Runtime) sobek.Value{
+ "get": j.Get,
+ "getAll": j.GetAll,
+ "set": j.Set,
+ "del": j.Del,
+ }), nil
+}
+
+// Get returns the cookie for the given option.
+func (j *CookieJar) Get(call sobek.FunctionCall, rt *sobek.Runtime) sobek.Value {
+ opt, err := cast.ToStringMapStringE(call.Argument(0).Export())
+ if err != nil {
+ js.Throw(rt, errors.New("get parameter must be an object containing name, url"))
+ }
+ u, err := url.Parse(opt["url"])
+ if err != nil {
+ js.Throw(rt, err)
+ }
+ cookies := j.CookieJar.Cookies(u)
+ name := opt["name"]
+ for _, cookie := range cookies {
+ if cookie.Name == name {
+ return toObj(cookie, rt)
+ }
+ }
+ if len(cookies) > 0 {
+ return toObj(cookies[0], rt)
+ }
+ return sobek.Null()
+}
+
+// GetAll returns the cookies for the given option.
+func (j *CookieJar) GetAll(call sobek.FunctionCall, rt *sobek.Runtime) sobek.Value {
+ opt, err := cast.ToStringMapStringE(call.Argument(0).Export())
+ if err != nil {
+ js.Throw(rt, errors.New("getAll parameter must be an object containing name, url"))
+ }
+ u, err := url.Parse(opt["url"])
+ if err != nil {
+ js.Throw(rt, err)
+ }
+ return toObjs(j.CookieJar.Cookies(u), rt)
+}
+
+// Set handles the receipt of the cookies in a reply for the given option.
+func (j *CookieJar) Set(call sobek.FunctionCall, rt *sobek.Runtime) (ret sobek.Value) {
+ u, err := url.Parse(call.Argument(0).String())
+ if err != nil {
+ js.Throw(rt, errors.New("set first parameter must be url string"))
+ }
+ var cookies []*http.Cookie
+ switch e := call.Argument(1).Export().(type) {
+ case map[string]any:
+ cookies = append(cookies, toCookie(e))
+ case []any:
+ for _, cookie := range cookies {
+ cookies = append(cookies, toCookie(cast.ToStringMap(cookie)))
+ }
+ default:
+ js.Throw(rt, errors.New("set second parameter must be cookie object"))
+ }
+ if len(cookies) == 0 {
+ return sobek.Undefined()
+ }
+
+ j.CookieJar.SetCookies(u, cookies)
+ return sobek.Undefined()
+}
+
+// Del handles the receipt of the cookies in a reply for the given URL.
+func (j *CookieJar) Del(call sobek.FunctionCall, rt *sobek.Runtime) sobek.Value {
+ u, err := url.Parse(call.Argument(0).String())
+ if err != nil {
+ js.Throw(rt, err)
+ }
+ j.CookieJar.RemoveCookie(u)
+ return sobek.Undefined()
+}
+
+var sameSiteMapping = [...]string{
+ http.SameSiteDefaultMode: "",
+ http.SameSiteLaxMode: "lax",
+ http.SameSiteStrictMode: "strict",
+ http.SameSiteNoneMode: "none",
+}
+
+func toObj(cookie *http.Cookie, rt *sobek.Runtime) sobek.Value {
+ o := rt.NewObject()
+ _ = o.Set("domain", rt.ToValue(cookie.Domain))
+ _ = o.Set("expires", rt.ToValue(cookie.Expires.Unix()))
+ _ = o.Set("name", rt.ToValue(cookie.Name))
+ _ = o.Set("path", rt.ToValue(cookie.Path))
+ _ = o.Set("sameSite", rt.ToValue(sameSiteMapping[cookie.SameSite]))
+ _ = o.Set("secure", rt.ToValue(cookie.Secure))
+ _ = o.Set("value", rt.ToValue(cookie.Value))
+ _ = o.Set("toString", func(sobek.FunctionCall) sobek.Value {
+ return rt.ToValue(cookie.String())
+ })
+ return o
+}
+
+func toObjs(cookies []*http.Cookie, rt *sobek.Runtime) sobek.Value {
+ ret := make([]sobek.Value, 0, len(cookies))
+ for _, cookie := range cookies {
+ ret = append(ret, toObj(cookie, rt))
+ }
+ return rt.ToValue(ret)
+}
+
+func toCookie(o map[string]any) *http.Cookie {
+ var sameSite = http.SameSiteDefaultMode
+ switch cast.ToString(o["sameSite"]) {
+ case "lax":
+ sameSite = http.SameSiteLaxMode
+ case "strict":
+ sameSite = http.SameSiteStrictMode
+ case "none":
+ sameSite = http.SameSiteNoneMode
+ }
+ expires := cast.ToInt64(o["expires"])
+ if expires == 0 {
+ expires = time.Now().Add(time.Hour * 72).Unix()
+ }
+ return &http.Cookie{
+ Domain: cast.ToString(o["domain"]),
+ Expires: time.Unix(expires, 0),
+ Name: cast.ToString(o["name"]),
+ Path: cast.ToString(o["path"]),
+ SameSite: sameSite,
+ Value: cast.ToString(o["value"]),
+ MaxAge: cast.ToInt(o["maxAge"]),
+ Secure: cast.ToBool(o["secure"]),
+ HttpOnly: cast.ToBool(o["httpOnly"]),
+ }
+}
diff --git a/js/modules/http/cookiejar_test.go b/js/modules/http/cookiejar_test.go
new file mode 100644
index 0000000..02e0d2d
--- /dev/null
+++ b/js/modules/http/cookiejar_test.go
@@ -0,0 +1,54 @@
+package http
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+
+ "github.com/grafana/sobek"
+ "github.com/shiroyk/ski"
+ "github.com/shiroyk/ski/js"
+ "github.com/shiroyk/ski/js/modulestest"
+ "github.com/stretchr/testify/assert"
+)
+
+func TestCookie(t *testing.T) {
+ t.Parallel()
+ vm := modulestest.New(t, js.WithInitial(func(rt *sobek.Runtime) {
+ jar := CookieJar{ski.NewCookieJar()}
+ instantiate, err := jar.Instantiate(rt)
+ if err != nil {
+ t.Fatal(err)
+ }
+ _ = rt.Set("cookieJar", instantiate)
+ client := http.Client{Jar: jar}
+ instance, _ := (&Http{&client}).Instantiate(rt)
+ _ = rt.Set("http", instance)
+ }))
+
+ ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if cookie, err := r.Cookie("foo"); err == nil {
+ _, err = fmt.Fprint(w, cookie.String())
+ assert.NoError(t, err)
+ return
+ }
+ w.WriteHeader(http.StatusOK)
+ }))
+ _ = vm.Runtime().Set("url", ts.URL)
+
+ _, err := vm.RunString(context.Background(), `
+ cookieJar.set("https://github.com", { name: "foo", value: "bar", path: "/", maxAge: 7200 });
+ assert.equal("bar", cookieJar.get({ url: "https://github.com" }).value);
+ cookieJar.del("https://github.com");
+ assert.true(!cookieJar.get({ url: "https://github.com" }), "cookie should be deleted");
+ cookieJar.set(url, { name: "foo", value: "bar", path: "/", maxAge: 7200 });
+ const res1 = http.get(url);
+ assert.equal(res1.text(), "foo=bar");
+ cookieJar.del(url);
+ const res2 = http.get(url);
+ assert.equal(res2.text(), "");
+ `)
+ assert.NoError(t, err)
+}
diff --git a/js/modules/http/form_data.go b/js/modules/http/form_data.go
index ea51061..5a74edf 100644
--- a/js/modules/http/form_data.go
+++ b/js/modules/http/form_data.go
@@ -2,98 +2,121 @@ package http
import (
"fmt"
+ "slices"
- "github.com/dop251/goja"
- "github.com/shiroyk/cloudcat"
- "github.com/shiroyk/cloudcat/js"
+ "github.com/grafana/sobek"
+ "github.com/shiroyk/ski"
+ "github.com/shiroyk/ski/js"
)
-// FileData wraps the file data and filename
-type FileData struct {
- Data []byte
- Filename string
+// fileData wraps the file data and filename
+type fileData struct {
+ data []byte
+ filename string
}
-// FormData provides a way to construct a set of key/value pairs representing form fields and their values.
+// formData provides a way to construct a set of key/value pairs representing form fields and their values.
// which can be sent using the http() method and encoding type were set to "multipart/form-data".
// Implement the https://developer.mozilla.org/en-US/docs/Web/API/FormData
-type FormData struct {
+type formData struct {
+ keys []string
data map[string][]any
}
-// FormDataConstructor FormData Constructor
-type FormDataConstructor struct{}
+// FormData Constructor
+type FormData struct{}
-// Exports returns module instance
-func (*FormDataConstructor) Exports() any {
- return func(call goja.ConstructorCall, vm *goja.Runtime) *goja.Object {
- param := call.Argument(0)
+// Instantiate returns module instance
+func (*FormData) Instantiate(rt *sobek.Runtime) (sobek.Value, error) {
+ return rt.ToValue(func(call sobek.ConstructorCall) *sobek.Object {
+ params := call.Argument(0)
- if goja.IsUndefined(param) {
- return vm.ToValue(FormData{make(map[string][]any)}).ToObject(vm)
+ var ret formData
+ if sobek.IsUndefined(params) {
+ ret.data = make(map[string][]any)
+ return ret.object(rt)
}
- var pa map[string]any
- var ok bool
- pa, ok = param.Export().(map[string]any)
- if !ok {
- js.Throw(vm, fmt.Errorf("unsupported type %T", param.Export()))
- }
-
- data := make(map[string][]any, len(pa))
+ object := params.ToObject(rt)
+ keys := object.Keys()
+ ret.keys = make([]string, 0, len(keys))
+ ret.data = make(map[string][]any, len(keys))
- for k, v := range pa {
- switch ve := v.(type) {
+ for _, key := range keys {
+ value, _ := js.Unwrap(object.Get(key))
+ switch ve := value.(type) {
case []byte:
// Default filename "blob".
- data[k] = []any{FileData{
- Data: ve,
- Filename: "blob",
+ ret.data[key] = []any{fileData{
+ data: ve,
+ filename: "blob",
}}
- case goja.ArrayBuffer:
+ case sobek.ArrayBuffer:
// Default filename "blob".
- data[k] = []any{FileData{
- Data: ve.Bytes(),
- Filename: "blob",
+ ret.data[key] = []any{fileData{
+ data: ve.Bytes(),
+ filename: "blob",
}}
case []any:
- data[k] = ve
+ ret.data[key] = ve
+ case nil:
+ ret.data[key] = nil
default:
- data[k] = []any{fmt.Sprintf("%v", ve)}
+ ret.data[key] = []any{fmt.Sprintf("%s", ve)}
}
+ ret.keys = append(ret.keys, key)
}
- return vm.ToValue(FormData{data}).ToObject(vm)
- }
+ return ret.object(rt)
+ }), nil
}
// Global it is a global module
-func (*FormDataConstructor) Global() {}
+func (*FormData) Global() {}
+
+func (f *formData) object(rt *sobek.Runtime) *sobek.Object {
+ obj := rt.ToValue(f).ToObject(rt)
+
+ _ = obj.SetSymbol(sobek.SymIterator, func(sobek.ConstructorCall) *sobek.Object {
+ var i int
+ it := rt.NewObject()
+ _ = it.Set("next", func(sobek.FunctionCall) sobek.Value {
+ if i < len(f.keys) {
+ key := f.keys[i]
+ i++
+ return rt.ToValue(iter{Value: rt.ToValue([2]any{key, f.data[key]})})
+ }
+ return rt.ToValue(iter{Done: true})
+ })
+ return it
+ })
+ return obj
+}
-// Append method of the FormData interface appends a new value onto an existing key inside a FormData object,
+// Append method of the formData interface appends a new value onto an existing key inside a formData object,
// or adds the key if it does not already exist.
-func (f *FormData) Append(name string, value any, filename string) (ret goja.Value) {
+func (f *formData) Append(name string, value any, filename string) sobek.Value {
if filename == "" {
// Default filename "blob".
filename = "blob"
}
- var ele []any
- var ok bool
- if ele, ok = f.data[name]; !ok {
+ ele, ok := f.data[name]
+ if !ok {
+ f.keys = append(f.keys, name)
ele = make([]any, 0)
}
switch v := value.(type) {
case []byte:
- ele = append(ele, FileData{
- Data: v,
- Filename: filename,
+ ele = append(ele, fileData{
+ data: v,
+ filename: filename,
})
- case goja.ArrayBuffer:
- ele = append(ele, FileData{
- Data: v.Bytes(),
- Filename: filename,
+ case sobek.ArrayBuffer:
+ ele = append(ele, fileData{
+ data: v.Bytes(),
+ filename: filename,
})
default:
ele = append(ele, fmt.Sprintf("%v", v))
@@ -101,36 +124,37 @@ func (f *FormData) Append(name string, value any, filename string) (ret goja.Val
f.data[name] = ele
- return
+ return sobek.Undefined()
}
-// Delete method of the FormData interface deletes a key and its value(s) from a FormData object.
-func (f *FormData) Delete(name string) {
+// Delete method of the formData interface deletes a key and its value(s) from a formData object.
+func (f *formData) Delete(name string) {
+ f.keys = slices.DeleteFunc(f.keys, func(k string) bool { return k == name })
delete(f.data, name)
}
-// Entries method returns an iterator which iterates through all key/value pairs contained in the FormData.
-func (f *FormData) Entries() any {
- entries := make([][2]any, 0, len(f.data))
- for k, v := range f.data {
- entries = append(entries, [2]any{k, v})
+// Entries method returns an iterator which iterates through all key/value pairs contained in the formData.
+func (f *formData) Entries() any {
+ entries := make([][2]any, 0, len(f.keys))
+ for _, key := range f.keys {
+ entries = append(entries, [2]any{key, f.data[key]})
}
return entries
}
-// Get method of the FormData interface returns the first value associated
-// with a given key from within a FormData object.
+// Get method of the formData interface returns the first value associated
+// with a given key from within a formData object.
// If you expect multiple values and want all of them, use the getAll() method instead.
-func (f *FormData) Get(name string) any {
+func (f *formData) Get(name string) any {
if v, ok := f.data[name]; ok {
return v[0]
}
return nil
}
-// GetAll method of the FormData interface returns all the values associated
-// with a given key from within a FormData object.
-func (f *FormData) GetAll(name string) any {
+// GetAll method of the formData interface returns all the values associated
+// with a given key from within a formData object.
+func (f *formData) GetAll(name string) any {
v, ok := f.data[name]
if ok {
return v
@@ -138,29 +162,33 @@ func (f *FormData) GetAll(name string) any {
return [0]any{}
}
-// Has method of the FormData interface returns whether a FormData object contains a certain key.
-func (f *FormData) Has(name string) bool {
+// Has method of the formData interface returns whether a formData object contains a certain key.
+func (f *formData) Has(name string) bool {
_, ok := f.data[name]
return ok
}
-// Keys method returns an iterator which iterates through all keys contained in the FormData.
+// Keys method returns an iterator which iterates through all keys contained in the formData.
// The keys are strings.
-func (f *FormData) Keys() any { return cloudcat.MapKeys(f.data) }
+func (f *formData) Keys() any { return f.keys }
-// Set method of the FormData interface sets a new value for an existing key inside a FormData object,
+// Set method of the formData interface sets a new value for an existing key inside a formData object,
// or adds the key/value if it does not already exist.
-func (f *FormData) Set(name string, value any, filename string) {
+func (f *formData) Set(name string, value any, filename string) {
if filename == "" {
filename = "blob"
}
+ if _, ok := f.data[name]; !ok {
+ f.keys = append(f.keys, name)
+ }
+
switch v := value.(type) {
- case goja.ArrayBuffer:
+ case sobek.ArrayBuffer:
f.data[name] = []any{
- FileData{
- Data: v.Bytes(),
- Filename: filename,
+ fileData{
+ data: v.Bytes(),
+ filename: filename,
},
}
default:
@@ -168,5 +196,5 @@ func (f *FormData) Set(name string, value any, filename string) {
}
}
-// Values method returns an iterator which iterates through all values contained in the FormData.
-func (f *FormData) Values() any { return cloudcat.MapValues(f.data) }
+// Values method returns an iterator which iterates through all values contained in the formData.
+func (f *formData) Values() any { return ski.MapValues(f.data) }
diff --git a/js/modules/http/form_data_test.go b/js/modules/http/form_data_test.go
index c68f2a2..5dcf543 100644
--- a/js/modules/http/form_data_test.go
+++ b/js/modules/http/form_data_test.go
@@ -2,46 +2,43 @@ package http
import (
"context"
- "fmt"
"testing"
- "github.com/shiroyk/cloudcat/js/modulestest"
+ "github.com/shiroyk/ski/js/modulestest"
"github.com/stretchr/testify/assert"
)
func TestFormData(t *testing.T) {
- ctx := context.Background()
vm := modulestest.New(t)
- _, _ = vm.Runtime().RunString(`const mp = new FormData({
+ _, err := vm.RunString(context.Background(), `
+ const form = new FormData({
'file': new Uint8Array([50]),
'name': 'foo'
- });`)
-
- testCase := []string{
- `try {
+ });
+ try {
new FormData(0);
- } catch (e) {
+ } catch (e) {
assert.true(e.toString().includes('unsupported type'))
- }`,
- `assert.equal(mp.get('name'), 'foo')`,
- `mp.append('file', new Uint8Array([51]).buffer);
- assert.equal(mp.getAll('file').length, 2)`,
- `mp.append('name', 'bar');
- assert.equal(mp.keys().length, 2);
- assert.equal(mp.get('name'), 'foo');`,
- `assert.equal(mp.entries().length, 2)`,
- `mp.delete('name');
- assert.equal(mp.getAll('name').length, 0)`,
- `assert.true(!mp.has('name'))`,
- `mp.set('name', 'foobar');
- assert.equal(mp.values().length, 2)`,
- }
-
- for i, s := range testCase {
- t.Run(fmt.Sprintf("Script%v", i), func(t *testing.T) {
- _, err := vm.RunString(ctx, s)
- assert.NoError(t, err)
- })
- }
+ }
+ assert.equal(form.get('name'), 'foo')
+ form.delete('name');
+ assert.equal(form.get('name'), null);
+ form.append('file', new Uint8Array([51]).buffer);
+ assert.equal(form.getAll('file').length, 2)
+ form.append('name', 'bar');
+ assert.equal(form.keys().length, 2);
+ assert.equal(form.get('name'), 'bar');
+ assert.equal(form.entries().length, 2)
+ form.delete('name');
+ assert.equal(form.getAll('name').length, 0)
+ assert.true(!form.has('name'))
+ form.set('name', 'foobar');
+ assert.equal(form.values().length, 2)
+ let str = "";
+ for (const [key, value] of form) {
+ str += key + ",";
+ }
+ assert.equal(str, 'file,name,')`)
+ assert.NoError(t, err)
}
diff --git a/js/modules/http/http.go b/js/modules/http/http.go
index 725d523..876e652 100644
--- a/js/modules/http/http.go
+++ b/js/modules/http/http.go
@@ -13,65 +13,76 @@ import (
urlpkg "net/url"
"strings"
- "github.com/dop251/goja"
- "github.com/shiroyk/cloudcat"
- "github.com/shiroyk/cloudcat/js"
- "github.com/shiroyk/cloudcat/plugin/jsmodule"
+ "github.com/grafana/sobek"
+ "github.com/shiroyk/ski"
+ "github.com/shiroyk/ski/js"
"github.com/spf13/cast"
- "golang.org/x/net/http/httpguts"
)
-// Module js module
-type Module struct{}
-
-// Exports returns module instance
-func (*Module) Exports() any {
- return &Http{cloudcat.MustResolve[cloudcat.Fetch]()}
-}
-
func init() {
- jsmodule.Register("http", new(Module))
- jsmodule.Register("fetch", new(FetchModule))
- jsmodule.Register("FormData", new(FormDataConstructor))
- jsmodule.Register("URLSearchParams", new(URLSearchParamsConstructor))
- jsmodule.Register("AbortController", new(AbortControllerConstructor))
- jsmodule.Register("AbortSignal", new(AbortSignalModule))
+ jar := ski.NewCookieJar()
+ fetch := ski.NewFetch().(*http.Client)
+ fetch.Jar = jar
+ js.Register("cookieJar", &CookieJar{jar})
+ js.Register("http", &Http{fetch})
+ js.Register("fetch", &Fetch{fetch})
+ js.Register("FormData", new(FormData))
+ js.Register("URLSearchParams", new(URLSearchParams))
+ js.Register("AbortController", new(AbortController))
+ js.Register("AbortSignal", new(AbortSignal))
}
-// FetchModule the global fetch() method starts the process of
+// Fetch the global Fetch() method starts the process of
// fetching a resource from the network, returning a promise
// which is fulfilled once the response is available.
// https://developer.mozilla.org/en-US/docs/Web/API/fetch
-type FetchModule struct{}
+type Fetch struct{ ski.Fetch }
-func (*FetchModule) Exports() any {
- fetch := cloudcat.MustResolveLazy[cloudcat.Fetch]()
- return func(call goja.FunctionCall, vm *goja.Runtime) goja.Value {
- req, signal := buildRequest(http.MethodGet, call, vm)
- return vm.ToValue(js.NewPromise(vm, func() (*http.Response, error) {
- if signal != nil {
- defer signal.abort() // release resources
- }
- return fetch().Do(req)
- }, func(res *http.Response, err error) (any, error) {
- if err != nil {
- return nil, err
- }
- return NewAsyncResponse(vm, res), nil
- }))
+func (fetch *Fetch) Instantiate(rt *sobek.Runtime) (sobek.Value, error) {
+ if fetch.Fetch == nil {
+ return nil, errors.New("Fetch can not nil")
}
+ return rt.ToValue(func(call sobek.FunctionCall, vm *sobek.Runtime) sobek.Value {
+ req, signal := buildRequest(http.MethodGet, call, vm)
+ return vm.ToValue(js.NewPromise(vm,
+ func() (*http.Response, error) {
+ if signal != nil {
+ defer signal.abort() // release resources
+ }
+ return fetch.Do(req)
+ },
+ func(res *http.Response, err error) (any, error) {
+ if err != nil {
+ return nil, err
+ }
+ return NewAsyncResponse(vm, res), nil
+ }))
+ }), nil
}
-func (*FetchModule) Global() {}
+func (*Fetch) Global() {}
// Http module for fetching resources (including across the network).
-type Http struct { //nolint
- fetch cloudcat.Fetch
+type Http struct{ ski.Fetch }
+
+func (h *Http) Instantiate(rt *sobek.Runtime) (sobek.Value, error) {
+ if h.Fetch == nil {
+ return nil, errors.New("Fetch can not nil")
+ }
+ return rt.ToValue(map[string]func(call sobek.FunctionCall, vm *sobek.Runtime) sobek.Value{
+ "get": h.Get,
+ "post": h.Post,
+ "put": h.Put,
+ "delete": h.Delete,
+ "patch": h.Patch,
+ "request": h.Request,
+ "head": h.Head,
+ }), nil
}
// Get Make a HTTP GET request.
-func (h *Http) Get(call goja.FunctionCall, vm *goja.Runtime) goja.Value {
- return doRequest(h.fetch, http.MethodGet, call, vm)
+func (h *Http) Get(call sobek.FunctionCall, vm *sobek.Runtime) sobek.Value {
+ return h.do(call, vm, http.MethodGet)
}
// Post Make a HTTP POST.
@@ -81,108 +92,108 @@ func (h *Http) Get(call goja.FunctionCall, vm *goja.Runtime) goja.Value {
// http.post(url, { body: new URLSearchParams({'key': 'foo', 'value': 'bar'}) })
// Send POST with json:
// http.post(url, { body: {'key': 'foo'} })
-func (h *Http) Post(call goja.FunctionCall, vm *goja.Runtime) goja.Value {
- return doRequest(h.fetch, http.MethodPost, call, vm)
+func (h *Http) Post(call sobek.FunctionCall, vm *sobek.Runtime) sobek.Value {
+ return h.do(call, vm, http.MethodPost)
}
// Put Make a HTTP PUT request.
-func (h *Http) Put(call goja.FunctionCall, vm *goja.Runtime) goja.Value {
- return doRequest(h.fetch, http.MethodPut, call, vm)
+func (h *Http) Put(call sobek.FunctionCall, vm *sobek.Runtime) sobek.Value {
+ return h.do(call, vm, http.MethodPut)
}
// Delete Make a HTTP DELETE request.
-func (h *Http) Delete(call goja.FunctionCall, vm *goja.Runtime) goja.Value {
- return doRequest(h.fetch, http.MethodDelete, call, vm)
+func (h *Http) Delete(call sobek.FunctionCall, vm *sobek.Runtime) sobek.Value {
+ return h.do(call, vm, http.MethodDelete)
}
// Patch Make a HTTP PATCH request.
-func (h *Http) Patch(call goja.FunctionCall, vm *goja.Runtime) goja.Value {
- return doRequest(h.fetch, http.MethodPatch, call, vm)
+func (h *Http) Patch(call sobek.FunctionCall, vm *sobek.Runtime) sobek.Value {
+ return h.do(call, vm, http.MethodPatch)
}
// Request Make a HTTP request.
-func (h *Http) Request(call goja.FunctionCall, vm *goja.Runtime) goja.Value {
- return doRequest(h.fetch, http.MethodGet, call, vm)
+func (h *Http) Request(call sobek.FunctionCall, vm *sobek.Runtime) sobek.Value {
+ return h.do(call, vm, http.MethodGet)
}
// Head Make a HTTP HEAD request.
-func (h *Http) Head(call goja.FunctionCall, vm *goja.Runtime) goja.Value {
- return doRequest(h.fetch, http.MethodHead, call, vm)
+func (h *Http) Head(call sobek.FunctionCall, vm *sobek.Runtime) sobek.Value {
+ return h.do(call, vm, http.MethodHead)
+}
+
+func (h *Http) do(call sobek.FunctionCall, vm *sobek.Runtime, method string) sobek.Value {
+ req, signal := buildRequest(method, call, vm)
+ if signal != nil {
+ defer signal.abort() // release resources
+ }
+
+ res, err := h.Do(req)
+ if err != nil {
+ js.Throw(vm, err)
+ }
+
+ return NewResponse(vm, res)
}
func buildRequest(
method string,
- call goja.FunctionCall,
- vm *goja.Runtime,
-) (req *http.Request, signal *AbortSignal) {
- url := call.Argument(0).String()
- opt := call.Argument(1)
- var body io.Reader
- var headers = make(map[string]string)
- var proxy *urlpkg.URL
- var err error
+ call sobek.FunctionCall,
+ vm *sobek.Runtime,
+) (req *http.Request, signal *abortSignal) {
+ var (
+ ctx = context.Background()
+ url = call.Argument(0).String()
+ options = call.Argument(1)
+ opt *sobek.Object
+ body io.Reader
+ headers = make(map[string]string)
+ err error
+ )
- if opt != nil && !goja.IsUndefined(opt) {
- options, assert := opt.Export().(map[string]any)
- if !assert {
- js.Throw(vm, errors.New("request options is invalid"))
- }
- if v, ok := options["method"]; ok {
- method, err = cast.ToStringE(v)
- if err != nil {
- js.Throw(vm, errors.New("request options method is invalid string"))
- }
- method = strings.ToUpper(method)
- if !validMethod(method) {
- js.Throw(vm, fmt.Errorf("request options method %v is invalid HTTP method", method))
- }
- }
- if v, ok := options["headers"]; ok {
- headers, err = cast.ToStringMapStringE(v)
- if err != nil {
- js.Throw(vm, errors.New("request options headers is invalid"))
- }
+ if sobek.IsUndefined(options) || sobek.IsNull(options) {
+ ctx = js.Context(vm)
+ goto NEW
+ }
+
+ opt = options.ToObject(vm)
+ if v := opt.Get("method"); v != nil {
+ method = strings.ToUpper(v.String())
+ }
+ if v := opt.Get("headers"); v != nil {
+ if headers, err = cast.ToStringMapStringE(v.Export()); err != nil {
+ js.Throw(vm, fmt.Errorf("options headers is invalid, %s", err))
}
- if v, ok := options["body"]; ok {
- body, err = handleBody(v, headers)
- if err != nil {
+ }
+ if method != http.MethodGet && method != http.MethodHead {
+ if v := opt.Get("body"); v != nil {
+ if body, err = processBody(v.Export(), headers); err != nil {
js.Throw(vm, err)
}
}
- if v, ok := options["cache"]; ok {
- str, err := cast.ToStringE(v)
- if err != nil {
- js.Throw(vm, errors.New("request options cache is invalid string"))
- }
- headers["Cache-Control"] = str
- headers["Pragma"] = str
- }
- if v, ok := options["proxy"]; ok {
- str, err := cast.ToStringE(v)
- if err != nil {
- js.Throw(vm, errors.New("request options proxy is invalid string"))
- }
- proxy, err = urlpkg.Parse(str)
- if err != nil {
- js.Throw(vm, errors.Join(errors.New("request options proxy is invalid URL"), err))
- }
- }
- if v, ok := options["signal"]; ok {
- signal, ok = v.(*AbortSignal)
- if !ok {
- js.Throw(vm, errors.New("request options signal is invalid AbortSignal"))
- }
- }
}
-
- var parent context.Context
- if signal != nil {
- parent = signal.ctx
+ if v := opt.Get("cache"); v != nil {
+ str := v.String()
+ headers["Cache-Control"] = str
+ headers["Pragma"] = str
+ }
+ if v := opt.Get("signal"); v != nil {
+ var ok bool
+ if signal, ok = v.Export().(*abortSignal); !ok {
+ js.Throw(vm, errors.New("options signal is not AbortSignal"))
+ }
+ ctx = signal.ctx
} else {
- parent = js.VMContext(vm)
+ ctx = js.Context(vm)
+ }
+ if v := opt.Get("proxy"); v != nil {
+ proxy, err := urlpkg.Parse(v.String())
+ if err != nil {
+ js.Throw(vm, fmt.Errorf("options proxy is invalid URL, %s", err))
+ }
+ ctx = ski.WithProxyURL(ctx, proxy)
}
- ctx := cloudcat.WithProxyURL(parent, proxy)
+NEW:
req, err = http.NewRequestWithContext(ctx, method, url, body)
if err != nil {
js.Throw(vm, err)
@@ -195,65 +206,27 @@ func buildRequest(
return
}
-func doRequest(
- fetch cloudcat.Fetch,
- method string,
- call goja.FunctionCall,
- vm *goja.Runtime,
-) goja.Value {
- req, signal := buildRequest(method, call, vm)
- if signal != nil {
- defer signal.abort() // release resources
- }
-
- res, err := fetch.Do(req)
- if err != nil {
- js.Throw(vm, err)
- }
-
- return NewResponse(vm, res)
-}
-
-func validMethod(method string) bool {
- /*
- Method = "OPTIONS" ; Section 9.2
- | "GET" ; Section 9.3
- | "HEAD" ; Section 9.4
- | "POST" ; Section 9.5
- | "PUT" ; Section 9.6
- | "DELETE" ; Section 9.7
- | "TRACE" ; Section 9.8
- | "CONNECT" ; Section 9.9
- | extension-method
- extension-method = token
- token = 1*
- */
- return len(method) > 0 && strings.IndexFunc(method, func(r rune) bool {
- return !httpguts.IsTokenRune(r)
- }) == -1
-}
-
-// handleBody process the send request body and set the content-type
-func handleBody(body any, headers map[string]string) (io.Reader, error) {
+// processBody process the send request body and set the content-type
+func processBody(body any, headers map[string]string) (io.Reader, error) {
switch data := body.(type) {
- case FormData:
+ case *formData:
buf := new(bytes.Buffer)
mpw := multipart.NewWriter(buf)
- for k, v := range data.data {
- for _, ve := range v {
- if f, ok := ve.(FileData); ok {
+ for _, key := range data.keys {
+ for _, value := range data.data[key] {
+ if f, ok := value.(fileData); ok {
// Creates a new form-data header with the provided field name and file name.
- fw, err := mpw.CreateFormFile(k, f.Filename)
+ fw, err := mpw.CreateFormFile(key, f.filename)
if err != nil {
return nil, err
}
// Write bytes to the part
- if _, err := fw.Write(f.Data); err != nil {
+ if _, err = fw.Write(f.data); err != nil {
return nil, err
}
} else {
// Write string value
- if err := mpw.WriteField(k, fmt.Sprintf("%v", v)); err != nil {
+ if err := mpw.WriteField(key, fmt.Sprintf("%v", key)); err != nil {
return nil, err
}
}
@@ -264,12 +237,12 @@ func handleBody(body any, headers map[string]string) (io.Reader, error) {
return nil, err
}
return buf, nil
- case URLSearchParams:
+ case *urlSearchParams:
headers["Content-Type"] = "application/x-www-form-url"
return strings.NewReader(data.encode()), nil
case string:
return strings.NewReader(data), nil
- case goja.ArrayBuffer:
+ case sobek.ArrayBuffer:
return bytes.NewReader(data.Bytes()), nil
case []byte:
return bytes.NewReader(data), nil
diff --git a/js/modules/http/http_test.go b/js/modules/http/http_test.go
index bc69c21..28eb46d 100644
--- a/js/modules/http/http_test.go
+++ b/js/modules/http/http_test.go
@@ -1,7 +1,6 @@
package http
import (
- "context"
"fmt"
"io"
"net/http"
@@ -11,14 +10,14 @@ import (
"testing"
"time"
- "github.com/shiroyk/cloudcat"
- "github.com/shiroyk/cloudcat/js"
- "github.com/shiroyk/cloudcat/js/modulestest"
+ "github.com/grafana/sobek"
+ "github.com/shiroyk/ski"
+ "github.com/shiroyk/ski/js"
+ "github.com/shiroyk/ski/js/modulestest"
"github.com/stretchr/testify/assert"
)
func TestHttp(t *testing.T) {
- cloudcat.Provide[cloudcat.Fetch](&http.Client{Transport: &http.Transport{Proxy: cloudcat.ProxyFromRequest}})
vm := createVM(t)
testCase := []string{
`assert.equal(http.get(url).text(), "");`,
@@ -65,14 +64,22 @@ func TestHttp(t *testing.T) {
for i, s := range testCase {
t.Run(fmt.Sprintf("Script%v", i), func(t *testing.T) {
- _, err := vm.RunString(context.Background(), s)
+ _, err := vm.Runtime().RunString(s)
assert.NoError(t, err)
})
}
}
-func createVM(t *testing.T) js.VM {
- vm := modulestest.New(t)
+var initial = js.WithInitial(func(rt *sobek.Runtime) {
+ client := http.Client{Transport: &http.Transport{Proxy: ski.ProxyFromRequest}}
+ instance, _ := (&Http{&client}).Instantiate(rt)
+ _ = rt.Set("http", instance)
+ f, _ := (&Fetch{&client}).Instantiate(rt)
+ _ = rt.Set("fetch", f)
+})
+
+func createVM(t *testing.T) modulestest.VM {
+ vm := modulestest.New(t, initial)
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodPut {
@@ -123,7 +130,6 @@ func createVM(t *testing.T) js.VM {
})
_, _ = vm.Runtime().RunString(fmt.Sprintf(`
- const http = require('cloudcat/http');
const url = "%s";
const proxyURL = "%s";
const fa = new Uint8Array([226, 153, 130, 239, 184, 142])`, ts.URL, proxy.URL))
diff --git a/js/modules/http/response.go b/js/modules/http/response.go
index 2551da6..89243a3 100644
--- a/js/modules/http/response.go
+++ b/js/modules/http/response.go
@@ -7,50 +7,73 @@ import (
"net/http"
"strings"
- "github.com/dop251/goja"
- "github.com/shiroyk/cloudcat/js"
+ "github.com/grafana/sobek"
+ "github.com/shiroyk/ski/js"
)
-func defineAccessorProperty(r *goja.Runtime, o *goja.Object, name string, v any) {
- _ = o.DefineAccessorProperty(name, r.ToValue(func(call goja.FunctionCall) goja.Value { return r.ToValue(v) }), nil, goja.FLAG_FALSE, goja.FLAG_FALSE)
+var errBodyAlreadyRead = errors.New("body stream already read")
+
+func defineGetter(r *sobek.Runtime, o *sobek.Object, name string, v func() any) {
+ _ = o.DefineAccessorProperty(name, r.ToValue(func(sobek.FunctionCall) sobek.Value {
+ return r.ToValue(v())
+ }), nil, sobek.FLAG_FALSE, sobek.FLAG_TRUE)
}
// NewResponse returns a new Response
-func NewResponse(vm *goja.Runtime, res *http.Response) goja.Value {
- defer res.Body.Close()
- body, err := io.ReadAll(res.Body)
- if err != nil {
- js.Throw(vm, err)
+func NewResponse(rt *sobek.Runtime, res *http.Response) sobek.Value {
+ var bodyUsed bool
+ js.OnDone(rt, func() {
+ if !bodyUsed {
+ res.Body.Close()
+ }
+ })
+ readBody := func() []byte {
+ if bodyUsed {
+ js.Throw(rt, errBodyAlreadyRead)
+ }
+ bodyUsed = true
+ defer res.Body.Close()
+ data, err := io.ReadAll(res.Body)
+ if err != nil {
+ js.Throw(rt, err)
+ }
+ return data
}
- object := vm.NewObject()
- _ = object.DefineAccessorProperty("body", vm.ToValue(func(call goja.FunctionCall) goja.Value {
- return vm.ToValue(vm.NewArrayBuffer(body))
- }), nil, goja.FLAG_FALSE, goja.FLAG_FALSE)
- defineAccessorProperty(vm, object, "bodyUsed", true)
- defineAccessorProperty(vm, object, "headers", joinHeader(res.Header))
- defineAccessorProperty(vm, object, "status", res.StatusCode)
- defineAccessorProperty(vm, object, "statusText", res.Status)
- defineAccessorProperty(vm, object, "ok", res.StatusCode >= 200 || res.StatusCode < 300)
- _ = object.Set("text", func(goja.FunctionCall) goja.Value { return vm.ToValue(string(body)) })
- _ = object.Set("json", func(goja.FunctionCall) goja.Value {
- var j any
- if err = json.Unmarshal(body, &j); err != nil {
- js.Throw(vm, err)
+
+ object := rt.NewObject()
+ defineGetter(rt, object, "body", func() any { return rt.NewArrayBuffer(readBody()) })
+ defineGetter(rt, object, "bodyUsed", func() any { return bodyUsed })
+ defineGetter(rt, object, "headers", func() any { return joinHeader(res.Header) })
+ defineGetter(rt, object, "status", func() any { return res.StatusCode })
+ defineGetter(rt, object, "statusText", func() any { return res.Status })
+ defineGetter(rt, object, "ok", func() any {
+ return res.StatusCode >= 200 && res.StatusCode < 300
+ })
+ _ = object.Set("text", func(sobek.FunctionCall) sobek.Value { return rt.ToValue(string(readBody())) })
+ _ = object.Set("json", func(call sobek.FunctionCall) sobek.Value {
+ var data any
+ if err := json.Unmarshal(readBody(), &data); err != nil {
+ js.Throw(rt, err)
}
- return vm.ToValue(j)
+ return rt.ToValue(data)
})
- _ = object.Set("arrayBuffer", func(goja.FunctionCall) goja.Value { return vm.ToValue(vm.NewArrayBuffer(body)) })
+ _ = object.Set("arrayBuffer", func(sobek.FunctionCall) sobek.Value { return rt.ToValue(rt.NewArrayBuffer(readBody())) })
return object
}
// NewAsyncResponse returns a new async Response
-func NewAsyncResponse(vm *goja.Runtime, res *http.Response) goja.Value {
+func NewAsyncResponse(rt *sobek.Runtime, res *http.Response) sobek.Value {
var bodyUsed bool
- object := vm.NewObject()
+ js.OnDone(rt, func() {
+ if !bodyUsed {
+ res.Body.Close()
+ }
+ })
+ object := rt.NewObject()
readBody := func() ([]byte, error) {
if bodyUsed {
- return nil, errors.New("body used already for")
+ return nil, errBodyAlreadyRead
}
bodyUsed = true
defer res.Body.Close()
@@ -61,20 +84,21 @@ func NewAsyncResponse(vm *goja.Runtime, res *http.Response) goja.Value {
return data, nil
}
- _ = object.DefineAccessorProperty("body", vm.ToValue(func(call goja.FunctionCall) goja.Value {
+ defineGetter(rt, object, "body", func() any {
if bodyUsed {
- js.Throw(vm, errors.New("body used already for"))
+ js.Throw(rt, errBodyAlreadyRead)
}
- return NewReadableStream(res.Body, vm, &bodyUsed)
- }), nil, goja.FLAG_FALSE, goja.FLAG_FALSE)
- defineAccessorProperty(vm, object, "bodyUsed", &bodyUsed)
- defineAccessorProperty(vm, object, "headers", joinHeader(res.Header))
- defineAccessorProperty(vm, object, "status", res.StatusCode)
- defineAccessorProperty(vm, object, "statusText", res.Status)
- defineAccessorProperty(vm, object, "ok", res.StatusCode >= 200 || res.StatusCode < 300)
-
- _ = object.Set("text", func(goja.FunctionCall) goja.Value {
- return vm.ToValue(js.NewPromise(vm, func() (any, error) {
+ return NewReadableStream(res.Body, rt, &bodyUsed)
+ })
+ defineGetter(rt, object, "bodyUsed", func() any { return bodyUsed })
+ defineGetter(rt, object, "headers", func() any { return joinHeader(res.Header) })
+ defineGetter(rt, object, "status", func() any { return res.StatusCode })
+ defineGetter(rt, object, "statusText", func() any { return res.Status })
+ defineGetter(rt, object, "ok", func() any {
+ return res.StatusCode >= 200 && res.StatusCode < 300
+ })
+ _ = object.Set("text", func(sobek.FunctionCall) sobek.Value {
+ return rt.ToValue(js.NewPromise(rt, func() (any, error) {
data, err := readBody()
if err != nil {
return nil, err
@@ -82,8 +106,8 @@ func NewAsyncResponse(vm *goja.Runtime, res *http.Response) goja.Value {
return string(data), nil
}))
})
- _ = object.Set("json", func(goja.FunctionCall) goja.Value {
- return vm.ToValue(js.NewPromise(vm, func() (any, error) {
+ _ = object.Set("json", func(sobek.FunctionCall) sobek.Value {
+ return rt.ToValue(js.NewPromise(rt, func() (any, error) {
data, err := readBody()
if err != nil {
return nil, err
@@ -95,13 +119,13 @@ func NewAsyncResponse(vm *goja.Runtime, res *http.Response) goja.Value {
return j, err
}))
})
- _ = object.Set("arrayBuffer", func(goja.FunctionCall) goja.Value {
- return vm.ToValue(js.NewPromise(vm, func() (any, error) {
+ _ = object.Set("arrayBuffer", func(sobek.FunctionCall) sobek.Value {
+ return rt.ToValue(js.NewPromise(rt, func() (any, error) {
data, err := readBody()
if err != nil {
return nil, err
}
- return vm.NewArrayBuffer(data), nil
+ return rt.NewArrayBuffer(data), nil
}))
})
return object
@@ -117,7 +141,7 @@ func joinHeader(header http.Header) map[string]string {
// NewReadableStream ReadableStream API
// https://developer.mozilla.org/en-US/docs/Web/API/ReadableStream
-func NewReadableStream(body io.ReadCloser, vm *goja.Runtime, bodyUsed *bool) *goja.Object {
+func NewReadableStream(body io.ReadCloser, vm *sobek.Runtime, bodyUsed *bool) *sobek.Object {
var lock bool
object := vm.NewObject()
_ = object.Set("cancel", func() {
@@ -125,9 +149,9 @@ func NewReadableStream(body io.ReadCloser, vm *goja.Runtime, bodyUsed *bool) *go
js.Throw(vm, err)
}
})
- _ = object.Set("getReader", func(call goja.FunctionCall) goja.Value {
+ _ = object.Set("getReader", func(call sobek.FunctionCall) sobek.Value {
if *bodyUsed {
- js.Throw(vm, errors.New("body used already for"))
+ js.Throw(vm, errBodyAlreadyRead)
}
*bodyUsed = true
if lock {
@@ -144,53 +168,49 @@ func NewReadableStream(body io.ReadCloser, vm *goja.Runtime, bodyUsed *bool) *go
return object
}
-type chunk struct {
- Value goja.Value
+type iter struct {
+ Value sobek.Value
Done bool
}
// NewReadableStreamDefaultReader Streams API
// https://developer.mozilla.org/en-US/docs/Web/API/ReadableStreamDefaultReader
// https://developer.mozilla.org/en-US/docs/Web/API/ReadableStreamBYOBReader
-func NewReadableStreamDefaultReader(body io.ReadCloser, vm *goja.Runtime, lock *bool) *goja.Object {
+func NewReadableStreamDefaultReader(body io.ReadCloser, vm *sobek.Runtime, lock *bool) *sobek.Object {
object := vm.NewObject()
- defineAccessorProperty(vm, object, "locked", &lock)
+ defineGetter(vm, object, "locked", func() any { return &lock })
_ = object.Set("cancel", func() {
if err := body.Close(); err != nil {
js.Throw(vm, err)
}
})
- _ = object.Set("read", func(call goja.FunctionCall) goja.Value {
+ _ = object.Set("read", func(call sobek.FunctionCall) sobek.Value {
var buffer []byte
- var value *goja.Object
- var view bool
- if goja.IsUndefined(call.Argument(0)) {
+ if sobek.IsUndefined(call.Argument(0)) {
buffer = make([]byte, 1024)
} else {
+ var view bool
buffer, view = call.Argument(0).Export().([]byte)
if !view {
js.Throw(vm, errors.New("read view is not TypedArray"))
}
- value = call.Argument(0).ToObject(vm)
}
- return vm.ToValue(js.NewPromise(vm, func() (int, error) { return body.Read(buffer) },
+ return vm.ToValue(js.NewPromise(vm,
+ func() (int, error) { return body.Read(buffer) },
func(n int, err error) (any, error) {
if err != nil {
if errors.Is(err, io.EOF) {
- return chunk{goja.Undefined(), true}, nil
+ return iter{sobek.Undefined(), true}, nil
}
return nil, err
}
- if !view {
- buffer = buffer[:n]
- value, err = vm.New(vm.Get("Uint8Array"), vm.ToValue(&buffer))
- if err != nil {
- js.Throw(vm, err)
- }
+ value, err := vm.New(vm.Get("Uint8Array"), vm.ToValue(vm.NewArrayBuffer(buffer[:n])))
+ if err != nil {
+ js.Throw(vm, err)
}
- return chunk{value, false}, nil
+ return iter{value, false}, nil
}))
})
diff --git a/js/modules/http/response_test.go b/js/modules/http/response_test.go
index daf1be8..01d9834 100644
--- a/js/modules/http/response_test.go
+++ b/js/modules/http/response_test.go
@@ -9,15 +9,12 @@ import (
"testing"
"time"
- "github.com/shiroyk/cloudcat"
- "github.com/shiroyk/cloudcat/js/modulestest"
+ "github.com/shiroyk/ski/js/modulestest"
"github.com/stretchr/testify/assert"
)
func TestResponse(t *testing.T) {
- ctx := context.Background()
- vm := modulestest.New(t)
- cloudcat.Provide[cloudcat.Fetch](http.DefaultClient)
+ vm := modulestest.New(t, initial)
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
@@ -37,9 +34,10 @@ func TestResponse(t *testing.T) {
}))
_ = vm.Runtime().Set("url", ts.URL)
- _, _ = vm.Runtime().RunString(`const http = require("cloudcat/http");`)
testCase := []string{
+ `const res = http.get(url+'/array');
+ assert.true(res.ok);`,
`const res = http.get(url+'/json');
assert.equal(res.json(), { "foo": "bar", "test": true });
assert.true(res.bodyUsed);
@@ -55,27 +53,28 @@ func TestResponse(t *testing.T) {
assert.equal(res.statusText, "200 OK");
assert.equal(res.headers["Content-Type"], "application/json");`,
`const res = http.get(url+'/text');
- assert.true(res.bodyUsed);
+ assert.true(!res.bodyUsed);
assert.true(res.ok);
assert.equal(res.statusText, "200 OK");
assert.equal(res.text(), "foo");
- assert.equal(res.arrayBuffer(), new Uint8Array([102, 111, 111]));
- assert.equal(res.arrayBuffer(), res.body);
- assert.equal(res.headers["Content-Type"], "text/plain");`,
+ assert.true(res.bodyUsed);
+ try {
+ res.text();
+ } catch (e) {
+ assert.true(e && e.toString().includes("body stream already read"));
+ }`,
}
for i, s := range testCase {
t.Run(fmt.Sprintf("Script%v", i), func(t *testing.T) {
- _, err := vm.RunString(ctx, s)
+ _, err := vm.Runtime().RunString(fmt.Sprintf(`{%s}`, s))
assert.NoError(t, err)
})
}
}
func TestAsyncResponse(t *testing.T) {
- ctx := context.Background()
- vm := modulestest.New(t)
- cloudcat.Provide[cloudcat.Fetch](http.DefaultClient)
+ vm := modulestest.New(t, initial)
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
@@ -118,7 +117,7 @@ func TestAsyncResponse(t *testing.T) {
size++;
return read();
};
- assert.equal(await read(), "0\r\n\x001\r\n\x002\r\n\x003\r\n\x004\r\n\x005\r\n\x00");
+ assert.equal(await read(), "0\r\n1\r\n2\r\n3\r\n4\r\n5\r\n");
assert.true(res.bodyUsed);
assert.true(res.ok);
assert.equal(res.status, 200);
@@ -169,8 +168,34 @@ func TestAsyncResponse(t *testing.T) {
for i, s := range testCase {
t.Run(fmt.Sprintf("Script%v", i), func(t *testing.T) {
- _, err := vm.RunString(ctx, s)
+ _, err := vm.RunString(context.Background(), s)
assert.NoError(t, err)
})
}
}
+
+type testBody struct {
+ closed bool
+}
+
+func (*testBody) Read(p []byte) (n int, err error) {
+ return 0, nil
+}
+
+func (b *testBody) Close() error {
+ b.closed = true
+ return nil
+}
+
+func TestAutoClose(t *testing.T) {
+ vm := modulestest.New(t)
+ body := new(testBody)
+ res := NewResponse(vm.Runtime(), &http.Response{Body: body, StatusCode: http.StatusOK})
+ ctx := context.WithValue(context.Background(), "res", res)
+ assert.False(t, body.closed)
+ v, err := vm.RunModule(ctx, `export default (ctx) => ctx.get('res').ok`)
+ if assert.NoError(t, err) {
+ assert.True(t, v.ToBoolean())
+ assert.True(t, body.closed)
+ }
+}
diff --git a/js/modules/http/signal.go b/js/modules/http/signal.go
index 20ac48e..37427e3 100644
--- a/js/modules/http/signal.go
+++ b/js/modules/http/signal.go
@@ -5,45 +5,44 @@ import (
"sync"
"time"
- "github.com/dop251/goja"
- "github.com/shiroyk/cloudcat/js"
+ "github.com/grafana/sobek"
+ "github.com/shiroyk/ski/js"
)
-// AbortController interface represents a controller object
+// abortController interface represents a controller object
// that allows you to abort one or more Web requests as and when desired.
// https://developer.mozilla.org/en-US/docs/Web/API/AbortController.
-type AbortController struct {
- Signal *AbortSignal
+type abortController struct {
+ Signal *abortSignal
Aborted bool
Reason string
}
-func (c *AbortController) Abort() {
+func (c *abortController) Abort() {
c.Signal.abort()
c.Aborted = c.Signal.Aborted
c.Reason = c.Signal.Reason
}
-// AbortControllerConstructor AbortController Constructor
-type AbortControllerConstructor struct{}
+// AbortController Constructor
+type AbortController struct{}
-// Exports AbortController Constructor
-func (*AbortControllerConstructor) Exports() any {
- return func(call goja.ConstructorCall, vm *goja.Runtime) *goja.Object {
- signal := new(AbortSignal)
- parent := js.VMContext(vm)
- signal.ctx, signal.cancel = context.WithCancel(parent)
- return vm.ToValue(&AbortController{Signal: signal}).ToObject(vm)
- }
+// Instantiate module
+func (*AbortController) Instantiate(rt *sobek.Runtime) (sobek.Value, error) {
+ return rt.ToValue(func(call sobek.ConstructorCall, vm *sobek.Runtime) *sobek.Object {
+ signal := new(abortSignal)
+ signal.ctx, signal.cancel = context.WithCancel(js.Context(vm))
+ return vm.ToValue(&abortController{Signal: signal}).ToObject(vm)
+ }), nil
}
// Global it is a global module
-func (*AbortControllerConstructor) Global() {}
+func (*AbortController) Global() {}
-// AbortSignal represents a signal object that allows you to communicate
+// abortSignal represents a signal object that allows you to communicate
// with http request and abort it.
// https://developer.mozilla.org/en-US/docs/Web/API/AbortSignal
-type AbortSignal struct {
+type abortSignal struct {
ctx context.Context
cancel context.CancelFunc
once sync.Once
@@ -51,7 +50,7 @@ type AbortSignal struct {
Reason string
}
-func (s *AbortSignal) abort() {
+func (s *abortSignal) abort() {
s.once.Do(func() {
s.Aborted = true
s.cancel()
@@ -61,24 +60,23 @@ func (s *AbortSignal) abort() {
})
}
-type AbortSignalModule struct{}
+type AbortSignal struct{}
-func (*AbortSignalModule) Exports() any { return new(abortSignalInstance) }
-
-func (*AbortSignalModule) Global() {}
-
-type abortSignalInstance struct{}
-
-func (s *abortSignalInstance) Abort(_ goja.FunctionCall, vm *goja.Runtime) goja.Value {
- signal := new(AbortSignal)
- signal.ctx, signal.cancel = context.WithCancel(context.Background())
- signal.abort()
- return vm.ToValue(signal).ToObject(vm)
+func (*AbortSignal) Instantiate(rt *sobek.Runtime) (sobek.Value, error) {
+ object := rt.NewObject()
+ _ = object.Set("abort", func(_ sobek.FunctionCall) sobek.Value {
+ signal := new(abortSignal)
+ signal.ctx, signal.cancel = context.WithCancel(context.Background())
+ signal.abort()
+ return rt.ToValue(signal).ToObject(rt)
+ })
+ _ = object.Set("timeout", func(call sobek.FunctionCall) sobek.Value {
+ timeout := call.Argument(0).ToInteger()
+ signal := new(abortSignal)
+ signal.ctx, signal.cancel = context.WithTimeout(js.Context(rt), time.Duration(timeout))
+ return rt.ToValue(signal).ToObject(rt)
+ })
+ return object, nil
}
-func (s *abortSignalInstance) Timeout(call goja.FunctionCall, vm *goja.Runtime) goja.Value {
- timeout := call.Argument(0).ToInteger()
- signal := new(AbortSignal)
- signal.ctx, signal.cancel = context.WithTimeout(js.VMContext(vm), time.Duration(timeout))
- return vm.ToValue(signal).ToObject(vm)
-}
+func (*AbortSignal) Global() {}
diff --git a/js/modules/http/signal_test.go b/js/modules/http/signal_test.go
index 12d6007..def118a 100644
--- a/js/modules/http/signal_test.go
+++ b/js/modules/http/signal_test.go
@@ -1,19 +1,17 @@
package http
import (
- "context"
"fmt"
"testing"
- "github.com/shiroyk/cloudcat/js/modulestest"
+ "github.com/shiroyk/ski/js/modulestest"
"github.com/stretchr/testify/assert"
)
func TestAbortSignal(t *testing.T) {
- ctx := context.Background()
vm := modulestest.New(t)
- testCase := []string{
+ testCases := []string{
`const controller = new AbortController();
controller.abort();
assert.equal(controller.reason, "context canceled");
@@ -26,9 +24,9 @@ func TestAbortSignal(t *testing.T) {
assert.true(!signal.aborted);`,
}
- for i, s := range testCase {
+ for i, s := range testCases {
t.Run(fmt.Sprintf("Script%v", i), func(t *testing.T) {
- _, err := vm.RunString(ctx, s)
+ _, err := vm.Runtime().RunString(fmt.Sprintf(`{%s}`, s))
assert.NoError(t, err)
})
}
diff --git a/js/modules/http/url_search_params.go b/js/modules/http/url_search_params.go
index f9b9065..04fec20 100644
--- a/js/modules/http/url_search_params.go
+++ b/js/modules/http/url_search_params.go
@@ -3,69 +3,100 @@ package http
import (
"fmt"
"net/url"
- "sort"
+ "reflect"
+ "slices"
"strings"
- "github.com/dop251/goja"
- "github.com/shiroyk/cloudcat"
- "github.com/shiroyk/cloudcat/js"
+ "github.com/grafana/sobek"
+ "github.com/shiroyk/ski"
+ "github.com/shiroyk/ski/js"
"github.com/spf13/cast"
)
-// The URLSearchParams defines utility methods to work with the query string of a URL,
+// The urlSearchParams defines utility methods to work with the query string of a URL,
// which can be sent using the http() method and encoding type were set to "application/x-www-form-url".
// Implement the https://developer.mozilla.org/en-US/docs/Web/API/URLSearchParams
-type URLSearchParams struct {
+type urlSearchParams struct {
+ keys []string
data map[string][]string
}
-// URLSearchParamsConstructor Constructor
-type URLSearchParamsConstructor struct{}
+// URLSearchParams Constructor
+type URLSearchParams struct{}
-// Exports instance URLSearchParams module
-func (*URLSearchParamsConstructor) Exports() any {
- return func(call goja.ConstructorCall, vm *goja.Runtime) *goja.Object {
- param := call.Argument(0)
+// Instantiate instance module
+func (*URLSearchParams) Instantiate(rt *sobek.Runtime) (sobek.Value, error) {
+ return rt.ToValue(func(call sobek.ConstructorCall) *sobek.Object {
+ params := call.Argument(0)
- if goja.IsUndefined(param) {
- return vm.ToValue(URLSearchParams{data: make(url.Values)}).ToObject(vm)
+ var ret urlSearchParams
+ if sobek.IsUndefined(params) {
+ ret.data = make(map[string][]string)
+ return ret.object(rt)
}
- var pa map[string]any
- var ok bool
- pa, ok = param.Export().(map[string]any)
- if !ok {
- js.Throw(vm, fmt.Errorf("unsupported type %T", param.Export()))
+ if params.ExportType().Kind() == reflect.String {
+ str := strings.TrimPrefix(params.String(), "?")
+ kvs := strings.Split(str, "&")
+ ret.data = make(map[string][]string, len(kvs))
+ for _, kv := range kvs {
+ k, v, _ := strings.Cut(kv, "=")
+ ret.Append(k, v)
+ }
+ return ret.object(rt)
}
- data := make(map[string][]string, len(pa))
- for k, v := range pa {
- if s, ok := v.([]any); ok {
- data[k] = cast.ToStringSlice(s)
+ object := params.ToObject(rt)
+ keys := object.Keys()
+ ret.keys = make([]string, 0, len(keys))
+ ret.data = make(map[string][]string, len(keys))
+
+ for _, key := range keys {
+ value, _ := js.Unwrap(object.Get(key))
+ if s, ok := value.([]any); ok {
+ ret.data[key] = cast.ToStringSlice(s)
} else {
- data[k] = []string{fmt.Sprintf("%v", v)}
+ ret.data[key] = []string{fmt.Sprintf("%s", value)}
}
+ ret.keys = append(ret.keys, key)
}
- return vm.ToValue(URLSearchParams{data: data}).ToObject(vm)
- }
+ return ret.object(rt)
+ }), nil
}
// Global it is a global module
-func (*URLSearchParamsConstructor) Global() {}
+func (*URLSearchParams) Global() {}
+
+func (u *urlSearchParams) object(rt *sobek.Runtime) *sobek.Object {
+ obj := rt.ToValue(u).ToObject(rt)
+
+ _ = obj.SetSymbol(sobek.SymIterator, func(sobek.ConstructorCall) *sobek.Object {
+ var i int
+ it := rt.NewObject()
+ _ = it.Set("next", func(sobek.FunctionCall) sobek.Value {
+ if i < len(u.keys) {
+ key := u.keys[i]
+ i++
+ return rt.ToValue(iter{Value: rt.ToValue([2]any{key, u.data[key]})})
+ }
+ return rt.ToValue(iter{Done: true})
+ })
+ return it
+ })
+ return obj
+}
// encode encodes the values into “URL encoded” form
// ("bar=baz&foo=qux") sorted by key.
-func (u *URLSearchParams) encode() string {
+func (u *urlSearchParams) encode() string {
if u.data == nil {
return ""
}
var buf strings.Builder
- keys := cloudcat.MapKeys(u.data)
- sort.Strings(keys)
- for _, k := range keys {
- vs := u.data[k]
- keyEscaped := url.QueryEscape(k)
+ for _, key := range u.keys {
+ vs := u.data[key]
+ keyEscaped := url.QueryEscape(key)
for _, v := range vs {
if buf.Len() > 0 {
buf.WriteByte('&')
@@ -78,38 +109,41 @@ func (u *URLSearchParams) encode() string {
return buf.String()
}
-// Append method of the URLSearchParams interface appends a specified key/value pair as a new search parameter.
-func (u *URLSearchParams) Append(name, value string) {
- u.data[name] = append(u.data[name], value)
+// Append method of the urlSearchParams interface appends a specified key/value pair as a new search parameter.
+func (u *urlSearchParams) Append(name, value string) {
+ values, ok := u.data[name]
+ if !ok {
+ u.keys = append(u.keys, name)
+ }
+ u.data[name] = append(values, value)
}
-// Delete method of the URLSearchParams interface deletes the given search parameter and all its associated values,
+// Delete method of the urlSearchParams interface deletes the given search parameter and all its associated values,
// from the list of all search parameters.
-func (u *URLSearchParams) Delete(name string) {
+func (u *urlSearchParams) Delete(name string) {
+ u.keys = slices.DeleteFunc(u.keys, func(k string) bool { return k == name })
delete(u.data, name)
}
-// Entries method of the URLSearchParams interface returns an iterator allowing iteration
+// Entries method of the urlSearchParams interface returns an iterator allowing iteration
// through all key/value pairs contained in this object.
// The iterator returns key/value pairs in the same order as they appear in the query string.
// The key and value of each pair are string objects.
-func (u *URLSearchParams) Entries() any {
- entries := make([][2]string, 0, len(u.data))
- for k, v := range u.data {
- for _, ve := range v {
- entries = append(entries, [2]string{k, ve})
- }
+func (u *urlSearchParams) Entries() any {
+ entries := make([][2]any, 0, len(u.keys))
+ for _, key := range u.keys {
+ entries = append(entries, [2]any{key, u.data[key]})
}
return entries
}
-// ForEach method of the URLSearchParams interface allows iteration
+// ForEach method of the urlSearchParams interface allows iteration
// through all values contained in this object via a callback function.
-func (u *URLSearchParams) ForEach(call goja.FunctionCall, vm *goja.Runtime) (ret goja.Value) {
+func (u *urlSearchParams) ForEach(call sobek.FunctionCall, vm *sobek.Runtime) (ret sobek.Value) {
arg := call.Argument(0)
- if callback, ok := goja.AssertFunction(arg); ok {
- for k, v := range u.data {
- if _, err := callback(goja.Undefined(), vm.ToValue(v), vm.ToValue(k), vm.ToValue(u)); err != nil {
+ if callback, ok := sobek.AssertFunction(arg); ok {
+ for _, key := range u.keys {
+ if _, err := callback(sobek.Undefined(), vm.ToValue(u.data[key]), vm.ToValue(key), vm.ToValue(u)); err != nil {
panic(vm.ToValue(err))
}
}
@@ -117,56 +151,55 @@ func (u *URLSearchParams) ForEach(call goja.FunctionCall, vm *goja.Runtime) (ret
return
}
-// Get method of the URLSearchParams interface returns the first value associated to the given search parameter.
-func (u *URLSearchParams) Get(name string) any {
+// Get method of the urlSearchParams interface returns the first value associated to the given search parameter.
+func (u *urlSearchParams) Get(name string) any {
if v, ok := u.data[name]; ok {
return v[0]
}
return nil
}
-// GetAll method of the URLSearchParams interface returns all the values associated
+// GetAll method of the urlSearchParams interface returns all the values associated
// with a given search parameter as an array.
-func (u *URLSearchParams) GetAll(name string) []string {
+func (u *urlSearchParams) GetAll(name string) []string {
if v, ok := u.data[name]; ok {
return v
}
return []string{}
}
-// Has method of the URLSearchParams interface returns a boolean value that indicates whether
+// Has method of the urlSearchParams interface returns a boolean value that indicates whether
// a parameter with the specified name exists.
-func (u *URLSearchParams) Has(name string) bool {
+func (u *urlSearchParams) Has(name string) bool {
_, ok := u.data[name]
return ok
}
-// Keys method of the URLSearchParams interface returns an iterator allowing iteration
+// Keys method of the urlSearchParams interface returns an iterator allowing iteration
// through all keys contained in this object. The keys are string objects.
-func (u *URLSearchParams) Keys() []string {
- return cloudcat.MapKeys(u.data)
-}
+func (u *urlSearchParams) Keys() []string { return u.keys }
-// Set method of the URLSearchParams interface sets the value associated
+// Set method of the urlSearchParams interface sets the value associated
// with a given search parameter to the given value.
// If there were several matching values, this method deletes the others.
// If the search parameter doesn't exist, this method creates it.
-func (u *URLSearchParams) Set(name, value string) {
+func (u *urlSearchParams) Set(name, value string) {
+ if _, ok := u.data[name]; !ok {
+ u.keys = append(u.keys, name)
+ }
u.data[name] = []string{value}
}
// Sort method sorts all key/value pairs contained in this object in place and returns undefined.
-func (u *URLSearchParams) Sort() {
- // Not implemented
-}
+func (u *urlSearchParams) Sort() { slices.Sort(u.keys) }
-// ToString method of the URLSearchParams interface returns a query string suitable for use in a URL.
-func (u *URLSearchParams) ToString() string {
+// ToString method of the urlSearchParams interface returns a query string suitable for use in a URL.
+func (u *urlSearchParams) ToString() string {
return u.encode()
}
-// Values method of the URLSearchParams interface returns an iterator allowing iteration through
+// Values method of the urlSearchParams interface returns an iterator allowing iteration through
// all values contained in this object. The values are string objects.
-func (u *URLSearchParams) Values() [][]string {
- return cloudcat.MapValues(u.data)
+func (u *urlSearchParams) Values() [][]string {
+ return ski.MapValues(u.data)
}
diff --git a/js/modules/http/url_search_params_test.go b/js/modules/http/url_search_params_test.go
index 23916a7..f520079 100644
--- a/js/modules/http/url_search_params_test.go
+++ b/js/modules/http/url_search_params_test.go
@@ -1,19 +1,17 @@
package http
import (
- "context"
"fmt"
"testing"
- "github.com/shiroyk/cloudcat/js/modulestest"
+ "github.com/shiroyk/ski/js/modulestest"
"github.com/stretchr/testify/assert"
)
func TestURLSearchParams(t *testing.T) {
- ctx := context.Background()
vm := modulestest.New(t)
- _, _ = vm.Runtime().RunString(`const form = new URLSearchParams({'name': 'foo'});form.sort();`)
+ _, _ = vm.Runtime().RunString(`const params = new URLSearchParams({'name': 'foo'});params.sort();`)
testCase := []string{
`try {
@@ -21,24 +19,35 @@ func TestURLSearchParams(t *testing.T) {
} catch (e) {
assert.true(e.toString().includes('unsupported type'))
}`,
- `form.forEach((v, k) => assert.true(v.length == 1))
- assert.equal(form.get('name'), 'foo')`,
- `form.append('name', 'bar');
- assert.equal(form.getAll('name').length, 2)`,
- `assert.equal(form.toString(), 'name=foo&name=bar')`,
- `form.append('value', 'zoo');
- assert.true(form.keys(), ['name', 'value'])`,
- `assert.equal(form.entries().length, 3)`,
- `form.delete('name');
- assert.equal(form.getAll('name').length, 0)`,
- `assert.true(!form.has('name'))`,
- `form.set('name', 'foobar');
- assert.equal(form.values().length, 2)`,
+ `assert.equal(new URLSearchParams('foo=1&bar=2').toString(), 'foo=1&bar=2')`,
+ `assert.equal(new URLSearchParams('?foo=1&bar=2').toString(), 'foo=1&bar=2')`,
+ `assert.equal(new URLSearchParams('https://example.com?foo=1&bar=2').toString(), 'https%3A%2F%2Fexample.com%3Ffoo=1&bar=2')`,
+ `params.forEach((v, k) => assert.true(v.length == 1))
+ assert.equal(params.get('name'), 'foo')`,
+ `params.append('name', 'bar');
+ assert.equal(params.getAll('name').length, 2)`,
+ `assert.equal(params.toString(), 'name=foo&name=bar')`,
+ `params.append('value', 'zoo');
+ assert.true(params.keys(), ['name', 'value'])`,
+ `assert.equal(params.entries().length, 2)`,
+ `params.delete('name');
+ assert.equal(params.getAll('name').length, 0)`,
+ `assert.true(!params.has('name'))`,
+ `params.set('name', 'foobar');
+ assert.equal(params.values().length, 2)`,
+ `params.append('000', '114');
+ params.sort();
+ assert.equal(params.toString(), '000=114&name=foobar&value=zoo')`,
+ `let str = "";
+ for (const [key, value] of params) {
+ str += key + "=" + value + ",";
+ }
+ assert.equal(str, '000=114,name=foobar,value=zoo,')`,
}
for i, s := range testCase {
t.Run(fmt.Sprintf("Script%v", i), func(t *testing.T) {
- _, err := vm.RunString(ctx, s)
+ _, err := vm.Runtime().RunString(s)
assert.NoError(t, err)
})
}
diff --git a/js/modules/main.go b/js/modules/main.go
deleted file mode 100644
index b8e7a77..0000000
--- a/js/modules/main.go
+++ /dev/null
@@ -1,9 +0,0 @@
-package jsmodules
-
-import (
- _ "github.com/shiroyk/cloudcat/js/modules/cache" // deep
- _ "github.com/shiroyk/cloudcat/js/modules/cookie"
- _ "github.com/shiroyk/cloudcat/js/modules/crypto"
- _ "github.com/shiroyk/cloudcat/js/modules/encoding"
- _ "github.com/shiroyk/cloudcat/js/modules/http"
-)
diff --git a/js/modulestest/vm.go b/js/modulestest/vm.go
index 5346d70..e4d8120 100644
--- a/js/modulestest/vm.go
+++ b/js/modulestest/vm.go
@@ -2,21 +2,37 @@
package modulestest
import (
+ "context"
"errors"
"testing"
- "github.com/dop251/goja"
- "github.com/shiroyk/cloudcat/js"
+ "github.com/grafana/sobek"
+ "github.com/shiroyk/ski/js"
"github.com/stretchr/testify/assert"
)
-// New returns a test VM instance
-func New(t *testing.T) js.VM {
- vm := js.NewVM()
- runtime := vm.Runtime()
+type VM struct{ js.VM }
- assertObject := runtime.NewObject()
- _ = assertObject.Set("equal", func(call goja.FunctionCall, vm *goja.Runtime) (ret goja.Value) {
+func (vm *VM) RunString(ctx context.Context, source string) (ret sobek.Value, err error) {
+ vm.Run(ctx, func() {
+ ret, err = vm.Runtime().RunString(source)
+ })
+ return
+}
+
+func (vm *VM) RunModule(ctx context.Context, source string) (ret sobek.Value, err error) {
+ module, err := vm.Loader().CompileModule("", source)
+ if err != nil {
+ return
+ }
+ return vm.VM.RunModule(ctx, module)
+}
+
+// New returns a test VM instance
+func New(t *testing.T, opts ...js.Option) VM {
+ vm := js.NewVM(append([]js.Option{js.WithModuleLoader(js.NewModuleLoader())}, opts...)...)
+ assertObject := vm.Runtime().NewObject()
+ _ = assertObject.Set("equal", func(call sobek.FunctionCall, vm *sobek.Runtime) (ret sobek.Value) {
a, err := js.Unwrap(call.Argument(0))
if err != nil {
js.Throw(vm, err)
@@ -26,7 +42,7 @@ func New(t *testing.T) js.VM {
js.Throw(vm, err)
}
var msg string
- if !goja.IsUndefined(call.Argument(2)) {
+ if !sobek.IsUndefined(call.Argument(2)) {
msg = call.Argument(2).String()
}
if !assert.Equal(t, b, a, msg) {
@@ -34,9 +50,9 @@ func New(t *testing.T) js.VM {
}
return
})
- _ = assertObject.Set("true", func(call goja.FunctionCall, vm *goja.Runtime) (ret goja.Value) {
+ _ = assertObject.Set("true", func(call sobek.FunctionCall, vm *sobek.Runtime) (ret sobek.Value) {
var msg string
- if !goja.IsUndefined(call.Argument(1)) {
+ if !sobek.IsUndefined(call.Argument(1)) {
msg = call.Argument(1).String()
}
if !assert.True(t, call.Argument(0).ToBoolean(), msg) {
@@ -45,7 +61,6 @@ func New(t *testing.T) js.VM {
return
})
- _ = runtime.Set("assert", assertObject)
-
- return vm
+ _ = vm.Runtime().Set("assert", assertObject)
+ return VM{vm}
}
diff --git a/js/require.go b/js/require.go
deleted file mode 100644
index e0364f9..0000000
--- a/js/require.go
+++ /dev/null
@@ -1,348 +0,0 @@
-package js
-
-import (
- "encoding/json"
- "errors"
- "fmt"
- "io"
- "net/http"
- "os"
- "path"
- "path/filepath"
- "strings"
- "syscall"
- "text/template"
-
- "github.com/dop251/goja"
- "github.com/dop251/goja/parser"
- "github.com/shiroyk/cloudcat"
- "github.com/shiroyk/cloudcat/plugin/jsmodule"
-)
-
-var (
- // ErrInvalidModule module is invalid
- ErrInvalidModule = errors.New("invalid module")
- // ErrIllegalModuleName module name is illegal
- ErrIllegalModuleName = errors.New("illegal module name")
- // ErrModuleFileDoesNotExist module not exist
- ErrModuleFileDoesNotExist = errors.New("module file does not exist")
-)
-
-// Copyright dop251/goja_nodejs, licensed under the MIT License.
-// NodeJS module search algorithm described by
-// https://nodejs.org/api/modules.html#modules_all_together
-
-// EnableRequire set runtime require module
-func EnableRequire(vm *goja.Runtime, path ...string) {
- req := &require{
- vm: vm,
- modules: make(map[string]*goja.Object),
- nodeModules: make(map[string]*goja.Object),
- globalFolders: path,
- fetch: cloudcat.MustResolveLazy[cloudcat.Fetch](),
- }
-
- _ = vm.Set("require", req.Require)
-}
-
-type require struct {
- vm *goja.Runtime
- modules map[string]*goja.Object
- nodeModules map[string]*goja.Object
- fetch func() cloudcat.Fetch
-
- globalFolders []string
-}
-
-// Require load a js module from path or URL
-func (r *require) Require(name string) (export goja.Value, err error) {
- var module *goja.Object
- switch {
- case name == "":
- err = ErrIllegalModuleName
- case isHTTP(name):
- module, err = r.resolveRemote(name)
- case strings.HasPrefix(name, jsmodule.ExtPrefix):
- return r.resolveNative(name)
- default:
- module, err = r.resolveFile(name)
- }
- if err != nil {
- return nil, err
- }
- return module.Get("exports"), nil
-}
-
-func (r *require) resolveNative(name string) (*goja.Object, error) {
- if native, ok := r.modules[name]; ok {
- return native, nil
- }
- if e, ok := jsmodule.GetModule(name); ok {
- mod := r.vm.ToValue(e.Exports()).ToObject(r.vm)
- r.modules[name] = mod
- return mod, nil
- }
- return nil, ErrIllegalModuleName
-}
-
-//nolint:nakedret
-func (r *require) resolveFile(modPath string) (module *goja.Object, err error) {
- origPath, modPath := modPath, path.Clean(modPath)
- if modPath == "" {
- return nil, ErrIllegalModuleName
- }
-
- var start string
- err = nil
- if path.IsAbs(origPath) {
- start = "/"
- } else {
- start = r.getCurrentModulePath()
- }
-
- p := path.Join(start, modPath)
-
- if strings.HasPrefix(origPath, "./") || //nolint:nestif
- strings.HasPrefix(origPath, "/") ||
- strings.HasPrefix(origPath, "../") ||
- origPath == "." || origPath == ".." {
- if module = r.modules[p]; module != nil {
- return
- }
- module, err = r.loadAsFileOrDirectory(p)
- if err == nil && module != nil {
- r.modules[p] = module
- }
- } else {
- if module = r.nodeModules[p]; module != nil {
- return
- }
- module, err = r.loadNodeModules(modPath, start)
- if err == nil && module != nil {
- r.nodeModules[p] = module
- }
- }
-
- if module == nil && err == nil {
- err = ErrInvalidModule
- }
- return
-}
-
-func (r *require) resolveRemote(name string) (module *goja.Object, err error) {
- data, err := r.fetchFile(name)
- if err != nil {
- return nil, err
- }
- if mod, exists := r.modules[name]; exists {
- return mod, nil
- }
-
- module = r.vm.NewObject()
- _ = module.Set("exports", r.vm.NewObject())
- r.modules[name] = module
-
- source := "(function(exports, require, module) {" + string(data) + "\n})"
- if err = r.compileModule(name, source, module); err != nil {
- delete(r.modules, name)
- return nil, err
- }
-
- return
-}
-
-func (r *require) fetchFile(name string) ([]byte, error) {
- req, err := http.NewRequest(http.MethodGet, name, nil)
- if err != nil {
- return nil, err
- }
- res, err := r.fetch().Do(req)
- if err != nil {
- return nil, err
- }
- body, err := io.ReadAll(res.Body)
- return body, err
-}
-
-func (r *require) loadAsFileOrDirectory(path string) (module *goja.Object, err error) {
- if module, err = r.loadAsFile(path); module != nil || err != nil {
- return
- }
-
- return r.loadAsDirectory(path)
-}
-
-func (r *require) loadAsFile(path string) (module *goja.Object, err error) {
- if module, err = r.loadModule(path); module != nil || err != nil {
- return
- }
-
- p := path + ".js"
- if module, err = r.loadModule(p); module != nil || err != nil {
- return
- }
-
- p = path + ".json"
- return r.loadModule(p)
-}
-
-func (r *require) loadIndex(modPath string) (module *goja.Object, err error) {
- p := path.Join(modPath, "index.js")
- if module, err = r.loadModule(p); module != nil || err != nil {
- return
- }
-
- p = path.Join(modPath, "index.json")
- return r.loadModule(p)
-}
-
-func (r *require) loadAsDirectory(modPath string) (module *goja.Object, err error) {
- p := path.Join(modPath, "package.json")
- buf, err := r.loadSource(p)
- if err != nil {
- return r.loadIndex(modPath)
- }
- var pkg struct {
- Main string `json:"main"`
- }
- err = json.Unmarshal(buf, &pkg)
- if err != nil || len(pkg.Main) == 0 {
- return r.loadIndex(modPath)
- }
-
- m := path.Join(modPath, pkg.Main)
- if module, err = r.loadAsFile(m); module != nil || err != nil {
- return
- }
-
- return r.loadIndex(m)
-}
-
-// loadSource is used loads files from the host's filesystem.
-func (r *require) loadSource(filename string) ([]byte, error) {
- if isHTTP(filename) {
- return r.fetchFile(filename)
- }
- data, err := os.ReadFile(filepath.FromSlash(filename))
- if err != nil {
- if os.IsNotExist(err) || errors.Is(err, syscall.EISDIR) {
- err = ErrModuleFileDoesNotExist
- }
- }
- return data, err
-}
-
-func (r *require) loadNodeModule(modPath, start string) (*goja.Object, error) {
- return r.loadAsFileOrDirectory(path.Join(start, modPath))
-}
-
-func (r *require) loadNodeModules(modPath, start string) (module *goja.Object, err error) {
- for _, dir := range r.globalFolders {
- if module, err = r.loadNodeModule(modPath, dir); module != nil || err != nil {
- return
- }
- }
- for {
- var p string
- if path.Base(start) != "node_modules" {
- p = path.Join(start, "node_modules")
- } else {
- p = start
- }
- if module, err = r.loadNodeModule(modPath, p); module != nil || err != nil {
- return
- }
- if start == ".." { // Dir('..') is '.'
- break
- }
- parent := path.Dir(start)
- if parent == start {
- break
- }
- start = parent
- }
-
- return nil, fmt.Errorf("not found module %s", modPath)
-}
-
-func (r *require) getCurrentModulePath() string {
- var buf [2]goja.StackFrame
- frames := r.vm.CaptureCallStack(2, buf[:0])
- if len(frames) < 2 {
- return "."
- }
- return path.Dir(frames[1].SrcName())
-}
-
-func (r *require) loadModule(path string) (*goja.Object, error) {
- module := r.modules[path]
- if module == nil {
- module = r.vm.NewObject()
- _ = module.Set("exports", r.vm.NewObject())
- r.modules[path] = module
- err := r.loadModuleFile(path, module)
- if err != nil {
- module = nil
- delete(r.modules, path)
- if errors.Is(err, ErrModuleFileDoesNotExist) {
- err = nil
- }
- }
- return module, err
- }
- return module, nil
-}
-
-func (r *require) loadModuleFile(p string, jsModule *goja.Object) error {
- buf, err := r.loadSource(p)
- if err != nil {
- return err
- }
- s := string(buf)
-
- if path.Ext(p) == ".json" {
- s = "module.exports = JSON.parse('" + template.JSEscapeString(s) + "')"
- }
-
- source := "(function(exports, require, module) {" + s + "\n})"
-
- return r.compileModule(p, source, jsModule)
-}
-
-func (r *require) compileModule(path, source string, jsModule *goja.Object) error {
- parsed, err := goja.Parse(path, source, parser.WithSourceMapLoader(r.loadSource))
- if err != nil {
- return err
- }
-
- prg, err := goja.CompileAST(parsed, false)
- if err != nil {
- return err
- }
-
- f, err := r.vm.RunProgram(prg)
- if err != nil {
- return err
- }
-
- if call, ok := goja.AssertFunction(f); ok {
- jsExports := jsModule.Get("exports")
- jsRequire := r.vm.Get("require")
-
- // Run the module source, with "jsExports" as "this",
- // "jsExports" as the "exports" variable, "jsRequire"
- // as the "require" variable and "jsModule" as the
- // "module" variable (Nodejs capable).
- _, err = call(jsExports, jsExports, jsRequire, jsModule)
- if err != nil {
- return err
- }
- return nil
- }
-
- return ErrInvalidModule
-}
-
-func isHTTP(name string) bool {
- return strings.HasPrefix(name, "http://") || strings.HasPrefix(name, "https://")
-}
diff --git a/js/require_test.go b/js/require_test.go
deleted file mode 100644
index 77fdd17..0000000
--- a/js/require_test.go
+++ /dev/null
@@ -1,60 +0,0 @@
-package js
-
-import (
- "context"
- "io"
- "net/http"
- "strconv"
- "strings"
- "testing"
-
- "github.com/shiroyk/cloudcat"
- "github.com/shiroyk/cloudcat/plugin/jsmodule"
- "github.com/stretchr/testify/assert"
-)
-
-type testFetcher struct{}
-
-func (*testFetcher) Do(*http.Request) (*http.Response, error) {
- return &http.Response{Body: io.NopCloser(strings.NewReader("module.exports = { foo: 'bar' }"))}, nil
-}
-
-type testRModule struct{}
-
-func (testRModule) Exports() any { return map[string]string{"key": "testr"} }
-
-type testRGModule struct{}
-
-func (testRGModule) Exports() any { return map[string]string{"key": "testrg"} }
-
-func (testRGModule) Global() {}
-
-func TestRequire(t *testing.T) {
- cloudcat.Provide[cloudcat.Fetch](new(testFetcher))
- jsmodule.Register("testr", new(testRModule))
- jsmodule.Register("testrg", new(testRGModule))
- vm := NewTestVM(t)
-
- testCases := []struct {
- script string
- }{
- {
- `const testr = require("cloudcat/testr");
- assert.equal(testr.key, "testr")`,
- },
- {
- `assert.equal(testrg.key, "testrg")`,
- },
- {
- `const foo = require("https://foo.com/foo.min.js");
- assert.equal(foo.foo, "bar")`,
- },
- }
-
- for i, testCase := range testCases {
- t.Run(strconv.Itoa(i), func(t *testing.T) {
- _, err := vm.RunString(context.Background(), testCase.script)
- assert.NoError(t, err)
- })
- }
-}
diff --git a/js/scheduler.go b/js/scheduler.go
new file mode 100644
index 0000000..dba4f4c
--- /dev/null
+++ b/js/scheduler.go
@@ -0,0 +1,191 @@
+package js
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "runtime"
+ "sync/atomic"
+ "time"
+
+ "log/slog"
+
+ "github.com/grafana/sobek"
+ "github.com/shiroyk/ski"
+)
+
+const (
+ // DefaultMaxTimeToWaitGetVM default retries time
+ DefaultMaxTimeToWaitGetVM = 500 * time.Millisecond
+ // DefaultMaxRetriesGetVM default retries times
+ DefaultMaxRetriesGetVM = 3
+)
+
+var (
+ _scheduler = new(atomic.Value)
+ // ErrSchedulerClosed the scheduler is closed error
+ ErrSchedulerClosed = errors.New("scheduler is closed")
+)
+
+func init() {
+ ski.Register("js", new_executor())
+ _scheduler.Store(NewScheduler(SchedulerOptions{
+ MaxVMs: uint(runtime.GOMAXPROCS(0)),
+ Loader: NewModuleLoader(),
+ }))
+}
+
+// SetScheduler set the default Scheduler
+func SetScheduler(scheduler Scheduler) { _scheduler.Store(scheduler) }
+
+// RunModule the sobek.CyclicModuleRecord
+func RunModule(ctx context.Context, module sobek.CyclicModuleRecord) (sobek.Value, error) {
+ vm, err := GetScheduler().Get()
+ if err != nil {
+ return nil, err
+ }
+ return vm.RunModule(ctx, module)
+}
+
+// GetScheduler get the default Scheduler
+func GetScheduler() Scheduler { return _scheduler.Load().(Scheduler) }
+
+// Scheduler the VM scheduler
+type Scheduler interface {
+ // Get the VM
+ Get() (VM, error)
+ // Shrink the available VM
+ Shrink()
+ // Loader the ModuleLoader
+ Loader() ModuleLoader
+ // Close the scheduler
+ Close() error
+}
+
+// SchedulerOptions options
+type SchedulerOptions struct {
+ InitialVMs uint `yaml:"initial-vms" json:"initialVMs"`
+ MaxVMs uint `yaml:"max-vms" json:"maxVMs"`
+ MaxRetriesGetVM uint `yaml:"max-retries-get-vm" json:"maxRetriesGetVM"`
+ MaxTimeToWaitGetVM time.Duration `yaml:"max-time-to-wait-get-vm" json:"maxTimeToWaitGetVM"`
+ Loader ModuleLoader `yaml:"-"` // module loader
+ VMOptions []Option `yaml:"-"` // options for NewVM
+}
+
+// NewScheduler returns a new Scheduler
+func NewScheduler(opt SchedulerOptions) Scheduler {
+ s := &schedulerImpl{
+ closed: new(atomic.Bool),
+ unInitVMs: new(atomic.Int32),
+ maxVMs: opt.MaxVMs,
+ maxRetriesGetVM: opt.MaxRetriesGetVM,
+ maxTimeToWaitGetVM: opt.MaxTimeToWaitGetVM,
+ loader: opt.Loader,
+ }
+ if s.maxVMs == 0 {
+ s.maxVMs = 1
+ }
+ if s.maxRetriesGetVM == 0 {
+ s.maxRetriesGetVM = DefaultMaxRetriesGetVM
+ }
+ if s.maxTimeToWaitGetVM == 0 {
+ s.maxTimeToWaitGetVM = DefaultMaxTimeToWaitGetVM
+ }
+ if s.loader == nil {
+ slog.Warn("js.ModuleLoader not provided, require and module will not working")
+ s.loader = emptyLoader{}
+ }
+ s.maxVMs = max(s.maxVMs, opt.InitialVMs)
+ s.vms = make(chan VM, s.maxVMs)
+ s.vmOpt = append(opt.VMOptions,
+ func(vm *vmImpl) {
+ vm.release = func() { s.release(vm) }
+ }, WithModuleLoader(opt.Loader))
+ for i := uint(0); i < opt.InitialVMs; i++ {
+ s.vms <- NewVM(s.vmOpt...)
+ }
+ s.unInitVMs.Store(int32(s.maxVMs - opt.InitialVMs))
+ return s
+}
+
+type schedulerImpl struct {
+ vms chan VM
+ maxVMs, maxRetriesGetVM uint
+ unInitVMs *atomic.Int32
+ closed *atomic.Bool
+ maxTimeToWaitGetVM time.Duration
+ loader ModuleLoader
+ vmOpt []Option
+}
+
+func (s *schedulerImpl) Loader() ModuleLoader { return s.loader }
+
+func (s *schedulerImpl) String() string {
+ text, _ := s.MarshalText()
+ return string(text)
+}
+
+func (s *schedulerImpl) MarshalText() ([]byte, error) {
+ return json.Marshal(map[string]any{
+ "available": len(s.vms),
+ "max": int(s.maxVMs),
+ "unInit": int(s.unInitVMs.Load()),
+ })
+}
+
+// Close the scheduler
+func (s *schedulerImpl) Close() error {
+ s.closed.Store(true)
+ close(s.vms)
+ return nil
+}
+
+// Get the VM
+func (s *schedulerImpl) Get() (VM, error) {
+ if s.unInitVMs.CompareAndSwap(int32(s.maxVMs), int32(s.maxVMs-1)) {
+ return NewVM(s.vmOpt...), nil
+ }
+
+ timer := time.NewTimer(s.maxTimeToWaitGetVM)
+
+ defer timer.Stop()
+
+ for i := uint(1); i <= s.maxRetriesGetVM; i++ {
+ select {
+ case vm, ok := <-s.vms:
+ if !ok {
+ return nil, ErrSchedulerClosed
+ }
+ return vm, nil
+ case <-timer.C:
+ if s.unInitVMs.Add(-1) >= 0 {
+ return NewVM(s.vmOpt...), nil
+ }
+ s.unInitVMs.Add(1)
+ slog.Warn(fmt.Sprintf("could not get VM in %v", time.Duration(i)*s.maxTimeToWaitGetVM))
+ timer.Reset(s.maxTimeToWaitGetVM)
+ }
+ }
+ return nil, fmt.Errorf("could not get VM in %v",
+ time.Duration(s.maxRetriesGetVM)*s.maxTimeToWaitGetVM)
+}
+
+func (s *schedulerImpl) Shrink() {
+ if len(s.vms) == 0 {
+ return
+ }
+ s.unInitVMs.Store(int32(s.maxVMs))
+ for i := 0; i <= len(s.vms); i++ {
+ _ = <-s.vms
+ }
+}
+
+// Release the VM
+func (s *schedulerImpl) release(vm VM) {
+ if s.closed.Load() {
+ return
+ }
+
+ s.vms <- vm
+}
diff --git a/js/scheduler_test.go b/js/scheduler_test.go
new file mode 100644
index 0000000..943d0a9
--- /dev/null
+++ b/js/scheduler_test.go
@@ -0,0 +1,61 @@
+package js
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestScheduler(t *testing.T) {
+ scheduler := NewScheduler(SchedulerOptions{InitialVMs: 2, MaxVMs: 4})
+ goroutineNum := 12
+ blockNum := 4
+ wg := new(sync.WaitGroup)
+
+ for i := 1; i <= goroutineNum; i++ {
+ wg.Add(1)
+ go func(i int) {
+ defer wg.Done()
+ timeout := time.Millisecond * 400
+ script := "1"
+ if i < blockNum {
+ script = `while(true){}`
+ timeout *= 2
+ }
+
+ vm, err := scheduler.Get()
+ if err != nil {
+ t.Errorf("scheduler %v: %v", i, err)
+ return
+ }
+
+ ctx, cancel := context.WithTimeout(context.Background(), timeout)
+ defer cancel()
+
+ vm.Run(ctx, func() {
+ _, err := vm.Runtime().RunString(script)
+ if err != nil && !errors.Is(err, context.DeadlineExceeded) && !errors.Is(err, context.Canceled) {
+ t.Errorf("run string %v: %v", i, err)
+ }
+ })
+ }(i)
+ }
+ wg.Wait()
+}
+
+func TestSchedulerShrink(t *testing.T) {
+ scheduler := NewScheduler(SchedulerOptions{InitialVMs: 2, MaxVMs: 4})
+ scheduler.Shrink()
+ assert.Equal(t, `{"available":0,"max":4,"unInit":4}`, scheduler.(fmt.Stringer).String())
+ start := time.Now()
+ _, _ = scheduler.Get()
+ _, _ = scheduler.Get()
+ took := time.Since(start)
+ assert.Equal(t, `{"available":0,"max":4,"unInit":2}`, scheduler.(fmt.Stringer).String())
+ assert.True(t, took < time.Millisecond*600)
+}
diff --git a/js/type.go b/js/type.go
deleted file mode 100644
index 854fc44..0000000
--- a/js/type.go
+++ /dev/null
@@ -1,33 +0,0 @@
-package js
-
-import (
- "reflect"
- "strings"
-)
-
-// Program The js program
-type Program struct {
- Code string
- Args map[string]any
-}
-
-// FieldNameMapper provides custom mapping between Go and JavaScript property names.
-type FieldNameMapper struct{}
-
-// FieldName returns a JavaScript name for the given struct field in the given type.
-// If this method returns "" the field becomes hidden.
-func (FieldNameMapper) FieldName(_ reflect.Type, f reflect.StructField) string {
- if v, ok := f.Tag.Lookup("js"); ok {
- if v == "-" {
- return ""
- }
- return v
- }
- return strings.ToLower(f.Name[0:1]) + f.Name[1:]
-}
-
-// MethodName returns a JavaScript name for the given method in the given type.
-// If this method returns "" the method becomes hidden.
-func (FieldNameMapper) MethodName(_ reflect.Type, m reflect.Method) string {
- return strings.ToLower(m.Name[0:1]) + m.Name[1:]
-}
diff --git a/js/utils.go b/js/utils.go
index 80ada2f..e2e864a 100644
--- a/js/utils.go
+++ b/js/utils.go
@@ -4,21 +4,20 @@ import (
"context"
"errors"
"fmt"
+ "log/slog"
+ "reflect"
+ "strings"
- "github.com/dop251/goja"
- "github.com/shiroyk/cloudcat/plugin/jsmodule"
- "github.com/spf13/cast"
+ "github.com/grafana/sobek"
)
-// vmContextKey the VM current context
-var vmContextKey = goja.NewSymbol("__ctx__")
-
// Throw js exception
-func Throw(vm *goja.Runtime, err error) {
- if e, ok := err.(*goja.Exception); ok { //nolint:errorlint
- panic(e)
+func Throw(rt *sobek.Runtime, err error) {
+ var ex *sobek.Exception
+ if errors.As(err, &ex) { //nolint:errorlint
+ panic(ex)
}
- panic(vm.ToValue(err))
+ panic(rt.ToValue(err))
}
// ToBytes tries to return a byte slice from compatible types.
@@ -28,46 +27,28 @@ func ToBytes(data any) ([]byte, error) {
return dt, nil
case string:
return []byte(dt), nil
- case goja.ArrayBuffer:
+ case sobek.ArrayBuffer:
return dt.Bytes(), nil
default:
- return nil, fmt.Errorf("invalid type %T, expected string, []byte or ArrayBuffer", data)
- }
-}
-
-// ToStrings tries to return a string slice or string from compatible types.
-func ToStrings(data any) (s any, err error) {
- switch dt := data.(type) {
- case string:
- return dt, nil
- case []string:
- return dt, nil
- case []byte:
- return string(dt), nil
- case []any:
- return cast.ToStringSliceE(dt)
- case goja.ArrayBuffer:
- return string(dt.Bytes()), nil
- default:
- return nil, fmt.Errorf("invalid type %T, expected string, string array or ArrayBuffer", data)
+ return nil, fmt.Errorf("expected string, []byte or ArrayBuffer, but got %T, ", data)
}
}
-// Unwrap the goja.Value to the raw value
-func Unwrap(value goja.Value) (any, error) {
+// Unwrap the sobek.Value to the raw value
+func Unwrap(value sobek.Value) (any, error) {
if value == nil {
return nil, nil
}
switch v := value.Export().(type) {
default:
return v, nil
- case goja.ArrayBuffer:
+ case sobek.ArrayBuffer:
return v.Bytes(), nil
- case *goja.Promise:
+ case *sobek.Promise:
switch v.State() {
- case goja.PromiseStateRejected:
+ case sobek.PromiseStateRejected:
return nil, errors.New(v.Result().String())
- case goja.PromiseStateFulfilled:
+ case sobek.PromiseStateFulfilled:
return v.Result().Export(), nil
default:
return nil, errors.New("unexpected promise pending state")
@@ -75,23 +56,83 @@ func Unwrap(value goja.Value) (any, error) {
}
}
-// VMContext returns the current context of the goja.Runtime
-func VMContext(runtime *goja.Runtime) context.Context {
- ctx := context.Background()
- if v := runtime.GlobalObject().GetSymbol(vmContextKey); v != nil {
- if c, ok := v.Export().(context.Context); ok {
- ctx = c
+// ModuleCallable return the sobek.CyclicModuleRecord default export as sobek.Callable.
+func ModuleCallable(rt *sobek.Runtime, resolve sobek.HostResolveImportedModuleFunc, module sobek.CyclicModuleRecord) (sobek.Callable, error) {
+ instance := rt.GetModuleInstance(module)
+ if instance == nil {
+ if err := module.Link(); err != nil {
+ return nil, err
}
+ promise := rt.CyclicModuleRecordEvaluate(module, resolve)
+ switch promise.State() {
+ case sobek.PromiseStateRejected:
+ return nil, promise.Result().Export().(error)
+ case sobek.PromiseStateFulfilled:
+ default:
+ }
+ instance = rt.GetModuleInstance(module)
+ }
+ value := instance.GetBindingValue("default")
+ call, ok := sobek.AssertFunction(value)
+ if !ok {
+ return nil, errors.New("module default export is not a function")
}
- return ctx
+ return call, nil
}
-// InitGlobalModule init all global modules
-func InitGlobalModule(runtime *goja.Runtime) {
- // Init global modules
- for _, extension := range jsmodule.AllModules() {
- if mod, ok := extension.Module.(jsmodule.Global); ok {
- _ = runtime.Set(extension.Name, mod.Exports())
+// Context returns the current context of the sobek.Runtime
+func Context(rt *sobek.Runtime) context.Context {
+ if v := self(rt).ctx.Export().(*vmctx).ctx; v != nil {
+ return v
+ }
+ return context.Background()
+}
+
+// OnDone add a function to execute when the VM has finished running.
+// eg: close resources...
+func OnDone(rt *sobek.Runtime, job func()) { self(rt).eventloop.OnDone(job) }
+
+// InitGlobalModule init all implement the Global modules
+func InitGlobalModule(rt *sobek.Runtime) {
+ for name, mod := range AllModule() {
+ if mod, ok := mod.(Global); ok {
+ instance, err := mod.Instantiate(rt)
+ if err != nil {
+ slog.Warn(fmt.Sprintf("instantiate global js module %s failed: %s", name, err))
+ continue
+ }
+ _ = rt.Set(name, instance)
}
}
}
+
+func FreezeObject(rt *sobek.Runtime, obj sobek.Value) error {
+ global := rt.GlobalObject().Get("Object").ToObject(rt)
+ freeze, ok := sobek.AssertFunction(global.Get("freeze"))
+ if !ok {
+ panic("failed to get the Object.freeze function from the runtime")
+ }
+ _, err := freeze(sobek.Undefined(), obj)
+ return err
+}
+
+// FieldNameMapper provides custom mapping between Go and JavaScript property names.
+type FieldNameMapper struct{}
+
+// FieldName returns a JavaScript name for the given struct field in the given type.
+// If this method returns "" the field becomes hidden.
+func (FieldNameMapper) FieldName(_ reflect.Type, f reflect.StructField) string {
+ if v, ok := f.Tag.Lookup("js"); ok {
+ if v == "-" {
+ return ""
+ }
+ return v
+ }
+ return strings.ToLower(f.Name[0:1]) + f.Name[1:]
+}
+
+// MethodName returns a JavaScript name for the given method in the given type.
+// If this method returns "" the method becomes hidden.
+func (FieldNameMapper) MethodName(_ reflect.Type, m reflect.Method) string {
+ return strings.ToLower(m.Name[0:1]) + m.Name[1:]
+}
diff --git a/js/vm.go b/js/vm.go
index 9e15701..255d0f5 100644
--- a/js/vm.go
+++ b/js/vm.go
@@ -3,148 +3,265 @@ package js
import (
"bytes"
"context"
- "errors"
"fmt"
"log/slog"
+ "reflect"
"runtime/debug"
- "github.com/dop251/goja"
- "github.com/shiroyk/cloudcat/plugin"
+ "github.com/grafana/sobek"
+ "github.com/shiroyk/ski"
)
-var errInitExecutor = errors.New("initializing JavaScript VM executor failed")
-
// VM the js runtime.
// An instance of VM can only be used by a single goroutine at a time.
type VM interface {
- // Run the js program
- Run(context.Context, Program) (goja.Value, error)
- // RunString the js string
- RunString(context.Context, string) (goja.Value, error)
- // Runtime the js runtime
- Runtime() *goja.Runtime
+ // RunModule run the sobek.CyclicModuleRecord.
+ // To compile the module, sobek.ParseModule or ModuleLoader.CompileModule
+ RunModule(ctx context.Context, module sobek.CyclicModuleRecord) (sobek.Value, error)
+ // Run execute the given function in the EventLoop.
+ // when context done interrupt VM execution and release the VM.
+ // This is usually used when needs to be called repeatedly many times.
+ // like this:
+ //
+ // func main() {
+ // scheduler := js.NewScheduler(js.SchedulerOptions{
+ // InitialVMs: 2,
+ // Loader: js.NewModuleLoader(),
+ // })
+ // run := func(ctx context.Context, scheduler js.Scheduler) int64 {
+ // vm, err := scheduler.Get()
+ // if err != nil {
+ // panic(err)
+ // }
+ // rt := vm.Runtime()
+ //
+ // module, err := scheduler.Loader().CompileModule("sum", "module.exports = (a, b) => a + b")
+ // if err != nil {
+ // panic(module)
+ // }
+ //
+ // sum, err := js.ModuleCallable(rt, module)
+ // if err != nil {
+ // panic(err)
+ // }
+ //
+ // var total int64
+ // vm.Run(ctx, func() {
+ // for i := 0; i < 8; i++ {
+ // v, err := sum(sobek.Undefined(), rt.ToValue(i), rt.ToValue(total))
+ // if err != nil {
+ // panic(err)
+ // }
+ // total = v.ToInteger()
+ // }
+ // })
+ //
+ // return total
+ // }
+ //
+ // ctx, cancel := context.WithTimeout(context.Background(), time.Second*30)
+ // defer cancel()
+ //
+ // fmt.Println(run(ctx, scheduler))
+ // }
+ Run(context.Context, func())
+ // Context return the ja context object of NewContext
+ Context() sobek.Value
+ // Loader return the ModuleLoader
+ Loader() ModuleLoader
+ // Runtime return the js runtime
+ Runtime() *sobek.Runtime
}
-type vmImpl struct {
- runtime *goja.Runtime
- eventloop *EventLoop
- executor goja.Callable
- done chan struct{}
+type Option func(*vmImpl)
+
+// WithInitial call sobek.Runtime on VM create, be care require and module not working when init.
+func WithInitial(fn func(*sobek.Runtime)) Option {
+ return func(vm *vmImpl) { fn(vm.runtime) }
+}
+
+// WithModuleLoader set a ModuleLoader, if not present require and es module will not work.
+func WithModuleLoader(loader ModuleLoader) Option {
+ return func(vm *vmImpl) { vm.loader = loader }
}
// NewVM creates a new JavaScript VM
-// Initialize the EventLoop, require, global module, console
-func NewVM(modulePath ...string) VM {
- runtime := goja.New()
- runtime.SetFieldNameMapper(FieldNameMapper{})
- EnableRequire(runtime, modulePath...)
- InitGlobalModule(runtime)
- EnableConsole(runtime)
-
- // TODO: any better way?
- eval := `(function(ctx, code){with(ctx){return eval(code)}})`
- program := goja.MustCompile("eval", eval, false)
- callable, err := runtime.RunProgram(program)
- if err != nil {
- panic(errInitExecutor)
+// Initialize the EventLoop, global module, console.
+func NewVM(opts ...Option) VM {
+ rt := sobek.New()
+ rt.SetFieldNameMapper(FieldNameMapper{})
+ EnableConsole(rt)
+ InitGlobalModule(rt)
+ vm := &vmImpl{
+ runtime: rt,
+ eventloop: NewEventLoop(),
+ ctx: NewContext(context.Background(), rt),
}
- executor, ok := goja.AssertFunction(callable)
- if !ok {
- panic(errInitExecutor)
+ for _, opt := range opts {
+ opt(vm)
+ }
+ if vm.release == nil {
+ vm.release = func() {}
+ }
+ if vm.loader == nil {
+ vm.loader = new(emptyLoader)
}
- //keys, _ := runtime.RunString("Object.keys(this)")
- //globalKeys := cast.ToStringSlice(keys.Export())
+ vm.loader.EnableRequire(rt).EnableImportModuleDynamically(rt)
+ _ = rt.GlobalObject().SetSymbol(symbolVM, &vmself{vm})
- return &vmImpl{
- runtime,
- NewEventLoop(runtime),
- executor,
- make(chan struct{}, 1),
+ return vm
+}
+
+type (
+ vmImpl struct {
+ runtime *sobek.Runtime
+ eventloop *EventLoop
+ ctx sobek.Value
+ release func()
+ loader ModuleLoader
}
+
+ vmctx struct{ ctx context.Context }
+
+ vmself struct{ vm *vmImpl }
+)
+
+// Loader return the ModuleLoader
+func (vm *vmImpl) Loader() ModuleLoader { return vm.loader }
+
+// Runtime return the js runtime
+func (vm *vmImpl) Runtime() *sobek.Runtime { return vm.runtime }
+
+func (vm *vmImpl) Context() sobek.Value { return vm.ctx }
+
+// RunModule run the sobek.CyclicModuleRecord.
+// The module default export must be a function.
+func (vm *vmImpl) RunModule(ctx context.Context, module sobek.CyclicModuleRecord) (ret sobek.Value, err error) {
+ vm.Run(ctx, func() {
+ var call sobek.Callable
+ call, err = ModuleCallable(vm.runtime, vm.loader.ResolveModule, module)
+ if err != nil {
+ return
+ }
+ ret, err = call(sobek.Undefined(), vm.ctx)
+ })
+ return
}
-// Run the js program
-func (vm *vmImpl) Run(ctx context.Context, p Program) (ret goja.Value, err error) {
- // resets the interrupt flag.
- vm.runtime.ClearInterrupt()
+// Run execute the given function in the EventLoop.
+// when context done interrupt VM execution and release the VM.
+// This is usually used when needs to be called repeatedly many times.
+// like this:
+//
+// func main() {
+// scheduler := js.NewScheduler(js.SchedulerOptions{
+// InitialVMs: 2,
+// Loader: js.NewModuleLoader(),
+// })
+// run := func(ctx context.Context, scheduler js.Scheduler) int64 {
+// vm, err := scheduler.Get()
+// if err != nil {
+// panic(err)
+// }
+// rt := vm.Runtime()
+//
+// module, err := scheduler.Loader().CompileModule("sum", "module.exports = (a, b) => a + b")
+// if err != nil {
+// panic(module)
+// }
+//
+// sum, err := js.ModuleCallable(rt, module)
+// if err != nil {
+// panic(err)
+// }
+//
+// var total int64
+// vm.Run(ctx, func() {
+// for i := 0; i < 8; i++ {
+// v, err := sum(sobek.Undefined(), rt.ToValue(i), rt.ToValue(total))
+// if err != nil {
+// panic(err)
+// }
+// total = v.ToInteger()
+// }
+// })
+//
+// return total
+// }
+//
+// ctx, cancel := context.WithTimeout(context.Background(), time.Second*30)
+// defer cancel()
+//
+// fmt.Println(run(ctx, scheduler))
+// }
+func (vm *vmImpl) Run(ctx context.Context, task func()) {
defer func() {
- vm.eventloop.WaitOnRegistered()
-
- if r := recover(); r != nil {
+ if x := recover(); x != nil {
stack := vm.runtime.CaptureCallStack(20, nil)
buf := new(bytes.Buffer)
for _, frame := range stack {
frame.Write(buf)
}
- slog.Error(fmt.Sprintf("vm run error %s", r),
- "stack", string(debug.Stack()), "js stack", buf.String())
+ ski.Logger(ctx).Error(fmt.Sprintf("vm run error: %s", x),
+ slog.String("go_stack", string(debug.Stack())),
+ slog.String("js_stack", buf.String()))
}
-
- _ = vm.runtime.GlobalObject().DeleteSymbol(vmContextKey)
- vm.done <- struct{}{} // End of run
+ vm.ctx.Export().(*vmctx).ctx = context.Background()
+ vm.release()
}()
+ // resets the interrupt flag.
+ vm.runtime.ClearInterrupt()
+ vm.ctx.Export().(*vmctx).ctx = ctx
go func() {
select {
case <-ctx.Done():
- // Interrupt running JavaScript.
+ // interrupt the running JavaScript.
vm.runtime.Interrupt(ctx.Err())
- // Release vm
- GetScheduler().Release(vm)
- return
- case <-vm.done:
- // Release vm
- GetScheduler().Release(vm)
+ // stop the event loop.
+ vm.eventloop.Stop()
return
}
}()
- args := p.Args
- if args == nil {
- args = make(map[string]any, 1)
- }
- if ctx, ok := ctx.(*plugin.Context); ok {
- args["cat"] = NewCtxWrapper(vm, ctx)
- }
- _ = vm.runtime.GlobalObject().SetSymbol(vmContextKey, ctx)
-
- err = vm.eventloop.Start(func() error {
- ret, err = vm.executor(goja.Undefined(), vm.runtime.ToValue(args), vm.runtime.ToValue(p.Code))
- return err
- })
-
- return
+ vm.eventloop.Start(task)
}
-// RunString the js string
-func (vm *vmImpl) RunString(ctx context.Context, s string) (goja.Value, error) {
- return vm.Run(ctx, Program{Code: s})
-}
-
-// Runtime the js runtime
-func (vm *vmImpl) Runtime() *goja.Runtime { return vm.runtime }
-
-// NewPromise returns the new promise with the async function.
-// must be called on the EventLoop.
+// NewPromise return a sobek.Promise object.
+// The second argument is a long-running asynchronous task that will be executed in a child goroutine.
+// The third optional argument is a callback that will be executed in the main goroutine.
+// Additional arguments will be ignored.
// like this:
//
// func main() {
+// server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+// w.WriteHeader(http.StatusOK)
+// _, _ = w.Write([]byte(`{"foo":"bar"}`))
+// }))
+// defer server.Close()
+//
// vm := js.NewVM()
// ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
// defer cancel()
//
-// goFunc := func(call goja.FunctionCall, rt *goja.Runtime) goja.Value {
-// return rt.ToValue(js.NewPromise(rt, func() (any, error) {
-// time.Sleep(time.Second)
-// return max(call.Argument(0).ToInteger(), call.Argument(1).ToInteger()), nil
-// }))
+// fetch := func(call sobek.FunctionCall, rt *sobek.Runtime) sobek.Value {
+// return rt.ToValue(js.NewPromise(rt,
+// func() (*http.Response, error) { return http.Get(call.Argument(0).String()) },
+// func(res *http.Response, err error) (any, error) {
+// defer res.Body.Close()
+// data, err := io.ReadAll(res.Body)
+// if err != nil {
+// return nil, err
+// }
+// return string(data), nil
+// }))
// }
-// _ = vm.Runtime().Set("max", goFunc)
+// _ = vm.Runtime().Set("fetch", fetch)
//
// start := time.Now()
//
-// result, err := vm.RunString(ctx, `max(1, 2)`)
+// result, err := vm.Runtime().RunString(ctx, fmt.Sprintf(`fetch("%s")`, server.URL))
// if err != nil {
// panic(err)
// }
@@ -154,21 +271,24 @@ func (vm *vmImpl) Runtime() *goja.Runtime { return vm.runtime }
// }
//
// fmt.Println(value)
-// fmt.Println(time.Now().Sub(start))
+// fmt.Println(time.Since(start))
// }
-func NewPromise[T any](runtime *goja.Runtime, async func() (T, error), then ...func(T, error) (any, error)) *goja.Promise {
- callback := NewEnqueueCallback(runtime)
+func NewPromise[T any](runtime *sobek.Runtime, async func() (T, error), then ...func(T, error) (any, error)) *sobek.Promise {
+ enqueue := self(runtime).eventloop.EnqueueJob()
promise, resolve, reject := runtime.NewPromise()
- thenFun := func(r T, e error) (any, error) {
- return r, e
- }
+ thenFun := func(r T, e error) (any, error) { return r, e }
if len(then) > 0 {
thenFun = then[0]
}
go func() {
+ defer func() {
+ if x := recover(); x != nil {
+ reject(x)
+ }
+ }()
result, err := async()
- callback(func() error {
+ enqueue(func() {
var value any = result
value, err = thenFun(result, err)
if err != nil {
@@ -176,40 +296,60 @@ func NewPromise[T any](runtime *goja.Runtime, async func() (T, error), then ...f
} else {
resolve(value)
}
- return nil
})
}()
return promise
}
-// NewEnqueueCallback signals to the event loop that you are going to do some
-// asynchronous work off the main thread and that you may need to execute some
-// code back on the main thread when you are done.
-// see EventLoop.RegisterCallback.
-//
-// func doAsyncWork(runtime *goja.Runtime) *goja.Promise {
-// enqueueCallback := js.NewEnqueueCallback(runtime)
-// promise, resolve, reject := runtime.NewPromise()
-//
-// // Do the actual async work in a new independent goroutine, but make
-// // sure that the Promise resolution is done on the main thread:
-//
-// go func() {
-// // Also make sure to abort early if the context is cancelled, so
-// // the VM is not stuck when the scenario ends or Ctrl+C is used:
-// result, err := doTheActualAsyncWork()
-// enqueueCallback(func() error {
-// if err != nil {
-// reject(err)
-// } else {
-// resolve(result)
-// }
-// return nil // do not abort the iteration
-// })
-// }()
-// return promise
-// }
-func NewEnqueueCallback(runtime *goja.Runtime) EnqueueCallback {
- return runtime.GlobalObject().GetSymbol(enqueueCallbackSymbol).Export().(func() EnqueueCallback)()
+var (
+ reflectTypeCtx = reflect.TypeOf((*vmctx)(nil))
+ reflectTypeVmself = reflect.TypeOf((*vmself)(nil))
+ symbolVM = sobek.NewSymbol("Symbol.__vm__")
+)
+
+// NewContext create the js context object
+func NewContext(ctx context.Context, rt *sobek.Runtime) *sobek.Object {
+ ret := rt.ToValue(&vmctx{ctx}).ToObject(rt)
+ proto := rt.NewObject()
+ _ = ret.SetPrototype(proto)
+ err := FreezeObject(rt, ret)
+ if err != nil {
+ panic(err)
+ }
+
+ _ = proto.Set("get", func(call sobek.FunctionCall) sobek.Value {
+ return rt.ToValue(toCtx(rt, call.This).Value(call.Argument(0).Export()))
+ })
+ _ = proto.Set("set", func(call sobek.FunctionCall) sobek.Value {
+ e := toCtx(rt, call.This)
+ if c, ok := e.(ski.Context); ok {
+ c.SetValue(call.Argument(0).Export(), call.Argument(1).Export())
+ return rt.ToValue(true)
+ }
+ return rt.ToValue(false)
+ })
+ _ = proto.Set("toString", func(call sobek.FunctionCall) sobek.Value {
+ return rt.ToValue("[context]")
+ })
+ return ret
+}
+
+func toCtx(rt *sobek.Runtime, v sobek.Value) context.Context {
+ if v.ExportType() == reflectTypeCtx {
+ if u := v.Export().(*vmctx); u != nil && u.ctx != nil {
+ return u.ctx
+ }
+ }
+ panic(rt.NewTypeError(`value of "this" must be of type vmctx`))
+}
+
+// self get VM self
+func self(rt *sobek.Runtime) *vmImpl {
+ value := rt.GlobalObject().GetSymbol(symbolVM)
+ if value.ExportType() == reflectTypeVmself {
+ return value.Export().(*vmself).vm
+ }
+ panic(rt.NewTypeError(`symbol value of "VM" must be of type vmself, ` +
+ `this shouldn't happen, maybe not call from VM.Runtime`))
}
diff --git a/js/vm_test.go b/js/vm_test.go
index 1862af3..3effa01 100644
--- a/js/vm_test.go
+++ b/js/vm_test.go
@@ -1,41 +1,54 @@
package js
import (
+ "bytes"
"context"
- "errors"
+ "log/slog"
"testing"
"time"
+ _ "unsafe"
- "github.com/dop251/goja"
- "github.com/shiroyk/cloudcat/plugin"
+ "github.com/grafana/sobek"
+ "github.com/shiroyk/ski"
"github.com/stretchr/testify/assert"
)
-func TestVMRunString(t *testing.T) {
+func TestVMContext(t *testing.T) {
t.Parallel()
- vm := NewTestVM(t)
+ ctx := context.WithValue(context.Background(), "foo", "bar")
+ vm := NewVM(WithModuleLoader(NewModuleLoader()))
+
+ v, err := runMod(ctx, vm, `module.exports = (ctx) => ctx.get('foo')`)
+ if assert.NoError(t, err) {
+ assert.Equal(t, "bar", v.Export())
+ assert.Equal(t, context.Background(), Context(vm.Runtime()))
+ }
+}
+
+func TestVMRunModule(t *testing.T) {
+ t.Parallel()
+ vm := NewVM()
testCases := []struct {
script string
want any
}{
- {"2", 2},
- {"let a = 1; a + 2", 3},
- {"(() => 4)()", 4},
- {"[5]", []any{int64(5)}},
- {"let a = {'key':'foo'}; a", map[string]any{"key": "foo"}},
- {"JSON.stringify({'key':7})", `{"key":7}`},
- {"JSON.stringify([8])", `[8]`},
- {"(async () => 9)()", 9},
+ {"export default () => 1", 1},
+ {"export default function () {let a = 1; return a + 1}", 2},
+ {"export default async () => 3", 3},
+ {"const a = async () => 5; let b = await a(); export default () => b - 1", 4},
+ {"export default () => 3 + 2", 5},
}
for _, c := range testCases {
t.Run(c.script, func(t *testing.T) {
- v, err := vm.RunString(context.Background(), c.script)
- assert.NoError(t, err)
- vv, err := Unwrap(v)
- assert.NoError(t, err)
- assert.EqualValues(t, c.want, vv)
+ v, err := runMod(context.Background(), vm, c.script)
+ if assert.NoError(t, err) {
+ vv, err := Unwrap(v)
+ if assert.NoError(t, err) {
+ assert.EqualValues(t, c.want, vv)
+ }
+ }
})
}
}
@@ -45,45 +58,32 @@ func TestTimeout(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*200)
defer cancel()
- _, err := NewTestVM(t).RunString(ctx, `while(true){}`)
+ start := time.Now()
+ _, err := runMod(ctx, NewVM(), "export default () => {while(true){}}")
+ took := time.Since(start)
assert.ErrorIs(t, err, context.DeadlineExceeded)
+ assert.Greater(t, time.Millisecond*300, took)
}
-func TestVMRunWithContext(t *testing.T) {
+func TestWithInitial(t *testing.T) {
t.Parallel()
- {
- vm := NewTestVM(t)
- ctx, cancel := context.WithTimeout(context.Background(), time.Second)
- defer cancel()
- _ = vm.Runtime().Set("testContext", func(call goja.FunctionCall, vm *goja.Runtime) goja.Value {
- return vm.ToValue(VMContext(vm))
- })
- v, err := vm.RunString(ctx, "testContext()")
- assert.NoError(t, err)
- assert.Equal(t, ctx, v.Export())
- assert.Equal(t, context.Background(), VMContext(vm.Runtime()))
- }
- {
- vm := NewTestVM(t)
- ctx := plugin.NewContext(plugin.ContextOptions{})
- _ = vm.Runtime().Set("testContext", func(call goja.FunctionCall, vm *goja.Runtime) goja.Value {
- return vm.ToValue(VMContext(vm))
- })
- v, err := vm.RunString(ctx, "testContext()")
- assert.NoError(t, err)
- assert.Equal(t, ctx, v.Export())
- assert.Equal(t, context.Background(), VMContext(vm.Runtime()))
+ vm := NewVM(WithInitial(func(rt *sobek.Runtime) {
+ _ = rt.Set("init", true)
+ }))
+ v, err := runMod(context.Background(), vm, `export default () => init`)
+ if assert.NoError(t, err) {
+ assert.Equal(t, true, v.Export())
}
}
func TestNewPromise(t *testing.T) {
t.Parallel()
- vm := NewTestVM(t)
+ vm := NewVM()
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
defer cancel()
- goFunc := func(call goja.FunctionCall, rt *goja.Runtime) goja.Value {
+ goFunc := func(call sobek.FunctionCall, rt *sobek.Runtime) sobek.Value {
return rt.ToValue(NewPromise(rt, func() (any, error) {
time.Sleep(time.Second)
return max(call.Argument(0).ToInteger(), call.Argument(1).ToInteger()), nil
@@ -93,51 +93,53 @@ func TestNewPromise(t *testing.T) {
start := time.Now()
- result, err := vm.RunString(ctx, `max(1, 2)`)
- if err != nil {
- assert.NoError(t, err)
- }
- value, err := Unwrap(result)
- if err != nil {
- assert.NoError(t, err)
+ result, err := runMod(ctx, vm, `export default () => max(1, 2)`)
+ if assert.NoError(t, err) {
+ value, err := Unwrap(result)
+ if assert.NoError(t, err) {
+ assert.EqualValues(t, 2, value)
+ assert.EqualValues(t, 1, int(time.Now().Sub(start).Seconds()))
+ }
}
- assert.EqualValues(t, 2, value)
- assert.EqualValues(t, 1, int(time.Now().Sub(start).Seconds()))
}
-func NewTestVM(t *testing.T) VM {
- vm := NewVM()
+type testScheduler struct{ vm VM }
- assertObject := vm.Runtime().NewObject()
- _ = assertObject.Set("equal", func(call goja.FunctionCall, vm *goja.Runtime) (ret goja.Value) {
- a, err := Unwrap(call.Argument(0))
- if err != nil {
- Throw(vm, err)
- }
- b, err := Unwrap(call.Argument(1))
- if err != nil {
- Throw(vm, err)
- }
- var msg string
- if !goja.IsUndefined(call.Argument(2)) {
- msg = call.Argument(2).String()
- }
- if !assert.Equal(t, b, a, msg) {
- Throw(vm, errors.New("not equal"))
- }
- return
+func (t *testScheduler) release(vm VM) { t.vm = vm }
+func (*testScheduler) Get() (VM, error) { return nil, nil }
+func (*testScheduler) Close() error { return nil }
+
+func TestVMPanic(t *testing.T) {
+ t.Parallel()
+ scheduler := new(testScheduler)
+ vm := NewVM(func(vm *vmImpl) {
+ vm.release = func() { scheduler.release(vm) }
})
- _ = assertObject.Set("true", func(call goja.FunctionCall, vm *goja.Runtime) (ret goja.Value) {
- var msg string
- if !goja.IsUndefined(call.Argument(1)) {
- msg = call.Argument(1).String()
- }
- if !assert.True(t, call.Argument(0).ToBoolean(), msg) {
- Throw(vm, errors.New("should be true"))
- }
- return
+
+ ctx, cancel := context.WithTimeout(ski.NewContext(context.Background(), nil), time.Second)
+ defer cancel()
+
+ log := new(bytes.Buffer)
+
+ logger := slog.New(slog.NewTextHandler(log, nil))
+
+ _ = vm.Runtime().Set("some", func() {
+ OnDone(vm.Runtime(), func() { assert.Equal(t, Context(vm.Runtime()), ctx) })
+ OnDone(vm.Runtime(), func() { panic("some panic") })
})
- _ = vm.Runtime().Set("assert", assertObject)
+ _, err := runMod(ski.WithLogger(ctx, logger), vm, `export default () => {some(); (() => other.error)()}`)
+ if assert.Error(t, err) {
+ assert.ErrorContains(t, err, "other is not defined")
+ assert.NotNil(t, scheduler.vm)
+ assert.Equal(t, context.Background(), Context(vm.Runtime()))
+ assert.Contains(t, log.String(), "vm run error: some panic")
+ }
+}
- return vm
+func runMod(ctx context.Context, vm VM, script string) (sobek.Value, error) {
+ mod, err := vm.Loader().CompileModule("", script)
+ if err != nil {
+ return nil, err
+ }
+ return vm.RunModule(ctx, mod)
}
diff --git a/parsers/gq/bench_gq_test.go b/parsers/gq/bench_gq_test.go
deleted file mode 100644
index 6bf3d22..0000000
--- a/parsers/gq/bench_gq_test.go
+++ /dev/null
@@ -1,16 +0,0 @@
-package gq
-
-import (
- "testing"
-)
-
-func BenchmarkParser(b *testing.B) {
- b.StartTimer()
- for i := 0; i < b.N; i++ {
- _, err := gq.GetString(ctx, content, `.body ul a -> parent(li) -> slice(0) -> next(.selected) -> join(-)`)
- if err != nil {
- b.Fatal(err)
- }
- }
- b.StopTimer()
-}
diff --git a/parsers/gq/buildin_function_test.go b/parsers/gq/buildin_function_test.go
deleted file mode 100644
index cd2960f..0000000
--- a/parsers/gq/buildin_function_test.go
+++ /dev/null
@@ -1,195 +0,0 @@
-package gq
-
-import (
- "testing"
-
- "github.com/stretchr/testify/assert"
-)
-
-func TestBuildInFuncGet(t *testing.T) {
- t.Parallel()
- if _, err := gq.GetString(ctx, content, `-> get`); err == nil {
- t.Error("Unexpected function error")
- }
-
- if _, err := gq.GetString(ctx, content, `.body #a1 -> set(key111)`); err != nil {
- t.Error(err)
- }
-
- assertGetString(t, `-> get(key111) -> child`, "Google")
-}
-
-func TestBuildInFuncSet(t *testing.T) {
- t.Parallel()
- if _, err := gq.GetString(ctx, content, `-> set`); err == nil {
- t.Fatal("Unexpected function error")
- }
-
- if _, err := gq.GetString(ctx, content, `-> set(v1, 'v1 ')`); err != nil {
- t.Error(err)
- }
-
- if _, err := gq.GetString(ctx, content, `.body #a1 -> text -> set(key222)`); err != nil {
- t.Error(err)
- }
-}
-
-func TestBuildInFuncText(t *testing.T) {
- t.Parallel()
-
- assertGetString(t, `#main #n1 -> text`, "1")
-
- assertGetString(t, `#main #n1`, "1")
-}
-
-func TestBuildInFuncAttr(t *testing.T) {
- t.Parallel()
- if _, err := gq.GetString(ctx, content, `#main #n1 -> text -> attr`); err == nil {
- t.Fatal("Unexpected function error")
- }
-
- if _, err := gq.GetString(ctx, content, `-> attr()`); err == nil {
- t.Fatal("Unexpected null argument")
- }
-
- assertGetString(t, `#main #n1 -> attr(class)`, "one even row")
-
- assertGetString(t, `#main #n1 -> attr(empty, default)`, "default")
-}
-
-func TestBuildInFuncJoin(t *testing.T) {
- t.Parallel()
- assertGetString(t, `#main div -> join(' < ')`, "1 < 2 < 3 < 4 < 5 < 6")
-
- assertGetString(t, `#main div -> join("")`, "123456")
-
- assertGetString(t, `#main div -> join('')`, "123456")
-}
-
-func TestBuildInFuncHref(t *testing.T) {
- t.Parallel()
- if _, err := gq.GetString(ctx, content, `.body ul #a4 -> text -> href`); err == nil {
- t.Fatal("Unexpected function error")
- }
-
- assertGetString(t, `.body ul #a4 a -> href`, "https://localhost/home")
-
- assertGetString(t, `.body ul #a4 a -> href(path)`, "https://localhost/path/home")
-
- assertGetString(t, `.body ul #a4 a -> href(path/)`, "https://localhost/path/home")
-
- assertGetString(t, `.body ul #a4 a -> href(/path/)`, "https://localhost/path/home")
-
- _, err := gq.GetString(ctx, content, `#main #n1 -> href`)
- assert.Error(t, err)
-}
-
-func TestBuildInFuncHtml(t *testing.T) {
- t.Parallel()
- if _, err := gq.GetString(ctx, content, `-> html(test)`); err == nil {
- t.Fatal("Unexpected function error")
- }
-
- assertGetString(t, `.body ul a -> html`, "Google\nGithub\nGolang\nHome")
-
- assertGetString(t, `.body ul a -> slice(0) -> html(true)`,
- `Google `)
-}
-
-func TestBuildInFuncPrev(t *testing.T) {
- t.Parallel()
- if _, err := gq.GetString(ctx, content, `#foot #nf3 -> text -> prev`); err == nil {
- t.Fatal("Unexpected function error")
- }
-
- assertGetString(t, `#foot #nf3 -> prev`, "f2")
-
- assertGetString(t, `#foot #nf3 -> prev(#nf1)`, "f2")
-}
-
-func TestBuildInFuncNext(t *testing.T) {
- t.Parallel()
- if _, err := gq.GetString(ctx, content, `#foot #nf2 -> text -> next`); err == nil {
- t.Fatal("Unexpected function error")
- }
-
- assertGetString(t, `#foot #nf2 -> next`, "f3")
-
- assertGetString(t, `#foot #nf2 -> next(#nf4)`, "f3")
-}
-
-func TestBuildInFuncSlice(t *testing.T) {
- t.Parallel()
- if _, err := gq.GetString(ctx, content, `-> slice`); err == nil {
- t.Fatal("Unexpected function error")
- }
-
- if _, err := gq.GetString(ctx, content, `#main div -> text -> slice(0)`); err == nil {
- t.Fatal("Unexpected function error")
- }
-
- assertGetString(t, `#main div -> slice(0)`, "1")
-
- assertGetString(t, `#main div -> slice(-1)`, "6")
-
- assertGetString(t, `#main div -> slice(0, 3)`, "1\n2\n3")
-
- assertGetString(t, `#main div -> slice(0, -2)`, "1\n2\n3\n4")
-}
-
-func TestBuildInFuncChild(t *testing.T) {
- t.Parallel()
- if _, err := gq.GetString(ctx, content, `.body ul -> text -> child`); err == nil {
- t.Fatal("Unexpected function error")
- }
-
- assertGetString(t, `.body ul li -> child(a)`, "Google\nGithub\nGolang\nHome")
-
- assertGetString(t, `.body ul li -> child`, "Google\nGithub\nGolang\nHome")
-}
-
-func TestBuildInFuncParent(t *testing.T) {
- t.Parallel()
- if _, err := gq.GetString(ctx, content, `.body ul -> text -> parent`); err == nil {
- t.Fatal("Unexpected function error")
- }
-
- assertGetString(t, `.body ul a -> parent(#a1) -> attr(id)`, "a1")
-
- assertGetString(t, `.body ul a -> parent -> attr(id)`, "a1\na2\na3\na4")
-}
-
-func TestBuildInFuncParents(t *testing.T) {
- t.Parallel()
- if _, err := gq.GetString(ctx, content, `.body ul -> text -> parents`); err == nil {
- t.Fatal("Unexpected type")
- }
-
- if _, err := gq.GetString(ctx, content, `.body ul .selected -> parents(div, test)`); err == nil {
- t.Fatal("Unexpected argument")
- }
-
- assertGetString(t, `.body ul .selected -> parents(div, true) -> attr(id)`, "url")
-
- assertGetString(t, `.body ul .selected -> parents -> slice(0) -> attr(id)`, "url")
-}
-
-func TestBuildInFuncPrefix(t *testing.T) {
- t.Parallel()
-
- assertGetString(t, `#main #n1 -> text -> prefix(A)`, "A1")
-
- assertGetString(t, `#main #n1 -> prefix(B)`, "B1")
-
- assertGetStrings(t, `#main div -> slice(0, 2) -> text -> prefix(-)`, []string{"-1", "-2"})
-}
-
-func TestBuildInFuncSuffix(t *testing.T) {
- t.Parallel()
-
- assertGetString(t, `#main #n1 -> text -> suffix(A)`, "1A")
-
- assertGetString(t, `#main #n1 -> suffix(B)`, "1B")
-
- assertGetStrings(t, `.body a -> slice(0, 2) -> text -> suffix(.com)`, []string{"Google.com", "Github.com"})
-}
diff --git a/parsers/gq/gq.go b/parsers/gq/gq.go
deleted file mode 100644
index b80823d..0000000
--- a/parsers/gq/gq.go
+++ /dev/null
@@ -1,201 +0,0 @@
-// Package gq the goquery parser
-package gq
-
-import (
- "strings"
-
- "github.com/PuerkitoBio/goquery"
- "github.com/shiroyk/cloudcat/plugin"
- "github.com/shiroyk/cloudcat/plugin/parser"
- "github.com/spf13/cast"
-)
-
-// Key the gq parser register key.
-const Key string = "gq"
-
-// Parser the goquery parser
-type Parser struct {
- parseFuncs FuncMap
-}
-
-// NewParser creates a new goquery Parser with the given FuncMap.
-func NewParser(funcs FuncMap) parser.Parser {
- p := &Parser{parseFuncs: builtins()}
- for k, v := range funcs {
- p.parseFuncs[k] = v
- }
- return p
-}
-
-// init register the goquery Parser with default builtins funcs.
-func init() {
- parser.Register(Key, &Parser{parseFuncs: builtins()})
-}
-
-// GetString gets the string of the content with the given arguments.
-//
-// content := ``
-// GetString(ctx, content, "ul li") returns "1\n2"
-func (p *Parser) GetString(ctx *plugin.Context, content any, arg string) (ret string, err error) {
- nodes, err := getSelection(content)
- if err != nil {
- return
- }
-
- rule, funcs, err := parseRuleFunctions(p.parseFuncs, arg)
- if err != nil {
- return
- }
-
- var node any = nodes.Find(rule)
-
- for _, fun := range funcs {
- node, err = p.parseFuncs[fun.name](ctx, node, fun.args...)
- if err != nil || node == nil {
- return ret, err
- }
- }
-
- node, err = Join(ctx, node, "\n")
- if err != nil {
- return
- }
-
- return node.(string), nil
-}
-
-// GetStrings gets the strings of the content with the given arguments.
-//
-// content := ``
-// GetStrings(ctx, content, "ul li") returns []string{"1", "2"}
-func (p *Parser) GetStrings(ctx *plugin.Context, content any, arg string) (ret []string, err error) {
- nodes, err := getSelection(content)
- if err != nil {
- return
- }
-
- rule, funcs, err := parseRuleFunctions(p.parseFuncs, arg)
- if err != nil {
- return
- }
-
- var node any = nodes.Find(rule)
-
- for _, fun := range funcs {
- node, err = p.parseFuncs[fun.name](ctx, node, fun.args...)
- if err != nil || node == nil {
- return nil, err
- }
- }
-
- if sel, ok := node.(*goquery.Selection); ok {
- str := make([]string, sel.Length())
- sel.EachWithBreak(func(i int, sel *goquery.Selection) bool {
- str[i] = strings.TrimSpace(sel.Text())
- return true
- })
- return str, nil
- }
- return cast.ToStringSliceE(node)
-}
-
-// GetElement gets the element of the content with the given arguments.
-//
-// content := ``
-// GetElement(ctx, content, "ul li") returns "1 \n2 "
-func (p *Parser) GetElement(ctx *plugin.Context, content any, arg string) (ret string, err error) {
- nodes, err := getSelection(content)
- if err != nil {
- return
- }
-
- rule, funcs, err := parseRuleFunctions(p.parseFuncs, arg)
- if err != nil {
- return
- }
-
- var node any = nodes.Find(rule)
-
- for _, fun := range funcs {
- node, err = p.parseFuncs[fun.name](ctx, node, fun.args...)
- if err != nil || node == nil {
- return ret, err
- }
- }
-
- if sel, ok := node.(*goquery.Selection); ok {
- return goquery.OuterHtml(sel)
- }
-
- return cast.ToStringE(node)
-}
-
-// GetElements gets the elements of the content with the given arguments.
-//
-// content := ``
-// GetElements(ctx, content, "ul li") returns []string{"1 ", "2 "}
-func (p *Parser) GetElements(ctx *plugin.Context, content any, arg string) (ret []string, err error) {
- nodes, err := getSelection(content)
- if err != nil {
- return
- }
-
- rule, funcs, err := parseRuleFunctions(p.parseFuncs, arg)
- if err != nil {
- return
- }
-
- var node any = nodes.Find(rule)
-
- for _, fun := range funcs {
- node, err = p.parseFuncs[fun.name](ctx, node, fun.args...)
- if err != nil || node == nil {
- return nil, err
- }
- }
-
- if sel, ok := node.(*goquery.Selection); ok {
- objs := make([]string, sel.Length())
- sel.EachWithBreak(func(i int, sel *goquery.Selection) bool {
- if objs[i], err = goquery.OuterHtml(sel); err != nil {
- return false
- }
- return true
- })
- if err != nil {
- return
- }
- return objs, nil
- }
- return cast.ToStringSliceE(node)
-}
-
-// getSelection converts content to goquery.Selection
-func getSelection(content any) (*goquery.Selection, error) {
- switch data := content.(type) {
- default:
- str, err := cast.ToStringE(content)
- if err != nil {
- return nil, err
- }
- doc, err := goquery.NewDocumentFromReader(strings.NewReader(str))
- if err != nil {
- return nil, err
- }
- return doc.Selection, nil
- case nil:
- return new(goquery.Selection), nil
- case []string:
- doc, err := goquery.NewDocumentFromReader(strings.NewReader(strings.Join(data, "\n")))
- if err != nil {
- return nil, err
- }
- return doc.Selection, nil
- case string:
- doc, err := goquery.NewDocumentFromReader(strings.NewReader(data))
- if err != nil {
- return nil, err
- }
- return doc.Selection, nil
- }
-}
diff --git a/parsers/gq/gq_test.go b/parsers/gq/gq_test.go
deleted file mode 100644
index 58dfeac..0000000
--- a/parsers/gq/gq_test.go
+++ /dev/null
@@ -1,191 +0,0 @@
-package gq
-
-import (
- "flag"
- "fmt"
- "os"
- "testing"
-
- "log/slog"
-
- "github.com/shiroyk/cloudcat/plugin"
- "github.com/shiroyk/cloudcat/plugin/parser"
- "github.com/stretchr/testify/assert"
-)
-
-var (
- gq Parser
- ctx *plugin.Context
- content = `
-
-
- Tests for siblings
-
-
-
-
1
-
2
-
3
-
4
-
5
-
6
-
Golang `, ``)
- assert.NoError(t, err)
-
- sel, _ := gq.GetElement(ctx, content, `#main .row`)
- _, err = gq.GetString(ctx, sel, ``)
- assert.NoError(t, err)
-}
-
-func TestGetString(t *testing.T) {
- t.Parallel()
- assertGetString(t, `#main .row -> text`, "1\n2\n3\n4\n5\n6")
-
- assertGetString(t, `.body ul a -> parent(li) -> attr(id) -> join(-)`, "a1-a2-a3-a4")
-
- assertGetString(t, `script -> slice(0) -> attr(type)`, "text/javascript")
-}
-
-func TestGetStrings(t *testing.T) {
- t.Parallel()
- assertGetStrings(t, `.body ul li -> child(a) -> attr(title)`, []string{"Google page", "Github page", "Golang page", "Home page"})
-
- assertGetStrings(t, `.body ul a`, []string{"Google", "Github", "Golang", "Home"})
-}
-
-func TestGetElement(t *testing.T) {
- t.Parallel()
- assertGetElement(t, `.body ul a -> parents(li)`, `Google f1
`,
- `f2
`,
- `f3
`,
- })
-
- assertGetElements(t, `#foot div -> slice(0, 3) -> text`, []string{"f1", "f2", "f3"})
-}
-
-func TestExternalFunc(t *testing.T) {
- {
- fun := func(logger *slog.Logger) GFunc {
- return func(_ *plugin.Context, content any, args ...string) (any, error) {
- logger.Info(fmt.Sprintf("result type was %T", content))
- return content, nil
- }
- }
- p := NewParser(FuncMap{"logger": fun(slog.Default())})
- _, err := p.GetString(ctx, content, ".body ul a -> logger -> text")
- assert.NoError(t, err)
- }
-
- {
- fun := func(_ *plugin.Context, content any, args ...string) (any, error) {
- return nil, nil
- }
- p := NewParser(FuncMap{"nil": fun})
- _, err := p.GetString(ctx, content, ".body ul a -> nil -> text")
- assert.NoError(t, err)
- _, err = p.GetStrings(ctx, content, ".body ul a -> nil -> text")
- assert.NoError(t, err)
- _, err = p.GetElement(ctx, content, ".body ul a -> nil -> text")
- assert.NoError(t, err)
- _, err = p.GetElements(ctx, content, ".body ul a -> nil -> text")
- assert.NoError(t, err)
- }
-}
diff --git a/parsers/js/README.md b/parsers/js/README.md
deleted file mode 100644
index b688aa7..0000000
--- a/parsers/js/README.md
+++ /dev/null
@@ -1,3 +0,0 @@
-
-## References
-- [goja](https://github.com/dop251/goja)
\ No newline at end of file
diff --git a/parsers/js/js.go b/parsers/js/js.go
deleted file mode 100644
index 5747d14..0000000
--- a/parsers/js/js.go
+++ /dev/null
@@ -1,104 +0,0 @@
-// Package js the js parser
-package js
-
-import (
- "encoding/json"
-
- "github.com/shiroyk/cloudcat/js"
- "github.com/shiroyk/cloudcat/plugin"
- "github.com/shiroyk/cloudcat/plugin/parser"
- "github.com/spf13/cast"
-)
-
-// Parser the js parser
-type Parser struct{}
-
-const key string = "js"
-
-func init() {
- parser.Register(key, new(Parser))
-}
-
-// GetString gets the string of the content with the given arguments.
-// returns the string result.
-func (p *Parser) GetString(ctx *plugin.Context, content any, arg string) (ret string, err error) {
- return getString(ctx, content, arg)
-}
-
-// GetStrings gets the strings of the content with the given arguments.
-// returns the slice of string result.
-func (p *Parser) GetStrings(ctx *plugin.Context, content any, arg string) (ret []string, err error) {
- return getStrings(ctx, content, arg)
-}
-
-// GetElement gets the element of the content with the given arguments.
-// returns the string result.
-func (p *Parser) GetElement(ctx *plugin.Context, content any, arg string) (string, error) {
- return getString(ctx, content, arg)
-}
-
-// GetElements gets the elements of the content with the given arguments.
-// returns the slice of string result.
-func (p *Parser) GetElements(ctx *plugin.Context, content any, arg string) ([]string, error) {
- return getStrings(ctx, content, arg)
-}
-
-func getString(ctx *plugin.Context, content any, script string) (ret string, err error) {
- result, err := js.Run(ctx, js.Program{Code: script, Args: map[string]any{
- "content": content,
- }})
- if err != nil {
- return ret, err
- }
-
- value, err := js.Unwrap(result)
- if err != nil {
- return ret, err
- }
-
- switch value.(type) {
- case map[string]any, []any:
- bytes, err := json.Marshal(value)
- if err != nil {
- return ret, err
- }
- return string(bytes), nil
- case nil:
- return ret, nil
- default:
- return cast.ToStringE(value)
- }
-}
-
-func getStrings(ctx *plugin.Context, content any, script string) (ret []string, err error) {
- result, err := js.Run(ctx, js.Program{Code: script, Args: map[string]any{
- "content": content,
- }})
- if err != nil {
- return nil, err
- }
-
- value, err := js.Unwrap(result)
- if err != nil {
- return nil, err
- }
- if value == nil {
- return nil, nil
- }
-
- slice, ok := value.([]any)
- if !ok {
- slice = []any{value}
- }
-
- ret = make([]string, len(slice))
- for i, v := range slice {
- if s, ok := v.(string); ok {
- ret[i] = s
- } else {
- bytes, _ := json.Marshal(v)
- ret[i] = string(bytes)
- }
- }
- return
-}
diff --git a/parsers/js/js_test.go b/parsers/js/js_test.go
deleted file mode 100644
index d629efc..0000000
--- a/parsers/js/js_test.go
+++ /dev/null
@@ -1,95 +0,0 @@
-package js
-
-import (
- "flag"
- "os"
- "testing"
-
- "github.com/shiroyk/cloudcat/plugin"
- "github.com/shiroyk/cloudcat/plugin/parser"
- "github.com/stretchr/testify/assert"
-)
-
-var (
- jsParser Parser
- ctx *plugin.Context
-)
-
-func TestMain(m *testing.M) {
- flag.Parse()
- ctx = plugin.NewContext(plugin.ContextOptions{
- URL: "http://localhost/home",
- })
- code := m.Run()
- os.Exit(code)
-}
-
-func TestParser(t *testing.T) {
- if _, ok := parser.GetParser(key); !ok {
- t.Fatal("schema not registered")
- }
-}
-
-func TestGetString(t *testing.T) {
- {
- str, err := jsParser.GetString(ctx, "a", `(async () => content + 1)()`)
- if err != nil {
- t.Fatal(err)
- }
-
- assert.Equal(t, "a1", str)
- }
-
- {
- str, err := jsParser.GetString(ctx, "", `(async () => ({"test":"1"}))()`)
- if err != nil {
- t.Fatal(err)
- }
-
- assert.JSONEq(t, `{"test":"1"}`, str)
- }
-}
-
-func TestGetStrings(t *testing.T) {
- {
- str, err := jsParser.GetStrings(ctx, `["a1"]`,
- `new Promise((r, j) => {
- let s = JSON.parse(content);
- s.push('a2');
- r(s)
- });`)
- if err != nil {
- t.Fatal(err)
- }
-
- assert.Equal(t, []string{"a1", "a2"}, str)
- }
-
- {
- str, err := jsParser.GetStrings(ctx, "", `[{"foo":"1"}, {"bar":"1"}, 19]`)
- if err != nil {
- t.Fatal(err)
- }
-
- assert.Equal(t, []string{`{"foo":"1"}`, `{"bar":"1"}`, "19"}, str)
- }
-}
-
-func TestGetElement(t *testing.T) {
- ele, err := jsParser.GetElement(ctx, ``, `cat.setVar('size', 1 + 2);cat.getVar('size');`)
- if err != nil {
- t.Fatal(err)
- }
-
- assert.Equal(t, "3", ele)
-}
-
-func TestGetElements(t *testing.T) {
- t.Parallel()
- ele, err := jsParser.GetElements(ctx, ``, `[1, 2]`)
- if err != nil {
- t.Fatal(err)
- }
-
- assert.Equal(t, []string{"1", "2"}, ele)
-}
diff --git a/parsers/json/json.go b/parsers/json/json.go
deleted file mode 100644
index 4304fd6..0000000
--- a/parsers/json/json.go
+++ /dev/null
@@ -1,113 +0,0 @@
-// Package json the json parser
-package json
-
-import (
- "fmt"
- "strings"
-
- "github.com/ohler55/ojg/jp"
- "github.com/ohler55/ojg/oj"
- "github.com/shiroyk/cloudcat/plugin"
- "github.com/shiroyk/cloudcat/plugin/parser"
- "github.com/spf13/cast"
-)
-
-// Parser the json parser
-type Parser struct{}
-
-const key string = "json"
-
-func init() {
- parser.Register(key, new(Parser))
-}
-
-// GetString gets the string of the content with the given arguments.
-//
-// content := `{"keys": [{"key":"foo"},{"key":"bar"}]}`
-// GetString(ctx, content, "$.key[*].key") returns "foo\nbar"
-func (p Parser) GetString(_ *plugin.Context, content any, arg string) (string, error) {
- obj, err := getDoc(content, arg)
- if err != nil {
- return "", err
- }
-
- str := make([]string, len(obj))
- var ok bool
-
- for i, o := range obj {
- if str[i], ok = o.(string); !ok {
- str[i] = oj.JSON(o)
- }
- }
-
- return strings.Join(str, "\n"), nil
-}
-
-// GetStrings gets the strings of the content with the given arguments.
-//
-// content := `{"keys": [{"key":"foo"},{"key":"bar"}]}`
-// GetStrings(ctx, content, "$.key[*].key") returns []string{"foo", "bar"}
-func (p Parser) GetStrings(_ *plugin.Context, content any, arg string) ([]string, error) {
- obj, err := getDoc(content, arg)
- if err != nil {
- return nil, err
- }
-
- str := make([]string, len(obj))
- var ok bool
-
- for i, o := range obj {
- if str[i], ok = o.(string); !ok {
- str[i] = oj.JSON(o)
- }
- }
-
- return str, nil
-}
-
-// GetElement gets the element of the content with the given arguments.
-// sames as the GetString.
-func (p Parser) GetElement(ctx *plugin.Context, content any, arg string) (string, error) {
- return p.GetString(ctx, content, arg)
-}
-
-// GetElements gets the elements of the content with the given arguments.
-// sames as the GetStrings.
-func (p Parser) GetElements(ctx *plugin.Context, content any, arg string) ([]string, error) {
- return p.GetStrings(ctx, content, arg)
-}
-
-func getDoc(content any, arg string) ([]any, error) {
- var err error
- var doc any
- switch data := content.(type) {
- default:
- str, err := cast.ToStringE(content)
- if err != nil {
- return nil, err
- }
- if doc, err = oj.ParseString(str); err != nil {
- return nil, err
- }
- case nil:
- return nil, nil
- case []string:
- if len(data) == 0 {
- return nil, fmt.Errorf("unexpected content %s", content)
- }
- if doc, err = oj.ParseString(data[0]); err != nil {
- return nil, err
- }
- case string:
- if doc, err = oj.ParseString(data); err != nil {
- return nil, err
- }
- }
-
- x, err := jp.ParseString(arg)
- if err != nil {
- return nil, err
- }
-
- return x.Get(doc), nil
-}
diff --git a/parsers/json/json_test.go b/parsers/json/json_test.go
deleted file mode 100644
index ccc1b95..0000000
--- a/parsers/json/json_test.go
+++ /dev/null
@@ -1,148 +0,0 @@
-package json
-
-import (
- "flag"
- "os"
- "testing"
-
- "github.com/shiroyk/cloudcat/plugin"
- "github.com/shiroyk/cloudcat/plugin/parser"
- "github.com/stretchr/testify/assert"
-)
-
-var (
- json Parser
- ctx *plugin.Context
- content = `
-{
- "store": {
- "book": [
- {
- "category": "reference",
- "author": "Nigel Rees",
- "title": "Sayings of the Century",
- "price": 8.95
- },
- {
- "category": "fiction",
- "author": "Evelyn Waugh",
- "title": "Sword of Honour",
- "price": 12.99
- },
- {
- "category": "fiction",
- "author": "Herman Melville",
- "title": "Moby Dick",
- "isbn": "0-553-21311-3",
- "price": 8.99
- },
- {
- "category": "fiction",
- "author": "J. R. R. Tolkien",
- "title": "The Lord of the Rings",
- "isbn": "0-395-19395-8",
- "price": 22.99
- }
- ],
- "bicycle": {
- "color": "red",
- "price": 19.95
- }
- },
- "expensive": 10
-}`
-)
-
-func TestMain(m *testing.M) {
- flag.Parse()
- ctx = plugin.NewContext(plugin.ContextOptions{})
- code := m.Run()
- os.Exit(code)
-}
-
-func assertString(t *testing.T, arg string, expected string) {
- str, err := json.GetString(ctx, content, arg)
- if err != nil {
- t.Fatal(err)
- }
-
- assert.Equal(t, expected, str)
-}
-
-func assertStrings(t *testing.T, arg string, expected []string) {
- str, err := json.GetStrings(ctx, content, arg)
- if err != nil {
- t.Fatal(err)
- }
-
- assert.Equal(t, expected, str)
-}
-
-func TestParser(t *testing.T) {
- t.Parallel()
- if _, ok := parser.GetParser(key); !ok {
- t.Fatal("schema not registered")
- }
-
- contents := []any{`][`, `}{`}
- for _, ct := range contents {
- _, err := json.GetString(ctx, ct, ``)
- assert.ErrorContains(t, err, "unexpected")
- }
-
- if _, err := json.GetString(ctx, &contents[len(contents)-1], ""); err == nil {
- t.Fatal("Unexpected type")
- }
-}
-
-func TestGetString(t *testing.T) {
- t.Parallel()
- assertString(t, `$.store.book[*].author`, "Nigel Rees\nEvelyn Waugh\nHerman Melville\nJ. R. R. Tolkien")
-}
-
-func TestGetStrings(t *testing.T) {
- t.Parallel()
- assertStrings(t, `$...book[0].price`, []string{"8.95"})
-
- assertStrings(t, `$...book[-1].price`, []string{"22.99"})
-}
-
-func TestGetElement(t *testing.T) {
- t.Parallel()
- if _, err := json.GetElement(ctx, content, `$$$`); err == nil {
- t.Fatal("Unexpected path")
- }
-
- assertString(t, `$.store.book[-1].price`, "22.99")
-
- str1, err := json.GetElement(ctx, content, `$.store.book[?(@.price > 20)]`)
- if err != nil {
- t.Fatal(err)
- }
-
- str2, err := json.GetElement(ctx, str1, `$.title`)
- if err != nil {
- t.Fatal(err)
- }
- if str2 != `The Lord of the Rings` {
- t.Fatalf("Unexpected string %s", str2)
- }
-}
-
-func TestGetElements(t *testing.T) {
- t.Parallel()
- assertStrings(t, `$.store.book[?(@.price < 10)].isbn`, []string{`0-553-21311-3`})
-
- str1, err := json.GetElements(ctx, content, `$.store.book[3]`)
- if err != nil {
- t.Fatal(err)
- }
-
- str2, err := json.GetElement(ctx, str1[0], `$.category`)
- if err != nil {
- t.Fatal(err)
- }
- if str2 != `fiction` {
- t.Fatalf("Unexpected string %s", str2)
- }
-}
diff --git a/parsers/main.go b/parsers/main.go
deleted file mode 100644
index 8469318..0000000
--- a/parsers/main.go
+++ /dev/null
@@ -1,9 +0,0 @@
-package parsers
-
-import (
- _ "github.com/shiroyk/cloudcat/parsers/gq" // deep
- _ "github.com/shiroyk/cloudcat/parsers/js"
- _ "github.com/shiroyk/cloudcat/parsers/json"
- _ "github.com/shiroyk/cloudcat/parsers/regex"
- _ "github.com/shiroyk/cloudcat/parsers/xpath"
-)
diff --git a/parsers/regex/regex.go b/parsers/regex/regex.go
deleted file mode 100644
index 6464fcc..0000000
--- a/parsers/regex/regex.go
+++ /dev/null
@@ -1,174 +0,0 @@
-// Package regex the regexp parser
-package regex
-
-import (
- "fmt"
- "strconv"
- "strings"
-
- "github.com/dlclark/regexp2"
- "github.com/shiroyk/cloudcat/plugin"
- "github.com/shiroyk/cloudcat/plugin/parser"
- "github.com/spf13/cast"
-)
-
-// Parser the regexp2 parser
-type Parser struct{}
-
-const key string = "regex"
-
-func init() {
- parser.Register(key, new(Parser))
-}
-
-// GetString gets the string of the content with the given arguments.
-// replace the string with the given regexp.
-func (p Parser) GetString(_ *plugin.Context, content any, arg string) (string, error) {
- re, replace, start, count, err := parseRegexp(arg)
- if err != nil {
- return "", err
- }
-
- var str string
- switch conv := content.(type) {
- case string:
- str = conv
- case []string:
- str = strings.Join(conv, "")
- default:
- str, err = cast.ToStringE(conv)
- if err != nil {
- return "", err
- }
- }
-
- return re.Replace(str, replace, start, count)
-}
-
-// GetStrings gets the strings of the content with the given arguments.
-// replace each string of the slice with the given regexp.
-func (p Parser) GetStrings(_ *plugin.Context, content any, arg string) ([]string, error) {
- re, replace, start, count, err := parseRegexp(arg)
- if err != nil {
- return nil, err
- }
-
- var str []string
- switch conv := content.(type) {
- case string:
- str = []string{conv}
- case []string:
- str = conv
- default:
- str, err = cast.ToStringSliceE(conv)
- if err != nil {
- return nil, err
- }
- }
-
- for i := 0; i < len(str); i++ {
- str[i], err = re.Replace(str[i], replace, start, count)
- if err != nil {
- return nil, err
- }
- }
- return str, nil
-}
-
-// GetElement gets the element of the content with the given arguments.
-// sames as GetString.
-func (p Parser) GetElement(ctx *plugin.Context, content any, arg string) (string, error) {
- return p.GetString(ctx, content, arg)
-}
-
-// GetElements gets the elements of the content with the given arguments.
-// sames as GetStrings.
-func (p Parser) GetElements(ctx *plugin.Context, content any, arg string) ([]string, error) {
- return p.GetStrings(ctx, content, arg)
-}
-
-type tokenState int
-
-const (
- commonState tokenState = iota
- searchState
- replaceState
- flagState
-)
-
-var reOptMap = map[string]regexp2.RegexOptions{
- "i": regexp2.IgnoreCase,
- "m": regexp2.Multiline,
- "n": regexp2.ExplicitCapture,
- "c": regexp2.Compiled,
- "s": regexp2.Singleline,
- "x": regexp2.IgnorePatternWhitespace,
- "r": regexp2.RightToLeft,
- "d": regexp2.Debug,
- "e": regexp2.ECMAScript,
- "u": regexp2.Unicode,
-}
-
-//nolint:gocognit
-func parseRegexp(arg string) (re *regexp2.Regexp, replace string, start, count int, err error) {
- state := commonState
- pattern := strings.Builder{}
- start = -1
- count = -1
- var offset int
- var regex string
- var reOpt int32
-
- for offset < len(arg) {
- ch := arg[offset]
- offset++
- switch ch {
- default:
- if state == flagState {
- if i, ok := reOptMap[string(ch)]; ok {
- reOpt |= int32(i)
- } else if ch >= '0' && ch <= '9' || ch == '-' || ch == ',' {
- pattern.WriteByte(ch)
- }
- } else {
- pattern.WriteByte(ch)
- }
- case '\\':
- if nextCh := arg[offset]; nextCh == '/' {
- pattern.WriteByte(nextCh)
- offset++
- } else {
- pattern.WriteByte(ch)
- }
- case '/':
- switch state {
- case commonState:
- state = searchState
- case searchState:
- state = replaceState
- regex = pattern.String()
- pattern.Reset()
- case replaceState:
- state = flagState
- replace = pattern.String()
- pattern.Reset()
- default:
- return nil, "", start, count, fmt.Errorf("/ character must escaped")
- }
- }
- }
-
- if pattern.Len() > 0 {
- s1, s2, _ := strings.Cut(pattern.String(), ",")
- start, err = strconv.Atoi(s1)
- if err != nil {
- start = -1
- }
- count, err = strconv.Atoi(s2)
- if err != nil {
- count = -1
- }
- }
-
- return regexp2.MustCompile(regex, regexp2.RegexOptions(reOpt)), replace, start, count, nil
-}
diff --git a/parsers/regex/regex_test.go b/parsers/regex/regex_test.go
deleted file mode 100644
index d212ff9..0000000
--- a/parsers/regex/regex_test.go
+++ /dev/null
@@ -1,58 +0,0 @@
-package regex
-
-import (
- "testing"
-
- "github.com/shiroyk/cloudcat/plugin/parser"
- "github.com/stretchr/testify/assert"
-)
-
-var (
- re Parser
- testCase = []struct{ re, str, want string }{
- {`/[0-9]/`, `114i`, "i"},
- {`/[0-9]/i/`, `114`, "iii"},
- {`/\\//`, `1/`, "1"},
- {`/[a-z]/1/`, `aaa`, "111"},
- {`/olang/olang/i`, `GoLAnG`, "Golang"},
- {`/[^ ]+\s(?)/${time}/`, `08/10/99 16:00`, "16:00"},
- {`/D\.(.+)/David $1/`, `D.Bau`, "David Bau"},
- {`/a/b/{0,2}`, `aaaaa`, "bbaaa"},
- {`/a/b/{3,2}`, `aaaaa`, "aaabb"},
- {`/[a-z]/1/i{3,3}`, `aaaBBB`, "aaa111"},
- {`/a/stuff/{-1,-1}`, `a test a blah and a`, "stuff test stuff blstuffh stuffnd stuff"},
- {`/(\p{Sc}\s?)?(\d+\.?((?<=\.)\d+)?)(?(1)|\s?\p{Sc})?/$2/`, `$17.43 €2 16.33 £0.98 0.43 £43 12€ 17`, "17.43 2 16.33 0.98 0.43 43 12 17"},
- }
-)
-
-func TestParser(t *testing.T) {
- if _, ok := parser.GetParser(key); !ok {
- t.Fatal("parser not registered")
- }
-}
-
-func TestGetString(t *testing.T) {
- t.Parallel()
- for _, s := range testCase {
- t.Run(s.re, func(t *testing.T) {
- str, err := re.GetString(nil, s.str, s.re)
- if err != nil {
- t.Error(err)
- }
- assert.Equal(t, s.want, str)
- })
- }
-}
-
-func TestGetStrings(t *testing.T) {
- t.Parallel()
- for _, s := range testCase {
- t.Run(s.re, func(t *testing.T) {
- str, err := re.GetStrings(nil, []string{s.str}, s.re)
- if err != nil {
- t.Error(err)
- }
- assert.Equal(t, s.want, str[0])
- })
- }
-}
diff --git a/parsers/xpath/bench_xpath_test.go b/parsers/xpath/bench_xpath_test.go
deleted file mode 100644
index 5d2318f..0000000
--- a/parsers/xpath/bench_xpath_test.go
+++ /dev/null
@@ -1,16 +0,0 @@
-package xpath
-
-import (
- "testing"
-)
-
-func BenchmarkParser(b *testing.B) {
- b.StartTimer()
- for i := 0; i < b.N; i++ {
- _, err := xpath.GetString(ctx, ``, `//div[@class="body"]/ul//a/@title`)
- if err != nil {
- b.Fatal(err)
- }
- }
- b.StopTimer()
-}
diff --git a/parsers/xpath/xpath.go b/parsers/xpath/xpath.go
deleted file mode 100644
index f0c4857..0000000
--- a/parsers/xpath/xpath.go
+++ /dev/null
@@ -1,148 +0,0 @@
-// Package xpath the xpath parser
-package xpath
-
-import (
- "strings"
-
- "github.com/antchfx/htmlquery"
- "github.com/shiroyk/cloudcat/plugin"
- "github.com/shiroyk/cloudcat/plugin/parser"
- "github.com/spf13/cast"
- "golang.org/x/net/html"
-)
-
-// Parser the xpath parser
-type Parser struct{}
-
-const key string = "xpath"
-
-func init() {
- parser.Register(key, new(Parser))
-}
-
-// GetString gets the string of the content with the given arguments.
-//
-// content := ``
-// GetString(ctx, content, "//li/text()") returns "1\n2"
-func (p Parser) GetString(_ *plugin.Context, content any, arg string) (string, error) {
- nodes, err := getHTMLNode(content, arg)
- if err != nil {
- return "", err
- }
-
- if len(nodes) == 0 {
- return "", nil
- }
-
- str := strings.Builder{}
- str.WriteString(htmlquery.InnerText(nodes[0]))
- for _, node := range nodes[1:] {
- str.WriteString("\n")
- str.WriteString(htmlquery.InnerText(node))
- }
-
- return str.String(), nil
-}
-
-// GetStrings gets the strings of the content with the given arguments.
-//
-// content := ``
-// GetStrings(ctx, content, "//li/text()") returns []string{"1", "2"}
-func (p Parser) GetStrings(_ *plugin.Context, content any, arg string) ([]string, error) {
- nodes, err := getHTMLNode(content, arg)
- if err != nil {
- return nil, err
- }
-
- if len(nodes) == 0 {
- return nil, nil
- }
-
- result := make([]string, len(nodes))
- for i, node := range nodes {
- result[i] = htmlquery.InnerText(node)
- }
-
- return result, err
-}
-
-// GetElement gets the element of the content with the given arguments.
-//
-// content := ``
-// GetStrings(ctx, content, "//li..") returns "1 \n2 "
-func (p Parser) GetElement(_ *plugin.Context, content any, arg string) (string, error) {
- nodes, err := getHTMLNode(content, arg)
- if err != nil {
- return "", err
- }
-
- if len(nodes) == 0 {
- return "", nil
- }
-
- str := strings.Builder{}
- str.WriteString(htmlquery.OutputHTML(nodes[0], true))
- for _, node := range nodes[1:] {
- str.WriteString("\n")
- str.WriteString(htmlquery.OutputHTML(node, true))
- }
-
- return str.String(), nil
-}
-
-// GetElements gets the elements of the content with the given arguments.
-//
-// content := ``
-// GetStrings(ctx, content, "//li..") returns []string{"1 ", "2 "}
-func (p Parser) GetElements(_ *plugin.Context, content any, arg string) ([]string, error) {
- nodes, err := getHTMLNode(content, arg)
- if err != nil {
- return nil, err
- }
-
- if len(nodes) == 0 {
- return nil, nil
- }
-
- str := make([]string, len(nodes))
- for i, node := range nodes {
- str[i] = htmlquery.OutputHTML(node, true)
- }
-
- return str, nil
-}
-
-func getHTMLNode(content any, arg string) ([]*html.Node, error) {
- var err error
- var node *html.Node
- switch data := content.(type) {
- default:
- str, err := cast.ToStringE(content)
- if err != nil {
- return nil, err
- }
- node, err = html.Parse(strings.NewReader(str))
- if err != nil {
- return nil, err
- }
- case nil:
- return nil, nil
- case []string:
- node, err = html.Parse(strings.NewReader(strings.Join(data, "\n")))
- if err != nil {
- return nil, err
- }
- case string:
- node, err = html.Parse(strings.NewReader(data))
- if err != nil {
- return nil, err
- }
- }
-
- htmlNode, err := htmlquery.QueryAll(node, arg)
- if err != nil {
- return nil, err
- }
-
- return htmlNode, nil
-}
diff --git a/parsers/xpath/xpath_test.go b/parsers/xpath/xpath_test.go
deleted file mode 100644
index e1e5849..0000000
--- a/parsers/xpath/xpath_test.go
+++ /dev/null
@@ -1,162 +0,0 @@
-package xpath
-
-import (
- "flag"
- "os"
- "testing"
-
- "github.com/shiroyk/cloudcat/plugin"
- "github.com/shiroyk/cloudcat/plugin/parser"
- "github.com/stretchr/testify/assert"
-)
-
-var (
- xpath Parser
- ctx *plugin.Context
- content = `
-
-
-
- Tests for siblings
-
-
-
-
1
-
2
-
3
-
4
-
5
-
6
-
Golang `, `//a`)
- if err != nil {
- t.Error(err)
- }
-
- sel, _ := xpath.GetElement(ctx, content, `//div[@class="body"]`)
- _, err = xpath.GetString(ctx, sel, `//a/text()`)
- if err != nil {
- t.Error(err)
- }
-}
-
-func TestGetString(t *testing.T) {
- t.Parallel()
- if o, _ := xpath.GetStrings(ctx, content, `///`); o != nil {
- t.Fatal("Unexpected type")
- }
-
- str1, err := xpath.GetString(ctx, content, `//div[@id="main"]/div[contains(@class, "row")]/text()`)
- if err != nil {
- t.Fatal(err)
- }
- assert.Equal(t, "1\n2\n3\n4\n5\n6", str1)
-
- str2, err := xpath.GetString(ctx, content, `//div[@class="body"]/ul/li/@id`)
- if err != nil {
- t.Fatal(err)
- }
- assert.Equal(t, "a1\na2\na3", str2)
-
- js, err := xpath.GetString(ctx, content, `//script[1]`)
- if err != nil {
- t.Fatal(err)
- }
- assert.NotEmpty(t, js)
-}
-
-func TestGetStrings(t *testing.T) {
- t.Parallel()
- if o, _ := xpath.GetStrings(ctx, content, `//unknown`); o != nil {
- t.Fatal("Unexpected type")
- }
-
- str1, err := xpath.GetStrings(ctx, content, `//div[@class="body"]/ul//a/@title`)
- if err != nil {
- t.Fatal(err)
- }
- assert.Equal(t, []string{"Google page", "Github page", "Golang page"}, str1)
-
- str2, err := xpath.GetStrings(ctx, content, `//div[@class="body"]/ul//a`)
- if err != nil {
- t.Fatal(err)
- }
- assert.Equal(t, []string{"Google", "Github", "Golang"}, str2)
-}
-
-func TestGetElement(t *testing.T) {
- t.Parallel()
- if o, _ := xpath.GetElement(ctx, content, `//unknown`); o != "" {
- t.Fatal("Unexpected type")
- }
-
- object, err := xpath.GetElement(ctx, content, `//div[@class="body"]/ul//a/..`)
- if err != nil {
- t.Fatal(err)
- }
- assert.Equal(t, `Google Github Golang one even row ", "two odd row ",
- "three even row ", "four odd row ",
- "five even row odder ", "six odd row ",
- }, objects)
-}
diff --git a/plugin/context.go b/plugin/context.go
deleted file mode 100644
index 46b125f..0000000
--- a/plugin/context.go
+++ /dev/null
@@ -1,115 +0,0 @@
-package plugin
-
-import (
- "context"
- "fmt"
- "log/slog"
- "net/url"
- "sync"
- "time"
-)
-
-const (
- // DefaultTimeout The Context default timeout one minute.
- DefaultTimeout = time.Minute
-)
-
-// Context The Parser context
-type Context struct {
- context.Context // set to non-nil by the first cancel call
- parent context.Context // the parent context
- cancelFunc context.CancelFunc
- logger *slog.Logger
- value *sync.Map
- baseURL, url string
-}
-
-// ContextOptions The Context options
-type ContextOptions struct {
- Parent context.Context // the parent context
- Timeout time.Duration // the context timeout, default DefaultTimeout.
- Logger *slog.Logger // the context logger, default slog.Default if nil.
- Values map[any]any // the values
- URL string // the analyzer URL
-}
-
-// NewContext creates a new Context with ContextOptions
-func NewContext(opt ContextOptions) *Context {
- ctx := &Context{
- value: new(sync.Map),
- logger: opt.Logger,
- parent: opt.Parent,
- }
- if ctx.logger == nil {
- ctx.logger = slog.Default()
- }
- if ctx.parent == nil {
- ctx.parent = context.Background()
- }
- for k, v := range opt.Values {
- ctx.value.Store(k, v)
- }
- if opt.URL != "" {
- ctx.url = opt.URL
- if u, err := url.Parse(ctx.url); err == nil {
- ctx.baseURL = fmt.Sprintf("%s://%s", u.Scheme, u.Host)
- }
- }
-
- timeout := DefaultTimeout
- if opt.Timeout > 0 {
- timeout = opt.Timeout
- }
- ctx.Context, ctx.cancelFunc = context.WithTimeout(ctx.parent, timeout)
- return ctx
-}
-
-// Cancel this context releases resources associated with it, so code should
-// call cancel as soon as the operations running in this Context complete.
-func (c *Context) Cancel() {
- if c.cancelFunc != nil {
- c.cancelFunc()
- }
-}
-
-// ClearValue clean all values
-func (c *Context) ClearValue() {
- c.value = new(sync.Map)
-}
-
-// Value returns the value associated with this context for key, or nil
-// if no value is associated with key. Successive calls to Value with
-// the same key returns the same result.
-func (c *Context) Value(key any) any {
- if v, ok := c.value.Load(key); ok {
- return v
- }
- return c.parent.Value(key)
-}
-
-// GetValue returns the value associated with this context for key, or nil
-// if no value is associated with key. Successive calls to Value with
-// the same key returns the same result.
-func (c *Context) GetValue(key any) (any, bool) {
- return c.value.Load(key)
-}
-
-// SetValue value associated with key is val.
-func (c *Context) SetValue(key any, value any) {
- c.value.Store(key, value)
-}
-
-// Logger returns the logger, if ContextOptions.Logger is nil return slog.Default
-func (c *Context) Logger() *slog.Logger {
- return c.logger
-}
-
-// BaseURL returns the baseURL string
-func (c *Context) BaseURL() string {
- return c.baseURL
-}
-
-// URL returns the absolute URL string
-func (c *Context) URL() string {
- return c.url
-}
diff --git a/plugin/context_test.go b/plugin/context_test.go
deleted file mode 100644
index 14eaed8..0000000
--- a/plugin/context_test.go
+++ /dev/null
@@ -1,74 +0,0 @@
-package plugin
-
-import (
- "context"
- "log/slog"
- "testing"
- "time"
-
- "github.com/stretchr/testify/assert"
-)
-
-func TestContext(t *testing.T) {
- t.Parallel()
- url := "https://example.com/some/path?offset=1"
- baseURL := "https://example.com"
- logger := slog.Default().With(slog.String("source", "ctx"))
- ctx := NewContext(ContextOptions{
- URL: url,
- Logger: logger,
- Timeout: time.Minute,
- Values: map[any]any{
- "key1": "value1",
- },
- })
- defer ctx.Cancel()
-
- assert.NotNil(t, ctx.Logger())
- assert.Equal(t, ctx.Logger(), logger)
- assert.Equal(t, ctx.URL(), url)
- assert.Equal(t, ctx.BaseURL(), baseURL)
- assert.Equal(t, ctx.Value("key1"), "value1")
- assert.Nil(t, ctx.Value("notExists"))
-
- if _, ok := ctx.Deadline(); !ok {
- t.Error("deadline not set")
- }
- key := "test"
- value := "1"
- ctx.SetValue(key, value)
- if v, ok := ctx.GetValue(key); ok {
- assert.Equalf(t, v, value, "want %v, got %v", value, v)
- }
- if v := ctx.Value(key); v != value {
- t.Errorf("want %v, got %v", value, v)
- }
- ctx.ClearValue()
- assert.Nil(t, ctx.Value(key), "values should be nil")
-
- ctx.Cancel()
- assert.ErrorIs(t, ctx.Err(), context.Canceled)
-
- <-ctx.Done()
-
- ctx1 := NewContext(ContextOptions{Timeout: time.Nanosecond})
- <-ctx1.Done()
- assert.ErrorIs(t, ctx1.Err(), context.DeadlineExceeded)
-}
-
-func TestParentContext(t *testing.T) {
- t.Parallel()
- type k string
- key := k("parentKey")
- value := "foo"
- valueCtx := context.WithValue(context.Background(), key, value)
- parent, cancel := context.WithTimeout(valueCtx, time.Minute)
-
- ctx := NewContext(ContextOptions{Parent: parent})
- assert.Equal(t, value, ctx.Value(key))
- cancel()
-
- time.Sleep(time.Millisecond)
-
- assert.ErrorIs(t, ctx.Err(), context.Canceled)
-}
diff --git a/plugin/go.mod b/plugin/go.mod
deleted file mode 100644
index 225d4e8..0000000
--- a/plugin/go.mod
+++ /dev/null
@@ -1,13 +0,0 @@
-module github.com/shiroyk/cloudcat/plugin
-
-go 1.21
-
-require github.com/stretchr/testify v1.8.4
-
-require (
- github.com/davecgh/go-spew v1.1.1 // indirect
- github.com/kr/pretty v0.3.1 // indirect
- github.com/pmezard/go-difflib v1.0.0 // indirect
- gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
- gopkg.in/yaml.v3 v3.0.1 // indirect
-)
diff --git a/plugin/go.sum b/plugin/go.sum
deleted file mode 100644
index 9b7f750..0000000
--- a/plugin/go.sum
+++ /dev/null
@@ -1,22 +0,0 @@
-github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
-github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
-github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
-github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
-github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
-github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
-github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
-github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
-github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
-github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
-gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
-gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
-gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
diff --git a/plugin/internal/ext/extension.go b/plugin/internal/ext/extension.go
deleted file mode 100644
index 3de4dbc..0000000
--- a/plugin/internal/ext/extension.go
+++ /dev/null
@@ -1,176 +0,0 @@
-// Package ext the extension manager
-package ext
-
-import (
- "encoding/json"
- "errors"
- "fmt"
- "reflect"
- "runtime"
- "runtime/debug"
- "strings"
- "sync"
-)
-
-var (
- mx sync.RWMutex
- extensions = make(map[ExtensionType]map[string]*Extension)
-)
-
-// ExtensionType The type of extension
-type ExtensionType uint
-
-const (
- // JSExtension The modules.Module
- JSExtension ExtensionType = iota + 1
- // ParserExtension The parser.Parser.
- ParserExtension
-)
-
-func (e ExtensionType) String() string {
- switch e {
- case JSExtension:
- return "js"
- case ParserExtension:
- return "parser"
- default:
- return ""
- }
-}
-
-// Extension a generic container.
-type Extension struct {
- Name, Path, Version string
- Type ExtensionType
- Module any
-}
-
-func (e Extension) String() string {
- return fmt.Sprintf("%s [%s] %s %s ", e.Name, e.Type, e.Version, e.Path)
-}
-
-// MarshalJSON encodes to JSON
-func (e Extension) MarshalJSON() ([]byte, error) {
- return json.Marshal(map[string]string{
- "name": e.Name,
- "path": e.Path,
- "version": e.Version,
- "type": e.Type.String(),
- })
-}
-
-// Register a new extension with the given name and type. This function will
-// panic if an unsupported extension type is provided, or if an extension of the
-// same type and name is already registered.
-func Register(name string, typ ExtensionType, mod any) {
- mx.Lock()
- defer mx.Unlock()
-
- if mod == nil {
- panic(errors.New("extension cannot be nil"))
- }
-
- exts, ok := extensions[typ]
- if !ok {
- panic(fmt.Sprintf("unsupported extension type: %T", typ))
- }
-
- path, version := extractModuleInfo(mod)
-
- exts[name] = &Extension{
- Name: name,
- Type: typ,
- Module: mod,
- Path: path,
- Version: version,
- }
-}
-
-// Get returns all extensions of the specified type.
-func Get(typ ExtensionType) map[string]*Extension {
- mx.RLock()
- defer mx.RUnlock()
-
- exts, ok := extensions[typ]
- if !ok {
- panic(fmt.Sprintf("unsupported extension type: %T", typ))
- }
-
- result := make(map[string]*Extension, len(exts))
-
- for name, ext := range exts {
- result[name] = ext
- }
-
- return result
-}
-
-// GetName returns extension of the specified type and name.
-func GetName(typ ExtensionType, name string) (ext *Extension, ok bool) {
- mx.RLock()
- defer mx.RUnlock()
-
- exts, ok := extensions[typ]
- if !ok {
- panic(fmt.Sprintf("unsupported extension type: %T", typ))
- }
- ext, ok = exts[name]
- return
-}
-
-// GetAll returns all extensions.
-func GetAll() []*Extension {
- mx.RLock()
- defer mx.RUnlock()
-
- js, parser := extensions[JSExtension], extensions[ParserExtension]
- result := make([]*Extension, 0, len(js)+len(parser))
-
- for _, e := range js {
- result = append(result, e)
- }
- for _, e := range parser {
- result = append(result, e)
- }
-
- return result
-}
-
-// extractModuleInfo attempts to return the package path and version of the Go
-// module that created the given value.
-func extractModuleInfo(mod any) (path, version string) {
- t := reflect.TypeOf(mod)
-
- switch t.Kind() {
- case reflect.Ptr, reflect.Struct:
- if t.Elem() != nil {
- path = t.Elem().PkgPath()
- }
- case reflect.Func:
- path = runtime.FuncForPC(reflect.ValueOf(mod).Pointer()).Name()
- default:
- return
- }
-
- buildInfo, ok := debug.ReadBuildInfo()
- if !ok {
- return
- }
-
- for _, dep := range buildInfo.Deps {
- depPath := strings.TrimSpace(dep.Path)
- if strings.HasPrefix(path, depPath) {
- if dep.Replace != nil {
- return depPath, dep.Replace.Version
- }
- return depPath, dep.Version
- }
- }
-
- return
-}
-
-func init() {
- extensions[JSExtension] = make(map[string]*Extension)
- extensions[ParserExtension] = make(map[string]*Extension)
-}
diff --git a/plugin/jsmodule/module.go b/plugin/jsmodule/module.go
deleted file mode 100644
index f71f0f9..0000000
--- a/plugin/jsmodule/module.go
+++ /dev/null
@@ -1,44 +0,0 @@
-// Package jsmodule the JS module
-package jsmodule
-
-import (
- "github.com/shiroyk/cloudcat/plugin/internal/ext"
-)
-
-const (
- // ExtPrefix common module prefix
- ExtPrefix = "cloudcat/"
-)
-
-// Module is what a module needs to return
-type Module interface {
- Exports() any // module instance
-}
-
-// Global is it a global module
-// When the module implements the interface it will be loaded into the global
-// when the js.VM is initialized.
-type Global interface {
- Module
- Global() // is it a global module
-}
-
-// Register the given mod as an external JavaScript module that can be imported
-// by name.
-func Register(name string, mod Module) {
- if _, ok := mod.(any).(Global); !ok {
- name = ExtPrefix + name
- }
- ext.Register(name, ext.JSExtension, mod)
-}
-
-func GetModule(name string) (Module, bool) {
- if m, ok := ext.GetName(ext.JSExtension, name); ok {
- return m.Module.(Module), true
- }
- return nil, false
-}
-
-func AllModules() map[string]*ext.Extension {
- return ext.Get(ext.JSExtension)
-}
diff --git a/plugin/jsmodule/module_test.go b/plugin/jsmodule/module_test.go
deleted file mode 100644
index 22c6067..0000000
--- a/plugin/jsmodule/module_test.go
+++ /dev/null
@@ -1,34 +0,0 @@
-package jsmodule
-
-import (
- "testing"
-)
-
-type testModule struct{}
-
-func (t testModule) Exports() any { return map[string]string{"foo": "module"} }
-
-type testGlobalModule struct{}
-
-func (t testGlobalModule) Exports() any { return map[string]string{"foo": "global"} }
-func (t testGlobalModule) Global() {}
-
-func TestModule(t *testing.T) {
- t.Parallel()
-
- moduleKey := "testModule"
- if _, ok := GetModule(ExtPrefix + moduleKey); !ok {
- Register(moduleKey, new(testModule))
- }
- if _, ok := GetModule(ExtPrefix + moduleKey); !ok {
- t.Fatal("unable get module")
- }
-
- globalModuleKey := "testModule"
- if _, ok := GetModule(globalModuleKey); !ok {
- Register(globalModuleKey, new(testGlobalModule))
- }
- if _, ok := GetModule(globalModuleKey); !ok {
- t.Fatal("unable get global module")
- }
-}
diff --git a/plugin/parser/parser.go b/plugin/parser/parser.go
deleted file mode 100644
index 220fc3f..0000000
--- a/plugin/parser/parser.go
+++ /dev/null
@@ -1,56 +0,0 @@
-// Package parser the schema parser
-package parser
-
-import (
- "github.com/shiroyk/cloudcat/plugin"
- "github.com/shiroyk/cloudcat/plugin/internal/ext"
-)
-
-// Parser the content schema
-type Parser interface {
- // GetString gets the string of the content with the given arguments.
- // e.g.:
- //
- // content := ``
- // GetString(ctx, content, "ul li") returns "1\n2"
- //
- GetString(*plugin.Context, any, string) (string, error)
- // GetStrings gets the strings of the content with the given arguments.
- // e.g.:
- //
- // content := ``
- // GetStrings(ctx, content, "ul li") returns []string{"1", "2"}
- //
- GetStrings(*plugin.Context, any, string) ([]string, error)
- // GetElement gets the element of the content with the given arguments.
- // e.g.:
- //
- // content := ``
- // GetElement(ctx, content, "ul li") returns "1 \n2 "
- //
- GetElement(*plugin.Context, any, string) (string, error)
- // GetElements gets the elements of the content with the given arguments.
- // e.g.:
- //
- // content := ``
- // GetElements(ctx, content, "ul li") returns []string{"1 ", "2 "}
- //
- GetElements(*plugin.Context, any, string) ([]string, error)
-}
-
-// Register registers the Parser with the given key Parser
-func Register(key string, parser Parser) {
- ext.Register(key, ext.ParserExtension, parser)
-}
-
-// GetParser returns a Parser with the given key
-func GetParser(key string) (Parser, bool) {
- if p, ok := ext.GetName(ext.ParserExtension, key); ok {
- return p.Module.(Parser), true
- }
- return nil, false
-}
-
-func AllParsers() map[string]*ext.Extension {
- return ext.Get(ext.ParserExtension)
-}
diff --git a/plugin/parser/parser_test.go b/plugin/parser/parser_test.go
deleted file mode 100644
index 276bd98..0000000
--- a/plugin/parser/parser_test.go
+++ /dev/null
@@ -1,35 +0,0 @@
-package parser
-
-import (
- "testing"
-
- "github.com/shiroyk/cloudcat/plugin"
-)
-
-type testParser struct{}
-
-func (t *testParser) GetString(*plugin.Context, any, string) (string, error) {
- return "", nil
-}
-
-func (t *testParser) GetStrings(*plugin.Context, any, string) ([]string, error) {
- return nil, nil
-}
-
-func (t *testParser) GetElement(*plugin.Context, any, string) (string, error) {
- return "", nil
-}
-
-func (t *testParser) GetElements(*plugin.Context, any, string) ([]string, error) {
- return nil, nil
-}
-
-func TestRegister(t *testing.T) {
- t.Parallel()
- if _, ok := GetParser("test"); !ok {
- Register("test", new(testParser))
- }
- if _, ok := GetParser("test"); !ok {
- t.Fatal("unable get parser")
- }
-}
diff --git a/plugin/plugin.go b/plugin/plugin.go
deleted file mode 100644
index 86a5ea0..0000000
--- a/plugin/plugin.go
+++ /dev/null
@@ -1,8 +0,0 @@
-package plugin
-
-import (
- "github.com/shiroyk/cloudcat/plugin/internal/ext"
-)
-
-// GetAll returns all plugins.
-func GetAll() []*ext.Extension { return ext.GetAll() }
diff --git a/plugin/plugin_unix.go b/plugin/plugin_unix.go
deleted file mode 100644
index 9b61dbd..0000000
--- a/plugin/plugin_unix.go
+++ /dev/null
@@ -1,31 +0,0 @@
-//go:build unix
-
-package plugin
-
-import (
- "errors"
- "fmt"
- "os"
- "path/filepath"
- "plugin"
-)
-
-func LoadPlugin(dir string) (size int, err error) {
- entries, err := os.ReadDir(dir)
- if err != nil {
- return size, err
- }
- loadErr := make([]error, 0)
- for _, entry := range entries {
- if entry.IsDir() || filepath.Ext(entry.Name()) != ".so" {
- continue
- }
- _, err = plugin.Open(filepath.Join(dir, entry.Name()))
- if err != nil {
- loadErr = append(loadErr, fmt.Errorf("error opening %s: %v", entry.Name(), err))
- continue
- }
- size++
- }
- return size, errors.Join(loadErr...)
-}
diff --git a/plugin/plugin_windows.go b/plugin/plugin_windows.go
deleted file mode 100644
index 0e5eac8..0000000
--- a/plugin/plugin_windows.go
+++ /dev/null
@@ -1,12 +0,0 @@
-//go:build windows
-
-package plugin
-
-import (
- "log/slog"
-)
-
-func LoadPlugin(dir string) (size int, err error) {
- slog.Warn("plugin are only supported on Linux, FreeBSD, and macOS. see https://pkg.go.dev/plugin")
- return
-}
diff --git a/parsers/regex/README.md b/regex/README.md
similarity index 100%
rename from parsers/regex/README.md
rename to regex/README.md
diff --git a/regex/regex.go b/regex/regex.go
new file mode 100644
index 0000000..ea591b6
--- /dev/null
+++ b/regex/regex.go
@@ -0,0 +1,294 @@
+// Package regex the regexp executor
+package regex
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "math"
+ "strconv"
+ "strings"
+
+ "github.com/dlclark/regexp2"
+ "github.com/shiroyk/ski"
+)
+
+func init() {
+ ski.Register("regex.replace", new_replace())
+ ski.Register("regex.match", new_match())
+ ski.Register("regex.assert", new_assert())
+}
+
+type tokenState int
+
+const (
+ commonState tokenState = iota
+ searchState
+ replaceState
+ flagState
+)
+
+var reOptMap = map[string]regexp2.RegexOptions{
+ "i": regexp2.IgnoreCase,
+ "m": regexp2.Multiline,
+ "n": regexp2.ExplicitCapture,
+ "c": regexp2.Compiled,
+ "s": regexp2.Singleline,
+ "x": regexp2.IgnorePatternWhitespace,
+ "r": regexp2.RightToLeft,
+ "d": regexp2.Debug,
+ "e": regexp2.ECMAScript,
+ "u": regexp2.Unicode,
+}
+
+type _replace struct {
+ *regexp2.Regexp
+ replace string
+ start, count int
+}
+
+func new_replace() ski.NewExecutor {
+ return ski.StringExecutor(func(str string) (ski.Executor, error) {
+ re, replace, start, count, err := Compile(str)
+ if err != nil {
+ return nil, err
+ }
+ s, err := strconv.Atoi(start)
+ if err != nil {
+ s = -1
+ }
+ c, err := strconv.Atoi(count)
+ if err != nil {
+ c = -1
+ }
+ return _replace{re, replace, s, c}, nil
+ })
+}
+
+func (r _replace) Exec(_ context.Context, arg any) (any, error) {
+ switch conv := arg.(type) {
+ case string:
+ return r.Replace(conv, r.replace, r.start, r.count)
+ case ski.Iterator:
+ if conv.Len() == 0 {
+ return nil, nil
+ }
+ _, ok := conv.At(0).(string)
+ if !ok {
+ return nil, fmt.Errorf("regex.replace unsupported type %T", arg)
+ }
+ ret := make([]string, 0, conv.Len())
+ for i := 0; i < conv.Len(); i++ {
+ v, err := r.Replace(conv.At(i).(string), r.replace, r.start, r.count)
+ if err != nil {
+ return nil, err
+ }
+ ret = append(ret, v)
+ }
+ return ret, nil
+ case fmt.Stringer:
+ return r.Replace(conv.String(), r.replace, r.start, r.count)
+ case nil:
+ return nil, nil
+ default:
+ return nil, fmt.Errorf("regex.replace unsupported type %T", arg)
+ }
+}
+
+type _match struct {
+ *regexp2.Regexp
+ start, end int
+}
+
+func new_match() ski.NewExecutor {
+ return ski.StringExecutor(func(str string) (ski.Executor, error) {
+ re, _, start, end, err := Compile(str)
+ if err != nil {
+ return nil, err
+ }
+ s, _ := strconv.Atoi(start)
+ e, _ := strconv.Atoi(end)
+ return _match{re, s, e}, nil
+ })
+}
+
+func (r _match) Exec(_ context.Context, arg any) (any, error) {
+ var str string
+ switch t := arg.(type) {
+ case nil:
+ return nil, nil
+ case string:
+ str = t
+ case []string:
+ str = t[0]
+ case fmt.Stringer:
+ str = t.String()
+ default:
+ return nil, fmt.Errorf("regex.match unsupported type %T", arg)
+ }
+
+ all := r.findAllString(str)
+ if len(all) == 0 {
+ return nil, nil
+ }
+
+ // get the groups from start to end
+ start, end := r.start, r.end
+ if start < 0 {
+ start += len(all)
+ }
+ if end == math.MaxInt {
+ end = len(all)
+ } else if end < 0 {
+ end += len(all)
+ }
+ if start > len(all) || end > len(all) || start < 0 || end < 0 {
+ return nil, nil
+ }
+ if start >= end {
+ return all[start], nil
+ }
+
+ return all[start : end+1], nil
+}
+
+func (r _match) findAllString(s string) []string {
+ var matches []string
+ m, _ := r.FindStringMatch(s)
+ for m != nil {
+ matches = append(matches, m.String())
+ m, _ = r.FindNextMatch(m)
+ }
+ return matches
+}
+
+type _assert struct {
+ *regexp2.Regexp
+ err error
+}
+
+func new_assert() ski.NewExecutor {
+ return ski.StringExecutor(func(str string) (ski.Executor, error) {
+ re, msg, _, _, err := Compile(str)
+ if err != nil {
+ return nil, err
+ }
+ var ret _assert
+ ret.Regexp = re
+ if len(msg) > 0 {
+ ret.err = errors.New(msg)
+ }
+ return ret, nil
+ })
+}
+
+func (r _assert) assert(str string) error {
+ ok, err := r.MatchString(str)
+ if err != nil {
+ if r.err != nil {
+ return r.err
+ }
+ return fmt.Errorf(`assert failed %s`, err)
+ }
+ if !ok {
+ if r.err != nil {
+ return r.err
+ }
+ return errors.New(`assert failed`)
+ }
+ return nil
+}
+
+func (r _assert) Exec(_ context.Context, arg any) (any, error) {
+ var err error
+ switch conv := arg.(type) {
+ case string:
+ err = r.assert(conv)
+ case []string:
+ for _, str := range conv {
+ err = r.assert(str)
+ if err != nil {
+ return nil, err
+ }
+ }
+ case fmt.Stringer:
+ err = r.assert(conv.String())
+ default:
+ return nil, fmt.Errorf("regex.assert unsupported type %T", arg)
+ }
+ if err != nil {
+ return nil, err
+ }
+ return arg, nil
+}
+
+// Compile the pattern `/regex/replace/options{start,count}` or `/regex/options{start,count}`
+func Compile(arg string) (re *regexp2.Regexp, replace string, start, count string, err error) {
+ state := commonState
+ pattern := strings.Builder{}
+ pattern.Grow(len(arg))
+ var offset int
+ var regex string
+ var reOpt int32
+
+ for offset < len(arg) {
+ ch := arg[offset]
+ offset++
+ switch ch {
+ default:
+ if state == flagState {
+ if i, ok := reOptMap[string(ch)]; ok {
+ reOpt |= int32(i)
+ } else if ch >= '0' && ch <= '9' || ch == '-' || ch == ',' {
+ pattern.WriteByte(ch)
+ }
+ } else {
+ pattern.WriteByte(ch)
+ }
+ case '\\':
+ if nextCh := arg[offset]; nextCh == '/' {
+ pattern.WriteByte(nextCh)
+ offset++
+ } else {
+ pattern.WriteByte(ch)
+ }
+ case '/':
+ switch state {
+ case commonState:
+ state = searchState
+ case searchState:
+ state = replaceState
+ regex = pattern.String()
+ pattern.Reset()
+ case replaceState:
+ state = flagState
+ replace = pattern.String()
+ pattern.Reset()
+ default:
+ err = fmt.Errorf("/ character must escaped")
+ return
+ }
+ }
+ }
+
+ if state == replaceState && pattern.Len() > 0 {
+ flags := pattern.String()
+ pattern.Reset()
+ for i := 0; i < len(flags); i++ {
+ ch := flags[i]
+ if f, ok := reOptMap[string(ch)]; ok {
+ reOpt |= int32(f)
+ } else if ch >= '0' && ch <= '9' || ch == '-' || ch == ',' {
+ pattern.WriteByte(ch)
+ }
+ }
+ }
+
+ if pattern.Len() > 0 {
+ start, count, _ = strings.Cut(pattern.String(), ",")
+ pattern.Reset()
+ }
+
+ re, err = regexp2.Compile(regex, regexp2.RegexOptions(reOpt))
+ return
+}
diff --git a/regex/regex_test.go b/regex/regex_test.go
new file mode 100644
index 0000000..9b5ba5f
--- /dev/null
+++ b/regex/regex_test.go
@@ -0,0 +1,103 @@
+package regex
+
+import (
+ "context"
+ "testing"
+
+ "github.com/shiroyk/ski"
+ "github.com/stretchr/testify/assert"
+)
+
+func TestReplace(t *testing.T) {
+ t.Parallel()
+ testCases := []struct{ re, str, want string }{
+ {`/[0-9]/`, `114i`, "i"},
+ {`/[0-9]/i/`, `114`, "iii"},
+ {`/\\//`, `1/`, "1"},
+ {`/[a-z]/1/`, `aaa`, "111"},
+ {`/olang/olang/i`, `GoLAnG`, "Golang"},
+ {`/[^ ]+\s(?)/${time}/`, `08/10/99 16:00`, "16:00"},
+ {`/D\.(.+)/David $1/`, `D.Bau`, "David Bau"},
+ {`/a/b/{0,2}`, `aaaaa`, "bbaaa"},
+ {`/a/b/{3,2}`, `aaaaa`, "aaabb"},
+ {`/[a-z]/1/i{3,3}`, `aaaBBB`, "aaa111"},
+ {`/a/stuff/{-1,-1}`, `a test a blah and a`, "stuff test stuff blstuffh stuffnd stuff"},
+ {`/(\p{Sc}\s?)?(\d+\.?((?<=\.)\d+)?)(?(1)|\s?\p{Sc})?/$2/`, `$17.43 €2 16.33 £0.98 0.43 £43 12€ 17`, "17.43 2 16.33 0.98 0.43 43 12 17"},
+ }
+ for _, testCase := range testCases {
+ t.Run(testCase.re, func(t *testing.T) {
+ exec, err := new_replace()(ski.String(testCase.re))
+ if err != nil {
+ t.Fatal(err)
+ }
+ v, err := exec.Exec(context.Background(), testCase.str)
+ if err != nil {
+ t.Fatal(err)
+ }
+ assert.Equal(t, testCase.want, v)
+ })
+ }
+}
+
+func TestMatch(t *testing.T) {
+ t.Parallel()
+ testCases := []struct {
+ re, str string
+ want any
+ }{
+ {`/\d+/`, `114a514b1919`, "114"},
+ {`/\d+/1`, `114a514b1919`, "514"},
+ {`/\d+/-1`, `114a514b1919`, "1919"},
+ {`/\d+/-1919`, `114a514b1919`, nil},
+ {`/\d+/{-114,-1919}`, `114a514b1919`, nil},
+ {`/\d+/{0,1}`, `114a514b1919`, []string{"114", "514"}},
+ {`/\d+/{5,6}`, `114a514b1919`, nil},
+ {`/[a-z]+/{0,2}`, `114a514b1919c810`, []string{"a", "b", "c"}},
+ {`/[a-z]+/i{1,2}`, `114A514B1919C810`, []string{"B", "C"}},
+ {`/\\//`, `1/`, "/"},
+ {`/[a-z]/`, `aaa`, "a"},
+ {`/[a-z]+/`, `ABCxyz`, "xyz"},
+ }
+ for _, testCase := range testCases {
+ t.Run(testCase.re, func(t *testing.T) {
+ exec, err := new_match()(ski.String(testCase.re))
+ if err != nil {
+ t.Fatal(err)
+ }
+ v, err := exec.Exec(context.Background(), testCase.str)
+ if err != nil {
+ t.Fatal(err)
+ }
+ assert.Equal(t, testCase.want, v)
+ })
+ }
+}
+
+func TestAssert(t *testing.T) {
+ t.Parallel()
+ testCases := []struct {
+ re, str string
+ valid bool
+ }{
+ {`/[0-9]/`, `114i`, true},
+ {`/[a-z]/`, `abc`, true},
+ {`/[a-z]/i`, `ABC`, true},
+ {`/[a-z]/`, `123`, false},
+ {`/[a-z]/check failed/`, `123`, false},
+ {`/[a-z]/check failed/i`, `123`, false},
+ }
+ for _, testCase := range testCases {
+ t.Run(testCase.re, func(t *testing.T) {
+ exec, err := new_assert()(ski.String(testCase.re))
+ if err != nil {
+ t.Fatal(err)
+ }
+ _, err = exec.Exec(context.Background(), testCase.str)
+ if testCase.valid {
+ assert.NoError(t, err)
+ } else {
+ assert.NotNil(t, err)
+ }
+ })
+ }
+}
diff --git a/sample/env/README.md b/sample/env/README.md
deleted file mode 100644
index 6445e45..0000000
--- a/sample/env/README.md
+++ /dev/null
@@ -1,14 +0,0 @@
-# Env Plugin
-A cloudcat js plugin for reading environment variables.
-### Build the plugin
-```shell
-go build -buildmode=plugin -o env.so
-```
-### Plugin usage
-```shell
-export FOO=BAR
-cat << EOF | cloudcat -p $(pwd) -d -s -
-require("cloudcat/env").get("FOO")
-EOF
-# "BAR"
-```
\ No newline at end of file
diff --git a/sample/env/env.go b/sample/env/env.go
deleted file mode 100644
index 2e356a9..0000000
--- a/sample/env/env.go
+++ /dev/null
@@ -1,19 +0,0 @@
-package main
-
-import (
- "os"
-
- "github.com/shiroyk/cloudcat/plugin/jsmodule"
-)
-
-type Module struct{}
-
-func (m Module) Exports() any { return new(Env) }
-
-func init() {
- jsmodule.Register("env", new(Module))
-}
-
-type Env struct{}
-
-func (e Env) Get(key string) string { return os.Getenv(key) }
diff --git a/sample/prefix/README.md b/sample/prefix/README.md
deleted file mode 100644
index b00b5cd..0000000
--- a/sample/prefix/README.md
+++ /dev/null
@@ -1,13 +0,0 @@
-# Prefix Plugin
-A cloudcat parser plugin for adding string prefix.
-### Build the plugin
-```shell
-go build -buildmode=plugin -o prefix.so
-```
-### Plugin usage
-```shell
-cat << EOF | cloudcat -p $(pwd) run -s -
-cat.getString("prefix", "...", "test");
-EOF
-# "...test"
-```
\ No newline at end of file
diff --git a/sample/prefix/prefix.go b/sample/prefix/prefix.go
deleted file mode 100644
index 195667e..0000000
--- a/sample/prefix/prefix.go
+++ /dev/null
@@ -1,38 +0,0 @@
-package main
-
-import (
- "fmt"
-
- "github.com/shiroyk/cloudcat/plugin"
- "github.com/shiroyk/cloudcat/plugin/parser"
-)
-
-type Parser struct{}
-
-func init() {
- parser.Register("prefix", new(Parser))
-}
-
-func (p Parser) GetString(_ *plugin.Context, content any, arg string) (string, error) {
- if str, ok := content.(string); ok {
- return arg + str, nil
- }
- return "", fmt.Errorf("content must be a string")
-}
-
-func (p Parser) GetStrings(_ *plugin.Context, content any, arg string) ([]string, error) {
- if str, ok := content.([]string); ok {
- for i := range str {
- str[i] = arg + str[i]
- }
- }
- return nil, fmt.Errorf("content must be a string slice")
-}
-
-func (p Parser) GetElement(_ *plugin.Context, content any, arg string) (string, error) {
- return p.GetString(nil, content, arg)
-}
-
-func (p Parser) GetElements(_ *plugin.Context, content any, arg string) ([]string, error) {
- return p.GetStrings(nil, content, arg)
-}
diff --git a/schema.go b/schema.go
index 96d5e2e..511695b 100644
--- a/schema.go
+++ b/schema.go
@@ -1,581 +1,475 @@
-package cloudcat
+package ski
import (
+ "context"
+ "encoding/json"
"errors"
"fmt"
- "slices"
+ "log/slog"
"strings"
- "github.com/shiroyk/cloudcat/plugin"
- "github.com/shiroyk/cloudcat/plugin/parser"
+ "github.com/spf13/cast"
"gopkg.in/yaml.v3"
)
-var (
- // ErrInvalidSchema invalid schema error
- ErrInvalidSchema = errors.New("invalid schema")
- // ErrInvalidAction invalid action error
- ErrInvalidAction = errors.New("invalid action")
- // ErrInvalidStep invalid step error
- ErrInvalidStep = errors.New("invalid step")
-)
+func init() {
+ Register("kind", new_kind())
+ Register("map", new_map)
+ Register("each", new_each)
+ Register("pipe", new_pipe)
+ Register("or", new_or)
+ Register("debug", new_debug)
+ Register("string.join", new_string_join)
+ Register("json.parse", new_json_parse)
+ Register("json.string", new_json_string)
+}
-// Type The property type.
-type Type string
+// Iterator is an interface for iterators
+type Iterator interface {
+ // Len returns the length of the iterator
+ Len() int
+ // At returns the element at the given index, or nil if the index is out of range
+ At(idx int) any
+}
-const (
- // StringType The Type of string.
- StringType Type = "string"
- // NumberType The Type of number.
- NumberType Type = "number"
- // IntegerType The Type of integer.
- IntegerType Type = "integer"
- // BooleanType The Type of boolean.
- BooleanType Type = "boolean"
- // ObjectType The Type of object.
- ObjectType Type = "object"
- // ArrayType The Type of array.
- ArrayType Type = "array"
-)
+// NewIterator creates a new iterator
+func NewIterator[T any](v []T) Iterator { return _iter[T](v) }
-// ToType parses the schema type.
-func ToType(s any) (Type, error) {
- switch s {
- case "string":
- return StringType, nil
- case "array":
- return ArrayType, nil
- case "object":
- return ObjectType, nil
- case "number":
- return NumberType, nil
- case "integer":
- return IntegerType, nil
- case "boolean":
- return BooleanType, nil
+type _iter[T any] []T
+
+func (i _iter[T]) Len() int { return len(i) }
+func (i _iter[T]) At(idx int) any {
+ if idx < 0 || idx >= len(i) {
+ return nil
}
- return "", fmt.Errorf("invalid type %s", s)
+ return i[idx]
}
-// Operator The Action operator.
-type Operator string
+type Kind uint
const (
- // OperatorAnd The Operator of and.
- // Action result A, B; Join the A + B.
- OperatorAnd Operator = "and"
- // OperatorOr The Operator of or.
- // Action result A, B; if result A is nil return B else return A.
- OperatorOr Operator = "or"
- // OperatorNot The Operator of not.
- // Action result A, B; if result A is not nil return B else return nil.
- OperatorNot Operator = "not"
+ KindAny Kind = iota
+ KindBool
+ KindInt // int32
+ KindInt64
+ KindFloat // float 32
+ KindFloat64
+ KindString
)
-// Schema The schema.
-type Schema struct {
- Type Type `yaml:"type"`
- Format Type `yaml:"format,omitempty"`
- Init Action `yaml:"init,omitempty"`
- Rule Action `yaml:"rule,omitempty"`
- Properties Property `yaml:"properties,omitempty"`
+func new_kind() NewExecutor {
+ return StringExecutor(func(str string) (Executor, error) {
+ var k Kind
+ if err := k.UnmarshalText([]byte(str)); err != nil {
+ return nil, err
+ }
+ return k, nil
+ })
}
-// Property The Schema property.
-type Property map[string]Schema
-
-// NewSchema returns a new Schema with the given Type.
-// The first argument is the Schema.Type, second is the Schema.Format.
-func NewSchema(types ...Type) *Schema {
- switch {
- case len(types) == 0:
- panic("schema must have type")
- case len(types) == 1:
- return &Schema{
- Type: types[0],
- }
- default:
- return &Schema{
- Type: types[0],
- Format: types[1],
- }
- }
+var kindNames = [...]string{
+ KindAny: "any",
+ KindBool: "bool",
+ KindInt: "int",
+ KindInt64: "int64",
+ KindFloat: "float",
+ KindFloat64: "float64",
+ KindString: "string",
}
-// SetProperty set the Property to Schema.Properties.
-func (schema *Schema) SetProperty(m Property) *Schema {
- schema.Properties = m
- return schema
+func (k Kind) String() string { return kindNames[k] }
+
+func (k Kind) MarshalText() (text []byte, err error) { return []byte(kindNames[k]), nil }
+
+func (k *Kind) UnmarshalText(text []byte) error {
+ switch string(text) {
+ case "", "any":
+ *k = KindAny
+ case "bool":
+ *k = KindBool
+ case "int", "int32":
+ *k = KindInt
+ case "int64":
+ *k = KindInt64
+ case "float", "float32":
+ *k = KindFloat
+ case "float64":
+ *k = KindFloat64
+ case "string":
+ *k = KindString
+ default:
+ return fmt.Errorf("unknown kind %s", text)
+ }
+ return nil
}
-// AddProperty append a field string with Schema to Schema.Properties.
-func (schema *Schema) AddProperty(field string, s Schema) *Schema {
- if schema.Properties == nil {
- property := make(map[string]Schema)
- schema.Properties = property
+func (k Kind) Exec(_ context.Context, v any) (any, error) {
+ switch k {
+ case KindBool:
+ return cast.ToBoolE(v)
+ case KindInt:
+ return cast.ToInt32E(v)
+ case KindInt64:
+ return cast.ToInt64E(v)
+ case KindFloat:
+ return cast.ToFloat32E(v)
+ case KindFloat64:
+ return cast.ToFloat64E(v)
+ case KindString:
+ return cast.ToStringE(v)
+ default:
+ return v, nil
}
+}
- schema.Properties[field] = s
+type compiler struct {
+ exec Executor
+ meta func(node *yaml.Node, exec Executor, isParser bool) Executor
+}
- return schema
+func (c compiler) newError(message string, node *yaml.Node, err error) error {
+ if err != nil {
+ message = fmt.Sprintf("%s: %s", message, err)
+ }
+ return fmt.Errorf("line %d column %d %s", node.Line, node.Column, message)
}
-// SetInit set the Init Action to Schema.Init.
-func (schema *Schema) SetInit(action Action) *Schema {
- schema.Init = action
- return schema
+// UnmarshalYAML compile the Executor from the YAML string.
+func (c *compiler) UnmarshalYAML(node *yaml.Node) error {
+ exec, err := c.compileNode(node)
+ if err != nil {
+ return err
+ }
+ switch len(exec) {
+ case 1:
+ c.exec = exec[0]
+ default:
+ c.exec = c.piping(exec)
+ }
+ return nil
}
-// SetRule set the Init Action to Schema.Rule.
-func (schema *Schema) SetRule(action Action) *Schema {
- schema.Rule = action
- return schema
+// piping return the first arg if the length is 1, else return _pipe
+func (c compiler) piping(args []Executor) Executor {
+ if len(args) == 1 {
+ return args[0]
+ }
+ return _pipe(args)
}
-// UnmarshalYAML decodes the Schema from yaml
-func (schema *Schema) UnmarshalYAML(node *yaml.Node) (err error) {
- *schema, err = buildSchema(node)
- return
+// compileExecutor return the Executor with the key and values
+func (c compiler) compileExecutor(k, v *yaml.Node) (Executor, error) {
+ key := strings.TrimPrefix(k.Value, "$")
+ init, ok := GetExecutor(key)
+ if !ok {
+ return nil, c.newError("executor not found", k, errors.New(key))
+ }
+ args, err := c.compileNode(v)
+ if err != nil {
+ return nil, err
+ }
+ exec, err := init(args...)
+ if err != nil {
+ return nil, c.newError(key, k, err)
+ }
+ if c.meta != nil {
+ return c.meta(k, exec, false), nil
+ }
+ return exec, nil
}
-// buildSchema builds a Schema
-func buildSchema(node *yaml.Node) (schema Schema, err error) {
+func (c compiler) compileNode(node *yaml.Node) ([]Executor, error) {
switch node.Kind {
- case yaml.SequenceNode:
- return buildStringSchema(node)
case yaml.MappingNode:
- typed := slices.ContainsFunc(node.Content,
- func(node *yaml.Node) bool {
- return node.Value == "type"
- })
- if typed {
- return buildTypedSchema(node)
- }
- return buildStringSchema(node)
+ return c.compileMapping(node)
+ case yaml.SequenceNode:
+ return c.compileSequence(node)
+ case yaml.ScalarNode:
+ return []Executor{String(node.Value)}, nil
case yaml.AliasNode:
- return buildSchema(node.Alias)
+ return c.compileNode(node.Alias)
default:
- err = ErrInvalidSchema
+ return nil, c.newError("invalid node type", node, nil)
}
- return
}
-// buildStringSchema builds a StringType Schema
-func buildStringSchema(node *yaml.Node) (schema Schema, err error) {
- schema.Type = StringType
- schema.Rule, err = actionDecode(node)
- if err != nil {
- return
- }
- if len(node.Tag) > 2 && node.Tag[0] == '!' && node.Tag[1] != '!' {
- tags := strings.Split(node.Tag[1:], "/")
- schema.Type, err = ToType(tags[0])
- if len(tags) > 1 {
- schema.Format, err = ToType(tags[1])
- }
+func (c compiler) compileSequence(node *yaml.Node) ([]Executor, error) {
+ args := make([]Executor, 0, len(node.Content))
+ for _, item := range node.Content {
+ items, err := c.compileNode(item)
if err != nil {
- return schema, fmt.Errorf("invalid tag %s", node.Tag)
+ return nil, err
}
+ args = append(args, c.piping(items))
}
- return
+ return args, nil
}
-// buildTypedSchema builds a specific Type Schema
-//
-//nolint:nakedret
-func buildTypedSchema(node *yaml.Node) (schema Schema, err error) {
+func (c compiler) compileMapping(node *yaml.Node) ([]Executor, error) {
+ if len(node.Content) == 0 || len(node.Content)%2 != 0 {
+ return nil, c.newError("mapping node requires at least two elements", node, nil)
+ }
+
+ if strings.HasPrefix(node.Content[0].Value, "$") {
+ ret := make([]Executor, 0, len(node.Content)/2)
+ for i := 0; i < len(node.Content); i += 2 {
+ exec, err := c.compileExecutor(node.Content[i], node.Content[i+1])
+ if err != nil {
+ return nil, err
+ }
+ ret = append(ret, exec)
+ }
+ return ret, nil
+ }
+
+ ret := make([]Executor, 0, len(node.Content)/2)
for i := 0; i < len(node.Content); i += 2 {
- field, value := node.Content[i], node.Content[i+1]
- switch field.Value {
- case "type":
- schema.Type, err = ToType(value.Value)
- case "format":
- schema.Format, err = ToType(value.Value)
- case "init":
- schema.Init, err = actionDecode(value)
- case "rule":
- schema.Rule, err = actionDecode(value)
- case "properties":
- if len(value.Content) == 2 {
- schema.Properties = make(Property, 2)
- k, v := value.Content[0], value.Content[1]
- if k.Kind == yaml.MappingNode {
- schema.Properties["$key"], err = buildSchema(k)
- schema.Properties["$value"], err = buildSchema(v)
- return
- }
- schema.Properties[k.Value], err = buildSchema(v)
- return
+ keyNode, valueNode := node.Content[i], node.Content[i+1]
+ key := String(keyNode.Value)
+
+ if valueNode.Kind != yaml.MappingNode {
+ child, err := c.compileNode(valueNode)
+ if err != nil {
+ return nil, err
}
- schema.Properties = make(Property, len(value.Content)/2)
- for j := 0; j < len(value.Content); j += 2 {
- k, v := value.Content[j], value.Content[j+1]
- schema.Properties[k.Value], err = buildSchema(v)
- if err != nil {
- return
- }
+ ret = append(ret, key, c.piping(child))
+ continue
+ }
+
+ if len(valueNode.Content) == 2 {
+ exec, err := c.compileExecutor(valueNode.Content[0], valueNode.Content[1])
+ if err != nil {
+ return nil, err
}
+ ret = append(ret, key, exec)
+ continue
}
- if err != nil {
- return
+
+ pipe := make(_pipe, 0, len(valueNode.Content)/2)
+ for j := 0; j < len(valueNode.Content); j += 2 {
+ exec, err := c.compileExecutor(valueNode.Content[j], valueNode.Content[j+1])
+ if err != nil {
+ return nil, err
+ }
+ pipe = append(pipe, exec)
}
+ ret = append(ret, key, pipe)
}
-
- return
+ return ret, nil
}
-// MarshalYAML encodes the Schema
-func (schema Schema) MarshalYAML() (any, error) {
- if schema.Type == StringType &&
- schema.Init == nil {
- return schema.Rule, nil
- }
- s := make(map[string]any, 5)
- s["type"] = schema.Type
- if schema.Format != "" {
- s["format"] = schema.Format
- }
- if schema.Init != nil {
- s["init"] = schema.Init
- }
- if schema.Rule != nil {
- s["rule"] = schema.Rule
- }
- if len(schema.Properties) > 0 {
- s["properties"] = schema.Properties
- }
- return s, nil
-}
+type Option func(*compiler)
-// MarshalText encodes the receiver into UTF-8-encoded text and returns the result.
-func (schema Schema) MarshalText() ([]byte, error) {
- if schema.Type == "" {
- return nil, nil
- }
- return yaml.Marshal(schema)
-}
+type Meta = func(node *yaml.Node, exec Executor, isParser bool) Executor
-// UnmarshalText must be able to decode the form generated by MarshalText.
-func (schema *Schema) UnmarshalText(text []byte) error {
- return yaml.Unmarshal(text, schema)
+// WithMeta with the meta message
+func WithMeta(meta Meta) Option {
+ return func(c *compiler) { c.meta = meta }
}
-// Action The Schema Action
-type Action interface {
- // Left returns the left Action
- Left() Action
- // Right returns the right Action
- Right() Action
+// Compile the Executor with the Option.
+func Compile(str string, opts ...Option) (Executor, error) {
+ c := new(compiler)
+ for _, opt := range opts {
+ opt(c)
+ }
+ if err := yaml.Unmarshal([]byte(str), c); err != nil {
+ return nil, err
+ }
+ return c.exec, nil
}
-// Step The Action of step
-type Step struct{ K, V string }
+// String the Executor for string value
+type String string
-// MarshalYAML encodes to yaml
-func (s Step) MarshalYAML() (any, error) {
- return map[string]string{s.K: s.V}, nil
-}
+func (k String) Exec(_ context.Context, _ any) (any, error) { return k.String(), nil }
-// Steps slice of Step
-type Steps []Step
+func (k String) String() string { return string(k) }
-// NewSteps return new Steps
-func NewSteps(str ...string) *Steps {
- if len(str)%2 != 0 {
- panic(ErrInvalidStep)
- }
- steps := make(Steps, 0, len(str)/2)
- for i := 0; i < len(str); i += 2 {
- steps = append(steps, Step{str[i], str[i+1]})
+type _map []Executor
+
+func new_map(args ...Executor) (Executor, error) {
+ m := _map(args)
+ if len(m)%2 != 0 {
+ m = append(m, Raw(nil))
}
- return &steps
+ return m, nil
}
-// Left returns the left Action
-func (s *Steps) Left() Action { return nil }
-
-// Right returns the right Action
-func (s *Steps) Right() Action { return nil }
+func (m _map) Exec(ctx context.Context, arg any) (any, error) {
+ var ret map[string]any
-func (s *Steps) UnmarshalYAML(value *yaml.Node) error {
- switch value.Kind {
- case yaml.MappingNode:
- *s = make(Steps, 0, len(value.Content)/2)
- for i := 0; i < len(value.Content); i += 2 {
- k, v := value.Content[i], value.Content[i+1]
- if v.Kind == yaml.AliasNode {
- v = v.Alias
+ exec := func(a any) {
+ for i := 0; i < len(m); i += 2 {
+ k, err := m[i].Exec(ctx, a)
+ if err != nil {
+ continue
}
- if k.Kind != yaml.ScalarNode || v.Kind != yaml.ScalarNode {
- return ErrInvalidStep
+ ks, err := cast.ToStringE(k)
+ if err != nil {
+ continue
}
- *s = append(*s, Step{k.Value, v.Value})
+ v, _ := m[i+1].Exec(ctx, a)
+ ret[ks] = v
}
- case yaml.SequenceNode:
- *s = make(Steps, 0, len(value.Content))
- for _, node := range value.Content {
- if node.Kind != yaml.MappingNode {
- return ErrInvalidStep
- }
- steps := new(Steps)
- if err := node.Decode(steps); err != nil {
- return err
- }
- *s = append(*s, *steps...)
+ }
+
+ switch s := arg.(type) {
+ case Iterator:
+ ret = make(map[string]any, s.Len())
+ for i := 0; i < s.Len(); i++ {
+ exec(s.At(i))
}
+ return ret, nil
default:
- return ErrInvalidStep
+ ret = make(map[string]any, len(m)/2)
+ exec(arg)
+ return ret, nil
}
- return nil
}
-func (s Steps) MarshalYAML() (any, error) {
- if len(s) == 1 {
- return s[0], nil
+type _each struct{ Executor }
+
+func new_each(args ...Executor) (Executor, error) {
+ if len(args) != 1 {
+ return nil, errors.New("each needs 1 parameter")
}
- return s, nil
+ return _each{args[0]}, nil
}
-// And Action node of Operator and
-type And struct{ l, r Action }
-
-// NewAnd create new And action with left and right Action
-func NewAnd(left, right Action) *And { return &And{left, right} }
-func (a And) Left() Action { return a.l }
-func (a And) Right() Action { return a.r }
-func (a And) String() string { return string(OperatorAnd) }
-func (a And) MarshalYAML() (any, error) { return [...]any{a.l, a.String(), a.r}, nil }
-
-// Or Action node of Operator or
-type Or struct{ l, r Action }
-
-// NewOr create new Or action with left and right Action
-func NewOr(left, right Action) *Or { return &Or{left, right} }
-func (a Or) Left() Action { return a.l }
-func (a Or) Right() Action { return a.r }
-func (a Or) String() string { return string(OperatorOr) }
-func (a Or) MarshalYAML() (any, error) { return [...]any{a.l, a.String(), a.r}, nil }
-
-// Not Action node of Operator not
-type Not struct{ l, r Action }
-
-// NewNot create new Not action with left and right Action
-func NewNot(left, right Action) *Not { return &Not{left, right} }
-func (a Not) Left() Action { return a.l }
-func (a Not) Right() Action { return a.r }
-func (a Not) String() string { return string(OperatorNot) }
-func (a Not) MarshalYAML() (any, error) { return [...]any{a.l, a.String(), a.r}, nil }
-
-// actionDecode decodes the Action from yaml.node
-func actionDecode(value *yaml.Node) (ret Action, err error) {
- if value.Kind == yaml.DocumentNode {
- return nil, ErrInvalidAction
- }
- if value.Kind == yaml.AliasNode {
- value = value.Alias
- }
- multiStep := value.Kind == yaml.SequenceNode &&
- !slices.ContainsFunc(value.Content, func(e *yaml.Node) bool {
- return e.Kind == yaml.ScalarNode
- })
- if value.Kind == yaml.MappingNode || multiStep {
- steps := new(Steps)
- if err = value.Decode(steps); err != nil {
- return
+func (each _each) Exec(ctx context.Context, arg any) (any, error) {
+ switch s := arg.(type) {
+ case Iterator:
+ ret := make([]any, 0, s.Len())
+ for i := 0; i < s.Len(); i++ {
+ v, _ := each.Executor.Exec(ctx, s.At(i))
+ ret = append(ret, v)
+ }
+ return NewIterator(ret), nil
+ default:
+ v, err := each.Executor.Exec(ctx, arg)
+ if err != nil {
+ return nil, nil
}
- return steps, nil
+ return NewIterator([]any{v}), nil
}
+}
+
+// Raw the Executor for raw value, return the original value
+func Raw(arg any) Executor { return _raw{arg} }
+
+type _raw struct{ any }
+
+func (raw _raw) Exec(context.Context, any) (any, error) { return raw.any, nil }
- var op string
- var left Action
- for _, node := range value.Content {
- switch node.Kind {
- case yaml.MappingNode, yaml.SequenceNode:
- var leaf Action
- leaf, err = actionDecode(node)
+type _pipe []Executor
+
+func new_pipe(args ...Executor) (Executor, error) { return _pipe(args), nil }
+
+func (pipe _pipe) Exec(ctx context.Context, v any) (any, error) {
+ switch len(pipe) {
+ case 0:
+ return nil, nil
+ case 1:
+ return pipe[0].Exec(ctx, v)
+ default:
+ ret, err := pipe[0].Exec(ctx, v)
+ if err != nil || ret == nil {
+ return nil, err
+ }
+ for _, s := range pipe[1:] {
+ ret, err = s.Exec(ctx, ret)
if err != nil {
- return
- }
- if left == nil {
- left = leaf
- continue
+ return nil, err
}
- ret, err = toActionOp(op, left, leaf)
- left = nil
- case yaml.ScalarNode:
- op = node.Value
- default:
- continue
}
+ return ret, nil
+ }
+}
+
+type _or []Executor
+
+func new_or(args ...Executor) (Executor, error) { return _or(args), nil }
+func (or _or) Exec(ctx context.Context, arg any) (any, error) {
+ for _, exec := range or {
+ v, err := exec.Exec(ctx, arg)
if err != nil {
- return
+ continue
+ }
+ if v != nil {
+ return v, nil
}
}
+ return nil, nil
+}
+
+type _debug string
- if left != nil {
- return toActionOp(op, ret, left)
+func new_debug(args ...Executor) (Executor, error) {
+ if len(args) > 0 {
+ return _debug(ExecToString(args[0])), nil
}
+ return _debug(""), nil
+}
- return
+func (debug _debug) Exec(ctx context.Context, v any) (any, error) {
+ Logger(ctx).LogAttrs(ctx, slog.LevelDebug, string(debug), slog.Any("value", v))
+ return v, nil
}
-// toActionOp parser the Operator string returns an operator Action
-func toActionOp(op string, left, right Action) (Action, error) {
- switch Operator(strings.ToLower(op)) {
- case OperatorAnd:
- return NewAnd(left, right), nil
- case OperatorOr:
- return NewOr(left, right), nil
- case OperatorNot:
- return NewNot(left, right), nil
- default:
- return nil, fmt.Errorf("invalid operator %v", op)
+type _string_join string
+
+func new_string_join(args ...Executor) (Executor, error) {
+ if len(args) > 0 {
+ return _string_join(ExecToString(args[0])), nil
}
+ return _string_join(""), nil
}
-// GetString run the action returns a string
-func GetString(ctx *plugin.Context, node Action, content any) (string, error) {
- ret, err := WalkAction(node, func(steps Steps) ([]string, error) {
- var err error
- cur := content
- for _, step := range steps {
- p, ok := parser.GetParser(step.K)
- if !ok {
- return nil, fmt.Errorf("parser %s not found", step.K)
- }
- cur, err = p.GetString(ctx, cur, step.V)
+func (sep _string_join) Exec(_ context.Context, arg any) (any, error) {
+ switch s := arg.(type) {
+ case Iterator:
+ ret := make([]string, 0, s.Len())
+ for i := 0; i < s.Len(); i++ {
+ s, err := cast.ToStringE(s.At(i))
if err != nil {
- return nil, fmt.Errorf("parser %s: %s", step.K, err)
+ return nil, err
}
+ ret = append(ret, s)
}
- ret := cur.(string)
- if len(ret) == 0 {
- return nil, nil
- }
- return []string{ret}, nil
- })
- if err != nil {
- return "", err
+ return strings.Join(ret, string(sep)), nil
+ case []string:
+ return strings.Join(s, string(sep)), nil
+ case string:
+ return s, nil
+ default:
+ return nil, fmt.Errorf("expected string or []string, but got type %T", arg)
}
- return strings.Join(ret, ""), nil
}
-// GetStrings run the action returns a slice of string
-func GetStrings(ctx *plugin.Context, node Action, content any) ([]string, error) {
- return WalkAction(node, func(steps Steps) ([]string, error) {
- var err error
- ret := content
- for _, step := range steps {
- p, ok := parser.GetParser(step.K)
- if !ok {
- return nil, fmt.Errorf("parser %s not found", step.K)
- }
- ret, err = p.GetStrings(ctx, ret, step.V)
- if err != nil {
- return nil, fmt.Errorf("parser %s: %s", step.K, err)
- }
- }
- return ret.([]string), nil
- })
-}
+type _json_parse struct{}
-// GetElement run the action returns an element string
-func GetElement(ctx *plugin.Context, node Action, content any) (string, error) {
- ret, err := WalkAction(node, func(steps Steps) ([]string, error) {
- var err error
- cur := content
- for _, step := range steps {
- p, ok := parser.GetParser(step.K)
- if !ok {
- return nil, fmt.Errorf("parser %s not found", step.K)
- }
- cur, err = p.GetElement(ctx, cur, step.V)
- if err != nil {
- return nil, fmt.Errorf("parser %s: %s", step.K, err)
- }
- }
- ret := cur.(string)
- if len(ret) == 0 {
- return nil, nil
- }
- return []string{ret}, nil
- })
+func new_json_parse(_ ...Executor) (Executor, error) { return _json_parse{}, nil }
+
+func (_json_parse) Exec(_ context.Context, v any) (any, error) {
+ s, err := cast.ToStringE(v)
if err != nil {
- return "", err
+ return nil, err
}
- return strings.Join(ret, ""), nil
+ var ret any
+ err = json.Unmarshal([]byte(s), &ret)
+ return ret, err
}
-// GetElements run the action returns a slice of element string
-func GetElements(ctx *plugin.Context, node Action, content any) ([]string, error) {
- return WalkAction(node, func(steps Steps) ([]string, error) {
- var err error
- ret := content
- for _, step := range steps {
- p, ok := parser.GetParser(step.K)
- if !ok {
- return nil, fmt.Errorf("parser %s not found", step.K)
- }
- ret, err = p.GetElements(ctx, ret, step.V)
- if err != nil {
- return nil, fmt.Errorf("parser %s: %s", step.K, err)
- }
- }
- return ret.([]string), nil
- })
-}
+type _json_string struct{}
-// WalkAction preorder traversal of Action.
-func WalkAction(node Action, walk func(Steps) ([]string, error)) (ret []string, err error) {
- var left []string
- var stack []Action
- for len(stack) > 0 || node != nil {
- // traverse the left subtree and push the node to the stack
- for node != nil {
- stack = append(stack, node)
- node = node.Left()
- }
- // pop the stack and walk the node
- node = stack[len(stack)-1]
- stack = stack[:len(stack)-1]
- switch node.(type) {
- case *And:
- if len(stack) == 0 {
- // join the left subtree result to ret
- ret = append(ret, left...)
- left = nil
- }
- case *Or:
- if len(left) > 0 {
- // discard right subtree if left subtree result is not empty
- node = nil
- if len(stack) == 0 {
- ret = append(ret, left...)
- left = nil
- }
- continue
- }
- case *Not:
- if len(left) == 0 {
- // discard right subtree if left subtree result is empty
- node = nil
- continue
- }
- left = nil
- case *Steps:
- var cur []string
- cur, err = walk(*node.(*Steps))
- if err != nil {
- return nil, err
- }
- if len(stack) == 0 {
- ret = append(ret, cur...)
- } else {
- left = append(left, cur...)
- }
- }
- node = node.Right()
+func new_json_string(_ ...Executor) (Executor, error) { return _json_string{}, nil }
+
+func (_json_string) Exec(_ context.Context, v any) (any, error) {
+ data, err := json.Marshal(v)
+ if err != nil {
+ return nil, err
}
- return
+ return string(data), nil
}
diff --git a/schema_test.go b/schema_test.go
index 0dfe62b..1312781 100644
--- a/schema_test.go
+++ b/schema_test.go
@@ -1,598 +1,197 @@
-package cloudcat
+package ski
import (
+ "bytes"
+ "context"
+ "fmt"
+ "log/slog"
+ "reflect"
"strconv"
"testing"
- "github.com/shiroyk/cloudcat/plugin"
- "github.com/shiroyk/cloudcat/plugin/parser"
+ "github.com/spf13/cast"
"github.com/stretchr/testify/assert"
"gopkg.in/yaml.v3"
)
-func TestSchemaYaml(t *testing.T) {
- t.Parallel()
+type _inc struct{}
+
+func (_inc) Exec(_ context.Context, v any) (ret any, err error) {
+ return cast.ToInt(v) + 1, nil
+}
+
+type _dec struct{}
+
+func (_dec) Exec(_ context.Context, v any) (ret any, err error) {
+ return cast.ToInt(v) - 1, nil
+}
+func TestExecutor(t *testing.T) {
+ ctx := context.WithValue(context.Background(), "foo", "bar")
testCases := []struct {
- Yaml string
- Schema *Schema
+ e Executor
+ arg any
+ want any
}{
- {
- `
-{ p: foo }`, NewSchema(StringType).
- SetRule(NewSteps("p", "foo")),
- },
- {
- `
-- p: foo
- p: bar`, NewSchema(StringType).
- SetRule(NewSteps("p", "foo", "p", "bar")),
- },
- {
- `
-- p: foo
-- p: bar
-- p: title`, NewSchema(StringType).
- SetRule(NewSteps("p", "foo", "p", "bar", "p", "title")),
- },
- {
- `
-- p: foo
-- not
-- p: title`, NewSchema(StringType).
- SetRule(NewNot(NewSteps("p", "foo"), NewSteps("p", "title"))),
- },
- {
- `
-- p: foo
-- or
-- p: title`, NewSchema(StringType).
- SetRule(NewOr(NewSteps("p", "foo"), NewSteps("p", "title"))),
- },
- {
- `
-- p: foo
-- and
-- p: bar
-- or
-- p: body`, NewSchema(StringType).
- SetRule(NewOr(
- NewAnd(NewSteps("p", "foo"), NewSteps("p", "bar")),
- NewSteps("p", "body"),
- )),
- },
- {
- `
-- p: foo
-- not
-- p: bar
-- or
-- p: body`, NewSchema(StringType).
- SetRule(NewOr(
- NewNot(NewSteps("p", "foo"), NewSteps("p", "bar")),
- NewSteps("p", "body"),
- )),
- },
- {
- `
-- - p: foo
- - p: bar
-- or
-- - p: title
- - p: body`, NewSchema(StringType).
- SetRule(NewOr(NewSteps("p", "foo", "p", "bar"), NewSteps("p", "title", "p", "body"))),
- },
- {
- `
-!integer { p: foo }`, NewSchema(IntegerType).
- SetRule(NewSteps("p", "foo")),
- },
- {
- `
-type: integer
-rule: { p: foo }`, NewSchema(IntegerType).
- SetRule(NewSteps("p", "foo")),
- },
- {
- `
-type: number
-rule: { p: foo }`, NewSchema(NumberType).
- SetRule(NewSteps("p", "foo")),
- },
- {
- `
-type: boolean
-rule: { p: foo }`, NewSchema(BooleanType).
- SetRule(NewSteps("p", "foo")),
- },
- {
- `
-type: object
-properties:
- context:
- type: string
- format: boolean
- rule: { p: foo }`, NewSchema(ObjectType).
- AddProperty("context", *NewSchema(StringType, BooleanType).
- SetRule(NewSteps("p", "foo"))),
- },
- {
- `
-type: object
-properties:
- context: !string/boolean { p: foo }`, NewSchema(ObjectType).
- AddProperty("context", *NewSchema(StringType, BooleanType).
- SetRule(NewSteps("p", "foo"))),
- },
- {
- `
-type: array
-init: { p: foo }
-properties:
- context:
- type: string
- format: integer
- rule: { p: foo }`, NewSchema(ArrayType).
- SetInit(NewSteps("p", "foo")).
- AddProperty("context", *NewSchema(StringType, IntegerType).
- SetRule(NewSteps("p", "foo"))),
- },
- {
- `
-type: object
-init: { p: foo }
-properties:
- context:
- type: number
- rule: { p: foo }`, NewSchema(ObjectType).
- SetInit(NewSteps("p", "foo")).
- AddProperty("context", *NewSchema(NumberType).
- SetRule(NewSteps("p", "foo"))),
- },
- {
- `
-type: object
-init: { p: foo }
-properties:
- context: !number { p: foo }`, NewSchema(ObjectType).
- SetInit(NewSteps("p", "foo")).
- AddProperty("context", *NewSchema(NumberType).
- SetRule(NewSteps("p", "foo"))),
- },
- {
- `
-type: object
-properties:
- ? p: foo
- : p: bar`, NewSchema(ObjectType).
- AddProperty("$key", *NewSchema(StringType).
- SetRule(NewSteps("p", "foo"))).
- AddProperty("$value", *NewSchema(StringType).
- SetRule(NewSteps("p", "bar"))),
- },
- {
- `
-type: object
-properties:
- ? p: foo
- : type: integer
- rule: { p: bar }`, NewSchema(ObjectType).
- AddProperty("$key", *NewSchema(StringType).
- SetRule(NewSteps("p", "foo"))).
- AddProperty("$value", *NewSchema(IntegerType).
- SetRule(NewSteps("p", "bar"))),
- },
- {
- `
-type: object
-properties:
- $key: { p: foo }
- $value: { p: bar }`, NewSchema(ObjectType).
- AddProperty("$key", *NewSchema(StringType).
- SetRule(NewSteps("p", "foo"))).
- AddProperty("$value", *NewSchema(StringType).
- SetRule(NewSteps("p", "bar"))),
- },
- {
- `
-type: object
-init:
- - p: foo
- - not
- - p: bar
-properties:
- context:
- type: number
- rule: { p: foo }`, NewSchema(ObjectType).
- SetInit(NewNot(NewSteps("p", "foo"), NewSteps("p", "bar"))).
- AddProperty("context", *NewSchema(NumberType).
- SetRule(NewSteps("p", "foo"))),
- },
- {
- `
-type: object
-init:
- - p: foo
- - or
- - p: bar
-properties:
- context:
- type: number
- rule: { p: foo }`, NewSchema(ObjectType).
- SetInit(NewOr(NewSteps("p", "foo"), NewSteps("p", "bar"))).
- AddProperty("context", *NewSchema(NumberType).
- SetRule(NewSteps("p", "foo"))),
- },
- {
- `
-type: object
-init:
- - p: foo
- - or
- - p: bar
-properties:
- a: !number { p: foo }
- b:
- type: boolean
- rule: { p: foo }`, NewSchema(ObjectType).
- SetInit(NewOr(NewSteps("p", "foo"), NewSteps("p", "bar"))).
- AddProperty("a", *NewSchema(NumberType).
- SetRule(NewSteps("p", "foo"))).
- AddProperty("b", *NewSchema(BooleanType).
- SetRule(NewSteps("p", "foo"))),
- },
- {
- `
-type: object
-properties:
- a: !boolean { p: &p foo }
- b: { p: *p }`, NewSchema(ObjectType).
- AddProperty("a", *NewSchema(BooleanType).
- SetRule(NewSteps("p", "foo"))).
- AddProperty("b", *NewSchema(StringType).
- SetRule(NewSteps("p", "foo"))),
- },
- {
- `
-type: object
-properties:
- a: &a
- type: boolean
- rule: { p: foo }
- b: *a`, NewSchema(ObjectType).
- AddProperty("a", *NewSchema(BooleanType).
- SetRule(NewSteps("p", "foo"))).
- AddProperty("b", *NewSchema(BooleanType).
- SetRule(NewSteps("p", "foo"))),
- },
- {
- `
-type: object
-properties:
- a:
- type: array
- rule: &a
- - { p: foo }
- - and
- - { p: foo }
- b:
- type: array
- rule: *a`, NewSchema(ObjectType).
- AddProperty("a", *NewSchema(ArrayType).
- SetRule(NewAnd(NewSteps("p", "foo"), NewSteps("p", "foo")))).
- AddProperty("b", *NewSchema(ArrayType).
- SetRule(NewAnd(NewSteps("p", "foo"), NewSteps("p", "foo")))),
- },
+ {_raw{nil}, 1, nil},
+ {_map{_raw{"k"}, _raw{nil}}, 1, map[string]any{"k": nil}},
+ {_pipe{_inc{}, _inc{}, _dec{}}, 1, 2},
+ {KindInt, "1", int32(1)},
+ {_or{_raw{nil}, _raw{"b"}}, nil, "b"},
+ {_map{_raw{"k"}, _inc{}}, 0, map[string]any{"k": 1}},
+ {_each{_inc{}}, _iter[any]{"1", "2", "3"}, _iter[any]{2, 3, 4}},
+ {_map{_raw{"k"}, _inc{}}, _iter[any]{1}, map[string]any{"k": 2}},
+ {_string_join(""), _iter[any]{"1", "2", "3"}, "123"},
+ {_pipe{_each{KindString}, _string_join("")}, _iter[any]{1, 2, 3}, "123"},
+ {_pipe{_each{_inc{}}, _each{_inc{}}}, _iter[any]{1, 2, 3}, _iter[any]{3, 4, 5}},
+ {_each{_map{_raw{"k"}, _inc{}}}, _iter[any]{1}, _iter[any]{map[string]any{"k": 2}}},
+ {_map{_raw{"k"}, _json_parse{}}, `{"foo": "bar"}`, map[string]any{"k": map[string]any{"foo": "bar"}}},
}
-
- for i, test := range testCases {
+ for i, c := range testCases {
t.Run(strconv.Itoa(i), func(t *testing.T) {
- s := new(Schema)
- err := yaml.Unmarshal([]byte(test.Yaml), s)
- if err != nil {
- t.Fatal(err)
+ v, err := c.e.Exec(ctx, c.arg)
+ if assert.NoError(t, err) {
+ assert.Equal(t, c.want, v)
}
- assert.Equal(t, test.Schema, s)
})
}
}
-type testParser struct{}
-
-func (t *testParser) GetString(_ *plugin.Context, content any, arg string) (string, error) {
- if str, ok := content.(string); ok {
- if str == arg {
- return str, nil
- }
+func TestDebug(t *testing.T) {
+ data := new(bytes.Buffer)
+ ctx := WithLogger(context.Background(), slog.New(slog.NewTextHandler(data, &slog.HandlerOptions{Level: slog.LevelDebug})))
+ v, err := _pipe{_debug("before"), _inc{}, _debug("after")}.Exec(ctx, 1)
+ if assert.NoError(t, err) {
+ assert.Equal(t, v, 2)
}
- return "", nil
+ assert.Regexp(t, "msg=before value=1 | msg=after value=2", data.String())
}
-func (t *testParser) GetStrings(_ *plugin.Context, content any, arg string) ([]string, error) {
- if str, ok := content.(string); ok {
- if str == arg {
- return []string{str}, nil
- }
- }
- return nil, nil
+type meta struct {
+ exec Executor
+ line, column int
}
-func (t *testParser) GetElement(ctx *plugin.Context, content any, arg string) (string, error) {
- return t.GetString(ctx, content, arg)
+func (m *meta) Exec(ctx context.Context, arg any) (any, error) {
+ v, err := m.exec.Exec(ctx, arg)
+ if err != nil {
+ return nil, fmt.Errorf("line %d column %d: %s", m.line, m.column, err)
+ }
+ return v, nil
}
-func (t *testParser) GetElements(ctx *plugin.Context, content any, arg string) ([]string, error) {
- return t.GetStrings(ctx, content, arg)
-}
+type errexec struct{}
-type unknown struct{ act Action }
+func new_errexec(_ ...Executor) (Executor, error) { return new(errexec), nil }
-func (u *unknown) UnmarshalYAML(value *yaml.Node) (err error) {
- u.act, err = actionDecode(value)
- return
+func (errexec) Exec(context.Context, any) (any, error) {
+ return nil, fmt.Errorf("some error")
}
-func TestActions(t *testing.T) {
- t.Parallel()
-
- parser.Register("act", new(testParser))
- ctx := plugin.NewContext(plugin.ContextOptions{})
-
- testCases := []struct {
- acts string
- want any
- str bool
- }{
- {
- `
-- act: 1
-`, `1`, true,
- },
- {
- `
-- act: 2
-`, ``, true,
- },
- {
- `
-- act: 1
-- and
-- act: 1
-`, `11`, true,
- },
- {
- `
-- act: 1
-- not
-- act: 1
-`, `1`, true,
- },
- {
- `
-- act: 2
-- not
-- act: 1
-`, ``, true,
- },
- {
- `
-- act: 1
-- and
-- act: 1
-- and
-- act: 1
-`, `111`, true,
- },
- {
- `
-- act: 1
-- and
-- act: 1
-`, []string{`1`, `1`}, false,
- },
- {
- `
-- act: 2
-- or
-- act: 1
-`, `1`, true,
- },
- {
- `
-- act: 2
-- or
-- act: 2
-- or
-- act: 1
-`, `1`, true,
- },
- {
- `
-- act: 2
-- or
-- act: 1
-`, []string{`1`}, false,
- },
- {
- `
-- act: 1
-- or
-- act: 2
-- and
-- act: 1
-`, `11`, true,
- },
- {
- `
-- act: 1
-- and
-- act: 2
-- or
-- act: 1
-`, `1`, true,
- },
- {
- `
-- act: 1
-- and
-- act: 2
-- or
-- act: 1
-`, []string{`1`}, false,
- },
- {
- `
-- - act: 1
- - and
- - act: 1
-- and
-- act: 2
-- or
-- - act: 2
- - or
- - act: 1
-`, `11`, true,
- },
- {
- `
-- - act: 2
- - or
- - act: 1
-- and
-- act: 1
-- or
-- - act: 2
- - and
- - act: 1
-`, `11`, true,
- },
- {
- `
-- - act: 2
- - or
- - act: 1
-- or
-- - act: 1
- - and
- - act: 1
-`, `1`, true,
- },
- {
- `
-- - act: 1
- - or
- - act: 2
-- or
-- - act: 2
- - or
- - act: 1
-`, `1`, true,
- },
- {
- `
-- - act: 2
- - and
- - act: 2
-- or
-- - act: 2
- - or
- - act: 1
-`, `1`, true,
- },
- {
- `
-- - act: 2
- - and
- - act: 1
-- and
-- - act: 2
- - or
- - act: 1
-`, `11`, true,
- },
- {
- `
-- - act: 2
- - or
- - act: 1
-- not
-- - act: 2
- - or
- - act: 1
-`, `1`, true,
- },
- {
- `
-- - act: 2
- - and
- - act: 2
-- not
-- - act: 2
- - or
- - act: 1
-`, ``, true,
- },
- {
- `
-- - act: 1
- - and
- - act: 1
-- and
-- - act: 2
- - or
- - act: 2
- - or
- - act: 1
-`, `111`, true,
- },
- {
- `
-- - act: 1
- - and
- - act: 1
-- and
-- act: 1
-- and
-- - act: 1
- - and
- - act: 1
-`, `11111`, true,
- },
- {
- `
-- - act: 1
- - and
- - act: 1
-- and
-- act: 1
-- and
-- - act: 1
- - and
- - act: 1
-`, []string{`1`, `1`, `1`, `1`, `1`}, false,
- },
+func TestWithMetaWrap(t *testing.T) {
+ Register("error", new_errexec)
+ v, err := Compile(`$error: ...`,
+ WithMeta(func(node *yaml.Node, exec Executor, isParser bool) Executor {
+ return &meta{exec, node.Line, node.Column}
+ }))
+ if assert.NoError(t, err) {
+ _, err = v.Exec(context.Background(), ``)
+ assert.ErrorContains(t, err, "line 1 column 1: some error")
}
+}
- for i, testCase := range testCases {
- t.Run(strconv.Itoa(i), func(t *testing.T) {
- u := new(unknown)
- err := yaml.Unmarshal([]byte(testCase.acts), u)
- if err != nil {
- t.Fatal(err)
- }
- var result any
- if testCase.str {
- result, err = GetString(ctx, u.act, "1")
- if err != nil {
- t.Error(err)
- }
- } else {
- result, err = GetStrings(ctx, u.act, "1")
- if err != nil {
- t.Error(err)
- }
+// deepEqual recursively checks if the values (dereferencing pointers) are the same.
+func deepEqual(x, y any) bool {
+ v1, v2 := reflect.ValueOf(x), reflect.ValueOf(y)
+ if v1.Kind() == reflect.Ptr && v2.Kind() == reflect.Ptr {
+ return deepEqual(v1.Elem().Interface(), v2.Elem().Interface())
+ }
+ if v1.Kind() == reflect.Ptr {
+ v1 = v1.Elem()
+ x = v1.Interface()
+ }
+ if v2.Kind() == reflect.Ptr {
+ v2 = v2.Elem()
+ y = v2.Interface()
+ }
+ if v1.Kind() == reflect.Slice && v2.Kind() == reflect.Slice {
+ if v1.Len() != v2.Len() {
+ return false
+ }
+ for i := 0; i < v1.Len(); i++ {
+ if !deepEqual(v1.Index(i).Interface(), v2.Index(i).Interface()) {
+ return false
}
- assert.EqualValues(t, testCase.want, result, testCase.acts)
- })
+ }
+ return true
}
+ return reflect.DeepEqual(x, y)
+}
+
+func TestCompileBuildIn(t *testing.T) {
+ t.Parallel()
+ t.Run("top pipe 1", func(t *testing.T) {
+ expect := _pipe{
+ _map{String("title"), _debug("text")},
+ _or{String("title"), _debug("text")}}
+ exec, err := Compile(`
+$map: &alias
+ title:
+ $debug: text
+$or: *alias`)
+ if assert.NoError(t, err) {
+ assert.True(t, deepEqual(expect, exec))
+ }
+ })
+
+ t.Run("top pipe 2", func(t *testing.T) {
+ expect := _pipe{
+ _map{String("title"), _debug("text")},
+ _map{String("title"), _debug("text")}}
+ exec, err := Compile(`
+- $map: &alias
+ title:
+ $debug: text
+- $map: *alias`)
+ if assert.NoError(t, err) {
+ assert.True(t, deepEqual(expect, exec))
+ }
+ })
+
+ t.Run("mapping pipe", func(t *testing.T) {
+ expect := _map{String("size"), _pipe{_debug("the size"), KindInt64}}
+ exec, err := Compile(`
+$map:
+ size:
+ $debug: the size
+ $kind: int64`)
+ if assert.NoError(t, err) {
+ assert.True(t, deepEqual(expect, exec))
+ }
+ })
+
+ t.Run("sequence pipe", func(t *testing.T) {
+ expect := _map{String("size"), _pipe{_debug("the size"), KindInt64}}
+ exec, err := Compile(`
+$map:
+ size:
+ - $debug: the size
+ - $kind: int64`)
+ if assert.NoError(t, err) {
+ assert.True(t, deepEqual(expect, exec))
+ }
+ })
+
+ t.Run("pipe", func(t *testing.T) {
+ expect := _map{String("size"), _pipe{_debug("the size"), KindInt}}
+ exec, err := Compile(`
+$map:
+ size:
+ $pipe:
+ - $debug: the size
+ - $kind: int32`)
+ if assert.NoError(t, err) {
+ assert.True(t, deepEqual(expect, exec))
+ }
+ })
}
diff --git a/ski/README.md b/ski/README.md
new file mode 100644
index 0000000..8282705
--- /dev/null
+++ b/ski/README.md
@@ -0,0 +1,64 @@
+# ski
+
+## Install
+```shell
+go install github.com/shiroyk/ski/ski
+```
+
+## Run executor
+```shell
+cat << 'EOF' | ski -m -
+$fetch: https://news.ycombinator.com/best
+$xpath.element: //*[@id="hnmain"]/tbody/tr[3]/td/table
+$gq.elements: -> zip('.athing', '.athing + tr')
+$each:
+ $map:
+ index:
+ $gq: .rank
+ $regex: /[^\d]/
+ $kind: int
+ title:
+ $gq: .titleline>:first-child
+ by:
+ $gq: .hnuser
+ age:
+ $gq: .age
+ comments:
+ $gq: .subline>:last-child
+ $regex: /[^\d]/
+ $kind: int
+EOF
+```
+
+## Run script
+```shell
+cat << EOF | ski -s -
+import http from "ski/http";
+import gq from "executor/gq";
+
+export default () => {
+ let res = http.get('https://news.ycombinator.com/best');
+
+ const index = gq('.rank');
+ const title = gq('.titleline>:first-child');
+ const by = gq('.hnuser');
+ const age = gq('.age');
+ const comments = gq('.subline>:last-child');
+
+ const stories = gq.elements("#hnmain tbody -> slice(2) -> child('tr:not(.spacer,.morespace,:last-child)')").exec(res.text());
+ return stories?.reduce((acc, v, i, arr) => {
+ if (i % 2 === 0) {
+ let item = arr.slice(i, i + 2);
+ acc.push({
+ index: parseInt(index.exec(item)?.replace(/[^\d]+/g, ''), 10),
+ title: title.exec(item),
+ by: by.exec(item),
+ age: age.exec(item),
+ comments: parseInt(comments.exec(item)?.replace(/[^\d]+/g, ''), 10)
+ });
+ }
+ return acc;
+ }, []);
+}
+EOF
+```
\ No newline at end of file
diff --git a/ski/main.go b/ski/main.go
new file mode 100644
index 0000000..4a8bc65
--- /dev/null
+++ b/ski/main.go
@@ -0,0 +1,195 @@
+package main
+
+import (
+ "context"
+ "encoding/json"
+ "flag"
+ "fmt"
+ "io"
+ "log/slog"
+ "net/http"
+ "os"
+ "path/filepath"
+ "strings"
+ "time"
+
+ "github.com/shiroyk/ski"
+ "github.com/shiroyk/ski/js"
+
+ _ "github.com/shiroyk/ski/js/modules/cache"
+ _ "github.com/shiroyk/ski/js/modules/crypto"
+ _ "github.com/shiroyk/ski/js/modules/encoding"
+ _ "github.com/shiroyk/ski/js/modules/http"
+
+ _ "github.com/shiroyk/ski/gq"
+ _ "github.com/shiroyk/ski/jq"
+ _ "github.com/shiroyk/ski/regex"
+ _ "github.com/shiroyk/ski/xpath"
+)
+
+const defaultTimeout = time.Minute
+
+var (
+ scriptFlag = flag.String("s", "", "run script")
+ modelFlag = flag.String("m", "", "run model")
+ timeoutFlag = flag.Duration("t", defaultTimeout, "run timeout")
+ outputFlag = flag.String("o", "", "write to file instead of stdout")
+ versionFlag = flag.Bool("v", false, "output version")
+)
+
+type _fetch string
+
+func new_fetch() ski.NewExecutor {
+ return ski.StringExecutor(func(str string) (ski.Executor, error) {
+ return _fetch(str), nil
+ })
+}
+
+func (f _fetch) Exec(ctx context.Context, _ any) (any, error) {
+ method, url, found := strings.Cut(string(f), " ")
+ if !found {
+ url = string(f)
+ method = http.MethodGet
+ }
+
+ req, err := http.NewRequestWithContext(ctx, method, url, nil)
+ if err != nil {
+ return nil, err
+ }
+ req.Header.Set("User-Agent", "ski")
+
+ res, err := http.DefaultClient.Do(req)
+ if err != nil {
+ return nil, err
+ }
+
+ defer res.Body.Close()
+ data, err := io.ReadAll(res.Body)
+ if err != nil {
+ return nil, err
+ }
+ return string(data), nil
+}
+
+func runModel() (err error) {
+ var bytes []byte
+ if *modelFlag == "-" {
+ bytes, err = io.ReadAll(os.Stdin)
+ } else {
+ bytes, err = os.ReadFile(*modelFlag) //nolint:gosec
+ }
+ if err != nil {
+ return
+ }
+ fmt.Println(string(bytes))
+
+ ski.Register("fetch", new_fetch())
+
+ executor, err := ski.Compile(string(bytes))
+ if err != nil {
+ return err
+ }
+
+ timeout := defaultTimeout
+ if timeoutFlag != nil {
+ timeout = *timeoutFlag
+ }
+
+ ctx, cancel := context.WithTimeout(context.Background(), timeout)
+ defer cancel()
+
+ ret, err := executor.Exec(ski.WithLogger(ctx, slog.New(loggerHandler())), nil)
+ if err != nil {
+ return err
+ }
+
+ return outputJSON(ret)
+}
+
+func runScript() (err error) {
+ var bytes []byte
+ if *scriptFlag == "-" {
+ bytes, err = io.ReadAll(os.Stdin)
+ } else {
+ bytes, err = os.ReadFile(*scriptFlag) //nolint:gosec
+ }
+ if err != nil {
+ return
+ }
+
+ timeout := defaultTimeout
+ if timeoutFlag != nil {
+ timeout = *timeoutFlag
+ }
+
+ ctx, cancel := context.WithTimeout(context.Background(), timeout)
+ defer cancel()
+ ctx = ski.NewContext(ctx, nil)
+
+ vm, err := js.GetScheduler().Get()
+ if err != nil {
+ return err
+ }
+ module, err := vm.Loader().CompileModule("js", string(bytes))
+ if err != nil {
+ return err
+ }
+
+ ret, err := vm.RunModule(ski.WithLogger(ctx, slog.New(loggerHandler())), module)
+ if err != nil {
+ return err
+ }
+
+ v, err := js.Unwrap(ret)
+ if err != nil {
+ return err
+ }
+
+ return outputJSON(v)
+}
+
+func loggerHandler() slog.Handler {
+ return slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelDebug})
+}
+
+func outputJSON(data any) (err error) {
+ bytes, err := json.MarshalIndent(data, "", "\t")
+ if err != nil {
+ return err
+ }
+
+ if *outputFlag == "" {
+ fmt.Println(string(bytes)) //nolint:forbidigo
+ return
+ }
+
+ ext := filepath.Ext(*outputFlag)
+ if ext == "" {
+ *outputFlag += ".json"
+ }
+ return os.WriteFile(*outputFlag, bytes, 0o600)
+}
+
+func main() {
+ flag.Parse()
+
+ if *versionFlag {
+ fmt.Println(fmt.Sprintf("ski %v/%v", Version, CommitSHA))
+ os.Exit(0)
+ return
+ }
+
+ if scriptFlag != nil && *scriptFlag != "" {
+ if err := runScript(); err != nil {
+ fmt.Println(err.Error())
+ os.Exit(1)
+ }
+ } else if modelFlag != nil && *modelFlag != "" {
+ if err := runModel(); err != nil {
+ fmt.Println(err.Error())
+ os.Exit(1)
+ }
+ } else {
+ flag.Usage()
+ }
+}
diff --git a/cmd/version.go b/ski/version.go
similarity index 100%
rename from cmd/version.go
rename to ski/version.go
diff --git a/utils.go b/utils.go
index d4c280c..42c64e0 100644
--- a/utils.go
+++ b/utils.go
@@ -1,24 +1,48 @@
-package cloudcat
+package ski
import (
- "net/http"
+ "context"
+ "errors"
+ "fmt"
+ "log/slog"
)
-// ZeroOr if value is zero value returns the defaultValue
-func ZeroOr[T comparable](value, defaultValue T) T {
- var zero T
- if zero == value {
- return defaultValue
+var loggerKey byte
+
+// Logger get slog.Logger from the context
+func Logger(ctx context.Context) *slog.Logger {
+ if logger := ctx.Value(&loggerKey); logger != nil {
+ return logger.(*slog.Logger)
+ }
+ return slog.Default()
+}
+
+// WithLogger set the slog.Logger to context
+func WithLogger(ctx context.Context, logger *slog.Logger) context.Context {
+ return WithValue(ctx, &loggerKey, logger)
+}
+
+// ExecToString convert Executor to string if it implements fmt.Stringer
+func ExecToString(exec Executor) string {
+ switch t := exec.(type) {
+ case fmt.Stringer:
+ return t.String()
+ case _raw:
+ if s, ok := t.any.(string); ok {
+ return s
+ }
}
- return value
+ return ""
}
-// EmptyOr if slice is empty returns the defaultValue
-func EmptyOr[T any](value, defaultValue []T) []T {
- if len(value) == 0 {
- return defaultValue
+// StringExecutor to create a new Executor with string argument
+func StringExecutor(fn func(str string) (Executor, error)) NewExecutor {
+ return func(args ...Executor) (Executor, error) {
+ if len(args) == 0 {
+ return nil, errors.New("needs 1 string argument")
+ }
+ return fn(ExecToString(args[0]))
}
- return value
}
// MapKeys returns the keys of the map m.
@@ -40,37 +64,3 @@ func MapValues[M ~map[K]V, K comparable, V any](m M) []V {
}
return r
}
-
-// ParseCookie parses the cookie string and return a slice http.Cookie.
-func ParseCookie(cookies string) []*http.Cookie {
- header := http.Header{}
- header.Add("Cookie", cookies)
- req := http.Request{Header: header}
- return req.Cookies()
-}
-
-// ParseSetCookie parses the set-cookie strings and return a slice http.Cookie.
-func ParseSetCookie(cookies ...string) []*http.Cookie {
- header := http.Header{}
- for _, cookie := range cookies {
- header.Add("Set-Cookie", cookie)
- }
- res := http.Response{Header: header}
- return res.Cookies()
-}
-
-// CookieToString returns the slice string of the slice http.Cookie.
-func CookieToString(cookies []*http.Cookie) []string {
- switch len(cookies) {
- case 0:
- return nil
- case 1:
- return []string{cookies[0].String()}
- }
-
- ret := make([]string, 0, len(cookies))
- for _, cookie := range cookies {
- ret = append(ret, cookie.String())
- }
- return ret
-}
diff --git a/utils_test.go b/utils_test.go
deleted file mode 100644
index ae237c0..0000000
--- a/utils_test.go
+++ /dev/null
@@ -1,62 +0,0 @@
-package cloudcat
-
-import (
- "net/http"
- "testing"
- "time"
-
- "github.com/stretchr/testify/assert"
-)
-
-func TestZeroOr(t *testing.T) {
- assert.Equal(t, 1, ZeroOr(0, 1))
-}
-
-func TestEmptyOr(t *testing.T) {
- assert.Equal(t, []int{1}, EmptyOr([]int{}, []int{1}))
-}
-
-func TestParseCookie(t *testing.T) {
- maxAge := "name=Test;id=123"
- assert.Equal(t, []string{"name=Test", "id=123"}, CookieToString(ParseCookie(maxAge)))
-}
-
-func TestParseSetCookie(t *testing.T) {
- var parseCookiesTests = []struct {
- String string
- Cookies []*http.Cookie
- }{
- {
- "Cookie-1=v$1",
- []*http.Cookie{{Name: "Cookie-1", Value: "v$1", Raw: "Cookie-1=v$1"}},
- },
- {
- "NID=99=MaIh2c9H-Mzwz-; expires=Wed, 07-Jun-2023 19:52:03 GMT; path=/; domain=.google.com; HttpOnly",
- []*http.Cookie{{
- Name: "NID",
- Value: "99=MaIh2c9H-Mzwz-",
- Path: "/",
- Domain: ".google.com",
- HttpOnly: true,
- Expires: time.Date(2023, 6, 7, 19, 52, 3, 0, time.UTC),
- RawExpires: "Wed, 07-Jun-2023 19:52:03 GMT",
- Raw: "NID=99=MaIh2c9H-Mzwz-; expires=Wed, 07-Jun-2023 19:52:03 GMT; path=/; domain=.google.com; HttpOnly",
- }},
- },
- {
- ".ASPXAUTH=7E3AA; expires=Wed, 07-Jun-2023 19:58:08 GMT; path=/; HttpOnly",
- []*http.Cookie{{
- Name: ".ASPXAUTH",
- Value: "7E3AA",
- Path: "/",
- Expires: time.Date(2023, 6, 7, 19, 58, 8, 0, time.UTC),
- RawExpires: "Wed, 07-Jun-2023 19:58:08 GMT",
- HttpOnly: true,
- Raw: ".ASPXAUTH=7E3AA; expires=Wed, 07-Jun-2023 19:58:08 GMT; path=/; HttpOnly",
- }},
- },
- }
- for _, tt := range parseCookiesTests {
- assert.Equal(t, tt.Cookies, ParseSetCookie(tt.String))
- }
-}
diff --git a/parsers/xpath/README.md b/xpath/README.md
similarity index 100%
rename from parsers/xpath/README.md
rename to xpath/README.md
diff --git a/xpath/xpath.go b/xpath/xpath.go
new file mode 100644
index 0000000..d0bf0d9
--- /dev/null
+++ b/xpath/xpath.go
@@ -0,0 +1,119 @@
+// Package xpath the xpath executor
+package xpath
+
+import (
+ "context"
+ "fmt"
+ "strings"
+
+ "github.com/antchfx/htmlquery"
+ "github.com/antchfx/xpath"
+ "github.com/shiroyk/ski"
+ "golang.org/x/net/html"
+)
+
+func init() {
+ ski.Register("xpath", new_value())
+ ski.Register("xpath.element", new_element())
+ ski.Register("xpath.elements", new_elements())
+}
+
+func new_value() ski.NewExecutor {
+ return ski.StringExecutor(func(str string) (ski.Executor, error) {
+ ex, err := xpath.Compile(str)
+ if err != nil {
+ return nil, err
+ }
+ return expr{ex, value}, nil
+ })
+}
+
+func new_element() ski.NewExecutor {
+ return ski.StringExecutor(func(str string) (ski.Executor, error) {
+ ex, err := xpath.Compile(str)
+ if err != nil {
+ return nil, err
+ }
+ return expr{ex, element}, nil
+ })
+}
+
+func new_elements() ski.NewExecutor {
+ return ski.StringExecutor(func(str string) (ski.Executor, error) {
+ ex, err := xpath.Compile(str)
+ if err != nil {
+ return nil, err
+ }
+ return expr{ex, elements}, nil
+ })
+}
+
+type expr struct {
+ *xpath.Expr
+ ret func([]*html.Node) (any, error)
+}
+
+func (e expr) Exec(_ context.Context, arg any) (any, error) {
+ node, err := htmlNode(arg)
+ if err != nil {
+ return nil, err
+ }
+ return e.ret(htmlquery.QuerySelectorAll(node, e.Expr))
+}
+
+func value(nodes []*html.Node) (any, error) {
+ switch len(nodes) {
+ case 0:
+ return nil, nil
+ case 1:
+ return htmlquery.InnerText(nodes[0]), nil
+ default:
+ str := make([]string, len(nodes))
+ for i, node := range nodes {
+ str[i] = htmlquery.InnerText(node)
+ }
+ return ski.NewIterator(str), nil
+ }
+}
+
+func element(nodes []*html.Node) (any, error) {
+ if len(nodes) == 0 {
+ return nil, nil
+ }
+ return nodes[0], nil
+}
+
+func elements(nodes []*html.Node) (any, error) {
+ if len(nodes) == 0 {
+ return nil, nil
+ }
+ return ski.NewIterator(nodes), nil
+}
+
+func htmlNode(content any) (*html.Node, error) {
+ switch data := content.(type) {
+ default:
+ return nil, fmt.Errorf("unexpected type %T", content)
+ case nil:
+ return &html.Node{Type: html.DocumentNode}, nil
+ case ski.Iterator:
+ root := &html.Node{Type: html.DocumentNode}
+ if data.Len() == 0 {
+ return root, nil
+ }
+ for i := 0; i < data.Len(); i++ {
+ n, err := htmlNode(data.At(i))
+ if err != nil {
+ return nil, err
+ }
+ root.AppendChild(n)
+ }
+ return root, nil
+ case *html.Node:
+ return data, nil
+ case []string:
+ return html.Parse(strings.NewReader(strings.Join(data, "\n")))
+ case string:
+ return html.Parse(strings.NewReader(data))
+ }
+}
diff --git a/xpath/xpath_test.go b/xpath/xpath_test.go
new file mode 100644
index 0000000..0c86561
--- /dev/null
+++ b/xpath/xpath_test.go
@@ -0,0 +1,139 @@
+package xpath
+
+import (
+ "bytes"
+ "context"
+ "fmt"
+ "testing"
+
+ "github.com/shiroyk/ski"
+ "github.com/stretchr/testify/assert"
+ "golang.org/x/net/html"
+)
+
+var (
+ ctx = context.Background()
+ content = `
+
+
+
+ Tests for siblings
+
+
+
+
1
+
2
+
3
+
4
+
5
+
6
+
Google one even row ", "two odd row ",
+ "three even row ", "four odd row ",
+ "five even row odder ", "six odd row ",
+ })
+}