From 4097cfb937a1303701ac6bf2a27cc75d80b989cf Mon Sep 17 00:00:00 2001 From: Reuben Miller Date: Sun, 18 Feb 2024 00:14:59 +0100 Subject: [PATCH 1/8] feat(pipe): support reading pretty printed json from the pipeline --- pkg/iterator/pipe.go | 19 ++++++-- pkg/stream/stream.go | 95 +++++++++++++++++++++++++++++++++++++++ pkg/stream/stream_test.go | 74 ++++++++++++++++++++++++++++++ 3 files changed, 184 insertions(+), 4 deletions(-) create mode 100644 pkg/stream/stream.go create mode 100644 pkg/stream/stream_test.go diff --git a/pkg/iterator/pipe.go b/pkg/iterator/pipe.go index 2d0e99269..064b26169 100644 --- a/pkg/iterator/pipe.go +++ b/pkg/iterator/pipe.go @@ -12,6 +12,7 @@ import ( "errors" "github.com/reubenmiller/go-c8y-cli/v2/pkg/jsonUtilities" + "github.com/reubenmiller/go-c8y-cli/v2/pkg/stream" "github.com/tidwall/gjson" ) @@ -69,6 +70,7 @@ type PipeIterator struct { filter Filter opts *PipeOptions reader *bufio.Reader + stream *stream.InputStreamer } // IsBound return true if the iterator is bound @@ -80,11 +82,14 @@ func (i *PipeIterator) IsBound() bool { func (i *PipeIterator) GetNext() (line []byte, input interface{}, err error) { i.mu.Lock() defer i.mu.Unlock() - line, err = i.reader.ReadBytes('\n') - line = bytes.TrimSpace(line) - if err != nil { - return line, line, err + line, err = i.stream.Read() + if len(line) > 0 && errors.Is(err, io.EOF) { + // Don't return io.EOF if the line includes a value + // TODO: Ideally this should be changed so the reader of the + // iterator handles io.EOF correctly (e.g. if there is still a value process it, then + // react to the io.EOF) + err = nil } if i.filter != nil { @@ -175,6 +180,9 @@ func NewPipeIterator(in io.Reader, filter ...Filter) (Iterator, error) { return &PipeIterator{ reader: reader, + stream: &stream.InputStreamer{ + Buffer: reader, + }, filter: pipelineFilter, }, nil } @@ -210,6 +218,9 @@ func NewJSONPipeIterator(in io.Reader, pipeOpts *PipeOptions, filter ...Filter) return &PipeIterator{ reader: reader, + stream: &stream.InputStreamer{ + Buffer: reader, + }, filter: pipelineFilter, opts: pipeOpts, }, nil diff --git a/pkg/stream/stream.go b/pkg/stream/stream.go new file mode 100644 index 000000000..41a63c0b0 --- /dev/null +++ b/pkg/stream/stream.go @@ -0,0 +1,95 @@ +package stream + +import ( + "bufio" + "bytes" + "io" + "unicode" +) + +// InputStreamer an input streamer which breaks down a buffer into input values +type InputStreamer struct { + Buffer *bufio.Reader +} + +func (r *InputStreamer) isJSONObject() (bool, error) { + c, _, err := r.Buffer.ReadRune() + if err == io.EOF { + return false, err + } + r.Buffer.UnreadRune() + return c == '{', err +} + +func (r *InputStreamer) consumeWhitespace() error { + for { + c, _, err := r.Buffer.ReadRune() + if err == io.EOF { + return err + } + if !unicode.IsSpace(c) { + r.Buffer.UnreadRune() + break + } + } + return nil +} + +// ReadJSONObject read the next JSON object +func (r *InputStreamer) ReadJSONObject() ([]byte, error) { + out := bytes.Buffer{} + brackets := 0 + var err error + + if err := r.consumeWhitespace(); err != nil { + return nil, err + } + + for { + c, _, rErr := r.Buffer.ReadRune() + if rErr == io.EOF { + err = io.EOF + break + } + switch c { + case '{': + brackets++ + case '}': + brackets-- + } + out.WriteRune(c) + if brackets == 0 { + break + } + } + return out.Bytes(), err +} + +// ReadLine reads the next chunk of text until the next newline char +func (r *InputStreamer) ReadLine() ([]byte, error) { + return r.Buffer.ReadBytes('\n') +} + +// Read reads the next delimited value (either text or JSON object) +func (r *InputStreamer) Read() (output []byte, err error) { + if err := r.consumeWhitespace(); err != nil { + return output, err + } + + var isJSON bool + isJSON, err = r.isJSONObject() + if err != nil { + return output, err + } + + if isJSON { + output, err = r.ReadJSONObject() + } else { + output, err = r.ReadLine() + output = bytes.TrimSpace(output) + if err != nil { + return output, err + } + } + return output, err +} diff --git a/pkg/stream/stream_test.go b/pkg/stream/stream_test.go new file mode 100644 index 000000000..b953ccc1f --- /dev/null +++ b/pkg/stream/stream_test.go @@ -0,0 +1,74 @@ +package stream + +import ( + "bufio" + "io" + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func createBuffer(v string) *bufio.Reader { + return bufio.NewReader(strings.NewReader(v)) +} + +func Test_ScanJSONObject_Consuming(t *testing.T) { + data := []struct { + Input string + Output []string + Error error + }{ + { + Input: ` + {"name":"1"} + `, + Output: []string{ + `{"name":"1"}`, + }, + Error: nil, + }, + { + Input: ` + {"name":"1"}{"name":"2"} +one +two + + `, + Output: []string{ + `{"name":"1"}`, + `{"name":"2"}`, + `one`, + `two`, + }, + Error: nil, + }, + { + Input: `{"name":{"1":{"2":{"3":{"4":{"5":"value"}}}}}}{"name":"2"} + + `, + Output: []string{ + `{"name":{"1":{"2":{"3":{"4":{"5":"value"}}}}}}`, + `{"name":"2"}`, + }, + Error: nil, + }, + { + Input: `{"name":"1"`, + Output: []string{ + `{"name":"1"`, + }, + Error: io.EOF, + }, + } + for _, testcase := range data { + s := InputStreamer{ + Buffer: createBuffer(testcase.Input), + } + + for _, iOutput := range testcase.Output { + out, _ := s.Read() + assert.Equal(t, iOutput, string(out)) + } + } +} From 430c4c3cf1d4257008be52a22df837f717eeb1b0 Mon Sep 17 00:00:00 2001 From: Reuben Miller Date: Sun, 18 Feb 2024 00:31:44 +0100 Subject: [PATCH 2/8] add system tests for mixed stdin parsing --- tests/manual/pipeline/mixed_pipeline.txt | 4 +++ tests/manual/pipeline/piping_pretty_json.yaml | 25 +++++++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 tests/manual/pipeline/mixed_pipeline.txt create mode 100644 tests/manual/pipeline/piping_pretty_json.yaml diff --git a/tests/manual/pipeline/mixed_pipeline.txt b/tests/manual/pipeline/mixed_pipeline.txt new file mode 100644 index 000000000..6fd55ed05 --- /dev/null +++ b/tests/manual/pipeline/mixed_pipeline.txt @@ -0,0 +1,4 @@ +{"name": "device01"}{"name": "device02"} + {"name": "device03"} + device04 +device05 \ No newline at end of file diff --git a/tests/manual/pipeline/piping_pretty_json.yaml b/tests/manual/pipeline/piping_pretty_json.yaml new file mode 100644 index 000000000..223cf06cd --- /dev/null +++ b/tests/manual/pipeline/piping_pretty_json.yaml @@ -0,0 +1,25 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/reubenmiller/commander/feat/handle-nested-files/schema.json +# +# Piping pretty printed json +# +config: + env: + C8Y_SETTINGS_DEFAULTS_CACHE: true + C8Y_SETTINGS_CACHE_METHODS: GET PUT POST + C8Y_SETTINGS_DEFAULTS_CACHETTL: 100h + C8Y_SETTINGS_DEFAULTS_DRYFORMAT: json + +tests: + It supports reading mixed piped input: + command: | + cat manual/pipeline/mixed_pipeline.txt | + c8y devices create --dry | + c8y util show --select body -o json -c + exit-code: 0 + stdout: + exactly: | + {"body":{"c8y_IsDevice":{},"name":"device01"}} + {"body":{"c8y_IsDevice":{},"name":"device02"}} + {"body":{"c8y_IsDevice":{},"name":"device03"}} + {"body":{"c8y_IsDevice":{},"name":"device04"}} + {"body":{"c8y_IsDevice":{},"name":"device05"}} From 670b0e8d24d719b4eb7c42fcce3e2e6be98cf14e Mon Sep 17 00:00:00 2001 From: Reuben Miller Date: Sun, 18 Feb 2024 00:45:51 +0100 Subject: [PATCH 3/8] add pretty printed example --- tests/manual/pipeline/mixed_pipeline.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/manual/pipeline/mixed_pipeline.txt b/tests/manual/pipeline/mixed_pipeline.txt index 6fd55ed05..b67e34fd8 100644 --- a/tests/manual/pipeline/mixed_pipeline.txt +++ b/tests/manual/pipeline/mixed_pipeline.txt @@ -1,4 +1,6 @@ {"name": "device01"}{"name": "device02"} {"name": "device03"} device04 -device05 \ No newline at end of file +{ + "name": "device05" +} \ No newline at end of file From fe5e4d9e70ec73ee30481f71dabe0f67f13892d5 Mon Sep 17 00:00:00 2001 From: Reuben Miller Date: Sun, 18 Feb 2024 12:48:27 +0100 Subject: [PATCH 4/8] support ignore braces in quoted strings --- pkg/stream/stream.go | 16 ++++++++++++++-- pkg/stream/stream_test.go | 4 ++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/pkg/stream/stream.go b/pkg/stream/stream.go index 41a63c0b0..3128ff907 100644 --- a/pkg/stream/stream.go +++ b/pkg/stream/stream.go @@ -45,6 +45,9 @@ func (r *InputStreamer) ReadJSONObject() ([]byte, error) { return nil, err } + // Simple json object parser + var prev rune + quote := 0 for { c, _, rErr := r.Buffer.ReadRune() if rErr == io.EOF { @@ -52,15 +55,24 @@ func (r *InputStreamer) ReadJSONObject() ([]byte, error) { break } switch c { + case '"': + if prev != '\\' { + quote = (quote + 1) % 2 + } case '{': - brackets++ + if quote == 0 { + brackets++ + } case '}': - brackets-- + if quote == 0 { + brackets-- + } } out.WriteRune(c) if brackets == 0 { break } + prev = c } return out.Bytes(), err } diff --git a/pkg/stream/stream_test.go b/pkg/stream/stream_test.go index b953ccc1f..5a4d4b0a8 100644 --- a/pkg/stream/stream_test.go +++ b/pkg/stream/stream_test.go @@ -21,10 +21,10 @@ func Test_ScanJSONObject_Consuming(t *testing.T) { }{ { Input: ` - {"name":"1"} + {"name":"1 {literal \" value}"} `, Output: []string{ - `{"name":"1"}`, + `{"name":"1 {literal \" value}"}`, }, Error: nil, }, From e46a3519a9d9b5316a2afdc668b404dc3488873c Mon Sep 17 00:00:00 2001 From: Reuben Miller Date: Sun, 18 Feb 2024 14:10:44 +0100 Subject: [PATCH 5/8] option to ignore or preserve empty lines --- pkg/stream/stream.go | 42 ++++++++- pkg/stream/stream_test.go | 191 ++++++++++++++++++++++++++------------ 2 files changed, 170 insertions(+), 63 deletions(-) diff --git a/pkg/stream/stream.go b/pkg/stream/stream.go index 3128ff907..6f954f349 100644 --- a/pkg/stream/stream.go +++ b/pkg/stream/stream.go @@ -9,7 +9,8 @@ import ( // InputStreamer an input streamer which breaks down a buffer into input values type InputStreamer struct { - Buffer *bufio.Reader + IgnoreEmptyLines bool + Buffer *bufio.Reader } func (r *InputStreamer) isJSONObject() (bool, error) { @@ -34,6 +35,19 @@ func (r *InputStreamer) consumeWhitespace() error { } return nil } +func (r *InputStreamer) consumeWhitespaceOnLine() error { + for { + c, _, err := r.Buffer.ReadRune() + if err == io.EOF { + return err + } + if !(c == ' ' || c == '\t') { + r.Buffer.UnreadRune() + break + } + } + return nil +} // ReadJSONObject read the next JSON object func (r *InputStreamer) ReadJSONObject() ([]byte, error) { @@ -74,9 +88,24 @@ func (r *InputStreamer) ReadJSONObject() ([]byte, error) { } prev = c } + + // Consume a trailing newline if present + r.consumeIf('\n') return out.Bytes(), err } +// ReadLine reads the next chunk of text until the next newline char +// if not put it back on the buffer +func (r *InputStreamer) consumeIf(c rune) error { + v, _, err := r.Buffer.ReadRune() + if err == nil { + if v != c { + return r.Buffer.UnreadRune() + } + } + return nil +} + // ReadLine reads the next chunk of text until the next newline char func (r *InputStreamer) ReadLine() ([]byte, error) { return r.Buffer.ReadBytes('\n') @@ -84,8 +113,15 @@ func (r *InputStreamer) ReadLine() ([]byte, error) { // Read reads the next delimited value (either text or JSON object) func (r *InputStreamer) Read() (output []byte, err error) { - if err := r.consumeWhitespace(); err != nil { - return output, err + if r.IgnoreEmptyLines { + if err := r.consumeWhitespace(); err != nil { + return output, err + } + + } else { + if err := r.consumeWhitespaceOnLine(); err != nil { + return output, err + } } var isJSON bool diff --git a/pkg/stream/stream_test.go b/pkg/stream/stream_test.go index 5a4d4b0a8..381f1ecfa 100644 --- a/pkg/stream/stream_test.go +++ b/pkg/stream/stream_test.go @@ -9,66 +9,137 @@ import ( "github.com/stretchr/testify/assert" ) -func createBuffer(v string) *bufio.Reader { - return bufio.NewReader(strings.NewReader(v)) +func newTestStreamer(v string, ignoreEmptyLines bool) *InputStreamer { + return &InputStreamer{ + Buffer: bufio.NewReader(strings.NewReader(v)), + IgnoreEmptyLines: ignoreEmptyLines, + } } -func Test_ScanJSONObject_Consuming(t *testing.T) { - data := []struct { - Input string - Output []string - Error error - }{ - { - Input: ` - {"name":"1 {literal \" value}"} - `, - Output: []string{ - `{"name":"1 {literal \" value}"}`, - }, - Error: nil, - }, - { - Input: ` - {"name":"1"}{"name":"2"} -one -two - - `, - Output: []string{ - `{"name":"1"}`, - `{"name":"2"}`, - `one`, - `two`, - }, - Error: nil, - }, - { - Input: `{"name":{"1":{"2":{"3":{"4":{"5":"value"}}}}}}{"name":"2"} - - `, - Output: []string{ - `{"name":{"1":{"2":{"3":{"4":{"5":"value"}}}}}}`, - `{"name":"2"}`, - }, - Error: nil, - }, - { - Input: `{"name":"1"`, - Output: []string{ - `{"name":"1"`, - }, - Error: io.EOF, - }, - } - for _, testcase := range data { - s := InputStreamer{ - Buffer: createBuffer(testcase.Input), - } - - for _, iOutput := range testcase.Output { - out, _ := s.Read() - assert.Equal(t, iOutput, string(out)) - } - } +func Test_ReadSimpleWithoutEmptyLines(t *testing.T) { + input := strings.TrimSpace(` +1 + +2 +3 +`) + s := newTestStreamer(input, true) + + var obj []byte + var err error + + obj, err = s.Read() + assert.Nil(t, err) + assert.Equal(t, "1", string(obj)) + + obj, err = s.Read() + assert.Nil(t, err) + assert.Equal(t, "2", string(obj)) + + obj, err = s.Read() + assert.ErrorIs(t, err, io.EOF) + assert.Equal(t, "3", string(obj)) +} + +func Test_Read_Simple(t *testing.T) { + input := strings.TrimSpace(` +1 + +2 +3 +`) + s := newTestStreamer(input, false) + + var obj []byte + var err error + + obj, err = s.Read() + assert.Nil(t, err) + assert.Equal(t, "1", string(obj)) + + obj, err = s.Read() + assert.Nil(t, err) + assert.Equal(t, "", string(obj)) + + obj, err = s.Read() + assert.Nil(t, err) + assert.Equal(t, "2", string(obj)) + + obj, err = s.Read() + assert.ErrorIs(t, err, io.EOF) + assert.Equal(t, "3", string(obj)) +} + +func Test_Read_Complex_JSON(t *testing.T) { + input := strings.TrimSpace(` +{"name":"1 {literal \" value}"} +{"name":{"1":{"2":{"3":{"4":{"5":"value"}}}}}}{"name":"3"} +`) + s := newTestStreamer(input, false) + + var obj []byte + var err error + + obj, err = s.Read() + assert.Nil(t, err) + assert.Equal(t, `{"name":"1 {literal \" value}"}`, string(obj)) + + obj, err = s.Read() + assert.Nil(t, err) + assert.Equal(t, `{"name":{"1":{"2":{"3":{"4":{"5":"value"}}}}}}`, string(obj)) + + obj, err = s.Read() + assert.Nil(t, err) + assert.Equal(t, `{"name":"3"}`, string(obj)) +} + +func Test_ReadPartialJSON(t *testing.T) { + input := strings.TrimSpace(` +{"name":"1" +`) + s := newTestStreamer(input, false) + + var obj []byte + var err error + + obj, err = s.Read() + assert.ErrorIs(t, err, io.EOF) + assert.Equal(t, `{"name":"1"`, string(obj)) +} + +func Test_Read_Mixed(t *testing.T) { + input := strings.TrimLeft(` +{"name":"1"}{"name":"2"} + +3 +4 +`, "\n\t ") + s := newTestStreamer(input, false) + + var obj []byte + var err error + + obj, err = s.Read() + assert.Nil(t, err) + assert.Equal(t, `{"name":"1"}`, string(obj)) + + obj, err = s.Read() + assert.Nil(t, err) + assert.Equal(t, `{"name":"2"}`, string(obj)) + + obj, err = s.Read() + assert.Nil(t, err) + assert.Equal(t, "", string(obj)) + + obj, err = s.Read() + assert.Nil(t, err) + assert.Equal(t, "3", string(obj)) + + obj, err = s.Read() + assert.Nil(t, err) + assert.Equal(t, "4", string(obj)) + + obj, err = s.Read() + assert.ErrorIs(t, err, io.EOF) + assert.Equal(t, "", string(obj)) } From 2ec1b9a43b0580143bf86cefcdac29bd336e1525 Mon Sep 17 00:00:00 2001 From: Reuben Miller Date: Sun, 18 Feb 2024 14:20:28 +0100 Subject: [PATCH 6/8] support stripping double quotes --- pkg/stream/stream.go | 13 ++++++++++++- pkg/stream/stream_test.go | 2 +- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/pkg/stream/stream.go b/pkg/stream/stream.go index 6f954f349..651935b7d 100644 --- a/pkg/stream/stream.go +++ b/pkg/stream/stream.go @@ -134,10 +134,21 @@ func (r *InputStreamer) Read() (output []byte, err error) { output, err = r.ReadJSONObject() } else { output, err = r.ReadLine() - output = bytes.TrimSpace(output) if err != nil { return output, err } + output = r.formatLine(output) } return output, err } + +func (r *InputStreamer) formatLine(b []byte) []byte { + b = bytes.TrimSpace(b) + + // If has surrounding quotes then strip them + // as it improves compatibility with jq output when not using the -r option, e.g. `echo '{"key":"1234"}' | jq '.key' | c8y util show` + if bytes.HasPrefix(b, []byte("\"")) && bytes.HasSuffix(b, []byte("\"")) { + b = bytes.Trim(b, "\"") + } + return b +} diff --git a/pkg/stream/stream_test.go b/pkg/stream/stream_test.go index 381f1ecfa..ed885e6ae 100644 --- a/pkg/stream/stream_test.go +++ b/pkg/stream/stream_test.go @@ -45,7 +45,7 @@ func Test_Read_Simple(t *testing.T) { input := strings.TrimSpace(` 1 -2 +"2" 3 `) s := newTestStreamer(input, false) From 35edc04d7f1d5e0c9113cafaa2acdc5fc4709f4d Mon Sep 17 00:00:00 2001 From: Reuben Miller Date: Mon, 19 Feb 2024 17:12:39 +0100 Subject: [PATCH 7/8] use json lib to parse quoted strings --- pkg/stream/stream.go | 15 +++++++++------ pkg/stream/stream_test.go | 5 +++++ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/pkg/stream/stream.go b/pkg/stream/stream.go index 651935b7d..a9ce6942b 100644 --- a/pkg/stream/stream.go +++ b/pkg/stream/stream.go @@ -3,6 +3,7 @@ package stream import ( "bufio" "bytes" + "encoding/json" "io" "unicode" ) @@ -143,12 +144,14 @@ func (r *InputStreamer) Read() (output []byte, err error) { } func (r *InputStreamer) formatLine(b []byte) []byte { - b = bytes.TrimSpace(b) - - // If has surrounding quotes then strip them - // as it improves compatibility with jq output when not using the -r option, e.g. `echo '{"key":"1234"}' | jq '.key' | c8y util show` - if bytes.HasPrefix(b, []byte("\"")) && bytes.HasSuffix(b, []byte("\"")) { - b = bytes.Trim(b, "\"") + // Prase json strings (e.g. quoted strings) + // as it improves compatibility with jq output when not using the -r option, + // e.g. `echo '{"key":"1234"}' | jq '.key' | c8y util show` + var strValue string + if err := json.Unmarshal(b, &strValue); err == nil { + return []byte(strValue) } + + b = bytes.TrimSpace(b) return b } diff --git a/pkg/stream/stream_test.go b/pkg/stream/stream_test.go index ed885e6ae..cac64d9b7 100644 --- a/pkg/stream/stream_test.go +++ b/pkg/stream/stream_test.go @@ -46,6 +46,7 @@ func Test_Read_Simple(t *testing.T) { 1 "2" + 1.23 3 `) s := newTestStreamer(input, false) @@ -65,6 +66,10 @@ func Test_Read_Simple(t *testing.T) { assert.Nil(t, err) assert.Equal(t, "2", string(obj)) + obj, err = s.Read() + assert.Nil(t, err) + assert.Equal(t, "1.23", string(obj)) + obj, err = s.Read() assert.ErrorIs(t, err, io.EOF) assert.Equal(t, "3", string(obj)) From 75e7236725e84b47ae6a008f3aa82b4006a1f11a Mon Sep 17 00:00:00 2001 From: Reuben Miller Date: Mon, 19 Feb 2024 18:02:41 +0100 Subject: [PATCH 8/8] add json string test case --- tests/manual/pipeline/mixed_pipeline.txt | 3 ++- tests/manual/pipeline/piping_pretty_json.yaml | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/manual/pipeline/mixed_pipeline.txt b/tests/manual/pipeline/mixed_pipeline.txt index b67e34fd8..58dbe6b07 100644 --- a/tests/manual/pipeline/mixed_pipeline.txt +++ b/tests/manual/pipeline/mixed_pipeline.txt @@ -1,6 +1,7 @@ {"name": "device01"}{"name": "device02"} {"name": "device03"} device04 +"device05" { - "name": "device05" + "name": "device06" } \ No newline at end of file diff --git a/tests/manual/pipeline/piping_pretty_json.yaml b/tests/manual/pipeline/piping_pretty_json.yaml index 223cf06cd..7656fae61 100644 --- a/tests/manual/pipeline/piping_pretty_json.yaml +++ b/tests/manual/pipeline/piping_pretty_json.yaml @@ -23,3 +23,4 @@ tests: {"body":{"c8y_IsDevice":{},"name":"device03"}} {"body":{"c8y_IsDevice":{},"name":"device04"}} {"body":{"c8y_IsDevice":{},"name":"device05"}} + {"body":{"c8y_IsDevice":{},"name":"device06"}}