feat: support high-performance generic-call SDK for protobuf protocol (…

…#37)

.github/workflows/benchmark-linux-x64.yml

@@ -1,6 +1,6 @@
 name: Benchmark Linux-X64
 
-on: push
+on: [ push, pull_request ]
 
 jobs:
   build:
@@ -30,6 +30,7 @@ jobs:
           go get github.com/thrift-iterator/go@latest
           go get github.com/cloudwego/kitex/pkg/remote/codec/thrift@latest
           go get github.com/cloudwego/kitex/pkg/generic@latest
+          go get github.com/cloudwego/kitex/pkg/retry@latest
           go get github.com/cloudwego/frugal@latest
           go test -race github.com/cloudwego/dynamicgo/testdata
 

.github/workflows/push-check-linux-compat.yml

@@ -1,6 +1,6 @@
 name: Compatibility Test
 
-on: push
+on: [ push, pull_request ]
 
 jobs:
   build:
@@ -31,4 +31,7 @@ jobs:
           go test -race github.com/cloudwego/dynamicgo/thrift/generic
           go test -race github.com/cloudwego/dynamicgo/conv/t2j
           go test -race github.com/cloudwego/dynamicgo/http
-          go test -race github.com/cloudwego/dynamicgo/internal/json
+          go test -race github.com/cloudwego/dynamicgo/internal/json
+          go test -race github.com/cloudwego/dynamicgo/conv/j2p
+          go test -race github.com/cloudwego/dynamicgo/conv/p2j
+          go test -race github.com/cloudwego/dynamicgo/proto/generic

.github/workflows/push-check-linux-x64.yml

@@ -1,6 +1,6 @@
 name: Push Check Linux-X64
 
-on: push
+on: [ push, pull_request ]
 
 jobs:
   build:

README.md

@@ -2,7 +2,9 @@
 Dynamically operating data for Go. Aiming at reducing serialization/deserializtion process thus it can be fast as much as possible.
 
 ## Introduction
-See [introduction.md](introduction.md).
+Dynamic-Go for Thrift protocol: [introduction.md](introduction.md).
+
+Dynamic-Go for Protobuf protocol: [introduction.md](./proto/INTRODUCTION.md)
 
 ## Usage
 ### thrift
@@ -19,6 +21,22 @@ The meta data about message transportation, including caller, address, log-id, e
 #### thrift/annotation 
 Built-in implementation of thrid-party annotations, see [thrift_idl_annotation_standards](https://www.cloudwego.io/docs/kitex/tutorials/advanced-feature/generic-call/thrift_idl_annotation_standards/). They are mainly used for `conv` (protocol convertion) modules. 
 
+### proto
+Protobuf IDL parser and message operators. It can parse protobuf IDL in runtime and handle protobuf data in generic way.
+[DOC](proto/README.md)
+
+#### proto/generic
+Reflection APIs to search, modify, deserialize, serialize protobuf value **with or without** runtime descriptor.
+[DOC](proto/generic/README.md)
+
+#### proto/protowire
+Protobuf data encode and decode APIs. It parses and formats the low-level raw wire encoding. It is modified from Protobuf official code [`encoding/protowire`](https://pkg.go.dev/google.golang.org/protobuf/encoding/protowire).
+[DOC](proto/protowire/README.md)
+
+#### proto/binary
+BinaryProtocol tool for Protobuf Protocol. It can read, wirte and skip fields directly on binary data of protobuf message.
+[DOC](proto/binary/README.md)
+
 ### http
 Http request/response wrapper interfaces. They are mainly used to pass http values on `http<>thrift` conversion. 
 [DOC](http/README.md)
@@ -34,7 +52,14 @@ Convert JSON value or JSON-body HTTP request into thrift message.
 Convert thrift message to JSON value or JSON-body HTTP response.
 [DOC](conv/t2j/README.md)
 
+#### conv/j2p
+Convert JSON value into protobuf message.
+[DOC](conv/j2p/README.md)
+
+#### conv/p2j
+Convert protobuf message into JSON value.
+[DOC](conv/p2j/README.md)
 
 ## Requirements
 - Go 1.16~1.20
-- Amd64 arch CPU
+- Amd64 arch CPU

conv/j2p/README.md

@@ -0,0 +1,122 @@
+<!-- Code generated by gomarkdoc. DO NOT EDIT -->
+
+# j2p
+
+```go
+import "github.com/cloudwego/dynamicgo/conv/j2p"
+```
+
+## Index
+
+- [type BinaryConv](<#BinaryConv>)
+  - [func NewBinaryConv(opts conv.Options) BinaryConv](<#NewBinaryConv>)
+  - [func (self *BinaryConv) Do(ctx context.Context, desc *proto.TypeDescriptor, jbytes []byte) (tbytes []byte, err error)](<#BinaryConv.Do>)
+  - [func (self *BinaryConv) DoInto(ctx context.Context, desc *proto.TypeDescriptor, jbytes []byte, buf *[]byte) error](<#BinaryConv.DoInto>)
+
+
+<a name="BinaryConv"></a>
+## type [BinaryConv](<https://github.com/khan-yin/dynamicgo/blob/main/conv/j2p/conv.go#L13-L15>)
+
+BinaryConv is a converter from json to protobuf binary
+
+```go
+type BinaryConv struct {
+    // contains filtered or unexported fields
+}
+```
+
+<a name="NewBinaryConv"></a>
+### func [NewBinaryConv](<https://github.com/khan-yin/dynamicgo/blob/main/conv/j2p/conv.go#L18>)
+
+```go
+func NewBinaryConv(opts conv.Options) BinaryConv
+```
+
+NewBinaryConv returns a new BinaryConv
+
+<a name="BinaryConv.Do"></a>
+### func (*BinaryConv) [Do](<https://github.com/khan-yin/dynamicgo/blob/main/conv/j2p/conv.go#L26>)
+
+```go
+func (self *BinaryConv) Do(ctx context.Context, desc *proto.TypeDescriptor, jbytes []byte) (tbytes []byte, err error)
+```
+
+Do converts json bytes (jbytes) to protobuf binary (tbytes) desc is the protobuf type descriptor of the protobuf binary, usually it the request Message type
+
+<details><summary>Example</summary>
+<p>
+
+
+
+```go
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"reflect"
+
+	"github.com/cloudwego/dynamicgo/conv"
+	"github.com/cloudwego/dynamicgo/testdata/kitex_gen/pb/example2"
+	"google.golang.org/protobuf/encoding/protowire"
+)
+
+var opts = conv.Options{}
+
+func main() {
+	// get descriptor and data
+	desc := getExampleDesc()
+	data := getExampleData()
+
+	// make BinaryConv
+	cv := NewBinaryConv(opts)
+
+	// do conversion
+	out, err := cv.Do(context.Background(), desc, data)
+	if err != nil {
+		panic(err)
+	}
+
+	// validate result
+	exp := &example2.ExampleReq{}
+	err = json.Unmarshal(data, exp)
+	if err != nil {
+		panic(err)
+	}
+	act := &example2.ExampleReq{}
+	l := 0
+	dataLen := len(out)
+	// fastRead to get target struct
+	for l < dataLen {
+		id, wtyp, tagLen := protowire.ConsumeTag(out)
+		if tagLen < 0 {
+			panic("parseTag failed")
+		}
+		l += tagLen
+		out = out[tagLen:]
+		offset, err := act.FastRead(out, int8(wtyp), int32(id))
+		if err != nil {
+			panic(err)
+		}
+		out = out[offset:]
+		l += offset
+	}
+	if !reflect.DeepEqual(exp, act) {
+		panic("not equal")
+	}
+}
+```
+
+</p>
+</details>
+
+<a name="BinaryConv.DoInto"></a>
+### func (*BinaryConv) [DoInto](<https://github.com/khan-yin/dynamicgo/blob/main/conv/j2p/conv.go#L55>)
+
+```go
+func (self *BinaryConv) DoInto(ctx context.Context, desc *proto.TypeDescriptor, jbytes []byte, buf *[]byte) error
+```
+
+DoInto behaves like Do, but it writes the result to buffer directly instead of returning a new buffer
+
+Generated by [gomarkdoc](<https://github.com/princjef/gomarkdoc>)

conv/j2p/api_amd64.go

@@ -0,0 +1,26 @@
+//go:build amd64 && go1.16
+// +build amd64,go1.16
+
+// Copyright 2023 CloudWeGo Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package j2p
+
+import (
+	"github.com/chenzhuoyu/base64x"
+)
+
+func decodeBase64(src string) ([]byte, error) {
+	return base64x.StdEncoding.DecodeString(src)
+}

conv/j2p/api_compat.go

@@ -0,0 +1,24 @@
+//go:build !amd64 || !go1.16
+// +build !amd64 !go1.16
+
+// Copyright 2023 CloudWeGo Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package j2p
+
+import 	"encoding/base64"
+
+func decodeBase64(src string) ([]byte, error) {
+	return base64.StdEncoding.DecodeString(src)
+}

conv/j2p/conv.go

@@ -0,0 +1,71 @@
+package j2p
+
+import (
+	"context"
+
+	"github.com/cloudwego/dynamicgo/conv"
+	"github.com/cloudwego/dynamicgo/http"
+	"github.com/cloudwego/dynamicgo/meta"
+	"github.com/cloudwego/dynamicgo/proto"
+)
+
+// BinaryConv is a converter from json to protobuf binary
+type BinaryConv struct {
+	opts conv.Options
+}
+
+// NewBinaryConv returns a new BinaryConv
+func NewBinaryConv(opts conv.Options) BinaryConv {
+	return BinaryConv{
+		opts: opts,
+	}
+}
+
+// Do converts json bytes (jbytes) to protobuf binary (tbytes)
+// desc is the protobuf type descriptor of the protobuf binary, usually it the request Message type
+func (self *BinaryConv) Do(ctx context.Context, desc *proto.TypeDescriptor, jbytes []byte) (tbytes []byte, err error) {
+	buf := conv.NewBytes()
+
+	// req alloc but not use
+	var req http.RequestGetter
+	if self.opts.EnableHttpMapping {
+		reqi := ctx.Value(conv.CtxKeyHTTPRequest)
+		if reqi != nil {
+			reqi, ok := reqi.(http.RequestGetter)
+			if !ok {
+				return nil, newError(meta.ErrInvalidParam, "invalid http.RequestGetter", nil)
+			}
+			req = reqi
+		} else {
+			return nil, newError(meta.ErrInvalidParam, "EnableHttpMapping but no http response in context", nil)
+		}
+	}
+
+	err = self.do(ctx, jbytes, desc, buf, req)
+	if err == nil && len(*buf) > 0 {
+		tbytes = make([]byte, len(*buf))
+		copy(tbytes, *buf)
+	}
+
+	conv.FreeBytes(buf)
+	return
+}
+
+// DoInto behaves like Do, but it writes the result to buffer directly instead of returning a new buffer
+func (self *BinaryConv) DoInto(ctx context.Context, desc *proto.TypeDescriptor, jbytes []byte, buf *[]byte) error {
+	// req alloc but not use
+	var req http.RequestGetter
+	if self.opts.EnableHttpMapping {
+		reqi := ctx.Value(conv.CtxKeyHTTPRequest)
+		if reqi != nil {
+			reqi, ok := reqi.(http.RequestGetter)
+			if !ok {
+				return newError(meta.ErrInvalidParam, "invalid http.RequestGetter", nil)
+			}
+			req = reqi
+		} else {
+			return newError(meta.ErrInvalidParam, "EnableHttpMapping but no http response in context", nil)
+		}
+	}
+	return self.do(ctx, jbytes, desc, buf, req)
+}