diff --git a/deployments/db.js b/deployments/db.js index 288b2d92..ef6e407f 100644 --- a/deployments/db.js +++ b/deployments/db.js @@ -67,11 +67,32 @@ db.createCollection( "label", { } } } } ); - +db.createCollection( "view", { + validator: { $jsonSchema: { + bsonType: "object", + required: [ "id","domain","project","display","criteria" ], + properties: { + id: { + bsonType: "string", + }, + domain: { + bsonType: "string" + }, + project: { + bsonType: "string" + }, + criteria: { + bsonType: "string" + } + } + } } +} ); //index db.kv.createIndex({"id": 1}, { unique: true } ); db.kv.createIndex({key: 1, label_id: 1,domain:1,project:1},{ unique: true }); db.label.createIndex({"id": 1}, { unique: true } ); db.label.createIndex({format: 1,domain:1,project:1},{ unique: true }); +db.view.createIndex({"id": 1}, { unique: true } ); +db.view.createIndex({display:1,domain:1,project:1},{ unique: true }); //db config db.setProfilingLevel(1, {slowms: 80, sampleRate: 1} ); \ No newline at end of file diff --git a/pkg/model/db_schema.go b/pkg/model/db_schema.go index 4feaf420..196484ec 100644 --- a/pkg/model/db_schema.go +++ b/pkg/model/db_schema.go @@ -42,3 +42,12 @@ type KVDoc struct { Domain string `json:"domain,omitempty" yaml:"domain,omitempty"` //redundant } + +//ViewDoc is db struct +type ViewDoc struct { + ID string `json:"id,omitempty" bson:"id,omitempty" yaml:"id,omitempty" swag:"string"` + Display string `json:"display,omitempty" yaml:"display,omitempty"` + Project string `json:"project,omitempty" yaml:"project,omitempty"` + Domain string `json:"domain,omitempty" yaml:"domain,omitempty"` + Criteria string `json:"criteria,omitempty" yaml:"criteria,omitempty"` +} diff --git a/pkg/model/kv.go b/pkg/model/kv.go index a9a8952a..8ee7374c 100644 --- a/pkg/model/kv.go +++ b/pkg/model/kv.go @@ -48,3 +48,9 @@ type LabelHistoryResponse struct { KVs []*KVDoc `json:"data,omitempty"` Revision int `json:"revision"` } + +//ViewResponse represents the view list +type ViewResponse struct { + Total int `json:"total,omitempty"` + Data []*ViewDoc `json:"data,omitempty"` +} diff --git a/server/handler/noop_auth_handler.go b/server/handler/noop_auth_handler.go index b4a3833d..801a402d 100644 --- a/server/handler/noop_auth_handler.go +++ b/server/handler/noop_auth_handler.go @@ -36,7 +36,7 @@ func newDomainResolver() handler.Handler { return &NoopAuthHandler{} } -//Name is handler name +//Display is handler name func (bk *NoopAuthHandler) Name() string { return "auth-handler" } diff --git a/server/service/mongo/session/session.go b/server/service/mongo/session/session.go index 58f9289b..e170b2e0 100644 --- a/server/service/mongo/session/session.go +++ b/server/service/mongo/session/session.go @@ -46,9 +46,9 @@ const ( CollectionKV = "kv" CollectionKVRevision = "kv_revision" CollectionCounter = "counter" - - DefaultTimeout = 5 * time.Second - DefaultValueType = "text" + CollectionView = "view" + DefaultTimeout = 5 * time.Second + DefaultValueType = "text" ) //db errors @@ -59,9 +59,14 @@ var ( ErrTooMany = errors.New("key with labels should be only one") ErrKeyMustNotEmpty = errors.New("must supply key if you want to get exact one result") - ErrKVIDIsNil = errors.New("kvID id is nil") + ErrIDIsNil = errors.New("id is empty") ErrKvIDAndLabelIDNotMatch = errors.New("kvID and labelID do not match") ErrRootCAMissing = errors.New("rootCAFile is empty in config file") + + ErrViewCreation = errors.New("can not create view") + ErrViewUpdate = errors.New("can not update view") + ErrViewNotExist = errors.New("view not exists") + ErrViewFinding = errors.New("view search error") ) var client *mongo.Client @@ -132,3 +137,17 @@ func Init() error { func GetDB() *mongo.Database { return db } + +func CreateView(ctx context.Context, view, source string, pipeline mongo.Pipeline) error { + sr := GetDB().RunCommand(ctx, + bson.D{ + {"create", view}, + {"viewOn", source}, + {"pipeline", pipeline}, + }) + if sr.Err() != nil { + openlogging.Error("can not create view: " + sr.Err().Error()) + return ErrViewCreation + } + return nil +} diff --git a/server/service/mongo/view/view_dao.go b/server/service/mongo/view/view_dao.go new file mode 100644 index 00000000..90d43149 --- /dev/null +++ b/server/service/mongo/view/view_dao.go @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package view + +import ( + "context" + "github.com/apache/servicecomb-kie/pkg/model" + "github.com/apache/servicecomb-kie/server/service/mongo/session" + "github.com/go-mesh/openlogging" + uuid "github.com/satori/go.uuid" + "go.mongodb.org/mongo-driver/bson" +) + +func create(ctx context.Context, viewDoc *model.ViewDoc) error { + viewDoc.ID = uuid.NewV4().String() + viewDoc.Criteria = "" //TODO parse pipe line to sql-like lang + _, err := session.GetDB().Collection(session.CollectionView).InsertOne(ctx, viewDoc) + if err != nil { + openlogging.Error("can not insert view collection: " + err.Error()) + return session.ErrViewCreation + } + return nil +} +func findOne(ctx context.Context, viewID, domain, project string) (*model.ViewDoc, error) { + filter := bson.M{"domain": domain, + "project": project, + "id": viewID} + sr := session.GetDB().Collection(session.CollectionView).FindOne(ctx, filter) + if sr.Err() != nil { + openlogging.Error("can not insert view collection: " + sr.Err().Error()) + return nil, sr.Err() + } + result := &model.ViewDoc{} + err := sr.Decode(result) + if err != nil { + openlogging.Error("decode error: " + err.Error()) + return nil, err + } + if result.ID == viewID { + return result, nil + } + return nil, session.ErrViewNotExist +} diff --git a/server/service/mongo/view/view_service.go b/server/service/mongo/view/view_service.go new file mode 100644 index 00000000..e6a688a5 --- /dev/null +++ b/server/service/mongo/view/view_service.go @@ -0,0 +1,171 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package view + +import ( + "context" + "github.com/apache/servicecomb-kie/pkg/model" + "github.com/apache/servicecomb-kie/server/service" + "github.com/apache/servicecomb-kie/server/service/mongo/session" + "github.com/go-mesh/openlogging" + uuid "github.com/satori/go.uuid" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "time" +) + +//Service operate data in mongodb +type Service struct { + timeout time.Duration +} + +//Create insert a view data and create a mongo db view +func (s *Service) Create(ctx context.Context, viewDoc *model.ViewDoc, options ...service.FindOption) (*model.ViewDoc, error) { + if viewDoc.Domain == "" { + return nil, session.ErrMissingDomain + } + var pipeline mongo.Pipeline = []bson.D{ + {{"$match", bson.D{{"domain", viewDoc.Domain}}}}, + {{"$match", bson.D{{"project", viewDoc.Project}}}}, + } + opts := service.FindOptions{} + for _, o := range options { + o(&opts) + } + if opts.Key != "" { + pipeline = append(pipeline, bson.D{{"$match", bson.D{{"key", opts.Key}}}}) + } + if len(opts.Labels) != 0 { + for k, v := range opts.Labels { + pipeline = append(pipeline, bson.D{{"$match", bson.D{{"labels." + k, v}}}}) + } + } + viewDoc.ID = uuid.NewV4().String() + viewDoc.Criteria = "" //TODO parse pipe line to sql-like lang + err := create(ctx, viewDoc) + if err != nil { + openlogging.Error("can not insert view collection: " + err.Error()) + return nil, session.ErrViewCreation + } + err = session.CreateView(ctx, generateViewName(viewDoc.ID, viewDoc.Domain, viewDoc.Project), session.CollectionKV, pipeline) + if err != nil { + openlogging.Error("can not create view: " + err.Error()) + return nil, session.ErrViewCreation + } + return viewDoc, nil +} + +func (s *Service) Update(ctx context.Context, viewDoc *model.ViewDoc) error { + if viewDoc.Domain == "" { + return session.ErrMissingDomain + } + if viewDoc.Project == "" { + return session.ErrMissingProject + } + if viewDoc.ID == "" { + return session.ErrIDIsNil + } + oldView, err := findOne(ctx, viewDoc.ID, viewDoc.Domain, viewDoc.Project) + if err != nil { + return err + } + if viewDoc.Display != "" { + oldView.Display = viewDoc.Display + } + if viewDoc.Criteria != "" { + oldView.Criteria = viewDoc.Criteria + } + _, err = session.GetDB().Collection(session.CollectionView).UpdateOne(ctx, bson.M{"domain": oldView.Domain, + "project": oldView.Project, + "id": oldView.ID}, + bson.D{ + {"$set", bson.D{ + {"name", oldView.Display}, + {"criteria", oldView.Criteria}, + }}, + }) + if err != nil { + openlogging.Error("can not update view: " + err.Error()) + return session.ErrViewUpdate + } + return nil +} +func (s *Service) List(ctx context.Context, domain, project string, opts ...service.FindOption) (*model.ViewResponse, error) { + option := service.FindOptions{} + for _, o := range opts { + o(&option) + } + collection := session.GetDB().Collection(session.CollectionView) + filter := bson.M{"domain": domain, "project": project} + mOpt := options.Find() + if option.Limit != 0 { + mOpt = mOpt.SetLimit(option.Limit) + } + if option.Offset != 0 { + mOpt = mOpt.SetSkip(option.Offset) + } + cur, err := collection.Find(ctx, filter, mOpt) + if err != nil { + openlogging.Error("can not find view: " + err.Error()) + return nil, session.ErrViewFinding + } + result := &model.ViewResponse{} + for cur.Next(ctx) { + v := &model.ViewDoc{} + if err := cur.Decode(v); err != nil { + openlogging.Error("decode error: " + err.Error()) + return nil, err + } + result.Data = append(result.Data, v) + } + return result, nil +} +func (s *Service) GetContent(ctx context.Context, id, domain, project string, opts ...service.FindOption) (*model.ViewResponse, error) { + option := service.FindOptions{} + for _, o := range opts { + o(&option) + } + mOpt := options.Find() + if option.Limit != 0 { + mOpt = mOpt.SetLimit(option.Limit) + } + if option.Offset != 0 { + mOpt = mOpt.SetSkip(option.Offset) + } + collection := session.GetDB().Collection(generateViewName(id, domain, project)) + cur, err := collection.Find(ctx, bson.D{}, mOpt) + if err != nil { + openlogging.Error("can not find view content: " + err.Error()) + return nil, session.ErrViewFinding + } + result := &model.ViewResponse{} + for cur.Next(ctx) { + v := &model.ViewDoc{} + if err := cur.Decode(v); err != nil { + openlogging.Error("decode error: " + err.Error()) + return nil, err + } + result.Data = append(result.Data, v) + } + return result, nil +} + +func generateViewName(id, domain, project string) string { + return domain + "::" + project + "::" + id +} diff --git a/server/service/mongo/view/view_service_test.go b/server/service/mongo/view/view_service_test.go new file mode 100644 index 00000000..9c3c2541 --- /dev/null +++ b/server/service/mongo/view/view_service_test.go @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package view_test + +import ( + "context" + "github.com/apache/servicecomb-kie/pkg/model" + "github.com/apache/servicecomb-kie/server/config" + "github.com/apache/servicecomb-kie/server/service" + "github.com/apache/servicecomb-kie/server/service/mongo/kv" + "github.com/apache/servicecomb-kie/server/service/mongo/session" + "github.com/apache/servicecomb-kie/server/service/mongo/view" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestGet(t *testing.T) { + var err error + config.Configurations = &config.Config{DB: config.DB{URI: "mongodb://kie:123@127.0.0.1:27017/kie"}} + err = session.Init() + assert.NoError(t, err) + kvsvc := &kv.Service{} + t.Run("put view data", func(t *testing.T) { + kv, err := kvsvc.CreateOrUpdate(context.TODO(), &model.KVDoc{ + Key: "timeout", + Value: "2s", + Labels: map[string]string{ + "app": "mall", + "service": "cart", + "view": "test", + }, + Domain: "default", + Project: "test", + }) + assert.NoError(t, err) + assert.NotEmpty(t, kv.ID) + + kv, err = kvsvc.CreateOrUpdate(context.TODO(), &model.KVDoc{ + Key: "timeout", + Value: "2s", + Labels: map[string]string{ + "app": "mall", + }, + Domain: "default", + Project: "test", + }) + assert.NoError(t, err) + assert.NotEmpty(t, kv.ID) + + kv, err = kvsvc.CreateOrUpdate(context.TODO(), &model.KVDoc{ + Key: "retry", + Value: "2", + Labels: map[string]string{ + "app": "mall", + }, + Domain: "default", + Project: "test", + }) + assert.NoError(t, err) + assert.NotEmpty(t, kv.ID) + }) + + svc := &view.Service{} + t.Run("create and get view", func(t *testing.T) { + view1, err := svc.Create(context.TODO(), &model.ViewDoc{ + Display: "timeout_config", + Project: "test", + Domain: "default", + }, service.WithKey("timeout")) + assert.NoError(t, err) + assert.NotEmpty(t, view1.ID) + view2, err := svc.Create(context.TODO(), &model.ViewDoc{ + Display: "mall_config", + Project: "test", + Domain: "default", + }, service.WithLabels(map[string]string{ + "app": "mall", + })) + assert.NoError(t, err) + assert.NotEmpty(t, view2.ID) + + resp1, err := svc.GetContent(context.TODO(), view1.ID, "default", "test") + assert.NoError(t, err) + assert.Equal(t, 2, len(resp1.Data)) + + resp2, err := svc.GetContent(context.TODO(), view2.ID, "default", "test") + assert.NoError(t, err) + assert.Equal(t, 3, len(resp2.Data)) + }) + +} diff --git a/server/service/service.go b/server/service/service.go index 67fb9e87..27a1abca 100644 --- a/server/service/service.go +++ b/server/service/service.go @@ -33,7 +33,7 @@ var ( //db errors var ( - ErrKeyNotExists = errors.New("key with labels does not exits") + ErrKeyNotExists = errors.New("can not find any key value") ErrRevisionNotExist = errors.New("revision does not exist") ) @@ -55,5 +55,14 @@ type Revision interface { GetRevision(ctx context.Context, domain string) (int64, error) } +//view create update and get view data +type View interface { + Create(ctx context.Context, viewDoc *model.ViewDoc, options ...FindOption) error + Update(ctx context.Context, viewDoc *model.ViewDoc) error + //TODO + //List(ctx context.Context, domain, project string, options ...FindOption) ([]*model.ViewDoc, error) + GetContent(ctx context.Context, id, domain, project string, options ...FindOption) ([]*model.KVResponse, error) +} + //Init init db session type Init func() error