Skip to content

Commit

Permalink
chore: remove embedding
Browse files Browse the repository at this point in the history
  • Loading branch information
@zensh
zensh committed Dec 1, 2023
1 parent e74590b commit e229256
Show file tree
Hide file tree
Showing 5 changed files with 128 additions and 194 deletions.
48 changes: 22 additions & 26 deletions src/api/creation.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,10 +115,6 @@ func (a *Creation) Update(ctx *gear.Context) error {
if err != nil {
return err
}
// we can update price in status 2
// if *creation.Status != 0 && *creation.Status != 1 {
// return gear.ErrBadRequest.WithMsg("cannot update creation, status is not 0 or 1")
// }

output, err := a.blls.Writing.UpdateCreation(ctx, input)
if err != nil {
Expand Down Expand Up @@ -607,7 +603,6 @@ func (a *Creation) summarize(gctx context.Context, gid, cid util.ID, auditLog *b
return nil, err
}

sess := gear.CtxValue[middleware.Session](gctx)
teInput := &bll.TEInput{
GID: gid,
CID: cid,
Expand All @@ -623,27 +618,28 @@ func (a *Creation) summarize(gctx context.Context, gid, cid util.ID, auditLog *b

// summary will not generated by ai if the content is too short
// tokens will be 0
if summary.Tokens > 0 {
go logging.Run(func() logging.Log {
now := time.Now()
_, err := a.blls.Jarvis.Embedding(gctx, teInput)
log := logging.Log{
"action": "embedding",
"rid": sess.RID,
"uid": sess.UserID.String(),
"gid": gid.String(),
"cid": cid.String(),
"language": *creation.Language,
"version": *creation.Version,
"elapsed": time.Since(now) / 1e6,
}

if err != nil {
log["error"] = err.Error()
}
return log
})
}
// sess := gear.CtxValue[middleware.Session](gctx)
// if summary.Tokens > 0 {
// go logging.Run(func() logging.Log {
// now := time.Now()
// _, err := a.blls.Jarvis.Embedding(gctx, teInput)
// log := logging.Log{
// "action": "embedding",
// "rid": sess.RID,
// "uid": sess.UserID.String(),
// "gid": gid.String(),
// "cid": cid.String(),
// "language": *creation.Language,
// "version": *creation.Version,
// "elapsed": time.Since(now) / 1e6,
// }

// if err != nil {
// log["error"] = err.Error()
// }
// return log
// })
// }

if len(summary.Keywords) > 5 {
summary.Keywords = summary.Keywords[:5]
Expand Down
155 changes: 50 additions & 105 deletions src/api/jarvis.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,24 +65,24 @@ func (a *Jarvis) Search(ctx *gear.Context) error {
wg.Add(2)

now := time.Now()
semanticElapsed := int64(0)
// semanticElapsed := int64(0)
literalElapsed := int64(0)

var semanticOutput []*bll.EmbeddingSearchOutput
go logging.Run(func() logging.Log {
defer wg.Done()
// var semanticOutput []*bll.EmbeddingSearchOutput
// go logging.Run(func() logging.Log {
// defer wg.Done()

semanticInput := &bll.EmbeddingSearchInput{
Input: input.Q,
Public: true,
GID: input.GID,
Language: input.Language,
}
// semanticInput := &bll.EmbeddingSearchInput{
// Input: input.Q,
// Public: true,
// GID: input.GID,
// Language: input.Language,
// }

semanticOutput = a.blls.Jarvis.EmbeddingSearch(ctx, semanticInput)
semanticElapsed = int64(time.Since(now)) / 1e6
return nil
})
// semanticOutput = a.blls.Jarvis.EmbeddingSearch(ctx, semanticInput)
// semanticElapsed = int64(time.Since(now)) / 1e6
// return nil
// })

var literalOutput bll.SearchOutput
go logging.Run(func() logging.Log {
Expand All @@ -94,15 +94,15 @@ func (a *Jarvis) Search(ctx *gear.Context) error {
})

wg.Wait()
logging.SetTo(ctx, "semanticResults", len(semanticOutput))
// logging.SetTo(ctx, "semanticResults", len(semanticOutput))
logging.SetTo(ctx, "literalResults", len(literalOutput.Hits))
logging.SetTo(ctx, "semanticElapsed", semanticElapsed)
// logging.SetTo(ctx, "semanticElapsed", semanticElapsed)
logging.SetTo(ctx, "literalElapsed", literalElapsed)

output.Hits = make([]bll.SearchDocument, 0, len(semanticOutput)+len(literalOutput.Hits))
output.Hits = make([]bll.SearchDocument, 0, len(literalOutput.Hits))
// append(output.Hits, literalOutput.Hits...)
output.Languages = literalOutput.Languages
resMap := make(map[util.ID]int, len(semanticOutput)+len(literalOutput.Hits))
resMap := make(map[util.ID]int, len(literalOutput.Hits))
for i, item := range literalOutput.Hits {
j, ok := resMap[item.CID]
if ok && item.Language != lang {
Expand All @@ -118,36 +118,36 @@ func (a *Jarvis) Search(ctx *gear.Context) error {
}
}

for _, item := range semanticOutput {
j, ok := resMap[item.CID]
if ok && item.Language != lang {
continue
}

if doc, err := a.blls.Writing.ImplicitGetPublication(ctx, &bll.ImplicitQueryPublication{
CID: item.CID,
Language: item.Language,
Fields: "status,updated_at,title,summary",
}, nil); err == nil && *doc.Status == 2 {
v := bll.SearchDocument{
GID: doc.GID,
CID: doc.CID,
Language: doc.Language,
Version: doc.Version,
UpdatedAt: *doc.UpdatedAt,
Kind: 1,
Title: *doc.Title,
Summary: *doc.Summary,
}

if ok {
output.Hits[j] = v
} else {
output.Hits = append(output.Hits, v)
resMap[item.CID] = len(output.Hits) - 1
}
}
}
// for _, item := range semanticOutput {
// j, ok := resMap[item.CID]
// if ok && item.Language != lang {
// continue
// }

// if doc, err := a.blls.Writing.ImplicitGetPublication(ctx, &bll.ImplicitQueryPublication{
// CID: item.CID,
// Language: item.Language,
// Fields: "status,updated_at,title,summary",
// }, nil); err == nil && *doc.Status == 2 {
// v := bll.SearchDocument{
// GID: doc.GID,
// CID: doc.CID,
// Language: doc.Language,
// Version: doc.Version,
// UpdatedAt: *doc.UpdatedAt,
// Kind: 1,
// Title: *doc.Title,
// Summary: *doc.Summary,
// }

// if ok {
// output.Hits[j] = v
// } else {
// output.Hits = append(output.Hits, v)
// resMap[item.CID] = len(output.Hits) - 1
// }
// }
// }
}

(&output).LoadGroups(func(ids ...util.ID) []bll.GroupInfo {
Expand Down Expand Up @@ -186,28 +186,8 @@ func (a *Jarvis) GroupSearch(ctx *gear.Context) error {
wg.Add(2)

now := time.Now()
semanticElapsed := int64(0)
literalElapsed := int64(0)

var semanticOutput []*bll.EmbeddingSearchOutput
go logging.Run(func() logging.Log {
defer wg.Done()

semanticInput := &bll.EmbeddingSearchInput{
Input: input.Q,
Public: false,
GID: input.GID,
}

if input.Language != nil {
semanticInput.Language = input.Language
}

semanticOutput = a.blls.Jarvis.EmbeddingSearch(ctx, semanticInput)
semanticElapsed = int64(time.Since(now)) / 1e6
return nil
})

var literalOutput bll.SearchOutput
go logging.Run(func() logging.Log {
defer wg.Done()
Expand All @@ -218,15 +198,12 @@ func (a *Jarvis) GroupSearch(ctx *gear.Context) error {
})

wg.Wait()
logging.SetTo(ctx, "semanticResults", len(semanticOutput))
logging.SetTo(ctx, "literalResults", len(literalOutput.Hits))
logging.SetTo(ctx, "semanticElapsed", semanticElapsed)
logging.SetTo(ctx, "literalElapsed", literalElapsed)

output.Hits = make([]bll.SearchDocument, 0, len(semanticOutput)+len(literalOutput.Hits))
// append(output.Hits, literalOutput.Hits...)
output.Hits = make([]bll.SearchDocument, 0, len(literalOutput.Hits))
output.Languages = literalOutput.Languages
resMap := make(map[util.ID]int, len(semanticOutput)+len(literalOutput.Hits))
resMap := make(map[util.ID]int, len(literalOutput.Hits))
for i, item := range literalOutput.Hits {
j, ok := resMap[item.CID]
if ok && item.Language != lang {
Expand All @@ -242,38 +219,6 @@ func (a *Jarvis) GroupSearch(ctx *gear.Context) error {
}
}

for _, item := range semanticOutput {
j, ok := resMap[item.CID]
if ok && item.Language != lang {
continue
}

if doc, err := a.blls.Writing.ImplicitGetPublication(ctx, &bll.ImplicitQueryPublication{
GID: util.Ptr(item.GID),
CID: item.CID,
Language: item.Language,
Fields: "updated_at,title,summary",
}, nil); err == nil {
v := bll.SearchDocument{
GID: doc.GID,
CID: doc.CID,
Language: doc.Language,
Version: doc.Version,
UpdatedAt: *doc.UpdatedAt,
Kind: 1,
Title: *doc.Title,
Summary: *doc.Summary,
}

if ok {
output.Hits[j] = v
} else {
output.Hits = append(output.Hits, v)
resMap[item.CID] = len(output.Hits) - 1
}
}
}

(&output).LoadGroups(func(ids ...util.ID) []bll.GroupInfo {
return a.blls.Userbase.LoadGroupInfo(ctx, ids...)
})
Expand Down
23 changes: 8 additions & 15 deletions src/api/publication.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -558,15 +558,11 @@ func (a *Publication) Update(ctx *gear.Context) error {
return err
}

publication, err := a.checkWritePermission(ctx, input.GID, input.CID, input.Language, input.Version)
_, err := a.checkWritePermission(ctx, input.GID, input.CID, input.Language, input.Version)
if err != nil {
return err
}

if *publication.Status != 0 {
return gear.ErrBadRequest.WithMsg("cannot update publication, status is not 0")
}

output, err := a.blls.Writing.UpdatePublication(ctx, input)
if err != nil {
return gear.ErrInternalServerError.From(err)
Expand Down Expand Up @@ -632,9 +628,6 @@ func (a *Publication) ListPublished(ctx *gear.Context) error {
return gear.ErrInternalServerError.From(err)
}

// output.Result.LoadCreators(func(ids ...util.ID) []bll.UserInfo {
// return a.blls.Userbase.LoadUserInfo(ctx, ids...)
// })
output.Result.LoadGroups(func(ids ...util.ID) []bll.GroupInfo {
return a.blls.Userbase.LoadGroupInfo(ctx, ids...)
})
Expand Down Expand Up @@ -875,13 +868,13 @@ func (a *Publication) Publish(ctx *gear.Context) error {
return gear.ErrInternalServerError.From(err)
}

gctx := middleware.WithGlobalCtx(ctx)
go a.blls.Jarvis.EmbeddingPublic(gctx, &bll.TEInput{
GID: input.GID,
CID: input.CID,
Language: input.Language,
Version: input.Version,
})
// gctx := middleware.WithGlobalCtx(ctx)
// go a.blls.Jarvis.EmbeddingPublic(gctx, &bll.TEInput{
// GID: input.GID,
// CID: input.CID,
// Language: input.Language,
// Version: input.Version,
// })

if _, err = a.blls.Logbase.Log(ctx, bll.LogActionPublicationPublish, 1, input.GID, &bll.LogPayload{
GID: input.GID,
Expand Down
Loading

0 comments on commit e229256

Please sign in to comment.