Knowledge (#241)

* Knowledge UX revamp Signed-off-by: Daishan Peng <[email protected]> --------- Signed-off-by: Daishan Peng <[email protected]>
obot-platform · Oct 21, 2024 · 3308e96 · 3308e96
1 parent 8de45f4
commit 3308e96
Show file tree

Hide file tree

Showing 35 changed files with 2,042 additions and 1,301 deletions.
diff --git a/apiclient/types/knowledge.go b/apiclient/types/knowledge.go
@@ -11,13 +11,15 @@ type KnowledgeFile struct {
 	IngestionStatus           IngestionStatus           `json:"ingestionStatus,omitempty"`
 	FileDetails               FileDetails               `json:"fileDetails,omitempty"`
 	UploadID                  string                    `json:"uploadID,omitempty"`
+	Approved                  *bool                     `json:"approved,omitempty"`
 }
 
 type FileDetails struct {
 	FilePath  string `json:"filePath,omitempty"`
 	URL       string `json:"url,omitempty"`
 	UpdatedAt string `json:"updatedAt,omitempty"`
 	Checksum  string `json:"checksum,omitempty"`
+	Ingested  bool   `json:"ingested,omitempty"`
 }
 
 type IngestionStatus struct {

diff --git a/apiclient/types/remoteKnowledgeSource.go b/apiclient/types/remoteKnowledgeSource.go
@@ -21,27 +21,24 @@ type RemoteKnowledgeSource struct {
 
 type RemoteKnowledgeSourceManifest struct {
 	SyncSchedule               string `json:"syncSchedule,omitempty"`
+	AutoApprove                *bool  `json:"autoApprove,omitempty"`
 	RemoteKnowledgeSourceInput `json:",inline"`
 }
 
 type RemoteKnowledgeSourceList List[RemoteKnowledgeSource]
 
 type RemoteKnowledgeSourceInput struct {
-	DisableIngestionAfterSync bool                      `json:"disableIngestionAfterSync,omitempty"`
-	SourceType                RemoteKnowledgeSourceType `json:"sourceType,omitempty"`
-	Exclude                   []string                  `json:"exclude,omitempty"`
-	OneDriveConfig            *OneDriveConfig           `json:"onedriveConfig,omitempty"`
-	NotionConfig              *NotionConfig             `json:"notionConfig,omitempty"`
-	WebsiteCrawlingConfig     *WebsiteCrawlingConfig    `json:"websiteCrawlingConfig,omitempty"`
+	SourceType            RemoteKnowledgeSourceType `json:"sourceType,omitempty"`
+	OneDriveConfig        *OneDriveConfig           `json:"onedriveConfig,omitempty"`
+	NotionConfig          *NotionConfig             `json:"notionConfig,omitempty"`
+	WebsiteCrawlingConfig *WebsiteCrawlingConfig    `json:"websiteCrawlingConfig,omitempty"`
 }
 
 type OneDriveConfig struct {
 	SharedLinks []string `json:"sharedLinks,omitempty"`
 }
 
-type NotionConfig struct {
-	Pages []string `json:"pages,omitempty"`
-}
+type NotionConfig struct{}
 
 type WebsiteCrawlingConfig struct {
 	URLs []string `json:"urls,omitempty"`
@@ -56,6 +53,12 @@ type RemoteKnowledgeSourceState struct {
 type OneDriveLinksConnectorState struct {
 	Folders FolderSet            `json:"folders,omitempty"`
 	Files   map[string]FileState `json:"files,omitempty"`
+	Links   map[string]LinkState `json:"links,omitempty"`
+}
+
+type LinkState struct {
+	IsFolder bool   `json:"isFolder,omitempty"`
+	Name     string `json:"name,omitempty"`
 }
 
 type FileState struct {
@@ -75,7 +78,11 @@ type NotionPage struct {
 }
 
 type WebsiteCrawlingConnectorState struct {
-	ScrapeJobIds map[string]string `json:"scrapeJobIds"`
-	Folders      FolderSet         `json:"folders"`
-	Pages        map[string]Item   `json:"pages"`
+	ScrapeJobIds map[string]string      `json:"scrapeJobIds"`
+	Folders      FolderSet              `json:"folders"`
+	Pages        map[string]PageDetails `json:"pages"`
+}
+
+type PageDetails struct {
+	ParentURL string `json:"parentUrl"`
 }
diff --git a/apiclient/types/zz_generated.deepcopy.go b/apiclient/types/zz_generated.deepcopy.go
diff --git a/pkg/api/handlers/agent.go b/pkg/api/handlers/agent.go
@@ -184,6 +184,29 @@ func (a *AgentHandler) UploadKnowledge(req api.Context) error {
 	return uploadKnowledge(req, a.gptscript, agent.Status.KnowledgeSetNames...)
 }
 
+func (a *AgentHandler) ApproveKnowledgeFile(req api.Context) error {
+	var body struct {
+		Approve bool `json:"approve"`
+	}
+
+	if err := req.Read(&body); err != nil {
+		return err
+	}
+	var file v1.KnowledgeFile
+	if err := req.Storage.Get(req.Context(), kclient.ObjectKey{
+		Namespace: req.Namespace(),
+		Name:      req.PathValue("file_id"),
+	}, &file); err != nil {
+		return err
+	}
+
+	if file.Spec.Approved == nil || *file.Spec.Approved != body.Approve {
+		file.Spec.Approved = &body.Approve
+		return req.Storage.Update(req.Context(), &file)
+	}
+	return nil
+}
+
 func (a *AgentHandler) DeleteKnowledge(req api.Context) error {
 	var agent v1.Agent
 	if err := req.Get(&agent, req.PathValue("id")); err != nil {

diff --git a/pkg/api/handlers/files.go b/pkg/api/handlers/files.go
@@ -114,6 +114,7 @@ func uploadKnowledgeToWorkspace(req api.Context, gClient *gptscript.GPTScript, w
 		Spec: v1.KnowledgeFileSpec{
 			FileName:      filename,
 			WorkspaceName: ws.Name,
+			Approved:      &[]bool{true}[0],
 		},
 	}
 
@@ -137,6 +138,7 @@ func convertKnowledgeFile(file v1.KnowledgeFile, ws v1.Workspace) types.Knowledg
 		RemoteKnowledgeSourceID:   file.Spec.RemoteKnowledgeSourceName,
 		RemoteKnowledgeSourceType: file.Spec.RemoteKnowledgeSourceType,
 		UploadID:                  file.Status.UploadID,
+		Approved:                  file.Spec.Approved,
 	}
 }
 

diff --git a/pkg/api/handlers/remoteknowledgesource.go b/pkg/api/handlers/remoteknowledgesource.go
@@ -209,8 +209,9 @@ func checkConfigChanged(input types.RemoteKnowledgeSourceInput, remoteKnowledgeS
 		return !equality.Semantic.DeepEqual(*input.OneDriveConfig, *remoteKnowledgeSource.Spec.Manifest.OneDriveConfig)
 	}
 
-	if input.NotionConfig != nil && remoteKnowledgeSource.Spec.Manifest.NotionConfig != nil {
-		return !equality.Semantic.DeepEqual(*input.NotionConfig, *remoteKnowledgeSource.Spec.Manifest.NotionConfig)
+	if remoteKnowledgeSource.Spec.Manifest.SourceType == types.RemoteKnowledgeSourceTypeNotion {
+		// we never resync notion on update, this is because by default we sync every page that it has access to
+		return false
 	}
 
 	if input.WebsiteCrawlingConfig != nil && remoteKnowledgeSource.Spec.Manifest.WebsiteCrawlingConfig != nil {

diff --git a/pkg/api/router/router.go b/pkg/api/router/router.go
@@ -38,6 +38,7 @@ func Router(services *services.Services) (http.Handler, error) {
 	// Agent knowledge files
 	mux.HandleFunc("GET /api/agents/{id}/knowledge", agents.Knowledge)
 	mux.HandleFunc("POST /api/agents/{id}/knowledge/{file}", agents.UploadKnowledge)
+	mux.HandleFunc("PUT /api/agents/{id}/knowledge/{file_id}/approve", agents.ApproveKnowledgeFile)
 	mux.HandleFunc("DELETE /api/agents/{id}/knowledge/{file...}", agents.DeleteKnowledge)
 
 	mux.HandleFunc("POST /api/agents/{agent_id}/remote-knowledge-sources", agents.CreateRemoteKnowledgeSource)