Skip to content

Commit

Permalink
bug: fix datasets packaging issues
Browse files Browse the repository at this point in the history
  • Loading branch information
@ibuildthecloud
ibuildthecloud committed Nov 6, 2024
1 parent f65234f commit 4e8d398
Show file tree
Hide file tree
Showing 4 changed files with 39 additions and 43 deletions.
6 changes: 3 additions & 3 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,11 @@ require (
github.com/google/go-github/v62 v62.0.0
github.com/google/uuid v1.6.0
github.com/gptscript-ai/cmd v0.0.0-20240907001148-ffd49061124a
github.com/gptscript-ai/go-gptscript v0.9.6-0.20241028114905-1fce3cceae35
github.com/gptscript-ai/gptscript v0.9.6-0.20241106011514-d21c001097ef
github.com/gptscript-ai/go-gptscript v0.9.6-0.20241106212914-ba040ce8f47b
github.com/gptscript-ai/gptscript v0.9.6-0.20241106194024-266bc58956c6
github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de
github.com/oauth2-proxy/oauth2-proxy/v7 v7.0.0-00010101000000-000000000000
github.com/otto8-ai/kinm v0.0.0-20241104221322-9ecacf74dda7
github.com/otto8-ai/kinm v0.0.0-20241106213623-2a34e5d88241
github.com/otto8-ai/nah v0.0.0-20241103035236-f2497c45ddb0
github.com/otto8-ai/otto8/apiclient v0.0.0-00010101000000-000000000000
github.com/otto8-ai/otto8/logger v0.0.0-00010101000000-000000000000
Expand Down
20 changes: 6 additions & 14 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,8 +67,6 @@ github.com/ProtonMail/go-crypto v1.0.0 h1:LRuvITjQWX+WIfr930YHG2HNfjR1uOfyf5vE0k
github.com/ProtonMail/go-crypto v1.0.0/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0=
github.com/a8m/envsubst v1.4.2 h1:4yWIHXOLEJHQEFd4UjrWDrYeYlV7ncFWJOCBRLOZHQg=
github.com/a8m/envsubst v1.4.2/go.mod h1:MVUTQNGQ3tsjOOtKCNd+fl8RzhsXcDvvAEzkhGtlsbY=
github.com/adrg/xdg v0.5.1 h1:Im8iDbEFARltY09yOJlSGu4Asjk2vF85+3Dyru8uJ0U=
github.com/adrg/xdg v0.5.1/go.mod h1:nlTsY+NNiCBGCK2tpm09vRqfVzrc2fLmXGpBLF0zlTQ=
github.com/adrg/xdg v0.5.3 h1:xRnxJXne7+oWDatRhR1JLnvuccuIeCoBu2rtuLqQB78=
github.com/adrg/xdg v0.5.3/go.mod h1:nlTsY+NNiCBGCK2tpm09vRqfVzrc2fLmXGpBLF0zlTQ=
github.com/alecthomas/assert/v2 v2.7.0 h1:QtqSACNS3tF7oasA8CU6A6sXZSBDqnm7RfpLl9bZqbE=
Expand Down Expand Up @@ -132,12 +130,8 @@ github.com/chai2010/gettext-go v1.0.2 h1:1Lwwip6Q2QGsAdl/ZKPCwTe9fe0CjlUbqj5bFNS
github.com/chai2010/gettext-go v1.0.2/go.mod h1:y+wnP2cHYaVj19NZhYKAwEMH2CI1gNHeQQ+5AjwawxA=
github.com/charmbracelet/glamour v0.8.0 h1:tPrjL3aRcQbn++7t18wOpgLyl8wrOHUEDS7IZ68QtZs=
github.com/charmbracelet/glamour v0.8.0/go.mod h1:ViRgmKkf3u5S7uakt2czJ272WSg2ZenlYEZXT2x7Bjw=
github.com/charmbracelet/lipgloss v0.13.1 h1:Oik/oqDTMVA01GetT4JdEC033dNzWoQHdWnHnQmXE2A=
github.com/charmbracelet/lipgloss v0.13.1/go.mod h1:zaYVJ2xKSKEnTEEbX6uAHabh2d975RJ+0yfkFpRBz5U=
github.com/charmbracelet/lipgloss v1.0.0 h1:O7VkGDvqEdGi93X+DeqsQ7PKHDgtQfF8j8/O2qFMQNg=
github.com/charmbracelet/lipgloss v1.0.0/go.mod h1:U5fy9Z+C38obMs+T+tJqst9VGzlOYGj4ri9reL3qUlo=
github.com/charmbracelet/x/ansi v0.4.0 h1:NqwHA4B23VwsDn4H3VcNX1W1tOmgnvY1NDx5tOXdnOU=
github.com/charmbracelet/x/ansi v0.4.0/go.mod h1:dk73KoMTT5AX5BsX0KrqhsTqAnhZZoCBjs7dGWp4Ktw=
github.com/charmbracelet/x/ansi v0.4.5 h1:LqK4vwBNaXw2AyGIICa5/29Sbdq58GbGdFngSexTdRM=
github.com/charmbracelet/x/ansi v0.4.5/go.mod h1:dk73KoMTT5AX5BsX0KrqhsTqAnhZZoCBjs7dGWp4Ktw=
github.com/charmbracelet/x/exp/golden v0.0.0-20240806155701-69247e0abc2a h1:G99klV19u0QnhiizODirwVksQB91TJKV/UaTnACcG30=
Expand Down Expand Up @@ -354,12 +348,10 @@ github.com/gptscript-ai/chat-completion-client v0.0.0-20241104122544-5fe75f07c13
github.com/gptscript-ai/chat-completion-client v0.0.0-20241104122544-5fe75f07c131/go.mod h1:7P/o6/IWa1KqsntVf68hSnLKuu3+xuqm6lYhch1w4jo=
github.com/gptscript-ai/cmd v0.0.0-20240907001148-ffd49061124a h1:LX7AOcbBoTnUk/233EARaZMnyQO66yXM9lyigniich4=
github.com/gptscript-ai/cmd v0.0.0-20240907001148-ffd49061124a/go.mod h1:DJAo1xTht1LDkNYFNydVjTHd576TC7MlpsVRl3oloVw=
github.com/gptscript-ai/go-gptscript v0.9.6-0.20241028114905-1fce3cceae35 h1:ljdLe1+L7+NR/W4+wXM9xOJULGQfa+Y5v0W40bGdRtk=
github.com/gptscript-ai/go-gptscript v0.9.6-0.20241028114905-1fce3cceae35/go.mod h1:/FVuLwhz+sIfsWUgUHWKi32qT0i6+IXlUlzs70KKt/Q=
github.com/gptscript-ai/gptscript v0.9.6-0.20241105182510-2a9f664b71f9 h1:GMlIcttohgfNC6DmxkIAmI9h5xMMRKjK0+4W+01K36U=
github.com/gptscript-ai/gptscript v0.9.6-0.20241105182510-2a9f664b71f9/go.mod h1:1ECuES7S+IjL4oua0nJzytsWw45tCCv750T44+/JczQ=
github.com/gptscript-ai/gptscript v0.9.6-0.20241106011514-d21c001097ef h1:0kquCAYiiiZSTcXuHklEXPcvmxDQjIt5YwQw6dmHkfQ=
github.com/gptscript-ai/gptscript v0.9.6-0.20241106011514-d21c001097ef/go.mod h1:1ECuES7S+IjL4oua0nJzytsWw45tCCv750T44+/JczQ=
github.com/gptscript-ai/go-gptscript v0.9.6-0.20241106212914-ba040ce8f47b h1:adIh3EnMTlC19t2k1IJoOtF6me/hPxks2GxSSgB7oEw=
github.com/gptscript-ai/go-gptscript v0.9.6-0.20241106212914-ba040ce8f47b/go.mod h1:/FVuLwhz+sIfsWUgUHWKi32qT0i6+IXlUlzs70KKt/Q=
github.com/gptscript-ai/gptscript v0.9.6-0.20241106194024-266bc58956c6 h1:3jgCbFqu/aedukkYFJEjQkBQ5wq5SCEDvXEzux4KgcU=
github.com/gptscript-ai/gptscript v0.9.6-0.20241106194024-266bc58956c6/go.mod h1:1ECuES7S+IjL4oua0nJzytsWw45tCCv750T44+/JczQ=
github.com/gptscript-ai/tui v0.0.0-20240923192013-172e51ccf1d6 h1:vkgNZVWQgbE33VD3z9WKDwuu7B/eJVVMMPM62ixfCR8=
github.com/gptscript-ai/tui v0.0.0-20240923192013-172e51ccf1d6/go.mod h1:frrl/B+ZH3VSs3Tqk2qxEIIWTONExX3tuUa4JsVnqx4=
github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 h1:pdN6V1QBWetyv/0+wjACpqVH+eVULgEjkurDLq3goeM=
Expand Down Expand Up @@ -530,8 +522,8 @@ github.com/onsi/ginkgo/v2 v2.20.2 h1:7NVCeyIWROIAheY21RLS+3j2bb52W0W82tkberYytp4
github.com/onsi/ginkgo/v2 v2.20.2/go.mod h1:K9gyxPIlb+aIvnZ8bd9Ak+YP18w3APlR+5coaZoE2ag=
github.com/onsi/gomega v1.34.2 h1:pNCwDkzrsv7MS9kpaQvVb1aVLahQXyJ/Tv5oAZMI3i8=
github.com/onsi/gomega v1.34.2/go.mod h1:v1xfxRgk0KIsG+QOdm7p8UosrOzPYRo60fd3B/1Dukc=
github.com/otto8-ai/kinm v0.0.0-20241104221322-9ecacf74dda7 h1:7FIUowoW1S8+8w+12/yFzMMrgE/F/2EqzR631GKXUEE=
github.com/otto8-ai/kinm v0.0.0-20241104221322-9ecacf74dda7/go.mod h1:DOegE8RKFYiE0nmLdkJ3YDR668njuEmmqHE4q5qVNqU=
github.com/otto8-ai/kinm v0.0.0-20241106213623-2a34e5d88241 h1:pR1P/rTKR2egA6JtU8sv6YiE4BwszMbxmm3WoEagy9s=
github.com/otto8-ai/kinm v0.0.0-20241106213623-2a34e5d88241/go.mod h1:DOegE8RKFYiE0nmLdkJ3YDR668njuEmmqHE4q5qVNqU=
github.com/otto8-ai/nah v0.0.0-20241103035236-f2497c45ddb0 h1:I+4dYCCPY58dbABkbtEGtLQTx/tJvbZgkfcLj889yQM=
github.com/otto8-ai/nah v0.0.0-20241103035236-f2497c45ddb0/go.mod h1:mo2Dz5pAyZhAVvo+kQEzXZlLzb0SaNzhIate7BQs4Vg=
github.com/otto8-ai/oauth2-proxy/v7 v7.0.0-20241008204315-265dabe17f43 h1:zuroXr125FGRsaj9/9brmtWfkiu3XkNoB78owd8lAo4=
Expand Down
24 changes: 7 additions & 17 deletions pkg/events/events.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ func NewEmitter(client kclient.WithWatch) *Emitter {

type liveState struct {
Prg *gptscript.Program
Frames *Frames
Frames *gptscript.CallFrames
Progress *types.Progress
Done bool
}
Expand All @@ -58,8 +58,6 @@ type WatchOptions struct {
WaitForThread bool
}

type Frames map[string]gptscript.CallFrame

type callFramePrintState struct {
Outputs []gptscript.Output
InputPrinted bool
Expand All @@ -86,7 +84,7 @@ func newPrintState(oldState *printState) *printState {
}
}

func (e *Emitter) Submit(run *v1.Run, prg *gptscript.Program, frames Frames) {
func (e *Emitter) Submit(run *v1.Run, prg *gptscript.Program, frames gptscript.CallFrames) {
e.liveStateLock.Lock()
defer e.liveStateLock.Unlock()

Expand Down Expand Up @@ -341,7 +339,7 @@ func (e *Emitter) printRun(ctx context.Context, state *printState, run v1.Run, r
}
var (
prg gptscript.Program
callFrames = Frames{}
callFrames = gptscript.CallFrames{}
)
if len(runState.Spec.Program) != 0 {
if err := gz.Decompress(&prg, runState.Spec.Program); err != nil {
Expand Down Expand Up @@ -571,24 +569,16 @@ func (e *Emitter) findNextRun(ctx context.Context, run v1.Run, follow bool) (*v1

}

func (e *Emitter) callToEvents(ctx context.Context, namespace, runID string, prg *gptscript.Program, frames Frames, printed *printState, out chan types.Progress) error {
var (
parent gptscript.CallFrame
)
for _, frame := range frames {
if frame.ParentID == "" {
parent = frame
break
}
}
func (e *Emitter) callToEvents(ctx context.Context, namespace, runID string, prg *gptscript.Program, frames gptscript.CallFrames, printed *printState, out chan types.Progress) error {
parent := frames.ParentCallFrame()
if parent.ID == "" || parent.Start.IsZero() {
return nil
}

return e.printCall(ctx, namespace, runID, prg, &parent, frames, printed, out)
}

func getStepTemplateInvoke(prg *gptscript.Program, call *gptscript.CallFrame, frames Frames) *types.StepTemplateInvoke {
func getStepTemplateInvoke(prg *gptscript.Program, call *gptscript.CallFrame, frames gptscript.CallFrames) *types.StepTemplateInvoke {
if len(call.Tool.InputFilters) == 0 {
return nil
}
Expand Down Expand Up @@ -620,7 +610,7 @@ func getStepTemplateInvoke(prg *gptscript.Program, call *gptscript.CallFrame, fr
return nil
}

func (e *Emitter) printCall(ctx context.Context, namespace, runID string, prg *gptscript.Program, call *gptscript.CallFrame, frames Frames, lastPrint *printState, out chan types.Progress) error {
func (e *Emitter) printCall(ctx context.Context, namespace, runID string, prg *gptscript.Program, call *gptscript.CallFrame, frames gptscript.CallFrames, lastPrint *printState, out chan types.Progress) error {
printed := lastPrint.frames[call.ID]
lastOutputs := printed.Outputs

Expand Down
32 changes: 23 additions & 9 deletions pkg/services/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"github.com/gptscript-ai/go-gptscript"
"github.com/gptscript-ai/gptscript/pkg/cache"
gptscriptai "github.com/gptscript-ai/gptscript/pkg/gptscript"
"github.com/gptscript-ai/gptscript/pkg/loader"
"github.com/gptscript-ai/gptscript/pkg/sdkserver"
"github.com/otto8-ai/nah"
"github.com/otto8-ai/nah/pkg/leader"
Expand All @@ -36,6 +37,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apiserver/pkg/authentication/authenticator"
"k8s.io/apiserver/pkg/authentication/request/union"

// Setup baaah logging
_ "github.com/otto8-ai/nah/pkg/logrus"
)
Expand All @@ -53,7 +55,7 @@ type Config struct {
ToolRegistry string `usage:"The tool reference for the tool registry" default:"github.com/otto8-ai/tools"`
WorkspaceProviderType string `usage:"The type of workspace provider to use for non-knowledge workspaces" default:"directory" env:"OTTO_WORKSPACE_PROVIDER_TYPE"`
WorkspaceTool string `usage:"The tool reference for the workspace provider" default:"github.com/gptscript-ai/workspace-provider"`
DatasetTool string `usage:"The tool reference for the dataset provider" default:"github.com/gptscript-ai/datasets"`
DatasetsTool string `usage:"The tool reference for the dataset provider" default:"github.com/gptscript-ai/datasets"`
HelperModel string `usage:"The model used to generate names and descriptions" default:"gpt-4o-mini"`
AWSKMSKeyARN string `usage:"The ARN of the AWS KMS key to use for encrypting credential storage" env:"OTTO_AWS_KMS_KEY_ARN" name:"aws-kms-key-arn"`
EncryptionConfigFile string `usage:"The path to the encryption configuration file" default:"./encryption.yaml"`
Expand Down Expand Up @@ -81,12 +83,24 @@ type Services struct {
GatewayServer *gserver.Server
}

func newGPTScript(ctx context.Context, workspaceTool, datasetsTool string) (*gptscript.GPTScript, error) {
const (
defaultDatasetsTool = "github.com/gptscript-ai/datasets"
defaultToolsRegistry = "github.com/otto8-ai/tools"
)

func newGPTScript(ctx context.Context, workspaceTool, datasetsTool, toolsRegistry string) (*gptscript.GPTScript, error) {
if datasetsTool != defaultDatasetsTool {
loader.Remap[defaultDatasetsTool] = datasetsTool
}
if toolsRegistry != defaultToolsRegistry {
loader.Remap[defaultToolsRegistry] = toolsRegistry
}

if os.Getenv("GPTSCRIPT_URL") != "" {
return gptscript.NewGPTScript(gptscript.GlobalOptions{
URL: os.Getenv("GPTSCRIPT_URL"),
DatasetToolRepo: datasetsTool,
WorkspaceTool: workspaceTool,
URL: os.Getenv("GPTSCRIPT_URL"),
WorkspaceTool: workspaceTool,
DatasetTool: datasetsTool,
})
}

Expand All @@ -112,9 +126,9 @@ func newGPTScript(ctx context.Context, workspaceTool, datasetsTool string) (*gpt
}

return gptscript.NewGPTScript(gptscript.GlobalOptions{
URL: url,
DatasetToolRepo: datasetsTool,
WorkspaceTool: workspaceTool,
URL: url,
WorkspaceTool: workspaceTool,
DatasetTool: datasetsTool,
})
}

Expand Down Expand Up @@ -168,7 +182,7 @@ func New(ctx context.Context, config Config) (*Services, error) {
config.UIHostname = "https://" + config.UIHostname
}

c, err := newGPTScript(ctx, config.WorkspaceTool, config.DatasetTool)
c, err := newGPTScript(ctx, config.WorkspaceTool, config.DatasetsTool, config.ToolRegistry)
if err != nil {
return nil, err
}
Expand Down

0 comments on commit 4e8d398

Please sign in to comment.