enhance/multiple tool registries remix

.github/workflows/docker-build-and-push.yml

@@ -42,7 +42,7 @@ jobs:
         username: ${{ secrets.DOCKERHUB_USERNAME }}
         password: ${{ secrets.DOCKERHUB_TOKEN }}
 
-    - name: Build and push Docker image
+    - name: Build and push OSS Docker image
       uses: depot/build-push-action@v1
       with:
         project: bbqjs4tj1g
@@ -53,15 +53,34 @@ jobs:
           ${{ github.ref_type == 'tag' && !contains(github.ref_name, '-rc') && format('docker.io/obot/{0}:{1}', github.event.repository.name, github.ref_name) || '' }}
         platforms: linux/amd64,linux/arm64
 
+    - name: Build and push enterprise Docker image
+      uses: depot/build-push-action@v1
+      with:
+        project: bbqjs4tj1g
+        context: .
+        push: true
+        secrets: |
+          "GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}"
+        build-args: |
+          TOOL_REGISTRY_REPOS='github.com/obot-platform/tools,github.com/obot-platform/enterprise-tools'
+        tags: |
+          ghcr.io/${{ github.repository }}-enterprise:${{ github.ref_name }}
+        platforms: linux/amd64,linux/arm64
+
     - name: Setup crane
       uses: imjasonh/[email protected]
 
-    - name: Copy to latest tag
+    - name: Copy OSS image to latest tag
       if: ${{ github.ref_type == 'tag' && !contains(github.ref_name, '-rc') }}
       run: |
         crane tag ghcr.io/${{ github.repository }}:${{ github.ref_name }} latest
         crane tag docker.io/obot/${{ github.event.repository.name }}:${{ github.ref_name }} latest
 
+    - name: Copy Enterprise image to latest tag
+      if: ${{ github.ref_type == 'tag' && !contains(github.ref_name, '-rc') }}
+      run: |
+        crane tag ghcr.io/${{ github.repository }}-enterprise:${{ github.ref_name }} latest
+
     - name: Deploy to Test Render
       if: ${{ env.DEPLOY_TO_TEST == 'true' }}
       uses: joelwmale/[email protected]

DEVELOPMENT.md

@@ -50,7 +50,7 @@ Obot has a set of packaged tools. These tools are in the repo `github.com/obot-p
 
 1. Clone `github.com/obot-platform/tools` to your local machine.
 2. In the root directory of the tools repo on your local machine, run `make build`.
-3. Run the Obot server, either with `make dev` or in your IDE, with the `OBOT_SERVER_TOOL_REGISTRY` environment variable set to the root directory of the tools repo.
+3. Run the Obot server, either with `make dev` or in your IDE, with the `GPTSCRIPT_TOOL_REMAP` environment variable set to `github.com/obot-platform/tools=<local-tools-fork-root-directory>`; e.g. If you cloned the tools repo to the directory "above" the Obot repo, you'd use `GPTSCRIPT_TOOL_REMAP='github.com/obot-platform/tools=../tools' make dev`.
 
 Now, any time one of these tools is run, your local copy will be used.
 

Dockerfile

@@ -1,3 +1,4 @@
+# syntax=docker/dockerfile:1
 FROM cgr.dev/chainguard/wolfi-base AS base
 
 RUN apk add --no-cache go make git npm pnpm
@@ -12,14 +13,16 @@ RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store \
     make all
 
 FROM base AS tools
+ARG TOOL_REGISTRY_REPOS='github.com/obot-platform/tools'
 RUN apk add --no-cache curl python-3.13 py3.13-pip
 WORKDIR /app
 COPY . .
 RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store \
     --mount=type=cache,target=/root/.cache/go-build \
     --mount=type=cache,target=/root/.cache/uv \
     --mount=type=cache,target=/root/go/pkg/mod \
-    UV_LINK_MODE=copy BIN_DIR=/bin make package-tools
+    --mount=type=secret,id=GITHUB_TOKEN,env=GITHUB_TOKEN \
+    UV_LINK_MODE=copy BIN_DIR=/bin TOOL_REGISTRY_REPOS=$TOOL_REGISTRY_REPOS make package-tools
 
 FROM cgr.dev/chainguard/postgres:latest-dev AS build-pgvector
 RUN apk add build-base git postgresql-dev
@@ -42,6 +45,7 @@ COPY --from=build-pgvector /usr/share/postgresql17/extension/vector* /usr/share/
 
 RUN apk add --no-cache git python-3.13 py3.13-pip openssh-server npm bash tini procps libreoffice docker
 COPY --chmod=0755 /tools/package-chrome.sh /
+
 RUN /package-chrome.sh && rm /package-chrome.sh
 RUN sed -E 's/^#(PermitRootLogin)no/\1yes/' /etc/ssh/sshd_config -i
 RUN ssh-keygen -A
@@ -54,13 +58,9 @@ COPY --from=bin /app/bin/obot /bin/
 
 EXPOSE 22
 # libreoffice executables
-ENV PATH=/obot-tools/venv/bin:$PATH:/usr/lib/libreoffice/program
+ENV PATH=$PATH:/usr/lib/libreoffice/program
 ENV HOME=/data
 ENV XDG_CACHE_HOME=/data/cache
-ENV GPTSCRIPT_SYSTEM_TOOLS_DIR=/obot-tools/
-ENV OBOT_SERVER_WORKSPACE_TOOL=/obot-tools/workspace-provider
-ENV OBOT_SERVER_DATASETS_TOOL=/obot-tools/datasets
-ENV OBOT_SERVER_TOOL_REGISTRY=/obot-tools
 ENV OBOT_SERVER_ENCRYPTION_CONFIG_FILE=/encryption.yaml
 ENV BAAAH_THREADINESS=20
 ENV TERM=vt100

pkg/controller/controller.go

@@ -41,7 +41,7 @@ func (c *Controller) PreStart(ctx context.Context) error {
 }
 
 func (c *Controller) PostStart(ctx context.Context) error {
-	go c.toolRefHandler.PollRegistry(ctx, c.services.Router.Backend())
+	go c.toolRefHandler.PollRegistries(ctx, c.services.Router.Backend())
 	return c.toolRefHandler.EnsureOpenAIEnvCredentialAndDefaults(ctx, c.services.Router.Backend())
 }
 

pkg/controller/handlers/toolreference/toolreference.go

@@ -46,15 +46,18 @@ type Handler struct {
 	gptClient     *gptscript.GPTScript
 	dispatcher    *dispatcher.Dispatcher
 	supportDocker bool
-	registryURL   string
+	registryURLs  []string
 }
 
-func New(gptClient *gptscript.GPTScript, dispatcher *dispatcher.Dispatcher,
-	registryURL string, supportDocker bool) *Handler {
+func New(gptClient *gptscript.GPTScript,
+	dispatcher *dispatcher.Dispatcher,
+	registryURLs []string,
+	supportDocker bool,
+) *Handler {
 	return &Handler{
 		gptClient:     gptClient,
 		dispatcher:    dispatcher,
-		registryURL:   registryURL,
+		registryURLs:  registryURLs,
 		supportDocker: supportDocker,
 	}
 }
@@ -66,13 +69,13 @@ func isValidTool(tool gptscript.Tool) bool {
 	return tool.Name != "" && (tool.Type == "" || tool.Type == "tool")
 }
 
-func (h *Handler) toolsToToolReferences(ctx context.Context, toolType types.ToolReferenceType, entries map[string]indexEntry) (result []client.Object) {
+func (h *Handler) toolsToToolReferences(ctx context.Context, toolType types.ToolReferenceType, registryURL string, entries map[string]indexEntry) (result []client.Object) {
 	annotations := map[string]string{
 		"obot.obot.ai/timestamp": time.Now().String(),
 	}
 	for name, entry := range entries {
 		if ref, ok := strings.CutPrefix(entry.Reference, "./"); ok {
-			entry.Reference = h.registryURL + "/" + ref
+			entry.Reference = registryURL + "/" + ref
 		}
 
 		if entry.All {
@@ -152,8 +155,8 @@ func (h *Handler) toolsToToolReferences(ctx context.Context, toolType types.Tool
 	return
 }
 
-func (h *Handler) readRegistry(ctx context.Context) (index, error) {
-	run, err := h.gptClient.Run(ctx, h.registryURL, gptscript.Options{})
+func (h *Handler) readRegistry(ctx context.Context, registryURL string) (index, error) {
+	run, err := h.gptClient.Run(ctx, registryURL, gptscript.Options{})
 	if err != nil {
 		return index{}, err
 	}
@@ -173,21 +176,31 @@ func (h *Handler) readRegistry(ctx context.Context) (index, error) {
 }
 
 func (h *Handler) readFromRegistry(ctx context.Context, c client.Client) error {
-	index, err := h.readRegistry(ctx)
-	if err != nil {
-		return err
-	}
+	var (
+		toAdd []client.Object
+		errs  []error
+	)
+	for _, registryURL := range h.registryURLs {
+		index, err := h.readRegistry(ctx, registryURL)
+		if err != nil {
+			errs = append(errs, fmt.Errorf("failed to read registry %s: %w", registryURL, err))
+			continue
+		}
 
-	var toAdd []client.Object
+		toAdd = append(toAdd, h.toolsToToolReferences(ctx, types.ToolReferenceTypeSystem, registryURL, index.System)...)
+		toAdd = append(toAdd, h.toolsToToolReferences(ctx, types.ToolReferenceTypeModelProvider, registryURL, index.ModelProviders)...)
+		toAdd = append(toAdd, h.toolsToToolReferences(ctx, types.ToolReferenceTypeTool, registryURL, index.Tools)...)
+		toAdd = append(toAdd, h.toolsToToolReferences(ctx, types.ToolReferenceTypeStepTemplate, registryURL, index.StepTemplates)...)
+		toAdd = append(toAdd, h.toolsToToolReferences(ctx, types.ToolReferenceTypeKnowledgeDataSource, registryURL, index.KnowledgeDataSources)...)
+		toAdd = append(toAdd, h.toolsToToolReferences(ctx, types.ToolReferenceTypeKnowledgeDocumentLoader, registryURL, index.KnowledgeDocumentLoaders)...)
+	}
 
-	toAdd = append(toAdd, h.toolsToToolReferences(ctx, types.ToolReferenceTypeSystem, index.System)...)
-	toAdd = append(toAdd, h.toolsToToolReferences(ctx, types.ToolReferenceTypeModelProvider, index.ModelProviders)...)
-	toAdd = append(toAdd, h.toolsToToolReferences(ctx, types.ToolReferenceTypeTool, index.Tools)...)
-	toAdd = append(toAdd, h.toolsToToolReferences(ctx, types.ToolReferenceTypeStepTemplate, index.StepTemplates)...)
-	toAdd = append(toAdd, h.toolsToToolReferences(ctx, types.ToolReferenceTypeKnowledgeDataSource, index.KnowledgeDataSources)...)
-	toAdd = append(toAdd, h.toolsToToolReferences(ctx, types.ToolReferenceTypeKnowledgeDocumentLoader, index.KnowledgeDocumentLoaders)...)
+	if len(errs) > 0 {
+		// Don't accidentally delete tool references for registry URLs that failed to be read.
+		return errors.Join(errs...)
+	}
 
-	if len(toAdd) == 0 {
+	if len(toAdd) < 1 {
 		// Don't accidentally delete all the tool references
 		return nil
 	}
@@ -199,8 +212,8 @@ func normalize(names ...string) string {
 	return strings.ToLower(strings.ReplaceAll(strings.ReplaceAll(strings.Join(names, "-"), " ", "-"), "_", "-"))
 }
 
-func (h *Handler) PollRegistry(ctx context.Context, c client.Client) {
-	if h.registryURL == "" {
+func (h *Handler) PollRegistries(ctx context.Context, c client.Client) {
+	if len(h.registryURLs) < 1 {
 		return
 	}
 
@@ -216,7 +229,7 @@ func (h *Handler) PollRegistry(ctx context.Context, c client.Client) {
 	defer t.Stop()
 	for {
 		if err := h.readFromRegistry(ctx, c); err != nil {
-			log.Errorf("Failed to read from registry: %v", err)
+			log.Errorf("Failed to read from registries: %v", err)
 		}
 
 		select {

pkg/controller/routes.go

@@ -29,8 +29,12 @@ func (c *Controller) setupRoutes() error {
 
 	workflowExecution := workflowexecution.New(c.services.Invoker)
 	workflowStep := workflowstep.New(c.services.Invoker)
-	toolRef := toolreference.New(c.services.GPTClient, c.services.ModelProviderDispatcher,
-		c.services.ToolRegistryURL, c.services.SupportDocker)
+	toolRef := toolreference.New(
+		c.services.GPTClient,
+		c.services.ModelProviderDispatcher,
+		c.services.ToolRegistryURLs,
+		c.services.SupportDocker,
+	)
 	workspace := workspace.New(c.services.GPTClient, c.services.WorkspaceProviderType)
 	knowledgeset := knowledgeset.New(c.services.Invoker)
 	knowledgesource := knowledgesource.NewHandler(c.services.Invoker, c.services.GPTClient)

pkg/credstores/credstore.go

@@ -7,6 +7,8 @@ import (
 	"os/exec"
 	"strings"
 
+	"github.com/gptscript-ai/gptscript/pkg/input"
+	"github.com/gptscript-ai/gptscript/pkg/loader"
 	"github.com/obot-platform/obot/logger"
 )
 
@@ -17,16 +19,16 @@ type Options struct {
 
 var log = logger.Package()
 
-func Init(ctx context.Context, toolsRegistry, dsn string, opts Options) (string, []string, error) {
+func Init(ctx context.Context, toolRegistries []string, dsn string, opts Options) (string, []string, error) {
 	if err := setupKMS(ctx, opts.AWSKMSKeyARN, opts.EncryptionConfigFile); err != nil {
 		return "", nil, fmt.Errorf("failed to setup kms: %w", err)
 	}
 
 	switch {
 	case strings.HasPrefix(dsn, "sqlite://"):
-		return setupSQLite(toolsRegistry, dsn)
+		return setupSQLite(toolRegistries, dsn)
 	case strings.HasPrefix(dsn, "postgres://"):
-		return setupPostgres(toolsRegistry, dsn)
+		return setupPostgres(toolRegistries, dsn)
 	default:
 		return "", nil, fmt.Errorf("unsupported database for credentials %s", dsn)
 	}
@@ -69,13 +71,18 @@ func setupKMS(ctx context.Context, arn, configFile string) error {
 	return nil
 }
 
-func setupPostgres(toolRegistry, dsn string) (string, []string, error) {
-	return toolRegistry + "/credential-stores/postgres", []string{
+func setupPostgres(toolRegistries []string, dsn string) (string, []string, error) {
+	toolRef, err := resolveToolRef(toolRegistries, "credential-stores/postgres")
+	if err != nil {
+		return "", nil, err
+	}
+
+	return toolRef, []string{
 		"GPTSCRIPT_POSTGRES_DSN=" + dsn,
 	}, nil
 }
 
-func setupSQLite(toolRegistry, dsn string) (string, []string, error) {
+func setupSQLite(toolRegistries []string, dsn string) (string, []string, error) {
 	dbFile, ok := strings.CutPrefix(dsn, "sqlite://file:")
 	if !ok {
 		return "", nil, fmt.Errorf("invalid sqlite dsn, must start with sqlite://file: %s", dsn)
@@ -88,7 +95,33 @@ func setupSQLite(toolRegistry, dsn string) (string, []string, error) {
 
 	dbFile = strings.TrimSuffix(dbFile, ".db") + "-credentials.db"
 
-	return toolRegistry + "/credential-stores/sqlite", []string{
+	toolRef, err := resolveToolRef(toolRegistries, "credential-stores/sqlite")
+	if err != nil {
+		return "", nil, err
+	}
+
+	return toolRef, []string{
 		"GPTSCRIPT_SQLITE_FILE=" + dbFile,
 	}, nil
 }
+
+func resolveToolRef(toolRegistries []string, relToolPath string) (string, error) {
+	for _, toolRegistry := range toolRegistries {
+		if remapped := loader.Remap[toolRegistry]; remapped != "" {
+			toolRegistry = remapped
+		}
+
+		// This doesn't support registry references with revisions; e.g. `<registry>@<revision>`
+		ref := fmt.Sprintf("%s/%s", toolRegistry, relToolPath)
+		content, err := input.FromLocation(ref+"/tool.gpt", true)
+		if err != nil || content == "" {
+			continue
+		}
+
+		// Note: We could parse the content here to be extra sure the tool we're looking for exists,
+		// but this is probably good enough for now.
+		return ref, nil
+	}
+
+	return "", fmt.Errorf("%q not found in provided tool registries", relToolPath)
+}