From f0add11b02628ea9d69655447e2502f1f956e319 Mon Sep 17 00:00:00 2001 From: sanjay920 Date: Tue, 17 Dec 2024 17:21:53 -0800 Subject: [PATCH 1/9] init vllm model provider --- vllm-model-provider/go.mod | 5 +++ vllm-model-provider/go.sum | 2 + vllm-model-provider/main.go | 31 +++++++++++++++ vllm-model-provider/server/server.go | 59 ++++++++++++++++++++++++++++ vllm-model-provider/tool.gpt | 11 ++++++ 5 files changed, 108 insertions(+) create mode 100644 vllm-model-provider/go.mod create mode 100644 vllm-model-provider/go.sum create mode 100644 vllm-model-provider/main.go create mode 100644 vllm-model-provider/server/server.go create mode 100644 vllm-model-provider/tool.gpt diff --git a/vllm-model-provider/go.mod b/vllm-model-provider/go.mod new file mode 100644 index 00000000..d1ffb77d --- /dev/null +++ b/vllm-model-provider/go.mod @@ -0,0 +1,5 @@ +module github.com/obot-platform/tools/openai-model-provider + +go 1.23.4 + +require github.com/gptscript-ai/chat-completion-client v0.0.0-20241127005108-02b41e1cd02e diff --git a/vllm-model-provider/go.sum b/vllm-model-provider/go.sum new file mode 100644 index 00000000..e6ef4500 --- /dev/null +++ b/vllm-model-provider/go.sum @@ -0,0 +1,2 @@ +github.com/gptscript-ai/chat-completion-client v0.0.0-20241127005108-02b41e1cd02e h1:Nj9709xPbjAPLOsdR/Ik4zaJfpU4O6AEP/R6o9h30CE= +github.com/gptscript-ai/chat-completion-client v0.0.0-20241127005108-02b41e1cd02e/go.mod h1:7P/o6/IWa1KqsntVf68hSnLKuu3+xuqm6lYhch1w4jo= diff --git a/vllm-model-provider/main.go b/vllm-model-provider/main.go new file mode 100644 index 00000000..f492240a --- /dev/null +++ b/vllm-model-provider/main.go @@ -0,0 +1,31 @@ +package main + +import ( + "fmt" + "os" + + "github.com/obot-platform/tools/vllm-model-provider/server" +) + +func main() { + apiKey := os.Getenv("OBOT_VLLM_MODEL_PROVIDER_API_KEY") + if apiKey == "" { + fmt.Println("OBOT_VLLM_MODEL_PROVIDER_API_KEY environment variable not set") + os.Exit(1) + } + + endpoint := os.Getenv("OBOT_VLLM_MODEL_PROVIDER_ENDPOINT") + if endpoint == "" { + fmt.Println("OBOT_VLLM_MODEL_PROVIDER_ENDPOINT environment variable not set") + os.Exit(1) + } + + port := os.Getenv("PORT") + if port == "" { + port = "8000" + } + + if err := server.Run(apiKey, port); err != nil { + panic(err) + } +} diff --git a/vllm-model-provider/server/server.go b/vllm-model-provider/server/server.go new file mode 100644 index 00000000..3a128a69 --- /dev/null +++ b/vllm-model-provider/server/server.go @@ -0,0 +1,59 @@ +package server + +import ( + "errors" + "fmt" + "net/http" + "net/http/httputil" + "os" +) + +func Run(apiKey, port string) error { + endpoint := os.Getenv("OBOT_VLLM_MODEL_PROVIDER_ENDPOINT") + if endpoint == "" { + return fmt.Errorf("OBOT_VLLM_MODEL_PROVIDER_ENDPOINT environment variable not set") + } + + mux := http.NewServeMux() + + s := &server{ + apiKey: apiKey, + port: port, + endpoint: endpoint, + } + + mux.HandleFunc("/{$}", s.healthz) + mux.Handle("GET /v1/models", &httputil.ReverseProxy{ + Director: s.proxy, + }) + mux.Handle("/{path...}", &httputil.ReverseProxy{ + Director: s.proxy, + }) + + httpServer := &http.Server{ + Addr: "127.0.0.1:" + port, + Handler: mux, + } + + if err := httpServer.ListenAndServe(); !errors.Is(err, http.ErrServerClosed) { + return err + } + + return nil +} + +type server struct { + apiKey, port, endpoint string +} + +func (s *server) healthz(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte("http://127.0.0.1:" + s.port)) +} + +func (s *server) proxy(req *http.Request) { + req.URL.Host = s.endpoint + req.URL.Scheme = "https" + req.Host = req.URL.Host + + req.Header.Set("Authorization", "Bearer "+s.apiKey) +} diff --git a/vllm-model-provider/tool.gpt b/vllm-model-provider/tool.gpt new file mode 100644 index 00000000..d671316f --- /dev/null +++ b/vllm-model-provider/tool.gpt @@ -0,0 +1,11 @@ +Name: vLLM +Description: Model Provider for vLLM +Metadata: envVars: OBOT_VLLM_MODEL_PROVIDER_ENDPOINT,OBOT_VLLM_MODEL_PROVIDER_API_KEY +Model Provider: true +Credential: ../model-provider-credential as vllm-model-provider + +#!sys.daemon ${GPTSCRIPT_TOOL_DIR}/bin/gptscript-go-tool + +--- +!metadata:*:icon +https://raw.githubusercontent.com/vllm-project/media-kit/refs/heads/main/vLLM-Full-Logo.svg \ No newline at end of file From 07d7c737c45321b9c4d7391743738a21c4a5eda8 Mon Sep 17 00:00:00 2001 From: sanjay920 Date: Wed, 18 Dec 2024 16:12:49 -0800 Subject: [PATCH 2/9] update vllm model provider --- index.yaml | 2 + vllm-model-provider/go.mod | 4 +- vllm-model-provider/go.sum | 4 +- vllm-model-provider/server/server.go | 151 +++++++++++++++++++-------- 4 files changed, 114 insertions(+), 47 deletions(-) diff --git a/index.yaml b/index.yaml index 5fed66ac..9f5a6e80 100644 --- a/index.yaml +++ b/index.yaml @@ -90,3 +90,5 @@ modelProviders: reference: ./groq-model-provider voyage-model-provider: reference: ./voyage-model-provider + vllm-model-provider: + reference: ./vllm-model-provider diff --git a/vllm-model-provider/go.mod b/vllm-model-provider/go.mod index d1ffb77d..b7a66a73 100644 --- a/vllm-model-provider/go.mod +++ b/vllm-model-provider/go.mod @@ -1,5 +1,5 @@ -module github.com/obot-platform/tools/openai-model-provider +module github.com/sanjay920/tools/vllm-model-provider go 1.23.4 -require github.com/gptscript-ai/chat-completion-client v0.0.0-20241127005108-02b41e1cd02e +require github.com/gptscript-ai/chat-completion-client v0.0.0-20241216203633-5c0178fb89ed // indirect diff --git a/vllm-model-provider/go.sum b/vllm-model-provider/go.sum index e6ef4500..1d96f708 100644 --- a/vllm-model-provider/go.sum +++ b/vllm-model-provider/go.sum @@ -1,2 +1,2 @@ -github.com/gptscript-ai/chat-completion-client v0.0.0-20241127005108-02b41e1cd02e h1:Nj9709xPbjAPLOsdR/Ik4zaJfpU4O6AEP/R6o9h30CE= -github.com/gptscript-ai/chat-completion-client v0.0.0-20241127005108-02b41e1cd02e/go.mod h1:7P/o6/IWa1KqsntVf68hSnLKuu3+xuqm6lYhch1w4jo= +github.com/gptscript-ai/chat-completion-client v0.0.0-20241216203633-5c0178fb89ed h1:qMHm0IYpKgmw4KHX76RMB/duSICxo7IZuimPCKb0qG4= +github.com/gptscript-ai/chat-completion-client v0.0.0-20241216203633-5c0178fb89ed/go.mod h1:7P/o6/IWa1KqsntVf68hSnLKuu3+xuqm6lYhch1w4jo= diff --git a/vllm-model-provider/server/server.go b/vllm-model-provider/server/server.go index 3a128a69..5c149f81 100644 --- a/vllm-model-provider/server/server.go +++ b/vllm-model-provider/server/server.go @@ -1,59 +1,124 @@ package server import ( - "errors" - "fmt" - "net/http" - "net/http/httputil" - "os" + "bytes" + "compress/gzip" + "crypto/tls" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/httputil" + "os" + "strings" + + "github.com/gptscript-ai/chat-completion-client" ) func Run(apiKey, port string) error { - endpoint := os.Getenv("OBOT_VLLM_MODEL_PROVIDER_ENDPOINT") - if endpoint == "" { - return fmt.Errorf("OBOT_VLLM_MODEL_PROVIDER_ENDPOINT environment variable not set") - } - - mux := http.NewServeMux() - - s := &server{ - apiKey: apiKey, - port: port, - endpoint: endpoint, - } - - mux.HandleFunc("/{$}", s.healthz) - mux.Handle("GET /v1/models", &httputil.ReverseProxy{ - Director: s.proxy, - }) - mux.Handle("/{path...}", &httputil.ReverseProxy{ - Director: s.proxy, - }) - - httpServer := &http.Server{ - Addr: "127.0.0.1:" + port, - Handler: mux, - } - - if err := httpServer.ListenAndServe(); !errors.Is(err, http.ErrServerClosed) { - return err - } - - return nil + endpoint := os.Getenv("OBOT_VLLM_MODEL_PROVIDER_ENDPOINT") + if endpoint == "" { + return fmt.Errorf("OBOT_VLLM_MODEL_PROVIDER_ENDPOINT environment variable not set") + } + + // Remove https:// or http:// if present + endpoint = strings.TrimPrefix(endpoint, "https://") + endpoint = strings.TrimPrefix(endpoint, "http://") + + mux := http.NewServeMux() + + s := &server{ + apiKey: apiKey, + port: port, + endpoint: endpoint, + } + + mux.HandleFunc("/{$}", s.healthz) + mux.Handle("GET /v1/models", &httputil.ReverseProxy{ + Director: s.proxy, + ModifyResponse: s.rewriteModelsResponse, + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + }, + }) + mux.Handle("/{path...}", &httputil.ReverseProxy{ + Director: s.proxy, + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + }, + }) + + httpServer := &http.Server{ + Addr: "127.0.0.1:" + port, + Handler: mux, + } + + if err := httpServer.ListenAndServe(); !errors.Is(err, http.ErrServerClosed) { + return err + } + + return nil } type server struct { - apiKey, port, endpoint string + apiKey, port, endpoint string } func (s *server) healthz(w http.ResponseWriter, _ *http.Request) { - _, _ = w.Write([]byte("http://127.0.0.1:" + s.port)) + _, _ = w.Write([]byte("http://127.0.0.1:" + s.port)) } -func (s *server) proxy(req *http.Request) { - req.URL.Host = s.endpoint - req.URL.Scheme = "https" - req.Host = req.URL.Host +func (s *server) rewriteModelsResponse(resp *http.Response) error { + if resp.StatusCode != http.StatusOK { + return nil + } - req.Header.Set("Authorization", "Bearer "+s.apiKey) + originalBody := resp.Body + defer originalBody.Close() + + if resp.Header.Get("Content-Encoding") == "gzip" { + var err error + originalBody, err = gzip.NewReader(originalBody) + if err != nil { + return fmt.Errorf("failed to create gzip reader: %w", err) + } + defer originalBody.Close() + resp.Header.Del("Content-Encoding") + } + + var models openai.ModelsList + if err := json.NewDecoder(originalBody).Decode(&models); err != nil { + return fmt.Errorf("failed to decode models response: %w, %d, %v", err, resp.StatusCode, resp.Header) + } + + // Set all models as LLM + for i, model := range models.Models { + if model.Metadata == nil { + model.Metadata = make(map[string]string) + } + model.Metadata["usage"] = "llm" + models.Models[i] = model + } + + b, err := json.Marshal(models) + if err != nil { + return fmt.Errorf("failed to marshal models response: %w", err) + } + + resp.Body = io.NopCloser(bytes.NewReader(b)) + resp.Header.Set("Content-Length", fmt.Sprintf("%d", len(b))) + return nil } + +func (s *server) proxy(req *http.Request) { + req.URL.Host = s.endpoint + req.URL.Scheme = "https" + req.Host = req.URL.Host + + req.Header.Set("Authorization", "Bearer "+s.apiKey) +} \ No newline at end of file From f885ff4be984a7984fde8077f7d9b25bae8f9600 Mon Sep 17 00:00:00 2001 From: sanjay920 Date: Wed, 18 Dec 2024 16:30:12 -0800 Subject: [PATCH 3/9] update tool.gpt for vllm --- vllm-model-provider/tool.gpt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vllm-model-provider/tool.gpt b/vllm-model-provider/tool.gpt index d671316f..5f2f9fce 100644 --- a/vllm-model-provider/tool.gpt +++ b/vllm-model-provider/tool.gpt @@ -8,4 +8,4 @@ Credential: ../model-provider-credential as vllm-model-provider --- !metadata:*:icon -https://raw.githubusercontent.com/vllm-project/media-kit/refs/heads/main/vLLM-Full-Logo.svg \ No newline at end of file +/admin/assets/vllm-logo.svg \ No newline at end of file From eda048693baa219cbae31253b4594e8f7e0a9f07 Mon Sep 17 00:00:00 2001 From: sanjay920 Date: Thu, 19 Dec 2024 16:41:54 -0800 Subject: [PATCH 4/9] Update vllm-model-provider/go.mod Co-authored-by: Grant Linville --- vllm-model-provider/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vllm-model-provider/go.mod b/vllm-model-provider/go.mod index b7a66a73..e5ff1d7d 100644 --- a/vllm-model-provider/go.mod +++ b/vllm-model-provider/go.mod @@ -1,4 +1,4 @@ -module github.com/sanjay920/tools/vllm-model-provider +module github.com/obot-platform/tools/vllm-model-provider go 1.23.4 From 563f798abcdb42ac7fa04ac867a61138e4a38a1b Mon Sep 17 00:00:00 2001 From: sanjay920 Date: Thu, 19 Dec 2024 16:53:12 -0800 Subject: [PATCH 5/9] store the endpoint as a *url.URL instead of a string --- vllm-model-provider/server/server.go | 220 ++++++++++++++------------- 1 file changed, 115 insertions(+), 105 deletions(-) diff --git a/vllm-model-provider/server/server.go b/vllm-model-provider/server/server.go index 5c149f81..5811c88a 100644 --- a/vllm-model-provider/server/server.go +++ b/vllm-model-provider/server/server.go @@ -1,124 +1,134 @@ package server import ( - "bytes" - "compress/gzip" - "crypto/tls" - "encoding/json" - "errors" - "fmt" - "io" - "net/http" - "net/http/httputil" - "os" - "strings" - - "github.com/gptscript-ai/chat-completion-client" + "bytes" + "compress/gzip" + "crypto/tls" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/httputil" + "net/url" + "os" + "path" + + openai "github.com/gptscript-ai/chat-completion-client" ) func Run(apiKey, port string) error { - endpoint := os.Getenv("OBOT_VLLM_MODEL_PROVIDER_ENDPOINT") - if endpoint == "" { - return fmt.Errorf("OBOT_VLLM_MODEL_PROVIDER_ENDPOINT environment variable not set") - } - - // Remove https:// or http:// if present - endpoint = strings.TrimPrefix(endpoint, "https://") - endpoint = strings.TrimPrefix(endpoint, "http://") - - mux := http.NewServeMux() - - s := &server{ - apiKey: apiKey, - port: port, - endpoint: endpoint, - } - - mux.HandleFunc("/{$}", s.healthz) - mux.Handle("GET /v1/models", &httputil.ReverseProxy{ - Director: s.proxy, - ModifyResponse: s.rewriteModelsResponse, - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: true, - }, - }, - }) - mux.Handle("/{path...}", &httputil.ReverseProxy{ - Director: s.proxy, - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: true, - }, - }, - }) - - httpServer := &http.Server{ - Addr: "127.0.0.1:" + port, - Handler: mux, - } - - if err := httpServer.ListenAndServe(); !errors.Is(err, http.ErrServerClosed) { - return err - } - - return nil + endpointStr := os.Getenv("OBOT_VLLM_MODEL_PROVIDER_ENDPOINT") + if endpointStr == "" { + return fmt.Errorf("OBOT_VLLM_MODEL_PROVIDER_ENDPOINT environment variable not set") + } + + // Parse the endpoint URL + endpoint, err := url.Parse(endpointStr) + if err != nil { + return fmt.Errorf("invalid endpoint URL: %w", err) + } + + // Ensure scheme is set + if endpoint.Scheme == "" { + endpoint.Scheme = "https" + } + + mux := http.NewServeMux() + + s := &server{ + apiKey: apiKey, + port: port, + endpoint: endpoint, + } + + mux.HandleFunc("/{$}", s.healthz) + mux.Handle("GET /v1/models", &httputil.ReverseProxy{ + Director: s.proxy, + ModifyResponse: s.rewriteModelsResponse, + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + }, + }) + mux.Handle("/{path...}", &httputil.ReverseProxy{ + Director: s.proxy, + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + }, + }) + + httpServer := &http.Server{ + Addr: "127.0.0.1:" + port, + Handler: mux, + } + + if err := httpServer.ListenAndServe(); !errors.Is(err, http.ErrServerClosed) { + return err + } + + return nil } type server struct { - apiKey, port, endpoint string + apiKey, port string + endpoint *url.URL } func (s *server) healthz(w http.ResponseWriter, _ *http.Request) { - _, _ = w.Write([]byte("http://127.0.0.1:" + s.port)) + _, _ = w.Write([]byte("http://127.0.0.1:" + s.port)) } func (s *server) rewriteModelsResponse(resp *http.Response) error { - if resp.StatusCode != http.StatusOK { - return nil - } - - originalBody := resp.Body - defer originalBody.Close() - - if resp.Header.Get("Content-Encoding") == "gzip" { - var err error - originalBody, err = gzip.NewReader(originalBody) - if err != nil { - return fmt.Errorf("failed to create gzip reader: %w", err) - } - defer originalBody.Close() - resp.Header.Del("Content-Encoding") - } - - var models openai.ModelsList - if err := json.NewDecoder(originalBody).Decode(&models); err != nil { - return fmt.Errorf("failed to decode models response: %w, %d, %v", err, resp.StatusCode, resp.Header) - } - - // Set all models as LLM - for i, model := range models.Models { - if model.Metadata == nil { - model.Metadata = make(map[string]string) - } - model.Metadata["usage"] = "llm" - models.Models[i] = model - } - - b, err := json.Marshal(models) - if err != nil { - return fmt.Errorf("failed to marshal models response: %w", err) - } - - resp.Body = io.NopCloser(bytes.NewReader(b)) - resp.Header.Set("Content-Length", fmt.Sprintf("%d", len(b))) - return nil + if resp.StatusCode != http.StatusOK { + return nil + } + + originalBody := resp.Body + defer originalBody.Close() + + if resp.Header.Get("Content-Encoding") == "gzip" { + var err error + originalBody, err = gzip.NewReader(originalBody) + if err != nil { + return fmt.Errorf("failed to create gzip reader: %w", err) + } + defer originalBody.Close() + resp.Header.Del("Content-Encoding") + } + + var models openai.ModelsList + if err := json.NewDecoder(originalBody).Decode(&models); err != nil { + return fmt.Errorf("failed to decode models response: %w, %d, %v", err, resp.StatusCode, resp.Header) + } + + // Set all models as LLM + for i, model := range models.Models { + if model.Metadata == nil { + model.Metadata = make(map[string]string) + } + model.Metadata["usage"] = "llm" + models.Models[i] = model + } + + b, err := json.Marshal(models) + if err != nil { + return fmt.Errorf("failed to marshal models response: %w", err) + } + + resp.Body = io.NopCloser(bytes.NewReader(b)) + resp.Header.Set("Content-Length", fmt.Sprintf("%d", len(b))) + return nil } func (s *server) proxy(req *http.Request) { - req.URL.Host = s.endpoint - req.URL.Scheme = "https" - req.Host = req.URL.Host + u := *s.endpoint + u.Path = path.Join(u.Path, req.URL.Path) + req.URL = &u + req.Host = req.URL.Host - req.Header.Set("Authorization", "Bearer "+s.apiKey) -} \ No newline at end of file + req.Header.Set("Authorization", "Bearer "+s.apiKey) +} From e1311997b5716045f157538723522ee413ad1029 Mon Sep 17 00:00:00 2001 From: sanjay920 Date: Thu, 19 Dec 2024 16:55:37 -0800 Subject: [PATCH 6/9] pass endpoint in main.go --- vllm-model-provider/main.go | 2 +- vllm-model-provider/server/server.go | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/vllm-model-provider/main.go b/vllm-model-provider/main.go index f492240a..e13fd1c8 100644 --- a/vllm-model-provider/main.go +++ b/vllm-model-provider/main.go @@ -25,7 +25,7 @@ func main() { port = "8000" } - if err := server.Run(apiKey, port); err != nil { + if err := server.Run(apiKey, endpoint, port); err != nil { panic(err) } } diff --git a/vllm-model-provider/server/server.go b/vllm-model-provider/server/server.go index 5811c88a..1c54c6b7 100644 --- a/vllm-model-provider/server/server.go +++ b/vllm-model-provider/server/server.go @@ -11,16 +11,14 @@ import ( "net/http" "net/http/httputil" "net/url" - "os" "path" openai "github.com/gptscript-ai/chat-completion-client" ) -func Run(apiKey, port string) error { - endpointStr := os.Getenv("OBOT_VLLM_MODEL_PROVIDER_ENDPOINT") +func Run(apiKey, endpointStr, port string) error { if endpointStr == "" { - return fmt.Errorf("OBOT_VLLM_MODEL_PROVIDER_ENDPOINT environment variable not set") + return fmt.Errorf("endpoint is required") } // Parse the endpoint URL From cf142783a632aaabf34b47524cb0397f82ce43dc Mon Sep 17 00:00:00 2001 From: sanjay920 Date: Thu, 19 Dec 2024 17:05:05 -0800 Subject: [PATCH 7/9] update server.go --- vllm-model-provider/server/server.go | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/vllm-model-provider/server/server.go b/vllm-model-provider/server/server.go index 1c54c6b7..07e6c837 100644 --- a/vllm-model-provider/server/server.go +++ b/vllm-model-provider/server/server.go @@ -3,7 +3,6 @@ package server import ( "bytes" "compress/gzip" - "crypto/tls" "encoding/json" "errors" "fmt" @@ -27,9 +26,13 @@ func Run(apiKey, endpointStr, port string) error { return fmt.Errorf("invalid endpoint URL: %w", err) } - // Ensure scheme is set + // More intelligent scheme handling if endpoint.Scheme == "" { - endpoint.Scheme = "https" + if endpoint.Hostname() == "localhost" || endpoint.Hostname() == "127.0.0.1" { + endpoint.Scheme = "http" + } else { + endpoint.Scheme = "https" + } } mux := http.NewServeMux() @@ -44,19 +47,9 @@ func Run(apiKey, endpointStr, port string) error { mux.Handle("GET /v1/models", &httputil.ReverseProxy{ Director: s.proxy, ModifyResponse: s.rewriteModelsResponse, - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: true, - }, - }, }) mux.Handle("/{path...}", &httputil.ReverseProxy{ Director: s.proxy, - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: true, - }, - }, }) httpServer := &http.Server{ From 136316c649877707ba29a7ec179483be85b7f50c Mon Sep 17 00:00:00 2001 From: sanjay920 Date: Thu, 19 Dec 2024 17:05:44 -0800 Subject: [PATCH 8/9] update go.mod --- vllm-model-provider/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vllm-model-provider/go.mod b/vllm-model-provider/go.mod index e5ff1d7d..04ea51d8 100644 --- a/vllm-model-provider/go.mod +++ b/vllm-model-provider/go.mod @@ -2,4 +2,4 @@ module github.com/obot-platform/tools/vllm-model-provider go 1.23.4 -require github.com/gptscript-ai/chat-completion-client v0.0.0-20241216203633-5c0178fb89ed // indirect +require github.com/gptscript-ai/chat-completion-client v0.0.0-20241216203633-5c0178fb89ed From c4e69fb3d61852a2b0caa8023fa483f595538e56 Mon Sep 17 00:00:00 2001 From: sanjay920 Date: Fri, 20 Dec 2024 09:33:33 -0800 Subject: [PATCH 9/9] update server.go --- vllm-model-provider/server/server.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/vllm-model-provider/server/server.go b/vllm-model-provider/server/server.go index 07e6c837..802d406c 100644 --- a/vllm-model-provider/server/server.go +++ b/vllm-model-provider/server/server.go @@ -16,17 +16,12 @@ import ( ) func Run(apiKey, endpointStr, port string) error { - if endpointStr == "" { - return fmt.Errorf("endpoint is required") - } - // Parse the endpoint URL endpoint, err := url.Parse(endpointStr) if err != nil { - return fmt.Errorf("invalid endpoint URL: %w", err) + return fmt.Errorf("Invalid endpoint URL %q: %w", endpointStr, err) } - // More intelligent scheme handling if endpoint.Scheme == "" { if endpoint.Hostname() == "localhost" || endpoint.Hostname() == "127.0.0.1" { endpoint.Scheme = "http"