Merge pull request #24 from carlson-svg/main

adding docs and example worker-config.json
runpod-workers · Oct 16, 2024 · 9c9d80a · 9c9d80a
2 parents 21c7646 + 77fe109
commit 9c9d80a
Show file tree

Hide file tree

Showing 2 changed files with 213 additions and 0 deletions.
diff --git a/README.md b/README.md
@@ -18,6 +18,103 @@
 5. Add any dependencies to the `requirements.txt` file.
 6. Add any other build time scripts to the`builder` directory, for example, downloading models.
 7. Update the `Dockerfile` to include any additional dependencies.
+8. Replace the template `worker-config.json` file with your own (the template one is an example from our [vLLM worker](https://github.com/runpod-workers/worker-vllm)).
+
+### 🔧 | Worker Config
+
+The `worker-config.json` is a JSON file that is used to build the form that helps users configure their serverless endpoint on the RunPod Web Interface.
+
+Note: This is a new feature and only works for workers that use one model 
+
+<details>
+<summary>Writing your worker-config.json</summary>
+
+The JSON consists of two main parts, schema and versions.
+- `schema`: Here you specify the form fields that will be displayed to the user.
+  - `env_var_name`: The name of the environment variable that is being set using the form field.
+  - `value`: This is the default value of the form field. It will be shown in the UI as such unless the user changes it.
+  - `title`: This is the title of the form field in the UI.
+  - `description`: This is the description of the form field in the UI.
+  - `required`: This is a boolean that specifies if the form field is required.
+  - `type`: This is the type of the form field. Options are:
+    - `text`: Environment variable is a string so user inputs text in form field.
+    - `select`: User selects one option from the dropdown. You must provide the `options` key value pair after type if using this.
+    - `toggle`: User toggles between true and false.
+    - `number`: User inputs a number in the form field.
+  - `options`: Specify the options the user can select from if the type is `select`. DO NOT include this unless the `type` is `select`.
+- `versions`: This is where you call the form fields specified in `schema` and organize them into categories.
+  - `imageName`: This is the name of the Docker image that will be used to run the serverless endpoint.
+  - `minimumCudaVersion`: This is the minimum CUDA version that is required to run the serverless endpoint.
+  - `categories`: This is where you call the keys of the form fields specified in `schema` and organize them into categories. Each category is a toggle list of forms on the Web UI.
+    - `title`: This is the title of the category in the UI.
+    - `settings`: This is the array of settings schemas specified in `schema` associated with the category.
+
+<details>
+<summary>Example of schema</summary>
+
+```json
+{
+  "schema": {
+    "TOKENIZER": {
+      "env_var_name": "TOKENIZER",
+      "value": "",
+      "title": "Tokenizer",
+      "description": "Name or path of the Hugging Face tokenizer to use.",
+      "required": false,
+      "type": "text"
+    }, 
+    "TOKENIZER_MODE": {
+      "env_var_name": "TOKENIZER_MODE",
+      "value": "auto",
+      "title": "Tokenizer Mode",
+      "description": "The tokenizer mode.",
+      "required": false,
+      "type": "select",
+      "options": [
+        { "value": "auto", "label": "auto" },
+        { "value": "slow", "label": "slow" }
+      ]
+    },
+    ...
+  }
+}
+```
+</details>
+
+<details>
+<summary>Example of versions</summary>
+
+```json
+{
+  "versions": {
+    "0.5.4": {
+      "imageName": "runpod/worker-v1-vllm:v1.2.0stable-cuda12.1.0",
+      "minimumCudaVersion": "12.1",
+      "categories": [
+        {
+          "title": "LLM Settings",
+          "settings": [
+            "TOKENIZER", "TOKENIZER_MODE", "OTHER_SETTINGS_SCHEMA_KEYS_YOU_HAVE_SPECIFIED_0", ...
+          ]
+        },
+        {
+          "title": "Tokenizer Settings",
+          "settings": [
+            "OTHER_SETTINGS_SCHEMA_KEYS_0", "OTHER_SETTINGS_SCHEMA_KEYS_1", ...
+          ]
+        },
+        ...
+      ]
+    }
+  }
+}
+```
+</details>
+</details>
+
+
+
+
 
 ### ⚙️ | CI/CD (GitHub Actions)
 

diff --git a/worker-config.json b/worker-config.json
@@ -0,0 +1,116 @@
+{
+    "versions": {
+        "version_number": {
+            "imageName": "DOCKER_IMAGE_NAME:TAG",
+            "minimumCudaVersion": "MINUMUM_CUDA_VERSION",
+            "categories": [
+                {
+                    "title": "CATEGORY_GROUP_TITLE_DISPLAYED_IN_UI",
+                    "settings": [
+                        "SCHEMA_SETTING_KEY_0", "SCHEMA_SETTING_KEY_2", "SCHEMA_SETTING_KEY_3",
+                        "SCHEMA_SETTING_KEY_4"
+                    ]
+                },
+                {
+                    "title": "CATEGORY_GROUP_TITLE_DISPLAYED_IN_UI_2",
+                    "settings": [
+                        "SCHEMA_SETTING_KEY_5", "SCHEMA_SETTING_KEY_6", "SCHEMA_SETTING_KEY_7"
+                    ]
+                },
+                {
+                    "title": "CATEGORY_GROUP_TITLE_DISPLAYED_IN_UI_3",
+                    "settings": [
+                        "SCHEMA_SETTING_KEY_8", "SCHEMA_SETTING_KEY_9"
+                    ]
+                }
+            ]
+        },
+        "0.5.4": {
+            "imageName": "runpod/ai-api-sdxl:1.2.1",
+            "minimumCudaVersion": "11.8",
+            "categories": [
+                {
+                    "title": "LLM Settings",
+                    "settings": [
+                        "TOKENIZER", "MAX_NUM_SEQS", "LOAD_FORMAT"
+                    ]
+                },
+                {
+                    "title": "Serverless Settings",
+                    "settings": [
+                        "DISABLE_LOG_STATS"
+                    ]
+                }
+            ]
+        },
+        "0.4.2": {
+            "imageName": "runpod/ai-api-sdxl:1.0.5",
+            "minimumCudaVersion": "11.6",
+            "categories": [
+                {
+                    "title": "LLM Settings",
+                    "settings": [
+                        "TOKENIZER"
+                    ]
+                },
+                {
+                    "title": "Serverless Settings",
+                    "settings": [
+                        "DISABLE_LOG_STATS"
+                    ]
+                }
+            ]
+        }
+    },
+    "schema": {
+        "SCHEMA_SETTING_KEY": {
+            "env_var_name": "ENV_VAR_NAME_ON_DEPLOYMENT",
+            "value": "DEFAULT_VALUE",
+            "title": "TITLE_DISPLAYED_IN_UI_FORM",
+            "description": "DESCRIPTION_DISPLAYED_IN_UI_FORM",
+            "required": "BOOLEAN_IF_ENV_VAR_MUST_BE_SET_BY_USER",
+            "type": "TYPE_OF_FORM_INPUT_FIELD (text, number, toggle, or select))"
+        },
+        "TOKENIZER": {
+            "env_var_name": "TOKENIZER",
+            "value": "",
+            "title": "Tokenizer",
+            "description": "Name or path of the Hugging Face tokenizer to use.",
+            "required": false,
+            "type": "text"
+        },
+        "MAX_NUM_SEQS": {
+            "env_var_name": "MAX_NUM_SEQS",
+            "value": 256,
+            "title": "Max Num Seqs",
+            "description": "Maximum number of sequences per iteration.",
+            "required": false,
+            "type": "number"
+        },
+        "DISABLE_LOG_STATS": {
+            "env_var_name": "DISABLE_LOG_STATS",
+            "value": false,
+            "title": "Disable Log Stats",
+            "description": "Disable logging statistics.",
+            "required": false,
+            "type": "toggle"
+        },
+        "LOAD_FORMAT": {
+            "env_var_name": "LOAD_FORMAT",
+            "value": "auto",
+            "title": "Load Format",
+            "description": "The format of the model weights to load.",
+            "required": false,
+            "type": "select",
+            "options": [
+              { "value": "auto", "label": "auto" },
+              { "value": "pt", "label": "pt" },
+              { "value": "safetensors", "label": "safetensors" },
+              { "value": "npcache", "label": "npcache" },
+              { "value": "dummy", "label": "dummy" },
+              { "value": "tensorizer", "label": "tensorizer" },
+              { "value": "bitsandbytes", "label": "bitsandbytes" }
+            ]
+        }
+    }
+}