VLLM_qwq_32b_awq copy.json

{
    "version": "0.1",
    "type": "container",
    "meta": {
      "trigger": "cli"
    },
    "ops": [
      {
        "type": "container/run",
        "id": "vllm",
        "args": {
            "entrypoint": ["python3", "-m", "vllm.entrypoints.openai.api_server"],
            "cmd": [
              "--model",
              "KirillR/QwQ-32B-Preview-AWQ",
              "--quantization",
              "awq_marlin",
              "--served-model-name",
              "QwQ-32B-Preview",
              "--trust-remote-code", 
              "--tokenizer-mode",
              "auto"
            ],
          "image": "docker.io/vllm/vllm-openai:latest",
          "gpu": true,
          "expose": 8000
        }
      }
    ]
  }