Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added Llama 3 8B support #156

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Added Llama 3 8B support #156

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions docker-compose-gguf.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,16 +3,16 @@ version: '3.6'
services:
llama-gpt-api:
# Pin to llama-cpp-python 0.1.80 with GGUF support
image: ghcr.io/abetlen/llama-cpp-python:latest@sha256:de0fd227f348b5e43d4b5b7300f1344e712c14132914d1332182e9ecfde502b2
image: ghcr.io/abetlen/llama-cpp-python:v0.2.63
restart: on-failure
volumes:
- './models:/models'
- './api:/api'
ports:
- 3001:8000
environment:
MODEL: '/models/${MODEL_NAME:-code-llama-2-7b-chat.gguf}'
MODEL_DOWNLOAD_URL: '${MODEL_DOWNLOAD_URL:-https://huggingface.co/TheBloke/CodeLlama-7B-Instruct-GGUF/resolve/main/codellama-7b-instruct.Q4_K_M.gguf}'
MODEL: '/models/${MODEL_NAME:-Meta-Llama-3-8B-Instruct.Q4_K_M.gguf}'
MODEL_DOWNLOAD_URL: '${MODEL_DOWNLOAD_URL:-https://huggingface.co/QuantFactory/Meta-Llama-3-8B-Instruct-GGUF/resolve/main/Meta-Llama-3-8B-Instruct.Q4_K_M.gguf}'
N_GQA: '${N_GQA:-1}'
USE_MLOCK: 1
cap_add:
Expand All @@ -31,7 +31,7 @@ services:
environment:
- 'OPENAI_API_KEY=sk-XXXXXXXXXXXXXXXXXXXX'
- 'OPENAI_API_HOST=http://llama-gpt-api:8000'
- 'DEFAULT_MODEL=/models/${MODEL_NAME:-llama-2-7b-chat.bin}'
- 'DEFAULT_MODEL=/models/${MODEL_NAME:-Meta-Llama-3-8B-Instruct.Q4_K_M.gguf}'
- 'NEXT_PUBLIC_DEFAULT_SYSTEM_PROMPT=${DEFAULT_SYSTEM_PROMPT:-"You are a helpful and friendly AI assistant. Respond very concisely."}'
- 'WAIT_HOSTS=llama-gpt-api:8000'
- 'WAIT_TIMEOUT=${WAIT_TIMEOUT:-3600}'
7 changes: 7 additions & 0 deletions run.sh
View file
Open in desktop
Copy link

@derek-palmer derek-palmer May 9, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd recommend updating the echo to include llama3-8b with the intro of the model to reflect the current availability of models. :)

derek-palmer@f64fc2f

Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,13 @@ model_type="gguf"

# Export the model value as an environment variable
case $model in
llama3-8b)
export MODEL_NAME="Meta-Llama-3-8B-Instruct.Q4_K_M.gguf"
export MODEL_DOWNLOAD_URL="https://huggingface.co/QuantFactory/Meta-Llama-3-8B-Instruct-GGUF/resolve/main/Meta-Llama-3-8B-Instruct.Q4_K_M.gguf"
export WAIT_TIMEOUT=3600
export N_GQA=1
model_type="gguf"
;;
7b)
export MODEL_NAME="llama-2-7b-chat.bin"
export MODEL_DOWNLOAD_URL="https://huggingface.co/TheBloke/Nous-Hermes-Llama-2-7B-GGML/resolve/main/nous-hermes-llama-2-7b.ggmlv3.q4_0.bin"
Expand Down
8 changes: 8 additions & 0 deletions ui/types/openai.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ export enum OpenAIModelID {
GPT_4 = 'gpt-4',
GPT_4_32K = 'gpt-4-32k',

LLAMA_3_8b_Q4_K_M = '/models/Meta-Llama-3-8B-Instruct.Q4_K_M.gguf',

LLAMA_7B_CHAT_GGMLV3_Q4_0 = '/models/llama-2-7b-chat.bin',
LLAMA_13B_CHAT_GGMLV3_Q4_0 = '/models/llama-2-13b-chat.bin',
LLAMA_70B_CHAT_GGMLV3_Q4_0 = '/models/llama-2-70b-chat.bin',
Expand All @@ -34,6 +36,12 @@ export enum OpenAIModelID {
export const fallbackModelID = OpenAIModelID.LLAMA_7B_CHAT_GGMLV3_Q4_0;

export const OpenAIModels: Record<OpenAIModelID, OpenAIModel> = {
[OpenAIModelID.LLAMA_3_8b_Q4_K_M]: {
id: OpenAIModelID.LLAMA_3_8b_Q4_K_M,
name: 'LLAMA 3 8B',
maxLength: 12000,
tokenLimit: 4000,
},
[OpenAIModelID.GPT_3_5]: {
id: OpenAIModelID.GPT_3_5,
name: 'GPT-3.5',
Expand Down