forked from Mozilla-Ocho/llamafile
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'Mozilla-Ocho:main' into main
- Loading branch information
Showing
162 changed files
with
55,172 additions
and
23,496 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,53 @@ | ||
| #!/bin/sh | ||
|
|
||
| find_nvcc() { | ||
| CC=$(command -v nvcc 2>/dev/null) && return | ||
| CC="$CUDA_PATH/bin/nvcc" | ||
| [ -x "$CC" ] && return | ||
| CC="/opt/cuda/bin/nvcc" | ||
| [ -x "$CC" ] && return | ||
| CC="/usr/local/cuda/bin/nvcc" | ||
| [ -x "$CC" ] && return | ||
| return 1 | ||
| } | ||
|
|
||
| find_hipcc() { | ||
| CC=$(command -v hipcc 2>/dev/null) && return | ||
| CC="$HIP_PATH/bin/hipcc" | ||
| [ -x "$CC" ] && return | ||
| CC="/opt/rocm/bin/hipcc" | ||
| [ -x "$CC" ] && return | ||
| CC="/usr/local/rocm/bin/hipcc" | ||
| [ -x "$CC" ] && return | ||
| return 1 | ||
| } | ||
|
|
||
| if find_hipcc; then | ||
| VENDOR=AMD | ||
| FLAGS= | ||
| elif find_nvcc; then | ||
| VENDOR=NVIDIA | ||
| FLAGS="--forward-unknown-to-host-compiler" | ||
| else | ||
| echo 'error: need either hipcc (AMD) or nvcc (NVIDIA) on $PATH' >&2 | ||
| exit 1 | ||
| fi | ||
|
|
||
| FIRST=1 | ||
| for x; do | ||
| if [ $FIRST -eq 1 ]; then | ||
| set -- | ||
| FIRST=0 | ||
| fi | ||
| if [ $VENDOR = AMD ]; then | ||
| if [ x"$x" = x"-lcublas" ]; then | ||
| set -- "$@" -lhipblas -lrocblas | ||
| continue | ||
| elif [ x"$x" = x"--use_fast_math" ]; then | ||
| continue | ||
| fi | ||
| fi | ||
| set -- "$@" "$x" | ||
| done | ||
|
|
||
| exec "$CC" $FLAGS "$@" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,64 +1,92 @@ | ||
| #!/bin/sh | ||
| FILE=$1 | ||
| SCRIPTNAME=${0##*/} | ||
| BIN=${0%/*} | ||
| PROG=${0##*/} | ||
|
|
||
| if [ -z "$FILE" ]; then | ||
| echo "Usage: $SCRIPTNAME <gguf file or url> [cli|server|both]" | ||
| if [ x"$1" = x"--help" ]; then | ||
| echo "Usage: $PROG <gguf file or url>" | ||
| echo | ||
| echo "This program converts GGUF weights into a llamafile." | ||
| echo "Your .llamafile is outputted to the current directory." | ||
| echo | ||
| echo "You can supply either a .gguf filename, or the URL to" | ||
| echo "download one from an online service like Hugging Face." | ||
| echo | ||
| echo "When you run this program, it's recommended that you've" | ||
| echo "downloaded or installed an official llamafile-VERSION.zip" | ||
| echo "from https://github.com/Mozilla-Ocho/llamafile/releases" | ||
| echo "because they include prebuilt DLLs for CUDA and ROCm." | ||
| echo "You can verify your llamafile has them w/ unzip -vl" | ||
| exit 0 | ||
| fi | ||
|
|
||
| abort() { | ||
| echo "conversion terminated." >&2 | ||
| exit 1 | ||
| } | ||
|
|
||
| # find paths of golden llamafile binaries | ||
| # | ||
| # 1. if user downloaded `llamafile-VERSION.zip`, extracted it, and ran | ||
| # `./llamafile-VERSION/bin/llamafile-convert` directly, then we can | ||
| # support that by looking for a `llamafile` in the same bin folder. | ||
| # | ||
| # 2. otherwise, perform a $PATH lookup for llamafile | ||
| # | ||
| LLAMAFILE="$BIN/llamafile" | ||
| if [ ! -x "$LLAMAFILE" ]; then | ||
| LLAMAFILE=$(command -v llamafile) || abort | ||
| fi | ||
| ZIPALIGN="$BIN/zipalign" | ||
| if [ ! -x "$ZIPALIGN" ]; then | ||
| ZIPALIGN=$(command -v zipalign) || abort | ||
| fi | ||
|
|
||
| # get path of downloader program | ||
| if WGET=$(command -v wget 2>/dev/null); then | ||
| DOWNLOAD=$WGET | ||
| DOWNLOAD_ARGS=-O | ||
| elif CURL=$(command -v curl 2>/dev/null); then | ||
| DOWNLOAD=$CURL | ||
| DOWNLOAD_ARGS=-fLo | ||
| else | ||
| echo "$PROG: fatal error: you need to install either wget or curl" >&2 | ||
| echo "please download https://cosmo.zip/pub/cosmos/bin/wget and put it on the system path" >&2 | ||
| abort | ||
| fi | ||
|
|
||
| # get first program argument | ||
| FILE=$1 | ||
| if [ -z "$FILE" ]; then | ||
| echo "$PROG: missing operand (pass --help for help)" >&2 | ||
| abort | ||
| fi | ||
|
|
||
| # if the file starts with http | ||
| SHOULD_DELETE=0 | ||
| if [ x"$FILE" != x"${FILE#http*}" ]; then | ||
| # download the file | ||
| # if the filename contains ?download=true, remove it | ||
| FILE=$(echo $FILE | sed 's/?download=true//g') | ||
| # get the filename | ||
| FILENAME=$(echo $FILE | sed 's/.*\///g') | ||
| echo "Downloading $FILENAME" >&2 | ||
| if WGET=$(command -v wget 2>/dev/null); then | ||
| DOWNLOAD=$WGET | ||
| DOWNLOAD_ARGS=-O | ||
| elif CURL=$(command -v curl 2>/dev/null); then | ||
| DOWNLOAD=$CURL | ||
| DOWNLOAD_ARGS=-fLo | ||
| else | ||
| printf '%s\n' "$0: fatal error: you need to install either wget or curl" >&2 | ||
| printf '%s\n' "please download https://cosmo.zip/pub/cosmos/bin/wget and put it on the system path" >&2 | ||
| abort | ||
| fi | ||
| "${DOWNLOAD}" ${DOWNLOAD_ARGS} $FILENAME $FILE | ||
| # get the filename | ||
| FILE=$FILENAME | ||
| URL=$FILE | ||
| URL=${URL%?download=true} # strip "?download=true" suffix | ||
| FILE=${URL##*/} # local file is basename of url | ||
| echo "Downloading $FILE" >&2 | ||
| "${DOWNLOAD}" ${DOWNLOAD_ARGS} "$FILE" "$URL" || abort | ||
| SHOULD_DELETE=1 | ||
| fi | ||
|
|
||
| # replace .gguf with .llamafile | ||
| LLAMAFILE_NAME=$(echo $FILE | sed 's/.gguf/.llamafile/g') | ||
| LLAMAFILE_PATH=$(command -v llamafile) | ||
| CLI_ARGS="-m | ||
| $FILE | ||
| # create output in current directory | ||
| echo "Using $LLAMAFILE as golden llamafile binary" >&2 | ||
| OUTPUT=${FILE##*/} # basename | ||
| OUTPUT="${OUTPUT%.gguf}.llamafile" | ||
| echo "Converting $FILE to $OUTPUT" >&2 | ||
| cp -f "$LLAMAFILE" "$OUTPUT" || abort | ||
| printf %s "-m | ||
| ${FILE##*/} | ||
| ... | ||
| " | ||
|
|
||
| convert() { | ||
| echo "Converting $FILE to $LLAMAFILE_NAME" | ||
| # print CLI args to .args | ||
| printf %s "$CLI_ARGS" > .args | ||
| cp $LLAMAFILE_PATH $LLAMAFILE_NAME | ||
| zipalign -j0 $LLAMAFILE_NAME $FILE .args | ||
| } | ||
| " > .args | ||
| "$ZIPALIGN" -j0 "$OUTPUT" "$FILE" .args || abort | ||
|
|
||
| cleanup() { | ||
| echo "Cleaning up" | ||
| rm -f .args | ||
| # remove the downloaded file | ||
| rm -f $FILE | ||
| echo "Done" | ||
| } | ||
|
|
||
| abort() { | ||
| printf '%s\n' "conversion terminated." >&2 | ||
| exit 1 | ||
| } | ||
|
|
||
| convert || abort | ||
| cleanup | ||
| # cleanup | ||
| rm -f .args | ||
| if [ $SHOULD_DELETE -eq 1 ]; then | ||
| rm -f "$FILE" | ||
| fi | ||
| echo "Success. You may now run ./$OUTPUT" >&2 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,6 +1,6 @@ | ||
| #!/bin/sh | ||
| if printf '%s\n' "$*" | grep aarch64 >/dev/null 2>&1; then | ||
| exec aarch64-unknown-cosmo-objdump "$@" | ||
| exec aarch64-unknown-cosmo-objdump $1 ${2%/*}/.aarch64/${2##*/} | ||
| else | ||
| exec x86_64-unknown-cosmo-objdump "$@" | ||
| fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.