From 9f6bedea7b0d7a17db22eedc8a49c6938209a27a Mon Sep 17 00:00:00 2001 From: Paul Rock Date: Sun, 26 May 2024 22:14:32 +0300 Subject: [PATCH 1/3] Support of converting local models added to convert-hf-to-gguf-update.py --- convert-hf-to-gguf-update.py | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/convert-hf-to-gguf-update.py b/convert-hf-to-gguf-update.py index 84b72348dc579..3fc8783fa0c42 100755 --- a/convert-hf-to-gguf-update.py +++ b/convert-hf-to-gguf-update.py @@ -26,6 +26,7 @@ import os import pathlib import re +import shutil import requests import sys @@ -106,12 +107,26 @@ def download_model(model): if tokt == TOKENIZER_TYPE.SPM: files.append("tokenizer.model") - for file in files: - save_path = f"models/tokenizers/{name}/{file}" - if os.path.isfile(save_path): - logger.info(f"{name}: File {save_path} already exists - skipping") - continue - download_file_with_auth(f"{repo}/resolve/main/{file}", token, save_path) + if os.path.isdir(repo): + # Если repo это путь на файловой системе, копируем директорию + for file in files: + src_path = os.path.join(repo, file) + dst_path = f"models/tokenizers/{name}/{file}" + if os.path.isfile(dst_path): + logger.info(f"{name}: File {dst_path} already exists - skipping") + continue + if os.path.isfile(src_path): + shutil.copy2(src_path, dst_path) + logger.info(f"{name}: Copied {src_path} to {dst_path}") + else: + logger.warning(f"{name}: Source file {src_path} does not exist") + else: + for file in files: + save_path = f"models/tokenizers/{name}/{file}" + if os.path.isfile(save_path): + logger.info(f"{name}: File {save_path} already exists - skipping") + continue + download_file_with_auth(f"{repo}/resolve/main/{file}", token, save_path) for model in models: From 150111f419e2d7f8c126a10978ebc963918b34ed Mon Sep 17 00:00:00 2001 From: Paul Rock Date: Sun, 26 May 2024 22:37:35 +0300 Subject: [PATCH 2/3] Description fixed --- convert-hf-to-gguf-update.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/convert-hf-to-gguf-update.py b/convert-hf-to-gguf-update.py index 3fc8783fa0c42..efa8d6dc68897 100755 --- a/convert-hf-to-gguf-update.py +++ b/convert-hf-to-gguf-update.py @@ -108,7 +108,7 @@ def download_model(model): files.append("tokenizer.model") if os.path.isdir(repo): - # Если repo это путь на файловой системе, копируем директорию + # If repo is a path on the file system, copy the directory for file in files: src_path = os.path.join(repo, file) dst_path = f"models/tokenizers/{name}/{file}" @@ -121,6 +121,7 @@ def download_model(model): else: logger.warning(f"{name}: Source file {src_path} does not exist") else: + # If repo is a URL, download the files for file in files: save_path = f"models/tokenizers/{name}/{file}" if os.path.isfile(save_path): From 98038bdc8ae9ae2b0cbd28798554948581a1d029 Mon Sep 17 00:00:00 2001 From: pasha Date: Wed, 11 Sep 2024 14:16:32 +0300 Subject: [PATCH 3/3] shutil added to imports --- convert_hf_to_gguf_update.py | 1 + 1 file changed, 1 insertion(+) diff --git a/convert_hf_to_gguf_update.py b/convert_hf_to_gguf_update.py index 5304db60fe51a..59a0b81a18880 100755 --- a/convert_hf_to_gguf_update.py +++ b/convert_hf_to_gguf_update.py @@ -31,6 +31,7 @@ import requests import sys import json +import shutil from hashlib import sha256 from enum import IntEnum, auto