From 52d0567402cd086f690053cda7ce307bd4f77990 Mon Sep 17 00:00:00 2001 From: brian khuu Date: Fri, 3 May 2024 12:35:14 +1000 Subject: [PATCH] *.py: add compilade warning suggestions and style fixes --- convert-hf-to-gguf.py | 4 ++-- convert-llama-ggml-to-gguf.py | 2 +- gguf-py/scripts/gguf-convert-endian.py | 2 +- requirements.txt | 1 + scripts/run-with-preset.py | 2 +- 5 files changed, 6 insertions(+), 5 deletions(-) diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index 170be0233439c..2dfd4201048d9 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -2363,7 +2363,7 @@ def set_vocab(self): # TODO: this is a hack, should be fixed # https://github.com/ggerganov/llama.cpp/pull/6745#issuecomment-2067687048 special_vocab.special_token_ids["eos"] = self._try_get_sft_eos(tokenizer) - logger.debug(f"Replace eos:{old_eos} with a special token:{special_vocab.special_token_ids['eos']} \ + logger.warning(f"Replace eos:{old_eos} with a special token:{special_vocab.special_token_ids['eos']} \ in chat mode so that the conversation can end normally.") special_vocab.add_to_gguf(self.gguf_writer) @@ -2691,7 +2691,7 @@ def set_vocab(self): else: # Use the GPT-NeoX tokenizer when no tokenizer files are present tokenizer_path = Path(sys.path[0]) / "models" / "ggml-vocab-gpt-neox.gguf" - logger.debug(f"Using tokenizer from '{os.path.relpath(tokenizer_path, os.getcwd())}'") + logger.warning(f"Using tokenizer from '{os.path.relpath(tokenizer_path, os.getcwd())}'") neox_reader = gguf.GGUFReader(tokenizer_path, "r") field = neox_reader.get_field(gguf.Keys.Tokenizer.MODEL) diff --git a/convert-llama-ggml-to-gguf.py b/convert-llama-ggml-to-gguf.py index 97e6ff85323c6..9349de3b3b498 100755 --- a/convert-llama-ggml-to-gguf.py +++ b/convert-llama-ggml-to-gguf.py @@ -402,7 +402,7 @@ def handle_args(): help="directory containing tokenizer.model, if separate from model file - only meaningful with --model-metadata-dir") parser.add_argument("--vocabtype", default="spm,hfft", help="vocab format - only meaningful with --model-metadata-dir and/or --vocab-dir (default: spm,hfft)") - parser.add_argument("--verbose", action="store_true", help="increase output verbosity") + parser.add_argument("--verbose", action="store_true", help="increase output verbosity") return parser.parse_args() diff --git a/gguf-py/scripts/gguf-convert-endian.py b/gguf-py/scripts/gguf-convert-endian.py index f182778d82195..b698af0fe7631 100755 --- a/gguf-py/scripts/gguf-convert-endian.py +++ b/gguf-py/scripts/gguf-convert-endian.py @@ -119,7 +119,7 @@ def main() -> None: "--dry-run", action="store_true", help="Don't actually change anything", ) - parser.add_argument("--verbose", action="store_true", help="increase output verbosity") + parser.add_argument("--verbose", action="store_true", help="increase output verbosity") args = parser.parse_args(None if len(sys.argv) > 1 else ["--help"]) diff --git a/requirements.txt b/requirements.txt index c82fde9393872..fc1e28278b695 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,7 @@ # -r ./requirements/requirements-convert.txt + -r ./requirements/requirements-convert-hf-to-gguf.txt -r ./requirements/requirements-convert-hf-to-gguf-update.txt -r ./requirements/requirements-convert-llama-ggml-to-gguf.txt diff --git a/scripts/run-with-preset.py b/scripts/run-with-preset.py index 3c62c557f5a00..e986a36045549 100755 --- a/scripts/run-with-preset.py +++ b/scripts/run-with-preset.py @@ -59,7 +59,7 @@ parser.add_argument("yaml_files", nargs="*", help="Arbitrary number of YAML files from which to read preset values. " "If two files specify the same values the later one will be used.") -parser.add_argument("--verbose", action="store_true", help="increase output verbosity") +parser.add_argument("--verbose", action="store_true", help="increase output verbosity") known_args, unknown_args = parser.parse_known_args()