diff --git a/convert.py b/convert.py index fa174788e4c868..54ad8ca0b6cb43 100755 --- a/convert.py +++ b/convert.py @@ -23,7 +23,7 @@ from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor from dataclasses import dataclass from pathlib import Path -from typing import TYPE_CHECKING, Any, Callable, ClassVar, IO, Iterable, Literal, Protocol, TypeVar, runtime_checkable +from typing import TYPE_CHECKING, Any, Callable, ClassVar, IO, Iterable, Literal, Protocol, TypeVar, runtime_checkable, Optional import numpy as np from sentencepiece import SentencePieceProcessor @@ -336,6 +336,7 @@ def load(model_plus: ModelPlus) -> Params: return params + @dataclass class Metadata: name: Optional[str] = None @@ -370,10 +371,12 @@ def load(metadata_path: Path) -> "Metadata": return metadata + # # vocab # + @runtime_checkable class BaseVocab(Protocol): tokenizer_model: ClassVar[str]