Skip to content

Commit

Permalink
feat: add auto packaging api
Browse files Browse the repository at this point in the history
  • Loading branch information
percevalw committed Oct 26, 2023
1 parent e5eef12 commit a6b7e0b
Show file tree
Hide file tree
Showing 4 changed files with 685 additions and 5 deletions.
38 changes: 37 additions & 1 deletion edsnlp/core/pipeline.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@
Dict,
Iterable,
List,
Literal,
Mapping,
Optional,
Sequence,
Set,
Expand Down Expand Up @@ -257,7 +259,7 @@ def add_pipe(
if hasattr(pipe, "name"):
if name is not None and name != pipe.name:
raise ValueError(
"The provided name does not match the name of the component."
f"The provided name {name!r} does not match the name of the component {pipe.name!r}."
)
else:
name = pipe.name
Expand Down Expand Up @@ -828,6 +830,9 @@ def deserialize_tensors(path: Path):
import safetensors.torch

torch_components = dict(self.torch_components())
if len(torch_components) == 0 and not path.exists():
return

for file_name in path.iterdir():
pipe_names = file_name.stem.split("+")
if any(pipe_name in torch_components for pipe_name in pipe_names):
Expand Down Expand Up @@ -957,6 +962,37 @@ def __exit__(ctx_self, type, value, traceback):
self._disabled = disable
return context()

def package(
self,
name: Optional[str] = None,
root_dir: Union[str, Path] = ".",
artifacts_name: str = "artifacts",
check_dependencies: bool = False,
project_type: Optional[Literal["poetry", "setuptools"]] = None,
version: str = "0.1.0",
metadata: Optional[Dict[str, Any]] = {},
distributions: Optional[Sequence[Literal["wheel", "sdist"]]] = ["wheel"],
config_settings: Optional[Mapping[str, Union[str, Sequence[str]]]] = None,
isolation: bool = True,
skip_build_dependency_check: bool = False,
):
from edsnlp.utils.package import package

return package(
pipeline=self,
name=name,
root_dir=root_dir,
artifacts_name=artifacts_name,
check_dependencies=check_dependencies,
project_type=project_type,
version=version,
metadata=metadata,
distributions=distributions,
config_settings=config_settings,
isolation=isolation,
skip_build_dependency_check=skip_build_dependency_check,
)


def blank(
lang: str,
Expand Down
6 changes: 2 additions & 4 deletions edsnlp/pipelines/misc/measurements/measurements.py
Original file line number Diff line number Diff line change
Expand Up @@ -529,10 +529,8 @@ def __init__(
self.after_snippet_limit = after_snippet_limit

# MEASURES
for measure_config in measurements:
name = measure_config["name"]
unit = measure_config["unit"]
self.measure_names[self.unit_registry.parse_unit(unit)[0]] = name
for m in measurements:
self.measure_names[self.unit_registry.parse_unit(m["unit"])[0]] = m["name"]

if span_setter is None:
span_setter = {
Expand Down
Loading

0 comments on commit a6b7e0b

Please sign in to comment.