Skip to content

Commit

Permalink
Enable deploying at HEAD for demo as well as staging. (#1089)
Browse files Browse the repository at this point in the history
  • Loading branch information
@brilee
brilee authored Jan 19, 2024
1 parent 1ecef77 commit d1c1f6a
Show file tree
Hide file tree
Showing 3 changed files with 94 additions and 93 deletions.
79 changes: 58 additions & 21 deletions lilac/deploy.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
"""
import os
import shutil
import subprocess
import tempfile
from importlib import resources
Expand All @@ -30,6 +31,8 @@ def deploy_project(
project_dir: Optional[str] = None,
concepts: Optional[list[str]] = None,
skip_concept_upload: Optional[bool] = False,
deploy_at_head: bool = False,
skip_ts_build: bool = False,
create_space: Optional[bool] = False,
load_on_space: Optional[bool] = False,
hf_space_storage: Optional[Union[Literal['small'], Literal['medium'], Literal['large']]] = None,
Expand All @@ -43,6 +46,9 @@ def deploy_project(
project_dir: The project directory to grab data from. Defaults to `env.LILAC_PROJECT_DIR`.
concepts: The names of concepts to upload. Defaults to all concepts.
skip_concept_upload: When true, skips uploading concepts.
deploy_at_head: If true, deploys the latest code from your machine. Otherwise, deploys from
PyPI's latest published package.
skip_ts_build: (Only relevant when deploy_at_head=True) - Skips building frontend assets.
create_space: When True, creates the HuggingFace space if it doesnt exist. The space will be
created with the storage type defined by --hf_space_storage.
load_on_space: When True, loads the datasets from your project in the space and does not upload
Expand All @@ -67,6 +73,7 @@ def deploy_project(
'--project_dir or the environment variable `LILAC_PROJECT_DIR` must be defined.'
)

project_config = project_config or read_project_config(project_dir)
hf_api = HfApi(token=hf_token)

operations: list[Union[CommitOperationDelete, CommitOperationAdd]] = deploy_project_operations(
Expand All @@ -76,6 +83,8 @@ def deploy_project(
project_config=project_config,
concepts=concepts,
skip_concept_upload=skip_concept_upload,
deploy_at_head=deploy_at_head,
skip_ts_build=skip_ts_build,
create_space=create_space,
load_on_space=load_on_space,
hf_space_storage=hf_space_storage,
Expand All @@ -98,15 +107,16 @@ def deploy_project_operations(
hf_api: 'HfApi',
project_dir: str,
hf_space: str,
project_config: Optional[Config] = None,
project_config: Config,
concepts: Optional[list[str]] = None,
skip_concept_upload: Optional[bool] = False,
deploy_at_head: bool = False,
skip_ts_build: bool = False,
create_space: Optional[bool] = False,
load_on_space: Optional[bool] = False,
hf_space_storage: Optional[Union[Literal['small'], Literal['medium'], Literal['large']]] = None,
) -> list:
"""The commit operations for a project deployment."""
project_config = project_config or read_project_config(project_dir)
try:
from huggingface_hub import CommitOperationAdd, CommitOperationDelete
from huggingface_hub.utils._errors import RepositoryNotFoundError
Expand Down Expand Up @@ -163,10 +173,9 @@ def deploy_project_operations(
)
)

##
## Create the empty wheel directory. If uploading a local wheel, use scripts.deploy_staging.
##
operations.extend(_make_wheel_dir(hf_api, hf_space))
operations.extend(
_make_wheel_dir(hf_api, hf_space, deploy_at_head=deploy_at_head, skip_ts_build=skip_ts_build)
)

##
## Upload the HuggingFace application file (README.md) with uploaded datasets.
Expand Down Expand Up @@ -244,10 +253,14 @@ def deploy_project_operations(
return operations


def _make_wheel_dir(api: Any, hf_space: str) -> list:
"""Creates the wheel directory README. This does not upload local wheels.
def _make_wheel_dir(
api: Any, hf_space: str, deploy_at_head: bool, skip_ts_build: bool = False
) -> list:
"""Makes the wheel directory for the HuggingFace Space commit.
For local wheels, use deploy_local.
An empty directory (README only) is used by default; the dockerfile will then fall back to the
public pip package. When deploy_at_head is True, a wheel with the latest code on your machine is
built and uploaded.
"""
try:
from huggingface_hub import CommitOperationAdd, CommitOperationDelete, HfApi
Expand All @@ -261,29 +274,53 @@ def _make_wheel_dir(api: Any, hf_space: str) -> list:

operations: list[Union[CommitOperationDelete, CommitOperationAdd]] = []

# Make an empty readme in py_dist_dir.
os.makedirs(PY_DIST_DIR, exist_ok=True)
with open(os.path.join(PY_DIST_DIR, 'README.md'), 'w') as f:
f.write(
'This directory is used for locally built whl files.\n'
'We write a README.md to ensure an empty folder is uploaded when there is no whl.'
)

readme_contents = (
'This directory is used for locally built whl files.\n'
'We write a README.md to ensure an empty folder is uploaded when there is no whl.'
).encode()

# Remove everything that exists in dist.
# Clean the remote dist dir.
remote_readme_filepath = os.path.join(PY_DIST_DIR, 'README.md')
if hf_api.file_exists(hf_space, remote_readme_filepath, repo_type='space'):
operations.append(CommitOperationDelete(path_in_repo=f'{PY_DIST_DIR}/'))

# Add a file to the dist/ dir so that it's not empty.
readme_contents = (
'This directory is used for locally built whl files.\n'
'We write a README.md to ensure an empty folder is uploaded when there is no whl.'
).encode()
operations.append(
# The path in the remote doesn't os.path.join as it is specific to Linux.
CommitOperationAdd(path_in_repo=f'{PY_DIST_DIR}/README.md', path_or_fileobj=readme_contents)
)

if deploy_at_head:
##
## Build the web server Svelte & TypeScript.
##
if not skip_ts_build:
log('Building webserver...')
run('./scripts/build_server_prod.sh')

# Clean local dist dir before building
if os.path.exists(PY_DIST_DIR):
shutil.rmtree(PY_DIST_DIR)
os.makedirs(PY_DIST_DIR, exist_ok=True)

# Build the wheel for pip.
# We have to bump the version number so that Dockerfile sees this wheel as "newer" than PyPI.
current_lilac_version = run('poetry version -s', capture_output=True).stdout.strip()
temp_new_version = '1337.0.0'

run(f'poetry version "{temp_new_version}"')
run('poetry build -f wheel')
run(f'poetry version "{current_lilac_version}"')

# Add wheel files to the commit.
for upload_file in os.listdir(PY_DIST_DIR):
operations.append(
CommitOperationAdd(
path_in_repo=os.path.join(PY_DIST_DIR, upload_file),
path_or_fileobj=os.path.join(PY_DIST_DIR, upload_file),
)
)
return operations


Expand Down
19 changes: 19 additions & 0 deletions scripts/deploy_demo.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,21 @@
is_flag=True,
default=False,
)
@click.option(
'--deploy_at_head',
help='Deploy the current working directory, instead of latest PyPI release.',
type=bool,
is_flag=True,
default=False,
)
@click.option(
'--skip_ts_build',
help='Skip building the web server TypeScript. '
'Useful to speed up deploy_at_head if you are only changing python or data.',
type=bool,
is_flag=True,
default=False,
)
@click.option(
'--skip_deploy',
help='Skip deploying to HuggingFace. Useful to test locally.',
Expand Down Expand Up @@ -97,6 +112,8 @@ def deploy_demo(
skip_sync: bool,
skip_load: bool,
skip_data_upload: bool,
deploy_at_head: bool,
skip_ts_build: bool,
skip_deploy: bool,
create_space: bool,
dataset: Optional[list[str]] = None,
Expand Down Expand Up @@ -161,6 +178,8 @@ def deploy_demo(
concepts=[],
# We only use public concepts in demos.
skip_concept_upload=True,
deploy_at_head=deploy_at_head,
skip_ts_build=skip_ts_build,
create_space=create_space,
hf_space_storage='small',
load_on_space=False,
Expand Down
89 changes: 17 additions & 72 deletions scripts/deploy_staging.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,18 +21,14 @@
"""

import os
import shutil
import subprocess
from typing import Optional, Union
from typing import Optional

import click
from huggingface_hub import CommitOperationAdd, CommitOperationDelete, HfApi
from lilac.data.dataset_storage_utils import upload
from lilac.deploy import PY_DIST_DIR, deploy_project_operations
from lilac.deploy import deploy_project
from lilac.env import env
from lilac.project import read_project_config
from lilac.utils import get_hf_dataset_repo_id, log
from lilac.utils import get_hf_dataset_repo_id


@click.command()
Expand Down Expand Up @@ -75,7 +71,7 @@ def deploy_staging(
hf_space: Optional[str] = None,
dataset: Optional[list[str]] = None,
concept: Optional[list[str]] = None,
skip_ts_build: Optional[bool] = False,
skip_ts_build: bool = False,
skip_concept_upload: Optional[bool] = False,
create_space: Optional[bool] = False,
) -> None:
Expand All @@ -86,16 +82,7 @@ def deploy_staging(
if not hf_space:
raise ValueError('Must specify --hf_space or set env.HF_STAGING_DEMO_REPO')

operations: list[Union[CommitOperationDelete, CommitOperationAdd]] = []

hf_api = HfApi(token=env('HF_ACCESS_TOKEN'))

##
## Build the web server Svelte & TypeScript.
##
if not skip_ts_build:
log('Building webserver...')
run('./scripts/build_server_prod.sh')
hf_token = env('HF_ACCESS_TOKEN')

# When datasets are not defined, don't upload any datasets.
if dataset is None:
Expand All @@ -113,69 +100,27 @@ def deploy_staging(
project_dir=project_dir,
url_or_repo=get_hf_dataset_repo_id(*hf_space.split('/'), *d.split('/')),
public=False,
hf_token=hf_api.token,
hf_token=hf_token,
)

# For staging deployments, we strip down the project config to only the specified datasets
project_config = read_project_config(project_dir)
project_config.datasets = [
d for d in project_config.datasets if f'{d.namespace}/{d.name}' in dataset
]

operations.extend(
deploy_project_operations(
hf_api,
project_config=project_config,
project_dir=project_dir,
hf_space=hf_space,
concepts=concept,
skip_concept_upload=skip_concept_upload,
hf_space_storage=None,
create_space=create_space,
)
deploy_project(
hf_space,
project_config=project_config,
project_dir=project_dir,
concepts=concept,
skip_concept_upload=skip_concept_upload,
deploy_at_head=True,
skip_ts_build=skip_ts_build,
hf_space_storage=None,
create_space=create_space,
hf_token=hf_token,
)

# Unconditionally remove dist. dist is unconditionally uploaded so it is empty when using
# the public package.
if os.path.exists(PY_DIST_DIR):
shutil.rmtree(PY_DIST_DIR)
os.makedirs(PY_DIST_DIR, exist_ok=True)

# Build the wheel for pip.
# We have to change the version to a dev version so that the huggingface demo does not try to
# install the public pip package.
current_lilac_version = run('poetry version -s', capture_output=True).stdout.strip()
# Bump the version to something extremely large so we never accidentally install the same version
# when a bump happens.
temp_new_version = '1337.0.0'

run(f'poetry version "{temp_new_version}"')
run('poetry build -f wheel')
run(f'poetry version "{current_lilac_version}"')

for upload_file in os.listdir(PY_DIST_DIR):
operations.append(
CommitOperationAdd(
path_in_repo=os.path.join(PY_DIST_DIR, upload_file),
path_or_fileobj=os.path.join(PY_DIST_DIR, upload_file),
)
)

# Atomically commit all the operations so we don't kick the server multiple times.
hf_api.create_commit(
repo_id=hf_space,
repo_type='space',
operations=operations,
commit_message='Push to HF space',
)

log(f'Done! View your space at https://huggingface.co/spaces/{hf_space}')


def run(cmd: str, capture_output=False) -> subprocess.CompletedProcess[str]:
"""Run a command and return the result."""
return subprocess.run(cmd, shell=True, check=True, capture_output=capture_output, text=True)


if __name__ == '__main__':
deploy_staging()

0 comments on commit d1c1f6a

Please sign in to comment.