Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
update some release tests to use den launcher
Browse files Browse the repository at this point in the history
Alexandra Belousov authored and Alexandra Belousov committed Dec 10, 2024
1 parent 92b337c commit 956242b
Showing 3 changed files with 82 additions and 16 deletions.
6 changes: 5 additions & 1 deletion tests/conftest.py
Original file line number Diff line number Diff line change
@@ -260,7 +260,11 @@ def event_loop():
unnamed_resource, # noqa: F401
)

from tests.fixtures.static_cluster_fixtures import static_cpu_pwd_cluster # noqa: F401
from tests.fixtures.static_cluster_fixtures import ( # noqa: F401
static_cpu_pwd_cluster,
static_cpu_pwd_cluster_den_launcher,
static_gpu_pwd_cluster_den_launcher,
)


# ----------------- Folders -----------------
58 changes: 43 additions & 15 deletions tests/fixtures/static_cluster_fixtures.py
Original file line number Diff line number Diff line change
@@ -1,49 +1,77 @@
from enum import Enum
from typing import Union

import pytest

import runhouse as rh
from runhouse.resources.hardware.utils import LauncherType

from tests.conftest import init_args
from tests.utils import test_env


@pytest.fixture(scope="session")
def static_cpu_pwd_cluster():
sky_cluster = rh.cluster(
"aws-cpu-password", instance_type="CPU:4", provider="aws"
class computeType(str, Enum):
cpu = "cpu"
gpu = "gpu"


def setup_static_cluster(
launcher: Union[LauncherType, str] = None,
compute_type: computeType = computeType.cpu,
):
instance_type = "CPU:4" if compute_type == computeType.cpu else "g5.xlarge"
cluster = rh.cluster(
f"aws-{compute_type}-password",
instance_type=instance_type,
provider="aws",
launcher=launcher,
).save()
if not sky_cluster.is_up():
sky_cluster.up()
if not cluster.is_up():
cluster.up()

# set up password on remote
sky_cluster.run(
cluster.run(
[
[
'sudo sed -i "/^[^#]*PasswordAuthentication[[:space:]]no/c\PasswordAuthentication yes" '
"/etc/ssh/sshd_config"
]
]
)
sky_cluster.run(["sudo /etc/init.d/ssh force-reload"])
sky_cluster.run(["sudo /etc/init.d/ssh restart"])
sky_cluster.run(
cluster.run(["sudo /etc/init.d/ssh force-reload"])
cluster.run(["sudo /etc/init.d/ssh restart"])
cluster.run(
["(echo 'cluster-pass' && echo 'cluster-pass') | sudo passwd ubuntu"]
)
sky_cluster.run(["pip uninstall skypilot runhouse -y", "pip install pytest"])
sky_cluster.run(["rm -rf runhouse/"])
cluster.run(["pip uninstall skypilot runhouse -y", "pip install pytest"])
cluster.run(["rm -rf runhouse/"])

# instantiate byo cluster with password
ssh_creds = {
"ssh_user": "ubuntu",
"ssh_private_key": "~/.ssh/sky-key",
"password": "cluster-pass",
}
args = dict(
name="static-cpu-password", host=[sky_cluster.head_ip], ssh_creds=ssh_creds
)
args = dict(name="static-cpu-password", host=[cluster.head_ip], ssh_creds=ssh_creds)
c = rh.cluster(**args).save()
c.restart_server(resync_rh=True)
init_args[id(c)] = args

test_env().to(c)

return c


@pytest.fixture(scope="session")
def static_cpu_pwd_cluster():
return setup_static_cluster()


@pytest.fixture(scope="session")
def static_cpu_pwd_cluster_den_launcher():
return setup_static_cluster(launcher=LauncherType.DEN)


@pytest.fixture(scope="session")
def static_gpu_pwd_cluster_den_launcher():
return setup_static_cluster(launcher=LauncherType.DEN, compute_type=computeType.gpu)
34 changes: 34 additions & 0 deletions tests/test_resources/test_modules/test_module.py
Original file line number Diff line number Diff line change
@@ -253,6 +253,40 @@ def nested_call_logs_stream_helper(slow_numpy_array):
@pytest.mark.moduletest
class TestModule:

MAP_FIXTURES = {"resource": "cluster"}

UNIT = {"cluster": ["named_cluster"]}
LOCAL = {
"cluster": [
"docker_cluster_pk_ssh_no_auth", # Represents private dev use case
"docker_cluster_pk_ssh_den_auth", # Helps isolate Auth issues
"docker_cluster_pk_http_exposed", # Represents within VPC use case
"docker_cluster_pwd_ssh_no_auth",
],
}
MINIMAL = {
"cluster": [
"static_cpu_pwd_cluster",
"static_cpu_pwd_cluster_den_launcher",
]
}
RELEASE = {
"cluster": [
"static_cpu_pwd_cluster_den_launcher", # tests modules on a den-launched cpu cluster
"static_gpu_pwd_cluster_den_launcher", # tests modules on a den-launched gpu cluster
"static_cpu_pwd_cluster", # tests modules on a local (sky) -launched cpu cluster
]
}
MAXIMAL = {
"cluster": [
"docker_cluster_pk_ssh_no_auth",
"docker_cluster_pk_ssh_den_auth",
"docker_cluster_pwd_ssh_no_auth",
"static_cpu_pwd_cluster",
"multinode_cpu_docker_conda_cluster",
]
}

# --------- integration tests ---------
@pytest.mark.level("local")
def test_call_module_method(self, cluster):

0 comments on commit 956242b

Please sign in to comment.