diff --git a/frontend/src/types/backend.d.ts b/frontend/src/types/backend.d.ts index dfe74648d..f09bf1b20 100644 --- a/frontend/src/types/backend.d.ts +++ b/frontend/src/types/backend.d.ts @@ -13,6 +13,7 @@ declare type TBackendType = | 'remote' | 'oci' | 'runpod' + | 'seeweb' | 'tensordock' | 'vastai' | 'cloudrift' diff --git a/mkdocs/docs/concepts/backends.md b/mkdocs/docs/concepts/backends.md index 92d0f72e9..f179af1d0 100644 --- a/mkdocs/docs/concepts/backends.md +++ b/mkdocs/docs/concepts/backends.md @@ -994,6 +994,34 @@ projects: +### Seeweb + +Log into the [Seeweb](https://www.seeweb.it/en) control panel, go to the +`Calculation` section, open `API Token`, and click `New token` to generate one. + +Then, go ahead and configure the backend: + +
+ +```yaml +projects: + - name: main + backends: + - type: seeweb + creds: + type: api_token + api_token: ... +``` + +
+ +By default, `dstack` uses all Seeweb regions. To limit them, specify `regions` +(for example, `it-fr2`). + +`dstack` checks [live plan capacity](https://docs.seeweb.it/en/hosting/cloudserver/rest-api/API-Endpoints/Plan/) +before provisioning. A compatible NVIDIA driver image is selected from the images allowed by the +chosen plan. + ### OCI There are two ways to configure OCI: using client credentials or using the default credentials. diff --git a/mkdocs/docs/reference/server/config.yml.md b/mkdocs/docs/reference/server/config.yml.md index fdc3f4c7c..e6b535908 100644 --- a/mkdocs/docs/reference/server/config.yml.md +++ b/mkdocs/docs/reference/server/config.yml.md @@ -341,6 +341,23 @@ to configure [backends](../../concepts/backends.md) and other [server-level sett type: required: true +##### `projects[n].backends[type=seeweb]` { #seeweb data-toc-label="seeweb" } + +#SCHEMA# dstack._internal.core.backends.seeweb.models.SeewebBackendConfigWithCreds + overrides: + show_root_heading: false + type: + required: true + item_id_prefix: seeweb- + +###### `projects[n].backends[type=seeweb].creds` { #seeweb-creds data-toc-label="creds" } + +#SCHEMA# dstack._internal.core.backends.seeweb.models.SeewebAPITokenCreds + overrides: + show_root_heading: false + type: + required: true + ##### `projects[n].backends[type=amddevcloud]` { #amddevcloud data-toc-label="amddevcloud" } #SCHEMA# dstack._internal.core.backends.digitalocean_base.models.BaseDigitalOceanBackendConfigWithCreds diff --git a/src/dstack/_internal/core/backends/configurators.py b/src/dstack/_internal/core/backends/configurators.py index fe30d42e7..2fa3267be 100644 --- a/src/dstack/_internal/core/backends/configurators.py +++ b/src/dstack/_internal/core/backends/configurators.py @@ -135,6 +135,13 @@ except ImportError: pass +try: + from dstack._internal.core.backends.seeweb.configurator import SeewebConfigurator + + _CONFIGURATOR_CLASSES.append(SeewebConfigurator) +except ImportError: + pass + try: from dstack._internal.core.backends.vastai.configurator import VastAIConfigurator diff --git a/src/dstack/_internal/core/backends/models.py b/src/dstack/_internal/core/backends/models.py index a7bb8c9ad..a27966e49 100644 --- a/src/dstack/_internal/core/backends/models.py +++ b/src/dstack/_internal/core/backends/models.py @@ -66,6 +66,10 @@ RunpodBackendConfig, RunpodBackendConfigWithCreds, ) +from dstack._internal.core.backends.seeweb.models import ( + SeewebBackendConfig, + SeewebBackendConfigWithCreds, +) from dstack._internal.core.backends.slurm.models import ( SlurmBackendConfig, SlurmBackendConfigWithCreds, @@ -105,6 +109,7 @@ NebiusBackendConfig, OCIBackendConfig, RunpodBackendConfig, + SeewebBackendConfig, TensorDockBackendConfig, VastAIBackendConfig, VerdaBackendConfig, @@ -133,6 +138,7 @@ OCIBackendConfigWithCreds, NebiusBackendConfigWithCreds, RunpodBackendConfigWithCreds, + SeewebBackendConfigWithCreds, TensorDockBackendConfigWithCreds, VastAIBackendConfigWithCreds, VultrBackendConfigWithCreds, @@ -159,6 +165,7 @@ OCIBackendConfigWithCreds, NebiusBackendFileConfigWithCreds, RunpodBackendConfigWithCreds, + SeewebBackendConfigWithCreds, TensorDockBackendConfigWithCreds, VastAIBackendConfigWithCreds, VultrBackendConfigWithCreds, diff --git a/src/dstack/_internal/core/backends/seeweb/__init__.py b/src/dstack/_internal/core/backends/seeweb/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/dstack/_internal/core/backends/seeweb/api_client.py b/src/dstack/_internal/core/backends/seeweb/api_client.py new file mode 100644 index 000000000..d7dcca141 --- /dev/null +++ b/src/dstack/_internal/core/backends/seeweb/api_client.py @@ -0,0 +1,231 @@ +import hashlib +from dataclasses import dataclass +from typing import Any, Dict, Optional + +import requests + +from dstack._internal.core.errors import ( + BackendError, + BackendInvalidCredentialsError, + NoCapacityError, +) +from dstack._internal.utils.logging import get_logger + +logger = get_logger(__name__) + +API_URL = "https://api.seeweb.it/ecs/v2" +TIMEOUT = 30 + + +@dataclass(frozen=True) +class SeewebPlanAvailability: + regions: frozenset[str] + images: tuple[str, ...] + + +class SeewebApiClient: + """Minimal client for the Seeweb ECS REST API (https://api.seeweb.it/ecs/v2). + + Uses ``requests`` directly instead of the ``ecsapi`` SDK: ``ecsapi`` requires pydantic v2, + while dstack is pinned to pydantic v1, so the two cannot share an environment. + """ + + def __init__(self, api_token: str): + self.api_token = api_token + + def _request( + self, method: str, path: str, json: Optional[Dict[str, Any]] = None + ) -> requests.Response: + try: + response = requests.request( + method=method, + url=API_URL + path, + headers={"X-APITOKEN": self.api_token}, + json=json, + timeout=TIMEOUT, + ) + except requests.RequestException as e: + raise BackendError(f"Seeweb API request failed: {e}") from e + if response.status_code in (requests.codes.unauthorized, requests.codes.forbidden): + raise BackendInvalidCredentialsError(_response_message(response)) + return response + + def validate_api_key(self) -> bool: + try: + response = self._request("GET", "/regions") + except BackendInvalidCredentialsError: + return False + _raise_api_error(response) + return True + + def get_or_create_ssh_key(self, public_key: str) -> str: + """Return the label of a Seeweb SSH key matching ``public_key``, creating it if needed.""" + canonical_key = _canonicalize_ssh_public_key(public_key) + label = "dstack-" + hashlib.sha256(canonical_key.encode()).hexdigest()[:20] + response = self._request("GET", "/sshkeys") + data = _response_data(response) + for key in data.get("pubkeys", []): + if not isinstance(key, dict): + continue + try: + existing_key = _canonicalize_ssh_public_key(str(key.get("key", ""))) + except ValueError: + continue + if existing_key == canonical_key: + existing_label = key.get("label") + if isinstance(existing_label, str): + return existing_label + created = self._request( + "POST", + "/sshkeys", + json={"key": public_key.strip(), "label": label}, + ) + _response_data(created) + return label + + def get_available_plans(self) -> dict[str, SeewebPlanAvailability]: + """Return current regions and allowed images for each activatable plan.""" + response = self._request("GET", "/plans/availables") + data = _response_data(response) + available = {} + for plan in data.get("plans", []): + if not isinstance(plan, dict) or not isinstance(plan.get("name"), str): + continue + regions = set() + raw_regions = plan.get("region_availables") or plan.get("region_available") or [] + if isinstance(raw_regions, list): + for region in raw_regions: + if isinstance(region, dict) and isinstance(region.get("region"), str): + regions.add(region["region"]) + images = [] + raw_images = plan.get("os_availables") or [] + if isinstance(raw_images, list): + for image in raw_images: + if ( + isinstance(image, dict) + and isinstance(image.get("name"), str) + and image.get("active_flag") is not False + ): + images.append(image["name"]) + available[plan["name"]] = SeewebPlanAvailability( + regions=frozenset(regions), + images=tuple(images), + ) + return available + + def get_available_plan_regions(self) -> set[tuple[str, str]]: + """Return the set of ``(plan_name, region)`` pairs that can currently be provisioned. + + Seeweb's ``/plans`` catalog lists a plan's regions, but a plan/region is only actually + creatable if it appears in ``/plans/availables`` ``region_availables`` (i.e. has capacity). + """ + return { + (plan_name, region) + for plan_name, availability in self.get_available_plans().items() + for region in availability.regions + } + + def create_server(self, body: Dict[str, Any]) -> tuple[Dict[str, Any], Optional[int]]: + """Create a server. Returns ``(server_dict, action_id)``. + + Capacity errors become ``NoCapacityError`` so dstack can fall through to another offer. + """ + response = self._request("POST", "/servers", json=body) + data = _response_data(response, capacity_errors=True) + if not isinstance(data, dict) or "server" not in data: + raise BackendError("Seeweb create-server response did not include a server") + server = data["server"] + if not isinstance(server, dict) or not isinstance(server.get("name"), str): + raise BackendError("Seeweb create-server response contained an invalid server") + action = data.get("action_id", data.get("action")) + if isinstance(action, dict): + action = action.get("id") + action_id = action if isinstance(action, int) else None + return server, action_id + + def get_action(self, action_id: int) -> Optional[Dict[str, Any]]: + response = self._request("GET", f"/actions/{action_id}") + if response.status_code == requests.codes.not_found: + return None + data = _response_data(response) + action = data.get("action") + return action if isinstance(action, dict) else None + + def get_server(self, name: str) -> Optional[Dict[str, Any]]: + response = self._request("GET", f"/servers/{name}") + if response.status_code == requests.codes.not_found: + return None + data = _response_data(response) + server = data.get("server") + return server if isinstance(server, dict) else None + + def delete_server(self, name: str) -> None: + """Delete a server, treating a missing server as success (idempotent).""" + response = self._request("DELETE", f"/servers/{name}") + if response.status_code == requests.codes.not_found: + logger.debug("Seeweb server %s already deleted", name) + return + _response_data(response) + + +def _response_data( + response: requests.Response, *, capacity_errors: bool = False +) -> Dict[str, Any]: + try: + data = response.json() + except requests.JSONDecodeError as e: + if not response.ok: + raise BackendError(_response_message(response)) from e + raise BackendError("Seeweb API returned a non-JSON response") from e + if not isinstance(data, dict): + raise BackendError("Seeweb API returned an invalid response") + status = str(data.get("status", "ok")).lower() + if not response.ok or status not in {"ok", "success"}: + message = _response_message(response, data) + if capacity_errors and _is_capacity_error(message): + raise NoCapacityError(message) + raise BackendError(message) + return data + + +def _raise_api_error(response: requests.Response) -> None: + _response_data(response) + + +def _response_message(response: requests.Response, data: Optional[Dict[str, Any]] = None) -> str: + if data is None: + try: + parsed = response.json() + data = parsed if isinstance(parsed, dict) else None + except requests.JSONDecodeError: + data = None + if data is not None: + for field in ("message", "error", "detail"): + value = data.get(field) + if value: + return str(value) + text = response.text.strip() + return text if text else f"Seeweb API returned HTTP {response.status_code}" + + +def _is_capacity_error(message: str) -> bool: + normalized = message.lower() + return any( + marker in normalized + for marker in ( + "not available", + "no capacity", + "lack of resources", + "not activatable", + "no host", + ) + ) + + +def _canonicalize_ssh_public_key(public_key: str) -> str: + parts = public_key.strip().split() + if len(parts) < 2 or not ( + parts[0].startswith("ssh-") or parts[0].startswith("ecdsa-") or parts[0].startswith("sk-") + ): + raise ValueError("Invalid SSH public key") + return " ".join(parts[:2]) diff --git a/src/dstack/_internal/core/backends/seeweb/backend.py b/src/dstack/_internal/core/backends/seeweb/backend.py new file mode 100644 index 000000000..209a059fe --- /dev/null +++ b/src/dstack/_internal/core/backends/seeweb/backend.py @@ -0,0 +1,16 @@ +from dstack._internal.core.backends.base.backend import Backend +from dstack._internal.core.backends.seeweb.compute import SeewebCompute +from dstack._internal.core.backends.seeweb.models import SeewebConfig +from dstack._internal.core.models.backends.base import BackendType + + +class SeewebBackend(Backend): + TYPE = BackendType.SEEWEB + COMPUTE_CLASS = SeewebCompute + + def __init__(self, config: SeewebConfig): + self.config = config + self._compute = SeewebCompute(self.config) + + def compute(self) -> SeewebCompute: + return self._compute diff --git a/src/dstack/_internal/core/backends/seeweb/compute.py b/src/dstack/_internal/core/backends/seeweb/compute.py new file mode 100644 index 000000000..9dd384458 --- /dev/null +++ b/src/dstack/_internal/core/backends/seeweb/compute.py @@ -0,0 +1,292 @@ +import shlex +from typing import List, Optional + +import gpuhunt +from gpuhunt.providers.seeweb import SeewebProvider + +from dstack._internal.core.backends.base.backend import Compute +from dstack._internal.core.backends.base.compute import ( + ComputeWithCreateInstanceSupport, + ComputeWithFilteredOffersCached, + ComputeWithInstanceVolumesSupport, + ComputeWithMultinodeSupport, + ComputeWithPrivilegedSupport, + generate_unique_instance_name, + get_shim_commands, +) +from dstack._internal.core.backends.base.offers import get_catalog_offers +from dstack._internal.core.backends.seeweb.api_client import ( + SeewebApiClient, + SeewebPlanAvailability, +) +from dstack._internal.core.backends.seeweb.models import SeewebConfig +from dstack._internal.core.errors import BackendError, NoCapacityError, ProvisioningError +from dstack._internal.core.models.backends.base import BackendType +from dstack._internal.core.models.common import CoreModel +from dstack._internal.core.models.instances import ( + InstanceAvailability, + InstanceConfiguration, + InstanceOfferWithAvailability, +) +from dstack._internal.core.models.placement import PlacementGroup +from dstack._internal.core.models.runs import JobProvisioningData, Requirements +from dstack._internal.utils.logging import get_logger + +logger = get_logger(__name__) + +# Seeweb auto-generates the server name; notes are limited, so keep the label short. +MAX_INSTANCE_NAME_LEN = 60 + +# Seeweb statuses that mean provisioning failed (matched case-insensitively). +FAILED_SERVER_STATUSES = {"failed", "error", "deleted", "deleting"} +FAILED_ACTION_STATUSES = {"failed", "error", "cancelled", "canceled"} +COMPLETED_ACTION_STATUSES = {"completed", "complete", "success", "succeeded"} + +GPU_IMAGE_PREFERENCE = ( + "ubuntu-2204-uefi-nvidia-driver", + "ubuntu-2204-nvidia-driver", +) +CPU_IMAGE_PREFERENCE = ("ubuntu-2204",) + + +class SeewebCompute( + ComputeWithCreateInstanceSupport, + ComputeWithFilteredOffersCached, + ComputeWithPrivilegedSupport, + ComputeWithInstanceVolumesSupport, + ComputeWithMultinodeSupport, + Compute, +): + def __init__(self, config: SeewebConfig): + super().__init__() + self.config = config + self.api_client = SeewebApiClient(config.creds.api_token) + + def _make_catalog(self) -> gpuhunt.Catalog: + # Seeweb pricing requires auth, so query the provider live with the project's token + # (like the vastai/jarvislabs backends) instead of relying on the published catalog. + catalog = gpuhunt.Catalog(balance_resources=False, auto_reload=False) + catalog.add_provider(SeewebProvider(token=self.config.creds.api_token)) + return catalog + + def get_offers_by_requirements( + self, requirements: Requirements, full_offers: bool + ) -> List[InstanceOfferWithAvailability]: + offers = get_catalog_offers( + backend=BackendType.SEEWEB, + locations=self.config.regions or None, + requirements=requirements, + catalog=self._make_catalog(), + ) + # A plan can be in the catalog but sold out: mark only currently-creatable + # (plan, region) pairs as available so dstack never tries an unavailable card. + available = self.api_client.get_available_plan_regions() + return [ + offer.with_availability( + availability=InstanceAvailability.AVAILABLE + if (offer.instance.name, offer.region) in available + else InstanceAvailability.NOT_AVAILABLE + ) + for offer in offers + ] + + def create_instance( + self, + instance_offer: InstanceOfferWithAvailability, + instance_config: InstanceConfiguration, + placement_group: Optional[PlacementGroup], + ) -> JobProvisioningData: + instance_name = generate_unique_instance_name( + instance_config, max_length=MAX_INSTANCE_NAME_LEN + ) + public_keys = instance_config.get_public_keys() + gpus = instance_offer.instance.resources.gpus + is_gpu = len(gpus) > 0 + plan_name = instance_offer.instance.name + availability = self.api_client.get_available_plans().get(plan_name) + image = _select_image( + availability=availability, + plan_name=plan_name, + region=instance_offer.region, + is_gpu=is_gpu, + ) + + commands = _setup_commands(authorized_keys=public_keys, is_gpu=is_gpu) + shim_commands = get_shim_commands(arch=instance_offer.instance.resources.cpu_arch) + # The Seeweb GPU image reboots after its first-boot package upgrade. Do not start + # the shim before that reboot, otherwise dstack can submit a GPU job while the + # userspace NVIDIA libraries no longer match the still-loaded kernel module. + commands += shim_commands[:-1] + commands += _persist_shim_commands() + user_customize = "#!/bin/bash\nset -euo pipefail\n" + "\n".join(commands) + + body = { + "plan": plan_name, + "image": image, + "location": instance_offer.region, + "notes": instance_name, + "user_customize": user_customize, + } + # Register the project SSH key so Seeweb injects it too (the startup script also does, + # but this covers the window before the script runs). + if public_keys: + body["ssh_key"] = self.api_client.get_or_create_ssh_key(public_keys[0]) + + server, action_id = self.api_client.create_server(body) + + return JobProvisioningData( + backend=instance_offer.backend, + instance_type=instance_offer.instance, + instance_id=server["name"], + hostname=None, + internal_ip=None, + region=instance_offer.region, + price=instance_offer.price, + username="root", + ssh_port=22, + ssh_proxy=None, + dockerized=True, + backend_data=SeewebInstanceBackendData(action_id=action_id).json(), + ) + + def update_provisioning_data( + self, + provisioning_data: JobProvisioningData, + project_ssh_public_key: str, + project_ssh_private_key: str, + ): + backend_data = SeewebInstanceBackendData.load(provisioning_data.backend_data) + action_completed = backend_data.action_id is None + if backend_data.action_id is not None: + action = self.api_client.get_action(backend_data.action_id) + if action is None: + return + action_status = str(action.get("status") or "").lower() + if action_status in FAILED_ACTION_STATUSES: + raise ProvisioningError( + f"Seeweb action {backend_data.action_id} entered status {action_status!r}" + ) + action_completed = action_status in COMPLETED_ACTION_STATUSES + if not action_completed: + return + + server = self.api_client.get_server(provisioning_data.instance_id) + if server is None: + # The server endpoint can lag briefly after the creation action completes. + return + status = (server.get("status") or "").lower() + if status in FAILED_SERVER_STATUSES: + raise ProvisioningError( + f"Seeweb server {provisioning_data.instance_id} entered status {status!r}" + ) + ipv4 = server.get("ipv4") + if action_completed and isinstance(ipv4, str) and ipv4: + provisioning_data.hostname = ipv4 + + def terminate_instance( + self, instance_id: str, region: str, backend_data: Optional[str] = None + ) -> None: + self.api_client.delete_server(instance_id) + + +def _select_image( + *, + availability: Optional[SeewebPlanAvailability], + plan_name: str, + region: str, + is_gpu: bool, +) -> str: + """Choose an image actually allowed by the plan, preferring the NVIDIA UEFI image.""" + if availability is None or region not in availability.regions: + raise NoCapacityError(f"Seeweb plan {plan_name} is not available in region {region}") + preference = GPU_IMAGE_PREFERENCE if is_gpu else CPU_IMAGE_PREFERENCE + for image in preference: + if image in availability.images: + return image + workload = "GPU" if is_gpu else "CPU" + raise BackendError( + f"Seeweb plan {plan_name} has no supported {workload} image in region {region}" + ) + + +def _setup_commands(authorized_keys: List[str], is_gpu: bool) -> List[str]: + key_commands = [] + for key in dict.fromkeys(key.strip() for key in authorized_keys if key.strip()): + key_commands.append(f"printf '%s\\n' {shlex.quote(key)} >> /root/.ssh/authorized_keys") + commands = [ + "install -d -m 0700 /root/.ssh", + "touch /root/.ssh/authorized_keys", + "chmod 0600 /root/.ssh/authorized_keys", + *key_commands, + "export DEBIAN_FRONTEND=noninteractive", + ] + commands += _install_docker_commands() + if is_gpu: + commands += _install_nvidia_container_toolkit_commands() + return commands + + +def _install_docker_commands() -> List[str]: + return [ + "install -d -m 0755 /etc/apt/keyrings", + "curl --max-time 60 -fsSL https://download.docker.com/linux/ubuntu/gpg" + " | gpg --batch --yes --dearmor -o /etc/apt/keyrings/docker.gpg", + "chmod a+r /etc/apt/keyrings/docker.gpg", + 'echo "deb [arch=$(dpkg --print-architecture)' + " signed-by=/etc/apt/keyrings/docker.gpg]" + ' https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"' + " | tee /etc/apt/sources.list.d/docker.list > /dev/null", + "apt-get update", + "apt-get --assume-yes install docker-ce docker-ce-cli containerd.io" + " docker-compose-plugin jq", + "systemctl enable --now docker.service", + ] + + +def _install_nvidia_container_toolkit_commands() -> List[str]: + return [ + "apt-get --assume-yes install nvidia-container-toolkit", + "nvidia-ctk runtime configure --runtime=docker", + # `systemctl restart` can report non-zero under the Seeweb cloud-init runner even when + # Docker comes up fine; keep it non-fatal so the chained runner/shim commands still run. + "systemctl restart docker.service || true", + ] + + +def _persist_shim_commands() -> List[str]: + """Make the shim survive the reboot performed by Seeweb's GPU image updates.""" + unit_lines = [ + "[Unit]", + "Description=dstack shim", + "Wants=network-online.target", + "After=network-online.target docker.service", + "", + "[Service]", + "EnvironmentFile=/etc/dstack-shim.env", + "ExecStart=/usr/local/bin/dstack-shim", + "Restart=always", + "RestartSec=3", + "", + "[Install]", + "WantedBy=multi-user.target", + ] + write_unit = "printf '%s\\n' " + " ".join(map(shlex.quote, unit_lines)) + return [ + "env | grep '^DSTACK_' > /etc/dstack-shim.env", + "chmod 0600 /etc/dstack-shim.env", + f"{write_unit} > /etc/systemd/system/dstack-shim.service", + "systemctl daemon-reload", + # Do not start the service on the initial boot. Seeweb's GPU image performs an + # automatic reboot after cloud-init, and the service must become reachable only then. + "systemctl enable dstack-shim.service", + ] + + +class SeewebInstanceBackendData(CoreModel): + action_id: Optional[int] = None + + @classmethod + def load(cls, raw: Optional[str]) -> "SeewebInstanceBackendData": + if raw is None: + return cls() + return cls.__response__.parse_raw(raw) diff --git a/src/dstack/_internal/core/backends/seeweb/configurator.py b/src/dstack/_internal/core/backends/seeweb/configurator.py new file mode 100644 index 000000000..f71a9347c --- /dev/null +++ b/src/dstack/_internal/core/backends/seeweb/configurator.py @@ -0,0 +1,59 @@ +import json + +from dstack._internal.core.backends.base.configurator import ( + BackendRecord, + Configurator, + raise_invalid_credentials_error, +) +from dstack._internal.core.backends.seeweb.api_client import SeewebApiClient +from dstack._internal.core.backends.seeweb.backend import SeewebBackend +from dstack._internal.core.backends.seeweb.models import ( + SeewebBackendConfig, + SeewebBackendConfigWithCreds, + SeewebConfig, + SeewebCreds, + SeewebStoredConfig, +) +from dstack._internal.core.models.backends.base import BackendType + + +class SeewebConfigurator( + Configurator[ + SeewebBackendConfig, + SeewebBackendConfigWithCreds, + ] +): + TYPE = BackendType.SEEWEB + BACKEND_CLASS = SeewebBackend + + def validate_config(self, config: SeewebBackendConfigWithCreds, default_creds_enabled: bool): + if not SeewebApiClient(config.creds.api_token).validate_api_key(): + raise_invalid_credentials_error(fields=[["creds", "api_token"]]) + + def create_backend( + self, project_name: str, config: SeewebBackendConfigWithCreds + ) -> BackendRecord: + return BackendRecord( + config=SeewebStoredConfig( + **SeewebBackendConfig.__response__.parse_obj(config).dict() + ).json(), + auth=SeewebCreds.parse_obj(config.creds).json(), + ) + + def get_backend_config_with_creds(self, record: BackendRecord) -> SeewebBackendConfigWithCreds: + config = self._get_config(record) + return SeewebBackendConfigWithCreds.__response__.parse_obj(config) + + def get_backend_config_without_creds(self, record: BackendRecord) -> SeewebBackendConfig: + config = self._get_config(record) + return SeewebBackendConfig.__response__.parse_obj(config) + + def get_backend(self, record: BackendRecord) -> SeewebBackend: + config = self._get_config(record) + return SeewebBackend(config=config) + + def _get_config(self, record: BackendRecord) -> SeewebConfig: + return SeewebConfig.__response__( + **json.loads(record.config), + creds=SeewebCreds.parse_raw(record.auth), + ) diff --git a/src/dstack/_internal/core/backends/seeweb/models.py b/src/dstack/_internal/core/backends/seeweb/models.py new file mode 100644 index 000000000..2bb2007fc --- /dev/null +++ b/src/dstack/_internal/core/backends/seeweb/models.py @@ -0,0 +1,42 @@ +from typing import Annotated, List, Literal, Optional, Union + +from pydantic import Field + +from dstack._internal.core.models.common import CoreModel + + +class SeewebAPITokenCreds(CoreModel): + type: Annotated[Literal["api_token"], Field(description="The type of credentials")] = ( + "api_token" + ) + api_token: Annotated[str, Field(description="The Seeweb ECS API token")] + + +AnySeewebCreds = SeewebAPITokenCreds +SeewebCreds = AnySeewebCreds + + +class SeewebBackendConfig(CoreModel): + type: Annotated[Literal["seeweb"], Field(description="The type of backend")] = "seeweb" + regions: Annotated[ + Optional[List[str]], + Field( + description="The list of Seeweb regions (e.g. it-mi2, it-fr2, ch-lug1, hr-zag1)." + " Omit to use all regions" + ), + ] = None + + +class SeewebBackendConfigWithCreds(SeewebBackendConfig): + creds: Annotated[AnySeewebCreds, Field(description="The credentials")] + + +AnySeewebBackendConfig = Union[SeewebBackendConfig, SeewebBackendConfigWithCreds] + + +class SeewebStoredConfig(SeewebBackendConfig): + pass + + +class SeewebConfig(SeewebStoredConfig): + creds: AnySeewebCreds diff --git a/src/dstack/_internal/core/models/backends/base.py b/src/dstack/_internal/core/models/backends/base.py index 419b2cc70..b42b7bf57 100644 --- a/src/dstack/_internal/core/models/backends/base.py +++ b/src/dstack/_internal/core/models/backends/base.py @@ -21,6 +21,7 @@ class BackendType(str, enum.Enum): NEBIUS (BackendType): Nebius AI Cloud OCI (BackendType): Oracle Cloud Infrastructure RUNPOD (BackendType): Runpod Cloud + SEEWEB (BackendType): Seeweb Cloud Server GPU TENSORDOCK (BackendType): TensorDock Marketplace VASTAI (BackendType): Vast.ai Marketplace VERDA (BackendType): Verda Cloud @@ -47,6 +48,7 @@ class BackendType(str, enum.Enum): NEBIUS = "nebius" OCI = "oci" RUNPOD = "runpod" + SEEWEB = "seeweb" TENSORDOCK = "tensordock" VASTAI = "vastai" VERDA = "verda" diff --git a/src/dstack/_internal/server/background/pipeline_tasks/common.py b/src/dstack/_internal/server/background/pipeline_tasks/common.py index 56e0dc5f9..3d0327075 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/common.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/common.py @@ -12,6 +12,9 @@ def get_provisioning_timeout(backend_type: BackendType, instance_type_name: str) return timedelta(minutes=30) if backend_type == BackendType.RUNPOD: return timedelta(minutes=20) + if backend_type == BackendType.SEEWEB: + # Seeweb GPU servers install drivers/Docker on first boot, which can exceed 10 minutes. + return timedelta(minutes=20) if backend_type == BackendType.KUBERNETES: return timedelta(minutes=20) if backend_type == BackendType.SLURM: diff --git a/src/tests/_internal/core/backends/seeweb/__init__.py b/src/tests/_internal/core/backends/seeweb/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/tests/_internal/core/backends/seeweb/test_api_client.py b/src/tests/_internal/core/backends/seeweb/test_api_client.py new file mode 100644 index 000000000..80cf38ab2 --- /dev/null +++ b/src/tests/_internal/core/backends/seeweb/test_api_client.py @@ -0,0 +1,181 @@ +import pytest +import requests + +from dstack._internal.core.backends.seeweb.api_client import API_URL, SeewebApiClient +from dstack._internal.core.errors import ( + BackendError, + BackendInvalidCredentialsError, + NoCapacityError, +) + + +def test_validate_api_key_and_auth_header(requests_mock): + requests_mock.get(f"{API_URL}/regions", json={"status": "ok", "regions": []}) + + assert SeewebApiClient("token").validate_api_key() is True + assert requests_mock.last_request.headers["X-APITOKEN"] == "token" + + +def test_validate_api_key_returns_false_for_unauthorized(requests_mock): + requests_mock.get(f"{API_URL}/regions", status_code=401, json={"message": "unauthorized"}) + + assert SeewebApiClient("bad").validate_api_key() is False + + +def test_request_wraps_network_errors(requests_mock): + requests_mock.get(f"{API_URL}/regions", exc=requests.ConnectTimeout("timed out")) + + with pytest.raises(BackendError, match="Seeweb API request failed"): + SeewebApiClient("token").validate_api_key() + + +def test_forbidden_raises_invalid_credentials(requests_mock): + requests_mock.get(f"{API_URL}/servers/server", status_code=403, text="forbidden") + + with pytest.raises(BackendInvalidCredentialsError): + SeewebApiClient("bad").get_server("server") + + +def test_get_available_plans_parses_regions_and_active_images(requests_mock): + requests_mock.get( + f"{API_URL}/plans/availables", + json={ + "status": "ok", + "plans": [ + { + "name": "ECS1GPU7", + "region_availables": [{"region": "it-fr2"}], + "os_availables": [ + { + "name": "ubuntu-2204-uefi-nvidia-driver", + "active_flag": True, + }, + {"name": "old-image", "active_flag": False}, + ], + }, + { + "name": "ECS1GPU12", + "region_available": [{"region": "it-mi2"}], + "os_availables": [], + }, + ], + }, + ) + + plans = SeewebApiClient("token").get_available_plans() + + assert plans["ECS1GPU7"].regions == frozenset({"it-fr2"}) + assert plans["ECS1GPU7"].images == ("ubuntu-2204-uefi-nvidia-driver",) + assert plans["ECS1GPU12"].regions == frozenset({"it-mi2"}) + assert SeewebApiClient("token").get_available_plan_regions() == { + ("ECS1GPU7", "it-fr2"), + ("ECS1GPU12", "it-mi2"), + } + + +def test_get_or_create_ssh_key_reuses_key_ignoring_comment(requests_mock): + requests_mock.get( + f"{API_URL}/sshkeys", + json={ + "status": "ok", + "pubkeys": [ + { + "label": "existing", + "key": "ssh-ed25519 AAAATEST old-comment", + } + ], + }, + ) + + label = SeewebApiClient("token").get_or_create_ssh_key("ssh-ed25519 AAAATEST new-comment") + + assert label == "existing" + assert requests_mock.call_count == 1 + + +def test_get_or_create_ssh_key_creates_missing_key(requests_mock): + requests_mock.get(f"{API_URL}/sshkeys", json={"status": "ok", "pubkeys": []}) + requests_mock.post(f"{API_URL}/sshkeys", json={"status": "ok"}) + public_key = "ssh-ed25519 AAAATEST comment with spaces" + + label = SeewebApiClient("token").get_or_create_ssh_key(public_key) + + assert label.startswith("dstack-") + assert requests_mock.last_request.json() == {"key": public_key, "label": label} + + +def test_get_or_create_ssh_key_rejects_invalid_key(requests_mock): + with pytest.raises(ValueError, match="Invalid SSH public key"): + SeewebApiClient("token").get_or_create_ssh_key('invalid "key"') + + assert requests_mock.call_count == 0 + + +def test_create_server_parses_action_id(requests_mock): + requests_mock.post( + f"{API_URL}/servers", + json={ + "status": "ok", + "action": 35, + "server": {"name": "ec-test", "status": "Booting"}, + }, + ) + + server, action_id = SeewebApiClient("token").create_server( + { + "plan": "ECS1GPU7", + "image": "ubuntu-2204-uefi-nvidia-driver", + "location": "it-fr2", + } + ) + + assert server["name"] == "ec-test" + assert action_id == 35 + + +@pytest.mark.parametrize("status_code", [200, 400]) +def test_create_server_maps_capacity_errors(requests_mock, status_code): + requests_mock.post( + f"{API_URL}/servers", + status_code=status_code, + json={"status": "error", "message": "Plan is not available in this region"}, + ) + + with pytest.raises(NoCapacityError, match="not available"): + SeewebApiClient("token").create_server({}) + + +def test_create_server_maps_other_errors(requests_mock): + requests_mock.post( + f"{API_URL}/servers", + status_code=500, + json={"status": "error", "message": "internal failure"}, + ) + + with pytest.raises(BackendError, match="internal failure"): + SeewebApiClient("token").create_server({}) + + +def test_get_action_and_server(requests_mock): + requests_mock.get( + f"{API_URL}/actions/35", + json={"status": "ok", "action": {"id": 35, "status": "completed"}}, + ) + requests_mock.get( + f"{API_URL}/servers/ec-test", + json={"status": "ok", "server": {"name": "ec-test", "ipv4": "192.0.2.1"}}, + ) + client = SeewebApiClient("token") + + assert client.get_action(35)["status"] == "completed" + assert client.get_server("ec-test")["ipv4"] == "192.0.2.1" + + +def test_delete_server_is_idempotent(requests_mock): + requests_mock.delete( + f"{API_URL}/servers/missing", + status_code=404, + json={"status": "error", "message": "not found"}, + ) + + SeewebApiClient("token").delete_server("missing") diff --git a/src/tests/_internal/core/backends/seeweb/test_compute.py b/src/tests/_internal/core/backends/seeweb/test_compute.py new file mode 100644 index 000000000..39b3c0cd5 --- /dev/null +++ b/src/tests/_internal/core/backends/seeweb/test_compute.py @@ -0,0 +1,295 @@ +import shlex +from unittest.mock import MagicMock, patch + +import pytest +from gpuhunt import RawCatalogItem + +from dstack._internal.core.backends.base.compute import ( + ComputeWithInstanceVolumesSupport, + ComputeWithMultinodeSupport, + ComputeWithPrivilegedSupport, +) +from dstack._internal.core.backends.seeweb.api_client import SeewebPlanAvailability +from dstack._internal.core.backends.seeweb.compute import ( + SeewebCompute, + SeewebInstanceBackendData, + _select_image, +) +from dstack._internal.core.backends.seeweb.models import SeewebAPITokenCreds, SeewebConfig +from dstack._internal.core.errors import BackendError, NoCapacityError, ProvisioningError +from dstack._internal.core.models.backends.base import BackendType +from dstack._internal.core.models.instances import ( + Disk, + Gpu, + InstanceAvailability, + InstanceConfiguration, + InstanceOfferWithAvailability, + InstanceType, + Resources, + SSHKey, +) +from dstack._internal.core.models.runs import JobProvisioningData + + +def _config(regions=None) -> SeewebConfig: + return SeewebConfig(regions=regions, creds=SeewebAPITokenCreds(api_token="tok")) + + +def _compute(regions=None) -> SeewebCompute: + compute = SeewebCompute(_config(regions)) + compute.api_client = MagicMock() + return compute + + +def _raw(name: str, gpu: str, location: str = "it-fr2") -> RawCatalogItem: + return RawCatalogItem( + instance_name=name, + location=location, + price=0.38, + cpu=4, + memory=32.0, + gpu_count=1, + gpu_name=gpu, + gpu_memory=24.0, + gpu_vendor="nvidia", + spot=False, + disk_size=100.0, + ) + + +def _gpu_offer(name: str = "ECS1GPU7") -> InstanceOfferWithAvailability: + return InstanceOfferWithAvailability( + backend=BackendType.SEEWEB, + instance=InstanceType( + name=name, + resources=Resources( + cpus=8, + memory_mib=32 * 1024, + gpus=[Gpu(name="L40S", memory_mib=48 * 1024)], + spot=False, + disk=Disk(size_mib=500 * 1024), + ), + ), + region="it-fr2", + price=0.85, + availability=InstanceAvailability.AVAILABLE, + ) + + +def _instance_config(public_key: str = "ssh-ed25519 AAAATEST test") -> InstanceConfiguration: + return InstanceConfiguration( + project_name="project", + instance_name="seeweb-test", + user="user", + ssh_keys=[SSHKey(public=public_key)], + ) + + +def _provisioning_data(action_id: int | None = 35) -> JobProvisioningData: + return JobProvisioningData( + backend=BackendType.SEEWEB, + instance_type=_gpu_offer().instance, + instance_id="ec-test", + hostname=None, + internal_ip=None, + region="it-fr2", + price=0.85, + username="root", + ssh_port=22, + dockerized=True, + ssh_proxy=None, + backend_data=SeewebInstanceBackendData(action_id=action_id).json(), + ) + + +def test_vm_capability_mixins_are_enabled(): + assert issubclass(SeewebCompute, ComputeWithPrivilegedSupport) + assert issubclass(SeewebCompute, ComputeWithInstanceVolumesSupport) + assert issubclass(SeewebCompute, ComputeWithMultinodeSupport) + + +def test_get_offers_marks_only_creatable_as_available(): + raws = [_raw("ECS1GPU6", "L4"), _raw("ECS1GPU11", "A30")] + with ( + patch( + "dstack._internal.core.backends.seeweb.compute.SeewebProvider.get", + return_value=raws, + ), + patch( + "dstack._internal.core.backends.seeweb.api_client.SeewebApiClient" + ".get_available_plan_regions", + return_value={("ECS1GPU6", "it-fr2")}, + ), + ): + offers = SeewebCompute(_config()).get_offers_by_requirements( + requirements=None, full_offers=False + ) + + by_name = {offer.instance.name: offer for offer in offers} + assert by_name["ECS1GPU6"].availability == InstanceAvailability.AVAILABLE + assert by_name["ECS1GPU11"].availability == InstanceAvailability.NOT_AVAILABLE + assert by_name["ECS1GPU6"].instance.resources.gpus[0].name == "L4" + + +def test_get_offers_filters_configured_regions(): + raws = [ + _raw("ECS1GPU7", "L40S", "it-fr2"), + _raw("ECS1GPU7", "L40S", "it-mi2"), + ] + with ( + patch( + "dstack._internal.core.backends.seeweb.compute.SeewebProvider.get", + return_value=raws, + ), + patch( + "dstack._internal.core.backends.seeweb.api_client.SeewebApiClient" + ".get_available_plan_regions", + return_value={("ECS1GPU7", "it-fr2"), ("ECS1GPU7", "it-mi2")}, + ), + ): + offers = SeewebCompute(_config(["it-mi2"])).get_offers_by_requirements( + requirements=None, full_offers=False + ) + + assert [offer.region for offer in offers] == ["it-mi2"] + + +def test_select_image_prefers_uefi_image(): + availability = SeewebPlanAvailability( + regions=frozenset({"it-fr2"}), + images=( + "ubuntu-2204-nvidia-driver", + "ubuntu-2204-uefi-nvidia-driver", + ), + ) + + assert ( + _select_image( + availability=availability, + plan_name="ECS1GPU7", + region="it-fr2", + is_gpu=True, + ) + == "ubuntu-2204-uefi-nvidia-driver" + ) + + +def test_select_image_rejects_unavailable_plan(): + with pytest.raises(NoCapacityError, match="not available"): + _select_image( + availability=None, + plan_name="ECS1GPU7", + region="it-fr2", + is_gpu=True, + ) + + +def test_select_image_rejects_unsupported_images(): + availability = SeewebPlanAvailability( + regions=frozenset({"it-fr2"}), + images=("debian-12",), + ) + + with pytest.raises(BackendError, match="no supported GPU image"): + _select_image( + availability=availability, + plan_name="ECS1GPU7", + region="it-fr2", + is_gpu=True, + ) + + +def test_create_instance_uses_allowed_image_and_quotes_ssh_key(): + compute = _compute() + public_key = "ssh-ed25519 AAAATEST comment with ' quote" + compute.api_client.get_available_plans.return_value = { + "ECS1GPU7": SeewebPlanAvailability( + regions=frozenset({"it-fr2"}), + images=("ubuntu-2204-uefi-nvidia-driver",), + ) + } + compute.api_client.get_or_create_ssh_key.return_value = "dstack-key" + compute.api_client.create_server.return_value = ({"name": "ec-test"}, 35) + + with ( + patch( + "dstack._internal.core.backends.seeweb.compute.generate_unique_instance_name", + return_value="dstack-seeweb-test", + ), + patch( + "dstack._internal.core.backends.seeweb.compute.get_shim_commands", + return_value=["prepare-shim", "start-shim"], + ), + ): + provisioning_data = compute.create_instance( + _gpu_offer(), _instance_config(public_key), placement_group=None + ) + + body = compute.api_client.create_server.call_args.args[0] + assert body["plan"] == "ECS1GPU7" + assert body["location"] == "it-fr2" + assert body["image"] == "ubuntu-2204-uefi-nvidia-driver" + assert body["ssh_key"] == "dstack-key" + assert f"printf '%s\\n' {shlex.quote(public_key)}" in body["user_customize"] + assert 'echo "ssh-' not in body["user_customize"] + assert "prepare-shim" in body["user_customize"] + assert "start-shim" not in body["user_customize"] + assert "EnvironmentFile=/etc/dstack-shim.env" in body["user_customize"] + assert body["user_customize"].endswith("systemctl enable dstack-shim.service") + assert provisioning_data.instance_id == "ec-test" + assert provisioning_data.hostname is None + assert provisioning_data.username == "root" + assert SeewebInstanceBackendData.load(provisioning_data.backend_data).action_id == 35 + + +def test_update_provisioning_data_waits_for_action(): + compute = _compute() + compute.api_client.get_action.return_value = {"status": "in-progress"} + provisioning_data = _provisioning_data() + + compute.update_provisioning_data(provisioning_data, "public", "private") + + assert provisioning_data.hostname is None + compute.api_client.get_server.assert_not_called() + + +def test_update_provisioning_data_sets_hostname_after_completed_action(): + compute = _compute() + compute.api_client.get_action.return_value = {"status": "completed"} + compute.api_client.get_server.return_value = { + "name": "ec-test", + "status": "Booted", + "ipv4": "192.0.2.1", + } + provisioning_data = _provisioning_data() + + compute.update_provisioning_data(provisioning_data, "public", "private") + + assert provisioning_data.hostname == "192.0.2.1" + + +def test_update_provisioning_data_raises_for_failed_action(): + compute = _compute() + compute.api_client.get_action.return_value = {"status": "failed"} + + with pytest.raises(ProvisioningError, match="entered status 'failed'"): + compute.update_provisioning_data(_provisioning_data(), "public", "private") + + +def test_update_provisioning_data_raises_for_failed_server(): + compute = _compute() + compute.api_client.get_server.return_value = { + "name": "ec-test", + "status": "Error", + } + + with pytest.raises(ProvisioningError, match="entered status 'error'"): + compute.update_provisioning_data(_provisioning_data(action_id=None), "public", "private") + + +def test_terminate_instance_delegates_to_idempotent_api(): + compute = _compute() + + compute.terminate_instance("ec-test", "it-fr2") + + compute.api_client.delete_server.assert_called_once_with("ec-test") diff --git a/src/tests/_internal/core/backends/seeweb/test_configurator.py b/src/tests/_internal/core/backends/seeweb/test_configurator.py new file mode 100644 index 000000000..543982a19 --- /dev/null +++ b/src/tests/_internal/core/backends/seeweb/test_configurator.py @@ -0,0 +1,70 @@ +from unittest.mock import patch + +import pytest + +from dstack._internal.core.backends.configurators import ( + get_configurator, + list_available_backend_types, +) +from dstack._internal.core.backends.seeweb.backend import SeewebBackend +from dstack._internal.core.backends.seeweb.compute import SeewebCompute +from dstack._internal.core.backends.seeweb.configurator import SeewebConfigurator +from dstack._internal.core.backends.seeweb.models import ( + SeewebAPITokenCreds, + SeewebBackendConfigWithCreds, +) +from dstack._internal.core.errors import BackendInvalidCredentialsError +from dstack._internal.core.models.backends.base import BackendType + +VALIDATE = "dstack._internal.core.backends.seeweb.api_client.SeewebApiClient.validate_api_key" + + +class TestSeewebConfigurator: + def test_registered(self): + assert BackendType.SEEWEB in list_available_backend_types() + assert isinstance(get_configurator(BackendType.SEEWEB), SeewebConfigurator) + + def test_validate_config_valid(self): + config = SeewebBackendConfigWithCreds( + creds=SeewebAPITokenCreds(api_token="valid"), + regions=["it-mi2"], + ) + with patch(VALIDATE) as validate_mock: + validate_mock.return_value = True + SeewebConfigurator().validate_config(config, default_creds_enabled=True) + + def test_validate_config_invalid_creds(self): + config = SeewebBackendConfigWithCreds( + creds=SeewebAPITokenCreds(api_token="invalid"), + regions=["it-mi2"], + ) + with ( + patch(VALIDATE) as validate_mock, + pytest.raises(BackendInvalidCredentialsError) as exc_info, + ): + validate_mock.return_value = False + SeewebConfigurator().validate_config(config, default_creds_enabled=True) + assert exc_info.value.fields == [["creds", "api_token"]] + + def test_config_round_trip_strips_creds(self): + config = SeewebBackendConfigWithCreds( + creds=SeewebAPITokenCreds(api_token="secret-token"), + regions=["it-mi2", "it-fr2"], + ) + configurator = SeewebConfigurator() + record = configurator.create_backend("proj", config) + + # Creds are stored separately from the non-sensitive config. + assert "secret-token" not in record.config + assert "secret-token" in record.auth + + with_creds = configurator.get_backend_config_with_creds(record) + assert with_creds.creds.api_token == "secret-token" + assert with_creds.regions == ["it-mi2", "it-fr2"] + + without_creds = configurator.get_backend_config_without_creds(record) + assert not hasattr(without_creds, "creds") + + backend = configurator.get_backend(record) + assert isinstance(backend, SeewebBackend) + assert isinstance(backend.compute(), SeewebCompute) diff --git a/src/tests/_internal/server/routers/test_backends.py b/src/tests/_internal/server/routers/test_backends.py index 910c62dbf..9aed7be00 100644 --- a/src/tests/_internal/server/routers/test_backends.py +++ b/src/tests/_internal/server/routers/test_backends.py @@ -98,6 +98,7 @@ async def test_returns_backend_types(self, client: AsyncClient): "oci", "runpod", "slurm", + "seeweb", "vastai", "verda", "vultr",