From 7631d71a258f35b82141d7d260eb7dbe41665c8a Mon Sep 17 00:00:00 2001 From: peterschmidt85 Date: Sun, 1 Feb 2026 22:15:23 +0100 Subject: [PATCH 1/4] [UX] Make `dstack fleet` show more useful information --- src/dstack/_internal/cli/utils/fleet.py | 350 ++++++++++--- src/tests/_internal/cli/utils/test_fleet.py | 531 ++++++++++++++++++++ 2 files changed, 806 insertions(+), 75 deletions(-) create mode 100644 src/tests/_internal/cli/utils/test_fleet.py diff --git a/src/dstack/_internal/cli/utils/fleet.py b/src/dstack/_internal/cli/utils/fleet.py index 3d04100c8a..4443fa66a1 100644 --- a/src/dstack/_internal/cli/utils/fleet.py +++ b/src/dstack/_internal/cli/utils/fleet.py @@ -1,11 +1,12 @@ -from typing import List +from typing import Any, Dict, List, Optional, Union from rich.table import Table from dstack._internal.cli.utils.common import add_row_from_dict, console from dstack._internal.core.models.backends.base import BackendType -from dstack._internal.core.models.fleets import Fleet, FleetStatus -from dstack._internal.core.models.instances import InstanceStatus +from dstack._internal.core.models.fleets import Fleet, FleetNodesSpec, FleetStatus +from dstack._internal.core.models.instances import Instance, InstanceStatus +from dstack._internal.core.models.resources import GPUSpec, ResourcesSpec from dstack._internal.utils.common import DateFormatter, pretty_date @@ -14,93 +15,292 @@ def print_fleets_table(fleets: List[Fleet], verbose: bool = False) -> None: console.print() +def _format_nodes(nodes: Optional[FleetNodesSpec]) -> str: + """Format nodes spec as '0..1', '3', '2..10', etc.""" + if nodes is None: + return "-" + if nodes.min == nodes.max: + return str(nodes.min) + if nodes.max is None: + return f"{nodes.min}.." + return f"{nodes.min}..{nodes.max}" + + +def _format_backends(backends: Optional[List[BackendType]]) -> str: + if backends is None or len(backends) == 0: + return "*" + return ", ".join(b.value.replace("remote", "ssh") for b in backends) + + +def _format_range(min_val: Optional[Any], max_val: Optional[Any]) -> str: + if min_val is None and max_val is None: + return "" + if min_val == max_val: + return str(min_val) + if max_val is None: + return f"{min_val}.." + if min_val is None: + return f"..{max_val}" + return f"{min_val}..{max_val}" + + +def _format_fleet_gpu(resources: Optional[ResourcesSpec]) -> str: + """Extract GPU-only info from fleet requirements, handling ranges.""" + if resources is None or resources.gpu is None: + return "-" + + gpu: GPUSpec = resources.gpu + + # Check if there's actually a GPU requirement + count = gpu.count + if count is None or (count.min == 0 and (count.max is None or count.max == 0)): + return "-" + + parts = [] + + # GPU name(s) + if gpu.name: + parts.append(",".join(gpu.name)) + else: + parts.append("gpu") + + # GPU memory (range) + if gpu.memory is not None: + mem_str = _format_range(gpu.memory.min, gpu.memory.max) + if mem_str: + parts.append(mem_str) + + # GPU count (range) + count_str = _format_range(count.min, count.max) + if count_str: + parts.append(count_str) + + return ":".join(parts) + + +def _format_fleet_status(fleet: Fleet) -> str: + status = fleet.status + status_text = status.value + + color_map = { + FleetStatus.SUBMITTED: "grey", + FleetStatus.ACTIVE: "white", + FleetStatus.TERMINATING: "deep_sky_blue1", + FleetStatus.TERMINATED: "grey", + FleetStatus.FAILED: "indian_red1", + } + color = color_map.get(status, "white") + is_finished = status in [FleetStatus.TERMINATED, FleetStatus.FAILED] + status_style = f"bold {color}" if not is_finished else color + return f"[{status_style}]{status_text}[/]" + + +def _format_instance_status(instance: Instance) -> str: + """Format instance status with colors and health info.""" + status = instance.status + status_text = status.value + + total_blocks = instance.total_blocks + busy_blocks = instance.busy_blocks + if ( + status in [InstanceStatus.IDLE, InstanceStatus.BUSY] + and total_blocks is not None + and total_blocks > 1 + ): + status_text = f"{busy_blocks}/{total_blocks} {InstanceStatus.BUSY.value}" + + # Add health status + health_suffix = "" + if status in [InstanceStatus.IDLE, InstanceStatus.BUSY]: + if instance.unreachable: + health_suffix = " (unreachable)" + elif not instance.health_status.is_healthy(): + health_suffix = f" ({instance.health_status.value})" + + color_map = { + InstanceStatus.PENDING: "deep_sky_blue1", + InstanceStatus.PROVISIONING: "deep_sky_blue1", + InstanceStatus.IDLE: "sea_green3", + InstanceStatus.BUSY: "white", + InstanceStatus.TERMINATING: "deep_sky_blue1", + InstanceStatus.TERMINATED: "grey", + } + color = color_map.get(status, "white") + is_finished = status == InstanceStatus.TERMINATED + status_style = f"bold {color}" if not is_finished else color + return f"[{status_style}]{status_text}{health_suffix}[/]" + + +def _format_backend(backend: Optional[BackendType], region: Optional[str]) -> str: + if backend is None: + return "-" + backend_str = backend.value + if backend == BackendType.REMOTE: + backend_str = "ssh" + if region: + backend_str += f" ({region})" + return backend_str + + +def _format_price(price: Optional[float]) -> str: + if price is None: + return "-" + return f"${price:.4f}".rstrip("0").rstrip(".") + + +def _format_instance_gpu(instance: Instance) -> str: + if instance.instance_type is None: + return "-" + if ( + instance.backend == BackendType.REMOTE + and instance.status in [InstanceStatus.PENDING, InstanceStatus.PROVISIONING] + ): + return "-" + return instance.instance_type.resources.pretty_format(gpu_only=True, include_spot=False) or "-" + + +def _format_instance_resources(instance: Instance) -> str: + if instance.instance_type is None: + return "-" + if ( + instance.backend == BackendType.REMOTE + and instance.status in [InstanceStatus.PENDING, InstanceStatus.PROVISIONING] + ): + return "-" + return instance.instance_type.resources.pretty_format(include_spot=False) + + def get_fleets_table( fleets: List[Fleet], verbose: bool = False, format_date: DateFormatter = pretty_date ) -> Table: table = Table(box=None) - table.add_column("FLEET", no_wrap=True) + + # Columns + table.add_column("NAME", style="bold", no_wrap=True) + table.add_column("NODES") if verbose: - table.add_column("RESERVATION") - table.add_column("INSTANCE") + table.add_column("RESOURCES") + else: + table.add_column("GPU") + table.add_column("SPOT") table.add_column("BACKEND") - if verbose: - table.add_column("REGION") - table.add_column("RESOURCES") table.add_column("PRICE") - table.add_column("STATUS") - table.add_column("CREATED") - + table.add_column("STATUS", no_wrap=True) + table.add_column("CREATED", no_wrap=True) if verbose: table.add_column("ERROR") for fleet in fleets: - for i, instance in enumerate(fleet.instances): - resources = "" - if instance.instance_type is not None and ( - instance.backend != BackendType.REMOTE - or instance.status not in [InstanceStatus.PENDING, InstanceStatus.PROVISIONING] - ): - resources = instance.instance_type.resources.pretty_format(include_spot=True) - - status = instance.status.value - total_blocks = instance.total_blocks - busy_blocks = instance.busy_blocks - if ( - instance.status in [InstanceStatus.IDLE, InstanceStatus.BUSY] - and total_blocks is not None - and total_blocks > 1 - ): - status = f"{busy_blocks}/{total_blocks} {InstanceStatus.BUSY.value}" - if instance.status in [InstanceStatus.IDLE, InstanceStatus.BUSY]: - if instance.unreachable: - status += "\n(unreachable)" - elif not instance.health_status.is_healthy(): - status += f"\n({instance.health_status.value})" - - backend = instance.backend or "" - if backend == "remote": - backend = "ssh" - - region = "" - if instance.region: - region = f"{instance.region}" - if verbose: - if instance.availability_zone: - region += f" ({instance.availability_zone})" - else: - backend += f" ({instance.region})" - error = "" - if instance.status == InstanceStatus.TERMINATED and instance.termination_reason: - error = f"{instance.termination_reason}" - row = { - "FLEET": fleet.name if i == 0 else "", - "RESERVATION": fleet.spec.configuration.reservation or "" if i == 0 else "", - "INSTANCE": str(instance.instance_num), - "BACKEND": backend, - "REGION": region, - "RESOURCES": resources, - "PRICE": f"${instance.price:.4f}".rstrip("0").rstrip(".") - if instance.price is not None - else "", - "STATUS": status, + # Fleet row + config = fleet.spec.configuration + merged_profile = fleet.spec.merged_profile + + # Detect SSH fleet vs backend fleet + is_ssh_fleet = config.ssh_config is not None + + if is_ssh_fleet: + # SSH fleet: fixed number of hosts, no cloud billing + nodes = str(len(config.ssh_config.hosts)) + backend = "ssh" + spot_policy = "-" + max_price = "-" + else: + # Backend fleet: dynamic nodes, cloud billing + nodes = _format_nodes(config.nodes) + backend = _format_backends(config.backends) + spot_policy = "-" + if merged_profile and merged_profile.spot_policy: + spot_policy = merged_profile.spot_policy.value + # Format as "$0..$X.XX" range, or "-" if not set + if merged_profile and merged_profile.max_price is not None: + max_price = f"$0..{_format_price(merged_profile.max_price)}" + else: + max_price = "-" + + # In verbose mode, append placement to nodes if cluster + if verbose and config.placement and config.placement.value == "cluster": + nodes = f"{nodes} (cluster)" + + fleet_row: Dict[Union[str, int], Any] = { + "NAME": fleet.name, + "NODES": nodes, + "BACKEND": backend, + "PRICE": max_price, + "SPOT": spot_policy, + "STATUS": _format_fleet_status(fleet), + "CREATED": format_date(fleet.created_at), + } + + if verbose: + fleet_row["RESOURCES"] = config.resources.pretty_format() if config.resources else "-" + fleet_row["ERROR"] = "" + else: + fleet_row["GPU"] = _format_fleet_gpu(config.resources) + + add_row_from_dict(table, fleet_row) + + # Instance rows (indented) + for instance in fleet.instances: + # Check if this is an SSH instance + is_ssh_instance = instance.backend == BackendType.REMOTE + + # Format backend with region (and AZ in verbose mode) + if verbose and instance.availability_zone: + # In verbose mode, show AZ instead of region (AZ is more specific) + backend_with_region = _format_backend(instance.backend, instance.availability_zone) + else: + backend_with_region = _format_backend(instance.backend, instance.region) + + # Get spot info from instance resources (not applicable to SSH) + if is_ssh_instance: + instance_spot = "-" + instance_price = "-" + else: + instance_spot = "-" + if ( + instance.instance_type is not None + and instance.instance_type.resources is not None + ): + instance_spot = "spot" if instance.instance_type.resources.spot else "on-demand" + instance_price = _format_price(instance.price) + + instance_row: Dict[Union[str, int], Any] = { + "NAME": f" instance={instance.instance_num}", + "NODES": "", + "BACKEND": backend_with_region, + "PRICE": instance_price, + "SPOT": instance_spot, + "STATUS": _format_instance_status(instance), "CREATED": format_date(instance.created), - "ERROR": error, } - add_row_from_dict(table, row) + if verbose: + instance_row["RESOURCES"] = _format_instance_resources(instance) + error = "" + if instance.status == InstanceStatus.TERMINATED and instance.termination_reason: + error = instance.termination_reason + instance_row["ERROR"] = error + else: + instance_row["GPU"] = _format_instance_gpu(instance) + + add_row_from_dict(table, instance_row, style="secondary") + + # If fleet has no instances and is not terminating, show placeholder if len(fleet.instances) == 0 and fleet.status != FleetStatus.TERMINATING: - row = { - "FLEET": fleet.name, - "RESERVATION": "-", - "INSTANCE": "-", - "BACKEND": "-", - "REGION": "-", - "RESOURCES": "-", - "PRICE": "-", - "STATUS": "-", - "CREATED": format_date(fleet.created_at), - "ERROR": "-", + empty_row: Dict[Union[str, int], Any] = { + "NAME": " (no instances)", + "NODES": "", + "BACKEND": "", + "PRICE": "", + "SPOT": "", + "STATUS": "", + "CREATED": "", } - add_row_from_dict(table, row) + if verbose: + empty_row["RESOURCES"] = "" + empty_row["ERROR"] = "" + else: + empty_row["GPU"] = "" + add_row_from_dict(table, empty_row, style="secondary") return table diff --git a/src/tests/_internal/cli/utils/test_fleet.py b/src/tests/_internal/cli/utils/test_fleet.py new file mode 100644 index 0000000000..1ee93a472a --- /dev/null +++ b/src/tests/_internal/cli/utils/test_fleet.py @@ -0,0 +1,531 @@ +import re +from datetime import datetime, timezone +from typing import List, Optional +from uuid import uuid4 + +import pytest +from rich.table import Table +from rich.text import Text + +from dstack._internal.cli.utils.fleet import get_fleets_table +from dstack._internal.core.models.backends.base import BackendType +from dstack._internal.core.models.fleets import ( + Fleet, + FleetConfiguration, + FleetNodesSpec, + FleetSpec, + FleetStatus, + InstanceGroupPlacement, + SSHHostParams, + SSHParams, +) +from dstack._internal.core.models.health import HealthStatus +from dstack._internal.core.models.instances import ( + Disk, + Gpu, + Instance, + InstanceStatus, + InstanceType, + Resources, + SSHKey, +) +from dstack._internal.core.models.profiles import Profile, SpotPolicy +from dstack._internal.core.models.resources import GPUSpec, Range, ResourcesSpec + + +def _strip_rich_markup(text: str) -> str: + return re.sub(r"\[[^\]]*\]([^\[]*)\[/[^\]]*\]", r"\1", text) + + +def get_table_cells(table: Table) -> list[dict[str, str]]: + rows = [] + + if not table.columns: + return rows + + num_rows = len(table.columns[0]._cells) + + for row_idx in range(num_rows): + row = {} + for col in table.columns: + col_name = str(col.header) + if row_idx < len(col._cells): + cell_value = col._cells[row_idx] + if isinstance(cell_value, Text): + row[col_name] = cell_value.plain + else: + text = str(cell_value) + row[col_name] = _strip_rich_markup(text) + else: + row[col_name] = "" + rows.append(row) + + return rows + + +def get_table_cell_style(table: Table, column_name: str, row_idx: int = 0) -> Optional[str]: + for col in table.columns: + if str(col.header) == column_name: + if row_idx < len(col._cells): + cell_value = col._cells[row_idx] + if isinstance(cell_value, Text): + return str(cell_value.style) if cell_value.style else None + text = str(cell_value) + match = re.search(r"\[([^\]]+)\][^\[]*\[/\]", text) + if match: + return match.group(1) + return None + return None + + +def create_test_instance( + instance_num: int = 0, + backend: BackendType = BackendType.AWS, + region: str = "us-east-1", + status: InstanceStatus = InstanceStatus.IDLE, + price: Optional[float] = 0.50, + spot: bool = False, + gpu_name: Optional[str] = None, + gpu_count: int = 0, + gpu_memory_mib: int = 0, +) -> Instance: + gpus = [] + if gpu_count > 0 and gpu_name: + gpus = [Gpu(name=gpu_name, memory_mib=gpu_memory_mib)] * gpu_count + + resources = Resources( + cpus=4, + memory_mib=16384, + gpus=gpus, + spot=spot, + disk=Disk(size_mib=102400), + ) + instance_type = InstanceType(name="test-instance", resources=resources) + + return Instance( + id=uuid4(), + project_name="test-project", + name=f"instance-{instance_num}", + instance_num=instance_num, + backend=backend, + region=region, + status=status, + price=price, + instance_type=instance_type, + created=datetime(2023, 1, 2, 3, 4, 5, tzinfo=timezone.utc), + ) + + +def create_backend_fleet( + name: str = "test-fleet", + nodes_min: int = 0, + nodes_max: int = 2, + backends: Optional[List[BackendType]] = None, + spot_policy: SpotPolicy = SpotPolicy.AUTO, + max_price: Optional[float] = None, + placement: Optional[InstanceGroupPlacement] = None, + gpu_count_min: int = 0, + gpu_count_max: int = 0, + instances: Optional[List[Instance]] = None, + status: FleetStatus = FleetStatus.ACTIVE, +) -> Fleet: + nodes = FleetNodesSpec(min=nodes_min, target=nodes_min, max=nodes_max) + + gpu_spec = None + if gpu_count_max > 0: + gpu_spec = GPUSpec(count=Range[int](min=gpu_count_min, max=gpu_count_max)) + + resources = ResourcesSpec(gpu=gpu_spec) if gpu_spec else ResourcesSpec() + + config = FleetConfiguration( + name=name, + nodes=nodes, + backends=backends, + placement=placement, + resources=resources, + ) + + profile = Profile(name="default", spot_policy=spot_policy, max_price=max_price) + + spec = FleetSpec( + configuration=config, + configuration_path="fleet.dstack.yml", + profile=profile, + ) + + return Fleet( + id=uuid4(), + name=name, + project_name="test-project", + spec=spec, + created_at=datetime(2023, 1, 2, 3, 4, 5, tzinfo=timezone.utc), + status=status, + instances=instances or [], + ) + + +def create_ssh_fleet( + name: str = "ssh-fleet", + hosts: Optional[List[str]] = None, + placement: Optional[InstanceGroupPlacement] = None, + instances: Optional[List[Instance]] = None, + status: FleetStatus = FleetStatus.ACTIVE, +) -> Fleet: + if hosts is None: + hosts = ["10.0.0.1", "10.0.0.2"] + + ssh_key = SSHKey(public="ssh-rsa AAAA...", private="-----BEGIN PRIVATE KEY-----\n...") + ssh_config = SSHParams( + user="ubuntu", + ssh_key=ssh_key, + hosts=[SSHHostParams(hostname=h) for h in hosts], + network=None, + ) + + config = FleetConfiguration( + name=name, + ssh_config=ssh_config, + placement=placement, + ) + + spec = FleetSpec( + configuration=config, + configuration_path="fleet.dstack.yml", + profile=Profile(name="default"), + ) + + return Fleet( + id=uuid4(), + name=name, + project_name="test-project", + spec=spec, + created_at=datetime(2023, 1, 2, 3, 4, 5, tzinfo=timezone.utc), + status=status, + instances=instances or [], + ) + + +class TestGetFleetsTable: + """Tests for get_fleets_table function.""" + + def test_backend_fleet_without_verbose(self): + """Test backend fleet display without verbose mode.""" + instance = create_test_instance( + instance_num=0, + backend=BackendType.AWS, + region="us-east-1", + status=InstanceStatus.IDLE, + price=0.50, + spot=True, + ) + fleet = create_backend_fleet( + name="my-cloud", + nodes_min=0, + nodes_max=4, + backends=[BackendType.AWS], + spot_policy=SpotPolicy.AUTO, + instances=[instance], + ) + + table = get_fleets_table([fleet], verbose=False) + cells = get_table_cells(table) + + assert len(cells) == 2 # 1 fleet row + 1 instance row + + # Fleet row + fleet_row = cells[0] + assert fleet_row["NAME"] == "my-cloud" + assert fleet_row["NODES"] == "0..4" + assert fleet_row["BACKEND"] == "aws" + assert fleet_row["SPOT"] == "auto" + assert fleet_row["PRICE"] == "-" # no max_price set + assert fleet_row["STATUS"] == "active" + + # Instance row + instance_row = cells[1] + assert "instance=0" in instance_row["NAME"] + assert instance_row["BACKEND"] == "aws (us-east-1)" + assert instance_row["SPOT"] == "spot" + assert instance_row["PRICE"] == "$0.5" + assert instance_row["STATUS"] == "idle" + + def test_backend_fleet_with_verbose(self): + """Test backend fleet display with verbose mode.""" + instance = create_test_instance( + instance_num=0, + backend=BackendType.GCP, + region="us-west4", + status=InstanceStatus.BUSY, + price=1.25, + spot=False, + ) + fleet = create_backend_fleet( + name="my-cloud", + nodes_min=1, + nodes_max=1, + backends=[BackendType.GCP], + spot_policy=SpotPolicy.ONDEMAND, + max_price=2.0, + placement=InstanceGroupPlacement.CLUSTER, + instances=[instance], + ) + + table = get_fleets_table([fleet], verbose=True) + cells = get_table_cells(table) + + assert len(cells) == 2 + + # Fleet row + fleet_row = cells[0] + assert fleet_row["NAME"] == "my-cloud" + assert fleet_row["NODES"] == "1 (cluster)" + assert fleet_row["BACKEND"] == "gcp" + assert fleet_row["SPOT"] == "on-demand" + assert fleet_row["PRICE"] == "$0..$2" + assert fleet_row["STATUS"] == "active" + + # Instance row + instance_row = cells[1] + assert "instance=0" in instance_row["NAME"] + assert instance_row["BACKEND"] == "gcp (us-west4)" + assert instance_row["SPOT"] == "on-demand" + assert instance_row["PRICE"] == "$1.25" + + def test_ssh_fleet_without_verbose(self): + """Test SSH fleet display without verbose mode.""" + instance1 = create_test_instance( + instance_num=0, + backend=BackendType.REMOTE, + region="", + status=InstanceStatus.IDLE, + price=None, + spot=False, + gpu_name="L4", + gpu_count=1, + gpu_memory_mib=24576, + ) + instance2 = create_test_instance( + instance_num=1, + backend=BackendType.REMOTE, + region="", + status=InstanceStatus.BUSY, + price=None, + spot=False, + gpu_name="L4", + gpu_count=1, + gpu_memory_mib=24576, + ) + fleet = create_ssh_fleet( + name="my-ssh", + hosts=["10.0.0.1", "10.0.0.2"], + instances=[instance1, instance2], + ) + + table = get_fleets_table([fleet], verbose=False) + cells = get_table_cells(table) + + assert len(cells) == 3 # 1 fleet row + 2 instance rows + + # Fleet row + fleet_row = cells[0] + assert fleet_row["NAME"] == "my-ssh" + assert fleet_row["NODES"] == "2" # Number of hosts + assert fleet_row["BACKEND"] == "ssh" + assert fleet_row["SPOT"] == "-" + assert fleet_row["PRICE"] == "-" + assert fleet_row["STATUS"] == "active" + + # Instance rows + for i, instance_row in enumerate(cells[1:], start=0): + assert f"instance={i}" in instance_row["NAME"] + assert instance_row["BACKEND"] == "ssh" + assert instance_row["SPOT"] == "-" + assert instance_row["PRICE"] == "-" + + def test_ssh_fleet_with_verbose(self): + """Test SSH fleet display with verbose mode.""" + instance = create_test_instance( + instance_num=0, + backend=BackendType.REMOTE, + region="", + status=InstanceStatus.IDLE, + price=None, + spot=False, + ) + fleet = create_ssh_fleet( + name="my-ssh", + hosts=["10.0.0.1"], + placement=InstanceGroupPlacement.CLUSTER, + instances=[instance], + ) + + table = get_fleets_table([fleet], verbose=True) + cells = get_table_cells(table) + + assert len(cells) == 2 + + # Fleet row + fleet_row = cells[0] + assert fleet_row["NAME"] == "my-ssh" + assert fleet_row["NODES"] == "1 (cluster)" + assert fleet_row["BACKEND"] == "ssh" + assert fleet_row["SPOT"] == "-" + assert fleet_row["PRICE"] == "-" + + # Instance row + instance_row = cells[1] + assert "instance=0" in instance_row["NAME"] + assert instance_row["BACKEND"] == "ssh" + assert instance_row["SPOT"] == "-" + assert instance_row["PRICE"] == "-" + + def test_mixed_fleets(self): + """Test display of mixed backend and SSH fleets.""" + backend_instance = create_test_instance( + instance_num=0, + backend=BackendType.AWS, + region="us-east-1", + status=InstanceStatus.BUSY, + price=0.75, + spot=True, + ) + backend_fleet = create_backend_fleet( + name="cloud-fleet", + nodes_min=0, + nodes_max=2, + backends=[BackendType.AWS], + spot_policy=SpotPolicy.SPOT, + instances=[backend_instance], + ) + + ssh_instance = create_test_instance( + instance_num=0, + backend=BackendType.REMOTE, + region="", + status=InstanceStatus.IDLE, + price=None, + spot=False, + ) + ssh_fleet = create_ssh_fleet( + name="ssh-fleet", + hosts=["10.0.0.1"], + instances=[ssh_instance], + ) + + table = get_fleets_table([backend_fleet, ssh_fleet], verbose=False) + cells = get_table_cells(table) + + assert len(cells) == 4 # 2 fleet rows + 2 instance rows + + # Backend fleet + assert cells[0]["NAME"] == "cloud-fleet" + assert cells[0]["NODES"] == "0..2" + assert cells[0]["BACKEND"] == "aws" + assert cells[0]["SPOT"] == "spot" + + # Backend instance + assert "instance=0" in cells[1]["NAME"] + assert cells[1]["SPOT"] == "spot" + assert cells[1]["PRICE"] == "$0.75" + + # SSH fleet + assert cells[2]["NAME"] == "ssh-fleet" + assert cells[2]["NODES"] == "1" + assert cells[2]["BACKEND"] == "ssh" + assert cells[2]["SPOT"] == "-" + assert cells[2]["PRICE"] == "-" + + # SSH instance + assert "instance=0" in cells[3]["NAME"] + assert cells[3]["SPOT"] == "-" + assert cells[3]["PRICE"] == "-" + + def test_fleet_status_colors(self): + """Test that fleet statuses have correct colors.""" + # Add instances to avoid placeholder rows affecting row indices + active_instance = create_test_instance(instance_num=0, status=InstanceStatus.IDLE) + active_fleet = create_backend_fleet( + name="active", status=FleetStatus.ACTIVE, instances=[active_instance] + ) + + terminating_instance = create_test_instance(instance_num=0, status=InstanceStatus.TERMINATING) + terminating_fleet = create_backend_fleet( + name="terminating", status=FleetStatus.TERMINATING, instances=[terminating_instance] + ) + + table = get_fleets_table([active_fleet, terminating_fleet], verbose=False) + + # Active fleet (row 0) should be bold white + active_style = get_table_cell_style(table, "STATUS", 0) + assert active_style == "bold white" + + # Terminating fleet (row 2, after active fleet's instance) should be bold blue + terminating_style = get_table_cell_style(table, "STATUS", 2) + assert terminating_style == "bold deep_sky_blue1" + + def test_instance_status_colors(self): + """Test that instance statuses have correct colors.""" + idle_instance = create_test_instance(instance_num=0, status=InstanceStatus.IDLE) + busy_instance = create_test_instance(instance_num=1, status=InstanceStatus.BUSY) + + fleet = create_backend_fleet( + name="test", + instances=[idle_instance, busy_instance], + ) + + table = get_fleets_table([fleet], verbose=False) + + # Idle should be bold green + idle_style = get_table_cell_style(table, "STATUS", 1) + assert idle_style == "bold sea_green3" + + # Busy should be bold white + busy_style = get_table_cell_style(table, "STATUS", 2) + assert busy_style == "bold white" + + def test_empty_fleet_placeholder(self): + """Test that fleets with no instances show placeholder.""" + fleet = create_backend_fleet(name="empty-fleet", instances=[]) + + table = get_fleets_table([fleet], verbose=False) + cells = get_table_cells(table) + + assert len(cells) == 2 + assert cells[0]["NAME"] == "empty-fleet" + assert "(no instances)" in cells[1]["NAME"] + + def test_fleet_with_max_price(self): + """Test fleet with max_price configured.""" + fleet = create_backend_fleet( + name="priced-fleet", + max_price=5.0, + ) + + table = get_fleets_table([fleet], verbose=False) + cells = get_table_cells(table) + + assert cells[0]["PRICE"] == "$0..$5" + + def test_fleet_with_multiple_backends(self): + """Test fleet with multiple backends configured.""" + fleet = create_backend_fleet( + name="multi-backend", + backends=[BackendType.AWS, BackendType.GCP, BackendType.AZURE], + ) + + table = get_fleets_table([fleet], verbose=False) + cells = get_table_cells(table) + + assert cells[0]["BACKEND"] == "aws, gcp, azure" + + def test_fleet_with_any_backend(self): + """Test fleet with no backends configured (any backend).""" + fleet = create_backend_fleet( + name="any-backend", + backends=None, + ) + + table = get_fleets_table([fleet], verbose=False) + cells = get_table_cells(table) + + assert cells[0]["BACKEND"] == "*" From a0e8a2dc9d2bdd495619a5d2cd63d4e74d29fc74 Mon Sep 17 00:00:00 2001 From: peterschmidt85 Date: Sun, 1 Feb 2026 22:31:50 +0100 Subject: [PATCH 2/4] Do not show "(no instances" row placeholder for empty fleets; removed unecessary docstrings and comments. --- src/dstack/_internal/cli/utils/fleet.py | 18 ---------- src/tests/_internal/cli/utils/test_fleet.py | 37 +++------------------ 2 files changed, 4 insertions(+), 51 deletions(-) diff --git a/src/dstack/_internal/cli/utils/fleet.py b/src/dstack/_internal/cli/utils/fleet.py index 4443fa66a1..19527b9db9 100644 --- a/src/dstack/_internal/cli/utils/fleet.py +++ b/src/dstack/_internal/cli/utils/fleet.py @@ -285,22 +285,4 @@ def get_fleets_table( add_row_from_dict(table, instance_row, style="secondary") - # If fleet has no instances and is not terminating, show placeholder - if len(fleet.instances) == 0 and fleet.status != FleetStatus.TERMINATING: - empty_row: Dict[Union[str, int], Any] = { - "NAME": " (no instances)", - "NODES": "", - "BACKEND": "", - "PRICE": "", - "SPOT": "", - "STATUS": "", - "CREATED": "", - } - if verbose: - empty_row["RESOURCES"] = "" - empty_row["ERROR"] = "" - else: - empty_row["GPU"] = "" - add_row_from_dict(table, empty_row, style="secondary") - return table diff --git a/src/tests/_internal/cli/utils/test_fleet.py b/src/tests/_internal/cli/utils/test_fleet.py index 1ee93a472a..5cf1bd1c28 100644 --- a/src/tests/_internal/cli/utils/test_fleet.py +++ b/src/tests/_internal/cli/utils/test_fleet.py @@ -206,10 +206,7 @@ def create_ssh_fleet( class TestGetFleetsTable: - """Tests for get_fleets_table function.""" - def test_backend_fleet_without_verbose(self): - """Test backend fleet display without verbose mode.""" instance = create_test_instance( instance_num=0, backend=BackendType.AWS, @@ -232,7 +229,6 @@ def test_backend_fleet_without_verbose(self): assert len(cells) == 2 # 1 fleet row + 1 instance row - # Fleet row fleet_row = cells[0] assert fleet_row["NAME"] == "my-cloud" assert fleet_row["NODES"] == "0..4" @@ -241,7 +237,6 @@ def test_backend_fleet_without_verbose(self): assert fleet_row["PRICE"] == "-" # no max_price set assert fleet_row["STATUS"] == "active" - # Instance row instance_row = cells[1] assert "instance=0" in instance_row["NAME"] assert instance_row["BACKEND"] == "aws (us-east-1)" @@ -250,7 +245,6 @@ def test_backend_fleet_without_verbose(self): assert instance_row["STATUS"] == "idle" def test_backend_fleet_with_verbose(self): - """Test backend fleet display with verbose mode.""" instance = create_test_instance( instance_num=0, backend=BackendType.GCP, @@ -275,7 +269,6 @@ def test_backend_fleet_with_verbose(self): assert len(cells) == 2 - # Fleet row fleet_row = cells[0] assert fleet_row["NAME"] == "my-cloud" assert fleet_row["NODES"] == "1 (cluster)" @@ -284,7 +277,6 @@ def test_backend_fleet_with_verbose(self): assert fleet_row["PRICE"] == "$0..$2" assert fleet_row["STATUS"] == "active" - # Instance row instance_row = cells[1] assert "instance=0" in instance_row["NAME"] assert instance_row["BACKEND"] == "gcp (us-west4)" @@ -292,7 +284,6 @@ def test_backend_fleet_with_verbose(self): assert instance_row["PRICE"] == "$1.25" def test_ssh_fleet_without_verbose(self): - """Test SSH fleet display without verbose mode.""" instance1 = create_test_instance( instance_num=0, backend=BackendType.REMOTE, @@ -326,16 +317,14 @@ def test_ssh_fleet_without_verbose(self): assert len(cells) == 3 # 1 fleet row + 2 instance rows - # Fleet row fleet_row = cells[0] assert fleet_row["NAME"] == "my-ssh" - assert fleet_row["NODES"] == "2" # Number of hosts + assert fleet_row["NODES"] == "2" assert fleet_row["BACKEND"] == "ssh" assert fleet_row["SPOT"] == "-" assert fleet_row["PRICE"] == "-" assert fleet_row["STATUS"] == "active" - # Instance rows for i, instance_row in enumerate(cells[1:], start=0): assert f"instance={i}" in instance_row["NAME"] assert instance_row["BACKEND"] == "ssh" @@ -343,7 +332,6 @@ def test_ssh_fleet_without_verbose(self): assert instance_row["PRICE"] == "-" def test_ssh_fleet_with_verbose(self): - """Test SSH fleet display with verbose mode.""" instance = create_test_instance( instance_num=0, backend=BackendType.REMOTE, @@ -364,7 +352,6 @@ def test_ssh_fleet_with_verbose(self): assert len(cells) == 2 - # Fleet row fleet_row = cells[0] assert fleet_row["NAME"] == "my-ssh" assert fleet_row["NODES"] == "1 (cluster)" @@ -372,7 +359,6 @@ def test_ssh_fleet_with_verbose(self): assert fleet_row["SPOT"] == "-" assert fleet_row["PRICE"] == "-" - # Instance row instance_row = cells[1] assert "instance=0" in instance_row["NAME"] assert instance_row["BACKEND"] == "ssh" @@ -380,7 +366,6 @@ def test_ssh_fleet_with_verbose(self): assert instance_row["PRICE"] == "-" def test_mixed_fleets(self): - """Test display of mixed backend and SSH fleets.""" backend_instance = create_test_instance( instance_num=0, backend=BackendType.AWS, @@ -417,31 +402,26 @@ def test_mixed_fleets(self): assert len(cells) == 4 # 2 fleet rows + 2 instance rows - # Backend fleet assert cells[0]["NAME"] == "cloud-fleet" assert cells[0]["NODES"] == "0..2" assert cells[0]["BACKEND"] == "aws" assert cells[0]["SPOT"] == "spot" - # Backend instance assert "instance=0" in cells[1]["NAME"] assert cells[1]["SPOT"] == "spot" assert cells[1]["PRICE"] == "$0.75" - # SSH fleet assert cells[2]["NAME"] == "ssh-fleet" assert cells[2]["NODES"] == "1" assert cells[2]["BACKEND"] == "ssh" assert cells[2]["SPOT"] == "-" assert cells[2]["PRICE"] == "-" - # SSH instance assert "instance=0" in cells[3]["NAME"] assert cells[3]["SPOT"] == "-" assert cells[3]["PRICE"] == "-" def test_fleet_status_colors(self): - """Test that fleet statuses have correct colors.""" # Add instances to avoid placeholder rows affecting row indices active_instance = create_test_instance(instance_num=0, status=InstanceStatus.IDLE) active_fleet = create_backend_fleet( @@ -455,16 +435,14 @@ def test_fleet_status_colors(self): table = get_fleets_table([active_fleet, terminating_fleet], verbose=False) - # Active fleet (row 0) should be bold white active_style = get_table_cell_style(table, "STATUS", 0) assert active_style == "bold white" - # Terminating fleet (row 2, after active fleet's instance) should be bold blue + # Row 2 (after active fleet's instance) terminating_style = get_table_cell_style(table, "STATUS", 2) assert terminating_style == "bold deep_sky_blue1" def test_instance_status_colors(self): - """Test that instance statuses have correct colors.""" idle_instance = create_test_instance(instance_num=0, status=InstanceStatus.IDLE) busy_instance = create_test_instance(instance_num=1, status=InstanceStatus.BUSY) @@ -475,27 +453,22 @@ def test_instance_status_colors(self): table = get_fleets_table([fleet], verbose=False) - # Idle should be bold green idle_style = get_table_cell_style(table, "STATUS", 1) assert idle_style == "bold sea_green3" - # Busy should be bold white busy_style = get_table_cell_style(table, "STATUS", 2) assert busy_style == "bold white" - def test_empty_fleet_placeholder(self): - """Test that fleets with no instances show placeholder.""" + def test_empty_fleet(self): fleet = create_backend_fleet(name="empty-fleet", instances=[]) table = get_fleets_table([fleet], verbose=False) cells = get_table_cells(table) - assert len(cells) == 2 + assert len(cells) == 1 assert cells[0]["NAME"] == "empty-fleet" - assert "(no instances)" in cells[1]["NAME"] def test_fleet_with_max_price(self): - """Test fleet with max_price configured.""" fleet = create_backend_fleet( name="priced-fleet", max_price=5.0, @@ -507,7 +480,6 @@ def test_fleet_with_max_price(self): assert cells[0]["PRICE"] == "$0..$5" def test_fleet_with_multiple_backends(self): - """Test fleet with multiple backends configured.""" fleet = create_backend_fleet( name="multi-backend", backends=[BackendType.AWS, BackendType.GCP, BackendType.AZURE], @@ -519,7 +491,6 @@ def test_fleet_with_multiple_backends(self): assert cells[0]["BACKEND"] == "aws, gcp, azure" def test_fleet_with_any_backend(self): - """Test fleet with no backends configured (any backend).""" fleet = create_backend_fleet( name="any-backend", backends=None, From 310958cdc11503e242cccc31e821697888861b5a Mon Sep 17 00:00:00 2001 From: peterschmidt85 Date: Sun, 1 Feb 2026 22:33:36 +0100 Subject: [PATCH 3/4] Linter and pyright --- src/dstack/_internal/cli/utils/fleet.py | 24 ++++++++++----------- src/tests/_internal/cli/utils/test_fleet.py | 6 +++--- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/dstack/_internal/cli/utils/fleet.py b/src/dstack/_internal/cli/utils/fleet.py index 19527b9db9..869aba9e61 100644 --- a/src/dstack/_internal/cli/utils/fleet.py +++ b/src/dstack/_internal/cli/utils/fleet.py @@ -151,10 +151,10 @@ def _format_price(price: Optional[float]) -> str: def _format_instance_gpu(instance: Instance) -> str: if instance.instance_type is None: return "-" - if ( - instance.backend == BackendType.REMOTE - and instance.status in [InstanceStatus.PENDING, InstanceStatus.PROVISIONING] - ): + if instance.backend == BackendType.REMOTE and instance.status in [ + InstanceStatus.PENDING, + InstanceStatus.PROVISIONING, + ]: return "-" return instance.instance_type.resources.pretty_format(gpu_only=True, include_spot=False) or "-" @@ -162,10 +162,10 @@ def _format_instance_gpu(instance: Instance) -> str: def _format_instance_resources(instance: Instance) -> str: if instance.instance_type is None: return "-" - if ( - instance.backend == BackendType.REMOTE - and instance.status in [InstanceStatus.PENDING, InstanceStatus.PROVISIONING] - ): + if instance.backend == BackendType.REMOTE and instance.status in [ + InstanceStatus.PENDING, + InstanceStatus.PROVISIONING, + ]: return "-" return instance.instance_type.resources.pretty_format(include_spot=False) @@ -196,9 +196,7 @@ def get_fleets_table( merged_profile = fleet.spec.merged_profile # Detect SSH fleet vs backend fleet - is_ssh_fleet = config.ssh_config is not None - - if is_ssh_fleet: + if config.ssh_config is not None: # SSH fleet: fixed number of hosts, no cloud billing nodes = str(len(config.ssh_config.hosts)) backend = "ssh" @@ -261,7 +259,9 @@ def get_fleets_table( instance.instance_type is not None and instance.instance_type.resources is not None ): - instance_spot = "spot" if instance.instance_type.resources.spot else "on-demand" + instance_spot = ( + "spot" if instance.instance_type.resources.spot else "on-demand" + ) instance_price = _format_price(instance.price) instance_row: Dict[Union[str, int], Any] = { diff --git a/src/tests/_internal/cli/utils/test_fleet.py b/src/tests/_internal/cli/utils/test_fleet.py index 5cf1bd1c28..1c1df4df22 100644 --- a/src/tests/_internal/cli/utils/test_fleet.py +++ b/src/tests/_internal/cli/utils/test_fleet.py @@ -3,7 +3,6 @@ from typing import List, Optional from uuid import uuid4 -import pytest from rich.table import Table from rich.text import Text @@ -19,7 +18,6 @@ SSHHostParams, SSHParams, ) -from dstack._internal.core.models.health import HealthStatus from dstack._internal.core.models.instances import ( Disk, Gpu, @@ -428,7 +426,9 @@ def test_fleet_status_colors(self): name="active", status=FleetStatus.ACTIVE, instances=[active_instance] ) - terminating_instance = create_test_instance(instance_num=0, status=InstanceStatus.TERMINATING) + terminating_instance = create_test_instance( + instance_num=0, status=InstanceStatus.TERMINATING + ) terminating_fleet = create_backend_fleet( name="terminating", status=FleetStatus.TERMINATING, instances=[terminating_instance] ) From b977081c8b907760a36edfbcbbc096204a4c1aa7 Mon Sep 17 00:00:00 2001 From: peterschmidt85 Date: Wed, 4 Feb 2026 12:06:34 +0100 Subject: [PATCH 4/4] PR review feedback --- src/dstack/_internal/cli/utils/fleet.py | 236 ++++++++++++------------ 1 file changed, 118 insertions(+), 118 deletions(-) diff --git a/src/dstack/_internal/cli/utils/fleet.py b/src/dstack/_internal/cli/utils/fleet.py index 869aba9e61..fdca4270ab 100644 --- a/src/dstack/_internal/cli/utils/fleet.py +++ b/src/dstack/_internal/cli/utils/fleet.py @@ -15,6 +15,124 @@ def print_fleets_table(fleets: List[Fleet], verbose: bool = False) -> None: console.print() +def get_fleets_table( + fleets: List[Fleet], verbose: bool = False, format_date: DateFormatter = pretty_date +) -> Table: + table = Table(box=None) + + # Columns + table.add_column("NAME", style="bold", no_wrap=True) + table.add_column("NODES") + if verbose: + table.add_column("RESOURCES") + else: + table.add_column("GPU") + table.add_column("SPOT") + table.add_column("BACKEND") + table.add_column("PRICE") + table.add_column("STATUS", no_wrap=True) + table.add_column("CREATED", no_wrap=True) + if verbose: + table.add_column("ERROR") + + for fleet in fleets: + # Fleet row + config = fleet.spec.configuration + merged_profile = fleet.spec.merged_profile + + # Detect SSH fleet vs backend fleet + if config.ssh_config is not None: + # SSH fleet: fixed number of hosts, no cloud billing + nodes = str(len(config.ssh_config.hosts)) + backend = "ssh" + spot_policy = "-" + max_price = "-" + else: + # Backend fleet: dynamic nodes, cloud billing + nodes = _format_nodes(config.nodes) + backend = _format_backends(config.backends) + spot_policy = "-" + if merged_profile and merged_profile.spot_policy: + spot_policy = merged_profile.spot_policy.value + # Format as "$0..$X.XX" range, or "-" if not set + if merged_profile and merged_profile.max_price is not None: + max_price = f"$0..{_format_price(merged_profile.max_price)}" + else: + max_price = "-" + + # In verbose mode, append placement to nodes if cluster + if verbose and config.placement and config.placement.value == "cluster": + nodes = f"{nodes} (cluster)" + + fleet_row: Dict[Union[str, int], Any] = { + "NAME": fleet.name, + "NODES": nodes, + "BACKEND": backend, + "PRICE": max_price, + "SPOT": spot_policy, + "STATUS": _format_fleet_status(fleet), + "CREATED": format_date(fleet.created_at), + } + + if verbose: + fleet_row["RESOURCES"] = config.resources.pretty_format() if config.resources else "-" + fleet_row["ERROR"] = "" + else: + fleet_row["GPU"] = _format_fleet_gpu(config.resources) + + add_row_from_dict(table, fleet_row) + + # Instance rows (indented) + for instance in fleet.instances: + # Check if this is an SSH instance + is_ssh_instance = instance.backend == BackendType.REMOTE + + # Format backend with region (and AZ in verbose mode) + if verbose and instance.availability_zone: + # In verbose mode, show AZ instead of region (AZ is more specific) + backend_with_region = _format_backend(instance.backend, instance.availability_zone) + else: + backend_with_region = _format_backend(instance.backend, instance.region) + + # Get spot info from instance resources (not applicable to SSH) + if is_ssh_instance: + instance_spot = "-" + instance_price = "-" + else: + instance_spot = "-" + if ( + instance.instance_type is not None + and instance.instance_type.resources is not None + ): + instance_spot = ( + "spot" if instance.instance_type.resources.spot else "on-demand" + ) + instance_price = _format_price(instance.price) + + instance_row: Dict[Union[str, int], Any] = { + "NAME": f" instance={instance.instance_num}", + "NODES": "", + "BACKEND": backend_with_region, + "PRICE": instance_price, + "SPOT": instance_spot, + "STATUS": _format_instance_status(instance), + "CREATED": format_date(instance.created), + } + + if verbose: + instance_row["RESOURCES"] = _format_instance_resources(instance) + error = "" + if instance.status == InstanceStatus.TERMINATED and instance.termination_reason: + error = instance.termination_reason + instance_row["ERROR"] = error + else: + instance_row["GPU"] = _format_instance_gpu(instance) + + add_row_from_dict(table, instance_row, style="secondary") + + return table + + def _format_nodes(nodes: Optional[FleetNodesSpec]) -> str: """Format nodes spec as '0..1', '3', '2..10', etc.""" if nodes is None: @@ -168,121 +286,3 @@ def _format_instance_resources(instance: Instance) -> str: ]: return "-" return instance.instance_type.resources.pretty_format(include_spot=False) - - -def get_fleets_table( - fleets: List[Fleet], verbose: bool = False, format_date: DateFormatter = pretty_date -) -> Table: - table = Table(box=None) - - # Columns - table.add_column("NAME", style="bold", no_wrap=True) - table.add_column("NODES") - if verbose: - table.add_column("RESOURCES") - else: - table.add_column("GPU") - table.add_column("SPOT") - table.add_column("BACKEND") - table.add_column("PRICE") - table.add_column("STATUS", no_wrap=True) - table.add_column("CREATED", no_wrap=True) - if verbose: - table.add_column("ERROR") - - for fleet in fleets: - # Fleet row - config = fleet.spec.configuration - merged_profile = fleet.spec.merged_profile - - # Detect SSH fleet vs backend fleet - if config.ssh_config is not None: - # SSH fleet: fixed number of hosts, no cloud billing - nodes = str(len(config.ssh_config.hosts)) - backend = "ssh" - spot_policy = "-" - max_price = "-" - else: - # Backend fleet: dynamic nodes, cloud billing - nodes = _format_nodes(config.nodes) - backend = _format_backends(config.backends) - spot_policy = "-" - if merged_profile and merged_profile.spot_policy: - spot_policy = merged_profile.spot_policy.value - # Format as "$0..$X.XX" range, or "-" if not set - if merged_profile and merged_profile.max_price is not None: - max_price = f"$0..{_format_price(merged_profile.max_price)}" - else: - max_price = "-" - - # In verbose mode, append placement to nodes if cluster - if verbose and config.placement and config.placement.value == "cluster": - nodes = f"{nodes} (cluster)" - - fleet_row: Dict[Union[str, int], Any] = { - "NAME": fleet.name, - "NODES": nodes, - "BACKEND": backend, - "PRICE": max_price, - "SPOT": spot_policy, - "STATUS": _format_fleet_status(fleet), - "CREATED": format_date(fleet.created_at), - } - - if verbose: - fleet_row["RESOURCES"] = config.resources.pretty_format() if config.resources else "-" - fleet_row["ERROR"] = "" - else: - fleet_row["GPU"] = _format_fleet_gpu(config.resources) - - add_row_from_dict(table, fleet_row) - - # Instance rows (indented) - for instance in fleet.instances: - # Check if this is an SSH instance - is_ssh_instance = instance.backend == BackendType.REMOTE - - # Format backend with region (and AZ in verbose mode) - if verbose and instance.availability_zone: - # In verbose mode, show AZ instead of region (AZ is more specific) - backend_with_region = _format_backend(instance.backend, instance.availability_zone) - else: - backend_with_region = _format_backend(instance.backend, instance.region) - - # Get spot info from instance resources (not applicable to SSH) - if is_ssh_instance: - instance_spot = "-" - instance_price = "-" - else: - instance_spot = "-" - if ( - instance.instance_type is not None - and instance.instance_type.resources is not None - ): - instance_spot = ( - "spot" if instance.instance_type.resources.spot else "on-demand" - ) - instance_price = _format_price(instance.price) - - instance_row: Dict[Union[str, int], Any] = { - "NAME": f" instance={instance.instance_num}", - "NODES": "", - "BACKEND": backend_with_region, - "PRICE": instance_price, - "SPOT": instance_spot, - "STATUS": _format_instance_status(instance), - "CREATED": format_date(instance.created), - } - - if verbose: - instance_row["RESOURCES"] = _format_instance_resources(instance) - error = "" - if instance.status == InstanceStatus.TERMINATED and instance.termination_reason: - error = instance.termination_reason - instance_row["ERROR"] = error - else: - instance_row["GPU"] = _format_instance_gpu(instance) - - add_row_from_dict(table, instance_row, style="secondary") - - return table