From 44d1e246c7536c3a7a46554d75aabea97706c6e5 Mon Sep 17 00:00:00 2001 From: andweng Date: Sat, 21 Nov 2026 22:29:23 +0000 Subject: [PATCH] Kubernetes: add proxy_jump.image The jump pod always ran testcontainers/sshd:1.3.0. proxy_jump.image sets another image, e.g., one built on an approved base, from an internal registry, or with a mandated SSH crypto policy. It defaults to the current image and, like the other proxy_jump fields, is excluded from requests when unset so new clients keep working with older servers. Closes #4508 --- mkdocs/docs/concepts/backends.md | 16 +++++++++++- .../core/backends/kubernetes/compute.py | 6 +++-- .../core/backends/kubernetes/models.py | 13 ++++++++++ .../_internal/core/compatibility/backends.py | 2 ++ .../core/backends/kubernetes/test_compute.py | 25 +++++++++++++++++-- .../core/compatibility/test_backends.py | 12 +++++++++ 6 files changed, 69 insertions(+), 5 deletions(-) diff --git a/mkdocs/docs/concepts/backends.md b/mkdocs/docs/concepts/backends.md index df1d4820b..c22bb2b2b 100644 --- a/mkdocs/docs/concepts/backends.md +++ b/mkdocs/docs/concepts/backends.md @@ -1296,13 +1296,27 @@ projects: Each `tolerations` item is `key[=value][:effect]`, the `kubectl taint` format, so a taint added with `kubectl taint` can be copied as is. Without `=value`, any value is tolerated; without `:effect`, any effect. If `tolerations` isn't set, `dstack` tolerates the GPU node taints when every node is tainted, but not if `node_selector` is set. - !!! warning "Changing `cpu`, `memory`, `node_selector`, and `tolerations`" + By default, the jump pod runs `testcontainers/sshd`. To run your own image instead, e.g., one built on your organization's base image, scanned by your registry, or with your SSH policy, set `proxy_jump.image`: + + ```yaml + contexts: + - name: gpu-cluster-a + proxy_jump: + image: registry.example.com/sshd:1.0@sha256:... + ``` + + The image must run OpenSSH `sshd` at `/usr/sbin/sshd` as root and provide `/bin/sh` and `ssh-keygen`: `dstack` writes the authorized keys, regenerates the host keys with `ssh-keygen -A`, and starts `sshd`. Pin the image by digest. + + !!! warning "Changing `image`, `cpu`, `memory`, `node_selector`, and `tolerations`" New values don't apply to an existing jump pod. To apply them, update the jump pod's ReplicaSet in the context's namespace and delete the pod so that Kubernetes recreates it: ```shell # cpu and memory kubectl -n set resources replicaset dstack--ssh-jump-pod \ --requests=cpu=1,memory=2Gi --limits=cpu=1,memory=2Gi + # image + kubectl -n set image replicaset dstack--ssh-jump-pod \ + dstack--ssh-jump-pod-container=registry.example.com/sshd:1.0@sha256:... # node_selector and tolerations: edit spec.template.spec.affinity and .tolerations kubectl -n edit replicaset dstack--ssh-jump-pod # Recreate the pod diff --git a/src/dstack/_internal/core/backends/kubernetes/compute.py b/src/dstack/_internal/core/backends/kubernetes/compute.py index a0105291d..d9d125505 100644 --- a/src/dstack/_internal/core/backends/kubernetes/compute.py +++ b/src/dstack/_internal/core/backends/kubernetes/compute.py @@ -46,6 +46,7 @@ from dstack._internal.core.backends.kubernetes.features import resource_groups from dstack._internal.core.backends.kubernetes.models import ( JUMP_POD_DEFAULT_CPU, + JUMP_POD_DEFAULT_IMAGE, JUMP_POD_DEFAULT_MEMORY, KubernetesConfig, ) @@ -137,7 +138,6 @@ logger = get_logger(__name__) -JUMP_POD_IMAGE = "testcontainers/sshd:1.3.0@sha256:c50c0f59554dcdb2d9e5e705112144428ae9d04ac0af6322b365a18e24213a6a" JUMP_POD_SSH_PORT = 22 JUMP_POD_USER = "root" # Under disk pressure, the kubelet first evicts pods whose disk usage exceeds their @@ -271,6 +271,7 @@ def run_job( jump_pod_service_name=jump_pod_service_name, jump_pod_port=cluster.proxy_jump.port, authorized_keys=project_authorized_keys, + image=cluster.proxy_jump.image or JUMP_POD_DEFAULT_IMAGE, cpu_request_limit=cluster.proxy_jump.cpu or JUMP_POD_DEFAULT_CPU, memory_request_limit=cluster.proxy_jump.memory or JUMP_POD_DEFAULT_MEMORY, node_selector=cluster.proxy_jump.node_selector, @@ -949,6 +950,7 @@ def _create_jump_pod_service_if_not_exists( jump_pod_service_name: str, jump_pod_port: Optional[int], authorized_keys: list[str], + image: str, cpu_request_limit: str, memory_request_limit: str, node_selector: Optional[list[str]], @@ -1064,7 +1066,7 @@ def _create_jump_pod_service_if_not_exists( containers=[ client.V1Container( name=f"{jump_pod_name}-container", - image=JUMP_POD_IMAGE, + image=image, command=["/bin/sh"], args=["-c", " && ".join(commands)], ports=[ diff --git a/src/dstack/_internal/core/backends/kubernetes/models.py b/src/dstack/_internal/core/backends/kubernetes/models.py index 15603c24e..8783d8675 100644 --- a/src/dstack/_internal/core/backends/kubernetes/models.py +++ b/src/dstack/_internal/core/backends/kubernetes/models.py @@ -8,6 +8,9 @@ # Measured with the jump pod image (OpenSSH 9.9). The container takes ~9 MiB plus ~1.6 MiB per # tunneled SSH session (two sshd-session processes), so 1Gi fits ~600 sessions. Each running job # holds one or more sessions per server replica. At 500m, it tunnels ~130-180 MiB/s. +# The image must provide `/bin/sh`, `ssh-keygen`, and OpenSSH `sshd` at `/usr/sbin/sshd`, run as +# root, and generate host keys with `ssh-keygen -A` (see `_get_jump_pod_commands()`). +JUMP_POD_DEFAULT_IMAGE = "testcontainers/sshd:1.3.0@sha256:c50c0f59554dcdb2d9e5e705112144428ae9d04ac0af6322b365a18e24213a6a" JUMP_POD_DEFAULT_CPU = "500m" JUMP_POD_DEFAULT_MEMORY = "1Gi" @@ -19,6 +22,16 @@ class KubernetesProxyJumpConfig(CoreModel): port: Annotated[ Optional[int], Field(description="Any port accessible outside of the cluster") ] = None + image: Annotated[ + Optional[str], + Field( + description=( + "The jump pod's image. It must run OpenSSH `sshd` at `/usr/sbin/sshd` as root" + " and provide `/bin/sh` and `ssh-keygen`." + f" Defaults to `{JUMP_POD_DEFAULT_IMAGE}`" + ) + ), + ] = None cpu: Annotated[ Optional[str], Field( diff --git a/src/dstack/_internal/core/compatibility/backends.py b/src/dstack/_internal/core/compatibility/backends.py index 72e58d8e5..9d7eb6302 100644 --- a/src/dstack/_internal/core/compatibility/backends.py +++ b/src/dstack/_internal/core/compatibility/backends.py @@ -39,6 +39,8 @@ def _get_kubernetes_proxy_jump_excludes( ) -> IncludeExcludeStrDictType: # A field is excluded only if it's unset in all `proxy_jumps` excludes: IncludeExcludeStrDictType = {} + if all(proxy_jump.image is None for proxy_jump in proxy_jumps): + excludes["image"] = True if all(proxy_jump.cpu is None for proxy_jump in proxy_jumps): excludes["cpu"] = True if all(proxy_jump.memory is None for proxy_jump in proxy_jumps): diff --git a/src/tests/_internal/core/backends/kubernetes/test_compute.py b/src/tests/_internal/core/backends/kubernetes/test_compute.py index 071eeff95..7973803a7 100644 --- a/src/tests/_internal/core/backends/kubernetes/test_compute.py +++ b/src/tests/_internal/core/backends/kubernetes/test_compute.py @@ -18,7 +18,11 @@ _get_pod_spec_parameters_for_gpu, _KubernetesSkipOfferCache, ) -from dstack._internal.core.backends.kubernetes.models import KubeconfigConfig, KubernetesConfig +from dstack._internal.core.backends.kubernetes.models import ( + JUMP_POD_DEFAULT_IMAGE, + KubeconfigConfig, + KubernetesConfig, +) from dstack._internal.core.backends.kubernetes.resources import ( KubernetesOfferBackendData, build_base_labels, @@ -231,6 +235,7 @@ def call( api: Mock, apps_api: Mock, jump_pod_port: Optional[int] = None, + image: str = JUMP_POD_DEFAULT_IMAGE, cpu: str = "500m", memory: str = "1Gi", node_selector: Optional[list[str]] = None, @@ -245,6 +250,7 @@ def call( jump_pod_service_name=JUMP_POD_SERVICE_NAME, jump_pod_port=jump_pod_port, authorized_keys=["ssh-ed25519 AAAA project"], + image=image, cpu_request_limit=cpu, memory_request_limit=memory, node_selector=node_selector, @@ -257,7 +263,12 @@ def get_pod_spec(self, apps_api: Mock) -> client.V1PodSpec: return replica_set.spec.template.spec def assert_replica_set_created( - self, apps_api: Mock, *, cpu: str = "500m", memory: str = "1Gi" + self, + apps_api: Mock, + *, + image: str = JUMP_POD_DEFAULT_IMAGE, + cpu: str = "500m", + memory: str = "1Gi", ) -> None: apps_api.create_namespaced_replica_set.assert_called_once() replica_set = apps_api.create_namespaced_replica_set.call_args.kwargs["body"] @@ -265,6 +276,7 @@ def assert_replica_set_created( assert replica_set.spec.replicas == 1 assert replica_set.spec.selector.match_labels == JUMP_POD_SELECTOR assert replica_set.spec.template.metadata.labels == JUMP_POD_LABELS + assert replica_set.spec.template.spec.containers[0].image == image resources = replica_set.spec.template.spec.containers[0].resources # Equal CPU and memory requests and limits for the Guaranteed QoS class, and # an ephemeral-storage request without a limit @@ -303,6 +315,15 @@ def test_creates_replica_set_and_service_if_not_found(self): self.assert_replica_set_created(apps_api) self.assert_service_created(api, selector=JUMP_POD_SELECTOR, node_port=None) + def test_sets_jump_pod_image(self): + api, apps_api = self.make_apis() + + self.call(api, apps_api, image="registry.example.com/sshd:1@sha256:" + "0" * 64) + + self.assert_replica_set_created( + apps_api, image="registry.example.com/sshd:1@sha256:" + "0" * 64 + ) + def test_sets_jump_pod_resources(self): api, apps_api = self.make_apis() diff --git a/src/tests/_internal/core/compatibility/test_backends.py b/src/tests/_internal/core/compatibility/test_backends.py index 5ce7038ee..b33614336 100644 --- a/src/tests/_internal/core/compatibility/test_backends.py +++ b/src/tests/_internal/core/compatibility/test_backends.py @@ -65,6 +65,18 @@ def test_keeps_set_kubernetes_proxy_jump_resources(self): assert _request_body(config)["proxy_jump"] == {"hostname": None, "port": None, "cpu": "1"} + def test_keeps_set_kubernetes_proxy_jump_image(self): + config = KubernetesBackendConfigWithCreds( + kubeconfig=KubeconfigConfig(data="mocked"), + proxy_jump=KubernetesProxyJumpConfig(image="registry.example.com/sshd:1"), + ) + + assert _request_body(config)["proxy_jump"] == { + "hostname": None, + "port": None, + "image": "registry.example.com/sshd:1", + } + def test_keeps_set_kubernetes_proxy_jump_placement(self): config = KubernetesBackendConfigWithCreds( kubeconfig=KubeconfigConfig(data="mocked"),