skypilot-org · aylei · Jan 24, 2026 · Dec 26, 2025 · Dec 26, 2025 · Dec 26, 2025
diff --git a/sky/provision/kubernetes/constants.py b/sky/provision/kubernetes/constants.py
@@ -21,5 +21,8 @@
 TAG_POD_INITIALIZED = 'skypilot-initialized'
 TAG_SKYPILOT_DEPLOYMENT_NAME = 'skypilot-deployment-name'
 
+# Default name of the primary workload container in SkyPilot Ray pods.
+RAY_NODE_CONTAINER_NAME = 'ray-node'
+
 # Pod phases that are not holding PVCs
 PVC_NOT_HOLD_POD_PHASES = ['Succeeded', 'Failed']
diff --git a/sky/provision/kubernetes/instance.py b/sky/provision/kubernetes/instance.py
@@ -895,7 +895,8 @@ def _pre_init_thread(new_node):
         pod_name = new_node.metadata.name
         logger.info(f'{"-"*20}Start: Pre-init in pod {pod_name!r} {"-"*20}')
         runner = command_runner.KubernetesCommandRunner(
-            ((namespace, context), pod_name))
+            ((namespace, context), pod_name),
+            container=k8s_constants.RAY_NODE_CONTAINER_NAME)
 
         # Run the combined pre-init command
         rc, stdout, _ = runner.run(pre_init_cmd,
@@ -966,10 +967,10 @@ def _create_namespaced_pod_with_retries(namespace: str, pod_spec: dict,
 
             # Remove the AppArmor annotation
             annotations = pod_spec.get('metadata', {}).get('annotations', {})
-            if ('container.apparmor.security.beta.kubernetes.io/ray-node'
-                    in annotations):
-                del annotations[
-                    'container.apparmor.security.beta.kubernetes.io/ray-node']
+            apparmor_key = ('container.apparmor.security.beta.kubernetes.io/'
+                            f'{k8s_constants.RAY_NODE_CONTAINER_NAME}')
+            if apparmor_key in annotations:
+                del annotations[apparmor_key]
                 pod_spec['metadata']['annotations'] = annotations
                 logger.info('AppArmor annotation removed from Pod spec.')
             else:
@@ -1057,8 +1058,6 @@ def _create_pods(region: str, cluster_name: str, cluster_name_on_cloud: str,
     if to_create_deployment:
         deployment_spec = pod_spec.pop('deployment_spec')
         pvc_spec = pod_spec.pop('pvc_spec')
-        assert len(pod_spec['spec']['containers']) == 1, (
-            'Only one container is supported for deployment')
 
     tags = ray_tag_filter(cluster_name_on_cloud)
 
@@ -1595,7 +1594,13 @@ def get_cluster_info(
             head_pod_name = pod_name
             head_spec = pod.spec
             assert head_spec is not None, pod
-            cpu_request = head_spec.containers[0].resources.requests['cpu']
+            primary_container = kubernetes_utils.get_pod_primary_container(pod)
+            resources = getattr(primary_container, 'resources', None)
+            requests = (getattr(resources, 'requests', None)
+                        if resources else None)
+            limits = (getattr(resources, 'limits', None) if resources else None)
+            cpu_request = ((requests or {}).get('cpu') or
+                           (limits or {}).get('cpu'))
 
     if cpu_request is None:
         raise RuntimeError(f'Pod {cluster_name_on_cloud}-head not found'
@@ -1609,7 +1614,8 @@ def get_cluster_info(
     get_k8s_ssh_user_cmd = 'echo "SKYPILOT_SSH_USER: $(whoami)"'
     assert head_pod_name is not None
     runner = command_runner.KubernetesCommandRunner(
-        ((namespace, context), head_pod_name))
+        ((namespace, context), head_pod_name),
+        container=k8s_constants.RAY_NODE_CONTAINER_NAME)
     rc, stdout, stderr = runner.run(get_k8s_ssh_user_cmd,
                                     require_outputs=True,
                                     separate_stderr=True,
@@ -2110,14 +2116,19 @@ def get_command_runners(
 
         node_list = [((namespace, context), pod_name)]
         head_runner = command_runner.KubernetesCommandRunner(
-            node_list[0], deployment=deployment, **credentials)
+            node_list[0],
+            deployment=deployment,
+            container=k8s_constants.RAY_NODE_CONTAINER_NAME,
+            **credentials)
         runners.append(head_runner)
 
     node_list = [((namespace, context), pod_name)
                  for pod_name in instances.keys()
                  if pod_name != cluster_info.head_instance_id]
     runners.extend(
         command_runner.KubernetesCommandRunner.make_runner_list(
-            node_list, **credentials))
+            node_list,
+            container=k8s_constants.RAY_NODE_CONTAINER_NAME,
+            **credentials))
 
     return runners
diff --git a/sky/provision/kubernetes/utils.py b/sky/provision/kubernetes/utils.py
@@ -3700,6 +3700,28 @@ def from_cluster(
         )
 
 
+def get_pod_primary_container(
+    pod: Any,
+    *,
+    primary_name: str = kubernetes_constants.RAY_NODE_CONTAINER_NAME,
+):
+    """Return the primary workload container for a SkyPilot pod.
+
+    Pods may include sidecars (e.g., log shippers). Kubernetes preserves the
+    ordering of the `containers` list as authored, but mutating webhooks can
+    inject additional containers. Callers should not rely on containers[0].
+    """
+    spec = getattr(pod, 'spec', None)
+    containers = getattr(spec, 'containers', None) if spec is not None else None
+    if not containers:
+        pod_name = getattr(getattr(pod, 'metadata', None), 'name', '<unknown>')
+        raise ValueError(f'Pod {pod_name!r} has no containers.')
+    for container in containers:
+        if getattr(container, 'name', None) == primary_name:
+            return container
+    return containers[0]
+
+
 def process_skypilot_pods(
     pods: List[Any],
     context: Optional[str] = None
@@ -3737,14 +3759,18 @@ def process_skypilot_pods(
                 start_time = pod.status.start_time.timestamp()
 
             # Parse resources
+            primary_container = get_pod_primary_container(pod)
+            resources = getattr(primary_container, 'resources', None)
+            requests = getattr(resources, 'requests',
+                               None) if resources else None
             cpu_request = parse_cpu_or_gpu_resource(
-                pod.spec.containers[0].resources.requests.get('cpu', '0'))
+                (requests.get('cpu', '0') if requests is not None else '0'))
             memory_request = parse_memory_resource(
-                pod.spec.containers[0].resources.requests.get('memory', '0'),
+                (requests.get('memory', '0') if requests is not None else '0'),
                 unit='G')
             gpu_count = parse_cpu_or_gpu_resource(
-                pod.spec.containers[0].resources.requests.get(
-                    get_gpu_resource_key(context), '0'))
+                (requests.get(get_gpu_resource_key(context), '0')
+                 if requests is not None else '0'))
             gpu_name = None
             if gpu_count > 0:
                 label_formatter, _ = (detect_gpu_label_formatter(context))

diff --git a/sky/utils/command_runner.py b/sky/utils/command_runner.py
@@ -1290,6 +1290,7 @@ def __init__(
         self,
         node: Tuple[Tuple[str, Optional[str]], str],
         deployment: Optional[str] = None,
+        container: Optional[str] = None,
         **kwargs,
     ):
         """Initialize KubernetesCommandRunner.
@@ -1301,11 +1302,19 @@ def __init__(
 
         Args:
             node: The namespace and pod_name of the remote machine.
+            deployment: If set, run commands against `deployment/<deployment>`
+                instead of `pod/<pod_name>`.
+            container: If set, run commands inside the given container name via
+                `kubectl exec -c <container>`. This is recommended for
+                multi-container pods (e.g., when sidecars are injected) to
+                ensure commands target the primary workload container (such as
+                `ray-node`).
         """
         del kwargs
         super().__init__(node)
         (self.namespace, self.context), self.pod_name = node
         self.deployment = deployment
+        self.container = container
 
     @property
     def node_id(self) -> str:
@@ -1427,6 +1436,8 @@ def run(
             kubectl_args += ['--kubeconfig', '/dev/null']
 
         kubectl_args += [self.kube_identifier]
+        if self.container is not None:
+            kubectl_args += ['-c', self.container]
 
         if ssh_mode == SshMode.LOGIN:
             assert isinstance(cmd, list), 'cmd must be a list for login mode.'
@@ -1530,7 +1541,9 @@ def rsync(
             log_path=log_path,
             stream_logs=stream_logs,
             max_retry=max_retry,
-            prefix_command=f'chmod +x {helper_path} && ',
+            prefix_command=(f'chmod +x {helper_path} && ' + (
+                '' if self.container is None else
+                f'SKYPILOT_K8S_EXEC_CONTAINER={shlex.quote(self.container)} ')),
             # rsync with `kubectl` as the rsh command will cause ~/xx parsed as
             # /~/xx, so we need to replace ~ with the remote home directory. We
             # only need to do this when ~ is at the beginning of the path.

diff --git a/sky/utils/command_runner.pyi b/sky/utils/command_runner.pyi
@@ -245,6 +245,7 @@ class KubernetesCommandRunner(CommandRunner):
         self,
         node: Tuple[Tuple[str, Optional[str]], str],
         deployment: Optional[str] = ...,
+        container: Optional[str] = ...,
         **kwargs,
     ) -> None:
         ...

diff --git a/sky/utils/config_utils.py b/sky/utils/config_utils.py
@@ -13,9 +13,9 @@
 # maps the field name to the patch merge key.
 # pylint: disable=line-too-long
 # Ref: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#podspec-v1-core
-# NOTE: field containers and imagePullSecrets are not included deliberately for
-# backward compatibility (we only support one container per pod now).
+# NOTE: field imagePullSecrets are not included deliberately for backward compatibility
 _PATCH_MERGE_KEYS = {
+    'containers': 'name',
     'initContainers': 'name',
     'ephemeralContainers': 'name',
     'volumes': 'name',
@@ -207,9 +207,15 @@ def merge_k8s_configs(
     Updates nested dictionaries instead of replacing them.
     If a list is encountered, it will be appended to the base_config list.
 
-    An exception is when the key is 'containers', in which case the
-    first container in the list will be fetched and merge_dict will be
-    called on it with the first container in the base_config list.
+    For fields with Kubernetes patch merge strategy (containers, volumes, env,
+    etc.), items are merged by their patch merge key (e.g., 'name' for
+    containers). If an item with the same key exists in base_config, it is
+    merged; otherwise, the new item is appended.
+
+    Special handling for 'containers': for backward compatibility, if a
+    container in the override does not have a 'name' field, it is merged
+    into the first container in the base config (legacy behavior). If the
+    container has a 'name' field, patch merge by name is used.
     """
     for key, value in override_config.items():
         (next_allowed_override_keys, next_disallowed_override_keys
@@ -222,12 +228,11 @@ def merge_k8s_configs(
         elif isinstance(value, list) and key in base_config:
             assert isinstance(base_config[key], list), \
                 f'Expected {key} to be a list, found {base_config[key]}'
-            if key in ['containers', 'imagePullSecrets']:
-                # If the key is 'containers' or 'imagePullSecrets, we take the
-                # first and only container/secret in the list and merge it, as
-                # we only support one container per pod.
+            if key == 'imagePullSecrets':
+                # For imagePullSecrets, merge the first item from override
+                # into the first item in base (legacy behavior).
                 assert len(value) == 1, \
-                    f'Expected only one container, found {value}'
+                    f'Expected only one imagePullSecret, found {value}'
                 merge_k8s_configs(base_config[key][0], value[0],
                                   next_allowed_override_keys,
                                   next_disallowed_override_keys)
@@ -238,6 +243,7 @@ def merge_k8s_configs(
                 for override_item in value:
                     override_item_name = override_item.get(patch_merge_key)
                     if override_item_name is not None:
+                        # Item has a name - use patch merge by name
                         existing_base_item = next(
                             (v for v in base_config[key]
                              if v.get(patch_merge_key) == override_item_name),
@@ -246,6 +252,12 @@ def merge_k8s_configs(
                             merge_k8s_configs(existing_base_item, override_item)
                         else:
                             base_config[key].append(override_item)
+                    elif key == 'containers' and base_config[key]:
+                        # Backward compatibility for containers: if no name is
+                        # specified, merge into the first container (index 0)
+                        merge_k8s_configs(base_config[key][0], override_item,
+                                          next_allowed_override_keys,
+                                          next_disallowed_override_keys)
                     else:
                         base_config[key].append(override_item)
             else:

diff --git a/sky/utils/kubernetes/rsync_helper.sh b/sky/utils/kubernetes/rsync_helper.sh
@@ -31,6 +31,8 @@ echo "namespace: $namespace" >&2
 context=$(echo $namespace_context | grep '+' >/dev/null && echo $namespace_context | cut -d+ -f2- || echo "")
 echo "context: $context" >&2
 context_lower=$(echo "$context" | tr '[:upper:]' '[:lower:]')
+container="${SKYPILOT_K8S_EXEC_CONTAINER:-ray-node}"
+echo "container: $container" >&2
 
 # Check if the resource is a pod or a deployment (or other type)
 if [[ "$pod" == *"/"* ]]; then
@@ -49,9 +51,9 @@ fi
 if [ -z "$context" ] || [ "$context_lower" = "none" ]; then
     # If context is none, it means we are using incluster auth. In this case,
     # we need to set KUBECONFIG to /dev/null to avoid using kubeconfig file.
-    kubectl_cmd_base="kubectl exec \"$resource_type/$resource_name\" -n \"$namespace\" --kubeconfig=/dev/null --"
+    kubectl_cmd_base="kubectl exec \"$resource_type/$resource_name\" -n \"$namespace\" -c \"$container\" --kubeconfig=/dev/null --"
 else
-    kubectl_cmd_base="kubectl exec \"$resource_type/$resource_name\" -n \"$namespace\" --context=\"$context\" --"
+    kubectl_cmd_base="kubectl exec \"$resource_type/$resource_name\" -n \"$namespace\" -c \"$container\" --context=\"$context\" --"
 fi
 
 # Execute command on remote pod, waiting for rsync to be available first.

diff --git a/tests/smoke_tests/test_cluster_job.py b/tests/smoke_tests/test_cluster_job.py
@@ -2748,6 +2748,62 @@ def test_kubernetes_pod_config_change_detection():
         os.unlink(task_yaml_2_path)
 
 
+@pytest.mark.kubernetes
+def test_kubernetes_pod_config_sidecar():
+    """Test Kubernetes pod_config with sidecar container injection.
+
+    This test verifies that SkyPilot correctly handles pods with multiple
+    containers (sidecars) by:
+    1. Launching a cluster with a sidecar container via pod_config
+    2. Verifying the pod has both ray-node and sidecar containers
+    3. Verifying sky exec commands run in the ray-node container
+    4. Verifying the sidecar container is running
+    """
+    name = smoke_tests_utils.get_cluster_name()
+    name_on_cloud = common_utils.make_cluster_name_on_cloud(
+        name, sky.Kubernetes.max_cluster_name_length())
+
+    template_str = pathlib.Path(
+        'tests/test_yamls/test_k8s_pod_config_sidecar.yaml.j2').read_text()
+    template = jinja2.Template(template_str)
+    task_yaml_content = template.render()
+
+    with tempfile.NamedTemporaryFile(suffix='.yaml', mode='w',
+                                     delete=False) as f:
+        f.write(task_yaml_content)
+        f.flush()
+        task_yaml_path = f.name
+
+        test = smoke_tests_utils.Test(
+            'kubernetes_pod_config_sidecar',
+            [
+                smoke_tests_utils.launch_cluster_for_cloud_cmd(
+                    'kubernetes', name),
+                # Launch SkyPilot cluster with sidecar
+                f'sky launch -y -c {name} --infra kubernetes '
+                f'{smoke_tests_utils.LOW_RESOURCE_ARG} {task_yaml_path}',
+                # Verify pod has 2 containers (ray-node and sidecar)
+                smoke_tests_utils.run_cloud_cmd_on_cluster(
+                    name,
+                    f'kubectl get pod -l skypilot-cluster-name={name_on_cloud} '
+                    '-o jsonpath=\'{.items[0].spec.containers[*].name}\' | '
+                    'grep -E "ray-node.*sidecar|sidecar.*ray-node"'),
+                # Verify sky exec runs in ray-node container
+                f'sky exec {name} "echo CONTAINER_CHECK: ray-node is working"',
+                # Verify sidecar is running
+                smoke_tests_utils.run_cloud_cmd_on_cluster(
+                    name,
+                    f'kubectl logs -l skypilot-cluster-name={name_on_cloud} '
+                    '-c sidecar --tail=5 | grep "sidecar running"'),
+            ],
+            f'sky down -y {name} && '
+            f'{smoke_tests_utils.down_cluster_for_cloud_cmd(name)}',
+            timeout=10 * 60,
+        )
+        smoke_tests_utils.run_one_test(test)
+        os.unlink(task_yaml_path)
+
+
 # ---------- Testing Kubernetes set_pod_resource_limits ----------
 @pytest.mark.kubernetes
 def test_kubernetes_set_pod_resource_limits():

diff --git a/tests/test_yamls/test_k8s_pod_config_sidecar.yaml.j2 b/tests/test_yamls/test_k8s_pod_config_sidecar.yaml.j2
@@ -0,0 +1,16 @@
+config:
+  kubernetes:
+    pod_config:
+      spec:
+        containers:
+        - name: sidecar
+          image: busybox:latest
+          command: ["sh", "-c", "while true; do echo 'sidecar running'; sleep 60; done"]
+          resources:
+            requests:
+              cpu: "100m"
+              memory: "64Mi"
+
+run: |
+  echo "Testing multi-container pod with sidecar"
+  echo "Hostname: $(hostname)"