apiVersion: "trainer.kubeflow.org/v1alpha1"
kind: "ClusterTrainingRuntime"
metadata:
  annotations:
    {}
  labels:
    trainer.kubeflow.org/framework: "torch"
  name: "torch-distributed"
spec:
  mlPolicy:
    numNodes: 4
    torch:
      {}
  template:
    spec:
      replicatedJobs:
        -
          name: "node"
          template:
            metadata:
              labels:
                trainer.kubeflow.org/trainjob-ancestor-step: "trainer"
            spec:
              template:
                spec:
                  containers:
                    -
                      image: "pytorch/pytorch:2.11.0-cuda12.8-cudnn9-runtime@sha256:eee11b3b3872a8c838e35ef48f08b2d5def2080902c7f666831310ca1a0ef2be"
                      name: "node"
                      env:
                        -
                          name: "AI_API_KEY"
                          valueFrom:
                            secretKeyRef:
                              key: "api-key"
                              name: "training-provider-credentials"
                  nodeSelector:
                    nvidia.com/gpu.present: "true"
                  tolerations:
                    -
                      operator: "Exists"