Kueue values#

Values under hw-kueue configure Kueue, the job queueing controller, and the queues Hopsworks schedules jobs with.

Generated from the Hopsworks Helm chart 5.2.0-alpha-1791549041 (Hopsworks 5.2.0).

Deployed according to the first of these values that is set: global._hopsworks.kueue.enabled, global._hopsworks.full_platform.

Upstream charts

Values under hw-kueue.kueue go to kueue 0.12.2 from https://repo.hops.works/master/kueue/.

Only the values Hopsworks sets under hw-kueue.kueue are listed on this page. Any other value of the chart can be set there too; the link opens its documentation for the version Hopsworks pins.

General#

Defaults as YAML

hw-kueue:
  fullnameOverride: kueue
  hopsworkslib: {}
  imagePullPolicy: Always
  kueue:
    controllerManager:
      featureGates:
      - enabled: false
        name: TopologyAwareScheduling
      imagePullSecrets: []
      livenessProbe:
        failureThreshold: 3
        initialDelaySeconds: 15
        periodSeconds: 20
        successThreshold: 1
        timeoutSeconds: 1
      manager:
        containerSecurityContext:
          allowPrivilegeEscalation: false
          capabilities:
            drop:
            - ALL
        image:
          pullPolicy: Always
          repository: docker.hops.works/registry.k8s.io/kueue/kueue
        podAnnotations: {}
        podSecurityContext:
          runAsNonRoot: true
          seccompProfile:
            type: RuntimeDefault
        resources:
          limits:
            cpu: '2'
            memory: 512Mi
          requests:
            cpu: 500m
            memory: 512Mi
      podDisruptionBudget:
        enabled: false
        minAvailable: 1
      readinessProbe:
        failureThreshold: 3
        initialDelaySeconds: 5
        periodSeconds: 10
        successThreshold: 1
        timeoutSeconds: 1
      replicas: 1
      topologySpreadConstraints: []
    enableCertManager: false
    enableKueueViz: false
    enablePrometheus: false
    enableVisibilityAPF: false
    enableVisibilityServerAuth: true
    fullnameOverride: kueue
    kubernetesClusterDomain: cluster.local
    metrics:
      prometheusNamespace: monitoring
      serviceMonitor:
        tlsConfig:
          insecureSkipVerify: true
    metricsService:
      annotations: {}
      ports:
      - name: metrics
        port: 8443
        protocol: TCP
        targetPort: 8443
      type: ClusterIP
    nameOverride: kueue
    webhookService:
      ipDualStack:
        enabled: false
        ipFamilies:
        - IPv6
        - IPv4
        ipFamilyPolicy: PreferDualStack
      ports:
      - port: 443
        protocol: TCP
        targetPort: 9443
      type: ClusterIP
  nameOverride: kueue
  resourceFlavours:
  - annotations: {}
    labels: {}
    name: default-flavor
    spec:
      nodeLabels:
        cloud.provider.com/region: europe
      nodeTaints: {}
      tolerations: {}
      topologyName: default
  topologies:
  - levels:
    - nodeLabel: cloud.provider.com/region
    - nodeLabel: cloud.provider.com/zone
    - nodeLabel: kubernetes.io/hostname
    name: default

hw-kueue #

Type object, default {}. override hw-kueue values

hw-kueue.fullnameOverride #

Type string, default "kueue".

hw-kueue.hopsworkslib #

Type object, default {}. override hopsworkslib values

hw-kueue.imagePullPolicy #

Type string, default "Always".

hw-kueue.kueue #

Type object, passed to the kueue 0.12.2 chart, whose other values are documented there. override kueue values

Default

controllerManager:
  featureGates:
  - enabled: false
    name: TopologyAwareScheduling
  imagePullSecrets: []
  livenessProbe:
    failureThreshold: 3
    initialDelaySeconds: 15
    periodSeconds: 20
    successThreshold: 1
    timeoutSeconds: 1
  manager:
    containerSecurityContext:
      allowPrivilegeEscalation: false
      capabilities:
        drop:
        - ALL
    image:
      pullPolicy: Always
      repository: docker.hops.works/registry.k8s.io/kueue/kueue
    podAnnotations: {}
    podSecurityContext:
      runAsNonRoot: true
      seccompProfile:
        type: RuntimeDefault
    resources:
      limits:
        cpu: '2'
        memory: 512Mi
      requests:
        cpu: 500m
        memory: 512Mi
  podDisruptionBudget:
    enabled: false
    minAvailable: 1
  readinessProbe:
    failureThreshold: 3
    initialDelaySeconds: 5
    periodSeconds: 10
    successThreshold: 1
    timeoutSeconds: 1
  replicas: 1
  topologySpreadConstraints: []
enableCertManager: false
enableKueueViz: false
enablePrometheus: false
enableVisibilityAPF: false
enableVisibilityServerAuth: true
fullnameOverride: kueue
kubernetesClusterDomain: cluster.local
metrics:
  prometheusNamespace: monitoring
  serviceMonitor:
    tlsConfig:
      insecureSkipVerify: true
metricsService:
  annotations: {}
  ports:
  - name: metrics
    port: 8443
    protocol: TCP
    targetPort: 8443
  type: ClusterIP
nameOverride: kueue
webhookService:
  ipDualStack:
    enabled: false
    ipFamilies:
    - IPv6
    - IPv4
    ipFamilyPolicy: PreferDualStack
  ports:
  - port: 443
    protocol: TCP
    targetPort: 9443
  type: ClusterIP

hw-kueue.nameOverride #

Type string, default "kueue".

hw-kueue.resourceFlavours #

Type list. List of ResourceFlavors

Default

- annotations: {}
  labels: {}
  name: default-flavor
  spec:
    nodeLabels:
      cloud.provider.com/region: europe
    nodeTaints: {}
    tolerations: {}
    topologyName: default

hw-kueue.topologies[0].levels[0].nodeLabel #

Type string, default "cloud.provider.com/region".

hw-kueue.topologies[0].levels[1].nodeLabel #

Type string, default "cloud.provider.com/zone".

hw-kueue.topologies[0].levels[2].nodeLabel #

Type string, default "kubernetes.io/hostname".

hw-kueue.topologies[0].name #

Type string, default "default".

clusterQueues#

Defaults as YAML

hw-kueue:
  clusterQueues:
  - annotations: {}
    labels: {}
    name: other
    spec:
      cohort: cluster
      fairSharing:
        weight: '1'
      namespaceSelector: {}
      queueingStrategy: BestEffortFIFO
      resourceGroups:
      - coveredResources:
        - cpu
        - memory
        - pods
        - nvidia.com/gpu
        flavors:
        - name: default-flavor
          resources:
          - name: cpu
            nominalQuota: 0
          - name: memory
            nominalQuota: 0
          - name: pods
            nominalQuota: 0
          - name: nvidia.com/gpu
            nominalQuota: 0

hw-kueue.clusterQueues[0].annotations #: Type object, default {}.
hw-kueue.clusterQueues[0].labels #: Type object, default {}.
hw-kueue.clusterQueues[0].name #: Type string, default "other".
hw-kueue.clusterQueues[0].spec.cohort #: Type string, default "cluster".
hw-kueue.clusterQueues[0].spec.fairSharing.weight #: Type string, default "1".
hw-kueue.clusterQueues[0].spec.namespaceSelector #: Type object, default {}.
hw-kueue.clusterQueues[0].spec.queueingStrategy #: Type string, default "BestEffortFIFO".
hw-kueue.clusterQueues[0].spec.resourceGroups[0].coveredResources[0] #: Type string, default "cpu".
hw-kueue.clusterQueues[0].spec.resourceGroups[0].coveredResources[1] #: Type string, default "memory".
hw-kueue.clusterQueues[0].spec.resourceGroups[0].coveredResources[2] #: Type string, default "pods".
hw-kueue.clusterQueues[0].spec.resourceGroups[0].coveredResources[3] #: Type string, default "nvidia.com/gpu".
hw-kueue.clusterQueues[0].spec.resourceGroups[0].flavors[0].name #: Type string, default "default-flavor".
hw-kueue.clusterQueues[0].spec.resourceGroups[0].flavors[0].resources[0].name #: Type string, default "cpu".
hw-kueue.clusterQueues[0].spec.resourceGroups[0].flavors[0].resources[0].nominalQuota #: Type int, default 0.
hw-kueue.clusterQueues[0].spec.resourceGroups[0].flavors[0].resources[1].name #: Type string, default "memory".
hw-kueue.clusterQueues[0].spec.resourceGroups[0].flavors[0].resources[1].nominalQuota #: Type int, default 0.
hw-kueue.clusterQueues[0].spec.resourceGroups[0].flavors[0].resources[2].name #: Type string, default "pods".
hw-kueue.clusterQueues[0].spec.resourceGroups[0].flavors[0].resources[2].nominalQuota #: Type int, default 0.
hw-kueue.clusterQueues[0].spec.resourceGroups[0].flavors[0].resources[3].name #: Type string, default "nvidia.com/gpu".
hw-kueue.clusterQueues[0].spec.resourceGroups[0].flavors[0].resources[3].nominalQuota #: Type int, default 0.

cohorts#

Defaults as YAML

hw-kueue:
  cohorts:
  - annotations: {}
    labels: {}
    name: cluster
    spec:
      resourceGroups:
      - coveredResources:
        - cpu
        - memory
        - pods
        - nvidia.com/gpu
        flavors:
        - name: default-flavor
          resources:
          - name: cpu
            nominalQuota: 100
          - name: memory
            nominalQuota: 200Gi
          - name: pods
            nominalQuota: 100
          - name: nvidia.com/gpu
            nominalQuota: 50

hw-kueue.cohorts[0].annotations #: Type object, default {}.
hw-kueue.cohorts[0].labels #: Type object, default {}.
hw-kueue.cohorts[0].name #: Type string, default "cluster".
hw-kueue.cohorts[0].spec.resourceGroups[0].coveredResources[0] #: Type string, default "cpu".
hw-kueue.cohorts[0].spec.resourceGroups[0].coveredResources[1] #: Type string, default "memory".
hw-kueue.cohorts[0].spec.resourceGroups[0].coveredResources[2] #: Type string, default "pods".
hw-kueue.cohorts[0].spec.resourceGroups[0].coveredResources[3] #: Type string, default "nvidia.com/gpu".
hw-kueue.cohorts[0].spec.resourceGroups[0].flavors[0].name #: Type string, default "default-flavor".
hw-kueue.cohorts[0].spec.resourceGroups[0].flavors[0].resources[0].name #: Type string, default "cpu".
hw-kueue.cohorts[0].spec.resourceGroups[0].flavors[0].resources[0].nominalQuota #: Type int, default 100.
hw-kueue.cohorts[0].spec.resourceGroups[0].flavors[0].resources[1].name #: Type string, default "memory".
hw-kueue.cohorts[0].spec.resourceGroups[0].flavors[0].resources[1].nominalQuota #: Type string, default "200Gi".
hw-kueue.cohorts[0].spec.resourceGroups[0].flavors[0].resources[2].name #: Type string, default "pods".
hw-kueue.cohorts[0].spec.resourceGroups[0].flavors[0].resources[2].nominalQuota #: Type int, default 100.
hw-kueue.cohorts[0].spec.resourceGroups[0].flavors[0].resources[3].name #: Type string, default "nvidia.com/gpu".
hw-kueue.cohorts[0].spec.resourceGroups[0].flavors[0].resources[3].nominalQuota #: Type int, default 50.

installJob#

Defaults as YAML

hw-kueue:
  installJob:
    configMapMountPath: /mnt/kueue-resources
    configMapName: kueue-objects-install-job-resources
    name: kueue-obj
    resources:
      limits:
        cpu: 100m
        memory: 250Mi
      requests:
        cpu: 100m
        memory: 250Mi
    ttlSecondsAfterFinished: null

hw-kueue.installJob.configMapMountPath #: Type string, default "/mnt/kueue-resources".
hw-kueue.installJob.configMapName #: Type string, default "kueue-objects-install-job-resources".
hw-kueue.installJob.name #: Type string, default "kueue-obj". install job name
hw-kueue.installJob.resources.limits.cpu #: Type string, default "100m".
hw-kueue.installJob.resources.limits.memory #: Type string, default "250Mi".
hw-kueue.installJob.resources.requests.cpu #: Type string, default "100m".
hw-kueue.installJob.resources.requests.memory #: Type string, default "250Mi".
hw-kueue.installJob.ttlSecondsAfterFinished #: Type string, default nil. TTL in seconds for the install-kueue-object Job. Overrides global default.

managerConfig#

Defaults as YAML

hw-kueue:
  managerConfig:
    clientConnection:
      burst: 100
      qps: 50
    controller:
      groupKindConcurrency:
        ClusterQueue.kueue.x-k8s.io: 1
        Job.batch: 5
        LocalQueue.kueue.x-k8s.io: 1
        Pod: 5
        ResourceFlavor.kueue.x-k8s.io: 1
        Workload.kueue.x-k8s.io: 5
    excludedNamespaces:
    - kube-system
    - kueue-system
    - kyverno
    fairSharing:
      enable: true
      preemptionStrategies:
      - LessThanOrEqualToFinalShare
      - LessThanInitialShare
    health:
      healthProbeBindAddress: 8081
    integrations:
      frameworks:
      - batch/job
      - kubeflow.org/mpijob
      - ray.io/rayjob
      - ray.io/raycluster
      - jobset.x-k8s.io/jobset
      - kubeflow.org/paddlejob
      - kubeflow.org/pytorchjob
      - kubeflow.org/tfjob
      - kubeflow.org/xgboostjob
      - workload.codeflare.dev/appwrapper
      - pod
      - deployment
      - statefulset
    leaderElection:
      leaderElect: true
      resourceName: c1f6bfd2.kueue.x-k8s.io
    manageJobsWithoutQueueName: false
    metrics:
      bindAddress: 8443
    webhook:
      port: 9443

hw-kueue.managerConfig.clientConnection.burst #: Type int, default 100.
hw-kueue.managerConfig.clientConnection.qps #: Type int, default 50.
hw-kueue.managerConfig.controller.groupKindConcurrency."ClusterQueue.kueue.x-k8s.io" #: Type int, default 1.
hw-kueue.managerConfig.controller.groupKindConcurrency."Job.batch" #: Type int, default 5.
hw-kueue.managerConfig.controller.groupKindConcurrency."LocalQueue.kueue.x-k8s.io" #: Type int, default 1.
hw-kueue.managerConfig.controller.groupKindConcurrency."ResourceFlavor.kueue.x-k8s.io" #: Type int, default 1.
hw-kueue.managerConfig.controller.groupKindConcurrency."Workload.kueue.x-k8s.io" #: Type int, default 5.
hw-kueue.managerConfig.controller.groupKindConcurrency.Pod #: Type int, default 5.
hw-kueue.managerConfig.excludedNamespaces[0] #: Type string, default "kube-system".
hw-kueue.managerConfig.excludedNamespaces[1] #: Type string, default "kueue-system".
hw-kueue.managerConfig.excludedNamespaces[2] #: Type string, default "kyverno".
hw-kueue.managerConfig.fairSharing.enable #: Type bool, default true.
hw-kueue.managerConfig.fairSharing.preemptionStrategies[0] #: Type string, default "LessThanOrEqualToFinalShare".
hw-kueue.managerConfig.fairSharing.preemptionStrategies[1] #: Type string, default "LessThanInitialShare".
hw-kueue.managerConfig.health.healthProbeBindAddress #: Type int, default 8081.
hw-kueue.managerConfig.integrations.frameworks[0] #: Type string, default "batch/job".
hw-kueue.managerConfig.integrations.frameworks[10] #: Type string, default "pod".
hw-kueue.managerConfig.integrations.frameworks[11] #: Type string, default "deployment".
hw-kueue.managerConfig.integrations.frameworks[12] #: Type string, default "statefulset".
hw-kueue.managerConfig.integrations.frameworks[1] #: Type string, default "kubeflow.org/mpijob".
hw-kueue.managerConfig.integrations.frameworks[2] #: Type string, default "ray.io/rayjob".
hw-kueue.managerConfig.integrations.frameworks[3] #: Type string, default "ray.io/raycluster".
hw-kueue.managerConfig.integrations.frameworks[4] #: Type string, default "jobset.x-k8s.io/jobset".
hw-kueue.managerConfig.integrations.frameworks[5] #: Type string, default "kubeflow.org/paddlejob".
hw-kueue.managerConfig.integrations.frameworks[6] #: Type string, default "kubeflow.org/pytorchjob".
hw-kueue.managerConfig.integrations.frameworks[7] #: Type string, default "kubeflow.org/tfjob".
hw-kueue.managerConfig.integrations.frameworks[8] #: Type string, default "kubeflow.org/xgboostjob".
hw-kueue.managerConfig.integrations.frameworks[9] #: Type string, default "workload.codeflare.dev/appwrapper".
hw-kueue.managerConfig.leaderElection.leaderElect #: Type bool, default true.
hw-kueue.managerConfig.leaderElection.resourceName #: Type string, default "c1f6bfd2.kueue.x-k8s.io".
hw-kueue.managerConfig.manageJobsWithoutQueueName #: Type bool, default false.
hw-kueue.managerConfig.metrics.bindAddress #: Type int, default 8443.
hw-kueue.managerConfig.webhook.port #: Type int, default 9443.

rbac#

Defaults as YAML

hw-kueue:
  rbac:
    clusterqueue:
      enabled: true
    cohort:
      enabled: true
    localqueue:
      enabled: true
    resourceFlavors:
      enabled: true
    topologies:
      enabled: true
    workload:
      enabled: true

hw-kueue.rbac.clusterqueue.enabled #: Type bool, default true.
hw-kueue.rbac.cohort.enabled #: Type bool, default true.
hw-kueue.rbac.localqueue.enabled #: Type bool, default true.
hw-kueue.rbac.resourceFlavors.enabled #: Type bool, default true.
hw-kueue.rbac.topologies.enabled #: Type bool, default true.
hw-kueue.rbac.workload.enabled #: Type bool, default true.