1 # ============LICENSE_START=======================================================
2 # Copyright (C) 2021 The Nordix Foundation. All rights reserved.
3 # ================================================================================
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at
8 # http://www.apache.org/licenses/LICENSE-2.0
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
16 # SPDX-License-Identifier: Apache-2.0
17 # ============LICENSE_END=========================================================
19 #################################################################################################################
20 # The deployment for the rook operator
21 # Contains the common settings for most Kubernetes deployments.
22 # For example, to create the rook-ceph cluster:
23 # kubectl create -f crds.yaml -f common.yaml -f operator.yaml
24 # kubectl create -f cluster.yaml
26 # Also see other operator sample files for variations of operator.yaml:
27 # - operator-openshift.yaml: Common settings for running in OpenShift
28 ###############################################################################################################
30 # Rook Ceph Operator Config ConfigMap
31 # Use this ConfigMap to override Rook-Ceph Operator configurations.
32 # NOTE! Precedence will be given to this config if the same Env Var config also exists in the
33 # Operator Deployment.
34 # To move a configuration(s) from the Operator Deployment to this ConfigMap, add the config
35 # here. It is recommended to then remove it from the Deployment to eliminate any future confusion.
40 name: rook-ceph-operator-config
41 # should be in the namespace of the operator
42 namespace: "{{ rook_namespace }}" # namespace:operator
44 # Enable the CSI driver.
45 # To run the non-default version of the CSI driver, see the override-able image properties in operator.yaml
46 ROOK_CSI_ENABLE_CEPHFS: "true"
47 # Enable the default version of the CSI RBD driver. To start another version of the CSI driver, see image properties below.
48 ROOK_CSI_ENABLE_RBD: "true"
49 ROOK_CSI_ENABLE_GRPC_METRICS: "false"
51 # Set logging level for csi containers.
52 # Supported values from 0 to 5. 0 for general useful logs, 5 for trace level verbosity.
55 # OMAP generator will generate the omap mapping between the PV name and the RBD image.
56 # CSI_ENABLE_OMAP_GENERATOR need to be enabled when we are using rbd mirroring feature.
57 # By default OMAP generator sidecar is deployed with CSI provisioner pod, to disable
59 # CSI_ENABLE_OMAP_GENERATOR: "false"
61 # set to false to disable deployment of snapshotter container in CephFS provisioner pod.
62 CSI_ENABLE_CEPHFS_SNAPSHOTTER: "true"
64 # set to false to disable deployment of snapshotter container in RBD provisioner pod.
65 CSI_ENABLE_RBD_SNAPSHOTTER: "true"
67 # Enable cephfs kernel driver instead of ceph-fuse.
68 # If you disable the kernel client, your application may be disrupted during upgrade.
69 # See the upgrade guide: https://rook.io/docs/rook/master/ceph-upgrade.html
70 # NOTE! cephfs quota is not supported in kernel version < 4.17
71 CSI_FORCE_CEPHFS_KERNEL_CLIENT: "true"
73 # (Optional) policy for modifying a volume's ownership or permissions when the RBD PVC is being mounted.
74 # supported values are documented at https://kubernetes-csi.github.io/docs/support-fsgroup.html
75 CSI_RBD_FSGROUPPOLICY: "ReadWriteOnceWithFSType"
77 # (Optional) policy for modifying a volume's ownership or permissions when the CephFS PVC is being mounted.
78 # supported values are documented at https://kubernetes-csi.github.io/docs/support-fsgroup.html
79 CSI_CEPHFS_FSGROUPPOLICY: "ReadWriteOnceWithFSType"
81 # (Optional) Allow starting unsupported ceph-csi image
82 ROOK_CSI_ALLOW_UNSUPPORTED_VERSION: "false"
83 # The default version of CSI supported by Rook will be started. To change the version
84 # of the CSI driver to something other than what is officially supported, change
85 # these images to the desired release of the CSI driver.
86 # ROOK_CSI_CEPH_IMAGE: "quay.io/cephcsi/cephcsi:v3.2.0"
87 ROOK_CSI_CEPH_IMAGE: "{{ cephcsi_repository }}:{{ cephcsi_version }}"
88 # ROOK_CSI_REGISTRAR_IMAGE: "k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.0.1"
89 ROOK_CSI_REGISTRAR_IMAGE: "{{ csi_node_driver_registrar_repository }}:{{ csi_node_driver_registrar_version }}"
90 # ROOK_CSI_RESIZER_IMAGE: "k8s.gcr.io/sig-storage/csi-resizer:v1.0.0"
91 ROOK_CSI_RESIZER_IMAGE: "{{ csi_resizer_repository }}:{{ csi_resizer_version }}"
92 # ROOK_CSI_PROVISIONER_IMAGE: "k8s.gcr.io/sig-storage/csi-provisioner:v2.0.0"
93 ROOK_CSI_PROVISIONER_IMAGE: "{{ csi_provisioner_repository }}:{{ csi_provisioner_version }}"
94 # ROOK_CSI_SNAPSHOTTER_IMAGE: "k8s.gcr.io/sig-storage/csi-snapshotter:v3.0.0"
95 ROOK_CSI_SNAPSHOTTER_IMAGE: "{{ csi_snapshotter_repository }}:{{ csi_snapshotter_version }}"
96 # ROOK_CSI_ATTACHER_IMAGE: "k8s.gcr.io/sig-storage/csi-attacher:v3.0.0"
97 ROOK_CSI_ATTACHER_IMAGE: "{{ csi_attacher_repository }}:{{ csi_attacher_version }}"
99 # (Optional) set user created priorityclassName for csi plugin pods.
100 # CSI_PLUGIN_PRIORITY_CLASSNAME: "system-node-critical"
102 # (Optional) set user created priorityclassName for csi provisioner pods.
103 # CSI_PROVISIONER_PRIORITY_CLASSNAME: "system-cluster-critical"
105 # CSI CephFS plugin daemonset update strategy, supported values are OnDelete and RollingUpdate.
106 # Default value is RollingUpdate.
107 # CSI_CEPHFS_PLUGIN_UPDATE_STRATEGY: "OnDelete"
108 # CSI RBD plugin daemonset update strategy, supported values are OnDelete and RollingUpdate.
109 # Default value is RollingUpdate.
110 # CSI_RBD_PLUGIN_UPDATE_STRATEGY: "OnDelete"
112 # kubelet directory path, if kubelet configured to use other than /var/lib/kubelet path.
113 # ROOK_CSI_KUBELET_DIR_PATH: "/var/lib/kubelet"
115 # Labels to add to the CSI CephFS Deployments and DaemonSets Pods.
116 # ROOK_CSI_CEPHFS_POD_LABELS: "key1=value1,key2=value2"
117 # Labels to add to the CSI RBD Deployments and DaemonSets Pods.
118 # ROOK_CSI_RBD_POD_LABELS: "key1=value1,key2=value2"
120 # (Optional) Ceph Provisioner NodeAffinity.
121 CSI_PROVISIONER_NODE_AFFINITY: "{{ rook_storage_label }}=true"
122 # (Optional) CEPH CSI provisioner tolerations list. Put here list of taints you want to tolerate in YAML format.
123 # CSI provisioner would be best to start on the same nodes as other ceph daemons.
124 CSI_PROVISIONER_TOLERATIONS: |
125 - key: "{{ rook_storage_label }}"
127 - key: node-role.kubernetes.io/master
129 # - effect: NoSchedule
130 # key: node-role.kubernetes.io/controlplane
132 # - effect: NoExecute
133 # key: node-role.kubernetes.io/etcd
135 # (Optional) Ceph CSI plugin NodeAffinity.
136 # CSI_PLUGIN_NODE_AFFINITY: "role=storage-node; storage=rook, ceph"
137 CSI_PLUGIN_NODE_AFFINITY: "{{ rook_storage_label }}=false"
138 # (Optional) CEPH CSI plugin tolerations list. Put here list of taints you want to tolerate in YAML format.
139 # CSI plugins need to be started on all the nodes where the clients need to mount the storage.
140 # CSI_PLUGIN_TOLERATIONS: |
141 # - effect: NoSchedule
142 # key: node-role.kubernetes.io/controlplane
144 # - effect: NoExecute
145 # key: node-role.kubernetes.io/etcd
148 # (Optional) CEPH CSI RBD provisioner resource requirement list, Put here list of resource
149 # requests and limits you want to apply for provisioner pod
150 # CSI_RBD_PROVISIONER_RESOURCE: |
151 # - name : csi-provisioner
159 # - name : csi-resizer
167 # - name : csi-attacher
175 # - name : csi-snapshotter
183 # - name : csi-rbdplugin
191 # - name : liveness-prometheus
199 # (Optional) CEPH CSI RBD plugin resource requirement list, Put here list of resource
200 # requests and limits you want to apply for plugin pod
201 # CSI_RBD_PLUGIN_RESOURCE: |
202 # - name : driver-registrar
210 # - name : csi-rbdplugin
218 # - name : liveness-prometheus
226 # (Optional) CEPH CSI CephFS provisioner resource requirement list, Put here list of resource
227 # requests and limits you want to apply for provisioner pod
228 # CSI_CEPHFS_PROVISIONER_RESOURCE: |
229 # - name : csi-provisioner
237 # - name : csi-resizer
245 # - name : csi-attacher
253 # - name : csi-cephfsplugin
261 # - name : liveness-prometheus
269 # (Optional) CEPH CSI CephFS plugin resource requirement list, Put here list of resource
270 # requests and limits you want to apply for plugin pod
271 # CSI_CEPHFS_PLUGIN_RESOURCE: |
272 # - name : driver-registrar
280 # - name : csi-cephfsplugin
288 # - name : liveness-prometheus
297 # Configure CSI CSI Ceph FS grpc and liveness metrics port
298 # CSI_CEPHFS_GRPC_METRICS_PORT: "9091"
299 # CSI_CEPHFS_LIVENESS_METRICS_PORT: "9081"
300 # Configure CSI RBD grpc and liveness metrics port
301 # CSI_RBD_GRPC_METRICS_PORT: "9090"
302 # CSI_RBD_LIVENESS_METRICS_PORT: "9080"
304 # Whether the OBC provisioner should watch on the operator namespace or not, if not the namespace of the cluster will be used
305 ROOK_OBC_WATCH_OPERATOR_NAMESPACE: "true"
307 # (Optional) Admission controller NodeAffinity.
308 # ADMISSION_CONTROLLER_NODE_AFFINITY: "role=storage-node; storage=rook, ceph"
309 # (Optional) Admission controller tolerations list. Put here list of taints you want to tolerate in YAML format.
310 # Admission controller would be best to start on the same nodes as other ceph daemons.
311 # ADMISSION_CONTROLLER_TOLERATIONS: |
312 # - effect: NoSchedule
313 # key: node-role.kubernetes.io/controlplane
315 # - effect: NoExecute
316 # key: node-role.kubernetes.io/etcd
319 # Some other config values need to be set in this ConfigMap
324 name: rook-config-override
325 namespace: "{{ rook_namespace }}" # namespace:cluster
329 osd_pool_default_size = {{ rook_ceph_osd_pool_default_size }}
330 osd_pool_default_min_size = {{ rook_ceph_osd_pool_default_min_size }}
331 mon_warn_on_pool_no_redundancy = {{ rook_ceph_mon_warn_on_no_pool_redundancy }}
333 # OLM: BEGIN OPERATOR DEPLOYMENT
337 name: rook-ceph-operator
338 namespace: "{{ rook_namespace }}" # namespace:operator
341 storage-backend: ceph
345 app: rook-ceph-operator
350 app: rook-ceph-operator
352 serviceAccountName: rook-ceph-system
354 - name: rook-ceph-operator
355 image: {{ rook_repository }}:{{ rook_version }}
356 args: ["ceph", "operator"]
358 - mountPath: /var/lib/rook
360 - mountPath: /etc/ceph
361 name: default-config-dir
363 # If the operator should only watch for cluster CRDs in the same namespace, set this to "true".
364 # If this is not set to true, the operator will watch for cluster CRDs in all namespaces.
365 - name: ROOK_CURRENT_NAMESPACE_ONLY
367 # To disable RBAC, uncomment the following:
368 # - name: RBAC_ENABLED
370 # Rook Agent toleration. Will tolerate all taints with all keys.
371 # Choose between NoSchedule, PreferNoSchedule and NoExecute:
372 # - name: AGENT_TOLERATION
373 # value: "NoSchedule"
374 # (Optional) Rook Agent toleration key. Set this to the key of the taint you want to tolerate
375 # - name: AGENT_TOLERATION_KEY
376 # value: "<KeyOfTheTaintToTolerate>"
377 # (Optional) Rook Agent tolerations list. Put here list of taints you want to tolerate in YAML format.
378 - name: AGENT_TOLERATIONS
380 - key: "{{ rook_storage_label }}"
382 # - effect: NoSchedule
383 # key: node-role.kubernetes.io/controlplane
385 # - effect: NoExecute
386 # key: node-role.kubernetes.io/etcd
388 # (Optional) Rook Agent priority class name to set on the pod(s)
389 # - name: AGENT_PRIORITY_CLASS_NAME
390 # value: "<PriorityClassName>"
391 # (Optional) Rook Agent NodeAffinity.
392 - name: AGENT_NODE_AFFINITY
393 value: "{{ rook_storage_label }}=true"
394 # (Optional) Rook Agent mount security mode. Can by `Any` or `Restricted`.
395 # `Any` uses Ceph admin credentials by default/fallback.
396 # For using `Restricted` you must have a Ceph secret in each namespace storage should be consumed from and
397 # set `mountUser` to the Ceph user, `mountSecret` to the Kubernetes secret name.
398 # to the namespace in which the `mountSecret` Kubernetes secret namespace.
399 # - name: AGENT_MOUNT_SECURITY_MODE
401 # Set the path where the Rook agent can find the flex volumes
402 # - name: FLEXVOLUME_DIR_PATH
403 # value: "<PathToFlexVolumes>"
404 # Set the path where kernel modules can be found
405 # - name: LIB_MODULES_DIR_PATH
406 # value: "<PathToLibModules>"
407 # Mount any extra directories into the agent container
408 # - name: AGENT_MOUNTS
409 # value: "somemount=/host/path:/container/path,someothermount=/host/path2:/container/path2"
410 # Rook Discover toleration. Will tolerate all taints with all keys.
411 # Choose between NoSchedule, PreferNoSchedule and NoExecute:
412 # - name: DISCOVER_TOLERATION
413 # value: "NoSchedule"
414 # (Optional) Rook Discover toleration key. Set this to the key of the taint you want to tolerate
415 # - name: DISCOVER_TOLERATION_KEY
416 # value: "<KeyOfTheTaintToTolerate>"
417 # (Optional) Rook Discover tolerations list. Put here list of taints you want to tolerate in YAML format.
418 - name: DISCOVER_TOLERATIONS
420 - key: "{{ rook_storage_label }}"
422 # - effect: NoSchedule
423 # key: node-role.kubernetes.io/controlplane
425 # - effect: NoExecute
426 # key: node-role.kubernetes.io/etcd
428 # (Optional) Rook Discover priority class name to set on the pod(s)
429 # - name: DISCOVER_PRIORITY_CLASS_NAME
430 # value: "<PriorityClassName>"
431 # (Optional) Discover Agent NodeAffinity.
432 - name: DISCOVER_AGENT_NODE_AFFINITY
433 value: "{{ rook_storage_label }}=true"
434 # (Optional) Discover Agent Pod Labels.
435 # - name: DISCOVER_AGENT_POD_LABELS
436 # value: "key1=value1,key2=value2"
437 # Allow rook to create multiple file systems. Note: This is considered
438 # an experimental feature in Ceph as described at
439 # http://docs.ceph.com/docs/master/cephfs/experimental-features/#multiple-filesystems-within-a-ceph-cluster
440 # which might cause mons to crash as seen in https://github.com/rook/rook/issues/1027
441 - name: ROOK_ALLOW_MULTIPLE_FILESYSTEMS
444 # The logging level for the operator: INFO | DEBUG
445 - name: ROOK_LOG_LEVEL
448 # The duration between discovering devices in the rook-discover daemonset.
449 - name: ROOK_DISCOVER_DEVICES_INTERVAL
452 # Whether to start pods as privileged that mount a host path, which includes the Ceph mon and osd pods.
453 # Set this to true if SELinux is enabled (e.g. OpenShift) to workaround the anyuid issues.
454 # For more details see https://github.com/rook/rook/issues/1314#issuecomment-355799641
455 - name: ROOK_HOSTPATH_REQUIRES_PRIVILEGED
458 # In some situations SELinux relabelling breaks (times out) on large filesystems, and doesn't work with cephfs ReadWriteMany volumes (last relabel wins).
459 # Disable it here if you have similar issues.
460 # For more details see https://github.com/rook/rook/issues/2417
461 - name: ROOK_ENABLE_SELINUX_RELABELING
464 # In large volumes it will take some time to chown all the files. Disable it here if you have performance issues.
465 # For more details see https://github.com/rook/rook/issues/2254
466 - name: ROOK_ENABLE_FSGROUP
469 # Disable automatic orchestration when new devices are discovered
470 - name: ROOK_DISABLE_DEVICE_HOTPLUG
473 # Provide customised regex as the values using comma. For eg. regex for rbd based volume, value will be like "(?i)rbd[0-9]+".
474 # In case of more than one regex, use comma to separate between them.
475 # Default regex will be "(?i)dm-[0-9]+,(?i)rbd[0-9]+,(?i)nbd[0-9]+"
476 # Add regex expression after putting a comma to blacklist a disk
477 # If value is empty, the default regex will be used.
478 - name: DISCOVER_DAEMON_UDEV_BLACKLIST
479 value: "(?i)dm-[0-9]+,(?i)rbd[0-9]+,(?i)nbd[0-9]+"
481 # Whether to enable the flex driver. By default it is enabled and is fully supported, but will be deprecated in some future release
482 # in favor of the CSI driver.
483 - name: ROOK_ENABLE_FLEX_DRIVER
486 # Whether to start the discovery daemon to watch for raw storage devices on nodes in the cluster.
487 # This daemon does not need to run if you are only going to create your OSDs based on StorageClassDeviceSets with PVCs.
488 - name: ROOK_ENABLE_DISCOVERY_DAEMON
491 # Time to wait until the node controller will move Rook pods to other
492 # nodes after detecting an unreachable node.
493 # Pods affected by this setting are:
494 # mgr, rbd, mds, rgw, nfs, PVC based mons and osds, and ceph toolbox
495 # The value used in this variable replaces the default value of 300 secs
496 # added automatically by k8s as Toleration for
497 # <node.kubernetes.io/unreachable>
498 # The total amount of time to reschedule Rook pods in healthy nodes
499 # before detecting a <not ready node> condition will be the sum of:
500 # --> node-monitor-grace-period: 40 seconds (k8s kube-controller-manager flag)
501 # --> ROOK_UNREACHABLE_NODE_TOLERATION_SECONDS: 5 seconds
502 - name: ROOK_UNREACHABLE_NODE_TOLERATION_SECONDS
505 # The name of the node to pass with the downward API
509 fieldPath: spec.nodeName
510 # The pod name to pass with the downward API
514 fieldPath: metadata.name
515 # The pod namespace to pass with the downward API
516 - name: POD_NAMESPACE
519 fieldPath: metadata.namespace
521 # Uncomment it to run lib bucket provisioner in multithreaded mode
522 #- name: LIB_BUCKET_PROVISIONER_THREADS
525 # Uncomment it to run rook operator on the host network
530 - name: default-config-dir
532 # OLM: END OPERATOR DEPLOYMENT