support cephfs-mirror feature

group_vars/cephfsmirrors.yml.sample

@@ -0,0 +1,51 @@
+---
+# Variables here are applicable to all host groups NOT roles
+
+# Dummy variable to avoid error because ansible does not recognize the
+# file as a good configuration file when no variable in it.
+dummy:
+
+#########
+# SETUP #
+#########
+
+# Even though cephfs-mirror nodes should not have the admin key
+# at their disposal, some people might want to have it
+# distributed on cephfs-mirror nodes. Setting 'copy_admin_key' to 'true'
+# will copy the admin key to the /etc/ceph/ directory.
+#copy_admin_key: false
+
+#################
+# CONFIGURATION #
+#################
+
+# Automatically enable mgr mirroring module before starting daemon(s).
+#ceph_cephfs_mirror_enable_mgr_module: true
+
+# CephX identity for local cephfs-mirror daemon.
+#ceph_cephfs_mirror_local_user: "client.cephfs-mirror.{{ ansible_facts['hostname'] }}"
+
+# Caps for cephfs-mirror daemon user.
+#ceph_cephfs_mirror_local_user_caps:
+#  mon: "allow r"
+#  mds: "allow r"
+#  osd: "allow rwx"
+#  mgr: "allow r"
+
+##########
+# DOCKER #
+##########
+
+#ceph_cephfs_mirror_docker_memory_limit: "{{ ansible_facts['memtotal_mb'] }}m"
+#ceph_cephfs_mirror_docker_cpu_limit: 1
+#ceph_cephfs_mirror_docker_extra_env:
+
+###########
+# SYSTEMD #
+###########
+# ceph_cephfs_mirror_systemd_overrides will override the systemd settings
+# for the ceph-cephfs-mirror services.
+# For example,to set "PrivateDevices=false" you can specify:
+# ceph_cephfs_mirror_systemd_overrides:
+#   Service:
+#     PrivateDevices: false

infrastructure-playbooks/purge-cluster.yml

@@ -352,6 +352,36 @@
         - '@.service'
         - '.target'
 
+- name: Purge ceph cephfs-mirror cluster
+  hosts: cephfsmirrors
+  gather_facts: false # Already gathered previously
+  become: true
+  tasks:
+    - name: Stop ceph cephfs-mirror (non-containerized)
+      ansible.builtin.service:
+        name: "cephfs-mirror@{{ ceph_cephfs_mirror_daemon_id | default('mirror') }}"
+        state: stopped
+        enabled: false
+      failed_when: false
+      when: not containerized_deployment | bool
+
+    - name: Stop ceph cephfs-mirror (containerized)
+      ansible.builtin.service:
+        name: "ceph-cephfs-mirror@{{ ceph_cephfs_mirror_daemon_id | default('mirror') }}"
+        state: stopped
+        enabled: false
+      failed_when: false
+      when: containerized_deployment | bool
+
+    - name: Remove ceph cephfs-mirror service files
+      ansible.builtin.file:
+        path: "/etc/systemd/system/{{ item }}"
+        state: absent
+      loop:
+        - "cephfs-mirror@.service"
+        - "ceph-cephfs-mirror@.service"
+        - "ceph-cephfs-mirror.target"
+
 
 - name: Purge ceph osd cluster
   vars:
@@ -825,6 +855,7 @@
       - ceph-radosgw
       - ceph-grafana-dashboards
       - rbd-mirror
+      - cephfs-mirror
     ceph_remaining_packages:
       - libcephfs2
       - librados2
@@ -846,6 +877,7 @@
     - mdss
     - rgws
     - rbdmirrors
+    - cephfsmirrors
     - nfss
     - clients
     - mgrs

infrastructure-playbooks/rolling_update.yml

@@ -46,6 +46,7 @@
     - "{{ rgw_group_name|default('rgws') }}"
     - "{{ mgr_group_name|default('mgrs') }}"
     - "{{ rbdmirror_group_name|default('rbdmirrors') }}"
+    - "{{ cephfsmirror_group_name|default('cephfsmirrors') }}"
     - "{{ nfs_group_name|default('nfss') }}"
     - "{{ client_group_name|default('clients') }}"
     - "{{ monitoring_group_name|default('monitoring') }}"
@@ -967,6 +968,63 @@
       ansible.builtin.import_role:
         name: ceph-rbd-mirror
 
+- name: Upgrade ceph cephfs mirror node
+  vars:
+    upgrade_ceph_packages: true
+  hosts: "{{ cephfsmirror_group_name|default('cephfsmirrors') }}"
+  tags: cephfsmirrors
+  serial: 1
+  become: true
+  gather_facts: false
+  tasks:
+    - name: Stop ceph cephfs mirror (containerized)
+      ansible.builtin.systemd:
+        name: "ceph-cephfs-mirror@{{ ceph_cephfs_mirror_daemon_id }}"
+        state: stopped
+        enabled: false
+        masked: true
+      failed_when: false
+      when: containerized_deployment | bool
+
+    - name: Stop ceph cephfs mirror (non-containerized)
+      ansible.builtin.systemd:
+        name: "cephfs-mirror@{{ ceph_cephfs_mirror_daemon_id }}"
+        state: stopped
+        enabled: false
+        masked: true
+      failed_when: false
+      when: not containerized_deployment | bool
+
+    - name: Import ceph-defaults role
+      ansible.builtin.import_role:
+        name: ceph-defaults
+
+    - name: Import ceph-facts role
+      ansible.builtin.import_role:
+        name: ceph-facts
+
+    - name: Import ceph-handler role
+      ansible.builtin.import_role:
+        name: ceph-handler
+
+    - name: Import ceph-common role
+      ansible.builtin.import_role:
+        name: ceph-common
+      when: not containerized_deployment | bool
+
+    - name: Import ceph-container-common role
+      ansible.builtin.import_role:
+        name: ceph-container-common
+      when: containerized_deployment | bool
+
+    - name: Import ceph-config role
+      ansible.builtin.import_role:
+        name: ceph-config
+
+    - name: Import ceph-cephfs-mirror role
+      ansible.builtin.import_role:
+        name: ceph-cephfs-mirror
+
 
 - name: Upgrade ceph nfs node
   vars:
@@ -1069,6 +1127,7 @@
     - "{{ mds_group_name | default('mdss') }}"
     - "{{ rgw_group_name | default('rgws') }}"
     - "{{ rbdmirror_group_name | default('rbdmirrors') }}"
+    - "{{ cephfsmirror_group_name | default('cephfsmirrors') }}"
     - "{{ mgr_group_name | default('mgrs') }}"
   tags:
     - post_upgrade
@@ -1109,6 +1168,7 @@
     - "{{ mds_group_name | default('mdss') }}"
     - "{{ rgw_group_name | default('rgws') }}"
     - "{{ rbdmirror_group_name | default('rbdmirrors') }}"
+    - "{{ cephfsmirror_group_name | default('cephfsmirrors') }}"
     - "{{ mgr_group_name | default('mgrs') }}"
   tags:
     - post_upgrade
@@ -1187,6 +1247,7 @@
     - "{{ rgw_group_name|default('rgws') }}"
     - "{{ mgr_group_name|default('mgrs') }}"
     - "{{ rbdmirror_group_name|default('rbdmirrors') }}"
+    - "{{ cephfsmirror_group_name|default('cephfsmirrors') }}"
     - "{{ nfs_group_name|default('nfss') }}"
     - "{{ monitoring_group_name|default('monitoring') }}"
   tags: monitoring

infrastructure-playbooks/shrink-cephfsmirror.yml

@@ -0,0 +1,138 @@
+---
+# This playbook removes a CephFS mirror daemon from your cluster on the given
+# node.
+#
+# Use it like this:
+# ansible-playbook shrink-cephfsmirror.yml -e cephfsmirror_to_kill=cephfs-mirror01
+#     Prompts for confirmation to shrink, defaults to no and
+#     doesn't shrink the cluster. yes shrinks the cluster.
+#
+# ansible-playbook -e ireallymeanit=yes|no shrink-cephfsmirror.yml
+#     Overrides the prompt using -e option. Can be used in
+#     automation scripts to avoid interactive prompt.
+
+- name: Gather facts and check the init system
+  hosts:
+    - mons
+    - cephfsmirrors
+  become: true
+  tasks:
+    - name: Gather facts on MONs and CephFS mirrors
+      ansible.builtin.debug:
+        msg: gather facts on MONs and CephFS mirrors
+
+- name: Confirm whether user really meant to remove cephfs mirror from the ceph cluster
+  hosts: mons[0]
+  become: true
+  vars_prompt:
+    - name: ireallymeanit  # noqa: name[casing]
+      prompt: Are you sure you want to shrink the cluster?
+      default: 'no'
+      private: false
+  pre_tasks:
+    - name: Import ceph-defaults role
+      ansible.builtin.import_role:
+        name: ceph-defaults
+
+    - name: Import ceph-facts role
+      ansible.builtin.import_role:
+        name: ceph-facts
+        tasks_from: container_binary
+
+    - name: Exit playbook, if no cephfsmirror was given
+      ansible.builtin.fail:
+        msg: "cephfsmirror_to_kill must be declared.
+          Exiting shrink-cephfsmirror playbook, no cephfs-mirror was removed.
+          On the command line when invoking the playbook, you can use
+          -e cephfsmirror_to_kill=cephfs-mirror01 argument. You can only remove
+          a single cephfs-mirror each time the playbook runs."
+      when: cephfsmirror_to_kill is not defined
+
+    - name: Exit playbook, if the cephfsmirror is not part of the inventory
+      ansible.builtin.fail:
+        msg: >
+          It seems that the host given is not part of your inventory,
+          please make sure it is.
+      when: cephfsmirror_to_kill not in groups[cephfsmirror_group_name]
+
+    - name: Exit playbook, if user did not mean to shrink cluster
+      ansible.builtin.fail:
+        msg: "Exiting shrink-cephfsmirror playbook, no cephfs-mirror was removed.
+          To shrink the cluster, either say 'yes' on the prompt or
+          use `-e ireallymeanit=yes` on the command line when
+          invoking the playbook"
+      when: ireallymeanit != 'yes'
+
+    - name: Set_fact container_exec_cmd for mon0
+      when: containerized_deployment | bool
+      ansible.builtin.set_fact:
+        container_exec_cmd: "{{ container_binary }} exec ceph-mon-{{ ansible_facts['hostname'] }}"
+
+    - name: Exit playbook, if can not connect to the cluster
+      ansible.builtin.command: "{{ container_exec_cmd | default('') }} timeout 5 ceph --cluster {{ cluster }} fs snapshot mirror daemon status -f json"
+      register: cephfs_mirror_status
+      changed_when: false
+      failed_when: false
+      until: cephfs_mirror_status is succeeded
+      retries: 5
+      delay: 2
+
+    - name: Set_fact cephfsmirror_to_kill_hostname
+      ansible.builtin.set_fact:
+        cephfsmirror_to_kill_hostname: "{{ hostvars[cephfsmirror_to_kill]['ansible_facts']['hostname'] }}"
+
+    - name: Set_fact cephfs_mirror_daemon_id
+      ansible.builtin.set_fact:
+        cephfs_mirror_daemon_id: "{{ hostvars[cephfsmirror_to_kill]['ceph_cephfs_mirror_daemon_id'] | default('mirror') }}"
+
+  tasks:
+    - name: Stop cephfs-mirror service (non-containerized)
+      ansible.builtin.service:
+        name: "cephfs-mirror@{{ cephfs_mirror_daemon_id }}"
+        state: stopped
+        enabled: false
+      delegate_to: "{{ cephfsmirror_to_kill }}"
+      failed_when: false
+      when: not containerized_deployment | bool
+
+    - name: Stop ceph cephfs-mirror service (containerized)
+      ansible.builtin.service:
+        name: "ceph-cephfs-mirror@{{ cephfs_mirror_daemon_id }}"
+        state: stopped
+        enabled: false
+      delegate_to: "{{ cephfsmirror_to_kill }}"
+      failed_when: false
+      when: containerized_deployment | bool
+
+    - name: Ensure cephfs-mirror service is masked (non-containerized)
+      ansible.builtin.systemd:
+        name: "cephfs-mirror@{{ cephfs_mirror_daemon_id }}"
+        masked: true
+      delegate_to: "{{ cephfsmirror_to_kill }}"
+      failed_when: false
+      when: not containerized_deployment | bool
+
+    - name: Ensure ceph cephfs-mirror service is masked (containerized)
+      ansible.builtin.systemd:
+        name: "ceph-cephfs-mirror@{{ cephfs_mirror_daemon_id }}"
+        masked: true
+      delegate_to: "{{ cephfsmirror_to_kill }}"
+      failed_when: false
+      when: containerized_deployment | bool
+
+  post_tasks:
+    - name: Check cephfs-mirror process on removed host
+      ansible.builtin.command: pgrep -af cephfs-mirror
+      delegate_to: "{{ cephfsmirror_to_kill }}"
+      register: cephfsmirror_process
+      changed_when: false
+      failed_when: false
+
+    - name: Fail if cephfs-mirror process is still running
+      ansible.builtin.fail:
+        msg: "cephfs-mirror process is still running on {{ cephfsmirror_to_kill }}"
+      when: cephfsmirror_process.rc == 0
+
+    - name: Show cephfs-mirror daemon status from cluster
+      ansible.builtin.command: "{{ container_exec_cmd | default('') }} ceph --cluster {{ cluster }} fs snapshot mirror daemon status"
+      changed_when: false