diff --git a/group_vars/cephfsmirrors.yml.sample b/group_vars/cephfsmirrors.yml.sample new file mode 100644 index 0000000000..349468038b --- /dev/null +++ b/group_vars/cephfsmirrors.yml.sample @@ -0,0 +1,51 @@ +--- +# Variables here are applicable to all host groups NOT roles + +# Dummy variable to avoid error because ansible does not recognize the +# file as a good configuration file when no variable in it. +dummy: + +######### +# SETUP # +######### + +# Even though cephfs-mirror nodes should not have the admin key +# at their disposal, some people might want to have it +# distributed on cephfs-mirror nodes. Setting 'copy_admin_key' to 'true' +# will copy the admin key to the /etc/ceph/ directory. +#copy_admin_key: false + +################# +# CONFIGURATION # +################# + +# Automatically enable mgr mirroring module before starting daemon(s). +#ceph_cephfs_mirror_enable_mgr_module: true + +# CephX identity for local cephfs-mirror daemon. +#ceph_cephfs_mirror_local_user: "client.cephfs-mirror.{{ ansible_facts['hostname'] }}" + +# Caps for cephfs-mirror daemon user. +#ceph_cephfs_mirror_local_user_caps: +# mon: "allow r" +# mds: "allow r" +# osd: "allow rwx" +# mgr: "allow r" + +########## +# DOCKER # +########## + +#ceph_cephfs_mirror_docker_memory_limit: "{{ ansible_facts['memtotal_mb'] }}m" +#ceph_cephfs_mirror_docker_cpu_limit: 1 +#ceph_cephfs_mirror_docker_extra_env: + +########### +# SYSTEMD # +########### +# ceph_cephfs_mirror_systemd_overrides will override the systemd settings +# for the ceph-cephfs-mirror services. +# For example,to set "PrivateDevices=false" you can specify: +# ceph_cephfs_mirror_systemd_overrides: +# Service: +# PrivateDevices: false diff --git a/infrastructure-playbooks/purge-cluster.yml b/infrastructure-playbooks/purge-cluster.yml index d1a4114f9a..be795eabd1 100644 --- a/infrastructure-playbooks/purge-cluster.yml +++ b/infrastructure-playbooks/purge-cluster.yml @@ -352,6 +352,36 @@ - '@.service' - '.target' +- name: Purge ceph cephfs-mirror cluster + hosts: cephfsmirrors + gather_facts: false # Already gathered previously + become: true + tasks: + - name: Stop ceph cephfs-mirror (non-containerized) + ansible.builtin.service: + name: "cephfs-mirror@{{ ceph_cephfs_mirror_daemon_id | default('mirror') }}" + state: stopped + enabled: false + failed_when: false + when: not containerized_deployment | bool + + - name: Stop ceph cephfs-mirror (containerized) + ansible.builtin.service: + name: "ceph-cephfs-mirror@{{ ceph_cephfs_mirror_daemon_id | default('mirror') }}" + state: stopped + enabled: false + failed_when: false + when: containerized_deployment | bool + + - name: Remove ceph cephfs-mirror service files + ansible.builtin.file: + path: "/etc/systemd/system/{{ item }}" + state: absent + loop: + - "cephfs-mirror@.service" + - "ceph-cephfs-mirror@.service" + - "ceph-cephfs-mirror.target" + - name: Purge ceph osd cluster vars: @@ -825,6 +855,7 @@ - ceph-radosgw - ceph-grafana-dashboards - rbd-mirror + - cephfs-mirror ceph_remaining_packages: - libcephfs2 - librados2 @@ -846,6 +877,7 @@ - mdss - rgws - rbdmirrors + - cephfsmirrors - nfss - clients - mgrs diff --git a/infrastructure-playbooks/rolling_update.yml b/infrastructure-playbooks/rolling_update.yml index 839609a728..fdaede49d8 100644 --- a/infrastructure-playbooks/rolling_update.yml +++ b/infrastructure-playbooks/rolling_update.yml @@ -46,6 +46,7 @@ - "{{ rgw_group_name|default('rgws') }}" - "{{ mgr_group_name|default('mgrs') }}" - "{{ rbdmirror_group_name|default('rbdmirrors') }}" + - "{{ cephfsmirror_group_name|default('cephfsmirrors') }}" - "{{ nfs_group_name|default('nfss') }}" - "{{ client_group_name|default('clients') }}" - "{{ monitoring_group_name|default('monitoring') }}" @@ -967,6 +968,63 @@ ansible.builtin.import_role: name: ceph-rbd-mirror +- name: Upgrade ceph cephfs mirror node + vars: + upgrade_ceph_packages: true + hosts: "{{ cephfsmirror_group_name|default('cephfsmirrors') }}" + tags: cephfsmirrors + serial: 1 + become: true + gather_facts: false + tasks: + - name: Stop ceph cephfs mirror (containerized) + ansible.builtin.systemd: + name: "ceph-cephfs-mirror@{{ ceph_cephfs_mirror_daemon_id }}" + state: stopped + enabled: false + masked: true + failed_when: false + when: containerized_deployment | bool + + - name: Stop ceph cephfs mirror (non-containerized) + ansible.builtin.systemd: + name: "cephfs-mirror@{{ ceph_cephfs_mirror_daemon_id }}" + state: stopped + enabled: false + masked: true + failed_when: false + when: not containerized_deployment | bool + + - name: Import ceph-defaults role + ansible.builtin.import_role: + name: ceph-defaults + + - name: Import ceph-facts role + ansible.builtin.import_role: + name: ceph-facts + + - name: Import ceph-handler role + ansible.builtin.import_role: + name: ceph-handler + + - name: Import ceph-common role + ansible.builtin.import_role: + name: ceph-common + when: not containerized_deployment | bool + + - name: Import ceph-container-common role + ansible.builtin.import_role: + name: ceph-container-common + when: containerized_deployment | bool + + - name: Import ceph-config role + ansible.builtin.import_role: + name: ceph-config + + - name: Import ceph-cephfs-mirror role + ansible.builtin.import_role: + name: ceph-cephfs-mirror + - name: Upgrade ceph nfs node vars: @@ -1069,6 +1127,7 @@ - "{{ mds_group_name | default('mdss') }}" - "{{ rgw_group_name | default('rgws') }}" - "{{ rbdmirror_group_name | default('rbdmirrors') }}" + - "{{ cephfsmirror_group_name | default('cephfsmirrors') }}" - "{{ mgr_group_name | default('mgrs') }}" tags: - post_upgrade @@ -1109,6 +1168,7 @@ - "{{ mds_group_name | default('mdss') }}" - "{{ rgw_group_name | default('rgws') }}" - "{{ rbdmirror_group_name | default('rbdmirrors') }}" + - "{{ cephfsmirror_group_name | default('cephfsmirrors') }}" - "{{ mgr_group_name | default('mgrs') }}" tags: - post_upgrade @@ -1187,6 +1247,7 @@ - "{{ rgw_group_name|default('rgws') }}" - "{{ mgr_group_name|default('mgrs') }}" - "{{ rbdmirror_group_name|default('rbdmirrors') }}" + - "{{ cephfsmirror_group_name|default('cephfsmirrors') }}" - "{{ nfs_group_name|default('nfss') }}" - "{{ monitoring_group_name|default('monitoring') }}" tags: monitoring diff --git a/infrastructure-playbooks/shrink-cephfsmirror.yml b/infrastructure-playbooks/shrink-cephfsmirror.yml new file mode 100644 index 0000000000..630d899aa3 --- /dev/null +++ b/infrastructure-playbooks/shrink-cephfsmirror.yml @@ -0,0 +1,138 @@ +--- +# This playbook removes a CephFS mirror daemon from your cluster on the given +# node. +# +# Use it like this: +# ansible-playbook shrink-cephfsmirror.yml -e cephfsmirror_to_kill=cephfs-mirror01 +# Prompts for confirmation to shrink, defaults to no and +# doesn't shrink the cluster. yes shrinks the cluster. +# +# ansible-playbook -e ireallymeanit=yes|no shrink-cephfsmirror.yml +# Overrides the prompt using -e option. Can be used in +# automation scripts to avoid interactive prompt. + +- name: Gather facts and check the init system + hosts: + - mons + - cephfsmirrors + become: true + tasks: + - name: Gather facts on MONs and CephFS mirrors + ansible.builtin.debug: + msg: gather facts on MONs and CephFS mirrors + +- name: Confirm whether user really meant to remove cephfs mirror from the ceph cluster + hosts: mons[0] + become: true + vars_prompt: + - name: ireallymeanit # noqa: name[casing] + prompt: Are you sure you want to shrink the cluster? + default: 'no' + private: false + pre_tasks: + - name: Import ceph-defaults role + ansible.builtin.import_role: + name: ceph-defaults + + - name: Import ceph-facts role + ansible.builtin.import_role: + name: ceph-facts + tasks_from: container_binary + + - name: Exit playbook, if no cephfsmirror was given + ansible.builtin.fail: + msg: "cephfsmirror_to_kill must be declared. + Exiting shrink-cephfsmirror playbook, no cephfs-mirror was removed. + On the command line when invoking the playbook, you can use + -e cephfsmirror_to_kill=cephfs-mirror01 argument. You can only remove + a single cephfs-mirror each time the playbook runs." + when: cephfsmirror_to_kill is not defined + + - name: Exit playbook, if the cephfsmirror is not part of the inventory + ansible.builtin.fail: + msg: > + It seems that the host given is not part of your inventory, + please make sure it is. + when: cephfsmirror_to_kill not in groups[cephfsmirror_group_name] + + - name: Exit playbook, if user did not mean to shrink cluster + ansible.builtin.fail: + msg: "Exiting shrink-cephfsmirror playbook, no cephfs-mirror was removed. + To shrink the cluster, either say 'yes' on the prompt or + use `-e ireallymeanit=yes` on the command line when + invoking the playbook" + when: ireallymeanit != 'yes' + + - name: Set_fact container_exec_cmd for mon0 + when: containerized_deployment | bool + ansible.builtin.set_fact: + container_exec_cmd: "{{ container_binary }} exec ceph-mon-{{ ansible_facts['hostname'] }}" + + - name: Exit playbook, if can not connect to the cluster + ansible.builtin.command: "{{ container_exec_cmd | default('') }} timeout 5 ceph --cluster {{ cluster }} fs snapshot mirror daemon status -f json" + register: cephfs_mirror_status + changed_when: false + failed_when: false + until: cephfs_mirror_status is succeeded + retries: 5 + delay: 2 + + - name: Set_fact cephfsmirror_to_kill_hostname + ansible.builtin.set_fact: + cephfsmirror_to_kill_hostname: "{{ hostvars[cephfsmirror_to_kill]['ansible_facts']['hostname'] }}" + + - name: Set_fact cephfs_mirror_daemon_id + ansible.builtin.set_fact: + cephfs_mirror_daemon_id: "{{ hostvars[cephfsmirror_to_kill]['ceph_cephfs_mirror_daemon_id'] | default('mirror') }}" + + tasks: + - name: Stop cephfs-mirror service (non-containerized) + ansible.builtin.service: + name: "cephfs-mirror@{{ cephfs_mirror_daemon_id }}" + state: stopped + enabled: false + delegate_to: "{{ cephfsmirror_to_kill }}" + failed_when: false + when: not containerized_deployment | bool + + - name: Stop ceph cephfs-mirror service (containerized) + ansible.builtin.service: + name: "ceph-cephfs-mirror@{{ cephfs_mirror_daemon_id }}" + state: stopped + enabled: false + delegate_to: "{{ cephfsmirror_to_kill }}" + failed_when: false + when: containerized_deployment | bool + + - name: Ensure cephfs-mirror service is masked (non-containerized) + ansible.builtin.systemd: + name: "cephfs-mirror@{{ cephfs_mirror_daemon_id }}" + masked: true + delegate_to: "{{ cephfsmirror_to_kill }}" + failed_when: false + when: not containerized_deployment | bool + + - name: Ensure ceph cephfs-mirror service is masked (containerized) + ansible.builtin.systemd: + name: "ceph-cephfs-mirror@{{ cephfs_mirror_daemon_id }}" + masked: true + delegate_to: "{{ cephfsmirror_to_kill }}" + failed_when: false + when: containerized_deployment | bool + + post_tasks: + - name: Check cephfs-mirror process on removed host + ansible.builtin.command: pgrep -af cephfs-mirror + delegate_to: "{{ cephfsmirror_to_kill }}" + register: cephfsmirror_process + changed_when: false + failed_when: false + + - name: Fail if cephfs-mirror process is still running + ansible.builtin.fail: + msg: "cephfs-mirror process is still running on {{ cephfsmirror_to_kill }}" + when: cephfsmirror_process.rc == 0 + + - name: Show cephfs-mirror daemon status from cluster + ansible.builtin.command: "{{ container_exec_cmd | default('') }} ceph --cluster {{ cluster }} fs snapshot mirror daemon status" + changed_when: false diff --git a/roles/ceph-cephfs-mirror/LICENSE b/roles/ceph-cephfs-mirror/LICENSE new file mode 100644 index 0000000000..acee72b2b5 --- /dev/null +++ b/roles/ceph-cephfs-mirror/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [2014] [Sébastien Han] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/roles/ceph-cephfs-mirror/README.md b/roles/ceph-cephfs-mirror/README.md new file mode 100644 index 0000000000..2a0de1556d --- /dev/null +++ b/roles/ceph-cephfs-mirror/README.md @@ -0,0 +1,3 @@ +# Ansible role: ceph-cephfs-mirror + +Deploys CephFS mirror daemon(s) for snapshot mirroring. diff --git a/roles/ceph-cephfs-mirror/defaults/main.yml b/roles/ceph-cephfs-mirror/defaults/main.yml new file mode 100644 index 0000000000..8ed0614cc6 --- /dev/null +++ b/roles/ceph-cephfs-mirror/defaults/main.yml @@ -0,0 +1,48 @@ +--- +######### +# SETUP # +######### + +# Even though cephfs-mirror nodes should not have the admin key +# at their disposal, some people might want to have it +# distributed on cephfs-mirror nodes. Setting 'copy_admin_key' to 'true' +# will copy the admin key to the /etc/ceph/ directory. +copy_admin_key: false + +################# +# CONFIGURATION # +################# + +# Automatically enable mgr mirroring module before starting daemon(s). +ceph_cephfs_mirror_enable_mgr_module: true + +# Daemon ID used by cephfs-mirror (--id). +ceph_cephfs_mirror_daemon_id: "mirror" + +# CephX identity for local cephfs-mirror daemon. +ceph_cephfs_mirror_local_user: "client.{{ ceph_cephfs_mirror_daemon_id }}" + +# Caps for cephfs-mirror daemon user. Keep broad by default for compatibility. +ceph_cephfs_mirror_local_user_caps: + mon: "profile cephfs-mirror" + mds: "allow r" + osd: "allow rw tag cephfs metadata=*, allow r tag cephfs data=*" + mgr: "allow r" + +########## +# DOCKER # +########## + +ceph_cephfs_mirror_docker_memory_limit: "{{ ansible_facts['memtotal_mb'] }}m" +ceph_cephfs_mirror_docker_cpu_limit: 1 +ceph_cephfs_mirror_docker_extra_env: + +########### +# SYSTEMD # +########### +# ceph_cephfs_mirror_systemd_overrides will override the systemd settings +# for the ceph-cephfs-mirror services. +# For example,to set "PrivateDevices=false" you can specify: +# ceph_cephfs_mirror_systemd_overrides: +# Service: +# PrivateDevices: false diff --git a/roles/ceph-cephfs-mirror/files/ceph-cephfs-mirror.target b/roles/ceph-cephfs-mirror/files/ceph-cephfs-mirror.target new file mode 100644 index 0000000000..aa0bbc307a --- /dev/null +++ b/roles/ceph-cephfs-mirror/files/ceph-cephfs-mirror.target @@ -0,0 +1,7 @@ +[Unit] +Description=ceph target allowing to start/stop all ceph-cephfs-mirror@.service instances at once +PartOf=ceph.target +Before=ceph.target + +[Install] +WantedBy=multi-user.target ceph.target diff --git a/roles/ceph-cephfs-mirror/handlers/main.yml b/roles/ceph-cephfs-mirror/handlers/main.yml new file mode 100644 index 0000000000..a1ec1d0a7e --- /dev/null +++ b/roles/ceph-cephfs-mirror/handlers/main.yml @@ -0,0 +1,6 @@ +--- +- name: Restart ceph cephfs mirrors + ansible.builtin.systemd: + name: "{{ 'ceph-cephfs-mirror@' + ceph_cephfs_mirror_daemon_id if containerized_deployment | bool else 'cephfs-mirror@' + ceph_cephfs_mirror_daemon_id }}" + state: restarted + daemon_reload: true diff --git a/roles/ceph-cephfs-mirror/meta/main.yml b/roles/ceph-cephfs-mirror/meta/main.yml new file mode 100644 index 0000000000..835c714244 --- /dev/null +++ b/roles/ceph-cephfs-mirror/meta/main.yml @@ -0,0 +1,14 @@ +--- +galaxy_info: + company: Red Hat + author: ceph-ansible contributors + description: Installs CephFS Mirror daemon + license: Apache + min_ansible_version: '2.7' + platforms: + - name: EL + versions: + - 'all' + galaxy_tags: + - system +dependencies: [] diff --git a/roles/ceph-cephfs-mirror/tasks/main.yml b/roles/ceph-cephfs-mirror/tasks/main.yml new file mode 100644 index 0000000000..f8f02e38fc --- /dev/null +++ b/roles/ceph-cephfs-mirror/tasks/main.yml @@ -0,0 +1,107 @@ +--- +- name: Install dependencies + ansible.builtin.package: + name: cephfs-mirror + state: present + register: result + until: result is succeeded + when: not containerized_deployment | bool + tags: package-install + +- name: Enable mgr mirroring module + ansible.builtin.command: "{{ hostvars[groups[mon_group_name][0]]['container_exec_cmd'] | default('') }} ceph --cluster {{ cluster }} mgr module enable mirroring" + changed_when: false + failed_when: false + delegate_to: "{{ groups[mon_group_name][0] }}" + run_once: true + when: + - ceph_cephfs_mirror_enable_mgr_module | bool + - groups.get(mon_group_name, []) | length > 0 + +- name: Cephx tasks + when: cephx | bool + block: + - name: Create cephfs-mirror daemon keyring + ceph_key: + name: "{{ ceph_cephfs_mirror_local_user }}" + cluster: "{{ cluster }}" + user: client.admin + user_key: "/etc/ceph/{{ cluster }}.client.admin.keyring" + caps: "{{ ceph_cephfs_mirror_local_user_caps }}" + dest: "/etc/ceph/{{ cluster }}.{{ ceph_cephfs_mirror_local_user }}.keyring" + import_key: true + owner: "{{ ceph_uid if containerized_deployment | bool else 'ceph' }}" + group: "{{ ceph_uid if containerized_deployment | bool else 'ceph' }}" + mode: "{{ ceph_keyring_permissions }}" + no_log: "{{ no_log_on_ceph_key_tasks }}" + environment: + CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment | bool else None }}" + CEPH_CONTAINER_BINARY: "{{ container_binary }}" + delegate_to: "{{ groups.get(mon_group_name)[0] }}" + + - name: Get cephfs-mirror keyring from ceph monitor + ceph_key_info: + name: "{{ ceph_cephfs_mirror_local_user }}" + cluster: "{{ cluster }}" + output_format: plain + state: info + environment: + CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment | bool else None }}" + CEPH_CONTAINER_BINARY: "{{ container_binary }}" + register: _cephfs_mirror_key + delegate_to: "{{ groups.get(mon_group_name)[0] }}" + no_log: "{{ no_log_on_ceph_key_tasks }}" + + - name: Copy ceph key + ansible.builtin.copy: + dest: "/etc/ceph/{{ cluster }}.{{ ceph_cephfs_mirror_local_user }}.keyring" + content: "{{ _cephfs_mirror_key.stdout + '\n' }}" + owner: "{{ ceph_uid if containerized_deployment | bool else 'ceph' }}" + group: "{{ ceph_uid if containerized_deployment | bool else 'ceph' }}" + mode: "{{ ceph_keyring_permissions }}" + no_log: "{{ no_log_on_ceph_key_tasks }}" + +- name: Ensure systemd service override directory exists + ansible.builtin.file: + state: directory + path: "/etc/systemd/system/ceph-cephfs-mirror@.service.d/" + mode: "0755" + when: + - ceph_cephfs_mirror_systemd_overrides is defined + - ansible_facts['service_mgr'] == 'systemd' + +- name: Add ceph-cephfs-mirror systemd service overrides + openstack.config_template.config_template: + src: "ceph-cephfs-mirror.service.d-overrides.j2" + dest: "/etc/systemd/system/ceph-cephfs-mirror@.service.d/ceph-cephfs-mirror-systemd-overrides.conf" + config_overrides: "{{ ceph_cephfs_mirror_systemd_overrides | default({}) }}" + config_type: "ini" + when: + - ceph_cephfs_mirror_systemd_overrides is defined + - ansible_facts['service_mgr'] == 'systemd' + +- name: Include_tasks start_container_cephfs_mirror.yml + ansible.builtin.include_tasks: start_container_cephfs_mirror.yml + when: containerized_deployment | bool + +- name: Include_tasks systemd.yml + ansible.builtin.include_tasks: systemd.yml + when: containerized_deployment | bool + +- name: Start and add cephfs-mirror service instance (containerized) + ansible.builtin.service: + name: "ceph-cephfs-mirror@{{ ceph_cephfs_mirror_daemon_id }}" + state: started + enabled: true + masked: false + changed_when: false + when: containerized_deployment | bool + +- name: Start and add cephfs-mirror service instance (packaged non-containerized) + ansible.builtin.service: + name: "cephfs-mirror@{{ ceph_cephfs_mirror_daemon_id }}" + state: started + enabled: true + masked: false + changed_when: false + when: not containerized_deployment | bool diff --git a/roles/ceph-cephfs-mirror/tasks/start_container_cephfs_mirror.yml b/roles/ceph-cephfs-mirror/tasks/start_container_cephfs_mirror.yml new file mode 100644 index 0000000000..b7d8ffa436 --- /dev/null +++ b/roles/ceph-cephfs-mirror/tasks/start_container_cephfs_mirror.yml @@ -0,0 +1,11 @@ +--- +- name: Include_tasks systemd.yml + ansible.builtin.include_tasks: systemd.yml + +- name: Systemd start cephfs mirror container + ansible.builtin.systemd: + name: ceph-cephfs-mirror@{{ ceph_cephfs_mirror_daemon_id }} + state: started + enabled: true + masked: false + daemon_reload: true diff --git a/roles/ceph-cephfs-mirror/tasks/systemd.yml b/roles/ceph-cephfs-mirror/tasks/systemd.yml new file mode 100644 index 0000000000..4b4b3b3f10 --- /dev/null +++ b/roles/ceph-cephfs-mirror/tasks/systemd.yml @@ -0,0 +1,23 @@ +--- +- name: Generate systemd unit file + ansible.builtin.template: + src: "{{ role_path }}/templates/ceph-cephfs-mirror.service.j2" + dest: /etc/systemd/system/ceph-cephfs-mirror@.service + owner: "root" + group: "root" + mode: "0644" + when: containerized_deployment | bool + +- name: Generate systemd ceph-cephfs-mirror target file + ansible.builtin.copy: + src: ceph-cephfs-mirror.target + dest: /etc/systemd/system/ceph-cephfs-mirror.target + mode: "0644" + when: containerized_deployment | bool + +- name: Enable ceph-cephfs-mirror.target + ansible.builtin.service: + name: ceph-cephfs-mirror.target + enabled: true + daemon_reload: true + when: containerized_deployment | bool diff --git a/roles/ceph-cephfs-mirror/templates/ceph-cephfs-mirror.service.d-overrides.j2 b/roles/ceph-cephfs-mirror/templates/ceph-cephfs-mirror.service.d-overrides.j2 new file mode 100644 index 0000000000..e2bb153305 --- /dev/null +++ b/roles/ceph-cephfs-mirror/templates/ceph-cephfs-mirror.service.d-overrides.j2 @@ -0,0 +1 @@ +# {{ ansible_managed }} diff --git a/roles/ceph-cephfs-mirror/templates/ceph-cephfs-mirror.service.j2 b/roles/ceph-cephfs-mirror/templates/ceph-cephfs-mirror.service.j2 new file mode 100644 index 0000000000..8e0f3ca3bc --- /dev/null +++ b/roles/ceph-cephfs-mirror/templates/ceph-cephfs-mirror.service.j2 @@ -0,0 +1,64 @@ +[Unit] +Description=CephFS mirror daemon +PartOf=ceph-cephfs-mirror.target +{% if container_binary == 'docker' and containerized_deployment | bool %} +After=docker.service network-online.target local-fs.target time-sync.target +Requires=docker.service +{% else %} +After=network-online.target local-fs.target time-sync.target +{% endif %} +Wants=network-online.target local-fs.target time-sync.target + +[Service] +EnvironmentFile=-/etc/environment +{% if containerized_deployment | bool %} +{% if container_binary == 'podman' %} +ExecStartPre=-/usr/bin/rm -f /%t/%n-pid /%t/%n-cid +ExecStartPre=-/usr/bin/{{ container_binary }} rm --storage ceph-cephfs-mirror-{{ ceph_cephfs_mirror_daemon_id }}-{{ ansible_facts['hostname'] }} +ExecStartPre=-/usr/bin/mkdir -p /var/log/ceph +{% else %} +ExecStartPre=-/usr/bin/{{ container_binary }} stop ceph-cephfs-mirror-{{ ceph_cephfs_mirror_daemon_id }}-{{ ansible_facts['hostname'] }} +{% endif %} +ExecStartPre=-/usr/bin/{{ container_binary }} rm -f ceph-cephfs-mirror-{{ ceph_cephfs_mirror_daemon_id }}-{{ ansible_facts['hostname'] }} +ExecStart=/usr/bin/{{ container_binary }} run --rm --net=host \ +{% if container_binary == 'podman' %} + -d --log-driver journald --conmon-pidfile /%t/%n-pid --cidfile /%t/%n-cid \ +{% endif %} + --pids-limit={{ 0 if container_binary == 'podman' else -1 }} \ + --memory={{ ceph_cephfs_mirror_docker_memory_limit }} \ + --cpus={{ ceph_cephfs_mirror_docker_cpu_limit }} \ + --security-opt label=disable \ + -v /etc/ceph:/etc/ceph:z \ + -v /var/run/ceph:/var/run/ceph:z \ + -v /etc/localtime:/etc/localtime:ro \ + -v /var/log/ceph:/var/log/ceph:z \ + -e TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES={{ ceph_tcmalloc_max_total_thread_cache }} \ + --name=ceph-cephfs-mirror-{{ ceph_cephfs_mirror_daemon_id }}-{{ ansible_facts['hostname'] }} \ + {{ ceph_cephfs_mirror_docker_extra_env }} \ + --entrypoint=/usr/bin/cephfs-mirror \ + {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \ + --id {{ ceph_cephfs_mirror_daemon_id }} --cluster {{ cluster }} -f +{% if container_binary == 'podman' %} +ExecStop=-/usr/bin/sh -c "/usr/bin/{{ container_binary }} rm -f `cat /%t/%n-cid`" +{% else %} +ExecStopPost=-/usr/bin/{{ container_binary }} stop ceph-cephfs-mirror-{{ ceph_cephfs_mirror_daemon_id }}-{{ ansible_facts['hostname'] }} +{% endif %} +{% else %} +ExecStart=/usr/bin/cephfs-mirror --id {{ ceph_cephfs_mirror_daemon_id }} --cluster {{ cluster }} -f +{% endif %} +{% if containerized_deployment | bool %} +KillMode=none +{% else %} +KillMode=control-group +{% endif %} +Restart=always +RestartSec=10s +TimeoutStartSec=120 +TimeoutStopSec=15 +{% if containerized_deployment | bool and container_binary == 'podman' %} +Type=forking +PIDFile=/%t/%n-pid +{% endif %} + +[Install] +WantedBy=ceph.target diff --git a/site.yml.sample b/site.yml.sample index 8811d3cfc5..b01ae12740 100644 --- a/site.yml.sample +++ b/site.yml.sample @@ -18,6 +18,7 @@ - rgws - nfss - rbdmirrors + - cephfsmirrors - clients - mgrs - monitoring @@ -394,6 +395,44 @@ status: "Complete" end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" +- hosts: cephfsmirrors + gather_facts: false + become: True + any_errors_fatal: true + pre_tasks: + - name: set ceph cephfs mirror install 'In Progress' + run_once: true + set_stats: + data: + installer_phase_ceph_cephfs_mirror: + status: "In Progress" + start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" + + tasks: + - import_role: + name: ceph-defaults + tags: ['ceph_update_config'] + - import_role: + name: ceph-facts + tags: ['ceph_update_config'] + - import_role: + name: ceph-handler + tags: ['ceph_update_config'] + - import_role: + name: ceph-config + tags: ['ceph_update_config'] + - import_role: + name: ceph-cephfs-mirror + + post_tasks: + - name: set ceph cephfs mirror install 'Complete' + run_once: true + set_stats: + data: + installer_phase_ceph_cephfs_mirror: + status: "Complete" + end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" + - hosts: - rgwloadbalancers gather_facts: false @@ -438,6 +477,7 @@ - mdss - rgws - rbdmirrors + - cephfsmirrors - mgrs gather_facts: false @@ -478,6 +518,7 @@ - mdss - rgws - rbdmirrors + - cephfsmirrors - clients - mgrs - monitoring diff --git a/tests/conftest.py b/tests/conftest.py index 07e35e2654..c4f08083d7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -213,6 +213,8 @@ def pytest_collection_modifyitems(session, config, items): item.add_marker(pytest.mark.mgrs) elif "rbd-mirror" in test_path: item.add_marker(pytest.mark.rbdmirrors) + elif "cephfs-mirror" in test_path: + item.add_marker(pytest.mark.cephfsmirrors) elif "rgw" in test_path: item.add_marker(pytest.mark.rgws) elif "nfs" in test_path: diff --git a/tests/functional/all_daemons/container/hosts b/tests/functional/all_daemons/container/hosts index 51d488ccbd..892658b9ec 100644 --- a/tests/functional/all_daemons/container/hosts +++ b/tests/functional/all_daemons/container/hosts @@ -31,3 +31,6 @@ rbd-mirror0 [monitoring] mon0 +[cephfsmirrors] +cephfs-mirror0 ansible_host=client0 + diff --git a/tests/functional/all_daemons/hosts b/tests/functional/all_daemons/hosts index 8e2019776a..bae3f2ef07 100644 --- a/tests/functional/all_daemons/hosts +++ b/tests/functional/all_daemons/hosts @@ -29,5 +29,8 @@ client1 [rbdmirrors] rbd-mirror0 +[cephfsmirrors] +cephfs-mirror0 ansible_host=client0 + [ceph_monitoring] mon0 diff --git a/tests/functional/shrink_cephfsmirror/Vagrantfile b/tests/functional/shrink_cephfsmirror/Vagrantfile new file mode 120000 index 0000000000..706a5bb470 --- /dev/null +++ b/tests/functional/shrink_cephfsmirror/Vagrantfile @@ -0,0 +1 @@ +../../../Vagrantfile \ No newline at end of file diff --git a/tests/functional/shrink_cephfsmirror/ceph-override.json b/tests/functional/shrink_cephfsmirror/ceph-override.json new file mode 120000 index 0000000000..fe2ff40d62 --- /dev/null +++ b/tests/functional/shrink_cephfsmirror/ceph-override.json @@ -0,0 +1 @@ +../all_daemons/ceph-override.json \ No newline at end of file diff --git a/tests/functional/shrink_cephfsmirror/container/Vagrantfile b/tests/functional/shrink_cephfsmirror/container/Vagrantfile new file mode 120000 index 0000000000..16076e4245 --- /dev/null +++ b/tests/functional/shrink_cephfsmirror/container/Vagrantfile @@ -0,0 +1 @@ +../../../../Vagrantfile \ No newline at end of file diff --git a/tests/functional/shrink_cephfsmirror/container/ceph-override.json b/tests/functional/shrink_cephfsmirror/container/ceph-override.json new file mode 120000 index 0000000000..8417cc0c98 --- /dev/null +++ b/tests/functional/shrink_cephfsmirror/container/ceph-override.json @@ -0,0 +1 @@ +../../all_daemons/ceph-override.json \ No newline at end of file diff --git a/tests/functional/shrink_cephfsmirror/container/group_vars/all b/tests/functional/shrink_cephfsmirror/container/group_vars/all new file mode 100644 index 0000000000..0ec43c7533 --- /dev/null +++ b/tests/functional/shrink_cephfsmirror/container/group_vars/all @@ -0,0 +1,18 @@ +--- +# this is only here to let the CI tests know +# that this scenario is using docker +docker: True +public_network: "192.168.87.0/24" +cluster_network: "192.168.88.0/24" +containerized_deployment: True +ceph_mon_docker_subnet: "{{ public_network }}" +ceph_conf_overrides: + global: + mon_allow_pool_size_one: true + mon_warn_on_pool_no_redundancy: false + osd_pool_default_size: 1 +dashboard_enabled: False +copy_admin_key: True +ceph_docker_registry: quay.io +ceph_docker_image: ceph/daemon-base +ceph_docker_image_tag: latest-main \ No newline at end of file diff --git a/tests/functional/shrink_cephfsmirror/container/group_vars/mons b/tests/functional/shrink_cephfsmirror/container/group_vars/mons new file mode 100644 index 0000000000..0e679799d5 --- /dev/null +++ b/tests/functional/shrink_cephfsmirror/container/group_vars/mons @@ -0,0 +1,3 @@ +--- +create_crush_tree: False +crush_rule_config: False \ No newline at end of file diff --git a/tests/functional/shrink_cephfsmirror/container/group_vars/osds b/tests/functional/shrink_cephfsmirror/container/group_vars/osds new file mode 100644 index 0000000000..27268d4065 --- /dev/null +++ b/tests/functional/shrink_cephfsmirror/container/group_vars/osds @@ -0,0 +1,9 @@ +--- +osd_objectstore: "bluestore" +lvm_volumes: + - data: data-lv1 + data_vg: test_group + - data: data-lv2 + data_vg: test_group + db: journal1 + db_vg: journals \ No newline at end of file diff --git a/tests/functional/shrink_cephfsmirror/container/hosts b/tests/functional/shrink_cephfsmirror/container/hosts new file mode 100644 index 0000000000..de243961b8 --- /dev/null +++ b/tests/functional/shrink_cephfsmirror/container/hosts @@ -0,0 +1,8 @@ +[mons] +mon0 + +[osds] +osd0 + +[cephfsmirrors] +cephfs-mirror0 ansible_host=client0 diff --git a/tests/functional/shrink_cephfsmirror/container/vagrant_variables.yml b/tests/functional/shrink_cephfsmirror/container/vagrant_variables.yml new file mode 100644 index 0000000000..8100c6ce75 --- /dev/null +++ b/tests/functional/shrink_cephfsmirror/container/vagrant_variables.yml @@ -0,0 +1,60 @@ +--- + +# DEPLOY CONTAINERIZED DAEMONS +docker: True + +# DEFINE THE NUMBER OF VMS TO RUN +mon_vms: 1 +osd_vms: 1 +mds_vms: 0 +rgw_vms: 0 +nfs_vms: 0 +grafana_server_vms: 0 +rbd_mirror_vms: 0 +client_vms: 1 +mgr_vms: 0 + +# SUBNETS TO USE FOR THE VMS +public_subnet: 192.168.87 +cluster_subnet: 192.168.88 + +# MEMORY +# set 1024 for CentOS +memory: 1024 + +# Disks +# For libvirt use disks: "[ '/dev/vdb', '/dev/vdc' ]" +# For CentOS7 use disks: "[ '/dev/sda', '/dev/sdb' ]" +disks: "[ '/dev/sda', '/dev/sdb' ]" + +# VAGRANT BOX +# Ceph boxes are *strongly* suggested. They are under better control and will +# not get updated frequently unless required for build systems. These are (for +# now): +# +# * ceph/ubuntu-xenial +# +# Ubuntu: ceph/ubuntu-xenial bento/ubuntu-16.04 or ubuntu/trusty64 or ubuntu/wily64 +# CentOS: bento/centos-7.1 or puppetlabs/centos-7.0-64-puppet +# libvirt CentOS: centos/7 +# parallels Ubuntu: parallels/ubuntu-14.04 +# Debian: deb/jessie-amd64 - be careful the storage controller is named 'SATA Controller' +# For more boxes have a look at: +# - https://atlas.hashicorp.com/boxes/search?utf8=✓&sort=&provider=virtualbox&q= +# - https://download.gluster.org/pub/gluster/purpleidea/vagrant/ +vagrant_box: centos/stream9 +#client_vagrant_box: centos/stream9 +#ssh_private_key_path: "~/.ssh/id_rsa" +# The sync directory changes based on vagrant box +# Set to /home/vagrant/sync for Centos/7, /home/{ user }/vagrant for openstack and defaults to /vagrant +#vagrant_sync_dir: /home/vagrant/sync +vagrant_sync_dir: /vagrant +# Disables synced folder creation. Not needed for testing, will skip mounting +# the vagrant directory on the remote box regardless of the provider. +vagrant_disable_synced_folder: true +# VAGRANT URL +# This is a URL to download an image from an alternate location. vagrant_box +# above should be set to the filename of the image. +# Fedora virtualbox: https://download.fedoraproject.org/pub/fedora/linux/releases/22/Cloud/x86_64/Images/Fedora-Cloud-Base-Vagrant-22-20150521.x86_64.vagrant-virtualbox.box +# Fedora libvirt: https://download.fedoraproject.org/pub/fedora/linux/releases/22/Cloud/x86_64/Images/Fedora-Cloud-Base-Vagrant-22-20150521.x86_64.vagrant-libvirt.box +# vagrant_box_url: https://download.fedoraproject.org/pub/fedora/linux/releases/22/Cloud/x86_64/Images/Fedora-Cloud-Base-Vagrant-22-20150521.x86_64.vagrant-virtualbox.box diff --git a/tests/functional/shrink_cephfsmirror/group_vars/all b/tests/functional/shrink_cephfsmirror/group_vars/all new file mode 100644 index 0000000000..1a90c972b7 --- /dev/null +++ b/tests/functional/shrink_cephfsmirror/group_vars/all @@ -0,0 +1,13 @@ +--- +ceph_origin: repository +ceph_repository: community +public_network: "192.168.85.0/24" +cluster_network: "192.168.86.0/24" +osd_objectstore: "bluestore" +copy_admin_key: true +ceph_conf_overrides: + global: + mon_allow_pool_size_one: true + mon_warn_on_pool_no_redundancy: false + osd_pool_default_size: 1 +dashboard_enabled: False \ No newline at end of file diff --git a/tests/functional/shrink_cephfsmirror/group_vars/mons b/tests/functional/shrink_cephfsmirror/group_vars/mons new file mode 100644 index 0000000000..0e679799d5 --- /dev/null +++ b/tests/functional/shrink_cephfsmirror/group_vars/mons @@ -0,0 +1,3 @@ +--- +create_crush_tree: False +crush_rule_config: False \ No newline at end of file diff --git a/tests/functional/shrink_cephfsmirror/group_vars/osds b/tests/functional/shrink_cephfsmirror/group_vars/osds new file mode 100644 index 0000000000..3ec1d6e4c1 --- /dev/null +++ b/tests/functional/shrink_cephfsmirror/group_vars/osds @@ -0,0 +1,11 @@ +--- +os_tuning_params: + - { name: fs.file-max, value: 26234859 } +osd_objectstore: "bluestore" +lvm_volumes: + - data: data-lv1 + data_vg: test_group + - data: data-lv2 + data_vg: test_group + db: journal1 + db_vg: journals \ No newline at end of file diff --git a/tests/functional/shrink_cephfsmirror/hosts b/tests/functional/shrink_cephfsmirror/hosts new file mode 100644 index 0000000000..de243961b8 --- /dev/null +++ b/tests/functional/shrink_cephfsmirror/hosts @@ -0,0 +1,8 @@ +[mons] +mon0 + +[osds] +osd0 + +[cephfsmirrors] +cephfs-mirror0 ansible_host=client0 diff --git a/tests/functional/shrink_cephfsmirror/vagrant_variables.yml b/tests/functional/shrink_cephfsmirror/vagrant_variables.yml new file mode 100644 index 0000000000..0ce26c837b --- /dev/null +++ b/tests/functional/shrink_cephfsmirror/vagrant_variables.yml @@ -0,0 +1,73 @@ +--- + +# DEPLOY CONTAINERIZED DAEMONS +docker: false + +# DEFINE THE NUMBER OF VMS TO RUN +mon_vms: 1 +osd_vms: 1 +mds_vms: 0 +rgw_vms: 0 +nfs_vms: 0 +grafana_server_vms: 0 +rbd_mirror_vms: 0 +client_vms: 1 +mgr_vms: 0 + +# INSTALL SOURCE OF CEPH +# valid values are 'stable' and 'dev' +ceph_install_source: stable + +# SUBNETS TO USE FOR THE VMS +public_subnet: 192.168.85 +cluster_subnet: 192.168.86 + +# MEMORY +# set 1024 for CentOS +memory: 1024 + +# Ethernet interface name +# use eth1 for libvirt and ubuntu precise, enp0s8 for CentOS and ubuntu xenial +eth: 'eth1' + +# Disks +# For libvirt use disks: "[ '/dev/vdb', '/dev/vdc' ]" +# For CentOS7 use disks: "[ '/dev/sda', '/dev/sdb' ]" +disks: "[ '/dev/sdb', '/dev/sdc' ]" + +# VAGRANT BOX +# Ceph boxes are *strongly* suggested. They are under better control and will +# not get updated frequently unless required for build systems. These are (for +# now): +# +# * ceph/ubuntu-xenial +# +# Ubuntu: ceph/ubuntu-xenial bento/ubuntu-16.04 or ubuntu/trusty64 or ubuntu/wily64 +# CentOS: bento/centos-7.1 or puppetlabs/centos-7.0-64-puppet +# libvirt CentOS: centos/7 +# parallels Ubuntu: parallels/ubuntu-14.04 +# Debian: deb/jessie-amd64 - be careful the storage controller is named 'SATA Controller' +# For more boxes have a look at: +# - https://atlas.hashicorp.com/boxes/search?utf8=✓&sort=&provider=virtualbox&q= +# - https://download.gluster.org/pub/gluster/purpleidea/vagrant/ +vagrant_box: centos/stream9 +#ssh_private_key_path: "~/.ssh/id_rsa" +# The sync directory changes based on vagrant box +# Set to /home/vagrant/sync for Centos/7, /home/{ user }/vagrant for openstack and defaults to /vagrant +#vagrant_sync_dir: /home/vagrant/sync +vagrant_sync_dir: /vagrant +# Disables synced folder creation. Not needed for testing, will skip mounting +# the vagrant directory on the remote box regardless of the provider. +vagrant_disable_synced_folder: true +# VAGRANT URL +# This is a URL to download an image from an alternate location. vagrant_box +# above should be set to the filename of the image. +# Fedora virtualbox: https://download.fedoraproject.org/pub/fedora/linux/releases/22/Cloud/x86_64/Images/Fedora-Cloud-Base-Vagrant-22-20150521.x86_64.vagrant-virtualbox.box +# Fedora libvirt: https://download.fedoraproject.org/pub/fedora/linux/releases/22/Cloud/x86_64/Images/Fedora-Cloud-Base-Vagrant-22-20150521.x86_64.vagrant-libvirt.box +# vagrant_box_url: https://download.fedoraproject.org/pub/fedora/linux/releases/22/Cloud/x86_64/Images/Fedora-Cloud-Base-Vagrant-22-20150521.x86_64.vagrant-virtualbox.box + +os_tuning_params: + - { name: fs.file-max, value: 26234859 } + +# VM prefix name, need to match the hostname +# label_prefix: ceph diff --git a/tests/functional/subset_update/container/hosts b/tests/functional/subset_update/container/hosts index 8823f1a85b..ccd86cf576 100644 --- a/tests/functional/subset_update/container/hosts +++ b/tests/functional/subset_update/container/hosts @@ -15,3 +15,6 @@ osd2 [rgws] rgw0 rgw1 + +[cephfsmirrors] +cephfs-mirror0 ansible_host=client0 diff --git a/tests/functional/subset_update/container/vagrant_variables.yml b/tests/functional/subset_update/container/vagrant_variables.yml index cd342bd2d3..3a25151d64 100644 --- a/tests/functional/subset_update/container/vagrant_variables.yml +++ b/tests/functional/subset_update/container/vagrant_variables.yml @@ -11,7 +11,7 @@ rgw_vms: 2 nfs_vms: 0 grafana_server_vms: 0 rbd_mirror_vms: 0 -client_vms: 0 +client_vms: 1 mgr_vms: 0 # SUBNETS TO USE FOR THE VMS diff --git a/tests/functional/subset_update/hosts b/tests/functional/subset_update/hosts index ce63629468..ccd86cf576 100644 --- a/tests/functional/subset_update/hosts +++ b/tests/functional/subset_update/hosts @@ -16,3 +16,5 @@ osd2 rgw0 rgw1 +[cephfsmirrors] +cephfs-mirror0 ansible_host=client0 diff --git a/tests/functional/subset_update/vagrant_variables.yml b/tests/functional/subset_update/vagrant_variables.yml index f6331dda54..01655ff50f 100644 --- a/tests/functional/subset_update/vagrant_variables.yml +++ b/tests/functional/subset_update/vagrant_variables.yml @@ -11,7 +11,7 @@ rgw_vms: 2 nfs_vms: 0 grafana_server_vms: 0 rbd_mirror_vms: 0 -client_vms: 0 +client_vms: 1 mgr_vms: 0 # INSTALL SOURCE OF CEPH diff --git a/tests/functional/tests/cephfs-mirror/__init__.py b/tests/functional/tests/cephfs-mirror/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/functional/tests/cephfs-mirror/test_cephfs_mirror.py b/tests/functional/tests/cephfs-mirror/test_cephfs_mirror.py new file mode 100644 index 0000000000..05a17ae630 --- /dev/null +++ b/tests/functional/tests/cephfs-mirror/test_cephfs_mirror.py @@ -0,0 +1,29 @@ +import pytest + + +class TestCephFSMirror(object): + + @pytest.mark.cephfsmirrors + @pytest.mark.no_docker + def test_cephfs_mirror_is_installed(self, host): + assert host.package("cephfs-mirror").is_installed + + @pytest.mark.cephfsmirrors + def test_cephfs_mirror_service_enabled_and_running(self, node, host): + daemon_id = node["vars"].get("ceph_cephfs_mirror_daemon_id", "mirror") + service_name = ( + f"ceph-cephfs-mirror@{daemon_id}" + if node["docker"] + else f"cephfs-mirror@{daemon_id}" + ) + service = host.service(service_name) + assert service.is_enabled + assert service.is_running + + @pytest.mark.cephfsmirrors + def test_cephfs_mirror_keyring_exists(self, node, host, setup): + cluster = setup["cluster_name"] + daemon_id = node["vars"].get("ceph_cephfs_mirror_daemon_id", "mirror") + keyring = host.file(f"/etc/ceph/{cluster}.client.{daemon_id}.keyring") + assert keyring.exists + assert keyring.is_file diff --git a/tests/pytest.ini b/tests/pytest.ini index d4c15634be..227860fd46 100644 --- a/tests/pytest.ini +++ b/tests/pytest.ini @@ -14,5 +14,6 @@ markers = nfss: for nfs nodes osds: for osd nodes rbdmirrors: for rbdmirror nodes + cephfsmirrors: for cephfs-mirror nodes rgws: for rgw nodes grafanas: for grafana nodes diff --git a/tox-subset_update.ini b/tox-subset_update.ini index f4b6e0f213..3489bfb87f 100644 --- a/tox-subset_update.ini +++ b/tox-subset_update.ini @@ -108,6 +108,15 @@ commands= ceph_docker_registry_username={env:DOCKER_HUB_USERNAME} \ ceph_docker_registry_password={env:DOCKER_HUB_PASSWORD} \ " +# upgrade cephfs-mirror daemons + ansible-playbook -vv --diff -i {changedir}/{env:INVENTORY} {toxinidir}/infrastructure-playbooks/rolling_update.yml --tags=cephfsmirrors --extra-vars "\ + ireallymeanit=yes \ + ceph_dev_branch={env:UPDATE_CEPH_DEV_BRANCH:main} \ + ceph_dev_sha1={env:UPDATE_CEPH_DEV_SHA1:latest} \ + ceph_docker_registry_auth=True \ + ceph_docker_registry_username={env:DOCKER_HUB_USERNAME} \ + ceph_docker_registry_password={env:DOCKER_HUB_PASSWORD} \ + " # post upgrade actions ansible-playbook -vv --diff -i {changedir}/{env:INVENTORY} {toxinidir}/infrastructure-playbooks/rolling_update.yml --tags=post_upgrade --extra-vars "\ ireallymeanit=yes \ diff --git a/tox.ini b/tox.ini index 32f2f9b09f..39498a01a8 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = {centos,ubuntu,rocky}-{container,non_container}-{all_daemons,all_daemons_ipv6,collocation,lvm_osds,shrink_mon,shrink_mgr,shrink_mds,shrink_rbdmirror,shrink_rgw,lvm_batch,add_mons,add_mgrs,add_mdss,add_rbdmirrors,add_rgws,purge,storage_inventory,lvm_auto_discovery,all_in_one,cephadm_adopt,purge_dashboard} +envlist = {centos,ubuntu,rocky}-{container,non_container}-{all_daemons,all_daemons_ipv6,collocation,lvm_osds,shrink_mon,shrink_mgr,shrink_mds,shrink_rbdmirror,shrink_cephfsmirror,shrink_rgw,lvm_batch,add_mons,add_mgrs,add_mdss,add_rbdmirrors,add_rgws,purge,storage_inventory,lvm_auto_discovery,all_in_one,cephadm_adopt,purge_dashboard} centos-non_container-{switch_to_containers} infra_lv_create migrate_ceph_disk_to_ceph_volume @@ -150,6 +150,13 @@ commands= rbdmirror_to_kill={env:RBDMIRROR_TO_KILL:rbd-mirror0} \ " +[shrink-cephfsmirror] +commands= + ansible-playbook -vv --diff -i {changedir}/{env:INVENTORY} {toxinidir}/infrastructure-playbooks/shrink-cephfsmirror.yml --extra-vars "\ + ireallymeanit=yes \ + cephfsmirror_to_kill={env:CEPHFSMIRROR_TO_KILL:cephfs-mirror0} \ + " + [shrink-rgw] commands= ansible-playbook -vv --diff -i {changedir}/{env:INVENTORY} {toxinidir}/infrastructure-playbooks/shrink-rgw.yml --extra-vars "\ @@ -297,6 +304,7 @@ setenv= shrink_mgr: MGR_TO_KILL = mgr1 shrink_mon: MON_TO_KILL = mon2 shrink_rbdmirror: RBDMIRROR_TO_KILL = rbd-mirror0 + shrink_cephfsmirror: CEPHFSMIRROR_TO_KILL = cephfs-mirror0 shrink_rgw: RGW_TO_KILL = rgw0.rgw0 CEPH_DOCKER_IMAGE_TAG = latest-main @@ -315,6 +323,7 @@ changedir= shrink_mgr: {toxinidir}/tests/functional/shrink_mgr{env:CONTAINER_DIR:} shrink_mds: {toxinidir}/tests/functional/shrink_mds{env:CONTAINER_DIR:} shrink_rbdmirror: {toxinidir}/tests/functional/shrink_rbdmirror{env:CONTAINER_DIR:} + shrink_cephfsmirror: {toxinidir}/tests/functional/shrink_cephfsmirror{env:CONTAINER_DIR:} shrink_rgw: {toxinidir}/tests/functional/shrink_rgw{env:CONTAINER_DIR:} # tests a 1 mon, 1 osd, 1 mds and 1 rgw centos7 cluster using docker collocation: {toxinidir}/tests/functional/collocation{env:CONTAINER_DIR:} @@ -376,6 +385,7 @@ commands= shrink_mgr: {[shrink-mgr]commands} shrink_mds: {[shrink-mds]commands} shrink_rbdmirror: {[shrink-rbdmirror]commands} + shrink_cephfsmirror: {[shrink-cephfsmirror]commands} shrink_rgw: {[shrink-rgw]commands} add_mons: {[add-mons]commands} add_mgrs: {[add-mgrs]commands}