diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 56865aca..1b6794ac 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -43,7 +43,7 @@ concurrency: jobs: molecule: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 strategy: max-parallel: 2 fail-fast: false @@ -54,13 +54,15 @@ jobs: os: - rockylinux8 - ubuntu20 + - ubuntu22 + # Single ansible-core version for all tests; use one supported on Jammy (focal/jammy/noble generations) ansible-core: - - "2.11.12" + - "2.18.12" # Only run centos7 for stable, as not valid for new release include: - st2_repo: stable os: centos7 - ansible-core: 2.11.12 + ansible-core: "2.18.12" steps: - name: Checkout the repository @@ -70,6 +72,7 @@ jobs: run: | python3 -m pip install --upgrade pip python3 -m pip install --upgrade "requests<=2.29.0" + python3 -m pip install --upgrade "ansible-compat>=2.2.0,<3" python3 -m pip install --upgrade ansible-core==${{ matrix.ansible-core }} python3 -m pip install --upgrade molecule==${{ env.MOLECULE_VERSION }} python3 -m pip install --upgrade molecule-docker==${{ env.MOLECULE_DOCKER_VERSION }} @@ -86,7 +89,7 @@ jobs: name: Slack notification for failed master builds if: always() needs: [molecule] - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - name: Workflow conclusion # this step creates an environment variable WORKFLOW_CONCLUSION and is the most reliable way to check the status of previous jobs diff --git a/.gitignore b/.gitignore index 6e53cc9d..7704691d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,17 @@ +# CI-matching molecule venv (Python 3.11+, ansible-core 2.18) +.venv-ci/ + # KitchenCI .kitchen/ .kitchen.local.yml +build/kitchen/.bundle/ # Ansible Retry *.retry +# Ansible local files +.ansible/ + # Vagrant .vagrant/ @@ -12,4 +19,5 @@ *.log # Virtual Environment -.venv \ No newline at end of file +.venv +.venv-molecule/ \ No newline at end of file diff --git a/.tool-versions b/.tool-versions new file mode 100644 index 00000000..bf031553 --- /dev/null +++ b/.tool-versions @@ -0,0 +1,2 @@ +# Match E2E CI: ansible-core 2.18.x requires Python 3.11+ +python 3.11.12 diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..b807b994 --- /dev/null +++ b/Makefile @@ -0,0 +1,92 @@ +# Match E2E CI (ubuntu-22.04, ansible-core 2.18.12, molecule 3.6.1, molecule-docker 1.1.0). +# Requires Python 3.11+ (ansible-core 2.18.x). +# +# Internal CA: set EXTRA_CA_BUNDLE to your CA cert (e.g. ~/SammyCA.pem) so pip and +# molecule trust internal hosts. The Makefile builds a combined bundle (system + extra) +# and sets SSL_CERT_FILE for the venv. Example: +# make molecule-env EXTRA_CA_BUNDLE=~/SammyCA.pem + +PYTHON ?= python3.11 +VENV := .venv-ci +ANSIBLE_CORE := 2.18.12 +MOLECULE_VER := 3.6.1 +MOLECULE_DOCKER_VER := 1.1.0 + +# Combined CA bundle when using internal CA (system/certifi + EXTRA_CA_BUNDLE) +CA_BUNDLE := $(VENV)/.ca-bundle.crt + +.PHONY: molecule-env molecule-test molecule-converge molecule-destroy help ca-bundle + +help: + @echo "Targets:" + @echo " make molecule-env Create/update $(VENV) with CI deps (Python 3.11+, ansible-core $(ANSIBLE_CORE))." + @echo " make molecule-test Run molecule test (default SCENARIO=ubuntu22)." + @echo " make molecule-converge Run molecule converge for SCENARIO." + @echo " make molecule-destroy Run molecule destroy for SCENARIO." + @echo "Override: SCENARIO=ubuntu20 PYTHON=python3.12 EXTRA_CA_BUNDLE=~/SammyCA.pem" + +# Build combined CA bundle so SSL verifies both public and internal (e.g. Artifactory) certs. +# Uses system/certifi default bundle + EXTRA_CA_BUNDLE. Only built when EXTRA_CA_BUNDLE is set. +$(CA_BUNDLE): + @mkdir -p $(VENV) + @if [ -n "$(EXTRA_CA_BUNDLE)" ]; then \ + extra="$(EXTRA_CA_BUNDLE)"; \ + case "$$extra" in ~*) extra="$(HOME)$${extra#\~}";; esac; \ + [ -f "$$extra" ] || { echo "EXTRA_CA_BUNDLE not found: $$extra"; exit 1; }; \ + if [ ! -f "$$extra" ]; then echo "EXTRA_CA_BUNDLE not found: $(EXTRA_CA_BUNDLE)"; exit 1; fi; \ + default=$$($(PYTHON) -c "import ssl, os; p=ssl.get_default_verify_paths(); f=getattr(p,'openssl_cafile',None); print(f if f and os.path.isfile(f) else '')" 2>/dev/null); \ + [ -n "$$default" ] || default=$$($(PYTHON) -c "import certifi; print(certifi.where())" 2>/dev/null); \ + [ -n "$$default" ] || { echo "Could not find system CA bundle (install certifi?)"; exit 1; }; \ + cat "$$default" "$$extra" > $(CA_BUNDLE); \ + echo "Using CA bundle: $(CA_BUNDLE) (system + $$extra)"; \ + else \ + : > $(CA_BUNDLE); \ + fi + +$(VENV)/.stamp: requirements-ci.txt $(CA_BUNDLE) + $(PYTHON) -c 'import sys; assert sys.version_info >= (3, 11), "Need Python 3.11+ (ansible-core 2.18)"' + $(PYTHON) -m venv $(VENV) + @if [ -n "$(EXTRA_CA_BUNDLE)" ] && [ -f $(CA_BUNDLE) ] && [ -s $(CA_BUNDLE) ]; then \ + export SSL_CERT_FILE="$(CURDIR)/$(CA_BUNDLE)" REQUESTS_CA_BUNDLE="$(CURDIR)/$(CA_BUNDLE)"; \ + $(VENV)/bin/pip install --upgrade pip; \ + $(VENV)/bin/pip install -r requirements-ci.txt; \ + else \ + $(VENV)/bin/pip install --upgrade pip; \ + $(VENV)/bin/pip install -r requirements-ci.txt; \ + fi + @if [ -n "$(EXTRA_CA_BUNDLE)" ] && [ -f $(CA_BUNDLE) ] && [ -s $(CA_BUNDLE) ]; then \ + export SSL_CERT_FILE="$(CURDIR)/$(CA_BUNDLE)" REQUESTS_CA_BUNDLE="$(CURDIR)/$(CA_BUNDLE)"; \ + $(VENV)/bin/ansible-galaxy collection install -r test/requirements.yml; \ + else \ + $(VENV)/bin/ansible-galaxy collection install -r test/requirements.yml; \ + fi + touch $(VENV)/.stamp + +molecule-env: $(VENV)/.stamp + @echo "Use: $(VENV)/bin/molecule test --scenario-name " + @$(VENV)/bin/ansible --version + +# Ensure PATH uses CI venv so molecule and ansible-playbook both use .venv-ci +MOLECULE_PATH := $(CURDIR)/$(VENV)/bin:$(PATH) + +# Export CA bundle for molecule so Ansible/requests inside molecule also trust internal CA +molecule-test: molecule-env + @if [ -n "$(EXTRA_CA_BUNDLE)" ] && [ -f $(CA_BUNDLE) ] && [ -s $(CA_BUNDLE) ]; then \ + export SSL_CERT_FILE="$(CURDIR)/$(CA_BUNDLE)" REQUESTS_CA_BUNDLE="$(CURDIR)/$(CA_BUNDLE)"; \ + fi; \ + export PATH="$(MOLECULE_PATH)"; \ + $(VENV)/bin/molecule test --scenario-name $(or $(SCENARIO),ubuntu22) + +molecule-converge: molecule-env + @if [ -n "$(EXTRA_CA_BUNDLE)" ] && [ -f $(CA_BUNDLE) ] && [ -s $(CA_BUNDLE) ]; then \ + export SSL_CERT_FILE="$(CURDIR)/$(CA_BUNDLE)" REQUESTS_CA_BUNDLE="$(CURDIR)/$(CA_BUNDLE)"; \ + fi; \ + export PATH="$(MOLECULE_PATH)"; \ + $(VENV)/bin/molecule converge --scenario-name $(or $(SCENARIO),ubuntu22) + +molecule-destroy: molecule-env + @if [ -n "$(EXTRA_CA_BUNDLE)" ] && [ -f $(CA_BUNDLE) ] && [ -s $(CA_BUNDLE) ]; then \ + export SSL_CERT_FILE="$(CURDIR)/$(CA_BUNDLE)" REQUESTS_CA_BUNDLE="$(CURDIR)/$(CA_BUNDLE)"; \ + fi; \ + export PATH="$(MOLECULE_PATH)"; \ + $(VENV)/bin/molecule destroy --scenario-name $(or $(SCENARIO),ubuntu22) diff --git a/README.md b/README.md index 8744c765..95e25317 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@ ## Supported platforms * Ubuntu Focal (20.04) +* Ubuntu Jammy (22.04) * RHEL 7 and CentOS 7 * RHEL 8 and Rocky Linux 8 @@ -39,7 +40,7 @@ Below is the list of variables which you can redefine in your playbook, or inven | **st2repo** | `st2repo_name` | `stable` | StackStorm PackageCloud repository to install. [`stable`](https://packagecloud.io/StackStorm/stable/), [`unstable`](https://packagecloud.io/StackStorm/unstable/), [`staging-stable`](https://packagecloud.io/StackStorm/staging-stable/), [`staging-unstable`](https://packagecloud.io/StackStorm/staging-unstable/) | **st2** -| `st2_version` | `latest` | StackStorm version to install. `present` to install available package, `latest` to get automatic updates, or pin it to numeric version like `2.2.0` or with revision like `2.2.0-1` +| `st2_version` | `3.9.0` | StackStorm version to install. `present` to install available package, `latest` to get automatic updates, or pin it to numeric version like `3.9.0` or with revision like `3.9.0-1` | `st2_config` | `{}` | Hash with StackStorm configuration settings to set in [`st2.conf`](https://github.com/StackStorm/st2/blob/master/conf/st2.conf.sample) ini file. | `st2_system_user` | `stanley` | System user from which st2 will execute local/remote shell actions. | `st2_system_user_in_sudoers` | `yes`| Add `st2_system_user` to the sudoers (recommended for most `st2` features to work). @@ -82,14 +83,14 @@ echo "stackstorm.example.com" > inventory ansible-playbook --inventory inventory stackstorm.yml ``` -> Keeping the `latest` version is useful to update StackStorm by re-running the playbook, since it will reinstall (upgrade) st2 when there is new version available. +> The default installs StackStorm 3.9.0. To get automatic updates on re-run, set `st2_version=latest`. If you want a different pinned version, pass it via `--extra-vars`. This is default behavior. If you do not want updates, consider pinning specific version and revision numbers. Install a specific version of st2 with pinned revision number: ```sh -ansible-playbook stackstorm.yml --extra-vars "st2_version=2.2.0 st2_revision=8" +ansible-playbook stackstorm.yml --extra-vars='st2_version=3.9.0' ``` ## Installing behind a proxy @@ -110,6 +111,7 @@ There are a few requirements when developing on `ansible-st2`. These are the platforms we must support (must pass end-to-end testing): * Ubuntu Focal (20.04) +* Ubuntu Jammy (22.04) * CentOS 7 * Rocky Linux 8 * RHEL 7 (via AWS) @@ -118,8 +120,28 @@ These are the platforms we must support (must pass end-to-end testing): The playbooks and roles must support Ansible idempotence. That is, re-running the `stackstorm.yml` playbook must end with the result `changed=0.*failed=0` (provided that versions and revisions are pinned). +### Molecule (E2E) tests + +To run the same E2E tests as CI locally (Python 3.11+, ansible-core 2.18.12, molecule 3.6.1): + +```sh +make molecule-env # one-time: create .venv-ci and install deps +make molecule-test # run full test (default: ubuntu22) +``` + +Override the scenario: `make molecule-test SCENARIO=ubuntu20`. The project uses [asdf](https://asdf-vm.com/) `.tool-versions` for Python 3.11; if you use asdf, run `asdf install` in the repo first. + +**Internal CA (e.g. Artifactory):** If pip or Ansible must trust an internal CA, pass your PEM file so the Makefile builds a combined bundle (system + your CA) and uses it for SSL: + +```sh +make molecule-env EXTRA_CA_BUNDLE=~/SammyCA.pem +make molecule-test EXTRA_CA_BUNDLE=~/SammyCA.pem +``` + +You only need `EXTRA_CA_BUNDLE` on the same invocations where SSL is used (env creation and any molecule run that talks to internal hosts). + For local development, there is a [Vagrantfile](Vagrantfile) available. -By default, the following command will set up an ubuntu20 box (`ubuntu/focal64`): +By default, the following command will set up an ubuntu22 box (`ubuntu/jammy64`): ```sh vagrant up @@ -141,8 +163,8 @@ You may be interested in other methods to deploy StackStorm: * [Puppet Module](https://github.com/stackstorm/puppet-st2) * Manual Instructions - * [Ubuntu Bionic (18.04)](https://docs.stackstorm.com/install/u18.html) * [Ubuntu Focal (20.04)](https://docs.stackstorm.com/install/u20.html) + * [Ubuntu Jammy (22.04)](https://docs.stackstorm.com/install/u22.html) * [RHEL 8 / Rocky Linux 8](https://docs.stackstorm.com/install/rhel8.html) * [RHEL 7 / CentOS 7](https://docs.stackstorm.com/install/rhel7.html) diff --git a/Vagrantfile b/Vagrantfile index 9568f1a6..edb2a44d 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -6,6 +6,10 @@ VIRTUAL_MACHINES = { :hostname => 'ansible-st2-ubuntu20', :box => 'ubuntu/focal64', }, + :ubuntu22 => { + :hostname => 'ansible-st2-ubuntu22', + :box => 'ubuntu/jammy64', + }, :centos7 => { :hostname => 'ansible-st2-centos7', :box => 'centos/7', diff --git a/meta/main.yml b/meta/main.yml index dbfeb844..5801e320 100644 --- a/meta/main.yml +++ b/meta/main.yml @@ -17,6 +17,7 @@ galaxy_info: - name: Ubuntu versions: - focal + - jammy - name: EL versions: - 7 diff --git a/molecule/ubuntu22/Dockerfile.ubuntu22.j2 b/molecule/ubuntu22/Dockerfile.ubuntu22.j2 new file mode 100644 index 00000000..1c7e2255 --- /dev/null +++ b/molecule/ubuntu22/Dockerfile.ubuntu22.j2 @@ -0,0 +1,5 @@ +FROM stackstorm/packagingtest:jammy-systemd + +RUN set -eu \ + && useradd --create-home --user-group {{ item.username }} \ + && echo "{{ item.username }} ALL=(ALL) NOPASSWD: ALL" >/etc/sudoers.d/{{ item.username }} diff --git a/molecule/ubuntu22/molecule.yml b/molecule/ubuntu22/molecule.yml new file mode 100644 index 00000000..225fcdeb --- /dev/null +++ b/molecule/ubuntu22/molecule.yml @@ -0,0 +1,62 @@ +--- +dependency: + name: galaxy + +driver: + name: docker + +platforms: + - name: ubuntu-22.04 + groups: [test] + pre_build_image: false + image: stackstorm/packagingtest:jammy-systemd + dockerfile: Dockerfile.ubuntu22.j2 + privileged: true + command: /sbin/init + # /tmp left off tmpfs to avoid remote_tmp mkdir/echo issues in container + tmpfs: [/run] + volumes: [/sys/fs/cgroup:/sys/fs/cgroup:rw] + cgroupns_mode: host + username: molecule + +provisioner: + name: ansible + env: + ANSIBLE_ROLES_PATH: ../../roles + playbooks: + converge: ../../stackstorm.yml + inventory: + group_vars: + test: + ansible_user: molecule + st2_auth_username: testu + st2_auth_password: testp + st2chatops_hubot_adapter: slack + st2chatops_config: + HUBOT_SLACK_TOKEN: "{{ lookup('ansible.builtin.env', 'HUBOT_SLACK_TOKEN') }}" + st2_rbac_enable: true + +verifier: + name: ansible + +scenario: + create_sequence: + - create + check_sequence: + - destroy + - create + - converge + - check + - destroy + converge_sequence: + - create + - converge + destroy_sequence: + - destroy + test_sequence: + - destroy + - syntax + - create + - converge + - idempotence + - destroy diff --git a/requirements-ci.txt b/requirements-ci.txt new file mode 100644 index 00000000..ecc0fa00 --- /dev/null +++ b/requirements-ci.txt @@ -0,0 +1,7 @@ +# Match .github/workflows/e2e.yaml (ansible-core 2.18.12, Python 3.11+) +# ansible-compat 3+ renamed runtime.exec -> run; molecule 3.6.1 needs .exec +requests<=2.29.0 +ansible-compat>=2.2.0,<3 +ansible-core==2.18.12 +molecule==3.6.1 +molecule-docker==1.1.0 diff --git a/roles/StackStorm.mongodb/meta/main.yml b/roles/StackStorm.mongodb/meta/main.yml index 92381916..4d6c7ef1 100644 --- a/roles/StackStorm.mongodb/meta/main.yml +++ b/roles/StackStorm.mongodb/meta/main.yml @@ -9,6 +9,7 @@ galaxy_info: - name: Ubuntu versions: - focal + - jammy - name: EL versions: - 7 diff --git a/roles/StackStorm.nginx/meta/main.yml b/roles/StackStorm.nginx/meta/main.yml index 33de2c73..8c25dc26 100644 --- a/roles/StackStorm.nginx/meta/main.yml +++ b/roles/StackStorm.nginx/meta/main.yml @@ -10,6 +10,7 @@ galaxy_info: - name: Ubuntu versions: - focal + - jammy - name: EL versions: - 7 diff --git a/roles/StackStorm.nodejs/meta/main.yml b/roles/StackStorm.nodejs/meta/main.yml index 56629a01..3ba60fd5 100644 --- a/roles/StackStorm.nodejs/meta/main.yml +++ b/roles/StackStorm.nodejs/meta/main.yml @@ -10,6 +10,7 @@ galaxy_info: - name: Ubuntu versions: - focal + - jammy - name: EL versions: - 7 diff --git a/roles/StackStorm.rabbitmq/meta/main.yml b/roles/StackStorm.rabbitmq/meta/main.yml index 68674674..7dcd5424 100644 --- a/roles/StackStorm.rabbitmq/meta/main.yml +++ b/roles/StackStorm.rabbitmq/meta/main.yml @@ -9,6 +9,7 @@ galaxy_info: - name: Ubuntu versions: - focal + - jammy - name: EL versions: - 7 diff --git a/roles/StackStorm.redis/meta/main.yml b/roles/StackStorm.redis/meta/main.yml index 2f4c75af..10416010 100644 --- a/roles/StackStorm.redis/meta/main.yml +++ b/roles/StackStorm.redis/meta/main.yml @@ -9,6 +9,7 @@ galaxy_info: - name: Ubuntu versions: - focal + - jammy - name: EL versions: - 7 diff --git a/roles/StackStorm.st2/defaults/main.yml b/roles/StackStorm.st2/defaults/main.yml index 43fbeb70..28da6ac5 100644 --- a/roles/StackStorm.st2/defaults/main.yml +++ b/roles/StackStorm.st2/defaults/main.yml @@ -1,7 +1,7 @@ --- -# StackStorm version to install. `present` to install available package, `latest` to get automatic updates or pin it to numeric version like `2.2.0`. -# On Debian need to specify revision of package like `2.2.0-1`. -st2_version: latest +# StackStorm version to install. `present` to install available package, `latest` to get automatic updates or pin it to numeric version like `3.9.0`. +# On Debian need to specify revision of package like `3.9.0-1`. +st2_version: "3.9.0" # Hash with StackStorm configuration settings to set in 'st2.conf' ini file # See https://github.com/StackStorm/st2/blob/master/conf/st2.conf.sample for a full list diff --git a/roles/StackStorm.st2/meta/main.yml b/roles/StackStorm.st2/meta/main.yml index b26cbebe..6debae3f 100644 --- a/roles/StackStorm.st2/meta/main.yml +++ b/roles/StackStorm.st2/meta/main.yml @@ -9,6 +9,7 @@ galaxy_info: - name: Ubuntu versions: - focal + - jammy - name: EL versions: - 7 diff --git a/roles/StackStorm.st2/vars/debian_22.yml b/roles/StackStorm.st2/vars/debian_22.yml new file mode 100644 index 00000000..2383fae6 --- /dev/null +++ b/roles/StackStorm.st2/vars/debian_22.yml @@ -0,0 +1,3 @@ +# List of python3 variables (Ubuntu 22.04 Jammy) +--- +passlib: python3-passlib diff --git a/roles/StackStorm.st2chatops/meta/main.yml b/roles/StackStorm.st2chatops/meta/main.yml index ee6e637c..164a2718 100644 --- a/roles/StackStorm.st2chatops/meta/main.yml +++ b/roles/StackStorm.st2chatops/meta/main.yml @@ -9,6 +9,7 @@ galaxy_info: - name: Ubuntu versions: - focal + - jammy - name: EL versions: - 7 diff --git a/roles/StackStorm.st2repo/meta/main.yml b/roles/StackStorm.st2repo/meta/main.yml index 5413c211..d9e6e216 100644 --- a/roles/StackStorm.st2repo/meta/main.yml +++ b/roles/StackStorm.st2repo/meta/main.yml @@ -9,6 +9,7 @@ galaxy_info: - name: Ubuntu versions: - focal + - jammy - name: EL versions: - 7 diff --git a/roles/StackStorm.st2smoketests/meta/main.yml b/roles/StackStorm.st2smoketests/meta/main.yml index d55f9f06..17a30ad7 100644 --- a/roles/StackStorm.st2smoketests/meta/main.yml +++ b/roles/StackStorm.st2smoketests/meta/main.yml @@ -9,6 +9,7 @@ galaxy_info: - name: Ubuntu versions: - focal + - jammy - name: EL versions: - 7 diff --git a/roles/StackStorm.st2web/meta/main.yml b/roles/StackStorm.st2web/meta/main.yml index 53bc4516..14afb7ce 100644 --- a/roles/StackStorm.st2web/meta/main.yml +++ b/roles/StackStorm.st2web/meta/main.yml @@ -10,6 +10,7 @@ galaxy_info: - name: Ubuntu versions: - focal + - jammy - name: EL versions: - 7