diff --git a/CHANGELOG.md b/CHANGELOG.md index e43c0a4d..06d07cc7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +* **plugin:graylog_input, plugin:graylog_index_set**: New in-house modules that drive the Graylog REST API directly. They support `--check` (dry-run preview) and `--diff` (per-field before/after), report `changed=true` only when a field actually differs, and replace the previous `ansible.builtin.uri` chains in the `graylog_server` role. * **plugin:platform_select**: New filter plugin for selecting a value from a platform-keyed dictionary by OS family / distribution / version. * **role:alternatives**: Support managing `subcommands` (slaves/followers) and the Red Hat-only `family` grouping. The role now also ensures the alternatives tooling is installed (`chkconfig` on RHEL 8, `alternatives` on RHEL 9/10; bundled with `dpkg` on Debian/Ubuntu), and can be included without variables as a no-op. * **role:redis**: Add template for version 8.8 @@ -31,6 +32,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Breaking Changes +* **role:graylog_server**: System inputs and index sets now follow the standard `__group_var` / `__host_var` pattern with `state: present/absent` per entry, and the on-demand tag `graylog_server:configure_defaults` has been split into `graylog_server:configure_system_inputs` and `graylog_server:configure_system_index_sets`. The role no longer ships a default index set. Operators must define at least one index set in the inventory and mark one as `default: true`. The four default system inputs (Beats, GELF TCP, GELF UDP, Syslog UDP) are preserved in `graylog_server__system_inputs__role_var`. Migration: rename `graylog_server__system_default_index_set` (dict) to a list entry under `graylog_server__system_index_sets__host_var` (or `__group_var`), add `default: true` and `state: 'present'`. Rename `graylog_server__system_inputs` (list) to `graylog_server__system_inputs__host_var` (or `__group_var`); add `state: 'present'` to each entry. * **role:minio_client, role:objectstore_backup**: Both roles and their playbooks (`playbooks/minio_client.yml`, `playbooks/objectstore_backup.yml`) have been removed, along with the corresponding role blocks in `playbooks/setup_nextcloud.yml` and the `setup_nextcloud__skip_minio_client` / `setup_nextcloud__skip_objectstore_backup` variables. MinIO Server has been archived as no-longer-maintained since February 2026, and we are moving away from using object storage for critical data. Users relying on these roles must replace the MinIO-based object-store backup with their own solution (e.g. `rclone`); the `mc` binary, its config under `/etc/mc/`, the `objectstore-backup` systemd timer/service, and `/usr/local/bin/mc-mirror.sh` are no longer managed by lfops and will remain on existing hosts until removed manually ([#241](https://github.com/Linuxfabrik/lfops/issues/241)). * **role:infomaniak_vm**: Always create a managed port for every entry in `infomaniak_vm__networks`, even when no `fixed_ip` is set. Previously only networks with a `fixed_ip` got a managed port; networks without one relied on OpenStack's auto-created port. To avoid creating unused (but billed) managed ports on VMs provisioned under the old behavior, make sure to manually rename the existing port in OpenStack to match the `port_name`. Note that this port will not survive VM deletion / detachment, since it was automatically created and therefore is owned by OpenStack, not the user. diff --git a/plugins/module_utils/graylog.py b/plugins/module_utils/graylog.py new file mode 100644 index 00000000..a7e1ac8f --- /dev/null +++ b/plugins/module_utils/graylog.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8; py-indent-offset: 4 -*- +# +# Author: Linuxfabrik GmbH, Zurich, Switzerland +# Contact: info (at) linuxfabrik (dot) ch +# https://www.linuxfabrik.ch/ +# License: The Unlicense, see LICENSE file. + +"""Shared client and diff helpers for the linuxfabrik.lfops.graylog_* modules. + +Wraps `ansible.module_utils.urls.fetch_url` with the headers and auth shape +Graylog expects (basic auth, `X-Requested-By`, JSON bodies). All HTTP errors +surface as `GraylogAPIError` so the modules can fail uniformly. +""" + +from __future__ import absolute_import, division, print_function + +__metaclass__ = type + +import base64 +import json + +from ansible.module_utils.urls import fetch_url + + +DEFAULT_HEADERS = { + 'Accept': 'application/json', + 'X-Requested-By': 'ansible-lfops', +} + + +class GraylogAPIError(Exception): + """Raised on any non-2xx response from the Graylog API.""" + + def __init__(self, status, url, body): + self.status = status + self.url = url + self.body = body + super().__init__(f'HTTP {status} from {url}: {body}') + + +class GraylogClient: + """Thin REST client around `fetch_url`. One instance per module call.""" + + def __init__(self, module, url, username, password, validate_certs=True, timeout=30): + self._module = module + self._base = url.rstrip('/') + self._validate_certs = validate_certs + self._timeout = timeout + token = base64.b64encode(f'{username}:{password}'.encode('utf-8')).decode('ascii') + self._auth_header = f'Basic {token}' + + # --- low-level ------------------------------------------------------------ + + def _request(self, method, path, body=None, expected_status=None): + url = self._base + path + headers = dict(DEFAULT_HEADERS) + headers['Authorization'] = self._auth_header + data = None + if body is not None: + headers['Content-Type'] = 'application/json' + data = json.dumps(body).encode('utf-8') + + resp, info = fetch_url( + self._module, + url, + data=data, + headers=headers, + method=method, + timeout=self._timeout, + ) + status = info.get('status', -1) + + if expected_status is not None: + allowed = expected_status if isinstance(expected_status, (list, tuple, set)) else (expected_status,) + if status not in allowed: + raise GraylogAPIError(status, url, info.get('body') or info.get('msg') or '') + elif status < 200 or status >= 300: + raise GraylogAPIError(status, url, info.get('body') or info.get('msg') or '') + + if resp is None: + return None + raw = resp.read() + if not raw: + return None + try: + return json.loads(raw.decode('utf-8')) + except (ValueError, AttributeError) as exc: + raise GraylogAPIError(status, url, f'invalid JSON: {exc}') + + # --- verbs ---------------------------------------------------------------- + + def get(self, path, expected_status=200): + return self._request('GET', path, expected_status=expected_status) + + def post(self, path, body, expected_status=(200, 201)): + return self._request('POST', path, body=body, expected_status=expected_status) + + def put(self, path, body=None, expected_status=(200, 201, 204)): + return self._request('PUT', path, body=body, expected_status=expected_status) + + def delete(self, path, expected_status=(200, 204)): + return self._request('DELETE', path, expected_status=expected_status) + + +# --- diff helpers ------------------------------------------------------------ + + +def strip_keys(entry, keys): + """Return a shallow copy of `entry` with the given role-only `keys` removed. + + Replaces the role's `dict2items | rejectattr | items2dict` chain. + """ + if not entry: + return {} + return {k: v for k, v in entry.items() if k not in keys} + + +def _equal(a, b): + """Recursive equality. Treats dicts and lists structurally; falls back to + `==` for scalars. Used by `diff_changed_fields` so nested config dicts + (e.g. Graylog input `configuration`, index-set `data_tiering`) compare + correctly. + """ + if isinstance(a, dict) and isinstance(b, dict): + if set(a.keys()) != set(b.keys()): + return False + return all(_equal(a[k], b[k]) for k in a) + if isinstance(a, list) and isinstance(b, list): + if len(a) != len(b): + return False + return all(_equal(x, y) for x, y in zip(a, b)) + return a == b + + +def diff_changed_fields(current, desired, ignore=None): + """Return `(before, after)` containing only the keys whose values differ. + + Keys present in `desired` are checked against `current`; keys not in + `desired` are not compared (we never want to "unset" a field the user + did not mention). Keys listed in `ignore` are skipped entirely. Suitable + for an Ansible `result["diff"]` payload that highlights just the user- + visible delta on update. + """ + if ignore is None: + ignore = () + before = {} + after = {} + current = current or {} + for key, desired_value in (desired or {}).items(): + if key in ignore: + continue + current_value = current.get(key) + if not _equal(current_value, desired_value): + before[key] = current_value + after[key] = desired_value + return before, after diff --git a/plugins/modules/graylog_index_set.py b/plugins/modules/graylog_index_set.py new file mode 100644 index 00000000..029f7659 --- /dev/null +++ b/plugins/modules/graylog_index_set.py @@ -0,0 +1,390 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8; py-indent-offset: 4 -*- +# +# Author: Linuxfabrik GmbH, Zurich, Switzerland +# Contact: info (at) linuxfabrik (dot) ch +# https://www.linuxfabrik.ch/ +# License: The Unlicense, see LICENSE file. + +from __future__ import absolute_import, division, print_function + +__metaclass__ = type + +DOCUMENTATION = r''' +--- +module: graylog_index_set +short_description: Create, update or delete a Graylog index set +version_added: 'in the next LFOps release.' +description: + - Manages a single Graylog index set end-to-end (create, update, delete, set-as-default) against the Graylog REST API. + - Identification is by I(index_prefix); the value must be unique within the Graylog instance. Re-running the task with the same I(index_prefix) updates the existing index set in place and reports C(changed=true) only when one of the diffable fields actually differs. + - I(index_prefix) is immutable after creation; changing it in inventory produces a new index set rather than renaming the existing one. + - When I(default=true), the module additionally calls C(PUT /api/system/indices/index_sets/{id}/default) after a successful create/update. The default flag itself does not need to be sent to the create/update endpoint and is only used by this module to drive the extra call. + - Supports check mode (C(--check)) and diff mode (C(--diff)). +author: + - Linuxfabrik GmbH, Zurich, Switzerland (info (at) linuxfabrik (dot) ch) +requirements: + - python >= 3.6 +options: + url: + description: Base URL of the Graylog HTTP interface, e.g. C(http://127.0.0.1:9000). The C(/api) prefix is appended automatically. + type: str + required: true + username: + description: Username for HTTP basic authentication against the Graylog API. + type: str + required: true + password: + description: Password for HTTP basic authentication. + type: str + required: true + no_log: true + validate_certs: + description: Whether to verify the TLS certificate of the Graylog endpoint. + type: bool + required: false + default: true + index_prefix: + description: Unique prefix used in indices belonging to this index set. Identity key for create/update/delete. + type: str + required: true + title: + description: Human-readable name of the index set. Required on create. + type: str + required: false + description: + description: Free-text description. + type: str + required: false + shards: + description: Number of shards per index. Required on create. + type: int + required: false + replicas: + description: Number of replicas per index. Required on create. + type: int + required: false + writable: + description: Whether this index set is writable. + type: bool + required: false + index_analyzer: + description: Elasticsearch / OpenSearch analyzer. Required on create. + type: str + required: false + index_optimization_disabled: + description: Whether to skip force-merge after rotation. + type: bool + required: false + index_optimization_max_num_segments: + description: Maximum number of segments per index after optimization. + type: int + required: false + field_type_refresh_interval: + description: Field-type refresh interval in milliseconds. + type: int + required: false + use_legacy_rotation: + description: C(false) (recommended) uses the I(data_tiering) field. C(true) falls back to the legacy I(rotation_strategy) / I(retention_strategy) configuration, which Graylog has announced will be deprecated. + type: bool + required: false + default: false + data_tiering: + description: Data-tiering configuration. Mandatory when I(use_legacy_rotation=false). + type: dict + required: false + rotation_strategy: + description: Legacy rotation strategy configuration. Mandatory when I(use_legacy_rotation=true). + type: dict + required: false + rotation_strategy_class: + description: Legacy rotation strategy class. Mandatory when I(use_legacy_rotation=true). + type: str + required: false + retention_strategy: + description: Legacy retention strategy configuration. Mandatory when I(use_legacy_rotation=true). + type: dict + required: false + retention_strategy_class: + description: Legacy retention strategy class. Mandatory when I(use_legacy_rotation=true). + type: str + required: false + default: + description: When C(true), also set this index set as the Graylog default after create/update. Exactly one index set should be marked as default across the inventory. + type: bool + required: false + default: false + state: + description: + - C(present) creates the index set when missing, updates it in place when present. + - C(absent) deletes the index set identified by I(index_prefix). When the index set does not exist, the module exits with C(changed=false). + type: str + choices: ['absent', 'present'] + default: 'present' +''' + +EXAMPLES = r''' +- name: 'Create the default catch-all index set' + linuxfabrik.lfops.graylog_index_set: + url: 'http://127.0.0.1:9000' + username: 'admin' + password: '{{ graylog_admin_password }}' + title: 'Default' + description: 'Default catch-all index set; 25-30 days - managed by Ansible - do not edit' + index_prefix: 'default' + shards: 1 + replicas: 0 + writable: true + index_analyzer: 'standard' + index_optimization_disabled: false + index_optimization_max_num_segments: 1 + field_type_refresh_interval: 5000 + use_legacy_rotation: false + data_tiering: + type: 'hot_only' + index_lifetime_min: 'P25D' + index_lifetime_max: 'P30D' + default: true + state: 'present' + +- name: 'Drop the access index set' + linuxfabrik.lfops.graylog_index_set: + url: 'http://127.0.0.1:9000' + username: 'admin' + password: '{{ graylog_admin_password }}' + index_prefix: 'access' + state: 'absent' + +# Preview a change without writing: +# ansible-playbook ... --check --diff --tags graylog_server:configure_system_index_sets +''' + +RETURN = r''' +index_set: + description: + - On create or update, the index-set object as returned by Graylog. On delete, the last known state of the index set. Empty dict when there was nothing to delete. + - In check mode, a synthetic preview reflecting what the run would have written. + type: dict + returned: always +diff: + description: Standard Ansible diff structure. C(before) / C(after) contain only the fields whose values actually change. + type: dict + returned: when changed +''' + + +from ansible.module_utils.basic import AnsibleModule +from ansible_collections.linuxfabrik.lfops.plugins.module_utils.graylog import ( + GraylogAPIError, + GraylogClient, + diff_changed_fields, +) + + +# Fields the user can set in the inventory. +_DIFF_FIELDS = ( + 'title', + 'description', + 'index_prefix', + 'shards', + 'replicas', + 'writable', + 'index_analyzer', + 'index_optimization_disabled', + 'index_optimization_max_num_segments', + 'field_type_refresh_interval', + 'use_legacy_rotation', + 'data_tiering', + 'rotation_strategy', + 'rotation_strategy_class', + 'retention_strategy', + 'retention_strategy_class', +) + +# Server-managed / read-only fields that come back in GET responses but are +# not user input. +_IGNORE_ON_DIFF = ( + 'id', + 'creation_date', + 'can_be_default', + 'index_template_type', + 'field_restrictions', + 'field_type_profile', + 'default', +) + + +def find_by_prefix(index_sets, prefix): + """Return the first index-set dict whose `index_prefix` matches, or None.""" + for item in index_sets or []: + if item.get('index_prefix') == prefix: + return item + return None + + +def normalize_current(item): + """Reduce a GET-side index-set dict to the fields we diff against.""" + if not item: + return {} + return {field: item.get(field) for field in _DIFF_FIELDS} + + +def build_desired(params): + """Return the dict we POST/PUT, with `None`/omitted params left out.""" + desired = {} + for field in _DIFF_FIELDS: + value = params.get(field) + if value is None: + continue + desired[field] = value + return desired + + +def main(): + argument_spec = { + 'url': {'type': 'str', 'required': True}, + 'username': {'type': 'str', 'required': True}, + 'password': {'type': 'str', 'required': True, 'no_log': True}, + 'validate_certs': {'type': 'bool', 'default': True}, + 'index_prefix': {'type': 'str', 'required': True}, + 'title': {'type': 'str'}, + 'description': {'type': 'str'}, + 'shards': {'type': 'int'}, + 'replicas': {'type': 'int'}, + 'writable': {'type': 'bool'}, + 'index_analyzer': {'type': 'str'}, + 'index_optimization_disabled': {'type': 'bool'}, + 'index_optimization_max_num_segments': {'type': 'int'}, + 'field_type_refresh_interval': {'type': 'int'}, + 'use_legacy_rotation': {'type': 'bool', 'default': False}, + 'data_tiering': {'type': 'dict'}, + 'rotation_strategy': {'type': 'dict'}, + 'rotation_strategy_class': {'type': 'str'}, + 'retention_strategy': {'type': 'dict'}, + 'retention_strategy_class': {'type': 'str'}, + 'default': {'type': 'bool', 'default': False}, + 'state': {'type': 'str', 'choices': ['absent', 'present'], 'default': 'present'}, + } + module = AnsibleModule( + argument_spec=argument_spec, + required_if=[ + ( + 'state', 'present', + ['title', 'description', 'shards', 'replicas', 'writable', + 'index_analyzer', 'index_optimization_disabled', + 'index_optimization_max_num_segments', + 'field_type_refresh_interval'], + False, + ), + ], + supports_check_mode=True, + ) + + state = module.params['state'] + if state == 'present': + if module.params['use_legacy_rotation']: + missing = [ + k for k in ('rotation_strategy', 'rotation_strategy_class', + 'retention_strategy', 'retention_strategy_class') + if module.params.get(k) is None + ] + if missing: + module.fail_json( + msg=f'use_legacy_rotation=true requires: {", ".join(missing)}', + ) + else: + if module.params.get('data_tiering') is None: + module.fail_json( + msg='use_legacy_rotation=false (the default) requires `data_tiering`.', + ) + + client = GraylogClient( + module, + url=module.params['url'], + username=module.params['username'], + password=module.params['password'], + validate_certs=module.params['validate_certs'], + ) + prefix = module.params['index_prefix'] + set_default = module.params['default'] + + try: + listing = client.get('/api/system/indices/index_sets') + except GraylogAPIError as exc: + module.fail_json(msg=f'Could not list index sets: {exc}') + + current = find_by_prefix((listing or {}).get('index_sets') or [], prefix) + current_compare = normalize_current(current) + desired = build_desired(module.params) + + # --- absent -------------------------------------------------------------- + if state == 'absent': + if current is None: + module.exit_json(changed=False, index_set={}, diff={'before': {}, 'after': {}}) + diff = {'before': current_compare, 'after': {}} + if module.check_mode: + module.exit_json(changed=True, index_set=current, diff=diff) + try: + client.delete(f"/api/system/indices/index_sets/{current['id']}", expected_status=204) + except GraylogAPIError as exc: + module.fail_json(msg=f'Could not delete index set {prefix!r}: {exc}') + module.exit_json(changed=True, index_set=current, diff=diff) + + # --- present, create ----------------------------------------------------- + if current is None: + diff = {'before': {}, 'after': dict(desired)} + if set_default: + diff['after']['default'] = True + if module.check_mode: + module.exit_json(changed=True, index_set=desired, diff=diff) + try: + created = client.post('/api/system/indices/index_sets', body=desired, expected_status=200) + except GraylogAPIError as exc: + module.fail_json(msg=f'Could not create index set {prefix!r}: {exc}') + if set_default and created and created.get('id'): + try: + client.put(f"/api/system/indices/index_sets/{created['id']}/default", expected_status=200) + except GraylogAPIError as exc: + module.fail_json(msg=f'Created {prefix!r} but failed to set it as default: {exc}') + module.exit_json(changed=True, index_set=created or desired, diff=diff) + + # --- present, update ----------------------------------------------------- + before, after = diff_changed_fields(current_compare, desired, ignore=_IGNORE_ON_DIFF) + # Track default-flip separately: GET returns `default: true|false`, the + # write endpoint does not accept it, so compare here and drive an extra + # PUT below if needed. + default_now = bool(current.get('default')) + default_change = set_default != default_now if set_default else False + if default_change: + before['default'] = default_now + after['default'] = True + + if not before and not after: + module.exit_json(changed=False, index_set=current, diff={'before': {}, 'after': {}}) + + diff = {'before': before, 'after': after} + if module.check_mode: + module.exit_json(changed=True, index_set=current, diff=diff) + + # Only PUT if non-default fields actually changed (i.e. anything other + # than the `default` flip, which is its own endpoint). + field_changes = {k: v for k, v in after.items() if k != 'default'} + if field_changes: + body = dict(desired) + body['id'] = current['id'] + try: + client.put(f"/api/system/indices/index_sets/{current['id']}", body=body, expected_status=200) + except GraylogAPIError as exc: + module.fail_json(msg=f'Updating index set {prefix!r} failed: {exc}') + + if default_change: + try: + client.put(f"/api/system/indices/index_sets/{current['id']}/default", expected_status=200) + except GraylogAPIError as exc: + module.fail_json(msg=f'Could not set index set {prefix!r} as default: {exc}') + + module.exit_json(changed=True, index_set=current, diff=diff) + + +if __name__ == '__main__': + main() diff --git a/plugins/modules/graylog_input.py b/plugins/modules/graylog_input.py new file mode 100644 index 00000000..b0055bdc --- /dev/null +++ b/plugins/modules/graylog_input.py @@ -0,0 +1,291 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8; py-indent-offset: 4 -*- +# +# Author: Linuxfabrik GmbH, Zurich, Switzerland +# Contact: info (at) linuxfabrik (dot) ch +# https://www.linuxfabrik.ch/ +# License: The Unlicense, see LICENSE file. + +from __future__ import absolute_import, division, print_function + +__metaclass__ = type + +DOCUMENTATION = r''' +--- +module: graylog_input +short_description: Create, update or delete a Graylog system input +version_added: 'in the next LFOps release.' +description: + - Manages a single Graylog system input end-to-end (create, update, delete) against the Graylog REST API. + - Identification is by I(title); the value must be unique within the Graylog instance. Re-running the task with the same I(title) updates the existing input in place and reports C(changed=true) only when one of the diffable fields actually differs. + - The C(port) inside I(configuration) cannot be changed once an input exists; changing it in inventory would update the existing entry in place via PUT, which the Graylog API rejects. Renaming I(title) is treated as "delete the old entry and create a new one"; the module does not detect renames. + - Supports check mode (C(--check)) and diff mode (C(--diff)). +author: + - Linuxfabrik GmbH, Zurich, Switzerland (info (at) linuxfabrik (dot) ch) +requirements: + - python >= 3.6 +options: + url: + description: + - Base URL of the Graylog HTTP interface, e.g. C(http://127.0.0.1:9000). The C(/api) prefix is appended automatically. + type: str + required: true + username: + description: + - Username for HTTP basic authentication against the Graylog API. Typically the Graylog root user. + type: str + required: true + password: + description: + - Password for HTTP basic authentication. + type: str + required: true + no_log: true + validate_certs: + description: + - Whether to verify the TLS certificate of the Graylog endpoint. Only relevant for C(https://) URLs. + type: bool + required: false + default: true + title: + description: + - Human-readable name of the input. Used as the idempotency key, so it must be unique on the Graylog instance. + type: str + required: true + type: + description: + - Fully qualified Graylog input class, e.g. C(org.graylog2.inputs.gelf.udp.GELFUDPInput). Required when creating a new input. + type: str + required: false + configuration: + description: + - Input-specific configuration, e.g. C(bind_address), C(port), C(number_worker_threads). See the Graylog API browser (C(/api-browser#?route=get-/system/inputs/types/all)) for the per-type fields. The C(port) cannot change after creation. + type: dict + required: false + global: + description: + - Whether the input runs on every node (C(true)) or only on the node named in I(node) (C(false)). + - Required when creating a new input. + type: bool + required: false + node: + description: + - Node ID to bind the input to when I(global=false). Ignored when I(global=true). + type: str + required: false + state: + description: + - C(present) creates the input when missing, updates it in place when present (modulo the immutable I(configuration.port)). + - C(absent) deletes the input identified by I(title). When the input does not exist, the module exits with C(changed=false). + type: str + choices: ['absent', 'present'] + default: 'present' +''' + +EXAMPLES = r''' +- name: 'Create a GELF UDP input on every node' + linuxfabrik.lfops.graylog_input: + url: 'http://127.0.0.1:9000' + username: 'admin' + password: '{{ graylog_admin_password }}' + title: 'Gelf (12201/UDP - managed by Ansible - do not edit)' + type: 'org.graylog2.inputs.gelf.udp.GELFUDPInput' + configuration: + bind_address: '0.0.0.0' + port: 12201 + decompress_size_limit: 8388608 + number_worker_threads: 4 + override_source: '' + recv_buffer_size: 1048576 + global: true + state: 'present' + +- name: 'Retire the legacy Syslog input' + linuxfabrik.lfops.graylog_input: + url: 'http://127.0.0.1:9000' + username: 'admin' + password: '{{ graylog_admin_password }}' + title: 'Syslog (1514/UDP - managed by Ansible - do not edit)' + state: 'absent' + +# Preview a change without writing: +# ansible-playbook ... --check --diff --tags graylog_server:configure_system_inputs +''' + +RETURN = r''' +input: + description: + - On create or update, the input object as returned by Graylog. On delete, the last known state of the input. Empty dict when there was nothing to delete. + - In check mode, a synthetic preview reflecting what the run would have written. + type: dict + returned: always +diff: + description: Standard Ansible diff structure. C(before) / C(after) contain only the fields whose values actually change. + type: dict + returned: when changed +''' + + +from ansible.module_utils.basic import AnsibleModule +from ansible_collections.linuxfabrik.lfops.plugins.module_utils.graylog import ( + GraylogAPIError, + GraylogClient, + diff_changed_fields, +) + + +# Fields the user can set in the inventory. These are the ones we compare and +# the ones we send on POST/PUT. +_DIFF_FIELDS = ('title', 'type', 'global', 'node', 'configuration') + +# Server-managed fields that come back in GET responses but are not user input. +_IGNORE_ON_DIFF = ('id', 'created_at', 'creator_user_id', 'static_fields') + + +def find_by_title(inputs, title): + """Return the first input dict whose `title` matches, or None.""" + for item in inputs or []: + if item.get('title') == title: + return item + return None + + +def _unwrap_secret(value): + """Graylog stores secret-bearing fields like `tls_key_password` as + `{"encrypted_value": "...", "salt": "..."}` on read, while POST/PUT bodies + accept a plain string. When the secret is empty, unwrap the dict to `''` + so the diff does not fire on every run. + + A non-empty `encrypted_value` is left as-is because the plain user-supplied + value cannot be compared to the encrypted form; the module treats such + secrets as write-once (a subsequent run will diff and re-send, which + Graylog stores idempotently). + """ + if ( + isinstance(value, dict) + and set(value.keys()) == {'encrypted_value', 'salt'} + and value.get('encrypted_value') == '' + ): + return '' + return value + + +def normalize_current(item): + """Reduce a GET-side input dict to the fields we diff against. + + The Graylog GET response uses `attributes` for the per-input configuration + block, while POST/PUT bodies use `configuration`. Mirror so both sides + compare under the same key, and unwrap Graylog's empty-secret dicts back + to plain strings. + """ + if not item: + return {} + configuration = item.get('attributes', item.get('configuration')) or {} + configuration = {k: _unwrap_secret(v) for k, v in configuration.items()} + return { + 'title': item.get('title'), + 'type': item.get('type'), + 'global': item.get('global'), + 'node': item.get('node'), + 'configuration': configuration, + } + + +def build_desired(params): + """Return the dict we POST/PUT, with `None`/omitted params left out.""" + desired = {} + for field in _DIFF_FIELDS: + value = params.get(field) + if value is None: + continue + desired[field] = value + return desired + + +def main(): + # `global` is a Python keyword, so we build argument_spec via a dict literal. + argument_spec = { + 'url': {'type': 'str', 'required': True}, + 'username': {'type': 'str', 'required': True}, + 'password': {'type': 'str', 'required': True, 'no_log': True}, + 'validate_certs': {'type': 'bool', 'default': True}, + 'title': {'type': 'str', 'required': True}, + 'type': {'type': 'str'}, + 'configuration': {'type': 'dict'}, + 'global': {'type': 'bool'}, + 'node': {'type': 'str'}, + 'state': {'type': 'str', 'choices': ['absent', 'present'], 'default': 'present'}, + } + module = AnsibleModule( + argument_spec=argument_spec, + required_if=[ + ('state', 'present', ['type', 'configuration', 'global'], False), + ], + supports_check_mode=True, + ) + + client = GraylogClient( + module, + url=module.params['url'], + username=module.params['username'], + password=module.params['password'], + validate_certs=module.params['validate_certs'], + ) + title = module.params['title'] + state = module.params['state'] + + try: + listing = client.get('/api/system/inputs') + except GraylogAPIError as exc: + module.fail_json(msg=f'Could not list inputs: {exc}') + + current = find_by_title((listing or {}).get('inputs') or [], title) + current_compare = normalize_current(current) + desired = build_desired(module.params) + + # --- absent -------------------------------------------------------------- + if state == 'absent': + if current is None: + module.exit_json(changed=False, input={}, diff={'before': {}, 'after': {}}) + diff = {'before': current_compare, 'after': {}} + if module.check_mode: + module.exit_json(changed=True, input=current, diff=diff) + try: + client.delete(f"/api/system/inputs/{current['id']}", expected_status=204) + except GraylogAPIError as exc: + module.fail_json(msg=f'Could not delete input {title!r}: {exc}') + module.exit_json(changed=True, input=current, diff=diff) + + # --- present, create ----------------------------------------------------- + if current is None: + diff = {'before': {}, 'after': dict(desired)} + if module.check_mode: + module.exit_json(changed=True, input=desired, diff=diff) + try: + created = client.post('/api/system/inputs', body=desired, expected_status=201) + except GraylogAPIError as exc: + module.fail_json(msg=f'Could not create input {title!r}: {exc}') + module.exit_json(changed=True, input=created or desired, diff=diff) + + # --- present, update ----------------------------------------------------- + before, after = diff_changed_fields(current_compare, desired, ignore=_IGNORE_ON_DIFF) + if not before and not after: + module.exit_json(changed=False, input=current, diff={'before': {}, 'after': {}}) + + diff = {'before': before, 'after': after} + if module.check_mode: + module.exit_json(changed=True, input=current, diff=diff) + + # Graylog's InputCreateRequest builder rejects `id` in the PUT body + # ("Unable to map property id"); the id only goes in the URL path. Index + # sets behave the opposite way - see graylog_index_set. + try: + client.put(f"/api/system/inputs/{current['id']}", body=desired, expected_status=201) + except GraylogAPIError as exc: + module.fail_json(msg=f'Could not update input {title!r}: {exc}') + + module.exit_json(changed=True, input=current, diff=diff) + + +if __name__ == '__main__': + main() diff --git a/roles/graylog_server/README.md b/roles/graylog_server/README.md index 8329f3a4..eaa7a55d 100644 --- a/roles/graylog_server/README.md +++ b/roles/graylog_server/README.md @@ -1,13 +1,19 @@ # Ansible Role linuxfabrik.lfops.graylog_server -This role installs and configures a [Graylog](https://www.graylog.org) server. Optionally, it allows the creation of a cluster setup. +This role installs and configures a [Graylog](https://www.graylog.org) server. Optionally, it allows the creation of a cluster setup. On-demand, the role can also create or remove Graylog system inputs and index sets via the Graylog API; see [Post-Installation Steps](#post-installation-steps). -Additionally this role creates default "System Inputs" and a Linuxfabrik default "index set". -Note that this role does NOT let you specify a particular Graylog Server version. It simply installs the latest available Graylog Server version from the repos configured in the system. If you want or need to install a specific Graylog Server version, use the [linuxfabrik.lfops.repo_graylog](https://github.com/Linuxfabrik/lfops/tree/main/roles/repo_graylog) beforehand. +*Available since LFOps `2.0.0`.* -*Available since LFOps `2.0.0`.* +## How the Role Behaves + +* The role does not pin the Graylog Server version: it installs whatever the configured repository currently offers. To install a specific version, configure the repository accordingly beforehand (e.g. via [linuxfabrik.lfops.repo_graylog](https://github.com/Linuxfabrik/lfops/tree/main/roles/repo_graylog)). +* `/etc/graylog/server/server.conf` and the sysconfig opts file are fully templated and re-rendered on every run, with a timestamped backup kept next to each file, so out-of-band manual edits are overwritten. Manage all settings through the role variables. +* The role asserts that `graylog_server__password_secret` is at least 16 characters before running any other task. +* System inputs and index sets are managed only on demand, only on the leader node, and only when the corresponding tag (`graylog_server:configure_system_inputs` / `graylog_server:configure_system_index_sets`) is selected. See [Post-Installation Steps](#post-installation-steps). +* The role ships default system inputs (Beats, GELF TCP, GELF UDP, Syslog UDP) in `__role_var` but **no default index sets** - a single catch-all set is unsuitable for any non-trivial deployment. The operator must define at least one index set in the inventory and mark exactly one as `default: true`. +* The role does not manage the firewall or TLS certificates. Open the listener ports and provide certificates separately. ## Known Limitations @@ -36,6 +42,22 @@ Manual steps: * If you're not using a versioned MongoDB repository, protect MongoDB from being updated with newer minor and major versions by running the [dnf_versionlock](https://github.com/Linuxfabrik/lfops/blob/main/playbooks/dnf_versionlock.yml) playbook (role: [linuxfabrik.lfops.dnf_versionlock](https://github.com/Linuxfabrik/lfops/tree/main/roles/dnf_versionlock)). +## Post-Installation Steps + +Configure the CA for the Data Nodes in the "Graylog Initial Setup" wizard then wait for Graylog to start. + +System inputs and index sets are managed on demand via the Graylog API. The role ships with default system inputs in `graylog_server__system_inputs__role_var` but **no default index sets**: rotation, retention, shards and replicas are configured per index set, so a single catch-all set is unsuitable as soon as you have mixed log sources, compliance retention requirements or wildly different volumes. You must define one or more index sets in the inventory, split by use case (typical split: an audit set with long retention, an application set with medium retention, an access/syslog set with short retention). The example in [Optional Role Variables](#optional-role-variables) below uses **data tiering** (Graylog's current model since 6.0); the legacy `rotation_strategy` / `retention_strategy` fields are still supported via `use_legacy_rotation: true` but Graylog has announced they will be deprecated. + +To apply, run: + +* `--tags graylog_server:configure_system_inputs` to create, update or delete the configured system inputs. +* `--tags graylog_server:configure_system_index_sets` to create, update or delete the configured index sets. + +Both tasks only run on the leader node (`graylog_server__is_leader: true`) to avoid duplicates. Entries are matched by `title` (inputs) and `index_prefix` (index sets); changing other fields on an existing entry updates it in place. To remove an entry, override it in your inventory with `state: 'absent'`. Note that the input `port` and `index_prefix` can not be changed after creation; modifying them in the inventory will create a new entry instead of updating the existing one. + +`--check` and `--diff` are supported: a dry-run with both flags prints the per-field delta (only the changed keys) and writes nothing. + + ## Tags `graylog_server` @@ -54,11 +76,17 @@ Manual steps: * Creates the message journal directory. * Triggers: graylog-server.service restart. -`graylog_server:configure_defaults` +`graylog_server:configure_system_index_sets` + +* Only executed on demand. +* Creates, updates and deletes Graylog index sets via the API. +* Sets the index set marked with `default: true` as the default in Graylog. +* Triggers: none. + +`graylog_server:configure_system_inputs` * Only executed on demand. -* Configures Graylog system inputs via the API. -* Creates and sets the default index set. +* Creates, updates and deletes Graylog system inputs via the API. * Triggers: none. `graylog_server:state` @@ -169,21 +197,44 @@ graylog_server__root_user: * Type: Number. * Default: `2000` -`graylog_server__system_default_index_set` +`graylog_server__system_index_sets__host_var` / `graylog_server__system_index_sets__group_var` -* Creates a default index set. Used with the `graylog_server:configure_defaults` tag. -* Type: Dictionary. -* Default: See [defaults/main.yml](https://github.com/Linuxfabrik/lfops/blob/main/roles/graylog_server/defaults/main.yml) +* Index sets to create, update or delete via the Graylog API. Used with the `graylog_server:configure_system_index_sets` tag. +* The role intentionally ships no defaults; the sysadmin must define at least one index set in the inventory and mark exactly one as `default: true`. See [Post-Installation Steps](#post-installation-steps) for the rationale. +* Type: List of dictionaries. +* Default: `[]` * Subkeys: - * `creation_date`: + * `data_tiering`: - * Mandatory. Date in ISO 8601 format. - * Type: String. + * Mandatory when `use_legacy_rotation` is `false` (recommended). Data tiering is Graylog's current rotation/retention model and replaces the legacy `rotation_strategy` / `retention_strategy` configuration. The legacy strategies still work but Graylog has announced they will be deprecated. + * Type: Dictionary. + * Subkeys: + + * `type`: + + * Mandatory. `'hot_only'` for Graylog Open. `'hot_warm'` requires Graylog Enterprise and additional `warm_tier_*` settings. + * Type: String. + + * `index_lifetime_min`: + + * Mandatory. Minimum time data must stay in the hot tier before becoming eligible for the next lifecycle action. ISO 8601 duration, e.g. `'P7D'` for 7 days. + * Type: String. + + * `index_lifetime_max`: + + * Mandatory. Maximum time data may stay in the hot tier; once reached the next lifecycle action (rotation, deletion or migration to warm) is enforced. ISO 8601 duration, e.g. `'P30D'` for 30 days. + * Type: String. + + * `default`: + + * Optional. If `true`, this index set is set as the Graylog default after create/update. Only one entry should set this. + * Type: Bool. + * Default: `false` * `description`: - * Mandatory. Description of index set. + * Mandatory. Description of the index set. * Type: String. * `field_type_refresh_interval`: @@ -208,7 +259,7 @@ graylog_server__root_user: * `index_prefix`: - * Mandatory. A unique prefix used in indices belonging to this index set. The prefix must start with a letter or number, and can only contain letters, numbers, `_`, `-` and `+`. + * Mandatory. A unique prefix used in indices belonging to this index set. The prefix must start with a letter or number, and can only contain letters, numbers, `_`, `-` and `+`. Used as the identity key for create/update/delete; cannot be changed after creation (changing it creates a new index set). It must not start with the same word as another index_prefix, e.g. `lfops` and `lfops02` would conflict. * Type: String. * `replicas`: @@ -218,7 +269,7 @@ graylog_server__root_user: * `retention_strategy`: - * Mandatory. Retention strategy configuration. + * Legacy. Mandatory only when `use_legacy_rotation` is `true`; ignored otherwise. Prefer `data_tiering`. * Type: Dictionary. * Subkeys: @@ -234,12 +285,12 @@ graylog_server__root_user: * `retention_strategy_class`: - * Mandatory. Retention strategy class to clean up old indices. + * Legacy. Mandatory only when `use_legacy_rotation` is `true`; ignored otherwise. Prefer `data_tiering`. * Type: String. * `rotation_strategy`: - * Mandatory. Rotation strategy configuration. + * Legacy. Mandatory only when `use_legacy_rotation` is `true`; ignored otherwise. Prefer `data_tiering`. * Type: Dictionary. * Subkeys: @@ -260,7 +311,7 @@ graylog_server__root_user: * `rotation_strategy_class`: - * Mandatory. Graylog uses multiple indices to store documents in. You can configure the strategy it uses to determine when to rotate the currently active write index. + * Legacy. Mandatory only when `use_legacy_rotation` is `true`; ignored otherwise. Prefer `data_tiering`. * Type: String. * `shards`: @@ -268,36 +319,54 @@ graylog_server__root_user: * Mandatory. Number of shards used per index in this index set. Never set this higher than the number of data nodes. * Type: Number. + * `state`: + + * Optional. State of the index set, one of `present`, `absent`. + * Type: String. + * Default: `'present'` + * `title`: * Mandatory. Descriptive name of the index set. * Type: String. + * `use_legacy_rotation`: + + * Optional. `false` (recommended) activates `data_tiering`; `true` falls back to the legacy `rotation_strategy` / `retention_strategy` configuration, which Graylog has announced will be deprecated. + * Type: Bool. + * Default: `false` + * `writable`: * Mandatory. Whether this index set is writable. * Type: Bool. -`graylog_server__system_inputs` +`graylog_server__system_inputs__host_var` / `graylog_server__system_inputs__group_var` -* Creates system inputs. Used with the `graylog_server:configure_defaults` tag. +* System inputs to create, update or delete via the Graylog API. Used with the `graylog_server:configure_system_inputs` tag. * Type: List of dictionaries. -* Default: See [defaults/main.yml](https://github.com/Linuxfabrik/lfops/blob/main/roles/graylog_server/defaults/main.yml) +* Default: `Beats (5044/TCP)`, `Gelf (12201/TCP)`, `Gelf (12201/UDP)` and `Syslog (1514/UDP)`. See [defaults/main.yml](https://github.com/Linuxfabrik/lfops/blob/main/roles/graylog_server/defaults/main.yml) for the full content. * Subkeys: * `configuration`: - * Mandatory. Specific configuration of corresponding input. Please refer to the [API documentation](https://go2docs.graylog.org/current/setting_up_graylog/rest_api.html). + * Mandatory. Input-specific configuration; see the [API documentation](https://go2docs.graylog.org/current/setting_up_graylog/rest_api.html). The `port` cannot be changed after creation (changing it creates a new input). * Type: Dictionary. * `global`: - * Mandatory. Whether this input should start on all nodes. + * Mandatory. Whether this input should start on all nodes. Each entry must either set `global: true` or assign a `node`, otherwise Graylog creates the input but never starts it; the role asserts this. * Type: Bool. + * `state`: + + * Optional. State of the input, one of `present`, `absent`. + * Type: String. + * Default: `'present'` + * `title`: - * Mandatory. The title for this input. + * Mandatory. The title for this input. Used as the identity key for create/update/delete. * Type: String. * `type`: @@ -334,87 +403,76 @@ graylog_server__mongodb_uri: 'mongodb://graylog01.example.com:27017,username:pas graylog_server__opts: '-Xms2g -Xmx2g -server -XX:+UseG1GC -XX:-OmitStackTraceInFastThrow' graylog_server__service_enabled: false graylog_server__stale_leader_timeout_ms: 10000 -graylog_server__system_default_index_set: - creation_date: '{{ ansible_date_time.iso8601 }}' - description: 'One index per day; 365 indices max' - field_type_refresh_interval: 5000 - index_analyzer: 'standard' - index_optimization_disabled: false - index_optimization_max_num_segments: 1 - index_prefix: 'lfops-default' - replicas: 0 - retention_strategy: - max_number_of_indices: 365 - type: 'org.graylog2.indexer.retention.strategies.DeletionRetentionStrategyConfig' - retention_strategy_class: 'org.graylog2.indexer.retention.strategies.DeletionRetentionStrategy' - rotation_strategy: - rotation_period: 'P1D' - rotate_empty_index_set: false - type: 'org.graylog2.indexer.rotation.strategies.TimeBasedRotationStrategyConfig' - rotation_strategy_class: 'org.graylog2.indexer.rotation.strategies.TimeBasedRotationStrategy' - shards: 3 - title: 'Linuxfabrik Index Set (managed by Ansible - do not edit)' - writable: true -graylog_server__system_inputs: - - configuration: - bind_address: '0.0.0.0' - number_worker_threads: 4 - override_source: '' - port: 5044 - recv_buffer_size: 1048576 - tcp_keepalive: false - tls_cert_file: '' - tls_client_auth: 'disabled' - tls_client_auth_cert_file: '' - tls_enable: false - tls_key_file: '' - tls_key_password: '' - global: true - title: 'Beats (5044/TCP - managed by Ansible - do not edit)' - type: 'org.graylog.plugins.beats.Beats2Input' - - configuration: +graylog_server__system_index_sets__host_var: + # catch-all set, marked as the Graylog default; everything that no stream rule + # routes elsewhere lands here. Data tiering keeps data for 25-30 days. + - title: 'Default' + description: 'Default catch-all index set; 25-30 days - managed by Ansible - do not edit' + default: true + use_legacy_rotation: false + data_tiering: + type: 'hot_only' + index_lifetime_min: 'P25D' + index_lifetime_max: 'P30D' + field_type_refresh_interval: 5000 + index_analyzer: 'standard' + index_optimization_disabled: false + index_optimization_max_num_segments: 1 + index_prefix: 'default' + replicas: 0 + shards: 1 + state: 'present' + writable: true + # audit logs, long retention + - title: 'Audit' + description: 'Audit logs; 360-365 days - managed by Ansible - do not edit' + use_legacy_rotation: false + data_tiering: + type: 'hot_only' + index_lifetime_min: 'P360D' + index_lifetime_max: 'P365D' + field_type_refresh_interval: 5000 + index_analyzer: 'standard' + index_optimization_disabled: false + index_optimization_max_num_segments: 1 + index_prefix: 'audit' + replicas: 0 + shards: 1 + state: 'present' + writable: true + # high-volume access/syslog logs, short retention + - title: 'Access' + description: 'Access/syslog logs; 10-14 days - managed by Ansible - do not edit' + use_legacy_rotation: false + data_tiering: + type: 'hot_only' + index_lifetime_min: 'P10D' + index_lifetime_max: 'P14D' + field_type_refresh_interval: 5000 + index_analyzer: 'standard' + index_optimization_disabled: false + index_optimization_max_num_segments: 1 + index_prefix: 'access' + replicas: 0 + shards: 1 + state: 'present' + writable: true +graylog_server__system_inputs__host_var: + # add an additional input alongside the role defaults + - title: 'Gelf Audit (12202/UDP - managed by Ansible - do not edit)' + configuration: bind_address: '0.0.0.0' + port: 12202 decompress_size_limit: 8388608 - max_message_size: 2097152 number_worker_threads: 4 override_source: '' - port: 12201 recv_buffer_size: 1048576 - tcp_keepalive: false - tls_cert_file: '' - tls_client_auth: 'disabled' - tls_client_auth_cert_file: '' - tls_enable: false - tls_key_file: '' - tls_key_password: '' - use_null_delimiter: true global: true - title: 'Gelf (12201/TCP - managed by Ansible - do not edit)' - type: 'org.graylog2.inputs.gelf.tcp.GELFTCPInput' - - configuration: - bind_address: '0.0.0.0' - decompress_size_limit: 8388608 - number_worker_threads: 4 - override_source: '' - port: 12201 - recv_buffer_size: 1048576 - global: true - title: 'Gelf (12201/UDP - managed by Ansible - do not edit)' + state: 'present' type: 'org.graylog2.inputs.gelf.udp.GELFUDPInput' - - configuration: - allow_override_date: true - bind_address: '0.0.0.0' - decompress_size_limit: 8388608 - expand_structured_data: false - force_rdns: false - number_worker_threads: 4 - override_source: '' - port: 1514 - recv_buffer_size: 1048576 - store_full_message: false - global: true - title: 'Syslog (1514/UDP - managed by Ansible - do not edit)' - type: 'org.graylog2.inputs.syslog.udp.SyslogUDPInput' + # opt out of one of the role defaults: + - title: 'Syslog (1514/UDP - managed by Ansible - do not edit)' + state: 'absent' graylog_server__timezone: 'Europe/Zurich' graylog_server__trusted_proxies: - '127.0.0.1/32' @@ -425,9 +483,9 @@ graylog_server__trusted_proxies: ## Troubleshooting -Q: `/bin/sh: /opt/python-venv/pymongo/bin/python3: No such file or directory` +**`/bin/sh: /opt/python-venv/pymongo/bin/python3: No such file or directory`** -A: You either have to run the whole playbook, or python_venv directly: `ansible-playbook --inventory myinv linuxfabrik.lfops.setup_graylog_server --tags python_venv` +* The `pymongo` virtualenv that the role's MongoDB-touching tasks rely on is missing. Run the full playbook, or just the `python_venv` role: `ansible-playbook --inventory myinv linuxfabrik.lfops.setup_graylog_server --tags python_venv`. ## License diff --git a/roles/graylog_server/defaults/main.yml b/roles/graylog_server/defaults/main.yml index 16413a08..986c4ebb 100644 --- a/roles/graylog_server/defaults/main.yml +++ b/roles/graylog_server/defaults/main.yml @@ -10,29 +10,22 @@ graylog_server__stale_leader_timeout_ms: 2000 graylog_server__timezone: 'Europe/Zurich' graylog_server__trusted_proxies: [] -graylog_server__system_default_index_set: - creation_date: '{{ ansible_date_time.iso8601 }}' - description: 'One index per day; 365 indices max' - field_type_refresh_interval: 5000 - index_analyzer: 'standard' - index_optimization_disabled: false - index_optimization_max_num_segments: 1 - index_prefix: 'lfops-default' - replicas: 0 - retention_strategy: - max_number_of_indices: 365 - type: 'org.graylog2.indexer.retention.strategies.DeletionRetentionStrategyConfig' - retention_strategy_class: 'org.graylog2.indexer.retention.strategies.DeletionRetentionStrategy' - rotation_strategy: - rotation_period: 'P1D' - rotate_empty_index_set: false - type: 'org.graylog2.indexer.rotation.strategies.TimeBasedRotationStrategyConfig' - rotation_strategy_class: 'org.graylog2.indexer.rotation.strategies.TimeBasedRotationStrategy' - shards: 3 - title: 'Linuxfabrik Index Set (managed by Ansible - do not edit)' - writable: true +graylog_server__system_index_sets__dependent_var: [] +graylog_server__system_index_sets__group_var: [] +graylog_server__system_index_sets__host_var: [] +graylog_server__system_index_sets__role_var: [] +graylog_server__system_index_sets__combined_var: '{{ ( + graylog_server__system_index_sets__role_var + + graylog_server__system_index_sets__dependent_var + + graylog_server__system_index_sets__group_var + + graylog_server__system_index_sets__host_var + ) | linuxfabrik.lfops.combine_lod(unique_key="index_prefix") + }}' -graylog_server__system_inputs: +graylog_server__system_inputs__dependent_var: [] +graylog_server__system_inputs__group_var: [] +graylog_server__system_inputs__host_var: [] +graylog_server__system_inputs__role_var: - configuration: bind_address: '0.0.0.0' number_worker_threads: 4 @@ -47,6 +40,7 @@ graylog_server__system_inputs: tls_key_file: '' tls_key_password: '' global: true + state: 'present' title: 'Beats (5044/TCP - managed by Ansible - do not edit)' type: 'org.graylog.plugins.beats.Beats2Input' - configuration: @@ -66,6 +60,7 @@ graylog_server__system_inputs: tls_key_password: '' use_null_delimiter: true global: true + state: 'present' title: 'Gelf (12201/TCP - managed by Ansible - do not edit)' type: 'org.graylog2.inputs.gelf.tcp.GELFTCPInput' - configuration: @@ -76,6 +71,7 @@ graylog_server__system_inputs: port: 12201 recv_buffer_size: 1048576 global: true + state: 'present' title: 'Gelf (12201/UDP - managed by Ansible - do not edit)' type: 'org.graylog2.inputs.gelf.udp.GELFUDPInput' - configuration: @@ -90,8 +86,16 @@ graylog_server__system_inputs: recv_buffer_size: 1048576 store_full_message: false global: true + state: 'present' title: 'Syslog (1514/UDP - managed by Ansible - do not edit)' type: 'org.graylog2.inputs.syslog.udp.SyslogUDPInput' +graylog_server__system_inputs__combined_var: '{{ ( + graylog_server__system_inputs__role_var + + graylog_server__system_inputs__dependent_var + + graylog_server__system_inputs__group_var + + graylog_server__system_inputs__host_var + ) | linuxfabrik.lfops.combine_lod(unique_key="title") + }}' # --------------------------------------------------- diff --git a/roles/graylog_server/tasks/main.yml b/roles/graylog_server/tasks/main.yml index ed1dd66e..d4bf5cfc 100644 --- a/roles/graylog_server/tasks/main.yml +++ b/roles/graylog_server/tasks/main.yml @@ -116,115 +116,71 @@ - block: - - name: 'Validate that each graylog_server__system_inputs entry is global or assigned to a node' + - name: 'Validate that each system input entry is global or assigned to a node' ansible.builtin.assert: that: - '(item["global"] | default(false) | bool) or (item["node"] | default("") | length > 0)' - fail_msg: 'Each graylog_server__system_inputs entry must set "global: true" or assign a "node", otherwise Graylog creates the input but never starts it' + fail_msg: 'Each system input must set "global: true" or assign a "node", otherwise Graylog creates the input but never starts it' quiet: true - loop: '{{ graylog_server__system_inputs }}' + loop: '{{ graylog_server__system_inputs__combined_var }}' loop_control: label: '{{ item["title"] | default(item["type"]) }}' + when: + - 'item["state"] | default("present") != "absent"' - - name: 'Get all inputs' - ansible.builtin.uri: - url: 'http://{{ graylog_server__http_bind_address }}:{{ graylog_server__http_bind_port }}/api/system/inputs' - user: '{{ graylog_server__root_user["username"] }}' + - name: 'Manage Graylog system inputs' + linuxfabrik.lfops.graylog_input: + url: 'http://{{ graylog_server__http_bind_address }}:{{ graylog_server__http_bind_port }}' + username: '{{ graylog_server__root_user["username"] }}' password: '{{ graylog_server__root_user["password"] }}' - method: 'GET' - force_basic_auth: true - # status_code: 200 - register: 'graylog_server__input_result' - changed_when: false # no actual config change - check_mode: false # run task even if `--check` is specified + title: '{{ item["title"] }}' + type: '{{ item["type"] | default(omit) }}' + configuration: '{{ item["configuration"] | default(omit) }}' + global: '{{ item["global"] | default(omit) }}' + node: '{{ item["node"] | default(omit) }}' + state: '{{ item["state"] | default("present") }}' + loop: '{{ graylog_server__system_inputs__combined_var }}' + loop_control: + label: '{{ item["title"] }}, state={{ item["state"] | default("present") }}' - - name: 'Graylog inputs:' - ansible.builtin.debug: - var: 'graylog_server__input_result.json.inputs' + when: + - 'graylog_server__is_leader | bool' # only run this against one host, else we get duplicate inputs + tags: + - 'never' + - 'graylog_server:configure_system_inputs' - - name: 'Terminate input on this node' - ansible.builtin.uri: - url: 'http://{{ graylog_server__http_bind_address }}:{{ graylog_server__http_bind_port }}/api/system/inputs/{{ item }}' - user: '{{ graylog_server__root_user["username"] }}' - password: '{{ graylog_server__root_user["password"] }}' - method: 'DELETE' - force_basic_auth: true - status_code: 204 - headers: - X-Requested-By: 'cli' - loop: '{{ graylog_server__input_result | community.general.json_query("json.inputs[*].id") }}' - when: - - 'graylog_server__is_leader | bool' # only run this against one host, else we get duplicate inputs - - 'graylog_server__input_result.json.inputs is defined and graylog_server__input_result.json.inputs | length > 0' - - name: 'Launch input on this node' - ansible.builtin.uri: - url: 'http://{{ graylog_server__http_bind_address }}:{{ graylog_server__http_bind_port }}/api/system/inputs' - user: '{{ graylog_server__root_user["username"] }}' - password: '{{ graylog_server__root_user["password"] }}' - method: 'POST' - body: '{{ item | to_json }}' - force_basic_auth: true - status_code: 201 - body_format: 'json' - headers: - Accept: 'application/json' - X-Requested-By: 'cli' - loop: '{{ graylog_server__system_inputs }}' - - - name: 'Get a list of all index sets' - ansible.builtin.uri: - url: 'http://{{ graylog_server__http_bind_address }}:{{ graylog_server__http_bind_port }}/api/system/indices/index_sets' - user: '{{ graylog_server__root_user["username"] }}' - password: '{{ graylog_server__root_user["password"] }}' - method: 'GET' - force_basic_auth: true - # status_code: 200 - register: 'graylog_server__get_index_sets_result' - changed_when: false # no actual config change - check_mode: false # run task even if `--check` is specified - - - name: 'Create index set' - ansible.builtin.uri: - url: 'http://{{ graylog_server__http_bind_address }}:{{ graylog_server__http_bind_port }}/api/system/indices/index_sets' - user: '{{ graylog_server__root_user["username"] }}' - password: '{{ graylog_server__root_user["password"] }}' - method: 'POST' - body: '{{ graylog_server__system_default_index_set | to_json }}' - force_basic_auth: true - # status_code: 200 - body_format: 'json' - headers: - Accept: 'application/json' - X-Requested-By: 'cli' - when: "graylog_server__get_index_sets_result | community.general.json_query(\"json.index_sets[?index_prefix==`\" ~ graylog_server__system_default_index_set['index_prefix'] ~ \"`].id\") | length == 0" - - - name: 'Get a list of all index sets' - ansible.builtin.uri: - url: 'http://{{ graylog_server__http_bind_address }}:{{ graylog_server__http_bind_port }}/api/system/indices/index_sets' - user: '{{ graylog_server__root_user["username"] }}' - password: '{{ graylog_server__root_user["password"] }}' - method: 'GET' - force_basic_auth: true - # status_code: 200 - register: 'graylog_server__get_index_sets_result' - changed_when: false # no actual config change - check_mode: false # run task even if `--check` is specified +- block: - - name: 'Set default index set' - ansible.builtin.uri: - url: "http://{{ graylog_server__http_bind_address }}:{{ graylog_server__http_bind_port }}/api/system/indices/index_sets/{{ graylog_server__get_index_sets_result | community.general.json_query(\"json.index_sets[?index_prefix==`\" ~ graylog_server__system_default_index_set['index_prefix'] ~ \"`].id\") | regex_replace(\"\\[\\'|\\'\\]\", \"\") }}/default" - user: '{{ graylog_server__root_user["username"] }}' + - name: 'Manage Graylog index sets' + linuxfabrik.lfops.graylog_index_set: + url: 'http://{{ graylog_server__http_bind_address }}:{{ graylog_server__http_bind_port }}' + username: '{{ graylog_server__root_user["username"] }}' password: '{{ graylog_server__root_user["password"] }}' - method: 'PUT' - force_basic_auth: true - # status_code: 200 - headers: - X-Requested-By: 'cli' - when: "graylog_server__get_index_sets_result | community.general.json_query(\"json.index_sets[?index_prefix==`\" ~ graylog_server__system_default_index_set['index_prefix'] ~ \"`].id\") | length > 0" + index_prefix: '{{ item["index_prefix"] }}' + title: '{{ item["title"] | default(omit) }}' + description: '{{ item["description"] | default(omit) }}' + shards: '{{ item["shards"] | default(omit) }}' + replicas: '{{ item["replicas"] | default(omit) }}' + writable: '{{ item["writable"] | default(omit) }}' + index_analyzer: '{{ item["index_analyzer"] | default(omit) }}' + index_optimization_disabled: '{{ item["index_optimization_disabled"] | default(omit) }}' + index_optimization_max_num_segments: '{{ item["index_optimization_max_num_segments"] | default(omit) }}' + field_type_refresh_interval: '{{ item["field_type_refresh_interval"] | default(omit) }}' + use_legacy_rotation: '{{ item["use_legacy_rotation"] | default(omit) }}' + data_tiering: '{{ item["data_tiering"] | default(omit) }}' + rotation_strategy: '{{ item["rotation_strategy"] | default(omit) }}' + rotation_strategy_class: '{{ item["rotation_strategy_class"] | default(omit) }}' + retention_strategy: '{{ item["retention_strategy"] | default(omit) }}' + retention_strategy_class: '{{ item["retention_strategy_class"] | default(omit) }}' + default: '{{ item["default"] | default(omit) }}' + state: '{{ item["state"] | default("present") }}' + loop: '{{ graylog_server__system_index_sets__combined_var }}' + loop_control: + label: '{{ item["index_prefix"] }}, state={{ item["state"] | default("present") }}' when: - - 'graylog_server__is_leader | bool' # only run this against one host, else we get duplicate inputs + - 'graylog_server__is_leader | bool' # only run this against one host, else we get duplicate index sets tags: - 'never' - - 'graylog_server:configure_defaults' + - 'graylog_server:configure_system_index_sets' diff --git a/tests/unit/plugins/module_utils/test_graylog.py b/tests/unit/plugins/module_utils/test_graylog.py new file mode 100644 index 00000000..ff9faa25 --- /dev/null +++ b/tests/unit/plugins/module_utils/test_graylog.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8; py-indent-offset: 4 -*- +# +# Author: Linuxfabrik GmbH, Zurich, Switzerland +# Contact: info (at) linuxfabrik (dot) ch +# https://www.linuxfabrik.ch/ +# License: The Unlicense, see LICENSE file. + +"""Unit tests for the graylog module_util (pure diff helpers). + +The collection import is wired up by tests/conftest.py. +""" + +from __future__ import absolute_import, division, print_function + +__metaclass__ = type + +import unittest + +from ansible_collections.linuxfabrik.lfops.plugins.module_utils import graylog + + +class TestStripKeys(unittest.TestCase): + + def test_drops_listed_keys(self): + self.assertEqual( + graylog.strip_keys({'a': 1, 'state': 'present', 'b': 2}, ('state',)), + {'a': 1, 'b': 2}, + ) + + def test_handles_empty(self): + self.assertEqual(graylog.strip_keys({}, ('state',)), {}) + self.assertEqual(graylog.strip_keys(None, ('state',)), {}) + + def test_no_op_when_keys_missing(self): + self.assertEqual( + graylog.strip_keys({'a': 1}, ('state', 'default')), + {'a': 1}, + ) + + +class TestEqual(unittest.TestCase): + + def test_scalar(self): + self.assertTrue(graylog._equal(1, 1)) + self.assertFalse(graylog._equal(1, 2)) + self.assertTrue(graylog._equal('a', 'a')) + + def test_nested_dict(self): + a = {'x': {'y': [1, 2]}} + b = {'x': {'y': [1, 2]}} + self.assertTrue(graylog._equal(a, b)) + + def test_nested_dict_diff(self): + a = {'x': {'y': [1, 2]}} + b = {'x': {'y': [1, 3]}} + self.assertFalse(graylog._equal(a, b)) + + def test_list_order_sensitive(self): + # diff_changed_fields treats lists positionally to avoid masking order + # changes in things like rotation/retention configuration. + self.assertFalse(graylog._equal([1, 2], [2, 1])) + + +class TestDiffChangedFields(unittest.TestCase): + + def test_no_change(self): + before, after = graylog.diff_changed_fields( + {'a': 1, 'b': 2}, {'a': 1, 'b': 2}, + ) + self.assertEqual((before, after), ({}, {})) + + def test_scalar_change(self): + before, after = graylog.diff_changed_fields( + {'a': 1, 'b': 2}, {'a': 1, 'b': 9}, + ) + self.assertEqual(before, {'b': 2}) + self.assertEqual(after, {'b': 9}) + + def test_nested_change(self): + before, after = graylog.diff_changed_fields( + {'configuration': {'port': 5044}}, + {'configuration': {'port': 5045}}, + ) + self.assertEqual(before, {'configuration': {'port': 5044}}) + self.assertEqual(after, {'configuration': {'port': 5045}}) + + def test_ignore_skips_field(self): + before, after = graylog.diff_changed_fields( + {'id': 'abc', 'title': 'old'}, + {'id': 'abc', 'title': 'new'}, + ignore=('id',), + ) + self.assertEqual(before, {'title': 'old'}) + self.assertEqual(after, {'title': 'new'}) + + def test_desired_only_keys(self): + # Keys present in current but absent from desired are never reported; + # the module never wants to "unset" something the inventory did not + # mention. + before, after = graylog.diff_changed_fields( + {'a': 1, 'extra': 'server_managed'}, + {'a': 1}, + ) + self.assertEqual((before, after), ({}, {})) + + def test_field_added_in_desired(self): + before, after = graylog.diff_changed_fields( + {'a': 1}, + {'a': 1, 'b': 2}, + ) + self.assertEqual(before, {'b': None}) + self.assertEqual(after, {'b': 2}) + + def test_both_empty(self): + self.assertEqual(graylog.diff_changed_fields({}, {}), ({}, {})) + self.assertEqual(graylog.diff_changed_fields(None, None), ({}, {})) + + +class TestGraylogAPIError(unittest.TestCase): + + def test_carries_fields(self): + err = graylog.GraylogAPIError(400, 'http://x/y', 'bad request') + self.assertEqual(err.status, 400) + self.assertEqual(err.url, 'http://x/y') + self.assertEqual(err.body, 'bad request') + self.assertIn('HTTP 400', str(err)) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/unit/plugins/modules/test_graylog_index_set.py b/tests/unit/plugins/modules/test_graylog_index_set.py new file mode 100644 index 00000000..2e59f725 --- /dev/null +++ b/tests/unit/plugins/modules/test_graylog_index_set.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8; py-indent-offset: 4 -*- +# +# Author: Linuxfabrik GmbH, Zurich, Switzerland +# Contact: info (at) linuxfabrik (dot) ch +# https://www.linuxfabrik.ch/ +# License: The Unlicense, see LICENSE file. + +"""Unit tests for the graylog_index_set pure helpers. + +The collection import is wired up by tests/conftest.py. +""" + +from __future__ import absolute_import, division, print_function + +__metaclass__ = type + +import unittest + +from ansible_collections.linuxfabrik.lfops.plugins.modules import graylog_index_set as mod + + +class TestFindByPrefix(unittest.TestCase): + + def test_match(self): + sets = [{'index_prefix': 'audit'}, {'index_prefix': 'access'}] + self.assertEqual(mod.find_by_prefix(sets, 'access'), {'index_prefix': 'access'}) + + def test_no_match_returns_none(self): + self.assertIsNone(mod.find_by_prefix([{'index_prefix': 'audit'}], 'access')) + + def test_empty_list(self): + self.assertIsNone(mod.find_by_prefix([], 'access')) + self.assertIsNone(mod.find_by_prefix(None, 'access')) + + +class TestNormalizeCurrent(unittest.TestCase): + + def test_keeps_only_diff_fields(self): + current = { + 'id': 'srv-managed', + 'creation_date': '2026-01-01T00:00:00Z', + 'can_be_default': True, + 'title': 'Audit', + 'index_prefix': 'audit', + 'shards': 1, + 'replicas': 0, + 'writable': True, + 'data_tiering': {'type': 'hot_only', + 'index_lifetime_min': 'P360D', + 'index_lifetime_max': 'P365D'}, + 'use_legacy_rotation': False, + 'description': 'Audit logs', + 'index_analyzer': 'standard', + 'index_optimization_disabled': False, + 'index_optimization_max_num_segments': 1, + 'field_type_refresh_interval': 5000, + } + normalized = mod.normalize_current(current) + self.assertNotIn('id', normalized) + self.assertNotIn('creation_date', normalized) + self.assertNotIn('can_be_default', normalized) + self.assertEqual(normalized['title'], 'Audit') + self.assertEqual(normalized['data_tiering']['index_lifetime_max'], 'P365D') + + +class TestBuildDesired(unittest.TestCase): + + def test_drops_none(self): + params = { + 'index_prefix': 'audit', + 'title': 'Audit', + 'description': None, + 'use_legacy_rotation': False, + 'data_tiering': {'type': 'hot_only'}, + 'shards': 1, + 'replicas': 0, + 'writable': True, + 'index_analyzer': 'standard', + 'index_optimization_disabled': False, + 'index_optimization_max_num_segments': 1, + 'field_type_refresh_interval': 5000, + } + desired = mod.build_desired(params) + self.assertNotIn('description', desired) + self.assertEqual(desired['index_prefix'], 'audit') + self.assertEqual(desired['data_tiering'], {'type': 'hot_only'}) + + def test_ignores_role_only_and_connection_keys(self): + params = { + 'index_prefix': 'audit', + 'url': 'http://x', + 'username': 'admin', + 'state': 'present', + 'default': True, + } + desired = mod.build_desired(params) + # default + state + connection keys are role-only / module-only and + # must never be POSTed/PUT'd to Graylog. + self.assertEqual(set(desired.keys()), {'index_prefix'}) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/unit/plugins/modules/test_graylog_input.py b/tests/unit/plugins/modules/test_graylog_input.py new file mode 100644 index 00000000..560a543a --- /dev/null +++ b/tests/unit/plugins/modules/test_graylog_input.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8; py-indent-offset: 4 -*- +# +# Author: Linuxfabrik GmbH, Zurich, Switzerland +# Contact: info (at) linuxfabrik (dot) ch +# https://www.linuxfabrik.ch/ +# License: The Unlicense, see LICENSE file. + +"""Unit tests for the graylog_input pure helpers. + +The collection import is wired up by tests/conftest.py. +""" + +from __future__ import absolute_import, division, print_function + +__metaclass__ = type + +import unittest + +from ansible_collections.linuxfabrik.lfops.plugins.modules import graylog_input as mod + + +class TestFindByTitle(unittest.TestCase): + + def test_match(self): + inputs = [{'title': 'a'}, {'title': 'b'}] + self.assertEqual(mod.find_by_title(inputs, 'b'), {'title': 'b'}) + + def test_no_match_returns_none(self): + self.assertIsNone(mod.find_by_title([{'title': 'a'}], 'b')) + + def test_empty_list(self): + self.assertIsNone(mod.find_by_title([], 'b')) + self.assertIsNone(mod.find_by_title(None, 'b')) + + +class TestNormalizeCurrent(unittest.TestCase): + + def test_attributes_remapped_to_configuration(self): + # Graylog GET responses use `attributes` for what POST/PUT bodies call + # `configuration`; the normalizer mirrors the field so the diff + # compares like with like. + current = { + 'id': 'abc', + 'title': 'Gelf UDP', + 'type': 'org.graylog2.inputs.gelf.udp.GELFUDPInput', + 'global': True, + 'node': None, + 'attributes': {'port': 12201, 'bind_address': '0.0.0.0'}, + } + normalized = mod.normalize_current(current) + self.assertEqual(normalized['configuration'], {'port': 12201, 'bind_address': '0.0.0.0'}) + self.assertEqual(normalized['title'], 'Gelf UDP') + self.assertEqual(normalized['type'], 'org.graylog2.inputs.gelf.udp.GELFUDPInput') + self.assertTrue(normalized['global']) + + def test_empty(self): + self.assertEqual(mod.normalize_current({}), {}) + self.assertEqual(mod.normalize_current(None), {}) + + def test_empty_secret_dict_unwrapped(self): + # Graylog stores secret-bearing fields like `tls_key_password` as + # {"encrypted_value": "", "salt": ""} on read but accepts a plain + # string on write. Without unwrapping, the diff fires every run. + current = { + 'title': 't', + 'attributes': { + 'port': 5044, + 'tls_key_password': {'encrypted_value': '', 'salt': ''}, + }, + } + normalized = mod.normalize_current(current) + self.assertEqual(normalized['configuration']['tls_key_password'], '') + + def test_non_empty_secret_dict_left_as_is(self): + # A non-empty encrypted_value cannot be compared to the user's plain + # value; preserve the dict so the diff continues to fire (write-once + # semantics) rather than silently masking a real secret change. + current = { + 'title': 't', + 'attributes': { + 'tls_key_password': {'encrypted_value': 'abc', 'salt': 'xyz'}, + }, + } + normalized = mod.normalize_current(current) + self.assertEqual( + normalized['configuration']['tls_key_password'], + {'encrypted_value': 'abc', 'salt': 'xyz'}, + ) + + +class TestBuildDesired(unittest.TestCase): + + def test_drops_none_values(self): + params = { + 'title': 'a', + 'type': 'foo', + 'configuration': {'port': 1}, + 'global': True, + 'node': None, + } + desired = mod.build_desired(params) + self.assertNotIn('node', desired) + self.assertEqual(desired['title'], 'a') + self.assertEqual(desired['configuration'], {'port': 1}) + + def test_ignores_unknown_keys(self): + # build_desired only picks up _DIFF_FIELDS; ancillary AnsibleModule + # params like `url`/`username`/`state` must not leak into the body. + params = { + 'title': 'a', + 'url': 'http://x', + 'username': 'admin', + 'state': 'present', + } + desired = mod.build_desired(params) + self.assertEqual(set(desired.keys()), {'title'}) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/unit/test_plugin_docs.py b/tests/unit/test_plugin_docs.py index 18cd3d04..fb9eb5bc 100644 --- a/tests/unit/test_plugin_docs.py +++ b/tests/unit/test_plugin_docs.py @@ -73,8 +73,12 @@ def _extract_doc_constants(source): def _iter_description_problems(obj, path=''): """Yield human-readable paths where a `description` is not str / list[str].""" if isinstance(obj, dict): + # Inside an `options` / `suboptions` collection the dict keys are + # option names (one of which may legitimately be "description"), not + # field names. Skip the description field-name check at that level. + in_options_collection = path.endswith('.options') or path.endswith('.suboptions') for key, value in obj.items(): - if key == 'description': + if key == 'description' and not in_options_collection: if isinstance(value, str): pass elif isinstance(value, list):