diff --git a/common/roles/defaults/defaults/main.yaml b/common/roles/defaults/defaults/main.yaml index 1efc47f29..78ae30cc0 100644 --- a/common/roles/defaults/defaults/main.yaml +++ b/common/roles/defaults/defaults/main.yaml @@ -98,6 +98,8 @@ metal_stack_release: capms_controller_name: "docker-images.metal-stack.kubernetes.cluster-api-provider-metal-stack-controller.name" capms_controller_tag: "docker-images.metal-stack.kubernetes.cluster-api-provider-metal-stack-controller.tag" # third party + alloy_image_tag: "docker-images.third-party.partition.alloy.tag" + alloy_image_name: "docker-images.third-party.partition.alloy.name" event_exporter_name: "docker-images.third-party.control-plane.event-exporter.name" event_exporter_tag: "docker-images.third-party.control-plane.event-exporter.tag" nsq_image_tag: "docker-images.third-party.control-plane.nsq.tag" diff --git a/partition/README.md b/partition/README.md index d61154e8e..4d1c9a3f6 100644 --- a/partition/README.md +++ b/partition/README.md @@ -40,7 +40,7 @@ You can look up all the default values [here](partition-defaults/main.yaml). | [metal-bmc](roles/metal-bmc) | Deploys metal-bmc | | [metal-core](roles/metal-core) | Deploys metal-core | | [pixiecore](roles/pixiecore) | Deploys pixiecore | -| [promtail](roles/promtail) | Deploys promtail | +| [alloy](roles/alloy) | Deploys alloy | ## Examples diff --git a/partition/roles/alloy/README.md b/partition/roles/alloy/README.md new file mode 100644 index 000000000..e6defbcbe --- /dev/null +++ b/partition/roles/alloy/README.md @@ -0,0 +1,213 @@ +# alloy + +Deploys [Grafana Alloy](https://grafana.com/docs/alloy/latest/) in a systemd-managed Docker container. + +This role replaces the deprecated `promtail` role. Alloy is configured to forward logs to Loki and exposes its own metrics endpoint for scraping by Prometheus. See [Migration from `promtail`](#migration-from-promtail) for step-by-step migration instructions. + +## Configuration + +The Alloy configuration is driven by a single template `templates/config.alloy.j2`, which contains all supported sections as conditional blocks. At deploy time the template is rendered into `{{ alloy_config_host_dir }}/config.alloy`, with each section included only when the corresponding name appears in `alloy_config_snippets`. + +The sections can be configured with variables defined in the [Variables](#variables) section below. This allows you to enable only the features you need without having to maintain a full custom config. + +### Available snippets + +The snippets are used in our default configs to provide a sensible out-of-the-box setup and can be configured with the variables described in the next section to fit most use cases without modification. + +If your needs are more custom (e.g. you have a non-standard log source, or want to use advanced Alloy features not covered by the existing snippets), you can follow the [Customizing the config](#customizing-the-config) section below to either add your own snippet or bypass the snippet system entirely with a custom raw config. + +| Snippet | Description | +| -------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `alloy-meta` | Meta-monitoring: forwards Alloy's own logs to Loki with `job=alloy`. Redundant when `journal` is enabled — journald already captures Alloy's stdout/stderr. Log level and format are always configured via the base config regardless of this snippet. | +| `docker` | Scrapes Docker container logs on the host via the Docker socket and forwards them to Loki with `job=docker` and `container` labels. Assumes the `json-file` log driver (the default). If containers are systemd-managed or use the `journald` log driver, their logs are already captured by `journal` — enabling both snippets in that case will produce duplicate log entries in Loki. Note: log entries not yet shipped can be lost if Alloy is down when a log rotation occurs and the rotated file is subsequently deleted. | +| `journal` | Scrapes the systemd journal via the journal API and forwards to Loki with `job=systemd-journal`, `unit`, and `level` labels. Automatically discovers the journal regardless of storage mode (volatile or persistent). On standard systemd hosts, journald captures syslog messages as well, so this snippet typically covers everything `syslog` would. Enabling both may produce duplicate log entries in Loki. Note: with a volatile journal (`/run/log/journal`), any entries not yet shipped are lost on reboot — use persistent journal storage to avoid this. | +| `journal-file` | Like `journal`, but reads from an explicit directory path (`alloy_journal_path`, default `/var/log/journal`) instead of the journal API. Produces the same labels (`job=systemd-journal`, `unit`, `level`). Prefer `journal` for new deployments — use `journal-file` only when migrating from promtail and needing the `legacy_position` block to resume cursor state. Do not enable both `journal` and `journal-file` simultaneously — this will produce duplicate log entries. | +| `syslog` | Tails `/var/log/syslog` and forwards to Loki with `job=syslog`. Suited for hosts without journald (e.g. minimal switch OS images). On standard systemd hosts, prefer `journal` — enabling both may produce duplicate log entries in Loki. | + +### Variables + +| Name | Mandatory | Default | Description | +| ----------------------------------- | -------------------------------------- | ---------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| alloy_config_host_dir | | `/etc/alloy` | The location of the alloy config on the host | +| alloy_image_name | yes | | Image name of alloy | +| alloy_image_tag | yes | | Image tag of alloy | +| alloy_loki_write_endpoints | yes (unless `alloy_config_raw` is set) | | List of Loki push endpoints. Each entry: `{url, remote_timeout?: , basic_auth?: {username, password}}` | +| alloy_docker_log_driver | | `json-file` | Docker log driver for the alloy container | +| alloy_config_snippets | | `[]` | List of built-in snippet names to enable. Available: `syslog`, `journal`, `journal-file`, `docker`, `alloy-meta` | +| alloy_config_custom_snippets | | `[]` | List of paths to custom Alloy River snippet templates. Resolved via Ansible's template search path (relative to the playbook's `templates/` directory, or absolute). Appended after built-in snippets. See [Customizing the config](#customizing-the-config). | +| alloy_port | | `12345` | Port for Alloy metrics and HTTP API | +| alloy_migrate_from_promtail | | `false` | Enable migration mode: imports cursor state from the legacy promtail positions file on first start. Set to `true` when migrating from promtail; leave `false` for fresh deployments. Without this, Alloy starts from the current tail and previously shipped log data will be re-shipped. | +| alloy_syslog_legacy_positions_file | | `/var/log/promtail-positions.yaml` | Path to the legacy promtail positions file. Used by `syslog` when `alloy_migrate_from_promtail` is `true`. | +| alloy_journal_path | | `/var/log/journal` | Path to the persistent journal directory used by the `journal-file` snippet | +| alloy_journal_legacy_positions_file | | `/var/log/promtail-positions.yaml` | Path to the legacy promtail positions file. Used by `journal-file` when `alloy_migrate_from_promtail` is `true`. | +| alloy_journal_legacy_position_name | yes (migration) | | Job name from the old promtail journal scrape_config. Must match `job_name` in your old promtail config. Required when `alloy_migrate_from_promtail` is `true` and `journal-file` is used. | +| alloy_config_raw | | | Full Alloy River config as a string. When set, bypasses snippet assembly entirely — `alloy_loki_write_endpoints` and `alloy_config_snippets` are ignored. | +| promtail_migrate_stop | | `false` | Stop and disable the promtail systemd service. Use during alloy cutover to stop promtail without removing its files. Implied by `promtail_migrate_cleanup`. | +| promtail_migrate_cleanup | | `false` | Remove all promtail remnants: stops and disables the service, removes the systemd unit file, deletes `promtail_config_host_dir` and the legacy positions files. Only set this once migration to alloy is fully complete. | + +### Meta-monitoring (for Alloy itself) + +#### Metrics + +Alloy always exposes Prometheus metrics on `0.0.0.0:{{ alloy_port }}/metrics`, regardless of which snippets are enabled. Add the host targets to `prometheus_alloy_targets` in your inventory to have the prometheus role scrape them. + +Scraped metrics will carry the following labels (set by the prometheus role, not the alloy role): + +| Label | Value | +| ----------- | ------------------------------------------------------------------------------- | +| `job` | `alloy` | +| `instance` | hostname of the target (port stripped) | +| `partition` | `metal_partition_id` (from Prometheus `external_labels`) | +| `replica` | `inventory_hostname` of the Prometheus host (from Prometheus `external_labels`) | + +#### Labels + +All log entries carry `host` (set to `inventory_hostname`) and `partition` (set to `metal_partition_id`) as `external_labels` on `loki.write "default"`, so they are appended to every log entry regardless of snippet. Per-snippet labels are: + +| Snippet | Labels | +| -------------- | -------------------------------------- | +| `alloy-meta` | `job=alloy` | +| `docker` | `job=docker`, `container` | +| `journal` | `job=systemd-journal`, `unit`, `level` | +| `journal-file` | `job=systemd-journal`, `unit`, `level` | +| `syslog` | `job=syslog` | + +#### Logs + +Alloy's own logs are captured in two ways depending on your snippet configuration: + +- **If `journal` is enabled** — Alloy runs as a systemd-managed Docker container, so its stdout/stderr are captured by journald automatically. No additional snippet is needed. +- **If `journal` is not enabled** (e.g. you only use `syslog` or no log snippet at all) — enable the `alloy-meta` snippet to forward Alloy's own logs to Loki explicitly. + +Alloy's log level (`info`) and format (`logfmt`) are always configured via the base config, regardless of which snippets are enabled. The `alloy-meta` snippet only adds the Loki forwarding pipeline on top. + +### Cursor/positions persistence + +`loki.source.file` and `loki.source.journal` track their read positions in small YAML positions files written under `--storage.path`, which is set to `/var/lib/alloy` (bind-mounted from the host). These files stay in the low-KB range regardless of log volume and survive container restarts, preventing duplicate log shipments. + +When migrating from promtail, set `alloy_migrate_from_promtail: true` to import cursor state from promtail's positions file on first start and continue from where promtail left off. The exact mechanism depends on the snippet: + +| Snippet | Cursor mechanism | Promtail migration path | +| -------------- | -------------------------------------------------------- | ---------------------------------------------------------------------------- | +| `syslog` | Positions file via `loki.source.file` | `alloy_syslog_legacy_positions_file` | +| `journal` | Positions file via `loki.source.journal` (journal API) | None — first start re-reads a small journal window once | +| `journal-file` | Positions file via `loki.source.journal` (explicit path) | `alloy_journal_legacy_positions_file` + `alloy_journal_legacy_position_name` | + +**`syslog`:** On first start after migration, Alloy reads `alloy_syslog_legacy_positions_file` (default: `/var/log/promtail-positions.yaml`) to continue tailing `/var/log/syslog` from where promtail left off. No log lines are re-shipped. After this Alloy tracks positions in its own format. If the file does not exist on the host, this has no effect. + +**`journal-file`:** On first start after migration, Alloy reads the legacy positions file via the `legacy_position` block using `alloy_journal_legacy_positions_file` and `alloy_journal_legacy_position_name`. The name must match the `job_name` of the journal scrape config in your old promtail config. If the file does not exist on the host, this has no effect. + +**`journal`:** No promtail-compatible migration path. The first start after migration re-reads a small window of the journal — this is a one-time event and subsequent restarts are safe once the cursor is written. + +Once migration is complete, the old promtail positions files can be cleaned up from hosts — Alloy will already be tracking positions in its own format from that point. + +> **Note:** Alloy also has an experimental [WAL feature for `loki.write`](https://grafana.com/docs/alloy/latest/reference/components/loki/loki.write/#wal) that buffers outgoing log entries on disk when Loki is temporarily unavailable. It is disabled by default and not used by this role. It is not needed for `syslog`, `journal-file`, or `journal` with persistent storage, as Alloy resumes from the saved cursor after a restart. For `docker`, it reduces the risk of losing entries during log rotation. For `journal` with volatile storage, it provides no benefit — entries are lost on reboot regardless since the journal itself is wiped. + +## Customizing the config + +Your options, from least to most invasive: + +**Option A — Custom snippets via inventory** (no role change required) + +Add a Jinja2 template file to your playbook's `templates/` directory and reference it via `alloy_config_custom_snippets`: + +```yaml +alloy_config_custom_snippets: + - my_custom_source.alloy.j2 +``` + +The path is resolved through Ansible's normal template search path (relative to the playbook's `templates/` directory, or an absolute path). The snippet is rendered after all built-in snippets and has access to all Ansible variables on the host. It does not need to define `loki.write "default"` — that is already in the base template. + +Example `templates/my_custom_source.alloy.j2`: + +```river +loki.source.file "custom" { + targets = [{ __path__ = "/var/log/myapp/*.log", job = "myapp" }] + forward_to = [loki.write.default.receiver] +} +``` + +**Option B — Contribute a built-in snippet to this role** (requires editing this repo) + +Add a new file `templates/snippets/.alloy.j2` and reference it by name in `alloy_config_snippets`: + +```yaml +alloy_config_snippets: + - alloy-meta + - docker +``` + +A snippet can use any Ansible variables available on the host. It does not need to define `loki.write "default"` as it is already in the base template. + +Contributions of new snippets are welcome — since all snippets are opt-in via `alloy_config_snippets`, adding one to the role has no impact on existing deployments. + +**Option C — Use `alloy_config_raw`** (inventory only, no role change) + +Set `alloy_config_raw` to a full Alloy River config string in your inventory. The role will write it verbatim and skip snippet assembly entirely. `alloy_loki_write_endpoints`, `alloy_config_snippets`, and `alloy_config_custom_snippets` are ignored. You own the complete config, including the base `loki.write "default"` block. + +## Migration from `promtail` + +Alloy allows reproducing the same log forwarding behavior as promtail via the [available snippets](#available-snippets), but it is a different tool with a different internal data model. Label names, pipeline stages, and how metrics are exposed might differ from promtail. **Review your existing Loki dashboards and alerting rules after the migration and adapt them where necessary.** + +Use the inventory flags below to control the deployment based on your situation: + +| Scenario | `alloy_migrate_from_promtail` | `promtail_migrate_stop` | Notes | +| ------------------------------------ | ----------------------------- | ----------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Fresh deployment** | `false` | `false` | No prior promtail. Alloy starts from the current log tail. | +| **Existing promtail — parallel run** | `true` | `false` | Both services ship logs simultaneously; Alloy resumes from promtail's cursor. Expect duplicate log entries during the overlap. Use to verify Alloy in production before cutting over. | +| **Existing promtail — cutover** | `true` | `true` | Alloy starts resuming from promtail's cursor; promtail service is stopped and disabled. Promtail files and container are left in place. There is a brief gap between promtail stopping and Alloy starting — entries written in this window are not shipped immediately but are safe as long as the log source persists across the gap (journal file, syslog on disk). With a volatile journal this is unlikely to matter in practice since the system stays running, but a reboot in this window would lose those entries. | + +1. **Add the `alloy` role** to your playbook. Keep `promtail` alongside it for a gradual migration, or remove it for a hard cut-over. + +2. **Configure the Loki endpoint.** Set `alloy_loki_write_endpoints` in your inventory: + + ```yaml + alloy_loki_write_endpoints: + - url: "http://loki.{{ metal_control_plane_ingress_dns }}:8080/loki/api/v1/push" + remote_timeout: 10s + basic_auth: + username: "promtail" + password: "test" + ``` + + The `username` and `password` here must match `logging_ingress_loki_basic_auth_user` and `logging_ingress_loki_basic_auth_password` configured in the control-plane [logging role](../../../control-plane/roles/logging/README.md). The default username on the control-plane side is `promtail` and can optionally be changed there — update both sides together to avoid auth failures. + +3. **Choose snippets** that match your legacy `promtail_scrape_configs` — see [Available snippets](#available-snippets). If the available snippets are close enough to your old setup, enable the relevant ones and adjust with the provided variables. Otherwise, continue with step 4. + + Set `alloy_migrate_from_promtail: true` to import cursor state from the legacy promtail positions file so Alloy resumes from where promtail left off. See [Cursor/positions persistence](#cursorpositions-persistence) for per-snippet details and the relevant variables. + **Without this, larger amounts of previously shipped entries may be re-shipped.** + + If your environment deploys Alloy on different host groups with different scrape needs (e.g. leaf switches vs. management servers), set `alloy_config_snippets` in per-group inventory files: + + ```text + group_vars/leaves/alloy.yaml ← alloy_config_snippets: [journal] + group_vars/mgmtservers/alloy.yaml ← alloy_config_snippets: [journal-file] + group_vars/partition/alloy.yaml ← alloy_loki_write_endpoints: [...] (shared by all) + ``` + +4. **Convert custom promtail scrape configs** _(optional — only needed if you extended the promtail setup beyond what the available snippets cover)_. Use the [`alloy convert`](https://grafana.com/docs/alloy/latest/reference/cli/convert/) CLI (`--source-format=promtail`) to get a starting point. Be aware of the [converter limitations](https://grafana.com/docs/alloy/latest/set-up/migrate/from-promtail/#limitations) — review the output manually before use. + + You can run the converter without installing Alloy locally: + + ```bash + docker run \ + -v ./promtail.yaml:/etc/promtail/promtail.yaml \ + -v ./config.alloy:/etc/alloy/config.alloy \ + grafana/alloy:latest \ + convert --source-format=promtail --output=/etc/alloy/config.alloy /etc/promtail/promtail.yaml + ``` + + The input must be a fully rendered promtail config (no Ansible variables). Substitute real values before running the converter, then replace hardcoded host-specific values (hostnames, partition IDs, URLs, credentials) with Ansible variables (e.g. `{{ inventory_hostname }}`, `{{ metal_partition_id }}`) in the output. + - **If the config is generally useful** (not environment-specific), consider contributing it as a new snippet to this role — see [Option B in Customizing the config](#customizing-the-config). + - **If the config is specific to your environment**, use `alloy_config_raw` instead — see [Option A in Customizing the config](#customizing-the-config). Note that with `alloy_config_raw` you own the complete config, including the `loki.write "default"` block and its endpoint configuration; `alloy_loki_write_endpoints` is ignored. + +5. **Update Prometheus inventory:** rename `prometheus_promtail_targets` → `prometheus_alloy_targets` (port `12345` instead of `9080`). + +6. **Verify** that logs arrive in Loki with the correct labels. Check that existing dashboards and alerts still work as expected and adapt them for any label or metric name changes. + +7. **Cut over from promtail** _(parallel run only)_. Set `promtail_migrate_stop: true` in your inventory and re-run the playbook — the alloy role will stop and disable the service. You can also set `alloy_migrate_from_promtail: false` at this point since the cursor state has already been imported on first start. + + When you are ready, clean up all promtail remnants by setting `promtail_migrate_cleanup: true` and re-running the playbook — the alloy role will stop and disable the service, remove the systemd unit file, config directory, and positions files. Once cleanup is complete, remove the `promtail` role from the playbook entirely. + We do not recommend doing the cleanup at the same time as the alloy deployment and migration steps, as the promtail positions file may be needed for the migration and the cleanup removes it. If you anyway choose to set the cleanup flag at the same time as the migration, make sure that the alloy role runs first to avoid migration failures due to the missing positions files. + + The `promtail` role is deprecated and will be removed in a future release. Once all environments are migrated, remove the `promtail` role from the playbook. diff --git a/partition/roles/alloy/defaults/main.yaml b/partition/roles/alloy/defaults/main.yaml new file mode 100644 index 000000000..2e912db6c --- /dev/null +++ b/partition/roles/alloy/defaults/main.yaml @@ -0,0 +1,67 @@ +--- +alloy_config_host_dir: "/etc/alloy" +alloy_docker_log_driver: json-file + +# List of Loki push endpoints used by the base loki.write component. +# Each entry: {url: , remote_timeout?: , basic_auth?: {username, password}} +# alloy_loki_write_endpoints: +# - url: http://loki.example.com:8080/loki/api/v1/push +# remote_timeout: 10s +# basic_auth: +# username: "username" +# password: "password" + +# List of built-in snippet names to enable. Each entry must correspond to +# templates/snippets/.alloy.j2 in this role. +# Available snippets: syslog, journal, journal-file, docker, alloy-meta +alloy_config_snippets: [] + +# List of paths to custom Alloy River config snippets. Each entry is a +# template path resolved via Ansible's normal template search path (i.e. +# relative to the playbook's templates/ directory, or an absolute path). +# Custom snippets are appended after all built-in snippets. +# Example: +# alloy_config_custom_snippets: +# - my_custom_source.alloy.j2 +alloy_config_custom_snippets: [] + +# Provide a fully custom Alloy River config as a string to bypass the +# snippet assembly system entirely. When set, alloy_loki_write_endpoints +# and alloy_config_snippets are ignored. +# alloy_config_raw: | + +# Port to listen for metrics and HTTP API traffic on +alloy_port: 12345 + +# Enable migration mode: imports cursor state from the legacy promtail positions file +# on first start so Alloy resumes from where promtail left off. Set to true when +# migrating from promtail; leave false for fresh deployments. +alloy_migrate_from_promtail: false + +# Path to the legacy promtail positions file used by the syslog snippet on first start +# after migration to continue tailing /var/log/syslog from where promtail left off. +# Only active when alloy_migrate_from_promtail is true. +alloy_syslog_legacy_positions_file: "/var/log/promtail-positions.yaml" + +# Path to the persistent journal directory used by the journal-file snippet. +alloy_journal_path: "/var/log/journal" + +# Path to the legacy promtail positions file used by the journal-file snippet on first +# start after migration to resume from where promtail left off. +# Only active when alloy_migrate_from_promtail is true. +alloy_journal_legacy_positions_file: "/var/log/promtail-positions.yaml" + +# Job name used in the legacy promtail positions file for the journal scrape config. +# Must match the job_name of the journal scrape_config in your old promtail config. +# Required when alloy_migrate_from_promtail is true and the journal-file snippet is used. +# No default — must be set explicitly. +# alloy_journal_legacy_position_name: + +# Stop and disable the promtail systemd service during alloy cutover. +# Implied by promtail_migrate_cleanup. +promtail_migrate_stop: false + +# Remove all promtail remnants: stops and disables the service, +# removes the systemd unit file, deletes promtail_config_host_dir and +# the legacy positions files. Only set once migration to alloy is fully complete. +promtail_migrate_cleanup: false diff --git a/partition/roles/alloy/handlers/main.yaml b/partition/roles/alloy/handlers/main.yaml new file mode 100644 index 000000000..8056a839d --- /dev/null +++ b/partition/roles/alloy/handlers/main.yaml @@ -0,0 +1,10 @@ +--- +- name: restart alloy + service: + name: alloy + enabled: true + state: restarted + +- name: reload systemd + systemd: + daemon_reload: true diff --git a/partition/roles/alloy/meta/main.yml b/partition/roles/alloy/meta/main.yml new file mode 100644 index 000000000..d275533f2 --- /dev/null +++ b/partition/roles/alloy/meta/main.yml @@ -0,0 +1,18 @@ +--- +galaxy_info: + role_name: alloy + author: metal-stack + description: Deploys alloy. + license: MIT + min_ansible_version: "2.10" + galaxy_tags: [] + + platforms: + - name: Debian + versions: + - all + +dependencies: + - role: ansible-common + - role: metal-roles/common/roles/defaults + - role: metal-roles/partition/roles/defaults diff --git a/partition/roles/alloy/tasks/main.yaml b/partition/roles/alloy/tasks/main.yaml new file mode 100644 index 000000000..231d3aae8 --- /dev/null +++ b/partition/roles/alloy/tasks/main.yaml @@ -0,0 +1,142 @@ +--- +- name: Gather release versions + setup_yaml: + +- name: Check mandatory variables for this role are set + assert: + fail_msg: "not all mandatory variables given, check role documentation" + quiet: true + that: + - alloy_image_tag is defined + - alloy_image_name is defined + - alloy_config_raw is defined or (alloy_loki_write_endpoints is defined and (alloy_loki_write_endpoints | length > 0)) + +- name: Check alloy_journal_legacy_position_name is set when migrating journal-file from promtail + assert: + fail_msg: "alloy_journal_legacy_position_name must be set — it must match the job_name of the journal scrape_config in your old promtail config" + quiet: true + that: + - alloy_journal_legacy_position_name is defined + when: + - "'journal-file' in alloy_config_snippets" + - alloy_migrate_from_promtail + +- name: Check alloy_config_raw and alloy_config_snippets are not both set + assert: + fail_msg: "alloy_config_raw and alloy_config_snippets/alloy_config_custom_snippets cannot both be set — use either raw config or snippets, not both" + quiet: true + that: + - not (alloy_config_raw is defined and (alloy_config_snippets | length > 0 or alloy_config_custom_snippets | length > 0)) + +- name: Check alloy_config_snippets only contains supported snippets + assert: + fail_msg: "alloy_config_snippets contains unsupported snippet(s): {{ alloy_config_snippets | difference(['syslog', 'journal', 'journal-file', 'docker', 'alloy-meta']) | join(', ') }}" + quiet: true + that: + - alloy_config_snippets | difference(['syslog', 'journal', 'journal-file', 'docker', 'alloy-meta']) | length == 0 + +- name: Check journal and journal-file are not both enabled + assert: + fail_msg: "journal and journal-file snippets cannot both be enabled — this will produce duplicate log entries in Loki" + quiet: true + that: + - not ('journal' in alloy_config_snippets and 'journal-file' in alloy_config_snippets) + +- name: Create alloy config directory + file: + path: "{{ alloy_config_host_dir }}" + state: directory + owner: "nobody" + group: "nogroup" + mode: "0755" + +- name: Template alloy config + template: + src: config.alloy.j2 + dest: "{{ alloy_config_host_dir }}/config.alloy" + notify: restart alloy + when: alloy_config_raw is not defined + +- name: Write raw alloy config + copy: + content: "{{ alloy_config_raw }}" + dest: "{{ alloy_config_host_dir }}/config.alloy" + notify: restart alloy + when: alloy_config_raw is defined + +- name: Create alloy storage directory + file: + path: /var/lib/alloy + state: directory + owner: "nobody" + group: "nogroup" + mode: "0755" + +- name: Deploy alloy service + include_role: + name: ansible-common/roles/systemd-docker-service + vars: + systemd_service_name: alloy + systemd_docker_image_name: "{{ alloy_image_name }}" + systemd_docker_image_tag: "{{ alloy_image_tag }}" + systemd_service_after: docker.service + systemd_service_requires: docker.service + systemd_docker_ports: + - host_port: "{{ alloy_port }}" + target_port: "{{ alloy_port }}" + systemd_docker_volumes: + - "{{ alloy_config_host_dir }}:/etc/alloy:ro" + - "/var/lib/alloy:/var/lib/alloy" + - "/var/log:/var/log" + - "/run/log/journal:/run/log/journal:ro" + - "/etc/machine-id:/etc/machine-id:ro" + - "/var/run/docker.sock:/var/run/docker.sock" + systemd_docker_command: + - run + - --server.http.listen-addr=0.0.0.0:{{ alloy_port }} + - --storage.path=/var/lib/alloy + - --disable-reporting + - /etc/alloy/config.alloy + systemd_docker_log_driver: "{{ alloy_docker_log_driver }}" + +- name: Wait for alloy to listen on port + wait_for: + port: "{{ alloy_port }}" + timeout: 300 + msg: "alloy did not come up" + +# promtail migration +- name: Check if promtail systemd unit file is present + stat: + path: /etc/systemd/system/promtail.service + register: promtail_unit_file + when: promtail_migrate_stop or promtail_migrate_cleanup + +- name: Stop and disable promtail service + systemd: + name: promtail + state: stopped + enabled: false + when: + - promtail_migrate_stop or promtail_migrate_cleanup + - promtail_unit_file.stat.exists + +- name: Remove promtail systemd unit file + file: + path: /etc/systemd/system/promtail.service + state: absent + when: promtail_migrate_cleanup + notify: reload systemd + +- name: Remove promtail config directory + file: + path: "{{ promtail_config_host_dir }}" + state: absent + when: promtail_migrate_cleanup + +- name: Remove promtail positions file + file: + path: "{{ item }}" + state: absent + loop: "{{ [alloy_syslog_legacy_positions_file, alloy_journal_legacy_positions_file] | unique }}" + when: promtail_migrate_cleanup diff --git a/partition/roles/alloy/templates/config.alloy.j2 b/partition/roles/alloy/templates/config.alloy.j2 new file mode 100644 index 000000000..64fb580f0 --- /dev/null +++ b/partition/roles/alloy/templates/config.alloy.j2 @@ -0,0 +1,39 @@ +// Log level and format; write_to forwards Alloy's own logs to Loki when alloy-meta is enabled +logging { + level = "info" + format = "logfmt" +{% if "alloy-meta" in alloy_config_snippets %} + write_to = [loki.relabel.alloy_self.receiver] +{% endif %} +} + +// Loki write endpoint shared by all snippets; external_labels are appended to every log entry +loki.write "default" { + external_labels = { + host = "{{ inventory_hostname }}", + partition = "{{ metal_partition_id }}", + } +{% for endpoint in alloy_loki_write_endpoints %} + endpoint { + url = "{{ endpoint.url }}" +{% if endpoint.remote_timeout is defined %} + remote_timeout = "{{ endpoint.remote_timeout }}" +{% endif %} +{% if endpoint.basic_auth is defined %} + basic_auth { + username = "{{ endpoint.basic_auth.username }}" + password = "{{ endpoint.basic_auth.password }}" + } +{% endif %} + } +{% endfor %} +} + +{% for snippet in alloy_config_snippets %} +// snippet: {{ snippet }} +{% include "snippets/" + snippet + ".alloy.j2" %} +{% endfor %} +{% for snippet in alloy_config_custom_snippets %} +// snippet: {{ snippet }} +{% include snippet %} +{% endfor %} diff --git a/partition/roles/alloy/templates/snippets/alloy-meta.alloy.j2 b/partition/roles/alloy/templates/snippets/alloy-meta.alloy.j2 new file mode 100644 index 000000000..34bff47df --- /dev/null +++ b/partition/roles/alloy/templates/snippets/alloy-meta.alloy.j2 @@ -0,0 +1,9 @@ +// Meta-monitoring: forward Alloy's own logs to Loki +loki.relabel "alloy_self" { + forward_to = [loki.write.default.receiver] + + rule { + target_label = "job" + replacement = "alloy" + } +} diff --git a/partition/roles/alloy/templates/snippets/docker.alloy.j2 b/partition/roles/alloy/templates/snippets/docker.alloy.j2 new file mode 100644 index 000000000..71176376a --- /dev/null +++ b/partition/roles/alloy/templates/snippets/docker.alloy.j2 @@ -0,0 +1,27 @@ +discovery.docker "docker" { + host = "unix:///var/run/docker.sock" + refresh_interval = "5s" +} + +discovery.relabel "docker" { + targets = [] + + rule { + source_labels = ["__meta_docker_container_name"] + regex = "/(.*)" + target_label = "container" + } + + rule { + target_label = "job" + replacement = "docker" + } +} + +loki.source.docker "docker" { + host = "unix:///var/run/docker.sock" + targets = discovery.docker.docker.targets + forward_to = [loki.write.default.receiver] + relabel_rules = discovery.relabel.docker.rules + refresh_interval = "5s" +} diff --git a/partition/roles/alloy/templates/snippets/journal-file.alloy.j2 b/partition/roles/alloy/templates/snippets/journal-file.alloy.j2 new file mode 100644 index 000000000..09611c2a9 --- /dev/null +++ b/partition/roles/alloy/templates/snippets/journal-file.alloy.j2 @@ -0,0 +1,32 @@ +discovery.relabel "journal" { + targets = [] + + rule { + source_labels = ["__journal__systemd_unit"] + target_label = "unit" + } + + rule { + source_labels = ["__journal_priority_keyword"] + target_label = "level" + } +} + +loki.source.journal "journal" { + path = "{{ alloy_journal_path }}" + relabel_rules = discovery.relabel.journal.rules + forward_to = [loki.write.default.receiver] + labels = { + job = "systemd-journal", + } +{% if alloy_migrate_from_promtail %} + // migration only: resumes reading from where promtail left off on first start. + // ignored if the file does not exist on the host. + // once alloy has written its own positions file, this can be removed. + // https://grafana.com/docs/alloy/latest/reference/components/loki/loki.source.journal/#legacy_position + legacy_position { + file = "{{ alloy_journal_legacy_positions_file }}" + name = "{{ alloy_journal_legacy_position_name }}" + } +{% endif %} +} diff --git a/partition/roles/alloy/templates/snippets/journal.alloy.j2 b/partition/roles/alloy/templates/snippets/journal.alloy.j2 new file mode 100644 index 000000000..f2f8d2cfb --- /dev/null +++ b/partition/roles/alloy/templates/snippets/journal.alloy.j2 @@ -0,0 +1,21 @@ +discovery.relabel "journal" { + targets = [] + + rule { + source_labels = ["__journal__systemd_unit"] + target_label = "unit" + } + + rule { + source_labels = ["__journal_priority_keyword"] + target_label = "level" + } +} + +loki.source.journal "journal" { + relabel_rules = discovery.relabel.journal.rules + forward_to = [loki.write.default.receiver] + labels = { + job = "systemd-journal", + } +} diff --git a/partition/roles/alloy/templates/snippets/syslog.alloy.j2 b/partition/roles/alloy/templates/snippets/syslog.alloy.j2 new file mode 100644 index 000000000..8b5be5509 --- /dev/null +++ b/partition/roles/alloy/templates/snippets/syslog.alloy.j2 @@ -0,0 +1,18 @@ +loki.source.file "syslog" { + targets = [{ + __path__ = "/var/log/syslog", + job = "syslog", + }] + forward_to = [loki.write.default.receiver] + + file_match { + enabled = true + } +{% if alloy_migrate_from_promtail %} + // migration only: resumes tailing from where promtail left off on first start. + // ignored if the file does not exist on the host. + // once alloy has written its own positions file, this can be removed. + // https://grafana.com/docs/alloy/latest/reference/components/loki/loki.source.file/#arguments + legacy_positions_file = "{{ alloy_syslog_legacy_positions_file }}" +{% endif %} +} diff --git a/partition/roles/monitoring/prometheus/README.md b/partition/roles/monitoring/prometheus/README.md index fe47ab040..e1fa459ca 100644 --- a/partition/roles/monitoring/prometheus/README.md +++ b/partition/roles/monitoring/prometheus/README.md @@ -22,6 +22,7 @@ you define them adequately as well. | prometheus_frr_exporter_targets | | FRR exporter targets to scrape from | | prometheus_metal_core_targets | | metal-core targets to scrape from | | prometheus_node_exporter_targets | | Node exporter targets to scrape from | +| prometheus_alloy_targets | | Alloy targets to scrape from (default port `12345`) | | prometheus_promtail_targets | | Promtail targets to scrape from | | prometheus_ping_targets | | Ping targets to scrape from | | prometheus_sonic_exporter_targets | | Sonic exporter targets to scrape from | diff --git a/partition/roles/monitoring/prometheus/defaults/main.yaml b/partition/roles/monitoring/prometheus/defaults/main.yaml index 092817f82..da28bcfc8 100644 --- a/partition/roles/monitoring/prometheus/defaults/main.yaml +++ b/partition/roles/monitoring/prometheus/defaults/main.yaml @@ -16,10 +16,16 @@ prometheus_alertmanager_target: prometheus_alertmanager_basic_auth_username: alertmanager prometheus_alertmanager_basic_auth_password: prometheus_remote_write: [] + # - url: "https://monitoring.{{ metal_control_plane_ingress_dns }}/api/v1/receive" + # basic_auth: + # username: thanos-receive + # password: "{{ basic_auth_pass }}" + prometheus_frr_exporter_targets: [] prometheus_metal_core_targets: [] prometheus_node_exporter_targets: [] prometheus_promtail_targets: [] +prometheus_alloy_targets: [] prometheus_ping_targets: [] prometheus_ipmi_exporter_targets: [] prometheus_sonic_exporter_targets: [] diff --git a/partition/roles/monitoring/prometheus/tasks/main.yaml b/partition/roles/monitoring/prometheus/tasks/main.yaml index 6aac6622e..d37c0bdae 100644 --- a/partition/roles/monitoring/prometheus/tasks/main.yaml +++ b/partition/roles/monitoring/prometheus/tasks/main.yaml @@ -120,7 +120,7 @@ set_fact: prometheus_promtail_sd: - labels: - job: frr + job: promtail targets: "{{ prometheus_promtail_targets }}" - name: create promtail.yaml @@ -129,6 +129,19 @@ dest: "{{ prometheus_config_host_dir }}/file_sd/promtail.yaml" notify: restart prometheus +- name: generate prometheus_alloy_sd + set_fact: + prometheus_alloy_sd: + - labels: + job: alloy + targets: "{{ prometheus_alloy_targets }}" + +- name: create alloy.yaml + copy: + content: "{{ prometheus_alloy_sd | to_nice_yaml(indent=2) }}" + dest: "{{ prometheus_config_host_dir }}/file_sd/alloy.yaml" + notify: restart prometheus + - name: generate prometheus_sonic_exporter_sd set_fact: prometheus_sonic_exporter_sd: diff --git a/partition/roles/monitoring/prometheus/templates/prometheus.yaml.j2 b/partition/roles/monitoring/prometheus/templates/prometheus.yaml.j2 index b7e6bf4af..d0f0b1517 100644 --- a/partition/roles/monitoring/prometheus/templates/prometheus.yaml.j2 +++ b/partition/roles/monitoring/prometheus/templates/prometheus.yaml.j2 @@ -88,6 +88,15 @@ scrape_configs: regex: (.*):\d+ target_label: instance +- job_name: alloy + file_sd_configs: + - files: + - '/etc/prometheus/file_sd/alloy.yaml' + relabel_configs: + - source_labels: [__address__] + regex: (.*):\d+ + target_label: instance + - job_name: sonic-exporter file_sd_configs: - files: diff --git a/partition/roles/promtail/README.md b/partition/roles/promtail/README.md index fcb2f4da8..2726a8a90 100644 --- a/partition/roles/promtail/README.md +++ b/partition/roles/promtail/README.md @@ -1,5 +1,8 @@ # promtail +> [!IMPORTANT] +> This role is deprecated and superseded by the `alloy` role. New deployments should use `metal-roles/partition/roles/alloy` instead. This role is kept for migration purposes only and may be removed in a future release. See the [alloy role README](../alloy/README.md) for migration instructions. + Deploys promtail in a systemd-managed Docker container. ## Variables diff --git a/partition/roles/promtail/meta/main.yml b/partition/roles/promtail/meta/main.yml index aaafedce6..054095902 100644 --- a/partition/roles/promtail/meta/main.yml +++ b/partition/roles/promtail/meta/main.yml @@ -2,7 +2,7 @@ galaxy_info: role_name: promtail author: metal-stack - description: Deploys promtail. + description: "DEPRECATED: Deploys promtail. Use metal-roles/partition/roles/alloy instead." license: MIT min_ansible_version: "2.10" galaxy_tags: [] diff --git a/partition/roles/promtail/tasks/main.yaml b/partition/roles/promtail/tasks/main.yaml index 5771124c4..1ae1311d0 100644 --- a/partition/roles/promtail/tasks/main.yaml +++ b/partition/roles/promtail/tasks/main.yaml @@ -1,4 +1,11 @@ --- +- name: Deprecation warning + ansible.builtin.fail: + msg: >- + The promtail role is deprecated and will be removed in a future release. + New deployments should use metal-roles/partition/roles/alloy instead - see https://github.com/metal-stack/metal-roles/tree/main/partition/roles/alloy#migration-from-promtail for instructions. + ignore_errors: true + - name: Gather release versions setup_yaml: