From 7e9458e4a48beca882647631ef49b5c1e3d87251 Mon Sep 17 00:00:00 2001
From: Matthias Hartmann <matthias.hartmann@x-cellent.com>
Date: Mon, 4 May 2026 15:58:42 +0200
Subject: [PATCH 01/41] feat: introduce alloy role to replace promtail in
 partition

---
 common/roles/defaults/defaults/main.yaml      |  2 +
 partition/roles/alloy/README.md               | 91 +++++++++++++++++++
 partition/roles/alloy/defaults/main.yaml      | 21 +++++
 partition/roles/alloy/handlers/main.yaml      |  6 ++
 partition/roles/alloy/meta/main.yml           | 17 ++++
 partition/roles/alloy/tasks/main.yaml         | 78 ++++++++++++++++
 .../roles/alloy/templates/config.alloy.j2     | 15 +++
 .../templates/snippets/alloy-self.alloy.j2    | 20 ++++
 .../templates/snippets/journal-logs.alloy.j2  | 25 +++++
 .../snippets/leaf-node-docker.alloy.j2        | 27 ++++++
 .../templates/snippets/syslog-logs.alloy.j2   | 19 ++++
 .../roles/monitoring/prometheus/README.md     |  1 +
 .../monitoring/prometheus/defaults/main.yaml  |  1 +
 .../monitoring/prometheus/tasks/main.yaml     | 13 +++
 .../prometheus/templates/prometheus.yaml.j2   |  9 ++
 partition/roles/promtail/README.md            |  2 +
 16 files changed, 347 insertions(+)
 create mode 100644 partition/roles/alloy/README.md
 create mode 100644 partition/roles/alloy/defaults/main.yaml
 create mode 100644 partition/roles/alloy/handlers/main.yaml
 create mode 100644 partition/roles/alloy/meta/main.yml
 create mode 100644 partition/roles/alloy/tasks/main.yaml
 create mode 100644 partition/roles/alloy/templates/config.alloy.j2
 create mode 100644 partition/roles/alloy/templates/snippets/alloy-self.alloy.j2
 create mode 100644 partition/roles/alloy/templates/snippets/journal-logs.alloy.j2
 create mode 100644 partition/roles/alloy/templates/snippets/leaf-node-docker.alloy.j2
 create mode 100644 partition/roles/alloy/templates/snippets/syslog-logs.alloy.j2

diff --git a/common/roles/defaults/defaults/main.yaml b/common/roles/defaults/defaults/main.yaml
index ac30dd1c3..cbbdd7661 100644
--- a/common/roles/defaults/defaults/main.yaml
+++ b/common/roles/defaults/defaults/main.yaml
@@ -96,6 +96,8 @@ metal_stack_release:
     capms_controller_name: "docker-images.metal-stack.kubernetes.cluster-api-provider-metal-stack-controller.name"
     capms_controller_tag: "docker-images.metal-stack.kubernetes.cluster-api-provider-metal-stack-controller.tag"
     # third party
+    alloy_image_tag: "docker-images.third-party.control-plane.alloy.tag"
+    alloy_image_name: "docker-images.third-party.control-plane.alloy.name"
     event_exporter_name: "docker-images.third-party.control-plane.event-exporter.name"
     event_exporter_tag: "docker-images.third-party.control-plane.event-exporter.tag"
     nsq_image_tag: "docker-images.third-party.control-plane.nsq.tag"
diff --git a/partition/roles/alloy/README.md b/partition/roles/alloy/README.md
new file mode 100644
index 000000000..657c5b417
--- /dev/null
+++ b/partition/roles/alloy/README.md
@@ -0,0 +1,91 @@
+# alloy
+
+Deploys [Grafana Alloy](https://grafana.com/docs/alloy/latest/) in a systemd-managed Docker container.
+
+This role replaces the `promtail` role. Alloy is configured to forward logs to Loki and exposes its own metrics endpoint for scraping by Prometheus.
+
+## Configuration layout
+
+The Alloy configuration is assembled from a base template plus a list of opt-in snippets:
+
+- `templates/config.alloy.j2` — base config, always rendered. Defines the shared `loki.write "default"` endpoint.
+- `templates/snippets/<name>.alloy.j2` — opt-in snippets, enabled via `alloy_config_snippets`.
+
+At deploy time each enabled snippet is rendered into `{{ alloy_config_host_dir }}/conf.d/` and concatenated into a single `config.alloy` file.
+
+Available snippets:
+
+| Snippet            | Description                                                                                |
+| ------------------ | ------------------------------------------------------------------------------------------ |
+| `alloy-self`       | Meta-monitoring: forwards Alloy's own logs to Loki with `job=alloy` and `node_name` labels |
+| `leaf-node-docker` | Scrapes Docker container logs on the host and forwards them to Loki                        |
+| `journal-logs`     | Scrapes the systemd journal and forwards to Loki; relabels `unit` from `__journal__systemd_unit` |
+| `syslog-logs`      | Tails `/var/log/syslog` and forwards to Loki with `job=syslog`, `partition`, and `host` labels |
+
+## Adding custom snippets
+
+You can extend the Alloy config with your own snippets without modifying this role.
+
+1. Create a Jinja2 template that renders valid [Alloy River syntax](https://grafana.com/docs/alloy/latest/reference/config-blocks/) in your own role or playbook files directory, e.g. `templates/snippets/my-custom.alloy.j2`.
+2. Add it to your tasks before this role runs, or use [`ansible.builtin.template`](https://docs.ansible.com/ansible/latest/collections/ansible/builtin/template_module.html) to render it directly into `{{ alloy_config_host_dir }}/conf.d/` on the target host with a filename starting with `60-` or higher so it sorts after the built-in snippets.
+3. Alternatively, add the template to this role's `templates/snippets/` and reference it in `alloy_config_snippets`:
+
+   ```yaml
+   alloy_config_snippets:
+     - alloy-self
+     - leaf-node-docker
+     - my-custom
+   ```
+
+   A snippet template can use any Ansible variables available on the host. It must not define `loki.write "default"` (already in the base config) but can freely reference `loki.write.default.receiver` as a forwarding target.
+
+## Variables
+
+| Name                  | Mandatory | Description                                                                  |
+| --------------------- | --------- | ---------------------------------------------------------------------------- |
+| alloy_config_host_dir   |           | The location of the alloy config on the host (default: `/etc/alloy`)         |
+| alloy_image_name        | yes       | Image name of alloy                                                          |
+| alloy_image_tag         | yes       | Image tag of alloy                                                           |
+| alloy_loki_write_endpoints | yes       | List of Loki push endpoints. Each entry: `{url, basic_auth?: {username, password}}` |
+| alloy_docker_log_driver |           | Docker log driver for the alloy container (default: `json-file`)             |
+| alloy_config_snippets   |           | List of snippet names to enable (default: `[alloy-self]`)  |
+| alloy_port              |           | Port for Alloy metrics and HTTP API (default: `12345`)                        |
+| alloy_config_raw        |           | Full Alloy River config as a string. When set, bypasses snippet assembly entirely — `alloy_loki_write_endpoints` and `alloy_config_snippets` are ignored. |
+
+## Meta-monitoring
+
+Alloy exposes its own Prometheus metrics on `0.0.0.0:12345/metrics` (configured via the systemd container command). Add the host targets to `prometheus_alloy_targets` in the prometheus role to scrape them.
+
+Alloy's own logs are forwarded to Loki when the `alloy-self` snippet is enabled.
+
+## Migration from `promtail`
+
+This role supersedes the `promtail` role. To migrate:
+
+1. Replace the `promtail` role with `alloy` in your playbook (e.g. `deploy_partition.yaml`).
+2. Remove the old `promtail_clients` / `promtail_scrape_configs` variables from your inventory. Set `alloy_loki_write_endpoints` to the Loki push endpoint(s) instead, e.g.:
+
+   ```yaml
+   alloy_loki_write_endpoints:
+     - url: "http://loki.{{ metal_control_plane_ingress_dns }}:8080/loki/api/v1/push"
+   ```
+3. If you have a custom promtail scrape config, convert it once with the [`alloy convert`](https://grafana.com/docs/alloy/latest/reference/cli/convert/) CLI (`--source-format=promtail`) and add the result as a new snippet under `templates/snippets/`, then enable it via `alloy_config_snippets`. Be aware of the [converter limitations](https://grafana.com/docs/alloy/latest/set-up/migrate/from-promtail/#limitations) — not all promtail features are supported, and the output should always be reviewed manually before use.
+
+   If the snippet system does not provide enough flexibility, you can bypass it entirely by setting `alloy_config_raw` to a full Alloy River config string. Ansible variables (e.g. `{{ inventory_hostname }}`) can be used inside the string and will be resolved at deploy time. See the `alloy_config_raw` entry in the Variables table above.
+4. Update Prometheus inventory: rename `prometheus_promtail_targets` → `prometheus_alloy_targets` (port `12345` instead of `9080`).
+5. If your environment deploys Alloy on different host groups with different scrape needs (e.g. leaf switches vs. management servers), set `alloy_config_snippets` in per-group inventory files rather than at the partition level:
+
+   ```
+   group_vars/leaves/alloy.yaml       ← alloy_config_snippets: [leaf-node-docker, alloy-self, syslog-logs]
+   group_vars/mgmtservers/alloy.yaml  ← alloy_config_snippets: [alloy-self, journal-logs]
+   group_vars/partition/alloy.yaml    ← alloy_loki_write_endpoints: [...] (shared by all)
+   ```
+
+6. Once the Alloy deployment is verified (logs arriving in Loki with correct labels), remove the old promtail container and config from the hosts:
+
+   ```bash
+   docker rm -f promtail
+   rm -rf /etc/promtail
+   ```
+
+   Keep the `promtail` role available in your playbook until all environments have been migrated.
diff --git a/partition/roles/alloy/defaults/main.yaml b/partition/roles/alloy/defaults/main.yaml
new file mode 100644
index 000000000..3f860c6cc
--- /dev/null
+++ b/partition/roles/alloy/defaults/main.yaml
@@ -0,0 +1,21 @@
+---
+alloy_config_host_dir: "/etc/alloy"
+alloy_docker_log_driver: json-file
+
+# List of Loki push endpoints used by the base loki.write component.
+# Each entry: {url: <push-url>, basic_auth?: {username, password}}
+# alloy_loki_write_endpoints:
+#   - url: http://loki.example.com:8080/loki/api/v1/push
+
+# List of snippet names to enable. Each entry must correspond to
+# templates/snippets/<name>.alloy.j2 in this role.
+alloy_config_snippets:
+  - alloy-self
+
+# Provide a fully custom Alloy River config as a string to bypass the
+# snippet assembly system entirely. When set, alloy_loki_write_endpoints
+# and alloy_config_snippets are ignored.
+# alloy_config_raw: |
+
+# Port to listen for metrics and HTTP API traffic on
+alloy_port: 12345
diff --git a/partition/roles/alloy/handlers/main.yaml b/partition/roles/alloy/handlers/main.yaml
new file mode 100644
index 000000000..56a671558
--- /dev/null
+++ b/partition/roles/alloy/handlers/main.yaml
@@ -0,0 +1,6 @@
+---
+- name: restart alloy
+  service:
+    name: alloy
+    enabled: true
+    state: restarted
diff --git a/partition/roles/alloy/meta/main.yml b/partition/roles/alloy/meta/main.yml
new file mode 100644
index 000000000..9dfb6e3b0
--- /dev/null
+++ b/partition/roles/alloy/meta/main.yml
@@ -0,0 +1,17 @@
+---
+galaxy_info:
+  role_name: alloy
+  author: metal-stack
+  description: Deploys alloy.
+  license: MIT
+  min_ansible_version: "2.10"
+  galaxy_tags: []
+
+  platforms:
+    - name: Debian
+      versions:
+        - all
+
+dependencies:
+  - role: metal-roles/common/roles/defaults
+  - role: metal-roles/partition/roles/defaults
diff --git a/partition/roles/alloy/tasks/main.yaml b/partition/roles/alloy/tasks/main.yaml
new file mode 100644
index 000000000..55b8d2ab9
--- /dev/null
+++ b/partition/roles/alloy/tasks/main.yaml
@@ -0,0 +1,78 @@
+---
+- name: Gather release versions
+  setup_yaml:
+
+- name: Check mandatory variables for this role are set
+  assert:
+    fail_msg: "not all mandatory variables given, check role documentation"
+    quiet: yes
+    that:
+      - alloy_image_tag is defined
+      - alloy_image_name is defined
+      - alloy_config_raw is defined or (alloy_loki_write_endpoints is defined and (alloy_loki_write_endpoints | length > 0))
+
+- name: Create alloy config directory
+  file:
+    path: "{{ alloy_config_host_dir }}"
+    state: directory
+    owner: "nobody"
+    group: "nogroup"
+    mode: "0755"
+
+- name: Create alloy config snippets directory
+  file:
+    path: "{{ alloy_config_host_dir }}/conf.d"
+    state: directory
+    owner: "nobody"
+    group: "nogroup"
+    mode: "0755"
+
+- name: Template alloy base configuration
+  template:
+    src: config.alloy.j2
+    dest: "{{ alloy_config_host_dir }}/conf.d/00-base.alloy"
+  notify: restart alloy
+  when: alloy_config_raw is not defined
+
+- name: Template alloy config snippets
+  template:
+    src: "snippets/{{ item }}.alloy.j2"
+    dest: "{{ alloy_config_host_dir }}/conf.d/50-{{ item }}.alloy"
+  loop: "{{ alloy_config_snippets }}"
+  notify: restart alloy
+  when: alloy_config_raw is not defined
+
+- name: Assemble alloy config
+  assemble:
+    src: "{{ alloy_config_host_dir }}/conf.d"
+    dest: "{{ alloy_config_host_dir }}/config.alloy"
+    delimiter: "\n"
+  notify: restart alloy
+  when: alloy_config_raw is not defined
+
+- name: Write raw alloy config
+  copy:
+    content: "{{ alloy_config_raw }}"
+    dest: "{{ alloy_config_host_dir }}/config.alloy"
+  notify: restart alloy
+  when: alloy_config_raw is defined
+
+- name: Deploy alloy service
+  include_role:
+    name: ansible-common/roles/systemd-docker-service
+  vars:
+    systemd_service_name: alloy
+    systemd_docker_image_name: "{{ alloy_image_name }}"
+    systemd_docker_image_tag: "{{ alloy_image_tag }}"
+    systemd_service_after: docker.service
+    systemd_service_requires: docker.service
+    systemd_docker_network: host
+    systemd_docker_volumes:
+      - "{{ alloy_config_host_dir }}:/etc/alloy:ro"
+      - "/var/log:/var/log"
+      - "/var/run/docker.sock:/var/run/docker.sock"
+    systemd_docker_command:
+      - run
+      - --server.http.listen-addr=0.0.0.0:{{ alloy_port }}
+      - /etc/alloy/config.alloy
+    systemd_docker_log_driver: "{{ alloy_docker_log_driver }}"
diff --git a/partition/roles/alloy/templates/config.alloy.j2 b/partition/roles/alloy/templates/config.alloy.j2
new file mode 100644
index 000000000..fea1ad991
--- /dev/null
+++ b/partition/roles/alloy/templates/config.alloy.j2
@@ -0,0 +1,15 @@
+// Loki write endpoint shared by all snippets
+loki.write "default" {
+{% for endpoint in alloy_loki_write_endpoints %}
+	endpoint {
+		url = "{{ endpoint.url }}"
+{% if endpoint.basic_auth is defined %}
+		basic_auth {
+			username = "{{ endpoint.basic_auth.username }}"
+			password = "{{ endpoint.basic_auth.password }}"
+		}
+{% endif %}
+	}
+{% endfor %}
+	external_labels = {}
+}
diff --git a/partition/roles/alloy/templates/snippets/alloy-self.alloy.j2 b/partition/roles/alloy/templates/snippets/alloy-self.alloy.j2
new file mode 100644
index 000000000..79d044c91
--- /dev/null
+++ b/partition/roles/alloy/templates/snippets/alloy-self.alloy.j2
@@ -0,0 +1,20 @@
+// Meta-monitoring: forward Alloy's own logs to Loki
+logging {
+	level    = "info"
+	format   = "logfmt"
+	write_to = [loki.relabel.alloy_self.receiver]
+}
+
+loki.relabel "alloy_self" {
+	forward_to = [loki.write.default.receiver]
+
+	rule {
+		target_label = "node_name"
+		replacement  = "{{ inventory_hostname }}"
+	}
+
+	rule {
+		target_label = "job"
+		replacement  = "alloy"
+	}
+}
diff --git a/partition/roles/alloy/templates/snippets/journal-logs.alloy.j2 b/partition/roles/alloy/templates/snippets/journal-logs.alloy.j2
new file mode 100644
index 000000000..2e9e03f04
--- /dev/null
+++ b/partition/roles/alloy/templates/snippets/journal-logs.alloy.j2
@@ -0,0 +1,25 @@
+loki.process "journal" {
+	forward_to = [loki.write.default.receiver]
+
+	stage.docker {}
+}
+
+discovery.relabel "journal" {
+	targets = []
+
+	rule {
+		source_labels = ["__journal__systemd_unit"]
+		target_label  = "unit"
+	}
+}
+
+loki.source.journal "journal" {
+	path          = "/var/log/journal"
+	relabel_rules = discovery.relabel.journal.rules
+	forward_to    = [loki.process.journal.receiver]
+	labels        = {
+		job       = "systemd-journal",
+		partition = "{{ metal_partition_id }}",
+		host      = "{{ inventory_hostname }}",
+	}
+}
diff --git a/partition/roles/alloy/templates/snippets/leaf-node-docker.alloy.j2 b/partition/roles/alloy/templates/snippets/leaf-node-docker.alloy.j2
new file mode 100644
index 000000000..40127a47b
--- /dev/null
+++ b/partition/roles/alloy/templates/snippets/leaf-node-docker.alloy.j2
@@ -0,0 +1,27 @@
+discovery.docker "leaf_node_docker" {
+	host             = "unix:///var/run/docker.sock"
+	refresh_interval = "5s"
+}
+
+discovery.relabel "leaf_node_docker" {
+	targets = []
+
+	rule {
+		source_labels = ["__meta_docker_container_name"]
+		regex         = "/(.*)"
+		target_label  = "container"
+	}
+
+	rule {
+		target_label = "node_name"
+		replacement  = "{{ inventory_hostname }}"
+	}
+}
+
+loki.source.docker "leaf_node_docker" {
+	host             = "unix:///var/run/docker.sock"
+	targets          = discovery.docker.leaf_node_docker.targets
+	forward_to       = [loki.write.default.receiver]
+	relabel_rules    = discovery.relabel.leaf_node_docker.rules
+	refresh_interval = "5s"
+}
diff --git a/partition/roles/alloy/templates/snippets/syslog-logs.alloy.j2 b/partition/roles/alloy/templates/snippets/syslog-logs.alloy.j2
new file mode 100644
index 000000000..8547da668
--- /dev/null
+++ b/partition/roles/alloy/templates/snippets/syslog-logs.alloy.j2
@@ -0,0 +1,19 @@
+loki.process "syslog" {
+	forward_to = [loki.write.default.receiver]
+
+	stage.docker {}
+}
+
+local.file_match "syslog" {
+	path_targets = [{
+		__path__  = "/var/log/syslog",
+		job       = "syslog",
+		partition = "{{ metal_partition_id }}",
+		host      = "{{ inventory_hostname }}",
+	}]
+}
+
+loki.source.file "syslog" {
+	targets    = local.file_match.syslog.targets
+	forward_to = [loki.process.syslog.receiver]
+}
diff --git a/partition/roles/monitoring/prometheus/README.md b/partition/roles/monitoring/prometheus/README.md
index fe47ab040..b46e07236 100644
--- a/partition/roles/monitoring/prometheus/README.md
+++ b/partition/roles/monitoring/prometheus/README.md
@@ -23,6 +23,7 @@ you define them adequately as well.
 | prometheus_metal_core_targets                    |           | metal-core targets to scrape from                                                                                                           |
 | prometheus_node_exporter_targets                 |           | Node exporter targets to scrape from                                                                                                        |
 | prometheus_promtail_targets                      |           | Promtail targets to scrape from                                                                                                             |
+| prometheus_alloy_targets                         |           | Alloy targets to scrape from (default port `12345`)                                                                                         |
 | prometheus_ping_targets                          |           | Ping targets to scrape from                                                                                                                 |
 | prometheus_sonic_exporter_targets                |           | Sonic exporter targets to scrape from                                                                                                       |
 | prometheus_blackbox_exporter_targets             |           | Blackbox exporter targets to scrape from                                                                                                    |
diff --git a/partition/roles/monitoring/prometheus/defaults/main.yaml b/partition/roles/monitoring/prometheus/defaults/main.yaml
index 2651f9421..ed3c3e864 100644
--- a/partition/roles/monitoring/prometheus/defaults/main.yaml
+++ b/partition/roles/monitoring/prometheus/defaults/main.yaml
@@ -22,6 +22,7 @@ prometheus_frr_exporter_targets: []
 prometheus_metal_core_targets: []
 prometheus_node_exporter_targets: []
 prometheus_promtail_targets: []
+prometheus_alloy_targets: []
 prometheus_ping_targets: []
 prometheus_ipmi_exporter_targets: []
 prometheus_sonic_exporter_targets: []
diff --git a/partition/roles/monitoring/prometheus/tasks/main.yaml b/partition/roles/monitoring/prometheus/tasks/main.yaml
index 4382bdde7..6431d7fe7 100644
--- a/partition/roles/monitoring/prometheus/tasks/main.yaml
+++ b/partition/roles/monitoring/prometheus/tasks/main.yaml
@@ -129,6 +129,19 @@
     dest: "{{ prometheus_config_host_dir }}/file_sd/promtail.yaml"
   notify: restart prometheus
 
+- name: generate prometheus_alloy_sd
+  set_fact:
+    prometheus_alloy_sd:
+      - labels:
+          job: alloy
+        targets: "{{ prometheus_alloy_targets }}"
+
+- name: create alloy.yaml
+  copy:
+    content: "{{ prometheus_alloy_sd | to_nice_yaml(indent=2) }}"
+    dest: "{{ prometheus_config_host_dir }}/file_sd/alloy.yaml"
+  notify: restart prometheus
+
 - name: generate prometheus_sonic_exporter_sd
   set_fact:
     prometheus_sonic_exporter_sd:
diff --git a/partition/roles/monitoring/prometheus/templates/prometheus.yaml.j2 b/partition/roles/monitoring/prometheus/templates/prometheus.yaml.j2
index b7e6bf4af..d0f0b1517 100644
--- a/partition/roles/monitoring/prometheus/templates/prometheus.yaml.j2
+++ b/partition/roles/monitoring/prometheus/templates/prometheus.yaml.j2
@@ -88,6 +88,15 @@ scrape_configs:
       regex: (.*):\d+
       target_label: instance
 
+- job_name: alloy
+  file_sd_configs:
+  - files:
+    - '/etc/prometheus/file_sd/alloy.yaml'
+  relabel_configs:
+    - source_labels: [__address__]
+      regex: (.*):\d+
+      target_label: instance
+
 - job_name: sonic-exporter
   file_sd_configs:
   - files:
diff --git a/partition/roles/promtail/README.md b/partition/roles/promtail/README.md
index fcb2f4da8..cbccf7b09 100644
--- a/partition/roles/promtail/README.md
+++ b/partition/roles/promtail/README.md
@@ -1,5 +1,7 @@
 # promtail
 
+> **Deprecated:** This role is superseded by the `alloy` role. New deployments should use `metal-roles/partition/roles/alloy` instead. This role is kept for migration purposes only and may be removed in a future release. See the [alloy role README](../alloy/README.md) for migration instructions.
+
 Deploys promtail in a systemd-managed Docker container.
 
 ## Variables

From 7354a0652d04a8c97e2e5c158f49159d371f066c Mon Sep 17 00:00:00 2001
From: Matthias Hartmann <matthias.hartmann@x-cellent.com>
Date: Tue, 5 May 2026 12:33:43 +0200
Subject: [PATCH 02/41] feat: use legacy_positions_file to avoid replaying
 syslog

---
 partition/roles/alloy/README.md               | 32 +++++++++++++++++++
 partition/roles/alloy/tasks/main.yaml         | 10 ++++++
 .../templates/snippets/syslog-logs.alloy.j2   |  5 +--
 3 files changed, 45 insertions(+), 2 deletions(-)

diff --git a/partition/roles/alloy/README.md b/partition/roles/alloy/README.md
index 657c5b417..7b6f3c758 100644
--- a/partition/roles/alloy/README.md
+++ b/partition/roles/alloy/README.md
@@ -22,6 +22,15 @@ Available snippets:
 | `journal-logs`     | Scrapes the systemd journal and forwards to Loki; relabels `unit` from `__journal__systemd_unit` |
 | `syslog-logs`      | Tails `/var/log/syslog` and forwards to Loki with `job=syslog`, `partition`, and `host` labels |
 
+### `journal-logs` vs `syslog-logs`
+
+On a modern systemd host, journald is a superset of syslog. `/var/log/syslog` is written by rsyslog consuming from journald — it is a filtered, text-rendered subset of the journal. Enabling both snippets on the same host will ship most messages twice and produce duplicates in Loki.
+
+**Pick one based on what the host runs:**
+
+- Use `journal-logs` on hosts with systemd/journald (standard Debian/Ubuntu partition nodes and management servers). It covers everything syslog would, plus systemd unit stdout/stderr.
+- Use `syslog-logs` on hosts without journald (some minimal switch OS images), or where rsyslog applies custom filtering that makes `/var/log/syslog` meaningfully different from the raw journal.
+
 ## Adding custom snippets
 
 You can extend the Alloy config with your own snippets without modifying this role.
@@ -71,7 +80,20 @@ This role supersedes the `promtail` role. To migrate:
    ```
 3. If you have a custom promtail scrape config, convert it once with the [`alloy convert`](https://grafana.com/docs/alloy/latest/reference/cli/convert/) CLI (`--source-format=promtail`) and add the result as a new snippet under `templates/snippets/`, then enable it via `alloy_config_snippets`. Be aware of the [converter limitations](https://grafana.com/docs/alloy/latest/set-up/migrate/from-promtail/#limitations) — not all promtail features are supported, and the output should always be reviewed manually before use.
 
+   You can run the converter without installing Alloy locally using Docker:
+
+   ```bash
+   docker run \
+     -v ./promtail.yaml:/etc/promtail/promtail.yaml \
+     -v ./config.alloy:/etc/alloy/config.alloy \
+     grafana/alloy:latest \
+       convert --source-format=promtail --output=/etc/alloy/config.alloy /etc/promtail/promtail.yaml
+   ```
+
+   Note: the input must be a fully rendered promtail config (no Ansible variables). Substitute real values before running the converter. After conversion, review the output and replace any hardcoded host-specific values (hostnames, partition IDs, URLs, credentials) with Ansible variables (e.g. `{{ inventory_hostname }}`, `{{ metal_partition_id }}`) so the snippet works correctly across all target hosts.
+
    If the snippet system does not provide enough flexibility, you can bypass it entirely by setting `alloy_config_raw` to a full Alloy River config string. Ansible variables (e.g. `{{ inventory_hostname }}`) can be used inside the string and will be resolved at deploy time. See the `alloy_config_raw` entry in the Variables table above.
+
 4. Update Prometheus inventory: rename `prometheus_promtail_targets` → `prometheus_alloy_targets` (port `12345` instead of `9080`).
 5. If your environment deploys Alloy on different host groups with different scrape needs (e.g. leaf switches vs. management servers), set `alloy_config_snippets` in per-group inventory files rather than at the partition level:
 
@@ -89,3 +111,13 @@ This role supersedes the `promtail` role. To migrate:
    ```
 
    Keep the `promtail` role available in your playbook until all environments have been migrated.
+
+### WAL and positions file persistence
+
+Alloy stores its write-ahead log (WAL) — including cursor state for `loki.source.file` and `loki.source.journal` — under `--storage.path`, which is set to `/var/lib/alloy` (bind-mounted from the host). This survives container restarts, preventing duplicate log shipments.
+
+The `syslog-logs` snippet sets `legacy_positions_file = "/var/log/promtail-positions.yaml"`. On first start after migration, Alloy reads the existing promtail positions file and continues tailing `/var/log/syslog` from where promtail left off. No log lines are re-shipped.
+
+`loki.source.journal` has no equivalent migration path — its cursor is WAL-only. The very first Alloy start after migration will re-read a portion of the journal (however much the kernel still has buffered, typically a few thousand lines). This is a one-time event and subsequent restarts are safe once the WAL is populated.
+
+Once the promtail role has been removed from all hosts, the `legacy_positions_file` line can be dropped from the snippet and the old `/var/log/promtail-positions.yaml` file cleaned up from hosts.
diff --git a/partition/roles/alloy/tasks/main.yaml b/partition/roles/alloy/tasks/main.yaml
index 55b8d2ab9..76a46dd92 100644
--- a/partition/roles/alloy/tasks/main.yaml
+++ b/partition/roles/alloy/tasks/main.yaml
@@ -57,6 +57,14 @@
   notify: restart alloy
   when: alloy_config_raw is defined
 
+- name: Create alloy storage directory
+  file:
+    path: /var/lib/alloy
+    state: directory
+    owner: "nobody"
+    group: "nogroup"
+    mode: "0755"
+
 - name: Deploy alloy service
   include_role:
     name: ansible-common/roles/systemd-docker-service
@@ -69,10 +77,12 @@
     systemd_docker_network: host
     systemd_docker_volumes:
       - "{{ alloy_config_host_dir }}:/etc/alloy:ro"
+      - "/var/lib/alloy:/var/lib/alloy"
       - "/var/log:/var/log"
       - "/var/run/docker.sock:/var/run/docker.sock"
     systemd_docker_command:
       - run
       - --server.http.listen-addr=0.0.0.0:{{ alloy_port }}
+      - --storage.path=/var/lib/alloy
       - /etc/alloy/config.alloy
     systemd_docker_log_driver: "{{ alloy_docker_log_driver }}"
diff --git a/partition/roles/alloy/templates/snippets/syslog-logs.alloy.j2 b/partition/roles/alloy/templates/snippets/syslog-logs.alloy.j2
index 8547da668..c65037907 100644
--- a/partition/roles/alloy/templates/snippets/syslog-logs.alloy.j2
+++ b/partition/roles/alloy/templates/snippets/syslog-logs.alloy.j2
@@ -14,6 +14,7 @@ local.file_match "syslog" {
 }
 
 loki.source.file "syslog" {
-	targets    = local.file_match.syslog.targets
-	forward_to = [loki.process.syslog.receiver]
+	targets               = local.file_match.syslog.targets
+	forward_to            = [loki.process.syslog.receiver]
+	legacy_positions_file = "/var/log/promtail-positions.yaml"
 }

From 17cb50e3010a25302c90db0bf9604cf7ceaf2667 Mon Sep 17 00:00:00 2001
From: Matthias Hartmann <matthias.hartmann@x-cellent.com>
Date: Tue, 5 May 2026 17:05:08 +0200
Subject: [PATCH 03/41] feat: loki write endpoints allow remote_timeout

---
 partition/roles/alloy/README.md                 | 9 +++++++--
 partition/roles/alloy/templates/config.alloy.j2 | 4 +++-
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/partition/roles/alloy/README.md b/partition/roles/alloy/README.md
index 7b6f3c758..20a134c02 100644
--- a/partition/roles/alloy/README.md
+++ b/partition/roles/alloy/README.md
@@ -55,7 +55,7 @@ You can extend the Alloy config with your own snippets without modifying this ro
 | alloy_config_host_dir   |           | The location of the alloy config on the host (default: `/etc/alloy`)         |
 | alloy_image_name        | yes       | Image name of alloy                                                          |
 | alloy_image_tag         | yes       | Image tag of alloy                                                           |
-| alloy_loki_write_endpoints | yes       | List of Loki push endpoints. Each entry: `{url, basic_auth?: {username, password}}` |
+| alloy_loki_write_endpoints | yes       | List of Loki push endpoints. Each entry: `{url, remote_timeout?: <duration>, basic_auth?: {username, password}}` |
 | alloy_docker_log_driver |           | Docker log driver for the alloy container (default: `json-file`)             |
 | alloy_config_snippets   |           | List of snippet names to enable (default: `[alloy-self]`)  |
 | alloy_port              |           | Port for Alloy metrics and HTTP API (default: `12345`)                        |
@@ -76,7 +76,12 @@ This role supersedes the `promtail` role. To migrate:
 
    ```yaml
    alloy_loki_write_endpoints:
-     - url: "http://loki.{{ metal_control_plane_ingress_dns }}:8080/loki/api/v1/push"
+   - url: "http://loki.{{ metal_control_plane_ingress_dns }}:8080/loki/api/v1/push"
+      remote_timeout: 10s
+      basic_auth:
+         username: "promtail"
+         password: "test"
+
    ```
 3. If you have a custom promtail scrape config, convert it once with the [`alloy convert`](https://grafana.com/docs/alloy/latest/reference/cli/convert/) CLI (`--source-format=promtail`) and add the result as a new snippet under `templates/snippets/`, then enable it via `alloy_config_snippets`. Be aware of the [converter limitations](https://grafana.com/docs/alloy/latest/set-up/migrate/from-promtail/#limitations) — not all promtail features are supported, and the output should always be reviewed manually before use.
 
diff --git a/partition/roles/alloy/templates/config.alloy.j2 b/partition/roles/alloy/templates/config.alloy.j2
index fea1ad991..7bb190c86 100644
--- a/partition/roles/alloy/templates/config.alloy.j2
+++ b/partition/roles/alloy/templates/config.alloy.j2
@@ -3,6 +3,9 @@ loki.write "default" {
 {% for endpoint in alloy_loki_write_endpoints %}
 	endpoint {
 		url = "{{ endpoint.url }}"
+{%if endpoint.remote_timeout is defined %}
+		remote_timeout = "{{ endpoint.remote_timeout }}"
+{% endif %}
 {% if endpoint.basic_auth is defined %}
 		basic_auth {
 			username = "{{ endpoint.basic_auth.username }}"
@@ -11,5 +14,4 @@ loki.write "default" {
 {% endif %}
 	}
 {% endfor %}
-	external_labels = {}
 }

From 04731ebdcdae8e6af7477893d49463b978155066 Mon Sep 17 00:00:00 2001
From: Matthias Hartmann <matthias.hartmann@x-cellent.com>
Date: Wed, 6 May 2026 15:28:35 +0200
Subject: [PATCH 04/41] docs(alloy): README with promtail migration guide

---
 partition/roles/alloy/README.md               | 160 ++++++++++--------
 partition/roles/alloy/defaults/main.yaml      |   8 +-
 .../roles/alloy/templates/config.alloy.j2     |   9 +
 ...lloy-self.alloy.j2 => alloy-meta.alloy.j2} |   6 -
 .../templates/snippets/journal-logs.alloy.j2  |   8 +-
 .../templates/snippets/syslog-logs.alloy.j2   |  12 +-
 6 files changed, 110 insertions(+), 93 deletions(-)
 rename partition/roles/alloy/templates/snippets/{alloy-self.alloy.j2 => alloy-meta.alloy.j2} (73%)

diff --git a/partition/roles/alloy/README.md b/partition/roles/alloy/README.md
index 20a134c02..62b9deb45 100644
--- a/partition/roles/alloy/README.md
+++ b/partition/roles/alloy/README.md
@@ -2,9 +2,9 @@
 
 Deploys [Grafana Alloy](https://grafana.com/docs/alloy/latest/) in a systemd-managed Docker container.
 
-This role replaces the `promtail` role. Alloy is configured to forward logs to Loki and exposes its own metrics endpoint for scraping by Prometheus.
+This role replaces the deprecated `promtail` role. Alloy is configured to forward logs to Loki and exposes its own metrics endpoint for scraping by Prometheus. See [Migration from `promtail`](#migration-from-promtail) for step-by-step migration instructions.
 
-## Configuration layout
+## Configuration
 
 The Alloy configuration is assembled from a base template plus a list of opt-in snippets:
 
@@ -13,79 +13,114 @@ The Alloy configuration is assembled from a base template plus a list of opt-in
 
 At deploy time each enabled snippet is rendered into `{{ alloy_config_host_dir }}/conf.d/` and concatenated into a single `config.alloy` file.
 
-Available snippets:
+The snippets can be configured with environment variables defined in the [Variables](#variables) section below. This allows you to enable only the features you need without having to maintain a full custom config.
 
-| Snippet            | Description                                                                                |
-| ------------------ | ------------------------------------------------------------------------------------------ |
-| `alloy-self`       | Meta-monitoring: forwards Alloy's own logs to Loki with `job=alloy` and `node_name` labels |
-| `leaf-node-docker` | Scrapes Docker container logs on the host and forwards them to Loki                        |
-| `journal-logs`     | Scrapes the systemd journal and forwards to Loki; relabels `unit` from `__journal__systemd_unit` |
-| `syslog-logs`      | Tails `/var/log/syslog` and forwards to Loki with `job=syslog`, `partition`, and `host` labels |
+### Available snippets
 
-### `journal-logs` vs `syslog-logs`
+The snippets are used in our default configs to provide a sensible out-of-the-box setup and can be configured with the variables described in the next section to fit most use cases without modification.
 
-On a modern systemd host, journald is a superset of syslog. `/var/log/syslog` is written by rsyslog consuming from journald — it is a filtered, text-rendered subset of the journal. Enabling both snippets on the same host will ship most messages twice and produce duplicates in Loki.
+If your needs are more custom (e.g. you have a non-standard log source, or want to use advanced Alloy features not covered by the existing snippets), you can follow the [Customizing the config](#customizing-the-config) section below to either add your own snippet or bypass the snippet system entirely with a custom raw config.
 
-**Pick one based on what the host runs:**
+| Snippet            | Description                                                                                                                                                                                                                                                                                                                           |
+| ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `alloy-meta`       | Meta-monitoring: forwards Alloy's own logs to Loki with `job=alloy` and `node_name` labels. Redundant when `journal-logs` is enabled — journald already captures Alloy's stdout/stderr. Log level info and format are always configured via the base config regardless of this snippet.                                               |
+| `leaf-node-docker` | Scrapes Docker container logs on the host via the Docker socket and forwards them to Loki. Assumes the `json-file` log driver (the default). If containers use the `journald` log driver instead, their logs are already captured by `journal-logs` — enabling both snippets in that case will produce duplicate log entries in Loki. |
+| `journal-logs`     | Scrapes the systemd journal and forwards to Loki; relabels `unit` from `__journal__systemd_unit`. On standard systemd hosts (Debian/Ubuntu), journald captures syslog messages as well, so this snippet typically covers everything `syslog-logs` would. Enabling both may produce duplicate log entries in Loki.                     |
+| `syslog-logs`      | Tails `/var/log/syslog` and forwards to Loki with `job=syslog`, `partition`, and `host` labels. Suited for hosts without journald (e.g. minimal switch OS images). On standard systemd hosts, prefer `journal-logs` — enabling both may produce duplicate log entries in Loki.                                                        |
 
-- Use `journal-logs` on hosts with systemd/journald (standard Debian/Ubuntu partition nodes and management servers). It covers everything syslog would, plus systemd unit stdout/stderr.
-- Use `syslog-logs` on hosts without journald (some minimal switch OS images), or where rsyslog applies custom filtering that makes `/var/log/syslog` meaningfully different from the raw journal.
+### Variables
 
-## Adding custom snippets
+| Name                          | Mandatory | Description                                                                                                                                               |
+| ----------------------------- | --------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| alloy_config_host_dir         |           | The location of the alloy config on the host (default: `/etc/alloy`)                                                                                      |
+| alloy_image_name              | yes       | Image name of alloy                                                                                                                                       |
+| alloy_image_tag               | yes       | Image tag of alloy                                                                                                                                        |
+| alloy_loki_write_endpoints    | yes       | List of Loki push endpoints. Each entry: `{url, remote_timeout?: <duration>, basic_auth?: {username, password}}`                                          |
+| alloy_docker_log_driver       |           | Docker log driver for the alloy container (default: `json-file`)                                                                                          |
+| alloy_config_snippets         |           | List of snippet names to enable (default: `[]`)                                                                                                           |
+| alloy_port                    |           | Port for Alloy metrics and HTTP API (default: `12345`)                                                                                                    |
+| alloy_promtail_positions_file |           | Path to the legacy promtail positions file (default: `/var/log/promtail-positions.yaml`). Used by `syslog-logs` on first start after migration.           |
+| alloy_config_raw              |           | Full Alloy River config as a string. When set, bypasses snippet assembly entirely — `alloy_loki_write_endpoints` and `alloy_config_snippets` are ignored. |
 
-You can extend the Alloy config with your own snippets without modifying this role.
+### Meta-monitoring (for Alloy itself)
 
-1. Create a Jinja2 template that renders valid [Alloy River syntax](https://grafana.com/docs/alloy/latest/reference/config-blocks/) in your own role or playbook files directory, e.g. `templates/snippets/my-custom.alloy.j2`.
-2. Add it to your tasks before this role runs, or use [`ansible.builtin.template`](https://docs.ansible.com/ansible/latest/collections/ansible/builtin/template_module.html) to render it directly into `{{ alloy_config_host_dir }}/conf.d/` on the target host with a filename starting with `60-` or higher so it sorts after the built-in snippets.
-3. Alternatively, add the template to this role's `templates/snippets/` and reference it in `alloy_config_snippets`:
+#### Metrics
 
-   ```yaml
-   alloy_config_snippets:
-     - alloy-self
-     - leaf-node-docker
-     - my-custom
-   ```
+Alloy always exposes Prometheus metrics on `0.0.0.0:{{ alloy_port }}/metrics`, regardless of which snippets are enabled. Add the host targets to `prometheus_alloy_targets` in the prometheus role to scrape them.
+
+#### Logs
+
+Alloy's own logs are captured in two ways depending on your snippet configuration:
+
+- **If `journal-logs` is enabled** — Alloy runs as a systemd-managed Docker container, so its stdout/stderr are captured by journald automatically. No additional snippet is needed.
+- **If `journal-logs` is not enabled** (e.g. you only use `syslog-logs` or no log snippet at all) — enable the `alloy-meta` snippet to forward Alloy's own logs to Loki explicitly.
 
-   A snippet template can use any Ansible variables available on the host. It must not define `loki.write "default"` (already in the base config) but can freely reference `loki.write.default.receiver` as a forwarding target.
+Alloy's log level (`info`) and format (`logfmt`) are always configured via the base config, regardless of which snippets are enabled. The `alloy-meta` snippet only adds the Loki forwarding pipeline on top.
+
+### WAL and positions file persistence
+
+Alloy stores its write-ahead log (WAL) — including cursor state for `loki.source.file` and `loki.source.journal` — under `--storage.path`, which is set to `/var/lib/alloy` (bind-mounted from the host). This survives container restarts, preventing duplicate log shipments.
 
-## Variables
+The `syslog-logs` snippet reads the legacy promtail positions file on first start after migration to continue tailing `/var/log/syslog` from where promtail left off. The path is configured via `alloy_promtail_positions_file` (default: `/var/log/promtail-positions.yaml`). No log lines are re-shipped. This only happens once — after the initial migration, Alloy updates the positions in its own WAL format.
 
-| Name                  | Mandatory | Description                                                                  |
-| --------------------- | --------- | ---------------------------------------------------------------------------- |
-| alloy_config_host_dir   |           | The location of the alloy config on the host (default: `/etc/alloy`)         |
-| alloy_image_name        | yes       | Image name of alloy                                                          |
-| alloy_image_tag         | yes       | Image tag of alloy                                                           |
-| alloy_loki_write_endpoints | yes       | List of Loki push endpoints. Each entry: `{url, remote_timeout?: <duration>, basic_auth?: {username, password}}` |
-| alloy_docker_log_driver |           | Docker log driver for the alloy container (default: `json-file`)             |
-| alloy_config_snippets   |           | List of snippet names to enable (default: `[alloy-self]`)  |
-| alloy_port              |           | Port for Alloy metrics and HTTP API (default: `12345`)                        |
-| alloy_config_raw        |           | Full Alloy River config as a string. When set, bypasses snippet assembly entirely — `alloy_loki_write_endpoints` and `alloy_config_snippets` are ignored. |
+`loki.source.journal` has no equivalent migration path — its cursor is WAL-only. The very first Alloy start after migration will re-read a portion of the journal. This is a one-time event and subsequent restarts are safe once the WAL is populated.
 
-## Meta-monitoring
+Once the migration is complete, the `legacy_positions_file` line can optionally be dropped from the snippet and the old positions file cleaned up from hosts — Alloy will simply start tracking positions in its own WAL format from that point.
 
-Alloy exposes its own Prometheus metrics on `0.0.0.0:12345/metrics` (configured via the systemd container command). Add the host targets to `prometheus_alloy_targets` in the prometheus role to scrape them.
+## Customizing the config
 
-Alloy's own logs are forwarded to Loki when the `alloy-self` snippet is enabled.
+There is no way to inject a custom snippet purely from inventory. Your options are:
+
+**Option A — Use `alloy_config_raw`** (inventory only, no role change)
+
+Set `alloy_config_raw` to a full Alloy River config string in your inventory. The role will write it verbatim and skip snippet assembly entirely. `alloy_loki_write_endpoints` and `alloy_config_snippets` are ignored. You own the complete config, including the base `loki.write "default"` block.
+
+**Option B — Add the snippet to this role** (requires editing this repo)
+
+Add the template to `templates/snippets/` and reference it by name in `alloy_config_snippets`:
+
+```yaml
+alloy_config_snippets:
+  - alloy-meta
+  - leaf-node-docker
+```
+
+A snippet template can use any Ansible variables available on the host. It does not need to define `loki.write "default"` as it is already in the base config.
+
+Contributions of new snippets are welcome — since all snippets are opt-in via `alloy_config_snippets`, adding one to the role has no impact on existing deployments.
 
 ## Migration from `promtail`
 
-This role supersedes the `promtail` role. To migrate:
+Alloy allows reproducing the same log forwarding behavior as promtail via the [available snippets](#available-snippets), but it is a different tool with a different internal data model. Label names, pipeline stages, and how metrics are exposed might differ from promtail. **Review your existing Loki dashboards and alerting rules after the migration and adapt them where necessary.**
+
+**Recommended approach — parallel run:** Deploy Alloy alongside the existing promtail installation first. Both will ship logs to Loki simultaneously, so expect duplicate log entries during this transition window. Once you have verified that logs arrive in Loki with the correct labels and dashboards show data correctly, remove promtail.
 
-1. Replace the `promtail` role with `alloy` in your playbook (e.g. `deploy_partition.yaml`).
-2. Remove the old `promtail_clients` / `promtail_scrape_configs` variables from your inventory. Set `alloy_loki_write_endpoints` to the Loki push endpoint(s) instead, e.g.:
+1. **Add the `alloy` role** to your playbook alongside `promtail` for a parallel run, or replace it directly if you prefer a hard cut-over.
+
+2. **Configure the Loki endpoint.** Set `alloy_loki_write_endpoints`:
 
    ```yaml
    alloy_loki_write_endpoints:
-   - url: "http://loki.{{ metal_control_plane_ingress_dns }}:8080/loki/api/v1/push"
-      remote_timeout: 10s
-      basic_auth:
+     - url: "http://loki.{{ metal_control_plane_ingress_dns }}:8080/loki/api/v1/push"
+       remote_timeout: 10s
+       basic_auth:
          username: "promtail"
          password: "test"
+   ```
+
+3. **Choose snippets** that match your legacy `promtail_scrape_configs` — see [Available snippets](#available-snippets). If the available snippets are close enough to your old setup, enable the relevant ones and adjust with the provided variables. Otherwise, continue with step 4.
 
+   If your environment deploys Alloy on different host groups with different scrape needs (e.g. leaf switches vs. management servers), set `alloy_config_snippets` in per-group inventory files:
+
+   ```text
+   group_vars/leaves/alloy.yaml       ← alloy_config_snippets: [leaf-node-docker, journal-logs]
+   group_vars/mgmtservers/alloy.yaml  ← alloy_config_snippets: [journal-logs]
+   group_vars/partition/alloy.yaml    ← alloy_loki_write_endpoints: [...] (shared by all)
    ```
-3. If you have a custom promtail scrape config, convert it once with the [`alloy convert`](https://grafana.com/docs/alloy/latest/reference/cli/convert/) CLI (`--source-format=promtail`) and add the result as a new snippet under `templates/snippets/`, then enable it via `alloy_config_snippets`. Be aware of the [converter limitations](https://grafana.com/docs/alloy/latest/set-up/migrate/from-promtail/#limitations) — not all promtail features are supported, and the output should always be reviewed manually before use.
 
-   You can run the converter without installing Alloy locally using Docker:
+4. **Convert custom promtail scrape configs** _(optional — only needed if you extended the promtail setup beyond what the available snippets cover)_. Use the [`alloy convert`](https://grafana.com/docs/alloy/latest/reference/cli/convert/) CLI (`--source-format=promtail`) to get a starting point. Be aware of the [converter limitations](https://grafana.com/docs/alloy/latest/set-up/migrate/from-promtail/#limitations) — review the output manually before use.
+
+   You can run the converter without installing Alloy locally:
 
    ```bash
    docker run \
@@ -95,34 +130,19 @@ This role supersedes the `promtail` role. To migrate:
        convert --source-format=promtail --output=/etc/alloy/config.alloy /etc/promtail/promtail.yaml
    ```
 
-   Note: the input must be a fully rendered promtail config (no Ansible variables). Substitute real values before running the converter. After conversion, review the output and replace any hardcoded host-specific values (hostnames, partition IDs, URLs, credentials) with Ansible variables (e.g. `{{ inventory_hostname }}`, `{{ metal_partition_id }}`) so the snippet works correctly across all target hosts.
+   The input must be a fully rendered promtail config (no Ansible variables). Substitute real values before running the converter, then replace hardcoded host-specific values (hostnames, partition IDs, URLs, credentials) with Ansible variables (e.g. `{{ inventory_hostname }}`, `{{ metal_partition_id }}`) in the output.
+   - **If the config is generally useful** (not environment-specific), consider contributing it as a new snippet to this role — see [Option B in Customizing the config](#customizing-the-config).
+   - **If the config is specific to your environment**, use `alloy_config_raw` instead — see [Option A in Customizing the config](#customizing-the-config). Note that with `alloy_config_raw` you own the complete config, including the `loki.write "default"` block and its endpoint configuration; `alloy_loki_write_endpoints` is ignored.
 
-   If the snippet system does not provide enough flexibility, you can bypass it entirely by setting `alloy_config_raw` to a full Alloy River config string. Ansible variables (e.g. `{{ inventory_hostname }}`) can be used inside the string and will be resolved at deploy time. See the `alloy_config_raw` entry in the Variables table above.
+5. **Update Prometheus inventory:** rename `prometheus_promtail_targets` → `prometheus_alloy_targets` (port `12345` instead of `9080`).
 
-4. Update Prometheus inventory: rename `prometheus_promtail_targets` → `prometheus_alloy_targets` (port `12345` instead of `9080`).
-5. If your environment deploys Alloy on different host groups with different scrape needs (e.g. leaf switches vs. management servers), set `alloy_config_snippets` in per-group inventory files rather than at the partition level:
+6. **Verify** that logs arrive in Loki with the correct labels. Check that existing dashboards and alerts still work as expected and adapt them for any label or metric name changes.
 
-   ```
-   group_vars/leaves/alloy.yaml       ← alloy_config_snippets: [leaf-node-docker, alloy-self, syslog-logs]
-   group_vars/mgmtservers/alloy.yaml  ← alloy_config_snippets: [alloy-self, journal-logs]
-   group_vars/partition/alloy.yaml    ← alloy_loki_write_endpoints: [...] (shared by all)
-   ```
-
-6. Once the Alloy deployment is verified (logs arriving in Loki with correct labels), remove the old promtail container and config from the hosts:
+7. **Remove the promtail container** from the migrated hosts once verified:
 
    ```bash
    docker rm -f promtail
    rm -rf /etc/promtail
    ```
 
-   Keep the `promtail` role available in your playbook until all environments have been migrated.
-
-### WAL and positions file persistence
-
-Alloy stores its write-ahead log (WAL) — including cursor state for `loki.source.file` and `loki.source.journal` — under `--storage.path`, which is set to `/var/lib/alloy` (bind-mounted from the host). This survives container restarts, preventing duplicate log shipments.
-
-The `syslog-logs` snippet sets `legacy_positions_file = "/var/log/promtail-positions.yaml"`. On first start after migration, Alloy reads the existing promtail positions file and continues tailing `/var/log/syslog` from where promtail left off. No log lines are re-shipped.
-
-`loki.source.journal` has no equivalent migration path — its cursor is WAL-only. The very first Alloy start after migration will re-read a portion of the journal (however much the kernel still has buffered, typically a few thousand lines). This is a one-time event and subsequent restarts are safe once the WAL is populated.
-
-Once the promtail role has been removed from all hosts, the `legacy_positions_file` line can be dropped from the snippet and the old `/var/log/promtail-positions.yaml` file cleaned up from hosts.
+   The `promtail` role remains in this repo (deprecated) and does not need to be removed from your playbook — simply stop including it for migrated environments.
diff --git a/partition/roles/alloy/defaults/main.yaml b/partition/roles/alloy/defaults/main.yaml
index 3f860c6cc..4b91998ba 100644
--- a/partition/roles/alloy/defaults/main.yaml
+++ b/partition/roles/alloy/defaults/main.yaml
@@ -9,8 +9,7 @@ alloy_docker_log_driver: json-file
 
 # List of snippet names to enable. Each entry must correspond to
 # templates/snippets/<name>.alloy.j2 in this role.
-alloy_config_snippets:
-  - alloy-self
+alloy_config_snippets: []
 
 # Provide a fully custom Alloy River config as a string to bypass the
 # snippet assembly system entirely. When set, alloy_loki_write_endpoints
@@ -19,3 +18,8 @@ alloy_config_snippets:
 
 # Port to listen for metrics and HTTP API traffic on
 alloy_port: 12345
+
+# Path to the legacy promtail positions file. Read once on first start after migration
+# to continue tailing /var/log/syslog from where promtail left off.
+# Can be removed from the snippet once all hosts have been migrated.
+alloy_promtail_positions_file: "/var/log/promtail-positions.yaml"
diff --git a/partition/roles/alloy/templates/config.alloy.j2 b/partition/roles/alloy/templates/config.alloy.j2
index 7bb190c86..37d827635 100644
--- a/partition/roles/alloy/templates/config.alloy.j2
+++ b/partition/roles/alloy/templates/config.alloy.j2
@@ -1,3 +1,12 @@
+// Log level and format; write_to forwards Alloy's own logs to Loki when alloy-meta is enabled
+logging {
+	level  = "info"
+	format = "logfmt"
+{% if "alloy-meta" in alloy_config_snippets %}
+	write_to = [loki.relabel.alloy_self.receiver]
+{% endif %}
+}
+
 // Loki write endpoint shared by all snippets
 loki.write "default" {
 {% for endpoint in alloy_loki_write_endpoints %}
diff --git a/partition/roles/alloy/templates/snippets/alloy-self.alloy.j2 b/partition/roles/alloy/templates/snippets/alloy-meta.alloy.j2
similarity index 73%
rename from partition/roles/alloy/templates/snippets/alloy-self.alloy.j2
rename to partition/roles/alloy/templates/snippets/alloy-meta.alloy.j2
index 79d044c91..550b0fcfc 100644
--- a/partition/roles/alloy/templates/snippets/alloy-self.alloy.j2
+++ b/partition/roles/alloy/templates/snippets/alloy-meta.alloy.j2
@@ -1,10 +1,4 @@
 // Meta-monitoring: forward Alloy's own logs to Loki
-logging {
-	level    = "info"
-	format   = "logfmt"
-	write_to = [loki.relabel.alloy_self.receiver]
-}
-
 loki.relabel "alloy_self" {
 	forward_to = [loki.write.default.receiver]
 
diff --git a/partition/roles/alloy/templates/snippets/journal-logs.alloy.j2 b/partition/roles/alloy/templates/snippets/journal-logs.alloy.j2
index 2e9e03f04..7d1cc783b 100644
--- a/partition/roles/alloy/templates/snippets/journal-logs.alloy.j2
+++ b/partition/roles/alloy/templates/snippets/journal-logs.alloy.j2
@@ -1,9 +1,3 @@
-loki.process "journal" {
-	forward_to = [loki.write.default.receiver]
-
-	stage.docker {}
-}
-
 discovery.relabel "journal" {
 	targets = []
 
@@ -16,7 +10,7 @@ discovery.relabel "journal" {
 loki.source.journal "journal" {
 	path          = "/var/log/journal"
 	relabel_rules = discovery.relabel.journal.rules
-	forward_to    = [loki.process.journal.receiver]
+	forward_to    = [loki.write.default.receiver]
 	labels        = {
 		job       = "systemd-journal",
 		partition = "{{ metal_partition_id }}",
diff --git a/partition/roles/alloy/templates/snippets/syslog-logs.alloy.j2 b/partition/roles/alloy/templates/snippets/syslog-logs.alloy.j2
index c65037907..b846cbc7f 100644
--- a/partition/roles/alloy/templates/snippets/syslog-logs.alloy.j2
+++ b/partition/roles/alloy/templates/snippets/syslog-logs.alloy.j2
@@ -1,9 +1,3 @@
-loki.process "syslog" {
-	forward_to = [loki.write.default.receiver]
-
-	stage.docker {}
-}
-
 local.file_match "syslog" {
 	path_targets = [{
 		__path__  = "/var/log/syslog",
@@ -15,6 +9,8 @@ local.file_match "syslog" {
 
 loki.source.file "syslog" {
 	targets               = local.file_match.syslog.targets
-	forward_to            = [loki.process.syslog.receiver]
-	legacy_positions_file = "/var/log/promtail-positions.yaml"
+	forward_to            = [loki.write.default.receiver]
+	// can be removed after migration is complete, as alloy will convert it to a new internal format and continue to update it
+	// https://grafana.com/docs/alloy/latest/reference/components/loki/loki.source.file/#arguments
+	legacy_positions_file = "{{ alloy_promtail_positions_file }}"
 }

From 1ce97d156d74cfdd6b84a698b1034e21a139ff5e Mon Sep 17 00:00:00 2001
From: Matthias Hartmann <matthias.hartmann@x-cellent.com>
Date: Wed, 6 May 2026 16:35:40 +0200
Subject: [PATCH 05/41] chore: remove prometheus promtail targets

---
 partition/README.md                                 |  2 +-
 partition/roles/monitoring/prometheus/README.md     |  1 -
 .../roles/monitoring/prometheus/defaults/main.yaml  |  1 -
 .../roles/monitoring/prometheus/tasks/main.yaml     | 13 -------------
 .../prometheus/templates/prometheus.yaml.j2         |  9 ---------
 5 files changed, 1 insertion(+), 25 deletions(-)

diff --git a/partition/README.md b/partition/README.md
index d61154e8e..4d1c9a3f6 100644
--- a/partition/README.md
+++ b/partition/README.md
@@ -40,7 +40,7 @@ You can look up all the default values [here](partition-defaults/main.yaml).
 | [metal-bmc](roles/metal-bmc)                 | Deploys metal-bmc                           |
 | [metal-core](roles/metal-core)               | Deploys metal-core                          |
 | [pixiecore](roles/pixiecore)                 | Deploys pixiecore                           |
-| [promtail](roles/promtail)                   | Deploys promtail                            |
+| [alloy](roles/alloy)                         | Deploys alloy                               |
 
 ## Examples
 
diff --git a/partition/roles/monitoring/prometheus/README.md b/partition/roles/monitoring/prometheus/README.md
index b46e07236..0851deadc 100644
--- a/partition/roles/monitoring/prometheus/README.md
+++ b/partition/roles/monitoring/prometheus/README.md
@@ -22,7 +22,6 @@ you define them adequately as well.
 | prometheus_frr_exporter_targets                  |           | FRR exporter targets to scrape from                                                                                                         |
 | prometheus_metal_core_targets                    |           | metal-core targets to scrape from                                                                                                           |
 | prometheus_node_exporter_targets                 |           | Node exporter targets to scrape from                                                                                                        |
-| prometheus_promtail_targets                      |           | Promtail targets to scrape from                                                                                                             |
 | prometheus_alloy_targets                         |           | Alloy targets to scrape from (default port `12345`)                                                                                         |
 | prometheus_ping_targets                          |           | Ping targets to scrape from                                                                                                                 |
 | prometheus_sonic_exporter_targets                |           | Sonic exporter targets to scrape from                                                                                                       |
diff --git a/partition/roles/monitoring/prometheus/defaults/main.yaml b/partition/roles/monitoring/prometheus/defaults/main.yaml
index ed3c3e864..1a0c992e8 100644
--- a/partition/roles/monitoring/prometheus/defaults/main.yaml
+++ b/partition/roles/monitoring/prometheus/defaults/main.yaml
@@ -21,7 +21,6 @@ prometheus_remote_write: []
 prometheus_frr_exporter_targets: []
 prometheus_metal_core_targets: []
 prometheus_node_exporter_targets: []
-prometheus_promtail_targets: []
 prometheus_alloy_targets: []
 prometheus_ping_targets: []
 prometheus_ipmi_exporter_targets: []
diff --git a/partition/roles/monitoring/prometheus/tasks/main.yaml b/partition/roles/monitoring/prometheus/tasks/main.yaml
index 6431d7fe7..0ba1418aa 100644
--- a/partition/roles/monitoring/prometheus/tasks/main.yaml
+++ b/partition/roles/monitoring/prometheus/tasks/main.yaml
@@ -116,19 +116,6 @@
   register: _result
   when: prometheus_ipmi_exporter_targets
 
-- name: generate prometheus_promtail_sd
-  set_fact:
-    prometheus_promtail_sd:
-      - labels:
-          job: frr
-        targets: "{{ prometheus_promtail_targets }}"
-
-- name: create promtail.yaml
-  copy:
-    content: "{{ prometheus_promtail_sd | to_nice_yaml(indent=2) }}"
-    dest: "{{ prometheus_config_host_dir }}/file_sd/promtail.yaml"
-  notify: restart prometheus
-
 - name: generate prometheus_alloy_sd
   set_fact:
     prometheus_alloy_sd:
diff --git a/partition/roles/monitoring/prometheus/templates/prometheus.yaml.j2 b/partition/roles/monitoring/prometheus/templates/prometheus.yaml.j2
index d0f0b1517..40df06927 100644
--- a/partition/roles/monitoring/prometheus/templates/prometheus.yaml.j2
+++ b/partition/roles/monitoring/prometheus/templates/prometheus.yaml.j2
@@ -79,15 +79,6 @@ scrape_configs:
       regex: (.*):\d+
       target_label: instance
 
-- job_name: promtail
-  file_sd_configs:
-  - files:
-    - '/etc/prometheus/file_sd/promtail.yaml'
-  relabel_configs:
-    - source_labels: [__address__]
-      regex: (.*):\d+
-      target_label: instance
-
 - job_name: alloy
   file_sd_configs:
   - files:

From ad2eacc531d208c8643e19cb3cb9ffed1b0928f3 Mon Sep 17 00:00:00 2001
From: Matthias Hartmann <matthias.hartmann@x-cellent.com>
Date: Wed, 6 May 2026 16:52:09 +0200
Subject: [PATCH 06/41] chore: simplify naming

---
 partition/roles/alloy/README.md               | 24 +++++++++----------
 ...journal-logs.alloy.j2 => journal.alloy.j2} |  0
 .../{syslog-logs.alloy.j2 => syslog.alloy.j2} |  0
 3 files changed, 12 insertions(+), 12 deletions(-)
 rename partition/roles/alloy/templates/snippets/{journal-logs.alloy.j2 => journal.alloy.j2} (100%)
 rename partition/roles/alloy/templates/snippets/{syslog-logs.alloy.j2 => syslog.alloy.j2} (100%)

diff --git a/partition/roles/alloy/README.md b/partition/roles/alloy/README.md
index 62b9deb45..d06382a99 100644
--- a/partition/roles/alloy/README.md
+++ b/partition/roles/alloy/README.md
@@ -21,12 +21,12 @@ The snippets are used in our default configs to provide a sensible out-of-the-bo
 
 If your needs are more custom (e.g. you have a non-standard log source, or want to use advanced Alloy features not covered by the existing snippets), you can follow the [Customizing the config](#customizing-the-config) section below to either add your own snippet or bypass the snippet system entirely with a custom raw config.
 
-| Snippet            | Description                                                                                                                                                                                                                                                                                                                           |
-| ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `alloy-meta`       | Meta-monitoring: forwards Alloy's own logs to Loki with `job=alloy` and `node_name` labels. Redundant when `journal-logs` is enabled — journald already captures Alloy's stdout/stderr. Log level info and format are always configured via the base config regardless of this snippet.                                               |
-| `leaf-node-docker` | Scrapes Docker container logs on the host via the Docker socket and forwards them to Loki. Assumes the `json-file` log driver (the default). If containers use the `journald` log driver instead, their logs are already captured by `journal-logs` — enabling both snippets in that case will produce duplicate log entries in Loki. |
-| `journal-logs`     | Scrapes the systemd journal and forwards to Loki; relabels `unit` from `__journal__systemd_unit`. On standard systemd hosts (Debian/Ubuntu), journald captures syslog messages as well, so this snippet typically covers everything `syslog-logs` would. Enabling both may produce duplicate log entries in Loki.                     |
-| `syslog-logs`      | Tails `/var/log/syslog` and forwards to Loki with `job=syslog`, `partition`, and `host` labels. Suited for hosts without journald (e.g. minimal switch OS images). On standard systemd hosts, prefer `journal-logs` — enabling both may produce duplicate log entries in Loki.                                                        |
+| Snippet            | Description                                                                                                                                                                                                                                                                                                                      |
+| ------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `alloy-meta`       | Meta-monitoring: forwards Alloy's own logs to Loki with `job=alloy` and `node_name` labels. Redundant when `journal` is enabled — journald already captures Alloy's stdout/stderr. Log level info and format are always configured via the base config regardless of this snippet.                                               |
+| `leaf-node-docker` | Scrapes Docker container logs on the host via the Docker socket and forwards them to Loki. Assumes the `json-file` log driver (the default). If containers use the `journald` log driver instead, their logs are already captured by `journal` — enabling both snippets in that case will produce duplicate log entries in Loki. |
+| `journal`          | Scrapes the systemd journal and forwards to Loki; relabels `unit` from `__journal__systemd_unit`. On standard systemd hosts (Debian/Ubuntu), journald captures syslog messages as well, so this snippet typically covers everything `syslog` would. Enabling both may produce duplicate log entries in Loki.                     |
+| `syslog`           | Tails `/var/log/syslog` and forwards to Loki with `job=syslog`, `partition`, and `host` labels. Suited for hosts without journald (e.g. minimal switch OS images). On standard systemd hosts, prefer `journal` — enabling both may produce duplicate log entries in Loki.                                                        |
 
 ### Variables
 
@@ -39,7 +39,7 @@ If your needs are more custom (e.g. you have a non-standard log source, or want
 | alloy_docker_log_driver       |           | Docker log driver for the alloy container (default: `json-file`)                                                                                          |
 | alloy_config_snippets         |           | List of snippet names to enable (default: `[]`)                                                                                                           |
 | alloy_port                    |           | Port for Alloy metrics and HTTP API (default: `12345`)                                                                                                    |
-| alloy_promtail_positions_file |           | Path to the legacy promtail positions file (default: `/var/log/promtail-positions.yaml`). Used by `syslog-logs` on first start after migration.           |
+| alloy_promtail_positions_file |           | Path to the legacy promtail positions file (default: `/var/log/promtail-positions.yaml`). Used by `syslog` on first start after migration.                |
 | alloy_config_raw              |           | Full Alloy River config as a string. When set, bypasses snippet assembly entirely — `alloy_loki_write_endpoints` and `alloy_config_snippets` are ignored. |
 
 ### Meta-monitoring (for Alloy itself)
@@ -52,8 +52,8 @@ Alloy always exposes Prometheus metrics on `0.0.0.0:{{ alloy_port }}/metrics`, r
 
 Alloy's own logs are captured in two ways depending on your snippet configuration:
 
-- **If `journal-logs` is enabled** — Alloy runs as a systemd-managed Docker container, so its stdout/stderr are captured by journald automatically. No additional snippet is needed.
-- **If `journal-logs` is not enabled** (e.g. you only use `syslog-logs` or no log snippet at all) — enable the `alloy-meta` snippet to forward Alloy's own logs to Loki explicitly.
+- **If `journal` is enabled** — Alloy runs as a systemd-managed Docker container, so its stdout/stderr are captured by journald automatically. No additional snippet is needed.
+- **If `journal` is not enabled** (e.g. you only use `syslog` or no log snippet at all) — enable the `alloy-meta` snippet to forward Alloy's own logs to Loki explicitly.
 
 Alloy's log level (`info`) and format (`logfmt`) are always configured via the base config, regardless of which snippets are enabled. The `alloy-meta` snippet only adds the Loki forwarding pipeline on top.
 
@@ -61,7 +61,7 @@ Alloy's log level (`info`) and format (`logfmt`) are always configured via the b
 
 Alloy stores its write-ahead log (WAL) — including cursor state for `loki.source.file` and `loki.source.journal` — under `--storage.path`, which is set to `/var/lib/alloy` (bind-mounted from the host). This survives container restarts, preventing duplicate log shipments.
 
-The `syslog-logs` snippet reads the legacy promtail positions file on first start after migration to continue tailing `/var/log/syslog` from where promtail left off. The path is configured via `alloy_promtail_positions_file` (default: `/var/log/promtail-positions.yaml`). No log lines are re-shipped. This only happens once — after the initial migration, Alloy updates the positions in its own WAL format.
+The `syslog` snippet reads the legacy promtail positions file on first start after migration to continue tailing `/var/log/syslog` from where promtail left off. The path is configured via `alloy_promtail_positions_file` (default: `/var/log/promtail-positions.yaml`). No log lines are re-shipped. This only happens once — after the initial migration, Alloy updates the positions in its own WAL format.
 
 `loki.source.journal` has no equivalent migration path — its cursor is WAL-only. The very first Alloy start after migration will re-read a portion of the journal. This is a one-time event and subsequent restarts are safe once the WAL is populated.
 
@@ -113,8 +113,8 @@ Alloy allows reproducing the same log forwarding behavior as promtail via the [a
    If your environment deploys Alloy on different host groups with different scrape needs (e.g. leaf switches vs. management servers), set `alloy_config_snippets` in per-group inventory files:
 
    ```text
-   group_vars/leaves/alloy.yaml       ← alloy_config_snippets: [leaf-node-docker, journal-logs]
-   group_vars/mgmtservers/alloy.yaml  ← alloy_config_snippets: [journal-logs]
+   group_vars/leaves/alloy.yaml       ← alloy_config_snippets: [leaf-node-docker, journal]
+   group_vars/mgmtservers/alloy.yaml  ← alloy_config_snippets: [journal]
    group_vars/partition/alloy.yaml    ← alloy_loki_write_endpoints: [...] (shared by all)
    ```
 
diff --git a/partition/roles/alloy/templates/snippets/journal-logs.alloy.j2 b/partition/roles/alloy/templates/snippets/journal.alloy.j2
similarity index 100%
rename from partition/roles/alloy/templates/snippets/journal-logs.alloy.j2
rename to partition/roles/alloy/templates/snippets/journal.alloy.j2
diff --git a/partition/roles/alloy/templates/snippets/syslog-logs.alloy.j2 b/partition/roles/alloy/templates/snippets/syslog.alloy.j2
similarity index 100%
rename from partition/roles/alloy/templates/snippets/syslog-logs.alloy.j2
rename to partition/roles/alloy/templates/snippets/syslog.alloy.j2

From ef67f5b25881bf76a654af4c0e95243b19679926 Mon Sep 17 00:00:00 2001
From: Matthias Hartmann <matthias.hartmann@x-cellent.com>
Date: Wed, 6 May 2026 17:55:19 +0200
Subject: [PATCH 07/41] fix: use correct alloy image from release

---
 common/roles/defaults/defaults/main.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/common/roles/defaults/defaults/main.yaml b/common/roles/defaults/defaults/main.yaml
index cbbdd7661..8613a8eaf 100644
--- a/common/roles/defaults/defaults/main.yaml
+++ b/common/roles/defaults/defaults/main.yaml
@@ -96,8 +96,8 @@ metal_stack_release:
     capms_controller_name: "docker-images.metal-stack.kubernetes.cluster-api-provider-metal-stack-controller.name"
     capms_controller_tag: "docker-images.metal-stack.kubernetes.cluster-api-provider-metal-stack-controller.tag"
     # third party
-    alloy_image_tag: "docker-images.third-party.control-plane.alloy.tag"
-    alloy_image_name: "docker-images.third-party.control-plane.alloy.name"
+    alloy_image_tag: "docker-images.third-party.partition.alloy.tag"
+    alloy_image_name: "docker-images.third-party.partition.alloy.name"
     event_exporter_name: "docker-images.third-party.control-plane.event-exporter.name"
     event_exporter_tag: "docker-images.third-party.control-plane.event-exporter.tag"
     nsq_image_tag: "docker-images.third-party.control-plane.nsq.tag"

From 673f6bee9b6fc5ffc6ae5f28fe13ee9a9e688103 Mon Sep 17 00:00:00 2001
From: Matthias Hartmann <matthias.hartmann@x-cellent.com>
Date: Thu, 7 May 2026 12:09:54 +0200
Subject: [PATCH 08/41] feat: journal-file alloy snippet allows migrating from
 promtail positions file

---
 partition/roles/alloy/README.md               | 59 ++++++++++++-------
 partition/roles/alloy/defaults/main.yaml      | 21 +++++--
 partition/roles/alloy/tasks/main.yaml         | 15 +++++
 .../templates/snippets/journal-file.alloy.j2  | 27 +++++++++
 .../alloy/templates/snippets/journal.alloy.j2 |  1 -
 .../alloy/templates/snippets/syslog.alloy.j2  |  5 +-
 6 files changed, 99 insertions(+), 29 deletions(-)
 create mode 100644 partition/roles/alloy/templates/snippets/journal-file.alloy.j2

diff --git a/partition/roles/alloy/README.md b/partition/roles/alloy/README.md
index d06382a99..26197538b 100644
--- a/partition/roles/alloy/README.md
+++ b/partition/roles/alloy/README.md
@@ -21,26 +21,30 @@ The snippets are used in our default configs to provide a sensible out-of-the-bo
 
 If your needs are more custom (e.g. you have a non-standard log source, or want to use advanced Alloy features not covered by the existing snippets), you can follow the [Customizing the config](#customizing-the-config) section below to either add your own snippet or bypass the snippet system entirely with a custom raw config.
 
-| Snippet            | Description                                                                                                                                                                                                                                                                                                                      |
-| ------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `alloy-meta`       | Meta-monitoring: forwards Alloy's own logs to Loki with `job=alloy` and `node_name` labels. Redundant when `journal` is enabled — journald already captures Alloy's stdout/stderr. Log level info and format are always configured via the base config regardless of this snippet.                                               |
-| `leaf-node-docker` | Scrapes Docker container logs on the host via the Docker socket and forwards them to Loki. Assumes the `json-file` log driver (the default). If containers use the `journald` log driver instead, their logs are already captured by `journal` — enabling both snippets in that case will produce duplicate log entries in Loki. |
-| `journal`          | Scrapes the systemd journal and forwards to Loki; relabels `unit` from `__journal__systemd_unit`. On standard systemd hosts (Debian/Ubuntu), journald captures syslog messages as well, so this snippet typically covers everything `syslog` would. Enabling both may produce duplicate log entries in Loki.                     |
-| `syslog`           | Tails `/var/log/syslog` and forwards to Loki with `job=syslog`, `partition`, and `host` labels. Suited for hosts without journald (e.g. minimal switch OS images). On standard systemd hosts, prefer `journal` — enabling both may produce duplicate log entries in Loki.                                                        |
+| Snippet            | Description                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 |
+| ------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `alloy-meta`       | Meta-monitoring: forwards Alloy's own logs to Loki with `job=alloy` and `node_name` labels. Redundant when `journal` is enabled — journald already captures Alloy's stdout/stderr. Log level info and format are always configured via the base config regardless of this snippet.                                                                                                                                                                                                                                          |
+| `leaf-node-docker` | Scrapes Docker container logs on the host via the Docker socket and forwards them to Loki. Assumes the `json-file` log driver (the default). If containers use the `journald` log driver instead, their logs are already captured by `journal` — enabling both snippets in that case will produce duplicate log entries in Loki.                                                                                                                                                                                            |
+| `journal`          | Scrapes the systemd journal via the journal API and forwards to Loki; relabels `unit` from `__journal__systemd_unit`. Automatically discovers the journal regardless of storage mode (volatile `/run/log/journal` or persistent `/var/log/journal`). On standard systemd hosts, journald captures syslog messages as well, so this snippet typically covers everything `syslog` would. Enabling both may produce duplicate log entries in Loki.                                                                             |
+| `journal-file`     | Like `journal`, but reads from an explicit directory path (`alloy_journal_path`, default `/var/log/journal`) instead of the journal API. Without an explicit path, `journal` already auto-discovers both `/var/log/journal` and `/run/log/journal`, so prefer `journal` for new deployments. Use `journal-file` only when migrating from promtail and needing the `legacy_position` block to resume cursor state. Do not enable both `journal` and `journal-file` simultaneously — this will produce duplicate log entries. |
+| `syslog`           | Tails `/var/log/syslog` and forwards to Loki with `job=syslog`, `partition`, and `host` labels. Suited for hosts without journald (e.g. minimal switch OS images). On standard systemd hosts, prefer `journal` — enabling both may produce duplicate log entries in Loki.                                                                                                                                                                                                                                                   |
 
 ### Variables
 
-| Name                          | Mandatory | Description                                                                                                                                               |
-| ----------------------------- | --------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| alloy_config_host_dir         |           | The location of the alloy config on the host (default: `/etc/alloy`)                                                                                      |
-| alloy_image_name              | yes       | Image name of alloy                                                                                                                                       |
-| alloy_image_tag               | yes       | Image tag of alloy                                                                                                                                        |
-| alloy_loki_write_endpoints    | yes       | List of Loki push endpoints. Each entry: `{url, remote_timeout?: <duration>, basic_auth?: {username, password}}`                                          |
-| alloy_docker_log_driver       |           | Docker log driver for the alloy container (default: `json-file`)                                                                                          |
-| alloy_config_snippets         |           | List of snippet names to enable (default: `[]`)                                                                                                           |
-| alloy_port                    |           | Port for Alloy metrics and HTTP API (default: `12345`)                                                                                                    |
-| alloy_promtail_positions_file |           | Path to the legacy promtail positions file (default: `/var/log/promtail-positions.yaml`). Used by `syslog` on first start after migration.                |
-| alloy_config_raw              |           | Full Alloy River config as a string. When set, bypasses snippet assembly entirely — `alloy_loki_write_endpoints` and `alloy_config_snippets` are ignored. |
+| Name                                | Mandatory | Description                                                                                                                                                                                             |
+| ----------------------------------- | --------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| alloy_config_host_dir               |           | The location of the alloy config on the host (default: `/etc/alloy`)                                                                                                                                    |
+| alloy_image_name                    | yes       | Image name of alloy                                                                                                                                                                                     |
+| alloy_image_tag                     | yes       | Image tag of alloy                                                                                                                                                                                      |
+| alloy_loki_write_endpoints          | yes       | List of Loki push endpoints. Each entry: `{url, remote_timeout?: <duration>, basic_auth?: {username, password}}`                                                                                        |
+| alloy_docker_log_driver             |           | Docker log driver for the alloy container (default: `json-file`)                                                                                                                                        |
+| alloy_config_snippets               |           | List of snippet names to enable (default: `[]`)                                                                                                                                                         |
+| alloy_port                          |           | Port for Alloy metrics and HTTP API (default: `12345`)                                                                                                                                                  |
+| alloy_syslog_legacy_positions_file  |           | Path to the legacy promtail positions file (default: `/var/log/promtail-positions.yaml`). Used by `syslog` on first start after migration.                                                              |
+| alloy_journal_legacy_positions_file |           | Path to the legacy promtail positions file (default: `/var/log/promtail-positions.yaml`). Used by `journal-file` on first start after migration.                                                        |
+| alloy_journal_path                  |           | Path to the persistent journal directory used by the `journal-file` snippet (default: `/var/log/journal`).                                                                                              |
+| alloy_journal_legacy_position_name  |           | Job name from the old promtail journal scrape_config, used by `journal-file` to resume from the legacy positions file (default: `journal`). Required when `alloy_journal_legacy_positions_file` is set. |
+| alloy_config_raw                    |           | Full Alloy River config as a string. When set, bypasses snippet assembly entirely — `alloy_loki_write_endpoints` and `alloy_config_snippets` are ignored.                                               |
 
 ### Meta-monitoring (for Alloy itself)
 
@@ -57,15 +61,24 @@ Alloy's own logs are captured in two ways depending on your snippet configuratio
 
 Alloy's log level (`info`) and format (`logfmt`) are always configured via the base config, regardless of which snippets are enabled. The `alloy-meta` snippet only adds the Loki forwarding pipeline on top.
 
-### WAL and positions file persistence
+### WAL and cursor persistence
 
 Alloy stores its write-ahead log (WAL) — including cursor state for `loki.source.file` and `loki.source.journal` — under `--storage.path`, which is set to `/var/lib/alloy` (bind-mounted from the host). This survives container restarts, preventing duplicate log shipments.
+When migrating from promtail, the cursor state from promtail's positions file can be imported into Alloy's WAL on first start after migration to continue from where promtail left off. The exact mechanism depends on the snippet:
 
-The `syslog` snippet reads the legacy promtail positions file on first start after migration to continue tailing `/var/log/syslog` from where promtail left off. The path is configured via `alloy_promtail_positions_file` (default: `/var/log/promtail-positions.yaml`). No log lines are re-shipped. This only happens once — after the initial migration, Alloy updates the positions in its own WAL format.
+| Snippet        | Cursor mechanism                                   | Promtail migration path                                                      |
+| -------------- | -------------------------------------------------- | ---------------------------------------------------------------------------- |
+| `syslog`       | WAL via `loki.source.file`                         | `alloy_syslog_legacy_positions_file`                                         |
+| `journal`      | WAL via `loki.source.journal` (journal API)        | None — first start re-reads a small journal window once                      |
+| `journal-file` | WAL via `loki.source.journal` (explicit file path) | `alloy_journal_legacy_positions_file` + `alloy_journal_legacy_position_name` |
 
-`loki.source.journal` has no equivalent migration path — its cursor is WAL-only. The very first Alloy start after migration will re-read a portion of the journal. This is a one-time event and subsequent restarts are safe once the WAL is populated.
+**`syslog`:** On first start after migration, Alloy reads `alloy_syslog_legacy_positions_file` (default: `/var/log/promtail-positions.yaml`) to continue tailing `/var/log/syslog` from where promtail left off. No log lines are re-shipped. After this Alloy tracks positions in its own WAL format. If the file does not exist on the host, this has no effect.
 
-Once the migration is complete, the `legacy_positions_file` line can optionally be dropped from the snippet and the old positions file cleaned up from hosts — Alloy will simply start tracking positions in its own WAL format from that point.
+**`journal-file`:** On first start after migration, Alloy reads the legacy positions file via the `legacy_position` block using `alloy_journal_legacy_positions_file` (default: `/var/log/promtail-positions.yaml`) and `alloy_journal_legacy_position_name` (default: `journal`). The name must match the `job_name` of the journal scrape config in your old promtail config. If the file does not exist on the host, this has no effect.
+
+**`journal`:** No promtail-compatible migration path. The first start after migration re-reads a small window of the journal — this is a one-time event and subsequent restarts are safe once the WAL is populated.
+
+Once migration is complete, the old promtail positions files can be cleaned up from hosts — Alloy will already be tracking positions in its own WAL format from that point.
 
 ## Customizing the config
 
@@ -110,11 +123,13 @@ Alloy allows reproducing the same log forwarding behavior as promtail via the [a
 
 3. **Choose snippets** that match your legacy `promtail_scrape_configs` — see [Available snippets](#available-snippets). If the available snippets are close enough to your old setup, enable the relevant ones and adjust with the provided variables. Otherwise, continue with step 4.
 
+   If you use `syslog` or `journal-file`, set the corresponding legacy positions file variable so Alloy resumes from where promtail left off — see [WAL and cursor persistence](#wal-and-cursor-persistence) for details.
+
    If your environment deploys Alloy on different host groups with different scrape needs (e.g. leaf switches vs. management servers), set `alloy_config_snippets` in per-group inventory files:
 
    ```text
    group_vars/leaves/alloy.yaml       ← alloy_config_snippets: [leaf-node-docker, journal]
-   group_vars/mgmtservers/alloy.yaml  ← alloy_config_snippets: [journal]
+   group_vars/mgmtservers/alloy.yaml  ← alloy_config_snippets: [journal-file]
    group_vars/partition/alloy.yaml    ← alloy_loki_write_endpoints: [...] (shared by all)
    ```
 
diff --git a/partition/roles/alloy/defaults/main.yaml b/partition/roles/alloy/defaults/main.yaml
index 4b91998ba..b864ca163 100644
--- a/partition/roles/alloy/defaults/main.yaml
+++ b/partition/roles/alloy/defaults/main.yaml
@@ -19,7 +19,20 @@ alloy_config_snippets: []
 # Port to listen for metrics and HTTP API traffic on
 alloy_port: 12345
 
-# Path to the legacy promtail positions file. Read once on first start after migration
-# to continue tailing /var/log/syslog from where promtail left off.
-# Can be removed from the snippet once all hosts have been migrated.
-alloy_promtail_positions_file: "/var/log/promtail-positions.yaml"
+# Path to the legacy promtail positions file used by the syslog snippet on first start
+# after migration to continue tailing /var/log/syslog from where promtail left off.
+# Has no effect if the file does not exist on the host.
+alloy_syslog_legacy_positions_file: "/var/log/promtail-positions.yaml"
+
+# Path to the persistent journal directory used by the journal-file snippet.
+alloy_journal_path: "/var/log/journal"
+
+# Path to the legacy promtail positions file used by the journal-file snippet on first
+# start after migration to resume from where promtail left off.
+# Has no effect if the file does not exist on the host.
+alloy_journal_legacy_positions_file: "/var/log/promtail-positions.yaml"
+
+# Job name used in the legacy promtail positions file for the journal scrape config.
+# Must match the job_name of the journal scrape_config in your old promtail config.
+# Required when alloy_journal_legacy_positions_file is set. Used by the journal-file snippet for migration only.
+alloy_journal_legacy_position_name: "journal"
diff --git a/partition/roles/alloy/tasks/main.yaml b/partition/roles/alloy/tasks/main.yaml
index 76a46dd92..2bbc56d09 100644
--- a/partition/roles/alloy/tasks/main.yaml
+++ b/partition/roles/alloy/tasks/main.yaml
@@ -11,6 +11,21 @@
       - alloy_image_name is defined
       - alloy_config_raw is defined or (alloy_loki_write_endpoints is defined and (alloy_loki_write_endpoints | length > 0))
 
+- name: Check journal-file legacy_position variables are either both set or both absent
+  assert:
+    fail_msg: "alloy_journal_legacy_positions_file and alloy_journal_legacy_position_name must either both be set or both be absent (alloy_journal_legacy_position_name must match the job_name of the journal scrape_config in your old promtail config)"
+    quiet: yes
+    that:
+      - (alloy_journal_legacy_positions_file is defined) == (alloy_journal_legacy_position_name is defined)
+  when: "'journal-file' in alloy_config_snippets"
+
+- name: Check journal and journal-file are not both enabled
+  assert:
+    fail_msg: "journal and journal-file snippets cannot both be enabled — this will produce duplicate log entries in Loki"
+    quiet: yes
+    that:
+      - not ('journal' in alloy_config_snippets and 'journal-file' in alloy_config_snippets)
+
 - name: Create alloy config directory
   file:
     path: "{{ alloy_config_host_dir }}"
diff --git a/partition/roles/alloy/templates/snippets/journal-file.alloy.j2 b/partition/roles/alloy/templates/snippets/journal-file.alloy.j2
new file mode 100644
index 000000000..91a3ec885
--- /dev/null
+++ b/partition/roles/alloy/templates/snippets/journal-file.alloy.j2
@@ -0,0 +1,27 @@
+discovery.relabel "journal" {
+	targets = []
+
+	rule {
+		source_labels = ["__journal__systemd_unit"]
+		target_label  = "unit"
+	}
+}
+
+loki.source.journal "journal" {
+	path          = "{{ alloy_journal_path }}"
+	relabel_rules = discovery.relabel.journal.rules
+	forward_to    = [loki.write.default.receiver]
+	labels        = {
+		job       = "systemd-journal",
+		partition = "{{ metal_partition_id }}",
+		host      = "{{ inventory_hostname }}",
+	}
+
+	// read once on first start after migration to resume from where promtail left off;
+	// ignored if the file does not exist. alloy then tracks the cursor in its own wal.
+	// https://grafana.com/docs/alloy/latest/reference/components/loki/loki.source.journal/#legacy_position
+	legacy_position {
+		file = "{{ alloy_journal_legacy_positions_file }}"
+		name = "{{ alloy_journal_legacy_position_name }}"
+	}
+}
diff --git a/partition/roles/alloy/templates/snippets/journal.alloy.j2 b/partition/roles/alloy/templates/snippets/journal.alloy.j2
index 7d1cc783b..7afc20d64 100644
--- a/partition/roles/alloy/templates/snippets/journal.alloy.j2
+++ b/partition/roles/alloy/templates/snippets/journal.alloy.j2
@@ -8,7 +8,6 @@ discovery.relabel "journal" {
 }
 
 loki.source.journal "journal" {
-	path          = "/var/log/journal"
 	relabel_rules = discovery.relabel.journal.rules
 	forward_to    = [loki.write.default.receiver]
 	labels        = {
diff --git a/partition/roles/alloy/templates/snippets/syslog.alloy.j2 b/partition/roles/alloy/templates/snippets/syslog.alloy.j2
index b846cbc7f..5be7068d1 100644
--- a/partition/roles/alloy/templates/snippets/syslog.alloy.j2
+++ b/partition/roles/alloy/templates/snippets/syslog.alloy.j2
@@ -10,7 +10,8 @@ local.file_match "syslog" {
 loki.source.file "syslog" {
 	targets               = local.file_match.syslog.targets
 	forward_to            = [loki.write.default.receiver]
-	// can be removed after migration is complete, as alloy will convert it to a new internal format and continue to update it
+	// read once on first start after migration to resume from where promtail left off;
+	// ignored if the file does not exist. alloy then tracks positions in its own wal.
 	// https://grafana.com/docs/alloy/latest/reference/components/loki/loki.source.file/#arguments
-	legacy_positions_file = "{{ alloy_promtail_positions_file }}"
+	legacy_positions_file = "{{ alloy_syslog_legacy_positions_file }}"
 }

From f830d3dd7c788a9559fddfba3f414c33447bf110 Mon Sep 17 00:00:00 2001
From: Matthias Hartmann <matthias.hartmann@x-cellent.com>
Date: Thu, 7 May 2026 14:26:43 +0200
Subject: [PATCH 09/41] fix: unify naming

---
 partition/roles/alloy/README.md               | 18 ++++++++---------
 partition/roles/alloy/tasks/main.yaml         |  2 ++
 .../templates/snippets/alloy-meta.alloy.j2    |  5 +++++
 ...f-node-docker.alloy.j2 => docker.alloy.j2} | 20 ++++++++++++++-----
 .../templates/snippets/journal-file.alloy.j2  |  2 +-
 .../alloy/templates/snippets/journal.alloy.j2 |  2 +-
 6 files changed, 33 insertions(+), 16 deletions(-)
 rename partition/roles/alloy/templates/snippets/{leaf-node-docker.alloy.j2 => docker.alloy.j2} (55%)

diff --git a/partition/roles/alloy/README.md b/partition/roles/alloy/README.md
index 26197538b..6ec9b1ac2 100644
--- a/partition/roles/alloy/README.md
+++ b/partition/roles/alloy/README.md
@@ -21,13 +21,13 @@ The snippets are used in our default configs to provide a sensible out-of-the-bo
 
 If your needs are more custom (e.g. you have a non-standard log source, or want to use advanced Alloy features not covered by the existing snippets), you can follow the [Customizing the config](#customizing-the-config) section below to either add your own snippet or bypass the snippet system entirely with a custom raw config.
 
-| Snippet            | Description                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 |
-| ------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `alloy-meta`       | Meta-monitoring: forwards Alloy's own logs to Loki with `job=alloy` and `node_name` labels. Redundant when `journal` is enabled — journald already captures Alloy's stdout/stderr. Log level info and format are always configured via the base config regardless of this snippet.                                                                                                                                                                                                                                          |
-| `leaf-node-docker` | Scrapes Docker container logs on the host via the Docker socket and forwards them to Loki. Assumes the `json-file` log driver (the default). If containers use the `journald` log driver instead, their logs are already captured by `journal` — enabling both snippets in that case will produce duplicate log entries in Loki.                                                                                                                                                                                            |
-| `journal`          | Scrapes the systemd journal via the journal API and forwards to Loki; relabels `unit` from `__journal__systemd_unit`. Automatically discovers the journal regardless of storage mode (volatile `/run/log/journal` or persistent `/var/log/journal`). On standard systemd hosts, journald captures syslog messages as well, so this snippet typically covers everything `syslog` would. Enabling both may produce duplicate log entries in Loki.                                                                             |
-| `journal-file`     | Like `journal`, but reads from an explicit directory path (`alloy_journal_path`, default `/var/log/journal`) instead of the journal API. Without an explicit path, `journal` already auto-discovers both `/var/log/journal` and `/run/log/journal`, so prefer `journal` for new deployments. Use `journal-file` only when migrating from promtail and needing the `legacy_position` block to resume cursor state. Do not enable both `journal` and `journal-file` simultaneously — this will produce duplicate log entries. |
-| `syslog`           | Tails `/var/log/syslog` and forwards to Loki with `job=syslog`, `partition`, and `host` labels. Suited for hosts without journald (e.g. minimal switch OS images). On standard systemd hosts, prefer `journal` — enabling both may produce duplicate log entries in Loki.                                                                                                                                                                                                                                                   |
+| Snippet        | Description                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 |
+| -------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `alloy-meta`   | Meta-monitoring: forwards Alloy's own logs to Loki with `job=alloy` and `node_name` labels. Redundant when `journal` is enabled — journald already captures Alloy's stdout/stderr. Log level info and format are always configured via the base config regardless of this snippet.                                                                                                                                                                                                                                          |
+| `docker`       | Scrapes Docker container logs on the host via the Docker socket and forwards them to Loki. Assumes the `json-file` log driver (the default). If containers use the `journald` log driver instead, their logs are already captured by `journal` — enabling both snippets in that case will produce duplicate log entries in Loki.                                                                                                                                                                                            |
+| `journal`      | Scrapes the systemd journal via the journal API and forwards to Loki; relabels `unit` from `__journal__systemd_unit`. Automatically discovers the journal regardless of storage mode (volatile `/run/log/journal` or persistent `/var/log/journal`). On standard systemd hosts, journald captures syslog messages as well, so this snippet typically covers everything `syslog` would. Enabling both may produce duplicate log entries in Loki.                                                                             |
+| `journal-file` | Like `journal`, but reads from an explicit directory path (`alloy_journal_path`, default `/var/log/journal`) instead of the journal API. Without an explicit path, `journal` already auto-discovers both `/var/log/journal` and `/run/log/journal`, so prefer `journal` for new deployments. Use `journal-file` only when migrating from promtail and needing the `legacy_position` block to resume cursor state. Do not enable both `journal` and `journal-file` simultaneously — this will produce duplicate log entries. |
+| `syslog`       | Tails `/var/log/syslog` and forwards to Loki with `job=syslog`, `partition`, and `host` labels. Suited for hosts without journald (e.g. minimal switch OS images). On standard systemd hosts, prefer `journal` — enabling both may produce duplicate log entries in Loki.                                                                                                                                                                                                                                                   |
 
 ### Variables
 
@@ -95,7 +95,7 @@ Add the template to `templates/snippets/` and reference it by name in `alloy_con
 ```yaml
 alloy_config_snippets:
   - alloy-meta
-  - leaf-node-docker
+  - docker
 ```
 
 A snippet template can use any Ansible variables available on the host. It does not need to define `loki.write "default"` as it is already in the base config.
@@ -128,7 +128,7 @@ Alloy allows reproducing the same log forwarding behavior as promtail via the [a
    If your environment deploys Alloy on different host groups with different scrape needs (e.g. leaf switches vs. management servers), set `alloy_config_snippets` in per-group inventory files:
 
    ```text
-   group_vars/leaves/alloy.yaml       ← alloy_config_snippets: [leaf-node-docker, journal]
+   group_vars/leaves/alloy.yaml       ← alloy_config_snippets: [docker, journal]
    group_vars/mgmtservers/alloy.yaml  ← alloy_config_snippets: [journal-file]
    group_vars/partition/alloy.yaml    ← alloy_loki_write_endpoints: [...] (shared by all)
    ```
diff --git a/partition/roles/alloy/tasks/main.yaml b/partition/roles/alloy/tasks/main.yaml
index 2bbc56d09..06953d670 100644
--- a/partition/roles/alloy/tasks/main.yaml
+++ b/partition/roles/alloy/tasks/main.yaml
@@ -94,6 +94,8 @@
       - "{{ alloy_config_host_dir }}:/etc/alloy:ro"
       - "/var/lib/alloy:/var/lib/alloy"
       - "/var/log:/var/log"
+      - "/run/log/journal:/run/log/journal:ro"
+      - "/etc/machine-id:/etc/machine-id:ro"
       - "/var/run/docker.sock:/var/run/docker.sock"
     systemd_docker_command:
       - run
diff --git a/partition/roles/alloy/templates/snippets/alloy-meta.alloy.j2 b/partition/roles/alloy/templates/snippets/alloy-meta.alloy.j2
index 550b0fcfc..20022438d 100644
--- a/partition/roles/alloy/templates/snippets/alloy-meta.alloy.j2
+++ b/partition/roles/alloy/templates/snippets/alloy-meta.alloy.j2
@@ -11,4 +11,9 @@ loki.relabel "alloy_self" {
 		target_label = "job"
 		replacement  = "alloy"
 	}
+
+	rule {
+		target_label = "partition"
+		replacement  = "{{ metal_partition_id }}"
+	}
 }
diff --git a/partition/roles/alloy/templates/snippets/leaf-node-docker.alloy.j2 b/partition/roles/alloy/templates/snippets/docker.alloy.j2
similarity index 55%
rename from partition/roles/alloy/templates/snippets/leaf-node-docker.alloy.j2
rename to partition/roles/alloy/templates/snippets/docker.alloy.j2
index 40127a47b..86730c018 100644
--- a/partition/roles/alloy/templates/snippets/leaf-node-docker.alloy.j2
+++ b/partition/roles/alloy/templates/snippets/docker.alloy.j2
@@ -1,9 +1,9 @@
-discovery.docker "leaf_node_docker" {
+discovery.docker "docker" {
 	host             = "unix:///var/run/docker.sock"
 	refresh_interval = "5s"
 }
 
-discovery.relabel "leaf_node_docker" {
+discovery.relabel "docker" {
 	targets = []
 
 	rule {
@@ -16,12 +16,22 @@ discovery.relabel "leaf_node_docker" {
 		target_label = "node_name"
 		replacement  = "{{ inventory_hostname }}"
 	}
+
+	rule {
+		target_label = "job"
+		replacement  = "docker"
+	}
+
+	rule {
+		target_label = "partition"
+		replacement  = "{{ metal_partition_id }}"
+	}
 }
 
-loki.source.docker "leaf_node_docker" {
+loki.source.docker "docker" {
 	host             = "unix:///var/run/docker.sock"
-	targets          = discovery.docker.leaf_node_docker.targets
+	targets          = discovery.docker.docker.targets
 	forward_to       = [loki.write.default.receiver]
-	relabel_rules    = discovery.relabel.leaf_node_docker.rules
+	relabel_rules    = discovery.relabel.docker.rules
 	refresh_interval = "5s"
 }
diff --git a/partition/roles/alloy/templates/snippets/journal-file.alloy.j2 b/partition/roles/alloy/templates/snippets/journal-file.alloy.j2
index 91a3ec885..8d5013870 100644
--- a/partition/roles/alloy/templates/snippets/journal-file.alloy.j2
+++ b/partition/roles/alloy/templates/snippets/journal-file.alloy.j2
@@ -12,7 +12,7 @@ loki.source.journal "journal" {
 	relabel_rules = discovery.relabel.journal.rules
 	forward_to    = [loki.write.default.receiver]
 	labels        = {
-		job       = "systemd-journal",
+		job       = "journal",
 		partition = "{{ metal_partition_id }}",
 		host      = "{{ inventory_hostname }}",
 	}
diff --git a/partition/roles/alloy/templates/snippets/journal.alloy.j2 b/partition/roles/alloy/templates/snippets/journal.alloy.j2
index 7afc20d64..6fbaa6790 100644
--- a/partition/roles/alloy/templates/snippets/journal.alloy.j2
+++ b/partition/roles/alloy/templates/snippets/journal.alloy.j2
@@ -11,7 +11,7 @@ loki.source.journal "journal" {
 	relabel_rules = discovery.relabel.journal.rules
 	forward_to    = [loki.write.default.receiver]
 	labels        = {
-		job       = "systemd-journal",
+		job       = "journal",
 		partition = "{{ metal_partition_id }}",
 		host      = "{{ inventory_hostname }}",
 	}

From 8b3c47ab11b3b49e92d3f93163fa1d9e42200368 Mon Sep 17 00:00:00 2001
From: Matthias Hartmann <matthias.hartmann@x-cellent.com>
Date: Thu, 7 May 2026 15:02:59 +0200
Subject: [PATCH 10/41] chore: unify label names

---
 partition/roles/alloy/templates/config.alloy.j2        |  6 +++++-
 .../roles/alloy/templates/snippets/alloy-meta.alloy.j2 | 10 ----------
 .../roles/alloy/templates/snippets/docker.alloy.j2     | 10 ----------
 .../alloy/templates/snippets/journal-file.alloy.j2     |  4 +---
 .../roles/alloy/templates/snippets/journal.alloy.j2    |  4 +---
 .../roles/alloy/templates/snippets/syslog.alloy.j2     |  6 ++----
 6 files changed, 9 insertions(+), 31 deletions(-)

diff --git a/partition/roles/alloy/templates/config.alloy.j2 b/partition/roles/alloy/templates/config.alloy.j2
index 37d827635..cf1560783 100644
--- a/partition/roles/alloy/templates/config.alloy.j2
+++ b/partition/roles/alloy/templates/config.alloy.j2
@@ -7,8 +7,12 @@ logging {
 {% endif %}
 }
 
-// Loki write endpoint shared by all snippets
+// Loki write endpoint shared by all snippets; external_labels are appended to every log entry
 loki.write "default" {
+	external_labels = {
+		host      = "{{ inventory_hostname }}",
+		partition = "{{ metal_partition_id }}",
+	}
 {% for endpoint in alloy_loki_write_endpoints %}
 	endpoint {
 		url = "{{ endpoint.url }}"
diff --git a/partition/roles/alloy/templates/snippets/alloy-meta.alloy.j2 b/partition/roles/alloy/templates/snippets/alloy-meta.alloy.j2
index 20022438d..34bff47df 100644
--- a/partition/roles/alloy/templates/snippets/alloy-meta.alloy.j2
+++ b/partition/roles/alloy/templates/snippets/alloy-meta.alloy.j2
@@ -2,18 +2,8 @@
 loki.relabel "alloy_self" {
 	forward_to = [loki.write.default.receiver]
 
-	rule {
-		target_label = "node_name"
-		replacement  = "{{ inventory_hostname }}"
-	}
-
 	rule {
 		target_label = "job"
 		replacement  = "alloy"
 	}
-
-	rule {
-		target_label = "partition"
-		replacement  = "{{ metal_partition_id }}"
-	}
 }
diff --git a/partition/roles/alloy/templates/snippets/docker.alloy.j2 b/partition/roles/alloy/templates/snippets/docker.alloy.j2
index 86730c018..71176376a 100644
--- a/partition/roles/alloy/templates/snippets/docker.alloy.j2
+++ b/partition/roles/alloy/templates/snippets/docker.alloy.j2
@@ -12,20 +12,10 @@ discovery.relabel "docker" {
 		target_label  = "container"
 	}
 
-	rule {
-		target_label = "node_name"
-		replacement  = "{{ inventory_hostname }}"
-	}
-
 	rule {
 		target_label = "job"
 		replacement  = "docker"
 	}
-
-	rule {
-		target_label = "partition"
-		replacement  = "{{ metal_partition_id }}"
-	}
 }
 
 loki.source.docker "docker" {
diff --git a/partition/roles/alloy/templates/snippets/journal-file.alloy.j2 b/partition/roles/alloy/templates/snippets/journal-file.alloy.j2
index 8d5013870..6de426067 100644
--- a/partition/roles/alloy/templates/snippets/journal-file.alloy.j2
+++ b/partition/roles/alloy/templates/snippets/journal-file.alloy.j2
@@ -12,9 +12,7 @@ loki.source.journal "journal" {
 	relabel_rules = discovery.relabel.journal.rules
 	forward_to    = [loki.write.default.receiver]
 	labels        = {
-		job       = "journal",
-		partition = "{{ metal_partition_id }}",
-		host      = "{{ inventory_hostname }}",
+		job = "systemd-journal",
 	}
 
 	// read once on first start after migration to resume from where promtail left off;
diff --git a/partition/roles/alloy/templates/snippets/journal.alloy.j2 b/partition/roles/alloy/templates/snippets/journal.alloy.j2
index 6fbaa6790..886dd89f4 100644
--- a/partition/roles/alloy/templates/snippets/journal.alloy.j2
+++ b/partition/roles/alloy/templates/snippets/journal.alloy.j2
@@ -11,8 +11,6 @@ loki.source.journal "journal" {
 	relabel_rules = discovery.relabel.journal.rules
 	forward_to    = [loki.write.default.receiver]
 	labels        = {
-		job       = "journal",
-		partition = "{{ metal_partition_id }}",
-		host      = "{{ inventory_hostname }}",
+		job = "systemd-journal",
 	}
 }
diff --git a/partition/roles/alloy/templates/snippets/syslog.alloy.j2 b/partition/roles/alloy/templates/snippets/syslog.alloy.j2
index 5be7068d1..0bfdf8a09 100644
--- a/partition/roles/alloy/templates/snippets/syslog.alloy.j2
+++ b/partition/roles/alloy/templates/snippets/syslog.alloy.j2
@@ -1,9 +1,7 @@
 local.file_match "syslog" {
 	path_targets = [{
-		__path__  = "/var/log/syslog",
-		job       = "syslog",
-		partition = "{{ metal_partition_id }}",
-		host      = "{{ inventory_hostname }}",
+		__path__ = "/var/log/syslog",
+		job      = "syslog",
 	}]
 }
 

From 3759237d77057bd70691dc18ba2b4a2ddcddf2a6 Mon Sep 17 00:00:00 2001
From: Matthias Hartmann <matthias.hartmann@x-cellent.com>
Date: Thu, 7 May 2026 17:21:33 +0200
Subject: [PATCH 11/41] feat: label for journal level filtering

---
 partition/roles/alloy/README.md               | 26 ++++++++++++++-----
 .../templates/snippets/journal-file.alloy.j2  |  5 ++++
 .../alloy/templates/snippets/journal.alloy.j2 |  5 ++++
 3 files changed, 29 insertions(+), 7 deletions(-)

diff --git a/partition/roles/alloy/README.md b/partition/roles/alloy/README.md
index 6ec9b1ac2..7f49e8cb5 100644
--- a/partition/roles/alloy/README.md
+++ b/partition/roles/alloy/README.md
@@ -21,13 +21,13 @@ The snippets are used in our default configs to provide a sensible out-of-the-bo
 
 If your needs are more custom (e.g. you have a non-standard log source, or want to use advanced Alloy features not covered by the existing snippets), you can follow the [Customizing the config](#customizing-the-config) section below to either add your own snippet or bypass the snippet system entirely with a custom raw config.
 
-| Snippet        | Description                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 |
-| -------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `alloy-meta`   | Meta-monitoring: forwards Alloy's own logs to Loki with `job=alloy` and `node_name` labels. Redundant when `journal` is enabled — journald already captures Alloy's stdout/stderr. Log level info and format are always configured via the base config regardless of this snippet.                                                                                                                                                                                                                                          |
-| `docker`       | Scrapes Docker container logs on the host via the Docker socket and forwards them to Loki. Assumes the `json-file` log driver (the default). If containers use the `journald` log driver instead, their logs are already captured by `journal` — enabling both snippets in that case will produce duplicate log entries in Loki.                                                                                                                                                                                            |
-| `journal`      | Scrapes the systemd journal via the journal API and forwards to Loki; relabels `unit` from `__journal__systemd_unit`. Automatically discovers the journal regardless of storage mode (volatile `/run/log/journal` or persistent `/var/log/journal`). On standard systemd hosts, journald captures syslog messages as well, so this snippet typically covers everything `syslog` would. Enabling both may produce duplicate log entries in Loki.                                                                             |
-| `journal-file` | Like `journal`, but reads from an explicit directory path (`alloy_journal_path`, default `/var/log/journal`) instead of the journal API. Without an explicit path, `journal` already auto-discovers both `/var/log/journal` and `/run/log/journal`, so prefer `journal` for new deployments. Use `journal-file` only when migrating from promtail and needing the `legacy_position` block to resume cursor state. Do not enable both `journal` and `journal-file` simultaneously — this will produce duplicate log entries. |
-| `syslog`       | Tails `/var/log/syslog` and forwards to Loki with `job=syslog`, `partition`, and `host` labels. Suited for hosts without journald (e.g. minimal switch OS images). On standard systemd hosts, prefer `journal` — enabling both may produce duplicate log entries in Loki.                                                                                                                                                                                                                                                   |
+| Snippet        | Description                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |
+| -------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `alloy-meta`   | Meta-monitoring: forwards Alloy's own logs to Loki with `job=alloy`. Redundant when `journal` is enabled — journald already captures Alloy's stdout/stderr. Log level and format are always configured via the base config regardless of this snippet.                                                                                                                                                                                                                                                                                                                  |
+| `docker`       | Scrapes Docker container logs on the host via the Docker socket and forwards them to Loki with `job=docker` and `container` labels. Assumes the `json-file` log driver (the default). If containers use the `journald` log driver instead, their logs are already captured by `journal` — enabling both snippets in that case will produce duplicate log entries in Loki.                                                                                                                                                                                               |
+| `journal`      | Scrapes the systemd journal via the journal API and forwards to Loki with `job=systemd-journal`, `unit` (from `_SYSTEMD_UNIT`), and `level` (from journal priority: `emerg`, `alert`, `crit`, `error`, `warning`, `notice`, `info`, `debug`). Automatically discovers the journal regardless of storage mode (volatile `/run/log/journal` or persistent `/var/log/journal`). On standard systemd hosts, journald captures syslog messages as well, so this snippet typically covers everything `syslog` would. Enabling both may produce duplicate log entries in Loki. |
+| `journal-file` | Like `journal`, but reads from an explicit directory path (`alloy_journal_path`, default `/var/log/journal`) instead of the journal API. Produces the same labels (`job=systemd-journal`, `unit`, `level`). Prefer `journal` for new deployments — use `journal-file` only when migrating from promtail and needing the `legacy_position` block to resume cursor state. Do not enable both `journal` and `journal-file` simultaneously — this will produce duplicate log entries.                                                                                       |
+| `syslog`       | Tails `/var/log/syslog` and forwards to Loki with `job=syslog`. Suited for hosts without journald (e.g. minimal switch OS images). On standard systemd hosts, prefer `journal` — enabling both may produce duplicate log entries in Loki.                                                                                                                                                                                                                                                                                                                               |
 
 ### Variables
 
@@ -52,6 +52,18 @@ If your needs are more custom (e.g. you have a non-standard log source, or want
 
 Alloy always exposes Prometheus metrics on `0.0.0.0:{{ alloy_port }}/metrics`, regardless of which snippets are enabled. Add the host targets to `prometheus_alloy_targets` in the prometheus role to scrape them.
 
+#### Labels
+
+All log entries carry `host` (set to `inventory_hostname`) and `partition` (set to `metal_partition_id`) as `external_labels` on `loki.write "default"`, so they are appended to every log entry regardless of snippet. Per-snippet labels are:
+
+| Snippet        | Labels                                 |
+| -------------- | -------------------------------------- |
+| `alloy-meta`   | `job=alloy`                            |
+| `docker`       | `job=docker`, `container`              |
+| `journal`      | `job=systemd-journal`, `unit`, `level` |
+| `journal-file` | `job=systemd-journal`, `unit`, `level` |
+| `syslog`       | `job=syslog`                           |
+
 #### Logs
 
 Alloy's own logs are captured in two ways depending on your snippet configuration:
diff --git a/partition/roles/alloy/templates/snippets/journal-file.alloy.j2 b/partition/roles/alloy/templates/snippets/journal-file.alloy.j2
index 6de426067..3c1f65d12 100644
--- a/partition/roles/alloy/templates/snippets/journal-file.alloy.j2
+++ b/partition/roles/alloy/templates/snippets/journal-file.alloy.j2
@@ -5,6 +5,11 @@ discovery.relabel "journal" {
 		source_labels = ["__journal__systemd_unit"]
 		target_label  = "unit"
 	}
+
+	rule {
+		source_labels = ["__journal_priority_keyword"]
+		target_label  = "level"
+	}
 }
 
 loki.source.journal "journal" {
diff --git a/partition/roles/alloy/templates/snippets/journal.alloy.j2 b/partition/roles/alloy/templates/snippets/journal.alloy.j2
index 886dd89f4..f2f8d2cfb 100644
--- a/partition/roles/alloy/templates/snippets/journal.alloy.j2
+++ b/partition/roles/alloy/templates/snippets/journal.alloy.j2
@@ -5,6 +5,11 @@ discovery.relabel "journal" {
 		source_labels = ["__journal__systemd_unit"]
 		target_label  = "unit"
 	}
+
+	rule {
+		source_labels = ["__journal_priority_keyword"]
+		target_label  = "level"
+	}
 }
 
 loki.source.journal "journal" {

From 5155e052ac57dba970ba68b37e3a303ff1930b8b Mon Sep 17 00:00:00 2001
From: Matthias Hartmann <matthias.hartmann@x-cellent.com>
Date: Tue, 12 May 2026 10:36:00 +0200
Subject: [PATCH 12/41] docs(alloy): improve cursor/WAL part

---
 partition/roles/alloy/README.md | 41 ++++++++++++++++++---------------
 1 file changed, 22 insertions(+), 19 deletions(-)

diff --git a/partition/roles/alloy/README.md b/partition/roles/alloy/README.md
index 7f49e8cb5..478b757ce 100644
--- a/partition/roles/alloy/README.md
+++ b/partition/roles/alloy/README.md
@@ -21,13 +21,13 @@ The snippets are used in our default configs to provide a sensible out-of-the-bo
 
 If your needs are more custom (e.g. you have a non-standard log source, or want to use advanced Alloy features not covered by the existing snippets), you can follow the [Customizing the config](#customizing-the-config) section below to either add your own snippet or bypass the snippet system entirely with a custom raw config.
 
-| Snippet        | Description                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |
-| -------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `alloy-meta`   | Meta-monitoring: forwards Alloy's own logs to Loki with `job=alloy`. Redundant when `journal` is enabled — journald already captures Alloy's stdout/stderr. Log level and format are always configured via the base config regardless of this snippet.                                                                                                                                                                                                                                                                                                                  |
-| `docker`       | Scrapes Docker container logs on the host via the Docker socket and forwards them to Loki with `job=docker` and `container` labels. Assumes the `json-file` log driver (the default). If containers use the `journald` log driver instead, their logs are already captured by `journal` — enabling both snippets in that case will produce duplicate log entries in Loki.                                                                                                                                                                                               |
-| `journal`      | Scrapes the systemd journal via the journal API and forwards to Loki with `job=systemd-journal`, `unit` (from `_SYSTEMD_UNIT`), and `level` (from journal priority: `emerg`, `alert`, `crit`, `error`, `warning`, `notice`, `info`, `debug`). Automatically discovers the journal regardless of storage mode (volatile `/run/log/journal` or persistent `/var/log/journal`). On standard systemd hosts, journald captures syslog messages as well, so this snippet typically covers everything `syslog` would. Enabling both may produce duplicate log entries in Loki. |
-| `journal-file` | Like `journal`, but reads from an explicit directory path (`alloy_journal_path`, default `/var/log/journal`) instead of the journal API. Produces the same labels (`job=systemd-journal`, `unit`, `level`). Prefer `journal` for new deployments — use `journal-file` only when migrating from promtail and needing the `legacy_position` block to resume cursor state. Do not enable both `journal` and `journal-file` simultaneously — this will produce duplicate log entries.                                                                                       |
-| `syslog`       | Tails `/var/log/syslog` and forwards to Loki with `job=syslog`. Suited for hosts without journald (e.g. minimal switch OS images). On standard systemd hosts, prefer `journal` — enabling both may produce duplicate log entries in Loki.                                                                                                                                                                                                                                                                                                                               |
+| Snippet        | Description                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         |
+| -------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `alloy-meta`   | Meta-monitoring: forwards Alloy's own logs to Loki with `job=alloy`. Redundant when `journal` is enabled — journald already captures Alloy's stdout/stderr. Log level and format are always configured via the base config regardless of this snippet.                                                                                                                                                                                                                                                                                                              |
+| `docker`       | Scrapes Docker container logs on the host via the Docker socket and forwards them to Loki with `job=docker` and `container` labels. Assumes the `json-file` log driver (the default). If containers use the `journald` log driver instead, their logs are already captured by `journal` — enabling both snippets in that case will produce duplicate log entries in Loki. Note: log entries not yet shipped can be lost if Alloy is down when a log rotation occurs and the rotated file is subsequently deleted.                                                   |
+| `journal`      | Scrapes the systemd journal via the journal API and forwards to Loki with `job=systemd-journal`, `unit`, and `level` labels. Automatically discovers the journal regardless of storage mode (volatile or persistent). On standard systemd hosts, journald captures syslog messages as well, so this snippet typically covers everything `syslog` would. Enabling both may produce duplicate log entries in Loki. Note: with a volatile journal (`/run/log/journal`), any entries not yet shipped are lost on reboot — use persistent journal storage to avoid this. |
+| `journal-file` | Like `journal`, but reads from an explicit directory path (`alloy_journal_path`, default `/var/log/journal`) instead of the journal API. Produces the same labels (`job=systemd-journal`, `unit`, `level`). Prefer `journal` for new deployments — use `journal-file` only when migrating from promtail and needing the `legacy_position` block to resume cursor state. Do not enable both `journal` and `journal-file` simultaneously — this will produce duplicate log entries.                                                                                   |
+| `syslog`       | Tails `/var/log/syslog` and forwards to Loki with `job=syslog`. Suited for hosts without journald (e.g. minimal switch OS images). On standard systemd hosts, prefer `journal` — enabling both may produce duplicate log entries in Loki.                                                                                                                                                                                                                                                                                                                           |
 
 ### Variables
 
@@ -73,24 +73,27 @@ Alloy's own logs are captured in two ways depending on your snippet configuratio
 
 Alloy's log level (`info`) and format (`logfmt`) are always configured via the base config, regardless of which snippets are enabled. The `alloy-meta` snippet only adds the Loki forwarding pipeline on top.
 
-### WAL and cursor persistence
+### Cursor/positions persistence
 
-Alloy stores its write-ahead log (WAL) — including cursor state for `loki.source.file` and `loki.source.journal` — under `--storage.path`, which is set to `/var/lib/alloy` (bind-mounted from the host). This survives container restarts, preventing duplicate log shipments.
-When migrating from promtail, the cursor state from promtail's positions file can be imported into Alloy's WAL on first start after migration to continue from where promtail left off. The exact mechanism depends on the snippet:
+`loki.source.file` and `loki.source.journal` track their read positions in small YAML positions files written under `--storage.path`, which is set to `/var/lib/alloy` (bind-mounted from the host). These files stay in the low-KB range regardless of log volume and survive container restarts, preventing duplicate log shipments.
 
-| Snippet        | Cursor mechanism                                   | Promtail migration path                                                      |
-| -------------- | -------------------------------------------------- | ---------------------------------------------------------------------------- |
-| `syslog`       | WAL via `loki.source.file`                         | `alloy_syslog_legacy_positions_file`                                         |
-| `journal`      | WAL via `loki.source.journal` (journal API)        | None — first start re-reads a small journal window once                      |
-| `journal-file` | WAL via `loki.source.journal` (explicit file path) | `alloy_journal_legacy_positions_file` + `alloy_journal_legacy_position_name` |
+When migrating from promtail, the cursor state from promtail's positions file can be imported on first start to continue from where promtail left off. The exact mechanism depends on the snippet:
 
-**`syslog`:** On first start after migration, Alloy reads `alloy_syslog_legacy_positions_file` (default: `/var/log/promtail-positions.yaml`) to continue tailing `/var/log/syslog` from where promtail left off. No log lines are re-shipped. After this Alloy tracks positions in its own WAL format. If the file does not exist on the host, this has no effect.
+| Snippet        | Cursor mechanism                                         | Promtail migration path                                                      |
+| -------------- | -------------------------------------------------------- | ---------------------------------------------------------------------------- |
+| `syslog`       | Positions file via `loki.source.file`                    | `alloy_syslog_legacy_positions_file`                                         |
+| `journal`      | Positions file via `loki.source.journal` (journal API)   | None — first start re-reads a small journal window once                      |
+| `journal-file` | Positions file via `loki.source.journal` (explicit path) | `alloy_journal_legacy_positions_file` + `alloy_journal_legacy_position_name` |
+
+**`syslog`:** On first start after migration, Alloy reads `alloy_syslog_legacy_positions_file` (default: `/var/log/promtail-positions.yaml`) to continue tailing `/var/log/syslog` from where promtail left off. No log lines are re-shipped. After this Alloy tracks positions in its own format. If the file does not exist on the host, this has no effect.
 
 **`journal-file`:** On first start after migration, Alloy reads the legacy positions file via the `legacy_position` block using `alloy_journal_legacy_positions_file` (default: `/var/log/promtail-positions.yaml`) and `alloy_journal_legacy_position_name` (default: `journal`). The name must match the `job_name` of the journal scrape config in your old promtail config. If the file does not exist on the host, this has no effect.
 
-**`journal`:** No promtail-compatible migration path. The first start after migration re-reads a small window of the journal — this is a one-time event and subsequent restarts are safe once the WAL is populated.
+**`journal`:** No promtail-compatible migration path. The first start after migration re-reads a small window of the journal — this is a one-time event and subsequent restarts are safe once the cursor is written.
+
+Once migration is complete, the old promtail positions files can be cleaned up from hosts — Alloy will already be tracking positions in its own format from that point.
 
-Once migration is complete, the old promtail positions files can be cleaned up from hosts — Alloy will already be tracking positions in its own WAL format from that point.
+> **Note:** Alloy also has an experimental [WAL feature for `loki.write`](https://grafana.com/docs/alloy/latest/reference/components/loki/loki.write/#wal) that buffers outgoing log entries on disk when Loki is temporarily unavailable. It is disabled by default and not used by this role. It is not needed for `syslog`, `journal-file`, or `journal` with persistent storage, as Alloy resumes from the saved cursor after a restart. For `docker`, it reduces the risk of losing entries during log rotation. For `journal` with volatile storage, it provides no benefit — entries are lost on reboot regardless since the journal itself is wiped.
 
 ## Customizing the config
 
@@ -135,7 +138,7 @@ Alloy allows reproducing the same log forwarding behavior as promtail via the [a
 
 3. **Choose snippets** that match your legacy `promtail_scrape_configs` — see [Available snippets](#available-snippets). If the available snippets are close enough to your old setup, enable the relevant ones and adjust with the provided variables. Otherwise, continue with step 4.
 
-   If you use `syslog` or `journal-file`, set the corresponding legacy positions file variable so Alloy resumes from where promtail left off — see [WAL and cursor persistence](#wal-and-cursor-persistence) for details.
+   If you use `syslog` or `journal-file`, set the corresponding legacy positions file variable so Alloy resumes from where promtail left off — see [Cursor/positions persistence](#cursorpositions-persistence) for details.
 
    If your environment deploys Alloy on different host groups with different scrape needs (e.g. leaf switches vs. management servers), set `alloy_config_snippets` in per-group inventory files:
 

From f5758a91e18c7d941703d9810dcf4d14fe06b784 Mon Sep 17 00:00:00 2001
From: Matthias Hartmann <matthias.hartmann@x-cellent.com>
Date: Tue, 12 May 2026 10:41:16 +0200
Subject: [PATCH 13/41] docs(promtail): add important admonition for promtail
 deprecation

---
 partition/roles/promtail/README.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/partition/roles/promtail/README.md b/partition/roles/promtail/README.md
index cbccf7b09..2726a8a90 100644
--- a/partition/roles/promtail/README.md
+++ b/partition/roles/promtail/README.md
@@ -1,6 +1,7 @@
 # promtail
 
-> **Deprecated:** This role is superseded by the `alloy` role. New deployments should use `metal-roles/partition/roles/alloy` instead. This role is kept for migration purposes only and may be removed in a future release. See the [alloy role README](../alloy/README.md) for migration instructions.
+> [!IMPORTANT]
+> This role is deprecated and superseded by the `alloy` role. New deployments should use `metal-roles/partition/roles/alloy` instead. This role is kept for migration purposes only and may be removed in a future release. See the [alloy role README](../alloy/README.md) for migration instructions.
 
 Deploys promtail in a systemd-managed Docker container.
 

From 73f7b6993030822b7a2dfe69ea9b53042dc837c4 Mon Sep 17 00:00:00 2001
From: Matthias Hartmann <matthias.hartmann@x-cellent.com>
Date: Tue, 12 May 2026 16:39:30 +0200
Subject: [PATCH 14/41] feat: add toggles and docs for different migration
 scenarios

---
 partition/roles/alloy/README.md               | 61 +++++++++++--------
 partition/roles/alloy/defaults/main.yaml      | 26 ++++++--
 partition/roles/alloy/handlers/main.yaml      |  1 +
 partition/roles/alloy/tasks/main.yaml         | 20 ++++--
 .../templates/snippets/journal-file.alloy.j2  |  8 ++-
 .../alloy/templates/snippets/syslog.alloy.j2  |  7 ++-
 partition/roles/promtail/README.md            | 17 +++---
 partition/roles/promtail/defaults/main.yaml   |  5 ++
 partition/roles/promtail/handlers/main.yaml   |  1 +
 partition/roles/promtail/tasks/main.yaml      |  4 ++
 10 files changed, 104 insertions(+), 46 deletions(-)

diff --git a/partition/roles/alloy/README.md b/partition/roles/alloy/README.md
index 478b757ce..bb8613fc7 100644
--- a/partition/roles/alloy/README.md
+++ b/partition/roles/alloy/README.md
@@ -13,7 +13,7 @@ The Alloy configuration is assembled from a base template plus a list of opt-in
 
 At deploy time each enabled snippet is rendered into `{{ alloy_config_host_dir }}/conf.d/` and concatenated into a single `config.alloy` file.
 
-The snippets can be configured with environment variables defined in the [Variables](#variables) section below. This allows you to enable only the features you need without having to maintain a full custom config.
+The snippets can be configured with variables defined in the [Variables](#variables) section below. This allows you to enable only the features you need without having to maintain a full custom config.
 
 ### Available snippets
 
@@ -31,20 +31,23 @@ If your needs are more custom (e.g. you have a non-standard log source, or want
 
 ### Variables
 
-| Name                                | Mandatory | Description                                                                                                                                                                                             |
-| ----------------------------------- | --------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| alloy_config_host_dir               |           | The location of the alloy config on the host (default: `/etc/alloy`)                                                                                                                                    |
-| alloy_image_name                    | yes       | Image name of alloy                                                                                                                                                                                     |
-| alloy_image_tag                     | yes       | Image tag of alloy                                                                                                                                                                                      |
-| alloy_loki_write_endpoints          | yes       | List of Loki push endpoints. Each entry: `{url, remote_timeout?: <duration>, basic_auth?: {username, password}}`                                                                                        |
-| alloy_docker_log_driver             |           | Docker log driver for the alloy container (default: `json-file`)                                                                                                                                        |
-| alloy_config_snippets               |           | List of snippet names to enable (default: `[]`)                                                                                                                                                         |
-| alloy_port                          |           | Port for Alloy metrics and HTTP API (default: `12345`)                                                                                                                                                  |
-| alloy_syslog_legacy_positions_file  |           | Path to the legacy promtail positions file (default: `/var/log/promtail-positions.yaml`). Used by `syslog` on first start after migration.                                                              |
-| alloy_journal_legacy_positions_file |           | Path to the legacy promtail positions file (default: `/var/log/promtail-positions.yaml`). Used by `journal-file` on first start after migration.                                                        |
-| alloy_journal_path                  |           | Path to the persistent journal directory used by the `journal-file` snippet (default: `/var/log/journal`).                                                                                              |
-| alloy_journal_legacy_position_name  |           | Job name from the old promtail journal scrape_config, used by `journal-file` to resume from the legacy positions file (default: `journal`). Required when `alloy_journal_legacy_positions_file` is set. |
-| alloy_config_raw                    |           | Full Alloy River config as a string. When set, bypasses snippet assembly entirely — `alloy_loki_write_endpoints` and `alloy_config_snippets` are ignored.                                               |
+| Name                                | Mandatory                              | Default                            | Description                                                                                                                                                                                                                                                                               |
+| ----------------------------------- | -------------------------------------- | ---------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| alloy_enabled                       |                                        | `false`                            | Deploy and start the Alloy service. Config files are always written regardless of this flag. Set to `false` to stage the config without starting the service.                                                                                                                             |
+| alloy_config_host_dir               |                                        | `/etc/alloy`                       | The location of the alloy config on the host                                                                                                                                                                                                                                              |
+| alloy_image_name                    | yes                                    |                                    | Image name of alloy                                                                                                                                                                                                                                                                       |
+| alloy_image_tag                     | yes                                    |                                    | Image tag of alloy                                                                                                                                                                                                                                                                        |
+| alloy_loki_write_endpoints          | yes (unless `alloy_config_raw` is set) |                                    | List of Loki push endpoints. Each entry: `{url, remote_timeout?: <duration>, basic_auth?: {username, password}}`                                                                                                                                                                          |
+| alloy_docker_log_driver             |                                        | `json-file`                        | Docker log driver for the alloy container                                                                                                                                                                                                                                                 |
+| alloy_config_snippets               |                                        | `[]`                               | List of snippet names to enable                                                                                                                                                                                                                                                           |
+| alloy_port                          |                                        | `12345`                            | Port for Alloy metrics and HTTP API                                                                                                                                                                                                                                                       |
+| alloy_migrate_from_promtail         |                                        | `false`                            | Enable migration mode: imports cursor state from the legacy promtail positions file on first start. Set to `true` when migrating from promtail; leave `false` for fresh deployments. Without this, Alloy starts from the current tail and previously shipped log data will be re-shipped. |
+| alloy_migrate_stop_promtail         |                                        | `false`                            | Stop and disable the promtail systemd service as part of this role run. Does not remove promtail files or images.                                                                                                                                                                         |
+| alloy_syslog_legacy_positions_file  |                                        | `/var/log/promtail-positions.yaml` | Path to the legacy promtail positions file. Used by `syslog` when `alloy_migrate_from_promtail` is `true`.                                                                                                                                                                                |
+| alloy_journal_path                  |                                        | `/var/log/journal`                 | Path to the persistent journal directory used by the `journal-file` snippet                                                                                                                                                                                                               |
+| alloy_journal_legacy_positions_file |                                        | `/var/log/promtail-positions.yaml` | Path to the legacy promtail positions file. Used by `journal-file` when `alloy_migrate_from_promtail` is `true`.                                                                                                                                                                          |
+| alloy_journal_legacy_position_name  | yes (migration)                        |                                    | Job name from the old promtail journal scrape_config. Must match `job_name` in your old promtail config. Required when `alloy_migrate_from_promtail` is `true` and `journal-file` is used.                                                                                                |
+| alloy_config_raw                    |                                        |                                    | Full Alloy River config as a string. When set, bypasses snippet assembly entirely — `alloy_loki_write_endpoints` and `alloy_config_snippets` are ignored.                                                                                                                                 |
 
 ### Meta-monitoring (for Alloy itself)
 
@@ -77,7 +80,7 @@ Alloy's log level (`info`) and format (`logfmt`) are always configured via the b
 
 `loki.source.file` and `loki.source.journal` track their read positions in small YAML positions files written under `--storage.path`, which is set to `/var/lib/alloy` (bind-mounted from the host). These files stay in the low-KB range regardless of log volume and survive container restarts, preventing duplicate log shipments.
 
-When migrating from promtail, the cursor state from promtail's positions file can be imported on first start to continue from where promtail left off. The exact mechanism depends on the snippet:
+When migrating from promtail, set `alloy_migrate_from_promtail: true` to import cursor state from promtail's positions file on first start and continue from where promtail left off. The exact mechanism depends on the snippet:
 
 | Snippet        | Cursor mechanism                                         | Promtail migration path                                                      |
 | -------------- | -------------------------------------------------------- | ---------------------------------------------------------------------------- |
@@ -87,7 +90,7 @@ When migrating from promtail, the cursor state from promtail's positions file ca
 
 **`syslog`:** On first start after migration, Alloy reads `alloy_syslog_legacy_positions_file` (default: `/var/log/promtail-positions.yaml`) to continue tailing `/var/log/syslog` from where promtail left off. No log lines are re-shipped. After this Alloy tracks positions in its own format. If the file does not exist on the host, this has no effect.
 
-**`journal-file`:** On first start after migration, Alloy reads the legacy positions file via the `legacy_position` block using `alloy_journal_legacy_positions_file` (default: `/var/log/promtail-positions.yaml`) and `alloy_journal_legacy_position_name` (default: `journal`). The name must match the `job_name` of the journal scrape config in your old promtail config. If the file does not exist on the host, this has no effect.
+**`journal-file`:** On first start after migration, Alloy reads the legacy positions file via the `legacy_position` block using `alloy_journal_legacy_positions_file` and `alloy_journal_legacy_position_name`. The name must match the `job_name` of the journal scrape config in your old promtail config. If the file does not exist on the host, this has no effect.
 
 **`journal`:** No promtail-compatible migration path. The first start after migration re-reads a small window of the journal — this is a one-time event and subsequent restarts are safe once the cursor is written.
 
@@ -121,11 +124,18 @@ Contributions of new snippets are welcome — since all snippets are opt-in via
 
 Alloy allows reproducing the same log forwarding behavior as promtail via the [available snippets](#available-snippets), but it is a different tool with a different internal data model. Label names, pipeline stages, and how metrics are exposed might differ from promtail. **Review your existing Loki dashboards and alerting rules after the migration and adapt them where necessary.**
 
-**Recommended approach — parallel run:** Deploy Alloy alongside the existing promtail installation first. Both will ship logs to Loki simultaneously, so expect duplicate log entries during this transition window. Once you have verified that logs arrive in Loki with the correct labels and dashboards show data correctly, remove promtail.
+Use the inventory flags below to control the deployment based on your situation:
 
-1. **Add the `alloy` role** to your playbook alongside `promtail` for a parallel run, or replace it directly if you prefer a hard cut-over.
+| Scenario                             | `alloy_enabled` | `alloy_migrate_from_promtail` | `alloy_migrate_stop_promtail` | `promtail_enabled` | Notes                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      |
+| ------------------------------------ | --------------- | ----------------------------- | ----------------------------- | ------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| **Fresh deployment**                 | `true`          | `false`                       | `false`                       | `false`            | No prior promtail. Alloy starts from the current log tail.                                                                                                                                                                                                                                                                                                                                                                                                                                                                 |
+| **Existing promtail — parallel run** | `true`          | `true`                        | `false`                       | `true`             | Both services ship logs simultaneously; Alloy resumes from promtail's cursor. Expect duplicate log entries during the overlap. Use to verify Alloy in production before cutting over.                                                                                                                                                                                                                                                                                                                                      |
+| **Existing promtail — cutover**      | `true`          | `true`                        | `true`                        | `false`            | Alloy starts resuming from promtail's cursor; promtail service is stopped and disabled. Promtail files and container are left in place. There is a brief gap between promtail stopping and Alloy starting — entries written in this window are not shipped immediately but are safe as long as the log source persists across the gap (journal file, syslog on disk). With a volatile journal this is unlikely to matter in practice since the system stays running, but a reboot in this window would lose those entries. |
+| **Stage config (dry run)**           | `false`         | `true` or `false`             | `false`                       | `true`             | Config files are written but the Alloy service is not started. Promtail keeps running unaffected. Setting `alloy_migrate_from_promtail: true` here is safe and recommended — the migration config (including `legacy_position`) is written to the leaf and inspectable before Alloy ever starts.                                                                                                                                                                                                                           |
 
-2. **Configure the Loki endpoint.** Set `alloy_loki_write_endpoints`:
+1. **Add the `alloy` role** to your playbook. Keep `promtail` alongside it for a gradual migration, or remove it for a hard cut-over.
+
+2. **Configure the Loki endpoint.** Set `alloy_loki_write_endpoints` in your inventory:
 
    ```yaml
    alloy_loki_write_endpoints:
@@ -138,7 +148,8 @@ Alloy allows reproducing the same log forwarding behavior as promtail via the [a
 
 3. **Choose snippets** that match your legacy `promtail_scrape_configs` — see [Available snippets](#available-snippets). If the available snippets are close enough to your old setup, enable the relevant ones and adjust with the provided variables. Otherwise, continue with step 4.
 
-   If you use `syslog` or `journal-file`, set the corresponding legacy positions file variable so Alloy resumes from where promtail left off — see [Cursor/positions persistence](#cursorpositions-persistence) for details.
+   Set `alloy_migrate_from_promtail: true` to import cursor state from the legacy promtail positions file so Alloy resumes from where promtail left off. See [Cursor/positions persistence](#cursorpositions-persistence) for per-snippet details and the relevant variables.
+   **Without this, larger amounts of previously shipped entries may be re-shipped.**
 
    If your environment deploys Alloy on different host groups with different scrape needs (e.g. leaf switches vs. management servers), set `alloy_config_snippets` in per-group inventory files:
 
@@ -168,11 +179,13 @@ Alloy allows reproducing the same log forwarding behavior as promtail via the [a
 
 6. **Verify** that logs arrive in Loki with the correct labels. Check that existing dashboards and alerts still work as expected and adapt them for any label or metric name changes.
 
-7. **Remove the promtail container** from the migrated hosts once verified:
+7. **Cut over from promtail** _(parallel run only)_. Set `alloy_migrate_stop_promtail: true` and `promtail_enabled: false` in your inventory and re-run the playbook — the role will stop and disable the promtail systemd service. You can also set `alloy_migrate_from_promtail: false` at this point since the cursor state has already been imported on first start.
+
+   When you are ready, clean up the stopped promtail container and its config:
 
    ```bash
-   docker rm -f promtail
+   docker rm promtail
    rm -rf /etc/promtail
    ```
 
-   The `promtail` role remains in this repo (deprecated) and does not need to be removed from your playbook — simply stop including it for migrated environments.
+   The `promtail` role is deprecated and will be removed in a future release. Once all environments are migrated, remove it from your playbook.
diff --git a/partition/roles/alloy/defaults/main.yaml b/partition/roles/alloy/defaults/main.yaml
index b864ca163..9ffb56531 100644
--- a/partition/roles/alloy/defaults/main.yaml
+++ b/partition/roles/alloy/defaults/main.yaml
@@ -1,9 +1,13 @@
 ---
+# Set to true to deploy and start the Alloy service.
+# Set to false to skip service deployment — useful when alloy is being staged or promtail is still running.
+alloy_enabled: false
+
 alloy_config_host_dir: "/etc/alloy"
 alloy_docker_log_driver: json-file
 
 # List of Loki push endpoints used by the base loki.write component.
-# Each entry: {url: <push-url>, basic_auth?: {username, password}}
+# Each entry: {url: <push-url>, remote_timeout?: <duration>, basic_auth?: {username, password}}
 # alloy_loki_write_endpoints:
 #   - url: http://loki.example.com:8080/loki/api/v1/push
 
@@ -19,9 +23,20 @@ alloy_config_snippets: []
 # Port to listen for metrics and HTTP API traffic on
 alloy_port: 12345
 
+# Enable migration mode: imports cursor state from the legacy promtail positions file
+# on first start so Alloy resumes from where promtail left off. Set to true when
+# migrating from promtail; leave false for fresh deployments.
+alloy_migrate_from_promtail: false
+
+# Stop and disable the promtail systemd service as part of this role run.
+# Use together with alloy_migrate_from_promtail: true for an immediate cutover,
+# or set independently after a successful parallel run to remove promtail.
+# Does not remove promtail files or images — cleanup is left to the operator.
+alloy_migrate_stop_promtail: false
+
 # Path to the legacy promtail positions file used by the syslog snippet on first start
 # after migration to continue tailing /var/log/syslog from where promtail left off.
-# Has no effect if the file does not exist on the host.
+# Only active when alloy_migrate_from_promtail is true.
 alloy_syslog_legacy_positions_file: "/var/log/promtail-positions.yaml"
 
 # Path to the persistent journal directory used by the journal-file snippet.
@@ -29,10 +44,11 @@ alloy_journal_path: "/var/log/journal"
 
 # Path to the legacy promtail positions file used by the journal-file snippet on first
 # start after migration to resume from where promtail left off.
-# Has no effect if the file does not exist on the host.
+# Only active when alloy_migrate_from_promtail is true.
 alloy_journal_legacy_positions_file: "/var/log/promtail-positions.yaml"
 
 # Job name used in the legacy promtail positions file for the journal scrape config.
 # Must match the job_name of the journal scrape_config in your old promtail config.
-# Required when alloy_journal_legacy_positions_file is set. Used by the journal-file snippet for migration only.
-alloy_journal_legacy_position_name: "journal"
+# Required when alloy_migrate_from_promtail is true and the journal-file snippet is used.
+# No default — must be set explicitly.
+# alloy_journal_legacy_position_name:
diff --git a/partition/roles/alloy/handlers/main.yaml b/partition/roles/alloy/handlers/main.yaml
index 56a671558..11ac110dd 100644
--- a/partition/roles/alloy/handlers/main.yaml
+++ b/partition/roles/alloy/handlers/main.yaml
@@ -4,3 +4,4 @@
     name: alloy
     enabled: true
     state: restarted
+  when: alloy_enabled | default(false)
diff --git a/partition/roles/alloy/tasks/main.yaml b/partition/roles/alloy/tasks/main.yaml
index 06953d670..77e3b8ed9 100644
--- a/partition/roles/alloy/tasks/main.yaml
+++ b/partition/roles/alloy/tasks/main.yaml
@@ -11,13 +11,15 @@
       - alloy_image_name is defined
       - alloy_config_raw is defined or (alloy_loki_write_endpoints is defined and (alloy_loki_write_endpoints | length > 0))
 
-- name: Check journal-file legacy_position variables are either both set or both absent
+- name: Check alloy_journal_legacy_position_name is set when migrating journal-file from promtail
   assert:
-    fail_msg: "alloy_journal_legacy_positions_file and alloy_journal_legacy_position_name must either both be set or both be absent (alloy_journal_legacy_position_name must match the job_name of the journal scrape_config in your old promtail config)"
+    fail_msg: "alloy_journal_legacy_position_name must be set — it must match the job_name of the journal scrape_config in your old promtail config"
     quiet: yes
     that:
-      - (alloy_journal_legacy_positions_file is defined) == (alloy_journal_legacy_position_name is defined)
-  when: "'journal-file' in alloy_config_snippets"
+      - alloy_journal_legacy_position_name is defined
+  when:
+    - "'journal-file' in alloy_config_snippets"
+    - alloy_migrate_from_promtail | default(false)
 
 - name: Check journal and journal-file are not both enabled
   assert:
@@ -26,6 +28,14 @@
     that:
       - not ('journal' in alloy_config_snippets and 'journal-file' in alloy_config_snippets)
 
+- name: Stop and disable promtail service
+  systemd:
+    name: promtail
+    state: stopped
+    enabled: false
+  when: alloy_migrate_stop_promtail | default(false)
+  ignore_errors: true  # service may not exist on hosts that never had promtail
+
 - name: Create alloy config directory
   file:
     path: "{{ alloy_config_host_dir }}"
@@ -79,6 +89,7 @@
     owner: "nobody"
     group: "nogroup"
     mode: "0755"
+  when: alloy_enabled | default(false)
 
 - name: Deploy alloy service
   include_role:
@@ -103,3 +114,4 @@
       - --storage.path=/var/lib/alloy
       - /etc/alloy/config.alloy
     systemd_docker_log_driver: "{{ alloy_docker_log_driver }}"
+  when: alloy_enabled | default(false)
diff --git a/partition/roles/alloy/templates/snippets/journal-file.alloy.j2 b/partition/roles/alloy/templates/snippets/journal-file.alloy.j2
index 3c1f65d12..2c439ee6d 100644
--- a/partition/roles/alloy/templates/snippets/journal-file.alloy.j2
+++ b/partition/roles/alloy/templates/snippets/journal-file.alloy.j2
@@ -19,12 +19,14 @@ loki.source.journal "journal" {
 	labels        = {
 		job = "systemd-journal",
 	}
-
-	// read once on first start after migration to resume from where promtail left off;
-	// ignored if the file does not exist. alloy then tracks the cursor in its own wal.
+{% if alloy_migrate_from_promtail | default(false) %}
+	// migration only: resumes reading from where promtail left off on first start.
+	// ignored if the file does not exist on the host.
+	// once alloy has written its own positions file, this can be removed.
 	// https://grafana.com/docs/alloy/latest/reference/components/loki/loki.source.journal/#legacy_position
 	legacy_position {
 		file = "{{ alloy_journal_legacy_positions_file }}"
 		name = "{{ alloy_journal_legacy_position_name }}"
 	}
+{% endif %}
 }
diff --git a/partition/roles/alloy/templates/snippets/syslog.alloy.j2 b/partition/roles/alloy/templates/snippets/syslog.alloy.j2
index 0bfdf8a09..775c25dd2 100644
--- a/partition/roles/alloy/templates/snippets/syslog.alloy.j2
+++ b/partition/roles/alloy/templates/snippets/syslog.alloy.j2
@@ -8,8 +8,11 @@ local.file_match "syslog" {
 loki.source.file "syslog" {
 	targets               = local.file_match.syslog.targets
 	forward_to            = [loki.write.default.receiver]
-	// read once on first start after migration to resume from where promtail left off;
-	// ignored if the file does not exist. alloy then tracks positions in its own wal.
+{% if alloy_migrate_from_promtail | default(false) %}
+	// migration only: resumes tailing from where promtail left off on first start.
+	// ignored if the file does not exist on the host.
+	// once alloy has written its own positions file, this can be removed.
 	// https://grafana.com/docs/alloy/latest/reference/components/loki/loki.source.file/#arguments
 	legacy_positions_file = "{{ alloy_syslog_legacy_positions_file }}"
+{% endif %}
 }
diff --git a/partition/roles/promtail/README.md b/partition/roles/promtail/README.md
index 2726a8a90..209a7724e 100644
--- a/partition/roles/promtail/README.md
+++ b/partition/roles/promtail/README.md
@@ -7,11 +7,12 @@ Deploys promtail in a systemd-managed Docker container.
 
 ## Variables
 
-| Name                       | Mandatory | Description                                                                                                          |
-| -------------------------- | --------- | -------------------------------------------------------------------------------------------------------------------- |
-| promtail_config_host_dir   |           | The location of the promtail config                                                                                  |
-| promtail_image_name        | yes       | Image version of the promtail                                                                                        |
-| promtail_image_tag         | yes       | Image tag of the promtail                                                                                            |
-| promtail_clients           | yes       | A list of clients for promtail, see <https://grafana.com/docs/loki/latest/send-data/promtail/configuration/#clients> |
-| promtail_scrape_configs    | yes       | A list containing the scrape configs                                                                                 |
-| promtail_docker_log_driver |           | Indicates where to write the docker logs to                                                                          |
+| Name                       | Mandatory | Description                                                                                                                                                                                                                                                                 |
+| -------------------------- | --------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| promtail_enabled           |           | Deploy and start the promtail service (default: `true`). Set to `false` during alloy migration to prevent this role from re-deploying promtail after `alloy_migrate_stop_promtail` has stopped it. Once migration is complete, remove this role from the playbook entirely. |
+| promtail_config_host_dir   |           | The location of the promtail config                                                                                                                                                                                                                                         |
+| promtail_image_name        | yes       | Image version of the promtail                                                                                                                                                                                                                                               |
+| promtail_image_tag         | yes       | Image tag of the promtail                                                                                                                                                                                                                                                   |
+| promtail_clients           | yes       | A list of clients for promtail, see <https://grafana.com/docs/loki/latest/send-data/promtail/configuration/#clients>                                                                                                                                                        |
+| promtail_scrape_configs    | yes       | A list containing the scrape configs                                                                                                                                                                                                                                        |
+| promtail_docker_log_driver |           | Indicates where to write the docker logs to                                                                                                                                                                                                                                 |
diff --git a/partition/roles/promtail/defaults/main.yaml b/partition/roles/promtail/defaults/main.yaml
index 4c5dd5b9e..9f80f5993 100644
--- a/partition/roles/promtail/defaults/main.yaml
+++ b/partition/roles/promtail/defaults/main.yaml
@@ -1,4 +1,9 @@
 ---
+# Set to false during alloy migration to prevent this role from re-deploying promtail
+# after alloy_migrate_stop_promtail has stopped it. Once migration is complete, remove
+# this role from the playbook entirely.
+promtail_enabled: true
+
 promtail_config_host_dir: "/etc/promtail"
 
 promtail_clients: []
diff --git a/partition/roles/promtail/handlers/main.yaml b/partition/roles/promtail/handlers/main.yaml
index 9d28fb13a..5e253bfd1 100644
--- a/partition/roles/promtail/handlers/main.yaml
+++ b/partition/roles/promtail/handlers/main.yaml
@@ -4,3 +4,4 @@
     name: promtail
     enabled: true
     state: restarted
+  when: promtail_enabled | default(true)
diff --git a/partition/roles/promtail/tasks/main.yaml b/partition/roles/promtail/tasks/main.yaml
index a1e7caf28..5f95c214f 100644
--- a/partition/roles/promtail/tasks/main.yaml
+++ b/partition/roles/promtail/tasks/main.yaml
@@ -11,6 +11,7 @@
       - promtail_image_tag is defined
       - promtail_image_name is defined
       - promtail_scrape_configs is defined and (promtail_scrape_configs|length>0)
+  when: promtail_enabled | default(true)
 
 - name: Create promtail config directory
   file:
@@ -19,12 +20,14 @@
     owner: "nobody"
     group: "nogroup"
     mode: "0755"
+  when: promtail_enabled | default(true)
 
 - name: Template promtail configuration file
   template:
     src: promtail.yaml.j2
     dest: "{{ promtail_config_host_dir }}/promtail.yaml"
   notify: restart promtail
+  when: promtail_enabled | default(true)
 
 - name: Deploy promtail service
   include_role:
@@ -43,3 +46,4 @@
     systemd_docker_command:
       - -config.file=/etc/promtail/promtail.yaml
     systemd_docker_log_driver: "{{ promtail_docker_log_driver }}"
+  when: promtail_enabled | default(true)

From 1b1d28fb360ca2fb18f9627f69dd2cdd1fc76ab9 Mon Sep 17 00:00:00 2001
From: Matthias Hartmann <matthias.hartmann@x-cellent.com>
Date: Tue, 12 May 2026 16:41:13 +0200
Subject: [PATCH 15/41] chore(metrics): bring back prometheus promtail job as
 it is optional

---
 partition/roles/alloy/README.md                     | 11 ++++++++++-
 .../roles/monitoring/prometheus/defaults/main.yaml  |  1 +
 .../roles/monitoring/prometheus/tasks/main.yaml     | 13 +++++++++++++
 .../prometheus/templates/prometheus.yaml.j2         |  9 +++++++++
 4 files changed, 33 insertions(+), 1 deletion(-)

diff --git a/partition/roles/alloy/README.md b/partition/roles/alloy/README.md
index bb8613fc7..55f1f2d6f 100644
--- a/partition/roles/alloy/README.md
+++ b/partition/roles/alloy/README.md
@@ -53,7 +53,16 @@ If your needs are more custom (e.g. you have a non-standard log source, or want
 
 #### Metrics
 
-Alloy always exposes Prometheus metrics on `0.0.0.0:{{ alloy_port }}/metrics`, regardless of which snippets are enabled. Add the host targets to `prometheus_alloy_targets` in the prometheus role to scrape them.
+Alloy always exposes Prometheus metrics on `0.0.0.0:{{ alloy_port }}/metrics`, regardless of which snippets are enabled. Add the host targets to `prometheus_alloy_targets` in your inventory to have the prometheus role scrape them.
+
+Scraped metrics will carry the following labels (set by the prometheus role, not the alloy role):
+
+| Label | Value |
+| --------- | ------------------------------------------------- |
+| `job` | `alloy` |
+| `instance` | hostname of the target (port stripped) |
+| `partition` | `metal_partition_id` (from Prometheus `external_labels`) |
+| `replica` | `inventory_hostname` of the Prometheus host (from Prometheus `external_labels`) |
 
 #### Labels
 
diff --git a/partition/roles/monitoring/prometheus/defaults/main.yaml b/partition/roles/monitoring/prometheus/defaults/main.yaml
index 1a0c992e8..ed3c3e864 100644
--- a/partition/roles/monitoring/prometheus/defaults/main.yaml
+++ b/partition/roles/monitoring/prometheus/defaults/main.yaml
@@ -21,6 +21,7 @@ prometheus_remote_write: []
 prometheus_frr_exporter_targets: []
 prometheus_metal_core_targets: []
 prometheus_node_exporter_targets: []
+prometheus_promtail_targets: []
 prometheus_alloy_targets: []
 prometheus_ping_targets: []
 prometheus_ipmi_exporter_targets: []
diff --git a/partition/roles/monitoring/prometheus/tasks/main.yaml b/partition/roles/monitoring/prometheus/tasks/main.yaml
index 0ba1418aa..85d2cd980 100644
--- a/partition/roles/monitoring/prometheus/tasks/main.yaml
+++ b/partition/roles/monitoring/prometheus/tasks/main.yaml
@@ -116,6 +116,19 @@
   register: _result
   when: prometheus_ipmi_exporter_targets
 
+- name: generate prometheus_promtail_sd
+  set_fact:
+    prometheus_promtail_sd:
+      - labels:
+          job: promtail
+        targets: "{{ prometheus_promtail_targets }}"
+
+- name: create promtail.yaml
+  copy:
+    content: "{{ prometheus_promtail_sd | to_nice_yaml(indent=2) }}"
+    dest: "{{ prometheus_config_host_dir }}/file_sd/promtail.yaml"
+  notify: restart prometheus
+
 - name: generate prometheus_alloy_sd
   set_fact:
     prometheus_alloy_sd:
diff --git a/partition/roles/monitoring/prometheus/templates/prometheus.yaml.j2 b/partition/roles/monitoring/prometheus/templates/prometheus.yaml.j2
index 40df06927..d0f0b1517 100644
--- a/partition/roles/monitoring/prometheus/templates/prometheus.yaml.j2
+++ b/partition/roles/monitoring/prometheus/templates/prometheus.yaml.j2
@@ -79,6 +79,15 @@ scrape_configs:
       regex: (.*):\d+
       target_label: instance
 
+- job_name: promtail
+  file_sd_configs:
+  - files:
+    - '/etc/prometheus/file_sd/promtail.yaml'
+  relabel_configs:
+    - source_labels: [__address__]
+      regex: (.*):\d+
+      target_label: instance
+
 - job_name: alloy
   file_sd_configs:
   - files:

From dccafd47fccaffb0a31da91ae5e50678ffa17e59 Mon Sep 17 00:00:00 2001
From: Matthias Hartmann <matthias.hartmann@x-cellent.com>
Date: Wed, 13 May 2026 15:11:53 +0200
Subject: [PATCH 16/41] chore: provide promtail_migrate_stop and
 promtail_migrate_cleanup variables for automated cleanup

---
 partition/roles/alloy/README.md             | 36 +++++++++------------
 partition/roles/alloy/defaults/main.yaml    |  6 ----
 partition/roles/alloy/tasks/main.yaml       |  8 -----
 partition/roles/promtail/README.md          | 20 ++++++------
 partition/roles/promtail/defaults/main.yaml | 12 ++++++-
 partition/roles/promtail/handlers/main.yaml |  4 +++
 partition/roles/promtail/tasks/main.yaml    | 34 +++++++++++++++++++
 7 files changed, 75 insertions(+), 45 deletions(-)

diff --git a/partition/roles/alloy/README.md b/partition/roles/alloy/README.md
index 55f1f2d6f..db693f4d4 100644
--- a/partition/roles/alloy/README.md
+++ b/partition/roles/alloy/README.md
@@ -24,7 +24,7 @@ If your needs are more custom (e.g. you have a non-standard log source, or want
 | Snippet        | Description                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         |
 | -------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
 | `alloy-meta`   | Meta-monitoring: forwards Alloy's own logs to Loki with `job=alloy`. Redundant when `journal` is enabled — journald already captures Alloy's stdout/stderr. Log level and format are always configured via the base config regardless of this snippet.                                                                                                                                                                                                                                                                                                              |
-| `docker`       | Scrapes Docker container logs on the host via the Docker socket and forwards them to Loki with `job=docker` and `container` labels. Assumes the `json-file` log driver (the default). If containers use the `journald` log driver instead, their logs are already captured by `journal` — enabling both snippets in that case will produce duplicate log entries in Loki. Note: log entries not yet shipped can be lost if Alloy is down when a log rotation occurs and the rotated file is subsequently deleted.                                                   |
+| `docker`       | Scrapes Docker container logs on the host via the Docker socket and forwards them to Loki with `job=docker` and `container` labels. Assumes the `json-file` log driver (the default). If containers are systemd-managed or use the `journald` log driver, their logs are already captured by `journal` — enabling both snippets in that case will produce duplicate log entries in Loki. Note: log entries not yet shipped can be lost if Alloy is down when a log rotation occurs and the rotated file is subsequently deleted.                                    |
 | `journal`      | Scrapes the systemd journal via the journal API and forwards to Loki with `job=systemd-journal`, `unit`, and `level` labels. Automatically discovers the journal regardless of storage mode (volatile or persistent). On standard systemd hosts, journald captures syslog messages as well, so this snippet typically covers everything `syslog` would. Enabling both may produce duplicate log entries in Loki. Note: with a volatile journal (`/run/log/journal`), any entries not yet shipped are lost on reboot — use persistent journal storage to avoid this. |
 | `journal-file` | Like `journal`, but reads from an explicit directory path (`alloy_journal_path`, default `/var/log/journal`) instead of the journal API. Produces the same labels (`job=systemd-journal`, `unit`, `level`). Prefer `journal` for new deployments — use `journal-file` only when migrating from promtail and needing the `legacy_position` block to resume cursor state. Do not enable both `journal` and `journal-file` simultaneously — this will produce duplicate log entries.                                                                                   |
 | `syslog`       | Tails `/var/log/syslog` and forwards to Loki with `job=syslog`. Suited for hosts without journald (e.g. minimal switch OS images). On standard systemd hosts, prefer `journal` — enabling both may produce duplicate log entries in Loki.                                                                                                                                                                                                                                                                                                                           |
@@ -42,7 +42,6 @@ If your needs are more custom (e.g. you have a non-standard log source, or want
 | alloy_config_snippets               |                                        | `[]`                               | List of snippet names to enable                                                                                                                                                                                                                                                           |
 | alloy_port                          |                                        | `12345`                            | Port for Alloy metrics and HTTP API                                                                                                                                                                                                                                                       |
 | alloy_migrate_from_promtail         |                                        | `false`                            | Enable migration mode: imports cursor state from the legacy promtail positions file on first start. Set to `true` when migrating from promtail; leave `false` for fresh deployments. Without this, Alloy starts from the current tail and previously shipped log data will be re-shipped. |
-| alloy_migrate_stop_promtail         |                                        | `false`                            | Stop and disable the promtail systemd service as part of this role run. Does not remove promtail files or images.                                                                                                                                                                         |
 | alloy_syslog_legacy_positions_file  |                                        | `/var/log/promtail-positions.yaml` | Path to the legacy promtail positions file. Used by `syslog` when `alloy_migrate_from_promtail` is `true`.                                                                                                                                                                                |
 | alloy_journal_path                  |                                        | `/var/log/journal`                 | Path to the persistent journal directory used by the `journal-file` snippet                                                                                                                                                                                                               |
 | alloy_journal_legacy_positions_file |                                        | `/var/log/promtail-positions.yaml` | Path to the legacy promtail positions file. Used by `journal-file` when `alloy_migrate_from_promtail` is `true`.                                                                                                                                                                          |
@@ -57,12 +56,12 @@ Alloy always exposes Prometheus metrics on `0.0.0.0:{{ alloy_port }}/metrics`, r
 
 Scraped metrics will carry the following labels (set by the prometheus role, not the alloy role):
 
-| Label | Value |
-| --------- | ------------------------------------------------- |
-| `job` | `alloy` |
-| `instance` | hostname of the target (port stripped) |
-| `partition` | `metal_partition_id` (from Prometheus `external_labels`) |
-| `replica` | `inventory_hostname` of the Prometheus host (from Prometheus `external_labels`) |
+| Label       | Value                                                                           |
+| ----------- | ------------------------------------------------------------------------------- |
+| `job`       | `alloy`                                                                         |
+| `instance`  | hostname of the target (port stripped)                                          |
+| `partition` | `metal_partition_id` (from Prometheus `external_labels`)                        |
+| `replica`   | `inventory_hostname` of the Prometheus host (from Prometheus `external_labels`) |
 
 #### Labels
 
@@ -135,12 +134,12 @@ Alloy allows reproducing the same log forwarding behavior as promtail via the [a
 
 Use the inventory flags below to control the deployment based on your situation:
 
-| Scenario                             | `alloy_enabled` | `alloy_migrate_from_promtail` | `alloy_migrate_stop_promtail` | `promtail_enabled` | Notes                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      |
-| ------------------------------------ | --------------- | ----------------------------- | ----------------------------- | ------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| **Fresh deployment**                 | `true`          | `false`                       | `false`                       | `false`            | No prior promtail. Alloy starts from the current log tail.                                                                                                                                                                                                                                                                                                                                                                                                                                                                 |
-| **Existing promtail — parallel run** | `true`          | `true`                        | `false`                       | `true`             | Both services ship logs simultaneously; Alloy resumes from promtail's cursor. Expect duplicate log entries during the overlap. Use to verify Alloy in production before cutting over.                                                                                                                                                                                                                                                                                                                                      |
-| **Existing promtail — cutover**      | `true`          | `true`                        | `true`                        | `false`            | Alloy starts resuming from promtail's cursor; promtail service is stopped and disabled. Promtail files and container are left in place. There is a brief gap between promtail stopping and Alloy starting — entries written in this window are not shipped immediately but are safe as long as the log source persists across the gap (journal file, syslog on disk). With a volatile journal this is unlikely to matter in practice since the system stays running, but a reboot in this window would lose those entries. |
-| **Stage config (dry run)**           | `false`         | `true` or `false`             | `false`                       | `true`             | Config files are written but the Alloy service is not started. Promtail keeps running unaffected. Setting `alloy_migrate_from_promtail: true` here is safe and recommended — the migration config (including `legacy_position`) is written to the leaf and inspectable before Alloy ever starts.                                                                                                                                                                                                                           |
+| Scenario                             | `alloy_enabled` | `alloy_migrate_from_promtail` | `promtail_migrate_stop` | `promtail_enabled` | Notes                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      |
+| ------------------------------------ | --------------- | ----------------------------- | ----------------------- | ------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| **Fresh deployment**                 | `true`          | `false`                       | `false`                 | `false`            | No prior promtail. Alloy starts from the current log tail.                                                                                                                                                                                                                                                                                                                                                                                                                                                                 |
+| **Existing promtail — parallel run** | `true`          | `true`                        | `false`                 | `true`             | Both services ship logs simultaneously; Alloy resumes from promtail's cursor. Expect duplicate log entries during the overlap. Use to verify Alloy in production before cutting over.                                                                                                                                                                                                                                                                                                                                      |
+| **Existing promtail — cutover**      | `true`          | `true`                        | `true`                  | `false`            | Alloy starts resuming from promtail's cursor; promtail service is stopped and disabled. Promtail files and container are left in place. There is a brief gap between promtail stopping and Alloy starting — entries written in this window are not shipped immediately but are safe as long as the log source persists across the gap (journal file, syslog on disk). With a volatile journal this is unlikely to matter in practice since the system stays running, but a reboot in this window would lose those entries. |
+| **Stage config (dry run)**           | `false`         | `true` or `false`             | `false`                 | `true`             | Config files are written but the Alloy service is not started. Promtail keeps running unaffected. Setting `alloy_migrate_from_promtail: true` here is safe and recommended — the migration config (including `legacy_position`) is written to the leaf and inspectable before Alloy ever starts.                                                                                                                                                                                                                           |
 
 1. **Add the `alloy` role** to your playbook. Keep `promtail` alongside it for a gradual migration, or remove it for a hard cut-over.
 
@@ -188,13 +187,8 @@ Use the inventory flags below to control the deployment based on your situation:
 
 6. **Verify** that logs arrive in Loki with the correct labels. Check that existing dashboards and alerts still work as expected and adapt them for any label or metric name changes.
 
-7. **Cut over from promtail** _(parallel run only)_. Set `alloy_migrate_stop_promtail: true` and `promtail_enabled: false` in your inventory and re-run the playbook — the role will stop and disable the promtail systemd service. You can also set `alloy_migrate_from_promtail: false` at this point since the cursor state has already been imported on first start.
+7. **Cut over from promtail** _(parallel run only)_. Set `promtail_migrate_stop: true` and `promtail_enabled: false` in your inventory and re-run the playbook — the promtail role will stop and disable the service. You can also set `alloy_migrate_from_promtail: false` at this point since the cursor state has already been imported on first start.
 
-   When you are ready, clean up the stopped promtail container and its config:
-
-   ```bash
-   docker rm promtail
-   rm -rf /etc/promtail
-   ```
+   When you are ready, clean up all promtail remnants by setting `promtail_migrate_cleanup: true` and re-running the playbook — the promtail role will remove the systemd unit file, container, config directory, and positions file. Then remove the `promtail` role from your playbook entirely.
 
    The `promtail` role is deprecated and will be removed in a future release. Once all environments are migrated, remove it from your playbook.
diff --git a/partition/roles/alloy/defaults/main.yaml b/partition/roles/alloy/defaults/main.yaml
index 9ffb56531..97ca752e3 100644
--- a/partition/roles/alloy/defaults/main.yaml
+++ b/partition/roles/alloy/defaults/main.yaml
@@ -28,12 +28,6 @@ alloy_port: 12345
 # migrating from promtail; leave false for fresh deployments.
 alloy_migrate_from_promtail: false
 
-# Stop and disable the promtail systemd service as part of this role run.
-# Use together with alloy_migrate_from_promtail: true for an immediate cutover,
-# or set independently after a successful parallel run to remove promtail.
-# Does not remove promtail files or images — cleanup is left to the operator.
-alloy_migrate_stop_promtail: false
-
 # Path to the legacy promtail positions file used by the syslog snippet on first start
 # after migration to continue tailing /var/log/syslog from where promtail left off.
 # Only active when alloy_migrate_from_promtail is true.
diff --git a/partition/roles/alloy/tasks/main.yaml b/partition/roles/alloy/tasks/main.yaml
index 77e3b8ed9..82159b84f 100644
--- a/partition/roles/alloy/tasks/main.yaml
+++ b/partition/roles/alloy/tasks/main.yaml
@@ -28,14 +28,6 @@
     that:
       - not ('journal' in alloy_config_snippets and 'journal-file' in alloy_config_snippets)
 
-- name: Stop and disable promtail service
-  systemd:
-    name: promtail
-    state: stopped
-    enabled: false
-  when: alloy_migrate_stop_promtail | default(false)
-  ignore_errors: true  # service may not exist on hosts that never had promtail
-
 - name: Create alloy config directory
   file:
     path: "{{ alloy_config_host_dir }}"
diff --git a/partition/roles/promtail/README.md b/partition/roles/promtail/README.md
index 209a7724e..537b8c1fd 100644
--- a/partition/roles/promtail/README.md
+++ b/partition/roles/promtail/README.md
@@ -7,12 +7,14 @@ Deploys promtail in a systemd-managed Docker container.
 
 ## Variables
 
-| Name                       | Mandatory | Description                                                                                                                                                                                                                                                                 |
-| -------------------------- | --------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| promtail_enabled           |           | Deploy and start the promtail service (default: `true`). Set to `false` during alloy migration to prevent this role from re-deploying promtail after `alloy_migrate_stop_promtail` has stopped it. Once migration is complete, remove this role from the playbook entirely. |
-| promtail_config_host_dir   |           | The location of the promtail config                                                                                                                                                                                                                                         |
-| promtail_image_name        | yes       | Image version of the promtail                                                                                                                                                                                                                                               |
-| promtail_image_tag         | yes       | Image tag of the promtail                                                                                                                                                                                                                                                   |
-| promtail_clients           | yes       | A list of clients for promtail, see <https://grafana.com/docs/loki/latest/send-data/promtail/configuration/#clients>                                                                                                                                                        |
-| promtail_scrape_configs    | yes       | A list containing the scrape configs                                                                                                                                                                                                                                        |
-| promtail_docker_log_driver |           | Indicates where to write the docker logs to                                                                                                                                                                                                                                 |
+| Name                       | Mandatory | Description                                                                                                                                                                                                                                                                                          |
+| -------------------------- | --------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| promtail_enabled           |           | Deploy and start the promtail service (default: `true`). Set to `false` during alloy migration to prevent this role from re-deploying promtail after `promtail_migrate_stop` has stopped it. Once migration is complete, remove this role from the playbook entirely.                                |
+| promtail_config_host_dir   |           | The location of the promtail config                                                                                                                                                                                                                                                                  |
+| promtail_image_name        | yes       | Image version of the promtail                                                                                                                                                                                                                                                                        |
+| promtail_image_tag         | yes       | Image tag of the promtail                                                                                                                                                                                                                                                                            |
+| promtail_clients           | yes       | A list of clients for promtail, see <https://grafana.com/docs/loki/latest/send-data/promtail/configuration/#clients>                                                                                                                                                                                 |
+| promtail_scrape_configs    | yes       | A list containing the scrape configs                                                                                                                                                                                                                                                                 |
+| promtail_docker_log_driver |           | Indicates where to write the docker logs to                                                                                                                                                                                                                                                          |
+| promtail_migrate_stop      |           | Stop and disable the promtail systemd service. Use during alloy cutover to stop promtail without removing its files. Implied by `promtail_migrate_cleanup`.                                                                                                                                          |
+| promtail_migrate_cleanup   |           | Remove all promtail remnants: stops and disables the service, removes the systemd unit file, removes the container, and deletes `promtail_config_host_dir` and `/var/log/promtail-positions.yaml`. Only set this once migration to alloy is fully complete, then remove this role from the playbook. |
diff --git a/partition/roles/promtail/defaults/main.yaml b/partition/roles/promtail/defaults/main.yaml
index 9f80f5993..3cfca150f 100644
--- a/partition/roles/promtail/defaults/main.yaml
+++ b/partition/roles/promtail/defaults/main.yaml
@@ -1,6 +1,6 @@
 ---
 # Set to false during alloy migration to prevent this role from re-deploying promtail
-# after alloy_migrate_stop_promtail has stopped it. Once migration is complete, remove
+# after promtail_migrate_stop has stopped it. Once migration is complete, remove
 # this role from the playbook entirely.
 promtail_enabled: true
 
@@ -10,3 +10,13 @@ promtail_clients: []
 promtail_scrape_configs: []
 
 promtail_docker_log_driver: json-file
+
+# Stop and disable the promtail systemd service. Use during alloy cutover to stop
+# promtail without removing its files. Implied by promtail_migrate_cleanup.
+promtail_migrate_stop: false
+
+# Remove all promtail remnants: stops and disables the service, removes the systemd
+# unit file (and triggers daemon-reload), removes the container, and deletes
+# promtail_config_host_dir and /var/log/promtail-positions.yaml.
+# Only set this once migration to alloy is fully complete, then remove this role from the playbook.
+promtail_migrate_cleanup: false
diff --git a/partition/roles/promtail/handlers/main.yaml b/partition/roles/promtail/handlers/main.yaml
index 5e253bfd1..01764881c 100644
--- a/partition/roles/promtail/handlers/main.yaml
+++ b/partition/roles/promtail/handlers/main.yaml
@@ -5,3 +5,7 @@
     enabled: true
     state: restarted
   when: promtail_enabled | default(true)
+
+- name: reload systemd
+  systemd:
+    daemon_reload: true
diff --git a/partition/roles/promtail/tasks/main.yaml b/partition/roles/promtail/tasks/main.yaml
index 5f95c214f..8853424f2 100644
--- a/partition/roles/promtail/tasks/main.yaml
+++ b/partition/roles/promtail/tasks/main.yaml
@@ -47,3 +47,37 @@
       - -config.file=/etc/promtail/promtail.yaml
     systemd_docker_log_driver: "{{ promtail_docker_log_driver }}"
   when: promtail_enabled | default(true)
+
+- name: Stop and disable promtail service
+  systemd:
+    name: promtail
+    state: stopped
+    enabled: false
+  when: promtail_migrate_stop | default(false) or promtail_migrate_cleanup | default(false)
+  ignore_errors: true  # service may not exist on all hosts
+
+- name: Remove promtail systemd unit file
+  file:
+    path: /etc/systemd/system/promtail.service
+    state: absent
+  when: promtail_migrate_cleanup | default(false)
+  notify: reload systemd
+
+- name: Remove promtail container
+  community.docker.docker_container:
+    name: promtail.service
+    state: absent
+  when: promtail_migrate_cleanup | default(false)
+  ignore_errors: true  # container may not exist
+
+- name: Remove promtail config directory
+  file:
+    path: "{{ promtail_config_host_dir }}"
+    state: absent
+  when: promtail_migrate_cleanup | default(false)
+
+- name: Remove promtail positions file
+  file:
+    path: /var/log/promtail-positions.yaml
+    state: absent
+  when: promtail_migrate_cleanup | default(false)

From d7434315614b3699032ad33ef2b837e7fcd25acb Mon Sep 17 00:00:00 2001
From: Matthias Hartmann <matthias.hartmann@x-cellent.com>
Date: Wed, 13 May 2026 17:09:25 +0200
Subject: [PATCH 17/41] chore: use recommended file_match block for syslog
 snippet

---
 .../roles/alloy/templates/snippets/syslog.alloy.j2   | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/partition/roles/alloy/templates/snippets/syslog.alloy.j2 b/partition/roles/alloy/templates/snippets/syslog.alloy.j2
index 775c25dd2..b35186fff 100644
--- a/partition/roles/alloy/templates/snippets/syslog.alloy.j2
+++ b/partition/roles/alloy/templates/snippets/syslog.alloy.j2
@@ -1,13 +1,13 @@
-local.file_match "syslog" {
-	path_targets = [{
+loki.source.file "syslog" {
+	targets = [{
 		__path__ = "/var/log/syslog",
 		job      = "syslog",
 	}]
-}
+	forward_to = [loki.write.default.receiver]
 
-loki.source.file "syslog" {
-	targets               = local.file_match.syslog.targets
-	forward_to            = [loki.write.default.receiver]
+	file_match {
+		enabled = true
+	}
 {% if alloy_migrate_from_promtail | default(false) %}
 	// migration only: resumes tailing from where promtail left off on first start.
 	// ignored if the file does not exist on the host.

From edd079ec4b58568d086ebdb3290b53240506891b Mon Sep 17 00:00:00 2001
From: Matthias Hartmann <matthias.hartmann@x-cellent.com>
Date: Tue, 19 May 2026 10:47:09 +0200
Subject: [PATCH 18/41] docs: Ansible meta deprecation and README

---
 partition/roles/alloy/README.md             | 8 ++++----
 partition/roles/promtail/defaults/main.yaml | 9 +++++----
 partition/roles/promtail/meta/main.yml      | 7 ++++++-
 partition/roles/promtail/tasks/main.yaml    | 7 +++++++
 4 files changed, 22 insertions(+), 9 deletions(-)

diff --git a/partition/roles/alloy/README.md b/partition/roles/alloy/README.md
index db693f4d4..1f4764c6f 100644
--- a/partition/roles/alloy/README.md
+++ b/partition/roles/alloy/README.md
@@ -141,7 +141,7 @@ Use the inventory flags below to control the deployment based on your situation:
 | **Existing promtail — cutover**      | `true`          | `true`                        | `true`                  | `false`            | Alloy starts resuming from promtail's cursor; promtail service is stopped and disabled. Promtail files and container are left in place. There is a brief gap between promtail stopping and Alloy starting — entries written in this window are not shipped immediately but are safe as long as the log source persists across the gap (journal file, syslog on disk). With a volatile journal this is unlikely to matter in practice since the system stays running, but a reboot in this window would lose those entries. |
 | **Stage config (dry run)**           | `false`         | `true` or `false`             | `false`                 | `true`             | Config files are written but the Alloy service is not started. Promtail keeps running unaffected. Setting `alloy_migrate_from_promtail: true` here is safe and recommended — the migration config (including `legacy_position`) is written to the leaf and inspectable before Alloy ever starts.                                                                                                                                                                                                                           |
 
-1. **Add the `alloy` role** to your playbook. Keep `promtail` alongside it for a gradual migration, or remove it for a hard cut-over.
+1. **Add the `alloy` role** to your playbook. Keep `promtail` alongside it for a gradual migration, or set `promtail_enabled: false` for a hard cut-over.
 
 2. **Configure the Loki endpoint.** Set `alloy_loki_write_endpoints` in your inventory:
 
@@ -162,7 +162,7 @@ Use the inventory flags below to control the deployment based on your situation:
    If your environment deploys Alloy on different host groups with different scrape needs (e.g. leaf switches vs. management servers), set `alloy_config_snippets` in per-group inventory files:
 
    ```text
-   group_vars/leaves/alloy.yaml       ← alloy_config_snippets: [docker, journal]
+   group_vars/leaves/alloy.yaml       ← alloy_config_snippets: [journal]
    group_vars/mgmtservers/alloy.yaml  ← alloy_config_snippets: [journal-file]
    group_vars/partition/alloy.yaml    ← alloy_loki_write_endpoints: [...] (shared by all)
    ```
@@ -189,6 +189,6 @@ Use the inventory flags below to control the deployment based on your situation:
 
 7. **Cut over from promtail** _(parallel run only)_. Set `promtail_migrate_stop: true` and `promtail_enabled: false` in your inventory and re-run the playbook — the promtail role will stop and disable the service. You can also set `alloy_migrate_from_promtail: false` at this point since the cursor state has already been imported on first start.
 
-   When you are ready, clean up all promtail remnants by setting `promtail_migrate_cleanup: true` and re-running the playbook — the promtail role will remove the systemd unit file, container, config directory, and positions file. Then remove the `promtail` role from your playbook entirely.
+   When you are ready, clean up all promtail remnants by setting `promtail_migrate_cleanup: true` and re-running the playbook — the promtail role will remove the systemd unit file, container, config directory, and positions file. Keep `promtail_enabled: false` in your inventory to prevent re-deployment.
 
-   The `promtail` role is deprecated and will be removed in a future release. Once all environments are migrated, remove it from your playbook.
+   The `promtail` role is deprecated and will be removed in a future release. Once all environments are migrated, keep `promtail_enabled: false` in your inventory.
diff --git a/partition/roles/promtail/defaults/main.yaml b/partition/roles/promtail/defaults/main.yaml
index 3cfca150f..fb838bc8e 100644
--- a/partition/roles/promtail/defaults/main.yaml
+++ b/partition/roles/promtail/defaults/main.yaml
@@ -1,7 +1,7 @@
 ---
-# Set to false during alloy migration to prevent this role from re-deploying promtail
-# after promtail_migrate_stop has stopped it. Once migration is complete, remove
-# this role from the playbook entirely.
+# Set to false to skip promtail deployment. Required for new deployments that use alloy
+# exclusively, and during alloy migration after promtail_migrate_stop has stopped the
+# service. Keep this set to false in your inventory once migration is complete.
 promtail_enabled: true
 
 promtail_config_host_dir: "/etc/promtail"
@@ -18,5 +18,6 @@ promtail_migrate_stop: false
 # Remove all promtail remnants: stops and disables the service, removes the systemd
 # unit file (and triggers daemon-reload), removes the container, and deletes
 # promtail_config_host_dir and /var/log/promtail-positions.yaml.
-# Only set this once migration to alloy is fully complete, then remove this role from the playbook.
+# Only set this once migration to alloy is fully complete, then set promtail_enabled: false
+# in your inventory to prevent re-deployment.
 promtail_migrate_cleanup: false
diff --git a/partition/roles/promtail/meta/main.yml b/partition/roles/promtail/meta/main.yml
index cea8ccd50..57d9ee96e 100644
--- a/partition/roles/promtail/meta/main.yml
+++ b/partition/roles/promtail/meta/main.yml
@@ -1,8 +1,13 @@
 ---
+deprecated:
+  removed_in: "a future release"
+  why: "Superseded by the alloy role."
+  alternative: "metal-roles/partition/roles/alloy"
+
 galaxy_info:
   role_name: promtail
   author: metal-stack
-  description: Deploys promtail.
+  description: "DEPRECATED: Deploys promtail. Use metal-roles/partition/roles/alloy instead."
   license: MIT
   min_ansible_version: "2.10"
   galaxy_tags: []
diff --git a/partition/roles/promtail/tasks/main.yaml b/partition/roles/promtail/tasks/main.yaml
index 8853424f2..1eae064ec 100644
--- a/partition/roles/promtail/tasks/main.yaml
+++ b/partition/roles/promtail/tasks/main.yaml
@@ -1,4 +1,11 @@
 ---
+- name: Deprecation warning
+  ansible.builtin.warn:
+    msg: >
+      The promtail role is deprecated and will be removed in a future release.
+      New deployments should use metal-roles/partition/roles/alloy instead.
+      See the alloy role README for migration instructions.
+
 - name: Gather release versions
   setup_yaml:
 

From 5210a30694ed6e95226d8d4cf5dcc96f8de14233 Mon Sep 17 00:00:00 2001
From: Matthias Hartmann <matthias.hartmann@x-cellent.com>
Date: Wed, 20 May 2026 12:19:50 +0200
Subject: [PATCH 19/41] chore: disable alloy reporting

---
 partition/roles/alloy/tasks/main.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/partition/roles/alloy/tasks/main.yaml b/partition/roles/alloy/tasks/main.yaml
index 82159b84f..71e8eb33c 100644
--- a/partition/roles/alloy/tasks/main.yaml
+++ b/partition/roles/alloy/tasks/main.yaml
@@ -104,6 +104,7 @@
       - run
       - --server.http.listen-addr=0.0.0.0:{{ alloy_port }}
       - --storage.path=/var/lib/alloy
+      - --disable-reporting
       - /etc/alloy/config.alloy
     systemd_docker_log_driver: "{{ alloy_docker_log_driver }}"
   when: alloy_enabled | default(false)

From 1d6a4d10c1f2b932d0c91083398937590bc3687c Mon Sep 17 00:00:00 2001
From: Matthias Hartmann <matthias.hartmann@x-cellent.com>
Date: Wed, 20 May 2026 13:33:31 +0200
Subject: [PATCH 20/41] fix: deprecation warning

---
 partition/roles/promtail/tasks/main.yaml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/partition/roles/promtail/tasks/main.yaml b/partition/roles/promtail/tasks/main.yaml
index 1eae064ec..72f75aa54 100644
--- a/partition/roles/promtail/tasks/main.yaml
+++ b/partition/roles/promtail/tasks/main.yaml
@@ -1,10 +1,11 @@
 ---
 - name: Deprecation warning
-  ansible.builtin.warn:
-    msg: >
+  ansible.builtin.fail:
+    msg: >-
       The promtail role is deprecated and will be removed in a future release.
       New deployments should use metal-roles/partition/roles/alloy instead.
       See the alloy role README for migration instructions.
+  ignore_errors: true
 
 - name: Gather release versions
   setup_yaml:

From dd19f15721a171d5429cd397992611de1d529d78 Mon Sep 17 00:00:00 2001
From: Matthias Hartmann <matthias.hartmann@x-cellent.com>
Date: Thu, 28 May 2026 11:46:38 +0200
Subject: [PATCH 21/41] chore: partition defaults

---
 partition/roles/alloy/defaults/main.yaml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/partition/roles/alloy/defaults/main.yaml b/partition/roles/alloy/defaults/main.yaml
index 97ca752e3..3b3d682c0 100644
--- a/partition/roles/alloy/defaults/main.yaml
+++ b/partition/roles/alloy/defaults/main.yaml
@@ -10,6 +10,10 @@ alloy_docker_log_driver: json-file
 # Each entry: {url: <push-url>, remote_timeout?: <duration>, basic_auth?: {username, password}}
 # alloy_loki_write_endpoints:
 #   - url: http://loki.example.com:8080/loki/api/v1/push
+#     remote_timeout: 10s
+#     basic_auth:
+#       username: "username"
+#       password: "password"
 
 # List of snippet names to enable. Each entry must correspond to
 # templates/snippets/<name>.alloy.j2 in this role.

From cefbe8184aba11e5a7425db8d05663d5da9dedf1 Mon Sep 17 00:00:00 2001
From: Matthias Hartmann <matthias.hartmann@x-cellent.com>
Date: Mon, 1 Jun 2026 17:49:53 +0200
Subject: [PATCH 22/41] fix: remove deprecation meta info

---
 partition/roles/promtail/meta/main.yml | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/partition/roles/promtail/meta/main.yml b/partition/roles/promtail/meta/main.yml
index 57d9ee96e..0f7f3141a 100644
--- a/partition/roles/promtail/meta/main.yml
+++ b/partition/roles/promtail/meta/main.yml
@@ -1,9 +1,4 @@
 ---
-deprecated:
-  removed_in: "a future release"
-  why: "Superseded by the alloy role."
-  alternative: "metal-roles/partition/roles/alloy"
-
 galaxy_info:
   role_name: promtail
   author: metal-stack

From 26bfc33f0232f89361308868d4ce4a8584a3aaa0 Mon Sep 17 00:00:00 2001
From: Matthias Hartmann <matthias.hartmann@x-cellent.com>
Date: Tue, 2 Jun 2026 10:35:58 +0200
Subject: [PATCH 23/41] docs: credential rename

---
 partition/roles/alloy/README.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/partition/roles/alloy/README.md b/partition/roles/alloy/README.md
index 1f4764c6f..978da1af5 100644
--- a/partition/roles/alloy/README.md
+++ b/partition/roles/alloy/README.md
@@ -154,6 +154,8 @@ Use the inventory flags below to control the deployment based on your situation:
          password: "test"
    ```
 
+   The `username` and `password` here must match `logging_ingress_loki_basic_auth_user` and `logging_ingress_loki_basic_auth_password` configured in the control-plane [logging role](../../../control-plane/roles/logging/README.md). The default username on the control-plane side is `promtail` and can optionally be changed there — update both sides together to avoid auth failures.
+
 3. **Choose snippets** that match your legacy `promtail_scrape_configs` — see [Available snippets](#available-snippets). If the available snippets are close enough to your old setup, enable the relevant ones and adjust with the provided variables. Otherwise, continue with step 4.
 
    Set `alloy_migrate_from_promtail: true` to import cursor state from the legacy promtail positions file so Alloy resumes from where promtail left off. See [Cursor/positions persistence](#cursorpositions-persistence) for per-snippet details and the relevant variables.

From a7a385388a5a03e4a6969f0d8aff91426855453e Mon Sep 17 00:00:00 2001
From: Matthias Hartmann <matthias.hartmann@x-cellent.com>
Date: Tue, 2 Jun 2026 11:16:06 +0200
Subject: [PATCH 24/41] chore: ansible lint

---
 partition/roles/alloy/tasks/main.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/partition/roles/alloy/tasks/main.yaml b/partition/roles/alloy/tasks/main.yaml
index 71e8eb33c..d4c5fb4d5 100644
--- a/partition/roles/alloy/tasks/main.yaml
+++ b/partition/roles/alloy/tasks/main.yaml
@@ -5,7 +5,7 @@
 - name: Check mandatory variables for this role are set
   assert:
     fail_msg: "not all mandatory variables given, check role documentation"
-    quiet: yes
+    quiet: true
     that:
       - alloy_image_tag is defined
       - alloy_image_name is defined
@@ -14,7 +14,7 @@
 - name: Check alloy_journal_legacy_position_name is set when migrating journal-file from promtail
   assert:
     fail_msg: "alloy_journal_legacy_position_name must be set — it must match the job_name of the journal scrape_config in your old promtail config"
-    quiet: yes
+    quiet: true
     that:
       - alloy_journal_legacy_position_name is defined
   when:
@@ -24,7 +24,7 @@
 - name: Check journal and journal-file are not both enabled
   assert:
     fail_msg: "journal and journal-file snippets cannot both be enabled — this will produce duplicate log entries in Loki"
-    quiet: yes
+    quiet: true
     that:
       - not ('journal' in alloy_config_snippets and 'journal-file' in alloy_config_snippets)
 

From 5ba4ded607ba7a46d70a74e81fa4c800d12e7794 Mon Sep 17 00:00:00 2001
From: Matthias Hartmann <matthias.hartmann@x-cellent.com>
Date: Wed, 3 Jun 2026 13:59:38 +0200
Subject: [PATCH 25/41] docs: finalize documentation

---
 partition/roles/alloy/README.md                          | 1 +
 partition/roles/monitoring/prometheus/defaults/main.yaml | 5 +++++
 partition/roles/promtail/tasks/main.yaml                 | 3 +--
 3 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/partition/roles/alloy/README.md b/partition/roles/alloy/README.md
index 978da1af5..a57d93b80 100644
--- a/partition/roles/alloy/README.md
+++ b/partition/roles/alloy/README.md
@@ -192,5 +192,6 @@ Use the inventory flags below to control the deployment based on your situation:
 7. **Cut over from promtail** _(parallel run only)_. Set `promtail_migrate_stop: true` and `promtail_enabled: false` in your inventory and re-run the playbook — the promtail role will stop and disable the service. You can also set `alloy_migrate_from_promtail: false` at this point since the cursor state has already been imported on first start.
 
    When you are ready, clean up all promtail remnants by setting `promtail_migrate_cleanup: true` and re-running the playbook — the promtail role will remove the systemd unit file, container, config directory, and positions file. Keep `promtail_enabled: false` in your inventory to prevent re-deployment.
+   We do not recommend doing the cleanup at the same time as the alloy deployment and migration steps, as the promtail positions file may be needed for the migration and the cleanup removes it. If you anyway choose to set the cleanup flag at the same time as the migration, make sure that the alloy role runs first to avoid migration failures due to the missing positions files.
 
    The `promtail` role is deprecated and will be removed in a future release. Once all environments are migrated, keep `promtail_enabled: false` in your inventory.
diff --git a/partition/roles/monitoring/prometheus/defaults/main.yaml b/partition/roles/monitoring/prometheus/defaults/main.yaml
index 2fd22572c..da28bcfc8 100644
--- a/partition/roles/monitoring/prometheus/defaults/main.yaml
+++ b/partition/roles/monitoring/prometheus/defaults/main.yaml
@@ -16,6 +16,11 @@ prometheus_alertmanager_target:
 prometheus_alertmanager_basic_auth_username: alertmanager
 prometheus_alertmanager_basic_auth_password:
 prometheus_remote_write: []
+  # - url: "https://monitoring.{{ metal_control_plane_ingress_dns }}/api/v1/receive"
+  #   basic_auth:
+  #     username: thanos-receive
+  #     password: "{{ basic_auth_pass }}"
+
 prometheus_frr_exporter_targets: []
 prometheus_metal_core_targets: []
 prometheus_node_exporter_targets: []
diff --git a/partition/roles/promtail/tasks/main.yaml b/partition/roles/promtail/tasks/main.yaml
index debdde1a8..025e036f4 100644
--- a/partition/roles/promtail/tasks/main.yaml
+++ b/partition/roles/promtail/tasks/main.yaml
@@ -3,8 +3,7 @@
   ansible.builtin.fail:
     msg: >-
       The promtail role is deprecated and will be removed in a future release.
-      New deployments should use metal-roles/partition/roles/alloy instead.
-      See the alloy role README for migration instructions.
+      New deployments should use metal-roles/partition/roles/alloy instead - see https://github.com/metal-stack/metal-roles/tree/main/partition/roles/alloy#migration-from-promtail for instructions.
   ignore_errors: true
 
 - name: Gather release versions

From 1b6642360b757df0e0bbe910d9527643b2e841a8 Mon Sep 17 00:00:00 2001
From: Matthias Hartmann <matthias.hartmann@x-cellent.com>
Date: Wed, 3 Jun 2026 14:04:50 +0200
Subject: [PATCH 26/41] fix: remove unnecessary alloy restarts

---
 partition/roles/alloy/tasks/main.yaml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/partition/roles/alloy/tasks/main.yaml b/partition/roles/alloy/tasks/main.yaml
index d4c5fb4d5..d91d8ffb4 100644
--- a/partition/roles/alloy/tasks/main.yaml
+++ b/partition/roles/alloy/tasks/main.yaml
@@ -48,7 +48,6 @@
   template:
     src: config.alloy.j2
     dest: "{{ alloy_config_host_dir }}/conf.d/00-base.alloy"
-  notify: restart alloy
   when: alloy_config_raw is not defined
 
 - name: Template alloy config snippets
@@ -56,7 +55,6 @@
     src: "snippets/{{ item }}.alloy.j2"
     dest: "{{ alloy_config_host_dir }}/conf.d/50-{{ item }}.alloy"
   loop: "{{ alloy_config_snippets }}"
-  notify: restart alloy
   when: alloy_config_raw is not defined
 
 - name: Assemble alloy config

From e1c6cadaafb1c3bf3bd6efad818e5c19f111f4d0 Mon Sep 17 00:00:00 2001
From: Matthias Hartmann <matthias.hartmann@x-cellent.com>
Date: Wed, 3 Jun 2026 14:05:58 +0200
Subject: [PATCH 27/41] Generated-By: [Claude Sonnet 4.6] find and remove stale
 alloy config snippets

---
 partition/roles/alloy/tasks/main.yaml | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/partition/roles/alloy/tasks/main.yaml b/partition/roles/alloy/tasks/main.yaml
index d91d8ffb4..29ed4f1d8 100644
--- a/partition/roles/alloy/tasks/main.yaml
+++ b/partition/roles/alloy/tasks/main.yaml
@@ -50,6 +50,22 @@
     dest: "{{ alloy_config_host_dir }}/conf.d/00-base.alloy"
   when: alloy_config_raw is not defined
 
+- name: Find existing alloy config snippet files
+  find:
+    paths: "{{ alloy_config_host_dir }}/conf.d"
+    patterns: "50-*.alloy"
+  register: existing_snippets
+  when: alloy_config_raw is not defined
+
+- name: Remove stale alloy config snippets
+  file:
+    path: "{{ item.path }}"
+    state: absent
+  loop: "{{ existing_snippets.files | default([]) }}"
+  when:
+    - alloy_config_raw is not defined
+    - item.path | basename | regex_replace('^50-(.+)\.alloy$', '\1') not in alloy_config_snippets
+
 - name: Template alloy config snippets
   template:
     src: "snippets/{{ item }}.alloy.j2"

From 8631ca83cf9ae9029ffbaa1261de519ed61cf64a Mon Sep 17 00:00:00 2001
From: Matthias Hartmann <matthias.hartmann@x-cellent.com>
Date: Thu, 11 Jun 2026 12:21:04 +0200
Subject: [PATCH 28/41] fix(alloy): depend on ansible-common

---
 partition/roles/alloy/meta/main.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/partition/roles/alloy/meta/main.yml b/partition/roles/alloy/meta/main.yml
index 9dfb6e3b0..d275533f2 100644
--- a/partition/roles/alloy/meta/main.yml
+++ b/partition/roles/alloy/meta/main.yml
@@ -13,5 +13,6 @@ galaxy_info:
         - all
 
 dependencies:
+  - role: ansible-common
   - role: metal-roles/common/roles/defaults
   - role: metal-roles/partition/roles/defaults

From a8ca3596fa5000adaaa5eb8a4926314e8d5bbc7f Mon Sep 17 00:00:00 2001
From: Matthias Hartmann <matthias.hartmann@x-cellent.com>
Date: Thu, 11 Jun 2026 12:23:58 +0200
Subject: [PATCH 29/41] fix(alloy) remove redundant default filters

---
 partition/roles/alloy/handlers/main.yaml                    | 2 +-
 partition/roles/alloy/tasks/main.yaml                       | 6 +++---
 .../roles/alloy/templates/snippets/journal-file.alloy.j2    | 2 +-
 partition/roles/alloy/templates/snippets/syslog.alloy.j2    | 2 +-
 4 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/partition/roles/alloy/handlers/main.yaml b/partition/roles/alloy/handlers/main.yaml
index 11ac110dd..f8ff20e45 100644
--- a/partition/roles/alloy/handlers/main.yaml
+++ b/partition/roles/alloy/handlers/main.yaml
@@ -4,4 +4,4 @@
     name: alloy
     enabled: true
     state: restarted
-  when: alloy_enabled | default(false)
+  when: alloy_enabled
diff --git a/partition/roles/alloy/tasks/main.yaml b/partition/roles/alloy/tasks/main.yaml
index 29ed4f1d8..0f3f9b95f 100644
--- a/partition/roles/alloy/tasks/main.yaml
+++ b/partition/roles/alloy/tasks/main.yaml
@@ -19,7 +19,7 @@
       - alloy_journal_legacy_position_name is defined
   when:
     - "'journal-file' in alloy_config_snippets"
-    - alloy_migrate_from_promtail | default(false)
+    - alloy_migrate_from_promtail
 
 - name: Check journal and journal-file are not both enabled
   assert:
@@ -95,7 +95,7 @@
     owner: "nobody"
     group: "nogroup"
     mode: "0755"
-  when: alloy_enabled | default(false)
+  when: alloy_enabled
 
 - name: Deploy alloy service
   include_role:
@@ -121,4 +121,4 @@
       - --disable-reporting
       - /etc/alloy/config.alloy
     systemd_docker_log_driver: "{{ alloy_docker_log_driver }}"
-  when: alloy_enabled | default(false)
+  when: alloy_enabled
diff --git a/partition/roles/alloy/templates/snippets/journal-file.alloy.j2 b/partition/roles/alloy/templates/snippets/journal-file.alloy.j2
index 2c439ee6d..09611c2a9 100644
--- a/partition/roles/alloy/templates/snippets/journal-file.alloy.j2
+++ b/partition/roles/alloy/templates/snippets/journal-file.alloy.j2
@@ -19,7 +19,7 @@ loki.source.journal "journal" {
 	labels        = {
 		job = "systemd-journal",
 	}
-{% if alloy_migrate_from_promtail | default(false) %}
+{% if alloy_migrate_from_promtail %}
 	// migration only: resumes reading from where promtail left off on first start.
 	// ignored if the file does not exist on the host.
 	// once alloy has written its own positions file, this can be removed.
diff --git a/partition/roles/alloy/templates/snippets/syslog.alloy.j2 b/partition/roles/alloy/templates/snippets/syslog.alloy.j2
index b35186fff..8b5be5509 100644
--- a/partition/roles/alloy/templates/snippets/syslog.alloy.j2
+++ b/partition/roles/alloy/templates/snippets/syslog.alloy.j2
@@ -8,7 +8,7 @@ loki.source.file "syslog" {
 	file_match {
 		enabled = true
 	}
-{% if alloy_migrate_from_promtail | default(false) %}
+{% if alloy_migrate_from_promtail %}
 	// migration only: resumes tailing from where promtail left off on first start.
 	// ignored if the file does not exist on the host.
 	// once alloy has written its own positions file, this can be removed.

From 0b1746cc7e5c69b4bcec7dd16fc78e8aef9417cc Mon Sep 17 00:00:00 2001
From: Matthias Hartmann <matthias.hartmann@x-cellent.com>
Date: Thu, 11 Jun 2026 12:37:59 +0200
Subject: [PATCH 30/41] fix(alloy) use port exposal instead of host network

---
 partition/roles/alloy/tasks/main.yaml | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/partition/roles/alloy/tasks/main.yaml b/partition/roles/alloy/tasks/main.yaml
index 0f3f9b95f..150790c52 100644
--- a/partition/roles/alloy/tasks/main.yaml
+++ b/partition/roles/alloy/tasks/main.yaml
@@ -106,7 +106,9 @@
     systemd_docker_image_tag: "{{ alloy_image_tag }}"
     systemd_service_after: docker.service
     systemd_service_requires: docker.service
-    systemd_docker_network: host
+    systemd_docker_ports:
+      - host_port: "{{ alloy_port }}"
+        target_port: "{{ alloy_port }}"
     systemd_docker_volumes:
       - "{{ alloy_config_host_dir }}:/etc/alloy:ro"
       - "/var/lib/alloy:/var/lib/alloy"
@@ -122,3 +124,10 @@
       - /etc/alloy/config.alloy
     systemd_docker_log_driver: "{{ alloy_docker_log_driver }}"
   when: alloy_enabled
+
+- name: Wait for alloy to listen on port
+  wait_for:
+    port: "{{ alloy_port }}"
+    timeout: 300
+    msg: "alloy did not come up"
+  when: alloy_enabled

From 2165b212299191fe0b685b6aca304c579dcb4279 Mon Sep 17 00:00:00 2001
From: Matthias Hartmann <matthias.hartmann@x-cellent.com>
Date: Thu, 11 Jun 2026 12:40:08 +0200
Subject: [PATCH 31/41] fix(alloy) assert that snippets and raw are not used at
 the same time

---
 partition/roles/alloy/tasks/main.yaml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/partition/roles/alloy/tasks/main.yaml b/partition/roles/alloy/tasks/main.yaml
index 150790c52..00039f4df 100644
--- a/partition/roles/alloy/tasks/main.yaml
+++ b/partition/roles/alloy/tasks/main.yaml
@@ -21,6 +21,13 @@
     - "'journal-file' in alloy_config_snippets"
     - alloy_migrate_from_promtail
 
+- name: Check alloy_config_raw and alloy_config_snippets are not both set
+  assert:
+    fail_msg: "alloy_config_raw and alloy_config_snippets cannot both be set — use either raw config or snippets, not both"
+    quiet: true
+    that:
+      - not (alloy_config_raw is defined and alloy_config_snippets | length > 0)
+
 - name: Check journal and journal-file are not both enabled
   assert:
     fail_msg: "journal and journal-file snippets cannot both be enabled — this will produce duplicate log entries in Loki"

From d6bd0f67efcb9a7ca7d325001c65fed91d3e1c07 Mon Sep 17 00:00:00 2001
From: Matthias Hartmann <matthias.hartmann@x-cellent.com>
Date: Thu, 11 Jun 2026 13:48:10 +0200
Subject: [PATCH 32/41] fix(alloy): simplify snippet handling

---
 partition/roles/alloy/README.md               |  17 +--
 partition/roles/alloy/tasks/main.yaml         |  40 +-----
 .../roles/alloy/templates/config.alloy.j2     | 122 ++++++++++++++++++
 .../templates/snippets/alloy-meta.alloy.j2    |   9 --
 .../alloy/templates/snippets/docker.alloy.j2  |  27 ----
 .../templates/snippets/journal-file.alloy.j2  |  32 -----
 .../alloy/templates/snippets/journal.alloy.j2 |  21 ---
 .../alloy/templates/snippets/syslog.alloy.j2  |  18 ---
 8 files changed, 129 insertions(+), 157 deletions(-)
 delete mode 100644 partition/roles/alloy/templates/snippets/alloy-meta.alloy.j2
 delete mode 100644 partition/roles/alloy/templates/snippets/docker.alloy.j2
 delete mode 100644 partition/roles/alloy/templates/snippets/journal-file.alloy.j2
 delete mode 100644 partition/roles/alloy/templates/snippets/journal.alloy.j2
 delete mode 100644 partition/roles/alloy/templates/snippets/syslog.alloy.j2

diff --git a/partition/roles/alloy/README.md b/partition/roles/alloy/README.md
index a57d93b80..6055528d9 100644
--- a/partition/roles/alloy/README.md
+++ b/partition/roles/alloy/README.md
@@ -6,14 +6,9 @@ This role replaces the deprecated `promtail` role. Alloy is configured to forwar
 
 ## Configuration
 
-The Alloy configuration is assembled from a base template plus a list of opt-in snippets:
+The Alloy configuration is driven by a single template `templates/config.alloy.j2`, which contains all supported sections as conditional blocks. At deploy time the template is rendered into `{{ alloy_config_host_dir }}/config.alloy`, with each section included only when the corresponding name appears in `alloy_config_snippets`.
 
-- `templates/config.alloy.j2` — base config, always rendered. Defines the shared `loki.write "default"` endpoint.
-- `templates/snippets/<name>.alloy.j2` — opt-in snippets, enabled via `alloy_config_snippets`.
-
-At deploy time each enabled snippet is rendered into `{{ alloy_config_host_dir }}/conf.d/` and concatenated into a single `config.alloy` file.
-
-The snippets can be configured with variables defined in the [Variables](#variables) section below. This allows you to enable only the features you need without having to maintain a full custom config.
+The sections can be configured with variables defined in the [Variables](#variables) section below. This allows you to enable only the features you need without having to maintain a full custom config.
 
 ### Available snippets
 
@@ -114,9 +109,9 @@ There is no way to inject a custom snippet purely from inventory. Your options a
 
 Set `alloy_config_raw` to a full Alloy River config string in your inventory. The role will write it verbatim and skip snippet assembly entirely. `alloy_loki_write_endpoints` and `alloy_config_snippets` are ignored. You own the complete config, including the base `loki.write "default"` block.
 
-**Option B — Add the snippet to this role** (requires editing this repo)
+**Option B — Add a section to this role** (requires editing this repo)
 
-Add the template to `templates/snippets/` and reference it by name in `alloy_config_snippets`:
+Add a new `{% if "<name>" in alloy_config_snippets %}...{% endif %}` block to `templates/config.alloy.j2` and reference it by name in `alloy_config_snippets`:
 
 ```yaml
 alloy_config_snippets:
@@ -124,9 +119,9 @@ alloy_config_snippets:
   - docker
 ```
 
-A snippet template can use any Ansible variables available on the host. It does not need to define `loki.write "default"` as it is already in the base config.
+A section can use any Ansible variables available on the host. It does not need to define `loki.write "default"` as it is already in the base template.
 
-Contributions of new snippets are welcome — since all snippets are opt-in via `alloy_config_snippets`, adding one to the role has no impact on existing deployments.
+Contributions of new sections are welcome — since all sections are opt-in via `alloy_config_snippets`, adding one to the role has no impact on existing deployments.
 
 ## Migration from `promtail`
 
diff --git a/partition/roles/alloy/tasks/main.yaml b/partition/roles/alloy/tasks/main.yaml
index 00039f4df..436b421c0 100644
--- a/partition/roles/alloy/tasks/main.yaml
+++ b/partition/roles/alloy/tasks/main.yaml
@@ -43,48 +43,10 @@
     group: "nogroup"
     mode: "0755"
 
-- name: Create alloy config snippets directory
-  file:
-    path: "{{ alloy_config_host_dir }}/conf.d"
-    state: directory
-    owner: "nobody"
-    group: "nogroup"
-    mode: "0755"
-
-- name: Template alloy base configuration
+- name: Template alloy config
   template:
     src: config.alloy.j2
-    dest: "{{ alloy_config_host_dir }}/conf.d/00-base.alloy"
-  when: alloy_config_raw is not defined
-
-- name: Find existing alloy config snippet files
-  find:
-    paths: "{{ alloy_config_host_dir }}/conf.d"
-    patterns: "50-*.alloy"
-  register: existing_snippets
-  when: alloy_config_raw is not defined
-
-- name: Remove stale alloy config snippets
-  file:
-    path: "{{ item.path }}"
-    state: absent
-  loop: "{{ existing_snippets.files | default([]) }}"
-  when:
-    - alloy_config_raw is not defined
-    - item.path | basename | regex_replace('^50-(.+)\.alloy$', '\1') not in alloy_config_snippets
-
-- name: Template alloy config snippets
-  template:
-    src: "snippets/{{ item }}.alloy.j2"
-    dest: "{{ alloy_config_host_dir }}/conf.d/50-{{ item }}.alloy"
-  loop: "{{ alloy_config_snippets }}"
-  when: alloy_config_raw is not defined
-
-- name: Assemble alloy config
-  assemble:
-    src: "{{ alloy_config_host_dir }}/conf.d"
     dest: "{{ alloy_config_host_dir }}/config.alloy"
-    delimiter: "\n"
   notify: restart alloy
   when: alloy_config_raw is not defined
 
diff --git a/partition/roles/alloy/templates/config.alloy.j2 b/partition/roles/alloy/templates/config.alloy.j2
index cf1560783..190ed682e 100644
--- a/partition/roles/alloy/templates/config.alloy.j2
+++ b/partition/roles/alloy/templates/config.alloy.j2
@@ -28,3 +28,125 @@ loki.write "default" {
 	}
 {% endfor %}
 }
+
+{% if "syslog" in alloy_config_snippets %}
+loki.source.file "syslog" {
+	targets = [{
+		__path__ = "/var/log/syslog",
+		job      = "syslog",
+	}]
+	forward_to = [loki.write.default.receiver]
+
+	file_match {
+		enabled = true
+	}
+{% if alloy_migrate_from_promtail %}
+	// migration only: resumes tailing from where promtail left off on first start.
+	// ignored if the file does not exist on the host.
+	// once alloy has written its own positions file, this can be removed.
+	// https://grafana.com/docs/alloy/latest/reference/components/loki/loki.source.file/#arguments
+	legacy_positions_file = "{{ alloy_syslog_legacy_positions_file }}"
+{% endif %}
+}
+{% endif %}
+
+{% if "journal" in alloy_config_snippets %}
+discovery.relabel "journal" {
+	targets = []
+
+	rule {
+		source_labels = ["__journal__systemd_unit"]
+		target_label  = "unit"
+	}
+
+	rule {
+		source_labels = ["__journal_priority_keyword"]
+		target_label  = "level"
+	}
+}
+
+loki.source.journal "journal" {
+	relabel_rules = discovery.relabel.journal.rules
+	forward_to    = [loki.write.default.receiver]
+	labels        = {
+		job = "systemd-journal",
+	}
+}
+{% endif %}
+
+{% if "journal-file" in alloy_config_snippets %}
+discovery.relabel "journal" {
+	targets = []
+
+	rule {
+		source_labels = ["__journal__systemd_unit"]
+		target_label  = "unit"
+	}
+
+	rule {
+		source_labels = ["__journal_priority_keyword"]
+		target_label  = "level"
+	}
+}
+
+loki.source.journal "journal" {
+	path          = "{{ alloy_journal_path }}"
+	relabel_rules = discovery.relabel.journal.rules
+	forward_to    = [loki.write.default.receiver]
+	labels        = {
+		job = "systemd-journal",
+	}
+{% if alloy_migrate_from_promtail %}
+	// migration only: resumes reading from where promtail left off on first start.
+	// ignored if the file does not exist on the host.
+	// once alloy has written its own positions file, this can be removed.
+	// https://grafana.com/docs/alloy/latest/reference/components/loki/loki.source.journal/#legacy_position
+	legacy_position {
+		file = "{{ alloy_journal_legacy_positions_file }}"
+		name = "{{ alloy_journal_legacy_position_name }}"
+	}
+{% endif %}
+}
+{% endif %}
+
+{% if "docker" in alloy_config_snippets %}
+discovery.docker "docker" {
+	host             = "unix:///var/run/docker.sock"
+	refresh_interval = "5s"
+}
+
+discovery.relabel "docker" {
+	targets = []
+
+	rule {
+		source_labels = ["__meta_docker_container_name"]
+		regex         = "/(.*)"
+		target_label  = "container"
+	}
+
+	rule {
+		target_label = "job"
+		replacement  = "docker"
+	}
+}
+
+loki.source.docker "docker" {
+	host             = "unix:///var/run/docker.sock"
+	targets          = discovery.docker.docker.targets
+	forward_to       = [loki.write.default.receiver]
+	relabel_rules    = discovery.relabel.docker.rules
+	refresh_interval = "5s"
+}
+{% endif %}
+
+{% if "alloy-meta" in alloy_config_snippets %}
+// Meta-monitoring: forward Alloy's own logs to Loki
+loki.relabel "alloy_self" {
+	forward_to = [loki.write.default.receiver]
+
+	rule {
+		target_label = "job"
+		replacement  = "alloy"
+	}
+}
+{% endif %}
diff --git a/partition/roles/alloy/templates/snippets/alloy-meta.alloy.j2 b/partition/roles/alloy/templates/snippets/alloy-meta.alloy.j2
deleted file mode 100644
index 34bff47df..000000000
--- a/partition/roles/alloy/templates/snippets/alloy-meta.alloy.j2
+++ /dev/null
@@ -1,9 +0,0 @@
-// Meta-monitoring: forward Alloy's own logs to Loki
-loki.relabel "alloy_self" {
-	forward_to = [loki.write.default.receiver]
-
-	rule {
-		target_label = "job"
-		replacement  = "alloy"
-	}
-}
diff --git a/partition/roles/alloy/templates/snippets/docker.alloy.j2 b/partition/roles/alloy/templates/snippets/docker.alloy.j2
deleted file mode 100644
index 71176376a..000000000
--- a/partition/roles/alloy/templates/snippets/docker.alloy.j2
+++ /dev/null
@@ -1,27 +0,0 @@
-discovery.docker "docker" {
-	host             = "unix:///var/run/docker.sock"
-	refresh_interval = "5s"
-}
-
-discovery.relabel "docker" {
-	targets = []
-
-	rule {
-		source_labels = ["__meta_docker_container_name"]
-		regex         = "/(.*)"
-		target_label  = "container"
-	}
-
-	rule {
-		target_label = "job"
-		replacement  = "docker"
-	}
-}
-
-loki.source.docker "docker" {
-	host             = "unix:///var/run/docker.sock"
-	targets          = discovery.docker.docker.targets
-	forward_to       = [loki.write.default.receiver]
-	relabel_rules    = discovery.relabel.docker.rules
-	refresh_interval = "5s"
-}
diff --git a/partition/roles/alloy/templates/snippets/journal-file.alloy.j2 b/partition/roles/alloy/templates/snippets/journal-file.alloy.j2
deleted file mode 100644
index 09611c2a9..000000000
--- a/partition/roles/alloy/templates/snippets/journal-file.alloy.j2
+++ /dev/null
@@ -1,32 +0,0 @@
-discovery.relabel "journal" {
-	targets = []
-
-	rule {
-		source_labels = ["__journal__systemd_unit"]
-		target_label  = "unit"
-	}
-
-	rule {
-		source_labels = ["__journal_priority_keyword"]
-		target_label  = "level"
-	}
-}
-
-loki.source.journal "journal" {
-	path          = "{{ alloy_journal_path }}"
-	relabel_rules = discovery.relabel.journal.rules
-	forward_to    = [loki.write.default.receiver]
-	labels        = {
-		job = "systemd-journal",
-	}
-{% if alloy_migrate_from_promtail %}
-	// migration only: resumes reading from where promtail left off on first start.
-	// ignored if the file does not exist on the host.
-	// once alloy has written its own positions file, this can be removed.
-	// https://grafana.com/docs/alloy/latest/reference/components/loki/loki.source.journal/#legacy_position
-	legacy_position {
-		file = "{{ alloy_journal_legacy_positions_file }}"
-		name = "{{ alloy_journal_legacy_position_name }}"
-	}
-{% endif %}
-}
diff --git a/partition/roles/alloy/templates/snippets/journal.alloy.j2 b/partition/roles/alloy/templates/snippets/journal.alloy.j2
deleted file mode 100644
index f2f8d2cfb..000000000
--- a/partition/roles/alloy/templates/snippets/journal.alloy.j2
+++ /dev/null
@@ -1,21 +0,0 @@
-discovery.relabel "journal" {
-	targets = []
-
-	rule {
-		source_labels = ["__journal__systemd_unit"]
-		target_label  = "unit"
-	}
-
-	rule {
-		source_labels = ["__journal_priority_keyword"]
-		target_label  = "level"
-	}
-}
-
-loki.source.journal "journal" {
-	relabel_rules = discovery.relabel.journal.rules
-	forward_to    = [loki.write.default.receiver]
-	labels        = {
-		job = "systemd-journal",
-	}
-}
diff --git a/partition/roles/alloy/templates/snippets/syslog.alloy.j2 b/partition/roles/alloy/templates/snippets/syslog.alloy.j2
deleted file mode 100644
index 8b5be5509..000000000
--- a/partition/roles/alloy/templates/snippets/syslog.alloy.j2
+++ /dev/null
@@ -1,18 +0,0 @@
-loki.source.file "syslog" {
-	targets = [{
-		__path__ = "/var/log/syslog",
-		job      = "syslog",
-	}]
-	forward_to = [loki.write.default.receiver]
-
-	file_match {
-		enabled = true
-	}
-{% if alloy_migrate_from_promtail %}
-	// migration only: resumes tailing from where promtail left off on first start.
-	// ignored if the file does not exist on the host.
-	// once alloy has written its own positions file, this can be removed.
-	// https://grafana.com/docs/alloy/latest/reference/components/loki/loki.source.file/#arguments
-	legacy_positions_file = "{{ alloy_syslog_legacy_positions_file }}"
-{% endif %}
-}

From db713a5f8a61ac43125c9b3c7f4dc55b79c3305d Mon Sep 17 00:00:00 2001
From: Matthias Hartmann <matthias.hartmann@x-cellent.com>
Date: Thu, 11 Jun 2026 17:24:06 +0200
Subject: [PATCH 33/41] review(promtail): old variable still present

---
 partition/roles/monitoring/prometheus/README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/partition/roles/monitoring/prometheus/README.md b/partition/roles/monitoring/prometheus/README.md
index 0851deadc..e1fa459ca 100644
--- a/partition/roles/monitoring/prometheus/README.md
+++ b/partition/roles/monitoring/prometheus/README.md
@@ -23,6 +23,7 @@ you define them adequately as well.
 | prometheus_metal_core_targets                    |           | metal-core targets to scrape from                                                                                                           |
 | prometheus_node_exporter_targets                 |           | Node exporter targets to scrape from                                                                                                        |
 | prometheus_alloy_targets                         |           | Alloy targets to scrape from (default port `12345`)                                                                                         |
+| prometheus_promtail_targets                      |           | Promtail targets to scrape from                                                                                                             |
 | prometheus_ping_targets                          |           | Ping targets to scrape from                                                                                                                 |
 | prometheus_sonic_exporter_targets                |           | Sonic exporter targets to scrape from                                                                                                       |
 | prometheus_blackbox_exporter_targets             |           | Blackbox exporter targets to scrape from                                                                                                    |

From 21282976c0a9d488c066960ade7b7d41b059b0fd Mon Sep 17 00:00:00 2001
From: Matthias Hartmann <matthias.hartmann@x-cellent.com>
Date: Thu, 11 Jun 2026 17:29:12 +0200
Subject: [PATCH 34/41] review(promtail): remove _enabled vars and move
 promtail cleanup to alloy role

---
 partition/roles/alloy/README.md             | 22 ++++++------
 partition/roles/alloy/defaults/main.yaml    | 13 ++++---
 partition/roles/alloy/tasks/main.yaml       | 38 +++++++++++++++++++--
 partition/roles/promtail/README.md          | 19 +++++------
 partition/roles/promtail/defaults/main.yaml | 16 ---------
 partition/roles/promtail/tasks/main.yaml    | 38 ---------------------
 6 files changed, 63 insertions(+), 83 deletions(-)

diff --git a/partition/roles/alloy/README.md b/partition/roles/alloy/README.md
index 6055528d9..cdfda5d76 100644
--- a/partition/roles/alloy/README.md
+++ b/partition/roles/alloy/README.md
@@ -28,7 +28,6 @@ If your needs are more custom (e.g. you have a non-standard log source, or want
 
 | Name                                | Mandatory                              | Default                            | Description                                                                                                                                                                                                                                                                               |
 | ----------------------------------- | -------------------------------------- | ---------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| alloy_enabled                       |                                        | `false`                            | Deploy and start the Alloy service. Config files are always written regardless of this flag. Set to `false` to stage the config without starting the service.                                                                                                                             |
 | alloy_config_host_dir               |                                        | `/etc/alloy`                       | The location of the alloy config on the host                                                                                                                                                                                                                                              |
 | alloy_image_name                    | yes                                    |                                    | Image name of alloy                                                                                                                                                                                                                                                                       |
 | alloy_image_tag                     | yes                                    |                                    | Image tag of alloy                                                                                                                                                                                                                                                                        |
@@ -42,6 +41,8 @@ If your needs are more custom (e.g. you have a non-standard log source, or want
 | alloy_journal_legacy_positions_file |                                        | `/var/log/promtail-positions.yaml` | Path to the legacy promtail positions file. Used by `journal-file` when `alloy_migrate_from_promtail` is `true`.                                                                                                                                                                          |
 | alloy_journal_legacy_position_name  | yes (migration)                        |                                    | Job name from the old promtail journal scrape_config. Must match `job_name` in your old promtail config. Required when `alloy_migrate_from_promtail` is `true` and `journal-file` is used.                                                                                                |
 | alloy_config_raw                    |                                        |                                    | Full Alloy River config as a string. When set, bypasses snippet assembly entirely — `alloy_loki_write_endpoints` and `alloy_config_snippets` are ignored.                                                                                                                                 |
+| promtail_migrate_stop               |                                        | `false`                            | Stop and disable the promtail systemd service. Use during alloy cutover to stop promtail without removing its files. Implied by `promtail_migrate_cleanup`.                                                                                                                               |
+| promtail_migrate_cleanup            |                                        | `false`                            | Remove all promtail remnants: stops and disables the service, removes the systemd unit file, removes the container, and deletes `promtail_config_host_dir` and `/var/log/promtail-positions.yaml`. Only set this once migration to alloy is fully complete.                               |
 
 ### Meta-monitoring (for Alloy itself)
 
@@ -129,14 +130,13 @@ Alloy allows reproducing the same log forwarding behavior as promtail via the [a
 
 Use the inventory flags below to control the deployment based on your situation:
 
-| Scenario                             | `alloy_enabled` | `alloy_migrate_from_promtail` | `promtail_migrate_stop` | `promtail_enabled` | Notes                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      |
-| ------------------------------------ | --------------- | ----------------------------- | ----------------------- | ------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| **Fresh deployment**                 | `true`          | `false`                       | `false`                 | `false`            | No prior promtail. Alloy starts from the current log tail.                                                                                                                                                                                                                                                                                                                                                                                                                                                                 |
-| **Existing promtail — parallel run** | `true`          | `true`                        | `false`                 | `true`             | Both services ship logs simultaneously; Alloy resumes from promtail's cursor. Expect duplicate log entries during the overlap. Use to verify Alloy in production before cutting over.                                                                                                                                                                                                                                                                                                                                      |
-| **Existing promtail — cutover**      | `true`          | `true`                        | `true`                  | `false`            | Alloy starts resuming from promtail's cursor; promtail service is stopped and disabled. Promtail files and container are left in place. There is a brief gap between promtail stopping and Alloy starting — entries written in this window are not shipped immediately but are safe as long as the log source persists across the gap (journal file, syslog on disk). With a volatile journal this is unlikely to matter in practice since the system stays running, but a reboot in this window would lose those entries. |
-| **Stage config (dry run)**           | `false`         | `true` or `false`             | `false`                 | `true`             | Config files are written but the Alloy service is not started. Promtail keeps running unaffected. Setting `alloy_migrate_from_promtail: true` here is safe and recommended — the migration config (including `legacy_position`) is written to the leaf and inspectable before Alloy ever starts.                                                                                                                                                                                                                           |
+| Scenario                             | `alloy_migrate_from_promtail` | `promtail_migrate_stop` | Notes                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      |
+| ------------------------------------ | ----------------------------- | ----------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| **Fresh deployment**                 | `false`                       | `false`                 | No prior promtail. Alloy starts from the current log tail.                                                                                                                                                                                                                                                                                                                                                                                                                                                                 |
+| **Existing promtail — parallel run** | `true`                        | `false`                 | Both services ship logs simultaneously; Alloy resumes from promtail's cursor. Expect duplicate log entries during the overlap. Use to verify Alloy in production before cutting over.                                                                                                                                                                                                                                                                                                                                      |
+| **Existing promtail — cutover**      | `true`                        | `true`                  | Alloy starts resuming from promtail's cursor; promtail service is stopped and disabled. Promtail files and container are left in place. There is a brief gap between promtail stopping and Alloy starting — entries written in this window are not shipped immediately but are safe as long as the log source persists across the gap (journal file, syslog on disk). With a volatile journal this is unlikely to matter in practice since the system stays running, but a reboot in this window would lose those entries. |
 
-1. **Add the `alloy` role** to your playbook. Keep `promtail` alongside it for a gradual migration, or set `promtail_enabled: false` for a hard cut-over.
+1. **Add the `alloy` role** to your playbook. Keep `promtail` alongside it for a gradual migration, or remove it for a hard cut-over.
 
 2. **Configure the Loki endpoint.** Set `alloy_loki_write_endpoints` in your inventory:
 
@@ -184,9 +184,9 @@ Use the inventory flags below to control the deployment based on your situation:
 
 6. **Verify** that logs arrive in Loki with the correct labels. Check that existing dashboards and alerts still work as expected and adapt them for any label or metric name changes.
 
-7. **Cut over from promtail** _(parallel run only)_. Set `promtail_migrate_stop: true` and `promtail_enabled: false` in your inventory and re-run the playbook — the promtail role will stop and disable the service. You can also set `alloy_migrate_from_promtail: false` at this point since the cursor state has already been imported on first start.
+7. **Cut over from promtail** _(parallel run only)_. Set `promtail_migrate_stop: true` in your inventory and re-run the playbook — the alloy role will stop and disable the service. You can also set `alloy_migrate_from_promtail: false` at this point since the cursor state has already been imported on first start.
 
-   When you are ready, clean up all promtail remnants by setting `promtail_migrate_cleanup: true` and re-running the playbook — the promtail role will remove the systemd unit file, container, config directory, and positions file. Keep `promtail_enabled: false` in your inventory to prevent re-deployment.
+   When you are ready, clean up all promtail remnants by setting `promtail_migrate_cleanup: true` and re-running the playbook — the alloy role will remove the systemd unit file, container, config directory, and positions file. Once cleanup is complete, remove the `promtail` role from the playbook entirely.
    We do not recommend doing the cleanup at the same time as the alloy deployment and migration steps, as the promtail positions file may be needed for the migration and the cleanup removes it. If you anyway choose to set the cleanup flag at the same time as the migration, make sure that the alloy role runs first to avoid migration failures due to the missing positions files.
 
-   The `promtail` role is deprecated and will be removed in a future release. Once all environments are migrated, keep `promtail_enabled: false` in your inventory.
+   The `promtail` role is deprecated and will be removed in a future release. Once all environments are migrated, remove the `promtail` role from the playbook.
diff --git a/partition/roles/alloy/defaults/main.yaml b/partition/roles/alloy/defaults/main.yaml
index 3b3d682c0..43584bbbc 100644
--- a/partition/roles/alloy/defaults/main.yaml
+++ b/partition/roles/alloy/defaults/main.yaml
@@ -1,8 +1,4 @@
 ---
-# Set to true to deploy and start the Alloy service.
-# Set to false to skip service deployment — useful when alloy is being staged or promtail is still running.
-alloy_enabled: false
-
 alloy_config_host_dir: "/etc/alloy"
 alloy_docker_log_driver: json-file
 
@@ -50,3 +46,12 @@ alloy_journal_legacy_positions_file: "/var/log/promtail-positions.yaml"
 # Required when alloy_migrate_from_promtail is true and the journal-file snippet is used.
 # No default — must be set explicitly.
 # alloy_journal_legacy_position_name:
+
+# Stop and disable the promtail systemd service during alloy cutover.
+# Implied by promtail_migrate_cleanup.
+promtail_migrate_stop: false
+
+# Remove all promtail remnants: stops and disables the service, removes the systemd unit
+# file, removes the container, and deletes promtail_config_host_dir and
+# /var/log/promtail-positions.yaml. Only set once migration to alloy is fully complete.
+promtail_migrate_cleanup: false
diff --git a/partition/roles/alloy/tasks/main.yaml b/partition/roles/alloy/tasks/main.yaml
index 436b421c0..866cc1027 100644
--- a/partition/roles/alloy/tasks/main.yaml
+++ b/partition/roles/alloy/tasks/main.yaml
@@ -64,7 +64,6 @@
     owner: "nobody"
     group: "nogroup"
     mode: "0755"
-  when: alloy_enabled
 
 - name: Deploy alloy service
   include_role:
@@ -92,11 +91,44 @@
       - --disable-reporting
       - /etc/alloy/config.alloy
     systemd_docker_log_driver: "{{ alloy_docker_log_driver }}"
-  when: alloy_enabled
 
 - name: Wait for alloy to listen on port
   wait_for:
     port: "{{ alloy_port }}"
     timeout: 300
     msg: "alloy did not come up"
-  when: alloy_enabled
+
+# promtail migration
+- name: Stop and disable promtail service
+  systemd:
+    name: promtail
+    state: stopped
+    enabled: false
+  when: promtail_migrate_stop or promtail_migrate_cleanup
+  ignore_errors: true  # service may not exist on all hosts
+
+- name: Remove promtail systemd unit file
+  file:
+    path: /etc/systemd/system/promtail.service
+    state: absent
+  when: promtail_migrate_cleanup
+  notify: reload systemd
+
+- name: Remove promtail container
+  community.docker.docker_container:
+    name: promtail.service
+    state: absent
+  when: promtail_migrate_cleanup
+  ignore_errors: true  # container may not exist
+
+- name: Remove promtail config directory
+  file:
+    path: "{{ promtail_config_host_dir }}"
+    state: absent
+  when: promtail_migrate_cleanup
+
+- name: Remove promtail positions file
+  file:
+    path: /var/log/promtail-positions.yaml
+    state: absent
+  when: promtail_migrate_cleanup
diff --git a/partition/roles/promtail/README.md b/partition/roles/promtail/README.md
index 537b8c1fd..2726a8a90 100644
--- a/partition/roles/promtail/README.md
+++ b/partition/roles/promtail/README.md
@@ -7,14 +7,11 @@ Deploys promtail in a systemd-managed Docker container.
 
 ## Variables
 
-| Name                       | Mandatory | Description                                                                                                                                                                                                                                                                                          |
-| -------------------------- | --------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| promtail_enabled           |           | Deploy and start the promtail service (default: `true`). Set to `false` during alloy migration to prevent this role from re-deploying promtail after `promtail_migrate_stop` has stopped it. Once migration is complete, remove this role from the playbook entirely.                                |
-| promtail_config_host_dir   |           | The location of the promtail config                                                                                                                                                                                                                                                                  |
-| promtail_image_name        | yes       | Image version of the promtail                                                                                                                                                                                                                                                                        |
-| promtail_image_tag         | yes       | Image tag of the promtail                                                                                                                                                                                                                                                                            |
-| promtail_clients           | yes       | A list of clients for promtail, see <https://grafana.com/docs/loki/latest/send-data/promtail/configuration/#clients>                                                                                                                                                                                 |
-| promtail_scrape_configs    | yes       | A list containing the scrape configs                                                                                                                                                                                                                                                                 |
-| promtail_docker_log_driver |           | Indicates where to write the docker logs to                                                                                                                                                                                                                                                          |
-| promtail_migrate_stop      |           | Stop and disable the promtail systemd service. Use during alloy cutover to stop promtail without removing its files. Implied by `promtail_migrate_cleanup`.                                                                                                                                          |
-| promtail_migrate_cleanup   |           | Remove all promtail remnants: stops and disables the service, removes the systemd unit file, removes the container, and deletes `promtail_config_host_dir` and `/var/log/promtail-positions.yaml`. Only set this once migration to alloy is fully complete, then remove this role from the playbook. |
+| Name                       | Mandatory | Description                                                                                                          |
+| -------------------------- | --------- | -------------------------------------------------------------------------------------------------------------------- |
+| promtail_config_host_dir   |           | The location of the promtail config                                                                                  |
+| promtail_image_name        | yes       | Image version of the promtail                                                                                        |
+| promtail_image_tag         | yes       | Image tag of the promtail                                                                                            |
+| promtail_clients           | yes       | A list of clients for promtail, see <https://grafana.com/docs/loki/latest/send-data/promtail/configuration/#clients> |
+| promtail_scrape_configs    | yes       | A list containing the scrape configs                                                                                 |
+| promtail_docker_log_driver |           | Indicates where to write the docker logs to                                                                          |
diff --git a/partition/roles/promtail/defaults/main.yaml b/partition/roles/promtail/defaults/main.yaml
index 4aaaf7fd4..ae9b3c03b 100644
--- a/partition/roles/promtail/defaults/main.yaml
+++ b/partition/roles/promtail/defaults/main.yaml
@@ -1,22 +1,6 @@
 ---
-# Set to false to skip promtail deployment. Required for new deployments that use alloy
-# exclusively, and during alloy migration after promtail_migrate_stop has stopped the
-# service. Keep this set to false in your inventory once migration is complete.
-promtail_enabled: true
-
 promtail_config_host_dir: "/etc/promtail"
 
 promtail_clients: []
 promtail_scrape_configs: []
 promtail_docker_log_driver: json-file
-
-# Stop and disable the promtail systemd service. Use during alloy cutover to stop
-# promtail without removing its files. Implied by promtail_migrate_cleanup.
-promtail_migrate_stop: false
-
-# Remove all promtail remnants: stops and disables the service, removes the systemd
-# unit file (and triggers daemon-reload), removes the container, and deletes
-# promtail_config_host_dir and /var/log/promtail-positions.yaml.
-# Only set this once migration to alloy is fully complete, then set promtail_enabled: false
-# in your inventory to prevent re-deployment.
-promtail_migrate_cleanup: false
diff --git a/partition/roles/promtail/tasks/main.yaml b/partition/roles/promtail/tasks/main.yaml
index 025e036f4..1ae1311d0 100644
--- a/partition/roles/promtail/tasks/main.yaml
+++ b/partition/roles/promtail/tasks/main.yaml
@@ -18,7 +18,6 @@
       - promtail_image_tag is defined
       - promtail_image_name is defined
       - promtail_scrape_configs is defined and (promtail_scrape_configs|length>0)
-  when: promtail_enabled | default(true)
 
 - name: Create promtail config directory
   file:
@@ -27,14 +26,12 @@
     owner: "nobody"
     group: "nogroup"
     mode: "0755"
-  when: promtail_enabled | default(true)
 
 - name: Template promtail configuration file
   template:
     src: promtail.yaml.j2
     dest: "{{ promtail_config_host_dir }}/promtail.yaml"
   notify: restart promtail
-  when: promtail_enabled | default(true)
 
 - name: Deploy promtail service
   include_role:
@@ -53,38 +50,3 @@
     systemd_docker_command:
       - -config.file=/etc/promtail/promtail.yaml
     systemd_docker_log_driver: "{{ promtail_docker_log_driver }}"
-  when: promtail_enabled | default(true)
-
-- name: Stop and disable promtail service
-  systemd:
-    name: promtail
-    state: stopped
-    enabled: false
-  when: promtail_migrate_stop | default(false) or promtail_migrate_cleanup | default(false)
-  ignore_errors: true  # service may not exist on all hosts
-
-- name: Remove promtail systemd unit file
-  file:
-    path: /etc/systemd/system/promtail.service
-    state: absent
-  when: promtail_migrate_cleanup | default(false)
-  notify: reload systemd
-
-- name: Remove promtail container
-  community.docker.docker_container:
-    name: promtail.service
-    state: absent
-  when: promtail_migrate_cleanup | default(false)
-  ignore_errors: true  # container may not exist
-
-- name: Remove promtail config directory
-  file:
-    path: "{{ promtail_config_host_dir }}"
-    state: absent
-  when: promtail_migrate_cleanup | default(false)
-
-- name: Remove promtail positions file
-  file:
-    path: /var/log/promtail-positions.yaml
-    state: absent
-  when: promtail_migrate_cleanup | default(false)

From f1ab72b8ff39a0484356034aea56966deb2171da Mon Sep 17 00:00:00 2001
From: Matthias Hartmann <matthias.hartmann@x-cellent.com>
Date: Fri, 12 Jun 2026 09:30:34 +0200
Subject: [PATCH 35/41] review(promtail): improve cleanup

---
 partition/roles/alloy/README.md          |  4 ++--
 partition/roles/alloy/defaults/main.yaml |  6 +++---
 partition/roles/alloy/tasks/main.yaml    | 21 +++++++++++----------
 3 files changed, 16 insertions(+), 15 deletions(-)

diff --git a/partition/roles/alloy/README.md b/partition/roles/alloy/README.md
index cdfda5d76..5db775d43 100644
--- a/partition/roles/alloy/README.md
+++ b/partition/roles/alloy/README.md
@@ -42,7 +42,7 @@ If your needs are more custom (e.g. you have a non-standard log source, or want
 | alloy_journal_legacy_position_name  | yes (migration)                        |                                    | Job name from the old promtail journal scrape_config. Must match `job_name` in your old promtail config. Required when `alloy_migrate_from_promtail` is `true` and `journal-file` is used.                                                                                                |
 | alloy_config_raw                    |                                        |                                    | Full Alloy River config as a string. When set, bypasses snippet assembly entirely — `alloy_loki_write_endpoints` and `alloy_config_snippets` are ignored.                                                                                                                                 |
 | promtail_migrate_stop               |                                        | `false`                            | Stop and disable the promtail systemd service. Use during alloy cutover to stop promtail without removing its files. Implied by `promtail_migrate_cleanup`.                                                                                                                               |
-| promtail_migrate_cleanup            |                                        | `false`                            | Remove all promtail remnants: stops and disables the service, removes the systemd unit file, removes the container, and deletes `promtail_config_host_dir` and `/var/log/promtail-positions.yaml`. Only set this once migration to alloy is fully complete.                               |
+| promtail_migrate_cleanup            |                                        | `false`                            | Remove all promtail remnants: stops and disables the service, removes the systemd unit file, deletes `promtail_config_host_dir` and the legacy positions files. Only set this once migration to alloy is fully complete.                                                                  |
 
 ### Meta-monitoring (for Alloy itself)
 
@@ -186,7 +186,7 @@ Use the inventory flags below to control the deployment based on your situation:
 
 7. **Cut over from promtail** _(parallel run only)_. Set `promtail_migrate_stop: true` in your inventory and re-run the playbook — the alloy role will stop and disable the service. You can also set `alloy_migrate_from_promtail: false` at this point since the cursor state has already been imported on first start.
 
-   When you are ready, clean up all promtail remnants by setting `promtail_migrate_cleanup: true` and re-running the playbook — the alloy role will remove the systemd unit file, container, config directory, and positions file. Once cleanup is complete, remove the `promtail` role from the playbook entirely.
+   When you are ready, clean up all promtail remnants by setting `promtail_migrate_cleanup: true` and re-running the playbook — the alloy role will stop and disable the service, remove the systemd unit file, config directory, and positions files. Once cleanup is complete, remove the `promtail` role from the playbook entirely.
    We do not recommend doing the cleanup at the same time as the alloy deployment and migration steps, as the promtail positions file may be needed for the migration and the cleanup removes it. If you anyway choose to set the cleanup flag at the same time as the migration, make sure that the alloy role runs first to avoid migration failures due to the missing positions files.
 
    The `promtail` role is deprecated and will be removed in a future release. Once all environments are migrated, remove the `promtail` role from the playbook.
diff --git a/partition/roles/alloy/defaults/main.yaml b/partition/roles/alloy/defaults/main.yaml
index 43584bbbc..908bb2644 100644
--- a/partition/roles/alloy/defaults/main.yaml
+++ b/partition/roles/alloy/defaults/main.yaml
@@ -51,7 +51,7 @@ alloy_journal_legacy_positions_file: "/var/log/promtail-positions.yaml"
 # Implied by promtail_migrate_cleanup.
 promtail_migrate_stop: false
 
-# Remove all promtail remnants: stops and disables the service, removes the systemd unit
-# file, removes the container, and deletes promtail_config_host_dir and
-# /var/log/promtail-positions.yaml. Only set once migration to alloy is fully complete.
+# Remove all promtail remnants: stops and disables the service, 
+# removes the systemd unit file, deletes promtail_config_host_dir and
+# the legacy positions files. Only set once migration to alloy is fully complete.
 promtail_migrate_cleanup: false
diff --git a/partition/roles/alloy/tasks/main.yaml b/partition/roles/alloy/tasks/main.yaml
index 866cc1027..f032a87d8 100644
--- a/partition/roles/alloy/tasks/main.yaml
+++ b/partition/roles/alloy/tasks/main.yaml
@@ -99,13 +99,20 @@
     msg: "alloy did not come up"
 
 # promtail migration
+- name: Check if promtail systemd unit file is present
+  stat:
+    path: /etc/systemd/system/promtail.service
+  register: promtail_unit_file
+  when: promtail_migrate_stop or promtail_migrate_cleanup
+
 - name: Stop and disable promtail service
   systemd:
     name: promtail
     state: stopped
     enabled: false
-  when: promtail_migrate_stop or promtail_migrate_cleanup
-  ignore_errors: true  # service may not exist on all hosts
+  when:
+    - promtail_migrate_stop or promtail_migrate_cleanup
+    - promtail_unit_file.stat.exists
 
 - name: Remove promtail systemd unit file
   file:
@@ -114,13 +121,6 @@
   when: promtail_migrate_cleanup
   notify: reload systemd
 
-- name: Remove promtail container
-  community.docker.docker_container:
-    name: promtail.service
-    state: absent
-  when: promtail_migrate_cleanup
-  ignore_errors: true  # container may not exist
-
 - name: Remove promtail config directory
   file:
     path: "{{ promtail_config_host_dir }}"
@@ -129,6 +129,7 @@
 
 - name: Remove promtail positions file
   file:
-    path: /var/log/promtail-positions.yaml
+    path: "{{ item }}"
     state: absent
+  loop: "{{ [alloy_syslog_legacy_positions_file, alloy_journal_legacy_positions_file] | unique }}"
   when: promtail_migrate_cleanup

From 876a2b6f31302bc64d1f81759a33edc5d195586b Mon Sep 17 00:00:00 2001
From: Matthias Hartmann <matthias.hartmann@x-cellent.com>
Date: Sat, 13 Jun 2026 00:12:21 +0200
Subject: [PATCH 36/41] chore: cleanup _enabled

---
 partition/roles/alloy/handlers/main.yaml    | 1 -
 partition/roles/promtail/handlers/main.yaml | 1 -
 2 files changed, 2 deletions(-)

diff --git a/partition/roles/alloy/handlers/main.yaml b/partition/roles/alloy/handlers/main.yaml
index f8ff20e45..56a671558 100644
--- a/partition/roles/alloy/handlers/main.yaml
+++ b/partition/roles/alloy/handlers/main.yaml
@@ -4,4 +4,3 @@
     name: alloy
     enabled: true
     state: restarted
-  when: alloy_enabled
diff --git a/partition/roles/promtail/handlers/main.yaml b/partition/roles/promtail/handlers/main.yaml
index 01764881c..9d15aa0cc 100644
--- a/partition/roles/promtail/handlers/main.yaml
+++ b/partition/roles/promtail/handlers/main.yaml
@@ -4,7 +4,6 @@
     name: promtail
     enabled: true
     state: restarted
-  when: promtail_enabled | default(true)
 
 - name: reload systemd
   systemd:

From 4ffdf3b07bb1a39d9f3a2d5533e22fdb1948ea1f Mon Sep 17 00:00:00 2001
From: Matthias Hartmann <matthias.hartmann@x-cellent.com>
Date: Mon, 15 Jun 2026 12:29:44 +0200
Subject: [PATCH 37/41] fix: reload systemd location

---
 partition/roles/alloy/handlers/main.yaml    | 4 ++++
 partition/roles/promtail/handlers/main.yaml | 4 ----
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/partition/roles/alloy/handlers/main.yaml b/partition/roles/alloy/handlers/main.yaml
index 56a671558..8056a839d 100644
--- a/partition/roles/alloy/handlers/main.yaml
+++ b/partition/roles/alloy/handlers/main.yaml
@@ -4,3 +4,7 @@
     name: alloy
     enabled: true
     state: restarted
+
+- name: reload systemd
+  systemd:
+    daemon_reload: true
diff --git a/partition/roles/promtail/handlers/main.yaml b/partition/roles/promtail/handlers/main.yaml
index 9d15aa0cc..9d28fb13a 100644
--- a/partition/roles/promtail/handlers/main.yaml
+++ b/partition/roles/promtail/handlers/main.yaml
@@ -4,7 +4,3 @@
     name: promtail
     enabled: true
     state: restarted
-
-- name: reload systemd
-  systemd:
-    daemon_reload: true

From 8e40fc0c0f95f9ca81c855fb97f4fcabc3657ab2 Mon Sep 17 00:00:00 2001
From: Matthias Hartmann <matthias.hartmann@x-cellent.com>
Date: Mon, 15 Jun 2026 17:21:07 +0200
Subject: [PATCH 38/41] feat: allow custom snippets

---
 partition/roles/alloy/README.md               |  37 +++--
 partition/roles/alloy/defaults/main.yaml      |  12 +-
 partition/roles/alloy/tasks/main.yaml         |   4 +-
 .../roles/alloy/templates/config.alloy.j2     | 129 +-----------------
 .../templates/snippets/alloy-meta.alloy.j2    |   9 ++
 .../alloy/templates/snippets/docker.alloy.j2  |  27 ++++
 .../templates/snippets/journal-file.alloy.j2  |  32 +++++
 .../alloy/templates/snippets/journal.alloy.j2 |  21 +++
 .../alloy/templates/snippets/syslog.alloy.j2  |  18 +++
 9 files changed, 156 insertions(+), 133 deletions(-)
 create mode 100644 partition/roles/alloy/templates/snippets/alloy-meta.alloy.j2
 create mode 100644 partition/roles/alloy/templates/snippets/docker.alloy.j2
 create mode 100644 partition/roles/alloy/templates/snippets/journal-file.alloy.j2
 create mode 100644 partition/roles/alloy/templates/snippets/journal.alloy.j2
 create mode 100644 partition/roles/alloy/templates/snippets/syslog.alloy.j2

diff --git a/partition/roles/alloy/README.md b/partition/roles/alloy/README.md
index 5db775d43..1ce815b7e 100644
--- a/partition/roles/alloy/README.md
+++ b/partition/roles/alloy/README.md
@@ -33,7 +33,8 @@ If your needs are more custom (e.g. you have a non-standard log source, or want
 | alloy_image_tag                     | yes                                    |                                    | Image tag of alloy                                                                                                                                                                                                                                                                        |
 | alloy_loki_write_endpoints          | yes (unless `alloy_config_raw` is set) |                                    | List of Loki push endpoints. Each entry: `{url, remote_timeout?: <duration>, basic_auth?: {username, password}}`                                                                                                                                                                          |
 | alloy_docker_log_driver             |                                        | `json-file`                        | Docker log driver for the alloy container                                                                                                                                                                                                                                                 |
-| alloy_config_snippets               |                                        | `[]`                               | List of snippet names to enable                                                                                                                                                                                                                                                           |
+| alloy_config_snippets               |                                        | `[]`                               | List of built-in snippet names to enable. Available: `syslog`, `journal`, `journal-file`, `docker`, `alloy-meta`                                                                                                                                                                          |
+| alloy_config_custom_snippets        |                                        | `[]`                               | List of paths to custom Alloy River snippet templates. Resolved via Ansible's template search path (relative to the playbook's `templates/` directory, or absolute). Appended after built-in snippets. See [Customizing the config](#customizing-the-config).                             |
 | alloy_port                          |                                        | `12345`                            | Port for Alloy metrics and HTTP API                                                                                                                                                                                                                                                       |
 | alloy_migrate_from_promtail         |                                        | `false`                            | Enable migration mode: imports cursor state from the legacy promtail positions file on first start. Set to `true` when migrating from promtail; leave `false` for fresh deployments. Without this, Alloy starts from the current tail and previously shipped log data will be re-shipped. |
 | alloy_syslog_legacy_positions_file  |                                        | `/var/log/promtail-positions.yaml` | Path to the legacy promtail positions file. Used by `syslog` when `alloy_migrate_from_promtail` is `true`.                                                                                                                                                                                |
@@ -104,15 +105,31 @@ Once migration is complete, the old promtail positions files can be cleaned up f
 
 ## Customizing the config
 
-There is no way to inject a custom snippet purely from inventory. Your options are:
+Your options, from least to most invasive:
 
-**Option A — Use `alloy_config_raw`** (inventory only, no role change)
+**Option A — Custom snippets via inventory** (no role change required)
 
-Set `alloy_config_raw` to a full Alloy River config string in your inventory. The role will write it verbatim and skip snippet assembly entirely. `alloy_loki_write_endpoints` and `alloy_config_snippets` are ignored. You own the complete config, including the base `loki.write "default"` block.
+Add a Jinja2 template file to your playbook's `templates/` directory and reference it via `alloy_config_custom_snippets`:
 
-**Option B — Add a section to this role** (requires editing this repo)
+```yaml
+alloy_config_custom_snippets:
+  - my_custom_source.alloy.j2
+```
+
+The path is resolved through Ansible's normal template search path (relative to the playbook's `templates/` directory, or an absolute path). The snippet is rendered after all built-in snippets and has access to all Ansible variables on the host. It does not need to define `loki.write "default"` — that is already in the base template.
+
+Example `templates/my_custom_source.alloy.j2`:
+
+```
+loki.source.file "custom" {
+  targets    = [{ __path__ = "/var/log/myapp/*.log", job = "myapp" }]
+  forward_to = [loki.write.default.receiver]
+}
+```
+
+**Option B — Contribute a built-in snippet to this role** (requires editing this repo)
 
-Add a new `{% if "<name>" in alloy_config_snippets %}...{% endif %}` block to `templates/config.alloy.j2` and reference it by name in `alloy_config_snippets`:
+Add a new file `templates/snippets/<name>.alloy.j2` and reference it by name in `alloy_config_snippets`:
 
 ```yaml
 alloy_config_snippets:
@@ -120,9 +137,13 @@ alloy_config_snippets:
   - docker
 ```
 
-A section can use any Ansible variables available on the host. It does not need to define `loki.write "default"` as it is already in the base template.
+A snippet can use any Ansible variables available on the host. It does not need to define `loki.write "default"` as it is already in the base template.
+
+Contributions of new snippets are welcome — since all snippets are opt-in via `alloy_config_snippets`, adding one to the role has no impact on existing deployments.
+
+**Option C — Use `alloy_config_raw`** (inventory only, no role change)
 
-Contributions of new sections are welcome — since all sections are opt-in via `alloy_config_snippets`, adding one to the role has no impact on existing deployments.
+Set `alloy_config_raw` to a full Alloy River config string in your inventory. The role will write it verbatim and skip snippet assembly entirely. `alloy_loki_write_endpoints`, `alloy_config_snippets`, and `alloy_config_custom_snippets` are ignored. You own the complete config, including the base `loki.write "default"` block.
 
 ## Migration from `promtail`
 
diff --git a/partition/roles/alloy/defaults/main.yaml b/partition/roles/alloy/defaults/main.yaml
index 908bb2644..2e912db6c 100644
--- a/partition/roles/alloy/defaults/main.yaml
+++ b/partition/roles/alloy/defaults/main.yaml
@@ -11,10 +11,20 @@ alloy_docker_log_driver: json-file
 #       username: "username"
 #       password: "password"
 
-# List of snippet names to enable. Each entry must correspond to
+# List of built-in snippet names to enable. Each entry must correspond to
 # templates/snippets/<name>.alloy.j2 in this role.
+# Available snippets: syslog, journal, journal-file, docker, alloy-meta
 alloy_config_snippets: []
 
+# List of paths to custom Alloy River config snippets. Each entry is a
+# template path resolved via Ansible's normal template search path (i.e.
+# relative to the playbook's templates/ directory, or an absolute path).
+# Custom snippets are appended after all built-in snippets.
+# Example:
+#   alloy_config_custom_snippets:
+#     - my_custom_source.alloy.j2
+alloy_config_custom_snippets: []
+
 # Provide a fully custom Alloy River config as a string to bypass the
 # snippet assembly system entirely. When set, alloy_loki_write_endpoints
 # and alloy_config_snippets are ignored.
diff --git a/partition/roles/alloy/tasks/main.yaml b/partition/roles/alloy/tasks/main.yaml
index f032a87d8..57ffdbde4 100644
--- a/partition/roles/alloy/tasks/main.yaml
+++ b/partition/roles/alloy/tasks/main.yaml
@@ -23,10 +23,10 @@
 
 - name: Check alloy_config_raw and alloy_config_snippets are not both set
   assert:
-    fail_msg: "alloy_config_raw and alloy_config_snippets cannot both be set — use either raw config or snippets, not both"
+    fail_msg: "alloy_config_raw and alloy_config_snippets/alloy_config_custom_snippets cannot both be set — use either raw config or snippets, not both"
     quiet: true
     that:
-      - not (alloy_config_raw is defined and alloy_config_snippets | length > 0)
+      - not (alloy_config_raw is defined and (alloy_config_snippets | length > 0 or alloy_config_custom_snippets | length > 0))
 
 - name: Check journal and journal-file are not both enabled
   assert:
diff --git a/partition/roles/alloy/templates/config.alloy.j2 b/partition/roles/alloy/templates/config.alloy.j2
index 190ed682e..51005dbe1 100644
--- a/partition/roles/alloy/templates/config.alloy.j2
+++ b/partition/roles/alloy/templates/config.alloy.j2
@@ -16,7 +16,7 @@ loki.write "default" {
 {% for endpoint in alloy_loki_write_endpoints %}
 	endpoint {
 		url = "{{ endpoint.url }}"
-{%if endpoint.remote_timeout is defined %}
+{% if endpoint.remote_timeout is defined %}
 		remote_timeout = "{{ endpoint.remote_timeout }}"
 {% endif %}
 {% if endpoint.basic_auth is defined %}
@@ -29,124 +29,9 @@ loki.write "default" {
 {% endfor %}
 }
 
-{% if "syslog" in alloy_config_snippets %}
-loki.source.file "syslog" {
-	targets = [{
-		__path__ = "/var/log/syslog",
-		job      = "syslog",
-	}]
-	forward_to = [loki.write.default.receiver]
-
-	file_match {
-		enabled = true
-	}
-{% if alloy_migrate_from_promtail %}
-	// migration only: resumes tailing from where promtail left off on first start.
-	// ignored if the file does not exist on the host.
-	// once alloy has written its own positions file, this can be removed.
-	// https://grafana.com/docs/alloy/latest/reference/components/loki/loki.source.file/#arguments
-	legacy_positions_file = "{{ alloy_syslog_legacy_positions_file }}"
-{% endif %}
-}
-{% endif %}
-
-{% if "journal" in alloy_config_snippets %}
-discovery.relabel "journal" {
-	targets = []
-
-	rule {
-		source_labels = ["__journal__systemd_unit"]
-		target_label  = "unit"
-	}
-
-	rule {
-		source_labels = ["__journal_priority_keyword"]
-		target_label  = "level"
-	}
-}
-
-loki.source.journal "journal" {
-	relabel_rules = discovery.relabel.journal.rules
-	forward_to    = [loki.write.default.receiver]
-	labels        = {
-		job = "systemd-journal",
-	}
-}
-{% endif %}
-
-{% if "journal-file" in alloy_config_snippets %}
-discovery.relabel "journal" {
-	targets = []
-
-	rule {
-		source_labels = ["__journal__systemd_unit"]
-		target_label  = "unit"
-	}
-
-	rule {
-		source_labels = ["__journal_priority_keyword"]
-		target_label  = "level"
-	}
-}
-
-loki.source.journal "journal" {
-	path          = "{{ alloy_journal_path }}"
-	relabel_rules = discovery.relabel.journal.rules
-	forward_to    = [loki.write.default.receiver]
-	labels        = {
-		job = "systemd-journal",
-	}
-{% if alloy_migrate_from_promtail %}
-	// migration only: resumes reading from where promtail left off on first start.
-	// ignored if the file does not exist on the host.
-	// once alloy has written its own positions file, this can be removed.
-	// https://grafana.com/docs/alloy/latest/reference/components/loki/loki.source.journal/#legacy_position
-	legacy_position {
-		file = "{{ alloy_journal_legacy_positions_file }}"
-		name = "{{ alloy_journal_legacy_position_name }}"
-	}
-{% endif %}
-}
-{% endif %}
-
-{% if "docker" in alloy_config_snippets %}
-discovery.docker "docker" {
-	host             = "unix:///var/run/docker.sock"
-	refresh_interval = "5s"
-}
-
-discovery.relabel "docker" {
-	targets = []
-
-	rule {
-		source_labels = ["__meta_docker_container_name"]
-		regex         = "/(.*)"
-		target_label  = "container"
-	}
-
-	rule {
-		target_label = "job"
-		replacement  = "docker"
-	}
-}
-
-loki.source.docker "docker" {
-	host             = "unix:///var/run/docker.sock"
-	targets          = discovery.docker.docker.targets
-	forward_to       = [loki.write.default.receiver]
-	relabel_rules    = discovery.relabel.docker.rules
-	refresh_interval = "5s"
-}
-{% endif %}
-
-{% if "alloy-meta" in alloy_config_snippets %}
-// Meta-monitoring: forward Alloy's own logs to Loki
-loki.relabel "alloy_self" {
-	forward_to = [loki.write.default.receiver]
-
-	rule {
-		target_label = "job"
-		replacement  = "alloy"
-	}
-}
-{% endif %}
+{% for snippet in alloy_config_snippets %}
+{% include "snippets/" + snippet + ".alloy.j2" %}
+{% endfor %}
+{% for snippet in alloy_config_custom_snippets %}
+{% include snippet %}
+{% endfor %}
diff --git a/partition/roles/alloy/templates/snippets/alloy-meta.alloy.j2 b/partition/roles/alloy/templates/snippets/alloy-meta.alloy.j2
new file mode 100644
index 000000000..34bff47df
--- /dev/null
+++ b/partition/roles/alloy/templates/snippets/alloy-meta.alloy.j2
@@ -0,0 +1,9 @@
+// Meta-monitoring: forward Alloy's own logs to Loki
+loki.relabel "alloy_self" {
+	forward_to = [loki.write.default.receiver]
+
+	rule {
+		target_label = "job"
+		replacement  = "alloy"
+	}
+}
diff --git a/partition/roles/alloy/templates/snippets/docker.alloy.j2 b/partition/roles/alloy/templates/snippets/docker.alloy.j2
new file mode 100644
index 000000000..71176376a
--- /dev/null
+++ b/partition/roles/alloy/templates/snippets/docker.alloy.j2
@@ -0,0 +1,27 @@
+discovery.docker "docker" {
+	host             = "unix:///var/run/docker.sock"
+	refresh_interval = "5s"
+}
+
+discovery.relabel "docker" {
+	targets = []
+
+	rule {
+		source_labels = ["__meta_docker_container_name"]
+		regex         = "/(.*)"
+		target_label  = "container"
+	}
+
+	rule {
+		target_label = "job"
+		replacement  = "docker"
+	}
+}
+
+loki.source.docker "docker" {
+	host             = "unix:///var/run/docker.sock"
+	targets          = discovery.docker.docker.targets
+	forward_to       = [loki.write.default.receiver]
+	relabel_rules    = discovery.relabel.docker.rules
+	refresh_interval = "5s"
+}
diff --git a/partition/roles/alloy/templates/snippets/journal-file.alloy.j2 b/partition/roles/alloy/templates/snippets/journal-file.alloy.j2
new file mode 100644
index 000000000..09611c2a9
--- /dev/null
+++ b/partition/roles/alloy/templates/snippets/journal-file.alloy.j2
@@ -0,0 +1,32 @@
+discovery.relabel "journal" {
+	targets = []
+
+	rule {
+		source_labels = ["__journal__systemd_unit"]
+		target_label  = "unit"
+	}
+
+	rule {
+		source_labels = ["__journal_priority_keyword"]
+		target_label  = "level"
+	}
+}
+
+loki.source.journal "journal" {
+	path          = "{{ alloy_journal_path }}"
+	relabel_rules = discovery.relabel.journal.rules
+	forward_to    = [loki.write.default.receiver]
+	labels        = {
+		job = "systemd-journal",
+	}
+{% if alloy_migrate_from_promtail %}
+	// migration only: resumes reading from where promtail left off on first start.
+	// ignored if the file does not exist on the host.
+	// once alloy has written its own positions file, this can be removed.
+	// https://grafana.com/docs/alloy/latest/reference/components/loki/loki.source.journal/#legacy_position
+	legacy_position {
+		file = "{{ alloy_journal_legacy_positions_file }}"
+		name = "{{ alloy_journal_legacy_position_name }}"
+	}
+{% endif %}
+}
diff --git a/partition/roles/alloy/templates/snippets/journal.alloy.j2 b/partition/roles/alloy/templates/snippets/journal.alloy.j2
new file mode 100644
index 000000000..f2f8d2cfb
--- /dev/null
+++ b/partition/roles/alloy/templates/snippets/journal.alloy.j2
@@ -0,0 +1,21 @@
+discovery.relabel "journal" {
+	targets = []
+
+	rule {
+		source_labels = ["__journal__systemd_unit"]
+		target_label  = "unit"
+	}
+
+	rule {
+		source_labels = ["__journal_priority_keyword"]
+		target_label  = "level"
+	}
+}
+
+loki.source.journal "journal" {
+	relabel_rules = discovery.relabel.journal.rules
+	forward_to    = [loki.write.default.receiver]
+	labels        = {
+		job = "systemd-journal",
+	}
+}
diff --git a/partition/roles/alloy/templates/snippets/syslog.alloy.j2 b/partition/roles/alloy/templates/snippets/syslog.alloy.j2
new file mode 100644
index 000000000..8b5be5509
--- /dev/null
+++ b/partition/roles/alloy/templates/snippets/syslog.alloy.j2
@@ -0,0 +1,18 @@
+loki.source.file "syslog" {
+	targets = [{
+		__path__ = "/var/log/syslog",
+		job      = "syslog",
+	}]
+	forward_to = [loki.write.default.receiver]
+
+	file_match {
+		enabled = true
+	}
+{% if alloy_migrate_from_promtail %}
+	// migration only: resumes tailing from where promtail left off on first start.
+	// ignored if the file does not exist on the host.
+	// once alloy has written its own positions file, this can be removed.
+	// https://grafana.com/docs/alloy/latest/reference/components/loki/loki.source.file/#arguments
+	legacy_positions_file = "{{ alloy_syslog_legacy_positions_file }}"
+{% endif %}
+}

From c83903a49fbb3653191be7d659fe15e44058f1e5 Mon Sep 17 00:00:00 2001
From: Matthias Hartmann <matthias.hartmann@x-cellent.com>
Date: Mon, 15 Jun 2026 17:39:07 +0200
Subject: [PATCH 39/41] fix: lint errors

---
 partition/roles/alloy/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/partition/roles/alloy/README.md b/partition/roles/alloy/README.md
index 1ce815b7e..e6defbcbe 100644
--- a/partition/roles/alloy/README.md
+++ b/partition/roles/alloy/README.md
@@ -120,7 +120,7 @@ The path is resolved through Ansible's normal template search path (relative to
 
 Example `templates/my_custom_source.alloy.j2`:
 
-```
+```river
 loki.source.file "custom" {
   targets    = [{ __path__ = "/var/log/myapp/*.log", job = "myapp" }]
   forward_to = [loki.write.default.receiver]

From 48d152b3ac946731774196d2e517a8c6cb4c9542 Mon Sep 17 00:00:00 2001
From: Matthias Hartmann <matthias.hartmann@x-cellent.com>
Date: Tue, 16 Jun 2026 13:21:05 +0200
Subject: [PATCH 40/41] chore: add comment for each snippet

---
 partition/roles/alloy/templates/config.alloy.j2 | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/partition/roles/alloy/templates/config.alloy.j2 b/partition/roles/alloy/templates/config.alloy.j2
index 51005dbe1..64fb580f0 100644
--- a/partition/roles/alloy/templates/config.alloy.j2
+++ b/partition/roles/alloy/templates/config.alloy.j2
@@ -30,8 +30,10 @@ loki.write "default" {
 }
 
 {% for snippet in alloy_config_snippets %}
+// snippet: {{ snippet }}
 {% include "snippets/" + snippet + ".alloy.j2" %}
 {% endfor %}
 {% for snippet in alloy_config_custom_snippets %}
+// snippet: {{ snippet }}
 {% include snippet %}
 {% endfor %}

From d8ecfb91a43e29ee62cb3869adce46d00636753d Mon Sep 17 00:00:00 2001
From: Matthias Hartmann <matthias.hartmann@x-cellent.com>
Date: Tue, 16 Jun 2026 13:23:21 +0200
Subject: [PATCH 41/41] chore: assert that only supported snippets are used

---
 partition/roles/alloy/tasks/main.yaml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/partition/roles/alloy/tasks/main.yaml b/partition/roles/alloy/tasks/main.yaml
index 57ffdbde4..231d3aae8 100644
--- a/partition/roles/alloy/tasks/main.yaml
+++ b/partition/roles/alloy/tasks/main.yaml
@@ -28,6 +28,13 @@
     that:
       - not (alloy_config_raw is defined and (alloy_config_snippets | length > 0 or alloy_config_custom_snippets | length > 0))
 
+- name: Check alloy_config_snippets only contains supported snippets
+  assert:
+    fail_msg: "alloy_config_snippets contains unsupported snippet(s): {{ alloy_config_snippets | difference(['syslog', 'journal', 'journal-file', 'docker', 'alloy-meta']) | join(', ') }}"
+    quiet: true
+    that:
+      - alloy_config_snippets | difference(['syslog', 'journal', 'journal-file', 'docker', 'alloy-meta']) | length == 0
+
 - name: Check journal and journal-file are not both enabled
   assert:
     fail_msg: "journal and journal-file snippets cannot both be enabled — this will produce duplicate log entries in Loki"