chore: replace partition promtail with alloy

common/roles/defaults/defaults/main.yaml

@@ -1,10 +1,10 @@
 ---
 metal_registry_auth_enabled: false
 metal_registry_auth_user: ""
 metal_registry_auth_password: ""
 metal_registry_url: "https://index.docker.io/v1/"
 
 metal_stack_release:
   mapping:
     # binaries
     metal_hammer_image_tag: "binaries.metal-stack.metal-hammer.version"
@@ -98,6 +98,8 @@
     capms_controller_name: "docker-images.metal-stack.kubernetes.cluster-api-provider-metal-stack-controller.name"
     capms_controller_tag: "docker-images.metal-stack.kubernetes.cluster-api-provider-metal-stack-controller.tag"
     # third party
+    alloy_image_tag: "docker-images.third-party.partition.alloy.tag"
+    alloy_image_name: "docker-images.third-party.partition.alloy.name"
     event_exporter_name: "docker-images.third-party.control-plane.event-exporter.name"
     event_exporter_tag: "docker-images.third-party.control-plane.event-exporter.tag"
     nsq_image_tag: "docker-images.third-party.control-plane.nsq.tag"

partition/README.md

@@ -40,7 +40,7 @@ You can look up all the default values [here](partition-defaults/main.yaml).
 | [metal-bmc](roles/metal-bmc)                 | Deploys metal-bmc                           |
 | [metal-core](roles/metal-core)               | Deploys metal-core                          |
 | [pixiecore](roles/pixiecore)                 | Deploys pixiecore                           |
-| [promtail](roles/promtail)                   | Deploys promtail                            |
+| [alloy](roles/alloy)                         | Deploys alloy                               |
 
 ## Examples
 

partition/roles/alloy/defaults/main.yaml

@@ -0,0 +1,52 @@
+---
+# Set to true to deploy and start the Alloy service.
+# Set to false to skip service deployment — useful when alloy is being staged or promtail is still running.
+alloy_enabled: false
+
+alloy_config_host_dir: "/etc/alloy"
+alloy_docker_log_driver: json-file
+
+# List of Loki push endpoints used by the base loki.write component.
+# Each entry: {url: <push-url>, remote_timeout?: <duration>, basic_auth?: {username, password}}
+# alloy_loki_write_endpoints:
+#   - url: http://loki.example.com:8080/loki/api/v1/push
+#     remote_timeout: 10s
+#     basic_auth:
+#       username: "username"
+#       password: "password"
+
+# List of snippet names to enable. Each entry must correspond to
+# templates/snippets/<name>.alloy.j2 in this role.
+alloy_config_snippets: []
+
+# Provide a fully custom Alloy River config as a string to bypass the
+# snippet assembly system entirely. When set, alloy_loki_write_endpoints
+# and alloy_config_snippets are ignored.
+# alloy_config_raw: |
+
+# Port to listen for metrics and HTTP API traffic on
+alloy_port: 12345
+
+# Enable migration mode: imports cursor state from the legacy promtail positions file
+# on first start so Alloy resumes from where promtail left off. Set to true when
+# migrating from promtail; leave false for fresh deployments.
+alloy_migrate_from_promtail: false
+
+# Path to the legacy promtail positions file used by the syslog snippet on first start
+# after migration to continue tailing /var/log/syslog from where promtail left off.
+# Only active when alloy_migrate_from_promtail is true.
+alloy_syslog_legacy_positions_file: "/var/log/promtail-positions.yaml"
+
+# Path to the persistent journal directory used by the journal-file snippet.
+alloy_journal_path: "/var/log/journal"
+
+# Path to the legacy promtail positions file used by the journal-file snippet on first
+# start after migration to resume from where promtail left off.
+# Only active when alloy_migrate_from_promtail is true.
+alloy_journal_legacy_positions_file: "/var/log/promtail-positions.yaml"
+
+# Job name used in the legacy promtail positions file for the journal scrape config.
+# Must match the job_name of the journal scrape_config in your old promtail config.
+# Required when alloy_migrate_from_promtail is true and the journal-file snippet is used.
+# No default — must be set explicitly.
+# alloy_journal_legacy_position_name:

partition/roles/alloy/handlers/main.yaml

@@ -0,0 +1,7 @@
+---
+- name: restart alloy
+  service:
+    name: alloy
+    enabled: true
+    state: restarted
+  when: alloy_enabled | default(false)

partition/roles/alloy/meta/main.yml

@@ -0,0 +1,17 @@
+---
+galaxy_info:
+  role_name: alloy
+  author: metal-stack
+  description: Deploys alloy.
+  license: MIT
+  min_ansible_version: "2.10"
+  galaxy_tags: []
+
+  platforms:
+    - name: Debian
+      versions:
+        - all
+
+dependencies:
+  - role: metal-roles/common/roles/defaults
+  - role: metal-roles/partition/roles/defaults

partition/roles/alloy/tasks/main.yaml

@@ -0,0 +1,124 @@
+---
+- name: Gather release versions
+  setup_yaml:
+
+- name: Check mandatory variables for this role are set
+  assert:
+    fail_msg: "not all mandatory variables given, check role documentation"
+    quiet: true
+    that:
+      - alloy_image_tag is defined
+      - alloy_image_name is defined
+      - alloy_config_raw is defined or (alloy_loki_write_endpoints is defined and (alloy_loki_write_endpoints | length > 0))
+
+- name: Check alloy_journal_legacy_position_name is set when migrating journal-file from promtail
+  assert:
+    fail_msg: "alloy_journal_legacy_position_name must be set — it must match the job_name of the journal scrape_config in your old promtail config"
+    quiet: true
+    that:
+      - alloy_journal_legacy_position_name is defined
+  when:
+    - "'journal-file' in alloy_config_snippets"
+    - alloy_migrate_from_promtail | default(false)
+
+- name: Check journal and journal-file are not both enabled
+  assert:
+    fail_msg: "journal and journal-file snippets cannot both be enabled — this will produce duplicate log entries in Loki"
+    quiet: true
+    that:
+      - not ('journal' in alloy_config_snippets and 'journal-file' in alloy_config_snippets)
+
+- name: Create alloy config directory
+  file:
+    path: "{{ alloy_config_host_dir }}"
+    state: directory
+    owner: "nobody"
+    group: "nogroup"
+    mode: "0755"
+
+- name: Create alloy config snippets directory
+  file:
+    path: "{{ alloy_config_host_dir }}/conf.d"
+    state: directory
+    owner: "nobody"
+    group: "nogroup"
+    mode: "0755"
+
+- name: Template alloy base configuration
+  template:
+    src: config.alloy.j2
+    dest: "{{ alloy_config_host_dir }}/conf.d/00-base.alloy"
+  when: alloy_config_raw is not defined
+
+- name: Find existing alloy config snippet files
+  find:
+    paths: "{{ alloy_config_host_dir }}/conf.d"
+    patterns: "50-*.alloy"
+  register: existing_snippets
+  when: alloy_config_raw is not defined
+
+- name: Remove stale alloy config snippets
+  file:
+    path: "{{ item.path }}"
+    state: absent
+  loop: "{{ existing_snippets.files | default([]) }}"
+  when:
+    - alloy_config_raw is not defined
+    - item.path | basename | regex_replace('^50-(.+)\.alloy$', '\1') not in alloy_config_snippets
+
+- name: Template alloy config snippets
+  template:
+    src: "snippets/{{ item }}.alloy.j2"
+    dest: "{{ alloy_config_host_dir }}/conf.d/50-{{ item }}.alloy"
+  loop: "{{ alloy_config_snippets }}"
+  when: alloy_config_raw is not defined
+
+- name: Assemble alloy config
+  assemble:
+    src: "{{ alloy_config_host_dir }}/conf.d"
+    dest: "{{ alloy_config_host_dir }}/config.alloy"
+    delimiter: "\n"
+  notify: restart alloy
+  when: alloy_config_raw is not defined
+
+- name: Write raw alloy config
+  copy:
+    content: "{{ alloy_config_raw }}"
+    dest: "{{ alloy_config_host_dir }}/config.alloy"
+  notify: restart alloy
+  when: alloy_config_raw is defined
+
+- name: Create alloy storage directory
+  file:
+    path: /var/lib/alloy
+    state: directory
+    owner: "nobody"
+    group: "nogroup"
+    mode: "0755"
+  when: alloy_enabled | default(false)
+
+- name: Deploy alloy service
+  include_role:
+    name: ansible-common/roles/systemd-docker-service
+  vars:
+    systemd_service_name: alloy
+    systemd_docker_image_name: "{{ alloy_image_name }}"
+    systemd_docker_image_tag: "{{ alloy_image_tag }}"
+    systemd_service_after: docker.service
+    systemd_service_requires: docker.service
+    systemd_docker_network: host
+    systemd_docker_volumes:
+      - "{{ alloy_config_host_dir }}:/etc/alloy:ro"
+      - "/var/lib/alloy:/var/lib/alloy"
+      - "/var/log:/var/log"
+      - "/run/log/journal:/run/log/journal:ro"
+      - "/etc/machine-id:/etc/machine-id:ro"
+      - "/var/run/docker.sock:/var/run/docker.sock"
+    systemd_docker_command:
+      - run
+      - --server.http.listen-addr=0.0.0.0:{{ alloy_port }}
+      - --storage.path=/var/lib/alloy
+      - --disable-reporting
+      - /etc/alloy/config.alloy
+    systemd_docker_log_driver: "{{ alloy_docker_log_driver }}"
+  when: alloy_enabled | default(false)

partition/roles/alloy/templates/config.alloy.j2

@@ -0,0 +1,30 @@
+// Log level and format; write_to forwards Alloy's own logs to Loki when alloy-meta is enabled
+logging {
+	level  = "info"
+	format = "logfmt"
+{% if "alloy-meta" in alloy_config_snippets %}
+	write_to = [loki.relabel.alloy_self.receiver]
+{% endif %}
+}
+
+// Loki write endpoint shared by all snippets; external_labels are appended to every log entry
+loki.write "default" {
+	external_labels = {
+		host      = "{{ inventory_hostname }}",
+		partition = "{{ metal_partition_id }}",
+	}
+{% for endpoint in alloy_loki_write_endpoints %}
+	endpoint {
+		url = "{{ endpoint.url }}"
+{%if endpoint.remote_timeout is defined %}
+		remote_timeout = "{{ endpoint.remote_timeout }}"
+{% endif %}
+{% if endpoint.basic_auth is defined %}
+		basic_auth {
+			username = "{{ endpoint.basic_auth.username }}"
+			password = "{{ endpoint.basic_auth.password }}"
+		}
+{% endif %}
+	}
+{% endfor %}
+}

partition/roles/alloy/templates/snippets/alloy-meta.alloy.j2

@@ -0,0 +1,9 @@
+// Meta-monitoring: forward Alloy's own logs to Loki
+loki.relabel "alloy_self" {
+	forward_to = [loki.write.default.receiver]
+
+	rule {
+		target_label = "job"
+		replacement  = "alloy"
+	}
+}

partition/roles/alloy/templates/snippets/docker.alloy.j2

@@ -0,0 +1,27 @@
+discovery.docker "docker" {
+	host             = "unix:///var/run/docker.sock"
+	refresh_interval = "5s"
+}
+
+discovery.relabel "docker" {
+	targets = []
+
+	rule {
+		source_labels = ["__meta_docker_container_name"]
+		regex         = "/(.*)"
+		target_label  = "container"
+	}
+
+	rule {
+		target_label = "job"
+		replacement  = "docker"
+	}
+}
+
+loki.source.docker "docker" {
+	host             = "unix:///var/run/docker.sock"
+	targets          = discovery.docker.docker.targets
+	forward_to       = [loki.write.default.receiver]
+	relabel_rules    = discovery.relabel.docker.rules
+	refresh_interval = "5s"
+}

partition/roles/alloy/templates/snippets/journal-file.alloy.j2

@@ -0,0 +1,32 @@
+discovery.relabel "journal" {
+	targets = []
+
+	rule {
+		source_labels = ["__journal__systemd_unit"]
+		target_label  = "unit"
+	}
+
+	rule {
+		source_labels = ["__journal_priority_keyword"]
+		target_label  = "level"
+	}
+}
+
+loki.source.journal "journal" {
+	path          = "{{ alloy_journal_path }}"
+	relabel_rules = discovery.relabel.journal.rules
+	forward_to    = [loki.write.default.receiver]
+	labels        = {
+		job = "systemd-journal",
+	}
+{% if alloy_migrate_from_promtail | default(false) %}
+	// migration only: resumes reading from where promtail left off on first start.
+	// ignored if the file does not exist on the host.
+	// once alloy has written its own positions file, this can be removed.
+	// https://grafana.com/docs/alloy/latest/reference/components/loki/loki.source.journal/#legacy_position
+	legacy_position {
+		file = "{{ alloy_journal_legacy_positions_file }}"
+		name = "{{ alloy_journal_legacy_position_name }}"
+	}
+{% endif %}
+}

partition/roles/alloy/templates/snippets/journal.alloy.j2

@@ -0,0 +1,21 @@
+discovery.relabel "journal" {
+	targets = []
+
+	rule {
+		source_labels = ["__journal__systemd_unit"]
+		target_label  = "unit"
+	}
+
+	rule {
+		source_labels = ["__journal_priority_keyword"]
+		target_label  = "level"
+	}
+}
+
+loki.source.journal "journal" {
+	relabel_rules = discovery.relabel.journal.rules
+	forward_to    = [loki.write.default.receiver]
+	labels        = {
+		job = "systemd-journal",
+	}
+}

partition/roles/alloy/templates/snippets/syslog.alloy.j2

@@ -0,0 +1,18 @@
+loki.source.file "syslog" {
+	targets = [{
+		__path__ = "/var/log/syslog",
+		job      = "syslog",
+	}]
+	forward_to = [loki.write.default.receiver]
+
+	file_match {
+		enabled = true
+	}
+{% if alloy_migrate_from_promtail | default(false) %}
+	// migration only: resumes tailing from where promtail left off on first start.
+	// ignored if the file does not exist on the host.
+	// once alloy has written its own positions file, this can be removed.
+	// https://grafana.com/docs/alloy/latest/reference/components/loki/loki.source.file/#arguments
+	legacy_positions_file = "{{ alloy_syslog_legacy_positions_file }}"
+{% endif %}
+}

partition/roles/monitoring/prometheus/README.md

@@ -22,7 +22,7 @@ you define them adequately as well.
 | prometheus_frr_exporter_targets                  |           | FRR exporter targets to scrape from                                                                                                         |
 | prometheus_metal_core_targets                    |           | metal-core targets to scrape from                                                                                                           |
 | prometheus_node_exporter_targets                 |           | Node exporter targets to scrape from                                                                                                        |
-| prometheus_promtail_targets                      |           | Promtail targets to scrape from                                                                                                             |
+| prometheus_alloy_targets                         |           | Alloy targets to scrape from (default port `12345`)                                                                                         |
 | prometheus_ping_targets                          |           | Ping targets to scrape from                                                                                                                 |
 | prometheus_sonic_exporter_targets                |           | Sonic exporter targets to scrape from                                                                                                       |
 | prometheus_blackbox_exporter_targets             |           | Blackbox exporter targets to scrape from                                                                                                    |

partition/roles/monitoring/prometheus/defaults/main.yaml

@@ -16,10 +16,16 @@ prometheus_alertmanager_target:
 prometheus_alertmanager_basic_auth_username: alertmanager
 prometheus_alertmanager_basic_auth_password:
 prometheus_remote_write: []
+  # - url: "https://monitoring.{{ metal_control_plane_ingress_dns }}/api/v1/receive"
+  #   basic_auth:
+  #     username: thanos-receive
+  #     password: "{{ basic_auth_pass }}"
+
 prometheus_frr_exporter_targets: []
 prometheus_metal_core_targets: []
 prometheus_node_exporter_targets: []
 prometheus_promtail_targets: []
+prometheus_alloy_targets: []
 prometheus_ping_targets: []
 prometheus_ipmi_exporter_targets: []
 prometheus_sonic_exporter_targets: []