add support for automated splunk update when new version is available

modules/enableit/common/manifests/monitor/splunk/forwarder.pp

@@ -2,9 +2,14 @@
 #
 # @param password_hash The password hash for the forwarder. This parameter is required.
 #
-# @param version The version of the Splunk forwarder. Defaults to '7.2.4'.
+# @param version The version of the Splunk forwarder to pin. When left undef (the default),
+#   the latest Splunk Universal Forwarder release is tracked automatically instead (checked
+#   via `splunk_forwarder_latest_version()`, cached for 6h). Must be set together with
+#   `build`, or left undef together with it - setting only one fails the catalog.
 #
-# @param build The build identifier. Defaults to '8a94541dcfac'.
+# @param build The build identifier matching `version`. Only used when `version` is given -
+#   ignored (and looked up automatically) when `version` is undef. Must be set together with
+#   `version`, or left undef together with it - setting only one fails the catalog.
 #
 # @param deploymentserver The deployment server URL. Defaults to undef.
 #
@@ -37,21 +42,32 @@
 # @groups addons addons
 #
 class common::monitor::splunk::forwarder (
-  String[1]             $password_hash,
-  Eit_types::Version    $version             = '7.2.4',
-  Optional[String]      $build               = '8a94541dcfac',
-  Stdlib::HTTPUrl       $deploymentserver    = undef,
-  Boolean               $seed_password       = true,
-  Integer               $log_keep_count      = 5,
-  Eit_types::Bytes      $log_max_file_size_b = 25000000,
-  Boolean               $enable              = false,
-  Boolean               $manage              = false,
-  Eit_types::Noop_Value $noop_value          = undef,
-  Hash[String[1], Hash] $addons              = {},
+  String[1]                    $password_hash,
+  Optional[Eit_types::Version] $version             = undef,
+  Optional[String]             $build               = undef,
+  Stdlib::HTTPUrl              $deploymentserver    = undef,
+  Boolean                      $seed_password       = true,
+  Integer                      $log_keep_count      = 5,
+  Eit_types::Bytes             $log_max_file_size_b = 25000000,
+  Boolean                      $enable              = false,
+  Boolean                      $manage              = false,
+  Eit_types::Noop_Value        $noop_value          = undef,
+  Hash[String[1], Hash]        $addons              = {},
 ) {
 
+  # Must be pinned together, or the unset one falls back to splunk::params's hardcoded default and mismatches.
+  if ($version =~ NotUndef) != ($build =~ NotUndef) {
+    fail("common::monitor::splunk::forwarder: version and build must both be set or both be left undef (got version=${version}, build=${build})")
+  }
+
+  # Left undef, the function tracks Splunk's latest published release instead of the pin (cached 6h).
+  $_latest = splunk_forwarder_latest_version($version, $build)
 
   if $manage {
+    class { 'profile::collector::splunk::forwarder':
+      version => $_latest['version'],
+      build   => $_latest['build'],
+    }
     contain profile::collector::splunk::forwarder
   }
 }

modules/enableit/functions/lib/puppet/functions/splunk_forwarder_latest_version.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "json"
+require "time"
+
+# @summary
+#
+#   Resolve the Splunk Universal Forwarder version+build to install. Given
+#   both version and build, returns them as-is (a pin). Given neither, looks
+#   up the latest release by scraping Splunk's own download page (there is
+#   no stable "latest" download URL or public API for this) - result is
+#   cached on disk (compiler-side) for CACHE_TTL_SECONDS so we don't hit
+#   Splunk's site on every catalog compile.
+#
+#   On any lookup failure (network, parse, Splunk page layout change) this
+#   falls back to a stale cache if one exists, or a hardcoded last-known-good
+#   version if not - it never raises, since that would break catalog
+#   compilation for every node using this class.
+#
+# Real module keyword (not the create_function do...end block below) so these get proper
+# lexical scoping - constants assigned directly inside create_function's block leak onto
+# top-level Object in the puppetserver JRuby, since Class.new-style blocks don't nest in
+# Module.nesting the way `module`/`class` do.
+module SplunkForwarderLatestVersion
+  DOWNLOAD_PAGE = "https://www.splunk.com/en_us/download/universal-forwarder.html"
+  CACHE_TTL_SECONDS = 6 * 60 * 60 # 6 hours
+  # Used only if there is no cache at all yet AND the live fetch also fails.
+  FALLBACK = { "version" => "10.4.2", "build" => "33c3bf42cd73" }.freeze
+
+  # Puppet[:vardir] is writable by the compiling process by construction, unlike
+  # /opt/obmondo/cache - which nothing in this repo creates or manages on the
+  # puppetserver (only on agent nodes, via common::init's $__opt_dir), so
+  # write_cache would silently fail there and force a live HTTP fetch on every
+  # catalog compile. Puppet[:vardir] is a runtime setting, not a static
+  # literal, so this is a method rather than a constant.
+  def self.cache_path
+    File.join(Puppet[:vardir], "splunk_forwarder_latest_version.json")
+  end
+end
+
+Puppet::Functions.create_function(:splunk_forwarder_latest_version) do
+  dispatch :latest do
+    required_param 'Optional[String[1]]', :version
+    required_param 'Optional[String[1]]', :build
+    return_type 'Struct[{version => String[1], build => String[1]}]'
+  end
+
+  def latest(version, build)
+    return { "version" => version, "build" => build } if version && build
+
+    cached = read_cache
+    return cached.slice("version", "build") if cached && !stale?(cached)
+
+    begin
+      fetched = fetch_latest
+      write_cache(fetched)
+      fetched
+    rescue StandardError => e
+      Puppet.warning("splunk_forwarder_latest_version: failed to fetch latest release (#{e.message}), " \
+                      "falling back to #{cached ? 'stale cache' : 'hardcoded default'}")
+      cached ? cached.slice("version", "build") : SplunkForwarderLatestVersion::FALLBACK
+    end
+  end
+
+  private
+
+  def fetch_latest
+    uri = URI(SplunkForwarderLatestVersion::DOWNLOAD_PAGE)
+    response = Net::HTTP.start(uri.host, uri.port, use_ssl: true, open_timeout: 5, read_timeout: 10) do |http|
+      http.get(uri)
+    end
+    raise "unexpected response #{response.code}" unless response.code.to_i == 200
+
+    # Match the linux x86_64 rpm link, e.g.
+    # .../releases/10.4.2/linux/splunkforwarder-10.4.2-33c3bf42cd73.x86_64.rpm
+    match = response.body.match(%r{splunkforwarder-(?<version>\d+(?:\.\d+)*)-(?<build>[0-9a-f]{12})\.x86_64\.rpm})
+    raise "could not find a version/build match on the download page" unless match
+
+    { "version" => match[:version], "build" => match[:build] }
+  end
+
+  def read_cache
+    return nil unless File.exist?(SplunkForwarderLatestVersion.cache_path)
+
+    JSON.parse(File.read(SplunkForwarderLatestVersion.cache_path))
+  rescue StandardError
+    nil
+  end
+
+  def stale?(cached)
+    (Time.now - Time.parse(cached["fetched_at"])) > SplunkForwarderLatestVersion::CACHE_TTL_SECONDS
+  rescue StandardError
+    true
+  end
+
+  def write_cache(data)
+    # No mkdir_p needed - Puppet[:vardir] always exists already. Unique per-writer temp filename,
+    # since puppetserver compiles many catalogs concurrently in one JVM (Process.pid alone
+    # wouldn't differentiate threads in the same process) - a shared fixed temp name would let
+    # concurrent writers splice/truncate each other's write before either renames it into place.
+    tmp_path = "#{SplunkForwarderLatestVersion.cache_path}.#{Process.pid}.#{Thread.current.object_id}.tmp"
+    File.write(tmp_path, data.merge("fetched_at" => Time.now.utc.iso8601).to_json)
+    File.rename(tmp_path, SplunkForwarderLatestVersion.cache_path)
+  rescue StandardError => e
+    Puppet.warning("splunk_forwarder_latest_version: failed to write cache (#{e.message})")
+  ensure
+    File.unlink(tmp_path) if tmp_path && File.exist?(tmp_path)
+  end
+end

modules/enableit/profile/manifests/collector/splunk/forwarder.pp

@@ -25,6 +25,9 @@
   Exec {
     noop => $noop_value,
   }
+  Service {
+    noop => $noop_value,
+  }
 
   # Create group
   group { 'splunkfwd':
@@ -53,6 +56,12 @@
     }
 
     class { 'splunk::forwarder':
+      # dpkg/rpm's `latest` reads the version straight from the staged source file (the archive we
+      # just downloaded for $version/$build) and, if it differs from what's installed, upgrades or
+      # downgrades in place via `dpkg -i`/`rpm -U --oldpackage` - no purge needed, and existing
+      # etc/system/local/* (incl. the seeded admin credentials) isn't touched since it's not a
+      # packaged conffile.
+      package_ensure   => 'latest',
       seed_password    => $seed_password,
       splunk_user      => 'splunkfwd',
       password_hash    => $password_hash,
@@ -68,6 +77,11 @@
       }
     }
 
+    # Default 300s exec timeout is too short for a fresh install's first start; override without patching the vendored module.
+    Exec <| title == 'splunk-forwarder-accept-tos' |> {
+      timeout => 900,
+    }
+
     splunkforwarder_deploymentclient { 'target-broker:deploymentServer':
       setting => 'targetUri',
       value   => $deploymentserver,