[11.x] Improves a visibility the case events routed to DLQ. (#1253)

CHANGELOG.md

@@ -1,3 +1,6 @@
+## 11.22.15
+ - Improves the logging experience when DLQ used [#1258](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1258)
+
 ## 11.22.14
  - Fix: replace deprecated `File.exists?` with `File.exist?` for Ruby 3.4 (JRuby 10) compatibility [#1238](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1238)
 

lib/logstash/plugin_mixins/elasticsearch/common.rb

@@ -241,7 +241,6 @@ def handle_dlq_response(message, action, status, response)
         @dlq_writer.write(event, "#{detailed_message}")
       else
         log_level = dig_value(response, 'index', 'error', 'type') == 'invalid_index_name_exception' ? :error : :warn
-
         @logger.public_send(log_level, message, status: status, action: action_params, response: response)
       end
     end
@@ -273,6 +272,7 @@ def submit(actions)
       end
 
       actions_to_retry = []
+      dlq_routed_stats = {}
       responses.each_with_index do |response,idx|
         action_type, action_props = response.first
 
@@ -294,6 +294,11 @@ def submit(actions)
         elsif @dlq_codes.include?(status)
           handle_dlq_response("Could not index event to Elasticsearch.", action, status, response)
           @document_level_metrics.increment(:dlq_routed)
+          if dlq_routed_stats.key?(status)
+            dlq_routed_stats[status][:count] += 1
+          else
+            dlq_routed_stats[status] = { :count => 1, :sample_event => { :action => action, :response => response } }
+          end
           next
         else
           # only log what the user whitelisted
@@ -303,6 +308,11 @@ def submit(actions)
         end
       end
 
+      if @dlq_writer && !dlq_routed_stats.empty?
+        total = dlq_routed_stats.values.sum { |entry| entry[:count] }
+        @logger.warn("Events could not be indexed and routing to DLQ", :count => total, :dlq_routed_stats => dlq_routed_stats)
+      end
+
       actions_to_retry
     end
 

logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '11.22.14'
+  s.version         = '11.22.15'
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"