Improves a visibility the case events routed to DLQ.

CHANGELOG.md

@@ -1,3 +1,6 @@
+## 12.1.3
+ - Improves the logging experience when DLQ used [#1253](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1253)
+
 ## 12.1.2
  - Fix: replace deprecated `File.exists?` with `File.exist?` for Ruby 3.4 (JRuby 10) compatibility [#1238](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1238)
 

lib/logstash/plugin_mixins/elasticsearch/common.rb

@@ -273,6 +273,7 @@ def submit(actions)
       end
 
       actions_to_retry = []
+      dlq_reouted_stats = {}
       responses.each_with_index do |response,idx|
         action_type, action_props = response.first
 
@@ -296,6 +297,11 @@ def submit(actions)
         elsif @dlq_codes.include?(status)
           handle_dlq_response("Could not index event to Elasticsearch.", action, status, response)
           @document_level_metrics.increment(:dlq_routed)
+          if dlq_reouted_stats.key?(status)
+            dlq_reouted_stats[status][:count] += 1
+          else
+            dlq_reouted_stats[status] = { :count => 1, :sample_event => { :action => action, :response => response } }
+          end
           next
         else
           # only log what the user whitelisted
@@ -305,6 +311,11 @@ def submit(actions)
         end
       end
 
+      if @dlq_writer && !dlq_reouted_stats.empty?
+        total = dlq_reouted_stats.values.sum { |entry| entry[:count] }
+        @logger.warn("Events could not be indexed and routing to DLQ", :count => total, :dlq_reouted_stats => dlq_reouted_stats)
+      end
+
       actions_to_retry
     end
 

logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '12.1.2'
+  s.version         = '12.1.3'
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"