Align backend with html2rss auto fallback

Author: gildesmarais

app/web/api/v1/contract.rb

@@ -10,7 +10,8 @@ module Contract # rubocop:disable Metrics/ModuleLength
           CODES = {
             unauthorized: Html2rss::Web::UnauthorizedError::CODE,
             forbidden: Html2rss::Web::ForbiddenError::CODE,
-            internal_server_error: Html2rss::Web::InternalServerError::CODE
+            internal_server_error: Html2rss::Web::InternalServerError::CODE,
+            extraction_empty: 'EXTRACTION_EMPTY'
           }.freeze
 
           ERROR_KINDS = {
@@ -50,7 +51,10 @@ module Contract # rubocop:disable Metrics/ModuleLength
 
           MESSAGES = {
             auto_source_disabled: 'Auto source feature is disabled',
-            health_check_failed: 'Health check failed'
+            health_check_failed: 'Health check failed',
+            extraction_empty:
+              'We could not extract feed items from this page yet. ' \
+              'Try a more specific listing URL or explicit selectors.'
           }.freeze
 
           class << self
@@ -92,14 +96,26 @@ def warning(code:, message:, retryable:, next_action:)
             # @param error [StandardError]
             # @return [Hash{Symbol=>Object}]
             def failure_metadata(error)
-              case error
-              when Html2rss::Web::AutoSourceDisabledError, Html2rss::Web::HealthCheckFailedError
-                non_retryable_server_failure_metadata
-              when Html2rss::Web::UnauthorizedError then auth_failure_metadata
-              when Html2rss::Web::BadRequestError, Html2rss::Web::ForbiddenError then input_failure_metadata
-              else
-                generic_failure_metadata(error)
-              end
+              return input_failure_metadata if input_failure?(error)
+              return non_retryable_server_failure_metadata if non_retryable_server_failure?(error)
+              return auth_failure_metadata if error.is_a?(Html2rss::Web::UnauthorizedError)
+
+              generic_failure_metadata(error)
+            end
+
+            # @param error [StandardError]
+            # @return [Boolean]
+            def input_failure?(error)
+              Html2rss::Web::ErrorClassification.auto_fallback_exhausted?(error) ||
+                error.is_a?(Html2rss::Web::BadRequestError) ||
+                error.is_a?(Html2rss::Web::ForbiddenError)
+            end
+
+            # @param error [StandardError]
+            # @return [Boolean]
+            def non_retryable_server_failure?(error)
+              error.is_a?(Html2rss::Web::AutoSourceDisabledError) ||
+                error.is_a?(Html2rss::Web::HealthCheckFailedError)
             end
 
             # @param error [StandardError]
@@ -147,14 +163,24 @@ def non_retryable_server_failure_metadata
             # @param error [StandardError]
             # @return [String]
             def client_message_for(error)
+              return MESSAGES[:extraction_empty] if extraction_empty_failure?(error)
+
               error.is_a?(Html2rss::Web::HttpError) ? error.message : Html2rss::Web::HttpError::DEFAULT_MESSAGE
             end
 
             # @param error [StandardError]
             # @return [String]
             def error_code_for(error)
+              return CODES[:extraction_empty] if extraction_empty_failure?(error)
+
               error.respond_to?(:code) ? error.code : CODES[:internal_server_error]
             end
+
+            # @param error [StandardError]
+            # @return [Boolean]
+            def extraction_empty_failure?(error)
+              Html2rss::Web::ErrorClassification.auto_fallback_exhausted?(error)
+            end
           end
         end
       end

app/web/api/v1/feed_status.rb

@@ -82,18 +82,7 @@ def conversion_attributes(result) # rubocop:disable Metrics/MethodLength
                   warnings: []
                 }
               when :empty
-                {
-                  readiness_phase: Contract::READINESS_PHASES[:feed_ready],
-                  preview_status: Contract::PREVIEW_STATUSES[:degraded],
-                  warnings: [
-                    Contract.warning(
-                      code: 'preview_partial',
-                      message: 'Preview content could not be fully verified.',
-                      retryable: true,
-                      next_action: Contract::NEXT_ACTIONS[:retry]
-                    )
-                  ]
-                }
+                empty_conversion_attributes(result)
               when :error
                 if result.error_kind == :network
                   transient_conversion_attributes
@@ -105,6 +94,46 @@ def conversion_attributes(result) # rubocop:disable Metrics/MethodLength
               end
             end
 
+            # @param result [Html2rss::Web::Feeds::Contracts::RenderResult]
+            # @return [Hash{Symbol=>Object}]
+            def empty_conversion_attributes(result)
+              return extraction_empty_conversion_attributes if result.error_kind == :extraction_empty
+
+              degraded_empty_conversion_attributes
+            end
+
+            # @return [Hash{Symbol=>Object}]
+            def degraded_empty_conversion_attributes # rubocop:disable Metrics/MethodLength
+              {
+                readiness_phase: Contract::READINESS_PHASES[:feed_ready],
+                preview_status: Contract::PREVIEW_STATUSES[:degraded],
+                warnings: [
+                  Contract.warning(
+                    code: 'preview_partial',
+                    message: 'Preview content could not be fully verified.',
+                    retryable: true,
+                    next_action: Contract::NEXT_ACTIONS[:retry]
+                  )
+                ]
+              }
+            end
+
+            # @return [Hash{Symbol=>Object}]
+            def extraction_empty_conversion_attributes # rubocop:disable Metrics/MethodLength
+              {
+                readiness_phase: Contract::READINESS_PHASES[:preview_unavailable],
+                preview_status: Contract::PREVIEW_STATUSES[:unavailable],
+                warnings: [
+                  Contract.warning(
+                    code: 'preview_partial',
+                    message: Contract::MESSAGES[:extraction_empty],
+                    retryable: false,
+                    next_action: Contract::NEXT_ACTIONS[:correct_input]
+                  )
+                ]
+              }
+            end
+
             # @return [Hash{Symbol=>Object}]
             def transient_conversion_attributes # rubocop:disable Metrics/MethodLength
               {

app/web/domain/auto_source.rb

@@ -19,9 +19,9 @@ def enabled?
         # @param name [String, nil]
         # @param url [String]
         # @param token_data [Hash{Symbol=>Object}] authenticated account data.
-        # @param strategy [String]
+        # @param strategy [String, nil]
         # @return [Html2rss::Web::Api::V1::FeedMetadata::Metadata, nil]
-        def create_stable_feed(name, url, token_data, strategy = Html2rss::RequestService.default_strategy_name.to_s)
+        def create_stable_feed(name, url, token_data, strategy = nil)
           return nil unless token_data && FeedAccess.url_allowed_for_username?(token_data[:username], url)
 
           feed_token = Auth.generate_feed_token(token_data[:username], url, strategy: strategy)
@@ -35,7 +35,7 @@ def create_stable_feed(name, url, token_data, strategy = Html2rss::RequestServic
         # @param name [String, nil]
         # @param url [String]
         # @param token_data [Hash{Symbol=>Object}]
-        # @param strategy [String]
+        # @param strategy [String, nil]
         # @param feed_token [String]
         # @return [Hash{Symbol=>Object}]
         def metadata_attributes(name, url, token_data, strategy, feed_token)

app/web/error_classification.rb

@@ -23,6 +23,14 @@ def network_error?(error)
           unwrap(error).any? { |candidate| NETWORK_ERROR_CLASS_NAMES.include?(candidate.class.name) }
         end
 
+        # @param error [StandardError, nil]
+        # @return [Boolean]
+        def auto_fallback_exhausted?(error)
+          return false unless defined?(::Html2rss::NoFeedItemsExtracted)
+
+          unwrap(error).any?(::Html2rss::NoFeedItemsExtracted)
+        end
+
         private
 
         # @param error [StandardError, nil]

app/web/errors/error_responder.rb

@@ -87,21 +87,33 @@ def error_log_line(request, error)
         # @param error [StandardError]
         # @return [String]
         def resolve_error_code(error)
+          return Api::V1::Contract::CODES[:extraction_empty] if extraction_empty_failure?(error)
+
           error.respond_to?(:code) ? error.code : INTERNAL_ERROR_CODE
         end
 
         # @param error [StandardError]
         # @return [Integer]
         def resolve_status(error)
+          return 422 if extraction_empty_failure?(error)
+
           error.respond_to?(:status) ? error.status : 500
         end
 
         # @param error [StandardError]
         # @return [String]
         def client_message_for(error)
+          return Api::V1::Contract::MESSAGES[:extraction_empty] if extraction_empty_failure?(error)
+
           error.is_a?(Html2rss::Web::HttpError) ? error.message : Html2rss::Web::HttpError::DEFAULT_MESSAGE
         end
 
+        # @param error [StandardError]
+        # @return [Boolean]
+        def extraction_empty_failure?(error)
+          Html2rss::Web::ErrorClassification.auto_fallback_exhausted?(error)
+        end
+
         # @param request [Rack::Request]
         # @param error [StandardError]
         # @return [void]

app/web/feeds/responder.rb

@@ -93,7 +93,7 @@ def render_result(result, representation)
           # @return [void]
           def emit_result(target_kind:, identifier:, resolved_source:, result:)
             return emit_success(target_kind:, identifier:, resolved_source:) if result.status == :ok
-            return emit_empty(target_kind:, identifier:, resolved_source:) if result.status == :empty
+            return emit_empty(target_kind:, identifier:, resolved_source:, result:) if result.status == :empty
 
             emit_failure(
               target_kind:,
@@ -110,9 +110,10 @@ def emit_result(target_kind:, identifier:, resolved_source:, result:)
           # @return [void]
           def emit_success(target_kind:, identifier:, resolved_source:)
             details = {
-              strategy: resolved_source.generator_input[:strategy],
               url: resolved_source.generator_input.dig(:channel, :url)
             }
+            strategy = resolved_source.generator_input[:strategy]
+            details[:strategy] = strategy if strategy
             details[:feed_name] = identifier if target_kind == :static
 
             Observability.emit(event_name: 'feed.render', outcome: 'success', details:, level: :info)
@@ -121,18 +122,28 @@ def emit_success(target_kind:, identifier:, resolved_source:)
           # @param target_kind [Symbol]
           # @param identifier [String]
           # @param resolved_source [Html2rss::Web::Feeds::Contracts::ResolvedSource]
+          # @param result [Html2rss::Web::Feeds::Contracts::RenderResult]
           # @return [void]
-          def emit_empty(target_kind:, identifier:, resolved_source:)
+          def emit_empty(target_kind:, identifier:, resolved_source:, result:)
             details = {
-              strategy: resolved_source.generator_input[:strategy],
               url: resolved_source.generator_input.dig(:channel, :url),
-              reason: 'content_extraction_empty'
+              reason: empty_reason_for(result)
             }
+            strategy = resolved_source.generator_input[:strategy]
+            details[:strategy] = strategy if strategy
             details[:feed_name] = identifier if target_kind == :static
 
             Observability.emit(event_name: 'feed.render', outcome: 'failure', details:, level: :warn)
           end
 
+          # @param result [Html2rss::Web::Feeds::Contracts::RenderResult]
+          # @return [String]
+          def empty_reason_for(result)
+            return 'content_extraction_empty' if result.error_kind == :extraction_empty
+
+            'feed_empty'
+          end
+
           # @param target_kind [Symbol]
           # @param identifier [String]
           # @param error [StandardError]

app/web/feeds/service.rb

@@ -30,6 +30,8 @@ def build_result(resolved_source, cache_key)
             feed = Html2rss.feed(resolved_source.generator_input)
             success_result(feed, resolved_source, cache_key)
           rescue StandardError => error
+            return empty_result(error, resolved_source, cache_key) if extraction_empty_error?(error)
+
             error_result(error, resolved_source, cache_key)
           end
 
@@ -69,7 +71,7 @@ def payload_for(feed, resolved_source)
               feed: feed,
               site_title: site_title_for(feed, resolved_source.generator_input.dig(:channel, :url)),
               url: resolved_source.generator_input.dig(:channel, :url),
-              strategy: resolved_source.generator_input[:strategy].to_s
+              strategy: resolved_source.generator_input[:strategy]&.to_s
             )
           end
 
@@ -98,6 +100,39 @@ def error_result(error, resolved_source, cache_key)
               error_kind: Html2rss::Web::ErrorClassification.network_error?(error) ? :network : :server
             )
           end
+
+          # @param error [Html2rss::NoFeedItemsExtracted]
+          # @param resolved_source [Html2rss::Web::Feeds::Contracts::ResolvedSource]
+          # @param cache_key [String]
+          # @return [Html2rss::Web::Feeds::Contracts::RenderResult]
+          def empty_result(error, resolved_source, cache_key)
+            Contracts::RenderResult.new(
+              status: :empty,
+              payload: payload_for_empty_result(resolved_source),
+              message: nil,
+              ttl_seconds: resolved_source.ttl_seconds,
+              cache_key: cache_key,
+              error_message: error.message,
+              error_kind: :extraction_empty
+            )
+          end
+
+          # @param error [StandardError]
+          # @return [Boolean]
+          def extraction_empty_error?(error)
+            Html2rss::Web::ErrorClassification.auto_fallback_exhausted?(error)
+          end
+
+          # @param resolved_source [Html2rss::Web::Feeds::Contracts::ResolvedSource]
+          # @return [Html2rss::Web::Feeds::Contracts::RenderPayload]
+          def payload_for_empty_result(resolved_source)
+            Contracts::RenderPayload.new(
+              feed: nil,
+              site_title: resolved_source.generator_input.dig(:channel, :url).to_s,
+              url: resolved_source.generator_input.dig(:channel, :url),
+              strategy: resolved_source.generator_input[:strategy]&.to_s
+            )
+          end
         end
       end
     end

app/web/feeds/source_resolver.rb

@@ -69,7 +69,6 @@ def static_cache_identity(feed_name, params)
           def static_generator_input(config, params)
             generator_input = config.dup
             generator_input[:params] = merged_static_params(config, params)
-            generator_input[:strategy] ||= Html2rss::RequestService.default_strategy_name.to_sym
             generator_input
           end
 
@@ -96,27 +95,34 @@ def ensure_auto_source_enabled!
           end
 
           # @param feed_token [Html2rss::Web::FeedToken]
-          # @return [String]
+          # @return [String, nil]
           def resolved_strategy(feed_token)
             strategy = feed_token.strategy.to_s.strip
-            strategy = Html2rss::RequestService.default_strategy_name.to_s if strategy.empty?
+            return nil if strategy.empty?
+            return strategy if strategy == default_strategy_name
+
             supported = Html2rss::RequestService.strategy_names.map(&:to_s)
             raise Html2rss::Web::BadRequestError, 'Unsupported strategy' unless supported.include?(strategy)
 
             strategy
           end
 
           # @param url [String]
-          # @param strategy [String]
+          # @param strategy [String, nil]
           # @return [Hash{Symbol=>Object}]
           def token_generator_input(url, strategy)
-            LocalConfig.global
-                       .slice(:stylesheets, :headers)
-                       .merge(
-                         strategy: strategy.to_sym,
-                         channel: { url: url },
-                         auto_source: {}
-                       )
+            global_config = LocalConfig.global
+            base_input = global_config.slice(:stylesheets, :headers)
+            generator_input = base_input.merge(channel: { url: url }, auto_source: {})
+            generator_input[:strategy] = strategy.to_sym if strategy
+            generator_input
+          end
+
+          # @return [String]
+          def default_strategy_name
+            return Html2rss::Config.default_strategy_name.to_s if Html2rss::Config.respond_to?(:default_strategy_name)
+
+            'auto'
           end
         end
       end

app/web/security/auth.rb

@@ -24,10 +24,10 @@ def authenticate(request)
 
         # @param username [String]
         # @param url [String]
-        # @param strategy [String]
+        # @param strategy [String, nil]
         # @param expires_in [Integer] seconds (default: 10 years)
         # @return [String, nil] signed feed token when generation succeeds.
-        def generate_feed_token(username, url, strategy:, expires_in: FeedToken::DEFAULT_EXPIRY)
+        def generate_feed_token(username, url, strategy: nil, expires_in: FeedToken::DEFAULT_EXPIRY)
           token = FeedToken.create_with_validation(
             username: username,
             url: url,