-
-
Notifications
You must be signed in to change notification settings - Fork 16
feat: redact sensitive feed data in structured logs #903
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 5 commits
c70861a
1f316d2
77b3358
40144e2
996fa53
fb6e123
2dc9b7b
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,82 @@ | ||
| # frozen_string_literal: true | ||
|
|
||
| require 'digest' | ||
| require 'html2rss/url' | ||
|
|
||
| module Html2rss | ||
| module Web | ||
| ## | ||
| # Sanitizes request paths and log payloads before they are emitted. | ||
| module LogSanitizer | ||
| FEED_TOKEN_ROUTE = %r{\A(/api/v1/feeds/)([^/?]+)\z} | ||
|
|
||
| class << self | ||
| # @param path [String, nil] | ||
| # @return [String, nil] | ||
| def sanitize_path(path) | ||
| return if path.nil? | ||
|
|
||
| path_string = path.to_s | ||
| suffix = feed_suffix(path_string) | ||
| token_path = suffix ? path_string.delete_suffix(suffix) : path_string | ||
|
|
||
| token_path.gsub(FEED_TOKEN_ROUTE, "\\1[REDACTED]#{suffix}") | ||
| end | ||
|
Comment on lines
+16
to
+20
|
||
|
|
||
| # @param details [Hash] | ||
| # @return [Hash] | ||
| def sanitize_details(details) | ||
| details.each_with_object({}) do |(key, value), sanitized| | ||
| sanitized[key] = sanitize_value(key, value) | ||
| end | ||
| end | ||
|
|
||
| private | ||
|
|
||
| # @param path [String] | ||
| # @return [String, nil] | ||
| def feed_suffix(path) | ||
| return '.json' if path.end_with?('.json') | ||
| return '.xml' if path.end_with?('.xml') | ||
| return '.rss' if path.end_with?('.rss') | ||
|
|
||
| nil | ||
| end | ||
|
|
||
| # @param key [Object] | ||
| # @param value [Object] | ||
| # @return [Object] | ||
| def sanitize_value(key, value) | ||
| return sanitize_details(value) if value.is_a?(Hash) | ||
| return value.map { |entry| sanitize_value(key, entry) } if value.is_a?(Array) | ||
| return sanitize_url(value) if url_key?(key) | ||
|
|
||
| value | ||
| end | ||
|
|
||
| # @param key [Object] | ||
| # @return [Boolean] | ||
| def url_key?(key) | ||
| key_name = key.to_s | ||
| key_name == 'url' || key_name.end_with?('_url', '_urls') | ||
| end | ||
|
|
||
| # @param value [Object] | ||
| # @return [Hash{Symbol=>Object}, Object] | ||
| def sanitize_url(value) | ||
| url = value.to_s | ||
| return value if url.empty? | ||
|
|
||
| normalized_url = Html2rss::Url.for_channel(url) | ||
| { | ||
| host: normalized_url.host, | ||
| scheme: normalized_url.scheme, | ||
| hash: Digest::SHA256.hexdigest(url)[0..11] | ||
| }.compact | ||
| rescue StandardError | ||
| { hash: Digest::SHA256.hexdigest(url)[0..11] } | ||
| end | ||
| end | ||
| end | ||
| end | ||
| end | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,91 @@ | ||
| # frozen_string_literal: true | ||
|
|
||
| require 'json' | ||
| require 'logger' | ||
| require 'time' | ||
|
|
||
| module Html2rss | ||
| module Web | ||
| ## | ||
| # Shared structured logger for application and middleware runtime events. | ||
| module AppLogger | ||
| class << self | ||
| # @return [Logger] | ||
| def logger | ||
| Thread.current[:app_logger] ||= build_logger | ||
| end | ||
|
|
||
| # @return [void] | ||
| def reset_logger! | ||
| Thread.current[:app_logger] = nil | ||
| end | ||
|
|
||
| private | ||
|
|
||
| # @return [Logger] | ||
| def build_logger | ||
| Logger.new($stdout).tap do |log| | ||
| log.formatter = method(:format_entry) | ||
| end | ||
| end | ||
|
|
||
| # @param severity [String] | ||
| # @param datetime [Time] | ||
| # @param _progname [String, nil] | ||
| # @param message [String] | ||
| # @return [String] | ||
| def format_entry(severity, datetime, _progname, message) | ||
| "#{base_payload(severity, datetime).merge(normalize_message(message)).to_json}\n" | ||
| end | ||
|
|
||
| # @param severity [String] | ||
| # @param datetime [Time] | ||
| # @return [Hash{Symbol=>Object}] | ||
| def base_payload(severity, datetime) | ||
| { | ||
| timestamp: datetime.iso8601, | ||
| level: severity, | ||
| service: 'html2rss-web' | ||
| } | ||
| end | ||
|
|
||
| # @param message [Object] | ||
| # @return [Hash{Symbol=>Object}] | ||
| def normalize_message(message) | ||
| parsed_json(message) || parse_logfmt(message.to_s) || { message: message.to_s } | ||
| end | ||
|
|
||
| # @param message [Object] | ||
| # @return [Hash{Symbol=>Object}, nil] | ||
| def parsed_json(message) | ||
| JSON.parse(message.to_s, symbolize_names: true) | ||
| rescue JSON::ParserError, TypeError | ||
| nil | ||
| end | ||
|
|
||
| # @param message [String] | ||
| # @return [Hash{Symbol=>Object}, nil] | ||
| def parse_logfmt(message) | ||
| pairs = message.scan(/([a-zA-Z0-9_.-]+)=("[^"]*"|\S+)/) | ||
| return nil if pairs.empty? | ||
|
|
||
| pairs.to_h do |key, raw_value| | ||
| [key.to_sym, normalize_logfmt_value(raw_value)] | ||
| end | ||
| end | ||
|
|
||
| # @param raw_value [String] | ||
| # @return [String, Integer, Float, TrueClass, FalseClass] | ||
| def normalize_logfmt_value(raw_value) | ||
| value = raw_value.delete_prefix('"').delete_suffix('"') | ||
| return true if value == 'true' | ||
| return false if value == 'false' | ||
| return value.to_i if value.match?(/\A-?\d+\z/) | ||
| return value.to_f if value.match?(/\A-?\d+\.\d+\z/) | ||
|
|
||
| value | ||
| end | ||
| end | ||
| end | ||
| end | ||
| end |
Uh oh!
There was an error while loading. Please reload this page.