Raise individual pull requests for group members that update-types rejects

silent/tests/testdata/vu-group-semver-existing-pr.txt

@@ -0,0 +1,67 @@
+# A dependency whose only available update is a major bump is excluded from a
+# minor/patch group by its update-types rules, so it must be raised as an
+# individual pull request.
+#
+# This has to keep working once the group already has an open pull request:
+# deferring the group to its own refresh job must not suppress the dependencies
+# that the group's update-types rules reject.
+
+dependabot update -f input.yml --local . --updater-image ghcr.io/dependabot/dependabot-updater-silent
+stdout -count=1 create_pull_request
+pr-created expected-individual.json
+
+-- manifest.json --
+{
+  "dependency-a": { "version": "1.2.3" },
+  "dependency-b": { "version": "1.2.3" }
+}
+
+-- expected-individual.json --
+{
+  "dependency-a": { "version": "1.2.3" },
+  "dependency-b": { "version": "2.0.0" }
+}
+
+-- dependency-a --
+{
+  "versions": [
+    "1.2.3",
+    "1.2.4",
+    "1.2.5"
+  ]
+}
+
+-- dependency-b --
+{
+  "versions": [
+    "1.2.3",
+    "2.0.0"
+  ]
+}
+
+-- input.yml --
+job:
+  package-manager: "silent"
+  cooldown:
+    default-days: 0
+  source:
+    directory: "/"
+    provider: example
+    hostname: 127.0.0.1
+    api-endpoint: http://127.0.0.1/api/v3
+    repo: dependabot/smoke-tests
+  dependency-groups:
+    - name: dev
+      rules:
+        update-types:
+          - minor
+          - patch
+  experiments:
+    individual-prs-for-semver-excluded-dependencies: true
+  existing-group-pull-requests:
+    - dependency-group-name: dev
+      pr-number: 123
+      dependencies:
+        - dependency-name: dependency-a
+          dependency-version: 1.2.5
+          directory: "/"

updater/lib/dependabot/dependency_snapshot.rb

@@ -4,6 +4,7 @@
 require "base64"
 require "sorbet-runtime"
 
+require "dependabot/experiments"
 require "dependabot/fetched_files"
 require "dependabot/file_parsers"
 require "dependabot/notices_helpers"
@@ -112,45 +113,43 @@ def job_group
       @dependency_group_engine.find_group(name: T.must(job.dependency_group_to_refresh))
     end
 
+    # `defer_update_types` is set by the callers that suppress individual updates for a whole group, i.e.
+    # the ones whose marking decides whether a dependency gets a pull request at all.
     sig do
       params(
         group: Dependabot::DependencyGroup,
-        excluding_dependencies: T::Hash[String, T::Set[String]]
+        excluding_dependencies: T::Hash[String, T::Set[String]],
+        defer_update_types: T::Boolean
       )
         .void
     end
-    def mark_group_handled(group, excluding_dependencies = {})
+    def mark_group_handled(group, excluding_dependencies = {}, defer_update_types: false)
       Dependabot.logger.info("Marking group '#{group.name}' as handled.")
 
       # When grouping by dependency name, we need to mark dependencies as handled
       # across ALL directories to prevent duplicate individual PRs
       group_by_name = group.group_by_dependency_name?
+      defer = defer_update_types && update_types_may_reject?(group)
 
       directories.each do |directory|
         @current_directory = directory
-
-        # add the existing dependencies in the group so individual updates don't try to update them
-        dependencies_in_existing_prs = dependencies_in_existing_pr_for_group(group)
-
-        dependencies_in_existing_prs = dependencies_in_existing_prs.filter do |dep|
-          # When grouping by name, include deps from all directories; otherwise filter by current directory
-          group_by_name || !dep.directory || dep.directory == directory
-        end
-
-        # also add dependencies that might be in the group, as a rebase would add them;
-        # this avoids individual PR creation that immediately is superseded by a group PR supersede
-        current_dependencies = group.dependencies.map(&:name).reject do |dep|
-          excluding_dependencies[directory]&.include?(dep)
-        end
-
-        add_handled_dependencies(
-          current_dependencies.concat(
-            dependencies_in_existing_prs.filter_map(&:name)
-          )
+        mark_group_handled_in_current_directory(
+          group, excluding_dependencies, group_by_name: group_by_name, defer: defer
         )
       end
     end
 
+    # The groups that may still claim this dependency once the semver level of its available update is
+    # known. An empty list means no group is waiting on it and it can be updated individually.
+    sig { params(dependency_name: String).returns(T::Array[Dependabot::DependencyGroup]) }
+    def groups_deferred_by_semver_rules(dependency_name)
+      assert_current_directory_set!
+      deferred = @deferred_group_dependencies[@current_directory]
+      return [] unless deferred
+
+      deferred[dependency_name] || []
+    end
+
     sig { params(dependency_names: T.any(String, T::Array[String])).void }
     def add_handled_dependencies(dependency_names)
       assert_current_directory_set!
@@ -204,6 +203,68 @@ def dependencies_in_existing_pr_for_group(group)
 
     private
 
+    sig do
+      params(
+        group: Dependabot::DependencyGroup,
+        excluding_dependencies: T::Hash[String, T::Set[String]],
+        group_by_name: T::Boolean,
+        defer: T::Boolean
+      )
+        .void
+    end
+    def mark_group_handled_in_current_directory(group, excluding_dependencies, group_by_name:, defer:)
+      directory = @current_directory
+
+      # add the existing dependencies in the group so individual updates don't try to update them
+      dependencies_in_existing_prs = dependencies_in_existing_pr_for_group(group).filter do |dep|
+        # When grouping by name, include deps from all directories; otherwise filter by current directory
+        group_by_name || !dep.directory || dep.directory == directory
+      end
+
+      # also add dependencies that might be in the group, as a rebase would add them;
+      # this avoids individual PR creation that immediately is superseded by a group PR supersede
+      current_dependencies = group.dependencies.map(&:name).reject do |dep|
+        excluding_dependencies[directory]&.include?(dep)
+      end
+
+      # A group that sets update-types only claims a dependency when the update available for it is at one
+      # of those semver levels, which is not known until an update checker has resolved the latest version.
+      # Marking those dependencies as handled here would silently drop the ones the group is going to
+      # reject, so the decision is deferred to the individual update run instead.
+      if defer
+        defer_to_group(group, current_dependencies)
+        current_dependencies = []
+      end
+
+      add_handled_dependencies(
+        current_dependencies.concat(
+          dependencies_in_existing_prs.filter_map(&:name)
+        )
+      )
+    end
+
+    sig { params(group: Dependabot::DependencyGroup).returns(T::Boolean) }
+    def update_types_may_reject?(group)
+      return false unless Dependabot::Experiments.enabled?(:individual_prs_for_semver_excluded_dependencies)
+
+      !group.update_types.nil?
+    end
+
+    sig { params(group: Dependabot::DependencyGroup, dependency_names: T::Array[String]).void }
+    def defer_to_group(group, dependency_names)
+      return if dependency_names.empty?
+
+      Dependabot.logger.info(
+        "Deferring to the update-types rules of group '#{group.name}': (#{dependency_names.join(', ')})."
+      )
+
+      deferred = (@deferred_group_dependencies[@current_directory] ||= {})
+      dependency_names.uniq.each do |name|
+        groups = (deferred[name] ||= [])
+        groups << group unless groups.include?(group)
+      end
+    end
+
     sig do
       params(job: Dependabot::Job, base_commit_sha: String, dependency_files: T::Array[Dependabot::DependencyFile]).void
     end
@@ -214,6 +275,10 @@ def initialize(job:, base_commit_sha:, dependency_files:) # rubocop:disable Metr
       @base_commit_sha = base_commit_sha
       @dependency_files = dependency_files
       @handled_dependencies = T.let({}, T::Hash[String, T::Set[String]])
+      @deferred_group_dependencies = T.let(
+        {},
+        T::Hash[String, T::Hash[String, T::Array[Dependabot::DependencyGroup]]]
+      )
       @current_directory = T.let("", String)
 
       @dependencies = T.let({}, T::Hash[String, T::Array[Dependabot::Dependency]])

updater/lib/dependabot/updater/group_update_creation.rb

@@ -5,6 +5,7 @@
 
 require "dependabot/dependency_change_builder"
 require "dependabot/updater/dependency_group_change_batch"
+require "dependabot/updater/semver_grouping_rules"
 require "dependabot/workspace"
 require "dependabot/updater/security_update_helpers"
 require "dependabot/service"
@@ -583,7 +584,6 @@ def all_versions_ignored?(dependency, checker)
       # then it should not be in the group, but be an individual PR, or in another group that fits it.
       # SemVer Grouping rules have to be applied after we have a checker, because we need to know the latest version.
       # Other rules are applied earlier in the process.
-      # rubocop:disable Metrics/AbcSize
       sig do
         params(
           group: Dependabot::DependencyGroup,
@@ -593,109 +593,11 @@ def all_versions_ignored?(dependency, checker)
           .returns(T::Boolean)
       end
       def semver_rules_allow_grouping?(group, dependency, checker)
-        update_types = group.update_types
-        # There are no group rules defined, so this dependency can be included in the group.
-        return true unless update_types
-
-        return cargo_semver_rules_allow_grouping?(group, dependency, checker) if job.package_manager == "cargo"
-
-        version_class = Dependabot::Utils.version_class_for_package_manager(job.package_manager)
-        unless version_class.correct?(dependency.version.to_s) && version_class.correct?(checker.latest_version)
-          return false
-        end
-
-        version = version_class.new(dependency.version.to_s)
-        latest_version = version_class.new(checker.latest_version)
-
-        # Not every version class implements .major, .minor, .patch so we calculate it here from the segments
-        latest = semver_segments(latest_version)
-        current = semver_segments(version)
-        # Ensure that semver components are of the same type and can be compared with each other.
-        return false unless %i(major minor patch).all? { |k| current[k].instance_of?(latest[k].class) }
-
-        return update_types.include?("major") if T.must(latest[:major]) > T.must(current[:major])
-        return update_types.include?("minor") if T.must(latest[:minor]) > T.must(current[:minor])
-        return update_types.include?("patch") if T.must(latest[:patch]) > T.must(current[:patch])
-
-        # some ecosystems don't do semver exactly, so anything lower gets individual for now
-        false
-      end
-      # rubocop:enable Metrics/AbcSize
-
-      sig do
-        params(
-          group: Dependabot::DependencyGroup,
-          dependency: Dependabot::Dependency,
-          checker: Dependabot::UpdateCheckers::Base
-        ).returns(T::Boolean)
-      end
-      def cargo_semver_rules_allow_grouping?(group, dependency, checker)
-        case dependency.metadata[:all_versions]
-        when Array
-          return cargo_locked_line_updates_allowed?(group, dependency, checker)
-        end
-
-        version_class = Dependabot::Utils.version_class_for_package_manager("cargo")
-        latest_version = checker.latest_version
-        return false unless version_class.correct?(dependency.version.to_s) && version_class.correct?(latest_version)
-
-        cargo_update_type_allowed?(
-          group,
-          version_class.new(dependency.version.to_s),
-          version_class.new(latest_version)
-        )
-      end
-
-      sig do
-        params(
-          group: Dependabot::DependencyGroup,
-          dependency: Dependabot::Dependency,
-          checker: Dependabot::UpdateCheckers::Base
-        ).returns(T::Boolean)
-      end
-      def cargo_locked_line_updates_allowed?(group, dependency, checker)
-        requirements = requirements_to_unlock(checker)
-        return false if requirements == :update_not_possible
-
-        updates = checker.updated_dependencies(requirements_to_unlock: requirements)
-                         .select { |updated| updated.name.casecmp?(dependency.name) }
-        return false if updates.empty?
-
-        version_class = Dependabot::Utils.version_class_for_package_manager("cargo")
-        updates.all? do |updated|
-          previous_version = updated.previous_version
-          version = updated.version
-          next false unless previous_version && version
-          next false unless version_class.correct?(previous_version) && version_class.correct?(version)
-
-          cargo_update_type_allowed?(
-            group,
-            version_class.new(previous_version),
-            version_class.new(version)
-          )
+        SemverGroupingRules.new(job: job).allow_grouping?(group, dependency, checker) do
+          requirements_to_unlock(checker)
         end
       end
 
-      sig { params(version: Gem::Version).returns(T::Hash[Symbol, Integer]) }
-      def semver_segments(version)
-        {
-          major: version.segments[0] || 0,
-          minor: version.segments[1] || 0,
-          patch: version.segments[2] || 0
-        }
-      end
-
-      sig { params(group: Dependabot::DependencyGroup, version: Gem::Version, latest_version: Gem::Version).returns(T::Boolean) }
-      def cargo_update_type_allowed?(group, version, latest_version)
-        return true unless Dependabot::Cargo::Version.respond_to?(:update_type)
-
-        actual_update_type = Dependabot::Cargo::Version.update_type(version.to_s, latest_version.to_s)
-        group_update_types = group.update_types
-        return true unless group_update_types
-
-        group_update_types.include?(actual_update_type)
-      end
-
       sig { params(checker: Dependabot::UpdateCheckers::Base).returns(Symbol) }
       def requirements_to_unlock(checker)
         if !checker.requirements_unlocked_or_can_be?

updater/lib/dependabot/updater/operations/group_update_all_versions.rb

@@ -96,7 +96,7 @@ def run_grouped_dependency_updates
               Dependabot.logger.info(
                 "Deferring creation of a new pull request. The existing pull request will update in a separate job."
               )
-              dependency_snapshot.mark_group_handled(group)
+              dependency_snapshot.mark_group_handled(group, defer_update_types: true)
               next
             end
 
@@ -106,7 +106,7 @@ def run_grouped_dependency_updates
           groups_without_pr.each do |group|
             dependency_change = run_grouped_update_for(group)
             # The update failed, add the suspected dependencies to the handled list so they don't update individually.
-            dependency_snapshot.mark_group_handled(group) if dependency_change.nil?
+            dependency_snapshot.mark_group_handled(group, defer_update_types: true) if dependency_change.nil?
           end
         end