From 545e3c37737968ad41171392db9307af69d8dda5 Mon Sep 17 00:00:00 2001 From: Darcy Date: Mon, 13 Apr 2026 19:28:59 +0800 Subject: [PATCH 1/3] Fix reachMinorInterval() starvation for cold partitions reachMinorInterval() uses table-level lastMinorOptimizingTime which is frequently reset by high-traffic partitions. This causes partitions with fewer small files (below minor trigger file count threshold) to never get minor optimized, even when their files accumulate over time. Add a cross-day fallback: when minorLeastInterval is less than one day and the last minor optimizing happened on a different calendar day, reachMinorInterval() returns true. This ensures every partition gets at least one minor optimization attempt per day. Also add Javadoc to SELF_OPTIMIZING_MINOR_TRIGGER_INTERVAL clarifying that it should be less than one day for the cross-day fallback to work. Closes #4055 --- .../plan/CommonPartitionEvaluator.java | 26 +++++++++++++++++-- .../apache/amoro/table/TableProperties.java | 6 +++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/plan/CommonPartitionEvaluator.java b/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/plan/CommonPartitionEvaluator.java index 2cd11cbd51..08a76c0090 100644 --- a/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/plan/CommonPartitionEvaluator.java +++ b/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/plan/CommonPartitionEvaluator.java @@ -35,6 +35,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.time.Instant; +import java.time.LocalDate; +import java.time.ZoneId; import java.util.List; import java.util.Set; @@ -412,8 +415,27 @@ public boolean isMinorNecessary() { } protected boolean reachMinorInterval() { - return config.getMinorLeastInterval() >= 0 - && planTime - lastMinorOptimizingTime > config.getMinorLeastInterval(); + if (config.getMinorLeastInterval() < 0) { + return false; + } + if (planTime - lastMinorOptimizingTime > config.getMinorLeastInterval()) { + return true; + } + // When minorLeastInterval is less than one day, use a cross-day fallback to ensure + // partitions with few small files still get optimized at least once per day, avoiding + // starvation caused by table-level lastMinorOptimizingTime being frequently reset by + // high-traffic partitions. See https://github.com/apache/amoro/issues/4055 + long oneDayMillis = 24 * 60 * 60 * 1000L; + if (config.getMinorLeastInterval() < oneDayMillis) { + return isDifferentDay(lastMinorOptimizingTime, planTime); + } + return false; + } + + private boolean isDifferentDay(long time1, long time2) { + LocalDate day1 = Instant.ofEpochMilli(time1).atZone(ZoneId.systemDefault()).toLocalDate(); + LocalDate day2 = Instant.ofEpochMilli(time2).atZone(ZoneId.systemDefault()).toLocalDate(); + return !day1.equals(day2); } protected boolean reachFullInterval() { diff --git a/amoro-format-iceberg/src/main/java/org/apache/amoro/table/TableProperties.java b/amoro-format-iceberg/src/main/java/org/apache/amoro/table/TableProperties.java index 015c68d480..bcf1d13ed5 100644 --- a/amoro-format-iceberg/src/main/java/org/apache/amoro/table/TableProperties.java +++ b/amoro-format-iceberg/src/main/java/org/apache/amoro/table/TableProperties.java @@ -110,8 +110,14 @@ private TableProperties() {} "self-optimizing.minor.trigger.file-count"; public static final int SELF_OPTIMIZING_MINOR_TRIGGER_FILE_CNT_DEFAULT = 12; + /** + * The minimum interval for triggering minor optimizing, in milliseconds. Should be less than one + * day (86400000 ms). When the interval is less than one day, a cross-day fallback mechanism + * ensures that partitions with few small files still get optimized at least once per day. + */ public static final String SELF_OPTIMIZING_MINOR_TRIGGER_INTERVAL = "self-optimizing.minor.trigger.interval"; + public static final int SELF_OPTIMIZING_MINOR_TRIGGER_INTERVAL_DEFAULT = 3600000; // 1 h public static final String SELF_OPTIMIZING_MAJOR_TRIGGER_DUPLICATE_RATIO = From 0cb2b1386b30c35461cdc2e3e5915f9792c43f25 Mon Sep 17 00:00:00 2001 From: Darcy Date: Mon, 13 Apr 2026 19:34:31 +0800 Subject: [PATCH 2/3] Remove extra blank line in TableProperties for style consistency --- .../src/main/java/org/apache/amoro/table/TableProperties.java | 1 - 1 file changed, 1 deletion(-) diff --git a/amoro-format-iceberg/src/main/java/org/apache/amoro/table/TableProperties.java b/amoro-format-iceberg/src/main/java/org/apache/amoro/table/TableProperties.java index bcf1d13ed5..8d52ae9b21 100644 --- a/amoro-format-iceberg/src/main/java/org/apache/amoro/table/TableProperties.java +++ b/amoro-format-iceberg/src/main/java/org/apache/amoro/table/TableProperties.java @@ -117,7 +117,6 @@ private TableProperties() {} */ public static final String SELF_OPTIMIZING_MINOR_TRIGGER_INTERVAL = "self-optimizing.minor.trigger.interval"; - public static final int SELF_OPTIMIZING_MINOR_TRIGGER_INTERVAL_DEFAULT = 3600000; // 1 h public static final String SELF_OPTIMIZING_MAJOR_TRIGGER_DUPLICATE_RATIO = From 4e4a405a7947a817f0463d45a5c5c7d0305553df Mon Sep 17 00:00:00 2001 From: Darcy Date: Tue, 14 Apr 2026 10:42:48 +0800 Subject: [PATCH 3/3] fix: restore blank line after SELF_OPTIMIZING_MINOR_TRIGGER_INTERVAL to pass spotless check --- .../src/main/java/org/apache/amoro/table/TableProperties.java | 1 + 1 file changed, 1 insertion(+) diff --git a/amoro-format-iceberg/src/main/java/org/apache/amoro/table/TableProperties.java b/amoro-format-iceberg/src/main/java/org/apache/amoro/table/TableProperties.java index 8d52ae9b21..bcf1d13ed5 100644 --- a/amoro-format-iceberg/src/main/java/org/apache/amoro/table/TableProperties.java +++ b/amoro-format-iceberg/src/main/java/org/apache/amoro/table/TableProperties.java @@ -117,6 +117,7 @@ private TableProperties() {} */ public static final String SELF_OPTIMIZING_MINOR_TRIGGER_INTERVAL = "self-optimizing.minor.trigger.interval"; + public static final int SELF_OPTIMIZING_MINOR_TRIGGER_INTERVAL_DEFAULT = 3600000; // 1 h public static final String SELF_OPTIMIZING_MAJOR_TRIGGER_DUPLICATE_RATIO =