diff --git a/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/plan/CommonPartitionEvaluator.java b/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/plan/CommonPartitionEvaluator.java index 2cd11cbd51..08a76c0090 100644 --- a/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/plan/CommonPartitionEvaluator.java +++ b/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/plan/CommonPartitionEvaluator.java @@ -35,6 +35,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.time.Instant; +import java.time.LocalDate; +import java.time.ZoneId; import java.util.List; import java.util.Set; @@ -412,8 +415,27 @@ public boolean isMinorNecessary() { } protected boolean reachMinorInterval() { - return config.getMinorLeastInterval() >= 0 - && planTime - lastMinorOptimizingTime > config.getMinorLeastInterval(); + if (config.getMinorLeastInterval() < 0) { + return false; + } + if (planTime - lastMinorOptimizingTime > config.getMinorLeastInterval()) { + return true; + } + // When minorLeastInterval is less than one day, use a cross-day fallback to ensure + // partitions with few small files still get optimized at least once per day, avoiding + // starvation caused by table-level lastMinorOptimizingTime being frequently reset by + // high-traffic partitions. See https://github.com/apache/amoro/issues/4055 + long oneDayMillis = 24 * 60 * 60 * 1000L; + if (config.getMinorLeastInterval() < oneDayMillis) { + return isDifferentDay(lastMinorOptimizingTime, planTime); + } + return false; + } + + private boolean isDifferentDay(long time1, long time2) { + LocalDate day1 = Instant.ofEpochMilli(time1).atZone(ZoneId.systemDefault()).toLocalDate(); + LocalDate day2 = Instant.ofEpochMilli(time2).atZone(ZoneId.systemDefault()).toLocalDate(); + return !day1.equals(day2); } protected boolean reachFullInterval() { diff --git a/amoro-format-iceberg/src/main/java/org/apache/amoro/table/TableProperties.java b/amoro-format-iceberg/src/main/java/org/apache/amoro/table/TableProperties.java index 0905acf428..745ab08413 100644 --- a/amoro-format-iceberg/src/main/java/org/apache/amoro/table/TableProperties.java +++ b/amoro-format-iceberg/src/main/java/org/apache/amoro/table/TableProperties.java @@ -114,8 +114,14 @@ private TableProperties() {} "self-optimizing.minor.trigger.file-count"; public static final int SELF_OPTIMIZING_MINOR_TRIGGER_FILE_CNT_DEFAULT = 12; + /** + * The minimum interval for triggering minor optimizing, in milliseconds. Should be less than one + * day (86400000 ms). When the interval is less than one day, a cross-day fallback mechanism + * ensures that partitions with few small files still get optimized at least once per day. + */ public static final String SELF_OPTIMIZING_MINOR_TRIGGER_INTERVAL = "self-optimizing.minor.trigger.interval"; + public static final int SELF_OPTIMIZING_MINOR_TRIGGER_INTERVAL_DEFAULT = 3600000; // 1 h public static final String SELF_OPTIMIZING_MAJOR_TRIGGER_DUPLICATE_RATIO =