Write daily IRs for incremental aggregation

aggregator/src/main/scala/ai/chronon/aggregator/row/RowAggregator.scala

@@ -70,6 +70,11 @@ class RowAggregator(val inputSchema: Seq[(String, DataType)], val aggregationPar
     .toArray
     .zip(columnAggregators.map(_.irType))
 
+  val incrementalOutputSchema: Array[(String, DataType)] = aggregationParts
+    .map(_.incrementalOutputColumnName)
+    .toArray
+    .zip(columnAggregators.map(_.irType))
+
   val outputSchema: Array[(String, DataType)] = aggregationParts
     .map(_.outputColumnName)
     .toArray

api/py/ai/chronon/group_by.py

@@ -390,6 +390,7 @@ def GroupBy(
     historical_backfill: Optional[bool] = None,
     deprecation_date: Optional[str] = None,
     description: Optional[str] = None,
+    is_incremental: Optional[bool] = None,
     **kwargs,
 ) -> ttypes.GroupBy:
     """
@@ -608,6 +609,7 @@ def _normalize_source(source):
         backfillStartDate=backfill_start_date,
         accuracy=accuracy,
         derivations=derivations,
+        isIncremental=is_incremental,
     )
     validate_group_by(group_by)
     return group_by

api/src/main/scala/ai/chronon/api/Extensions.scala

@@ -97,7 +97,7 @@ object Extensions {
     def cleanName: String = metaData.name.sanitize
 
     def outputTable = s"${metaData.outputNamespace}.${metaData.cleanName}"
-
+    def incrementalOutputTable = s"${metaData.outputNamespace}.${metaData.cleanName}_daily_inc"
     def preModelTransformsTable = s"${metaData.outputNamespace}.${metaData.cleanName}_pre_mt"
     def outputLabelTable = s"${metaData.outputNamespace}.${metaData.cleanName}_labels"
     def outputFinalView = s"${metaData.outputNamespace}.${metaData.cleanName}_labeled"
@@ -178,6 +178,10 @@ object Extensions {
 
     def outputColumnName =
       s"${aggregationPart.inputColumn}_$opSuffix${aggregationPart.window.suffix}${bucketSuffix}"
+
+    def incrementalOutputColumnName =
+      s"${aggregationPart.inputColumn}_$opSuffix${bucketSuffix}"
+
   }
 
   implicit class AggregationOps(aggregation: Aggregation) {

api/thrift/api.thrift

@@ -309,6 +309,7 @@ struct GroupBy {
     6: optional string backfillStartDate
     // Optional derivation list
     7: optional list<Derivation> derivations
+    8: optional bool isIncremental
 }
 
 struct JoinPart {

spark/src/main/scala/ai/chronon/spark/Comparison.scala

@@ -20,13 +20,32 @@ import org.slf4j.LoggerFactory
 import ai.chronon.online.Extensions.StructTypeOps
 import com.google.gson.{Gson, GsonBuilder}
 import org.apache.spark.sql.DataFrame
-import org.apache.spark.sql.types.{DecimalType, DoubleType, FloatType, MapType}
+import org.apache.spark.sql.types.{ArrayType, DecimalType, DoubleType, FloatType, MapType, StructType}
+import org.apache.spark.sql.functions.col
 
 import java.util
+import scala.collection.mutable
 
 object Comparison {
   @transient lazy val logger = LoggerFactory.getLogger(getClass)
 
+  // Flatten struct columns into individual columns so nested double fields can be compared with tolerance
+  private def flattenStructs(df: DataFrame): DataFrame = {
+    val flattenedSelects = df.schema.fields.toSeq.flatMap { field =>
+      field.dataType match {
+        case structType: StructType =>
+          // Flatten struct fields: struct_name.field_name -> struct_name_field_name
+          structType.fields.map { subField =>
+            col(s"${field.name}.${subField.name}").alias(s"${field.name}_${subField.name}")
+          }.toSeq
+        case _ =>
+          // Keep non-struct fields as-is
+          Seq(col(field.name))
+      }
+    }
+    df.select(flattenedSelects: _*)
+  }
+
   // used for comparison
   def sortedJson(m: Map[String, Any]): String = {
     if (m == null) return null
@@ -69,8 +88,12 @@ object Comparison {
         |""".stripMargin
     )
 
-    val prefixedExpectedDf = prefixColumnName(stringifyMaps(a), s"${aName}_")
-    val prefixedOutputDf = prefixColumnName(stringifyMaps(b), s"${bName}_")
+    // Flatten structs so nested double fields can be compared with tolerance
+    val aFlattened = flattenStructs(stringifyMaps(a))
+    val bFlattened = flattenStructs(stringifyMaps(b))
+
+    val prefixedExpectedDf = prefixColumnName(aFlattened, s"${aName}_")
+    val prefixedOutputDf = prefixColumnName(bFlattened, s"${bName}_")
 
     val joinExpr = keys
       .map(key => prefixedExpectedDf(s"${aName}_$key") <=> prefixedOutputDf(s"${bName}_$key"))
@@ -82,15 +105,16 @@ object Comparison {
     )
 
     var finalDf = joined
+    // Use flattened schema for comparison
     val comparisonColumns =
-      a.schema.fieldNames.toSet.diff(keys.toSet).toList.sorted
+      aFlattened.schema.fieldNames.toSet.diff(keys.toSet).toList.sorted
     val colOrder =
       keys.map(key => { finalDf(s"${aName}_$key").as(key) }) ++
         comparisonColumns.flatMap { col =>
           List(finalDf(s"${aName}_$col"), finalDf(s"${bName}_$col"))
         }
-    // double columns need to be compared approximately
-    val doubleCols = a.schema.fields
+    // double columns need to be compared approximately (now includes flattened struct fields)
+    val doubleCols = aFlattened.schema.fields
       .filter(field =>
         field.dataType == DoubleType || field.dataType == FloatType || field.dataType.isInstanceOf[DecimalType])
       .map(_.name)

spark/src/main/scala/ai/chronon/spark/DataRange.scala

@@ -54,6 +54,11 @@ case class PartitionRange(start: String, end: String)(implicit tableUtils: Table
     }
   }
 
+  def daysBetween: Int = {
+    if (start == null || end == null) 0
+    else Stream.iterate(start)(tableUtils.partitionSpec.after).takeWhile(_ <= end).size
+  }
+
   def isSingleDay: Boolean = {
     start == end
   }

spark/src/main/scala/ai/chronon/spark/Driver.scala

@@ -467,7 +467,8 @@ object Driver {
         tableUtils,
         args.stepDays.toOption,
         args.startPartitionOverride.toOption,
-        !args.runFirstHole()
+        !args.runFirstHole(),
+        Option(args.groupByConf.isIncremental).getOrElse(false)
       )
 
       if (args.shouldExport()) {