From 4ef14cd2abef45eb99237fbee4104c52c4eb5b42 Mon Sep 17 00:00:00 2001 From: Jouni Tuomisto Date: Sun, 26 Jul 2026 14:33:16 +0300 Subject: [PATCH 1/7] Longmont: Move trends from actions to nodes; stop using historical actions; delete redundant nodes --- configs/longmont-dev.yaml | 346 +++++++++++++++----------------------- 1 file changed, 133 insertions(+), 213 deletions(-) diff --git a/configs/longmont-dev.yaml b/configs/longmont-dev.yaml index 4504aad9..909a8d71 100644 --- a/configs/longmont-dev.yaml +++ b/configs/longmont-dev.yaml @@ -55,9 +55,7 @@ result_excels: - aviation_fuel_usage - building_class_mix - building_electricity_eui_by_action - - building_electricity_use_historical - building_natural_gas_eui_by_action - - building_natural_gas_use_historical - building_square_footage_new - building_square_footage_old - commercial_air_conditioned_square_footage @@ -1123,16 +1121,12 @@ nodes: quantity: floor_area unit: ft^2 output_dimensions: [emission_sectors] - input_datasets: - - id: longmont/building_square_footage - tags: [all] - forecast_from: 2025 - column: floor_area input_nodes: - id: building_square_footage_old tags: [old] + - id: population params: - formula: all + arithmetic_inverse(old) + formula: old * (ratio_to_max_hist_year(population) - 1) - id: building_square_footage_by_building_class name: Building square footage by building class @@ -1192,109 +1186,11 @@ nodes: drop_col: false output_nodes: - id: electricity_use + tags: [streetlights] from_dimensions: - id: site flatten: true -# TODO MAYBE REMOVE ALL THESE NODES --v - -- id: building_electricity_use_historical - name: Building electricity use (historical) - name_es-US: Consumo de electricidad en edificios (histórico) - type: generic.GenericNode - quantity: energy - unit: GWh/a - input_dimensions: [emission_sectors, site] - output_dimensions: [emission_sectors, site] - input_datasets: - - id: longmont/stationary_energy_consumption - column: electricity - filters: - - column: energy_carrier - value: electricity - -- id: building_electricity_use_per_resident - name: Building electricity use per resident - name_es-US: Consumo de electricidad en edificios por residente - type: formula.FormulaNode - quantity: per_capita - unit: GWh/a/cap - input_nodes: - - id: building_electricity_use_historical - from_dimensions: - - id: site - flatten: true - - id: population - to_dimensions: [] - input_dimensions: [emission_sectors] - output_dimensions: [emission_sectors] - params: - formula: extend_values(inventory_only(building_electricity_use_historical / - population)) - -- id: building_electricity_use_bau - name: Building electricity use (baseline) - name_es-US: Consumo de electricidad en edificios (referencia) - type: generic.GenericNode - quantity: energy - unit: GWh/a - input_nodes: - - id: building_electricity_use_per_resident - - id: population - to_dimensions: [] - input_dimensions: [emission_sectors] - output_dimensions: [emission_sectors] - -- id: building_natural_gas_use_historical - name: Building natural gas use (historical) - name_es-US: Consumo de gas natural en edificios (histórico) - type: buildings.HistoricalNode # FIXME - quantity: energy - unit: Mtherm/a - input_dimensions: [emission_sectors, site] - output_dimensions: [emission_sectors, site] - input_datasets: - - id: longmont/stationary_energy_consumption - column: natural_gas - filters: - - column: energy_carrier - value: natural_gas - input_dataset_processors: [LinearInterpolation] - -- id: building_natural_gas_use_per_resident - name: Building natural gas use per resident - name_es-US: Consumo de gas natural en edificios por residente - type: formula.FormulaNode - quantity: per_capita - unit: Mtherm/a/cap - input_nodes: - - id: building_natural_gas_use_historical - from_dimensions: - - id: site - flatten: true - - id: population - to_dimensions: [] - input_dimensions: [emission_sectors] - output_dimensions: [emission_sectors] - params: - formula: extend_values(inventory_only(building_natural_gas_use_historical / - population)) - -- id: building_natural_gas_use_bau - name: Building natural gas use (baseline) - name_es-US: Consumo de gas natural en edificios (referencia) - type: generic.GenericNode - quantity: energy - unit: Mtherm/a - input_nodes: - - id: building_natural_gas_use_per_resident - - id: population - to_dimensions: [] - input_dimensions: [emission_sectors] - output_dimensions: [emission_sectors] - -## TODO MAYBE REMOVE ALL THESE NODES ABOVE --^ - - id: building_natural_gas_eui_by_action name: Energy use intensity EUI baseline for natural gas by energy class type: formula.FormulaNode @@ -1309,8 +1205,7 @@ nodes: tags: [eui, extend_both_ways] output_dimensions: [emission_sectors] input_nodes: - - id: energy_use_intensity_trend - metrics: [natural_gas] + - id: energy_use_intensity_trend_natural_gas tags: [trend, extend_both_ways] - id: energy_use_intensity_improvement tags: [improvement] @@ -1358,8 +1253,7 @@ nodes: input_nodes: - id: energy_use_intensity_improvement tags: [improvement] - - id: energy_use_intensity_trend - metrics: [electricity] + - id: energy_use_intensity_trend_electricity tags: [trend] params: formula: (eui + trend) * complement(improvement) @@ -1398,15 +1292,16 @@ nodes: kilowatt-hours per year. description_es-US: Unidad: GWh/año = gigavatios hora al año = millones de kilovatios hora al año. - type: generic.ActionWithHistoryNode + type: formula.FormulaNode quantity: energy unit: GWh/a input_dimensions: [emission_sectors] output_dimensions: [emission_sectors] params: - operations: add,action_with_history + formula: (buildings + streetlights) * (1 - reduce_losses) input_nodes: - id: building_electricity_use + tags: [buildings] from_dimensions: - id: building_energy_class flatten: true @@ -2470,6 +2365,9 @@ nodes: input_datasets: - id: longmont/waste_data column: waste_landfilled + input_nodes: + - id: population + tags: [ratio_to_max_hist_year] - id: diversion_rate name: Diversion rate @@ -2562,6 +2460,9 @@ nodes: input_datasets: - id: longmont/waste_data column: waste_composted + input_nodes: + - id: population + tags: [ratio_to_max_hist_year] - id: composted_waste_emissions name: Composted waste emissions @@ -2638,6 +2539,9 @@ nodes: input_datasets: - id: longmont/waste_data column: waste_recycled + input_nodes: + - id: population + tags: [ratio_to_max_hist_year] - id: recycling_characterization name: Recycling characterization @@ -3054,6 +2958,113 @@ nodes: params: operations: get_single_dataset,extend_to_history +- id: energy_use_intensity_trend_natural_gas + name: Energy use intensity trend (natural gas) + name_es-US: Tendencia de la intensidad del uso de la energía (gas natural) + description: This action describes the assumed improvement in the energy use + intensity for the baseline scenario, i.e. changes that will occur anyway + without any activity from the city. + description_es-US: Esta acción describe la mejora supuesta en la intensidad + del uso de la energía para el escenario de referencia, es decir, los cambios + que se producirán de todos modos sin intervención alguna por parte de la + ciudad. + type: generic.GenericNode + quantity: consumption_factor + unit: thm/ft**2/a + output_dimensions: [emission_sectors] + input_datasets: + - id: longmont/energy_use_intensity_improvements + column: natural_gas + tags: [cleaned, extend_both_ways] + filters: + - column: action + value: none + forecast_from: 2025 + +- id: energy_use_intensity_trend_electricity + name: Energy use intensity trend (electricity) + name_es-US: Tendencia de la intensidad del uso de la energía (electricidad) + description: This action describes the assumed improvement in the energy use + intensity for the baseline scenario, i.e. changes that will occur anyway + without any activity from the city. + description_es-US: Esta acción describe la mejora supuesta en la intensidad + del uso de la energía para el escenario de referencia, es decir, los cambios + que se producirán de todos modos sin intervención alguna por parte de la + ciudad. + type: generic.GenericNode + quantity: consumption_factor + unit: kWh/ft**2/a + output_dimensions: [emission_sectors] + input_datasets: + - id: longmont/energy_use_intensity_improvements + column: electricity + tags: [cleaned, extend_both_ways] + filters: + - column: action + value: none + forecast_from: 2025 + +# Trends beyond the city's own control. + +- id: mpg_trend + name: Miles per gallon increasing trend + name_es-US: Tendencia al aumento de las millas recorridas por galón + description: The model assumes that internal combustion vehicles that use + gasoline and diesel will become more efficient, increase miles traveled per + gallon, over time. + description_es-US: El modelo supone que, con el tiempo, los vehículos de + combustión interna que usan gasolina y diésel serán más eficientes y + aumentarán las millas recorridas por galón. + type: simple.AdditiveNode + quantity: fuel_consumption + unit: MPG + input_datasets: + - id: longmont/energy_vehicle_and_waste_actions + forecast_from: 2025 + column: mpg_trend + output_nodes: + - id: transportation_miles_per_gallon + to_dimensions: + - id: energy_carrier + categories: [petrol] + - id: vehicle_type + categories: [car] + +- id: vmt_per_vehicle_trend + name: Vehicle miles per vehicle trend + name_es-US: Tendencia de millas recorridas por vehículo + description:

The model assumes that overtime there will be a decrease in + the annual miles traveled per vehicle.

Unit: mi/(yr. v) = + miles per year per vehicle

+ description_es-US:

El modelo supone que con el tiempo se producirá una + disminución de las millas anuales recorridas por vehículo.

+

Unidad: mi/(a.v) = millas anuales por vehículo

+ type: simple.AdditiveNode + quantity: mileage + unit: VMT/vehicle/a + input_datasets: + - id: longmont/energy_vehicle_and_waste_actions + forecast_from: 2025 + column: vmt_per_vehicle_trend + output_nodes: + - vehicle_miles_per_vehicle + +- id: electric_vehicle_efficiency_trend + name: Electric vehicle efficiency trend + name_es-US: Tendencia de la eficiencia de los vehículos eléctricos + description: The model assumes that electric vehicles will become more + efficient, increase in miles traveled per kilowatt hour of energy, overtime. + description_es-US: El modelo supone que, con el tiempo, los vehículos + eléctricos serán más eficientes y aumentarán los kilómetros recorridos por + kilovatio hora de energía. + type: simple.AdditiveNode + quantity: fuel_consumption + unit: kWh/VMT + input_datasets: + - id: longmont/energy_vehicle_and_waste_actions + forecast_from: 2025 + column: electric_vehicle_efficiency_trend + actions: - id: updated_building_code @@ -3404,7 +3415,7 @@ actions: forecast_from: 2025 output_nodes: - id: electricity_use - tags: [extend_to_history, complement, action_with_history] + tags: [reduce_losses] - id: leak_detection name: Adopt and enforce leak detection and repair @@ -3462,28 +3473,6 @@ actions: output_nodes: - diversion_rate -- id: energy_use_intensity_trend - name: Energy use intensity trend - name_es-US: Tendencia de la intensidad del uso de la energía - description: This action describes the assumed improvement in the energy use - intensity for the baseline scenario, i.e. changes that will occur anyway - without any activity from the city. - description_es-US: Esta acción describe la mejora supuesta en la intensidad - del uso de la energía para el escenario de referencia, es decir, los cambios - que se producirán de todos modos sin intervención alguna por parte de la - ciudad. - type: energy_saving.EnergyAction - quantity: consumption_factor - group: trend - input_datasets: - - id: longmont/energy_use_intensity_improvements - tags: [cleaned, extend_both_ways] - filters: - - column: action - value: none - forecast_from: 2025 - output_dimensions: [emission_sectors] - - id: vehicle_incentives name: Increase electric vehicles through vehicle incentives and improved infrastructure @@ -3600,70 +3589,6 @@ actions: # output_nodes: # - id: total_transit_vmt -# Trend actions. TODO Do we need these? - -- id: mpg_trend - name: Miles per gallon increasing trend - name_es-US: Tendencia al aumento de las millas recorridas por galón - description: The model assumes that internal combustion vehicles that use - gasoline and diesel will become more efficient, increase miles traveled per - gallon, over time. - description_es-US: El modelo supone que, con el tiempo, los vehículos de - combustión interna que usan gasolina y diésel serán más eficientes y - aumentarán las millas recorridas por galón. - type: simple.AdditiveAction - quantity: fuel_consumption - unit: MPG - group: trend - input_datasets: - - id: longmont/energy_vehicle_and_waste_actions - forecast_from: 2025 - column: mpg_trend - output_nodes: - - id: transportation_miles_per_gallon - to_dimensions: - - id: energy_carrier - categories: [petrol] - - id: vehicle_type - categories: [car] - -- id: vmt_per_vehicle_trend - name: Vehicle miles per vehicle trend - name_es-US: Tendencia de millas recorridas por vehículo - description:

The model assumes that overtime there will be a decrease in - the annual miles traveled per vehicle.

Unit: mi/(yr. v) = - miles per year per vehicle

- description_es-US:

El modelo supone que con el tiempo se producirá una - disminución de las millas anuales recorridas por vehículo.

-

Unidad: mi/(a.v) = millas anuales por vehículo

- type: simple.AdditiveAction - quantity: mileage - unit: VMT/vehicle/a - group: trend - input_datasets: - - id: longmont/energy_vehicle_and_waste_actions - forecast_from: 2025 - column: vmt_per_vehicle_trend - output_nodes: - - vehicle_miles_per_vehicle - -- id: electric_vehicle_efficiency_trend - name: Electric vehicle efficiency trend - name_es-US: Tendencia de la eficiencia de los vehículos eléctricos - description: The model assumes that electric vehicles will become more - efficient, increase in miles traveled per kilowatt hour of energy, overtime. - description_es-US: El modelo supone que, con el tiempo, los vehículos - eléctricos serán más eficientes y aumentarán los kilómetros recorridos por - kilovatio hora de energía. - type: simple.AdditiveAction - quantity: fuel_consumption - unit: kWh/VMT - group: trend - input_datasets: - - id: longmont/energy_vehicle_and_waste_actions - forecast_from: 2025 - column: electric_vehicle_efficiency_trend - pages: - id: home name: Longmont Greenhouse Gas Inventory @@ -3675,23 +3600,18 @@ scenarios: - id: baseline name: Business as Usual name_es-US: Normalidad - params: - - id: energy_use_intensity_trend.enabled - value: true - id: default default: true name: Climate Action Initiatives name_es-US: Iniciativas de acción climática all_actions_enabled: true - params: - - id: energy_use_intensity_trend.enabled - value: false -- id: historical_actions - name: Historical actions - all_actions_enabled: false - is_selectable: false - params: - - id: reduce_transmission_distribution_losses.enabled - value: true +# This was an exploration of how previous actions should work in historical years. Not needed now. +# - id: historical_actions +# name: Historical actions +# all_actions_enabled: false +# is_selectable: false +# params: +# - id: reduce_transmission_distribution_losses.enabled +# value: true From 9d8bfec9a6d6d833d835093b2e192d6c136d9a22 Mon Sep 17 00:00:00 2001 From: Jouni Tuomisto Date: Sun, 26 Jul 2026 14:34:06 +0300 Subject: [PATCH 2/7] Make DimensionalFlow.get_flow_node() more robust --- nodes/metric.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/nodes/metric.py b/nodes/metric.py index 28dfcb86..15f8113c 100644 --- a/nodes/metric.py +++ b/nodes/metric.py @@ -583,7 +583,7 @@ def from_action_node(cls, node: ActionNode) -> None | DimensionalFlow: # noqa: flow_nodes: dict[str, FlowNode] = {} - def get_flow_node(row: dict[str, Any], is_source: bool) -> FlowNode: + def get_flow_node(row: dict[str, Any], is_source: bool) -> FlowNode: # noqa: C901 path_parts = [] label_parts = [] @@ -621,6 +621,14 @@ def get_flow_node(row: dict[str, Any], is_source: bool) -> FlowNode: val_col = node.get_default_output_metric().column_id if sdf_exprs: sdf = sdf.filter(functools.reduce(lambda a, b: a & b, sdf_exprs)) # pyright: ignore[reportUnknownLambdaType] + # The source node (e.g. a ShiftAction's output_nodes[0]) may carry dimensions the + # shift itself never mentions in its source/dest categories (`dims` above comes from + # the shift's own dims, via df.primary_keys). Sum those out here so one row per year + # remains; this only affects this flow-diagram value, not the source node's own output + # or any actual effect computation elsewhere. + extra_dims = [dim_id for dim_id in sdf.dim_ids if dim_id not in dim_cats] + if extra_dims: + sdf = sdf.paths.sum_over_dims(extra_dims) sdf = sdf.select([YEAR_COLUMN, val_col]) assert not sdf.paths.index_has_duplicates() assert flow_node_id not in source_values From aae7c18811c5a53173471e47e5830c04d071ef4b Mon Sep 17 00:00:00 2001 From: Jouni Tuomisto Date: Sun, 26 Jul 2026 14:52:12 +0300 Subject: [PATCH 3/7] Longmont: Change ShiftActions to GenericActions with datasets --- configs/longmont-dev.yaml | 504 +++++++++++++++++++++++++++----------- 1 file changed, 360 insertions(+), 144 deletions(-) diff --git a/configs/longmont-dev.yaml b/configs/longmont-dev.yaml index 909a8d71..6dc8eb15 100644 --- a/configs/longmont-dev.yaml +++ b/configs/longmont-dev.yaml @@ -14,7 +14,7 @@ default_language: en supported_languages: [es-US] dataset_repo: url: https://github.com/kausaltech/dvctest.git - commit: ad50da5a21200b37695fc462081d7872840f6991 + commit: ef08a8359893a551927be5b739dac879ee16b5a6 dvc_remote: kausal-s3 name: 2025 Draft Longmont Greenhouse Gas Inventory name_es-US: 2025 Inventario de gases de efecto invernadero de Longmont @@ -1153,17 +1153,117 @@ nodes: params: formula: (old + new) * action_mix +- id: updated_building_code_stock + name: Updated building code (accumulated stock share) + description: Cumulative share of existing building stock brought under the + updated building code so far, built from the action's own yearly incremental + share (see updated_building_code) so that each year's floor area is weighted + by the code vintage in effect when it was brought under the code, not by the + current year's improvement figure. + type: formula.FormulaNode + quantity: fraction + unit: '%' + input_dimensions: [building_energy_class, emission_sectors] + output_dimensions: [building_energy_class, emission_sectors] + input_nodes: + - id: updated_building_code + tags: [flow] + params: + formula: cumulative(flow) + +- id: develop_beyond_building_code_stock + name: Develop beyond building code (accumulated stock share) + description: Cumulative share of new building stock developed beyond code so + far, built from the action's own yearly incremental share (see + develop_beyond_building_code) so that each year's floor area is weighted by + the vintage in effect when it was built, not by the current year's + improvement figure. + type: formula.FormulaNode + quantity: fraction + unit: '%' + input_dimensions: [building_energy_class, emission_sectors] + output_dimensions: [building_energy_class, emission_sectors] + input_nodes: + - id: develop_beyond_building_code + tags: [flow] + params: + formula: cumulative(flow) + +- id: benchmark_energy_use_stock + name: Benchmark energy use (accumulated stock share) + description: Cumulative share of commercial/industrial building stock brought + under the benchmarking program so far, built from the action's own yearly + incremental share (see benchmark_energy_use). + type: formula.FormulaNode + quantity: fraction + unit: '%' + input_dimensions: [building_energy_class, emission_sectors] + output_dimensions: [building_energy_class, emission_sectors] + input_nodes: + - id: benchmark_energy_use + tags: [flow] + params: + formula: cumulative(flow) + +- id: building_class_mix_none + name: Building class mix (no action taken, residual) + description: The complement of whatever share has been claimed by + updated_building_code, develop_beyond_building_code, and + benchmark_energy_use, so the three action shares plus this one always sum to + 100% for every (building_energy_class, emission_sectors) cell. + type: formula.FormulaNode + quantity: fraction + unit: '%' + input_dimensions: [building_energy_class, emission_sectors] + output_dimensions: [building_energy_class, emission_sectors] + input_nodes: + - id: updated_building_code_stock + tags: [ubc] + - id: develop_beyond_building_code_stock + tags: [dbbc] + - id: benchmark_energy_use_stock + tags: [beu] + params: + formula: complement(ubc + dbbc + beu) + - id: building_class_mix name: Building class mix - type: generic.GenericNode + type: formula.FormulaNode quantity: fraction unit: '%' input_dimensions: [action, building_energy_class, emission_sectors] output_dimensions: [action, building_energy_class, emission_sectors] - input_datasets: - - id: longmont/building_class_mix + input_nodes: + - id: updated_building_code_stock + tags: [ubc] + to_dimensions: + - id: building_energy_class + - id: emission_sectors + - id: action + categories: [updated_building_code] + - id: develop_beyond_building_code_stock + tags: [dbbc] + to_dimensions: + - id: building_energy_class + - id: emission_sectors + - id: action + categories: [develop_beyond_building_code] + - id: benchmark_energy_use_stock + tags: [beu] + to_dimensions: + - id: building_energy_class + - id: emission_sectors + - id: action + categories: [benchmark_energy_use] + - id: building_class_mix_none + tags: [none_share] + to_dimensions: + - id: building_energy_class + - id: emission_sectors + - id: action + categories: [none] params: - operations: get_single_dataset,add_to_existing_dims + formula: ubc + dbbc + beu + none_share - id: electricity_use_on_streetlights_and_internal_operations name: Electricity use on streetlights and internal operations @@ -1207,7 +1307,7 @@ nodes: input_nodes: - id: energy_use_intensity_trend_natural_gas tags: [trend, extend_both_ways] - - id: energy_use_intensity_improvement + - id: energy_use_intensity_improvement_vintage tags: [improvement] params: formula: (eui + trend) * complement(improvement) @@ -1251,7 +1351,7 @@ nodes: forecast_from: 2025 output_dimensions: [emission_sectors] input_nodes: - - id: energy_use_intensity_improvement + - id: energy_use_intensity_improvement_vintage tags: [improvement] - id: energy_use_intensity_trend_electricity tags: [trend] @@ -1565,57 +1665,64 @@ nodes: - id: stationary_fuel_consumption_emissions name: Stationary fuel consumption emissions name_es-US: Emisiones del consumo de combustible estacionario - description:

1) Stationary Diesel Use data was provided by CDPHE. Spreadsheet - on file.

2) Xcel Energy emission factor, electricity and natural gas usage - data, and renewable energy data is from Xcel's 2021 Community Energy Report for - Longmont, on file.

3) Longmont Power & Communications data provided. PDF - on file. The commercial buildings in Longmont's Weld County portion consumed 31,000,000 - kWh in 2021.

4) T&D losses are calculated for Xcel and Longmont based on - the EIA 861 (https://www.eia.gov/electricity/data/eia861/) reported data. Specifically - the Utility Operational Data_2021. Spreadsheet on file.

5) Propane data - provided by FerrellGas. Spreadsheet on file.

6) Houses heated with wood - were pulled from the US Census Bureau, data on file. It was assumed that all of - the houses heated by wood in Longmont fell within the Boulder County portion of - Longmont.

7) Longmont Power and Communications data was provided by Longmont - Power & Communications. PDF on file.

8) Longmont Power & Communications - and Lyons Municipal rooftop solar data was provided in kW of capacity rather than - kWh generated. NREL's PVWatts was used to estimate kWh generated by rooftop solar - systems. PDF of results on file. See https://pvwatts.nrel.gov/

10) Due - to Xcel's 15/15 rule, with less than 15 customers in the commercial and residential - data sets they cannot provide usage for those customers to avoid violating privacy - concerns. Therefore Lotus applied the average residential and commercial kWh use - from LPC to the 7 residential and commercial customers (respectively) in Xcel - territory to estimate use.

11) PRPA emission factor and transmission and - distribution loss rate for 2021 was provided. Email on file. Emissions factor - value represents Longmont's load share emissions rate without unbundled RECs.

- description_es-US:

1) Los datos sobre el consumo de diésel estacionario fueron - brindados por el CDPHE. Hoja de cálculo en el archivo.

2) El factor de - emisión de Xcel Energy, los datos de consumo de electricidad y gas natural y los - datos de energía renovable provienen del Informe de energía comunitaria 2021 de - Xcel para Longmont (en el archivo).

3) Datos facilitados por Longmont Power - & Communications. PDF en el archivo. Los edificios comerciales en la parte del - condado de Weld de Longmont consumieron 31.000.000 kWh en el año 2021.

4) - Las pérdidas de transmisión y distribución se calculan para Xcel y Longmont según - los datos de la EIA 861 (https://www.eia.gov/electricity/data/eia861/). Específicamente - los Datos operativos de servicios públicos 2021. Hoja de cálculo en el archivo.

-

5) Datos de propano facilitados por FerrellGas. Hoja de cálculo en el archivo.

-

6) Las casas con calefacción de madera fueron retiradas de la Oficina del Censo - de los Estados Unidos (datos en el archivo). Se supuso que todas las casas con - calefacción de madera se encontraban en la parte del condado de Boulder de Longmont.

-

7) Los datos de energía y comunicaciones de Longmont fueron facilitados por - Longmont Power & Communications. PDF en el archivo.

8) Los datos de sistemas - solares de techo de Longmont Power & Communications y Lyons Municipal se ofrecieron - en kW de capacidad en lugar de kWh generados. Los PVWatts del NREL se utilizaron - para estimar los kWh generados por los sistemas solares de techo. PDF de los resultados - en el archivo. Visite https://pvwatts.nrel.gov/

10) Debido a la regla 15/15 - de Xcel, con menos de 15 clientes en los conjuntos de datos comerciales y residenciales, - no pueden brindar el consumo de esos clientes para evitar problemas de privacidad. - Por lo tanto, Lotus aplicó el consumo promedio de kWh residencial y comercial - de LPC a los siete clientes residenciales y comerciales (respectivamente) en el - territorio de Xcel para estimar el consumo.

11) Se proporcionó la tasa - de pérdida de transmisión y distribución y el factor de emisión de la PRPA de - 2021. Correo electrónico en el archivo. El valor del factor de emisiones representa - la tasa de emisiones de carga compartida de Longmont sin REC desagregados.

+ description:

1) Stationary Diesel Use data was provided by CDPHE. + Spreadsheet on file.

2) Xcel Energy emission factor, electricity and + natural gas usage data, and renewable energy data is from Xcel's 2021 + Community Energy Report for Longmont, on file.

3) Longmont Power & + Communications data provided. PDF on file. The commercial buildings in + Longmont's Weld County portion consumed 31,000,000 kWh in 2021.

4) + T&D losses are calculated for Xcel and Longmont based on the EIA 861 + (https://www.eia.gov/electricity/data/eia861/) reported data. Specifically + the Utility Operational Data_2021. Spreadsheet on file.

5) Propane + data provided by FerrellGas. Spreadsheet on file.

6) Houses heated + with wood were pulled from the US Census Bureau, data on file. It was + assumed that all of the houses heated by wood in Longmont fell within the + Boulder County portion of Longmont.

7) Longmont Power and + Communications data was provided by Longmont Power & Communications. PDF on + file.

8) Longmont Power & Communications and Lyons Municipal rooftop + solar data was provided in kW of capacity rather than kWh generated. NREL's + PVWatts was used to estimate kWh generated by rooftop solar systems. PDF of + results on file. See https://pvwatts.nrel.gov/

10) Due to Xcel's + 15/15 rule, with less than 15 customers in the commercial and residential + data sets they cannot provide usage for those customers to avoid violating + privacy concerns. Therefore Lotus applied the average residential and + commercial kWh use from LPC to the 7 residential and commercial customers + (respectively) in Xcel territory to estimate use.

11) PRPA emission + factor and transmission and distribution loss rate for 2021 was provided. + Email on file. Emissions factor value represents Longmont's load share + emissions rate without unbundled RECs.

+ description_es-US:

1) Los datos sobre el consumo de diésel estacionario + fueron brindados por el CDPHE. Hoja de cálculo en el archivo.

2) El + factor de emisión de Xcel Energy, los datos de consumo de electricidad y gas + natural y los datos de energía renovable provienen del Informe de energía + comunitaria 2021 de Xcel para Longmont (en el archivo).

3) Datos + facilitados por Longmont Power & Communications. PDF en el archivo. Los + edificios comerciales en la parte del condado de Weld de Longmont + consumieron 31.000.000 kWh en el año 2021.

4) Las pérdidas de + transmisión y distribución se calculan para Xcel y Longmont según los datos + de la EIA 861 (https://www.eia.gov/electricity/data/eia861/). + Específicamente los Datos operativos de servicios públicos 2021. Hoja de + cálculo en el archivo.

5) Datos de propano facilitados por + FerrellGas. Hoja de cálculo en el archivo.

6) Las casas con + calefacción de madera fueron retiradas de la Oficina del Censo de los + Estados Unidos (datos en el archivo). Se supuso que todas las casas con + calefacción de madera se encontraban en la parte del condado de Boulder de + Longmont.

7) Los datos de energía y comunicaciones de Longmont fueron + facilitados por Longmont Power & Communications. PDF en el archivo.

+

8) Los datos de sistemas solares de techo de Longmont Power & + Communications y Lyons Municipal se ofrecieron en kW de capacidad en lugar + de kWh generados. Los PVWatts del NREL se utilizaron para estimar los kWh + generados por los sistemas solares de techo. PDF de los resultados en el + archivo. Visite https://pvwatts.nrel.gov/

10) Debido a la regla 15/15 + de Xcel, con menos de 15 clientes en los conjuntos de datos comerciales y + residenciales, no pueden brindar el consumo de esos clientes para evitar + problemas de privacidad. Por lo tanto, Lotus aplicó el consumo promedio de + kWh residencial y comercial de LPC a los siete clientes residenciales y + comerciales (respectivamente) en el territorio de Xcel para estimar el + consumo.

11) Se proporcionó la tasa de pérdida de transmisión y + distribución y el factor de emisión de la PRPA de 2021. Correo electrónico + en el archivo. El valor del factor de emisiones representa la tasa de + emisiones de carga compartida de Longmont sin REC desagregados.

type: formula.FormulaNode quantity: emissions unit: t_co2e/a @@ -1638,7 +1745,7 @@ nodes: - id: ghg - id: emission_sectors - id: scope - categories: [scope1] + categories: [scope1] params: formula: use * ef @@ -2417,7 +2524,8 @@ nodes: - id: gas_collection_rate to_dimensions: [] params: - formula: mass * complement(diversion_rate) * msw_characterization * ef * complement(gas_collection_rate) + formula: mass * complement(diversion_rate) * msw_characterization * ef * + complement(gas_collection_rate) output_nodes: - id: waste_emissions from_dimensions: @@ -2762,8 +2870,9 @@ nodes: - id: electric_vehicle_miles_traveled name: Electric vehicle miles traveled name_es-US: Millas recorridas por vehículos eléctricos - description_en: Amount of electric vehicle miles traveled replacing combustion engine miles. - Note! The energy carrier shows what energy carrier is replaced by electric vehicles. + description_en: Amount of electric vehicle miles traveled replacing combustion + engine miles. Note! The energy carrier shows what energy carrier is replaced + by electric vehicles. type: formula.FormulaNode quantity: mileage unit: VMT/a @@ -2781,11 +2890,13 @@ nodes: - id: vehicle_miles_traveled tags: [electric_vmt] params: - formula: vehicles * vmt_per_vehicle * extend_all(vmt / sum_dim(sum_dim(vmt, vehicle_type), energy_carrier)) + formula: vehicles * vmt_per_vehicle * extend_all(vmt / sum_dim(sum_dim(vmt, + vehicle_type), energy_carrier)) - id: vmt_change_due_to_public_transportation name: Change in vehicle miles traveled due to public transportation - name_es-US: Cambio en las millas recorridas por vehículos debido al transporte público + name_es-US: Cambio en las millas recorridas por vehículos debido al transporte + público type: formula.FormulaNode quantity: mileage unit: VMT/a @@ -2829,7 +2940,8 @@ nodes: - id: population tags: [pop] params: - formula: (extend_all(baseline) * ratio_to_max_hist_year(pop) - electric_vmt - transit) * complement(multimod) + formula: (extend_all(baseline) * ratio_to_max_hist_year(pop) - electric_vmt + - transit) * complement(multimod) - id: vehicle_fuel_use name: Vehicle fuel use @@ -2958,15 +3070,158 @@ nodes: params: operations: get_single_dataset,extend_to_history +- id: energy_use_intensity_improvement_updated_building_code + name: Energy use intensity improvement, updated_building_code (vintage-weighted) + description: Vintage-weighted average of energy_use_intensity_improvement's + updated_building_code figure -- each year's newly-brought-under-code floor + area is weighted by that year's achievable improvement, so floor area + brought under code in an earlier year keeps that year's rate rather than + being retroactively upgraded to the current year's rate. + type: formula.FormulaNode + quantity: fraction + unit: '%' + output_dimensions: [emission_sectors] + input_nodes: + - id: updated_building_code + tags: [flow] + from_dimensions: + - id: building_energy_class + categories: [existing] + flatten: true + - id: updated_building_code_stock + tags: [stock] + from_dimensions: + - id: building_energy_class + categories: [existing] + flatten: true + - id: energy_use_intensity_improvement + tags: [improvement] + from_dimensions: + - id: action + categories: [updated_building_code] + flatten: true + params: + formula: cumulative(flow * improvement) / (stock + 0.0000001) + +- id: energy_use_intensity_improvement_develop_beyond_building_code + name: Energy use intensity improvement, develop_beyond_building_code + (vintage-weighted) + description: Vintage-weighted average of energy_use_intensity_improvement's + develop_beyond_building_code figure -- see + energy_use_intensity_improvement_updated_building_code for the mechanism. + type: formula.FormulaNode + quantity: fraction + unit: '%' + output_dimensions: [emission_sectors] + input_nodes: + - id: develop_beyond_building_code + tags: [flow] + from_dimensions: + - id: building_energy_class + categories: [new] + flatten: true + - id: develop_beyond_building_code_stock + tags: [stock] + from_dimensions: + - id: building_energy_class + categories: [new] + flatten: true + - id: energy_use_intensity_improvement + tags: [improvement] + from_dimensions: + - id: action + categories: [develop_beyond_building_code] + flatten: true + params: + formula: cumulative(flow * improvement) / (stock + 0.0000001) + +- id: energy_use_intensity_improvement_benchmark_energy_use + name: Energy use intensity improvement, benchmark_energy_use (vintage-weighted) + description: Vintage-weighted average of energy_use_intensity_improvement's + benchmark_energy_use figure -- see + energy_use_intensity_improvement_updated_building_code for the mechanism. + benchmark_energy_use's flow is identical for existing and new building + stock, so either category can be used as the representative slice. + type: formula.FormulaNode + quantity: fraction + unit: '%' + output_dimensions: [emission_sectors] + input_nodes: + - id: benchmark_energy_use + tags: [flow] + from_dimensions: + - id: building_energy_class + categories: [existing] + flatten: true + - id: benchmark_energy_use_stock + tags: [stock] + from_dimensions: + - id: building_energy_class + categories: [existing] + flatten: true + - id: energy_use_intensity_improvement + tags: [improvement] + from_dimensions: + - id: action + categories: [benchmark_energy_use] + flatten: true + params: + formula: cumulative(flow * improvement) / (stock + 0.0000001) + +- id: energy_use_intensity_improvement_vintage + name: Energy use intensity improvement (vintage-weighted) + description: Per-action EUI improvement replacing + energy_use_intensity_improvement's raw current-year lookup with a + vintage-weighted average for updated_building_code, + develop_beyond_building_code, and benchmark_energy_use -- see + energy_use_intensity_improvement_updated_building_code for the mechanism. + none is unaffected (always 0, matching no action-driven improvement). + type: formula.FormulaNode + quantity: fraction + unit: '%' + input_dimensions: [emission_sectors, action] + output_dimensions: [emission_sectors, action] + input_nodes: + - id: energy_use_intensity_improvement + tags: [none_imp] + from_dimensions: + - id: action + categories: [none] + flatten: true + to_dimensions: + - id: emission_sectors + - id: action + categories: [none] + - id: energy_use_intensity_improvement_updated_building_code + tags: [ubc_imp] + to_dimensions: + - id: emission_sectors + - id: action + categories: [updated_building_code] + - id: energy_use_intensity_improvement_develop_beyond_building_code + tags: [dbbc_imp] + to_dimensions: + - id: emission_sectors + - id: action + categories: [develop_beyond_building_code] + - id: energy_use_intensity_improvement_benchmark_energy_use + tags: [beu_imp] + to_dimensions: + - id: emission_sectors + - id: action + categories: [benchmark_energy_use] + params: + formula: none_imp + ubc_imp + dbbc_imp + beu_imp + - id: energy_use_intensity_trend_natural_gas name: Energy use intensity trend (natural gas) name_es-US: Tendencia de la intensidad del uso de la energía (gas natural) - description: This action describes the assumed improvement in the energy use - intensity for the baseline scenario, i.e. changes that will occur anyway + description: This action describes the assumed improvement in the energy use + intensity for the baseline scenario, i.e. changes that will occur anyway without any activity from the city. - description_es-US: Esta acción describe la mejora supuesta en la intensidad + description_es-US: Esta acción describe la mejora supuesta en la intensidad del uso de la energía para el escenario de referencia, es decir, los cambios - que se producirán de todos modos sin intervención alguna por parte de la + que se producirán de todos modos sin intervención alguna por parte de la ciudad. type: generic.GenericNode quantity: consumption_factor @@ -2984,12 +3239,12 @@ nodes: - id: energy_use_intensity_trend_electricity name: Energy use intensity trend (electricity) name_es-US: Tendencia de la intensidad del uso de la energía (electricidad) - description: This action describes the assumed improvement in the energy use - intensity for the baseline scenario, i.e. changes that will occur anyway + description: This action describes the assumed improvement in the energy use + intensity for the baseline scenario, i.e. changes that will occur anyway without any activity from the city. - description_es-US: Esta acción describe la mejora supuesta en la intensidad + description_es-US: Esta acción describe la mejora supuesta en la intensidad del uso de la energía para el escenario de referencia, es decir, los cambios - que se producirán de todos modos sin intervención alguna por parte de la + que se producirán de todos modos sin intervención alguna por parte de la ciudad. type: generic.GenericNode quantity: consumption_factor @@ -3009,11 +3264,11 @@ nodes: - id: mpg_trend name: Miles per gallon increasing trend name_es-US: Tendencia al aumento de las millas recorridas por galón - description: The model assumes that internal combustion vehicles that use - gasoline and diesel will become more efficient, increase miles traveled per + description: The model assumes that internal combustion vehicles that use + gasoline and diesel will become more efficient, increase miles traveled per gallon, over time. - description_es-US: El modelo supone que, con el tiempo, los vehículos de - combustión interna que usan gasolina y diésel serán más eficientes y + description_es-US: El modelo supone que, con el tiempo, los vehículos de + combustión interna que usan gasolina y diésel serán más eficientes y aumentarán las millas recorridas por galón. type: simple.AdditiveNode quantity: fuel_consumption @@ -3033,11 +3288,11 @@ nodes: - id: vmt_per_vehicle_trend name: Vehicle miles per vehicle trend name_es-US: Tendencia de millas recorridas por vehículo - description:

The model assumes that overtime there will be a decrease in - the annual miles traveled per vehicle.

Unit: mi/(yr. v) = + description:

The model assumes that overtime there will be a decrease in + the annual miles traveled per vehicle.

Unit: mi/(yr. v) = miles per year per vehicle

- description_es-US:

El modelo supone que con el tiempo se producirá una - disminución de las millas anuales recorridas por vehículo.

+ description_es-US:

El modelo supone que con el tiempo se producirá una + disminución de las millas anuales recorridas por vehículo.

Unidad: mi/(a.v) = millas anuales por vehículo

type: simple.AdditiveNode quantity: mileage @@ -3052,10 +3307,10 @@ nodes: - id: electric_vehicle_efficiency_trend name: Electric vehicle efficiency trend name_es-US: Tendencia de la eficiencia de los vehículos eléctricos - description: The model assumes that electric vehicles will become more + description: The model assumes that electric vehicles will become more efficient, increase in miles traveled per kilowatt hour of energy, overtime. - description_es-US: El modelo supone que, con el tiempo, los vehículos - eléctricos serán más eficientes y aumentarán los kilómetros recorridos por + description_es-US: El modelo supone que, con el tiempo, los vehículos + eléctricos serán más eficientes y aumentarán los kilómetros recorridos por kilovatio hora de energía. type: simple.AdditiveNode quantity: fuel_consumption @@ -3081,28 +3336,16 @@ actions: de energía (EUI, por sus siglas en inglés) causada por esta acción.

Seguimiento de la aplicación:

' - type: shift.ShiftAction + type: simple.GenericAction quantity: fraction - unit: '%' + unit: '%/a' group: buildings - output_nodes: - - id: building_class_mix - tags: [add_to_existing_dims] + output_dimensions: [building_energy_class, emission_sectors] + input_datasets: + - id: longmont/building_actions + column: updated_building_code params: - - id: shift - unit: '%' - is_customizable: false - value: - - source: - categories: - action: none - building_energy_class: existing - dests: - - categories: - action: updated_building_code - building_energy_class: existing - amounts: - - {year: 2025, source_amount: -1.0, dest_amounts: [100]} + operations: get_single_dataset - id: develop_beyond_building_code name: Develop beyond building code @@ -3121,29 +3364,16 @@ actions: del consumo de energía (EUI, por sus siglas en inglés) causada por esta acción.

Seguimiento de la aplicación:

' - type: shift.ShiftAction + type: simple.GenericAction quantity: fraction - unit: '%' + unit: '%/a' group: buildings - output_nodes: - - id: building_class_mix - tags: [add_to_existing_dims] + output_dimensions: [building_energy_class, emission_sectors] + input_datasets: + - id: longmont/building_actions + column: develop_beyond_building_code params: - - id: shift - unit: '%' - is_customizable: false - value: - - source: - categories: - action: none - building_energy_class: new - dests: - - categories: - action: develop_beyond_building_code - building_energy_class: new - amounts: - - {year: 2025, source_amount: -90.0, dest_amounts: [100]} - - {year: 2026, source_amount: -0.0, dest_amounts: [100]} + operations: get_single_dataset - id: benchmark_energy_use name: Report and benchmark energy use in commercial buildings @@ -3161,30 +3391,16 @@ actions: de la intensidad del consumo de energía (EUI, por sus siglas en inglés) causada por esta acción.

Seguimiento de la aplicación:

' - type: shift.ShiftAction + type: simple.GenericAction quantity: fraction - unit: '%' + unit: '%/a' group: buildings - output_dimensions: [emission_sectors, building_energy_class, energy_carrier] - output_nodes: - - id: building_class_mix - tags: [add_to_existing_dims] + output_dimensions: [building_energy_class, emission_sectors] + input_datasets: + - id: longmont/building_actions + column: benchmark_energy_use params: - - id: shift - unit: '%' - is_customizable: false - value: - - source: - categories: - action: none - emission_sectors: commercial_industrial_energy - dests: - - categories: - action: benchmark_energy_use - emission_sectors: commercial_industrial_energy - amounts: - - {year: 2027, source_amount: -88.0, dest_amounts: [100]} - - {year: 2028, source_amount: -0.0, dest_amounts: [100]} + operations: get_single_dataset - id: efficiency_works_program name: Expand the Efficiency Works Program From cc7c612417cc78d92e0d8ddb109fe076b209e049 Mon Sep 17 00:00:00 2001 From: Jouni Tuomisto Date: Sun, 26 Jul 2026 18:37:45 +0300 Subject: [PATCH 4/7] Longmont: Fix additivity of building code and beyond building code --- configs/longmont-dev.yaml | 50 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/configs/longmont-dev.yaml b/configs/longmont-dev.yaml index 6dc8eb15..8e732a61 100644 --- a/configs/longmont-dev.yaml +++ b/configs/longmont-dev.yaml @@ -1228,6 +1228,28 @@ nodes: - id: building_class_mix name: Building class mix + description: "Floor area by building class
    +
  • old(t) = building_square_footage_old(t) [historical/baseline, existing stock]
    • +
  • new(t) = old(t) * (ratio_to_max_hist_year(population(t)) - 1) [scales with population growth]
+ + Building class mix — the share of floor area classified under each action, per (building energy class, emission sector) cell: +
  • flow_a(t) = yearly incremental share newly brought under action a
    • +
  • stock_a(t) = Σ_{t'≤t} flow_a(t') [cumulative, 'cumulative(flow)']
    • +
  • none(t) = 1 - Σ_a stock_a(t) [complement of the three actions]
+ where a ∈ {updated_building_code, develop_beyond_building_code, benchmark_energy_use}. + + Vintage-weighted EUI improvement — per action, weighted by when the floor area entered, not by the current year's rate: +
  • improvement_a(t) = Σ_{t'≤t} [flow_a(t') * improvement_table_a(t')] / Σ_{t'≤t} flow_a(t')
    • +
  • improvement_table_a(t') is the achievable EUI improvement for action a using the technology/code available in year t'. This means floor area brought under develop_beyond_building_code in 2025 keeps 2025's ~55%/58% rate forever, rather than being retroactively upgraded to whatever rate is current. improvement_none(t) = 0 always (no action → no improvement).
+ + EUI by action: +
  • EUI_a(t) = (eui_baseline + trend(t)) * (1 - improvement_a(t))
    • +
  • eui_baseline is the fixed observed baseline intensity; trend(t) is the autonomous/BAU improvement (additive, same regardless of action); improvement_a(t) is the vintage-weighted figure above (multiplicative discount via complement()).
+ + Total building energy use: +
  • Energy(t) = Σ_a [ floor_area_by_class(t, a) * EUI_a(t) ] * other multipliers (e.g. efficiency_works_program) + where floor_area_by_class(t, a) = (old(t) + new(t)) * building_class_mix(t, a).
" + type: formula.FormulaNode quantity: fraction unit: '%' @@ -3135,6 +3157,26 @@ nodes: params: formula: cumulative(flow * improvement) / (stock + 0.0000001) +- id: energy_use_intensity_improvement_develop_beyond_building_code_layered + name: Energy use intensity improvement, develop_beyond_building_code + (layered on updated_building_code) + description: develop_beyond_building_code's own vintage-weighted improvement + plus updated_building_code's, matching the original consultant + spreadsheet (Building Energy Model, row 99 equals row 61 -- both + actions' percentages are expressed against the same baseline, so they + add rather than each applying as an independent discount). + type: formula.FormulaNode + quantity: fraction + unit: '%' + output_dimensions: [emission_sectors] + input_nodes: + - id: energy_use_intensity_improvement_updated_building_code + tags: [ubc_imp] + - id: energy_use_intensity_improvement_develop_beyond_building_code + tags: [dbbc_imp] + params: + formula: ubc_imp + dbbc_imp + - id: energy_use_intensity_improvement_benchmark_energy_use name: Energy use intensity improvement, benchmark_energy_use (vintage-weighted) description: Vintage-weighted average of energy_use_intensity_improvement's @@ -3175,6 +3217,12 @@ nodes: vintage-weighted average for updated_building_code, develop_beyond_building_code, and benchmark_energy_use -- see energy_use_intensity_improvement_updated_building_code for the mechanism. + develop_beyond_building_code's improvement is layered on top of + updated_building_code's, matching the original consultant spreadsheet + (Building Energy Model, row 99 equals row 61 -- both actions' + percentages expressed against the same baseline), not applied + independently. benchmark_energy_use is not layered on + updated_building_code -- no evidence for that in the source spreadsheet. none is unaffected (always 0, matching no action-driven improvement). type: formula.FormulaNode quantity: fraction @@ -3198,7 +3246,7 @@ nodes: - id: emission_sectors - id: action categories: [updated_building_code] - - id: energy_use_intensity_improvement_develop_beyond_building_code + - id: energy_use_intensity_improvement_develop_beyond_building_code_layered tags: [dbbc_imp] to_dimensions: - id: emission_sectors From 27b27ca957631d75c5a486c0fa820119b3f11c0b Mon Sep 17 00:00:00 2001 From: Jouni Tuomisto Date: Sun, 26 Jul 2026 20:01:20 +0300 Subject: [PATCH 5/7] Longmont: Make beyond_building_code match excel logic --- configs/longmont-dev.yaml | 132 ++++++++++++++++++++++++++------------ 1 file changed, 91 insertions(+), 41 deletions(-) diff --git a/configs/longmont-dev.yaml b/configs/longmont-dev.yaml index 8e732a61..f4875919 100644 --- a/configs/longmont-dev.yaml +++ b/configs/longmont-dev.yaml @@ -1269,7 +1269,7 @@ nodes: - id: building_energy_class - id: emission_sectors - id: action - categories: [develop_beyond_building_code] + categories: [updated_building_code] - id: benchmark_energy_use_stock tags: [beu] to_dimensions: @@ -1318,8 +1318,8 @@ nodes: type: formula.FormulaNode quantity: consumption_factor unit: therm/ft**2/a - input_dimensions: [emission_sectors, action] - output_dimensions: [emission_sectors, action] + input_dimensions: [building_energy_class, emission_sectors, action] + output_dimensions: [building_energy_class, emission_sectors, action] input_datasets: - id: longmont/building_energy_use_intensity_eui column: natural_gas @@ -1354,6 +1354,7 @@ nodes: - id: building_natural_gas_eui_by_action tags: [eui] to_dimensions: + - id: building_energy_class - id: emission_sectors - id: action params: @@ -1364,8 +1365,8 @@ nodes: type: formula.FormulaNode quantity: consumption_factor unit: kWh/ft**2/a - input_dimensions: [emission_sectors, action] - output_dimensions: [emission_sectors, action] + input_dimensions: [building_energy_class, emission_sectors, action] + output_dimensions: [building_energy_class, emission_sectors, action] input_datasets: - id: longmont/building_energy_use_intensity_eui column: electricity @@ -1402,6 +1403,7 @@ nodes: - id: building_electricity_eui_by_action tags: [eui] to_dimensions: + - id: building_energy_class - id: emission_sectors - id: action params: @@ -3157,26 +3159,6 @@ nodes: params: formula: cumulative(flow * improvement) / (stock + 0.0000001) -- id: energy_use_intensity_improvement_develop_beyond_building_code_layered - name: Energy use intensity improvement, develop_beyond_building_code - (layered on updated_building_code) - description: develop_beyond_building_code's own vintage-weighted improvement - plus updated_building_code's, matching the original consultant - spreadsheet (Building Energy Model, row 99 equals row 61 -- both - actions' percentages are expressed against the same baseline, so they - add rather than each applying as an independent discount). - type: formula.FormulaNode - quantity: fraction - unit: '%' - output_dimensions: [emission_sectors] - input_nodes: - - id: energy_use_intensity_improvement_updated_building_code - tags: [ubc_imp] - - id: energy_use_intensity_improvement_develop_beyond_building_code - tags: [dbbc_imp] - params: - formula: ubc_imp + dbbc_imp - - id: energy_use_intensity_improvement_benchmark_energy_use name: Energy use intensity improvement, benchmark_energy_use (vintage-weighted) description: Vintage-weighted average of energy_use_intensity_improvement's @@ -3210,20 +3192,14 @@ nodes: params: formula: cumulative(flow * improvement) / (stock + 0.0000001) -- id: energy_use_intensity_improvement_vintage - name: Energy use intensity improvement (vintage-weighted) - description: Per-action EUI improvement replacing - energy_use_intensity_improvement's raw current-year lookup with a - vintage-weighted average for updated_building_code, - develop_beyond_building_code, and benchmark_energy_use -- see - energy_use_intensity_improvement_updated_building_code for the mechanism. - develop_beyond_building_code's improvement is layered on top of - updated_building_code's, matching the original consultant spreadsheet - (Building Energy Model, row 99 equals row 61 -- both actions' - percentages expressed against the same baseline), not applied - independently. benchmark_energy_use is not layered on - updated_building_code -- no evidence for that in the source spreadsheet. - none is unaffected (always 0, matching no action-driven improvement). +- id: energy_use_intensity_improvement_vintage_existing + name: Energy use intensity improvement (vintage-weighted, existing building + stock) + description: Per-action EUI improvement for existing building stock -- see + energy_use_intensity_improvement_updated_building_code for the vintage- + weighting mechanism. No develop_beyond_building_code term here -- it + doesn't apply to existing stock (see + energy_use_intensity_improvement_vintage_new). type: formula.FormulaNode quantity: fraction unit: '%' @@ -3246,12 +3222,53 @@ nodes: - id: emission_sectors - id: action categories: [updated_building_code] - - id: energy_use_intensity_improvement_develop_beyond_building_code_layered + - id: energy_use_intensity_improvement_benchmark_energy_use + tags: [beu_imp] + to_dimensions: + - id: emission_sectors + - id: action + categories: [benchmark_energy_use] + params: + formula: none_imp + ubc_imp + beu_imp + +- id: energy_use_intensity_improvement_vintage_new + name: Energy use intensity improvement (vintage-weighted, new building stock) + description: Per-action EUI improvement for new building stock. + develop_beyond_building_code has no category of its own here (matching + building_class_mix -- it applies to the same new-construction + population as updated_building_code, not a separate share, per the + original consultant spreadsheet's Building Energy Model rows 69-75 and + 112-119 both using the same affected-square-footage formula and + compliance rate). Its vintage-weighted improvement is instead added + into the updated_building_code slot here. + type: formula.FormulaNode + quantity: fraction + unit: '%' + input_dimensions: [emission_sectors, action] + output_dimensions: [emission_sectors, action] + input_nodes: + - id: energy_use_intensity_improvement + tags: [none_imp] + from_dimensions: + - id: action + categories: [none] + flatten: true + to_dimensions: + - id: emission_sectors + - id: action + categories: [none] + - id: energy_use_intensity_improvement_updated_building_code + tags: [ubc_imp] + to_dimensions: + - id: emission_sectors + - id: action + categories: [updated_building_code] + - id: energy_use_intensity_improvement_develop_beyond_building_code tags: [dbbc_imp] to_dimensions: - id: emission_sectors - id: action - categories: [develop_beyond_building_code] + categories: [updated_building_code] - id: energy_use_intensity_improvement_benchmark_energy_use tags: [beu_imp] to_dimensions: @@ -3261,6 +3278,39 @@ nodes: params: formula: none_imp + ubc_imp + dbbc_imp + beu_imp +- id: energy_use_intensity_improvement_vintage + name: Energy use intensity improvement (vintage-weighted) + description: Per-action EUI improvement replacing + energy_use_intensity_improvement's raw current-year lookup with a + vintage-weighted average for updated_building_code, + develop_beyond_building_code, and benchmark_energy_use -- combines + energy_use_intensity_improvement_vintage_existing and _new (see those + for the per-building-class breakdown; see + energy_use_intensity_improvement_updated_building_code for the + vintage-weighting mechanism itself). + type: formula.FormulaNode + quantity: fraction + unit: '%' + input_dimensions: [building_energy_class, emission_sectors, action] + output_dimensions: [building_energy_class, emission_sectors, action] + input_nodes: + - id: energy_use_intensity_improvement_vintage_existing + tags: [existing_slice] + to_dimensions: + - id: emission_sectors + - id: action + - id: building_energy_class + categories: [existing] + - id: energy_use_intensity_improvement_vintage_new + tags: [new_slice] + to_dimensions: + - id: emission_sectors + - id: action + - id: building_energy_class + categories: [new] + params: + formula: existing_slice + new_slice + - id: energy_use_intensity_trend_natural_gas name: Energy use intensity trend (natural gas) name_es-US: Tendencia de la intensidad del uso de la energía (gas natural) From 805e74eb163ab02612db647cb281a4e7eb92c6f2 Mon Sep 17 00:00:00 2001 From: Jouni Tuomisto Date: Mon, 27 Jul 2026 07:18:48 +0300 Subject: [PATCH 6/7] Longmont: Make benchmark_energy_use follow the excel logic to match values --- configs/longmont-dev.yaml | 244 ++++++++++++++++---------------------- 1 file changed, 104 insertions(+), 140 deletions(-) diff --git a/configs/longmont-dev.yaml b/configs/longmont-dev.yaml index f4875919..2b076da3 100644 --- a/configs/longmont-dev.yaml +++ b/configs/longmont-dev.yaml @@ -14,7 +14,7 @@ default_language: en supported_languages: [es-US] dataset_repo: url: https://github.com/kausaltech/dvctest.git - commit: ef08a8359893a551927be5b739dac879ee16b5a6 + commit: a7eacebcebdc8b131bc25a69fed8d8e38e2cedd7 dvc_remote: kausal-s3 name: 2025 Draft Longmont Greenhouse Gas Inventory name_es-US: 2025 Inventario de gases de efecto invernadero de Longmont @@ -1189,28 +1189,15 @@ nodes: params: formula: cumulative(flow) -- id: benchmark_energy_use_stock - name: Benchmark energy use (accumulated stock share) - description: Cumulative share of commercial/industrial building stock brought - under the benchmarking program so far, built from the action's own yearly - incremental share (see benchmark_energy_use). - type: formula.FormulaNode - quantity: fraction - unit: '%' - input_dimensions: [building_energy_class, emission_sectors] - output_dimensions: [building_energy_class, emission_sectors] - input_nodes: - - id: benchmark_energy_use - tags: [flow] - params: - formula: cumulative(flow) - - id: building_class_mix_none name: Building class mix (no action taken, residual) description: The complement of whatever share has been claimed by - updated_building_code, develop_beyond_building_code, and - benchmark_energy_use, so the three action shares plus this one always sum to - 100% for every (building_energy_class, emission_sectors) cell. + updated_building_code and develop_beyond_building_code, so the two action + shares plus this one always sum to 100% for every (building_energy_class, + emission_sectors) cell. benchmark_energy_use has no floor-area share here -- + it's now a direct GWh/a subtraction on building_electricity_use / + building_natural_gas_use, per request to keep it out of the EUI + path (see benchmark_energy_use's own node and README item 5). type: formula.FormulaNode quantity: fraction unit: '%' @@ -1221,34 +1208,37 @@ nodes: tags: [ubc] - id: develop_beyond_building_code_stock tags: [dbbc] - - id: benchmark_energy_use_stock - tags: [beu] params: - formula: complement(ubc + dbbc + beu) + formula: complement(ubc + dbbc) - id: building_class_mix name: Building class mix - description: "Floor area by building class
    -
  • old(t) = building_square_footage_old(t) [historical/baseline, existing stock]
    • -
  • new(t) = old(t) * (ratio_to_max_hist_year(population(t)) - 1) [scales with population growth]
- - Building class mix — the share of floor area classified under each action, per (building energy class, emission sector) cell: -
  • flow_a(t) = yearly incremental share newly brought under action a
    • -
  • stock_a(t) = Σ_{t'≤t} flow_a(t') [cumulative, 'cumulative(flow)']
    • -
  • none(t) = 1 - Σ_a stock_a(t) [complement of the three actions]
- where a ∈ {updated_building_code, develop_beyond_building_code, benchmark_energy_use}. - - Vintage-weighted EUI improvement — per action, weighted by when the floor area entered, not by the current year's rate: -
  • improvement_a(t) = Σ_{t'≤t} [flow_a(t') * improvement_table_a(t')] / Σ_{t'≤t} flow_a(t')
    • -
  • improvement_table_a(t') is the achievable EUI improvement for action a using the technology/code available in year t'. This means floor area brought under develop_beyond_building_code in 2025 keeps 2025's ~55%/58% rate forever, rather than being retroactively upgraded to whatever rate is current. improvement_none(t) = 0 always (no action → no improvement).
- - EUI by action: -
  • EUI_a(t) = (eui_baseline + trend(t)) * (1 - improvement_a(t))
    • -
  • eui_baseline is the fixed observed baseline intensity; trend(t) is the autonomous/BAU improvement (additive, same regardless of action); improvement_a(t) is the vintage-weighted figure above (multiplicative discount via complement()).
- - Total building energy use: -
  • Energy(t) = Σ_a [ floor_area_by_class(t, a) * EUI_a(t) ] * other multipliers (e.g. efficiency_works_program) - where floor_area_by_class(t, a) = (old(t) + new(t)) * building_class_mix(t, a).
" + description: "Floor area by building class
  • old(t) = building_square_footage_old(t)\ + \ [historical/baseline, existing stock]
  • new(t)\ + \ = old(t) * (ratio_to_max_hist_year(population(t)) - 1) [scales with population + growth]
\nBuilding class mix — the share of floor area classified under + each action, per (building energy class, emission sector) cell:
  • flow_a(t)\ + \ = yearly incremental share newly brought under action a
  • stock_a(t) + = Σ_{t'≤t} flow_a(t') [cumulative, 'cumulative(flow)']
    • +
  • none(t) = 1 - Σ_a stock_a(t) [complement + of the actions below]
where a ∈ {updated_building_code, develop_beyond_building_code}. + benchmark_energy_use is not one of these -- it has no floor-area share; it's a + direct GWh/a subtraction on building_electricity_use / building_natural_gas_use + instead (see benchmark_energy_use's own node).\nVintage-weighted EUI improvement + — per action, weighted by when the floor area entered, not by the current year's + rate:
  • improvement_a(t) = Σ_{t'≤t} [flow_a(t') * improvement_table_a(t')]\ + \ / Σ_{t'≤t} flow_a(t')
  • improvement_table_a(t') is the achievable EUI + improvement for action a using the technology/code available in year t'. This + means floor area brought under develop_beyond_building_code in 2025 keeps 2025's + ~55%/58% rate forever, rather than being retroactively upgraded to whatever rate + is current. improvement_none(t) = 0 always (no action → no improvement).
\n\ + EUI by action:
  • EUI_a(t) = (eui_baseline + trend(t)) * (1 - improvement_a(t))
    • +
  • eui_baseline is the fixed observed baseline intensity; trend(t) is the autonomous/BAU + improvement (additive, same regardless of action); improvement_a(t) is the vintage-weighted + figure above (multiplicative discount via complement()).
\nTotal building + energy use:
  • Energy(t) = Σ_a [ floor_area_by_class(t, a) * EUI_a(t) ] * + other multipliers (e.g. efficiency_works_program) where floor_area_by_class(t, + a) = (old(t) + new(t)) * building_class_mix(t, a).
" type: formula.FormulaNode quantity: fraction @@ -1270,13 +1260,6 @@ nodes: - id: emission_sectors - id: action categories: [updated_building_code] - - id: benchmark_energy_use_stock - tags: [beu] - to_dimensions: - - id: building_energy_class - - id: emission_sectors - - id: action - categories: [benchmark_energy_use] - id: building_class_mix_none tags: [none_share] to_dimensions: @@ -1285,7 +1268,7 @@ nodes: - id: action categories: [none] params: - formula: ubc + dbbc + beu + none_share + formula: ubc + dbbc + none_share - id: electricity_use_on_streetlights_and_internal_operations name: Electricity use on streetlights and internal operations @@ -1337,7 +1320,10 @@ nodes: - id: building_natural_gas_use name: Building natural gas use name_es-US: Consumo de gas natural en edificios - description: Unit: Mthm/yr. = million therms per year. + description: Unit: Mthm/yr. = million therms per year. Includes a + direct Mthm/a-scale subtraction of benchmark_energy_use's cumulative savings, + assigned to the existing building stock (see that action's own node for the + rationale). description_es-US: Unidad: Mthm/año = millones de termias al año. type: formula.FormulaNode quantity: energy @@ -1358,7 +1344,8 @@ nodes: - id: emission_sectors - id: action params: - formula: sum_dim(square_footage * eui, action) * efficiency_works_program + formula: sum_dim(square_footage * eui, action) * efficiency_works_program - + benchmark_natural_gas - id: building_electricity_eui_by_action name: Energy use intensity EUI for electricity by action @@ -1385,7 +1372,9 @@ nodes: name: Building electricity use name_es-US: Consumo de electricidad en edificios description: Unit: GWh/yr. = gigawatt-hours per year = million - kilowatt-hours per year. + kilowatt-hours per year. Includes a direct GWh/a-scale subtraction of + benchmark_energy_use's cumulative savings, assigned to the existing building + stock (see that action's own node for the rationale). description_es-US: Unidad: GWh/año = gigavatios hora al año = millones de kilovatios hora al año. type: formula.FormulaNode @@ -1407,7 +1396,8 @@ nodes: - id: emission_sectors - id: action params: - formula: sum_dim(square_footage * eui, action) * efficiency_works_program + formula: sum_dim(square_footage * eui, action) * efficiency_works_program - + benchmark_electricity - id: electricity_use name: Electricity use @@ -3095,11 +3085,12 @@ nodes: operations: get_single_dataset,extend_to_history - id: energy_use_intensity_improvement_updated_building_code - name: Energy use intensity improvement, updated_building_code (vintage-weighted) - description: Vintage-weighted average of energy_use_intensity_improvement's - updated_building_code figure -- each year's newly-brought-under-code floor - area is weighted by that year's achievable improvement, so floor area - brought under code in an earlier year keeps that year's rate rather than + name: Energy use intensity improvement, updated_building_code + (vintage-weighted) + description: Vintage-weighted average of energy_use_intensity_improvement's + updated_building_code figure -- each year's newly-brought-under-code floor + area is weighted by that year's achievable improvement, so floor area + brought under code in an earlier year keeps that year's rate rather than being retroactively upgraded to the current year's rate. type: formula.FormulaNode quantity: fraction @@ -3128,10 +3119,10 @@ nodes: formula: cumulative(flow * improvement) / (stock + 0.0000001) - id: energy_use_intensity_improvement_develop_beyond_building_code - name: Energy use intensity improvement, develop_beyond_building_code + name: Energy use intensity improvement, develop_beyond_building_code (vintage-weighted) - description: Vintage-weighted average of energy_use_intensity_improvement's - develop_beyond_building_code figure -- see + description: Vintage-weighted average of energy_use_intensity_improvement's + develop_beyond_building_code figure -- see energy_use_intensity_improvement_updated_building_code for the mechanism. type: formula.FormulaNode quantity: fraction @@ -3159,47 +3150,13 @@ nodes: params: formula: cumulative(flow * improvement) / (stock + 0.0000001) -- id: energy_use_intensity_improvement_benchmark_energy_use - name: Energy use intensity improvement, benchmark_energy_use (vintage-weighted) - description: Vintage-weighted average of energy_use_intensity_improvement's - benchmark_energy_use figure -- see - energy_use_intensity_improvement_updated_building_code for the mechanism. - benchmark_energy_use's flow is identical for existing and new building - stock, so either category can be used as the representative slice. - type: formula.FormulaNode - quantity: fraction - unit: '%' - output_dimensions: [emission_sectors] - input_nodes: - - id: benchmark_energy_use - tags: [flow] - from_dimensions: - - id: building_energy_class - categories: [existing] - flatten: true - - id: benchmark_energy_use_stock - tags: [stock] - from_dimensions: - - id: building_energy_class - categories: [existing] - flatten: true - - id: energy_use_intensity_improvement - tags: [improvement] - from_dimensions: - - id: action - categories: [benchmark_energy_use] - flatten: true - params: - formula: cumulative(flow * improvement) / (stock + 0.0000001) - - id: energy_use_intensity_improvement_vintage_existing - name: Energy use intensity improvement (vintage-weighted, existing building + name: Energy use intensity improvement (vintage-weighted, existing building stock) - description: Per-action EUI improvement for existing building stock -- see - energy_use_intensity_improvement_updated_building_code for the vintage- - weighting mechanism. No develop_beyond_building_code term here -- it - doesn't apply to existing stock (see - energy_use_intensity_improvement_vintage_new). + description: Per-action EUI improvement for existing building stock -- see + energy_use_intensity_improvement_updated_building_code for the vintage- + weighting mechanism. No develop_beyond_building_code term here -- it doesn't + apply to existing stock (see energy_use_intensity_improvement_vintage_new). type: formula.FormulaNode quantity: fraction unit: '%' @@ -3222,25 +3179,19 @@ nodes: - id: emission_sectors - id: action categories: [updated_building_code] - - id: energy_use_intensity_improvement_benchmark_energy_use - tags: [beu_imp] - to_dimensions: - - id: emission_sectors - - id: action - categories: [benchmark_energy_use] params: - formula: none_imp + ubc_imp + beu_imp + formula: none_imp + ubc_imp - id: energy_use_intensity_improvement_vintage_new name: Energy use intensity improvement (vintage-weighted, new building stock) - description: Per-action EUI improvement for new building stock. - develop_beyond_building_code has no category of its own here (matching - building_class_mix -- it applies to the same new-construction - population as updated_building_code, not a separate share, per the - original consultant spreadsheet's Building Energy Model rows 69-75 and - 112-119 both using the same affected-square-footage formula and - compliance rate). Its vintage-weighted improvement is instead added - into the updated_building_code slot here. + description: Per-action EUI improvement for new building stock. + develop_beyond_building_code has no category of its own here (matching + building_class_mix -- it applies to the same new-construction population as + updated_building_code, not a separate share, per the original consultant + spreadsheet's Building Energy Model rows 69-75 and 112-119 both using the + same affected-square-footage formula and compliance rate). Its + vintage-weighted improvement is instead added into the updated_building_code + slot here. type: formula.FormulaNode quantity: fraction unit: '%' @@ -3269,25 +3220,22 @@ nodes: - id: emission_sectors - id: action categories: [updated_building_code] - - id: energy_use_intensity_improvement_benchmark_energy_use - tags: [beu_imp] - to_dimensions: - - id: emission_sectors - - id: action - categories: [benchmark_energy_use] params: - formula: none_imp + ubc_imp + dbbc_imp + beu_imp + formula: none_imp + ubc_imp + dbbc_imp - id: energy_use_intensity_improvement_vintage name: Energy use intensity improvement (vintage-weighted) - description: Per-action EUI improvement replacing - energy_use_intensity_improvement's raw current-year lookup with a - vintage-weighted average for updated_building_code, - develop_beyond_building_code, and benchmark_energy_use -- combines - energy_use_intensity_improvement_vintage_existing and _new (see those - for the per-building-class breakdown; see - energy_use_intensity_improvement_updated_building_code for the - vintage-weighting mechanism itself). + description: Per-action EUI improvement replacing + energy_use_intensity_improvement's raw current-year lookup with a + vintage-weighted average for updated_building_code and + develop_beyond_building_code -- combines + energy_use_intensity_improvement_vintage_existing and _new (see those for + the per-building-class breakdown; see + energy_use_intensity_improvement_updated_building_code for the + vintage-weighting mechanism itself). benchmark_energy_use is not part of + this mechanism -- it's a direct GWh/a subtraction on + building_electricity_use / building_natural_gas_use instead (see that + action's own node). type: formula.FormulaNode quantity: fraction unit: '%' @@ -3489,16 +3437,32 @@ actions: de la intensidad del consumo de energía (EUI, por sus siglas en inglés) causada por esta acción.

Seguimiento de la aplicación:

' - type: simple.GenericAction - quantity: fraction - unit: '%/a' + type: simple.AdditiveAction group: buildings - output_dimensions: [building_energy_class, emission_sectors] + output_dimensions: [emission_sectors, building_energy_class] + output_metrics: + - id: electricity + quantity: energy + unit: GWh/a + column_id: benchmark_energy_use_electricity + - id: natural_gas + quantity: energy + unit: Mthm/a + column_id: benchmark_energy_use_natural_gas input_datasets: - id: longmont/building_actions - column: benchmark_energy_use - params: - operations: get_single_dataset + tags: [extend_all] + forecast_from: 2025 + filters: + - column: updated_building_code + - column: develop_beyond_building_code + output_nodes: + - id: building_electricity_use + metrics: [benchmark_energy_use_electricity] + tags: [benchmark_electricity] + - id: building_natural_gas_use + metrics: [benchmark_energy_use_natural_gas] + tags: [benchmark_natural_gas] - id: efficiency_works_program name: Expand the Efficiency Works Program From e8e6b285a5e6ca74965d51790f2a04d29753c897 Mon Sep 17 00:00:00 2001 From: Jouni Tuomisto Date: Mon, 27 Jul 2026 07:24:07 +0300 Subject: [PATCH 7/7] Improve handling of multiple metric columns in datasets --- common/polars.py | 13 +++++++++ nodes/datasets.py | 60 ++++++++++++++++++++++++++++++----------- nodes/defs/node_defs.py | 32 ++++++++++++++++++++++ nodes/spec_export.py | 14 ++++++---- 4 files changed, 98 insertions(+), 21 deletions(-) diff --git a/common/polars.py b/common/polars.py index ec0e45af..3f674892 100644 --- a/common/polars.py +++ b/common/polars.py @@ -1,5 +1,6 @@ from __future__ import annotations +import collections.abc import re import typing from dataclasses import dataclass @@ -181,6 +182,18 @@ def drop( *columns: ColumnNameOrSelector | Iterable[ColumnNameOrSelector], strict: bool = True, ) -> PathsDataFrame: + if strict: + requested: list[str] = [] + for col in columns: + if isinstance(col, str): + requested.append(col) + elif isinstance(col, collections.abc.Iterable): + requested.extend(c for c in col if isinstance(c, str)) + missing = [col for col in requested if col not in self.columns] + if missing: + raise Exception( + 'Cannot drop non-existent column(s) %s; available columns: %s' % (', '.join(missing), ', '.join(self.columns)) + ) meta = self.get_meta() df = super().drop(*columns, strict=strict) for col in list(meta.units.keys()): diff --git a/nodes/datasets.py b/nodes/datasets.py index d5c1c721..156ad830 100644 --- a/nodes/datasets.py +++ b/nodes/datasets.py @@ -243,6 +243,7 @@ def get_input_dataset_filter_adapter() -> TypeAdapter[InputDatasetFilterDef]: class FilterDatasetKwargs(DatasetKwargs): column: str | None + columns: list[str] | None filters: list[InputDatasetFilterDef] | None dropna: bool | None min_year: int | None @@ -254,6 +255,7 @@ class FilterDatasetKwargs(DatasetKwargs): @dataclass class DatasetWithFilters(Dataset, ABC): column: str | None = None + columns: list[str] | None = None filters: list[InputDatasetFilterDef] | None = None dropna: bool | None = None min_year: int | None = None @@ -268,6 +270,7 @@ def kwargs_from_def(cls, ds_def: InputDatasetDef) -> FilterDatasetKwargs: return FilterDatasetKwargs( **super().kwargs_from_def(ds_def), column=ds_def.column, + columns=ds_def.columns, filters=ds_def.filters, dropna=ds_def.dropna, min_year=ds_def.min_year, @@ -280,6 +283,8 @@ def __rich_repr__(self) -> RichReprResult: yield from super().__rich_repr__() if self.column is not None: yield 'column', self.column + if self.columns is not None: + yield 'columns', self.columns if self.filters is not None: yield 'filters', len(self.filters) @@ -438,6 +443,43 @@ def _operate_tags(self, df: ppl.PathsDataFrame) -> ppl.PathsDataFrame: df = df.paths.get_operation(tag)(df, context) return df + def _select_column(self, df: ppl.PathsDataFrame, cols: list[str]) -> tuple[ppl.PathsDataFrame, list[str]]: + assert self.column + if self.column not in cols: + available = ', '.join(cols) + raise DatasetError( + self, + "Column '%s' not found in dataset '%s'. Available columns: %s" % (self.column, self.id, available), + ) + df = df.with_columns(pl.col(self.column).alias(VALUE_COLUMN)) + df = df.filter(pl.col(VALUE_COLUMN).is_not_null()) + return df, [YEAR_COLUMN, VALUE_COLUMN, *df.dim_ids] + + def _select_columns(self, df: ppl.PathsDataFrame, cols: list[str]) -> tuple[ppl.PathsDataFrame, list[str]]: + assert self.columns + missing = [col for col in self.columns if col not in cols] + if missing: + available = ', '.join(cols) + raise DatasetError( + self, + "Column(s) %s not found in dataset '%s'. Available columns: %s" % (', '.join(missing), self.id, available), + ) + null_condition = pl.lit(True) # noqa: FBT003 + for col in self.columns: + null_condition = null_condition & pl.col(col).is_null() + df = df.filter(~null_condition) + all_null_dims = [dim_id for dim_id in df.dim_ids if df[dim_id].null_count() == len(df)] + if all_null_dims: + df = df.drop(all_null_dims) + return df, [YEAR_COLUMN, *self.columns, *df.dim_ids] + + def _select_dataset_columns(self, df: ppl.PathsDataFrame, cols: list[str]) -> tuple[ppl.PathsDataFrame, list[str]]: + if self.column: + return self._select_column(df, cols) + if self.columns: + return self._select_columns(df, cols) + return df, cols + @measure_dataset_call('dataset.filter', capture_df_result=True, capture_df_arg=True) def _filter_and_process_df(self, df: ppl.PathsDataFrame) -> ppl.PathsDataFrame: from nodes.defs.node_defs import RenameColumnDatasetFilterDef @@ -449,22 +491,7 @@ def _filter_and_process_df(self, df: ppl.PathsDataFrame) -> ppl.PathsDataFrame: df = self._rename_col_filter(df, filter_def) cols = list(df.columns) - - if self.column: - if self.column not in cols: - available = ', '.join(cols) - raise DatasetError( - self, - "Column '%s' not found in dataset '%s'. Available columns: %s" - % ( - self.column, - self.id, - available, - ), - ) - df = df.with_columns(pl.col(self.column).alias(VALUE_COLUMN)) - df = df.filter(pl.col(VALUE_COLUMN).is_not_null()) - cols = [YEAR_COLUMN, VALUE_COLUMN, *df.dim_ids] + df, cols = self._select_dataset_columns(df, cols) if YEAR_COLUMN in cols and YEAR_COLUMN not in df.primary_keys: df = df.add_to_index(YEAR_COLUMN) @@ -789,6 +816,7 @@ def hash_data(self) -> dict[str, Any]: extra_fields = [ 'input_dataset', 'column', + 'columns', 'filters', 'dropna', 'forecast_from', diff --git a/nodes/defs/node_defs.py b/nodes/defs/node_defs.py index 492d8355..b399f354 100644 --- a/nodes/defs/node_defs.py +++ b/nodes/defs/node_defs.py @@ -76,6 +76,13 @@ class SelectColumnDatasetTransformOp(BaseModel): column: str +class SelectColumnsDatasetTransformOp(BaseModel): + model_config = ConfigDict(extra='forbid') + + kind: Literal['select_columns'] = 'select_columns' + columns: list[str] + + class FilterColumnDatasetTransformOp(BaseModel): model_config = ConfigDict(extra='forbid') @@ -140,6 +147,7 @@ class ForecastFromDatasetTransformOp(BaseModel): type DatasetTransformOp = ( SelectColumnDatasetTransformOp + | SelectColumnsDatasetTransformOp | FilterColumnDatasetTransformOp | FilterDimensionDatasetTransformOp | RenameItemDatasetTransformOp @@ -179,6 +187,18 @@ class InputDatasetDef(I18nBaseModel): input_dataset: str | None = None """DVC dataset identifier override (when different from ``id``).""" column: str | None = None + columns: list[str] | None = None + """ + Explicit whitelist of metric columns to keep, for nodes that consume + several (but not all) metric columns from a shared multi-metric dataset + (e.g. a multimetric ``simple.AdditiveAction``). Every other unlisted + metric column is dropped, rows where all listed columns are null are + dropped, and any dimension column left entirely null by that row-drop is + dropped too -- the same "empty dimensions are dropped automatically" + behavior ``column`` gets, generalized to more than one column. Mutually + exclusive with ``column``, which aliases a single column to the internal + value column instead of keeping it under its own name. + """ forecast_from: int | None = None filters: list[InputDatasetFilterDef] = Field(default_factory=list) dropna: bool | None = None @@ -187,10 +207,18 @@ class InputDatasetDef(I18nBaseModel): unit: Unit | None = None output_dimensions: list[DimensionRef] | None = None + @model_validator(mode='after') + def validate_column_and_columns(self) -> InputDatasetDef: + if self.column is not None and self.columns is not None: + raise ValueError("'column' and 'columns' are mutually exclusive") + return self + def to_transform_pipeline(self) -> DatasetTransformPipelineDef: operations: list[DatasetTransformOp] = [] if self.column is not None: operations.append(SelectColumnDatasetTransformOp(column=self.column)) + if self.columns is not None: + operations.append(SelectColumnsDatasetTransformOp(columns=self.columns)) operations.extend(input_dataset_filter_to_transform_op(filter_def) for filter_def in self.filters) if self.forecast_from is not None: operations.append(ForecastFromDatasetTransformOp(year=self.forecast_from)) @@ -220,6 +248,8 @@ class DatasetPortSpec(I18nBaseModel): wide DVC datasets use human-readable column labels with spaces (e.g. "Trucks and lorries"). """ + columns: list[str] | None = None + """Mirrors ``InputDatasetDef.columns`` -- see there. Mutually exclusive with ``column``.""" forecast_from: int | None = None filters: list[InputDatasetFilterDef] = Field(default_factory=list) dropna: bool | None = None @@ -234,6 +264,7 @@ def from_input_dataset(cls, ds_def: InputDatasetDef) -> DatasetPortSpec: tags=ds_def.tags, input_dataset=ds_def.input_dataset, column=ds_def.column, + columns=ds_def.columns, forecast_from=ds_def.forecast_from, filters=ds_def.filters, dropna=ds_def.dropna, @@ -249,6 +280,7 @@ def to_input_dataset(self, *, id: DatasetIdentifier) -> InputDatasetDef: tags=self.tags, input_dataset=self.input_dataset, column=self.column, + columns=self.columns, forecast_from=self.forecast_from, filters=self.filters, dropna=self.dropna, diff --git a/nodes/spec_export.py b/nodes/spec_export.py index a4af2ce0..ab396f7f 100644 --- a/nodes/spec_export.py +++ b/nodes/spec_export.py @@ -354,12 +354,14 @@ def _dataset_binding_columns_for_node(node: Node, ds_instance: DatasetWithFilter """ Return dataset metric columns to expose as editor bindings. - Single-column dataset inputs carry ``column`` directly. Multi-metric - action datasets usually leave it unset and consume columns matching the - node's output metrics. + Single-column dataset inputs carry ``column`` directly; an explicit + ``columns`` whitelist is used as-is. Multi-metric action datasets that + leave both unset consume columns matching the node's output metrics. """ if ds_instance.column is not None: return [ds_instance.column] + if ds_instance.columns is not None: + return list(ds_instance.columns) columns: list[str] = [] seen: set[str] = set() for metric in node.output_metrics.values(): @@ -512,6 +514,7 @@ def _input_dataset_def_from_instance(ds: DatasetWithFilters) -> InputDatasetDef: tags=ds.tags or [], input_dataset=ds.input_dataset if isinstance(ds, DVCDataset) else None, column=ds.column, + columns=ds.columns, forecast_from=ds.forecast_from, filters=ds.filters or [], dropna=ds.dropna, @@ -706,7 +709,8 @@ def _resolve_dataset_ports( ports: list[DatasetPort] = [] spec = DatasetPortSpec.from_input_dataset(_input_dataset_def_from_instance(ds_instance)) metric_columns = _dataset_binding_columns_for_node(node, ds_instance) - if ds_instance.column is None: + explicit_columns = ds_instance.column is not None or ds_instance.columns is not None + if not explicit_columns: # Column-less bindings: the node consumes the full frame. Bind to every # metric the dataset actually exposes so ports stay accurate even when # the node renames columns post-load (e.g. HsyNode translating Finnish @@ -718,7 +722,7 @@ def _resolve_dataset_ports( for column in metric_columns: metric = metrics_by_schema_and_name.get((dataset_obj.schema.pk, column)) if metric is None: - if ds_instance.column is not None: + if explicit_columns: raise ValueError(f'No metric {column} in dataset {ds_instance.id} for node {node.id}') logger.debug('No metric %s in dataset %s for node %s; skipping dataset-port binding', column, ds_instance.id, node.id) continue