Regenerate ferc dbt schemas

catalyst-cooperative · Feb 6, 2025 · 012ba4a · 012ba4a
1 parent 79e2153
commit 012ba4a
Show file tree

Hide file tree

Showing 4 changed files with 127 additions and 34 deletions.
diff --git a/dbt/models/ferc1/out_ferc1__yearly_steam_plants_fuel_by_plant_sched402/schema.yml b/dbt/models/ferc1/out_ferc1__yearly_steam_plants_fuel_by_plant_sched402/schema.yml
@@ -1,5 +1,4 @@
 version: 2
-
 sources:
   - name: pudl
     tables:
@@ -8,6 +7,27 @@ sources:
           - check_row_counts_per_partition:
               table_name: out_ferc1__yearly_steam_plants_fuel_by_plant_sched402
               partition_column: report_year
+        columns:
+          - name: report_year
+          - name: utility_id_ferc1
+          - name: utility_id_pudl
+          - name: utility_name_ferc1
+          - name: plant_id_pudl
+          - name: plant_name_ferc1
+          - name: coal_fraction_cost
+          - name: coal_fraction_mmbtu
+          - name: fuel_cost
+          - name: fuel_mmbtu
+          - name: gas_fraction_cost
+          - name: gas_fraction_mmbtu
+          - name: nuclear_fraction_cost
+          - name: nuclear_fraction_mmbtu
+          - name: oil_fraction_cost
+          - name: oil_fraction_mmbtu
+          - name: primary_fuel_by_cost
+          - name: primary_fuel_by_mmbtu
+          - name: waste_fraction_cost
+          - name: waste_fraction_mmbtu
 
 models:
   - name: validate_ferc1__yearly_steam_plants_fuel_by_plant_sched402

diff --git a/dbt/models/ferc1/out_ferc1__yearly_steam_plants_sched402/schema.yml b/dbt/models/ferc1/out_ferc1__yearly_steam_plants_sched402/schema.yml
@@ -1,5 +1,4 @@
 version: 2
-
 sources:
   - name: pudl
     tables:
@@ -8,6 +7,62 @@ sources:
           - check_row_counts_per_partition:
               table_name: out_ferc1__yearly_steam_plants_sched402
               partition_column: report_year
+        columns:
+          - name: report_year
+          - name: utility_id_ferc1
+          - name: utility_id_pudl
+          - name: utility_name_ferc1
+          - name: plant_id_pudl
+          - name: plant_id_ferc1
+          - name: plant_name_ferc1
+          - name: asset_retirement_cost
+          - name: avg_num_employees
+          - name: capacity_factor
+          - name: capacity_mw
+          - name: capex_annual_addition
+          - name: capex_annual_addition_rolling
+          - name: capex_annual_per_kw
+          - name: capex_annual_per_mw
+          - name: capex_annual_per_mw_rolling
+          - name: capex_annual_per_mwh
+          - name: capex_annual_per_mwh_rolling
+          - name: capex_equipment
+          - name: capex_land
+          - name: capex_per_mw
+          - name: capex_structures
+          - name: capex_total
+          - name: capex_wo_retirement_total
+          - name: construction_type
+          - name: construction_year
+          - name: installation_year
+          - name: net_generation_mwh
+          - name: not_water_limited_capacity_mw
+          - name: opex_allowances
+          - name: opex_boiler
+          - name: opex_coolants
+          - name: opex_electric
+          - name: opex_engineering
+          - name: opex_fuel
+          - name: opex_fuel_per_mwh
+          - name: opex_misc_power
+          - name: opex_misc_steam
+          - name: opex_nonfuel_per_mwh
+          - name: opex_operations
+          - name: opex_per_mwh
+          - name: opex_plants
+          - name: opex_production_total
+          - name: opex_rents
+          - name: opex_steam
+          - name: opex_steam_other
+          - name: opex_structures
+          - name: opex_total_nonfuel
+          - name: opex_transfer
+          - name: peak_demand_mw
+          - name: plant_capability_mw
+          - name: plant_hours_connected_while_generating
+          - name: plant_type
+          - name: record_id
+          - name: water_limited_capacity_mw
 
 models:
   - name: validate_ferc1__yearly_steam_plants_sched402

diff --git a/dbt/seeds/row_counts.csv b/dbt/seeds/row_counts.csv
@@ -34,36 +34,6 @@ out_ferc1__yearly_steam_plants_fuel_by_plant_sched402,2020,716
 out_ferc1__yearly_steam_plants_fuel_by_plant_sched402,2021,751
 out_ferc1__yearly_steam_plants_fuel_by_plant_sched402,2022,765
 out_ferc1__yearly_steam_plants_fuel_by_plant_sched402,2023,759
-out_ferc1__yearly_steam_plants_sched402,1994,1411
-out_ferc1__yearly_steam_plants_sched402,1995,1448
-out_ferc1__yearly_steam_plants_sched402,1996,1395
-out_ferc1__yearly_steam_plants_sched402,1997,1328
-out_ferc1__yearly_steam_plants_sched402,1998,1339
-out_ferc1__yearly_steam_plants_sched402,1999,1310
-out_ferc1__yearly_steam_plants_sched402,2000,1308
-out_ferc1__yearly_steam_plants_sched402,2001,1174
-out_ferc1__yearly_steam_plants_sched402,2002,1108
-out_ferc1__yearly_steam_plants_sched402,2003,1033
-out_ferc1__yearly_steam_plants_sched402,2004,989
-out_ferc1__yearly_steam_plants_sched402,2005,1064
-out_ferc1__yearly_steam_plants_sched402,2006,1003
-out_ferc1__yearly_steam_plants_sched402,2007,1013
-out_ferc1__yearly_steam_plants_sched402,2008,1022
-out_ferc1__yearly_steam_plants_sched402,2009,1015
-out_ferc1__yearly_steam_plants_sched402,2010,1032
-out_ferc1__yearly_steam_plants_sched402,2011,1043
-out_ferc1__yearly_steam_plants_sched402,2012,1034
-out_ferc1__yearly_steam_plants_sched402,2013,1007
-out_ferc1__yearly_steam_plants_sched402,2014,997
-out_ferc1__yearly_steam_plants_sched402,2015,999
-out_ferc1__yearly_steam_plants_sched402,2016,957
-out_ferc1__yearly_steam_plants_sched402,2017,938
-out_ferc1__yearly_steam_plants_sched402,2018,922
-out_ferc1__yearly_steam_plants_sched402,2019,929
-out_ferc1__yearly_steam_plants_sched402,2020,945
-out_ferc1__yearly_steam_plants_sched402,2021,1046
-out_ferc1__yearly_steam_plants_sched402,2022,1070
-out_ferc1__yearly_steam_plants_sched402,2023,1096
 out_eia923__boiler_fuel,2008-01-01,6389
 out_eia923__boiler_fuel,2008-02-01,6358
 out_eia923__boiler_fuel,2008-03-01,6435
@@ -268,3 +238,33 @@ out_eia923__boiler_fuel,2024-09-01,6250
 out_eia923__boiler_fuel,2024-10-01,6250
 out_eia923__boiler_fuel,2024-11-01,6250
 out_eia923__boiler_fuel,2024-12-01,6250
+out_ferc1__yearly_steam_plants_sched402,1994,1411
+out_ferc1__yearly_steam_plants_sched402,1995,1448
+out_ferc1__yearly_steam_plants_sched402,1996,1395
+out_ferc1__yearly_steam_plants_sched402,1997,1328
+out_ferc1__yearly_steam_plants_sched402,1998,1339
+out_ferc1__yearly_steam_plants_sched402,1999,1310
+out_ferc1__yearly_steam_plants_sched402,2000,1308
+out_ferc1__yearly_steam_plants_sched402,2001,1174
+out_ferc1__yearly_steam_plants_sched402,2002,1108
+out_ferc1__yearly_steam_plants_sched402,2003,1033
+out_ferc1__yearly_steam_plants_sched402,2004,989
+out_ferc1__yearly_steam_plants_sched402,2005,1064
+out_ferc1__yearly_steam_plants_sched402,2006,1003
+out_ferc1__yearly_steam_plants_sched402,2007,1013
+out_ferc1__yearly_steam_plants_sched402,2008,1022
+out_ferc1__yearly_steam_plants_sched402,2009,1015
+out_ferc1__yearly_steam_plants_sched402,2010,1032
+out_ferc1__yearly_steam_plants_sched402,2011,1043
+out_ferc1__yearly_steam_plants_sched402,2012,1034
+out_ferc1__yearly_steam_plants_sched402,2013,1007
+out_ferc1__yearly_steam_plants_sched402,2014,997
+out_ferc1__yearly_steam_plants_sched402,2015,999
+out_ferc1__yearly_steam_plants_sched402,2016,957
+out_ferc1__yearly_steam_plants_sched402,2017,938
+out_ferc1__yearly_steam_plants_sched402,2018,922
+out_ferc1__yearly_steam_plants_sched402,2019,929
+out_ferc1__yearly_steam_plants_sched402,2020,945
+out_ferc1__yearly_steam_plants_sched402,2021,1046
+out_ferc1__yearly_steam_plants_sched402,2022,1070
+out_ferc1__yearly_steam_plants_sched402,2023,1096
diff --git a/devtools/dbt_helper.py b/devtools/dbt_helper.py
@@ -132,6 +132,7 @@ def generate_row_counts(
         .size()
         .reset_index(name="row_count")
         .rename(columns={partition_column: "partition"})
+        .astype({"partition": "str"})
     )
     new_row_counts["table_name"] = table_name
 
@@ -187,6 +188,20 @@ def _log_add_table_result(result: AddTableResult):
         logger.error(result.message)
 
 
+def _infer_partition_column(table_name: str) -> str:
+    all_columns = [c.name for c in PUDL_PACKAGE.get_resource(table_name).schema.fields]
+    if (
+        (partition_column := "report_year") in all_columns
+        or (partition_column := "report_date") in all_columns
+        or (partition_column := "datetime_utc") in all_columns
+    ):
+        return partition_column
+    raise RuntimeError(
+        f"Could not determine partition column for table {table_name}. "
+        "You can pass this in manually with the '--partition-column' option."
+    )
+
+
 def add_table(
     table_name: str,
     partition_column: str = "report_year",
@@ -196,6 +211,9 @@ def add_table(
     """Scaffold dbt yaml for a single table."""
     data_source = get_data_source(table_name)
 
+    if partition_column == "inferred":
+        partition_column = _infer_partition_column(table_name)
+
     _log_add_table_result(
         write_table_yaml(
             table_name, data_source, partition_column=partition_column, clobber=clobber
@@ -224,9 +242,9 @@ def dbt_helper():
 )
 @click.option(
     "--partition-column",
-    default="report_year",
+    default="inferred",
     type=str,
-    help="Column used to generate row count per partition test.",
+    help="Column used to generate row count per partition test. If 'inferred' the script will attempt to infer a reasonable partitioning column.",
 )
 @click.option(
     "--use-nightly-tables",