Refactor PUDL to use Pydantic v2

.github/workflows/pytest.yml

@@ -43,6 +43,9 @@ jobs:
           conda config --show
           printenv | sort
 
+      - name: Make input, output and dagster dirs
+        run: mkdir -p ${{ env.PUDL_OUTPUT }} ${{ env.PUDL_INPUT}} ${{ env.DAGSTER_HOME }}
+
       - name: Lint and build PUDL documentation with Sphinx
         run: |
           pip install --no-deps --editable .

.gitignore

@@ -8,6 +8,8 @@
 docs/data_dictionaries/pudl_db.rst
 .ipynb_checkpoints/
 .cache/
+.ruff_cache/
+.mypy_cache/
 .pytest_cache/*
 .DS_Store
 build/

devtools/materialize_asset.py

@@ -38,7 +38,7 @@ def main(asset_id):
                 config={
                     "resources": {
                         "dataset_settings": {
-                            "config": etl_fast_settings.dict(),
+                            "config": etl_fast_settings.model_dump(),
                         },
                     },
                 },

docs/conf.py

@@ -10,6 +10,7 @@
 
 import datetime
 import importlib.metadata
+import os
 import shutil
 from pathlib import Path
 
@@ -18,6 +19,11 @@
 from pudl.metadata.resources import RESOURCE_METADATA
 
 DOCS_DIR = Path(__file__).parent.resolve()
+if os.environ.get("READTHEDOCS"):
+    pudl_input = Path(os.environ["PUDL_INPUT"])
+    pudl_input.mkdir(parents=True, exist_ok=True)
+    pudl_output = Path(os.environ["PUDL_OUTPUT"])
+    pudl_output.mkdir(parents=True, exist_ok=True)
 
 # -- Path setup --------------------------------------------------------------
 # We are building and installing the pudl package in order to get access to

environments/conda-linux-64.lock.yml

@@ -1,6 +1,6 @@
 # Generated by conda-lock.
 # platform: linux-64
-# input_hash: 88ab6b06102bd362f7a93aacfa6fd9a41a0824c3982e85120db6a645c2c00351
+# input_hash: b9f8826532896e72ca49913a87e8be150e02e08901365edadcf42074ef6dec9c
 
 channels:
   - conda-forge
@@ -63,7 +63,7 @@ dependencies:
   - lzo=2.10=h516909a_1000
   - ncurses=6.4=h59595ed_2
   - nspr=4.35=h27087fc_0
-  - openssl=3.1.4=hd590300_0
+  - openssl=3.2.0=hd590300_0
   - pixman=0.42.2=h59595ed_0
   - pthread-stubs=0.4=h36c2ea0_1001
   - rdma-core=28.9=h59595ed_1
@@ -98,7 +98,7 @@ dependencies:
   - libprotobuf=4.24.4=hf27288f_0
   - libre2-11=2023.06.02=h7a70373_0
   - librttopo=1.1.0=h8917695_15
-  - libsqlite=3.44.1=h2797004_0
+  - libsqlite=3.44.2=h2797004_0
   - libssh2=1.11.0=h0841786_0
   - libxcb=1.15=h0b41bf4_0
   - libxml2=2.11.6=h232c23b_0
@@ -131,7 +131,7 @@ dependencies:
   - pandoc=3.1.3=h32600fe_0
   - python=3.11.6=hab00c5b_0_cpython
   - re2=2023.06.02=h2873b5e_0
-  - sqlite=3.44.1=h2c6b66d_0
+  - sqlite=3.44.2=h2c6b66d_0
   - xorg-libx11=1.8.7=h8ee46fc_0
   - aiofiles=23.2.1=pyhd8ed1ab_0
   - alabaster=0.7.13=pyhd8ed1ab_0
@@ -189,7 +189,7 @@ dependencies:
   - humanfriendly=10.0=pyhd8ed1ab_6
   - hupper=1.12=pyhd8ed1ab_0
   - hyperframe=6.0.1=pyhd8ed1ab_0
-  - idna=3.4=pyhd8ed1ab_0
+  - idna=3.5=pyhd8ed1ab_0
   - ijson=3.2.3=pyhd8ed1ab_0
   - imagesize=1.4.1=pyhd8ed1ab_0
   - iniconfig=2.0.0=pyhd8ed1ab_0
@@ -396,6 +396,7 @@ dependencies:
   - yarl=1.9.2=py311h459d7ec_1
   - addfips=0.4.0=pyhd8ed1ab_1
   - aniso8601=9.0.1=pyhd8ed1ab_0
+  - annotated-types=0.6.0=pyhd8ed1ab_0
   - argon2-cffi-bindings=21.2.0=py311h459d7ec_4
   - arrow=1.3.0=pyhd8ed1ab_0
   - async-timeout=4.0.3=pyhd8ed1ab_0
@@ -422,14 +423,14 @@ dependencies:
   - libnetcdf=4.9.2=nompi_h80fb2b6_112
   - libspatialite=5.1.0=h7385560_2
   - mako=1.3.0=pyhd8ed1ab_0
-  - numpy=1.26.0=py311h64a7726_0
+  - numpy=1.26.2=py311h64a7726_0
   - pbr=6.0.0=pyhd8ed1ab_0
   - pendulum=2.1.2=py311h459d7ec_6
   - poppler=23.11.0=h590f24d_0
   - prompt_toolkit=3.0.41=hd8ed1ab_0
   - psycopg2-binary=2.9.7=pyhd8ed1ab_1
   - pybtex=0.24.0=pyhd8ed1ab_2
-  - pydantic=1.10.13=py311h459d7ec_1
+  - pydantic-core=2.14.5=py311h46250e7_0
   - pyproj=3.6.1=py311h1facc83_4
   - pytest-console-scripts=1.4.1=pyhd8ed1ab_0
   - pytest-cov=4.1.0=pyhd8ed1ab_0
@@ -475,6 +476,7 @@ dependencies:
   - pandas=2.1.3=py311h320fe9a_0
   - pango=1.50.14=ha41ecd1_2
   - pybtex-docutils=1.0.3=py311h38be061_1
+  - pydantic=2.5.2=pyhd8ed1ab_0
   - pyopenssl=23.3.0=pyhd8ed1ab_0
   - readthedocs-sphinx-ext=2.2.3=pyhd8ed1ab_0
   - requests-toolbelt=0.10.1=pyhd8ed1ab_0
@@ -509,6 +511,7 @@ dependencies:
   - nbformat=5.9.2=pyhd8ed1ab_0
   - pandera-core=0.17.2=pyhd8ed1ab_1
   - pre-commit=3.5.0=pyha770c72_0
+  - pydantic-settings=2.1.0=pyhd8ed1ab_1
   - requests-oauthlib=1.3.1=pyhd8ed1ab_0
   - scikit-learn=1.3.2=py311hc009520_1
   - timezonefinder=6.2.0=py311h459d7ec_2