Merge branch 'dev' into eia176_start_implementation

docs/conf.py

@@ -80,15 +80,19 @@
 # we need to define these package to URL mappings:
 intersphinx_mapping = {
     "arrow": ("https://arrow.apache.org/docs/", None),
+    "dagster": ("https://docs.dagster.io/", None),
     "dask": ("https://docs.dask.org/en/latest/", None),
+    "datasette": ("https://docs.datasette.io/en/stable/", None),
     "geopandas": ("https://geopandas.org/en/stable/", None),
+    "hypothesis": ("https://hypothesis.readthedocs.io/en/latest/", None),
     "networkx": ("https://networkx.org/documentation/stable/", None),
     "numpy": ("https://numpy.org/doc/stable/", None),
     "pandas": ("https://pandas.pydata.org/pandas-docs/stable", None),
+    "pandera": ("https://pandera.readthedocs.io/en/stable/", None),
+    "pydantic": ("https://docs.pydantic.dev/latest/", None),
     "pytest": ("https://docs.pytest.org/en/latest/", None),
     "python": ("https://docs.python.org/3", None),
     "scipy": ("https://docs.scipy.org/doc/scipy/", None),
-    "setuptools": ("https://setuptools.pypa.io/en/latest/", None),
     "sklearn": ("https://scikit-learn.org/stable", None),
     "sqlalchemy": ("https://docs.sqlalchemy.org/en/latest/", None),
 }

docs/dev/dev_setup.rst

@@ -65,6 +65,14 @@ from your fork to your local computer where you'll be editing the code or docs.
 will download the whole history of the project, including the most recent version, and
 put it in a local directory where you can make changes.
 
+Note that we use a special merge method for our environment lockfiles, which you need
+to explicitly enable locally in your git configuration for the PUDL repository with this
+command. You only need to run it once, from within the cloned repo:
+
+.. code-block:: console
+
+    $ git config --local merge.ours.driver true
+
 -------------------------------------------------------------------------------
 Create the PUDL Dev Environment
 -------------------------------------------------------------------------------
@@ -79,40 +87,24 @@ listed in the project's ``pyproject.toml`` file.  The conda lockfile is updated
 automatically by a GitHub Action workflow that runs once a week, or any time
 ``pyproject.toml`` is changed.
 
-Use ``conda-lock`` to create a new ``pudl-dev`` environment and activate it:
-
-.. code-block:: console
-
-    $ mamba activate base
-    $ conda-lock install --name pudl-dev --mamba --dev environments/conda-lock.yml
-    $ mamba deactivate
-    $ mamba activate pudl-dev
-
-Now we need to install the PUDL package defined by the repository into the ``pudl-dev``
-conda environment. We use ``pip`` since this is just a local package, and since all of
-the required packages should have already been satisfied by the locked conda
-environment, we tell ``pip`` not to try and install any dependencies.
-
-.. code-block:: console
-
-    $ pip install --no-cache-dir --no-deps --editable .
-
--------------------------------------------------------------------------------
-Automating tasks with Make
--------------------------------------------------------------------------------
-We use the GNU build tool ``make`` to remember and automate some repetitive tasks in the
+We use a ``Makefile`` to remember and automate some common shared tasks in the
 PUDL repository, including creating and updating the ``pudl-dev`` conda environment. If
-you are on a Unix-based platform (Linux or MacOS) it should already be installed.
-Typically, rather than running the commands in the section above, you'll want to use
-the predefined ``make`` commands. If you'd like to learn more about how Makefiles work,
-check out `this excellent Makefile tutorial <https://makefiletutorial.com/>`__
+you are on a Unix-based platform (Linux or MacOS) the ``make`` command should already be
+installed. You'll typically want to use the predefined ``make`` commands rather than
+running the individual commands they wrap. If you'd like to learn more about how
+Makefiles work, check out `this excellent Makefile tutorial
+<https://makefiletutorial.com/>`__
 
-To create the ``pudl-dev`` environment and install the local PUDL package, run:
+To create the ``pudl-dev`` environment and install the local PUDL package using
+``make``, run:
 
 .. code-block:: console
 
     $ make install-pudl
 
+If you want to see all the bundled commands we've defined, open up the ``Makefile``.
+There's also some additional information in the :doc:`testing` documentation.
+
 -------------------------------------------------------------------------------
 Updating the PUDL Development Environment
 -------------------------------------------------------------------------------
@@ -130,9 +122,8 @@ creating it the first time:
     $ make install-pudl
 
 If you happen to be changing the dependencies listed in ``pyproject.toml`` and you want
-to re-create the conda lockfile from scratch, resolving for any newly updated
-dependencies, and then create a fresh ``pudl-dev`` environment using the new lockfile,
-you can use:
+to re-create the conda lockfile from scratch to include any newly defined dependencies,
+and then create a fresh ``pudl-dev`` environment using the new lockfile, you can do:
 
 .. code-block:: console
 
@@ -202,34 +193,30 @@ The scripts that run are configured in the ``.pre-commit-config.yaml`` file.
     * `Real Python Code Quality Tools and Best Practices <https://realpython.com/python-code-quality/>`__
       gives a good overview of available linters and static code analysis tools.
 
-Code and Docs Linters
-^^^^^^^^^^^^^^^^^^^^^
+Linting and Formatting
+^^^^^^^^^^^^^^^^^^^^^^
 
 * `ruff <https://docs.astral.sh/ruff/>`__ is a popular, fast Python
   `linting <https://en.wikipedia.org/wiki/Lint_(software)>`__ and autofix framework,
   with a large selection of rules that can be configured (often mirroring plugins
   originally developed for ``flake8``). We use it to check the formatting and syntax of
   the code and to ensure that we're all using modern python syntax, type hinting, etc.
+* We also use `ruff to format our code <https://docs.astral.sh/ruff/formatter/>`__. It
+  serves as a much faster drop-in replacement for longtime crowd favorite `black
+  <https://black.readthedocs.io/en/stable/>`__
 * `doc8 <https://github.com/PyCQA/doc8>`__ , lints our documentation files, which are
   written in the reStructuredText format and built by `Sphinx
   <https://www.sphinx-doc.org/en/master/>`__. This is the de-facto standard for Python
   documentation. The ``doc8`` tool checks for syntax errors and other formatting issues
   in the documentation source files under the ``docs/`` directory.
 
-Automatic Formatting
-^^^^^^^^^^^^^^^^^^^^
-We are using the the `black <https://black.readthedocs.io/en/stable/>`__ code formatter
-and style. It's automatically applied by oure pre-commit hooks, and can probably be
-integrated directly into your code editor.
-
 Linting Within Your Editor
 ^^^^^^^^^^^^^^^^^^^^^^^^^^
 If you are using an editor designed for Python development many of these code linting
 and formatting tools can be run automatically in the background while you write code or
 documentation. Popular editors that work with the above tools include:
 
-* `Visual Studio Code <https://code.visualstudio.com/>`__, from Microsoft (free, but
-   controlled by the hegemon).
+* `Visual Studio Code <https://code.visualstudio.com/>`__, from Microsoft (free, but...)
 * `NeoVim <https://neovim.io/>`__, (free and open source; for diehard Unix lovers)
 * `PyCharm <https://www.jetbrains.com/pycharm/>`__ (paid).
 * `Sublime Text <https://www.sublimetext.com/>`__ (paid).

environments/conda-linux-64.lock.yml

@@ -241,7 +241,7 @@ dependencies:
   - pure_eval=0.2.2=pyhd8ed1ab_0
   - pyasn1=0.5.0=pyhd8ed1ab_0
   - pycparser=2.21=pyhd8ed1ab_0
-  - pygments=2.17.1=pyhd8ed1ab_0
+  - pygments=2.17.2=pyhd8ed1ab_0
   - pyjwt=2.8.0=pyhd8ed1ab_0
   - pylev=1.4.0=pyhd8ed1ab_0
   - pyparsing=3.1.1=pyhd8ed1ab_0
@@ -294,7 +294,7 @@ dependencies:
   - uri-template=1.3.0=pyhd8ed1ab_0
   - uvloop=0.19.0=py311h460e60f_0
   - validators=0.22.0=pyhd8ed1ab_0
-  - wcwidth=0.2.11=pyhd8ed1ab_0
+  - wcwidth=0.2.12=pyhd8ed1ab_0
   - webcolors=1.13=pyhd8ed1ab_0
   - webencodings=0.5.1=pyhd8ed1ab_2
   - websocket-client=1.6.4=pyhd8ed1ab_0
@@ -315,7 +315,7 @@ dependencies:
   - asttokens=2.4.1=pyhd8ed1ab_0
   - async-lru=2.0.4=pyhd8ed1ab_0
   - aws-c-auth=0.7.7=h37ad1db_0
-  - aws-c-mqtt=0.9.9=h1387108_0
+  - aws-c-mqtt=0.9.10=h1387108_0
   - babel=2.13.1=pyhd8ed1ab_0
   - beautifulsoup4=4.12.2=pyha770c72_0
   - bleach=6.1.0=pyhd8ed1ab_0
@@ -398,8 +398,8 @@ dependencies:
   - argon2-cffi-bindings=21.2.0=py311h459d7ec_4
   - arrow=1.3.0=pyhd8ed1ab_0
   - async-timeout=4.0.3=pyhd8ed1ab_0
-  - aws-c-s3=0.3.24=hdb3bed3_1
-  - botocore=1.32.4=pyhd8ed1ab_0
+  - aws-c-s3=0.4.0=hdb3bed3_0
+  - botocore=1.32.5=pyhd8ed1ab_0
   - branca=0.7.0=pyhd8ed1ab_1
   - croniter=2.0.1=pyhd8ed1ab_0
   - cryptography=41.0.5=py311h63ff55d_0
@@ -433,7 +433,7 @@ dependencies:
   - pytest-console-scripts=1.4.1=pyhd8ed1ab_0
   - pytest-cov=4.1.0=pyhd8ed1ab_0
   - pytest-mock=3.12.0=pyhd8ed1ab_0
-  - pytest-xdist=3.4.0=pyhd8ed1ab_0
+  - pytest-xdist=3.5.0=pyhd8ed1ab_0
   - python-build=1.0.3=pyhd8ed1ab_0
   - requests=2.31.0=pyhd8ed1ab_0
   - rich=13.7.0=pyhd8ed1ab_0
@@ -448,7 +448,7 @@ dependencies:
   - alembic=1.12.1=pyhd8ed1ab_0
   - arelle-release=2.17.4=pyhd8ed1ab_0
   - argon2-cffi=23.1.0=pyhd8ed1ab_0
-  - aws-crt-cpp=0.24.7=hd0f6be0_2
+  - aws-crt-cpp=0.24.7=h08a389d_4
   - bottleneck=1.3.7=py311h1f0f07a_1
   - cachecontrol=0.13.1=pyhd8ed1ab_0
   - contourpy=1.2.0=py311h9547e67_0
@@ -487,7 +487,7 @@ dependencies:
   - uvicorn-standard=0.24.0=h38be061_0
   - virtualenv=20.24.6=pyhd8ed1ab_0
   - aws-sdk-cpp=1.11.182=h8beafcf_7
-  - boto3=1.29.4=pyhd8ed1ab_0
+  - boto3=1.29.5=pyhd8ed1ab_0
   - cachecontrol-with-filecache=0.13.1=pyhd8ed1ab_0
   - dagster=1.5.9=pyhd8ed1ab_0
   - datasette=0.64.4=pyhd8ed1ab_1