Merge remote-tracking branch 'origin' into tomnewton/upsream_output_e…

…ntity_hints Signed-off-by: Eduardo Apolinario <[email protected]>
Tom-Newton · Jan 30, 2024 · 5901cff · 5901cff
2 parents 6ae0e89 + 8861d0a
commit 5901cff
Show file tree

Hide file tree

Showing 124 changed files with 2,333 additions and 1,318 deletions.
diff --git a/.github/workflows/pythonbuild.yml b/.github/workflows/pythonbuild.yml
@@ -20,7 +20,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        python-version: ["3.8", "3.11"]
+        python-version: ["3.8", "3.11", "3.12"]
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}
@@ -39,6 +39,14 @@ jobs:
           make setup
           pip uninstall -y pandas
           pip freeze
+      - name: Run extras unit tests with coverage
+        # Skip this step if running on python 3.12 due to https://github.com/tensorflow/tensorflow/issues/62003
+        # and https://github.com/pytorch/pytorch/issues/110436
+        if: ${{ matrix.python-version != '3.12' }}
+        env:
+          PYTEST_OPTS: -n2
+        run: |
+          make unit_test_extras_codecov
       - name: Test with coverage
         env:
           PYTEST_OPTS: -n2
@@ -93,7 +101,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest]
-        python-version: ["3.8", "3.11"]
+        python-version: ["3.8", "3.11", "3.12"]
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}
@@ -128,10 +136,14 @@ jobs:
         os: [ubuntu-latest]
         # python 3.11 has intermittent issues with the docker build + push step
         # https://github.com/flyteorg/flytekit/actions/runs/5800978835/job/15724237979?pr=1579
-        python-version: ["3.8", "3.11"]
+        python-version: ["3.8", "3.11", "3.12"]
     steps:
       - uses: insightsengineering/disk-space-reclaimer@v1
+      # As described in https://github.com/pypa/setuptools_scm/issues/414, SCM needs git history
+      # and tags to work.
       - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v4
         with:
@@ -174,6 +186,7 @@ jobs:
         plugin-names:
           # Please maintain an alphabetical order in the following list
           - flytekit-airflow
+          - flytekit-async-fsspec
           - flytekit-aws-athena
           - flytekit-aws-batch
           # TODO: uncomment this when the sagemaker agent is implemented: https://github.com/flyteorg/flyte/issues/4079

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -22,3 +22,9 @@ repos:
     rev: 0.0.9
     hooks:
       - id: check_pdb_hook
+  - repo: https://github.com/codespell-project/codespell
+    rev: v2.2.6
+    hooks:
+      - id: codespell
+        additional_dependencies:
+          - tomli
diff --git a/Dockerfile b/Dockerfile
@@ -1,5 +1,5 @@
 ARG PYTHON_VERSION
-FROM python:${PYTHON_VERSION}-slim-buster
+FROM python:${PYTHON_VERSION}-slim-bookworm
 
 MAINTAINER Flyte Team <[email protected]>
 LABEL org.opencontainers.image.source=https://github.com/flyteorg/flytekit

diff --git a/Dockerfile.agent b/Dockerfile.agent
@@ -1,4 +1,4 @@
-FROM python:3.9-slim-buster
+FROM python:3.9-slim-bookworm
 
 MAINTAINER Flyte Team <[email protected]>
 LABEL org.opencontainers.image.source=https://github.com/flyteorg/flytekit

diff --git a/Dockerfile.dev b/Dockerfile.dev
@@ -6,7 +6,7 @@
 # $ pyflyte run --image localhost:30000/flytekittest:someversion
 
 ARG PYTHON_VERSION
-FROM python:${PYTHON_VERSION}-slim-buster
+FROM python:${PYTHON_VERSION}-slim-bookworm
 
 MAINTAINER Flyte Team <[email protected]>
 LABEL org.opencontainers.image.source https://github.com/flyteorg/flytekit
@@ -27,11 +27,13 @@ COPY . /flytekit
 # 3. Clean up the apt cache to reduce image size. Reference: https://gist.github.com/marvell/7c812736565928e602c4
 # 4. Create a non-root user 'flytekit' and set appropriate permissions for directories.
 RUN apt-get update && apt-get install build-essential vim libmagic1 git -y \
-    && pip install --no-cache-dir -e /flytekit \
-    && pip install --no-cache-dir -e /flytekit/plugins/flytekit-k8s-pod \
-    && pip install --no-cache-dir -e /flytekit/plugins/flytekit-deck-standard \
-    && pip install --no-cache-dir -e /flytekit/plugins/flytekit-flyin \
-    && pip install --no-cache-dir scikit-learn \
+    && pip install --no-cache-dir -U --pre \
+        flyteidl \
+        -e /flytekit \
+        -e /flytekit/plugins/flytekit-k8s-pod \
+        -e /flytekit/plugins/flytekit-deck-standard \
+        -e /flytekit/plugins/flytekit-flyin \
+        scikit-learn \
     && apt-get clean autoclean \
     && apt-get autoremove --yes \
     && rm -rf /var/lib/{apt,dpkg,cache,log}/ \

diff --git a/Makefile b/Makefile
@@ -24,6 +24,7 @@ update_boilerplate:
 
 .PHONY: setup
 setup: install-piptools ## Install requirements
+	pip install flyteidl --pre
 	pip install -r dev-requirements.in
 
 .PHONY: fmt
@@ -43,7 +44,8 @@ lint: ## Run linters
 
 .PHONY: spellcheck
 spellcheck:  ## Runs a spellchecker over all code and documentation
-	codespell -L "te,raison,fo" --skip="./docs/build,./.git"
+	# Configuration is in pyproject.toml
+	codespell
 
 .PHONY: test
 test: lint unit_test
@@ -52,12 +54,19 @@ test: lint unit_test
 unit_test_codecov:
 	$(MAKE) CODECOV_OPTS="--cov=./ --cov-report=xml --cov-append" unit_test
 
+.PHONY: unit_test_extras_codecov
+unit_test_extras_codecov:
+	$(MAKE) CODECOV_OPTS="--cov=./ --cov-report=xml --cov-append" unit_test_extras
+
 .PHONY: unit_test
 unit_test:
-	# Skip tensorflow tests and run them with the necessary env var set so that a working (albeit slower)
+	# Skip all extra tests and run them with the necessary env var set so that a working (albeit slower)
 	# library is used to serialize/deserialize protobufs is used.
-	$(PYTEST) -m "not sandbox_test" tests/flytekit/unit/ --ignore=tests/flytekit/unit/extras/tensorflow --ignore=tests/flytekit/unit/models ${CODECOV_OPTS} && \
-		PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python $(PYTEST) tests/flytekit/unit/extras/tensorflow ${CODECOV_OPTS}
+	$(PYTEST) -m "not sandbox_test" tests/flytekit/unit/ --ignore=tests/flytekit/unit/extras/ --ignore=tests/flytekit/unit/models ${CODECOV_OPTS}
+
+.PHONY: unit_test_extras
+unit_test_extras:
+	PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python $(PYTEST) tests/flytekit/unit/extras ${CODECOV_OPTS}
 
 .PHONY: test_serialization_codecov
 test_serialization_codecov:

diff --git a/dev-requirements.in b/dev-requirements.in
@@ -19,16 +19,14 @@ IPython
 keyrings.alt
 setuptools_scm
 
-# Only install tensorflow if not running on an arm Mac.
-tensorflow==2.8.1; python_version<'3.11' and (platform_machine!='arm64' or platform_system!='Darwin')
-# Tensorflow release candidate supports python 3.11
-tensorflow==2.13.0; python_version>='3.11' and (platform_machine!='arm64' or platform_system!='Darwin')
-
+# Tensorflow is not available for python 3.12 yet: https://github.com/tensorflow/tensorflow/issues/62003
+tensorflow; python_version<'3.12'
 # Newer versions of torch bring in nvidia dependencies that are not present in windows, so
 # we put this constraint while we do not have per-environment requirements files
 torch<=1.12.1; python_version<'3.11'
 # pytorch 2 supports python 3.11
-torch<=2.0.0; python_version>='3.11' or platform_system!='Windows'
+# pytorch 2 does not support 3.12 yet: https://github.com/pytorch/pytorch/issues/110436
+torch; python_version<'3.12'
 
 # TODO: Currently, the python-magic library causes build errors on Windows due to its dependency on DLLs for libmagic.
 # We have temporarily disabled this feature on Windows and are using python-magic for Mac OS and Linux instead.
@@ -47,3 +45,5 @@ pandas
 scikit-learn
 types-requests
 prometheus-client
+
+orjson
diff --git a/dev-requirements.txt b/dev-requirements.txt
@@ -10,11 +10,11 @@ absl-py==2.0.0
     # via
     #   tensorboard
     #   tensorflow
-adlfs==2023.10.0
+adlfs==2023.9.0
     # via flytekit
 aiobotocore==2.5.4
     # via s3fs
-aiohttp==3.8.6
+aiohttp==3.9.2
     # via
     #   adlfs
     #   aiobotocore
@@ -24,8 +24,6 @@ aioitertools==0.11.0
     # via aiobotocore
 aiosignal==1.3.1
     # via aiohttp
-appnope==0.1.3
-    # via ipython
 arrow==1.3.0
     # via cookiecutter
 asttokens==2.4.1
@@ -71,9 +69,7 @@ cfgv==3.4.0
 chardet==5.2.0
     # via binaryornot
 charset-normalizer==3.3.2
-    # via
-    #   aiohttp
-    #   requests
+    # via requests
 click==8.1.7
     # via
     #   cookiecutter
@@ -97,6 +93,7 @@ cryptography==41.0.5
     #   azure-storage-blob
     #   msal
     #   pyjwt
+    #   secretstorage
 dataclasses-json==0.5.9
     # via flytekit
 decorator==5.1.1
@@ -224,7 +221,11 @@ jaraco-classes==3.3.0
     #   keyrings-alt
 jedi==0.19.1
     # via ipython
-jinja2==3.1.2
+jeepney==0.8.0
+    # via
+    #   keyring
+    #   secretstorage
+jinja2==3.1.3
     # via cookiecutter
 jmespath==1.0.1
     # via botocore
@@ -297,7 +298,7 @@ nodeenv==1.8.0
     # via pre-commit
 numpy==1.23.5
     # via
-    #   flytekit
+    #   -r dev-requirements.in
     #   h5py
     #   keras-preprocessing
     #   opt-einsum
@@ -313,6 +314,8 @@ oauthlib==3.2.2
     #   requests-oauthlib
 opt-einsum==3.3.0
     # via tensorflow
+orjson==3.9.12
+    # via -r dev-requirements.in
 packaging==23.2
     # via
     #   docker
@@ -321,12 +324,12 @@ packaging==23.2
     #   pytest
     #   setuptools-scm
 pandas==1.5.3
-    # via flytekit
+    # via -r dev-requirements.in
 parso==0.8.3
     # via jedi
 pexpect==4.8.0
     # via ipython
-pillow==10.1.0
+pillow==10.2.0
     # via -r dev-requirements.in
 platformdirs==3.11.0
     # via virtualenv
@@ -411,6 +414,8 @@ python-dateutil==2.8.2
     #   pandas
 python-json-logger==2.0.7
     # via flytekit
+python-magic==0.4.27 ; platform_system == "Darwin" or platform_system == "Linux"
+    # via -r dev-requirements.in
 python-slugify==8.0.1
     # via cookiecutter
 pytimeparse==1.1.8
@@ -459,6 +464,8 @@ scikit-learn==1.3.2
     # via -r dev-requirements.in
 scipy==1.11.3
     # via scikit-learn
+secretstorage==3.3.3
+    # via keyring
 setuptools-scm==8.0.4
     # via -r dev-requirements.in
 six==1.16.0
@@ -473,8 +480,7 @@ six==1.16.0
     #   python-dateutil
     #   tensorflow
 sortedcontainers==2.4.0
-    # via
-    #   hypothesis
+    # via hypothesis
 stack-data==0.6.3
     # via ipython
 statsd==3.3.0
@@ -485,7 +491,7 @@ tensorboard-data-server==0.6.1
     # via tensorboard
 tensorboard-plugin-wit==1.8.1
     # via tensorboard
-tensorflow==2.8.1 ; python_version < "3.11" and (platform_machine != "arm64" or platform_system != "Darwin")
+tensorflow==2.8.1 ; python_version < "3.12"
     # via -r dev-requirements.in
 tensorflow-estimator==2.8.0
     # via tensorflow
@@ -503,8 +509,8 @@ tomli==2.0.1
     #   coverage
     #   mypy
     #   pytest
-    # . setuptools-scm
-torch==1.12.1 ; python_version >= "3.11" or platform_system != "Windows"
+    #   setuptools-scm
+torch==1.12.1 ; python_version < "3.12"
     # via -r dev-requirements.in
 traitlets==5.13.0
     # via
@@ -530,7 +536,7 @@ typing-extensions==4.8.0
     #   mashumaro
     #   mypy
     #   rich-click
-    # . setuptools-scm
+    #   setuptools-scm
     #   tensorflow
     #   torch
     #   typing-inspect

diff --git a/doc-requirements.in b/doc-requirements.in
@@ -11,6 +11,7 @@ sphinx-autoapi
 sphinx-copybutton
 sphinx_fontawesome
 sphinx-panels
+sphinx-reredirects
 sphinxcontrib-youtube==1.2.0
 cryptography
 google-api-core[grpc]
@@ -19,37 +20,8 @@ sphinx-tags
 sphinx-click
 retry
 mashumaro
+pydantic<2.0.0
 
 # Packages for Plugin docs
 # Package name              Plugin needing it
-botocore                    # fsspec
-fsspec                      # fsspec
-google-cloud                # bigquery
-google-cloud-bigquery       # bigquery
-markdown                    # deck
-plotly                      # deck
-pandas_profiling            # deck
-dolt_integrations           # dolt
-great-expectations          # greatexpectations
-datasets                    # huggingface
-kubernetes                  # k8s-pod
-modin                       # modin
-pandera                     # pandera
-papermill                   # papermill
-jupyter                     # papermill
-polars                      # polars
-pyspark                     # spark
-sqlalchemy                  # sqlalchemy
-torch                       # pytorch
-skl2onnx                    # onnxscikitlearn
-tf2onnx                     # onnxtensorflow
-tensorflow                  # onnxtensorflow
-whylogs==1.3.3              # whylogs
-whylabs-client              # whylogs
-ray==2.6.3                  # ray
-scikit-learn                # scikit-learn
-dask[distributed]           # dask
 vaex                        # vaex
-mlflow==2.7.0               # mlflow
-duckdb                      # duckdb
-snowflake-connector-python  # snowflake