Drop Python 3.8 and PyPy 3.9, add more CI tools, fix building docs

.bandit.yml

@@ -0,0 +1,3 @@
+skips:
+- B101
+exclude_dirs: ['tests']

.github/workflows/main.yml

@@ -9,7 +9,7 @@ jobs:
     - uses: actions/checkout@v4
     - uses: actions/setup-python@v5
       with:
-        python-version: "3.12"
+        python-version: "3.13"
     - uses: pre-commit/[email protected]
 
   tests:
@@ -21,18 +21,18 @@ jobs:
         - python-version: '3.12'  # Keep in sync with .readthedocs.yml
           env:
             TOXENV: docs
-        - python-version: '3.12'
+        - python-version: '3.13'
+          env:
+            TOXENV: pylint
+        - python-version: '3.13'
           env:
             TOXENV: twinecheck
-        - python-version: 3.8
+        - python-version: '3.13'
           env:
-            TOXENV: py
+            TOXENV: typing
         - python-version: 3.9
           env:
             TOXENV: py
-        - python-version: pypy-3.9
-          env:
-            TOXENV: py
         - python-version: pypy-3.10
           env:
             TOXENV: py
@@ -48,13 +48,13 @@ jobs:
         - python-version: '3.12'
           env:
             TOXENV: py
-        - python-version: '3.13.0-rc.1'
+        - python-version: '3.13'
           env:
             TOXENV: py
     steps:
       - uses: actions/checkout@v4
       - name: Install system libraries
-        if: contains(matrix.python-version, 'pypy') || contains(matrix.python-version, 'beta')
+        if: contains(matrix.python-version, 'pypy')
         run: |
           sudo apt-get update
           sudo apt-get install libxml2-dev libxslt-dev

.github/workflows/publish.yml

@@ -10,7 +10,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v5
       with:
-        python-version: 3.12
+        python-version: 3.13
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip

.pre-commit-config.yaml

@@ -1,11 +1,16 @@
 default_language_version:
-    python: python3.12
+    python: python3.13
 repos:
+  - hooks:
+      - id: bandit
+        args: [-r, -c, .bandit.yml]
+    repo: https://github.com/PyCQA/bandit
+    rev: 1.7.10
   - hooks:
       - id: black
         language_version: python3
     repo: https://github.com/ambv/black
-    rev: 24.8.0
+    rev: 24.10.0
   - hooks:
       - id: isort
         language_version: python3

docs/conf.py

@@ -14,8 +14,6 @@
 import sys
 from os import path
 
-import sphinx_rtd_theme
-
 # If your extensions are in another directory, add it here. If the directory
 # is relative to the documentation root, use os.path.abspath to make it
 # absolute, like shown here.
@@ -108,13 +106,6 @@
 # documentation.
 # html_theme_options = {}
 
-# Add any paths that contain custom themes here, relative to this directory.
-# Add path to the RTD explicitly to robustify builds (otherwise might
-# fail in a clean Debian build env)
-
-html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
-
-
 # The style sheet to use for HTML and HTML Help pages. A file of that name
 # must exist either in Sphinx' static/ path, or in one of the custom paths
 # given in html_static_path.

itemloaders/processors.py

@@ -159,6 +159,7 @@ def __call__(self, values: Any) -> Any:
         for value in values:
             if value is not None and value != "":
                 return value
+        return None
 
 
 class Identity:

pylintrc

@@ -0,0 +1,36 @@
+[MASTER]
+ignore=typing
+persistent=no
+
+[MESSAGES CONTROL]
+disable=broad-exception-caught,
+        c-extension-no-member,
+        consider-using-f-string,
+        disallowed-name,
+        duplicate-code,
+        fixme,
+        import-error,
+        import-outside-toplevel,
+        invalid-name,
+        line-too-long,
+        missing-class-docstring,
+        missing-function-docstring,
+        missing-module-docstring,
+        no-else-return,
+        no-member,
+        not-callable,
+        parse-error,
+        protected-access,
+        raise-missing-from,
+        redefined-builtin,
+        redefined-outer-name,
+        too-few-public-methods,
+        too-many-arguments,
+        too-many-lines,
+        too-many-positional-arguments,
+        too-many-public-methods,
+        unidiomatic-typecheck,
+        unused-argument,
+        use-a-generator,
+        wrong-import-order,
+        wrong-import-position,

setup.py

@@ -1,6 +1,6 @@
 from setuptools import find_packages, setup
 
-with open("README.rst") as f:
+with open("README.rst", encoding="utf-8") as f:
     long_description = f.read()
 
 setup(
@@ -30,7 +30,6 @@
         "Operating System :: OS Independent",
         "Programming Language :: Python",
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
@@ -39,7 +38,7 @@
         "Programming Language :: Python :: Implementation :: CPython",
         "Programming Language :: Python :: Implementation :: PyPy",
     ],
-    python_requires=">=3.8",
+    python_requires=">=3.9",
     install_requires=[
         # before updating these versions, be sure they are not higher than
         # scrapy's requirements

tests/test_base_loader.py

@@ -41,6 +41,7 @@ def validate_sku(value):
             # Let's assume a SKU is only digits.
             if value.isdigit():
                 return value
+            return None
 
         class MyLoader(ItemLoader):
             name_out = Compose(lambda vs: vs[0])  # take first which allows empty values
@@ -278,10 +279,11 @@ class CustomItemLoader(ItemLoader):
 
         il = CustomItemLoader()
         il.add_value("name", ["$10"])
+        expected_exc = None
         try:
             float("$10")
         except Exception as e:
-            expected_exc_str = str(e)
+            expected_exc = e
 
         exc = None
         try:
@@ -293,7 +295,7 @@ class CustomItemLoader(ItemLoader):
         assert "name" in s, s
         assert "$10" in s, s
         assert "ValueError" in s, s
-        assert expected_exc_str in s, s
+        assert str(expected_exc) in s, s
 
     def test_output_processor_using_classes(self):
         il = CustomItemLoader()

tests/test_output_processor.py

@@ -9,7 +9,7 @@ class TestOutputProcessorDict(unittest.TestCase):
     def test_output_processor(self):
         class TempDict(Dict[str, Any]):
             def __init__(self, *args, **kwargs):
-                super(TempDict, self).__init__(self, *args, **kwargs)
+                super().__init__(*args, **kwargs)
                 self.setdefault("temp", 0.3)
 
         class TempLoader(ItemLoader):

tests/test_select_jmes.py

@@ -18,8 +18,8 @@ class SelectJmesTestCase(unittest.TestCase):
     }
 
     def test_output(self):
-        for key in self.test_list_equals:
-            expr, test_list, expected = self.test_list_equals[key]
+        for key, value in self.test_list_equals.items():
+            expr, test_list, expected = value
             test = SelectJmes(expr)(test_list)
             self.assertEqual(
                 test, expected, msg=f"test {key!r} got {test} expected {expected}"

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py38,py39,py310,py311,py312
+envlist = py39,py310,py311,py312,py313
 
 [testenv]
 deps =
@@ -61,3 +61,10 @@ deps =
     pre-commit
 commands =
     pre-commit run {posargs:--all-files}
+
+[testenv:pylint]
+deps =
+    {[testenv]deps}
+    pylint==3.3.1
+commands =
+    pylint docs itemloaders tests setup.py