Got package to work as wanted using MANIFEST.in

.gitmodules

@@ -1,3 +1,3 @@
 [submodule "adslib"]
-	path = src/pyads/adslib
+	path = adslib
 	url = https://github.com/stlehmann/ADS.git

MANIFEST.in

@@ -0,0 +1,9 @@
+#
+# Recipe for extra files to pack for setuptools.
+# MANIFEST.in is a little outdated but the best solution to differentiate files between sdist and wheel builds.
+#
+
+graft adslib/
+global-exclude *.a *.o *.obj *.bin *.so
+prune obj/
+prune tests/

pyproject.toml

@@ -44,23 +44,21 @@ Repository = "https://github.com/stlehmann/pyads"
 Documentation = "https://pyads.readthedocs.io"
 
 [build-system]
-requires = ["setuptools >= 61.0", "setuptools-scm", "wheel"]
+requires = ["setuptools >= 61.0", "wheel"]
 build-backend = "setuptools.build_meta"
 
 [tool.setuptools]
 packages = ["pyads", "pyads.testserver"]
 package-dir = { "" = "src" }
 include-package-data = true
+# ^ needed for MANIFEST.in
 
-[tool.setuptools.package-data]
-pyads = ["adslib/**"]
-# The adslib source is not always needed, it is handled in `setup.py` instead
+#[tool.setuptools.package-data]
+# Package data (adslib/) is handled by MANIFEST.in instead
 
 [tool.setuptools.exclude-package-data]
-pyads = [
-    "*.a", "*.o", "obj/*", "*.bin", "*.so",
-]
-# ^ But always exclude build files from adslib source
+pyads = ["adslib/**"]
+# ^ Odd trick, put this excludes the adslib source again from a wheel build and from a pip install
 
 [tool.setuptools.dynamic]
 version = {attr = "pyads.__version__"}

setup.py

@@ -1,17 +1,15 @@
 from pathlib import Path
 from setuptools import setup
 from setuptools.command.install import install
-from setuptools.command.build_py import build_py
 from setuptools.command.build import build
 from wheel.bdist_wheel import bdist_wheel
 import sys
 import sysconfig
-import shutil
 import os
 import subprocess
 
 
-adslib_relative = "src/pyads/adslib"
+adslib_relative = "adslib"
 adslib_root = Path(__file__).parent.absolute() / adslib_relative
 
 
@@ -88,25 +86,13 @@ def get_tag(self):
         return impl_tag, abi_tag, plat_tag
 
 
-class CustomBuildPy(build_py):
-    """Skip adslib during source selection for wheel build."""
-
-    def run(self):
-        super().run()
-
-        # If this is run as part of a wheel build, simply remove "adslib/" from
-        # the build folder
-        if isinstance(self.distribution.get_command_obj('bdist_wheel'), bdist_wheel):
-            adslib_dir = Path(self.build_lib) / "pyads" / "adslib"
-            shutil.rmtree(adslib_dir)
-
-
 # noinspection PyTypeChecker
 setup(
     cmdclass={
         "build": CustomBuild,
-        "build_py": CustomBuildPy,
         "install": CustomInstall,
         "bdist_wheel": CustomBDistWheel,
     },
 )
+
+# Also see `MANIFEST.in`