Test MV3 model in CI wheel builds (#9192)

### Summary

As part of #9019, we want to
ensure we can export models on P0 hosts targeting P0 backends. So, let's
test this in our wheel creation. For now, we introduce macOS/Linux
XNNPACK export.

Given the wheel building GitHub workflow doesn't support using jobs as a
test step, we must go use a python script. This runs a similar test to
what we do in PRs:


https://github.com/pytorch/executorch/blob/1011fdc98bcd61063b845670fb1779635aa30e3a/.github/workflows/pull.yml#L85-L136

### Test plan

CI → Build {Linux, macOS} Wheels

cc @larryliu0820 @lucylq

Author: jathu

.ci/scripts/wheel/__init__.py

@@ -0,0 +1,11 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+# This file is sourced into the environment before building a pip wheel. It
+# should typically only contain shell variable assignments. Be sure to export
+# any variables so that subprocesses will see them.
+
+source "${GITHUB_WORKSPACE}/${REPOSITORY}/.ci/scripts/wheel/envvar_base.sh"

build/packaging/env_var_script_linux.sh .ci/scripts/wheel/envvar_base.sh

@@ -8,16 +8,7 @@
 # should typically only contain shell variable assignments. Be sure to export
 # any variables so that subprocesses will see them.
 
-# Enable pybindings so that users can execute ExecuTorch programs from python.
-export EXECUTORCH_BUILD_PYBIND=1
-
-# Ensure that CMAKE_ARGS is defined before referencing it. Defaults to empty
-# if not defined.
-export CMAKE_ARGS="${CMAKE_ARGS:-}"
-
-# Link the XNNPACK backend into the pybindings runtime so that users can execute
-# ExecuTorch programs that delegate to it.
-CMAKE_ARGS="${CMAKE_ARGS} -DEXECUTORCH_BUILD_XNNPACK=ON"
+source "${GITHUB_WORKSPACE}/${REPOSITORY}/.ci/scripts/wheel/envvar_base.sh"
 
 # When building for macOS, link additional backends into the pybindings runtime.
 CMAKE_ARGS="${CMAKE_ARGS} -DEXECUTORCH_BUILD_COREML=ON"

.ci/scripts/wheel/envvar_linux.sh

@@ -14,4 +14,4 @@ set -euxo pipefail
 # which does install them. Though we'd need to disable build isolation to be
 # able to see the installed torch package.
 
-pip install --progress-bar off -r requirements-dev.txt
+"${GITHUB_WORKSPACE}/${REPOSITORY}/install_requirements.sh"

build/packaging/env_var_script_macos.sh .ci/scripts/wheel/envvar_macos.sh

@@ -0,0 +1,66 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+import os
+import subprocess
+import sys
+from dataclasses import dataclass
+from functools import cache
+from typing import List
+
+
+@cache
+def _unsafe_get_env(key: str) -> str:
+    value = os.getenv(key)
+    if value is None:
+        raise RuntimeError(f"environment variable '{key}' is not set")
+    return value
+
+
+@cache
+def _repository_root_dir() -> str:
+    return os.path.join(
+        _unsafe_get_env("GITHUB_WORKSPACE"),
+        _unsafe_get_env("REPOSITORY"),
+    )
+
+
+# For some reason, we are unable to see the entire repo in the python path.
+# So manually add it.
+sys.path.append(_repository_root_dir())
+from examples.models import Backend, Model
+
+
+@dataclass
+class ModelTest:
+    model: Model
+    backend: Backend
+
+
+def run_tests(model_tests: List[ModelTest]) -> None:
+    # Why are we doing this envvar shenanigans? Since we build the testers, which
+    # uses buck, we cannot run as root. This is a sneaky of getting around that
+    # test.
+    #
+    # This can be reverted if either:
+    #   - We remove usage of buck in our builds
+    #   - We stop running the Docker image as root: https://github.com/pytorch/test-infra/issues/5091
+    envvars = os.environ.copy()
+    envvars.pop("HOME")
+
+    for model_test in model_tests:
+        subprocess.run(
+            [
+                os.path.join(_repository_root_dir(), ".ci/scripts/test_model.sh"),
+                str(model_test.model),
+                # What to build `executor_runner` with for testing.
+                "cmake",
+                str(model_test.backend),
+            ],
+            env=envvars,
+            check=True,
+            cwd=_repository_root_dir(),
+        )

build/packaging/post_build_script.sh .ci/scripts/wheel/post_build_script.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env python
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+import test_base
+from examples.models import Backend, Model
+
+if __name__ == "__main__":
+    test_base.run_tests(
+        model_tests=[
+            test_base.ModelTest(
+                model=Model.Mv3,
+                backend=Backend.XnnpackQuantizationDelegation,
+            )
+        ]
+    )

build/packaging/pre_build_script.sh .ci/scripts/wheel/pre_build_script.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env python
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+import test_base
+from examples.models import Backend, Model
+
+if __name__ == "__main__":
+    test_base.run_tests(
+        model_tests=[
+            test_base.ModelTest(
+                model=Model.Mv3,
+                backend=Backend.XnnpackQuantizationDelegation,
+            )
+        ]
+    )

.ci/scripts/wheel/test_base.py

@@ -4,7 +4,7 @@ name: Build Linux Wheels
 on:
   pull_request:
     paths:
-      - build/packaging/**
+      - .ci/**/*
       - .github/workflows/build-wheels-linux.yml
   push:
     branches:
@@ -39,9 +39,9 @@ jobs:
       matrix:
         include:
           - repository: pytorch/executorch
-            pre-script: build/packaging/pre_build_script.sh
-            post-script: build/packaging/post_build_script.sh
-            smoke-test-script: build/packaging/smoke_test.py
+            pre-script: .ci/scripts/wheel/pre_build_script.sh
+            post-script: .ci/scripts/wheel/post_build_script.sh
+            smoke-test-script: .ci/scripts/wheel/test_linux.py
             package-name: executorch
     name: ${{ matrix.repository }}
     uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main
@@ -51,11 +51,8 @@ jobs:
       test-infra-repository: pytorch/test-infra
       test-infra-ref: main
       build-matrix: ${{ needs.generate-matrix.outputs.matrix }}
-      # ExecuTorch only needs the first layer of submodules; override the
-      # "recursive" default to do less work, and to give the buck daemon fewer
-      # files to look at.
-      submodules: true
-      env-var-script: build/packaging/env_var_script_linux.sh
+      submodules: recursive
+      env-var-script: .ci/scripts/wheel/envvar_linux.sh
       pre-script: ${{ matrix.pre-script }}
       post-script: ${{ matrix.post-script }}
       package-name: ${{ matrix.package-name }}

.ci/scripts/wheel/test_linux.py

@@ -4,7 +4,7 @@ name: Build macOS Wheels
 on:
   pull_request:
     paths:
-      - build/packaging/**
+      - .ci/**/*
       - .github/workflows/build-wheels-macos.yml
   push:
     branches:
@@ -39,9 +39,9 @@ jobs:
       matrix:
         include:
           - repository: pytorch/executorch
-            pre-script: build/packaging/pre_build_script.sh
-            post-script: build/packaging/post_build_script.sh
-            smoke-test-script: build/packaging/smoke_test.py
+            pre-script: .ci/scripts/wheel/pre_build_script.sh
+            post-script: .ci/scripts/wheel/post_build_script.sh
+            smoke-test-script: .ci/scripts/wheel/test_macos.py
             package-name: executorch
     name: ${{ matrix.repository }}
     uses: pytorch/test-infra/.github/workflows/build_wheels_macos.yml@main
@@ -51,12 +51,9 @@ jobs:
       test-infra-repository: pytorch/test-infra
       test-infra-ref: main
       build-matrix: ${{ needs.generate-matrix.outputs.matrix }}
-      # ExecuTorch only needs the first layer of submodules; override the
-      # "recursive" default to do less work, and to give the buck daemon fewer
-      # files to look at.
-      submodules: true
+      submodules: recursive
       delocate-wheel: false
-      env-var-script: build/packaging/env_var_script_macos.sh
+      env-var-script: .ci/scripts/wheel/envvar_macos.sh
       pre-script: ${{ matrix.pre-script }}
       post-script: ${{ matrix.post-script }}
       package-name: ${{ matrix.package-name }}

.ci/scripts/wheel/test_macos.py

@@ -5,37 +5,81 @@
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
 
+from enum import Enum
+
+
+class Model(str, Enum):
+    Mul = "mul"
+    Linear = "linear"
+    Add = "add"
+    AddMul = "add_mul"
+    Softmax = "softmax"
+    Dl3 = "dl3"
+    Edsr = "edsr"
+    EmformerTranscribe = "emformer_transcribe"
+    EmformerPredict = "emformer_predict"
+    EmformerJoin = "emformer_join"
+    Llama2 = "llama2"
+    Llama = "llama"
+    Llama32VisionEncoder = "llama3_2_vision_encoder"
+    Lstm = "lstm"
+    MobileBert = "mobilebert"
+    Mv2 = "mv2"
+    Mv2Untrained = "mv2_untrained"
+    Mv3 = "mv3"
+    Vit = "vit"
+    W2l = "w2l"
+    Ic3 = "ic3"
+    Ic4 = "ic4"
+    ResNet18 = "resnet18"
+    ResNet50 = "resnet50"
+    Llava = "llava"
+    EfficientSam = "efficient_sam"
+    Qwen25 = "qwen2_5"
+    Phi4Mini = "phi-4-mini"
+
+    def __str__(self) -> str:
+        return self.value
+
+
+class Backend(str, Enum):
+    XnnpackQuantizationDelegation = "xnnpack-quantization-delegation"
+
+    def __str__(self) -> str:
+        return self.value
+
+
 MODEL_NAME_TO_MODEL = {
-    "mul": ("toy_model", "MulModule"),
-    "linear": ("toy_model", "LinearModule"),
-    "add": ("toy_model", "AddModule"),
-    "add_mul": ("toy_model", "AddMulModule"),
-    "softmax": ("toy_model", "SoftmaxModule"),
-    "dl3": ("deeplab_v3", "DeepLabV3ResNet50Model"),
-    "edsr": ("edsr", "EdsrModel"),
-    "emformer_transcribe": ("emformer_rnnt", "EmformerRnntTranscriberModel"),
-    "emformer_predict": ("emformer_rnnt", "EmformerRnntPredictorModel"),
-    "emformer_join": ("emformer_rnnt", "EmformerRnntJoinerModel"),
-    "llama2": ("llama", "Llama2Model"),
-    "llama": ("llama", "Llama2Model"),
-    "llama3_2_vision_encoder": ("llama3_2_vision", "FlamingoVisionEncoderModel"),
+    str(Model.Mul): ("toy_model", "MulModule"),
+    str(Model.Linear): ("toy_model", "LinearModule"),
+    str(Model.Add): ("toy_model", "AddModule"),
+    str(Model.AddMul): ("toy_model", "AddMulModule"),
+    str(Model.Softmax): ("toy_model", "SoftmaxModule"),
+    str(Model.Dl3): ("deeplab_v3", "DeepLabV3ResNet50Model"),
+    str(Model.Edsr): ("edsr", "EdsrModel"),
+    str(Model.EmformerTranscribe): ("emformer_rnnt", "EmformerRnntTranscriberModel"),
+    str(Model.EmformerPredict): ("emformer_rnnt", "EmformerRnntPredictorModel"),
+    str(Model.EmformerJoin): ("emformer_rnnt", "EmformerRnntJoinerModel"),
+    str(Model.Llama2): ("llama", "Llama2Model"),
+    str(Model.Llama): ("llama", "Llama2Model"),
+    str(Model.Llama32VisionEncoder): ("llama3_2_vision", "FlamingoVisionEncoderModel"),
     # TODO: This take too long to export on both Linux and MacOS (> 6 hours)
     # "llama3_2_text_decoder": ("llama3_2_vision", "Llama3_2Decoder"),
-    "lstm": ("lstm", "LSTMModel"),
-    "mobilebert": ("mobilebert", "MobileBertModelExample"),
-    "mv2": ("mobilenet_v2", "MV2Model"),
-    "mv2_untrained": ("mobilenet_v2", "MV2UntrainedModel"),
-    "mv3": ("mobilenet_v3", "MV3Model"),
-    "vit": ("torchvision_vit", "TorchVisionViTModel"),
-    "w2l": ("wav2letter", "Wav2LetterModel"),
-    "ic3": ("inception_v3", "InceptionV3Model"),
-    "ic4": ("inception_v4", "InceptionV4Model"),
-    "resnet18": ("resnet", "ResNet18Model"),
-    "resnet50": ("resnet", "ResNet50Model"),
-    "llava": ("llava", "LlavaModel"),
-    "efficient_sam": ("efficient_sam", "EfficientSAM"),
-    "qwen2_5": ("qwen2_5", "Qwen2_5Model"),
-    "phi-4-mini": ("phi-4-mini", "Phi4MiniModel"),
+    str(Model.Lstm): ("lstm", "LSTMModel"),
+    str(Model.MobileBert): ("mobilebert", "MobileBertModelExample"),
+    str(Model.Mv2): ("mobilenet_v2", "MV2Model"),
+    str(Model.Mv2Untrained): ("mobilenet_v2", "MV2UntrainedModel"),
+    str(Model.Mv3): ("mobilenet_v3", "MV3Model"),
+    str(Model.Vit): ("torchvision_vit", "TorchVisionViTModel"),
+    str(Model.W2l): ("wav2letter", "Wav2LetterModel"),
+    str(Model.Ic3): ("inception_v3", "InceptionV3Model"),
+    str(Model.Ic4): ("inception_v4", "InceptionV4Model"),
+    str(Model.ResNet18): ("resnet", "ResNet18Model"),
+    str(Model.ResNet50): ("resnet", "ResNet50Model"),
+    str(Model.Llava): ("llava", "LlavaModel"),
+    str(Model.EfficientSam): ("efficient_sam", "EfficientSAM"),
+    str(Model.Qwen25): ("qwen2_5", "Qwen2_5Model"),
+    str(Model.Phi4Mini): ("phi-4-mini", "Phi4MiniModel"),
 }
 
 __all__ = [

.github/workflows/build-wheels-linux.yml

@@ -1,4 +1,4 @@
-cmake  # For building binary targets in the wheel.
+cmake>=3.19  # For building binary targets in the wheel.
 pip>=23  # For building the pip package.
 pyyaml  # Imported by the kernel codegen tools.
 setuptools>=63  # For building the pip package contents.