Add unit test cases for benchmark

Author: iplitharas

Makefile

@@ -22,6 +22,9 @@ setup-local-dev: poetry-install install-hooks ## Setup the local environment
 test: ## Run locally pytest with coverage
 	pytest . -vv -p no:warnings --cov=. --cov-report=xml --cov-report=html
 
+test-local: ## Run pytest with coverage
+	pytest . -vv -p no:warnings --cov=.
+
 check: ## Run isort black and pylint in all files
 	isort src
 	black src

pyproject.toml

@@ -57,6 +57,9 @@ python_version = "3.10"
 strict_optional = true
 strict = true
 
+[project]
+project_urls = { "GitHub" = "https://github.com/iplitharas/pycompile" }
+
 
 
 

pytest.ini

@@ -1,3 +1,2 @@
 [pytest]
-addopts = --cov=./tests --cov-config=.coveragerc
 norecursedirs = src/examples

tests/conftest.py

@@ -24,6 +24,25 @@ def sample_python_file_fixture(tmp_path: Path) -> Path:
     return sample_folder
 
 
+@pytest.fixture
+def sample_python_file_with_test_fixture(tmp_path: Path) -> Path:
+    """
+    Pytest fixture to create a sample `.py` file with a corresponding
+    and return's test file and return its parent folder path.
+    """
+    # Create a sample folder within the `temp_path`
+    sample_folder = tmp_path / "sample"
+    sample_folder.mkdir()
+    # Create a sample file within the sample folder
+    python_file = sample_folder / "hello.py"
+    content = r"print('Im a python module')"
+    python_file.write_text(content)
+    test_file = sample_folder / "test_hello.py"
+    content = r"def test_hello():" r"    assert True"
+    test_file.write_text(content)
+    return sample_folder
+
+
 @pytest.fixture
 def sample_files_fixture(tmp_path: Path) -> Path:
     """

tests/unit/test_benchmark.py

@@ -0,0 +1,291 @@
+"""
+Test cases for the `Benchmark`
+"""
+import logging
+from unittest.mock import ANY, MagicMock, call, patch
+
+from src import CythonWrapper, NuitkaWrapper
+from src.benchmark import Benchmark
+
+MODULE = "src.benchmark"
+
+
+@patch(MODULE + ".run_pytest")
+@patch(MODULE + ".decorate_functions")
+def test_benchmark_run_cpu_bench(
+    mocked_decorate_functions,
+    mocked_run_pytest,
+    sample_python_file_with_test_fixture,
+    caplog,
+):
+    """
+    Given one `.py` file with a  test file within a directory
+    When we invoke the `cpu_bench` from `Benchmark`
+    Then we expect to see the `cpu_bench` in the logs,
+    the `decorate_functions`
+    and the run_pytest to be called once.
+    """
+    # Given
+    caplog.clear()
+    caplog.set_level(logging.INFO)
+    sample_folder = sample_python_file_with_test_fixture
+    assert len(list(sample_folder.iterdir())) == 2
+    # When
+    Benchmark.cpu_bench(input_path=sample_folder, engine="python")
+    # Then
+    assert "CPU benchmark using:`python`" in caplog.text
+    mocked_decorate_functions.assert_called_once()
+    mocked_run_pytest.assert_called_once()
+
+
+@patch(MODULE + ".run_pytest")
+@patch(MODULE + ".decorate_functions")
+def test_benchmark_run_memory_bench(
+    mocked_decorate_functions,
+    mocked_run_pytest,
+    sample_python_file_with_test_fixture,
+    caplog,
+):
+    """
+    Given one `.py` file with a  test file within a directory
+    When we invoke the `mem_bench` from `Benchmark`
+    Then we expect to see the `mem_bench` in the logs,
+    the `decore_functions` and the `run_pytest` be called once
+    """
+    # Given
+    caplog.clear()
+    caplog.set_level(logging.INFO)
+    sample_folder = sample_python_file_with_test_fixture
+    assert len(list(sample_folder.iterdir())) == 2
+    # When
+    Benchmark.mem_bench(
+        input_path=sample_folder, prof_func_name="test", engine="python"
+    )
+    # Then
+    assert "Memory benchmark using:`python`" in caplog.text
+    mocked_decorate_functions.assert_called_once()
+    mocked_run_pytest.assert_called_once()
+
+
+def test_benchmark_start_with_bench_type_both_and_without_compiling(
+    sample_python_file_with_test_fixture, caplog
+):
+    """
+    Given one `.py` file with a test file within a directory.
+    When we invoke the `start` method with an empty compilers list
+         and a `bench_type` of `both`
+    Then we expect only a mem and cpu benchmark to be executed.
+    """
+    # Given
+    caplog.clear()
+    caplog.set_level(logging.INFO)
+    sample_folder = sample_python_file_with_test_fixture
+    assert len(list(sample_folder.iterdir())) == 2
+    # When
+    benchmark = Benchmark(input_path=sample_folder)
+    benchmark.start(bench_type="both", compilers=[])
+    # Then
+    assert "Memory benchmark" in caplog.text
+    assert "CPU benchmark" in caplog.text
+
+
+def test_benchmark_start_with_bench_type_both_and_with_cython_compiling(
+    sample_python_file_with_test_fixture, caplog
+):
+    """
+    Given one `.py` file with a test file within a directory.
+    When we invoke the `start` method with compilers Cython
+         and a `bench_type` of `both`
+    Then we expect only a mem and cpu benchmark to be executed twice
+        one for python and one for cython.
+    """
+    # Given
+    caplog.clear()
+    caplog.set_level(logging.INFO)
+    sample_folder = sample_python_file_with_test_fixture
+    assert len(list(sample_folder.iterdir())) == 2
+    # When
+    benchmark = Benchmark(input_path=sample_folder)
+    benchmark.mem_bench = MagicMock()
+    benchmark.cpu_bench = MagicMock()
+    benchmark._compile = MagicMock()
+    benchmark.start(bench_type="both", compilers=[CythonWrapper()])
+    # Then
+    benchmark._compile.assert_called_once()
+    benchmark.mem_bench.assert_has_calls((ANY, ANY), any_order=True)
+    benchmark.cpu_bench.assert_has_calls((ANY, ANY), any_order=True)
+
+
+def test_benchmark_start_with_bench_type_both_and_with_cython__and_nuitka_compiling(
+    sample_python_file_with_test_fixture, caplog
+):
+    """
+    Given one `.py` file with a test file within a directory.
+    When we invoke the `start` method with compilers Cython and Nuitka
+         and a `bench_type` of `both`
+    Then we expect only a mem and cpu benchmark to be executed three times
+        one for python, one for cython and one for nuitka.
+    """
+    # Given
+    caplog.clear()
+    caplog.set_level(logging.INFO)
+    sample_folder = sample_python_file_with_test_fixture
+    assert len(list(sample_folder.iterdir())) == 2
+    # When
+    benchmark = Benchmark(input_path=sample_folder)
+    benchmark.mem_bench = MagicMock()
+    benchmark.cpu_bench = MagicMock()
+    benchmark._compile = MagicMock()
+    benchmark.start(
+        bench_type="both", compilers=[CythonWrapper(), NuitkaWrapper()]
+    )
+    # Then
+    benchmark._compile.assert_has_calls((ANY, ANY), any_order=True)
+    benchmark.mem_bench.assert_has_calls((ANY, ANY, ANY), any_order=True)
+    benchmark.cpu_bench.assert_has_calls((ANY, ANY, ANY), any_order=True)
+
+
+def test_benchmark_start_with_bench_type_memory_and_without_compiling(
+    sample_python_file_with_test_fixture, caplog
+):
+    """
+    Given one `.py` file with a test file within a directory.
+    When we invoke the `start` method with an empty compilers list
+        and a `bench_type` of `memory`
+    Then we expect only a mem benchmark to be executed.
+    """
+    # Given
+    caplog.clear()
+    caplog.set_level(logging.INFO)
+    sample_folder = sample_python_file_with_test_fixture
+    # When
+    benchmark = Benchmark(input_path=sample_folder)
+    benchmark.start(bench_type="memory", compilers=[])
+    # Then
+    assert "Memory benchmark" in caplog.text
+    assert "CPU benchmark" not in caplog.text
+
+
+def test_benchmark_start_with_bench_type_memory_and_with_cython__and_nuitka_compiling(
+    sample_python_file_with_test_fixture, caplog
+):
+    """
+    Given one `.py` file with a test file within a directory.
+    When we invoke the `start` method with compilers Cython
+        and a `bench_type` of `memory`
+    Then we expect only a mem benchmark to be executed the times
+    """
+    # Given
+    caplog.clear()
+    caplog.set_level(logging.INFO)
+    sample_folder = sample_python_file_with_test_fixture
+    # When
+    benchmark = Benchmark(input_path=sample_folder)
+    benchmark.mem_bench = MagicMock()
+    benchmark.cpu_bench = MagicMock()
+    benchmark._compile = MagicMock()
+    benchmark.start(
+        bench_type="memory", compilers=[CythonWrapper(), NuitkaWrapper()]
+    )
+    # Then
+    benchmark._compile.assert_has_calls((ANY, ANY), any_order=True)
+    benchmark.mem_bench.assert_has_calls((ANY, ANY, ANY), any_order=True)
+    benchmark.cpu_bench.assert_not_called()
+
+
+def test_benchmark_start_with_bench_type_memory_and_with_cython_compiling(
+    sample_python_file_with_test_fixture, caplog
+):
+    """
+    Given one `.py` file with a test file within a directory.
+    When we invoke the `start` method with compilers Cython
+        and a `bench_type` of `memory`
+    Then we expect only a mem benchmark to be executed twice
+    """
+    # Given
+    caplog.clear()
+    caplog.set_level(logging.INFO)
+    sample_folder = sample_python_file_with_test_fixture
+    # When
+    benchmark = Benchmark(input_path=sample_folder)
+    benchmark.mem_bench = MagicMock()
+    benchmark.cpu_bench = MagicMock()
+    benchmark._compile = MagicMock()
+    benchmark.start(bench_type="memory", compilers=[CythonWrapper()])
+    # Then
+    benchmark._compile.assert_called_once()
+    benchmark.mem_bench.assert_has_calls((ANY, ANY), any_order=True)
+    benchmark.cpu_bench.assert_not_called()
+
+
+def test_benchmark_start_with_bench_type_cpu_and_without_compiling(
+    sample_python_file_with_test_fixture, caplog
+):
+    """
+    Given one `.py` file with a test file within a directory.
+    When we invoke the `start` method with an empty compilers list
+        and a `bench_type` of `cpu`
+    Then we expect only a cpu benchmark to be executed.
+    """
+    # Given
+    caplog.clear()
+    caplog.set_level(logging.INFO)
+    sample_folder = sample_python_file_with_test_fixture
+    # When
+    benchmark = Benchmark(input_path=sample_folder)
+    benchmark.start(bench_type="cpu", compilers=[])
+    # Then
+    assert "Memory benchmark" not in caplog.text
+    assert "CPU benchmark" in caplog.text
+
+
+def test_benchmark_start_with_bench_type_cpu_and_with_cython_compiling(
+    sample_python_file_with_test_fixture, caplog
+):
+    """
+    Given one `.py` file with a test file within a directory.
+    When we invoke the `start` method with compilers Cython
+        and a `bench_type` of `cpu`
+    Then we expect only a cpu benchmark to be executed twice
+    """
+    # Given
+    caplog.clear()
+    caplog.set_level(logging.INFO)
+    sample_folder = sample_python_file_with_test_fixture
+    # When
+    benchmark = Benchmark(input_path=sample_folder)
+    benchmark.mem_bench = MagicMock()
+    benchmark.cpu_bench = MagicMock()
+    benchmark._compile = MagicMock()
+    benchmark.start(bench_type="cpu", compilers=[CythonWrapper()])
+    # Then
+    benchmark._compile.assert_called_once()
+    benchmark.cpu_bench.assert_has_calls((ANY, ANY), any_order=True)
+    benchmark.mem_bench.assert_not_called()
+
+
+def test_benchmark_start_with_bench_type_cpu_and_with_cython_and_nuitka_compiling(
+    sample_python_file_with_test_fixture, caplog
+):
+    """
+    Given one `.py` file with a test file within a directory.
+    When we invoke the `start` method with compilers Cython and Nuitka
+        and a `bench_type` of `cpu`
+    Then we expect only a cpu benchmark to be executed three times
+    """
+    # Given
+    caplog.clear()
+    caplog.set_level(logging.INFO)
+    sample_folder = sample_python_file_with_test_fixture
+    # When
+    benchmark = Benchmark(input_path=sample_folder)
+    benchmark.mem_bench = MagicMock()
+    benchmark.cpu_bench = MagicMock()
+    benchmark._compile = MagicMock()
+    benchmark.start(
+        bench_type="cpu", compilers=[CythonWrapper(), NuitkaWrapper()]
+    )
+    # Then
+    benchmark._compile.assert_has_calls((ANY, ANY), any_order=True)
+    benchmark.cpu_bench.assert_has_calls((ANY, ANY, ANY), any_order=True)
+    benchmark.mem_bench.assert_not_called()