Skip to content

Commit

Permalink
Start with adapting run_test_migration_recipe_yaml
Browse files Browse the repository at this point in the history
Write recipe.yaml not meta.yaml

Add schema to recipe yaml

Prepare test for the adapted migrator

Adapt `update_build_number` for recipe.yaml

fix: function signature to match baseclass

Update version.py

refactor: Improve `test_version_cupy`

refactor: Improve "run_test_migration"
  • Loading branch information
Hofer-Julian committed Aug 1, 2024
1 parent 76abc39 commit 612b83c
Show file tree
Hide file tree
Showing 13 changed files with 361 additions and 46 deletions.
15 changes: 10 additions & 5 deletions conda_forge_tick/feedstock_parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,7 @@ def _extract_requirements(meta_yaml, outputs_to_keep=None):
requirements_dict[section].update(
list(as_iterable(req.get(section, []) or [])),
)
test: "TestTypedDict" = block.get("test", {})
test: "TestTypedDict" = {} if block.get("test") is None else block.get("test")
requirements_dict["test"].update(test.get("requirements", []) or [])
requirements_dict["test"].update(test.get("requires", []) or [])
run_exports = (block.get("build", {}) or {}).get("run_exports", {})
Expand Down Expand Up @@ -350,6 +350,11 @@ def populate_feedstock_attributes(
parse_meta_yaml(meta_yaml, platform=plat, arch=arch)
for plat, arch in plat_archs
]
elif isinstance(recipe_yaml, str):
variant_yamls = [
parse_recipe_yaml(recipe_yaml, platform=plat, arch=arch)
for plat, arch in plat_archs
]
except Exception as e:
import traceback

Expand Down Expand Up @@ -378,11 +383,11 @@ def populate_feedstock_attributes(
if k.endswith("_meta_yaml") or k.endswith("_requirements"):
sub_graph.pop(k)

for k, v in zip(plat_archs, variant_yamls):
plat_arch_name = "_".join(k)
sub_graph[f"{plat_arch_name}_meta_yaml"] = v
for plat_arch, variant_yaml in zip(plat_archs, variant_yamls):
plat_arch_name = "_".join(plat_arch)
sub_graph[f"{plat_arch_name}_meta_yaml"] = variant_yaml
_, sub_graph[f"{plat_arch_name}_requirements"], _ = _extract_requirements(
v,
variant_yaml,
outputs_to_keep=BOOTSTRAP_MAPPINGS.get(name, None),
)

Expand Down
41 changes: 26 additions & 15 deletions conda_forge_tick/migrators/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import logging
import re
import typing
from pathlib import Path
from typing import Any, List, Sequence, Set

import dateutil.parser
Expand All @@ -14,7 +15,8 @@
from conda_forge_tick.lazy_json_backends import LazyJson
from conda_forge_tick.make_graph import make_outputs_lut_from_graph
from conda_forge_tick.path_lengths import cyclic_topological_sort
from conda_forge_tick.update_recipe import update_build_number
from conda_forge_tick.update_recipe import update_build_number_meta_yaml
from conda_forge_tick.update_recipe.build_number import update_build_number_recipe_yaml
from conda_forge_tick.utils import (
frozen_to_json_friendly,
get_bot_run_url,
Expand Down Expand Up @@ -439,7 +441,7 @@ def run_post_piggyback_migrations(
def migrate(
self, recipe_dir: str, attrs: "AttrsTypedDict", **kwargs: Any
) -> "MigrationUidTypedDict":
"""Perform the migration, updating the ``meta.yaml``
"""Perform the migration, updating the recipe
Parameters
----------
Expand Down Expand Up @@ -560,25 +562,34 @@ def order(
}
return cyclic_topological_sort(graph, top_level)

def set_build_number(self, filename: str) -> None:
def set_build_number(self, filename: str | Path) -> None:
"""Bump the build number of the specified recipe.
Parameters
----------
filename : str
Path the the meta.yaml
filename : str | Path
Path the the recipe file
"""
with open(filename) as f:
raw = f.read()

new_myaml = update_build_number(
raw,
self.new_build_number,
build_patterns=self.build_patterns,
)
filename = Path(filename)
raw = filename.read_text()

if filename.name == "meta.yaml":
new_yaml = update_build_number_meta_yaml(
raw,
self.new_build_number,
build_patterns=self.build_patterns,
)
elif filename.name == "recipe.yaml":
new_yaml = update_build_number_recipe_yaml(
raw,
self.new_build_number,
)
else:
raise ValueError(
f"`{filename=}` needs to be a `meta.yaml` or `recipe.yaml`."
)

with open(filename, "w") as f:
f.write(new_myaml)
filename.write_text(new_yaml)

def new_build_number(self, old_number: int) -> int:
"""Determine the new build number to use.
Expand Down
41 changes: 26 additions & 15 deletions conda_forge_tick/migrators/version.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import random
import typing
import warnings
from pathlib import Path
from typing import Any, List, Sequence

import conda.exceptions
Expand All @@ -14,9 +15,11 @@
from conda_forge_tick.contexts import FeedstockContext
from conda_forge_tick.migrators.core import Migrator
from conda_forge_tick.models.pr_info import MigratorName
from conda_forge_tick.os_utils import pushd
from conda_forge_tick.update_deps import get_dep_updates_and_hints
from conda_forge_tick.update_recipe import update_version
from conda_forge_tick.update_recipe import (
update_meta_yaml_version,
update_recipe_yaml_version,
)
from conda_forge_tick.utils import get_keys_default, sanitize_string

if typing.TYPE_CHECKING:
Expand Down Expand Up @@ -195,21 +198,29 @@ def migrate(
) -> "MigrationUidTypedDict":
version = attrs["new_version"]

with open(os.path.join(recipe_dir, "meta.yaml")) as fp:
raw_meta_yaml = fp.read()
meta_yaml_path = Path(recipe_dir, "meta.yaml")
recipe_yaml_path = Path(recipe_dir, "recipe.yaml")

updated_meta_yaml, errors = update_version(
raw_meta_yaml,
version,
hash_type=hash_type,
)

if len(errors) == 0 and updated_meta_yaml is not None:
with pushd(recipe_dir):
with open("meta.yaml", "w") as fp:
fp.write(updated_meta_yaml)
self.set_build_number("meta.yaml")
if meta_yaml_path.exists():
output_path = meta_yaml_path
raw_meta_yaml = meta_yaml_path.read_text()
updated_recipe, errors = update_meta_yaml_version(
raw_meta_yaml,
version,
hash_type=hash_type,
)
elif recipe_yaml_path.exists():
output_path = recipe_yaml_path
raw_recipe_yaml = recipe_yaml_path.read_text()
updated_recipe, errors = update_recipe_yaml_version(
raw_recipe_yaml,
version,
hash_type=hash_type,
)

if len(errors) == 0 and updated_recipe is not None:
output_path.write_text(updated_recipe)
self.set_build_number(output_path)
return super().migrate(recipe_dir, attrs)
else:
raise VersionMigrationError(
Expand Down
14 changes: 12 additions & 2 deletions conda_forge_tick/update_recipe/__init__.py
Original file line number Diff line number Diff line change
@@ -1,2 +1,12 @@
from .build_number import DEFAULT_BUILD_PATTERNS, update_build_number # noqa
from .version import update_version # noqa
from .build_number import (
DEFAULT_BUILD_PATTERNS as DEFAULT_BUILD_PATTERNS,
)
from .build_number import (
update_build_number_meta_yaml as update_build_number_meta_yaml,
)
from .build_number import (
update_build_number_recipe_yaml as update_build_number_recipe_yaml,
)

# noqa
from .version import update_meta_yaml_version, update_recipe_yaml_version # noqa
20 changes: 19 additions & 1 deletion conda_forge_tick/update_recipe/build_number.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,7 @@
import re
from typing import Callable

import yaml

DEFAULT_BUILD_PATTERNS = (
(re.compile(r"(\s*?)number:\s*([0-9]+)"), "number: {}"),
Expand All @@ -13,7 +16,11 @@
)


def update_build_number(raw_meta_yaml, new_build_number, build_patterns=None):
def update_build_number_meta_yaml(
raw_meta_yaml: str,
new_build_number: Callable[[str], str] | str,
build_patterns=None,
):
"""Update the build number for a recipe.
Parameters
Expand Down Expand Up @@ -51,3 +58,14 @@ def update_build_number(raw_meta_yaml, new_build_number, build_patterns=None):
raw_meta_yaml = "\n".join(lines) + "\n"

return raw_meta_yaml


def update_build_number_recipe_yaml(
raw_recipe_yaml: str, new_build_number: Callable[[str], str] | str
):
recipe = yaml.safe_load(raw_recipe_yaml)
if callable(new_build_number):
recipe["build"]["number"] = new_build_number(recipe["build"]["number"])
else:
recipe["build"]["number"] = new_build_number
return yaml.dump(recipe)
31 changes: 29 additions & 2 deletions conda_forge_tick/update_recipe/version.py
Original file line number Diff line number Diff line change
Expand Up @@ -379,7 +379,9 @@ def _try_to_update_version(cmeta: Any, src: str, hash_type: str):
return updated_version, errors


def update_version(raw_meta_yaml, version, hash_type="sha256"):
def update_meta_yaml_version(
raw_meta_yaml: str, version: str, hash_type: str = "sha256"
) -> tuple[str | None, set[str]]:
"""Update the version in a recipe.
Parameters
Expand All @@ -395,7 +397,7 @@ def update_version(raw_meta_yaml, version, hash_type="sha256"):
-------
updated_meta_yaml : str or None
The updated meta.yaml. Will be None if there is an error.
errors : str of str
errors : set of str
A set of strings giving any errors found when updating the
version. The set will be empty if there were no errors.
"""
Expand Down Expand Up @@ -527,3 +529,28 @@ def update_version(raw_meta_yaml, version, hash_type="sha256"):
else:
logger.critical("Recipe did not change in version migration!")
return None, errors


def update_recipe_yaml_version(
raw_recipe_yaml: str, version: str, hash_type: str = "sha256"
) -> tuple[str | None, set[str]]:
"""Update the version in a recipe.
Parameters
----------
raw_recipe_yaml : str
The recipe meta.yaml as a string.
version : str
The version of the recipe.
hash_type : str, optional
The kind of hash used on the source. Default is sha256.
Returns
-------
updated_recipe_yaml : str or None
The updated meta.yaml. Will be None if there is an error.
errors : set of str
A set of strings giving any errors found when updating the
version. The set will be empty if there were no errors.
"""
raise NotImplementedError()
39 changes: 34 additions & 5 deletions tests/test_build_number.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
import pytest

from conda_forge_tick.update_recipe import update_build_number
from conda_forge_tick.update_recipe import (
update_build_number_meta_yaml,
update_build_number_recipe_yaml,
)


@pytest.mark.parametrize(
Expand All @@ -12,8 +15,8 @@
("{% set build = 2 %}", "{% set build = 0 %}\n"),
],
)
def test_update_build_number(meta_yaml, new_meta_yaml):
out_meta_yaml = update_build_number(meta_yaml, 0)
def test_update_build_number_meta_yaml(meta_yaml, new_meta_yaml):
out_meta_yaml = update_build_number_meta_yaml(meta_yaml, 0)
assert out_meta_yaml == new_meta_yaml


Expand All @@ -26,6 +29,32 @@ def test_update_build_number(meta_yaml, new_meta_yaml):
("{% set build = 2 %}", "{% set build = 3 %}\n"),
],
)
def test_update_build_number_function(meta_yaml, new_meta_yaml):
out_meta_yaml = update_build_number(meta_yaml, lambda x: x + 1)
def test_update_build_number_meta_yaml_function(meta_yaml, new_meta_yaml):
out_meta_yaml = update_build_number_meta_yaml(meta_yaml, lambda x: x + 1)
assert out_meta_yaml == new_meta_yaml


def test_update_build_number_recipe_yaml():
in_yaml = """\
build:
number: 100
"""
expected_yaml = """\
build:
number: 0
"""
out_meta_yaml = update_build_number_recipe_yaml(in_yaml, 0)
assert out_meta_yaml == expected_yaml


def test_update_build_number_recipe_yaml_function():
in_yaml = """\
build:
number: 100
"""
expected_yaml = """\
build:
number: 101
"""
out_meta_yaml = update_build_number_recipe_yaml(in_yaml, lambda x: x + 1)
assert out_meta_yaml == expected_yaml
Loading

0 comments on commit 612b83c

Please sign in to comment.