Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Enable jpeg turbo for new recipes #2924

Open
wants to merge 8 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 41 additions & 29 deletions conda_forge_tick/migrators/arch.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import typing
from pathlib import Path
from textwrap import dedent
from typing import Any, Optional, Sequence

Expand All @@ -10,7 +11,6 @@
make_outputs_lut_from_graph,
)
from conda_forge_tick.migrators.core import GraphMigrator, _sanitized_muids
from conda_forge_tick.os_utils import pushd
from conda_forge_tick.utils import (
as_iterable,
frozen_to_json_friendly,
Expand Down Expand Up @@ -195,18 +195,26 @@ def filter(self, attrs: "AttrsTypedDict", not_bad_str_start: str = "") -> bool:
def migrate(
self, recipe_dir: str, attrs: "AttrsTypedDict", **kwargs: Any
) -> "MigrationUidTypedDict":
with pushd(recipe_dir + "/.."):
self.set_build_number("recipe/meta.yaml")
with open("conda-forge.yml") as f:
y = yaml_safe_load(f)
if "provider" not in y:
y["provider"] = {}
for k, v in self.arches.items():
if k not in y["provider"]:
y["provider"][k] = v

with open("conda-forge.yml", "w") as f:
yaml_safe_dump(y, f)
recipe_dir_p = Path(recipe_dir)
recipe_file = self.find_recipe(recipe_dir_p)
self.set_build_number(recipe_file)

conda_forge_yml = recipe_dir_p / "conda-forge.yml"
with conda_forge_yml.open() as f:
y = yaml_safe_load(f)

# we should do this recursively but the cf yaml is usually
# one key deep so this is fine
for k, v in self.additional_keys.items():
if isinstance(v, dict):
if k not in y:
y[k] = {}
for _k, _v in v.items():
y[k][_k] = _v
else:
y[k] = v
with conda_forge_yml.open("w") as f:
yaml_safe_dump(y, f)

return super().migrate(recipe_dir, attrs, **kwargs)

Expand Down Expand Up @@ -362,22 +370,26 @@ def filter(self, attrs: "AttrsTypedDict", not_bad_str_start: str = "") -> bool:
def migrate(
self, recipe_dir: str, attrs: "AttrsTypedDict", **kwargs: Any
) -> "MigrationUidTypedDict":
with pushd(recipe_dir + "/.."):
self.set_build_number("recipe/meta.yaml")
with open("conda-forge.yml") as f:
y = yaml_safe_load(f)
# we should do this recursively but the cf yaml is usually
# one key deep so this is fine
for k, v in self.additional_keys.items():
if isinstance(v, dict):
if k not in y:
y[k] = {}
for _k, _v in v.items():
y[k][_k] = _v
else:
y[k] = v
with open("conda-forge.yml", "w") as f:
yaml_safe_dump(y, f)
recipe_dir_p = Path(recipe_dir)
recipe_file = self.find_recipe(recipe_dir_p)
self.set_build_number(recipe_file)

conda_forge_yml = recipe_dir_p / "conda-forge.yml"
with conda_forge_yml.open() as f:
y = yaml_safe_load(f)

# we should do this recursively but the cf yaml is usually
# one key deep so this is fine
for k, v in self.additional_keys.items():
if isinstance(v, dict):
if k not in y:
y[k] = {}
for _k, _v in v.items():
y[k][_k] = _v
else:
y[k] = v
with conda_forge_yml.open("w") as f:
yaml_safe_dump(y, f)

return super().migrate(recipe_dir, attrs, **kwargs)

Expand Down
32 changes: 30 additions & 2 deletions conda_forge_tick/migrators/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import logging
import re
import typing
from pathlib import Path
from typing import Any, List, Sequence, Set

import dateutil.parser
Expand Down Expand Up @@ -560,12 +561,39 @@ def order(
}
return cyclic_topological_sort(graph, top_level)

def set_build_number(self, filename: str) -> None:
def find_recipe(self, recipe_dir: str | Path) -> Path:
"""
Find the recipe, either meta.yaml or recipe.yaml in the recipe_dir

Parameters
----------
recipe_dir : str or Path
Path to the recipe directory

Returns
-------
recipe_path : Path
Path to the recipe file

Raises
------
FileNotFoundError
If no recipe is found in the recipe_dir
"""
recipe_dir = Path(recipe_dir)
if recipe_dir.joinpath("meta.yaml").exists():
return recipe_dir.joinpath("meta.yaml")
elif recipe_dir.joinpath("recipe.yaml").exists():
return recipe_dir.joinpath("recipe.yaml")
else:
raise FileNotFoundError("No recipe found in %s" % recipe_dir)

def set_build_number(self, filename: str | Path) -> None:
"""Bump the build number of the specified recipe.

Parameters
----------
filename : str
filename : str or Path
Path the the meta.yaml
"""
with open(filename) as f:
Expand Down
15 changes: 15 additions & 0 deletions conda_forge_tick/migrators/cross_compile.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,9 @@ def migrate(self, recipe_dir: str, attrs: "AttrsTypedDict", **kwargs: Any) -> No
return
try:
with provide_source_code(recipe_dir) as cb_work_dir:
if not os.path.exists("meta.yaml") and os.path.exists("recipe.yaml"):
logger.info(f"Skipping {self.__class__.__name__} for recipe.yaml")
return
if cb_work_dir is None:
return
directories = set()
Expand Down Expand Up @@ -155,6 +158,9 @@ def filter(self, attrs: "AttrsTypedDict", not_bad_str_start: str = "") -> bool:
def migrate(self, recipe_dir: str, attrs: "AttrsTypedDict", **kwargs: Any) -> None:
host_reqs = attrs.get("requirements", {}).get("host", set())
with pushd(recipe_dir):
if not os.path.exists("meta.yaml") and os.path.exists("recipe.yaml"):
logger.info(f"Skipping {self.__class__.__name__} for recipe.yaml")
return
with open("meta.yaml") as f:
lines = f.readlines()
in_reqs = False
Expand Down Expand Up @@ -259,6 +265,9 @@ def filter(self, attrs: "AttrsTypedDict", not_bad_str_start: str = "") -> bool:

def migrate(self, recipe_dir: str, attrs: "AttrsTypedDict", **kwargs: Any) -> None:
with pushd(recipe_dir):
if not os.path.exists("meta.yaml") and os.path.exists("recipe.yaml"):
logger.info(f"Skipping {self.__class__.__name__} for recipe.yaml")
return
with open("meta.yaml") as fp:
meta_yaml = fp.readlines()

Expand Down Expand Up @@ -288,6 +297,9 @@ def filter(self, attrs: "AttrsTypedDict", not_bad_str_start: str = "") -> bool:

def migrate(self, recipe_dir: str, attrs: "AttrsTypedDict", **kwargs: Any) -> None:
with pushd(recipe_dir):
if not os.path.exists("meta.yaml") and os.path.exists("recipe.yaml"):
logger.info(f"Skipping {self.__class__.__name__} for recipe.yaml")
return
with open("meta.yaml") as fp:
meta_yaml = fp.readlines()

Expand Down Expand Up @@ -321,6 +333,9 @@ def filter(self, attrs: "AttrsTypedDict", not_bad_str_start: str = "") -> bool:

def migrate(self, recipe_dir: str, attrs: "AttrsTypedDict", **kwargs: Any) -> None:
with pushd(recipe_dir):
if not os.path.exists("meta.yaml") and os.path.exists("recipe.yaml"):
logger.info(f"Skipping {self.__class__.__name__} for recipe.yaml")
return
with open("meta.yaml") as fp:
meta_yaml = fp.readlines()

Expand Down
9 changes: 9 additions & 0 deletions conda_forge_tick/migrators/duplicate_lines.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
import logging
import os
import re
import typing
from typing import Any
Expand All @@ -9,6 +11,9 @@
from ..migrators_types import AttrsTypedDict


logger = logging.getLogger(__name__)


class DuplicateLinesCleanup(MiniMigrator):
regex_to_check = {
"noarch: generic": re.compile(r"^\s*noarch:\s*generic\s*$"),
Expand All @@ -29,6 +34,10 @@ def filter(self, attrs: "AttrsTypedDict", not_bad_str_start: str = "") -> bool:

def migrate(self, recipe_dir: str, attrs: "AttrsTypedDict", **kwargs: Any) -> None:
with pushd(recipe_dir):
if not os.path.exists("meta.yaml") and os.path.exists("recipe.yaml"):
logger.info(f"Skipping {self.__class__.__name__} for recipe.yaml")
return

with open("meta.yaml") as fp:
raw_yaml = fp.read()

Expand Down
7 changes: 7 additions & 0 deletions conda_forge_tick/migrators/extra_jinj2a_keys_cleanup.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
import logging
import os
import re
import typing
from typing import Any
Expand All @@ -8,6 +10,8 @@
if typing.TYPE_CHECKING:
from ..migrators_types import AttrsTypedDict

logger = logging.getLogger(__name__)


class ExtraJinja2KeysCleanup(MiniMigrator):
"""Remove extra Jinja2 keys that make things hard for the version migrator.
Expand Down Expand Up @@ -58,6 +62,9 @@ def _replace_jinja_key(self, key_name, lines):

def migrate(self, recipe_dir: str, attrs: "AttrsTypedDict", **kwargs: Any) -> None:
with pushd(recipe_dir):
if not os.path.exists("meta.yaml") and os.path.exists("recipe.yaml"):
logger.info(f"Skipping {self.__class__.__name__} for recipe.yaml")
return
with open("meta.yaml") as fp:
lines = fp.readlines()

Expand Down
8 changes: 8 additions & 0 deletions conda_forge_tick/migrators/jinja2_vars_cleanup.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
import logging
import os
import re
import typing
from typing import Any
Expand All @@ -8,6 +10,9 @@
if typing.TYPE_CHECKING:
from ..migrators_types import AttrsTypedDict


logger = logging.getLogger(__name__)

JINJA2_VAR_RE = re.compile(r"{{\s*(.*?)\s*}}")


Expand Down Expand Up @@ -38,6 +43,9 @@ def filter(self, attrs: "AttrsTypedDict", not_bad_str_start: str = "") -> bool:

def migrate(self, recipe_dir: str, attrs: "AttrsTypedDict", **kwargs: Any) -> None:
with pushd(recipe_dir):
if not os.path.exists("meta.yaml") and os.path.exists("recipe.yaml"):
logger.info(f"Skipping {self.__class__.__name__} for recipe.yaml")
return
with open("meta.yaml") as fp:
raw_yaml = fp.read()

Expand Down
11 changes: 4 additions & 7 deletions conda_forge_tick/migrators/jpegturbo.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,8 @@ def filter(self, attrs, not_bad_str_start=""):
return "jpeg" not in host_req

def migrate(self, recipe_dir, attrs, **kwargs):
fname = os.path.join(recipe_dir, "meta.yaml")
if os.path.exists(fname):
with open(fname) as fp:
lines = fp.readlines()
recipe_file = self.find_recipe(recipe_dir)

new_lines = _parse_jpeg(lines)
with open(fname, "w") as fp:
fp.write("".join(new_lines))
lines = recipe_file.read_text().splitlines(keepends=True)
new_lines = _parse_jpeg(lines)
recipe_file.write_text("".join(new_lines))
12 changes: 12 additions & 0 deletions conda_forge_tick/migrators/license.py
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,11 @@ def _scrape_license_string(pkg):
],
check=True,
)
if not os.path.exists("r-%s/meta.yaml" % pkg) and os.path.exists(
"r-%s/recipe.yaml" % pkg
):
logger.info("Skipping _scrape_license_string for recipe.yaml")
return
with open("r-%s/meta.yaml" % pkg) as fp:
in_about = False
meta_yaml = []
Expand Down Expand Up @@ -334,6 +339,13 @@ def migrate(self, recipe_dir: str, attrs: "AttrsTypedDict", **kwargs: Any) -> No
Copyright (c) 2017, Anthony Scopatz
Copyright (c) 2018, The Regro Developers
All rights reserved."""
if not os.path.exists("meta.yaml") and os.path.exists(
"recipe.yaml"
):
logger.info(
"Skipping _scrape_license_string for recipe.yaml"
)
return
with open("meta.yaml") as f:
raw = f.read()
lines = raw.splitlines()
Expand Down
7 changes: 7 additions & 0 deletions conda_forge_tick/migrators/pip_check.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
import io
import logging
import os
import re
import typing
from typing import Any
Expand All @@ -12,6 +14,8 @@
if typing.TYPE_CHECKING:
from ..migrators_types import AttrsTypedDict

logger = logging.getLogger(__name__)

# matches lines like 'key: val # [blah or blad]'
# giving back key, val, "blah or blad" in the groups
SELECTOR_RE = re.compile(r"^(\s*)(\S*):\s*(\S*)\s*#\s*\[(.*)\]")
Expand Down Expand Up @@ -159,6 +163,9 @@ def migrate(self, recipe_dir: str, attrs: "AttrsTypedDict", **kwargs: Any) -> No
with pushd(recipe_dir):
mapping = {}
groups = {}
if not os.path.exists("meta.yaml") and os.path.exists("recipe.yaml"):
logger.info(f"Skipping {self.__class__.__name__} for recipe.yaml")
return
with open("meta.yaml") as fp:
lines = []
for line in fp.readlines():
Expand Down
3 changes: 3 additions & 0 deletions conda_forge_tick/migrators/pip_wheel_dep.py
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,9 @@ def migrate(self, recipe_dir: str, attrs: "AttrsTypedDict", **kwargs: Any) -> No
handled_packages = set()

with pushd(recipe_dir):
if not os.path.exists("meta.yaml") and os.path.exists("recipe.yaml"):
logger.info(f"Skipping {self.__class__.__name__} for recipe.yaml")
return
with open("meta.yaml") as f:
lines = f.readlines()
in_reqs = False
Expand Down
7 changes: 7 additions & 0 deletions conda_forge_tick/migrators/r_ucrt.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
import logging
import os
import re
import typing
from typing import Any
Expand All @@ -8,6 +10,8 @@
if typing.TYPE_CHECKING:
from ..migrators_types import AttrsTypedDict

logger = logging.getLogger(__name__)


def _cleanup_raw_yaml(raw_yaml):
lines = []
Expand Down Expand Up @@ -47,6 +51,9 @@ def filter(self, attrs: "AttrsTypedDict", not_bad_str_start: str = "") -> bool:

def migrate(self, recipe_dir: str, attrs: "AttrsTypedDict", **kwargs: Any) -> None:
with pushd(recipe_dir):
if not os.path.exists("meta.yaml") and os.path.exists("recipe.yaml"):
logger.info(f"Skipping {self.__class__.__name__} for recipe.yaml")
return
with open("meta.yaml") as fp:
raw_yaml = fp.read()

Expand Down
5 changes: 5 additions & 0 deletions conda_forge_tick/migrators/replacement.py
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,11 @@ def filter(self, attrs: "AttrsTypedDict", not_bad_str_start: str = "") -> bool:
def migrate(
self, recipe_dir: str, attrs: "AttrsTypedDict", **kwargs: Any
) -> "MigrationUidTypedDict":
if not os.path.exists(os.path.join(recipe_dir, "meta.yaml")) and os.path.exists(
os.path.join(recipe_dir, "recipe.yaml")
):
logger.info(f"Skipping {self.__class__.__name__} for recipe.yaml")
return
with open(os.path.join(recipe_dir, "meta.yaml")) as f:
raw = f.read()
lines = raw.splitlines()
Expand Down
Loading
Loading