Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Run auto-tick with a single package only #2813

Draft
wants to merge 34 commits into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
34 commits
Select commit Hold shift + click to select a range
1ebe094
Reapply "Readd git backend"
ytausch Jun 13, 2024
411af0d
return feedstock_dir, not dirname
ytausch Jun 13, 2024
7c23852
add push_to_url
ytausch May 28, 2024
78e00bf
clean up auto_tick:run signature
ytausch May 28, 2024
304c087
add feedstock attributes to FeedstockContext
ytausch May 29, 2024
6ba48ce
add create_pull_request to git backend
ytausch May 30, 2024
289d404
git add and git commit
ytausch May 30, 2024
44420f8
add --allow-empty to git
ytausch May 30, 2024
79a9f38
add comment_on_pull_request
ytausch May 31, 2024
4a06d34
add rev-parse HEAD command
ytausch May 31, 2024
44c7257
add diffed_files to git CLI
ytausch May 31, 2024
cf2fbfd
feat: GitCLi supports hiding tokens
ytausch Jun 11, 2024
5af5db3
add more tests for token hiding
ytausch Jun 11, 2024
bf61beb
add feedstock-related data to FeedstockContext
ytausch Jun 13, 2024
71a2a95
use get_bot_token instead of sensitive_env
ytausch Jun 13, 2024
56f2759
token hiding: no context manager, automatically if token is known
ytausch Jun 13, 2024
bf16491
hide tokens in Git Backends automatically, push to repository
ytausch Jun 15, 2024
a0d7070
FIX: forking should do nothing if already exists
ytausch Jun 15, 2024
f481f43
get_remote_url now redirects forks to upstream
ytausch Jun 15, 2024
38c2e7e
detect duplicate pull requests
ytausch Jun 15, 2024
7fad40e
refactor auto_tick.run using the new git backend
ytausch Jun 15, 2024
6dcb67e
refactor: solvability checks in separate method
ytausch Jun 16, 2024
7a5bfca
use temporary directory instead of managing the local feedstock dir m…
ytausch Jun 16, 2024
b183dab
dependency injection for GitPlatformBackend
ytausch Jun 24, 2024
fc33822
small fixes
ytausch Jun 25, 2024
de76a1e
add feedstock attributes to FeedstockContext
ytausch May 29, 2024
359379b
add create_pull_request to git backend
ytausch May 30, 2024
a6755e0
hide tokens in Git Backends automatically, push to repository
ytausch Jun 15, 2024
320f62d
use temporary directory instead of managing the local feedstock dir m…
ytausch Jun 16, 2024
ada4150
dependency injection for GitPlatformBackend
ytausch Jun 24, 2024
56f64ed
add single package support for auto_tick, clean up dry run option
ytausch Jun 24, 2024
32737f7
fixes and TODOs
ytausch Jun 24, 2024
e734fdc
fix rebasing, fix running locally
ytausch Jun 24, 2024
2ed9925
remove unused parameter, add helpful debug info
ytausch Jun 24, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
891 changes: 561 additions & 330 deletions conda_forge_tick/auto_tick.py

Large diffs are not rendered by default.

13 changes: 11 additions & 2 deletions conda_forge_tick/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -149,11 +149,20 @@ def update_upstream_versions(


@main.command(name="auto-tick")
@click.argument(
"package",
required=False,
)
@pass_context
def auto_tick(ctx: CliContext) -> None:
def auto_tick(ctx: CliContext, package: str | None) -> None:
"""
Run the main bot logic that runs all migrations, updates the graph accordingly, and opens the corresponding PRs.

If PACKAGE is given, only run the bot for that package, otherwise run the bot for all packages.
"""
from . import auto_tick

auto_tick.main(ctx)
auto_tick.main(ctx, package=package)


@main.command(name="make-status-report")
Expand Down
100 changes: 94 additions & 6 deletions conda_forge_tick/contexts.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,17 @@
from __future__ import annotations

import os
import tempfile
import typing
from collections.abc import Iterator
from contextlib import contextmanager
from dataclasses import dataclass
from pathlib import Path

from networkx import DiGraph

from conda_forge_tick.lazy_json_backends import load
from conda_forge_tick.utils import get_keys_default

if typing.TYPE_CHECKING:
from conda_forge_tick.migrators_types import AttrsTypedDict
Expand All @@ -24,13 +31,12 @@ class MigratorSessionContext:
graph: DiGraph = None
smithy_version: str = ""
pinning_version: str = ""
dry_run: bool = True


@dataclass
@dataclass(frozen=True)
class FeedstockContext:
feedstock_name: str
attrs: "AttrsTypedDict"
attrs: AttrsTypedDict
_default_branch: str = None

@property
Expand All @@ -40,6 +46,88 @@ def default_branch(self):
else:
return self._default_branch

@default_branch.setter
def default_branch(self, v):
self._default_branch = v
@property
def git_repo_owner(self) -> str:
return "conda-forge"

@property
def git_repo_name(self) -> str:
return f"{self.feedstock_name}-feedstock"

@property
def git_href(self) -> str:
"""
A link to the feedstocks GitHub repository.
"""
return f"https://github.com/{self.git_repo_owner}/{self.git_repo_name}"

@property
def automerge(self) -> bool | str:
"""
Get the automerge setting of the feedstock.

Note: A better solution to implement this is to use the NodeAttributes Pydantic
model for the attrs field. This will be done in the future.
"""
return get_keys_default(
self.attrs,
["conda-forge.yml", "bot", "automerge"],
{},
False,
)

@property
def check_solvable(self) -> bool:
"""
Get the check_solvable setting of the feedstock.

Note: A better solution to implement this is to use the NodeAttributes Pydantic
model for the attrs field. This will be done in the future.
"""
return get_keys_default(
self.attrs,
["conda-forge.yml", "bot", "check_solvable"],
{},
False,
)

@contextmanager
def reserve_clone_directory(self) -> Iterator[ClonedFeedstockContext]:
"""
Reserve a temporary directory for the feedstock repository that will be available within the context manager.
The returned context object will contain the path to the feedstock repository in local_clone_dir.
After the context manager exits, the temporary directory will be deleted.
"""
with tempfile.TemporaryDirectory() as tmpdir:
local_clone_dir = Path(tmpdir) / self.git_repo_name
local_clone_dir.mkdir()
yield ClonedFeedstockContext(
**self.__dict__,
local_clone_dir=local_clone_dir,
)


@dataclass(frozen=True, kw_only=True)
class ClonedFeedstockContext(FeedstockContext):
"""
A FeedstockContext object that has reserved a temporary directory for the feedstock repository.
"""

# Implementation Note: Keep this class frozen or there will be consistency issues if someone modifies
# a ClonedFeedstockContext object in place - it will not be reflected in the original FeedstockContext object.
local_clone_dir: Path

@property
def git_repo_owner(self) -> str:
return "conda-forge"

@property
def git_repo_name(self) -> str:
return f"{self.feedstock_name}-feedstock"

@property
def git_href(self) -> str:
"""
A link to the feedstocks GitHub repository.
"""
return f"https://github.com/{self.git_repo_owner}/{self.git_repo_name}"
48 changes: 31 additions & 17 deletions conda_forge_tick/executors.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,21 @@ def __exit__(self, *args, **kwargs):
pass


TRLOCK = TRLock()
PRLOCK = DummyLock()
DRLOCK = DummyLock()
GIT_LOCK_THREAD = TRLock()
GIT_LOCK_PROCESS = DummyLock()
GIT_LOCK_DASK = DummyLock()


@contextlib.contextmanager
def lock_git_operation():
"""
A context manager to lock git operations - it can be acquired once per thread, once per process,
and once per dask worker.
Note that this is a reentrant lock, so it can be acquired multiple times by the same thread/process/worker.
"""

with GIT_LOCK_THREAD, GIT_LOCK_PROCESS, GIT_LOCK_DASK:
yield


logger = logging.getLogger(__name__)
Expand All @@ -27,10 +39,12 @@ def __exit__(self, *args, **kwargs):
class DaskRLock(DaskLock):
"""A reentrant lock for dask that is always blocking and never times out."""

def acquire(self):
if not hasattr(self, "_rcount"):
self._rcount = 0
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._rcount = 0
self._rdata = None

def acquire(self, *args):
self._rcount += 1

if self._rcount == 1:
Expand All @@ -39,29 +53,29 @@ def acquire(self):
return self._rdata

def release(self):
if not hasattr(self, "_rcount") or self._rcount == 0:
if self._rcount == 0:
raise RuntimeError("Lock not acquired so cannot be released!")

self._rcount -= 1

if self._rcount == 0:
delattr(self, "_rdata")
self._rdata = None
return super().release()
else:
return None


def _init_process(lock):
global PRLOCK
PRLOCK = lock
global GIT_LOCK_PROCESS
GIT_LOCK_PROCESS = lock


def _init_dask(lock):
global DRLOCK
# it appears we have to construct the locak by name instead
global GIT_LOCK_DASK
# it appears we have to construct the lock by name instead
# of passing the object itself
# otherwise dask uses a regular lock
DRLOCK = DaskRLock(name=lock)
GIT_LOCK_DASK = DaskRLock(name=lock)


@contextlib.contextmanager
Expand All @@ -70,8 +84,8 @@ def executor(kind: str, max_workers: int, daemon=True) -> typing.Iterator[Execut

This allows us to easily use other executors as needed.
"""
global DRLOCK
global PRLOCK
global GIT_LOCK_DASK
global GIT_LOCK_PROCESS

if kind == "thread":
with ThreadPoolExecutor(max_workers=max_workers) as pool_t:
Expand All @@ -85,7 +99,7 @@ def executor(kind: str, max_workers: int, daemon=True) -> typing.Iterator[Execut
initargs=(lock,),
) as pool_p:
yield pool_p
PRLOCK = DummyLock()
GIT_LOCK_PROCESS = DummyLock()
elif kind in ["dask", "dask-process", "dask-thread"]:
import dask
import distributed
Expand All @@ -101,6 +115,6 @@ def executor(kind: str, max_workers: int, daemon=True) -> typing.Iterator[Execut
with distributed.Client(cluster) as client:
client.run(_init_dask, "cftick")
yield ClientExecutor(client)
DRLOCK = DummyLock()
GIT_LOCK_DASK = DummyLock()
else:
raise NotImplementedError("That kind is not implemented")
Loading
Loading