diff --git a/NEWS.md b/NEWS.md index 73d93ceee..55e615bba 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,9 @@ # reticulate (development version) +- Reticulate-managed `uv` can now resolve system-installed Pythons, + supporting platforms where pre-built binaries are unavailable, such as + musl-based Alpine Linux (#1751, #1752). + - `uv_run_tool()` gains an `exclude_newer` argument (#1748). - Internal changes to support R-devel (4.5) (#1747). diff --git a/R/py_require.R b/R/py_require.R index 75cefb48c..05738c71a 100644 --- a/R/py_require.R +++ b/R/py_require.R @@ -90,13 +90,19 @@ #' rm -r "$(uv tool dir)" #' ``` #' -#' If `uv` is not installed, `reticulate` will automatically download `uv` and -#' store it along with ephemeral environments and all downloaded artifacts in -#' the `tools::R_user_dir("reticulate", "cache")` directory. To clear this -#' cache, simply delete the directory: +#' If `uv` is not installed, `reticulate` will automatically download it and +#' store it along with ephemeral environments in the +#' `tools::R_user_dir("reticulate", "cache")` directory. Python binaries +#' downloaded by `uv` to create ephemeral virtual environments are stored in +#' `tools::R_user_dir("reticulate", "data")`. To clear this cache, simply delete +#' these directories: #' #' ```r +#' # delete uv and ephemeral virtual environments #' unlink(tools::R_user_dir("reticulate", "cache"), recursive = TRUE) +#' +#' # delete python binaries +#' unlink(tools::R_user_dir("reticulate", "data"), recursive = TRUE) #' ``` #' #' @param packages A character vector of Python packages to be available during @@ -554,24 +560,28 @@ uv_binary <- function(bootstrap_install = TRUE) { !is.na(ver) && ver >= required_version } - uv <- Sys.getenv("RETICULATE_UV", NA) - if (is_usable_uv(uv)) { - return(path.expand(uv)) - } + repeat { + uv <- Sys.getenv("RETICULATE_UV", NA) + if (!is.na(uv)) { + if (uv == "managed") break else return(path.expand(uv)) + } - uv <- getOption("reticulate.uv_binary") - if (is_usable_uv(uv)) { - return(path.expand(uv)) - } + uv <- getOption("reticulate.uv_binary") + if (!is.null(uv)) { + if (uv == "managed") break else return(path.expand(uv)) + } - uv <- as.character(Sys.which("uv")) - if (is_usable_uv(uv)) { - return(path.expand(uv)) - } + uv <- as.character(Sys.which("uv")) + if (is_usable_uv(uv)) { + return(path.expand(uv)) + } - uv <- path.expand("~/.local/bin/uv") - if (is_usable_uv(uv)) { - return(path.expand(uv)) + uv <- path.expand("~/.local/bin/uv") + if (is_usable_uv(uv)) { + return(path.expand(uv)) + } + + break } uv <- reticulate_cache_dir("uv", "bin", if (is_windows()) "uv.exe" else "uv") @@ -601,7 +611,7 @@ uv_binary <- function(bootstrap_install = TRUE) { return(NULL) # stop("Unable to download Python dependencies. Please install `uv` manually.") } - if(debug <- Sys.getenv("_RETICULATE_DEBUG_UV_") == "1") + if (debug_uv <- Sys.getenv("_RETICULATE_DEBUG_UV_") == "1") system2 <- system2t if (is_windows()) { @@ -610,8 +620,8 @@ uv_binary <- function(bootstrap_install = TRUE) { system2("powershell", c( "-ExecutionPolicy", "ByPass", "-c", sprintf("irm %s | iex", utils::shortPathName(install_uv))), - stdout = if (debug) "" else FALSE, - stderr = if (debug) "" else FALSE + stdout = if (debug_uv) "" else FALSE, + stderr = if (debug_uv) "" else FALSE ) }) @@ -619,7 +629,9 @@ uv_binary <- function(bootstrap_install = TRUE) { Sys.chmod(install_uv, mode = "0755") withr::with_envvar(c("UV_UNMANAGED_INSTALL" = dirname(uv)), { - system2(install_uv, c(if (!debug) "--quiet")) + system2(install_uv, + stdout = if (debug_uv) "" else FALSE, + stderr = if (debug_uv) "" else FALSE) }) } @@ -641,13 +653,17 @@ uv_get_or_create_env <- function(packages = py_reqs_get("packages"), if (isTRUE(attr(uv, "reticulate-managed", TRUE))) c( UV_CACHE_DIR = reticulate_cache_dir("uv", "cache"), - UV_PYTHON_INSTALL_DIR = reticulate_cache_dir("uv", "python") + UV_PYTHON_INSTALL_DIR = reticulate_data_dir("uv", "python") ) )) resolved_python_version <- resolve_python_version(constraints = python_version, uv = uv) + if (!length(resolved_python_version)) { + return() # error? + } + # capture args; maybe used in error message later call_args <- list( packages = packages, @@ -672,7 +688,7 @@ uv_get_or_create_env <- function(packages = py_reqs_get("packages"), uv_args <- c( "run", "--no-project", - "--python-preference", "only-managed", + # "--python-preference", "managed", python_version, exclude_newer, packages, @@ -762,15 +778,15 @@ uv_run_tool <- function(tool, if (isTRUE(attr(uv, "reticulate-managed", TRUE))) c( UV_CACHE_DIR = reticulate_cache_dir("uv", "cache"), - UV_PYTHON_INSTALL_DIR = reticulate_cache_dir("uv", "python") + UV_PYTHON_INSTALL_DIR = reticulate_data_dir("uv", "python") ) )) + python <- resolve_python_version(constraints = python_version, uv = uv) system2(uv, c( "tool", "run", "--isolated", - "--python-preference=only-managed", - "--python", resolve_python_version(constraints = python_version), + if (length(python)) c("--python", python), if (length(exclude_newer)) c("--exclude-newer", exclude_newer), if (length(from)) c("--from", maybe_shQuote(from)), if (length(with)) c(rbind("--with", maybe_shQuote(with))), @@ -795,27 +811,83 @@ is_reticulate_managed_uv <- function(uv = uv_binary(bootstrap_install = FALSE)) - +# return a dataframe of python options sorted by default reticulate preference uv_python_list <- function(uv = uv_binary()) { + + if (isTRUE(attr(uv, "reticulate-managed", TRUE))) + withr::local_envvar(c( + UV_CACHE_DIR = reticulate_cache_dir("uv", "cache"), + UV_PYTHON_INSTALL_DIR = reticulate_data_dir("uv", "python") + )) + + + if (Sys.getenv("_RETICULATE_DEBUG_UV_") == "1") + system2 <- system2t + x <- system2(uv, c("python list", - "--python-preference only-managed", - "--only-downloads", + "--all-versions", + # "--only-downloads", + # "--only-installed", + # "--python-preference only-managed", + # "--python-preference only-system", "--color never", "--output-format json" ), stdout = TRUE ) - x <- jsonlite::parse_json(x) - x <- unlist(lapply(x, `[[`, "version")) - - # to parse default `--output-format text` - # x <- grep("^cpython-", x, value = TRUE) - # x <- sub("^cpython-([^-]+)-.*", "\\1", x) + x <- jsonlite::parse_json(x, simplifyVector = TRUE) + + x <- x[is.na(x$symlink) , ] # ignore local filesystem symlinks + x <- x[x$variant == "default", ] # ignore "freethreaded" + x <- x[x$implementation == "cpython", ] # ignore "pypy" + + x$is_prerelease <- x$version != paste(x$version_parts$major, + x$version_parts$minor, + x$version_parts$patch, + sep = ".") + # x <- x[!x$is_prerelease, ] # ignore versions like "3.14.0a5" + + # x$path is local file path, NA if not downloaded yet. + # x$url is populated if not downloaded yet. + is_uv_downloadable <- !is.na(x$url) + is_uv_downloaded <- grepl( + "/uv/python/", + normalizePath(x$path, winslash = "/", mustWork = FALSE), + fixed = TRUE + ) + x$is_uv_python <- is_uv_downloadable | is_uv_downloaded + + # order first to easily resolve the latest preferred patch for each minor version + x <- x[order( + !x$is_prerelease, + x$is_uv_python, + x$version_parts$major, + x$version_parts$minor, + x$version_parts$patch, + decreasing = TRUE + ), ] + + # Order so the latest patch level for each minor version appears first, + # prioritizing two versions behind the latest minor release. + # Sort by the distance of the minor version from the preferred minor version, + # breaking ties in favor of older minor versions. + latest_minor <- max(x$version_parts$minor[!x$is_prerelease]) + preferred_minor <- latest_minor - 2L + x$is_latest_patch <- !duplicated(x$version_parts[c("major", "minor")]) + + x <- x[order( + !x$is_prerelease, + x$is_uv_python, + x$is_latest_patch, + -abs(x$version_parts$minor - preferred_minor) + + (-0.5 * (x$version_parts$minor > preferred_minor)), + x$version_parts$major == 3L, + x$version_parts$minor, + x$version_parts$patch, + decreasing = TRUE + ), ] - xv <- numeric_version(x, strict = FALSE) - latest_minor_patch <- !duplicated(xv[, -3L]) & !is.na(xv) - x <- x[order(latest_minor_patch, xv, decreasing = TRUE)] x } @@ -837,7 +909,7 @@ uv_exec <- function(args, ...) { if (isTRUE(attr(uv, "reticulate-managed", TRUE))) c( UV_CACHE_DIR = reticulate_cache_dir("uv", "cache"), - UV_PYTHON_INSTALL_DIR = reticulate_cache_dir("uv", "python") + UV_PYTHON_INSTALL_DIR = reticulate_data_dir("uv", "python") ) )) @@ -849,32 +921,23 @@ resolve_python_version <- function(constraints = NULL, uv = uv_binary()) { constraints <- trimws(unlist(strsplit(constraints, ",", fixed = TRUE))) constraints <- constraints[nzchar(constraints)] - if (length(constraints) == 0) { - return(as.character(uv_python_list()[3L])) # default - } - - # reflect a direct version specification like "3.11" or "3.14.0a3" - if (length(constraints) == 1 && !substr(constraints, 1, 1) %in% c("=", ">", "<", "!")) { - return(constraints) - } - # We perform custom constraint resolution to prefer slightly older Python releases. # uv tends to select the latest version, which often lack package support # See: https://devguide.python.org/versions/ # Get latest patch for each minor version - candidates <- uv_python_list(uv) # E.g., candidates might be: - # c("3.13.1", "3.12.8", "3.11.11", "3.10.16", "3.9.21", "3.8.20") + # c("3.13.1", "3.12.8", "3.11.11", "3.10.16", "3.9.21", "3.8.20" , ...) + all_candidates <- candidates <- uv_python_list(uv)$version - # Reorder candidates to prefer stable versions over bleeding edge - ord <- as.integer(c(3, 4, 2, 5, 1)) - ord <- union(ord, seq_along(candidates)) - candidates <- candidates[ord] + if (length(constraints) == 0L) { + return(as.character(candidates[1L])) # default + } - # Maybe add non-latest patch levels to candidates if they're explicitly - # mentioned in constraints - append(candidates) <- sub("^[<>=!]{1,2}", "", constraints) + # reflect a direct version specification like "3.14.0a3" + if (length(constraints) == 1L && constraints %in% candidates) { + return(constraints) + } candidates <- numeric_version(candidates, strict = FALSE) candidates <- candidates[!is.na(candidates)] @@ -889,6 +952,7 @@ resolve_python_version <- function(constraints = NULL, uv = uv_binary()) { msg <- paste0( 'Requested Python version constraints could not be satisfied.\n', ' constraints: "', constraints, '"\n', + 'Available Python versions found: ', paste0(all_candidates, collapse = ", "), "\n", 'Hint: Call `py_require(python_version = , action = "set")` to replace constraints.' ) stop(msg) diff --git a/R/utils.R b/R/utils.R index 3e60cbc7a..e4db98a0d 100644 --- a/R/utils.R +++ b/R/utils.R @@ -691,6 +691,15 @@ reticulate_cache_dir <- function(...) { } +reticulate_data_dir <- function(...) { + root <- if (getRversion() > "4.0") { + tools::R_user_dir("reticulate", "data") + } else { + path.expand(user_data_dir("r-reticulate", NULL)) + } + + normalizePath(file.path(root, ...), mustWork = FALSE) +} user_data_dir <- function(...) { expand_env_vars(rappdirs::user_data_dir(...)) diff --git a/R/virtualenv.R b/R/virtualenv.R index 94eabe623..e28bf0c8e 100644 --- a/R/virtualenv.R +++ b/R/virtualenv.R @@ -683,23 +683,26 @@ as_version_constraint_checkers <- function(version) { version <- unlist(strsplit(version, ",", fixed = TRUE)) # given string like ">=3.8", match two groups, on ">=" and "3.8" - pattern <- "^([><=!]{0,2})\\s*([0-9.]*)" + pattern <- "^([><=!]{0,2})\\s*([0-9.a-zA-Z]*)" op <- sub(pattern, "\\1", version) op[op == ""] <- "==" - ver <- sub(pattern, "\\2", version) - ver <- numeric_version(ver) + ver_string <- sub(pattern, "\\2", version) + ver <- numeric_version(ver_string, strict = FALSE) - .mapply(function(op, ver) { + .mapply(function(op, ver, ver_string) { op <- as.symbol(op) force(ver) + force(ver_string) # return a "checker" function that takes a vector of versions and returns # a logical vector of if the version satisfies the constraint. rlang::zap_srcref(eval(bquote(function(x) { op <- .(op) ver <- .(ver) + if (is.na(ver)) + return(.(ver_string) %in% as.character(x)) x <- numeric_version(x) # if the constraint version is missing minor or patch level, set # to 0, so we can match on all, equivalent to pip style syntax like '3.8.*' @@ -710,7 +713,7 @@ as_version_constraint_checkers <- function(version) { } op(x, ver) }))) - }, list(op, ver), NULL) + }, list(op, ver, ver_string), NULL) } diff --git a/man/py_require.Rd b/man/py_require.Rd index 446998ca3..24ecd43d1 100644 --- a/man/py_require.Rd +++ b/man/py_require.Rd @@ -141,12 +141,18 @@ rm -r "$(uv python dir)" rm -r "$(uv tool dir)" }\if{html}{\out{}} -If \code{uv} is not installed, \code{reticulate} will automatically download \code{uv} and -store it along with ephemeral environments and all downloaded artifacts in -the \code{tools::R_user_dir("reticulate", "cache")} directory. To clear this -cache, simply delete the directory: - -\if{html}{\out{
}}\preformatted{unlink(tools::R_user_dir("reticulate", "cache"), recursive = TRUE) +If \code{uv} is not installed, \code{reticulate} will automatically download it and +store it along with ephemeral environments in the +\code{tools::R_user_dir("reticulate", "cache")} directory. Python binaries +downloaded by \code{uv} to create ephemeral virtual environments are stored in +\code{tools::R_user_dir("reticulate", "data")}. To clear this cache, simply delete +these directories: + +\if{html}{\out{
}}\preformatted{# delete uv and ephemeral virtual environments +unlink(tools::R_user_dir("reticulate", "cache"), recursive = TRUE) + +# delete python binaries +unlink(tools::R_user_dir("reticulate", "data"), recursive = TRUE) }\if{html}{\out{
}} } } diff --git a/tests/testthat/_snaps/py_require.md b/tests/testthat/_snaps/py_require.md index e36d00d2a..056532c8d 100644 --- a/tests/testthat/_snaps/py_require.md +++ b/tests/testthat/_snaps/py_require.md @@ -1,15 +1,18 @@ # Error requesting conflicting package versions Code - r_session(attach_namespace = TRUE, { + r_session({ + library(reticulate) py_require("numpy<2") py_require("numpy>=2") - uv_get_or_create_env() + import("numpy") + py_config() }) Output + > library(reticulate) > py_require("numpy<2") > py_require("numpy>=2") - > uv_get_or_create_env() + > import("numpy") × No solution found when resolving `--with` dependencies: ╰─▶ Because you require numpy<2 and numpy>=2, we can conclude that your requirements are unsatisfiable. @@ -20,6 +23,8 @@ ------------------------------------------------------------------------- Error in uv_get_or_create_env() : Call `py_require()` to remove or replace conflicting requirements. + Error: Installation of Python not found, Python bindings not loaded. + See the Python "Order of Discovery" here: https://rstudio.github.io/reticulate/articles/versions.html#order-of-discovery. Execution halted ------- session end ------- success: false @@ -162,6 +167,7 @@ Error in resolve_python_version(constraints = python_version, uv = uv) : Requested Python version constraints could not be satisfied. constraints: ">=3.10,<3.10" + Available Python versions found: 3.11.xx .... Hint: Call `py_require(python_version = , action = "set")` to replace constraints. Calls: uv_get_or_create_env -> resolve_python_version Execution halted diff --git a/tests/testthat/helper-py-require.R b/tests/testthat/helper-py-require.R index 9c53f1da2..62f798352 100644 --- a/tests/testthat/helper-py-require.R +++ b/tests/testthat/helper-py-require.R @@ -3,7 +3,12 @@ test_py_require_reset <- function() { } r_session <- function(exprs, echo = TRUE, color = FALSE, - attach_namespace = FALSE) { + attach_namespace = FALSE, + force_managed_python = TRUE) { + withr::local_envvar(c( + "VIRTUAL_ENV" = NA, + "RETICULATE_PYTHON" = if (force_managed_python) "managed" else NA + )) exprs <- substitute(exprs) if (!is.call(exprs)) stop("exprs must be a call") diff --git a/tests/testthat/test-py_require.R b/tests/testthat/test-py_require.R index bc985a110..2bbe286c0 100644 --- a/tests/testthat/test-py_require.R +++ b/tests/testthat/test-py_require.R @@ -8,10 +8,12 @@ test_that("Error requesting conflicting package versions", { # in the snapshot try(uv_get_or_create_env()) - expect_snapshot(r_session(attach_namespace = TRUE, { + expect_snapshot(r_session({ + library(reticulate) py_require("numpy<2") py_require("numpy>=2") - uv_get_or_create_env() + import("numpy") + py_config() })) }) @@ -108,7 +110,11 @@ test_that("Error requesting conflicting Python versions", { py_require(python_version = ">=3.10") py_require(python_version = "<3.10") uv_get_or_create_env() - })) + }), transform = function(x) { + sub("^Available Python versions found: 3\\.11\\..*", + "Available Python versions found: 3.11.xx ....", + x) + }) }) test_that("Simple tests", {