Merge pull request #1752 from rstudio/uv-permit-system-python

Support resolving system python versions by `py_require()`

NEWS.md

@@ -1,5 +1,9 @@
 # reticulate (development version)
 
+- Reticulate-managed `uv` can now resolve system-installed Pythons, 
+  supporting platforms where pre-built binaries are unavailable, such as 
+  musl-based Alpine Linux (#1751, #1752).
+
 - `uv_run_tool()` gains an `exclude_newer` argument (#1748).
 
 - Internal changes to support R-devel (4.5) (#1747).

R/py_require.R

@@ -90,13 +90,19 @@
 #' rm -r "$(uv tool dir)"
 #' ```
 #'
-#' If `uv` is not installed, `reticulate` will automatically download `uv` and
-#' store it along with ephemeral environments and all downloaded artifacts in
-#' the `tools::R_user_dir("reticulate", "cache")` directory. To clear this
-#' cache, simply delete the directory:
+#' If `uv` is not installed, `reticulate` will automatically download it and
+#' store it along with ephemeral environments in the
+#' `tools::R_user_dir("reticulate", "cache")` directory. Python binaries
+#' downloaded by `uv` to create ephemeral virtual environments are stored in
+#' `tools::R_user_dir("reticulate", "data")`. To clear this cache, simply delete
+#' these directories:
 #'
 #' ```r
+#' # delete uv and ephemeral virtual environments
 #' unlink(tools::R_user_dir("reticulate", "cache"), recursive = TRUE)
+#'
+#' # delete python binaries
+#' unlink(tools::R_user_dir("reticulate", "data"), recursive = TRUE)
 #' ```
 #'
 #' @param packages A character vector of Python packages to be available during
@@ -554,24 +560,28 @@ uv_binary <- function(bootstrap_install = TRUE) {
     !is.na(ver) && ver >= required_version
   }
 
-  uv <- Sys.getenv("RETICULATE_UV", NA)
-  if (is_usable_uv(uv)) {
-    return(path.expand(uv))
-  }
+  repeat {
+    uv <- Sys.getenv("RETICULATE_UV", NA)
+    if (!is.na(uv)) {
+      if (uv == "managed") break else return(path.expand(uv))
+    }
 
-  uv <- getOption("reticulate.uv_binary")
-  if (is_usable_uv(uv)) {
-    return(path.expand(uv))
-  }
+    uv <- getOption("reticulate.uv_binary")
+    if (!is.null(uv)) {
+      if (uv == "managed") break else return(path.expand(uv))
+    }
 
-  uv <- as.character(Sys.which("uv"))
-  if (is_usable_uv(uv)) {
-    return(path.expand(uv))
-  }
+    uv <- as.character(Sys.which("uv"))
+    if (is_usable_uv(uv)) {
+      return(path.expand(uv))
+    }
 
-  uv <- path.expand("~/.local/bin/uv")
-  if (is_usable_uv(uv)) {
-    return(path.expand(uv))
+    uv <- path.expand("~/.local/bin/uv")
+    if (is_usable_uv(uv)) {
+      return(path.expand(uv))
+    }
+
+    break
   }
 
   uv <- reticulate_cache_dir("uv", "bin", if (is_windows()) "uv.exe" else "uv")
@@ -601,7 +611,7 @@ uv_binary <- function(bootstrap_install = TRUE) {
       return(NULL)
       # stop("Unable to download Python dependencies. Please install `uv` manually.")
     }
-    if(debug <- Sys.getenv("_RETICULATE_DEBUG_UV_") == "1")
+    if (debug_uv <- Sys.getenv("_RETICULATE_DEBUG_UV_") == "1")
       system2 <- system2t
 
     if (is_windows()) {
@@ -610,16 +620,18 @@ uv_binary <- function(bootstrap_install = TRUE) {
         system2("powershell", c(
           "-ExecutionPolicy", "ByPass", "-c",
           sprintf("irm %s | iex", utils::shortPathName(install_uv))),
-          stdout = if (debug) "" else FALSE,
-          stderr = if (debug) "" else FALSE
+          stdout = if (debug_uv) "" else FALSE,
+          stderr = if (debug_uv) "" else FALSE
         )
       })
 
     } else {
 
       Sys.chmod(install_uv, mode = "0755")
       withr::with_envvar(c("UV_UNMANAGED_INSTALL" = dirname(uv)), {
-        system2(install_uv, c(if (!debug) "--quiet"))
+        system2(install_uv,
+                stdout = if (debug_uv) "" else FALSE,
+                stderr = if (debug_uv) "" else FALSE)
       })
 
     }
@@ -641,13 +653,17 @@ uv_get_or_create_env <- function(packages = py_reqs_get("packages"),
     if (isTRUE(attr(uv, "reticulate-managed", TRUE)))
       c(
         UV_CACHE_DIR = reticulate_cache_dir("uv", "cache"),
-        UV_PYTHON_INSTALL_DIR = reticulate_cache_dir("uv", "python")
+        UV_PYTHON_INSTALL_DIR = reticulate_data_dir("uv", "python")
       )
   ))
 
   resolved_python_version <-
     resolve_python_version(constraints = python_version, uv = uv)
 
+  if (!length(resolved_python_version)) {
+    return() # error?
+  }
+
   # capture args; maybe used in error message later
   call_args <- list(
     packages = packages,
@@ -672,7 +688,7 @@ uv_get_or_create_env <- function(packages = py_reqs_get("packages"),
   uv_args <- c(
     "run",
     "--no-project",
-    "--python-preference", "only-managed",
+    # "--python-preference", "managed",
     python_version,
     exclude_newer,
     packages,
@@ -762,15 +778,15 @@ uv_run_tool <- function(tool,
     if (isTRUE(attr(uv, "reticulate-managed", TRUE)))
       c(
         UV_CACHE_DIR = reticulate_cache_dir("uv", "cache"),
-        UV_PYTHON_INSTALL_DIR = reticulate_cache_dir("uv", "python")
+        UV_PYTHON_INSTALL_DIR = reticulate_data_dir("uv", "python")
       )
   ))
+  python <- resolve_python_version(constraints = python_version, uv = uv)
   system2(uv, c(
     "tool",
     "run",
     "--isolated",
-    "--python-preference=only-managed",
-    "--python", resolve_python_version(constraints = python_version),
+    if (length(python)) c("--python", python),
     if (length(exclude_newer)) c("--exclude-newer", exclude_newer),
     if (length(from)) c("--from", maybe_shQuote(from)),
     if (length(with)) c(rbind("--with", maybe_shQuote(with))),
@@ -795,27 +811,83 @@ is_reticulate_managed_uv <- function(uv = uv_binary(bootstrap_install = FALSE))
 
 
 
-
+# return a dataframe of python options sorted by default reticulate preference
 uv_python_list <- function(uv = uv_binary()) {
+
+  if (isTRUE(attr(uv, "reticulate-managed", TRUE)))
+    withr::local_envvar(c(
+      UV_CACHE_DIR = reticulate_cache_dir("uv", "cache"),
+      UV_PYTHON_INSTALL_DIR = reticulate_data_dir("uv", "python")
+    ))
+
+
+  if (Sys.getenv("_RETICULATE_DEBUG_UV_") == "1")
+    system2 <- system2t
+
   x <- system2(uv, c("python list",
-    "--python-preference only-managed",
-    "--only-downloads",
+    "--all-versions",
+    # "--only-downloads",
+    # "--only-installed",
+    # "--python-preference only-managed",
+    # "--python-preference only-system",
     "--color never",
     "--output-format json"
     ),
     stdout = TRUE
   )
 
-  x <- jsonlite::parse_json(x)
-  x <- unlist(lapply(x, `[[`, "version"))
-
-  # to parse default `--output-format text`
-  # x <- grep("^cpython-", x, value = TRUE)
-  # x <- sub("^cpython-([^-]+)-.*", "\\1", x)
+  x <- jsonlite::parse_json(x, simplifyVector = TRUE)
+
+  x <- x[is.na(x$symlink) , ]             # ignore local filesystem symlinks
+  x <- x[x$variant == "default", ]        # ignore "freethreaded"
+  x <- x[x$implementation == "cpython", ] # ignore "pypy"
+
+  x$is_prerelease <- x$version != paste(x$version_parts$major,
+                                        x$version_parts$minor,
+                                        x$version_parts$patch,
+                                        sep = ".")
+  # x <- x[!x$is_prerelease, ] # ignore versions like "3.14.0a5"
+
+  # x$path is local file path, NA if not downloaded yet.
+  # x$url is populated if not downloaded yet.
+  is_uv_downloadable <- !is.na(x$url)
+  is_uv_downloaded <- grepl(
+    "/uv/python/",
+    normalizePath(x$path, winslash = "/", mustWork = FALSE),
+    fixed = TRUE
+  )
+  x$is_uv_python <- is_uv_downloadable | is_uv_downloaded
+
+  # order first to easily resolve the latest preferred patch for each minor version
+  x <- x[order(
+    !x$is_prerelease,
+    x$is_uv_python,
+    x$version_parts$major,
+    x$version_parts$minor,
+    x$version_parts$patch,
+    decreasing = TRUE
+  ), ]
+
+  # Order so the latest patch level for each minor version appears first,
+  # prioritizing two versions behind the latest minor release.
+  # Sort by the distance of the minor version from the preferred minor version,
+  # breaking ties in favor of older minor versions.
+  latest_minor <- max(x$version_parts$minor[!x$is_prerelease])
+  preferred_minor <- latest_minor - 2L
+  x$is_latest_patch <- !duplicated(x$version_parts[c("major", "minor")])
+
+  x <- x[order(
+    !x$is_prerelease,
+    x$is_uv_python,
+    x$is_latest_patch,
+    -abs(x$version_parts$minor - preferred_minor) +
+      (-0.5 * (x$version_parts$minor > preferred_minor)),
+    x$version_parts$major == 3L,
+    x$version_parts$minor,
+    x$version_parts$patch,
+    decreasing = TRUE
+  ), ]
 
-  xv <- numeric_version(x, strict = FALSE)
-  latest_minor_patch <- !duplicated(xv[, -3L]) & !is.na(xv)
-  x <- x[order(latest_minor_patch, xv, decreasing = TRUE)]
   x
 }
 
@@ -837,7 +909,7 @@ uv_exec <- function(args, ...) {
     if (isTRUE(attr(uv, "reticulate-managed", TRUE)))
       c(
         UV_CACHE_DIR = reticulate_cache_dir("uv", "cache"),
-        UV_PYTHON_INSTALL_DIR = reticulate_cache_dir("uv", "python")
+        UV_PYTHON_INSTALL_DIR = reticulate_data_dir("uv", "python")
       )
   ))
 
@@ -849,32 +921,23 @@ resolve_python_version <- function(constraints = NULL, uv = uv_binary()) {
   constraints <- trimws(unlist(strsplit(constraints, ",", fixed = TRUE)))
   constraints <- constraints[nzchar(constraints)]
 
-  if (length(constraints) == 0) {
-    return(as.character(uv_python_list()[3L])) # default
-  }
-
-  # reflect a direct version specification like "3.11" or "3.14.0a3"
-  if (length(constraints) == 1 && !substr(constraints, 1, 1) %in% c("=", ">", "<", "!")) {
-    return(constraints)
-  }
-
   # We perform custom constraint resolution to prefer slightly older Python releases.
   # uv tends to select the latest version, which often lack package support
   # See: https://devguide.python.org/versions/
 
   # Get latest patch for each minor version
-  candidates <- uv_python_list(uv)
   # E.g., candidates might be:
-  #  c("3.13.1", "3.12.8", "3.11.11", "3.10.16", "3.9.21", "3.8.20")
+  #  c("3.13.1", "3.12.8", "3.11.11", "3.10.16", "3.9.21", "3.8.20" , ...)
+  all_candidates <- candidates <- uv_python_list(uv)$version
 
-  # Reorder candidates to prefer stable versions over bleeding edge
-  ord <- as.integer(c(3, 4, 2, 5, 1))
-  ord <- union(ord, seq_along(candidates))
-  candidates <- candidates[ord]
+  if (length(constraints) == 0L) {
+    return(as.character(candidates[1L])) # default
+  }
 
-  # Maybe add non-latest patch levels to candidates if they're explicitly
-  # mentioned in constraints
-  append(candidates) <- sub("^[<>=!]{1,2}", "", constraints)
+  # reflect a direct version specification like "3.14.0a3"
+  if (length(constraints) == 1L && constraints %in% candidates) {
+    return(constraints)
+  }
 
   candidates <- numeric_version(candidates, strict = FALSE)
   candidates <- candidates[!is.na(candidates)]
@@ -889,6 +952,7 @@ resolve_python_version <- function(constraints = NULL, uv = uv_binary()) {
     msg <- paste0(
       'Requested Python version constraints could not be satisfied.\n',
       '  constraints: "', constraints, '"\n',
+      'Available Python versions found: ', paste0(all_candidates, collapse = ", "), "\n",
       'Hint: Call `py_require(python_version = <string>, action = "set")` to replace constraints.'
     )
     stop(msg)

R/utils.R

@@ -691,6 +691,15 @@ reticulate_cache_dir <- function(...) {
 }
 
 
+reticulate_data_dir <- function(...) {
+  root <- if (getRversion() > "4.0") {
+    tools::R_user_dir("reticulate", "data")
+  } else {
+    path.expand(user_data_dir("r-reticulate", NULL))
+  }
+
+  normalizePath(file.path(root, ...), mustWork = FALSE)
+}
 
 user_data_dir <- function(...) {
   expand_env_vars(rappdirs::user_data_dir(...))

R/virtualenv.R

@@ -683,23 +683,26 @@ as_version_constraint_checkers <- function(version) {
   version <- unlist(strsplit(version, ",", fixed = TRUE))
 
   # given string like ">=3.8", match two groups, on ">=" and "3.8"
-  pattern <- "^([><=!]{0,2})\\s*([0-9.]*)"
+  pattern <- "^([><=!]{0,2})\\s*([0-9.a-zA-Z]*)"
 
   op <- sub(pattern, "\\1",  version)
   op[op == ""] <- "=="
 
-  ver <- sub(pattern, "\\2",  version)
-  ver <- numeric_version(ver)
+  ver_string <- sub(pattern, "\\2",  version)
+  ver <- numeric_version(ver_string, strict = FALSE)
 
-  .mapply(function(op, ver) {
+  .mapply(function(op, ver, ver_string) {
     op <- as.symbol(op)
     force(ver)
+    force(ver_string)
 
     # return a "checker" function that takes a vector of versions and returns
     # a logical vector of if the version satisfies the constraint.
     rlang::zap_srcref(eval(bquote(function(x) {
       op <- .(op)
       ver <- .(ver)
+      if (is.na(ver))
+        return(.(ver_string) %in% as.character(x))
       x <- numeric_version(x)
       # if the constraint version is missing minor or patch level, set
       # to 0, so we can match on all, equivalent to pip style syntax like '3.8.*'
@@ -710,7 +713,7 @@ as_version_constraint_checkers <- function(version) {
         }
       op(x, ver)
     })))
-  }, list(op, ver), NULL)
+  }, list(op, ver, ver_string), NULL)
 }