build-remote

#!/usr/bin/python3

"""
#
# This script triggers a build of WSL images, after optionally
# building a rootfs in launchpad
#
"""

# Copyright: 2021, Canonical Ltd.

# License: GPL-3
#  This program is free software: you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation, either version 3 of the License.
#  .
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY;without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
#  GNU General Public License for more details.
#  .
#  You should have received a copy of the GNU General Public License
#  along with this program. If not, see <https://www.gnu.org/licenses/>.
#  .
#  On Debian systems, the complete text of the GNU General
#  Public License version 3 can be found in "/usr/share/common-licenses/GPL-3".

import argparse
import logging
import os
import requests
import sys
import time
from lazr.restfulclient.errors import NotFound
from launchpadlib.launchpad import Launchpad

ACTION_API_BASE_URL = "https://api.github.com/repos/ubuntu/wsl/actions"

DISTRIBUTION = "Ubuntu"
SUPPORTED_ARCHES = ["amd64", "arm64"]


def set_logging(debugmode=False):
    """Initialize logging"""
    logging.basicConfig(
        level=logging.DEBUG if debugmode else logging.INFO,
        format="%(asctime)s %(levelname)s %(message)s")
    logging.debug("Debug mode enabled")


def strip_ppa(string):
    """Helper function for Python pre-3.9 to strip the ppa: suffix"""
    if string.startswith("ppa:"):
        return string[4:]
    return string


def lpinit(anonymous=False):
    """Initialize the connection to LP"""
    if anonymous:
        return Launchpad.login_anonymously('build-wsl', 'production', version='devel')
    else:
        return Launchpad.login_with('build-wsl', 'production', version='devel')


def appID_to_releases(lpconn):
    """returns the list of releases mapped by WSL_ID"""
    ubuntu = lpconn.distributions[DISTRIBUTION.lower()]

    r = {}

    latest_lts = ""
    for serie in ubuntu.series:
        if serie.status == "Active Development":
            r["UbuntuPreview"] = serie.name

        is_lts = ((int(serie.version.split('.')[0]) % 2 == 0) and serie.version.endswith('.04'))
        if is_lts:
            r["Ubuntu%sLTS" % serie.version] = serie.name
            latest_lts = serie.name

    r["Ubuntu"] = latest_lts
    return r


def rootfs_build(lpconn, release_name, packages_ppas, livecd_rootfs_ppa):
    """Request a launchpad rootfs build and create any livefs if not already present"""
    release = lpconn.load("/ubuntu/%s" % release_name)
    owner = lpconn.people["ubuntu-wsl-dev"]

    # Get or create livefs
    try:
        livefs = lpconn.livefses.getByName(distro_series=release, name="wsl", owner=owner)
        logging.debug("Found existing livefs for %s", release_name)
    except NotFound:
        logging.debug("Create a new livefs for %s", release_name)
        metadata = {"project": "ubuntu-cpc", "image_targets": ["wsl"]}
        livefs = lpconn.livefses.new(distro_series=release, metadata=metadata, name="wsl", owner=owner)

    logging.debug("Requesting rootfs build for release:\t%s, using:\nppas:\t%s\nlivecd-rootfs:\t%s\n", release_name, packages_ppas, livecd_rootfs_ppa)

    archive = lpconn.load("ubuntu/+archive/primary")
    if livecd_rootfs_ppa:
        livecd_rootfs_ppa_team, livecd_rootfs_ppa_name = livecd_rootfs_ppa.split("/")
        archive = lpconn.load("~%s/+archive/%s" % (livecd_rootfs_ppa_team, livecd_rootfs_ppa_name))

    builds = []
    for arch in SUPPORTED_ARCHES:
        logging.debug("Building on %s", arch)
        distro_arch_series = lpconn.load("/ubuntu/%s/%s" % (release_name, arch))
        builds.append(livefs.requestBuild(archive=archive, distro_arch_series=distro_arch_series,
                               pocket="Updates", metadata_override={"extra_ppas": packages_ppas}))

    return builds


def main():
    """Main routine"""
    args = _parse_arguments()
    set_logging(args.debug)
    logging.debug("arguments: %s", args)

    github_token = os.environ.get("WSL_GITHUB_TOKEN")
    if args.wsl_github_token:
        github_token = args.wsl_github_token
    if not github_token:
        logging.error("GitHub token not found in environment or passed with -t")
        sys.exit(1)

    # Use proxies for the request callse, if needed
    proxies = {}
    https_proxy = os.environ.get("HTTPS_PROXY")
    if args.https_proxy:
        https_proxy = args.https_proxy
    if https_proxy:
        proxies["https"] = https_proxy

    lpconn = lpinit(args.no_lp_login)

    releases = appID_to_releases(lpconn)
    selected_appID = None
    selected_release = None
    for appID in releases:
        if appID.lower() == args.appID.lower():
            selected_appID = appID
            selected_release = releases[appID]
            break

    if not selected_appID:
        logging.error("Unable to find a WSL release matching %s. Availables: %s", args.appID, releases)
        sys.exit(1)

    if args.ppas and args.rootfses:
        logging.error("--ppa and --with-rootfs are mutually exclusive")
        sys.exit(1)
    if not args.ppas and not args.rootfses:
        logging.error("one of --ppa (to build a rootfs first) or --with-rootfs (to download existing rootfs) is needed")
        sys.exit(1)

    # Sanitize input by accepting "ppa:" prefix
    ppas = []
    for ppa in args.ppas:
        ppas.append(strip_ppa(ppa))
    livecd_rootfs = strip_ppa(args.livecd_rootfs)

    rootfses = []

    if args.ppas:
        # ppas don’t have signature files
        args.rootfs_no_signature_check = True
        # Start building
        builds = rootfs_build(lpconn, selected_release, ppas, livecd_rootfs)
        #builds = [
        #    lpconn.load("/~ubuntu-wsl-dev/+livefs/ubuntu/impish/wsl/+build/287484"),
        #    lpconn.load("/~ubuntu-wsl-dev/+livefs/ubuntu/impish/wsl/+build/287485")
        #]

        logging.info("Rootfses are building:")
        for build in builds:
            logging.info("- %s", build.web_link)

        # Wait for builds
        still_building = True
        while still_building:
            time.sleep(30)
            logging.debug("checking if rootfs build status")
            still_building = False
            i = 0
            for build in builds:
                build = lpconn.load(build.self_link)
                builds[i] = build
                i += 1
                if build.buildstate not in ["Successfully built", "Failed to build",
                                        "Chroot problem", "Failed to upload", "Cancelled build"]:
                    still_building = True
                else:
                    logging.debug("%s is built with status: %s", build.web_link, build.buildstate)

        # Build artifacts list
        shouldExit = False
        for build in builds:
            if build.buildstate != "Successfully built":
                logging.error("%s is built with status: %s", build.web_link, build.buildstate)
                shouldExit = True
            arch = os.path.basename(build.distro_arch_series_link)
            for f in build.getFileUrls():
                if not f.endswith(".rootfs.tar.gz"):
                    continue
                rootfses.append("%s::%s" % (f, arch))
        if shouldExit:
            sys.exit(1)

    # Manually provided rootfs
    shouldExit = False
    for rootfs in args.rootfses:
        if not "::" in rootfs:
            for arch in SUPPORTED_ARCHES:
                if arch in rootfs:
                    rootfs = "%s::%s" % (rootfs, arch)
        if not "::" in rootfs:
            logging.error("Rootfs %s is not in the form <url>::<arch>", rootfs)
            shouldExit = True
        rootfses.append(rootfs)
    if shouldExit:
        sys.exit(1)

    # Prepare parameters for github builds
    headers = {
        "Accept": "application/vnd.github.v3+json",
        "Authorization": "token %s" % github_token,
        "Content-Type": "application/json;charset=utf-8",
        }

    rootfseschecksum = "yes"
    if args.rootfs_no_signature_check:
        rootfseschecksum = "no"
    upload = "no"
    if args.upload:
        upload = "yes"
    d = {
        "ref":"main", # WSL repo branch
        "inputs": {
            "appID": selected_appID,
            "rootfses": ",".join(rootfses),
            "rootfseschecksum": rootfseschecksum,
            "upload": upload,
        },
    }

    # Request github appbundle build
    logging.info("Requesting github appbundle build %s" % selected_appID)
    r = requests.post("%s/workflows/build-wsl.yaml/dispatches" % ACTION_API_BASE_URL, json=d, headers=headers, proxies=proxies)
    if r.status_code != requests.codes.ok and r.status_code != requests.codes.no_content:
        logging.error("Failed to start building the appxbundle on github: %s" % r.text)
        sys.exit(1)
    # Wait for the build to be listed
    time.sleep(1)
    r = requests.get("%s/runs" % ACTION_API_BASE_URL, headers=headers, proxies=proxies)
    if r.status_code != requests.codes.ok and r.status_code != requests.codes.no_content:
        logging.error("Failed to lookup latest appxbundle github build: %s" % r.text)
        sys.exit(1)
    run_id = r.json()["workflow_runs"][0]["id"]
    logging.info("GitHub appbundle build %s started at %s", selected_appID, r.json()["workflow_runs"][0]["html_url"])

    # Wait for builds
    while True:
        time.sleep(30)
        logging.debug("checking appxbundle build status")
        r = requests.get("%s/runs/%s" % (ACTION_API_BASE_URL, run_id), headers=headers, proxies=proxies)
        if r.status_code != requests.codes.ok and r.status_code != requests.codes.no_content:
            logging.error("Failed to lookup appxbundle build status: %s" % r.text)
            continue
        status = r.json()["status"]
        if status != "completed":
            continue
        break

    url = r.json()["html_url"]
    if r.json()["conclusion"] != "success":
        logging.error("Failed to build the appxbundle: %s (%s)", r.json()["conclusion"], url)
        sys.exit(1)

    print("Build ended successfully, you can download artifacts at %s" % url)

    return 0


# build-remote
#           <WSL_ID>
#                  -> with default current rootfses for WSL_ID, print appbundle url
#             --with-upload -> force upload to the store
#             --with-rootfs rootfs_url::arch (can have multiple).
#                warning: should have amd64 and arm64 for now, maybe confirmation?
#             --rootfs-ppa <ppa> (can have multiple)
#             --livecd-rootfs-ppa <ppa> (one)

def _parse_arguments():
    """Parse command-line args, returning an argparse dict."""

    # TODO: option for trigger a rebuild of official rootfses
    parser = argparse.ArgumentParser(description="Build wsl app packages. If --ppa is specified, the rootfs will be rebuilt first")
    parser.add_argument("appID", help="appID for appbundle")
    parser.add_argument("-d", "--debug", action="store_true", default=False,
                        help="enable debug mode")
    parser.add_argument("-r", "--with-rootfs", dest="rootfses", action="append", default=[],
                        help="rootfs to use for building the appbundle. Format is url::arch. Arch is deducted from url if available")
    parser.add_argument("--no-signature-check", dest="rootfs_no_signature_check", action="store_true", default=False,
                        help="disable rootfs SHA256SUMS download when specifying existing rootfs build. PPA builds don’t have this signature.")
    parser.add_argument("-p", "--ppa", dest="ppas", action="append", default=[],
                        help="ppas with rootfs contents to fetch from. Syntax is user/ppa")
    parser.add_argument("-l", "--livecd-rootfs-ppa", dest="livecd_rootfs", default="",
                        help="ppa to use livecd-rootfs from. Syntax is user/ppa")
    parser.add_argument("-u", "--with-upload", action="store_true", dest="upload", default=False,
                        help="upload the appbundle to the Microsoft Store")
    parser.add_argument("-t", "--github-token", dest="wsl_github_token",
                        help="Pass a github token with the repo:public_repo permission to run github worklows. Override WSL_GITHUB_TOKEN env variable")
    parser.add_argument("--no-lp-login", action="store_true", default=False,
                        help="login anonymously to Launchpad instead of authenticating")
    parser.add_argument("--https-proxy",
                        help="HTTPS proxy url, if applicable")

    return parser.parse_args()


if __name__ == "__main__":
    sys.exit(main())