From fb3c36b0f8998dbc973b4e8a5080c8ae123b6f89 Mon Sep 17 00:00:00 2001 From: Tony Hu Date: Mon, 3 Jun 2024 14:01:49 -0700 Subject: [PATCH 1/6] update AMI selection logics to use latest PyTorch version --- setup.py | 2 +- src/autogluon/cloud/utils/ec2.py | 55 +++++++++++++++++--------------- 2 files changed, 31 insertions(+), 26 deletions(-) diff --git a/setup.py b/setup.py index 5a6a5181..76fb20f2 100644 --- a/setup.py +++ b/setup.py @@ -68,7 +68,7 @@ def default_setup_args(*, version): AUTOGLUON: [ "LICENSE", ], - 'autogluon.cloud': ['default_cluster_configs/*.yaml'], + "autogluon.cloud": ["default_cluster_configs/*.yaml"], }, classifiers=[ "Development Status :: 4 - Beta", diff --git a/src/autogluon/cloud/utils/ec2.py b/src/autogluon/cloud/utils/ec2.py index a0b7889b..a2791d11 100644 --- a/src/autogluon/cloud/utils/ec2.py +++ b/src/autogluon/cloud/utils/ec2.py @@ -1,5 +1,7 @@ +import dateutil.parser as parser import os from functools import partial +import re from typing import Any, Dict, List, Optional import boto3 @@ -61,43 +63,46 @@ def delete_key_pair(key_name: str, local_path: Optional[str]): os.remove(local_path) -def get_latest_ami(ami_name: str = "Deep Learning AMI GPU PyTorch*Ubuntu*") -> str: - """ - Get the latest ami id +def parse_pytorch_version(name): + """Extracts the PyTorch version from the AMI name.""" + match = re.search(r"PyTorch (\d+\.\d+(\.\d+)?)", name) + return tuple(map(int, match.group(1).split("."))) if match else (0, 0, 0) - Parameter - --------- - ami_name: str, default = Deep Learning AMI GPU PyTorch*Ubuntu* - Name of the ami. Could be regex. - Return - ------ - str, - The latest ami id of the ami name being specified - """ - from dateutil import parser +def latest_torch_image(images): + """Finds the newest image based on PyTorch version and then creation date.""" - def newest_image(list_of_images: List[Dict[str, Any]]): - latest = None + def image_key(image): + version = parse_pytorch_version(image["Name"]) + creation_date = parser.parse(image["CreationDate"]) + return version, creation_date - for image in list_of_images: - if not latest: - latest = image - continue + return max(images, key=image_key) - if parser.parse(image["CreationDate"]) > parser.parse(latest["CreationDate"]): - latest = image - return latest +def get_latest_ami(ami_name="Deep Learning AMI GPU PyTorch*Ubuntu*"): + """ + Get the latest AMI ID based on PyTorch version and creation date. - ec2 = boto3.client("ec2") + Parameters + ---------- + ami_name : str, default="Deep Learning AMI GPU PyTorch*Ubuntu*" + Name of the AMI. Could be a regex pattern. + Returns + ------- + str + The latest AMI ID of the specified AMI name. + """ + ec2 = boto3.client("ec2") filters = [ {"Name": "name", "Values": [ami_name]}, {"Name": "owner-alias", "Values": ["amazon"]}, {"Name": "architecture", "Values": ["x86_64"]}, {"Name": "state", "Values": ["available"]}, ] + response = ec2.describe_images(Owners=["amazon"], Filters=filters) - source_image = newest_image(response["Images"]) - return source_image["ImageId"] + latest_image = latest_torch_image(response["Images"]) + + return latest_image["ImageId"] From 8ecadc2b9cf4ffc8ab8db0acefede9f68fef8ca1 Mon Sep 17 00:00:00 2001 From: Tony Hu Date: Mon, 3 Jun 2024 14:38:40 -0700 Subject: [PATCH 2/6] reformat --- src/autogluon/cloud/utils/ec2.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/autogluon/cloud/utils/ec2.py b/src/autogluon/cloud/utils/ec2.py index a2791d11..1ba1f70a 100644 --- a/src/autogluon/cloud/utils/ec2.py +++ b/src/autogluon/cloud/utils/ec2.py @@ -63,13 +63,13 @@ def delete_key_pair(key_name: str, local_path: Optional[str]): os.remove(local_path) -def parse_pytorch_version(name): +def parse_pytorch_version(name: str): """Extracts the PyTorch version from the AMI name.""" match = re.search(r"PyTorch (\d+\.\d+(\.\d+)?)", name) return tuple(map(int, match.group(1).split("."))) if match else (0, 0, 0) -def latest_torch_image(images): +def latest_torch_image(images: List[Dict[str, Any]]): """Finds the newest image based on PyTorch version and then creation date.""" def image_key(image): @@ -80,7 +80,7 @@ def image_key(image): return max(images, key=image_key) -def get_latest_ami(ami_name="Deep Learning AMI GPU PyTorch*Ubuntu*"): +def get_latest_ami(ami_name: str = "Deep Learning AMI GPU PyTorch*Ubuntu*") -> str: """ Get the latest AMI ID based on PyTorch version and creation date. From 02197057ba34afdd1d3264c9c67f2a2b06586033 Mon Sep 17 00:00:00 2001 From: Tony Hu Date: Mon, 3 Jun 2024 14:40:50 -0700 Subject: [PATCH 3/6] organize import --- src/autogluon/cloud/utils/ec2.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/autogluon/cloud/utils/ec2.py b/src/autogluon/cloud/utils/ec2.py index 1ba1f70a..ab8888a5 100644 --- a/src/autogluon/cloud/utils/ec2.py +++ b/src/autogluon/cloud/utils/ec2.py @@ -1,10 +1,10 @@ -import dateutil.parser as parser import os -from functools import partial import re +from functools import partial from typing import Any, Dict, List, Optional import boto3 +import dateutil.parser as parser from botocore.exceptions import ClientError From a26c711e1ab13acd2d4a304370c56aa06b522861 Mon Sep 17 00:00:00 2001 From: Prateek M Desai Date: Tue, 4 Jun 2024 11:55:34 -0700 Subject: [PATCH 4/6] [Fix] Fix workflow checkout actions (#118) Co-authored-by: Ubuntu --- .github/workflows/continuous_integration.yml | 54 ++++++++++++++++---- 1 file changed, 45 insertions(+), 9 deletions(-) diff --git a/.github/workflows/continuous_integration.yml b/.github/workflows/continuous_integration.yml index 36cbdae7..9975083d 100644 --- a/.github/workflows/continuous_integration.yml +++ b/.github/workflows/continuous_integration.yml @@ -47,7 +47,11 @@ jobs: if: ${{ github.event_name == 'pull_request_target' }} uses: actions/checkout@v3 with: - ref: "refs/pull/${{ github.event.number }}/merge" + ref: ${{ github.sha }} + - name: Fetch base branch + run: git fetch origin ${{ github.event.pull_request.base.ref }} + - name: Merge base branch into PR branch + run: git merge origin/${{ github.event.pull_request.base.ref }} - uses: actions/setup-python@v4 with: python-version: '3.10' @@ -77,7 +81,11 @@ jobs: if: ${{ github.event_name == 'pull_request_target' }} uses: actions/checkout@v3 with: - ref: "refs/pull/${{ github.event.number }}/merge" + ref: ${{ github.sha }} + - name: Fetch base branch + run: git fetch origin ${{ github.event.pull_request.base.ref }} + - name: Merge base branch into PR branch + run: git merge origin/${{ github.event.pull_request.base.ref }} - uses: actions/setup-python@v4 with: python-version: '3.10' @@ -109,7 +117,11 @@ jobs: if: ${{ github.event_name == 'pull_request_target' }} uses: actions/checkout@v3 with: - ref: "refs/pull/${{ github.event.number }}/merge" + ref: ${{ github.sha }} + - name: Fetch base branch + run: git fetch origin ${{ github.event.pull_request.base.ref }} + - name: Merge base branch into PR branch + run: git merge origin/${{ github.event.pull_request.base.ref }} - uses: actions/setup-python@v4 with: python-version: '3.10' @@ -141,7 +153,11 @@ jobs: if: ${{ github.event_name == 'pull_request_target' }} uses: actions/checkout@v3 with: - ref: "refs/pull/${{ github.event.number }}/merge" + ref: ${{ github.sha }} + - name: Fetch base branch + run: git fetch origin ${{ github.event.pull_request.base.ref }} + - name: Merge base branch into PR branch + run: git merge origin/${{ github.event.pull_request.base.ref }} - uses: actions/setup-python@v4 with: python-version: '3.10' @@ -173,7 +189,11 @@ jobs: if: ${{ github.event_name == 'pull_request_target' }} uses: actions/checkout@v3 with: - ref: "refs/pull/${{ github.event.number }}/merge" + ref: ${{ github.sha }} + - name: Fetch base branch + run: git fetch origin ${{ github.event.pull_request.base.ref }} + - name: Merge base branch into PR branch + run: git merge origin/${{ github.event.pull_request.base.ref }} - uses: actions/setup-python@v4 with: python-version: '3.10' @@ -205,7 +225,11 @@ jobs: if: ${{ github.event_name == 'pull_request_target' }} uses: actions/checkout@v3 with: - ref: "refs/pull/${{ github.event.number }}/merge" + ref: ${{ github.sha }} + - name: Fetch base branch + run: git fetch origin ${{ github.event.pull_request.base.ref }} + - name: Merge base branch into PR branch + run: git merge origin/${{ github.event.pull_request.base.ref }} - uses: actions/setup-python@v4 with: python-version: '3.10' @@ -237,7 +261,11 @@ jobs: if: ${{ github.event_name == 'pull_request_target' }} uses: actions/checkout@v3 with: - ref: "refs/pull/${{ github.event.number }}/merge" + ref: ${{ github.sha }} + - name: Fetch base branch + run: git fetch origin ${{ github.event.pull_request.base.ref }} + - name: Merge base branch into PR branch + run: git merge origin/${{ github.event.pull_request.base.ref }} - uses: actions/setup-python@v4 with: python-version: '3.8' @@ -265,7 +293,11 @@ jobs: if: ${{ github.event_name == 'pull_request_target' }} uses: actions/checkout@v3 with: - ref: "refs/pull/${{ github.event.number }}/merge" + ref: ${{ github.sha }} + - name: Fetch base branch + run: git fetch origin ${{ github.event.pull_request.base.ref }} + - name: Merge base branch into PR branch + run: git merge origin/${{ github.event.pull_request.base.ref }} - uses: actions/setup-python@v4 with: python-version: '3.8' @@ -299,7 +331,11 @@ jobs: if: ${{ github.event_name == 'pull_request_target' }} uses: actions/checkout@v3 with: - ref: "refs/pull/${{ github.event.number }}/merge" + ref: ${{ github.sha }} + - name: Fetch base branch + run: git fetch origin ${{ github.event.pull_request.base.ref }} + - name: Merge base branch into PR branch + run: git merge origin/${{ github.event.pull_request.base.ref }} - name: Setup Env Vars uses: ./.github/actions/setup-env-vars - uses: actions/setup-python@v4 From 65159cec9d27ea3a01d09e5a5f54cc83c22921b5 Mon Sep 17 00:00:00 2001 From: Tony Hu Date: Thu, 6 Jun 2024 15:36:10 -0700 Subject: [PATCH 5/6] update AG test version to 1.1.0 to match dependencies used in torch 2.2 base images --- .github/workflows/continuous_integration.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/continuous_integration.yml b/.github/workflows/continuous_integration.yml index 9975083d..9d06fa82 100644 --- a/.github/workflows/continuous_integration.yml +++ b/.github/workflows/continuous_integration.yml @@ -106,7 +106,7 @@ jobs: strategy: fail-fast: false matrix: - AG_VERSION: ["source", "0.7.0"] + AG_VERSION: ["source", "1.1.0"] needs: cloud_lint_check runs-on: ubuntu-latest steps: @@ -142,7 +142,7 @@ jobs: strategy: fail-fast: false matrix: - AG_VERSION: ["source", "0.7.0"] + AG_VERSION: ["source", "1.1.0"] needs: cloud_lint_check runs-on: ubuntu-latest steps: @@ -178,7 +178,7 @@ jobs: strategy: fail-fast: false matrix: - AG_VERSION: ["source", "0.7.0"] + AG_VERSION: ["source", "1.1.0"] needs: cloud_lint_check runs-on: ubuntu-latest steps: @@ -214,7 +214,7 @@ jobs: strategy: fail-fast: false matrix: - AG_VERSION: ["source", "0.7.0"] + AG_VERSION: ["source", "1.1.0"] needs: cloud_lint_check runs-on: ubuntu-latest steps: @@ -250,7 +250,7 @@ jobs: strategy: fail-fast: false matrix: - AG_VERSION: ["source", "0.7.0"] + AG_VERSION: ["source", "1.1.0"] needs: cloud_lint_check runs-on: ubuntu-latest steps: @@ -268,7 +268,7 @@ jobs: run: git merge origin/${{ github.event.pull_request.base.ref }} - uses: actions/setup-python@v4 with: - python-version: '3.8' + python-version: '3.10' - name: Check if changes beside docs uses: dorny/paths-filter@v2 id: changes @@ -300,7 +300,7 @@ jobs: run: git merge origin/${{ github.event.pull_request.base.ref }} - uses: actions/setup-python@v4 with: - python-version: '3.8' + python-version: '3.10' - name: Check if changes beside docs uses: dorny/paths-filter@v2 id: changes @@ -340,7 +340,7 @@ jobs: uses: ./.github/actions/setup-env-vars - uses: actions/setup-python@v4 with: - python-version: '3.8' + python-version: '3.10' - uses: r-lib/actions/setup-pandoc@v2 - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v1 From 45572e1e80dbbf1eeea56696c41d1121a4dd644b Mon Sep 17 00:00:00 2001 From: Tony Hu Date: Thu, 6 Jun 2024 20:46:34 -0700 Subject: [PATCH 6/6] Revert "update AG test version to 1.1.0 to match dependencies used in torch 2.2 base images" This reverts commit f3e38af6871ba15f50c10572d37d60a4155bde83. --- .github/workflows/continuous_integration.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/continuous_integration.yml b/.github/workflows/continuous_integration.yml index 9d06fa82..9975083d 100644 --- a/.github/workflows/continuous_integration.yml +++ b/.github/workflows/continuous_integration.yml @@ -106,7 +106,7 @@ jobs: strategy: fail-fast: false matrix: - AG_VERSION: ["source", "1.1.0"] + AG_VERSION: ["source", "0.7.0"] needs: cloud_lint_check runs-on: ubuntu-latest steps: @@ -142,7 +142,7 @@ jobs: strategy: fail-fast: false matrix: - AG_VERSION: ["source", "1.1.0"] + AG_VERSION: ["source", "0.7.0"] needs: cloud_lint_check runs-on: ubuntu-latest steps: @@ -178,7 +178,7 @@ jobs: strategy: fail-fast: false matrix: - AG_VERSION: ["source", "1.1.0"] + AG_VERSION: ["source", "0.7.0"] needs: cloud_lint_check runs-on: ubuntu-latest steps: @@ -214,7 +214,7 @@ jobs: strategy: fail-fast: false matrix: - AG_VERSION: ["source", "1.1.0"] + AG_VERSION: ["source", "0.7.0"] needs: cloud_lint_check runs-on: ubuntu-latest steps: @@ -250,7 +250,7 @@ jobs: strategy: fail-fast: false matrix: - AG_VERSION: ["source", "1.1.0"] + AG_VERSION: ["source", "0.7.0"] needs: cloud_lint_check runs-on: ubuntu-latest steps: @@ -268,7 +268,7 @@ jobs: run: git merge origin/${{ github.event.pull_request.base.ref }} - uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version: '3.8' - name: Check if changes beside docs uses: dorny/paths-filter@v2 id: changes @@ -300,7 +300,7 @@ jobs: run: git merge origin/${{ github.event.pull_request.base.ref }} - uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version: '3.8' - name: Check if changes beside docs uses: dorny/paths-filter@v2 id: changes @@ -340,7 +340,7 @@ jobs: uses: ./.github/actions/setup-env-vars - uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version: '3.8' - uses: r-lib/actions/setup-pandoc@v2 - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v1