From 2f85b882fcb42bb18190f664692409855f1b426d Mon Sep 17 00:00:00 2001
From: Andrew DiBiasio <mbox_user-github@yahoo.com>
Date: Wed, 7 Aug 2024 08:18:45 +0000
Subject: [PATCH 01/10] added aggregation logic for learning_curves artifacts

---
 .../bench/eval/aggregate/aggregate.py         |  63 +++++++++++
 src/autogluon/bench/eval/aggregate/results.py |  28 -----
 .../eval/benchmark_context/output_context.py  |  34 ++++++
 .../benchmark_context/output_suite_context.py |  18 +++
 .../bench/eval/benchmark_context/utils.py     | 106 +++++++++++++++++-
 .../eval/scripts/aggregate_amlb_results.py    |  51 +++++++--
 6 files changed, 263 insertions(+), 37 deletions(-)
 create mode 100644 src/autogluon/bench/eval/aggregate/aggregate.py
 delete mode 100644 src/autogluon/bench/eval/aggregate/results.py

diff --git a/src/autogluon/bench/eval/aggregate/aggregate.py b/src/autogluon/bench/eval/aggregate/aggregate.py
new file mode 100644
index 00000000..05fbfca8
--- /dev/null
+++ b/src/autogluon/bench/eval/aggregate/aggregate.py
@@ -0,0 +1,63 @@
+import logging
+
+from autogluon.bench.eval.benchmark_context.output_suite_context import OutputSuiteContext
+from autogluon.common.savers import save_pd
+
+logger = logging.getLogger(__name__)
+
+
+def aggregate(
+        s3_bucket: str, 
+        module: str, 
+        benchmark_name: str, 
+        artifact: str | None = "results", 
+        constraint: str | None = None,
+        include_infer_speed: bool = False ,
+        mode: str = "ray"
+    ) -> None:
+    """
+    Aggregates objects across an agbenchmark. Functionality depends on artifact specified:
+    
+    Params:
+    -------
+    s3_bucket: str
+        Name of the relevant s3_bucket
+    module: str
+        The name of the relevant autogluon module: can be one of ['tabular', 'timeseries', 'multimodal']
+    benchmark_name: str
+        The name of the relevant benchmark that was run
+    artifact: str
+        The desired artifact to be aggregatedL can be one of ['results', 'learning_curves']
+    constraint: str
+        Name of constraint used in benchmark
+    include_infer_speed: bool
+        Include inference speed in aggregation.
+    mode: str
+        Can be one of ['seq', 'ray'].
+        If seq, runs sequentially.
+        If ray, utilizes parallelization.
+    """
+    result_path = f"{module}/{benchmark_name}"
+    path_prefix = f"s3://{s3_bucket}/{result_path}/"
+    contains = f".{constraint}." if constraint else None
+
+    output_suite_context = OutputSuiteContext(
+        path=path_prefix,
+        contains=contains,
+        include_infer_speed=include_infer_speed,
+        mode=mode,
+    )
+
+    if artifact == "learning_curves":
+        save_path = f"s3://{s3_bucket}/aggregated/{result_path}/{artifact}"
+        artifact_path = output_suite_context.aggregate_learning_curves(save_path=save_path)
+    else:
+        aggregated_results_name = f"results_automlbenchmark_{constraint}_{benchmark_name}.csv"
+        results_df = output_suite_context.aggregate_results()
+
+        print(results_df)
+
+        artifact_path = f"s3://{s3_bucket}/aggregated/{result_path}/{aggregated_results_name}"
+        save_pd.save(path=artifact_path, df=results_df)
+
+    logger.info(f"Aggregated output saved to {artifact_path}!")
\ No newline at end of file
diff --git a/src/autogluon/bench/eval/aggregate/results.py b/src/autogluon/bench/eval/aggregate/results.py
deleted file mode 100644
index 2c14da64..00000000
--- a/src/autogluon/bench/eval/aggregate/results.py
+++ /dev/null
@@ -1,28 +0,0 @@
-import logging
-
-from autogluon.bench.eval.benchmark_context.output_suite_context import OutputSuiteContext
-from autogluon.common.savers import save_pd
-
-logger = logging.getLogger(__name__)
-
-
-def aggregate_results(s3_bucket, s3_prefix, version_name, constraint, include_infer_speed=False, mode="ray"):
-    contains = f".{constraint}." if constraint else None
-    result_path = f"{s3_prefix}{version_name}/"
-    path_prefix = f"s3://{s3_bucket}/{result_path}"
-
-    aggregated_results_name = f"results_automlbenchmark_{constraint}_{version_name}.csv"
-
-    output_suite_context = OutputSuiteContext(
-        path=path_prefix,
-        contains=contains,
-        include_infer_speed=include_infer_speed,
-        mode=mode,
-    )
-    results_df = output_suite_context.aggregate_results()
-
-    print(results_df)
-
-    save_path = f"s3://{s3_bucket}/aggregated/{result_path}{aggregated_results_name}"
-    save_pd.save(path=save_path, df=results_df)
-    logger.info(f"Aggregated results saved to {save_path}!")
diff --git a/src/autogluon/bench/eval/benchmark_context/output_context.py b/src/autogluon/bench/eval/benchmark_context/output_context.py
index 7a88587a..50f1b7ca 100644
--- a/src/autogluon/bench/eval/benchmark_context/output_context.py
+++ b/src/autogluon/bench/eval/benchmark_context/output_context.py
@@ -3,12 +3,16 @@
 from typing import Optional, Set, Union
 
 import boto3
+import logging
 import numpy as np
 import pandas as pd
 
 from autogluon.common.loaders import load_pd, load_pkl
 from autogluon.common.utils.s3_utils import s3_path_to_bucket_prefix
 
+from autogluon.bench.eval.benchmark_context.utils import copy_s3_object, get_s3_paths
+
+logger = logging.getLogger(__name__)
 
 class OutputContext:
     def __init__(self, path):
@@ -49,6 +53,10 @@ def path_results(self):
     def path_leaderboard(self):
         return self.path + "leaderboard.csv"
 
+    @property
+    def path_learning_curves(self):
+        return self.path + "learning_curves/"
+
     @property
     def path_model_failures(self):
         return self.path + "model_failures.csv"
@@ -98,6 +106,32 @@ def load_results(
 
     def load_leaderboard(self) -> pd.DataFrame:
         return load_pd.load(self.path_leaderboard)
+    
+    def load_learning_curves(self, save_path: str, suffix: str = "learning_curves.json") -> bool:
+        path = self.path_learning_curves
+        all_curves = get_s3_paths(path_prefix=path, suffix=suffix)
+
+        ok = True
+        for origin_path in all_curves:
+            dataset, fold = self.get_dataset_fold(origin_path)
+            destination_path = f"{save_path}/{dataset}/{fold}.json"
+            res = copy_s3_object(origin_path=origin_path, destination_path=destination_path)
+            ok &= res
+            if not res:
+                logger.log(f"Learning Curve artifact at {origin_path} could not be copied to {destination_path}")
+
+        return ok
+
+    def get_dataset_fold(self, path_str: str) -> tuple[str, str]:
+        parts = path_str.rstrip('/').split('/')
+        
+        if len(parts) < 3:
+            raise ValueError(f"Improper folder dimensions at {path_str}. Expected following path structure: .../dataset/fold/learning_curves.json")
+
+        # path pattern: .../dataset/fold/learning_curves.json
+        dataset, fold, _ = parts[-3:]
+
+        return dataset, fold
 
     def load_model_failures(self) -> pd.DataFrame:
         """Load and return the raw model failures file"""
diff --git a/src/autogluon/bench/eval/benchmark_context/output_suite_context.py b/src/autogluon/bench/eval/benchmark_context/output_suite_context.py
index f05b8824..b733ec16 100644
--- a/src/autogluon/bench/eval/benchmark_context/output_suite_context.py
+++ b/src/autogluon/bench/eval/benchmark_context/output_suite_context.py
@@ -2,6 +2,7 @@
 
 import copy
 import os
+import logging
 from typing import List, Optional, Set
 
 import pandas as pd
@@ -12,6 +13,8 @@
 from autogluon.bench.eval.benchmark_context.utils import get_s3_paths
 from autogluon.common.loaders import load_pd
 
+logger = logging.getLogger(__name__)
+
 DEFAULT_COLUMNS_TO_KEEP = [
     "id",
     "task",
@@ -175,6 +178,13 @@ def get_zeroshot_metadata_size_bytes(self, allow_exception=False) -> List[int]:
             allow_exception=allow_exception,
         )
 
+    def load_learning_curves(self, save_path: str) -> list[bool]:
+        return self._loop_func(
+            func=OutputContext.load_learning_curves,
+            input_list=self.output_contexts,
+            kwargs=dict(save_path=save_path),
+        )
+
     def filter_failures(self):
         amlb_info_list = self.get_amlb_info()
         output_contexts_valid = []
@@ -206,6 +216,14 @@ def aggregate_results(self, results_list: List[pd.DataFrame] | None = None) -> p
         results_df = pd.concat(results_list, ignore_index=True)
         return results_df
 
+    def aggregate_learning_curves(self, save_path: str) -> str:
+        outcomes = self.load_learning_curves(save_path=save_path)
+        failures = outcomes.count(False)
+        if failures > 0:
+            logger.log(f"WARNING: {failures} of {self.num_contexts} learning_curve artifacts failed to copy successfully from {self.path} to {save_path}")
+
+        return save_path
+
     def load_leaderboards(self) -> List[pd.DataFrame]:
         if self.num_contexts == 0:
             raise AssertionError("Empty output_contexts!")
diff --git a/src/autogluon/bench/eval/benchmark_context/utils.py b/src/autogluon/bench/eval/benchmark_context/utils.py
index a235d12f..504f18d4 100644
--- a/src/autogluon/bench/eval/benchmark_context/utils.py
+++ b/src/autogluon/bench/eval/benchmark_context/utils.py
@@ -1,11 +1,115 @@
+from __future__ import annotations
+
+import boto3
+
+from urllib.parse import urlparse
+
 from autogluon.common.loaders import load_s3
 from autogluon.common.utils import s3_utils
 
 
-def get_s3_paths(path_prefix: str, contains=None, suffix=None):
+def is_s3_url(path: str) -> bool:
+    """
+    Checks if path is a s3 uri.
+
+    Params:
+    -------
+    path: str
+        The path to check.
+
+    Returns:
+    --------
+    bool: whether the path is a s3 uri.
+    """
+    if (path[:2] == "s3") and ("://" in path[:6]):
+        return True 
+    return False
+
+
+def get_bucket_key(s3_uri: str) -> tuple[str, str]:
+    """
+    Retrieves the bucket and key from a s3 uri.
+
+    Params:
+    -------
+    origin_path: str
+        The path (s3 uri) to be parsed.
+
+    Returns:
+    --------
+    bucket_name: str
+        the associated bucket name
+    object_key: str
+        the associated key
+    """
+    if not is_s3_url(s3_uri):
+        raise ValueError("Invalid S3 URI scheme. It should be 's3'.")
+
+    parsed_uri = urlparse(s3_uri)
+    bucket_name = parsed_uri.netloc
+    object_key = parsed_uri.path.lstrip('/')
+
+    return bucket_name, object_key
+
+
+def get_s3_paths(path_prefix: str, contains: str | None = None, suffix: str | None = None) -> list[str]:
+    """
+    Gets all s3 paths in the path_prefix that contain 'contains'
+    and end with 'suffix.'
+
+    Params:
+    -------
+    path_prefix: str
+        The path prefix.
+    contains : Optional[str], default = None
+        Can be specified to limit the returned outputs.
+        For example, by specifying the constraint, such as ".1h8c."
+    suffix: str, default = None
+        Can be specified to limit the returned outputs.
+        For example, by specifying "leaderboard.csv" only objects ending
+        with this suffix will be included
+        If no suffix provided, will save all files in artifact directory.
+
+    Returns:
+    --------
+    List[str]: All s3 paths that adhere to the conditions passed in.
+    """
     bucket, prefix = s3_utils.s3_path_to_bucket_prefix(path_prefix)
     objects = load_s3.list_bucket_prefix_suffix_contains_s3(
         bucket=bucket, prefix=prefix, suffix=suffix, contains=contains
     )
     paths_full = [s3_utils.s3_bucket_prefix_to_path(bucket=bucket, prefix=file, version="s3") for file in objects]
     return paths_full
+
+
+def copy_s3_object(origin_path: str, destination_path: str) -> bool:
+    """
+    Copies s3 object from origin_path to destination_path
+
+    Params:
+    -------
+    origin_path: str
+        The path (s3 uri) to the original location of the object
+    destination_path: str
+        The path (s3 uri) to the intended destination location of the object
+
+    Returns:
+    --------
+    bool: whether the copy was successful.
+    """
+    origin_bucket, origin_key = get_bucket_key(origin_path)
+    destination_bucket, destination_key = get_bucket_key(destination_path)
+
+    try:
+        s3 = boto3.client('s3')
+        s3.copy_object(
+            Bucket=destination_bucket,
+            CopySource={'Bucket': origin_bucket, 'Key': origin_key},
+            Key=destination_key
+        )
+        return True
+    except:
+        pass
+
+    return False
+
diff --git a/src/autogluon/bench/eval/scripts/aggregate_amlb_results.py b/src/autogluon/bench/eval/scripts/aggregate_amlb_results.py
index de757a9e..5b195758 100644
--- a/src/autogluon/bench/eval/scripts/aggregate_amlb_results.py
+++ b/src/autogluon/bench/eval/scripts/aggregate_amlb_results.py
@@ -1,6 +1,6 @@
 import typer
 
-from autogluon.bench.eval.aggregate.results import aggregate_results
+from autogluon.bench.eval.aggregate.aggregate import aggregate
 
 app = typer.Typer()
 
@@ -12,6 +12,10 @@ def aggregate_amlb_results(
     benchmark_name: str = typer.Argument(
         help="Folder name of benchmark run in which all objects with path 'scores/results.csv' get aggregated."
     ),
+    artifact: str = typer.Option(
+        "results",
+        help="What should be saved, one of ['results', 'learning_curves'], default='results'"
+    ),
     constraint: str = typer.Option(
         None,
         help="Name of constraint used in benchmark, refer to https://github.com/openml/automlbenchmark/blob/master/resources/constraints.yaml. Not applicable when `module==multimodal`",
@@ -20,17 +24,48 @@ def aggregate_amlb_results(
     mode: str = typer.Option("ray", help='Aggregation mode: "seq" or "ray".'),
 ):
     """
-    Finds "scores/results.csv" under s3://<s3_bucket>/<module>/<benchmark_name> recursively with the constraint if provided,
-    and append all results into one file at s3://<s3_bucket>/aggregated/<module>/<benchmark_name>/results_automlbenchmark_<constraint>_<benchmark_name>.csv
+    Aggregates objects across an agbenchmark. Functionality depends on artifact specified:
+    
+    Params:
+    -------
+    s3_bucket: str
+        Name of the relevant s3_bucket
+    module: str
+        The name of the relevant autogluon module: can be one of ['tabular', 'timeseries', 'multimodal']
+    benchmark_name: str
+        The name of the relevant benchmark that was run
+    artifact: str
+        The desired artifact to be aggregatedL can be one of ['results', 'learning_curves']
+    constraint: str
+        Name of constraint used in benchmark
+    include_infer_speed: bool
+        Include inference speed in aggregation.
+    mode: str
+        Can be one of ['seq', 'ray'].
+        If seq, runs sequentially.
+        If ray, utilizes parallelization.
+
+    Artifact Outcomes: ['results', 'learning_curves']
+        results:
+            Finds "scores/results.csv" under s3://<s3_bucket>/<module>/<benchmark_name> recursively with the constraint if provided,
+            and append all results into one file at s3://<s3_bucket>/aggregated/<module>/<benchmark_name>/results_automlbenchmark_<constraint>_<benchmark_name>.csv
+
+            Example:
+                agbench aggregate-amlb-results autogluon-benchmark-metrics tabular ag_tabular_20230629T140546 --constraint test
+
+        learning_curves:
+            Finds specified learning_curves.json files under s3://<s3_bucket>/<module>/<benchmark_name> recursively with the constraint if provided,
+            and stores all artifacts in common directory at s3://<s3_bucket>/aggregated/<module>/<benchmark_name>/
 
-    Example:
-        agbench aggregate-amlb-results autogluon-benchmark-metrics tabular ag_tabular_20230629T140546 --constraint test
+            Example:
+                agbench aggregate-amlb-artifacts andrew-bench-dev tabular ag_bench_learning_curves_20240802T163522 --artifact learning_curves --constraint toy
     """
 
-    aggregate_results(
+    aggregate(
         s3_bucket=s3_bucket,
-        s3_prefix=f"{module}/",
-        version_name=benchmark_name,
+        module=module,
+        benchmark_name=benchmark_name,
+        artifact=artifact,
         constraint=constraint,
         include_infer_speed=include_infer_speed,
         mode=mode,

From bd5352797a88fab3f26782aa0f991bc02aed2048 Mon Sep 17 00:00:00 2001
From: Andrew DiBiasio <mbox_user-github@yahoo.com>
Date: Wed, 7 Aug 2024 08:26:27 +0000
Subject: [PATCH 02/10] docstring edits

---
 src/autogluon/bench/eval/scripts/aggregate_amlb_results.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/autogluon/bench/eval/scripts/aggregate_amlb_results.py b/src/autogluon/bench/eval/scripts/aggregate_amlb_results.py
index 5b195758..5b50f9d0 100644
--- a/src/autogluon/bench/eval/scripts/aggregate_amlb_results.py
+++ b/src/autogluon/bench/eval/scripts/aggregate_amlb_results.py
@@ -58,7 +58,7 @@ def aggregate_amlb_results(
             and stores all artifacts in common directory at s3://<s3_bucket>/aggregated/<module>/<benchmark_name>/
 
             Example:
-                agbench aggregate-amlb-artifacts andrew-bench-dev tabular ag_bench_learning_curves_20240802T163522 --artifact learning_curves --constraint toy
+                agbench aggregate-amlb-results autogluon-benchmark-metrics tabular ag_bench_learning_curves_20240802T163522 --artifact learning_curves --constraint toy
     """
 
     aggregate(

From b5e87f6f453f3675e0823b94cd9149372bf7b822 Mon Sep 17 00:00:00 2001
From: Andrew DiBiasio <mbox_user-github@yahoo.com>
Date: Wed, 7 Aug 2024 09:13:44 +0000
Subject: [PATCH 03/10] fixed linting errors

---
 .../bench/eval/aggregate/aggregate.py         | 23 ++++++++++---------
 .../eval/benchmark_context/output_context.py  | 11 +++++----
 .../benchmark_context/output_suite_context.py |  4 +++-
 .../bench/eval/benchmark_context/utils.py     | 11 ++++-----
 .../eval/scripts/aggregate_amlb_results.py    |  5 ++--
 5 files changed, 28 insertions(+), 26 deletions(-)

diff --git a/src/autogluon/bench/eval/aggregate/aggregate.py b/src/autogluon/bench/eval/aggregate/aggregate.py
index 05fbfca8..13d2d39b 100644
--- a/src/autogluon/bench/eval/aggregate/aggregate.py
+++ b/src/autogluon/bench/eval/aggregate/aggregate.py
@@ -7,23 +7,24 @@
 
 
 def aggregate(
-        s3_bucket: str, 
-        module: str, 
-        benchmark_name: str, 
-        artifact: str | None = "results", 
-        constraint: str | None = None,
-        include_infer_speed: bool = False ,
-        mode: str = "ray"
-    ) -> None:
+    s3_bucket: str,
+    module: str,
+    benchmark_name: str,
+    artifact: str | None = "results",
+    constraint: str | None = None,
+    include_infer_speed: bool = False,
+    mode: str = "ray",
+) -> None:
     """
     Aggregates objects across an agbenchmark. Functionality depends on artifact specified:
-    
+
     Params:
     -------
     s3_bucket: str
         Name of the relevant s3_bucket
     module: str
-        The name of the relevant autogluon module: can be one of ['tabular', 'timeseries', 'multimodal']
+        The name of the relevant autogluon module:
+        can be one of ['tabular', 'timeseries', 'multimodal']
     benchmark_name: str
         The name of the relevant benchmark that was run
     artifact: str
@@ -60,4 +61,4 @@ def aggregate(
         artifact_path = f"s3://{s3_bucket}/aggregated/{result_path}/{aggregated_results_name}"
         save_pd.save(path=artifact_path, df=results_df)
 
-    logger.info(f"Aggregated output saved to {artifact_path}!")
\ No newline at end of file
+    logger.info(f"Aggregated output saved to {artifact_path}!")
diff --git a/src/autogluon/bench/eval/benchmark_context/output_context.py b/src/autogluon/bench/eval/benchmark_context/output_context.py
index 50f1b7ca..687e6f71 100644
--- a/src/autogluon/bench/eval/benchmark_context/output_context.py
+++ b/src/autogluon/bench/eval/benchmark_context/output_context.py
@@ -14,6 +14,7 @@
 
 logger = logging.getLogger(__name__)
 
+
 class OutputContext:
     def __init__(self, path):
         """
@@ -106,7 +107,7 @@ def load_results(
 
     def load_leaderboard(self) -> pd.DataFrame:
         return load_pd.load(self.path_leaderboard)
-    
+
     def load_learning_curves(self, save_path: str, suffix: str = "learning_curves.json") -> bool:
         path = self.path_learning_curves
         all_curves = get_s3_paths(path_prefix=path, suffix=suffix)
@@ -123,10 +124,12 @@ def load_learning_curves(self, save_path: str, suffix: str = "learning_curves.js
         return ok
 
     def get_dataset_fold(self, path_str: str) -> tuple[str, str]:
-        parts = path_str.rstrip('/').split('/')
-        
+        parts = path_str.rstrip("/").split("/")
+
         if len(parts) < 3:
-            raise ValueError(f"Improper folder dimensions at {path_str}. Expected following path structure: .../dataset/fold/learning_curves.json")
+            raise ValueError(
+                f"Improper folder dimensions at {path_str}. Expected following path structure: .../dataset/fold/learning_curves.json"
+            )
 
         # path pattern: .../dataset/fold/learning_curves.json
         dataset, fold, _ = parts[-3:]
diff --git a/src/autogluon/bench/eval/benchmark_context/output_suite_context.py b/src/autogluon/bench/eval/benchmark_context/output_suite_context.py
index b733ec16..156ddb40 100644
--- a/src/autogluon/bench/eval/benchmark_context/output_suite_context.py
+++ b/src/autogluon/bench/eval/benchmark_context/output_suite_context.py
@@ -220,7 +220,9 @@ def aggregate_learning_curves(self, save_path: str) -> str:
         outcomes = self.load_learning_curves(save_path=save_path)
         failures = outcomes.count(False)
         if failures > 0:
-            logger.log(f"WARNING: {failures} of {self.num_contexts} learning_curve artifacts failed to copy successfully from {self.path} to {save_path}")
+            logger.log(
+                f"WARNING: {failures} of {self.num_contexts} learning_curve artifacts failed to copy successfully from {self.path} to {save_path}"
+            )
 
         return save_path
 
diff --git a/src/autogluon/bench/eval/benchmark_context/utils.py b/src/autogluon/bench/eval/benchmark_context/utils.py
index 504f18d4..59c618a1 100644
--- a/src/autogluon/bench/eval/benchmark_context/utils.py
+++ b/src/autogluon/bench/eval/benchmark_context/utils.py
@@ -22,7 +22,7 @@ def is_s3_url(path: str) -> bool:
     bool: whether the path is a s3 uri.
     """
     if (path[:2] == "s3") and ("://" in path[:6]):
-        return True 
+        return True
     return False
 
 
@@ -47,7 +47,7 @@ def get_bucket_key(s3_uri: str) -> tuple[str, str]:
 
     parsed_uri = urlparse(s3_uri)
     bucket_name = parsed_uri.netloc
-    object_key = parsed_uri.path.lstrip('/')
+    object_key = parsed_uri.path.lstrip("/")
 
     return bucket_name, object_key
 
@@ -101,15 +101,12 @@ def copy_s3_object(origin_path: str, destination_path: str) -> bool:
     destination_bucket, destination_key = get_bucket_key(destination_path)
 
     try:
-        s3 = boto3.client('s3')
+        s3 = boto3.client("s3")
         s3.copy_object(
-            Bucket=destination_bucket,
-            CopySource={'Bucket': origin_bucket, 'Key': origin_key},
-            Key=destination_key
+            Bucket=destination_bucket, CopySource={"Bucket": origin_bucket, "Key": origin_key}, Key=destination_key
         )
         return True
     except:
         pass
 
     return False
-
diff --git a/src/autogluon/bench/eval/scripts/aggregate_amlb_results.py b/src/autogluon/bench/eval/scripts/aggregate_amlb_results.py
index 5b50f9d0..42fa89c6 100644
--- a/src/autogluon/bench/eval/scripts/aggregate_amlb_results.py
+++ b/src/autogluon/bench/eval/scripts/aggregate_amlb_results.py
@@ -13,8 +13,7 @@ def aggregate_amlb_results(
         help="Folder name of benchmark run in which all objects with path 'scores/results.csv' get aggregated."
     ),
     artifact: str = typer.Option(
-        "results",
-        help="What should be saved, one of ['results', 'learning_curves'], default='results'"
+        "results", help="What should be saved, one of ['results', 'learning_curves'], default='results'"
     ),
     constraint: str = typer.Option(
         None,
@@ -25,7 +24,7 @@ def aggregate_amlb_results(
 ):
     """
     Aggregates objects across an agbenchmark. Functionality depends on artifact specified:
-    
+
     Params:
     -------
     s3_bucket: str

From 623c06e7027c9377ea86873161a5c78c8d0189e7 Mon Sep 17 00:00:00 2001
From: Andrew DiBiasio <mbox_user-github@yahoo.com>
Date: Wed, 7 Aug 2024 09:24:00 +0000
Subject: [PATCH 04/10] fixed import linting errors

---
 src/autogluon/bench/eval/benchmark_context/output_context.py | 5 ++---
 .../bench/eval/benchmark_context/output_suite_context.py     | 2 +-
 src/autogluon/bench/eval/benchmark_context/utils.py          | 4 ++--
 3 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/src/autogluon/bench/eval/benchmark_context/output_context.py b/src/autogluon/bench/eval/benchmark_context/output_context.py
index 687e6f71..22a67165 100644
--- a/src/autogluon/bench/eval/benchmark_context/output_context.py
+++ b/src/autogluon/bench/eval/benchmark_context/output_context.py
@@ -1,17 +1,16 @@
+import logging
 import zipfile
 from io import BytesIO, TextIOWrapper
 from typing import Optional, Set, Union
 
 import boto3
-import logging
 import numpy as np
 import pandas as pd
 
+from autogluon.bench.eval.benchmark_context.utils import copy_s3_object, get_s3_paths
 from autogluon.common.loaders import load_pd, load_pkl
 from autogluon.common.utils.s3_utils import s3_path_to_bucket_prefix
 
-from autogluon.bench.eval.benchmark_context.utils import copy_s3_object, get_s3_paths
-
 logger = logging.getLogger(__name__)
 
 
diff --git a/src/autogluon/bench/eval/benchmark_context/output_suite_context.py b/src/autogluon/bench/eval/benchmark_context/output_suite_context.py
index 156ddb40..b20cb324 100644
--- a/src/autogluon/bench/eval/benchmark_context/output_suite_context.py
+++ b/src/autogluon/bench/eval/benchmark_context/output_suite_context.py
@@ -1,8 +1,8 @@
 from __future__ import annotations
 
 import copy
-import os
 import logging
+import os
 from typing import List, Optional, Set
 
 import pandas as pd
diff --git a/src/autogluon/bench/eval/benchmark_context/utils.py b/src/autogluon/bench/eval/benchmark_context/utils.py
index 59c618a1..33225428 100644
--- a/src/autogluon/bench/eval/benchmark_context/utils.py
+++ b/src/autogluon/bench/eval/benchmark_context/utils.py
@@ -1,9 +1,9 @@
 from __future__ import annotations
 
-import boto3
-
 from urllib.parse import urlparse
 
+import boto3
+
 from autogluon.common.loaders import load_s3
 from autogluon.common.utils import s3_utils
 

From 01fb252b617aa53d37dda692610e411a5d8eeac7 Mon Sep 17 00:00:00 2001
From: Andrew DiBiasio <mbox_user-github@yahoo.com>
Date: Tue, 13 Aug 2024 23:09:22 +0000
Subject: [PATCH 05/10] move fold number to folder name instead of json file
 name

---
 src/autogluon/bench/eval/benchmark_context/output_context.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/autogluon/bench/eval/benchmark_context/output_context.py b/src/autogluon/bench/eval/benchmark_context/output_context.py
index 22a67165..7bb41665 100644
--- a/src/autogluon/bench/eval/benchmark_context/output_context.py
+++ b/src/autogluon/bench/eval/benchmark_context/output_context.py
@@ -114,7 +114,7 @@ def load_learning_curves(self, save_path: str, suffix: str = "learning_curves.js
         ok = True
         for origin_path in all_curves:
             dataset, fold = self.get_dataset_fold(origin_path)
-            destination_path = f"{save_path}/{dataset}/{fold}.json"
+            destination_path = f"{save_path}/{dataset}/{fold}/learning_curves.json"
             res = copy_s3_object(origin_path=origin_path, destination_path=destination_path)
             ok &= res
             if not res:

From ae92ade0a58859239b3467690fc71e92196a34e3 Mon Sep 17 00:00:00 2001
From: Andrew DiBiasio <mbox_user-github@yahoo.com>
Date: Thu, 15 Aug 2024 19:33:03 +0000
Subject: [PATCH 06/10] addressed pr comments

---
 .../bench/eval/aggregate/aggregate.py         | 23 +++++++++----------
 .../eval/benchmark_context/output_context.py  | 23 +++++++++++++++----
 .../eval/scripts/aggregate_amlb_results.py    | 15 ++++++++----
 3 files changed, 39 insertions(+), 22 deletions(-)

diff --git a/src/autogluon/bench/eval/aggregate/aggregate.py b/src/autogluon/bench/eval/aggregate/aggregate.py
index 13d2d39b..c9d05843 100644
--- a/src/autogluon/bench/eval/aggregate/aggregate.py
+++ b/src/autogluon/bench/eval/aggregate/aggregate.py
@@ -1,4 +1,5 @@
 import logging
+from typing import List
 
 from autogluon.bench.eval.benchmark_context.output_suite_context import OutputSuiteContext
 from autogluon.common.savers import save_pd
@@ -10,7 +11,7 @@ def aggregate(
     s3_bucket: str,
     module: str,
     benchmark_name: str,
-    artifact: str | None = "results",
+    artifacts: List[str] | None = ["results"],
     constraint: str | None = None,
     include_infer_speed: bool = False,
     mode: str = "ray",
@@ -49,16 +50,14 @@ def aggregate(
         mode=mode,
     )
 
-    if artifact == "learning_curves":
-        save_path = f"s3://{s3_bucket}/aggregated/{result_path}/{artifact}"
-        artifact_path = output_suite_context.aggregate_learning_curves(save_path=save_path)
-    else:
-        aggregated_results_name = f"results_automlbenchmark_{constraint}_{benchmark_name}.csv"
-        results_df = output_suite_context.aggregate_results()
-
-        print(results_df)
-
-        artifact_path = f"s3://{s3_bucket}/aggregated/{result_path}/{aggregated_results_name}"
-        save_pd.save(path=artifact_path, df=results_df)
+    for artifact in artifacts:
+        if artifact == "results":
+            aggregated_results_name = f"results_automlbenchmark_{constraint}_{benchmark_name}.csv"
+            results_df = output_suite_context.aggregate_results()
+            artifact_path = f"s3://{s3_bucket}/aggregated/{result_path}/{aggregated_results_name}"
+            save_pd.save(path=artifact_path, df=results_df)
+        elif artifact == "learning_curves":
+            save_path = f"s3://{s3_bucket}/aggregated/{result_path}/{artifact}"
+            artifact_path = output_suite_context.aggregate_learning_curves(save_path=save_path)
 
     logger.info(f"Aggregated output saved to {artifact_path}!")
diff --git a/src/autogluon/bench/eval/benchmark_context/output_context.py b/src/autogluon/bench/eval/benchmark_context/output_context.py
index 7bb41665..f44b433e 100644
--- a/src/autogluon/bench/eval/benchmark_context/output_context.py
+++ b/src/autogluon/bench/eval/benchmark_context/output_context.py
@@ -108,19 +108,32 @@ def load_leaderboard(self) -> pd.DataFrame:
         return load_pd.load(self.path_leaderboard)
 
     def load_learning_curves(self, save_path: str, suffix: str = "learning_curves.json") -> bool:
+        """
+        Params:
+        -------
+        save_path: str
+            the path to save all learning curve artifacts
+        suffix: str
+            the suffix matching all learning curves files
+
+        Returns:
+        --------
+        Whether all learning curve artifacts located at self.path_learning_curves
+        were successfully aggregated and copied to the save_path directory.
+        """
         path = self.path_learning_curves
         all_curves = get_s3_paths(path_prefix=path, suffix=suffix)
 
-        ok = True
+        all_copied_successfully = True
         for origin_path in all_curves:
             dataset, fold = self.get_dataset_fold(origin_path)
             destination_path = f"{save_path}/{dataset}/{fold}/learning_curves.json"
-            res = copy_s3_object(origin_path=origin_path, destination_path=destination_path)
-            ok &= res
-            if not res:
+            current_copied_successfully = copy_s3_object(origin_path=origin_path, destination_path=destination_path)
+            all_copied_successfully &= current_copied_successfully
+            if not current_copied_successfully:
                 logger.log(f"Learning Curve artifact at {origin_path} could not be copied to {destination_path}")
 
-        return ok
+        return all_copied_successfully
 
     def get_dataset_fold(self, path_str: str) -> tuple[str, str]:
         parts = path_str.rstrip("/").split("/")
diff --git a/src/autogluon/bench/eval/scripts/aggregate_amlb_results.py b/src/autogluon/bench/eval/scripts/aggregate_amlb_results.py
index 42fa89c6..dc3037e0 100644
--- a/src/autogluon/bench/eval/scripts/aggregate_amlb_results.py
+++ b/src/autogluon/bench/eval/scripts/aggregate_amlb_results.py
@@ -1,3 +1,5 @@
+from typing import List
+
 import typer
 
 from autogluon.bench.eval.aggregate.aggregate import aggregate
@@ -12,8 +14,8 @@ def aggregate_amlb_results(
     benchmark_name: str = typer.Argument(
         help="Folder name of benchmark run in which all objects with path 'scores/results.csv' get aggregated."
     ),
-    artifact: str = typer.Option(
-        "results", help="What should be saved, one of ['results', 'learning_curves'], default='results'"
+    artifact: List[str] = typer.Option(
+        ["results"], help="What should be saved, can be any of ['results', 'learning_curves'], default='results'"
     ),
     constraint: str = typer.Option(
         None,
@@ -33,8 +35,8 @@ def aggregate_amlb_results(
         The name of the relevant autogluon module: can be one of ['tabular', 'timeseries', 'multimodal']
     benchmark_name: str
         The name of the relevant benchmark that was run
-    artifact: str
-        The desired artifact to be aggregatedL can be one of ['results', 'learning_curves']
+    artifact: List[str]
+        The desired artifact(s) to be aggregated can be one of ['results', 'learning_curves']
     constraint: str
         Name of constraint used in benchmark
     include_infer_speed: bool
@@ -58,13 +60,16 @@ def aggregate_amlb_results(
 
             Example:
                 agbench aggregate-amlb-results autogluon-benchmark-metrics tabular ag_bench_learning_curves_20240802T163522 --artifact learning_curves --constraint toy
+
+                # to generate both
+                agbench aggregate-amlb-results autogluon-benchmark-metrics tabular ag_bench_learning_curves_20240802T163522 --artifact results learning_curves --constraint toy
     """
 
     aggregate(
         s3_bucket=s3_bucket,
         module=module,
         benchmark_name=benchmark_name,
-        artifact=artifact,
+        artifacts=artifact,
         constraint=constraint,
         include_infer_speed=include_infer_speed,
         mode=mode,

From 06bf520d5e6df99a12f268d5c64b3fff21f8e0ef Mon Sep 17 00:00:00 2001
From: Andrew DiBiasio <mbox_user-github@yahoo.com>
Date: Sun, 18 Aug 2024 05:13:09 +0000
Subject: [PATCH 07/10] removing redundant s3 utils

---
 .../bench/eval/aggregate/aggregate.py         | 33 ++++----
 .../eval/benchmark_context/output_context.py  | 25 +++---
 .../benchmark_context/output_suite_context.py | 11 +--
 .../bench/eval/benchmark_context/utils.py     | 78 -------------------
 4 files changed, 32 insertions(+), 115 deletions(-)

diff --git a/src/autogluon/bench/eval/aggregate/aggregate.py b/src/autogluon/bench/eval/aggregate/aggregate.py
index c9d05843..a3f8b86d 100644
--- a/src/autogluon/bench/eval/aggregate/aggregate.py
+++ b/src/autogluon/bench/eval/aggregate/aggregate.py
@@ -11,7 +11,7 @@ def aggregate(
     s3_bucket: str,
     module: str,
     benchmark_name: str,
-    artifacts: List[str] | None = ["results"],
+    artifacts: List[str] = ["results"],
     constraint: str | None = None,
     include_infer_speed: bool = False,
     mode: str = "ray",
@@ -28,8 +28,8 @@ def aggregate(
         can be one of ['tabular', 'timeseries', 'multimodal']
     benchmark_name: str
         The name of the relevant benchmark that was run
-    artifact: str
-        The desired artifact to be aggregatedL can be one of ['results', 'learning_curves']
+    artifacts: List[str]
+        The desired artifact to be aggregated can be one of ['results', 'learning_curves']
     constraint: str
         Name of constraint used in benchmark
     include_infer_speed: bool
@@ -50,14 +50,19 @@ def aggregate(
         mode=mode,
     )
 
-    for artifact in artifacts:
-        if artifact == "results":
-            aggregated_results_name = f"results_automlbenchmark_{constraint}_{benchmark_name}.csv"
-            results_df = output_suite_context.aggregate_results()
-            artifact_path = f"s3://{s3_bucket}/aggregated/{result_path}/{aggregated_results_name}"
-            save_pd.save(path=artifact_path, df=results_df)
-        elif artifact == "learning_curves":
-            save_path = f"s3://{s3_bucket}/aggregated/{result_path}/{artifact}"
-            artifact_path = output_suite_context.aggregate_learning_curves(save_path=save_path)
-
-    logger.info(f"Aggregated output saved to {artifact_path}!")
+    if "results" in artifacts:
+        aggregated_results_name = f"results_automlbenchmark_{constraint}_{benchmark_name}.csv"
+        results_df = output_suite_context.aggregate_results()
+        artifact_path = f"s3://{s3_bucket}/aggregated/{result_path}/{aggregated_results_name}"
+        save_pd.save(path=artifact_path, df=results_df)
+        logger.info(f"Aggregated output saved to {artifact_path}!")
+
+    if "learning_curves" in artifacts:
+        artifact_path = f"s3://{s3_bucket}/aggregated/{result_path}/learning_curves"
+        output_suite_context.aggregate_learning_curves(save_path=artifact_path)
+        logger.info(f"Aggregated output saved to {artifact_path}!")
+
+    valid_artifacts = ["results", "learning_curves"]
+    unknown_artifacts = [artifact for artifact in artifacts if artifact not in valid_artifacts]
+    if unknown_artifacts:
+        raise ValueError(f"Unknown artifacts: {unknown_artifacts}")
diff --git a/src/autogluon/bench/eval/benchmark_context/output_context.py b/src/autogluon/bench/eval/benchmark_context/output_context.py
index f44b433e..199ed03e 100644
--- a/src/autogluon/bench/eval/benchmark_context/output_context.py
+++ b/src/autogluon/bench/eval/benchmark_context/output_context.py
@@ -7,7 +7,7 @@
 import numpy as np
 import pandas as pd
 
-from autogluon.bench.eval.benchmark_context.utils import copy_s3_object, get_s3_paths
+from autogluon.bench.eval.benchmark_context.utils import get_s3_paths
 from autogluon.common.loaders import load_pd, load_pkl
 from autogluon.common.utils.s3_utils import s3_path_to_bucket_prefix
 
@@ -107,7 +107,7 @@ def load_results(
     def load_leaderboard(self) -> pd.DataFrame:
         return load_pd.load(self.path_leaderboard)
 
-    def load_learning_curves(self, save_path: str, suffix: str = "learning_curves.json") -> bool:
+    def load_learning_curves(self, save_path: str, suffix: str = "learning_curves.json") -> None:
         """
         Params:
         -------
@@ -115,25 +115,22 @@ def load_learning_curves(self, save_path: str, suffix: str = "learning_curves.js
             the path to save all learning curve artifacts
         suffix: str
             the suffix matching all learning curves files
-
-        Returns:
-        --------
-        Whether all learning curve artifacts located at self.path_learning_curves
-        were successfully aggregated and copied to the save_path directory.
         """
+        try:
+            # copy_s3_file method not yet in stable release of autogluon
+            from autogluon.common.utils.s3_utils import copy_s3_file
+        except:
+            raise ImportError(
+                f"Install AutoGluon from source to get access to copy_s3_file from autogluon.common.utils.s3_utils"
+            )
+
         path = self.path_learning_curves
         all_curves = get_s3_paths(path_prefix=path, suffix=suffix)
 
-        all_copied_successfully = True
         for origin_path in all_curves:
             dataset, fold = self.get_dataset_fold(origin_path)
             destination_path = f"{save_path}/{dataset}/{fold}/learning_curves.json"
-            current_copied_successfully = copy_s3_object(origin_path=origin_path, destination_path=destination_path)
-            all_copied_successfully &= current_copied_successfully
-            if not current_copied_successfully:
-                logger.log(f"Learning Curve artifact at {origin_path} could not be copied to {destination_path}")
-
-        return all_copied_successfully
+            copy_s3_file(origin_path=origin_path, destination_path=destination_path)
 
     def get_dataset_fold(self, path_str: str) -> tuple[str, str]:
         parts = path_str.rstrip("/").split("/")
diff --git a/src/autogluon/bench/eval/benchmark_context/output_suite_context.py b/src/autogluon/bench/eval/benchmark_context/output_suite_context.py
index b20cb324..be70a5a2 100644
--- a/src/autogluon/bench/eval/benchmark_context/output_suite_context.py
+++ b/src/autogluon/bench/eval/benchmark_context/output_suite_context.py
@@ -216,15 +216,8 @@ def aggregate_results(self, results_list: List[pd.DataFrame] | None = None) -> p
         results_df = pd.concat(results_list, ignore_index=True)
         return results_df
 
-    def aggregate_learning_curves(self, save_path: str) -> str:
-        outcomes = self.load_learning_curves(save_path=save_path)
-        failures = outcomes.count(False)
-        if failures > 0:
-            logger.log(
-                f"WARNING: {failures} of {self.num_contexts} learning_curve artifacts failed to copy successfully from {self.path} to {save_path}"
-            )
-
-        return save_path
+    def aggregate_learning_curves(self, save_path: str):
+        self.load_learning_curves(save_path=save_path)
 
     def load_leaderboards(self) -> List[pd.DataFrame]:
         if self.num_contexts == 0:
diff --git a/src/autogluon/bench/eval/benchmark_context/utils.py b/src/autogluon/bench/eval/benchmark_context/utils.py
index 33225428..a2b5e844 100644
--- a/src/autogluon/bench/eval/benchmark_context/utils.py
+++ b/src/autogluon/bench/eval/benchmark_context/utils.py
@@ -1,57 +1,9 @@
 from __future__ import annotations
 
-from urllib.parse import urlparse
-
-import boto3
-
 from autogluon.common.loaders import load_s3
 from autogluon.common.utils import s3_utils
 
 
-def is_s3_url(path: str) -> bool:
-    """
-    Checks if path is a s3 uri.
-
-    Params:
-    -------
-    path: str
-        The path to check.
-
-    Returns:
-    --------
-    bool: whether the path is a s3 uri.
-    """
-    if (path[:2] == "s3") and ("://" in path[:6]):
-        return True
-    return False
-
-
-def get_bucket_key(s3_uri: str) -> tuple[str, str]:
-    """
-    Retrieves the bucket and key from a s3 uri.
-
-    Params:
-    -------
-    origin_path: str
-        The path (s3 uri) to be parsed.
-
-    Returns:
-    --------
-    bucket_name: str
-        the associated bucket name
-    object_key: str
-        the associated key
-    """
-    if not is_s3_url(s3_uri):
-        raise ValueError("Invalid S3 URI scheme. It should be 's3'.")
-
-    parsed_uri = urlparse(s3_uri)
-    bucket_name = parsed_uri.netloc
-    object_key = parsed_uri.path.lstrip("/")
-
-    return bucket_name, object_key
-
-
 def get_s3_paths(path_prefix: str, contains: str | None = None, suffix: str | None = None) -> list[str]:
     """
     Gets all s3 paths in the path_prefix that contain 'contains'
@@ -80,33 +32,3 @@ def get_s3_paths(path_prefix: str, contains: str | None = None, suffix: str | No
     )
     paths_full = [s3_utils.s3_bucket_prefix_to_path(bucket=bucket, prefix=file, version="s3") for file in objects]
     return paths_full
-
-
-def copy_s3_object(origin_path: str, destination_path: str) -> bool:
-    """
-    Copies s3 object from origin_path to destination_path
-
-    Params:
-    -------
-    origin_path: str
-        The path (s3 uri) to the original location of the object
-    destination_path: str
-        The path (s3 uri) to the intended destination location of the object
-
-    Returns:
-    --------
-    bool: whether the copy was successful.
-    """
-    origin_bucket, origin_key = get_bucket_key(origin_path)
-    destination_bucket, destination_key = get_bucket_key(destination_path)
-
-    try:
-        s3 = boto3.client("s3")
-        s3.copy_object(
-            Bucket=destination_bucket, CopySource={"Bucket": origin_bucket, "Key": origin_key}, Key=destination_key
-        )
-        return True
-    except:
-        pass
-
-    return False

From 89fce00e112e7110819ad36e7a3e34a25c2256a6 Mon Sep 17 00:00:00 2001
From: Andrew DiBiasio <mbox_user-github@yahoo.com>
Date: Sun, 18 Aug 2024 05:14:53 +0000
Subject: [PATCH 08/10] update log message

---
 src/autogluon/bench/eval/aggregate/aggregate.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/autogluon/bench/eval/aggregate/aggregate.py b/src/autogluon/bench/eval/aggregate/aggregate.py
index a3f8b86d..6b8b0335 100644
--- a/src/autogluon/bench/eval/aggregate/aggregate.py
+++ b/src/autogluon/bench/eval/aggregate/aggregate.py
@@ -55,12 +55,12 @@ def aggregate(
         results_df = output_suite_context.aggregate_results()
         artifact_path = f"s3://{s3_bucket}/aggregated/{result_path}/{aggregated_results_name}"
         save_pd.save(path=artifact_path, df=results_df)
-        logger.info(f"Aggregated output saved to {artifact_path}!")
+        logger.info(f"Aggregated results output saved to {artifact_path}!")
 
     if "learning_curves" in artifacts:
         artifact_path = f"s3://{s3_bucket}/aggregated/{result_path}/learning_curves"
         output_suite_context.aggregate_learning_curves(save_path=artifact_path)
-        logger.info(f"Aggregated output saved to {artifact_path}!")
+        logger.info(f"Aggregated learning curves output saved to {artifact_path}!")
 
     valid_artifacts = ["results", "learning_curves"]
     unknown_artifacts = [artifact for artifact in artifacts if artifact not in valid_artifacts]

From 1925e950c5a6cdf8819db4a8b8d2dc218e663ab4 Mon Sep 17 00:00:00 2001
From: Andrew DiBiasio <mbox_user-github@yahoo.com>
Date: Sun, 18 Aug 2024 05:18:08 +0000
Subject: [PATCH 09/10] type hint updates

---
 .../bench/eval/benchmark_context/output_suite_context.py      | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/autogluon/bench/eval/benchmark_context/output_suite_context.py b/src/autogluon/bench/eval/benchmark_context/output_suite_context.py
index be70a5a2..0624a601 100644
--- a/src/autogluon/bench/eval/benchmark_context/output_suite_context.py
+++ b/src/autogluon/bench/eval/benchmark_context/output_suite_context.py
@@ -178,7 +178,7 @@ def get_zeroshot_metadata_size_bytes(self, allow_exception=False) -> List[int]:
             allow_exception=allow_exception,
         )
 
-    def load_learning_curves(self, save_path: str) -> list[bool]:
+    def load_learning_curves(self, save_path: str) -> list[None]:
         return self._loop_func(
             func=OutputContext.load_learning_curves,
             input_list=self.output_contexts,
@@ -216,7 +216,7 @@ def aggregate_results(self, results_list: List[pd.DataFrame] | None = None) -> p
         results_df = pd.concat(results_list, ignore_index=True)
         return results_df
 
-    def aggregate_learning_curves(self, save_path: str):
+    def aggregate_learning_curves(self, save_path: str) -> None:
         self.load_learning_curves(save_path=save_path)
 
     def load_leaderboards(self) -> List[pd.DataFrame]:

From 4a750c4d31d667089f1b22806246bcf949fb645e Mon Sep 17 00:00:00 2001
From: Andrew DiBiasio <mbox_user-github@yahoo.com>
Date: Sun, 18 Aug 2024 05:20:45 +0000
Subject: [PATCH 10/10] fixed doc string example

---
 src/autogluon/bench/eval/scripts/aggregate_amlb_results.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/autogluon/bench/eval/scripts/aggregate_amlb_results.py b/src/autogluon/bench/eval/scripts/aggregate_amlb_results.py
index dc3037e0..8dbd2e81 100644
--- a/src/autogluon/bench/eval/scripts/aggregate_amlb_results.py
+++ b/src/autogluon/bench/eval/scripts/aggregate_amlb_results.py
@@ -62,7 +62,7 @@ def aggregate_amlb_results(
                 agbench aggregate-amlb-results autogluon-benchmark-metrics tabular ag_bench_learning_curves_20240802T163522 --artifact learning_curves --constraint toy
 
                 # to generate both
-                agbench aggregate-amlb-results autogluon-benchmark-metrics tabular ag_bench_learning_curves_20240802T163522 --artifact results learning_curves --constraint toy
+                agbench aggregate-amlb-results autogluon-benchmark-metrics tabular ag_bench_learning_curves_20240802T163522 --artifact results --artifact learning_curves --constraint toy
     """
 
     aggregate(