2.使用 Jupyter Notebook 创建 HyperPod 集群.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2.使用 Jupyter Notebook 创建 HyperPod 集群"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages/pydantic/_internal/_fields.py:192: UserWarning: Field name \"json\" in \"MonitoringDatasetFormat\" shadows an attribute in parent \"Base\"\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">[02/12/25 09:43:30] </span><span style=\"color: #0069ff; text-decoration-color: #0069ff; font-weight: bold\">INFO    </span> Found credentials from IAM Role:                                   <a href=\"file:///home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages/botocore/credentials.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">credentials.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages/botocore/credentials.py#1075\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">1075</span></a>\n",
       "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         BaseNotebookInstanceEc2InstanceRole                                <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">                   </span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[2;36m[02/12/25 09:43:30]\u001b[0m\u001b[2;36m \u001b[0m\u001b[1;38;2;0;105;255mINFO    \u001b[0m Found credentials from IAM Role:                                   \u001b]8;id=250442;file:///home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages/botocore/credentials.py\u001b\\\u001b[2mcredentials.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=34575;file:///home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages/botocore/credentials.py#1075\u001b\\\u001b[2m1075\u001b[0m\u001b]8;;\u001b\\\n",
       "\u001b[2;36m                    \u001b[0m         BaseNotebookInstanceEc2InstanceRole                                \u001b[2m                   \u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sagemaker.config INFO - Not applying SDK defaults from location: /etc/xdg/sagemaker/config.yaml\n",
      "sagemaker.config INFO - Not applying SDK defaults from location: /home/ec2-user/.config/sagemaker/config.yaml\n"
     ]
    }
   ],
   "source": [
    "import boto3\n",
    "import sagemaker\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span><span style=\"color: #0069ff; text-decoration-color: #0069ff; font-weight: bold\">INFO    </span> Found credentials from IAM Role:                                   <a href=\"file:///home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages/botocore/credentials.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">credentials.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages/botocore/credentials.py#1075\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">1075</span></a>\n",
       "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         BaseNotebookInstanceEc2InstanceRole                                <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">                   </span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[2;36m                   \u001b[0m\u001b[2;36m \u001b[0m\u001b[1;38;2;0;105;255mINFO    \u001b[0m Found credentials from IAM Role:                                   \u001b]8;id=458778;file:///home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages/botocore/credentials.py\u001b\\\u001b[2mcredentials.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=673133;file:///home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages/botocore/credentials.py#1075\u001b\\\u001b[2m1075\u001b[0m\u001b]8;;\u001b\\\n",
       "\u001b[2;36m                    \u001b[0m         BaseNotebookInstanceEc2InstanceRole                                \u001b[2m                   \u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span><span style=\"color: #0069ff; text-decoration-color: #0069ff; font-weight: bold\">INFO    </span> Found credentials from IAM Role:                                   <a href=\"file:///home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages/botocore/credentials.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">credentials.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages/botocore/credentials.py#1075\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">1075</span></a>\n",
       "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         BaseNotebookInstanceEc2InstanceRole                                <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">                   </span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[2;36m                   \u001b[0m\u001b[2;36m \u001b[0m\u001b[1;38;2;0;105;255mINFO    \u001b[0m Found credentials from IAM Role:                                   \u001b]8;id=209958;file:///home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages/botocore/credentials.py\u001b\\\u001b[2mcredentials.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=597729;file:///home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages/botocore/credentials.py#1075\u001b\\\u001b[2m1075\u001b[0m\u001b]8;;\u001b\\\n",
       "\u001b[2;36m                    \u001b[0m         BaseNotebookInstanceEc2InstanceRole                                \u001b[2m                   \u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sagemaker role arn: arn:aws:iam::533267176336:role/CloudLabCampaignInfraStack-sagemaker--ExecutionRole-rC2M3kfw97Tz\n",
      "sagemaker bucket: sagemaker-us-west-2-533267176336\n",
      "sagemaker session region: us-west-2\n",
      "boto3 version: 1.36.18\n",
      "sagemaker version: 2.239.0\n"
     ]
    }
   ],
   "source": [
    "role = sagemaker.get_execution_role()  # execution role for the endpoint\n",
    "sess = sagemaker.session.Session()  # sagemaker session for interacting with different AWS APIs\n",
    "bucket = sess.default_bucket()  # bucket to house artifacts\n",
    "region = sess._region_name  # region name of the current SageMaker Studio environment\n",
    "\n",
    "\n",
    "print(f\"sagemaker role arn: {role}\")\n",
    "print(f\"sagemaker bucket: {sess.default_bucket()}\")\n",
    "print(f\"sagemaker session region: {sess.boto_region_name}\")\n",
    "print(f\"boto3 version: {boto3.__version__}\")\n",
    "print(f\"sagemaker version: {sagemaker.__version__}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 创建HyperPod集群\n",
    "### 将 lifecycle 配置文件上传到 S3 存储桶。在创建集群期间，在每个实例组中 HyperPod 运行它们。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- 写一个 Slurm 配置文件并将其另存为provisioning_parameters.json。在文件中，指定基本的 Slurm 配置参数，以便将 Slurm 节点正确分配给集群实例组。 \n",
    "- 本教程中，设置2个名为\n",
    "    - my-controller-group\n",
    "    - worker-group-1\n",
    "- 如以下示例配置所示。provisioning_parameters.json\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "local_code_dir = 'lifecycle-scripts'\n",
    "!rm -rf {local_code_dir}\n",
    "!mkdir -p {local_code_dir}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Writing lifecycle-scripts/provisioning_parameters.json\n"
     ]
    }
   ],
   "source": [
    "%%writefile {local_code_dir}/provisioning_parameters.json\n",
    "{\n",
    "    \"version\": \"1.0.0\",\n",
    "    \"workload_manager\": \"slurm\",\n",
    "    \"controller_group\": \"my-controller-group\",\n",
    "    \"worker_groups\": [\n",
    "        {\n",
    "            \"instance_group_name\": \"worker-group-1\",\n",
    "            \"partition_name\": \"partition-1\"\n",
    "        }\n",
    "    ]\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!rm -rf {local_code_dir}/.ipynb_checkpoints"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 将 awsome-distributed-training/ 中的 sample 配置文件和 provisioning_params.json 上传到s3目录中"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "upload: awsome-distributed-training/1.architectures/5.sagemaker-hyperpod/LifecycleScripts/base-config/hotfix/hold-lustre-client.sh to s3://sagemaker-us-west-2-533267176336/hyperpod/LifecycleScripts/hotfix/hold-lustre-client.sh\n",
      "upload: awsome-distributed-training/1.architectures/5.sagemaker-hyperpod/LifecycleScripts/base-config/config.py to s3://sagemaker-us-west-2-533267176336/hyperpod/LifecycleScripts/config.py\n",
      "upload: awsome-distributed-training/1.architectures/5.sagemaker-hyperpod/LifecycleScripts/base-config/shared_users_sample.txt to s3://sagemaker-us-west-2-533267176336/hyperpod/LifecycleScripts/shared_users_sample.txt\n",
      "upload: awsome-distributed-training/1.architectures/5.sagemaker-hyperpod/LifecycleScripts/base-config/setup_mariadb_accounting.sh to s3://sagemaker-us-west-2-533267176336/hyperpod/LifecycleScripts/setup_mariadb_accounting.sh\n",
      "upload: awsome-distributed-training/1.architectures/5.sagemaker-hyperpod/LifecycleScripts/base-config/setup_rds_accounting.sh to s3://sagemaker-us-west-2-533267176336/hyperpod/LifecycleScripts/setup_rds_accounting.sh\n",
      "upload: awsome-distributed-training/1.architectures/5.sagemaker-hyperpod/LifecycleScripts/base-config/apply_hotfix.sh to s3://sagemaker-us-west-2-533267176336/hyperpod/LifecycleScripts/apply_hotfix.sh\n",
      "upload: awsome-distributed-training/1.architectures/5.sagemaker-hyperpod/LifecycleScripts/base-config/lifecycle_script.py to s3://sagemaker-us-west-2-533267176336/hyperpod/LifecycleScripts/lifecycle_script.py\n",
      "upload: awsome-distributed-training/1.architectures/5.sagemaker-hyperpod/LifecycleScripts/base-config/mount_fsx.sh to s3://sagemaker-us-west-2-533267176336/hyperpod/LifecycleScripts/mount_fsx.sh\n",
      "upload: awsome-distributed-training/1.architectures/5.sagemaker-hyperpod/LifecycleScripts/base-config/hotfix/mock-gpu-driver-deb.sh to s3://sagemaker-us-west-2-533267176336/hyperpod/LifecycleScripts/hotfix/mock-gpu-driver-deb.sh\n",
      "upload: awsome-distributed-training/1.architectures/5.sagemaker-hyperpod/LifecycleScripts/base-config/start_slurm.sh to s3://sagemaker-us-west-2-533267176336/hyperpod/LifecycleScripts/start_slurm.sh\n",
      "upload: awsome-distributed-training/1.architectures/5.sagemaker-hyperpod/LifecycleScripts/base-config/on_create.sh to s3://sagemaker-us-west-2-533267176336/hyperpod/LifecycleScripts/on_create.sh\n",
      "upload: awsome-distributed-training/1.architectures/5.sagemaker-hyperpod/LifecycleScripts/base-config/add_users.sh to s3://sagemaker-us-west-2-533267176336/hyperpod/LifecycleScripts/add_users.sh\n",
      "upload: awsome-distributed-training/1.architectures/5.sagemaker-hyperpod/LifecycleScripts/base-config/setup_sssd.py to s3://sagemaker-us-west-2-533267176336/hyperpod/LifecycleScripts/setup_sssd.py\n",
      "upload: awsome-distributed-training/1.architectures/5.sagemaker-hyperpod/LifecycleScripts/base-config/utils/install_dcgm_exporter.sh to s3://sagemaker-us-west-2-533267176336/hyperpod/LifecycleScripts/utils/install_dcgm_exporter.sh\n",
      "upload: awsome-distributed-training/1.architectures/5.sagemaker-hyperpod/LifecycleScripts/base-config/utils/install_efa_node_exporter.sh to s3://sagemaker-us-west-2-533267176336/hyperpod/LifecycleScripts/utils/install_efa_node_exporter.sh\n",
      "upload: awsome-distributed-training/1.architectures/5.sagemaker-hyperpod/LifecycleScripts/base-config/utils/fsx_ubuntu.sh to s3://sagemaker-us-west-2-533267176336/hyperpod/LifecycleScripts/utils/fsx_ubuntu.sh\n",
      "upload: awsome-distributed-training/1.architectures/5.sagemaker-hyperpod/LifecycleScripts/base-config/utils/enroot.conf to s3://sagemaker-us-west-2-533267176336/hyperpod/LifecycleScripts/utils/enroot.conf\n",
      "upload: awsome-distributed-training/1.architectures/5.sagemaker-hyperpod/LifecycleScripts/base-config/utils/install_enroot_pyxis.sh to s3://sagemaker-us-west-2-533267176336/hyperpod/LifecycleScripts/utils/install_enroot_pyxis.sh\n",
      "upload: awsome-distributed-training/1.architectures/5.sagemaker-hyperpod/LifecycleScripts/base-config/utils/gen-keypair-ubuntu.sh to s3://sagemaker-us-west-2-533267176336/hyperpod/LifecycleScripts/utils/gen-keypair-ubuntu.sh\n",
      "upload: awsome-distributed-training/1.architectures/5.sagemaker-hyperpod/LifecycleScripts/base-config/utils/install_head_node_exporter.sh to s3://sagemaker-us-west-2-533267176336/hyperpod/LifecycleScripts/utils/install_head_node_exporter.sh\n",
      "upload: awsome-distributed-training/1.architectures/5.sagemaker-hyperpod/LifecycleScripts/base-config/utils/motd.sh to s3://sagemaker-us-west-2-533267176336/hyperpod/LifecycleScripts/utils/motd.sh\n",
      "upload: awsome-distributed-training/1.architectures/5.sagemaker-hyperpod/LifecycleScripts/base-config/utils/install_docker.sh to s3://sagemaker-us-west-2-533267176336/hyperpod/LifecycleScripts/utils/install_docker.sh\n",
      "upload: awsome-distributed-training/1.architectures/5.sagemaker-hyperpod/LifecycleScripts/base-config/utils/ssh-to-compute.sh to s3://sagemaker-us-west-2-533267176336/hyperpod/LifecycleScripts/utils/ssh-to-compute.sh\n",
      "upload: awsome-distributed-training/1.architectures/5.sagemaker-hyperpod/LifecycleScripts/base-config/utils/install_prometheus.sh to s3://sagemaker-us-west-2-533267176336/hyperpod/LifecycleScripts/utils/install_prometheus.sh\n",
      "upload: awsome-distributed-training/1.architectures/5.sagemaker-hyperpod/LifecycleScripts/base-config/utils/motd.txt to s3://sagemaker-us-west-2-533267176336/hyperpod/LifecycleScripts/utils/motd.txt\n",
      "upload: awsome-distributed-training/1.architectures/5.sagemaker-hyperpod/LifecycleScripts/base-config/utils/update_neuron_sdk.sh to s3://sagemaker-us-west-2-533267176336/hyperpod/LifecycleScripts/utils/update_neuron_sdk.sh\n",
      "upload: awsome-distributed-training/1.architectures/5.sagemaker-hyperpod/LifecycleScripts/base-config/utils/slurm_fix_plugstackconf.sh to s3://sagemaker-us-west-2-533267176336/hyperpod/LifecycleScripts/utils/slurm_fix_plugstackconf.sh\n",
      "upload: awsome-distributed-training/1.architectures/5.sagemaker-hyperpod/LifecycleScripts/base-config/utils/mount-s3.sh to s3://sagemaker-us-west-2-533267176336/hyperpod/LifecycleScripts/utils/mount-s3.sh\n",
      "upload: awsome-distributed-training/1.architectures/5.sagemaker-hyperpod/LifecycleScripts/base-config/utils/install_slurm_exporter.sh to s3://sagemaker-us-west-2-533267176336/hyperpod/LifecycleScripts/utils/install_slurm_exporter.sh\n",
      "upload: awsome-distributed-training/1.architectures/5.sagemaker-hyperpod/LifecycleScripts/base-config/utils/pam_adopt_cgroup_wheel.sh to s3://sagemaker-us-west-2-533267176336/hyperpod/LifecycleScripts/utils/pam_adopt_cgroup_wheel.sh\n",
      "upload: lifecycle-scripts/provisioning_parameters.json to s3://sagemaker-us-west-2-533267176336/hyperpod/LifecycleScripts/provisioning_parameters.json\n"
     ]
    }
   ],
   "source": [
    "!aws s3 cp --recursive awsome-distributed-training/1.architectures/5.sagemaker-hyperpod/LifecycleScripts/base-config/ s3://{bucket}/hyperpod/LifecycleScripts/\n",
    "!aws s3 cp --recursive lifecycle-scripts/ s3://{bucket}/hyperpod/LifecycleScripts/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 准备 create_cluster.json 文件，用于 CreateCluster"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from datetime import datetime\n",
    "import json\n",
    "\n",
    "cluster_name = f\"hyperpod-cluster-1\"\n",
    "\n",
    "SourceS3Uri = f\"s3://{bucket}/hyperpod/LifecycleScripts\"\n",
    "worker_instance = \"ml.g5.xlarge\"\n",
    "worker_count = 2\n",
    "\n",
    "create_cluster = \\\n",
    "{\n",
    "    \"ClusterName\": cluster_name,\n",
    "    \"InstanceGroups\": [\n",
    "        {\n",
    "            \"InstanceGroupName\": \"my-controller-group\",\n",
    "            \"InstanceType\": \"ml.c5.xlarge\",\n",
    "            \"InstanceCount\": 1,\n",
    "            \"LifeCycleConfig\": {\n",
    "              \"SourceS3Uri\": SourceS3Uri,\n",
    "              \"OnCreate\": \"on_create.sh\"\n",
    "            },\n",
    "            \"ExecutionRole\": role,\n",
    "            \"ThreadsPerCore\": 1\n",
    "        },\n",
    "        {\n",
    "            \"InstanceGroupName\": \"worker-group-1\",\n",
    "            \"InstanceType\": worker_instance,\n",
    "            \"InstanceCount\": worker_count,\n",
    "            \"LifeCycleConfig\": {\n",
    "              \"SourceS3Uri\": SourceS3Uri,\n",
    "              \"OnCreate\": \"on_create.sh\"\n",
    "            },\n",
    "            \"ExecutionRole\": role,\n",
    "            \"ThreadsPerCore\": 1\n",
    "        }\n",
    "    ]\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'ClusterName': 'hyperpod-cluster-1',\n",
       " 'InstanceGroups': [{'InstanceGroupName': 'my-controller-group',\n",
       "   'InstanceType': 'ml.c5.xlarge',\n",
       "   'InstanceCount': 1,\n",
       "   'LifeCycleConfig': {'SourceS3Uri': 's3://sagemaker-us-west-2-533267176336/hyperpod/LifecycleScripts',\n",
       "    'OnCreate': 'on_create.sh'},\n",
       "   'ExecutionRole': 'arn:aws:iam::533267176336:role/CloudLabCampaignInfraStack-sagemaker--ExecutionRole-rC2M3kfw97Tz',\n",
       "   'ThreadsPerCore': 1},\n",
       "  {'InstanceGroupName': 'worker-group-1',\n",
       "   'InstanceType': 'ml.g5.xlarge',\n",
       "   'InstanceCount': 2,\n",
       "   'LifeCycleConfig': {'SourceS3Uri': 's3://sagemaker-us-west-2-533267176336/hyperpod/LifecycleScripts',\n",
       "    'OnCreate': 'on_create.sh'},\n",
       "   'ExecutionRole': 'arn:aws:iam::533267176336:role/CloudLabCampaignInfraStack-sagemaker--ExecutionRole-rC2M3kfw97Tz',\n",
       "   'ThreadsPerCore': 1}]}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open(\"create_cluster.json\", \"w\") as f:\n",
    "    json.dump(create_cluster, f)\n",
    "create_cluster"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 运行以下命令创建集群"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "    \"ClusterArn\": \"arn:aws:sagemaker:us-west-2:533267176336:cluster/zr6altu3u2cj\"\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "!aws sagemaker create-cluster --cli-input-json file://~/SageMaker/LLM_Fintune_using_SageMaker_hyperpod_with_LLama_Factory/create_cluster.json"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 持续检测集群部署进度"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">[02/12/25 09:45:24] </span><span style=\"color: #0069ff; text-decoration-color: #0069ff; font-weight: bold\">INFO    </span> Found credentials from IAM Role:                                   <a href=\"file:///home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages/botocore/credentials.py\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">credentials.py</span></a><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">:</span><a href=\"file:///home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages/botocore/credentials.py#1075\" target=\"_blank\"><span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">1075</span></a>\n",
       "<span style=\"color: #7fbfbf; text-decoration-color: #7fbfbf\">                    </span>         BaseNotebookInstanceEc2InstanceRole                                <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">                   </span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[2;36m[02/12/25 09:45:24]\u001b[0m\u001b[2;36m \u001b[0m\u001b[1;38;2;0;105;255mINFO    \u001b[0m Found credentials from IAM Role:                                   \u001b]8;id=547770;file:///home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages/botocore/credentials.py\u001b\\\u001b[2mcredentials.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=457872;file:///home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages/botocore/credentials.py#1075\u001b\\\u001b[2m1075\u001b[0m\u001b]8;;\u001b\\\n",
       "\u001b[2;36m                    \u001b[0m         BaseNotebookInstanceEc2InstanceRole                                \u001b[2m                   \u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import time\n",
    "sm_client = boto3.client(\"sagemaker\")  # client to intreract with SageMaker"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Status: Creating\n",
      "Status: Creating\n",
      "Status: Creating\n",
      "Status: Creating\n",
      "Status: Creating\n",
      "Status: Creating\n",
      "Status: Creating\n",
      "Status: Creating\n",
      "Status: Creating\n",
      "Status: Creating\n",
      "Status: Creating\n",
      "Status: InService\n",
      "Arn: InService\n",
      "Status: InService\n"
     ]
    }
   ],
   "source": [
    "resp = sm_client.describe_cluster(\n",
    "    ClusterName=cluster_name\n",
    ")\n",
    "status = resp[\"ClusterStatus\"]\n",
    "print(\"Status: \" + status)\n",
    "\n",
    "while status == \"Creating\":\n",
    "    time.sleep(60)\n",
    "    resp = sm_client.describe_cluster(\n",
    "        ClusterName=cluster_name\n",
    "    )\n",
    "    status = resp[\"ClusterStatus\"]\n",
    "    print(\"Status: \" + status)\n",
    "\n",
    "print(\"Arn: \" + resp[\"ClusterStatus\"])\n",
    "print(\"Status: \" + status)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 列出集群 node 信息"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "    \"ClusterSummaries\": [\n",
      "        {\n",
      "            \"ClusterArn\": \"arn:aws:sagemaker:us-west-2:533267176336:cluster/zr6altu3u2cj\",\n",
      "            \"ClusterName\": \"hyperpod-cluster-1\",\n",
      "            \"CreationTime\": 1739353521.38,\n",
      "            \"ClusterStatus\": \"InService\",\n",
      "            \"TrainingPlanArns\": []\n",
      "        }\n",
      "    ]\n",
      "}\n",
      "{\n",
      "    \"ClusterNodeSummaries\": [\n",
      "        {\n",
      "            \"InstanceGroupName\": \"my-controller-group\",\n",
      "            \"InstanceId\": \"i-08f84209becd0c93f\",\n",
      "            \"InstanceType\": \"ml.c5.xlarge\",\n",
      "            \"LaunchTime\": 1739353528.489,\n",
      "            \"InstanceStatus\": {\n",
      "                \"Status\": \"Running\",\n",
      "                \"Message\": \"\"\n",
      "            }\n",
      "        },\n",
      "        {\n",
      "            \"InstanceGroupName\": \"worker-group-1\",\n",
      "            \"InstanceId\": \"i-0bd73cef642c655aa\",\n",
      "            \"InstanceType\": \"ml.g5.xlarge\",\n",
      "            \"LaunchTime\": 1739353529.663,\n",
      "            \"InstanceStatus\": {\n",
      "                \"Status\": \"Running\",\n",
      "                \"Message\": \"\"\n",
      "            }\n",
      "        },\n",
      "        {\n",
      "            \"InstanceGroupName\": \"worker-group-1\",\n",
      "            \"InstanceId\": \"i-0bec26502e13e7ca5\",\n",
      "            \"InstanceType\": \"ml.g5.xlarge\",\n",
      "            \"LaunchTime\": 1739353529.663,\n",
      "            \"InstanceStatus\": {\n",
      "                \"Status\": \"Running\",\n",
      "                \"Message\": \"\"\n",
      "            }\n",
      "        }\n",
      "    ]\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "!aws sagemaker list-clusters\n",
    "!aws sagemaker list-cluster-nodes --cluster-name {cluster_name} --region us-west-2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### (1) 使用 awsome-distributed-training/1.architectures/5.sagemaker-hyperpod/easy-ssh.sh 快速登录"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. 打开 notebook 上的终端 Terminal，依次执行以下命令：\n",
    "```bash\n",
    "sudo su\n",
    "cd /home/ec2-user/SageMaker/LLM_Fintune_using_SageMaker_hyperpod_with_LLama_Factory\n",
    "```\n",
    "2. 安装ssm plugin。\n",
    "```bash\n",
    "sudo yum install -y https://s3.amazonaws.com/session-manager-downloads/plugin/latest/linux_64bit/session-manager-plugin.rpm\n",
    "```\n",
    "\n",
    "3. 在此 Terminal 中输入 session-manager-plugin，检验是否成功安装。\n",
    "```bash\n",
    "session-manager-plugin\n",
    "\n",
    "```\n",
    "\n",
    "4. 将 session-manager-plugin 加入到环境变量中。\n",
    "```bash\n",
    "find / -name \"session-manager-plugin\"\n",
    "echo $PATH\n",
    "cp /usr/local/bin/session-manager-plugin /usr/bin\n",
    "session-manager-plugin\n",
    "```\n",
    "\n",
    "5. 拷贝easy-ssh.sh到当前用户目录。\n",
    "```bash\n",
    "cp awsome-distributed-training/1.architectures/5.sagemaker-hyperpod/easy-ssh.sh ./\n",
    "```\n",
    "\n",
    "6. 登录工作节点。\n",
    "```bash\n",
    "chmod +x easy-ssh.sh\n",
    "sudo su ec2-user\n",
    "cluster_name=hyperpod-cluster-1\n",
    "group=worker-group-1\n",
    "./easy-ssh.sh $cluster_name --controller-group $group\n",
    "```\n",
    "\n",
    "7. 登录进入之后，切换到 ubuntu 用户,运行 sinfo 查看当前集群状态。\n",
    "```bash\n",
    "sudo su ubuntu\n",
    "sinfo\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### (2) 上传训练脚本到 S3 bucket 中，之后 S3 bucket 会挂载到集群所有节点中，这样所有计算节点都可以访问训练代码和数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cp LLaMA-Factory/.github/CODE_OF_CONDUCT.md s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/.github/CODE_OF_CONDUCT.md\n",
      "cp LLaMA-Factory/.gitignore s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/.gitignore\n",
      "cp LLaMA-Factory/.env.local s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/.env.local\n",
      "cp LLaMA-Factory/.gitattributes s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/.gitattributes\n",
      "cp LLaMA-Factory/.github/workflows/publish.yml s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/.github/workflows/publish.yml\n",
      "cp LLaMA-Factory/.dockerignore s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/.dockerignore\n",
      "cp LLaMA-Factory/.github/workflows/tests.yml s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/.github/workflows/tests.yml\n",
      "cp LLaMA-Factory/.github/workflows/label_issue.yml s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/.github/workflows/label_issue.yml\n",
      "cp LLaMA-Factory/LICENSE s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/LICENSE\n",
      "cp LLaMA-Factory/MANIFEST.in s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/MANIFEST.in\n",
      "cp LLaMA-Factory/.github/CONTRIBUTING.md s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/.github/CONTRIBUTING.md\n",
      "cp LLaMA-Factory/.github/PULL_REQUEST_TEMPLATE.md s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/.github/PULL_REQUEST_TEMPLATE.md\n",
      "cp LLaMA-Factory/.github/ISSUE_TEMPLATE/bug-report.yml s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/.github/ISSUE_TEMPLATE/bug-report.yml\n",
      "cp LLaMA-Factory/data/dataset_info.json s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/data/dataset_info.json\n",
      "cp LLaMA-Factory/data/identity.json s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/data/identity.json\n",
      "cp LLaMA-Factory/data/belle_multiturn/belle_multiturn.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/data/belle_multiturn/belle_multiturn.py\n",
      "cp LLaMA-Factory/data/hh_rlhf_en/hh_rlhf_en.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/data/hh_rlhf_en/hh_rlhf_en.py\n",
      "cp LLaMA-Factory/data/README.md s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/data/README.md\n",
      "cp LLaMA-Factory/entry-multi-nodes.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/entry-multi-nodes.py\n",
      "cp LLaMA-Factory/examples/inference/qwen2_vl.yaml s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/examples/inference/qwen2_vl.yaml\n",
      "cp LLaMA-Factory/evaluation/ceval/mapping.json s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/evaluation/ceval/mapping.json\n",
      "cp LLaMA-Factory/examples/extras/loraplus/llama3_lora_sft.yaml s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/examples/extras/loraplus/llama3_lora_sft.yaml\n",
      "cp LLaMA-Factory/examples/inference/llama3_lora_sft.yaml s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/examples/inference/llama3_lora_sft.yaml\n",
      "cp LLaMA-Factory/docker/docker-npu/docker-compose.yml s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/docker/docker-npu/docker-compose.yml\n",
      "cp LLaMA-Factory/examples/deepspeed/ds_z0_config.json s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/examples/deepspeed/ds_z0_config.json\n",
      "cp LLaMA-Factory/examples/deepspeed/ds_z2_offload_config.json s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/examples/deepspeed/ds_z2_offload_config.json\n",
      "cp LLaMA-Factory/examples/extras/llama_pro/llama3_freeze_sft.yaml s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/examples/extras/llama_pro/llama3_freeze_sft.yaml\n",
      "cp LLaMA-Factory/examples/train_lora/llama3_lora_ppo.yaml s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/examples/train_lora/llama3_lora_ppo.yaml\n",
      "cp LLaMA-Factory/data/ultra_chat/ultra_chat.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/data/ultra_chat/ultra_chat.py\n",
      "cp LLaMA-Factory/docker/docker-cuda/docker-compose.yml s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/docker/docker-cuda/docker-compose.yml\n",
      "cp LLaMA-Factory/evaluation/cmmlu/mapping.json s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/evaluation/cmmlu/mapping.json\n",
      "cp LLaMA-Factory/.github/SECURITY.md s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/.github/SECURITY.md\n",
      "cp LLaMA-Factory/evaluation/mmlu/mmlu.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/evaluation/mmlu/mmlu.py\n",
      "cp LLaMA-Factory/assets/logo.png s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/assets/logo.png\n",
      "cp LLaMA-Factory/evaluation/ceval/ceval.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/evaluation/ceval/ceval.py\n",
      "cp LLaMA-Factory/data/mllm_video_demo.json s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/data/mllm_video_demo.json\n",
      "cp LLaMA-Factory/examples/deepspeed/ds_z3_offload_config.json s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/examples/deepspeed/ds_z3_offload_config.json\n",
      "cp LLaMA-Factory/examples/train_full/llama3_full_predict.yaml s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/examples/train_full/llama3_full_predict.yaml\n",
      "cp LLaMA-Factory/examples/extras/adam_mini/qwen2_full_sft.yaml s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/examples/extras/adam_mini/qwen2_full_sft.yaml\n",
      "cp LLaMA-Factory/examples/train_lora/llama3_lora_pretrain.yaml s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/examples/train_lora/llama3_lora_pretrain.yaml\n",
      "cp LLaMA-Factory/examples/train_full/llama3_full_sft_ds3.yaml s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/examples/train_full/llama3_full_sft_ds3.yaml\n",
      "cp LLaMA-Factory/examples/inference/llama3_vllm.yaml s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/examples/inference/llama3_vllm.yaml\n",
      "cp LLaMA-Factory/examples/train_lora/llama3_lora_sft_ds3.yaml s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/examples/train_lora/llama3_lora_sft_ds3.yaml\n",
      "cp LLaMA-Factory/assets/benchmark.svg s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/assets/benchmark.svg\n",
      "cp LLaMA-Factory/examples/inference/llama3.yaml s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/examples/inference/llama3.yaml\n",
      "cp LLaMA-Factory/data/mllm_demo.json s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/data/mllm_demo.json\n",
      "cp LLaMA-Factory/docker/docker-rocm/Dockerfile s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/docker/docker-rocm/Dockerfile\n",
      "cp LLaMA-Factory/examples/train_lora/llama3_lora_sft.yaml s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/examples/train_lora/llama3_lora_sft.yaml\n",
      "cp LLaMA-Factory/examples/extras/fsdp_qlora/llama3_lora_sft.yaml s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/examples/extras/fsdp_qlora/llama3_lora_sft.yaml\n",
      "cp LLaMA-Factory/evaluation/mmlu/mapping.json s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/evaluation/mmlu/mapping.json\n",
      "cp LLaMA-Factory/evaluation/cmmlu/cmmlu.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/evaluation/cmmlu/cmmlu.py\n",
      "cp LLaMA-Factory/entry_single_lora.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/entry_single_lora.py\n",
      "cp LLaMA-Factory/examples/train_lora/llava1_5_lora_sft.yaml s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/examples/train_lora/llava1_5_lora_sft.yaml\n",
      "cp LLaMA-Factory/examples/train_qlora/llama3_lora_sft_gptq.yaml s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/examples/train_qlora/llama3_lora_sft_gptq.yaml\n",
      "cp LLaMA-Factory/docker/docker-cuda/Dockerfile s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/docker/docker-cuda/Dockerfile\n",
      "cp LLaMA-Factory/examples/extras/pissa/init.sh s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/examples/extras/pissa/init.sh\n",
      "cp LLaMA-Factory/scripts/cal_mfu.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/scripts/cal_mfu.py\n",
      "cp LLaMA-Factory/examples/train_lora/llama3_lora_sft_ds0.yaml s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/examples/train_lora/llama3_lora_sft_ds0.yaml\n",
      "cp LLaMA-Factory/examples/train_lora/qwen2vl_lora_sft.yaml s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/examples/train_lora/qwen2vl_lora_sft.yaml\n",
      "cp LLaMA-Factory/Makefile s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/Makefile\n",
      "cp LLaMA-Factory/scripts/llama_pro.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/scripts/llama_pro.py\n",
      "cp LLaMA-Factory/README.md s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/README.md\n",
      "cp LLaMA-Factory/examples/README.md s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/examples/README.md\n",
      "cp LLaMA-Factory/examples/inference/llava1_5.yaml s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/examples/inference/llava1_5.yaml\n",
      "cp LLaMA-Factory/examples/train_qlora/llama3_lora_sft_aqlm.yaml s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/examples/train_qlora/llama3_lora_sft_aqlm.yaml\n",
      "cp LLaMA-Factory/examples/train_qlora/llama3_lora_sft_awq.yaml s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/examples/train_qlora/llama3_lora_sft_awq.yaml\n",
      "cp LLaMA-Factory/examples/train_lora/llama3_lora_reward.yaml s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/examples/train_lora/llama3_lora_reward.yaml\n",
      "cp LLaMA-Factory/examples/train_qlora/llama3_lora_sft_otfq.yaml s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/examples/train_qlora/llama3_lora_sft_otfq.yaml\n",
      "cp LLaMA-Factory/examples/merge_lora/qwen2vl_lora_sft.yaml s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/examples/merge_lora/qwen2vl_lora_sft.yaml\n",
      "cp LLaMA-Factory/examples/extras/badam/llama3_full_sft.yaml s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/examples/extras/badam/llama3_full_sft.yaml\n",
      "cp LLaMA-Factory/examples/deepspeed/ds_z3_config.json s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/examples/deepspeed/ds_z3_config.json\n",
      "cp LLaMA-Factory/examples/accelerate/fsdp_config.yaml s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/examples/accelerate/fsdp_config.yaml\n",
      "cp LLaMA-Factory/scripts/cal_ppl.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/scripts/cal_ppl.py\n",
      "cp LLaMA-Factory/src/api.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/api.py\n",
      "cp LLaMA-Factory/scripts/length_cdf.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/scripts/length_cdf.py\n",
      "cp LLaMA-Factory/CITATION.cff s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/CITATION.cff\n",
      "cp LLaMA-Factory/examples/merge_lora/llama3_gptq.yaml s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/examples/merge_lora/llama3_gptq.yaml\n",
      "cp LLaMA-Factory/examples/train_lora/qwen2vl_lora_dpo.yaml s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/examples/train_lora/qwen2vl_lora_dpo.yaml\n",
      "cp LLaMA-Factory/src/llamafactory/chat/hf_engine.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/chat/hf_engine.py\n",
      "cp LLaMA-Factory/src/llamafactory/data/collator.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/data/collator.py\n",
      "cp LLaMA-Factory/examples/extras/fsdp_qlora/train.sh s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/examples/extras/fsdp_qlora/train.sh\n",
      "cp LLaMA-Factory/src/llamafactory/chat/__init__.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/chat/__init__.py\n",
      "cp LLaMA-Factory/setup.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/setup.py\n",
      "cp LLaMA-Factory/requirements.txt s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/requirements.txt\n",
      "cp LLaMA-Factory/src/llamafactory/cli.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/cli.py\n",
      "cp LLaMA-Factory/src/llamafactory/data/processors/pairwise.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/data/processors/pairwise.py\n",
      "cp LLaMA-Factory/scripts/test_toolcall.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/scripts/test_toolcall.py\n",
      "cp LLaMA-Factory/src/llamafactory/extras/logging.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/extras/logging.py\n",
      "cp LLaMA-Factory/src/llamafactory/data/preprocess.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/data/preprocess.py\n",
      "cp LLaMA-Factory/src/llamafactory/data/processors/unsupervised.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/data/processors/unsupervised.py\n",
      "cp LLaMA-Factory/src/llamafactory/api/__init__.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/api/__init__.py\n",
      "cp LLaMA-Factory/src/llamafactory/eval/__init__.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/eval/__init__.py\n",
      "cp LLaMA-Factory/src/llamafactory/data/parser.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/data/parser.py\n",
      "cp LLaMA-Factory/src/llamafactory/data/__init__.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/data/__init__.py\n",
      "cp LLaMA-Factory/src/llamafactory/__init__.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/__init__.py\n",
      "cp LLaMA-Factory/examples/train_lora/llama3_lora_predict.yaml s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/examples/train_lora/llama3_lora_predict.yaml\n",
      "cp LLaMA-Factory/data/mllm_demo_data/2.avi s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/data/mllm_demo_data/2.avi\n",
      "cp LLaMA-Factory/examples/extras/llama_pro/expand.sh s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/examples/extras/llama_pro/expand.sh\n",
      "cp LLaMA-Factory/src/llamafactory/data/loader.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/data/loader.py\n",
      "cp LLaMA-Factory/examples/extras/pissa/llama3_lora_sft.yaml s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/examples/extras/pissa/llama3_lora_sft.yaml\n",
      "cp LLaMA-Factory/src/llamafactory/api/common.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/api/common.py\n",
      "cp LLaMA-Factory/src/llamafactory/eval/template.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/eval/template.py\n",
      "cp LLaMA-Factory/examples/train_lora/llama3_lora_kto.yaml s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/examples/train_lora/llama3_lora_kto.yaml\n",
      "cp LLaMA-Factory/data/README_zh.md s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/data/README_zh.md\n",
      "cp LLaMA-Factory/src/llamafactory/chat/chat_model.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/chat/chat_model.py\n",
      "cp LLaMA-Factory/scripts/cal_flops.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/scripts/cal_flops.py\n",
      "cp LLaMA-Factory/src/llamafactory/extras/packages.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/extras/packages.py\n",
      "cp LLaMA-Factory/scripts/llamafy_baichuan2.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/scripts/llamafy_baichuan2.py\n",
      "cp LLaMA-Factory/src/llamafactory/hparams/evaluation_args.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/hparams/evaluation_args.py\n",
      "cp LLaMA-Factory/examples/train_lora/llama3_lora_dpo.yaml s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/examples/train_lora/llama3_lora_dpo.yaml\n",
      "cp LLaMA-Factory/src/llamafactory/data/tool_utils.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/data/tool_utils.py\n",
      "cp LLaMA-Factory/data/mllm_demo_data/3.jpg s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/data/mllm_demo_data/3.jpg\n",
      "cp LLaMA-Factory/pyproject.toml s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/pyproject.toml\n",
      "cp LLaMA-Factory/src/llamafactory/train/__init__.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/train/__init__.py\n",
      "cp LLaMA-Factory/src/llamafactory/model/model_utils/__init__.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/model/model_utils/__init__.py\n",
      "cp LLaMA-Factory/examples/extras/mod/llama3_full_sft.yaml s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/examples/extras/mod/llama3_full_sft.yaml\n",
      "cp LLaMA-Factory/scripts/llamafy_qwen.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/scripts/llamafy_qwen.py\n",
      "cp LLaMA-Factory/src/llamafactory/hparams/finetuning_args.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/hparams/finetuning_args.py\n",
      "cp LLaMA-Factory/scripts/cal_lr.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/scripts/cal_lr.py\n",
      "cp LLaMA-Factory/sg_config_qlora.yaml s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/sg_config_qlora.yaml\n",
      "cp LLaMA-Factory/examples/extras/galore/llama3_full_sft.yaml s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/examples/extras/galore/llama3_full_sft.yaml\n",
      "cp LLaMA-Factory/src/llamafactory/data/processors/__init__.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/data/processors/__init__.py\n",
      "cp LLaMA-Factory/src/llamafactory/model/model_utils/checkpointing.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/model/model_utils/checkpointing.py\n",
      "cp LLaMA-Factory/docker/docker-rocm/docker-compose.yml s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/docker/docker-rocm/docker-compose.yml\n",
      "cp LLaMA-Factory/src/llamafactory/model/model_utils/longlora.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/model/model_utils/longlora.py\n",
      "cp LLaMA-Factory/src/llamafactory/model/model_utils/embedding.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/model/model_utils/embedding.py\n",
      "cp LLaMA-Factory/src/llamafactory/data/processors/pretrain.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/data/processors/pretrain.py\n",
      "cp LLaMA-Factory/src/llamafactory/chat/base_engine.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/chat/base_engine.py\n",
      "cp LLaMA-Factory/src/llamafactory/data/formatter.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/data/formatter.py\n",
      "cp LLaMA-Factory/scripts/pissa_init.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/scripts/pissa_init.py\n",
      "cp LLaMA-Factory/src/llamafactory/model/model_utils/rope.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/model/model_utils/rope.py\n",
      "cp LLaMA-Factory/src/llamafactory/extras/env.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/extras/env.py\n",
      "cp LLaMA-Factory/assets/wechat_npu.jpg s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/assets/wechat_npu.jpg\n",
      "cp LLaMA-Factory/src/llamafactory/chat/vllm_engine.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/chat/vllm_engine.py\n",
      "cp LLaMA-Factory/data/glaive_toolcall_en_demo.json s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/data/glaive_toolcall_en_demo.json\n",
      "cp LLaMA-Factory/src/llamafactory/train/callbacks.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/train/callbacks.py\n",
      "cp LLaMA-Factory/src/llamafactory/model/model_utils/valuehead.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/model/model_utils/valuehead.py\n",
      "cp LLaMA-Factory/examples/deepspeed/ds_z2_config.json s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/examples/deepspeed/ds_z2_config.json\n",
      "cp LLaMA-Factory/assets/wechat.jpg s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/assets/wechat.jpg\n",
      "cp LLaMA-Factory/src/llamafactory/hparams/data_args.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/hparams/data_args.py\n",
      "cp LLaMA-Factory/src/llamafactory/train/ppo/ppo_utils.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/train/ppo/ppo_utils.py\n",
      "cp LLaMA-Factory/src/llamafactory/model/adapter.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/model/adapter.py\n",
      "cp LLaMA-Factory/examples/train_full/qwen2vl_full_sft.yaml s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/examples/train_full/qwen2vl_full_sft.yaml\n",
      "cp LLaMA-Factory/src/llamafactory/model/model_utils/mod.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/model/model_utils/mod.py\n",
      "cp LLaMA-Factory/src/llamafactory/webui/__init__.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/webui/__init__.py\n",
      "cp LLaMA-Factory/src/llamafactory/model/model_utils/quantization.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/model/model_utils/quantization.py\n",
      "cp LLaMA-Factory/src/llamafactory/hparams/model_args.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/hparams/model_args.py\n",
      "cp LLaMA-Factory/src/llamafactory/extras/__init__.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/extras/__init__.py\n",
      "cp LLaMA-Factory/src/llamafactory/train/sft/metric.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/train/sft/metric.py\n",
      "cp LLaMA-Factory/src/llamafactory/train/sft/trainer.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/train/sft/trainer.py\n",
      "cp LLaMA-Factory/data/mllm_demo_data/2.jpg s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/data/mllm_demo_data/2.jpg\n",
      "cp LLaMA-Factory/src/llamafactory/webui/components/chatbot.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/webui/components/chatbot.py\n",
      "cp LLaMA-Factory/src/llamafactory/model/model_utils/attention.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/model/model_utils/attention.py\n",
      "cp LLaMA-Factory/src/llamafactory/train/rm/metric.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/train/rm/metric.py\n",
      "cp LLaMA-Factory/src/llamafactory/train/pt/__init__.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/train/pt/__init__.py\n",
      "cp LLaMA-Factory/src/llamafactory/hparams/generating_args.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/hparams/generating_args.py\n",
      "cp LLaMA-Factory/src/llamafactory/api/chat.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/api/chat.py\n",
      "cp LLaMA-Factory/src/llamafactory/model/model_utils/misc.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/model/model_utils/misc.py\n",
      "cp LLaMA-Factory/src/llamafactory/model/model_utils/packing.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/model/model_utils/packing.py\n",
      "cp LLaMA-Factory/src/llamafactory/webui/components/eval.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/webui/components/eval.py\n",
      "cp LLaMA-Factory/src/llamafactory/api/protocol.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/api/protocol.py\n",
      "cp LLaMA-Factory/src/llamafactory/webui/chatter.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/webui/chatter.py\n",
      "cp LLaMA-Factory/src/llamafactory/data/aligner.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/data/aligner.py\n",
      "cp LLaMA-Factory/src/llamafactory/train/tuner.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/train/tuner.py\n",
      "cp LLaMA-Factory/src/llamafactory/webui/components/top.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/webui/components/top.py\n",
      "cp LLaMA-Factory/src/llamafactory/data/processors/feedback.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/data/processors/feedback.py\n",
      "cp LLaMA-Factory/src/llamafactory/webui/common.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/webui/common.py\n",
      "cp LLaMA-Factory/src/llamafactory/model/model_utils/visual.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/model/model_utils/visual.py\n",
      "cp LLaMA-Factory/src/llamafactory/webui/components/export.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/webui/components/export.py\n",
      "cp LLaMA-Factory/tests/data/processors/test_pairwise.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/tests/data/processors/test_pairwise.py\n",
      "cp LLaMA-Factory/src/llamafactory/train/ppo/workflow.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/train/ppo/workflow.py\n",
      "cp LLaMA-Factory/src/llamafactory/train/dpo/workflow.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/train/dpo/workflow.py\n",
      "cp LLaMA-Factory/src/llamafactory/model/loader.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/model/loader.py\n",
      "cp LLaMA-Factory/src/llamafactory/model/__init__.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/model/__init__.py\n",
      "cp LLaMA-Factory/tests/data/processors/test_supervised.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/tests/data/processors/test_supervised.py\n",
      "cp LLaMA-Factory/src/train.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/train.py\n",
      "cp LLaMA-Factory/data/mllm_demo_data/1.jpg s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/data/mllm_demo_data/1.jpg\n",
      "cp LLaMA-Factory/README_zh.md s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/README_zh.md\n",
      "cp LLaMA-Factory/tests/data/processors/test_unsupervised.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/tests/data/processors/test_unsupervised.py\n",
      "cp LLaMA-Factory/src/llamafactory/extras/misc.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/extras/misc.py\n",
      "cp LLaMA-Factory/src/llamafactory/train/dpo/trainer.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/train/dpo/trainer.py\n",
      "cp LLaMA-Factory/tests/model/model_utils/test_attention.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/tests/model/model_utils/test_attention.py\n",
      "cp LLaMA-Factory/src/llamafactory/train/rm/trainer.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/train/rm/trainer.py\n",
      "cp LLaMA-Factory/src/llamafactory/train/ppo/__init__.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/train/ppo/__init__.py\n",
      "cp LLaMA-Factory/tests/data/processors/test_processor_utils.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/tests/data/processors/test_processor_utils.py\n",
      "cp LLaMA-Factory/src/llamafactory/train/kto/workflow.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/train/kto/workflow.py\n",
      "cp LLaMA-Factory/src/llamafactory/train/kto/__init__.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/train/kto/__init__.py\n",
      "cp LLaMA-Factory/tests/model/test_base.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/tests/model/test_base.py\n",
      "cp LLaMA-Factory/src/llamafactory/extras/ploting.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/extras/ploting.py\n",
      "cp LLaMA-Factory/src/llamafactory/model/model_utils/moe.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/model/model_utils/moe.py\n",
      "cp LLaMA-Factory/src/llamafactory/eval/evaluator.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/eval/evaluator.py\n",
      "cp LLaMA-Factory/src/llamafactory/webui/css.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/webui/css.py\n",
      "cp LLaMA-Factory/data/alpaca_zh_demo.json s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/data/alpaca_zh_demo.json\n",
      "cp LLaMA-Factory/src/llamafactory/model/patcher.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/model/patcher.py\n",
      "cp LLaMA-Factory/src/llamafactory/train/kto/trainer.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/train/kto/trainer.py\n",
      "cp LLaMA-Factory/src/llamafactory/data/data_utils.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/data/data_utils.py\n",
      "cp LLaMA-Factory/src/llamafactory/webui/components/__init__.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/webui/components/__init__.py\n",
      "cp LLaMA-Factory/tests/data/processors/test_feedback.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/tests/data/processors/test_feedback.py\n",
      "cp LLaMA-Factory/tests/eval/test_eval_template.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/tests/eval/test_eval_template.py\n",
      "cp LLaMA-Factory/tests/e2e/test_train.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/tests/e2e/test_train.py\n",
      "cp LLaMA-Factory/src/llamafactory/data/processors/processor_utils.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/data/processors/processor_utils.py\n",
      "cp LLaMA-Factory/src/llamafactory/train/rm/__init__.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/train/rm/__init__.py\n",
      "cp LLaMA-Factory/src/llamafactory/launcher.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/launcher.py\n",
      "cp LLaMA-Factory/src/llamafactory/train/pt/trainer.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/train/pt/trainer.py\n",
      "cp LLaMA-Factory/examples/train_lora/llama3_lora_eval.yaml s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/examples/train_lora/llama3_lora_eval.yaml\n",
      "cp LLaMA-Factory/tests/model/test_pissa.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/tests/model/test_pissa.py\n",
      "cp LLaMA-Factory/src/llamafactory/model/model_utils/unsloth.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/model/model_utils/unsloth.py\n",
      "cp LLaMA-Factory/src/llamafactory/train/sft/workflow.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/train/sft/workflow.py\n",
      "cp LLaMA-Factory/evaluation/mmlu/mmlu.zip s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/evaluation/mmlu/mmlu.zip\n",
      "cp LLaMA-Factory/tests/data/test_formatter.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/tests/data/test_formatter.py\n",
      "cp LLaMA-Factory/src/llamafactory/webui/components/infer.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/webui/components/infer.py\n",
      "cp LLaMA-Factory/src/llamafactory/webui/manager.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/webui/manager.py\n",
      "cp LLaMA-Factory/src/llamafactory/webui/components/data.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/webui/components/data.py\n",
      "cp LLaMA-Factory/tests/data/test_template.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/tests/data/test_template.py\n",
      "cp LLaMA-Factory/src/llamafactory/train/ppo/trainer.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/train/ppo/trainer.py\n",
      "cp LLaMA-Factory/tests/model/test_freeze.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/tests/model/test_freeze.py\n",
      "cp LLaMA-Factory/src/llamafactory/train/rm/workflow.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/train/rm/workflow.py\n",
      "cp LLaMA-Factory/tests/model/model_utils/test_packing.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/tests/model/model_utils/test_packing.py\n",
      "cp LLaMA-Factory/src/llamafactory/webui/interface.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/webui/interface.py\n",
      "cp LLaMA-Factory/src/llamafactory/data/mm_plugin.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/data/mm_plugin.py\n",
      "cp LLaMA-Factory/src/llamafactory/data/processors/supervised.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/data/processors/supervised.py\n",
      "cp LLaMA-Factory/scripts/loftq_init.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/scripts/loftq_init.py\n",
      "cp LLaMA-Factory/examples/merge_lora/llama3_lora_sft.yaml s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/examples/merge_lora/llama3_lora_sft.yaml\n",
      "cp LLaMA-Factory/tests/model/model_utils/test_checkpointing.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/tests/model/model_utils/test_checkpointing.py\n",
      "cp LLaMA-Factory/tests/e2e/test_chat.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/tests/e2e/test_chat.py\n",
      "cp LLaMA-Factory/examples/train_lora/llama3_preprocess.yaml s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/examples/train_lora/llama3_preprocess.yaml\n",
      "cp LLaMA-Factory/src/llamafactory/train/dpo/__init__.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/train/dpo/__init__.py\n",
      "cp LLaMA-Factory/src/webui.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/webui.py\n",
      "cp LLaMA-Factory/src/llamafactory/webui/engine.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/webui/engine.py\n",
      "cp LLaMA-Factory/src/llamafactory/model/model_utils/liger_kernel.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/model/model_utils/liger_kernel.py\n",
      "cp LLaMA-Factory/src/llamafactory/hparams/__init__.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/hparams/__init__.py\n",
      "cp LLaMA-Factory/data/wiki_demo.txt s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/data/wiki_demo.txt\n",
      "cp LLaMA-Factory/docker/docker-npu/Dockerfile s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/docker/docker-npu/Dockerfile\n",
      "cp LLaMA-Factory/src/llamafactory/api/app.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/api/app.py\n",
      "cp LLaMA-Factory/tests/data/test_collator.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/tests/data/test_collator.py\n",
      "cp LLaMA-Factory/src/llamafactory/train/pt/workflow.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/train/pt/workflow.py\n",
      "cp LLaMA-Factory/src/llamafactory/data/template.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/data/template.py\n",
      "cp LLaMA-Factory/data/c4_demo.json s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/data/c4_demo.json\n",
      "cp LLaMA-Factory/src/llamafactory/webui/components/train.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/webui/components/train.py\n",
      "cp LLaMA-Factory/evaluation/cmmlu/cmmlu.zip s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/evaluation/cmmlu/cmmlu.zip\n",
      "cp LLaMA-Factory/src/llamafactory/extras/constants.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/extras/constants.py\n",
      "cp LLaMA-Factory/examples/README_zh.md s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/examples/README_zh.md\n",
      "cp LLaMA-Factory/src/llamafactory/hparams/parser.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/hparams/parser.py\n",
      "cp LLaMA-Factory/data/mllm_demo_data/3.mp4 s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/data/mllm_demo_data/3.mp4\n",
      "cp LLaMA-Factory/data/glaive_toolcall_zh_demo.json s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/data/glaive_toolcall_zh_demo.json\n",
      "cp LLaMA-Factory/src/llamafactory/train/trainer_utils.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/train/trainer_utils.py\n",
      "cp LLaMA-Factory/tests/model/test_full.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/tests/model/test_full.py\n",
      "cp LLaMA-Factory/tests/data/test_mm_plugin.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/tests/data/test_mm_plugin.py\n",
      "cp LLaMA-Factory/src/llamafactory/train/sft/__init__.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/train/sft/__init__.py\n",
      "cp LLaMA-Factory/src/llamafactory/webui/locales.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/webui/locales.py\n",
      "cp LLaMA-Factory/src/llamafactory/train/test_utils.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/train/test_utils.py\n",
      "cp LLaMA-Factory/tests/model/test_lora.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/tests/model/test_lora.py\n",
      "cp LLaMA-Factory/src/llamafactory/webui/runner.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/webui/runner.py\n",
      "cp LLaMA-Factory/data/mllm_demo_data/1.mp4 s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/data/mllm_demo_data/1.mp4\n",
      "cp LLaMA-Factory/data/dpo_en_demo.json s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/data/dpo_en_demo.json\n",
      "cp LLaMA-Factory/data/dpo_zh_demo.json s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/data/dpo_zh_demo.json\n",
      "cp LLaMA-Factory/data/kto_en_demo.json s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/data/kto_en_demo.json\n",
      "cp LLaMA-Factory/evaluation/ceval/ceval.zip s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/evaluation/ceval/ceval.zip\n",
      "cp LLaMA-Factory/src/llamafactory/webui/utils.py s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/src/llamafactory/webui/utils.py\n",
      "cp LLaMA-Factory/data/alpaca_en_demo.json s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/data/alpaca_en_demo.json\n",
      "cp LLaMA-Factory/s5cmd s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/s5cmd\n",
      "upload: hyperpod-scripts/llama_factory_setup.sh to s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/llama_factory_setup.sh\n",
      "upload: hyperpod-scripts/train_single_lora.sh to s3://sagemaker-us-west-2-533267176336/hyperpod/LLaMA-Factory/train_single_lora.sh\n"
     ]
    }
   ],
   "source": [
    "!./s5cmd sync ./LLaMA-Factory s3://{bucket}/hyperpod/\n",
    "!aws s3 cp --recursive hyperpod-scripts/ s3://{bucket}/hyperpod/LLaMA-Factory/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### (3) 挂载S3\n",
    "- Hyperpod 集群特别适合大规模集群分布式训练，由于其提供了底层 IaaS 基础设施的接入，因此可以方便的使用业界流行的各种分布式框架，如 accelerate，Deepspeed…etc。\n",
    "- 与 EC2 实例一样，Hyperpod 集群实例上可以挂载各种共享存储，如 EFS，Lustre，S3 等，此处我们以 mount-s3 为例。\n",
    "- mount-s3 共享存储安装及挂载脚本示例：\n",
    "- 仍然ssh到集群中执行"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```bash\n",
    "###下载 s3mount\n",
    "cd ~\n",
    "\n",
    "srun -N2 \"wget\" \"https://s3.amazonaws.com/mountpoint-s3-release/latest/x86_64/mount-s3.deb\"\n",
    "srun -N2 sudo apt-get install -y  ./mount-s3.deb\n",
    "\n",
    "\n",
    "# 挂载到\"~/mnt\" 中， 注意实验中只用了1个计算节点，所以N 1,如果是多节点，则>1\n",
    "srun -N2 \"sudo\" \"mkdir\" \"/home/ubuntu/mnt\" \n",
    "\n",
    "# 在所有节点上挂载，注意region，account-id替换成您自己的aws region和 account id\n",
    "srun -N2 \"sudo\" \"mount-s3\" \"--allow-other\" \"--allow-overwrite\" \"sagemaker-us-west-2-605630858339\" \"/home/ubuntu/mnt\" \n",
    "\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### (4) 在集群上安装 LLaMA-Factory"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. 在 Terminal 中仍然保持登录到集群节点中，如果 session expired，请参考以下方式登录。\n",
    "```bash\n",
    "chmod +x easy-ssh.sh \n",
    "cluster_name=hyperpod-cluster-1\n",
    "group=worker-group-1\n",
    "./easy-ssh.sh $cluster_name --controller-group $group\n",
    "```\n",
    "\n",
    "2. 把S3 bucket的目录下的代码copy到本地目录。\n",
    "```bash\n",
    "sudo su ubuntu\n",
    "cd ~\n",
    "srun -N2 \"cp\" \"-r\" \"mnt/hyperpod/LLaMA-Factory\" \"LLaMA-Factory\"\n",
    "```\n",
    "\n",
    "3. 按照下面命令依次执行。\n",
    "```bash\n",
    "cd LLaMA-Factory\n",
    "srun -N2 \"rm\" \"-rf\" \"../miniconda3\"\n",
    "srun -N2 \"rm\" \"-rf\" \"Miniconda3-latest*\"\n",
    "srun -N2 \"bash\" \"llama_factory_setup.sh\" \n",
    "```\n",
    "\n",
    "4. 执行以下命令，确认LLaMA_Factory环境是否安装。\n",
    "```bash\n",
    "source  ../miniconda3/bin/activate\n",
    "source activate py310\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 至此，使用 Jupyter Notebook 创建 HyperPod 集群部分结束"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_python3",
   "language": "python",
   "name": "conda_python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}