S3 bucket access.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "e799ad23-7448-4fc6-8519-41fcbed97b26",
   "metadata": {},
   "outputs": [],
   "source": [
    "import fsspec\n",
    "import s3fs\n",
    "import xarray as xr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "64f71339-89e9-4741-abba-2f896560be72",
   "metadata": {},
   "outputs": [],
   "source": [
    "from fsspec.registry import known_implementations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "34910a8c-2071-4f94-a520-dd737a4eab03",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'file': {'class': 'fsspec.implementations.local.LocalFileSystem'},\n",
       " 'memory': {'class': 'fsspec.implementations.memory.MemoryFileSystem'},\n",
       " 'dropbox': {'class': 'dropboxdrivefs.DropboxDriveFileSystem',\n",
       "  'err': 'DropboxFileSystem requires \"dropboxdrivefs\",\"requests\" and \"dropbox\" to be installed'},\n",
       " 'http': {'class': 'fsspec.implementations.http.HTTPFileSystem',\n",
       "  'err': 'HTTPFileSystem requires \"requests\" and \"aiohttp\" to be installed'},\n",
       " 'https': {'class': 'fsspec.implementations.http.HTTPFileSystem',\n",
       "  'err': 'HTTPFileSystem requires \"requests\" and \"aiohttp\" to be installed'},\n",
       " 'zip': {'class': 'fsspec.implementations.zip.ZipFileSystem'},\n",
       " 'tar': {'class': 'fsspec.implementations.tar.TarFileSystem'},\n",
       " 'gcs': {'class': 'gcsfs.GCSFileSystem',\n",
       "  'err': 'Please install gcsfs to access Google Storage'},\n",
       " 'gs': {'class': 'gcsfs.GCSFileSystem',\n",
       "  'err': 'Please install gcsfs to access Google Storage'},\n",
       " 'gdrive': {'class': 'gdrivefs.GoogleDriveFileSystem',\n",
       "  'err': 'Please install gdrivefs for access to Google Drive'},\n",
       " 'sftp': {'class': 'fsspec.implementations.sftp.SFTPFileSystem',\n",
       "  'err': 'SFTPFileSystem requires \"paramiko\" to be installed'},\n",
       " 'ssh': {'class': 'fsspec.implementations.sftp.SFTPFileSystem',\n",
       "  'err': 'SFTPFileSystem requires \"paramiko\" to be installed'},\n",
       " 'ftp': {'class': 'fsspec.implementations.ftp.FTPFileSystem'},\n",
       " 'hdfs': {'class': 'fsspec.implementations.hdfs.PyArrowHDFS',\n",
       "  'err': 'pyarrow and local java libraries required for HDFS'},\n",
       " 'arrow_hdfs': {'class': 'fsspec.implementations.arrow.HadoopFileSystem',\n",
       "  'err': 'pyarrow and local java libraries required for HDFS'},\n",
       " 'webhdfs': {'class': 'fsspec.implementations.webhdfs.WebHDFS',\n",
       "  'err': 'webHDFS access requires \"requests\" to be installed'},\n",
       " 's3': {'class': 's3fs.S3FileSystem', 'err': 'Install s3fs to access S3'},\n",
       " 's3a': {'class': 's3fs.S3FileSystem', 'err': 'Install s3fs to access S3'},\n",
       " 'wandb': {'class': 'wandbfs.WandbFS',\n",
       "  'err': 'Install wandbfs to access wandb'},\n",
       " 'oci': {'class': 'ocifs.OCIFileSystem',\n",
       "  'err': 'Install ocifs to access OCI Object Storage'},\n",
       " 'adl': {'class': 'adlfs.AzureDatalakeFileSystem',\n",
       "  'err': 'Install adlfs to access Azure Datalake Gen1'},\n",
       " 'abfs': {'class': 'adlfs.AzureBlobFileSystem',\n",
       "  'err': 'Install adlfs to access Azure Datalake Gen2 and Azure Blob Storage'},\n",
       " 'az': {'class': 'adlfs.AzureBlobFileSystem',\n",
       "  'err': 'Install adlfs to access Azure Datalake Gen2 and Azure Blob Storage'},\n",
       " 'cached': {'class': 'fsspec.implementations.cached.CachingFileSystem'},\n",
       " 'blockcache': {'class': 'fsspec.implementations.cached.CachingFileSystem'},\n",
       " 'filecache': {'class': 'fsspec.implementations.cached.WholeFileCacheFileSystem'},\n",
       " 'simplecache': {'class': 'fsspec.implementations.cached.SimpleCacheFileSystem'},\n",
       " 'dask': {'class': 'fsspec.implementations.dask.DaskWorkerFileSystem',\n",
       "  'err': 'Install dask distributed to access worker file system'},\n",
       " 'dbfs': {'class': 'fsspec.implementations.dbfs.DatabricksFileSystem',\n",
       "  'err': 'Install the requests package to use the DatabricksFileSystem'},\n",
       " 'github': {'class': 'fsspec.implementations.github.GithubFileSystem',\n",
       "  'err': 'Install the requests package to use the github FS'},\n",
       " 'git': {'class': 'fsspec.implementations.git.GitFileSystem',\n",
       "  'err': 'Install pygit2 to browse local git repos'},\n",
       " 'smb': {'class': 'fsspec.implementations.smb.SMBFileSystem',\n",
       "  'err': 'SMB requires \"smbprotocol\" or \"smbprotocol[kerberos]\" installed'},\n",
       " 'jupyter': {'class': 'fsspec.implementations.jupyter.JupyterFileSystem',\n",
       "  'err': 'Jupyter FS requires requests to be installed'},\n",
       " 'jlab': {'class': 'fsspec.implementations.jupyter.JupyterFileSystem',\n",
       "  'err': 'Jupyter FS requires requests to be installed'},\n",
       " 'libarchive': {'class': 'fsspec.implementations.libarchive.LibArchiveFileSystem',\n",
       "  'err': 'LibArchive requires to be installed'},\n",
       " 'reference': {'class': 'fsspec.implementations.reference.ReferenceFileSystem'}}"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "known_implementations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "6f14362f-0df4-4cb4-af74-39a126f56f4a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<s3fs.core.S3FileSystem at 0x183ba07f0>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fs = s3fs.S3FileSystem(anon=True)\n",
    "fs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "ca57afe1-53bc-439f-86cd-d99bf9d675eb",
   "metadata": {},
   "outputs": [
    {
     "ename": "PermissionError",
     "evalue": "Access Denied",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mClientError\u001b[0m                               Traceback (most recent call last)",
      "File \u001b[0;32m/usr/local/miniconda3/envs/code-sprint-2022/lib/python3.10/site-packages/s3fs/core.py:614\u001b[0m, in \u001b[0;36mS3FileSystem._lsdir\u001b[0;34m(self, path, refresh, max_items, delimiter, prefix)\u001b[0m\n\u001b[1;32m    613\u001b[0m dircache \u001b[38;5;241m=\u001b[39m []\n\u001b[0;32m--> 614\u001b[0m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mfor\u001b[39;00m i \u001b[38;5;129;01min\u001b[39;00m it:\n\u001b[1;32m    615\u001b[0m     dircache\u001b[38;5;241m.\u001b[39mextend(i\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mCommonPrefixes\u001b[39m\u001b[38;5;124m\"\u001b[39m, []))\n",
      "File \u001b[0;32m/usr/local/miniconda3/envs/code-sprint-2022/lib/python3.10/site-packages/aiobotocore/paginate.py:32\u001b[0m, in \u001b[0;36mAioPageIterator.__anext__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     31\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[0;32m---> 32\u001b[0m     response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_make_request(current_kwargs)\n\u001b[1;32m     33\u001b[0m     parsed \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_extract_parsed_response(response)\n",
      "File \u001b[0;32m/usr/local/miniconda3/envs/code-sprint-2022/lib/python3.10/site-packages/aiobotocore/client.py:228\u001b[0m, in \u001b[0;36mAioBaseClient._make_api_call\u001b[0;34m(self, operation_name, api_params)\u001b[0m\n\u001b[1;32m    227\u001b[0m     error_class \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mexceptions\u001b[38;5;241m.\u001b[39mfrom_code(error_code)\n\u001b[0;32m--> 228\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m error_class(parsed_response, operation_name)\n\u001b[1;32m    229\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n",
      "\u001b[0;31mClientError\u001b[0m: An error occurred (AccessDenied) when calling the ListObjectsV2 operation: Access Denied",
      "\nThe above exception was the direct cause of the following exception:\n",
      "\u001b[0;31mPermissionError\u001b[0m                           Traceback (most recent call last)",
      "Input \u001b[0;32mIn [13]\u001b[0m, in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mfs\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mls\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mioos-code-sprint-2022\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/usr/local/miniconda3/envs/code-sprint-2022/lib/python3.10/site-packages/fsspec/asyn.py:85\u001b[0m, in \u001b[0;36msync_wrapper.<locals>.wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     82\u001b[0m \u001b[38;5;129m@functools\u001b[39m\u001b[38;5;241m.\u001b[39mwraps(func)\n\u001b[1;32m     83\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mwrapper\u001b[39m(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m     84\u001b[0m     \u001b[38;5;28mself\u001b[39m \u001b[38;5;241m=\u001b[39m obj \u001b[38;5;129;01mor\u001b[39;00m args[\u001b[38;5;241m0\u001b[39m]\n\u001b[0;32m---> 85\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43msync\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mloop\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfunc\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/usr/local/miniconda3/envs/code-sprint-2022/lib/python3.10/site-packages/fsspec/asyn.py:65\u001b[0m, in \u001b[0;36msync\u001b[0;34m(loop, func, timeout, *args, **kwargs)\u001b[0m\n\u001b[1;32m     63\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m FSTimeoutError \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mreturn_result\u001b[39;00m\n\u001b[1;32m     64\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(return_result, \u001b[38;5;167;01mBaseException\u001b[39;00m):\n\u001b[0;32m---> 65\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m return_result\n\u001b[1;32m     66\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m     67\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m return_result\n",
      "File \u001b[0;32m/usr/local/miniconda3/envs/code-sprint-2022/lib/python3.10/site-packages/fsspec/asyn.py:25\u001b[0m, in \u001b[0;36m_runner\u001b[0;34m(event, coro, result, timeout)\u001b[0m\n\u001b[1;32m     23\u001b[0m     coro \u001b[38;5;241m=\u001b[39m asyncio\u001b[38;5;241m.\u001b[39mwait_for(coro, timeout\u001b[38;5;241m=\u001b[39mtimeout)\n\u001b[1;32m     24\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m---> 25\u001b[0m     result[\u001b[38;5;241m0\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m coro\n\u001b[1;32m     26\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m ex:\n\u001b[1;32m     27\u001b[0m     result[\u001b[38;5;241m0\u001b[39m] \u001b[38;5;241m=\u001b[39m ex\n",
      "File \u001b[0;32m/usr/local/miniconda3/envs/code-sprint-2022/lib/python3.10/site-packages/s3fs/core.py:831\u001b[0m, in \u001b[0;36mS3FileSystem._ls\u001b[0;34m(self, path, detail, refresh)\u001b[0m\n\u001b[1;32m    829\u001b[0m     files \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_lsbuckets(refresh)\n\u001b[1;32m    830\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 831\u001b[0m     files \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_lsdir(path, refresh)\n\u001b[1;32m    832\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m files \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m/\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;129;01min\u001b[39;00m path:\n\u001b[1;32m    833\u001b[0m         files \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_lsdir(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_parent(path), refresh\u001b[38;5;241m=\u001b[39mrefresh)\n",
      "File \u001b[0;32m/usr/local/miniconda3/envs/code-sprint-2022/lib/python3.10/site-packages/s3fs/core.py:637\u001b[0m, in \u001b[0;36mS3FileSystem._lsdir\u001b[0;34m(self, path, refresh, max_items, delimiter, prefix)\u001b[0m\n\u001b[1;32m    635\u001b[0m         f[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mname\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m f[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mKey\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n\u001b[1;32m    636\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m ClientError \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[0;32m--> 637\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m translate_boto_error(e)\n\u001b[1;32m    639\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m delimiter \u001b[38;5;129;01mand\u001b[39;00m files:\n\u001b[1;32m    640\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdircache[path] \u001b[38;5;241m=\u001b[39m files\n",
      "\u001b[0;31mPermissionError\u001b[0m: Access Denied"
     ]
    }
   ],
   "source": [
    "fs.ls('ioos-code-sprint-2022')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "76ecf5b5-113a-4311-a4ca-2aaaebb5a1ef",
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Protocol not known: s3://ioos-code-sprint-2022",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "Input \u001b[0;32mIn [8]\u001b[0m, in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0m fs \u001b[38;5;241m=\u001b[39m \u001b[43mfsspec\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfilesystem\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43ms3://ioos-code-sprint-2022\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m      2\u001b[0m fs\n",
      "File \u001b[0;32m/usr/local/miniconda3/envs/code-sprint-2022/lib/python3.10/site-packages/fsspec/registry.py:252\u001b[0m, in \u001b[0;36mfilesystem\u001b[0;34m(protocol, **storage_options)\u001b[0m\n\u001b[1;32m    246\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mfilesystem\u001b[39m(protocol, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mstorage_options):\n\u001b[1;32m    247\u001b[0m     \u001b[38;5;124;03m\"\"\"Instantiate filesystems for given protocol and arguments\u001b[39;00m\n\u001b[1;32m    248\u001b[0m \n\u001b[1;32m    249\u001b[0m \u001b[38;5;124;03m    ``storage_options`` are specific to the protocol being chosen, and are\u001b[39;00m\n\u001b[1;32m    250\u001b[0m \u001b[38;5;124;03m    passed directly to the class.\u001b[39;00m\n\u001b[1;32m    251\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[0;32m--> 252\u001b[0m     \u001b[38;5;28mcls\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[43mget_filesystem_class\u001b[49m\u001b[43m(\u001b[49m\u001b[43mprotocol\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    253\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mcls\u001b[39m(\u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mstorage_options)\n",
      "File \u001b[0;32m/usr/local/miniconda3/envs/code-sprint-2022/lib/python3.10/site-packages/fsspec/registry.py:216\u001b[0m, in \u001b[0;36mget_filesystem_class\u001b[0;34m(protocol)\u001b[0m\n\u001b[1;32m    214\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m protocol \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m registry:\n\u001b[1;32m    215\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m protocol \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m known_implementations:\n\u001b[0;32m--> 216\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mProtocol not known: \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m%\u001b[39m protocol)\n\u001b[1;32m    217\u001b[0m     bit \u001b[38;5;241m=\u001b[39m known_implementations[protocol]\n\u001b[1;32m    218\u001b[0m     \u001b[38;5;28;01mtry\u001b[39;00m:\n",
      "\u001b[0;31mValueError\u001b[0m: Protocol not known: s3://ioos-code-sprint-2022"
     ]
    }
   ],
   "source": [
    "fs = fsspec.filesystem(\"s3://ioos-code-sprint-2022\")\n",
    "fs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "dfb529a2-dc8c-4fb3-820f-4711ca5838b6",
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'OpenFile' object has no attribute 'ls'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "Input \u001b[0;32mIn [3]\u001b[0m, in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mfs\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mls\u001b[49m()\n",
      "\u001b[0;31mAttributeError\u001b[0m: 'OpenFile' object has no attribute 'ls'"
     ]
    }
   ],
   "source": [
    "fs.ls()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ec344bbe-81ac-4d4a-a583-f12075baecce",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['__class__',\n",
       " '__del__',\n",
       " '__delattr__',\n",
       " '__dict__',\n",
       " '__dir__',\n",
       " '__doc__',\n",
       " '__enter__',\n",
       " '__eq__',\n",
       " '__exit__',\n",
       " '__format__',\n",
       " '__fspath__',\n",
       " '__ge__',\n",
       " '__getattribute__',\n",
       " '__gt__',\n",
       " '__hash__',\n",
       " '__init__',\n",
       " '__init_subclass__',\n",
       " '__le__',\n",
       " '__lt__',\n",
       " '__module__',\n",
       " '__ne__',\n",
       " '__new__',\n",
       " '__reduce__',\n",
       " '__reduce_ex__',\n",
       " '__repr__',\n",
       " '__setattr__',\n",
       " '__sizeof__',\n",
       " '__str__',\n",
       " '__subclasshook__',\n",
       " '__weakref__',\n",
       " 'close',\n",
       " 'compression',\n",
       " 'encoding',\n",
       " 'errors',\n",
       " 'fobjects',\n",
       " 'fs',\n",
       " 'full_name',\n",
       " 'mode',\n",
       " 'newline',\n",
       " 'open',\n",
       " 'path']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dir(fs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "4aa1333b-14c1-4e96-bbdc-68f1800f506d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"type\": \"Catalog\",\n",
      "  \"id\": \"DMAC-ZARR\",\n",
      "  \"stac_version\": \"1.0.0\",\n",
      "  \"description\": \"Experimental Catalog for Next-Gen DMAC\",\n",
      "  \"links\": [\n",
      "    {\n",
      "      \"rel\": \"root\",\n",
      "      \"href\": \"./catalog.json\",\n",
      "      \"type\": \"application/json\"\n",
      "    },\n",
      "    {\n",
      "      \"rel\": \"child\",\n",
      "      \"href\": \"./CBOFS/collection.json\",\n",
      "      \"type\": \"application/json\"\n",
      "    },\n",
      "    {\n",
      "      \"rel\": \"child\",\n",
      "      \"href\": \"./GFS/collection.json\",\n",
      "      \"type\": \"application/json\"\n",
      "    },\n",
      "    {\n",
      "      \"rel\": \"child\",\n",
      "      \"href\": \"./GFSWAVE/collection.json\",\n",
      "      \"type\": \"application/json\"\n",
      "    },\n",
      "    {\n",
      "      \"rel\": \"self\",\n",
      "      \"href\": \"s3://dmac-zarr/catalog.json\",\n",
      "      \"type\": \"application/json\"\n",
      "    }\n",
      "  ],\n",
      "  \"stac_extensions\": []\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "print(fs.cat_file(\"ioos-code-sprint-2022/catalog.json\").decode(\"utf-8\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "ca20241c-20a6-4c78-becc-441acc0e0915",
   "metadata": {},
   "outputs": [
    {
     "ename": "PermissionError",
     "evalue": "Access Denied",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mClientError\u001b[0m                               Traceback (most recent call last)",
      "File \u001b[0;32m/usr/local/miniconda3/envs/code-sprint-2022/lib/python3.10/site-packages/s3fs/core.py:614\u001b[0m, in \u001b[0;36mS3FileSystem._lsdir\u001b[0;34m(self, path, refresh, max_items, delimiter, prefix)\u001b[0m\n\u001b[1;32m    613\u001b[0m dircache \u001b[38;5;241m=\u001b[39m []\n\u001b[0;32m--> 614\u001b[0m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mfor\u001b[39;00m i \u001b[38;5;129;01min\u001b[39;00m it:\n\u001b[1;32m    615\u001b[0m     dircache\u001b[38;5;241m.\u001b[39mextend(i\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mCommonPrefixes\u001b[39m\u001b[38;5;124m\"\u001b[39m, []))\n",
      "File \u001b[0;32m/usr/local/miniconda3/envs/code-sprint-2022/lib/python3.10/site-packages/aiobotocore/paginate.py:32\u001b[0m, in \u001b[0;36mAioPageIterator.__anext__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     31\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[0;32m---> 32\u001b[0m     response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_make_request(current_kwargs)\n\u001b[1;32m     33\u001b[0m     parsed \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_extract_parsed_response(response)\n",
      "File \u001b[0;32m/usr/local/miniconda3/envs/code-sprint-2022/lib/python3.10/site-packages/aiobotocore/client.py:228\u001b[0m, in \u001b[0;36mAioBaseClient._make_api_call\u001b[0;34m(self, operation_name, api_params)\u001b[0m\n\u001b[1;32m    227\u001b[0m     error_class \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mexceptions\u001b[38;5;241m.\u001b[39mfrom_code(error_code)\n\u001b[0;32m--> 228\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m error_class(parsed_response, operation_name)\n\u001b[1;32m    229\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n",
      "\u001b[0;31mClientError\u001b[0m: An error occurred (AccessDenied) when calling the ListObjectsV2 operation: Access Denied",
      "\nThe above exception was the direct cause of the following exception:\n",
      "\u001b[0;31mPermissionError\u001b[0m                           Traceback (most recent call last)",
      "Input \u001b[0;32mIn [19]\u001b[0m, in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mfs\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mls\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mioos-code-sprint-2022\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/usr/local/miniconda3/envs/code-sprint-2022/lib/python3.10/site-packages/fsspec/asyn.py:85\u001b[0m, in \u001b[0;36msync_wrapper.<locals>.wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     82\u001b[0m \u001b[38;5;129m@functools\u001b[39m\u001b[38;5;241m.\u001b[39mwraps(func)\n\u001b[1;32m     83\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mwrapper\u001b[39m(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m     84\u001b[0m     \u001b[38;5;28mself\u001b[39m \u001b[38;5;241m=\u001b[39m obj \u001b[38;5;129;01mor\u001b[39;00m args[\u001b[38;5;241m0\u001b[39m]\n\u001b[0;32m---> 85\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43msync\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mloop\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfunc\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/usr/local/miniconda3/envs/code-sprint-2022/lib/python3.10/site-packages/fsspec/asyn.py:65\u001b[0m, in \u001b[0;36msync\u001b[0;34m(loop, func, timeout, *args, **kwargs)\u001b[0m\n\u001b[1;32m     63\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m FSTimeoutError \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mreturn_result\u001b[39;00m\n\u001b[1;32m     64\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(return_result, \u001b[38;5;167;01mBaseException\u001b[39;00m):\n\u001b[0;32m---> 65\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m return_result\n\u001b[1;32m     66\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m     67\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m return_result\n",
      "File \u001b[0;32m/usr/local/miniconda3/envs/code-sprint-2022/lib/python3.10/site-packages/fsspec/asyn.py:25\u001b[0m, in \u001b[0;36m_runner\u001b[0;34m(event, coro, result, timeout)\u001b[0m\n\u001b[1;32m     23\u001b[0m     coro \u001b[38;5;241m=\u001b[39m asyncio\u001b[38;5;241m.\u001b[39mwait_for(coro, timeout\u001b[38;5;241m=\u001b[39mtimeout)\n\u001b[1;32m     24\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m---> 25\u001b[0m     result[\u001b[38;5;241m0\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m coro\n\u001b[1;32m     26\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m ex:\n\u001b[1;32m     27\u001b[0m     result[\u001b[38;5;241m0\u001b[39m] \u001b[38;5;241m=\u001b[39m ex\n",
      "File \u001b[0;32m/usr/local/miniconda3/envs/code-sprint-2022/lib/python3.10/site-packages/s3fs/core.py:831\u001b[0m, in \u001b[0;36mS3FileSystem._ls\u001b[0;34m(self, path, detail, refresh)\u001b[0m\n\u001b[1;32m    829\u001b[0m     files \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_lsbuckets(refresh)\n\u001b[1;32m    830\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 831\u001b[0m     files \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_lsdir(path, refresh)\n\u001b[1;32m    832\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m files \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m/\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;129;01min\u001b[39;00m path:\n\u001b[1;32m    833\u001b[0m         files \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_lsdir(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_parent(path), refresh\u001b[38;5;241m=\u001b[39mrefresh)\n",
      "File \u001b[0;32m/usr/local/miniconda3/envs/code-sprint-2022/lib/python3.10/site-packages/s3fs/core.py:637\u001b[0m, in \u001b[0;36mS3FileSystem._lsdir\u001b[0;34m(self, path, refresh, max_items, delimiter, prefix)\u001b[0m\n\u001b[1;32m    635\u001b[0m         f[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mname\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m f[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mKey\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n\u001b[1;32m    636\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m ClientError \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[0;32m--> 637\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m translate_boto_error(e)\n\u001b[1;32m    639\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m delimiter \u001b[38;5;129;01mand\u001b[39;00m files:\n\u001b[1;32m    640\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdircache[path] \u001b[38;5;241m=\u001b[39m files\n",
      "\u001b[0;31mPermissionError\u001b[0m: Access Denied"
     ]
    }
   ],
   "source": [
    "fs.ls(\"ioos-code-sprint-2022\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "restful-grids",
   "language": "python",
   "name": "restful-grids"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}