scripts/01_get_datasets_for_dataverses.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "ab46773a-66e9-420a-9811-be81da1af05d",
   "metadata": {},
   "source": [
    "## Get Datasets For a List of Dataverses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "04143dc3-f6e0-4a21-9e70-ca2343a10189",
   "metadata": {},
   "outputs": [],
   "source": [
    "#!pip install -U \"pyDataverse==0.2.1\"\n",
    "import requests\n",
    "import pandas as pd\n",
    "import os\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "0ef3e750-3363-4d81-bb77-fef7b9bedf12",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_dataverse_datasets(dataverse_id):\n",
    "    # Set the base URL of your Dataverse instance\n",
    "    base_url = 'https://dataverse.harvard.edu/api'\n",
    "    \n",
    "    # Construct the API endpoint URL\n",
    "    url = f'{base_url}/dataverses/{dataverse_id}/contents'\n",
    "\n",
    "    # Make a GET request to retrieve the dataset information\n",
    "    response = requests.get(url)\n",
    "    dataset_json = response.json()\n",
    "    df = pd.DataFrame(dataset_json['data'])\n",
    "    \n",
    "    if not os.path.exists('datasets'):\n",
    "        os.makedirs('datasets')\n",
    "    \n",
    "    df.to_csv(f\"datasets/{dataverse_id}_datasets.csv\", index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "fefe9b4f-d159-4a4b-9095-f06aafdfa8f1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['YaleJREG', 'jri', 'JPE', 'JPE_Macroeconomics',\n",
       "       'JPE_Microeconomics', 'ntj', 'lcnapsa', 'asyafinajournal', 'raps',\n",
       "       'jhpe', 'josum', 'cybergeo', 'rbcp', 'pa', 'cjp', 'revistadados',\n",
       "       'NegotiationJournal', 'jpipe', 'rcfs', 'cps', 'rfs',\n",
       "       'securitystudies', 'politicsandgender', 'SOCRATESJOURNAL', 'ps',\n",
       "       'PAR', 'uidergisi', 'rac', 'se', 'IOJ', 'bpsr', 'JJPS', 'EDCC',\n",
       "       'cjip', 'JBPA', 'lsq', 'JAERE', 'perspectives', 'xps',\n",
       "       'health-psychology-bulletin', 'world-politics', 'LQ', 'the_review',\n",
       "       'culturalanalytics', 'opmi', 'jlc', 'qje', 'tpmnewsletter', 'LAPS',\n",
       "       'PittJWSR', 'cstp', 'JPublicPolicy', 'johd', 'regionalstatistics',\n",
       "       'jlsc', 'technology_science', 'polbehavior', 'isec', 'BJPolS',\n",
       "       'hsscomms', 'jop', 'ipsr-risp', 'jhr', 'joad', 'dib',\n",
       "       'researchandpolitics', 'jopd', 'PSRM', 'ajps', 'FPA', 'restat',\n",
       "       'internationalinteractions', 'pan', 'economics', 'isq'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv(\"../data/dataverse_socialscience.csv\")\n",
    "data.dataverse_id.dropna().unique()\n",
    "# sppq is on unc: https://dataverse.unc.edu/dataverse/sppq/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "a0c637a9-e0b3-4be7-9e67-3be19c9458cf",
   "metadata": {},
   "outputs": [],
   "source": [
    "#len(data.dataverse_id.dropna().unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "866c2298-9d9f-42d7-acb8-42dd36b5156b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing Dataverses: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████| 75/75 [02:14<00:00,  1.80s/it]\n"
     ]
    }
   ],
   "source": [
    "# Iterate over the list of dataverse_ids and get datasets for each\n",
    "dataverse_ids = data.dataverse_id.dropna().unique()\n",
    "\n",
    "for dataverse_id in tqdm(dataverse_ids, desc = 'Processing Dataverses'):\n",
    "    get_dataverse_datasets(dataverse_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "afb2e172-ecc5-495a-9cdf-8da67e02fe9e",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}