Ask_ChatGPT_to_Extract.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "808318b2-aaf5-4092-ba77-c4895ddf4a2d",
   "metadata": {},
   "source": [
    "# **Ask ChatGPT for One-Shot Mathematical Definitum Extraction**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "483e2533-7833-415e-99b0-e91ff866758f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting openai\n",
      "  Downloading openai-0.27.9-py3-none-any.whl (75 kB)\n",
      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.5/75.5 kB\u001b[0m \u001b[31m6.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting aiohttp\n",
      "  Downloading aiohttp-3.8.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.1 MB)\n",
      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.1/1.1 MB\u001b[0m \u001b[31m34.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: tqdm in /home/shufan/tools/anacnda3/lib/python3.8/site-packages (from openai) (4.64.1)\n",
      "Requirement already satisfied: requests>=2.20 in /home/shufan/tools/anacnda3/lib/python3.8/site-packages (from openai) (2.28.1)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /home/shufan/tools/anacnda3/lib/python3.8/site-packages (from requests>=2.20->openai) (3.4)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /home/shufan/tools/anacnda3/lib/python3.8/site-packages (from requests>=2.20->openai) (2023.5.7)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /home/shufan/tools/anacnda3/lib/python3.8/site-packages (from requests>=2.20->openai) (1.26.14)\n",
      "Requirement already satisfied: charset-normalizer<3,>=2 in /home/shufan/tools/anacnda3/lib/python3.8/site-packages (from requests>=2.20->openai) (2.0.4)\n",
      "Collecting async-timeout<5.0,>=4.0.0a3\n",
      "  Downloading async_timeout-4.0.3-py3-none-any.whl (5.7 kB)\n",
      "Collecting yarl<2.0,>=1.0\n",
      "  Downloading yarl-1.9.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (266 kB)\n",
      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m266.9/266.9 kB\u001b[0m \u001b[31m39.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: attrs>=17.3.0 in /home/shufan/tools/anacnda3/lib/python3.8/site-packages (from aiohttp->openai) (22.1.0)\n",
      "Collecting aiosignal>=1.1.2\n",
      "  Downloading aiosignal-1.3.1-py3-none-any.whl (7.6 kB)\n",
      "Collecting frozenlist>=1.1.1\n",
      "  Downloading frozenlist-1.4.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (220 kB)\n",
      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m220.1/220.1 kB\u001b[0m \u001b[31m34.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting multidict<7.0,>=4.5\n",
      "  Downloading multidict-6.0.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (121 kB)\n",
      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m121.3/121.3 kB\u001b[0m \u001b[31m18.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hInstalling collected packages: multidict, frozenlist, async-timeout, yarl, aiosignal, aiohttp, openai\n",
      "Successfully installed aiohttp-3.8.5 aiosignal-1.3.1 async-timeout-4.0.3 frozenlist-1.4.0 multidict-6.0.4 openai-0.27.9 yarl-1.9.2\n"
     ]
    }
   ],
   "source": [
    "#!pip install openai"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "629fca7d-6fbb-4dfc-9ace-70bd4c2faf54",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cpu\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.metrics import accuracy_score\n",
    "import torch\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from transformers import pipeline, AutoModelForTokenClassification, get_cosine_schedule_with_warmup, AutoTokenizer\n",
    "from seqeval.metrics import classification_report\n",
    "import seaborn as sns\n",
    "import os\n",
    "import openai\n",
    "from torch import cuda\n",
    "device = 'cuda' if cuda.is_available() else 'cpu'\n",
    "print(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "e2b16389-133e-457c-a2d7-8aab0b719fce",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "4ac2ed08-2ccd-4038-96f3-8a695db67e45",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Preprocessing the dataset\n",
    "\n",
    "Preliminary test shows that ChatGPT generated random IOB tags might not be aligned with the extracted terms.\n",
    "We decide to ask for the terms only. We evaluate later the precision (how many extracted terms are true) and recall(how many expected terms are extracted)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "8b28c1d3-1bb1-446c-bca1-1597b9d1ea9f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "13692"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(\"cleaned_plain-text_labeled_term+combined_no_ref_no_cit_def_same_len_only_must_conatin_B.csv\", delimiter=',')\n",
    "len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "88c1701b-87d2-4871-84f8-2fcaa7ff1412",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "average length =  70.69770669003798\n",
      "maximun length =  5266\n"
     ]
    }
   ],
   "source": [
    "all_texts = df['plain_text_def'].to_list()\n",
    "#check average tokenized sentence lenth\n",
    "all_text_lenlist = [len(e.split(' ')) for e in all_texts]\n",
    "print(\"average length = \", sum(all_text_lenlist) / len(all_text_lenlist))\n",
    "print(\"maximun length = \", max(all_text_lenlist))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "586efc96-2c12-42fb-934b-5733ad3d2acb",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['text_token_num']=all_text_lenlist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "58b30b2d-1efd-4e4d-947a-52b919b06cf0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeoAAAHpCAYAAABN+X+UAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAsjUlEQVR4nO3de3RU1aHH8d8kkwQIYSQ8MkQjRI0KJlgFiqAVEAg+0FJtQUFERa+IIPEtVQt6a6LcClpRLOgFH5dFvQu4dVWqRMVUBCsGU0lArAIKmhiUPGuYZGb2/YPm1CGERzLJ7JDvZ62sMufsmeyzTflyJnNmXMYYIwAAYKWoSE8AAAA0jlADAGAxQg0AgMUINQAAFiPUAABYjFADAGAxQg0AgMUItSRjjCorK8Ul5QAA2xBqSVVVVfJ4PKqqqor0VAAACEGoAQCwGKEGAMBihBoAAIsRagAALEaoAQCwGKEGAMBihBoAAIsRagAALEaoAQCwGKEGAMBihBoAAIsRagAALEaoAQCwGKEGAMBihBoAAIsRagAALEaoAQCwGKEGAMBihLoFGGPk9/tljIn0VAAAbRyhbgGBQEATns1TIBCI9FQAAG0coW4hUdHRkZ4CAOA4QKgBALAYoQYAwGKEGgAAixFqAAAsRqgBALAYoQYAwGKEGgAAixFqAAAsRqgBALAYoQYAwGKEGgAAixFqAAAsRqgBALAYoQYAwGKEGgAAixFqAAAsRqgBALAYoQYAwGKEGgAAixFqAAAsRqgBALAYoQYAwGKEGgAAixFqAAAsRqgBALAYoQYAwGKEGgAAixFqAAAsRqgBALAYoQYAwGKEGgAAixFqAAAsRqgBALAYoQYAwGKEGgAAixFqAAAsRqgBALAYoQYAwGKEGgAAixFqAAAsRqhbkN/vl9/vj/Q0AABtWERD7ff79eCDDyo1NVUdO3bUKaecokceeUTBYNAZY4zR3LlzlZycrI4dO2r48OEqKioKeRyfz6eZM2eqe/fuio+P1xVXXKE9e/a09uEAABB2EQ31448/rueee04LFy7Utm3bNG/ePP3Xf/2Xnn76aWfMvHnzNH/+fC1cuFCbNm2S1+vV6NGjVVVV5YzJysrS6tWrtWLFCq1fv17V1dUaO3asAoFAJA4LAICwcRljTKS++dixY5WUlKQXXnjB2XbVVVepU6dOevnll2WMUXJysrKysnTfffdJOnD2nJSUpMcff1y33HKLKioq1KNHD7388suaMGGCJOmbb75RSkqK1qxZozFjxjT4vj6fTz6fz7ldWVmplJQUVVRUqEuXLs0+Lr/fr2v+sF4vTz1Pbrdbbre72Y8JAGifInpGfcEFF+jtt9/WZ599Jkn6+9//rvXr1+vSSy+VJO3cuVMlJSXKzMx07hMXF6dhw4Zpw4YNkqT8/HzV1dWFjElOTlZ6eroz5mA5OTnyeDzOV0pKSksdIgAAzRLRU7377rtPFRUVOvPMMxUdHa1AIKBHH31U11xzjSSppKREkpSUlBRyv6SkJH355ZfOmNjYWHXt2rXBmPr7H2z27Nm68847ndv1Z9QAANgmoqH+4x//qFdeeUXLly/XWWedpYKCAmVlZSk5OVlTpkxxxrlcrpD7GWMabDvY4cbExcUpLi6u+QcAAEALi+hT3/fcc4/uv/9+XX311crIyNDkyZN1xx13KCcnR5Lk9XolqcGZcWlpqXOW7fV6VVtbq7KyskbHREIwEODSLABAs0U01D/88IOiokKnEB0d7VyelZqaKq/Xq9zcXGd/bW2t8vLyNHToUEnSgAEDFBMTEzKmuLhYhYWFzhgAANqqiD71ffnll+vRRx/VySefrLPOOksff/yx5s+frxtvvFHSgae8s7KylJ2drbS0NKWlpSk7O1udOnXSxIkTJUkej0dTp07VXXfdpW7duikxMVF33323MjIyNGrUqEgeHgAAzRbRUD/99NN66KGHNH36dJWWlio5OVm33HKLfvOb3zhj7r33XtXU1Gj69OkqKyvT4MGDtXbtWiUkJDhjFixYILfbrfHjx6umpkYjR47UsmXLFB0dHYnDAgAgbCJ6HbUtKisr5fF4wnod9YRn8/TijYPVoUMHrqMGADQZ7/UNAIDFCDUAABYj1AAAWIxQAwBgMUINAIDFCDUAABYj1AAAWIxQtxBjjPx+v7hMHQDQHIS6hZhgUNe/8IECgUCkpwIAaMMIdQty8RamAIBmItQtJBgI8LQ3AKDZCDUAABYj1AAAWIxQAwBgMUINAIDFCDUAABYj1C2EV30DAMKBUAMAYDFCDQCAxQg1AAAWI9QAAFiMUAMAYDFCDQCAxQg1AAAWI9QAAFiMUAMAYDFC3YKCgYD8fn+kpwEAaMMINQAAFiPUAABYjFADAGAxQg0AgMUINQAAFiPUAABYjFADAGAxQg0AgMUINQAAFiPUAABYjFADAGAxQg0AgMUINQAAFiPUAABYjFADAGAxQg0AgMUINQAAFiPUAABYjFADAGAxQg0AgMUINQAAFiPUAABYjFADAGAxQg0AgMUINQAAFiPUAABYjFADAGAxQg0AgMUINQAAFiPUAABYjFADAGAxQg0AgMUINQAAFiPUAABYjFADAGAxQt2CjDHy+/0yxkR6KgCANopQtyATDOqGpR8qEAhEeioAgDaKULcwV3R0pKcAAGjDCDUAABYj1AAAWIxQAwBgMUINAIDFCDUAABYj1AAAWIxQAwBgMUINAIDFCDUAABYj1AAAWIxQAwBgMUINAIDFCDUAABYj1AAAWCziof7666917bXXqlu3burUqZN+8pOfKD8/39lvjNHcuXOVnJysjh07avjw4SoqKgp5DJ/Pp5kzZ6p79+6Kj4/XFVdcoT179rT2oQAAEHYRDXVZWZnOP/98xcTE6C9/+Yu2bt2qJ554QieccIIzZt68eZo/f74WLlyoTZs2yev1avTo0aqqqnLGZGVlafXq1VqxYoXWr1+v6upqjR07VoFAIAJHBQBA+LiMMSZS3/z+++/X+++/r/fee++Q+40xSk5OVlZWlu677z5JB86ek5KS9Pjjj+uWW25RRUWFevTooZdfflkTJkyQJH3zzTdKSUnRmjVrNGbMmAaP6/P55PP5nNuVlZVKSUlRRUWFunTp0uzj8vv9+sUTbyhojOI6ddKr04fJ7XY3+3EBAO1PRM+oX3vtNQ0cOFC/+tWv1LNnT51zzjlasmSJs3/nzp0qKSlRZmamsy0uLk7Dhg3Thg0bJEn5+fmqq6sLGZOcnKz09HRnzMFycnLk8Xicr5SUlBY6QgAAmieiod6xY4cWLVqktLQ0vfnmm5o2bZpuv/12vfTSS5KkkpISSVJSUlLI/ZKSkpx9JSUlio2NVdeuXRsdc7DZs2eroqLC+dq9e3e4Dw0AgLCI6POxwWBQAwcOVHZ2tiTpnHPOUVFRkRYtWqTrrrvOGedyuULuZ4xpsO1ghxsTFxenuLi4Zs4eAICWF9Ez6l69eqlfv34h2/r27auvvvpKkuT1eiWpwZlxaWmpc5bt9XpVW1ursrKyRscAANBWRTTU559/vrZv3x6y7bPPPlPv3r0lSampqfJ6vcrNzXX219bWKi8vT0OHDpUkDRgwQDExMSFjiouLVVhY6IwBAKCtiuhT33fccYeGDh2q7OxsjR8/Xh9++KEWL16sxYsXSzrwlHdWVpays7OVlpamtLQ0ZWdnq1OnTpo4caIkyePxaOrUqbrrrrvUrVs3JSYm6u6771ZGRoZGjRoVycMDAKDZIhrqQYMGafXq1Zo9e7YeeeQRpaam6sknn9SkSZOcMffee69qamo0ffp0lZWVafDgwVq7dq0SEhKcMQsWLJDb7db48eNVU1OjkSNHatmyZYqOjo7EYQEAEDYRvY7aFpWVlfJ4PFxHDQCwTsTfQhQAADSOUAMAYDFCDQCAxQg1AAAWI9QAAFiMULcAv98vXkwPAAgHQg0AgMUINQAAFiPUAABYjFADAGAxQg0AgMUINQAAFiPUAABYjFADAGAxQg0AgMUINQAAFiPUAABYjFADAGAxQg0AgMUINQAAFiPUAABYjFADAGAxQt3CgoGA/H5/pKcBAGijCDUAABYj1AAAWIxQAwBgMUINAIDFCHUL48VkAIDmINQAAFiMUAMAYDFCDQCAxQg1AAAWa1KoTznlFH3//fcNtpeXl+uUU05p9qQAAMABTQr1rl27FAgEGmz3+Xz6+uuvmz0pAABwgPtYBr/22mvOn9988015PB7ndiAQ0Ntvv60+ffqEbXLHC7/fL7/fL7f7mJYbAIBjC/W4ceMkSS6XS1OmTAnZFxMToz59+uiJJ54I2+SOB8YY+f1+GWMiPRUAQBt0TE99B4NBBYNBnXzyySotLXVuB4NB+Xw+bd++XWPHjm2pubZJJhjU1GWbDvmrAgAAjqRJz8Xu3Lkz3PM4rrmioyM9BQBAG9XkX5q+/fbbevvtt50z6x/77//+72ZPDAAANDHUDz/8sB555BENHDhQvXr1ksvlCve8AACAmhjq5557TsuWLdPkyZPDPR8AAPAjTbqOura2VkOHDg33XAAAwEGaFOqbbrpJy5cvD/dcAADAQZr01Pf+/fu1ePFivfXWW+rfv79iYmJC9s+fPz8skwMAoL1rUqg/+eQT/eQnP5EkFRYWhuzjhWUAAIRPk0K9bt26cM8DAAAcAh9zCQCAxZp0Rj1ixIjDPsX9zjvvNHlCAADg35oU6vrfT9erq6tTQUGBCgsLG3xYBwAAaLomhXrBggWH3D537lxVV1c3a0IAAODfwvo76muvvZb3+QYAIIzCGuqNGzeqQ4cO4XxIAADatSY99X3llVeG3DbGqLi4WB999JEeeuihsEwMAAA0MdQejyfkdlRUlM444ww98sgjyszMDMvEAABAE0O9dOnScM8DAAAcQpNCXS8/P1/btm2Ty+VSv379dM4554RrXgAAQE0MdWlpqa6++mq9++67OuGEE2SMUUVFhUaMGKEVK1aoR48e4Z4nAADtUpNe9T1z5kxVVlaqqKhI+/btU1lZmQoLC1VZWanbb7893HMEAKDdatIZ9RtvvKG33npLffv2dbb169dPzzzzDC8mAwAgjJp0Rh0MBht8BrUkxcTEKBgMNntSAADggCaF+qKLLtKsWbP0zTffONu+/vpr3XHHHRo5cmTYJgcAQHvXpFAvXLhQVVVV6tOnj0499VSddtppSk1NVVVVlZ5++ulwzxEAgHarSb+jTklJ0ebNm5Wbm6tPP/1Uxhj169dPo0aNCvf8AABo147pjPqdd95Rv379VFlZKUkaPXq0Zs6cqdtvv12DBg3SWWedpffee69FJgoAQHt0TKF+8skndfPNN6tLly4N9nk8Ht1yyy2aP39+2CYHAEB7d0yh/vvf/66LL7640f2ZmZnKz89v9qQAAMABxxTqb7/99pCXZdVzu93au3dvsycFAAAOOKZQn3jiidqyZUuj+z/55BP16tWr2ZMCAAAHHFOoL730Uv3mN7/R/v37G+yrqanRnDlzNHbs2LBNDgCA9u6YLs968MEHtWrVKp1++umaMWOGzjjjDLlcLm3btk3PPPOMAoGAHnjggZaaKwAA7c4xhTopKUkbNmzQrbfeqtmzZ8sYI0lyuVwaM2aMnn32WSUlJbXIRAEAaI+O+Q1PevfurTVr1qisrEyff/65jDFKS0tT165dW2J+AAC0a016ZzJJ6tq1qwYNGhTOuQAAgIM06b2+AQBA6yDUAABYjFADAGAxQg0AgMUINQAAFiPUAABYzJpQ5+TkyOVyKSsry9lmjNHcuXOVnJysjh07avjw4SoqKgq5n8/n08yZM9W9e3fFx8friiuu0J49e1p59gAAtAwrQr1p0yYtXrxY/fv3D9k+b948zZ8/XwsXLtSmTZvk9Xo1evRoVVVVOWOysrK0evVqrVixQuvXr1d1dbXGjh2rQCDQ2ocBAEDYRTzU1dXVmjRpkpYsWRLy7mbGGD355JN64IEHdOWVVyo9PV0vvviifvjhBy1fvlySVFFRoRdeeEFPPPGERo0apXPOOUevvPKKtmzZorfeeqvR7+nz+VRZWRnyBQCAjSIe6ttuu02XXXaZRo0aFbJ9586dKikpUWZmprMtLi5Ow4YN04YNGyRJ+fn5qqurCxmTnJys9PR0Z8yh5OTkyOPxOF8pKSlhPioAAMIjoqFesWKFNm/erJycnAb7SkpKJKnBh3wkJSU5+0pKShQbG9vgfcZ/POZQZs+erYqKCudr9+7dzT0UAABaRJPf67u5du/erVmzZmnt2rXq0KFDo+NcLlfIbWNMg20HO9KYuLg4xcXFHduEAQCIgIidUefn56u0tFQDBgyQ2+2W2+1WXl6efv/738vtdjtn0gefGZeWljr7vF6vamtrVVZW1ugYAADasoiFeuTIkdqyZYsKCgqcr4EDB2rSpEkqKCjQKaecIq/Xq9zcXOc+tbW1ysvL09ChQyVJAwYMUExMTMiY4uJiFRYWOmMAAGjLIvbUd0JCgtLT00O2xcfHq1u3bs72rKwsZWdnKy0tTWlpacrOzlanTp00ceJESZLH49HUqVN11113qVu3bkpMTNTdd9+tjIyMBi9OAwCgLYpYqI/Gvffeq5qaGk2fPl1lZWUaPHiw1q5dq4SEBGfMggUL5Ha7NX78eNXU1GjkyJFatmyZoqOjIzhzAADCw2WMMZGeRKRVVlbK4/GooqJCXbp0afbj7d+/X798MldGB14M546N1YpbLzzsi+YAADiUiF9HfTzy+/3i3z8AgHAg1AAAWIxQAwBgMUINAIDFCDUAABYj1AAAWIxQAwBgMUINAIDFCDUAABYj1AAAWIxQAwBgMUINAIDFCDUAABYj1AAAWIxQAwBgMUINAIDFCDUAABYj1AAAWIxQAwBgMUINAIDFCDUAABYj1K0gGAjI7/dHehoAgDaIUAMAYDFCDQCAxQh1K+CpbwBAUxFqAAAsRqgBALAYoQYAwGKEGgAAixFqAAAsRqgBALAYoQYAwGKEGgAAixFqAAAsRqgBALAYoQYAwGKEupUYY+T3+2WMifRUAABtCKFuJYFAQBOezVMgEIj0VAAAbQihbkVR0dGRngIAoI0h1AAAWIxQAwBgMUINAIDFCDUAABYj1AAAWIxQtxKuoQYANAWhBgDAYoQaAACLEWoAACxGqAEAsBihBgDAYoS6FdR/chYAAMeKULcCEwxq2isfc3kWAOCYEepW4uKTswAATUCoAQCwGKEGAMBihBoAAIsRagAALEaoAQCwGKFuJcFAgMuzAADHjFADAGAxQg0AgMUINQAAFiPUAABYjFADAGAxQg0AgMUINQAAFiPUAABYjFADAGAxQg0AgMUINQAAFiPUAABYjFADAGAxQg0AgMUINQAAFiPUAABYjFADAGAxQg0AgMUINQAAFiPUAABYLKKhzsnJ0aBBg5SQkKCePXtq3Lhx2r59e8gYY4zmzp2r5ORkdezYUcOHD1dRUVHIGJ/Pp5kzZ6p79+6Kj4/XFVdcoT179rTmoQAA0CIiGuq8vDzddttt+uCDD5Sbmyu/36/MzEz985//dMbMmzdP8+fP18KFC7Vp0yZ5vV6NHj1aVVVVzpisrCytXr1aK1as0Pr161VdXa2xY8cqEAhE4rAAAAgblzHGRHoS9fbu3auePXsqLy9PF154oYwxSk5OVlZWlu677z5JB86ek5KS9Pjjj+uWW25RRUWFevTooZdfflkTJkyQJH3zzTdKSUnRmjVrNGbMmAbfx+fzyefzObcrKyuVkpKiiooKdenSpdnHUV1drQlPvyO5XHK5XAoGAnJFRSmuUye9On2Y3G53s78HAKB9sOp31BUVFZKkxMRESdLOnTtVUlKizMxMZ0xcXJyGDRumDRs2SJLy8/NVV1cXMiY5OVnp6enOmIPl5OTI4/E4XykpKS11SCGCgYD8fn+rfC8AwPHBmlAbY3TnnXfqggsuUHp6uiSppKREkpSUlBQyNikpydlXUlKi2NhYde3atdExB5s9e7YqKiqcr927d4f7cAAACAtrnoOdMWOGPvnkE61fv77BPpfLFXLbGNNg28EONyYuLk5xcXFNnywAAK3EijPqmTNn6rXXXtO6det00kknOdu9Xq8kNTgzLi0tdc6yvV6vamtrVVZW1ugYAADaqoiG2hijGTNmaNWqVXrnnXeUmpoasj81NVVer1e5ubnOttraWuXl5Wno0KGSpAEDBigmJiZkTHFxsQoLC50xAAC0VRF96vu2227T8uXL9ac//UkJCQnOmbPH41HHjh3lcrmUlZWl7OxspaWlKS0tTdnZ2erUqZMmTpzojJ06daruuusudevWTYmJibr77ruVkZGhUaNGtfoxGWN4wRgAIGwiGupFixZJkoYPHx6yfenSpbr++uslSffee69qamo0ffp0lZWVafDgwVq7dq0SEhKc8QsWLJDb7db48eNVU1OjkSNHatmyZYqOjm6tQ3EEAgFN/sP6o/o9OgAAR2LVddSRUllZKY/HE5brqP1+v656Mld1tbVyRUWFXEcdExenFbdeqA4dOoRp5gCA450VLyYDAACHRqhbEW94AgA4VoS6lfn9fmINADhqhBoAAIsRagAALEaoAQCwGKEGAMBihBoAAIsRagAALEaoAQCwGKFuRbzhCQDgWBFqAAAsRqgBALAYoQYAwGKEGgAAixFqAAAsRqgBALAYoQYAwGKEuhUZY+T3+2WMifRUAABtBKFuRSYY1H+8vFmBQCDSUwEAtBGEupW5oqMjPQUAQBtCqAEAsBihBgDAYoQaAACLEWoAACxGqAEAsBihBgDAYoQaAACLEWoAACxGqAEAsJg70hNob4KBgPbv3y9Jcrvdcrv5TwAAaBxn1K2MD+YAABwLQt3KTDCoqcs28cEcAICjQqgjgA/mAAAcLUINAIDFCDUAABYj1AAAWIxQAwBgMUINAIDFCDUAABYj1AAAWIxQAwBgMUINAIDFCDUAABYj1BEQDATk9/sjPQ0AQBtAqAEAsBihjgA+6hIAcLQIdQTwUZcAgKNFqCOEj7oEABwNQg0AgMUIdYTwym8AwNEg1AAAWIxQRwhn1ACAo0GoAQCwGKEGAMBihBoAAIsR6jA72ncc493JAABHg1BHiAkGNfXFj+Tz+Yg1AKBRhDqSXC5N/MN63koUANAoQh1hUbyVKADgMAh1hHE9NQDgcAh1BAUDAX4/DQA4LEIdQYQaAHAkhBoAAIsRagAALEaoI4wXkwEADodQAwBgMUINAIDFCHWEGWNUXl6uqqqqSE8FAGAhQh1hJhjUf/Ce3wCARhBqG7hcuumlfPl8Pl5YBgAIQahtERXFx14CABog1JYwwaCuXfw+n6QFAAhBqC3CJ2kBAA5GqC2zf/9+7d+/P9LTAABYglBbxBjD76kBACEItSWCgYCCgYBueOEDLtUCADgItSWCgYACfr+MpCnPb5TP54v0lAAAFiDUlgkGAlJUlIwxqqurU11dHWfXANCOHTehfvbZZ5WamqoOHTpowIABeu+99yI9pSYzxqiiokK/+N1fdNWTuSovL1dNTY2CwSDxBoB25rgI9R//+EdlZWXpgQce0Mcff6yf/exnuuSSS/TVV1+1+lzC8WIwEwzqpqUfKmiMAoGArlv8vq5+5l1VVlZq/MJ1+tXT72j//v3y+/1OvH/44QfV1NQQcAA4zrjMcfA3++DBg3Xuuedq0aJFzra+fftq3LhxysnJOeL9Kysr5fF4VFFRoS5dujRrLtXV1brm2TwF/H65oqLkcrkUDATkioqSCQYbbDvUn6NjYiRjnMeIdrtlgkHJGMnlUpTbrT9MPFs3v/iRlkwZqGnL/37gfm63lk/7maJ/dD22y+Vy/tftdh9yzuZf/yCIjo52xh+L5t4fxy9+NoDma/Ohrq2tVadOnfS///u/+sUvfuFsnzVrlgoKCpSXl9fgPj6fL+TFWhUVFTr55JO1e/fusIT6hj/8VUG/X/pXfM2/fu+sYLDBtqb+2cgoKtqtQF2t3HEdJGNkjJEJBuT611+K9dFfNHmQJCkuLk5utzvk/cTrb//Hix9o8ZTz5Ha7Gw16Pb/fL7/f74z1+/26Ycl7WnLDEHXo0OGI9/3x9z34cer3He5+x7KvpYTre0Zi7keruXOr/+97y0t/0ws3nm/NMdq85rDbj/++CpeEhIQj/iO2zf+kfvfddwoEAkpKSgrZnpSUpJKSkkPeJycnRw8//HCD7SkpKS0yx0g7Zc7RjTvx1837Pic+2Lz74/iVNDvSMwDsdDTP5Lb5UNc7+F8kxphG/5Uye/Zs3Xnnnc7tYDCoffv2qVu3bs1+eq6yslIpKSlhOTs/3rA2h8f6NI61OTzWp3G2r01CQsIRx7T5UHfv3l3R0dENzp5LS0sbnGXXi4uLU1xcXMi2E044Iazz6tKli5U/FDZgbQ6P9Wkca3N4rE/j2vLatPlXfcfGxmrAgAHKzc0N2Z6bm6uhQ4dGaFYAAIRHmz+jlqQ777xTkydP1sCBAzVkyBAtXrxYX331laZNmxbpqQEA0CzHRagnTJig77//Xo888oiKi4uVnp6uNWvWqHfv3q0+l7i4OM2ZM6fBU+tgbY6E9Wkca3N4rE/jjoe1afOXZwEAcDxr87+jBgDgeEaoAQCwGKEGAMBihBoAAIsR6jA6nj5qszF//etfdfnllys5OVkul0v/93//F7LfGKO5c+cqOTlZHTt21PDhw1VUVBQyxufzaebMmerevbvi4+N1xRVXaM+ePSFjysrKNHnyZHk8Hnk8Hk2ePFnl5eUtfHTNk5OTo0GDBikhIUE9e/bUuHHjtH379pAx7XV9Fi1apP79+ztvOjFkyBD95S9/cfa313VpTE5Ojlwul7Kyspxt7XWN5s6dK5fLFfLl9Xqd/e1iXQzCYsWKFSYmJsYsWbLEbN261cyaNcvEx8ebL7/8MtJTC6s1a9aYBx54wKxcudJIMqtXrw7Z/9hjj5mEhASzcuVKs2XLFjNhwgTTq1cvU1lZ6YyZNm2aOfHEE01ubq7ZvHmzGTFihDn77LON3+93xlx88cUmPT3dbNiwwWzYsMGkp6ebsWPHttZhNsmYMWPM0qVLTWFhoSkoKDCXXXaZOfnkk011dbUzpr2uz2uvvWZef/11s337drN9+3bz61//2sTExJjCwkJjTPtdl0P58MMPTZ8+fUz//v3NrFmznO3tdY3mzJljzjrrLFNcXOx8lZaWOvvbw7oQ6jD56U9/aqZNmxay7cwzzzT3339/hGbU8g4OdTAYNF6v1zz22GPOtv379xuPx2Oee+45Y4wx5eXlJiYmxqxYscIZ8/XXX5uoqCjzxhtvGGOM2bp1q5FkPvjgA2fMxo0bjSTz6aeftvBRhU9paamRZPLy8owxrM/Bunbtap5//nnW5UeqqqpMWlqayc3NNcOGDXNC3Z7XaM6cOebss88+5L72si489R0GtbW1ys/PV2ZmZsj2zMxMbdiwIUKzan07d+5USUlJyDrExcVp2LBhzjrk5+errq4uZExycrLS09OdMRs3bpTH49HgwYOdMeedd548Hk+bWs+KigpJUmJioiTWp14gENCKFSv0z3/+U0OGDGFdfuS2227TZZddplGjRoVsb+9r9I9//EPJyclKTU3V1VdfrR07dkhqP+tyXLwzWaQ15aM2j0f1x3qodfjyyy+dMbGxseratWuDMfX3LykpUc+ePRs8fs+ePdvMehpjdOedd+qCCy5Qenq6JNZny5YtGjJkiPbv36/OnTtr9erV6tevn/MXYXtdl3orVqzQ5s2btWnTpgb72vPPzuDBg/XSSy/p9NNP17fffqvf/va3Gjp0qIqKitrNuhDqMDqWj9o8njVlHQ4ec6jxbWk9Z8yYoU8++UTr169vsK+9rs8ZZ5yhgoIClZeXa+XKlZoyZYry8vKc/e11XSRp9+7dmjVrltauXasOHTo0Oq49rtEll1zi/DkjI0NDhgzRqaeeqhdffFHnnXeepON/XXjqOwya8lGbx6P6V2Iebh28Xq9qa2tVVlZ22DHffvttg8ffu3dvm1jPmTNn6rXXXtO6det00kknOdvb+/rExsbqtNNO08CBA5WTk6Ozzz5bTz31VLtfF+nA07OlpaUaMGCA3G633G638vLy9Pvf/15ut9uZf3teo3rx8fHKyMjQP/7xj3bzs0Oow4CP2jwgNTVVXq83ZB1qa2uVl5fnrMOAAQMUExMTMqa4uFiFhYXOmCFDhqiiokIffvihM+Zvf/ubKioqrF5PY4xmzJihVatW6Z133lFqamrI/va+Pgczxsjn87EukkaOHKktW7aooKDA+Ro4cKAmTZqkgoICnXLKKe1+jer5fD5t27ZNvXr1aj8/O6384rXjVv3lWS+88ILZunWrycrKMvHx8WbXrl2RnlpYVVVVmY8//th8/PHHRpKZP3+++fjjj53L0B577DHj8XjMqlWrzJYtW8w111xzyEslTjrpJPPWW2+ZzZs3m4suuuiQl0r079/fbNy40WzcuNFkZGRYc6lEY2699Vbj8XjMu+++G3IpyQ8//OCMaa/rM3v2bPPXv/7V7Ny503zyySfm17/+tYmKijJr1641xrTfdTmcH7/q25j2u0Z33XWXeffdd82OHTvMBx98YMaOHWsSEhKcv1vbw7oQ6jB65plnTO/evU1sbKw599xznctyjifr1q0zkhp8TZkyxRhz4HKJOXPmGK/Xa+Li4syFF15otmzZEvIYNTU1ZsaMGSYxMdF07NjRjB071nz11VchY77//nszadIkk5CQYBISEsykSZNMWVlZKx1l0xxqXSSZpUuXOmPa6/rceOONzv83evToYUaOHOlE2pj2uy6Hc3Co2+sa1V8XHRMTY5KTk82VV15pioqKnP3tYV34mEsAACzG76gBALAYoQYAwGKEGgAAixFqAAAsRqgBALAYoQYAwGKEGgAAixFqAAAsRqgBHNG7774rl8ul8vLySE8FaHcINdBKhg8frqysrIg+JsEF2h5CDQCAxQg10Aquv/565eXl6amnnpLL5ZLL5dKuXbu0detWXXrppercubOSkpI0efJkfffdd5IOnP3Gxsbqvffecx7niSeeUPfu3VVcXNzoYzZm165dGjFihCSpa9eucrlcuv766yUd+OjA22+/XT179lSHDh10wQUXaNOmTY0+Vk1NjS677DKdd9552rdvnyRp6dKl6tu3rzp06KAzzzxTzz77bMj3drlcWrVqlUaMGKFOnTrp7LPP1saNG49q/ZYtW6YTTjhBb775pvr27avOnTvr4osvVnFxsTPmUM8ujBs3zjlGSerTp49++9vf6rrrrlPnzp3Vu3dv/elPf9LevXv185//XJ07d1ZGRoY++uijo5oX0Coi/akgQHtQXl5uhgwZYm6++Wbn4y/37NljunfvbmbPnm22bdtmNm/ebEaPHm1GjBjh3O+ee+4xvXv3NuXl5aagoMDExcWZVatWNfqYP/7YvoP5/X6zcuVKI8ls377dFBcXm/LycmOMMbfffrtJTk42a9asMUVFRWbKlCmma9eu5vvvvzfG/PtT08rKykx5ebm54IILzKhRo0x1dbUxxpjFixebXr16mZUrV5odO3aYlStXmsTERLNs2TJjjDE7d+40ksyZZ55p/vznP5vt27ebX/7yl6Z3796mrq7uiOu3dOlSExMTY0aNGmU2bdpk8vPzTd++fc3EiROdMQd/2pQxxvz85z93PtnNGGN69+5tEhMTzXPPPWc+++wzc+utt5qEhARz8cUXm1dffdVs377djBs3zvTt29cEg8EjzgtoDYQaaCUHh+Shhx4ymZmZIWN2797thNQYY3w+nznnnHPM+PHjzVlnnWVuuummwz7mkfw4uPWqq6tNTEyM+Z//+R9nW21trUlOTjbz5s0Lud+nn35qzj77bHPllVcan8/njE9JSTHLly8P+V7/+Z//aYYMGWKM+Xeon3/+eWd/UVGRkWS2bdt2xHkvXbrUSDKff/65s+2ZZ54xSUlJzu2jDfW1117r3C4uLjaSzEMPPeRs27hxo5FkiouLjzgvoDW4I3QiD7R7+fn5WrdunTp37txg3xdffKHTTz9dsbGxeuWVV9S/f3/17t1bTz75ZNjn8cUXX6iurk7nn3++sy0mJkY//elPtW3btpCxo0aN0qBBg/Tqq68qOjpakrR3717t3r1bU6dO1c033+yM9fv98ng8Iffv37+/8+devXpJkkpLS3XmmWcecZ6dOnXSqaeeGnL/0tLSYzjShnNISkqSJGVkZDTYVlpaKq/Xe8yPD4QboQYiJBgM6vLLL9fjjz/eYF99xCRpw4YNkqR9+/Zp3759io+PD+s8zL8+kt7lcjXYfvC2yy67TCtXrtTWrVuduAWDQUnSkiVLNHjw4JDx9TGvFxMT4/y5/rHr738kP75v/f3r5y5JUVFRIbclqa6u7rCPUz+H5swLaGm8mAxoJbGxsQoEAs7tc889V0VFRerTp49OO+20kK/6GH/xxRe64447tGTJEp133nm67rrrQgJy8GMezRwkhdzntNNOU2xsrNavX+9sq6ur00cffaS+ffuG3P+xxx7TlClTNHLkSG3dulXSgTPQE088UTt27GhwHKmpqcewQs3To0ePkBeXBQIBFRYWttr3B1oKoQZaSZ8+ffS3v/1Nu3bt0nfffafbbrtN+/bt0zXXXKMPP/xQO3bs0Nq1a3XjjTcqEAgoEAho8uTJyszM1A033KClS5eqsLBQTzzxRKOPeaSzwN69e8vlcunPf/6z9u7dq+rqasXHx+vWW2/VPffcozfeeENbt27VzTffrB9++EFTp05t8Bi/+93vNGnSJF100UX69NNPJUlz585VTk6OnnrqKX322WfasmWLli5dqvnz54d3EQ/joosu0uuvv67XX39dn376qaZPn8714jguEGqgldx9992Kjo5Wv3791KNHD9XW1ur9999XIBDQmDFjlJ6erlmzZsnj8SgqKkqPPvqodu3apcWLF0uSvF6vnn/+eT344IMqKCg45GN+9dVXh53DiSeeqIcfflj333+/kpKSNGPGDEkHzpSvuuoqTZ48Weeee64+//xzvfnmm+rateshH2fBggUaP368LrroIn322We66aab9Pzzz2vZsmXKyMjQsGHDtGzZslY9o77xxhs1ZcoUXXfddRo2bJhSU1Ody9GAtsxlDv6lDgAAsAZn1AAAWIxQA8eRadOmqXPnzof8mjZtWqSn16hLLrmk0XlnZ2dHenpARPHUN3AcKS0tVWVl5SH3denSRT179mzlGR2dr7/+WjU1NYfcl5iYqMTExFaeEWAPQg0AgMV46hsAAIsRagAALEaoAQCwGKEGAMBihBoAAIsRagAALEaoAQCw2P8DxgM2ntQosaMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 500x500 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#check length distribution\n",
    "sns_dis = sns.displot(df['text_token_num'])\n",
    "sns_dis.savefig('all_text_len_dist.pdf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "18caedde-d3aa-472e-af36-cabea9f86624",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "13653"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df499 = df[df['text_token_num'] <= 500]\n",
    "len(df499)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "ea506da7-a32b-4c65-86bf-7f114744e484",
   "metadata": {},
   "outputs": [],
   "source": [
    "df499.to_csv('cleaned_plain-text_labeled_term+combined_no_ref_no_cit_def_same_len_only_must_conatin_B_less_than_500tokens.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "0088e4c4-72f7-46ad-b25b-c54cb044a3c9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "average length =  67.62455138064894\n",
      "maximun length =  497\n"
     ]
    }
   ],
   "source": [
    "print(\"average length = \", sum(df499['text_token_num']) / len(df499['text_token_num']))\n",
    "print(\"maximun length = \", max(df499['text_token_num']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "b3b157d6-b121-4743-a954-3355f217b691",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeoAAAHpCAYAAABN+X+UAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAytElEQVR4nO3df1RVdb7/8ddRfqiAJxDleBQVC1PUmrLGtL6jpmIlWct7r1OW2c26lj+Ksro51UTdgnImtVH7YXXVyXGY1lKnVjWm9sNyrMkwJgGTIbXUgbBCkEQQ+Hz/8LKHww8FBM4HeD7W2mtx9v6cfd77o5wXe+/P3ttljDECAABW6uTvAgAAQP0IagAALEZQAwBgMYIaAACLEdQAAFiMoAYAwGIENQAAFiOoG8gYo6KiInHZOQCgNRHUDXTs2DG53W4dO3bM36UAADoQghoAAIsR1AAAWIygBgDAYgQ1AAAWI6gBALAYQQ0AgMUIagAALEZQAwBgMYIaAACLEdQAAFiMoAYAwGIENQAAFiOoAQCwGEENAIDF/BrUH330ka699lp5vV65XC79+c9/9llujFFSUpK8Xq+6du2qsWPHKjMz06dNaWmp5s+fr8jISIWEhGjKlCk6dOiQT5uCggLNmDFDbrdbbrdbM2bM0NGjR1t46wAAOHt+DeqffvpJF154oZYvX17n8kWLFmnx4sVavny5du7cKY/Ho4kTJ/o8EzoxMVEbN25Uamqqtm/fruLiYiUkJKiiosJpM336dKWnp2vTpk3atGmT0tPTNWPGjBbfPgAAzpqxhCSzceNG53VlZaXxeDzm6aefduadOHHCuN1u8+KLLxpjjDl69KgJDAw0qampTpvDhw+bTp06mU2bNhljjMnKyjKSzKeffuq0+eSTT4wk89VXX9Vbz4kTJ0xhYaEzHTx40EgyhYWFzbXJAACckbXnqPfv36+8vDzFx8c784KDgzVmzBjt2LFDkpSWlqaTJ0/6tPF6vRo2bJjT5pNPPpHb7dbIkSOdNpdddpncbrfTpi4pKSnOoXK3263o6Ojm3kQAAM7I2qDOy8uTJEVFRfnMj4qKcpbl5eUpKChI4eHhp23Tq1evWuvv1auX06YuCxcuVGFhoTMdPHjwrLanJZSXlysrK8uZysvL/V0SAKCZBfi7gDNxuVw+r40xtebVVLNNXe3PtJ7g4GAFBwc3strWlZ2drdkr3lJoz74qPnJIL81NUFxcnL/LAgA0I2v3qD0ejyTV2uvNz8939rI9Ho/KyspUUFBw2jbfffddrfUfOXKk1t56WxTas6/c3hiF9uzr71IAAC3A2qCOiYmRx+PRli1bnHllZWXatm2bRo8eLUkaMWKEAgMDfdrk5uYqIyPDaTNq1CgVFhbqs88+c9r87W9/U2FhodMGAABb+fXQd3FxsXJycpzX+/fvV3p6uiIiItSvXz8lJiYqOTlZsbGxio2NVXJysrp166bp06dLktxut2bNmqUFCxaoR48eioiI0P3336/hw4drwoQJkqQhQ4boqquu0h133KGXXnpJkvRf//VfSkhI0Pnnn9/6Gw0AQCP4Nag///xzjRs3znl93333SZJmzpyp1atX68EHH1RJSYnmzJmjgoICjRw5Ups3b1ZYWJjzniVLliggIEDTpk1TSUmJxo8fr9WrV6tz585Omz/84Q+6++67ndHhU6ZMqffabQAAbOIyxhh/F9EWFBUVye12q7CwUN27d/d3OZKkrKwsLXg9XW5vjAr/uV/PTvsZg8kAoJ2x9hw1AAAgqAEAsBpBDQCAxQhqAAAsRlADAGAxghoAAIsR1AAAWIygBgDAYgQ1AAAWI6gBALAYQQ0AgMUIagAALEZQAwBgMYIaAACLEdQAAFiMoAYAwGIENQAAFiOoAQCwGEENAIDFCGoAACxGUAMAYDGCGgAAixHUAABYjKAGAMBiBDUAABYjqAEAsBhBDQCAxQhqAAAsRlADAGAxghoAAIsR1AAAWIygBgDAYgQ1AAAWI6gBALAYQQ0AgMUIagAALEZQAwBgMYIaAACLEdQAAFiMoAYAwGIENQAAFiOoAQCwGEENAIDFCGoAACxGUAMAYDGCGgAAixHUAABYjKAGAMBiBDUAABYjqAEAsBhBDQCAxQhqAAAsRlADAGAxghoAAIsR1AAAWIygBgDAYgQ1AAAWI6gBALAYQQ0AgMUIagAALEZQAwBgMYIaAACLEdQAAFiMoAYAwGIENQAAFiOoAQCwGEENAIDFCGoAACxGUAMAYDGCGgAAi1kd1OXl5XrkkUcUExOjrl27auDAgXriiSdUWVnptDHGKCkpSV6vV127dtXYsWOVmZnps57S0lLNnz9fkZGRCgkJ0ZQpU3To0KHW3hwAABrN6qB+5pln9OKLL2r58uXas2ePFi1apN/85jdatmyZ02bRokVavHixli9frp07d8rj8WjixIk6duyY0yYxMVEbN25Uamqqtm/fruLiYiUkJKiiosIfm2Wd8vJyZWVlOVN5ebm/SwIA/J8AfxdwOp988omuu+46TZ48WZI0YMAA/fGPf9Tnn38u6dTe9NKlS/Xwww9r6tSpkqQ1a9YoKipK69at0+zZs1VYWKhXX31Vr732miZMmCBJWrt2raKjo7V161ZNmjTJPxtnkezsbM1e8ZZCe/ZV8ZFDemluguLi4vxdFgBAlu9RX3HFFXrvvfeUnZ0tSfr73/+u7du365prrpEk7d+/X3l5eYqPj3feExwcrDFjxmjHjh2SpLS0NJ08edKnjdfr1bBhw5w2dSktLVVRUZHP1J6F9uwrtzdGoT37+rsUAEA1Vu9R//d//7cKCws1ePBgde7cWRUVFXrqqad04403SpLy8vIkSVFRUT7vi4qK0jfffOO0CQoKUnh4eK02Ve+vS0pKih5//PHm3BwAABrN6j3qP/3pT1q7dq3WrVunXbt2ac2aNfrtb3+rNWvW+LRzuVw+r40xtebVdKY2CxcuVGFhoTMdPHiw6RsCAEATWb1H/cADD+ihhx7SDTfcIEkaPny4vvnmG6WkpGjmzJnyeDySTu019+7d23lffn6+s5ft8XhUVlamgoICn73q/Px8jR49ut7PDg4OVnBwcEtsFgAADWb1HvXx48fVqZNviZ07d3Yuz4qJiZHH49GWLVuc5WVlZdq2bZsTwiNGjFBgYKBPm9zcXGVkZJw2qAEAsIHVe9TXXnutnnrqKfXr109Dhw7VF198ocWLF+u2226TdOqQd2JiopKTkxUbG6vY2FglJyerW7dumj59uiTJ7XZr1qxZWrBggXr06KGIiAjdf//9Gj58uDMKHAAAW1kd1MuWLdOjjz6qOXPmKD8/X16vV7Nnz9avf/1rp82DDz6okpISzZkzRwUFBRo5cqQ2b96ssLAwp82SJUsUEBCgadOmqaSkROPHj9fq1avVuXNnf2wWAAAN5jLGGH8X0RYUFRXJ7XarsLBQ3bt393c5kqSsrCwteD1dbm+MCv+5X89O+1mTrn9urvUAAJqf1eeoAQDo6AhqAAAsRlADAGAxghoAAIsR1AAAWIygBgDAYgQ1AAAWs/qGJzh75eXlzmNCJWnQoEEKCOCfHQDaCr6x26Hq4ZyTk6NFm/YorFe0io8c0ktzE7iZCQC0IQR1O5Sdna3ZK95SaM++yt+bpu79hsrtjfF3WQCAJuAcdTsV2rOv3N4YdYuI8ncpAICzQFADAGAxghoAAIsR1AAAWIzBZO1EZWWFcnJyJJ0a6c3TSwGgfSCo24njP+TqyTcOqEd0sTPSGwDQ9nHoux0JiezDSG8AaGcIagAALEZQAwBgMYIaAACLEdQAAFiMoAYAwGIENQAAFuM66g6k+k1RJJ5NDQBtAd/SHUj1m6LwbGoAaBsI6g6m6qYoAIC2gXPUAABYjKAGAMBiBDUAABYjqAEAsBhBDQCAxQhqAAAsRlADAGAxrqNuY8rLy5WdnS1JysnJkTHGzxUBAFoSQd3GZGdna/aKtxTas6/y96ape7+h/i4JANCCOPTdBoX27Cu3N0bdIqL8XQoAoIUR1AAAWIygBgDAYgQ1AAAWI6gBALAYQQ0AgMUIagAALEZQAwBgMYIaAACLEdQAAFiMoAYAwGIENQAAFiOoAQCwGEENAIDFCGoAACxGUAMAYDGCGgAAixHUAABYjKAGAMBiAf4uAP5RWVmhnJwcSVJOTo6MMX6uCABQF4K6gzr+Q66efOOAekQXK39vmrr3G+rvkgAAdeDQdwcWEtlHbm+MukVE+bsUAEA9CGoAACxGUAMAYDGCGgAAixHUAABYjKAGAMBiBDUAABYjqAEAsBhBDQCAxQhqAAAsRlADAGAxghoAAIsR1AAAWIygBgDAYgQ1AAAWsz6oDx8+rJtvvlk9evRQt27d9LOf/UxpaWnOcmOMkpKS5PV61bVrV40dO1aZmZk+6ygtLdX8+fMVGRmpkJAQTZkyRYcOHWrtTQEAoNGsDuqCggJdfvnlCgwM1F/+8hdlZWXp2Wef1TnnnOO0WbRokRYvXqzly5dr586d8ng8mjhxoo4dO+a0SUxM1MaNG5Wamqrt27eruLhYCQkJqqio8MNWAQDQcAH+LuB0nnnmGUVHR2vVqlXOvAEDBjg/G2O0dOlSPfzww5o6daokac2aNYqKitK6des0e/ZsFRYW6tVXX9Vrr72mCRMmSJLWrl2r6Ohobd26VZMmTWrVbQIAoDGs3qN+8803dckll+g//uM/1KtXL1100UV6+eWXneX79+9XXl6e4uPjnXnBwcEaM2aMduzYIUlKS0vTyZMnfdp4vV4NGzbMaVOX0tJSFRUV+UwAALQ2q4N63759euGFFxQbG6t3331Xd955p+6++279/ve/lyTl5eVJkqKionzeFxUV5SzLy8tTUFCQwsPD621Tl5SUFLndbmeKjo5uzk0DAKBBrA7qyspKXXzxxUpOTtZFF12k2bNn64477tALL7zg087lcvm8NsbUmlfTmdosXLhQhYWFznTw4MGmbwgAAE1kdVD37t1bcXFxPvOGDBmib7/9VpLk8XgkqdaecX5+vrOX7fF4VFZWpoKCgnrb1CU4OFjdu3f3mQAAaG1WB/Xll1+uvXv3+szLzs5W//79JUkxMTHyeDzasmWLs7ysrEzbtm3T6NGjJUkjRoxQYGCgT5vc3FxlZGQ4bQAAsJXVo77vvfdejR49WsnJyZo2bZo+++wzrVy5UitXrpR06pB3YmKikpOTFRsbq9jYWCUnJ6tbt26aPn26JMntdmvWrFlasGCBevTooYiICN1///0aPny4MwocAABbWR3Ul156qTZu3KiFCxfqiSeeUExMjJYuXaqbbrrJafPggw+qpKREc+bMUUFBgUaOHKnNmzcrLCzMabNkyRIFBARo2rRpKikp0fjx47V69Wp17tzZH5sFAECDWR3UkpSQkKCEhIR6l7tcLiUlJSkpKaneNl26dNGyZcu0bNmyFqgQAICWY/U5agAAOjqCGgAAixHUAABYzPpz1JDKy8uVnZ0tScrJyZExxs8VAQBaC0HdBmRnZ2v2ircU2rOv8vemqXu/of4uCQDQSjj03UaE9uwrtzdG3SLqv5saAKD9IagBALBYk4J64MCB+uGHH2rNP3r0qAYOHHjWRQEAgFOaFNQHDhxQRUVFrfmlpaU6fPjwWRcFAABOadRgsjfffNP5+d1335Xb7XZeV1RU6L333tOAAQOarTgAADq6RgX19ddfL+nUbTtnzpzpsywwMFADBgzQs88+22zFAQDQ0TUqqCsrKyWderzkzp07FRkZ2SJFAQCAU5p0HfX+/fubuw4AAFCHJt/w5L333tN7772n/Px8Z0+7yv/+7/+edWEAAKCJQf3444/riSee0CWXXKLevXvL5XI1d10AAEBNDOoXX3xRq1ev1owZM5q7HgAAUE2TrqMuKyvT6NGjm7sWAABQQ5OC+vbbb9e6deuauxYAAFBDkw59nzhxQitXrtTWrVt1wQUXKDAw0Gf54sWLm6U4+F/1R2xK0qBBgxQQwEPXAKC1NOkb98svv9TPfvYzSVJGRobPMgaWtS/VH7FZfOSQXpqboLi4OH+XBQAdRpOC+oMPPmjuOmCxqkdsAgBaH4+5BADAYk3aox43btxpD3G///77TS4IAAD8S5OCuur8dJWTJ08qPT1dGRkZtR7WgbalsrJCOTk5zuucnBwZY/xYEQB0bE0K6iVLltQ5PykpScXFxWdVEPzr+A+5evKNA+oRferfMX9vmrr3G+rnqgCg42rWc9Q333wz9/luB0Ii+8jtjZHbG6NuEVH+LgcAOrRmDepPPvlEXbp0ac5VAgDQoTXp0PfUqVN9XhtjlJubq88//1yPPvposxQGAACaGNRut9vndadOnXT++efriSeeUHx8fLMUBgAAmhjUq1atau46AABAHc7qps1paWnas2ePXC6X4uLidNFFFzVXXQAAQE0M6vz8fN1www368MMPdc4558gYo8LCQo0bN06pqanq2bNnc9cJAECH1KRR3/Pnz1dRUZEyMzP1448/qqCgQBkZGSoqKtLdd9/d3DXCElU3Q8nKylJWVpbKy8v9XRIAtHtN2qPetGmTtm7dqiFDhjjz4uLitGLFCgaTtWPVb4bCk7QAoHU0KagrKytrPYNakgIDA1VZWXnWRcFeVTdDAQC0jiYd+r7yyit1zz336J///Kcz7/Dhw7r33ns1fvz4ZisOAICOrklBvXz5ch07dkwDBgzQueeeq/POO08xMTE6duyYli1b1tw1AgDQYTXp0Hd0dLR27dqlLVu26KuvvpIxRnFxcZowYUJz1wcAQIfWqD3q999/X3FxcSoqKpIkTZw4UfPnz9fdd9+tSy+9VEOHDtXHH3/cIoUCANARNSqoly5dqjvuuEPdu3evtcztdmv27NlavHhxsxUHAEBH16ig/vvf/66rrrqq3uXx8fFKS0s766IAAMApjQrq7777rs7LsqoEBAToyJEjZ10UAAA4pVFB3adPH+3evbve5V9++aV69+591kUBAIBTGhXU11xzjX7961/rxIkTtZaVlJToscceU0JCQrMVBwBAR9eoy7MeeeQRbdiwQYMGDdK8efN0/vnny+Vyac+ePVqxYoUqKir08MMPt1StAAB0OI0K6qioKO3YsUN33XWXFi5cKGOMJMnlcmnSpEl6/vnnFRUV1SKFAgDQETX6hif9+/fXO++8o4KCAuXk5MgYo9jYWIWHh7dEfQAAdGhNujOZJIWHh+vSSy9tzloAAEANTbrXNwAAaB0ENQAAFiOoAQCwGEENAIDFCGoAACxGUAMAYDGCGgAAixHUAABYjKAGAMBiBDUAABYjqAEAsBhBDQCAxQhqAAAsRlADAGAxghoAAIsR1AAAWCzA3wWgbaqsrFBOTo7zetCgQQoI4L8TADQ3vlnRJMd/yNWTbxxQj+hiFR85pJfmJiguLs7fZQFAu0NQo8lCIvvI7Y3xdxkA0K5xjhoAAIsR1AAAWIxD32hW5eXlys7Odl4zyAwAzg7foGhW2dnZmr3iLYX27MsgMwBoBm3q0HdKSopcLpcSExOdecYYJSUlyev1qmvXrho7dqwyMzN93ldaWqr58+crMjJSISEhmjJlig4dOtTK1XccoT37yu2NUWjPvv4uBQDavDYT1Dt37tTKlSt1wQUX+MxftGiRFi9erOXLl2vnzp3yeDyaOHGijh075rRJTEzUxo0blZqaqu3bt6u4uFgJCQmqqKho7c0AAKBR2kRQFxcX66abbtLLL7+s8PBwZ74xRkuXLtXDDz+sqVOnatiwYVqzZo2OHz+udevWSZIKCwv16quv6tlnn9WECRN00UUXae3atdq9e7e2bt1a72eWlpaqqKjIZwIAoLW1iaCeO3euJk+erAkTJvjM379/v/Ly8hQfH+/MCw4O1pgxY7Rjxw5JUlpamk6ePOnTxuv1atiwYU6buqSkpMjtdjtTdHR0M28VAABnZn1Qp6amateuXUpJSam1LC8vT5IUFRXlMz8qKspZlpeXp6CgIJ898Zpt6rJw4UIVFhY608GDB892UwAAaDSrR30fPHhQ99xzjzZv3qwuXbrU287lcvm8NsbUmlfTmdoEBwcrODi4cQUDANDMrN6jTktLU35+vkaMGKGAgAAFBARo27Zt+t3vfqeAgABnT7rmnnF+fr6zzOPxqKysTAUFBfW2AQDAVlYH9fjx47V7926lp6c70yWXXKKbbrpJ6enpGjhwoDwej7Zs2eK8p6ysTNu2bdPo0aMlSSNGjFBgYKBPm9zcXGVkZDhtAACwldWHvsPCwjRs2DCfeSEhIerRo4czPzExUcnJyYqNjVVsbKySk5PVrVs3TZ8+XZLkdrs1a9YsLViwQD169FBERITuv/9+DR8+vNbgNAAAbGN1UDfEgw8+qJKSEs2ZM0cFBQUaOXKkNm/erLCwMKfNkiVLFBAQoGnTpqmkpETjx4/X6tWr1blzZz9WXr+at+HMycmRMcaPFQEA/KXNBfWHH37o89rlcikpKUlJSUn1vqdLly5atmyZli1b1rLFNZPqt+GUpPy9aereb6ifqwIA+EObC+qOouo2nJJUfITbnQJAR2X1YDIAADo6ghoAAIsR1AAAWIygBgDAYgQ1AAAWI6gBALAYQQ0AgMUIagAALEZQAwBgMYIaAACLEdQAAFiMoAYAwGIENQAAFiOoAQCwGI+5xFmrrKxQTk6OJCknJ0fGGD9XBADtB0GNs3b8h1w9+cYB9YguVv7eNHXvN9TfJQFAu8GhbzSLkMg+cntj1C0iyt+lAEC7QlADAGAxghoAAIsR1AAAWIygBgDAYgQ1AAAWI6gBALAYQQ0AgMUIagAALEZQAwBgMYIaAACLEdQAAFiMoAYAwGIENQAAFiOoAQCwGEENAIDFCGoAACxGUAMAYDGCGgAAixHUAABYjKAGAMBiAf4uAB1DeXm5srOzndeDBg1SQAD//QDgTPimRKvIzs7W7BVvKbRnXxUfOaSX5iYoLi7O32UBgPUIarSa0J595fbG+LsMAGhTOEcNAIDFCGoAACxGUAMAYDHOUaPFVFZWKCcnR5KUk5MjY4yfKwKAtoegRos5/kOunnzjgHpEFyt/b5q69xvq75IAoM3h0DdaVEhkH7m9MeoWEeXvUgCgTSKoAQCwGEENAIDFCGoAACzGYDK0uuqjwSXu+w0Ap8O3I1pd9dHg3PcbAE6PoIZfVI0GBwCcHueoAQCwGEENAIDFCGoAACxGUAMAYDEGk8GvuFQLAE6Pb0T4FZdqAcDpEdTwOy7VAoD6cY4aAACLEdQAAFiMoAYAwGIENQAAFmMwGazBpVoAUBvfgrAGl2oBQG0ENazCpVoA4Itz1AAAWIygBgDAYhz6RptQXl6u7Oxs5zUDzQB0FFbvUaekpOjSSy9VWFiYevXqpeuvv1579+71aWOMUVJSkrxer7p27aqxY8cqMzPTp01paanmz5+vyMhIhYSEaMqUKTp06FBrbgrOUnZ2tmaveEsLXk/X7BVv+YQ2ALRnVgf1tm3bNHfuXH366afasmWLysvLFR8fr59++slps2jRIi1evFjLly/Xzp075fF4NHHiRB07dsxpk5iYqI0bNyo1NVXbt29XcXGxEhISVFFR4Y/NQhOF9uwrtzdGoT37+rsUAGg1Vh873LRpk8/rVatWqVevXkpLS9MvfvELGWO0dOlSPfzww5o6daokac2aNYqKitK6des0e/ZsFRYW6tVXX9Vrr72mCRMmSJLWrl2r6Ohobd26VZMmTWr17QIAoKGs3qOuqbCwUJIUEREhSdq/f7/y8vIUHx/vtAkODtaYMWO0Y8cOSVJaWppOnjzp08br9WrYsGFOm7qUlpaqqKjIZwIAoLW1maA2xui+++7TFVdcoWHDhkmS8vLyJElRUVE+baOiopxleXl5CgoKUnh4eL1t6pKSkiK32+1M0dHRzbk5AAA0SJsJ6nnz5unLL7/UH//4x1rLXC6Xz2tjTK15NZ2pzcKFC1VYWOhMBw8ebFrhAACchTYR1PPnz9ebb76pDz74QH37/msgkcfjkaRae8b5+fnOXrbH41FZWZkKCgrqbVOX4OBgde/e3WcCAKC1WR3UxhjNmzdPGzZs0Pvvv6+YGN9bS8bExMjj8WjLli3OvLKyMm3btk2jR4+WJI0YMUKBgYE+bXJzc5WRkeG0AQDAVlaP+p47d67WrVunN954Q2FhYc6es9vtVteuXeVyuZSYmKjk5GTFxsYqNjZWycnJ6tatm6ZPn+60nTVrlhYsWKAePXooIiJC999/v4YPH+6MArdB9Rt65OTkyBjj54oAADawOqhfeOEFSdLYsWN95q9atUq33nqrJOnBBx9USUmJ5syZo4KCAo0cOVKbN29WWFiY037JkiUKCAjQtGnTVFJSovHjx2v16tXq3Llza23KGVXd0CO0Z1/l701T935D/V0SAMACVgd1Q/YqXS6XkpKSlJSUVG+bLl26aNmyZVq2bFkzVtf8qm7oUXyEu6YBAE6x+hw1AAAdHUENAIDFCGoAACxm9TlqdFyVlRXKyclxXjMSHkBHRVDDSsd/yNWTbxxQj+hiSWIkPIAOi6CGtUIi+8jtPXWTG0bCA+ioOEcNAIDFCGoAACxGUAMAYDGCGgAAixHUAABYjKAGAMBiBDUAABYjqAEAsBhBDQCAxQhqAAAsxi1E0ebUfGDHoEGDFBDAf2UA7RPfbmhzqj+wo/jIIb00N0FxcXH+LgsAWgRBjTap+gM7AKA94xw1AAAWY48a7UZ5ebmys7OdnyU55645jw2greKbC+1Gdna2Zq94S6E9+yp/b5o6dztHPaLP5Tw2gDaNoEa7Etqzr9zeGBUfOaSA0Ei5vTGMEgfQpvFthXaPUeIA2jKCGh0Co8QBtFWM+gYAwGLsUaNNq37+OScnR8YYP1cEAM2LoEabVv38c/7eNHXvN9TfJQFAs+LQN9q8qvPP3SKi/F0KADQ7ghoAAIsR1AAAWIxz1OhQuPkJgLaGbyh0KNz8BEBbQ1Cjw+HmJwDaEs5RAwBgMfaoAfk+IlPi3DUAe/BNBMj3EZmcuwZgE4Ia+D9Vj8gEAJtwjhoAAIsR1AAAWIygBgDAYpyjRofFIzIBtAUENTqs+h6RWfM2oxKXawHwH7550KFV3aWs+MghZ171AJfE5VoA/IqgBurAbUYB2IKgBs6AJ24B8Ce+bYAz4IlbAPyJoAYagEPhAPyFoAaaiAd5AGgNfKv4UfUveq7jbXt4kAeA1kBQ+1H1L/rq1/Gi7eBBHgBaGkHtZ1Vf9NWv44W9uJsZgNZGUAONUN/dzOrDeWwAZ4tvDKCR6rqbWX04jw3gbBHUQDM43U1ROI8N4GwQ1EAz4KYoAFoKQQ00E26KAqAlENRAM6tvZDj3DAfQFHxLAM2svpHh9R0eZ2Q4gNPh2wBoAfWNDK+aX3Ove9GmPQrrFX3a89sEOtAx8VsO+EFde901A7xKVSBzqRfQMRHUgJ/UtdddPcAl1QpkLvUCOh6CGrAMo8cBVEdQA+0U57SB9oHfWqCd4pw20D4Q1IDFGvu0rprPOOcwOtD2EdSAxRr7tC6ecQ60PwQ1YLm6RodX39MuLy+XJAUEBPjsRdfXXqr7fHX1vfHq66zrNee7gdbDbxrQBtXc0+7c7Rz1iD633r3o6u2PffeNHrx6qM477zxJdV+nXX2dknxec74baF0ENdBGVd9zDgiNPOMzsqu3f/KNv9f5pK+q67Srr1OSz+uzuWd5fSPRGaEO1I/fBKADqu9WpmcarCY1/pGeNQe41XW71NONUCfE0dF1qP/tzz//vH7zm98oNzdXQ4cO1dKlS/X//t//a9Uaan5pNeSLEWgpjR2sVuVMo8nrC+f6bpd6uhHq1UO8+mH76ufN6/tZ6pjBzh837UuH+Zf705/+pMTERD3//PO6/PLL9dJLL+nqq69WVlaW+vXr12p1MCoXtqnvASINcbpBbXWFc323S63+u1Dz0HrNAXJVh+1rnpuv6+fGBvvpAl+q+1D92a73TOs83WfU9/6GPuilyuk+u7lCvjU+o73qMD2zePFizZo1S7fffrskaenSpXr33Xf1wgsvKCUlpVVrqX4eEGjLzjSorTHnzetap6Raf9DWd26+vp8bE+z1/SzJJ/Rr/iHS1PU2ZJ1VfdDY91f1f3X1hWV9n11z4OHZ/PHRkM843Tobe9SkIX8YNGTMxOn+YGstHSKoy8rKlJaWpoceeshnfnx8vHbs2FHne0pLS1VaWuq8LiwslCQVFRWdVS3FxcU6ejhH5aXHdey7b9W5qFABnSrr/VlSg9q1xM/+/Gxb6uion92oOrq6VV56XBUnS6WyE87PRf/8uumf/X/rlNRs66pZX2N+lqTiI4f14Es5cvfyquBgtsJ6n6eu7rNbb0PWWdUHjX1/VZ8Vf/9PffllJxUXF2vfvn1K/tOH6npOTxUczFan4NDTfnb19Uuq9Z66fi45ekS/+uVYDRw4sNbnnekz6lvn6T67+udVV9+21lff6equXkfJ0SN6ZeFtGjx48FllQZWwsDC5XK7TNzIdwOHDh40k89e//tVn/lNPPWUGDRpU53see+wxI4mJiYmJianFpsLCwjNmWIfYo65S868WY0y9f8ksXLhQ9913n/O6srJSP/74o3r06HHmv37qUVRUpOjoaB08eFDdu3dv0jo6Avqp4eirhqGfGoZ+arjm6quwsLAztukQQR0ZGanOnTsrLy/PZ35+fr6ioqLqfE9wcLCCg4N95p1zzjnNUk/37t35JWgA+qnh6KuGoZ8ahn5quNboq04tunZLBAUFacSIEdqyZYvP/C1btmj06NF+qgoAgDPrEHvUknTfffdpxowZuuSSSzRq1CitXLlS3377re68805/lwYAQL06TFD/8pe/1A8//KAnnnhCubm5GjZsmN555x3179+/1WoIDg7WY489VuuQOnzRTw1HXzUM/dQw9FPDtWZfuYzh1lgAANiqQ5yjBgCgrSKoAQCwGEENAIDFCGoAACxGULeS559/XjExMerSpYtGjBihjz/+2N8ltaqPPvpI1157rbxer1wul/785z/7LDfGKCkpSV6vV127dtXYsWOVmZnp06a0tFTz589XZGSkQkJCNGXKFB061L4ebJKSkqJLL71UYWFh6tWrl66//nrt3bvXpw19dcoLL7ygCy64wLnhxKhRo/SXv/zFWU4/1S0lJUUul0uJiYnOPPrqlKSkJLlcLp/J4/E4y/3WT2d1E200SGpqqgkMDDQvv/yyycrKMvfcc48JCQkx33zzjb9LazXvvPOOefjhh8369euNJLNx40af5U8//bQJCwsz69evN7t37za//OUvTe/evU1RUZHT5s477zR9+vQxW7ZsMbt27TLjxo0zF154oSkvL2/lrWk5kyZNMqtWrTIZGRkmPT3dTJ482fTr188UFxc7beirU958803z9ttvm71795q9e/eaX/3qVyYwMNBkZGQYY+inunz22WdmwIAB5oILLjD33HOPM5++OuWxxx4zQ4cONbm5uc6Un5/vLPdXPxHUreDnP/+5ufPOO33mDR482Dz00EN+qsi/agZ1ZWWl8Xg85umnn3bmnThxwrjdbvPiiy8aY4w5evSoCQwMNKmpqU6bw4cPm06dOplNmza1Wu2tLT8/30gy27ZtM8bQV2cSHh5uXnnlFfqpDseOHTOxsbFmy5YtZsyYMU5Q01f/8thjj5kLL7ywzmX+7CcOfbewqkdsxsfH+8w/3SM2O5r9+/crLy/Pp4+Cg4M1ZswYp4/S0tJ08uRJnzZer1fDhg1r1/1Y9XjViIgISfRVfSoqKpSamqqffvpJo0aNop/qMHfuXE2ePFkTJkzwmU9f+frHP/4hr9ermJgY3XDDDdq3b58k//ZTh7kzmb98//33qqioqPXwj6ioqFoPCemoqvqhrj765ptvnDZBQUEKDw+v1aa99qMxRvfdd5+uuOIKDRs2TBJ9VdPu3bs1atQonThxQqGhodq4caPi4uKcL0X66ZTU1FTt2rVLO3furLWM/1P/MnLkSP3+97/XoEGD9N133+nJJ5/U6NGjlZmZ6dd+IqhbSWMesdlRNaWP2nM/zps3T19++aW2b99eaxl9dcr555+v9PR0HT16VOvXr9fMmTO1bds2Zzn9JB08eFD33HOPNm/erC5dutTbjr6Srr76aufn4cOHa9SoUTr33HO1Zs0aXXbZZZL8008c+m5hTXnEZkdTNarydH3k8XhUVlamgoKCetu0J/Pnz9ebb76pDz74QH379nXm01e+goKCdN555+mSSy5RSkqKLrzwQj333HP0UzVpaWnKz8/XiBEjFBAQoICAAG3btk2/+93vFBAQ4GwrfVVbSEiIhg8frn/84x9+/T9FULcwHrF5ZjExMfJ4PD59VFZWpm3btjl9NGLECAUGBvq0yc3NVUZGRrvqR2OM5s2bpw0bNuj9999XTEyMz3L66vSMMSotLaWfqhk/frx2796t9PR0Z7rkkkt00003KT09XQMHDqSv6lFaWqo9e/aod+/e/v0/1eRhaGiwqsuzXn31VZOVlWUSExNNSEiIOXDggL9LazXHjh0zX3zxhfniiy+MJLN48WLzxRdfOJeoPf3008btdpsNGzaY3bt3mxtvvLHOyx769u1rtm7danbt2mWuvPLKdnd5yF133WXcbrf58MMPfS4ROX78uNOGvjpl4cKF5qOPPjL79+83X375pfnVr35lOnXqZDZv3myMoZ9Op/qob2PoqyoLFiwwH374odm3b5/59NNPTUJCggkLC3O+q/3VTwR1K1mxYoXp37+/CQoKMhdffLFzuU1H8cEHHxhJtaaZM2caY05d+vDYY48Zj8djgoODzS9+8Quze/dun3WUlJSYefPmmYiICNO1a1eTkJBgvv32Wz9sTcupq48kmVWrVjlt6KtTbrvtNud3qmfPnmb8+PFOSBtDP51OzaCmr06pui46MDDQeL1eM3XqVJOZmeks91c/8ZhLAAAsxjlqAAAsRlADAGAxghoAAIsR1AAAWIygBgDAYgQ1AAAWI6gBALAYQQ0AgMUIagCN9uGHH8rlcuno0aP+LgVo9whqwE/Gjh2rxMREv66TwAXsR1ADAGAxghrwg1tvvVXbtm3Tc889J5fLJZfLpQMHDigrK0vXXHONQkNDFRUVpRkzZuj777+XdGrvNygoSB9//LGznmeffVaRkZHKzc2td531OXDggMaNGydJCg8Pl8vl0q233irp1OP97r77bvXq1UtdunTRFVdcoZ07d9a7rpKSEk2ePFmXXXaZfvzxR0nSqlWrNGTIEHXp0kWDBw/W888/7/PZLpdLGzZs0Lhx49StWzddeOGF+uSTTxrUf6tXr9Y555yjd999V0OGDFFoaKiuuuoq5ebmOm3qOrpw/fXXO9soSQMGDNCTTz6pW265RaGhoerfv7/eeOMNHTlyRNddd51CQ0M1fPhwff755w2qC2gRZ/VIDwBNcvToUTNq1Chzxx13OI+yPHTokImMjDQLFy40e/bsMbt27TITJ04048aNc973wAMPmP79+5ujR4+a9PR0ExwcbDZs2FDvOk/3aL3y8nKzfv16I8ns3bvX5ObmmqNHjxpjjLn77ruN1+s177zzjsnMzDQzZ8404eHh5ocffjDG/OtpaAUFBebo0aPmiiuuMBMmTDDFxcXGGGNWrlxpevfubdavX2/27dtn1q9fbyIiIszq1auNMcbs37/fSDKDBw82b731ltm7d6/593//d9O/f39z8uTJM/bfqlWrTGBgoJkwYYLZuXOnSUtLM0OGDDHTp0932tR8QpQxxlx33XXOE9uMMaZ///4mIiLCvPjiiyY7O9vcddddJiwszFx11VXm9ddfN3v37jXXX3+9GTJkiKmsrDxjXUBLIKgBP6kZJI8++qiJj4/3aXPw4EEnSI0xprS01Fx00UVm2rRpZujQoeb2228/7TrPpHrgVikuLjaBgYHmD3/4gzOvrKzMeL1es2jRIp/3ffXVV+bCCy80U6dONaWlpU776Ohos27dOp/P+p//+R8zatQoY8y/gvqVV15xlmdmZhpJZs+ePWese9WqVUaSycnJceatWLHCREVFOa8bGtQ333yz8zo3N9dIMo8++qgz75NPPjGSTG5u7hnrAlpCgJ925AHUkJaWpg8++EChoaG1ln399dcaNGiQgoKCtHbtWl1wwQXq37+/li5d2ux1fP311zp58qQuv/xyZ15gYKB+/vOfa8+ePT5tJ0yYoEsvvVSvv/66OnfuLEk6cuSIDh48qFmzZumOO+5w2paXl8vtdvu8/4ILLnB+7t27tyQpPz9fgwcPPmOd3bp107nnnuvz/vz8/EZsae0aoqKiJEnDhw+vNS8/P18ej6fR6wfOFkENWKKyslLXXnutnnnmmVrLqkJMknbs2CFJ+vHHH/Xjjz8qJCSkWesw//eIepfLVWt+zXmTJ0/W+vXrlZWV5YRbZWWlJOnll1/WyJEjfdpXhXmVwMBA5+eqdVe9/0yqv7fq/VW1S1KnTp18XkvSyZMnT7ueqhrOpi6guTGYDPCToKAgVVRUOK8vvvhiZWZmasCAATrvvPN8pqow/vrrr3Xvvffq5Zdf1mWXXaZbbrnFJ0BqrrMhNUjyec95552noKAgbd++3Zl38uRJff755xoyZIjP+59++mnNnDlT48ePV1ZWlqRTe6B9+vTRvn37am1HTExMI3ro7PTs2dNncFlFRYUyMjJa7fOB5kJQA34yYMAA/e1vf9OBAwf0/fffa+7cufrxxx9144036rPPPtO+ffu0efNm3XbbbaqoqFBFRYVmzJih+Ph4/ed//qdWrVqljIwMPfvss/Wu80x7gf3795fL5dJbb72lI0eOqLi4WCEhIbrrrrv0wAMPaNOmTcrKytIdd9yh48ePa9asWbXW8dvf/lY33XSTrrzySn311VeSpKSkJKWkpOi5555Tdna2du/erVWrVmnx4sXN24mnceWVV+rtt9/W22+/ra+++kpz5szhenG0SQQ14Cf333+/OnfurLi4OPXs2VNlZWX661//qoqKCk2aNEnDhg3TPffcI7fbrU6dOumpp57SgQMHtHLlSkmSx+PRK6+8okceeUTp6el1rvPbb789bQ19+vTR448/roceekhRUVGaN2+epFN7yv/2b/+mGTNm6OKLL1ZOTo7effddhYeH17meJUuWaNq0abryyiuVnZ2t22+/Xa+88opWr16t4cOHa8yYMVq9enWr7lHfdtttmjlzpm655RaNGTNGMTExzuVoQFviMjVP4gAAAGuwRw0AgMUIaqAdu/POOxUaGlrndOedd/q7vHpdffXV9dadnJzs7/KAVsWhb6Ady8/PV1FRUZ3Lunfvrl69erVyRQ1z+PBhlZSU1LksIiJCERERrVwR4D8ENQAAFuPQNwAAFiOoAQCwGEENAIDFCGoAACxGUAMAYDGCGgAAixHUAABY7P8Di0HmQ0ck8uoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 500x500 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns_dis_499 = sns.displot(df499['text_token_num'])\n",
    "sns_dis_499.savefig('text499_len_dist.pdf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "2472404f-22d1-4375-80f8-c58f42a93c78",
   "metadata": {},
   "outputs": [],
   "source": [
    "gen_data = df499[['plain_text_def','plain_text_term']].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "6d516d9a-6bb9-48ac-8f92-46cdf5b6496a",
   "metadata": {},
   "outputs": [],
   "source": [
    "gen_data = gen_data[-1024:]\n",
    "#gen_data.to_csv('data/test_GPT.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "5ba55d35-68fb-4e44-a459-47d2965382aa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "average length =  69.6083984375\n",
      "maximun length =  418\n"
     ]
    }
   ],
   "source": [
    "gen_texts = gen_data['plain_text_def'].to_list()\n",
    "#check average tokenized sentence lenth\n",
    "gen_text_lenlist = [len(e.split(' ')) for e in gen_texts]\n",
    "print(\"average length = \", sum(gen_text_lenlist) / len(gen_text_lenlist))\n",
    "print(\"maximun length = \", max(gen_text_lenlist))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "41de1cb9-eef9-4175-90ca-2b16f4a12d1b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeoAAAHpCAYAAABN+X+UAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAwP0lEQVR4nO3df1RVdb7/8ddR4OAPPInmwWMoeNPU/FGpmeYaJRUzf44z18oyK2tpJkqajYw1UbekXJPRaNa1GvFmjs1a/hinmhRLMcfupBgpWnpNzB/BpRJBDAHh8/2jr+d2QhTxcM4HfT7W2mu5P5/P3ue9P1av9j57n+0wxhgBAAArNQh2AQAAoHoENQAAFiOoAQCwGEENAIDFCGoAACxGUAMAYDGCGgAAixHUkowxKioqEo+UAwBsQ1BLOnnypFwul06ePBnsUgAA8EFQAwBgMYIaAACLEdQAAFiMoAYAwGIENQAAFiOoAQCwGEENAIDFCGoAACwW1KDesmWLRo4cKY/HI4fDobVr11YZ8+WXX2rUqFFyuVyKiIjQLbfcosOHD3v7S0tLlZCQoJYtW6pJkyYaNWqUjh49GsCjAACg7gQ1qE+dOqUePXpo0aJF5+z/+uuv1b9/f3Xq1EmbN2/WF198oaeeekrh4eHeMYmJiVqzZo1WrlyprVu3qri4WCNGjFBFRUWgDgMAgDrjMJb8wLXD4dCaNWs0ZswYb9tdd92l0NBQvf322+fcprCwUFdffbXefvtt3XnnnZKkb7/9VtHR0frggw80dOjQGn12UVGRXC6XCgsL1axZs0s+FgAA/MXa76grKyv1/vvvq2PHjho6dKhatWqlPn36+Fwez8zMVHl5ueLj471tHo9HXbt21bZt26rdd2lpqYqKinwWAABsZG1Q5+fnq7i4WC+88IJuv/12bdiwQb/+9a81duxYZWRkSJLy8vIUFham5s2b+2zrdruVl5dX7b5TUlLkcrm8S3R0dJ0eCwAAtWVtUFdWVkqSRo8erccee0w33HCD5syZoxEjRuj1118/77bGGDkcjmr7k5KSVFhY6F2OHDni19oBAPAXa4O6ZcuWCgkJUZcuXXzaO3fu7L3rOyoqSmVlZSooKPAZk5+fL7fbXe2+nU6nmjVr5rMAAGAja4M6LCxMvXv31r59+3za9+/fr3bt2kmSevbsqdDQUKWnp3v7c3NzlZ2drX79+gW0XgAA6kJIMD+8uLhYBw4c8K7n5OQoKytLkZGRatu2rWbPnq0777xTv/rVrxQXF6cPP/xQf//737V582ZJksvl0qRJkzRr1iy1aNFCkZGRevzxx9WtWzcNHjw4SEcFAID/BPXxrM2bNysuLq5K+8SJE5WWliZJ+vOf/6yUlBQdPXpU1113nZ555hmNHj3aO/b06dOaPXu2VqxYoZKSEg0aNEiLFy++qBvEeDwLAGAra56jDiaCGgBgq6Be+saFxbS/Vt8eO/9PonraXKNDBw+cdwwAoH4iqC337bGjGrtw03nHrE6o+vUBAODyYO1d3wAAgKAGAMBqBDUAABYjqAEAsBhBDQCAxQhqAAAsRlADAGAxghoAAIsR1AAAWIygBgDAYgQ1AAAWI6gBALAYQQ0AgMUIagAALEZQAwBgMYIaAACLEdQAAFiMoAYAwGIENQAAFiOoAQCwGEENAIDFCGoAACxGUAMAYDGCGgAAixHUAABYjKAGAMBiBDUAABYjqAEAsBhBDQCAxUKCXQAu3ZmKSoU5w887xtPmGh06eCBAFQEA/IWgvgyYygqNXZxx3jGrE+ICVA0AwJ+49A0AgMUIagAALEZQAwBgMYIaAACLEdQAAFiMoAYAwGIENQAAFiOoAQCwWFCDesuWLRo5cqQ8Ho8cDofWrl1b7djJkyfL4XAoNTXVp720tFQJCQlq2bKlmjRpolGjRuno0aN1WzgAAAES1KA+deqUevTooUWLFp133Nq1a/Wvf/1LHo+nSl9iYqLWrFmjlStXauvWrSouLtaIESNUUVFRV2UDABAwQf0J0WHDhmnYsGHnHXPs2DFNmzZN69ev1/Dhw336CgsL9dZbb+ntt9/W4MGDJUnLly9XdHS0Nm7cqKFDh55zn6WlpSotLfWuFxUVXeKRAABQN6z+jrqyslITJkzQ7Nmzdf3111fpz8zMVHl5ueLj471tHo9HXbt21bZt26rdb0pKilwul3eJjo6uk/oBALhUVgf1iy++qJCQEE2fPv2c/Xl5eQoLC1Pz5s192t1ut/Ly8qrdb1JSkgoLC73LkSNH/Fo3AAD+Yu3bszIzM/XKK69o586dcjgcF7WtMea82zidTjmdzkstEQCAOmftGfUnn3yi/Px8tW3bViEhIQoJCdE333yjWbNmKSYmRpIUFRWlsrIyFRQU+Gybn58vt9sdhKoBAPAva4N6woQJ2rVrl7KysryLx+PR7NmztX79eklSz549FRoaqvT0dO92ubm5ys7OVr9+/YJVOgAAfhPUS9/FxcU6cOCAdz0nJ0dZWVmKjIxU27Zt1aJFC5/xoaGhioqK0nXXXSdJcrlcmjRpkmbNmqUWLVooMjJSjz/+uLp16+a9CxwAgPosqEG9Y8cOxcXFeddnzpwpSZo4caLS0tJqtI+XX35ZISEhGjdunEpKSjRo0CClpaWpYcOGdVEyAAABFdSgHjhwoIwxNR5/6NChKm3h4eFauHChFi5c6MfKAACwg7XfUQMAAIIaAACrEdQAAFiMoAYAwGIENQAAFiOoAQCwGEENAIDFCGoAACxGUAMAYDGCGgAAixHUAABYjKAGAMBiBDUAABYjqAEAsBhBDQCAxQhqAAAsRlADAGAxghoAAIsR1AAAWIygBgDAYgQ1AAAWI6gBALAYQQ0AgMUIagAALEZQAwBgMYIaAACLEdQAAFiMoAYAwGIENQAAFiOoAQCwGEENAIDFCGoAACxGUAMAYDGCGgAAixHUAABYjKAGAMBiBDUAABYjqAEAsBhBDQCAxQhqAAAsRlADAGCxoAb1li1bNHLkSHk8HjkcDq1du9bbV15ert/97nfq1q2bmjRpIo/Ho/vuu0/ffvutzz5KS0uVkJCgli1bqkmTJho1apSOHj0a4CO5eDHtr1WYM/yCS/mZ8mCXCgAIopBgfvipU6fUo0cPPfDAA/rNb37j0/fjjz9q586deuqpp9SjRw8VFBQoMTFRo0aN0o4dO7zjEhMT9fe//10rV65UixYtNGvWLI0YMUKZmZlq2LBhoA+pxr49dlRjF2664Lh3p/QPQDUAAFsFNaiHDRumYcOGnbPP5XIpPT3dp23hwoW6+eabdfjwYbVt21aFhYV666239Pbbb2vw4MGSpOXLlys6OlobN27U0KFDz7nv0tJSlZaWeteLior8dEQAAPhXvfqOurCwUA6HQ1dddZUkKTMzU+Xl5YqPj/eO8Xg86tq1q7Zt21btflJSUuRyubxLdHR0XZcedGcqKmt0qT2m/bXBLhUA8DNBPaO+GKdPn9acOXM0fvx4NWvWTJKUl5ensLAwNW/e3Ges2+1WXl5etftKSkrSzJkzvetFRUWXfVibygqNXZxxwXGrE+ICUA0AoKbqRVCXl5frrrvuUmVlpRYvXnzB8cYYORyOavudTqecTqc/SwQAoE5Yf+m7vLxc48aNU05OjtLT071n05IUFRWlsrIyFRQU+GyTn58vt9sd6FIBAPA7q4P6bEj/z//8jzZu3KgWLVr49Pfs2VOhoaE+N53l5uYqOztb/fr1C3S5AAD4XVAvfRcXF+vAgQPe9ZycHGVlZSkyMlIej0e//e1vtXPnTr333nuqqKjwfu8cGRmpsLAwuVwuTZo0SbNmzVKLFi0UGRmpxx9/XN26dfPeBQ4AQH0W1KDesWOH4uL+7+alszd4TZw4UcnJyVq3bp0k6YYbbvDZbtOmTRo4cKAk6eWXX1ZISIjGjRunkpISDRo0SGlpaVY/Qw0AQE0FNagHDhwoY0y1/efrOys8PFwLFy7UwoUL/VkaAABWsPo7agAArnQENQAAFiOoAQCwGEENAIDFCGoAACxGUAMAYDGCGgAAixHUAABYjKAGAMBiBDUAABYjqAEAsBhBDQCAxQhqAAAsRlADAGAxghoAAIsR1AAAWIygBgDAYgQ1AAAWI6gBALAYQQ0AgMUIagAALEZQAwBgMYIaAACLEdQAAFiMoAYAwGIENQAAFiOoAQCwGEENAIDFCGoAACxGUAMAYDGCGgAAixHUAABYjKAGAMBiBDUAABYjqAEAsBhBDQCAxQhqAAAsRlADAGAxghoAAIsR1AAAWCyoQb1lyxaNHDlSHo9HDodDa9eu9ek3xig5OVkej0eNGjXSwIEDtWfPHp8xpaWlSkhIUMuWLdWkSRONGjVKR48eDeBRAABQd4Ia1KdOnVKPHj20aNGic/bPnz9fCxYs0KJFi7R9+3ZFRUVpyJAhOnnypHdMYmKi1qxZo5UrV2rr1q0qLi7WiBEjVFFREajDAACgzoQE88OHDRumYcOGnbPPGKPU1FTNnTtXY8eOlSQtW7ZMbrdbK1as0OTJk1VYWKi33npLb7/9tgYPHixJWr58uaKjo7Vx40YNHTo0YMcCAEBdsPY76pycHOXl5Sk+Pt7b5nQ6NWDAAG3btk2SlJmZqfLycp8xHo9HXbt29Y45l9LSUhUVFfksAADYyNqgzsvLkyS53W6fdrfb7e3Ly8tTWFiYmjdvXu2Yc0lJSZHL5fIu0dHRfq4eAAD/sDaoz3I4HD7rxpgqbb90oTFJSUkqLCz0LkeOHPFLrQAA+Ju1QR0VFSVJVc6M8/PzvWfZUVFRKisrU0FBQbVjzsXpdKpZs2Y+CwAANrI2qGNjYxUVFaX09HRvW1lZmTIyMtSvXz9JUs+ePRUaGuozJjc3V9nZ2d4xAADUZ0G967u4uFgHDhzwrufk5CgrK0uRkZFq27atEhMTNW/ePHXo0EEdOnTQvHnz1LhxY40fP16S5HK5NGnSJM2aNUstWrRQZGSkHn/8cXXr1s17FzgAAPVZUIN6x44diouL867PnDlTkjRx4kSlpaXpiSeeUElJiaZOnaqCggL16dNHGzZsUEREhHebl19+WSEhIRo3bpxKSko0aNAgpaWlqWHDhgE/HgAA/C2oQT1w4EAZY6rtdzgcSk5OVnJycrVjwsPDtXDhQi1cuLAOKgQAILis/Y4aAAAQ1AAAWI2gBgDAYgQ1AAAWI6gBALAYQQ0AgMUIagAALEZQAwBgMYIaAACLEdQAAFiMoAYAwGIENQAAFiOoAQCwGEENAIDFCGoAACxGUAMAYDGCGgAAixHUAABYjKAGAMBiBDUAABYjqAEAsFitgrp9+/b64YcfqrSfOHFC7du3v+SiAADAT2oV1IcOHVJFRUWV9tLSUh07duySiwIAAD8JuZjB69at8/55/fr1crlc3vWKigp99NFHiomJ8VtxAABc6S4qqMeMGSNJcjgcmjhxok9faGioYmJi9NJLL/mtOAAArnQXFdSVlZWSpNjYWG3fvl0tW7ask6IAAMBPLiqoz8rJyfF3HbDEmYpKhTnDzzvG0+YaHTp4IEAVAcCVrVZBLUkfffSRPvroI+Xn53vPtM/685//fMmFIThMZYXGLs4475jVCXEBqgYAUKugfuaZZ/Tss8+qV69eat26tRwOh7/rAgAAqmVQv/7660pLS9OECRP8XQ8AAPiZWj1HXVZWpn79+vm7FgAA8Au1CuqHHnpIK1as8HctAADgF2p16fv06dNasmSJNm7cqO7duys0NNSnf8GCBX4pDgCAK12tgnrXrl264YYbJEnZ2dk+fdxYBgCA/9QqqDdt2uTvOgAAwDnwmksAACxWqzPquLi4817i/vjjj2tdEAAA+D+1Cuqz30+fVV5erqysLGVnZ1d5WQcAAKi9WgX1yy+/fM725ORkFRcXX1JBAADg//j1O+p7772X3/kGAMCP/BrUn376qcLDz//mJQAAUHO1uvQ9duxYn3VjjHJzc7Vjxw499dRTfilMks6cOaPk5GS98847ysvLU+vWrXX//ffrySefVIMGDbyf/cwzz2jJkiUqKChQnz599Oqrr+r666/3Wx0AAARLrYLa5XL5rDdo0EDXXXednn32WcXHx/ulMEl68cUX9frrr2vZsmW6/vrrtWPHDj3wwANyuVyaMWOGJGn+/PlasGCB0tLS1LFjRz333HMaMmSI9u3bp4iICL/VAgBAMNQqqJcuXervOs7p008/1ejRozV8+HBJUkxMjP7yl79ox44dkn46m05NTdXcuXO9Z/nLli2T2+3WihUrNHny5IDUCQBAXbmk76gzMzO1fPlyvfPOO/r888/9VZNX//799dFHH2n//v2SpC+++EJbt27VHXfcIUnKyclRXl6ez1m80+nUgAEDtG3btmr3W1paqqKiIp8FAAAb1eqMOj8/X3fddZc2b96sq666SsYYFRYWKi4uTitXrtTVV1/tl+J+97vfqbCwUJ06dVLDhg1VUVGh559/XnfffbckKS8vT5Lkdrt9tnO73frmm2+q3W9KSoqeeeYZv9QIAEBdqtUZdUJCgoqKirRnzx4dP35cBQUFys7OVlFRkaZPn+634t59910tX75cK1as0M6dO7Vs2TL98Y9/1LJly3zG/fJX0owx5/3ltKSkJBUWFnqXI0eO+K1mAAD8qVZn1B9++KE2btyozp07e9u6dOmiV1991a83k82ePVtz5szRXXfdJUnq1q2bvvnmG6WkpGjixImKioqSJO8d4Wfl5+dXOcv+OafTKafT6bc6AQCoK7U6o66srKzyDmpJCg0NVWVl5SUXddaPP/7ofQzrrIYNG3o/IzY2VlFRUUpPT/f2l5WVKSMjQ/369fNbHQAABEutgvq2227TjBkz9O2333rbjh07pscee0yDBg3yW3EjR47U888/r/fff1+HDh3SmjVrtGDBAv3617+W9NMl78TERM2bN09r1qxRdna27r//fjVu3Fjjx4/3Wx0AAARLrS59L1q0SKNHj1ZMTIyio6PlcDh0+PBhdevWTcuXL/dbcQsXLtRTTz2lqVOnKj8/Xx6PR5MnT9Yf/vAH75gnnnhCJSUlmjp1qvcHTzZs2MAz1ACAy0Ktgjo6Olo7d+5Uenq6vvrqKxlj1KVLFw0ePNivxUVERCg1NVWpqanVjnE4HEpOTlZycrJfPxsAABtc1KXvjz/+WF26dPE+dzxkyBAlJCRo+vTp6t27t66//np98skndVIoAABXoosK6tTUVD388MNq1qxZlT6Xy6XJkydrwYIFfisOAIAr3UUF9RdffKHbb7+92v74+HhlZmZeclEAAOAnFxXU//u//3vOx7LOCgkJ0XfffXfJRQEAgJ9cVFC3adNGu3fvrrZ/165dPj88AgAALs1FBfUdd9yhP/zhDzp9+nSVvpKSEj399NMaMWKE34oDAOBKd1GPZz355JNavXq1OnbsqGnTpum6666Tw+HQl19+qVdffVUVFRWaO3duXdUKAMAV56KC2u12a9u2bXrkkUeUlJQkY4ykn55lHjp0qBYvXnze39gGAAAX56J/8KRdu3b64IMPVFBQoAMHDsgYow4dOqh58+Z1UR8AAFe0Wv0ymSQ1b95cvXv39mctAADgF2r1Ug4AABAYBDUAABYjqAEAsBhBDQCAxQhqAAAsRlADAGAxghoAAIsR1AAAWIygBgDAYgQ1AAAWI6gBALAYQQ0AgMUIagAALEZQAwBgsVq/5hJXrjMVlQpzhl9wnKfNNTp08EAAKgKAyxdBjYtmKis0dnHGBcetTogLQDUAcHnj0jcAABYjqAEAsBhBDQCAxQhqAAAsRlADAGAxghoAAIsR1AAAWIygBgDAYgQ1AAAWI6gBALAYQQ0AgMUIagAALEZQAwBgMYIaAACLEdQAAFjM+qA+duyY7r33XrVo0UKNGzfWDTfcoMzMTG+/MUbJycnyeDxq1KiRBg4cqD179gSxYgAA/MfqoC4oKNCtt96q0NBQ/eMf/9DevXv10ksv6aqrrvKOmT9/vhYsWKBFixZp+/btioqK0pAhQ3Ty5MngFQ4AgJ+EBLuA83nxxRcVHR2tpUuXettiYmK8fzbGKDU1VXPnztXYsWMlScuWLZPb7daKFSs0efLkc+63tLRUpaWl3vWioqK6OQAAAC6R1WfU69atU69evfTv//7vatWqlW688Ua98cYb3v6cnBzl5eUpPj7e2+Z0OjVgwABt27at2v2mpKTI5XJ5l+jo6Do9DgAAasvqoD548KBee+01dejQQevXr9eUKVM0ffp0/dd//ZckKS8vT5Lkdrt9tnO73d6+c0lKSlJhYaF3OXLkSN0dBAAAl8DqS9+VlZXq1auX5s2bJ0m68cYbtWfPHr322mu67777vOMcDofPdsaYKm0/53Q65XQ666ZoAAD8yOoz6tatW6tLly4+bZ07d9bhw4clSVFRUZJU5ew5Pz+/ylk2AAD1kdVBfeutt2rfvn0+bfv371e7du0kSbGxsYqKilJ6erq3v6ysTBkZGerXr19AawUAoC5Yfen7scceU79+/TRv3jyNGzdOn332mZYsWaIlS5ZI+umSd2JioubNm6cOHTqoQ4cOmjdvnho3bqzx48cHuXqcqahUmDP8vGM8ba7RoYMHAlQRANQ/Vgd17969tWbNGiUlJenZZ59VbGysUlNTdc8993jHPPHEEyopKdHUqVNVUFCgPn36aMOGDYqIiAhi5ZAkU1mhsYszzjtmdUJcgKoBgPrJ6qCWpBEjRmjEiBHV9jscDiUnJys5OTlwRQEAECBWf0cNAMCVjqAGAMBiBDUAABYjqAEAsBhBDQCAxQhqAAAsRlADAGAxghoAAIsR1AAAWIygBgDAYgQ1AAAWI6gBALAYQQ0AgMUIagAALEZQAwBgMYIaAACLEdQAAFiMoAYAwGIENQAAFiOoAQCwGEENAIDFCGoAACxGUAMAYDGCGgAAixHUAABYjKAGAMBiBDUAABYjqAEAsBhBDQCAxQhqAAAsRlADAGAxghoAAIsR1AAAWIygBgDAYgQ1AAAWI6gBALAYQQ0AgMUIagAALEZQAwBgMYIaAACLEdQAAFisXgV1SkqKHA6HEhMTvW3GGCUnJ8vj8ahRo0YaOHCg9uzZE7wicVHOVFQqzBl+3iWm/bXBLhMAgiYk2AXU1Pbt27VkyRJ1797dp33+/PlasGCB0tLS1LFjRz333HMaMmSI9u3bp4iIiCBVi5oylRUauzjjvGNWJ8QFqBoAsE+9OKMuLi7WPffcozfeeEPNmzf3thtjlJqaqrlz52rs2LHq2rWrli1bph9//FErVqyodn+lpaUqKiryWQAAsFG9COpHH31Uw4cP1+DBg33ac3JylJeXp/j4eG+b0+nUgAEDtG3btmr3l5KSIpfL5V2io6PrrHYAAC6F9UG9cuVK7dy5UykpKVX68vLyJElut9un3e12e/vOJSkpSYWFhd7lyJEj/i0aAAA/sfo76iNHjmjGjBnasGGDwsPDqx3ncDh81o0xVdp+zul0yul0+q1OAADqitVn1JmZmcrPz1fPnj0VEhKikJAQZWRk6E9/+pNCQkK8Z9K/PHvOz8+vcpYNAEB9ZHVQDxo0SLt371ZWVpZ36dWrl+655x5lZWWpffv2ioqKUnp6unebsrIyZWRkqF+/fkGsHAAA/7D60ndERIS6du3q09akSRO1aNHC256YmKh58+apQ4cO6tChg+bNm6fGjRtr/PjxwSgZAAC/sjqoa+KJJ55QSUmJpk6dqoKCAvXp00cbNmzgGWoAwGWh3gX15s2bfdYdDoeSk5OVnJwclHoAAKhLVn9HDQDAlY6gBgDAYgQ1AAAWI6gBALAYQQ0AgMUIagAALEZQAwBgMYIaAACLEdQAAFiMoAYAwGIENQAAFiOoAQCwGEENAIDF6t3bs+qDmPbX6ttjR887pvxMeYCqqf/OVFQqzBl+wXGeNtfo0MEDAagIAAKHoK4D3x47qrELN513zLtT+geomvrPVFZo7OKMC45bnRAXgGoAILC49A0AgMUIagAALEZQAwBgMYIaAACLEdQAAFiMoAYAwGIENQAAFiOoAQCwGEENAIDFCGoAACxGUAMAYDF+6xuXjZq8vIMXdwCobwhqXDZq8vIOXtwBoL7h0jcAABYjqAEAsBhBDQCAxQhqAAAsRlADAGAxghoAAIsR1AAAWIygBgDAYgQ1AAAWI6gBALAYQQ0AgMUIagAALGZ1UKekpKh3796KiIhQq1atNGbMGO3bt89njDFGycnJ8ng8atSokQYOHKg9e/YEqWIAAPzL6qDOyMjQo48+qv/+7/9Wenq6zpw5o/j4eJ06dco7Zv78+VqwYIEWLVqk7du3KyoqSkOGDNHJkyeDWDmuBDHtr1WYM/y8S0z7a4NdJoB6zurXXH744Yc+60uXLlWrVq2UmZmpX/3qVzLGKDU1VXPnztXYsWMlScuWLZPb7daKFSs0efLkYJSNK8S3x45q7MJN5x3DazUBXCqrz6h/qbCwUJIUGRkpScrJyVFeXp7i4+O9Y5xOpwYMGKBt27ZVu5/S0lIVFRX5LAAA2KjeBLUxRjNnzlT//v3VtWtXSVJeXp4kye12+4x1u93evnNJSUmRy+XyLtHR0XVXOAAAl6DeBPW0adO0a9cu/eUvf6nS53A4fNaNMVXafi4pKUmFhYXe5ciRI36vFwAAf7D6O+qzEhIStG7dOm3ZskXXXHONtz0qKkrST2fWrVu39rbn5+dXOcv+OafTKafTWXcFw1pnKioV5gy/4DhPm2t06OCBAFQEAOdndVAbY5SQkKA1a9Zo8+bNio2N9emPjY1VVFSU0tPTdeONN0qSysrKlJGRoRdffDEYJcNyprJCYxdnXHAcN4EBsIXVQf3oo49qxYoV+tvf/qaIiAjv984ul0uNGjWSw+FQYmKi5s2bpw4dOqhDhw6aN2+eGjdurPHjxwe5egAALp3VQf3aa69JkgYOHOjTvnTpUt1///2SpCeeeEIlJSWaOnWqCgoK1KdPH23YsEEREREBrhYAAP+zOqiNMRcc43A4lJycrOTk5LovCACAAKs3d30DAHAlsvqMGqjvuMscwKUiqIE6xF3mAC4VQQ2cQ03OhMvPlAeoGgBXMoIaOIeanAm/O6V/gKoBcCXjZjIAACxGUAMAYDGCGgAAixHUAABYjKAGAMBiBDUAABYjqAEAsBhBDQCAxQhqAAAsRlADAGAxghoAAIsR1AAAWIygBgDAYgQ1AAAWI6gBALAYQQ0AgMUIagAALEZQAwBgMYIaAACLhQS7AAA1E9P+Wn177OgFx3naXKNDBw8EoCIAgUBQA/XEt8eOauzCTRcctzohLgDVAAgUghq4zJypqFSYM/y8Y4yjgRym8oL7CvTZeU2uGnDFAFcaghq4zJjKCo1dnHHeMe9O6a87X996wX0F+uy8JlcNuGKAKw03kwEAYDHOqAEL1ORydfmZ8gBVA8AmBDVggZpergZw5eHSNwAAFuOMGkC1anJJnruwgbpFUAOoVk0uyXMXNlC3CGoAda6mv6p2ud8wx6/LoTYIagB1rqa/qna53zDHr8uhNriZDAAAi3FGDeCS8Aw4ULcIagCXhGfAgbpFUAO4LNXkxq2avJykPt/Yxc1rF8fWl8IQ1AAuSzW5casmLyepzzd2cfPaxbH1pTCXzc1kixcvVmxsrMLDw9WzZ0998sknwS4JAIBLdlmcUb/77rtKTEzU4sWLdeutt+o///M/NWzYMO3du1dt27YNdnkA6rGa3Cwn1ewyuj9vqqvJZVpbb+LjkvzFuSyCesGCBZo0aZIeeughSVJqaqrWr1+v1157TSkpKUGuDkB9VpOb5aSaXUb35011Nb20byMuyV+ceh/UZWVlyszM1Jw5c3za4+PjtW3btnNuU1paqtLSUu96YWGhJKmoqMgvNRljVF5y6kKDLjympuNs/Dx/7ova6/++/Ph5xpga/bvqt38PgzDvNTlGf/53pqZz6i81ql121uXvmiIiIuRwOC5YWL127NgxI8n885//9Gl//vnnTceOHc+5zdNPP20ksbCwsLCwBHUpLCy8YM7V+zPqs375fyTGmGr/LyUpKUkzZ870rldWVur48eNq0aLFhf/PRj+deUdHR+vIkSNq1qzZpRWOC2K+A4v5DhzmOrBsnO+IiIgLjqn3Qd2yZUs1bNhQeXl5Pu35+flyu93n3MbpdMrpdPq0XXXVVRf92c2aNbPmL/tKwHwHFvMdOMx1YNW3+a73j2eFhYWpZ8+eSk9P92lPT09Xv379glQVAAD+Ue/PqCVp5syZmjBhgnr16qW+fftqyZIlOnz4sKZMmRLs0gAAuCSXRVDfeeed+uGHH/Tss88qNzdXXbt21QcffKB27drVyec5nU49/fTTVS6fo24w34HFfAcOcx1Y9XW+HcYYE+wiAADAudX776gBALicEdQAAFiMoAYAwGIENQAAFiOoa4FXal66LVu2aOTIkfJ4PHI4HFq7dq1PvzFGycnJ8ng8atSokQYOHKg9e/b4jCktLVVCQoJatmypJk2aaNSoUTp69MJv5LnSpKSkqHfv3oqIiFCrVq00ZswY7du3z2cM8+0/r732mrp37+79UY2+ffvqH//4h7efua47KSkpcjgcSkxM9LZdFvN9ab+0feVZuXKlCQ0NNW+88YbZu3evmTFjhmnSpIn55ptvgl1avfLBBx+YuXPnmlWrVhlJZs2aNT79L7zwgomIiDCrVq0yu3fvNnfeeadp3bq1KSoq8o6ZMmWKadOmjUlPTzc7d+40cXFxpkePHubMmTMBPhq7DR061CxdutRkZ2ebrKwsM3z4cNO2bVtTXFzsHcN8+8+6devM+++/b/bt22f27dtnfv/735vQ0FCTnZ1tjGGu68pnn31mYmJiTPfu3c2MGTO87ZfDfBPUF+nmm282U6ZM8Wnr1KmTmTNnTpAqqv9+GdSVlZUmKirKvPDCC96206dPG5fLZV5//XVjjDEnTpwwoaGhZuXKld4xx44dMw0aNDAffvhhwGqvj/Lz840kk5GRYYxhvgOhefPm5s0332Su68jJkydNhw4dTHp6uhkwYIA3qC+X+ebS90U4+0rN+Ph4n/bzvVITFy8nJ0d5eXk+8+x0OjVgwADvPGdmZqq8vNxnjMfjUdeuXfm7uICzr3WNjIyUxHzXpYqKCq1cuVKnTp1S3759mes68uijj2r48OEaPHiwT/vlMt+XxS+TBcr333+vioqKKi/7cLvdVV4Kgto7O5fnmudvvvnGOyYsLEzNmzevMoa/i+oZYzRz5kz1799fXbt2lcR814Xdu3erb9++On36tJo2bao1a9aoS5cu3v/wM9f+s3LlSu3cuVPbt2+v0ne5/LNNUNfCxbxSE7VXm3nm7+L8pk2bpl27dmnr1q1V+phv/7nuuuuUlZWlEydOaNWqVZo4caIyMjK8/cy1fxw5ckQzZszQhg0bFB4eXu24+j7fXPq+CLV5pSYuXlRUlCSdd56joqJUVlamgoKCasfAV0JCgtatW6dNmzbpmmuu8bYz3/4XFhama6+9Vr169VJKSop69OihV155hbn2s8zMTOXn56tnz54KCQlRSEiIMjIy9Kc//UkhISHe+arv801QXwReqRkYsbGxioqK8pnnsrIyZWRkeOe5Z8+eCg0N9RmTm5ur7Oxs/i5+wRijadOmafXq1fr4448VGxvr08981z1jjEpLS5lrPxs0aJB2796trKws79KrVy/dc889ysrKUvv27S+P+Q7OPWz119nHs9566y2zd+9ek5iYaJo0aWIOHToU7NLqlZMnT5rPP//cfP7550aSWbBggfn888+9j7m98MILxuVymdWrV5vdu3ebu++++5yPVFxzzTVm48aNZufOnea2226z6pEKWzzyyCPG5XKZzZs3m9zcXO/y448/escw3/6TlJRktmzZYnJycsyuXbvM73//e9OgQQOzYcMGYwxzXdd+fte3MZfHfBPUtfDqq6+adu3ambCwMHPTTTd5H3NBzW3atMlIqrJMnDjRGPPTYxVPP/20iYqKMk6n0/zqV78yu3fv9tlHSUmJmTZtmomMjDSNGjUyI0aMMIcPHw7C0djtXPMsySxdutQ7hvn2nwcffND734err77aDBo0yBvSxjDXde2XQX05zDevuQQAwGJ8Rw0AgMUIagAALEZQAwBgMYIaAACLEdQAAFiMoAYAwGIENQAAFiOoAQCwGEEN4II2b94sh8OhEydOBLsU4IpDUAMBMnDgQCUmJgZ1nwQuUP8Q1AAAWIygBgLg/vvvV0ZGhl555RU5HA45HA4dOnRIe/fu1R133KGmTZvK7XZrwoQJ+v777yX9dPYbFhamTz75xLufl156SS1btlRubm61+6zOoUOHFBcXJ0lq3ry5HA6H7r//fklSaWmppk+frlatWik8PFz9+/fX9u3bq91XSUmJhg8frltuuUXHjx+XJC1dulSdO3dWeHi4OnXqpMWLF/t8tsPh0OrVqxUXF6fGjRurR48e+vTTT2s0f2lpabrqqqu0fv16de7cWU2bNtXtt9+u3Nxc75hzXV0YM2aM9xglKSYmRs8995zuu+8+NW3aVO3atdPf/vY3fffddxo9erSaNm2qbt26aceOHTWqCwiIYL8VBLgSnDhxwvTt29c8/PDD3tdMHj161LRs2dIkJSWZL7/80uzcudMMGTLExMXFebebPXu2adeunTlx4oTJysoyTqfTrF69utp9nu+1fGfOnDGrVq0yksy+fftMbm6uOXHihDHGmOnTpxuPx2M++OADs2fPHjNx4kTTvHlz88MPPxhj/u9tZwUFBebEiROmf//+ZvDgwaa4uNgYY8ySJUtM69atzapVq8zBgwfNqlWrTGRkpElLSzPGGJOTk2MkmU6dOpn33nvP7Nu3z/z2t7817dq1M+Xl5Recv6VLl5rQ0FAzePBgs337dpOZmWk6d+5sxo8f7x3zy7cmGWPM6NGjvW9kM8aYdu3amcjISPP666+b/fv3m0ceecRERESY22+/3fz1r381+/btM2PGjDGdO3c2lZWVF6wLCASCGgiQXwbJU089ZeLj433GHDlyxBukxhhTWlpqbrzxRjNu3Dhz/fXXm4ceeui8+7yQnwfuWcXFxSY0NNS888473raysjLj8XjM/Pnzfbb76quvTI8ePczYsWNNaWmpd3x0dLRZsWKFz2f9x3/8h+nbt68x5v+C+s033/T279mzx0gyX3755QXrXrp0qZFkDhw44G179dVXjdvt9q7XNKjvvfde73pubq6RZJ566ilv26effmokmdzc3AvWBQRCSJBO5IErXmZmpjZt2qSmTZtW6fv666/VsWNHhYWFafny5erevbvatWun1NRUv9fx9ddfq7y8XLfeequ3LTQ0VDfffLO+/PJLn7GDBw9W79699de//lUNGzaUJH333Xc6cuSIJk2apIcfftg79syZM3K5XD7bd+/e3fvn1q1bS5Ly8/PVqVOnC9bZuHFj/du//ZvP9vn5+RdxpFVrcLvdkqRu3bpVacvPz1dUVNRF7x/wN4IaCJLKykqNHDlSL774YpW+syEmSdu2bZMkHT9+XMePH1eTJk38Wof5/6+kdzgcVdp/2TZ8+HCtWrVKe/fu9YZbZWWlJOmNN95Qnz59fMafDfOzQkNDvX8+u++z21/Iz7c9u/3Z2iWpQYMGPuuSVF5eft79nK3hUuoC6ho3kwEBEhYWpoqKCu/6TTfdpD179igmJkbXXnutz3I2jL/++ms99thjeuONN3TLLbfovvvu8wmQX+6zJjVI8tnm2muvVVhYmLZu3eptKy8v144dO9S5c2ef7V944QVNnDhRgwYN0t69eyX9dAbapk0bHTx4sMpxxMbGXsQMXZqrr77a5+ayiooKZWdnB+zzgbpCUAMBEhMTo3/96186dOiQvv/+ez366KM6fvy47r77bn322Wc6ePCgNmzYoAcffFAVFRWqqKjQhAkTFB8frwceeEBLly5Vdna2XnrppWr3eaGzwHbt2snhcOi9997Td999p+LiYjVp0kSPPPKIZs+erQ8//FB79+7Vww8/rB9//FGTJk2qso8//vGPuueee3Tbbbfpq6++kiQlJycrJSVFr7zyivbv36/du3dr6dKlWrBggX8n8Txuu+02vf/++3r//ff11VdfaerUqTwvjssCQQ0EyOOPP66GDRuqS5cuuvrqq1VWVqZ//vOfqqio0NChQ9W1a1fNmDFDLpdLDRo00PPPP69Dhw5pyZIlkqSoqCi9+eabevLJJ5WVlXXOfR4+fPi8NbRp00bPPPOM5syZI7fbrWnTpkn66Uz5N7/5jSZMmKCbbrpJBw4c0Pr169W8efNz7ufll1/WuHHjdNttt2n//v166KGH9OabbyotLU3dunXTgAEDlJaWFtAz6gcffFATJ07UfffdpwEDBig2Ntb7OBpQnznML7/UAQAA1uCMGgAAixHUwGVkypQpatq06TmXKVOmBLu8ag0bNqzauufNmxfs8oCg4tI3cBnJz89XUVHROfuaNWumVq1aBbiimjl27JhKSkrO2RcZGanIyMgAVwTYg6AGAMBiXPoGAMBiBDUAABYjqAEAsBhBDQCAxQhqAAAsRlADAGAxghoAAIv9P853H0LoFdQOAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 500x500 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#check length distribution\n",
    "gen_data['text_token_num'] = gen_text_lenlist\n",
    "sns_dis_gen = sns.displot(gen_data['text_token_num'])\n",
    "sns_dis_gen.savefig('text_gen_len_dist.pdf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d3372cca-efba-4465-b772-d5752830fa07",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "8d5b0ba4-b00f-4d5f-adb2-6d2487e28be0",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Ask ChatGPT-3.5\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3e19c774-6bac-477d-8159-d9984d997c6e",
   "metadata": {},
   "outputs": [],
   "source": [
    "openai.api_key_path=\"API_KEY/TheoremKB_Shufan.txt\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "81746c20-3432-4fc6-9006-4f0f35d9460c",
   "metadata": {},
   "outputs": [],
   "source": [
    "gen_data = pd.read_csv('data/test_GPT.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "39dd9491-31e5-4f80-9141-97efa985fdc0",
   "metadata": {},
   "outputs": [],
   "source": [
    "response = openai.ChatCompletion.create(\n",
    "    model=\"gpt-3.5-turbo\",\n",
    "    messages=[\n",
    "    {\n",
    "      \"role\": \"system\",\n",
    "      \"content\": \"You will be provided with a block of text that might define one or multiple mathematical terms. Your task is to extract the defined term(s).  For example, the phrase “An interval of _n is called new if it cannot be obtained as the grafting of two intervals” defines “new”. It is possible that the sentence does not contain the defined term. You should only return me the terms that you find, separate the terms with ###.Please do not print anything else.\"\n",
    "    },\n",
    "    {\n",
    "      \"role\": \"user\",\n",
    "      \"content\": \"Given a word w let v be the letters of w to which the\\nthrough strings of the unique w-admissible half-pairing\\nare attached. Then v is a cyclic reduction of w – we\\nshall call it the standard cyclic reduction of w\\nand denote it w.\\n\"\n",
    "    }\n",
    "    ],\n",
    "    temperature=0,\n",
    "    max_tokens=128,\n",
    "    top_p=1,\n",
    "    frequency_penalty=0,\n",
    "    presence_penalty=0\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "80b6a91a-a37a-46ba-9340-8d293c4ed4d5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<OpenAIObject chat.completion id=chatcmpl-7s0fDMpRPIaPCSPcmLQvCjVsrtcpv at 0x7f41304b9ae0> JSON: {\n",
       "  \"id\": \"chatcmpl-7s0fDMpRPIaPCSPcmLQvCjVsrtcpv\",\n",
       "  \"object\": \"chat.completion\",\n",
       "  \"created\": 1693106619,\n",
       "  \"model\": \"gpt-3.5-turbo-0613\",\n",
       "  \"choices\": [\n",
       "    {\n",
       "      \"index\": 0,\n",
       "      \"message\": {\n",
       "        \"role\": \"assistant\",\n",
       "        \"content\": \"cyclic reduction###standard cyclic reduction\"\n",
       "      },\n",
       "      \"finish_reason\": \"stop\"\n",
       "    }\n",
       "  ],\n",
       "  \"usage\": {\n",
       "    \"prompt_tokens\": 165,\n",
       "    \"completion_tokens\": 7,\n",
       "    \"total_tokens\": 172\n",
       "  }\n",
       "}"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "response"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "75ee73c9-d90c-410f-8098-ef6de7aa7ac8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'cyclic reduction###standard cyclic reduction'"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "response.to_dict()[\"choices\"][0][\"message\"][\"content\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "4d2e05bd-c2e5-4e77-ae55-d715e99456da",
   "metadata": {},
   "outputs": [],
   "source": [
    "previous_res = pd.read_csv(\"GPT_results/res.csv\")\n",
    "previous_terms_string_list = previous_res['gpt-3.5-turbo terms'].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "1f40cabd-6874-4ef8-8459-e3ef8459489d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['G-poset###compatibility graph###C_P###vertex set###adjacent###element###comparable###G',\n",
       " 'G-poset###strong compatibility graph###C_P###vertex set###adjacent###element###comparable###e###y###[x]',\n",
       " 'good pair###bad pair',\n",
       " 'multiplicative###probability measures###partitions###sequence###positive numbers###counts###mutually independent###convex combination',\n",
       " 'ergodic###limit shape of partitions',\n",
       " 'simplicial map###simplicial complexes###vertices###simplex###nondegenerate###injective',\n",
       " 'minimal###nondegenerate###locally k-large',\n",
       " 't-uniform###minimal filling diagram###interior vertex###neighbours',\n",
       " 'triangulation###holes###planar double edge###face###edge###disc###cutting along###contains',\n",
       " 'NOA(n,d)###v_1###v_2###v_3###[n]###orthogonal array###symbols###column###array###ℕ',\n",
       " 'exceptional interval-poset###Hasse diagram###configuration###y###z###x',\n",
       " 'new###']"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "previous_terms_string_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "id": "31752a78-0318-4e03-bcaf-1af16f3e060c",
   "metadata": {},
   "outputs": [],
   "source": [
    "#terms_string_list = previous_terms_string_list\n",
    "terms_string_list = [] #when start from zero"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "id": "2a090346-a6ff-4126-97af-276db125e611",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 5.59 s, sys: 237 ms, total: 5.83 s\n",
      "Wall time: 29min 36s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "for def_txt in gen_data['plain_text_def'].tolist():\n",
    "    user_obj = {\n",
    "      \"role\": \"user\",\n",
    "      \"content\": def_txt\n",
    "    }\n",
    "    terms_string=\"\"\n",
    "    response = openai.ChatCompletion.create(\n",
    "        model=\"gpt-4\",#\"gpt-3.5-turbo\",\n",
    "        messages=[\n",
    "        {\n",
    "          \"role\": \"system\",\n",
    "          \"content\": \"You will be provided with a block of text that might define one or multiple mathematical terms. Your task is to extract the defined term(s).  For example, the phrase “An interval of _n is called new if it cannot be obtained as the grafting of two intervals” defines “new”. It is possible that the sentence does not contain the defined term. You should only return me the terms that you find, separate the terms with ###.Please do not print anything else.\"},\n",
    "        user_obj\n",
    "        ],\n",
    "        temperature=0,\n",
    "        max_tokens=128,\n",
    "        top_p=1,\n",
    "        frequency_penalty=0,\n",
    "        presence_penalty=0\n",
    "    )\n",
    "    terms_string = response.to_dict()[\"choices\"][0][\"message\"][\"content\"]\n",
    "    terms_string_list.append(terms_string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "id": "b76965d7-daec-44fe-b364-eae035f38d49",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_res = pd.DataFrame(terms_string_list,columns =['gpt-4 terms'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "id": "3f8e63ad-968b-4f84-b9ce-c96d9d8068b7",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_res.to_csv(\"GPT_results/gpt-4_res.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "e5eb2879-7ec6-4884-a688-70f077510327",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.25088"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "70*1024*0.0035/1000 # prix éstimé pour gpt-3.5-turbo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "b9ead788-5afc-43c0-b495-88cd2c38c142",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6.4512"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "70*1024*0.09/1000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "4ba1ad67-c90d-4b45-b15e-e7a841ed00da",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10.32192"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "70*1024*0.09/1000 * 0.40 / 0.25 # prix éstimé pour GPT-4"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0a2723a8-9675-423e-9f0a-87ed89ba33c0",
   "metadata": {},
   "source": [
    "## Calculate the precision and Recall of ChatGPT's annotation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "id": "5c515543-662e-4f4d-88e2-2827ea906b2d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>plain_text_def</th>\n",
       "      <th>plain_text_term</th>\n",
       "      <th>gpt-3.5-turbo terms</th>\n",
       "      <th>gpt-4 terms</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Let P be a G-poset. The compatibility graph of...</td>\n",
       "      <td>Compatibility graph</td>\n",
       "      <td>G-poset###compatibility graph###C_P###vertex s...</td>\n",
       "      <td>compatibility graph</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Let P be a G-poset. The strong compatibility g...</td>\n",
       "      <td>Strong compatibility graph</td>\n",
       "      <td>G-poset###strong compatibility graph###C_P###v...</td>\n",
       "      <td>strong compatibility graph</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>\\nLet p=p_1p_2⋯ p_n be a permutation. \\nWe say...</td>\n",
       "      <td>good pair;bad pair</td>\n",
       "      <td>good pair###bad pair</td>\n",
       "      <td>good pair###bad pair</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>\\nThe family of probability measures on partit...</td>\n",
       "      <td>multiplicative</td>\n",
       "      <td>multiplicative###probability measures###partit...</td>\n",
       "      <td>multiplicative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>\\nWe call a family of measures μ^(n) ergodic i...</td>\n",
       "      <td>ergodic;limit shape</td>\n",
       "      <td>ergodic###limit shape of partitions</td>\n",
       "      <td>ergodic###limit shape of partitions</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1019</th>\n",
       "      <td>Consider ⟨ G→ S, α→ A⟩, two adjacent vertices...</td>\n",
       "      <td>preferred direction;special claw corresponding...</td>\n",
       "      <td>special claw###subdivided medial graph</td>\n",
       "      <td>special claw</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1020</th>\n",
       "      <td>\\nAccording to Theorem &lt;ref&gt;, ^d has exactly d...</td>\n",
       "      <td>Cyclicity classes</td>\n",
       "      <td>cyclicity classes###</td>\n",
       "      <td>cyclicity classes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1021</th>\n",
       "      <td>By Wielandt number we mean the following funct...</td>\n",
       "      <td>Wielandt number</td>\n",
       "      <td>Wielandt number###</td>\n",
       "      <td>Wielandt number</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1022</th>\n",
       "      <td>\\nThe girth of , denoted it by g(),\\nis the sm...</td>\n",
       "      <td>Girth</td>\n",
       "      <td>girth###nonempty cycle</td>\n",
       "      <td>girth</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1023</th>\n",
       "      <td>\\nLet A∈. The cycles of (A) at which the maxim...</td>\n",
       "      <td>Critical graph</td>\n",
       "      <td>critical###nodes###edges###N_c(A)###E_c(A)###c...</td>\n",
       "      <td>critical###N_c(A)###E_c(A)###critical graph</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1024 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                         plain_text_def  \\\n",
       "0     Let P be a G-poset. The compatibility graph of...   \n",
       "1     Let P be a G-poset. The strong compatibility g...   \n",
       "2     \\nLet p=p_1p_2⋯ p_n be a permutation. \\nWe say...   \n",
       "3     \\nThe family of probability measures on partit...   \n",
       "4     \\nWe call a family of measures μ^(n) ergodic i...   \n",
       "...                                                 ...   \n",
       "1019   Consider ⟨ G→ S, α→ A⟩, two adjacent vertices...   \n",
       "1020  \\nAccording to Theorem <ref>, ^d has exactly d...   \n",
       "1021  By Wielandt number we mean the following funct...   \n",
       "1022  \\nThe girth of , denoted it by g(),\\nis the sm...   \n",
       "1023  \\nLet A∈. The cycles of (A) at which the maxim...   \n",
       "\n",
       "                                        plain_text_term  \\\n",
       "0                                   Compatibility graph   \n",
       "1                            Strong compatibility graph   \n",
       "2                                    good pair;bad pair   \n",
       "3                                        multiplicative   \n",
       "4                                   ergodic;limit shape   \n",
       "...                                                 ...   \n",
       "1019  preferred direction;special claw corresponding...   \n",
       "1020                                  Cyclicity classes   \n",
       "1021                                    Wielandt number   \n",
       "1022                                              Girth   \n",
       "1023                                     Critical graph   \n",
       "\n",
       "                                    gpt-3.5-turbo terms  \\\n",
       "0     G-poset###compatibility graph###C_P###vertex s...   \n",
       "1     G-poset###strong compatibility graph###C_P###v...   \n",
       "2                                  good pair###bad pair   \n",
       "3     multiplicative###probability measures###partit...   \n",
       "4                   ergodic###limit shape of partitions   \n",
       "...                                                 ...   \n",
       "1019             special claw###subdivided medial graph   \n",
       "1020                               cyclicity classes###   \n",
       "1021                                 Wielandt number###   \n",
       "1022                             girth###nonempty cycle   \n",
       "1023  critical###nodes###edges###N_c(A)###E_c(A)###c...   \n",
       "\n",
       "                                      gpt-4 terms  \n",
       "0                             compatibility graph  \n",
       "1                      strong compatibility graph  \n",
       "2                            good pair###bad pair  \n",
       "3                                  multiplicative  \n",
       "4             ergodic###limit shape of partitions  \n",
       "...                                           ...  \n",
       "1019                                 special claw  \n",
       "1020                            cyclicity classes  \n",
       "1021                              Wielandt number  \n",
       "1022                                        girth  \n",
       "1023  critical###N_c(A)###E_c(A)###critical graph  \n",
       "\n",
       "[1024 rows x 4 columns]"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gen_data['gpt-4 terms'] = df_res['gpt-4 terms']\n",
    "gen_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "id": "23d6bfb1-86c2-4724-97f9-8a1b68ac5991",
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_list = []\n",
    "for index, row in gen_data.iterrows():\n",
    "\n",
    "    #let's remove repeated terms in both lists, as our definitum detection requires high recall\n",
    "    expected_set = set(row['plain_text_term'].split(';'))\n",
    "    if '' in expected_set:\n",
    "        expected_set.remove('')\n",
    "    \n",
    "    extracted_set = set(row['gpt-4 terms'].split('###'))\n",
    "    if '' in extracted_set:\n",
    "        extracted_set.remove('')\n",
    "    \n",
    "    num_TP = 0\n",
    "    num_too_long = 0\n",
    "    num_cut_off = 0\n",
    "    for expected in expected_set:\n",
    "        for extracted in extracted_set:\n",
    "            if extracted.casefold() == expected.casefold():\n",
    "                num_TP = num_TP + 1\n",
    "            elif extracted.casefold() in expected.casefold():\n",
    "                num_cut_off = num_cut_off + 1\n",
    "            elif expected.casefold() in extracted.casefold():\n",
    "                num_too_long = num_too_long + 1\n",
    "    \n",
    "    eval_list.append({'True Term Num' : len(expected_set),\n",
    "                        'Extracted Term Num': len(extracted_set),\n",
    "                        'TP': num_TP,\n",
    "                        'Cut Off': num_cut_off,\n",
    "                        'Too Long': num_too_long})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "id": "1cc693a3-581b-4e47-a583-854be9d551c2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>True Term Num</th>\n",
       "      <th>Extracted Term Num</th>\n",
       "      <th>TP</th>\n",
       "      <th>Cut Off</th>\n",
       "      <th>Too Long</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1019</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1020</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1021</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1022</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1023</th>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1024 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      True Term Num  Extracted Term Num  TP  Cut Off  Too Long\n",
       "0                 1                   1   1        0         0\n",
       "1                 1                   1   1        0         0\n",
       "2                 2                   2   2        0         0\n",
       "3                 1                   1   1        0         0\n",
       "4                 2                   2   1        0         1\n",
       "...             ...                 ...  ..      ...       ...\n",
       "1019              2                   1   0        1         0\n",
       "1020              1                   1   1        0         0\n",
       "1021              1                   1   1        0         0\n",
       "1022              1                   1   1        0         0\n",
       "1023              1                   4   1        1         0\n",
       "\n",
       "[1024 rows x 5 columns]"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_eval = pd.DataFrame(eval_list)\n",
    "df_eval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "id": "2dbc9ecb-f2d5-439d-99d2-6282e20521e3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "846"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_eval['TP'].sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "id": "5d9a1879-5d57-4971-91d6-55d35b1c7c86",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1616"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_eval['True Term Num'].sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "id": "207f8855-0cc7-4a00-83b3-0bfe9f6c80d4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2245"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_eval['Extracted Term Num'].sum() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "id": "1aabfa0d-8b20-4bcf-9ab3-83f46a2a2dda",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.37683741648106905"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# precision /correct rate\n",
    "846 / 2245"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "id": "4cbe0755-3cad-40f0-99f8-cea24df83cf3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5235148514851485"
      ]
     },
     "execution_count": 124,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# recall\n",
    "846 / 1616"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "id": "5febb9c5-7df2-4698-94fd-f10feaba8464",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.30779510022271717"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# too long rate \n",
    "df_eval['Too Long'].sum() / 2245"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "id": "52db8df0-6da3-4df8-987c-1e0fbeb99713",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.055233853006681516"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# cut off rate \n",
    "df_eval['Cut Off'].sum() / 2245"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "id": "786e812c-9580-45c1-83f3-07513c1ff0c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.concat([gen_data, df_eval], axis=1).to_csv('GPT_results/gpt-4_res+eval.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "06bb47f2-fd10-4865-9cbd-2ce4429d9e14",
   "metadata": {},
   "source": [
    "## Correct eval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "b6ca6bcb-9420-47b6-be1d-14ed7586293c",
   "metadata": {},
   "outputs": [],
   "source": [
    "#df_res = pd.read_csv('GPT_results/gpt-4_res+eval.csv')\n",
    "df_res = pd.read_csv('GPT_results/Human_corrected_annotations+gpt_res.csv')\n",
    "\n",
    "eval_list =[]\n",
    "\n",
    "for index, row in df_res.iterrows():\n",
    "\n",
    "    #let's remove repeated terms, keeping 1616 unique out of 1660 terms\n",
    "    expected_list = set(row['plain_text_term'].split(';'))\n",
    "    while '' in expected_list:\n",
    "        expected_list.remove('')\n",
    "\n",
    "    extracted_string = row['gpt-4 terms']#['gpt-3.5-turbo terms']\n",
    "    if extracted_string != extracted_string: #nan\n",
    "        extracted_list = []\n",
    "    else:\n",
    "        extracted_list = extracted_string.split('###')\n",
    "        extracted_list = [x.strip() for x in extracted_list] \n",
    "        while '' in extracted_list:\n",
    "            extracted_list.remove('')\n",
    "    \n",
    "    num_TP = 0\n",
    "    num_too_long = 0\n",
    "    num_cut_off = 0\n",
    "    num_split_term = 0\n",
    "    TP_list = []\n",
    "    ST_list = [] \n",
    "    \n",
    "    for expected in expected_list:\n",
    "        for extracted in extracted_list:\n",
    "            \n",
    "            if extracted.casefold() == expected.casefold():\n",
    "                num_TP = num_TP + 1\n",
    "                TP_list.append(expected)\n",
    "            elif extracted.casefold() in expected.casefold():\n",
    "                num_cut_off = num_cut_off + 1\n",
    "            elif expected.casefold() in extracted.casefold():\n",
    "                num_too_long = num_too_long + 1\n",
    "                \n",
    "        expected_no_space = expected.replace(\" \",\"\")\n",
    "        extracted_no_space = (\"\".join(extracted_list)).replace(\" \",\"\")\n",
    "        if expected_no_space.casefold() in extracted_no_space.casefold(): # including TPs\n",
    "            num_split_term = num_split_term + 1\n",
    "            ST_list.append(expected)\n",
    "    \n",
    "    num_TP = num_TP - (len(TP_list) - len(set(TP_list)))\n",
    "    num_split_term = num_split_term - (len(ST_list) -len(set(ST_list)))\n",
    "    \n",
    "    eval_list.append({'True Term Num' : len(expected_list),\n",
    "                        'Extracted Term Num': len(extracted_list),\n",
    "                        'TP': num_TP,\n",
    "                        'Cut Off': num_cut_off,\n",
    "                        'Too Long': num_too_long,\n",
    "                        'Split Term': num_split_term})\n",
    "df_eval = pd.DataFrame(eval_list)\n",
    "df_eval['expected'] = df_res['plain_text_term']\n",
    "df_eval['extracted'] = df_res['gpt-4 terms']\n",
    "\n",
    "df_eval.to_csv('GPT_results/gpt-3.5-turbo_res+eval+ST_corrected.csv', index=False)\n",
    "#df_eval.to_csv('GPT_results/gpt-3.5-turbo_res+eval+ST.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c6fe7d91-cab3-4c3c-8a6d-fb0a488d443f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>plain_text_def</th>\n",
       "      <th>plain_text_term</th>\n",
       "      <th>gpt-3.5-turbo terms</th>\n",
       "      <th>gpt-4 terms</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Let P be a G-poset. The compatibility graph of...</td>\n",
       "      <td>Compatibility graph</td>\n",
       "      <td>G-poset###compatibility graph###C_P###vertex s...</td>\n",
       "      <td>compatibility graph</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Let P be a G-poset. The strong compatibility g...</td>\n",
       "      <td>Strong compatibility graph</td>\n",
       "      <td>G-poset###strong compatibility graph###C_P###v...</td>\n",
       "      <td>strong compatibility graph</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>\\nLet p=p_1p_2⋯ p_n be a permutation. \\nWe say...</td>\n",
       "      <td>good pair;bad pair</td>\n",
       "      <td>good pair###bad pair</td>\n",
       "      <td>good pair###bad pair</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>\\nThe family of probability measures on partit...</td>\n",
       "      <td>multiplicative</td>\n",
       "      <td>multiplicative###probability measures###partit...</td>\n",
       "      <td>multiplicative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>\\nWe call a family of measures μ^(n) ergodic i...</td>\n",
       "      <td>ergodic;limit shape</td>\n",
       "      <td>ergodic###limit shape of partitions</td>\n",
       "      <td>ergodic###limit shape of partitions</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>994</th>\n",
       "      <td>Consider ⟨ G→ S, α→ A⟩, two adjacent vertices...</td>\n",
       "      <td>preferred direction;special claw corresponding...</td>\n",
       "      <td>special claw###subdivided medial graph</td>\n",
       "      <td>special claw</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>995</th>\n",
       "      <td>\\nAccording to Theorem &lt;ref&gt;, ^d has exactly d...</td>\n",
       "      <td>Cyclicity classes</td>\n",
       "      <td>cyclicity classes###</td>\n",
       "      <td>cyclicity classes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>996</th>\n",
       "      <td>By Wielandt number we mean the following funct...</td>\n",
       "      <td>Wielandt number</td>\n",
       "      <td>Wielandt number###</td>\n",
       "      <td>Wielandt number</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997</th>\n",
       "      <td>\\nThe girth of , denoted it by g(),\\nis the sm...</td>\n",
       "      <td>Girth</td>\n",
       "      <td>girth###nonempty cycle</td>\n",
       "      <td>girth</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>998</th>\n",
       "      <td>\\nLet A∈. The cycles of (A) at which the maxim...</td>\n",
       "      <td>Critical graph</td>\n",
       "      <td>critical###nodes###edges###N_c(A)###E_c(A)###c...</td>\n",
       "      <td>critical###N_c(A)###E_c(A)###critical graph</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>999 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                        plain_text_def  \\\n",
       "0    Let P be a G-poset. The compatibility graph of...   \n",
       "1    Let P be a G-poset. The strong compatibility g...   \n",
       "2    \\nLet p=p_1p_2⋯ p_n be a permutation. \\nWe say...   \n",
       "3    \\nThe family of probability measures on partit...   \n",
       "4    \\nWe call a family of measures μ^(n) ergodic i...   \n",
       "..                                                 ...   \n",
       "994   Consider ⟨ G→ S, α→ A⟩, two adjacent vertices...   \n",
       "995  \\nAccording to Theorem <ref>, ^d has exactly d...   \n",
       "996  By Wielandt number we mean the following funct...   \n",
       "997  \\nThe girth of , denoted it by g(),\\nis the sm...   \n",
       "998  \\nLet A∈. The cycles of (A) at which the maxim...   \n",
       "\n",
       "                                       plain_text_term  \\\n",
       "0                                  Compatibility graph   \n",
       "1                           Strong compatibility graph   \n",
       "2                                   good pair;bad pair   \n",
       "3                                       multiplicative   \n",
       "4                                  ergodic;limit shape   \n",
       "..                                                 ...   \n",
       "994  preferred direction;special claw corresponding...   \n",
       "995                                  Cyclicity classes   \n",
       "996                                    Wielandt number   \n",
       "997                                              Girth   \n",
       "998                                     Critical graph   \n",
       "\n",
       "                                   gpt-3.5-turbo terms  \\\n",
       "0    G-poset###compatibility graph###C_P###vertex s...   \n",
       "1    G-poset###strong compatibility graph###C_P###v...   \n",
       "2                                 good pair###bad pair   \n",
       "3    multiplicative###probability measures###partit...   \n",
       "4                  ergodic###limit shape of partitions   \n",
       "..                                                 ...   \n",
       "994             special claw###subdivided medial graph   \n",
       "995                               cyclicity classes###   \n",
       "996                                 Wielandt number###   \n",
       "997                             girth###nonempty cycle   \n",
       "998  critical###nodes###edges###N_c(A)###E_c(A)###c...   \n",
       "\n",
       "                                     gpt-4 terms  \n",
       "0                            compatibility graph  \n",
       "1                     strong compatibility graph  \n",
       "2                           good pair###bad pair  \n",
       "3                                 multiplicative  \n",
       "4            ergodic###limit shape of partitions  \n",
       "..                                           ...  \n",
       "994                                 special claw  \n",
       "995                            cyclicity classes  \n",
       "996                              Wielandt number  \n",
       "997                                        girth  \n",
       "998  critical###N_c(A)###E_c(A)###critical graph  \n",
       "\n",
       "[999 rows x 4 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_res"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9ffa9c2a-486a-412f-81e2-069029bbd97c",
   "metadata": {},
   "source": [
    "### GPT4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "117bb2cc-83bd-415c-96b3-d6d0e23738f0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True Term Num: 1552\n",
      "Extracted Term Num: 2191\n",
      "True Positive: 945\n",
      "True Positive + split term: 1369\n",
      "Too Long: 573\n",
      "Cut Off: 137\n",
      "precision /correct rate: 0.6248288452761296\n",
      "recall: 0.8820876288659794\n"
     ]
    }
   ],
   "source": [
    "num_T = df_eval['True Term Num'].sum()\n",
    "print(\"True Term Num: \" + str(num_T))\n",
    "num_Ex = df_eval['Extracted Term Num'].sum()\n",
    "print(\"Extracted Term Num: \" + str(num_Ex))\n",
    "print(\"True Positive: \" + str(df_eval['TP'].sum()))\n",
    "num_ST = df_eval['Split Term'].sum()\n",
    "print(\"True Positive + split term: \" + str(num_ST))\n",
    "print(\"Too Long: \" + str(df_eval['Too Long'].sum()))\n",
    "print(\"Cut Off: \" + str(df_eval['Cut Off'].sum()))\n",
    "print(\"precision /correct rate: \" + str(num_ST / num_Ex))\n",
    "print(\"recall: \" + str(num_ST / num_T))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "070b7e6e-99c4-4b2c-9805-7f4ea7541131",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7314987977558107"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# GPT-4\n",
    "p = 0.6248288452761296\n",
    "r = 0.8820876288659794\n",
    "2 * p * r / (p + r)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d1398125-98dc-4896-9698-038c5b017440",
   "metadata": {},
   "source": [
    "### GPT3.5-turbo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "3288c0fd-a3fe-4ab0-9473-348f3fc7b0c6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True Term Num: 1552\n",
      "Extracted Term Num: 6687\n",
      "True Positive: 1071\n",
      "True Positive + split term: 1290\n",
      "Too Long: 338\n",
      "Cut Off: 653\n",
      "precision /correct rate: 0.19291161956034095\n",
      "recall: 0.8311855670103093\n"
     ]
    }
   ],
   "source": [
    "num_T = df_eval['True Term Num'].sum()\n",
    "print(\"True Term Num: \" + str(num_T))\n",
    "num_Ex = df_eval['Extracted Term Num'].sum()\n",
    "print(\"Extracted Term Num: \" + str(num_Ex))\n",
    "print(\"True Positive: \" + str(df_eval['TP'].sum()))\n",
    "num_ST = df_eval['Split Term'].sum()\n",
    "print(\"True Positive + split term: \" + str(num_ST))\n",
    "print(\"Too Long: \" + str(df_eval['Too Long'].sum()))\n",
    "print(\"Cut Off: \" + str(df_eval['Cut Off'].sum()))\n",
    "print(\"precision /correct rate: \" + str(num_ST / num_Ex))\n",
    "print(\"recall: \" + str(num_ST / num_T))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "cfb67237-2052-46c5-9ec0-a332afdd1224",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.31314479912610754"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# GPT-3.5\n",
    "p = 0.19291161956034095\n",
    "r = 0.8311855670103093\n",
    "2 * p * r / (p + r)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3eed6f99-c66c-4a6f-ad15-084f5c0d4172",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}