Preprocessing_corpus.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Preprocessing corpus.ipynb",
      "provenance": [],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "h6LrzmLNq-9J"
      },
      "source": [
        "### Preprocessing Data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "okoGmvssG8he"
      },
      "source": [
        "import warnings\n",
        "warnings.filterwarnings(\"ignore\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ICGtVcYAMN2p",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "cdc9fbc2-7bf4-4635-cfab-d551d7a5c21c"
      },
      "source": [
        "!wget https://nlp.stanford.edu/projects/snli/snli_1.0.zip"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2021-03-15 07:18:02--  https://nlp.stanford.edu/projects/snli/snli_1.0.zip\n",
            "Resolving nlp.stanford.edu (nlp.stanford.edu)... 171.64.67.140\n",
            "Connecting to nlp.stanford.edu (nlp.stanford.edu)|171.64.67.140|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 94550081 (90M) [application/zip]\n",
            "Saving to: ‘snli_1.0.zip’\n",
            "\n",
            "snli_1.0.zip        100%[===================>]  90.17M  2.79MB/s    in 30s     \n",
            "\n",
            "2021-03-15 07:18:32 (2.98 MB/s) - ‘snli_1.0.zip’ saved [94550081/94550081]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2yZhNwuEMSH3",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "33c21280-2214-4ff2-f9f5-4cae28575fe1"
      },
      "source": [
        "!unzip snli_1.0.zip"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Archive:  snli_1.0.zip\n",
            "   creating: snli_1.0/\n",
            "  inflating: snli_1.0/.DS_Store      \n",
            "   creating: __MACOSX/\n",
            "   creating: __MACOSX/snli_1.0/\n",
            "  inflating: __MACOSX/snli_1.0/._.DS_Store  \n",
            " extracting: snli_1.0/Icon           \n",
            "  inflating: __MACOSX/snli_1.0/._Icon  \n",
            "  inflating: snli_1.0/README.txt     \n",
            "  inflating: __MACOSX/snli_1.0/._README.txt  \n",
            "  inflating: snli_1.0/snli_1.0_dev.jsonl  \n",
            "  inflating: snli_1.0/snli_1.0_dev.txt  \n",
            "  inflating: snli_1.0/snli_1.0_test.jsonl  \n",
            "  inflating: snli_1.0/snli_1.0_test.txt  \n",
            "  inflating: snli_1.0/snli_1.0_train.jsonl  \n",
            "  inflating: snli_1.0/snli_1.0_train.txt  \n",
            "  inflating: __MACOSX/._snli_1.0     \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mQGaz4N1rEBK"
      },
      "source": [
        "### Imports"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "22E39JhyrRnY"
      },
      "source": [
        "import json\n",
        "import shutil\n",
        "\n",
        "import spacy"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tUpgecXjrdWG"
      },
      "source": [
        "nlp = spacy.load('en')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QQ1kkuzfr592"
      },
      "source": [
        "### Preprocess data with spacy"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QXTddcgKvb-M"
      },
      "source": [
        "def preprocess_text(text):\n",
        "    return ' '.join([str(w.lemma_) if w.lemma_ != '-PRON-' else str(w.lower_) for w in nlp(text.lower())])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "id": "FabYY8Izv-ZL",
        "outputId": "fc7697b1-1b04-4a5a-a67d-986a7a78343f"
      },
      "source": [
        "preprocess_text('He going to lakes')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'he go to lake'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vzpPD00drpHd",
        "outputId": "deece853-1b80-4e8b-c30c-78a78100a20d"
      },
      "source": [
        "for files in [\n",
        "              {'in':'.//snli_1.0//snli_1.0_train.jsonl', 'out':'.//train.jsonl'},\n",
        "              {'in':'.//snli_1.0//snli_1.0_test.jsonl', 'out':'.//test.jsonl'},\n",
        "              {'in':'.//snli_1.0//snli_1.0_dev.jsonl', 'out':'.//val.jsonl'}\n",
        "              ]:\n",
        "    print(files['in'])\n",
        "    with open(files['in']) as fin, open(files['out'], 'w') as fout:\n",
        "        for line in fin:\n",
        "            data = json.loads(line)\n",
        "            new_data = {\n",
        "                'sentence1':preprocess_text(data['sentence1']),\n",
        "                'sentence2':preprocess_text(data['sentence2']),\n",
        "                'gold_label':data['gold_label']\n",
        "            }\n",
        "            fout.write(json.dumps(new_data)+'\\n')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            ".//snli_1.0//snli_1.0_train.jsonl\n",
            ".//snli_1.0//snli_1.0_test.jsonl\n",
            ".//snli_1.0//snli_1.0_dev.jsonl\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CPIKAxP2sDNz"
      },
      "source": [
        "### Sent data on my google disk"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "id": "pkkjOrdSxz-Z",
        "outputId": "b55ae8ea-a38a-4b6e-e785-54c71f8f5602"
      },
      "source": [
        "shutil.copy('.//train.jsonl', './/drive//MyDrive//challenge//')\n",
        "shutil.copy('.//test.jsonl', './/drive//MyDrive//challenge//')\n",
        "shutil.copy('.//val.jsonl', './/drive//MyDrive//challenge//')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'.//drive//MyDrive//challenge//train.jsonl'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 26
        }
      ]
    }
  ]
}