Distributional+Semantics.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import io\n",
    "def get_unigrams(file_name):\n",
    "    unigrams = {}\n",
    "    with io.open(file_name, encoding='utf8', errors='ignore') as f:\n",
    "        for line in f:\n",
    "            tokens = line.strip().split()\n",
    "            for token in tokens:\n",
    "                token = token.lower()\n",
    "                try:\n",
    "                    unigrams[token]\n",
    "                except:\n",
    "                    unigrams[token] = 0\n",
    "                unigrams[token] += 1\n",
    "                \n",
    "    return unigrams\n",
    "\n",
    "def index_unigrams(unigrams):\n",
    "    new_unigrams = {}\n",
    "    reverse_unigrams = {}\n",
    "    for index, unigram in enumerate(unigrams):\n",
    "        new_unigrams[unigram] = index\n",
    "        reverse_unigrams[index] = unigram\n",
    "    return new_unigrams, reverse_unigrams\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: 'sample_corpus.txt'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-2-54c80de70002>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[0mfile_name\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m\"sample_corpus.txt\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0munigrams\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mget_unigrams\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfile_name\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      3\u001b[0m \u001b[0miunigrams\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mrunigrams\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mindex_unigrams\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0munigrams\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[0munigrams\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0msorted\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0munigrams\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkey\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mlambda\u001b[0m \u001b[0mx\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mx\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mreverse\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mTrue\u001b[0m \u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mpprint\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mpprint\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-1-c05595bf61ed>\u001b[0m in \u001b[0;36mget_unigrams\u001b[1;34m(file_name)\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mget_unigrams\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfile_name\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m     \u001b[0munigrams\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m{\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 4\u001b[1;33m     \u001b[1;32mwith\u001b[0m \u001b[0mio\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mopen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfile_name\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mencoding\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'utf8'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0merrors\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'ignore'\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      5\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mline\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      6\u001b[0m             \u001b[0mtokens\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mline\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstrip\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'sample_corpus.txt'"
     ]
    }
   ],
   "source": [
    "file_name = \"sample_corpus.txt\"\n",
    "unigrams = get_unigrams(file_name)\n",
    "iunigrams,runigrams = index_unigrams(unigrams)\n",
    "unigrams = sorted(unigrams.items(), key = lambda x: x[1], reverse = True )\n",
    "from pprint import pprint\n",
    "#pprint.pprint(iunigrams) # Figure out non-stop words\n",
    "dimensions = [x[0] for x in unigrams[100:3100]]\n",
    "idimensions = {x: index for index, x in enumerate(dimensions)}\n",
    "#pprint(dimensions)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy\n",
    "cmatrix = numpy.memmap(\"lsa.cmatrix\", dtype='float32', mode='w+', shape=(len(unigrams),len(dimensions)))\n",
    "print(cmatrix.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def populate_cmatrix(file_name, cmatrix, iunigrams, dimensions, window = 5):\n",
    "     e = 0\n",
    "     s = 0\n",
    "     with open(file_name, encoding='utf-8', errors='ignore') as f:\n",
    "        for index, line in enumerate(f):             \n",
    "            tokens = line.strip().split()\n",
    "            for indexj, token in enumerate(tokens):\n",
    "                token = token.lower()\n",
    "                lcontext = tokens[indexj - window:indexj]\n",
    "                rcontext = tokens[indexj + 1:index + window]\n",
    "                context = [tok.lower() for tok in lcontext + rcontext]\n",
    "                \n",
    "                try:\n",
    "                    unigram_index = iunigrams[token]                    \n",
    "                    for d in context:\n",
    "                        \n",
    "                        if d in dimensions:\n",
    "                            j = dimensions[d]\n",
    "                            cmatrix[unigram_index][j] += 1 \n",
    "                            s += 1\n",
    "                except:\n",
    "                    e += 1\n",
    "            \n",
    "            \n",
    "     print(e,s)\n",
    "                \n",
    "                "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from time import time\n",
    "s = time()\n",
    "populate_cmatrix(file_name, cmatrix, iunigrams, idimensions)\n",
    "e = time()\n",
    "print(e -s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "w1 = 'eat'\n",
    "w2 = 'drink'\n",
    "w3 = 'print'\n",
    "id1 = iunigrams[w1]\n",
    "id2 = iunigrams[w2]\n",
    "id3 = iunigrams[w3]\n",
    "print(id1, id2, id3)\n",
    "v1 = cmatrix[id1]\n",
    "v2 = cmatrix[id2]\n",
    "v3 = cmatrix[id3]\n",
    "\n",
    "print(v1, v2, v3)\n",
    "\n",
    "from scipy.spatial.distance import *\n",
    "print(euclidean(v1, v2))\n",
    "print(cosine(v1,v2))\n",
    "print(cosine(v1,v3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.decomposition import TruncatedSVD\n",
    "s = time()\n",
    "svd = TruncatedSVD(n_components=5, random_state=42)\n",
    "svd.fit(cmatrix)\n",
    "twod_cmatrix = svd.transform(cmatrix)\n",
    "e = time()\n",
    "print(e - s )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "v1_2d, v2_2d = twod_cmatrix[id1], twod_cmatrix[id2]\n",
    "id3 = iunigrams[w3]\n",
    "v3_2d = twod_cmatrix[id3]\n",
    "print(v1_2d, v2_2d, v3_2d)\n",
    "print(cosine(v1_2d, v2_2d), cosine(v1_2d, v3_2d))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%pylab inline\n",
    "import matplotlib.pyplot as plt\n",
    "v1_2d = v1_2d / numpy.linalg.norm(v1_2d)\n",
    "v2_2d = v2_2d / numpy.linalg.norm(v2_2d)\n",
    "v3_2d = v3_2d / numpy.linalg.norm(v3_2d)\n",
    "print ([v1_2d, v2_2d,v3_2d])\n",
    "colors = ['r','b','g']\n",
    "fig, axs = plt.subplots(1,1)\n",
    "for i, x in enumerate([v1_2d, v2_2d,v3_2d]):\n",
    "    a = plt.plot([0,x[0]],[0,x[1]],colors[i]+'-')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}