text-gensim/gensim-2-newsgroups.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LDA With Newsgroups\n",
    "\n",
    "Reference\n",
    "- https://www.kaggle.com/sumantindurkhya/topic-modeling-on-20-newsgroup-data-lsa-and-lda"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 1 - Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import fetch_20newsgroups\n",
    "\n",
    "categories = ['rec.sport.baseball', 'sci.crypt', 'comp.graphics', 'misc.forsale']\n",
    "data = fetch_20newsgroups(subset='train',categories=categories,shuffle=True, \n",
    "                          remove=('headers', 'footers', 'qutes'), random_state=123)\n",
    "data.keys()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 2 - Explore Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "news_df = pd.DataFrame({'text': data.data,\n",
    "                       'Target': data.target})\n",
    "\n",
    "news_df['Target_name'] = news_df['Target'].apply(lambda x: data.target_names[x])\n",
    "\n",
    "print ('news_df.shape :', news_df.shape)\n",
    "news_df.sample(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 3 - Cleanup Text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# remove non alphabetic characters\n",
    "# remove stopwords and lemmatize\n",
    "import re\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.stem.porter import PorterStemmer\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "\n",
    "def clean_text(sentence):\n",
    "    # remove non alphabetic sequences\n",
    "    pattern = re.compile(r'[^a-z]+')\n",
    "    sentence = sentence.lower()\n",
    "    sentence = pattern.sub(' ', sentence).strip()\n",
    "    \n",
    "    # Tokenize\n",
    "    word_list = word_tokenize(sentence)\n",
    "    \n",
    "    # stop words\n",
    "    stopwords_list = set(stopwords.words('english'))\n",
    "    # puctuation\n",
    "    # punct = set(string.punctuation)\n",
    "    \n",
    "    # remove stop words\n",
    "    word_list = [word for word in word_list if word not in stopwords_list]\n",
    "    # remove very small words, length < 3\n",
    "    # they don't contribute any useful information\n",
    "    word_list = [word for word in word_list if len(word) > 2]\n",
    "    # remove punctuation\n",
    "    # word_list = [word for word in word_list if word not in punct]\n",
    "    \n",
    "    # stemming\n",
    "    # ps  = PorterStemmer()\n",
    "    # word_list = [ps.stem(word) for word in word_list]\n",
    "    \n",
    "    # lemmatize\n",
    "    lemma = WordNetLemmatizer()\n",
    "    word_list = [lemma.lemmatize(word) for word in word_list]\n",
    "    return word_list\n",
    "    ## list to sentence\n",
    "    #sentence = ' '.join(word_list)\n",
    "    #return sentence\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time \n",
    "\n",
    "from tqdm import tqdm\n",
    "\n",
    "# we'll use tqdm to monitor progress of data cleaning process\n",
    "# create tqdm for pandas\n",
    "tqdm.pandas()\n",
    "# clean text data\n",
    "news_df['tokens'] = news_df['text'].progress_apply(lambda x: clean_text(str(x)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# now the 'text' column should be cleaned up\n",
    "news_df.sample(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "texts = news_df['tokens'].to_list()\n",
    "len(texts)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 5 - LDA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gensim import corpora\n",
    "from pprint import pprint\n",
    "import random\n",
    "\n",
    "# turn our tokenized documents into a id <-> term dictionary\n",
    "dictionary = corpora.Dictionary(texts)\n",
    "print ('dict: num_tokens : ', len(dictionary))\n",
    "for x in random.sample(dictionary.items(), 10):\n",
    "    print(x)\n",
    "print('-----')\n",
    "    \n",
    "# convert tokenized documents into a document-term matrix\n",
    "corpus = [dictionary.doc2bow(text) for text in texts]\n",
    "print ('corpus: num_docs : ', len(corpus))\n",
    "# for i, c in enumerate(corpus):\n",
    "#     print(\"doc\" , i, c)\n",
    "\n",
    "for x in random.sample(corpus, 3):\n",
    "    print (x)\n",
    "    print ()\n",
    "    \n",
    "## TODO : can you understand this output below of dictionary and corpus?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 6 - LDA Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time \n",
    "\n",
    "import gensim\n",
    "from gensim.models.ldamodel import LdaModel\n",
    "\n",
    "# generate LDA model\n",
    "## TODO : Experiment with these properties\n",
    "## topic_count : change this number and see what topics are coming up\n",
    "## passes : try changing this too\n",
    "\n",
    "# topic_count = len(categories)\n",
    "topic_count = 10\n",
    "\n",
    "ldamodel = LdaModel(corpus, num_topics=topic_count, id2word = dictionary, passes=20)\n",
    "print (ldamodel)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pprint import pprint\n",
    "\n",
    "pprint(ldamodel.print_topics())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}