hackernewsv4.py

# -*- coding: utf-8 -*-
"""HackerNewsV4.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1dtIIRFFq81O97i6azALZbqBYgoVsUKvD
"""

from warnings import filterwarnings
filterwarnings("ignore")

import pandas as pd
import json
import urllib
import requests

import numpy as np

html = urllib.request.urlopen('https://hacker-news.firebaseio.com/v0/item/2169746.json?print=pretty')
json.loads(html.read())

x = urllib.request.urlopen('https://hacker-news.firebaseio.com/v0/maxitem.json?print=pretty')
max_item = int(x.read())
print(max_item)

data = []
number_of_entries = 1000

min_item = max_item - number_of_entries
count = 0


for i in range(min_item, max_item):
    html = urllib.request.urlopen('https://hacker-news.firebaseio.com/v0/item/' + str(i) + '.json')
    data.append(json.loads(html.read()))
    count += 1
    if count % 100 == 0:
        print(f"Loaded {count} rows")

print (data[0])

data = [i for i in data if i is not None]

from pandas.io.json import json_normalize
df = pd.DataFrame.from_dict(data)
df.head(10)

print(df.shape)

df.columns

df_new = df.drop(columns = ['deleted', 'dead', 'descendants', 'score', 'kids', 'parent', 'title', 'url'])
df_new.head(10)

df_new['text'][0]

df_comments = df_new[df_new['type'] == 'comment']
df_comments['text']

import nltk
from nltk.tokenize import RegexpTokenizer
from nltk.stem import WordNetLemmatizer,PorterStemmer
from nltk.corpus import stopwords

nltk.download('stopwords')
nltk.download('wordnet')

import re
lemmatizer = WordNetLemmatizer()
stemmer = PorterStemmer() 

def preprocess(sentence):
    sentence=str(sentence)
    sentence = sentence.lower()
    sentence=sentence.replace('{html}',"") 
    cleanr = re.compile('<.*?>')
    cleantext = re.sub(cleanr, '', sentence)
    rem_url=re.sub(r'http\S+', '',cleantext)
    rem_num = re.sub('[0-9]+', '', rem_url)
    tokenizer = RegexpTokenizer(r'\w+')
    tokens = tokenizer.tokenize(rem_num)  
    filtered_words = [w for w in tokens if len(w) > 2 if not w in stopwords.words('english')]
    stem_words=[stemmer.stem(w) for w in filtered_words]
    lemma_words=[lemmatizer.lemmatize(w) for w in stem_words]
    return " ".join(filtered_words)


df_comments['clean_text']=df_comments['text'].map(lambda s:preprocess(s))

df_comments.head(10)

df_comments['by'].value_counts()

df_comments.shape

df_comments['text'] = df_comments['text'].astype(str)
df_comments['text'][:10]

df_comments['clean_text'] = df_comments['clean_text'].astype(str)
df_comments['clean_text'][:10]

import nltk
from nltk.corpus import stopwords

nltk.download('stopwords')

!pip install vaderSentiment

from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
analyzer = SentimentIntensityAnalyzer()

def get_compund_score(text):
        score = analyzer.polarity_scores(text)
        str(text)
        return score['compound']

get_compund_score(df_comments['text'][0])



get_compund_score(df_comments['clean_text'][0])

df_comments['vader_score'] = df_comments['text'].apply(get_compund_score)

df_comments['clean_vader_score'] = df_comments['clean_text'].apply(get_compund_score)

df_comments.head(10)

df_comments['sentiment'] = df_comments['vader_score'].apply(lambda c: 'positive' if c >=0 else 'negative')

df_comments['clean_sentiment'] = df_comments['clean_vader_score'].apply(lambda c: 'positive' if c >=0 else 'negative')

df_comments.head(25)

df_comments.to_csv('hn_sentiments.csv')

import pandas as pd
import json
import urllib
import requests

!pip install vaderSentiment

from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
analyzer = SentimentIntensityAnalyzer()

def get_compund_score(text):
        score = analyzer.polarity_scores(text)
        str(text)
        return score['compound']

def preprocess(sentence):
    sentence=str(sentence)
    sentence = sentence.lower()
    sentence=sentence.replace('{html}',"") 
    cleanr = re.compile('<.*?>')
    cleantext = re.sub(cleanr, '', sentence)
    rem_url=re.sub(r'http\S+', '',cleantext)
    rem_num = re.sub('[0-9]+', '', rem_url)
    tokenizer = RegexpTokenizer(r'\w+')
    tokens = tokenizer.tokenize(rem_num)  
    filtered_words = [w for w in tokens if len(w) > 2 if not w in stopwords.words('english')]
    stem_words=[stemmer.stem(w) for w in filtered_words]
    lemma_words=[lemmatizer.lemmatize(w) for w in stem_words]
    return " ".join(filtered_words)

import nltk
from nltk.corpus import stopwords

nltk.download('stopwords')

import pandas as pd
import json
import urllib
import requests

def get_score(entries):
  data = []
  for id in entries:
    html = urllib.request.urlopen('https://hacker-news.firebaseio.com/v0/item/' + str(id) + '.json')
    data.append(json.loads(html.read()))

  print(data[0])
  data = [i for i in data if i is not None]

  df = pd.DataFrame.from_dict(data)
  df_comments = df[df['type'] == 'comment']

  df_comments['clean_text']=df_comments['text'].map(lambda s:preprocess(s)) 
  df_comments['clean_vader_score'] = df_comments['clean_text'].apply(get_compund_score)

  return df_comments['clean_vader_score'].sum() #we can use mean()



import pandas as pd
import json
import urllib
import requests

def get_cummulative_score(username):

  data = []
  html = urllib.request.urlopen('https://hacker-news.firebaseio.com/v0/user/' + str(username) + '.json?print=pretty')
  data.append(json.loads(html.read()))
  df2 = pd.DataFrame.from_dict(data)
  entries =  (df2['submitted'][0])

  score = get_score(entries)
  return score

score = get_cummulative_score('jl')
print("Cummulative score for user ", score )

test_message = "This is really bad. Never going there again. Absolute worst"
print (get_compund_score(test_message))

test_message = "This is really good. Loved the place. Employees were super kind. Can't wait to go back again."
print (get_compund_score(test_message))