Python_Neo4J.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import csv\n",
    "\n",
    "from pprint import pprint\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "# from py2neo import Graph\n",
    "\n",
    "from openie import StanfordOpenIE\n",
    "from extract_entity import use_nltk_ner, use_spacy_ner, use_stanford_ner, use_allen_ner\n",
    "from extract_relation import AllanRE\n",
    "from resolve_coreference import AllenCR, StanfordCR, SpacyCR\n",
    "from utils.spell_check import levenshtein_ratio_and_distance\n",
    "\n",
    "from IPython.core.display import display, HTML\n",
    "from libs.gpr_pub import visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [],
   "source": [
    "''' Initialize variables '''\n",
    "# dataset = r\"data/input/restaurant_20_dataset.json\"\n",
    "dataset = r\"data/input/data_100/restaurant_dataset.json\"\n",
    "relation_triplets_file = r\"data/output/kg/relation_triplets.csv\"\n",
    "\n",
    "users_csv = r\"data/output/kg/users.csv\"\n",
    "restaurants_csv = r\"data/output/kg/restaurants.csv\"\n",
    "ratings_csv = r\"data/output/kg/ratings.csv\""
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "''' open restaurant dataset '''\n",
    "file = open(dataset,)\n",
    "restaurant_data = json.load(file)\n",
    "file.close()\n",
    "\n",
    "# check data sample\n",
    "# restaurant = restaurant_data['restaurants'][0]\n",
    "# print(restaurant)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "''' Test GraphDB '''\n",
    "# from module.neo4j.graph_db import GraphDB\n",
    "# graph = GraphDB(\"bolt://localhost:7687\", \"neo4j\", \"erclab\")\n",
    "# test connection\n",
    "# graph.run(\"UNWIND range(1, 3) AS n RETURN n, n * n as n_sq\")\n",
    "# graph.print_greeting(\"hello, world\")\n",
    "# graph.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "###############################################################################################################\n",
    "# Making tables in the Graph database from Yelp structured json dataset                                       #\n",
    "#                                                                                                             #\n",
    "# DB Schema is defined as follows:                                                                            #\n",
    "#                                                                                                             #\n",
    "# ##################  ##################  ############  ############  ##########  ###########  #############  #\n",
    "# # `Restaurant`   #  # `User`         #  # `Aspect` #  # `Review` #  # `City` #  # `State` #  # `Country` #  #\n",
    "# ##################  ##################  ############  ############  ##########  ###########  #############  #\n",
    "# # - ID           #  # - ID           #  # - ID     #  # - ID     #  # - ID   #  # - ID    #  # - ID      #  #\n",
    "# # - name         #  # - name         #  # - name   #  # - text   #  # - name #  # - name  #  # - name    #  #\n",
    "# # - address      #  # - review_count #  ############  ############  ##########  ###########  #############  #\n",
    "# # - postal_code  #  ##################                                                                      #\n",
    "# # - rating       #                                                                                          #\n",
    "# ##################                                                                                          #\n",
    "#                                                                                                             #\n",
    "###############################################################################################################\n",
    "\n",
    "#################################################\n",
    "#                                               #\n",
    "# make triples from yelp structured json data   #\n",
    "#                                               #\n",
    "# User       -> visit(date) -> Review           #\n",
    "# User       -> wrote(date) -> Review           #\n",
    "# User       -> rate(star)  -> Restaurant       #\n",
    "# User       -> has_friend  -> User             #\n",
    "# Restaurant -> has_review  -> Review           #\n",
    "# Restaurant -> has         -> Attribute/Aspect #\n",
    "# Restaurant -> in_city     -> City             #\n",
    "# City       -> in_state    -> State            #\n",
    "# State      -> in_country  -> Country          #\n",
    "#                                               #\n",
    "#################################################\n",
    "from module.neo4j.graph_db import GraphDB\n",
    "\n",
    "# graph = GraphDB(\"bolt://localhost:7687\", \"neo4j\", \"erclab\")\n",
    "graph = GraphDB(\"bolt://localhost:11012\", \"neo4j\", \"erclab\")\n",
    "\n",
    "# Inserting users\n",
    "users = restaurant_data['users']\n",
    "relation_triplets = []\n",
    "\n",
    "users_list = []\n",
    "restaurants_list = []\n",
    "ratings_list = []\n",
    "\n",
    "print(\"inserting users in DB\")\n",
    "print(\"Total users: \"+str(len(users)))\n",
    "i=0\n",
    "for user in users:\n",
    "    graph.create_user(user[\"user_id\"], user[\"name\"], user[\"gender\"], user[\"age\"], user[\"fans\"],\n",
    "                      user[\"review_count\"], user[\"average_stars\"])\n",
    "\n",
    "    users_list.append([i, user[\"user_id\"], user[\"name\"], user[\"gender\"], user[\"age\"], user[\"fans\"],\n",
    "                      user[\"review_count\"], user[\"average_stars\"], \"|\".join(fri for fri in user[\"friends\"])])\n",
    "    i+=1\n",
    "\n",
    "for user in users:\n",
    "    friends = user['friends']\n",
    "\n",
    "    if len(friends) == 0:\n",
    "        continue\n",
    "\n",
    "    for friend in friends:\n",
    "        # insert relation to GraphDB\n",
    "        graph.create_user_has_friend_relation(user[\"user_id\"], friend)\n",
    "        # insert relation to triplet list\n",
    "        relation_triplets.append([user[\"user_id\"], 'HAS_FRIEND', friend])\n",
    "\n",
    "    \n",
    "# Inserting restaurants\n",
    "restaurants = restaurant_data['restaurants']\n",
    "print(\"Inserting restaurants in DB\")\n",
    "print(\"Total restaurants: \"+str(len(restaurants)))\n",
    "i = 0\n",
    "for restaurant in restaurants:\n",
    "\n",
    "    # restaurant_photos = json.dumps(restaurant['photos'])\n",
    "    photos = []\n",
    "    if restaurant['photos']:\n",
    "        for photo in restaurant['photos']:\n",
    "            photos.append(photo['photo_id'])\n",
    "\n",
    "    restaurant_photos = \"[\"+\",\".join(p for p in photos)+\"]\"\n",
    "\n",
    "    graph.create_restaurant(restaurant[\"rest_id\"], restaurant[\"name\"], restaurant[\"address\"],\n",
    "                                restaurant[\"postal_code\"], restaurant[\"rating\"], restaurant_photos)\n",
    "\n",
    "    restaurants_list.append([i, restaurant[\"rest_id\"], restaurant[\"name\"], restaurant[\"postal_code\"],\n",
    "                           restaurant[\"city\"], restaurant[\"state\"], restaurant[\"country\"], restaurant[\"rating\"],\n",
    "                           \"|\".join(cat.strip() for cat in restaurant[\"category\"].split(\",\")\n",
    "                                    if cat.strip() != \"Restaurants\")])\n",
    "\n",
    "    # Inserting categories\n",
    "    categories = restaurant['category'].split(',')\n",
    "    for category in categories:\n",
    "        category = category.strip()\n",
    "        if category == 'Restaurants':\n",
    "            continue\n",
    "\n",
    "        category_id = \"_\".join(t.title() for t in category.split())\n",
    "        graph.create_category(category_id, category.title())\n",
    "        graph.create_restaurant_has_category_relation(restaurant[\"rest_id\"], category_id)\n",
    "        relation_triplets.append([restaurant[\"rest_id\"], 'HAS_CATEGORY', category_id])\n",
    "\n",
    "    # Inserting reviews\n",
    "    reviews = restaurant['reviews']\n",
    "    \n",
    "    print(\"[\"+str(i)+\"] Inserting reviews for restaurant \"+restaurant[\"rest_id\"]+\" - \"+restaurant[\"name\"]+\"in DB\")\n",
    "    print(\"Total reviews: \"+str(len(reviews)))\n",
    "    for review in reviews:\n",
    "        graph.create_review(review[\"review_id\"], review[\"text\"])\n",
    "\n",
    "        ratings_list.append([review[\"user_id\"], restaurant[\"rest_id\"], review[\"rating\"], review[\"date\"]])\n",
    "\n",
    "        # calculate visit count and avg rating\n",
    "        total_rating = 0\n",
    "        visit_count = 0\n",
    "        for rev in reviews:\n",
    "            if rev[\"user_id\"] == review[\"user_id\"]:\n",
    "                total_rating += rev[\"rating\"]\n",
    "                visit_count += 1\n",
    "                # print(\"visit_count: \"+str(visit_count)+\" - rating: \"+str(rev[\"rating\"])+\" - total_rating: \"+str(total_rating))\n",
    "        avg_rating = total_rating/visit_count\n",
    "\n",
    "        # for debugging about user multiple visit\n",
    "        # if visit_count > 1:\n",
    "        #     print(\"Visit Multiple Times: \"+review[\"user_id\"]+\" - count: \"+str(visit_count)+\" - avg_rate: \"+str(avg_rating))\n",
    "\n",
    "        # Add reviews relations to GraphDB\n",
    "        graph.create_restaurant_has_review_relation(restaurant[\"rest_id\"], review[\"review_id\"])\n",
    "        graph.create_user_write_review_relation(review[\"user_id\"], review[\"review_id\"], review[\"date\"])\n",
    "        graph.create_user_visit_restaurant_relation(review[\"user_id\"], restaurant[\"rest_id\"], visit_count)\n",
    "        graph.create_user_rate_restaurant_relation(review[\"user_id\"], restaurant[\"rest_id\"], avg_rating)\n",
    "\n",
    "        # Add reviews relations to triplet list\n",
    "        relation_triplets.append([restaurant[\"rest_id\"], 'HAS_REVIEW', review[\"review_id\"]])\n",
    "        relation_triplets.append([review[\"user_id\"], 'WRITE_REVIEW{date:'+str(review[\"date\"])+'}', review[\"review_id\"]])\n",
    "        relation_triplets.append([review[\"user_id\"], 'VISIT{count:'+str(visit_count)+'}', restaurant[\"rest_id\"]])\n",
    "        relation_triplets.append([review[\"user_id\"], 'RATE{star:'+str(avg_rating)+'}', restaurant[\"rest_id\"]])\n",
    "        \n",
    "    # Inserting city, state, country with relationships\n",
    "    # Remove state (Only use city and country)\n",
    "    city, state, country = restaurant[\"city\"], restaurant[\"state\"], restaurant[\"country\"]\n",
    "    unique_country = \"_\".join(cc for cc in country.split())\n",
    "    # unique_state = f'{state}-{unique_country}'\n",
    "    # unique_city = f'{city}-{state}-{unique_country}'\n",
    "    unique_city = f'{\"_\".join(cc for cc in city.split())}-{unique_country}'\n",
    "\n",
    "    # print(\"inserting city, state, country for restaurant \"+restaurant[\"rest_id\"]+\" - \"+restaurant[\"name\"]+\"in DB\")\n",
    "    print(\"Inserting city, country for restaurant \"+restaurant[\"rest_id\"]+\" - \"+restaurant[\"name\"]+\"in DB\")\n",
    "    graph.create_city(unique_city, city)\n",
    "    # graph.create_state(unique_state, state)\n",
    "    graph.create_country(unique_country, country)\n",
    "\n",
    "    # Add City Country relations to GraphDB\n",
    "    graph.create_restaurant_city_relation(restaurant[\"rest_id\"], unique_city)\n",
    "    # graph.create_city_state_relation(unique_city, unique_state)\n",
    "    graph.create_city_country_relation(unique_city, unique_country)\n",
    "\n",
    "    # Add City Country relations to triplet list\n",
    "    relation_triplets.append([restaurant[\"rest_id\"], 'LOCATED_IN', unique_city])\n",
    "    relation_triplets.append([unique_city, 'LOCATED_IN', unique_country])\n",
    "\n",
    "    i += 1\n",
    "\n",
    "graph.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "''' Store relation_triplets to csv file '''\n",
    "relation_triplets_df = pd.DataFrame(relation_triplets, columns=['subject', 'relation', 'object'])\n",
    "relation_triplets_df.to_csv(relation_triplets_file, index=False)\n",
    "\n",
    "users_df = pd.DataFrame(users_list, columns=['id', 'user_id', 'name', 'gender', 'age', 'fans', 'review_count',\n",
    "                                             'average_stars', 'friends'])\n",
    "users_df.to_csv(users_csv, index=False)\n",
    "\n",
    "restaurants_df = pd.DataFrame(restaurants_list, columns=['id', 'rest_id', 'name', 'postal_code', 'city', 'state',\n",
    "                                                         'country', 'rating', 'category'])\n",
    "restaurants_df.to_csv(restaurants_csv, index=False)\n",
    "\n",
    "ratings_df = pd.DataFrame(ratings_list, columns=['user_id', 'rest_id', 'rating', 'date'])\n",
    "ratings_df.to_csv(ratings_csv, index=False)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import glob\n",
    "import argparse\n",
    "import logging\n",
    "import pickle\n",
    "import re\n",
    "import json\n",
    "import csv\n",
    "from pprint import pprint\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "import spacy\n",
    "from spacy import displacy\n",
    "from spacy.matcher import Matcher\n",
    "from spacy.util import filter_spans\n",
    "from openie import StanfordOpenIE\n",
    "from extract_entity import use_nltk_ner, use_spacy_ner, use_stanford_ner, use_allen_ner\n",
    "from extract_relation import AllanRE\n",
    "from resolve_coreference import AllenCR, StanfordCR, SpacyCR\n",
    "from utils.spell_check import levenshtein_ratio_and_distance\n",
    "\n",
    "\n",
    "from IPython.core.display import display, HTML\n",
    "from libs.gpr_pub import visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# credit: https://github.com/wang-h/bert-relation-classification/blob/master/utils.py\n",
    "def clean_str(text):\n",
    "    # Clean the text\n",
    "    text = re.sub(r\"[^A-Za-z0-9^,!.\\/'+-=<>]\", \" \", text)\n",
    "    text = re.sub(r\"[0-9]\", \" \", text)\n",
    "    text = re.sub(r\"what's\", \"what is \", text)\n",
    "    text = re.sub(r\"that's\", \"that is \", text)\n",
    "    text = re.sub(r\"there's\", \"there is \", text)\n",
    "    text = re.sub(r\"it's\", \"it is \", text)\n",
    "    text = re.sub(r\"\\'s\", \" \", text)\n",
    "    text = re.sub(r\"\\'ve\", \" have \", text)\n",
    "    text = re.sub(r\"can't\", \"can not \", text)\n",
    "    text = re.sub(r\"n't\", \" not \", text)\n",
    "    text = re.sub(r\"i'm\", \"i am \", text)\n",
    "    text = re.sub(r\"\\'re\", \" are \", text)\n",
    "    text = re.sub(r\"\\'d\", \" would \", text)\n",
    "    text = re.sub(r\"\\'ll\", \" will \", text)\n",
    "    text = re.sub(r\",\", \" \", text)\n",
    "#     text = re.sub(r\"\\.\", \" \", text)\n",
    "    text = re.sub(r\"!\", \" ! \", text)\n",
    "    text = re.sub(r\"\\/\", \" \", text)\n",
    "    text = re.sub(r\"\\^\", \" ^ \", text)\n",
    "    text = re.sub(r\"\\+\", \" + \", text)\n",
    "    text = re.sub(r\"\\-\", \" - \", text)\n",
    "    text = re.sub(r\"\\=\", \" = \", text)\n",
    "    text = re.sub(r\"'\", \" \", text)\n",
    "    text = re.sub(r\"(\\d+)(k)\", r\"\\g<1>000\", text)\n",
    "    text = re.sub(r\":\", \" : \", text)\n",
    "    text = re.sub(r\" e g \", \" eg \", text)\n",
    "    text = re.sub(r\" b g \", \" bg \", text)\n",
    "    text = re.sub(r\" u s \", \" american \", text)\n",
    "    text = re.sub(r\"\\0s\", \"0\", text)\n",
    "    text = re.sub(r\" 9 11 \", \"911\", text)\n",
    "    text = re.sub(r\"e - mail\", \"email\", text)\n",
    "    text = re.sub(r\"j k\", \"jk\", text)\n",
    "    text = re.sub(r\"\\s{2,}\", \" \", text)\n",
    "    return text\n",
    "\n",
    "def replace_subject_entity(text, entity):\n",
    "    text = re.sub(r\"\\bI\\b|\\bi\\b|\\bWe\\b|\\bwe\\b\", entity, text)\n",
    "    return text\n",
    "\n",
    "# Clean reviews text\n",
    "def clean_reviews(restaurants_data, save_clean_data=False):    \n",
    "    rest_obj = dict()\n",
    "    restaurants = restaurants_data\n",
    "    for restaurant in restaurants:\n",
    "        for review in restaurant['reviews']:\n",
    "            review[\"text\"] = clean_str(review[\"text\"])\n",
    "            review[\"text\"] = replace_subject_entity(review[\"text\"], review[\"name\"])\n",
    "    \n",
    "    if save_clean_data:\n",
    "        rest_obj['restaurants'] = restaurants\n",
    "        with open('data/input/cleaned_reviews.json', 'w') as outfile:\n",
    "            json.dump(rest_obj, outfile)\n",
    "        \n",
    "    \n",
    "def split_sentence(text):\n",
    "    '''\n",
    "    splits review into a list of sentences using spacy's sentence parser\n",
    "    '''\n",
    "    nlp = spacy.load('en_core_web_sm')\n",
    "    review = nlp(text)\n",
    "    bag_sentence = []\n",
    "    start = 0\n",
    "    for token in review:\n",
    "        if token.sent_start:\n",
    "            bag_sentence.append(review[start:(token.i - 1)])\n",
    "            start = token.i\n",
    "        if token.i == len(review) - 1:\n",
    "            bag_sentence.append(review[start:(token.i + 1)])\n",
    "    return bag_sentence\n",
    "\n",
    "\n",
    "def triple_pruning(triples, ner_dict):\n",
    "    entity_set = set(ner_dict.keys())\n",
    "    final_triples = []\n",
    "\n",
    "    for row, col in triples.iterrows():\n",
    "        col['subject'] = col['subject'].strip()\n",
    "\n",
    "        # check if Named Entity in subject sentence fragment\n",
    "        # found_entity = False\n",
    "        # for named_entity in entity_set:\n",
    "        #     if named_entity in col['subject']:\n",
    "        #         col['subject'] = named_entity\n",
    "        #         found_entity = True\n",
    "        #\n",
    "        # if found_entity:\n",
    "        final_triples.append(('Node', col['subject'], col['relation'], 'Node', col['object']))\n",
    "\n",
    "    triple_df = pd.DataFrame(final_triples, columns=['Type1', 'Entity1', 'Relationship', 'Type2', 'Entity2']).drop_duplicates()\n",
    "    return triple_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "SUBJECTS = [\"nsubj\", \"nsubjpass\", \"csubj\", \"csubjpass\", \"agent\", \"expl\"]\n",
    "OBJECTS = [\"dobj\", \"dative\", \"attr\", \"oprd\"]\n",
    "ATTRS = [\"acomp\"]\n",
    "POSS_NOUN = [\"NOUN\", \"PROPN\", \"X\"]\n",
    "\n",
    "\n",
    "def get_compounds(tokens):\n",
    "    compounds = []\n",
    "    lefts = list(tokens.lefts)\n",
    "    compounds.extend([tok for tok in lefts if tok.dep_ == 'compound'])\n",
    "    return compounds\n",
    "\n",
    "\n",
    "def get_subject_compound(v):\n",
    "    subs = []\n",
    "    compounds = []\n",
    "    for tok in v.lefts:\n",
    "        if tok.dep_ in SUBJECTS and tok.pos_ != \"DET\":\n",
    "            compounds = get_compounds(tok)\n",
    "            compounds.extend([tok])\n",
    "    if compounds:\n",
    "        subs.extend(compounds)\n",
    "    return subs\n",
    "\n",
    "\n",
    "def get_object_compound(v):\n",
    "    objs = []\n",
    "    compounds = []\n",
    "    for tok in v.rights:\n",
    "        if tok.dep_ in OBJECTS:\n",
    "            compounds = get_compounds(tok)\n",
    "            compounds.extend([tok])\n",
    "    if compounds:\n",
    "        objs.extend(compounds)\n",
    "    return objs\n",
    "\n",
    "\n",
    "def get_attribute_compound(av):\n",
    "    objs = []\n",
    "    compounds = []\n",
    "    for tok in av.rights:\n",
    "        if tok.dep_ in ATTRS:\n",
    "            compounds = get_compounds(tok)\n",
    "            compounds.extend([tok])\n",
    "    if compounds:\n",
    "        objs.extend(compounds)\n",
    "    return objs\n",
    "\n",
    "\n",
    "def get_prep_object_compound(p):\n",
    "    p_objs = []\n",
    "    compounds = []\n",
    "    for tok in p.rights:\n",
    "        if tok.dep_ == 'pobj':\n",
    "            compounds = get_compounds(tok)\n",
    "            compounds.extend([tok])\n",
    "    if compounds:\n",
    "        p_objs.extend(compounds)\n",
    "    return p_objs\n",
    "\n",
    "\n",
    "def get_prep(p):\n",
    "    prep = False\n",
    "    for tok in p.rights:\n",
    "        if tok.dep_ == 'prep':\n",
    "            prep = tok\n",
    "    return prep\n",
    "\n",
    "nlp = spacy.load('en_core_web_sm')\n",
    "def get_lexical_triplets_pairs(text_doc, verbose=False):\n",
    "\n",
    "    doc = nlp(text_doc)\n",
    "\n",
    "#     print([(ent.text, ent.label_) for ent in doc.ents])\n",
    "\n",
    "    triplets = []\n",
    "\n",
    "    verbs = [tok for tok in doc if tok.pos_ == \"VERB\"]\n",
    "\n",
    "    # getting sub, verb, obj triples\n",
    "    # Angela, visit, silver spoon restaurant\n",
    "    for v in verbs:\n",
    "        subs = get_subject_compound(v)\n",
    "        objs = get_object_compound(v)\n",
    "\n",
    "        if subs and objs:\n",
    "            triplets.append([' '.join(str(i) for i in subs), v, ' '.join(str(i) for i in objs)])\n",
    "\n",
    "    # getting sub, verb_prep, p_obj triples\n",
    "    # Angela, visit_with, Angela's friends\n",
    "    for v in verbs:\n",
    "        subs = get_subject_compound(v)\n",
    "        prep = get_prep(v)\n",
    "\n",
    "        p_objs = False\n",
    "        if prep:\n",
    "            p_objs = get_prep_object_compound(prep)\n",
    "\n",
    "        if subs and p_objs:\n",
    "            triplets.append([' '.join(str(i) for i in subs), str(v)+'_'+str(prep), ' '.join(str(i) for i in p_objs)])\n",
    "\n",
    "    # getting sub, possession, obj triples\n",
    "    # Silver spoon restaurant, has, Chicken biryani\n",
    "    subs, objs = (False, False)\n",
    "    poss_nouns = [tok for tok in doc if tok.pos_ in POSS_NOUN]\n",
    "    for n in poss_nouns:\n",
    "        children = list(n.children)\n",
    "        for child in children:\n",
    "            if child.dep_ == 'poss' and child.pos_ in POSS_NOUN:\n",
    "                compounds = get_compounds(child)\n",
    "                compounds.extend([child])\n",
    "                subs = compounds\n",
    "\n",
    "                compounds = get_compounds(n)\n",
    "                compounds.extend([n])\n",
    "                objs = compounds\n",
    "\n",
    "                if subs and objs:\n",
    "                    triplets.append([' '.join(str(i) for i in subs), 'has', ' '.join(str(i) for i in objs)])\n",
    "\n",
    "    # getting sub, aux, complementary object triples\n",
    "    # food, are, good\n",
    "    subs, objs = (False, False)\n",
    "    aux_verbs = [tok for tok in doc if tok.pos_ == \"AUX\"]\n",
    "    for av in aux_verbs:\n",
    "        subs = get_subject_compound(av)\n",
    "        objs = get_attribute_compound(av)\n",
    "\n",
    "        if subs and objs:\n",
    "            triplets.append([' '.join(str(i) for i in subs), av, ' '.join(str(i) for i in objs)])\n",
    "\n",
    "#     pprint(triplets)\n",
    "    return triplets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import BertModel, BertForTokenClassification, BertTokenizer\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "\n",
    "class MenuNER:\n",
    "    \n",
    "    def __init__(self):\n",
    "        self.model = BertForTokenClassification.from_pretrained(\n",
    "            \"/home/muzamil/Projects/Python/ML/NLP/NER/BERT-NER/out_ner/\")\n",
    "        self.tokenizer = BertTokenizer.from_pretrained(\n",
    "            \"/home/muzamil/Projects/Python/ML/NLP/NER/MenuNER/model/FoodieBERT/cased_L-12_H-768_A-12\")\n",
    "        \n",
    "    def extract_menu_ner(self, restaurants, write_results=False):\n",
    "\n",
    "        menu_ner = dict()\n",
    "        for restaurant in restaurants['restaurants']:\n",
    "            for review in restaurant['reviews']:\n",
    "\n",
    "                sequence = review['text']\n",
    "                predict = self._extract_menu_ner(self.model, self.tokenizer, sequence)\n",
    "                menu_ner.update(predict)\n",
    "#                 print(predict)\n",
    "\n",
    "#         print(menu_ner)\n",
    "        if write_results:\n",
    "               # opening the csv file in 'w' mode\n",
    "            open_menu_extracted_file = open(menu_extracted_file, 'w')\n",
    "            writer = csv.DictWriter(open_menu_extracted_file, fieldnames=entity_headers)\n",
    "\n",
    "            writer.writeheader()\n",
    "            for key, value in menu_ner.items():\n",
    "                writer.writerow({entity_headers[0]: key,\n",
    "                                 entity_headers[1]: value})\n",
    "        \n",
    "    \n",
    "    def extract_menu_ner_single(self, sequence):\n",
    "        menu_ner = dict()\n",
    "        predict = self._extract_menu_ner(self.model, self.tokenizer, sequence)\n",
    "        menu_ner.update(predict)\n",
    "        return predict\n",
    "                \n",
    "    @staticmethod\n",
    "    def _extract_menu_ner(model, tokenizer, sequence):\n",
    "\n",
    "        label_list = [\n",
    "            \"O\",       # Outside of a named entity\n",
    "            \"B-MENU\",  # Beginning of a menu entity\n",
    "            \"I-MENU\",  # menu entity\n",
    "        ]\n",
    "        \n",
    "        # Bit of a hack to get the tokens with the special tokens\n",
    "        tokens = tokenizer.tokenize(tokenizer.decode(tokenizer.encode(sequence, max_length=512, truncation=True)))\n",
    "        inputs = tokenizer.encode(sequence, return_tensors=\"pt\")\n",
    "\n",
    "        predict = {}\n",
    "        if inputs.size()[1] > 512:\n",
    "            return predict\n",
    "\n",
    "        outputs = model(inputs)[0]\n",
    "        predictions = torch.argmax(outputs, dim=2)\n",
    "\n",
    "        full_token = ''\n",
    "        \n",
    "        for token, prediction in zip(tokens, predictions[0].tolist()):\n",
    "            if token != '[CLS]' and token != '[SEP]':\n",
    "                if prediction > 3:\n",
    "                    continue\n",
    "                if label_list[prediction-1] in [\"B-MENU\", \"I-MENU\"]:\n",
    "                    if token.startswith('##'):\n",
    "                        full_token = full_token + token.replace(\"##\", \"\")\n",
    "                    else:\n",
    "                        if full_token:\n",
    "                            full_token = full_token +\" \"+token\n",
    "                        else:\n",
    "                            full_token = token\n",
    "                elif full_token:\n",
    "                    if token.startswith('##'):\n",
    "                        full_token = full_token + token.replace(\"##\", \"\")\n",
    "                    else:\n",
    "                        predict[full_token] = \"MENU\"\n",
    "                        full_token = ''\n",
    "\n",
    "        # Make first letter capitan and all small case for MenuNER\n",
    "        predict = dict((key.title(), value) for (key, value) in predict.items())\n",
    "        return predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'chicken biryani': 'MENU', 'butter chicken': 'MENU'}\n"
     ]
    }
   ],
   "source": [
    "# Load MenuNER model to extract Food entities\n",
    "from module.ner.menu_ner import MenuNER\n",
    "\n",
    "menu_ner = MenuNER()\n",
    "review_text = \"Nice place busy during lunch time. \" \\\n",
    "              \"The chicken biryani is super butter chicken is very authentic.\"\n",
    "result = menu_ner.extract_menu_ner_single(review_text)\n",
    "print(result)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1\n",
    "output_path = \"./data/output/\"\n",
    "ner_pickles_op = output_path + \"ner/\"\n",
    "cr_pickles_op = output_path + \"cr/\"\n",
    "\n",
    "''' read restaurant knowledgebase data '''\n",
    "knowledge_base_entities = r\"data/kb/entities/\"\n",
    "kb_restaurant_file = knowledge_base_entities + \"restaurant.csv\"\n",
    "kb_menu_file = knowledge_base_entities + \"menu.csv\"\n",
    "kb_menu_file1 = knowledge_base_entities + \"menu1.csv\"\n",
    "kb_general_file = knowledge_base_entities + \"general.csv\"\n",
    "kb_restaurant_aspects_file = knowledge_base_entities + \"restaurant_aspects.csv\"\n",
    "kb_menu_aspects_file = knowledge_base_entities + \"menu_attrs.csv\"\n",
    "\n",
    "entity_headers = ['Name', 'Label']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 2\n",
    "''' Add restaurants to `kb_restaurant_file` '''\n",
    "\n",
    "data = []\n",
    "for restaurant in restaurant_data['restaurants']:\n",
    "    data.append((restaurant['name'], 'RESTAURANT'))\n",
    "\n",
    "rest_data_df = pd.DataFrame(data, columns=entity_headers).drop_duplicates()\n",
    "rest_data_df.to_csv(kb_restaurant_file, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 3\n",
    "''' Fixed some MENU entiies manually '''\n",
    "\n",
    "''' Remove Duplicate entities from MENU file '''\n",
    "df = pd.read_csv(kb_menu_file, sep=\",\")\n",
    "df.drop_duplicates(subset=None, inplace=True)\n",
    "df.to_csv(kb_menu_file, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 4\n",
    "''' make dataframes for kb '''\n",
    "kb_restaurant_df = pd.read_csv(kb_restaurant_file, header=0, names=entity_headers)\n",
    "kb_menu_df = pd.read_csv(kb_menu_file, header=0, names=entity_headers)\n",
    "# kb_general_df = pd.read_csv(kb_general_file, header=0, names=['Name', 'Label'])\n",
    "kb_restaurant_aspects_df = pd.read_csv(kb_restaurant_aspects_file, header=0, names=entity_headers)\n",
    "kb_menu_aspects_df = pd.read_csv(kb_menu_aspects_file, header=0, names=entity_headers)\n",
    "# print(kb_restaurant_aspects_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "''' Insert Menus in Neo4J database '''\n",
    "graph = GraphDB(\"bolt://localhost:7687\", \"neo4j\", \"erclab\")\n",
    "\n",
    "kb_menu_df_distinct = kb_menu_df.drop_duplicates()\n",
    "\n",
    "for i in kb_menu_df_distinct.index:\n",
    "    \n",
    "    menu_name = kb_menu_df_distinct['Name'][i]\n",
    "    menu_id_arr = menu_name.lower().split()\n",
    "    menu_id = \"_\".join(id for id in menu_id_arr)\n",
    "\n",
    "    # Inserting menus\n",
    "    graph.create_menu(menu_id, menu_name)\n",
    "\n",
    "graph.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "''' Insert Aspects in Neo4J database '''\n",
    "graph = GraphDB(\"bolt://localhost:7687\", \"neo4j\", \"erclab\")\n",
    "\n",
    "kb_restaurant_aspects_df_distinct = kb_restaurant_aspects_df.drop_duplicates()\n",
    "\n",
    "for i in kb_restaurant_aspects_df_distinct.index:\n",
    "    \n",
    "    aspect_name = kb_restaurant_aspects_df_distinct['Name'][i]\n",
    "    aspect_id_arr = aspect_name.lower().split()\n",
    "    aspect_id = \"_\".join(id for id in aspect_id_arr)\n",
    "\n",
    "    # Inserting menus\n",
    "    graph.create_aspect(aspect_id, aspect_name)\n",
    "\n",
    "graph.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# 5\n",
    "from module.processor.text_processor import clean_reviews\n",
    "# Clean reviews text\n",
    "clean_reviews(restaurant_data['restaurants'], save_clean_data=True)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 6\n",
    "''' open cleaned restaurant dataset '''\n",
    "restaurant_cleaned_reviews_file = r\"data/input/cleaned_reviews.json\"\n",
    "file = open(restaurant_cleaned_reviews_file,)\n",
    "restaurant_cleaned_reviews = json.load(file)\n",
    "file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 7\n",
    "''' coreference resolution '''\n",
    "allen_cr = AllenCR(True)\n",
    "predictor, nlp = allen_cr.load_models()\n",
    "\n",
    "rest_index=0\n",
    "rest_count = len(restaurant_cleaned_reviews['restaurants'])\n",
    "for restaurant in restaurant_cleaned_reviews['restaurants']:\n",
    "\n",
    "    rest_index += 1\n",
    "    print(\"Processing Restaurant \"+str(rest_index)+\"/\"+str(rest_count))\n",
    "    \n",
    "    reviews = restaurant['reviews']\n",
    "    rev_index=0\n",
    "    rev_count = len(reviews)\n",
    "    for review in reviews:\n",
    "        \n",
    "        rev_index +=1\n",
    "        print(\"\\t Processing Review \"+str(rev_index)+\"/\"+str(rev_count))\n",
    "        \n",
    "        dummy_sentence = review[\"name\"] + \" visit \" + restaurant[\"name\"] + \".\"\n",
    "        doc = dummy_sentence + \" \" + review[\"text\"]\n",
    "#         print(review[\"text\"])\n",
    "        clusters = predictor.predict(doc)['clusters']\n",
    "        nlp_doc = nlp(doc)\n",
    "        coref_resolved = allen_cr.improved_replace_corefs(nlp_doc, clusters)\n",
    "        \n",
    "        ## split doc into sentences and remove first sentence\n",
    "        #     # nlp_small.add_pipe(nlp_small.create_pipe('sentencizer'))\n",
    "        nlp_doc = nlp(coref_resolved)\n",
    "        sentences = [sent.string.strip() for sent in nlp_doc.sents]\n",
    "        ## remove dumy_sentence\n",
    "        sentences.pop(0)\n",
    "        rev = \" \".join([sent for sent in sentences])\n",
    "        review[\"text\"] = rev\n",
    "        \n",
    "#         print('resolved')\n",
    "#         print(review['text'])\n",
    "\n",
    "# print(restaurant_cleaned_reviews['restaurants'][0]['reviews'][0]['text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 8\n",
    "rest_obj = dict()\n",
    "rest_obj['restaurants'] = restaurant_cleaned_reviews['restaurants']\n",
    "with open('data/input/coref_resolved_reviews.json', 'w') as outfile:\n",
    "    json.dump(rest_obj, outfile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 9\n",
    "''' open coref_resolved_reviews restaurant dataset '''\n",
    "restaurant_coref_resolved_reviews_file = r\"data/input/data_100/coref_resolved_reviews.json\"\n",
    "file = open(restaurant_coref_resolved_reviews_file,)\n",
    "restaurant_cleaned_reviews = json.load(file)\n",
    "file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "kb_restaurant_dict = {}\n",
    "kb_restaurant_aspects_dict = {}\n",
    "kb_menu_dict = {}\n",
    "# kb_menu_aspects_dict = {}\n",
    "kb_general_dict = {}\n",
    "\n",
    "''' read restaurant NER dictionary '''\n",
    "with open(kb_restaurant_file, mode='r') as infile:\n",
    "    reader = csv.reader(infile)\n",
    "    i = 0\n",
    "    for rows in reader:\n",
    "        if i == 0:\n",
    "            i += 1\n",
    "            continue\n",
    "        kb_restaurant_dict[rows[0]] = rows[1]\n",
    "\n",
    "''' read restaurant aspects NER dictionary '''\n",
    "with open(kb_restaurant_aspects_file, mode='r') as infile:\n",
    "    reader = csv.reader(infile)\n",
    "    i = 0\n",
    "    for rows in reader:\n",
    "        if i == 0:\n",
    "            i += 1\n",
    "            continue\n",
    "        kb_restaurant_aspects_dict[rows[0]] = rows[1]\n",
    "\n",
    "''' read menu NER dictionary '''\n",
    "with open(kb_menu_file, mode='r') as infile:\n",
    "    reader = csv.reader(infile)\n",
    "    i = 0\n",
    "    for rows in reader:\n",
    "        if i == 0:\n",
    "            i += 1\n",
    "            continue\n",
    "        kb_menu_dict[rows[0]] = rows[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "outputs": [],
   "source": [
    "# 10\n",
    "# from module.processor.relation_processor import TripletProcessor\n",
    "from module.processor.relation_processor import TripletProcessor\n",
    "triplet_processor = TripletProcessor()\n",
    "triplet_processor.init_kb_dict(kb_restaurant_file, kb_restaurant_aspects_file,  kb_menu_file)\n",
    "triplet_processor.init_lexical_analyzer()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lexicon_file_path = 'data/opinion_lexicon-en/'\n",
    "\n",
    "def load_opinion_lexicon():\n",
    "    # Load opinion lexicon\n",
    "    neg_file = open(lexicon_file_path + \"negative-words.txt\", encoding=\"ISO-8859-1\")\n",
    "    pos_file = open(lexicon_file_path + \"positive-words.txt\", encoding=\"ISO-8859-1\")\n",
    "    neg = [line.strip() for line in neg_file.readlines()]\n",
    "    pos = [line.strip() for line in pos_file.readlines()]\n",
    "    opinion_words = neg + pos\n",
    "    return opinion_words, pos, neg\n",
    "\n",
    "opinion_words, pos, neg = load_opinion_lexicon()\n",
    "\n",
    "def process_triple_pruning(triples, rest_id, review, ner_dict):\n",
    "    # entity_set = set(ner_dict.keys())\n",
    "    final_triples = []\n",
    "\n",
    "    for row, col in triples.iterrows():\n",
    "        col['subject'] = col['subject'].strip()\n",
    "        col['object'] = col['object'].strip()\n",
    "\n",
    "        ''' Fixing Misspelling Menus '''\n",
    "        valid_menu = False\n",
    "        menu_subject = False\n",
    "        for key, value in kb_menu_dict.items():\n",
    "            if key.lower() == col['subject'].lower():\n",
    "                valid_menu = True\n",
    "                menu_subject = True\n",
    "                break\n",
    "                \n",
    "        if not valid_menu:\n",
    "            for key, value in kb_menu_dict.items():\n",
    "                subject_ratio = levenshtein_ratio_and_distance(key.lower(), col['subject'].lower(), \n",
    "                                                               ratio_calc=True)\n",
    "                if subject_ratio > 0.85:\n",
    "                    entity_word = menu_ner.extract_menu_ner_single(\n",
    "                        \"I like \"+col['subject'].lower()+\"from this restaurant.\")\n",
    "                    correction_word = menu_ner.extract_menu_ner_single(\n",
    "                        \"I like \"+key.lower()+\"from this restaurant.\")\n",
    "    #                 key_id = \"_\".join(k for k in key.split())\n",
    "                    if entity_word and correction_word:\n",
    "                        print(\"updating subject [\"+col['subject']+\"] with [\"+key+\"]\")\n",
    "                        col['subject'] = key\n",
    "                        menu_subject = True\n",
    "                        break\n",
    "        \n",
    "        valid_menu = False\n",
    "        menu_object = False\n",
    "        for key, value in kb_menu_dict.items():\n",
    "            if key.lower() == col['object'].lower():\n",
    "                valid_menu = True\n",
    "                menu_object = True\n",
    "                break\n",
    "\n",
    "        if not valid_menu:\n",
    "            for key, value in kb_menu_dict.items():\n",
    "                object_ratio  = levenshtein_ratio_and_distance(key.lower(), col['object'].lower(), ratio_calc=True)\n",
    "                if object_ratio > 0.85:\n",
    "                    entity_word = menu_ner.extract_menu_ner_single(\n",
    "                        \"I like \"+col['object'].lower()+\"from this restaurant.\")\n",
    "                    correction_word = menu_ner.extract_menu_ner_single(\n",
    "                        \"I like \"+key.lower()+\"from this restaurant.\")\n",
    "                    \n",
    "                    if entity_word and correction_word:\n",
    "                        print(\"updating object [\" + col['object'] + \"] with [\" + key + \"]\")\n",
    "                        col['object'] = key\n",
    "                        menu_object = True\n",
    "                        break\n",
    "        \n",
    "        ''' check if subject is a valid entity '''\n",
    "        valid_subject = False\n",
    "        \n",
    "        if menu_subject:\n",
    "            menu_id = \"_\".join(k for k in col['subject'].split())\n",
    "            col['subject'] = menu_id\n",
    "            valid_subject = True\n",
    "            \n",
    "        restaurant_subject = False\n",
    "        if not menu_subject:\n",
    "            for key, value in kb_restaurant_dict.items():\n",
    "                if key.lower() == col['subject'].lower():\n",
    "                    col['subject'] = rest_id\n",
    "                    restaurant_subject = True\n",
    "                    valid_subject = True\n",
    "                    break\n",
    "\n",
    "#         menu_subject = False\n",
    "#         if not restaurant_subject:\n",
    "#             for key, value in kb_menu_dict.items():\n",
    "#                 if key.lower() == col['subject'].lower():\n",
    "#                     menu_id = \"_\".join(k for k in col['subject'].split())\n",
    "#                     col['subject'] = menu_id\n",
    "#                     menu_subject = True\n",
    "#                     valid_subject = True\n",
    "#                     break\n",
    "        \n",
    "        user_subject = False\n",
    "        if not restaurant_subject:\n",
    "            if review['name'].lower() == col['subject'].lower():\n",
    "                col['subject'] = review['user_id']\n",
    "                user_subject = True\n",
    "                valid_subject = True\n",
    "\n",
    "                    \n",
    "        general_subject = False\n",
    "        if not user_subject:\n",
    "            for key, value in ner_dict.items():\n",
    "                if key.lower() == col['subject'].lower():\n",
    "                    general_subject = True\n",
    "                    valid_subject = True\n",
    "                    break\n",
    "\n",
    "        res_aspect_subject = False\n",
    "        if not general_subject:\n",
    "            for key, value in kb_restaurant_aspects_dict.items():\n",
    "                if key.lower() == col['subject'].lower():\n",
    "                    aspect_id = \"_\".join(k for k in col['subject'].split())\n",
    "                    col['subject'] = aspect_id\n",
    "                    res_aspect_subject = True\n",
    "                    valid_subject = True\n",
    "                    break\n",
    "\n",
    "        # menu_aspect_subject = False\n",
    "        # if not general_subject:\n",
    "        #     for key, value in kb_menu_aspects_dict.items():\n",
    "        #         if key.lower() == col['subject'].lower():\n",
    "        #             menu_aspect_subject = True\n",
    "        #             valid_subject = True\n",
    "        #             break\n",
    "\n",
    "        ''' check if object is a valid entity '''\n",
    "        valid_object = False\n",
    "        \n",
    "        if menu_object:\n",
    "            menu_id = \"_\".join(k for k in col['object'].split())\n",
    "            col['object'] = menu_id\n",
    "            valid_object = True\n",
    "            \n",
    "        restaurant_object = False\n",
    "        if not menu_object:\n",
    "            for key, value in kb_restaurant_dict.items():\n",
    "                if key.lower() == col['object'].lower():\n",
    "                    col['object'] = rest_id\n",
    "                    restaurant_object = True\n",
    "                    valid_object = True\n",
    "                    break\n",
    "\n",
    "#         menu_object = False\n",
    "#         if not restaurant_object:\n",
    "#             for key, value in kb_menu_dict.items():\n",
    "#                 if key.lower() == col['object'].lower():\n",
    "#                     menu_id = \"_\".join(k for k in col['object'].split())\n",
    "#                     col['object'] = menu_id\n",
    "#                     menu_object = True\n",
    "#                     valid_object = True\n",
    "#                     break\n",
    "\n",
    "        user_object = False\n",
    "        if not restaurant_object:\n",
    "            if review['name'].lower() == col['object'].lower():\n",
    "                col['object'] = review['user_id']\n",
    "                user_object = True\n",
    "                valid_object = True\n",
    "                \n",
    "        general_object = False\n",
    "        if not user_object:\n",
    "            for key, value in ner_dict.items():\n",
    "                if key.lower() == col['object'].lower():\n",
    "                    general_object = True\n",
    "                    valid_object = True\n",
    "                    break\n",
    "\n",
    "        res_aspect_object = False\n",
    "        if not general_object:\n",
    "            for key, value in kb_restaurant_aspects_dict.items():\n",
    "                if key.lower() == col['object'].lower():\n",
    "                    aspect_id = \"_\".join(k for k in col['object'].split())\n",
    "                    col['object'] = aspect_id\n",
    "                    res_aspect_object = True\n",
    "                    valid_object = True\n",
    "                    break\n",
    "\n",
    "        attr_obj = False\n",
    "        if not res_aspect_object:\n",
    "#             opinion_words, pos, neg = load_opinion_lexicon()\n",
    "            if col['object'].lower() in opinion_words:\n",
    "                attr_obj = True\n",
    "                valid_object = True\n",
    "\n",
    "#         print(col['subject']+\" : \"+col['object']+\" : \"+str(valid_subject)+\" : \"+str(valid_object))\n",
    "        if valid_subject and valid_object:\n",
    "            if menu_subject:\n",
    "                final_triples.append(('Node', rest_id, 'has_menu', 'Node', col['subject']))\n",
    "            if restaurant_subject and menu_object:\n",
    "                col['relation'] = 'has_menu'\n",
    "            if res_aspect_subject and (res_aspect_object or attr_obj):\n",
    "                col['relation'] = 'is'\n",
    "            final_triples.append(('Node', col['subject'], col['relation'], 'Node', col['object']))\n",
    "\n",
    "    triple_df = pd.DataFrame(final_triples, columns=['Type1', 'Entity1', 'Relationship', 'Type2', 'Entity2'])\\\n",
    "        .drop_duplicates()\n",
    "    return triple_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "           subject relation     object\n",
      "0  chicken biryani       is      super\n",
      "1   butter chicken       is  authentic\n"
     ]
    }
   ],
   "source": [
    "import spacy\n",
    "from module.ner.menu_ner import MenuNER\n",
    "from module.lexicon.lexical_analyzer import SpacyLexicalAnalyzer\n",
    "\n",
    "menu_ner = MenuNER()\n",
    "spacy_lexical_analyzer = SpacyLexicalAnalyzer()\n",
    "spacy_lexical_analyzer.load_spacy_models(\"small\")\n",
    "nlp = spacy.load('en_core_web_sm')\n",
    "\n",
    "ner_dict = {}\n",
    "\n",
    "review_text = \"Nice place busy during lunch time. \" \\\n",
    "              \"The chicken biryani is super, butter chicken is very authentic.\"\n",
    "doc = nlp(review_text)\n",
    "\n",
    "for x in doc.ents:\n",
    "    entity_span = x.text\n",
    "\n",
    "    has_restaurant_entity = False\n",
    "    i = 0\n",
    "    for kb_restaurant in kb_restaurant_df['Name']:\n",
    "        ratio = levenshtein_ratio_and_distance(kb_restaurant.lower(), entity_span.lower(), ratio_calc=True)\n",
    "        if ratio > 0.90:\n",
    "            has_restaurant_entity = True\n",
    "        if has_restaurant_entity:\n",
    "            break\n",
    "        i += 1\n",
    "\n",
    "    if has_restaurant_entity:\n",
    "        continue\n",
    "\n",
    "    if x.label_ not in [\"CARDINAL\", \"ORDINAL\"]:\n",
    "        ner_dict[x.text] = x.label_\n",
    "\n",
    "text = review_text\n",
    "tuple_pairs = spacy_lexical_analyzer.get_lexical_triplets_pairs(text)\n",
    "tuple_pairs_df = pd.DataFrame(tuple_pairs, columns=['subject', 'relation', 'object'])\n",
    "print(tuple_pairs_df)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import spacy\n",
    "from module.ner.menu_ner import MenuNER\n",
    "from module.lexicon.lexical_analyzer import SpacyLexicalAnalyzer\n",
    "from module.neo4j.graph_db import GraphDB\n",
    "\n",
    "# graph = GraphDB(\"bolt://localhost:7687\", \"neo4j\", \"erclab\")\n",
    "# graph = GraphDB(\"bolt://localhost:11012\", \"neo4j\", \"erclab\")\n",
    "\n",
    "menu_ner = MenuNER()\n",
    "spacy_lexical_analyzer = SpacyLexicalAnalyzer()\n",
    "spacy_lexical_analyzer.load_spacy_models(\"small\")\n",
    "triples_df = pd.DataFrame()\n",
    "nlp = spacy.load('en_core_web_sm')\n",
    "\n",
    "# Making data for GNN attribute based node embeddings\n",
    "user_menus_for_data = []\n",
    "restaurant_menu_aspect_for_data = []\n",
    "\n",
    "# Make list of dictionary for joint relation extraction\n",
    "joint_relations = []\n",
    "\n",
    "rest_index=0\n",
    "rest_count = len(restaurant_cleaned_reviews['restaurants'])\n",
    "for restaurant in restaurant_cleaned_reviews['restaurants']:\n",
    "\n",
    "    single_restaurant_menus = []\n",
    "    single_restaurant_aspects = []\n",
    "\n",
    "    rest_index += 1\n",
    "    # if rest_index == 2:\n",
    "    #     break\n",
    "    print(\"Processing Restaurant \"+str(rest_index)+\"/\"+str(rest_count))\n",
    "\n",
    "    reviews = restaurant['reviews']\n",
    "\n",
    "    rev_index=0\n",
    "    rev_count = len(reviews)\n",
    "    for review in reviews:\n",
    "\n",
    "        rev_index +=1\n",
    "        print(\"\\t Processing Review \"+str(rev_index)+\"/\"+str(rev_count))\n",
    "\n",
    "#         if rev_index < 6:\n",
    "#             continue\n",
    "\n",
    "        ner_dict = {}\n",
    "        doc = nlp(review[\"text\"])\n",
    "\n",
    "        for x in doc.ents:\n",
    "            entity_span = x.text\n",
    "\n",
    "            has_restaurant_entity = False\n",
    "            i = 0\n",
    "            for kb_restaurant in kb_restaurant_df['Name']:\n",
    "                ratio = levenshtein_ratio_and_distance(kb_restaurant.lower(), entity_span.lower(), ratio_calc=True)\n",
    "                if ratio > 0.90:\n",
    "                    has_restaurant_entity = True\n",
    "                if has_restaurant_entity:\n",
    "                    break\n",
    "                i += 1\n",
    "\n",
    "            if has_restaurant_entity:\n",
    "                continue\n",
    "\n",
    "            # has_menu_entity = False\n",
    "            # i = 0\n",
    "            # for kb_menu in kb_menu_df['Name']:\n",
    "            #     ratio = levenshtein_ratio_and_distance(kb_menu.lower(), entity_span.lower(), ratio_calc=True)\n",
    "            #     # print(menu.lower(), entity_span.lower(), str(ratio))\n",
    "            #     if ratio > 0.90:\n",
    "            #         has_menu_entity = True\n",
    "            #     if has_menu_entity:\n",
    "            #         break\n",
    "            #     i += 1\n",
    "            #\n",
    "            # if has_menu_entity:\n",
    "            #     continue\n",
    "\n",
    "            if x.label_ not in [\"CARDINAL\", \"ORDINAL\"]:\n",
    "                ner_dict[x.text] = x.label_\n",
    "\n",
    "        text = review[\"text\"]\n",
    "        tuple_pairs = spacy_lexical_analyzer.get_lexical_triplets_pairs(text)\n",
    "        tuple_pairs_df = pd.DataFrame(tuple_pairs, columns=['subject', 'relation', 'object'])\n",
    "        # pairs = list(set(tuple(sub) for sub in tuple_pairs))\n",
    "        rest = {\"rest_id\": restaurant[\"rest_id\"], \"name\": restaurant[\"name\"]}\n",
    "        tuple_pairs_prune, pro_rev, _menus, _aspects = triplet_processor.process_triple_pruning(menu_ner,\n",
    "                                                                    tuple_pairs_df, rest, review, ner_dict,\n",
    "                                                                    store_in_db=False,  graph_db=None)\n",
    "        triples_df = pd.concat([triples_df, tuple_pairs_prune])\n",
    "\n",
    "        # print(\"_menus: \"+str(len(_menus))+\" \"+str(len(_aspects)))\n",
    "        user_menus_for_data.append([pro_rev[\"user_id\"], \"|\".join(menu for menu in _menus)])\n",
    "        single_restaurant_menus.extend(_menus)\n",
    "        single_restaurant_aspects.extend(_aspects)\n",
    "\n",
    "        joint_relation = {\"sentText\": str(restaurant['name'])+\", \"+pro_rev['name']+\", \"+pro_rev['text'],\n",
    "                                \"relationMentions\": [ {\"em1Text\":triple[\"sub_ent\"],\n",
    "                                                       \"em2Text\":triple[\"obj_ent\"],\n",
    "                                                       \"label\":triple[\"relation\"]}\n",
    "                                                      for index, triple in tuple_pairs_prune.iterrows()\n",
    "                                                      if triple['sub_ent'] and triple['obj_ent']]}\n",
    "        if joint_relation['relationMentions']:\n",
    "            joint_relations.append(joint_relation)\n",
    "        # if rev_index == 7:\n",
    "        #     break\n",
    "\n",
    "    # print(\"single_restaurant_menus: \"+str(len(single_restaurant_menus))+\" \"+str(len(single_restaurant_aspects)))\n",
    "    single_restaurant_menus = list(set(single_restaurant_menus))\n",
    "    single_restaurant_aspects = list(set(single_restaurant_aspects))\n",
    "    # print(\"single_restaurant_menus: \"+str(len(single_restaurant_menus))+\" \"+str(len(single_restaurant_aspects)))\n",
    "    restaurant_menu_aspect_for_data.append([restaurant[\"rest_id\"], \"|\".join(menu for menu in single_restaurant_menus),\n",
    "                                \"|\".join(aspect for aspect in single_restaurant_aspects)])\n",
    "    # print(\"single_restaurant_menus: \"+str(len(joint_relations)))\n",
    "    # print(joint_relations)\n",
    "\n",
    "# {\n",
    "#     \"sentText\": \"restaurant_name, reviewer, review\"\n",
    "#     \"relationMentions\": [\n",
    "#     {\n",
    "#         \"em1Text\": \"Bananaman\",\n",
    "#         \"em2Text\": \"Brooke-Taylor\",\n",
    "#         \"label\": \"starring\"\n",
    "#     },\n",
    "#     {\"em1Text\": \"Bananaman\",\n",
    "#      \"em2Text\": \"Bright\",\n",
    "#      \"label\": \"creator\"\n",
    "#      }\n",
    "#     ]\n",
    "# }\n",
    "print(triples_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "triples_df.to_csv(\"data/output/kg/input_data.txt-out2.csv\", index=False)\n",
    "\n",
    "user_menus_for_data_df = pd.DataFrame(user_menus_for_data, columns=['user_id', 'menus'])\n",
    "user_menus_for_data_df.to_csv(\"data/output/kg/user_menu.csv\", index=False)\n",
    "\n",
    "restaurant_menu_aspect_for_data_df = pd.DataFrame(restaurant_menu_aspect_for_data, columns=['rest_id', 'menus',\n",
    "                                                                                            'aspects'])\n",
    "restaurant_menu_aspect_for_data_df.to_csv(\"data/output/kg/restaurant_menu_aspect.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "from pprint import pprint\n",
    "pprint(joint_relations)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "import json\n",
    "with open(\"data/output/kg/joint_relation_100.json\", 'w') as joint_relation_file:\n",
    "    json.dump(joint_relations, joint_relation_file)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "'''[{'sentText': 'Silver Spoon, Violet, The food is alright  not that great. Chicken briyani is very dry. '\n",
    "              'Just went there once.',\n",
    "  'relationMentions':\n",
    "    [\n",
    "        {'em1Text': 'Silver Spoon', 'em2Text': 'Food', 'label': 'HAS_ASPECT'},\n",
    "        {'em1Text': 'Great', 'em2Text': 'Silver Spoon', 'label': 'ASPECT_ATTR_FOR'},\n",
    "        {'em1Text': 'Food', 'em2Text': 'Great', 'label': 'IS'},\n",
    "        {'em1Text': 'Silver Spoon', 'em2Text': 'Chicken Biryani', 'label': 'HAS_MENU'},\n",
    "        {'em1Text': 'Dry', 'em2Text': 'Silver Spoon', 'label': 'MENU_ATTR_FOR'},\n",
    "        {'em1Text': 'Chicken Biryani', 'em2Text': 'Dry', 'label': 'IS'}\n",
    "    ]\n",
    "  },\n",
    " {'sentText': 'Silver Spoon, Marie, Marie have  really tried to like Silver Spoon but Silver Spoon disappoints '\n",
    "              'me more and more each time Marie visit. At first  the portions were generous and the food was '\n",
    "              'delicious. Today was the last time Marie will  be visiting  though. It was the third time Marie '\n",
    "              'was given three small bits of butter chicken in a combo when previous visits Marie was given '\n",
    "              'two heaping scoops. When my husband and Marie pointed it out to the server and asked for an '\n",
    "              'actual serving the server spooned three more pieces in but then disappeared into the back '\n",
    "              'with the container. the server reemerged a minute or so later and told my husband and Marie '\n",
    "              'the server had to check if it was ok. Marie get it if Marie were asking for a huge amount but '\n",
    "              'Marie were not. Just the same amount given to my husband and Marie when asking for the '\n",
    "              'vegetable curry. The thing that bugs me the most is that Marie have no clue what actually '\n",
    "              'happened to my container when the container was taken out of my sight. Wo not  be returning.',\n",
    "  'relationMentions':\n",
    "      [\n",
    "          {'em1Text': 'Silver Spoon', 'em2Text': 'Food', 'label': 'HAS_ASPECT'},\n",
    "          {'em1Text': 'Delicious', 'em2Text': 'Silver Spoon', 'label': 'ASPECT_ATTR_FOR'},\n",
    "          {'em1Text': 'Food', 'em2Text': 'Delicious', 'label': 'IS'}\n",
    "      ]\n",
    "  },\n",
    " {'sentText': \"Silver Spoon, Priyam, Silver Spoon is ideal for days you simply want to take out good  \"\n",
    "              \"fast  and already made food. Silver Spoon have a wide variety of Indian dishes \"\n",
    "              \"consisting of chicken  beef  goat  fish and vegetarian. Priyam especially like \"\n",
    "              \"Silver Spoon's Biryani  chicken and rice  special. their Biryani  chicken and rice  \"\n",
    "              \"special's delicious food at a good price. The food is laid out like a buffet for you \"\n",
    "              \"to see and you simply ask the server for the dishes you want. Personally Priyam like \"\n",
    "              \"taking out the meat dishes from here and eating with rice or rotis that Priyam already \"\n",
    "              \"have at home. Priyam's favourite dishes are Achari chicken  the goat  the fish pakoras \"\n",
    "              \"and the biryani. Silver Spoon is well maintained and clean making Silver Spoon more appealing.\",\n",
    "  'relationMentions':\n",
    "      [\n",
    "          {'em1Text': 'Silver Spoon', 'em2Text': 'Chicken', 'label': 'HAS_MENU'},\n",
    "          {'em1Text': 'Silver Spoon', 'em2Text': 'Ideal', 'label': 'IS'}\n",
    "      ]\n",
    "  },\n",
    " {'sentText': \"Silver Spoon, John, John were initially introduced to the food from Silver Spoon from a \"\n",
    "              \"group lunch event at work. John have been regulars ever since. Silver Spoon is a takeout \"\n",
    "              \"restaurant with a great selection of hot foods from Silver Spoon's display window. So no \"\n",
    "              \"waiting. Everything is fresh and has an authentic home made flavour. The rice is long grain \"\n",
    "              \"and light. The chicken pieces are tasty. The sauces are rich and tasty. John have always \"\n",
    "              \"found the staff friendly and helpful and willing to explain the various dishes. When you \"\n",
    "              \"get into your car with your food the smells are so delicious you cannot be sure you can get \"\n",
    "              \"home without trying some. Highly recommended.\",\n",
    "  'relationMentions':\n",
    "      [\n",
    "          {'em1Text': 'Silver Spoon', 'em2Text': 'Chicken Pieces', 'label': 'HAS_MENU'},\n",
    "          {'em1Text': 'Tasty', 'em2Text': 'Silver Spoon', 'label': 'MENU_ATTR_FOR'},\n",
    "          {'em1Text': 'Chicken Pieces', 'em2Text': 'Tasty', 'label': 'IS'},\n",
    "          {'em1Text': 'Silver Spoon', 'em2Text': 'Sauce', 'label': 'HAS_MENU'},\n",
    "          {'em1Text': 'Rich', 'em2Text': 'Silver Spoon', 'label': 'MENU_ATTR_FOR'},\n",
    "          {'em1Text': 'Sauce', 'em2Text': 'Rich', 'label': 'IS'}\n",
    "      ]\n",
    "  },\n",
    " {'sentText': 'Silver Spoon, Katya, Katya have  been here many times  and had varied experiences  but overall  Katya am a fan of Silver Spoon. Even though Silver Spoon have expanded to another location in Scarborough  so that there are now two Silver Spoons the Pickering location has managed to maintain consistent quality  although consistent quality can vary on days. Well  what restaurant does not  have that problem You might be suddenly short - staffed  someone down with the flu   or perhaps one of your deliveries did not  make it on time It is good food. Katya prefer the Pakistani cuisine :  Nihari Haleem not always available  and several of the biryanis. Silver Spoon are now offering Sindhi Biryani on weekends  and Katya m so pleased ! The only way that Katya could get Sindhi Biryani in the past was to make Sindhi Biryani Katya using a packaged spice blend. If you want to get Naan  Katya recommend the Butter Naan. Although the Butter Naan costs a bit more the butter enhancements really take Naan to the next level of browning and savoury tastes. Ok  things sit under the heat lamp but things do not seem to suffer that much  particularly something like Nihari  which is a stew - type dish that can withstand the warming table. Everything has always tasted fresh when Katya have ordered it. Katya find that the samosas are fairly standard  but there are other appetizers which are more exciting. For people who like the dark meat of the chicken Lollypop Chicken is great and consistently good. Katya also enjoy Shami Kabab. If you like fish  do not  skip the Fingerfish  which is consistently tasty. Whenever Katya have been there  which is perhaps once a month or two  for the past five years the portions have been generous. Katya am certainly a fan  and Silver Spoon fills an important niche in the local  Pickering community as a convenient place to pick up some Halal take - away of decent quality.', 'relationMentions': [{'em1Text': 'Silver Spoon', 'em2Text': 'Sindhi Biryani', 'label': 'HAS_MENU'}, {'em1Text': 'Silver Spoon', 'em2Text': 'Shami Kebab', 'label': 'HAS_MENU'}, {'em1Text': 'Katya', 'em2Text': 'Shami Kebab', 'label': 'ORDER'}, {'em1Text': 'Silver Spoon', 'em2Text': 'Lollypop Chicken', 'label': 'HAS_MENU'}, {'em1Text': 'Great', 'em2Text': 'Silver Spoon', 'label': 'MENU_ATTR_FOR'}, {'em1Text': 'Lollypop Chicken', 'em2Text': 'Great', 'label': 'IS'}]}, {'sentText': \"Silver Spoon, Heli, Oh  Silver Spoon. Heli have  tried to love Silver Spoon. Heli have  given Silver Spoon so many chances. But this just is not  working out anymore. To start :  the food is not  exactly terrible. the food just...not very good. Heli have  tried almost every meat dish  the chana masala  samosas  biriyanis  etc. Indian take - out is hardly known for Indian take - out's healthiness but when you order a veal karahi and get two sad chunks of meat floating in half a container in oil it is  a little disheartening. Add to that the fact that food is often sitting under a heat lamp for God knows how long... On the upside :  the chicken biriyani is good bang for your buck  -  especially during the week when the chicken biriyani is  on special. You get a fairly flavourful container packed with rice and meat. a fairly flavourful container packed with rice and meat  ready to go so you rarely have to wait more than a few minutes for your order. There was that one time they ran out of chicken biriyani  which is understandable when the chicken biriyani is  on special  but then offered me a  regular biriyani read : JUST rice  for the same price. Ummm  say what   Heli do love their beef samosas -  it is  just to bad that they NEVER HAVE ANY. Seriously  Heli have been here early in the evening  at random times during the week  and they have loads of sad looking veggie samosas  but no beef. Heli feel like the owners need to do a serious evaluation of what works and does not  work. If you notice you are  always selling out of a certain item  and NOT another   does not  it make sense to up a certain item while downsizing another Who knows. My biggest gripe was the chana masala incident. The last time Heli went in  Heli ordered a chana masala  which Heli m apathetic about  but my husband likes . The girl at the counter kind of made a face and said Oh... no. Heli do not  want a chana masala  which Heli m apathetic about but my husband likes. Heli was confused. There was a whole tray of a chana masala  which Heli m apathetic about but my husband likes there  brimming with chick - pea goodness. a chana masala  which Heli m apathetic about but my husband likes not... good today. What What does that mean  Is the taste just  off  or had a chana masala  which Heli m apathetic about but my husband likes spoiled  In either event why is there a full tray of  not good  chana masala here for customers to look at  and possible walk out the door with   Heli stared at a full tray of  not good  chana masala  and The girl at the counter motioned at a full tray of  not good  chana masala again Heli was just going to remove a full tray of  not good  chana masala from here anyway. But  um  Heli did not . And Heli was the only person in the store. From the wide glass window  before Heli even stepped in  Heli could see that there was no one else even there. She had not  even come over until Heli would  been standing there for a few minutes. It really bothers me that Heli still really do not  know what was so wrong with an entire tray of food that an entire tray of food would warrant being thrown out  -  and why an entire tray of food HADN T been thrown out immediately if an entire tray of food really could not  be saved. It was clear from the crispiness on the top you know that kind of layer of  kin  that develops   that an entire tray of food had been sitting for a while. Ultimately  Heli m really quite relieved the staff person told me before Heli bought a full tray of  not good  chana masala took a full tray of  not good  chana masala home and got diarrhea or something this would be a much harsher review if that had happened... And Heli guess that says something positive about the customer service  sorta... As more and more Indian food places -  both takeout and dine - in  -  emerge in the region  store owners need to realize that customers are going to eventually become more discerning. Heli really wanted this place to be a favorite  -  after all  this place is  very convenient for me  -  but  sorry Silver Spoon Silver Spoon just do not  make the cut.\", 'relationMentions': [{'em1Text': 'Silver Spoon', 'em2Text': 'Chana Masala', 'label': 'HAS_MENU'}, {'em1Text': 'Heli', 'em2Text': 'Chana Masala', 'label': 'ORDER'}, {'em1Text': 'Silver Spoon', 'em2Text': 'Beef', 'label': 'HAS_MENU'}, {'em1Text': 'Heli', 'em2Text': 'Beef', 'label': 'ORDER'}, {'em1Text': 'Silver Spoon', 'em2Text': 'Food', 'label': 'HAS_ASPECT'}, {'em1Text': 'Terrible', 'em2Text': 'Silver Spoon', 'label': 'ASPECT_ATTR_FOR'}, {'em1Text': 'Food', 'em2Text': 'Terrible', 'label': 'IS'}]}, {'sentText': \"Silver Spoon, Adnan, Silver Spoon do not have lots of crowd and it taste is different than other locations. Adnan would not say the food is completely bad but certainly not the best. Silver Spoon's biryani  the holy grail of southeast Asian food  is just so spicy and sometimes taste raw  like something is missing. Adnan would suggest Their biryani  the holy grail of southeast Asian food to masses  try before you buy.\", 'relationMentions': [{'em1Text': 'Silver Spoon', 'em2Text': 'Biryani', 'label': 'HAS_MENU'}, {'em1Text': 'Adnan', 'em2Text': 'Biryani', 'label': 'ORDER'}, {'em1Text': 'Silver Spoon', 'em2Text': 'Food', 'label': 'HAS_ASPECT'}, {'em1Text': 'Bad', 'em2Text': 'Silver Spoon', 'label': 'ASPECT_ATTR_FOR'}, {'em1Text': 'Food', 'em2Text': 'Bad', 'label': 'IS'}]}, {'sentText': \"Silver Spoon, Atif, Atif was first to review Silver Spoon. Atif gave Silver Spoon   stars and below review remains. Atif do feel it has fallen off big time. Atif have given you so many chances your staff is very nice. Atif's last straw was the cockroach on the fridge. Atif called the guy out from the back to show the cockroach on the fridge. the cockroach on the fridge was about mid adult size  with them wirey attennas. the cockroach on the fridge was crawling around  and Atif pictured the cockroach on the fridge being baked into Atif's shami kabab Atif just ordered. The food is on warmers. The food is either doused in oil to not get dry... Or The food is Dry. Atif have a sense of humour  and Atif literally have to never go back there ever again after posting this. Otherwise Atif will get curry cockroach to go ! Fail !  !  !  !  !  !  !  !  !  !  !  !  !  !\", 'relationMentions': [{'em1Text': 'Silver Spoon', 'em2Text': 'Staff', 'label': 'HAS_ASPECT'}, {'em1Text': 'Nice', 'em2Text': 'Silver Spoon', 'label': 'ASPECT_ATTR_FOR'}, {'em1Text': 'Staff', 'em2Text': 'Nice', 'label': 'IS'}]}]\n",
    "\n",
    "'''\n",
    "from module.ner.menu_ner import MenuNER\n",
    "menu_ner = MenuNER()\n",
    "\n",
    "with open('data/output/kg/joint_relation_100.json', 'r') as json_file:\n",
    "    json_data = json_file.read()\n",
    "    joint_relations = json.loads(json_data)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "'''\n",
    "Change title line code from MenuNER module\n",
    "'''\n",
    "from pprint import pprint\n",
    "\n",
    "kb_menu_dict = {}\n",
    "with open(kb_menu_file, mode='r') as infile:\n",
    "    reader = csv.reader(infile)\n",
    "    i = 0\n",
    "    for rows in reader:\n",
    "        if i == 0:\n",
    "            i += 1\n",
    "            continue\n",
    "        kb_menu_dict[rows[0]] = rows[1]\n",
    "\n",
    "print(str(len(kb_menu_dict)))\n",
    "\n",
    "def split_rest_user_from_text(text, sep, pos):\n",
    "    text = text.split(sep)\n",
    "    return text[0], sep.join(text[pos:]).strip()\n",
    "\n",
    "index = 0\n",
    "total_relations = len(joint_relations)\n",
    "\n",
    "for j_relation in joint_relations:\n",
    "    # if index < 1:\n",
    "    #     index += 1\n",
    "    #     continue\n",
    "    index += 1\n",
    "    print(\"processing review: \"+str(index)+\"/\"+str(total_relations))\n",
    "\n",
    "    rest_name, split_review = split_rest_user_from_text(j_relation['sentText'], ',', 2)\n",
    "    # print(split_review)\n",
    "\n",
    "    entity_dict = menu_ner.extract_menu_ner_single(split_review)\n",
    "    # pprint(entity_dict)\n",
    "\n",
    "    for k, v in entity_dict.items():\n",
    "        # print(k)\n",
    "        j_relation['sentText'] = j_relation['sentText'].replace(k, k.title())\n",
    "\n",
    "    for k, v in entity_dict.items():\n",
    "        # print(k)\n",
    "\n",
    "        valid_menu = False\n",
    "        for key, value in kb_menu_dict.items():\n",
    "            if key.lower() == k.lower():\n",
    "                valid_menu = True\n",
    "                # j_relation['sentText'] = j_relation['sentText'].replace(k, key.title())\n",
    "                k = key.title()\n",
    "                break\n",
    "\n",
    "        if not valid_menu:\n",
    "            for key, value in kb_menu_dict.items():\n",
    "                # Check if misspelled menu word\n",
    "                subject_ratio = levenshtein_ratio_and_distance(key.lower(), k.lower(), ratio_calc=True)\n",
    "                if subject_ratio > 0.85:\n",
    "                    # print(\"updating subject [\" + k + \"] with [\" + key + \"]\")\n",
    "                    j_relation['sentText'] = j_relation['sentText'].replace(k, key.title())\n",
    "                    # k = key.title()\n",
    "                    valid_menu = True\n",
    "                    break\n",
    "\n",
    "        if valid_menu:\n",
    "            menu_with_rel_found = False\n",
    "            for rel in j_relation['relationMentions']:\n",
    "                if rel['em1Text'].lower() == k.lower():\n",
    "                    rel['em1Text'] = rel['em1Text'].title()\n",
    "\n",
    "            for rel in j_relation['relationMentions']:\n",
    "                if rel['em2Text'].lower() == k.lower() and rel['label'] == 'HAS_MENU':\n",
    "                    # print('Menu relation already present')\n",
    "                    rel['em2Text'] = rel['em2Text'].title()\n",
    "                    menu_with_rel_found = True\n",
    "                    break\n",
    "\n",
    "            if not menu_with_rel_found:\n",
    "                # print('Menu relation not found')\n",
    "                j_relation['relationMentions'].append({'em1Text': rest_name,\n",
    "                                                       'em2Text': k.title(),\n",
    "                                                       'label': 'HAS_MENU'})\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "    # pprint(j_relation)\n",
    "    # break"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "with open('data/output/kg/joint_relation_100.json', 'r') as json_file:\n",
    "    json_data = json_file.read()\n",
    "    joint_relations = json.loads(json_data)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "import re\n",
    "def find_word(text, search):\n",
    "    res = re.findall('\\\\b'+search+'\\\\b', text, flags=re.IGNORECASE)\n",
    "    if len(res) > 0:\n",
    "        return True\n",
    "    else:\n",
    "        return False\n",
    "\n",
    "for relation in joint_relations:\n",
    "    relation['sentText'] = relation['sentText'].lower()\n",
    "    for rel_mention in relation['relationMentions']:\n",
    "        rel_mention['em1Text'] = rel_mention['em1Text'].lower()\n",
    "        rel_mention['em2Text'] = rel_mention['em2Text'].lower()\n",
    "        rel_mention['label'] = rel_mention['label'].lower()\n",
    "\n",
    "for relation in joint_relations:\n",
    "    relation_mentions = relation['relationMentions']\n",
    "    for rel_mention in relation_mentions[:]:\n",
    "        em1 = find_word(relation['sentText'], rel_mention['em1Text']) #relation['sentText'].find(rel_mention['em1Text'])\n",
    "        em2 = find_word(relation['sentText'], rel_mention['em2Text']) #relation['sentText'].find(rel_mention['em2Text'])\n",
    "\n",
    "        # if em1 == -1 or em2 == -1:\n",
    "        if not em1 or not em2:\n",
    "            relation_mentions.remove(rel_mention)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "import json\n",
    "with open(\"data/output/kg/joint_relation_100.json\", 'w') as joint_relation_file:\n",
    "    json.dump(joint_relations, joint_relation_file)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "import random\n",
    "\n",
    "random.shuffle(joint_relations)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "with open(\"data/output/kg/new_test.json\", 'w') as f:\n",
    "    item_no = 0\n",
    "    for item in joint_relations:\n",
    "        item_no += 1\n",
    "        if item_no == 401:\n",
    "            break\n",
    "        f.write(\"%s\\n\" % item)\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "with open(\"data/output/kg/new_valid.json\", 'w') as f:\n",
    "    item_no = 0\n",
    "    for item in joint_relations:\n",
    "        item_no += 1\n",
    "        if item_no < 401:\n",
    "            continue\n",
    "        if item_no == 801:\n",
    "            break\n",
    "        f.write(\"%s\\n\" % item)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "with open(\"data/output/kg/new_train.json\", 'w') as f:\n",
    "    item_no = 0\n",
    "    for item in joint_relations:\n",
    "        item_no += 1\n",
    "        if item_no < 801:\n",
    "            continue\n",
    "        f.write(\"%s\\n\" % item)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "'''\n",
    "If menus are null put empty string ('') and remove duplicate menus from the list\n",
    "'''\n",
    "user_menu_df = pd.read_csv(\"data/output/kg/user_menu.csv\")\n",
    "print(str(len(user_menu_df)))\n",
    "\n",
    "user_menu_df['menus'] = user_menu_df['menus'].apply(lambda x: x if not pd.isnull(x) else '')\n",
    "user_menu_df = user_menu_df.groupby(\"user_id\")\n",
    "\n",
    "user_merge_menus = []\n",
    "for key, item in user_menu_df:\n",
    "    group = user_menu_df.get_group(key)\n",
    "    group_menus = list(group[\"menus\"])\n",
    "    group_menus = group_menus[0].split(\"|\")\n",
    "    group_menus = set(group_menus)\n",
    "\n",
    "    user_merge_menus.append([list(group['user_id'])[0], '|'.join(str(menu) for menu in group_menus if menu != '')])\n",
    "\n",
    "print(str(len(user_menu_df)))\n",
    "\n",
    "user_menu_df = pd.DataFrame(user_merge_menus, columns=['user_id', 'menus'])\n",
    "user_menu_df.to_csv(\"data/output/kg/user_menu.csv\", index=False)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "'''\n",
    "Merge Menus from user_menu.csv to user.csv\n",
    "'''\n",
    "user_ori_df = pd.read_csv(users_csv)\n",
    "user_menu_temp_df = pd.read_csv(\"data/output/kg/user_menu.csv\")\n",
    "user_merged_df = pd.merge(user_ori_df, user_menu_temp_df, on=['user_id'], how='left')\n",
    "user_merged_df.to_csv(users_csv, index=False)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "'''\n",
    "Merge Menus & Aspects from restaurant_menu_aspect.csv to restaurant.csv\n",
    "'''\n",
    "\n",
    "rest_ori_df = pd.read_csv(restaurants_csv)\n",
    "rest_menu_aspect_temp_df = pd.read_csv(\"data/output/kg/restaurant_menu_aspect.csv\")\n",
    "rest_merged_df = pd.merge(rest_ori_df, rest_menu_aspect_temp_df, on=['rest_id'], how='left')\n",
    "rest_merged_df.to_csv(restaurants_csv, index=False)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "import datetime\n",
    "\n",
    "rating_df = pd.read_csv(ratings_csv)\n",
    "restaurant_df = pd.read_csv(restaurants_csv)\n",
    "user_df = pd.read_csv(users_csv)\n",
    "\n",
    "\n",
    "rating_df['date']=rating_df['date'].apply(lambda x: int(datetime.datetime.strptime(x, '%Y-%m-%d %H:%M:%S').strftime(\"%s\")))\n",
    "\n",
    "for item, row in user_df.iterrows():\n",
    "    rating_df['user_id'] = rating_df['user_id'].replace([row['user_id']], row['id'])\n",
    "\n",
    "for item, row in restaurant_df.iterrows():\n",
    "    rating_df['rest_id'] = rating_df['rest_id'].replace([row['rest_id']], row['id'])\n",
    "\n",
    "rating_df.to_csv(ratings_csv, index=False)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "name": "pycharm-742af70c",
   "language": "python",
   "display_name": "PyCharm (knowledge-graph)"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}