# import basic required libraries
import pandas as pd
import numpy as np
import os # for web and HTML
import requests
from bs4 import BeautifulSoup# create a dict of various
RSS feed link and their categories. Will iterate them one by one.
# Have mentioned only one feed for demo purposes
timesofindia = {'world':'http://timesofindia.indiatimes.com/rssfeeds/296589292.cms'}
for category, rsslink in timesofindia.items():
    print('Processing for category: {0}. \nRSS link: {1}'.format(category,rsslink))
    # get the webpage URL and read the html
    rssdata = requests.get(rsslink)
    #print(rssdata.content)
    soup = BeautifulSoup(rssdata.content)
    print(soup.prettify())

# get all news items. It has title, description, link, guid, pubdate for each news items.
# Lets call this items and we will iterate thru it
allitems = soup.find_all('item')# print one news item/healine to check
for item in range(len(allitems)):
    print('Processing news-item #:',item)
    title = allitems[item].title.text
    link = allitems[item].guid.text
    pubdate = allitems[item].pubdate.text
    print('TITLE:',title)
    print('LINK:',link)
    print('PUBDATE:',pubdate)

# Function to fetch each news link to get news essay
def fetch_news_text(link):
# read the html webpage and parse it
    soup = BeautifulSoup(requests.get(link).content, 'html.parser')
    # fetch the news article text box
    # these are with element <div class="_3WlLe clearfix">
    text_box = soup.find_all('div', attrs={'class':'_3WlLe clearfix'})
    # extract text and combine
    news_text = str(". ".join(t.text.strip() for t in text_box))
    return news_text# using the above function, process text
news_articles = [{'Feed':'timesofindia',
                  'Category':category,
                  'Headline':allitems[item].title.text,
                  'Link':allitems[item].guid.text,
                  'Pubdate':allitems[item].pubdate.text,
                  'NewsText': fetch_news_text(allitems[item].guid.text)}
                     for item in range(len(allitems))]news_articles = pd.DataFrame(news_articles)
news_articles.head(3)

# test normalize cleanup on one article
# clean_sentences = normalize_corpus([news_articles['NewsText'][0]])
clean_sentences = normalize_corpus(news_articles['NewsText'])

# define dict to hold a word and its vector
word_embeddings = {}# read the word embeddings file ~820MB
f = open('.\\GloVe\\glove.6B\\glove.6B.100d.txt', encoding='utf-8')
for line in f:
    values = line.split()
    word = values[0]
    coefs = np.asarray(values[1:], dtype='float32')
    word_embeddings[word] = coefs
f.close()# check the length
len(word_embeddings) # 400000

# create vector for each sentences
# list to hold vector
sentence_vectors = []# create vector for each clean normalized sentence
for i in clean_sentences:
    if len(i) != 0:
        v = sum([word_embeddings.get(w, np.zeros((100,))) for w in i.split()])/(len(i.split())+0.001)
    else:
        v = np.zeros((100,))
    sentence_vectors.append(v)print('Total vectors created:',len(sentence_vectors))

import networkx as nx
# build graph and get pagerank
nx_graph = nx.from_numpy_array(sim_mat)
scores = nx.pagerank(nx_graph)
# print final values of sentences
scores

{0: 0.0651816121717921,
 1: 0.0642861521750098,
 2: 0.06399116048715114,
 3: 0.06432009432128397,
 4: 0.06385988469675835,
 5: 0.06400525631019922,
 6: 0.06520921510891638,
 7: 0.06320537732857809,
 8: 0.06298228524215846,
 9: 0.06399491863786076,
 10: 0.0640726538022174,
 11: 0.06349704017361839,
 12: 0.06357060319536506,
 13: 0.057627597033478764,
 14: 0.058463972076477785,
 15: 0.05173217723913434}