-
Notifications
You must be signed in to change notification settings - Fork 0
/
get_tweets_update.py
120 lines (99 loc) · 3.78 KB
/
get_tweets_update.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
#In[]:
import io
import random
import string
import warnings
import pandas as pd
import numpy as np
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
import warnings
warnings.filterwarnings('ignore')
import nltk
from nltk.tokenize import sent_tokenize
from nltk.corpus import words
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
from nltk.stem import WordNetLemmatizer
from nltk.stem import PorterStemmer
from nltk.sentiment.vader import SentimentIntensityAnalyzer
from nltk.sentiment.util import *
# sklearn imports
from sklearn.model_selection import train_test_split
from sklearn.naive_bayes import MultinomialNB
from sklearn import metrics
# python imports
import re
import json
import os
from collections import Counter
import datetime as dt
# Visualization
from matplotlib import pyplot as plt
from matplotlib import ticker,dates
import seaborn as sns
from sklearn import feature_extraction, linear_model, model_selection, preprocessing
# from wordcloud import WordCloud
from tqdm import tqdm_notebook
# Saving models
import pickle
import tweepy
#In[]
def processTweetText(df):
'''
Step 1- remove links
Step 2- lower case
Step 3- remove punctuation
Step 4- remove stop words
'''
stop_words = set(stopwords.words('english'))
stop_words.update(['#coronaupdate','#corona','#stayhomestaysafe','#stayhomeandstaysafe','#stayathomeandstaysafe','#stayhomesavelives','#stayhome','#coronavirus', '#coronavirusoutbreak', '#coronavirusPandemic', '#covid19', '#covid_19', '#epitwitter', '#ihavecorona', 'amp', 'coronavirus', 'covid19'])
#removing links
df['text_ed'] = df['text'].apply(lambda x: re.sub(r"https\S+","",str(x)))
#removing twitter handles (@user)
df['text_ed'] = df['text_ed'].apply(lambda x: re.sub("@[\W]*","",str(x)))
#removing special characters, numbers, punctuations
df['text_ed'] = df['text_ed'].apply(lambda x: re.sub("[^a-zA-Z#]"," ",str(x)))
#lower case
df['text_ed'] = df['text_ed'].apply(lambda x: x.lower())
df['text_ed'] = df['text_ed'].apply(lambda x: ' '.join([word for word in x.split() if word not in stop_words]))
stemmer=PorterStemmer()
df['text_ed'] = df['text_ed'].apply(lambda x: ' '.join([stemmer.stem(word) for word in x.split() if word != ' ']))
return df
#In[]
def sentimentAnalysis(df):
#4 new columns added to dataframe-> neg,neu,pos,compound
sid = SentimentIntensityAnalyzer()
sentiment_scores = df['text_ed'].apply(lambda x: sid.polarity_scores(x))
sent_scores_df = pd.DataFrame(list(sentiment_scores))
df_new = pd.concat([df.reset_index(drop=True), sent_scores_df.reset_index(drop=True)], axis=1)
df_new['value']=df_new['compound'].apply(lambda x : 'neutral' if (x>-0.01 and x<0.01) else ('positive' if x>=0.01 else 'negative'))
return df_new
#In[]
def driver(tweets):
#In[]
#In[]
#auth = tweepy.OAuthHandler(consumer_key,consumer_secret)
#auth.set_access_token(access_token, access_token_secret)
#api = tweepy.API(auth,wait_on_rate_limit=True)
#In[]:
start = time.time()
#tweets = tweepy.Cursor(api.search,q="#covid" + " -filter:retweets",rpp=5,lang="en", tweet_mode='extended').items(10)
#In[]:
tweets_df = pd.DataFrame(columns=['id','text'])
for x in tweets:
tweets_df = tweets_df.append({'id': x.id , 'text': x.full_text}, ignore_index=True)
#print(tweets_df)
tweets_df = processTweetText(tweets_df)
senti = sentimentAnalysis(tweets_df)
# print(senti)
senti_ls = senti['value'].tolist()
tweet_id = senti['id'].tolist()
senti_dict = {}
for x in range(len(tweet_id)):
senti_dict[tweet_id[x]] = senti_ls[x]
# print('\n\n')
print(senti_dict)
end = time.time()
# print(end - start)
return senti_dict