twitter-sentiment-analysis.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport re\nimport string\n\n#preproccessing\nimport nltk\nfrom nltk.corpus import stopwords\nfrom nltk.tokenize import word_tokenize\nfrom nltk.stem import PorterStemmer\nfrom nltk.stem import WordNetLemmatizer\nfrom nltk import FreqDist\nfrom sklearn.model_selection import train_test_split \n\n#Visualization\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom wordcloud import WordCloud\n\n#feature extraction\nfrom sklearn.feature_extraction.text import CountVectorizer\nfrom sklearn.feature_extraction.text import TfidfVectorizer\n\n#machine learning libraries\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn import metrics","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stop_words = set(stopwords.words(\"english\"))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_excel(\"/kaggle/input/twitter-dataset/Tweets1.xlsx\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.describe()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.isnull().sum()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df = df.dropna(subset=[\"text\"])\ndf = df.dropna()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df1 = df[df.isna().any(axis=1)]\n# print (df1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df[\"selected_text\"] = np.where(df[\"selected_text\"].isnull() , df[\"text\"] , df[\"selected_text\"])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.isnull().sum()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(df.loc[[1157]],df.loc[[446]])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Text Preprocessing**","metadata":{}},{"cell_type":"code","source":"Z = df[\"selected_text\"].astype(\"string\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def preprocess_tweets(tweet):\n\n    #converting all tweest to lowercase\n    tweet =  tweet.lower()\n\n    #removing urls using regx\n    tweet = re.sub(r\"http\\S+|www\\S+|https\\S+\",\"\",tweet , flags = re.MULTILINE) #MULTILINE flag checks for the regx at the start of the string or even inside the string\n      \n    #remove punctuations\n    tweet =  tweet.translate(str.maketrans(\"\",\"\",string.punctuation))\n\n    #remove # and @\n    tweet = re.sub(r\"\\@\\w+|\\#\" , \"\",tweet)\n\n    #remove stopwords\n    tweet_tokens = word_tokenize(tweet)\n    filtered_words = [word for word in tweet_tokens if word not in stop_words]\n    \n    # stemming\n    ps = PorterStemmer()\n    stem_words = [ps.stem(words) for words in filtered_words]\n\n    #lemmatization\n    lemmatizer = WordNetLemmatizer()\n    lemma_words = [lemmatizer.lemmatize(words , pos = \"a\") for words in stem_words]\n\n    return \" \".join(lemma_words)\n\ndf[\"text_preprocessed\"] = Z.apply(preprocess_tweets)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df[\"text_preprocessed\"]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(df[\"text\"][446])\n# print(df[\"text_preprocessed\"][446])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df[\"sentiment\"]= df[\"sentiment\"].apply(lambda x:1 if x==\"positive\"else x)\n# df[\"sentiment\"]= df[\"sentiment\"].apply(lambda x:0 if x==\"negative\"else x)\n# df[\"sentiment\"]= df[\"sentiment\"].apply(lambda x:2 if x==\"neutral\"else x)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**EDA**","metadata":{}},{"cell_type":"code","source":"df.head(10)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df.dtypes)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[\"sentiment\"].value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = \"Positive\", \"Negative\" , \"Neutral\"\ncolors = sns.color_palette(\"bright\")[5:8]\nsizes = [df.sentiment[df[\"sentiment\"]==\"positive\"].count(), df.sentiment[df[\"sentiment\"]==\"negative\"].count(),df.sentiment[df[\"sentiment\"]==\"neutral\"].count()]\nexplode = (0, 0.1,0)\nfig1, ax1 = plt.subplots(figsize=(8, 6))\nax1.pie(sizes, labels=labels, autopct=\"%1.1f%%\",\n  startangle=200,colors = colors,explode=explode,wedgeprops={'linewidth': 3.0, 'edgecolor': 'white'},\n       textprops={'size': 'x-large'})\nax1.axis(\"equal\")\nplt.title(\"Proportion of Positive , Negative and Neutral Sentiments\", size = 15)\nplt.savefig(\"pie_chart.png\")\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#top 10 frequent words in tweets\nfreq_dist = FreqDist(df[\"text_preprocessed\"])\nprint(freq_dist.most_common(10))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#combining all the sentences into a single single sentence\nall_words = \" \".join([sentence for sentence in df[\"text_preprocessed\"]])\n\n#wordcloud\nwordcloud = WordCloud(width = 800 , height = 500,random_state = 42 ,max_font_size = 100).generate(all_words)\n\n#plotting graph\nplt.figure(figsize = (15,8))\nplt.imshow(wordcloud , interpolation = \"bilinear\")\nplt.axis(\"off\")\nplt.savefig(\"all_words.png\")\nplt.show()\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#top 10 frequent words in positive tweets\n\nfreq_dist_pos = FreqDist((df[\"text_preprocessed\"][df[\"sentiment\"] == \"positive\"]))\nprint(freq_dist_pos.most_common(10))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#frequent words visualization for positive words\nall_words_positive = \" \".join([sentence for sentence in df[\"text_preprocessed\"][df[\"sentiment\"]== \"positive\"]])\n\n#wordcloud\nwordcloud_positive = WordCloud(width = 800 , height = 500,random_state = 42 ,max_font_size = 100).generate(all_words_positive)\n\n#plotting graph\nplt.figure(figsize = (15,8))\nplt.imshow(wordcloud_positive , interpolation = \"bilinear\")\nplt.axis(\"off\")\nplt.savefig(\"Pos_words.png\")\nplt.show()\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#top 10 frequent words in negative tweets\nfreq_dist_negative = FreqDist((df[\"text_preprocessed\"][df[\"sentiment\"] == \"negative\"]))\nprint(freq_dist_negative.most_common(10))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#frequent words visualization for negative words\nall_words_negative = \" \".join([sentence for sentence in df[\"text_preprocessed\"][df[\"sentiment\"]== \"negative\"]])\n\n#wordcloud\nwordcloud_negative = WordCloud(width = 800 , height = 500,random_state = 42 ,max_font_size = 100).generate(all_words_negative)\n\n#plotting graph\nplt.figure(figsize = (15,8))\nplt.imshow(wordcloud_negative , interpolation = \"bilinear\")\nplt.axis(\"off\")\nplt.savefig(\"neg_words.png\")\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#top 10 frequent words in neutral tweets\nfreq_dist_neutral = FreqDist((df[\"text_preprocessed\"][df[\"sentiment\"] == \"neutral\"]))\nprint(freq_dist_neutral.most_common(10))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#frequent words visualization for neutral words\nall_words_neutral = \" \".join([sentence for sentence in df[\"text_preprocessed\"][df[\"sentiment\"]== \"neutral\"]])\n\n#wordcloud\nwordcloud_neutral = WordCloud(width = 800 , height = 500,random_state = 42 ,max_font_size = 100).generate(all_words_neutral)\n\n#plotting graph\nplt.figure(figsize = (15,8))\nplt.imshow(wordcloud_neutral , interpolation = \"bilinear\")\nplt.axis(\"off\")\nplt.savefig(\"neutral_words.png\")\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Feature Extraction**","metadata":{}},{"cell_type":"code","source":"x = df[\"text_preprocessed\"]\ny = df[\"sentiment\"]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(x.head(10))\nprint(y.head(10))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train,X_test,Y_train,Y_test = train_test_split(x,y,random_state = 42,test_size = 0.2)  ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"X_train: \",X_train.shape)\nprint(\"Y_train: \",Y_train.shape)\nprint(\"X_test: \",X_test.shape)\nprint(\"Y_test: \",Y_test.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"In information retrieval, TF-IDF, or TFIDF, short for term frequency–inverse document frequency, is a numerical statistic that is intended to reflect how important a word is to a document in a collection or corpus. It is often used as a weighting factor in searches of information retrieval, text mining, and user modeling. The TF-IDF value increases proportionally to the number of times a word appears in the document and is offset by the number of documents in the corpus that contain the word, which helps to adjust for the fact that some words appear more frequently in general.\n\nTF-IDF is one of the most popular term-weighting schemes today. A survey conducted in 2015 showed that 83% of text-based recommender systems in digital libraries use TF-IDF.","metadata":{}},{"cell_type":"markdown","source":"**TF-IDF Vectorizer**","metadata":{}},{"cell_type":"code","source":"# cv = CountVectorizer(ngram_range = (1,2))\ncv = TfidfVectorizer(ngram_range = (1,1),lowercase = True,analyzer = \"word\")\nX_train_vec = cv.fit_transform(X_train)\nX_test_vec = cv.transform(X_test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**LOGISTIC REGRESSION**","metadata":{}},{"cell_type":"code","source":"lr = LogisticRegression()\nlr.fit(X_train_vec,Y_train)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Accuracy score for LR**","metadata":{}},{"cell_type":"code","source":"lr_score = lr.score(X_test_vec, Y_test)\nprint(\"Results for Logistic Regression with CountVectorizer\")\nprint(lr_score)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Predicting the labels for test data\ny_pred_lr = lr.predict(X_test_vec)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#create confusion matrix\n# lr_matrix = metrics.confusion_matrix(Y_test, y_pred_lr)\n# index = [\"negative\",\"positive\",\"neutral\"]\n# lr_matrix = pd.DataFrame(lr_matrix,columns = index,index = index)\n# print(\"\\t\\tPredicted\\nActual\")\n# print(lr_matrix)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lr_confusion_matrix = pd.crosstab(Y_test, y_pred_lr, rownames=['Actual'], colnames=['Predicted'])\nprint (lr_confusion_matrix)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**KNN**","metadata":{}},{"cell_type":"code","source":"knn = KNeighborsClassifier(n_neighbors = 3)\nknn.fit(X_train_vec , Y_train)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Accuracy score for KNN**","metadata":{}},{"cell_type":"code","source":"knn_score = knn.score(X_test_vec , Y_test)\nprint(knn_score)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred_knn = knn.predict(X_test_vec)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"knn_confusion_matrix = pd.crosstab(Y_test, y_pred_knn, rownames=['Actual'], colnames=['Predicted'])\nprint (knn_confusion_matrix)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**DECISION TREE CLASSIFIER**","metadata":{}},{"cell_type":"code","source":"Dtc = DecisionTreeClassifier()\nDtc.fit(X_train_vec , Y_train)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Accuracy Score For Dtc**","metadata":{}},{"cell_type":"code","source":"Dtc_score = Dtc.score(X_test_vec,Y_test)\nprint(Dtc_score)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred_Dtc = Dtc.predict(X_test_vec)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Dtc_confusion_matrix = pd.crosstab(Y_test, y_pred_Dtc, rownames=['Actual'], colnames=['Predicted'])\nprint (Dtc_confusion_matrix)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}