2 - Full dataset classificator.Rmd

---
title: "2a - Full dataset classificator"
author: '100385774'
date: "2023-04-18"
output: html_document
---

```{r}
library(tidyverse)
library(data.table)
library(ggplot2)
library(stringr)
library(openxlsx)
library(lubridate)
library(emo)
library(stopwords)
library(tidytext)
library(caret)
library(textrecipes)
library(SnowballC)
library(rsample)
library(quanteda)
```


```{r}
XGB <- readRDS("data/XGB_model.rds")
```

```{r}
dfull <- fread("data/tweets.csv")
```

# Whole vocabulary construction

```{r}
dfull <- dfull %>% 
  select(c(id, text)) %>% 
  mutate(hate = 0,
  hate = case_when(hate == 0 ~ "No", 
                   hate == 1 ~ "Yes"),
  hate = factor(hate, levels = c("No", "Yes"))
)
```

```{r}
dfull <- dfull %>% 
  distinct(text, .keep_all = T) %>% 
  mutate(text = tolower(text),
         entities.mentions = ifelse(str_detect(text, "@.{1,15} "), 1, 0),
         entities.url = ifelse(str_detect(text, "https://.*\\b"), 1, 0),
         #text = str_replace_all(text, "[^[:alnum:]]", ""),
         text = str_remove_all(text, "https://.*\\b"),
         hateterm = str_detect(text, "(?:^|\\s)moros?\\b|(?:^|\\s)maric[oó]na?e?s?\\b|(?:^|\\s)monos?\\b|(?:^|\\s)putas?\\b|(?:^|\\s)hijos? de puta\\b|(?:^|\\s)hijos?deputa|(?:^|\\s)zorras?\\b|(?:^|\\s)panchitos?\\b"))
```

```{r}
dfull <- dfull %>% 
  mutate(text = str_replace_all(text, "🏻|🏼|🏽|🏾|🏿", ""),
         text = str_replace_all(text, "\\\\n|&lt", " "), 
         text = str_replace_all(text, "🤣+|😂+", " tokenrisa "),
         text = str_replace_all(text, "🐀+", " tokenrata "),
         text = str_replace_all(text, "😡+|🔪+|🤬+|👊+|🪓+|🤮+|💩+", " tokenenfado "),
         text = str_replace_all(text, "👏+|💪+|✊+|💪+|❤️+|🥰+|❤+|🤍+", " tokenpositivo "),
         text = ji_replace_all(text, ""),
         text = iconv(text, to = "UTF-8//IGNORE"),
         text = str_replace_all(text, "\\b(?:a*(?:ha*){2,}h?)\\b|\\b(?:a*(?:ja*)+j?)\\b|\\b(?:e*(?:je*)+j?)\\b|\\b(?:i*(?:ji+)+j?)\\b|\\b(?:A*(?:JA+)+J?)\\b|\\b(?:A*(?:HA+)+H?)\\b|\\b(?:e*(?:he+){2,}h?)\\b|\\b(?:Ja*(?:ja+)+j?)\\b|\\b(?:Je*(?:je+)+j?)\\b|\\bJa+\\b|\\b(?:Ji*(?:ji+)+j?)\\b|\\b(?:Ha*(?:ha+)+h?)\\b|\\b(?:Jo*(?:jo+)+j?)\\b|\\b(?:o*(?:jo+)+j?)\\b|\\b(?:a*((?:ja+)|(?:js+))+j?)\\b|\\b(?:A*((?:JA+)|(?:JS+))+J?)\\b|\\blo*l\\b", " tokenrisa "))
```

```{r}
dfull <- dfull %>% 
  mutate(entities.url = factor(entities.url, levels = c(0,1)),
         entities.mentions = factor(entities.mentions, levels = c(0,1)),
         hateterm = case_when(hateterm == F ~ 0, 
                              hateterm == T ~ 1),
         hateterm = factor(hateterm, levels = c(0,1)),
         text = str_replace_all(text, "@|#|\\|+", ""),
         text = stringi::stri_replace_all_regex(text, "[^[:alnum:][:space:]]+|\\p{So}+|[0-9]+", ""))
```


```{r}
#write.csv(dfull, file = "data/preprocessed.csv")
```


```{r}
set.seed(123)
dfull <- sample_n(dfull, 30000, replace=F)

```


```{r}

fullcorpus <- corpus(dfull$text, docvars = data.frame(hate = dfull$hate, hateterm = dfull$hateterm, mentions = dfull$entities.mentions, url = dfull$entities.url))
summary(fullcorpus)

```

```{r}
set.seed(1234)
id <- seq(1:nrow(dfull))
```

```{r}
dfmatfull <- fullcorpus %>%  
  tokens(remove_punct = T, remove_numbers = T, remove_separators = T) %>% 
  tokens_remove(stopwords("es", source = "nltk")) %>% 
  tokens_wordstem(language = "spanish") %>%
  #tokens_remove("[^[:alnum:]]+|[^\\p{L}\\p{N}]+") %>% 
  dfm() %>%
  dfm_tfidf() %>% 
  dfm_trim(min_docfreq=5,verbose=TRUE)
```

Sequential function: 

```{r}
batch_size <- 50000
n_tweets <- ndoc(dfmatfull)

all_predictions <- data.frame(No = numeric(0), Yes = numeric(0))

for (i in seq(1, n_tweets, by = batch_size)) {
  
  batch_indices <- i:min(i + batch_size - 1, n_tweets)
  dfmat_matched <- dfmatfull[batch_indices, ]
  dfmat_matched <- dfm_match(dfmat_matched, features = featnames(dfmat_train))
  predictions <- predict(XGB, newdata = dfmat_matched, type = "prob")
  all_predictions <- rbind(all_predictions, predictions)
}

dfull <- cbind(dfull, all_predictions)
```


```{r}
dfmat_matched <- dfm_match(dfmatfull, features=featnames(dfmat_train))
actual_class <- docvars(dfmat_matched, "hate")
#names = as.data.frame(featnames(dfmat_matched))
```


```{r}
predictions <- predict(XGB, dfmat_matched, type = "prob")
predicted_class.XGB <- as.factor(ifelse(predictions[,2] > 0.3, "Yes", "No"))
dfull$hate <- as.factor(ifelse(dfull[,10] > 0.3, "Yes", "No"))
```


```{r}
dfull <- fread("data/tweets.csv")
```

```{r}
dfull <- dfull %>% 
  distinct(text, .keep_all = T)
```

```{r}
#write.csv(dfull, file = "data/predicted.csv")
```