RandsSlackStats.Rmd

---
title: "Rand Slack Statistics"
author: "@alexis"
output: 
  pdf_document: 
    toc: yes
    includes:
      in_header: header.tex
---

```{r, echo=FALSE, message=FALSE, warning=FALSE}
library(knitr)
opts_chunk$set(echo=FALSE, message=FALSE, warning=FALSE, fig.width=12, fig.height=6)

library(reshape2); library(dplyr); library(lubridate)
library(ggplot2); library(scales)
library(jsonlite)

rm(list = ls())

listFiles <- dir('data/', '.zip|.json', full.names = TRUE, recursive = TRUE)

getJSON <- function(filenm) {
# extract the JSON content from the zipfile.  Assumes one .json per .zip, or .json file
  if (grepl(".zip", filenm)) {
    jsonFileName <- grep(".json", unzip(filenm, list = TRUE)$Name, value = TRUE)  # find .json file in the zipfile
    j <- fromJSON(unz(filenm, jsonFileName))
  } else {
    j <- fromJSON(filenm)
  }
  j
}

inJSON <- lapply(listFiles, getJSON)
extractJSON <- function(d) {
  dWeek <- paste(d$start, d$end, sep = " -\n ")
  dStart <- ymd(d$start)
  dEnd <- ymd(d$end)
  df <- melt(d$statistics) %>% 
    rename(user = L2, channel = L1, numPosts = value) %>% 
    filter(!grepl('\\$', user)) %>%
    mutate(period = dWeek, start = dStart, end = dEnd)
  df
}

df <- bind_rows(lapply(inJSON, extractJSON))

postsPerWeek <- df %>% 
   group_by(start, user) %>% 
   summarise(totPosts = sum(numPosts))

latestDate <- max(df$start)
start4weeks <- latestDate - dweeks(3.5)
earliestDate <- min(df$start)
```

\pagebreak

# Stats for last week
Week starting `r max(df$start)`.

## Top 20 channels and users
```{r}
plotDataLastWeek <- df %>% filter(start == latestDate)  

df.channelsLastWeek <- plotDataLastWeek %>% group_by(channel) %>% summarise(numPosts = sum(numPosts)) %>% 
  arrange(numPosts) %>% top_n(20)
df.channelsLastWeek %>% 
  mutate(channel = factor(channel, levels = df.channelsLastWeek$channel %>% unlist)) %>%  # reorder bars
  ggplot(aes(x = channel, y = numPosts)) + geom_bar(stat = "identity") + coord_flip()

df.usersLastWeek <- plotDataLastWeek %>% group_by(user) %>% summarise(numPosts = sum(numPosts)) %>% 
  arrange(numPosts) %>% top_n(20)
df.usersLastWeek %>% 
  mutate(user = factor(user, levels = df.usersLastWeek$user %>% unlist)) %>%
  ggplot(aes(x = user, y = numPosts)) + geom_bar(stat = "identity") + coord_flip()

```

# Overall participation over time

```{r}
df %>% group_by(start, end) %>% summarise(numPosts = sum(numPosts), numActiveUsers = length(user), numActiveChannels = length(unique(channel))) %>%
  select(start, end, numPosts, numActiveUsers, numActiveChannels) %>% melt(id.vars = c("start", "end")) %>%
  ggplot(aes(start, value)) + geom_line() + facet_wrap(~variable, ncol = 1, scales = "free_y") + 
  scale_y_continuous(label = comma)
```


# Top 20(-ish) users

Users who were in the Top 20 of messages posted, either since the beginning of the statistics (`r format(earliestDate, "%Y-%b-%d")`) or in the last 4 weeks (since `r format(start4weeks, "%Y-%b-%d")`).  Note this could be more than 20 people.

```{r, echo=FALSE}
# Top N per week
allTime <- df %>% 
  group_by(user) %>% 
  summarise(allTime = sum(numPosts)) %>% arrange(desc(allTime)) 

last4weeks <- df %>% filter(start > start4weeks) %>%
  group_by(user) %>% 
  summarise(last4weeks = sum(numPosts)) %>% arrange(desc(last4weeks)) 

# Pick everyone who has been in the top 20 in either all time or in the last 4 weeks
TopNtable <- distinct(bind_rows(select(top_n(allTime, 20, allTime), user), select(top_n(last4weeks, 20, last4weeks), user))) %>% 
                left_join(allTime) %>%
                left_join(last4weeks) %>%
                arrange(desc(allTime))

TopNusers <- select(TopNtable, user)

kable(TopNtable, row.names = TRUE, format.args = c(big.mark= "'"))

```

## Top20 users number of posts

```{r}
postsPerWeek %>% mutate(Top20 = ifelse(user %in% TopNusers$user, TRUE, FALSE)) %>%
  ggplot(aes(start, weight=totPosts, fill=Top20)) + geom_bar() +
  scale_y_continuous("Number of posts", label = comma) +
  theme(legend.position="bottom")

```

## Top20 users as a proportion of the total

```{r}
postsPerWeek %>% mutate(Top20 = ifelse(user %in% TopNusers$user, TRUE, FALSE)) %>% 
  group_by(start, Top20) %>%
  summarise(n = sum(totPosts)) %>%
  mutate(proportion = n / sum(n)) %>%
  ggplot(aes(start, weight=proportion, fill=Top20)) + geom_bar() +
  scale_y_continuous("Number of posts", label = percent) +
  geom_hline(aes(yintercept = 0.5)) +
  theme(legend.position="bottom")

```

# Activity over time per user

```{r, fig.width=8}
cohorts <- df %>% group_by(user) %>%
              summarise(first_post = min(start), last_post = max(start)) %>% 
              mutate(numWeeksActive = as.integer(difftime(last_post, first_post, units = "weeks")) + 1,
                     first_seen = format(first_post, "%Y-%m")) %>%
              inner_join(
                df %>% group_by(user, start) %>% summarise(postsPerWeek = sum(numPosts))    # number of posts per week
              ) %>%
              mutate(weekNo = as.integer(difftime(start, first_post, units = "weeks")))

# cohorts %>% select(user, first_seen, numWeeksActive) %>% distinct() %>%
#   ggplot(aes(numWeeksActive)) + geom_bar() + facet_wrap(~first_seen, ncol = 1, labeller = label_both) +
#   labs(title = "Number of users active since first spotted", x = "Weeks since first seen", y = "Number of users active")

cohorts %>% group_by(first_seen, weekNo) %>% summarise(n = n()) %>%
  ggplot(aes(weekNo, n, colour = first_seen)) + geom_line() + 
  scale_y_continuous(breaks = seq(10, 200, 10)) +
  labs(title = "Number of users active since first post", x = "Weeks since first posted", y = "Number of users active", colour = "First posted") +
  theme(legend.position = "bottom")

```

Notes:

* The data begins in 2015-09, so the top graph shows that many of the users who were active at the start of the data are still active
* For all following weeks, we can see that many users are only active in their first week, and only a small number keep posting after that

```{r, eval=FALSE}
oneWeekers <- cohorts %>% filter(numWeeksActive == 1) %>% left_join(df)

oneWeekers %>% group_by(user) %>% summarise(numPosts = sum(numPosts)) %>% select(user, numPosts) %>%
  ggplot(aes(numPosts)) + geom_histogram(binwidth = 1) 
oneWeekers %>% group_by(user, channel) %>% summarise(numPosts = sum(numPosts)) %>% select(user, channel, numPosts) %>%
  ggplot(aes(numPosts)) + geom_histogram(binwidth = 1) + facet_wrap(~channel)
```


# Detailed stats for the Top20 users

```{r, include=FALSE, fig.height=12}
# propPerChannel <- function(username) {
#   p <- df %>%
#       group_by(start, channel) %>% mutate(prop = numPosts/sum(numPosts)) %>%
#       filter(user == username) %>%
#       ggplot(aes(start, prop)) + geom_bar(stat="identity") + facet_wrap(~channel) +
#       scale_y_continuous("", limits = c(0,1), labels = percent, breaks = c(0, .25, .5, .75, 1)) + 
#       geom_text(aes(label = paste(numPosts), y = prop + 0.075), size = 3) +
#       labs(title = paste0("Proportion of posts in channel (@", username, ")"))
#   c(username, p)
# }

out <- NULL

for (username in TopNusers$user) {
  out <- c(out, knit_child('proportionMessagesPerGroup.Rmd'))
}

```
`r paste(out, collapse = '\n')`