library(tidyverse)
company_list <- read.csv2("employer_branding_company_list.csv")
company_list$Archiv.URL <- as.character(company_list$Archiv.URL)
company_list$URL.Branding.Seite <- as.character(company_list$URL.Branding.Seite)
company_list$ID_Einzelunternehmen <- as.character(company_list$ID_Einzelunternehmen)
company_list$Einzelunternehmen <- as.character(company_list$Einzelunternehmen)
knitr::kable(company_list, format = "markdown")

library(tm)
# Korpus laden
web_corpus <- VCorpus(DirSource(directory = "webpages/", encoding = "UTF-8",
mode = "text"))

# Korpus bereinigen
web_corpus <- tm_map(web_corpus, stripWhitespace)
web_corpus <- tm_map(web_corpus, content_transformer(tolower))
web_corpus <- tm_map(web_corpus, removeWords, c(stopwords("german"), "(m/w)", "(w/m)",
"(b.a.)", "mehr…", "..", "../",  "(m/f)", "()," , "aldi", "rewe", "obi", "süd",
 "bauhaus", "netto", "group", "mytoys", "metro", "gernsheim", "cash", "carry", "toom",
 "the", "and", "for", "our", "bonprix", "nord", "real", "müller", "globus", "diska",
 "markt", "zalando", "2018", "2017/2018", "sucht", "edeka", "hornbach", "dass", "otto",
 "company", "fachcentren", "berlin", "intersport", "germany", "0221", "elias", "gabi",
 "angela", "kik", "barry", "daniela", "waltrop", "gmbh", "manufactum", "köln",
 "ascdsc", "johannes", "rossmann", "försterling,", "isabelle", "warenhaus",
 "gmbhregion"))

# Korpus auswerten
dtm_texts <- DocumentTermMatrix(web_corpus, control = list(stemming = F))

# Korpus-Wortlisten in tidytext-Format umwandeln
library(tidytext)
dtm_texts_tidy <- tidy(dtm_texts)

# Informationen über die Unternehmen hinzufügen
library(stringi)
dtm_texts_tidy$ID_Hauptunternehmen <- as.numeric(
  stri_extract(dtm_texts_tidy$document, regex = "[^.]*"))
dtm_texts_tidy$ID_Einzelunternehmen <- (stri_extract(
  dtm_texts_tidy$document, regex = "[^_]*"))
dtm_texts_tidy <- dtm_texts_tidy %>% inner_join(company_list %>%
  select(ID_Hauptunternehmen, Hauptunternehmen) %>%
  distinct(ID_Hauptunternehmen, Hauptunternehmen))

dtm_texts_tidy <- dtm_texts_tidy %>% inner_join(company_list %>%
  select(ID_Einzelunternehmen, Einzelunternehmen) %>%
  distinct(ID_Einzelunternehmen, Einzelunternehmen))

library(wordcloud2)
set.seed(7712)
dtm_texts_tidy %>% select(term, count) %>% filter(count > 2) %>%
 wordcloud2(., fontFamily = "Times", color = "black")

## Download SentiWS hier: http://wortschatz.uni-leipzig.de/de/download
## Quelle Vorbereitung Wortliste https://www.inwt-statistics.de/blog-artikel-lesen/text-mining-part-3-sentiment-analyse.html
# Vorbereiten
sent <- c(
  # positive Wörter
  readLines("SentiWS_v1.8c/SentiWS_v1.8c_Positive.txt",
            encoding = "UTF-8"),
  # negative Wörter
  readLines("SentiWS_v1.8c/SentiWS_v1.8c_Negative.txt",
            encoding = "UTF-8")
) %>% lapply(function(x) {
  # Extrahieren der einzelnen Spalten
  res <- strsplit(x, "\t", fixed = TRUE)[[1]]
  return(data.frame(words = res[1], value = res[2],
                    stringsAsFactors = FALSE))
}) %>%
  bind_rows %>%
  mutate(words = gsub("\\|.*", "", words) %>% tolower,
         value = as.numeric(value)) %>%
  # manche Wörter kommen doppelt vor, hier nehmen wir den mittleren Wert
  group_by(words) %>% summarise(value = mean(value)) %>% ungroup

# Mit unserem Wortkorpus vereinen
dtm_texts_tidy <- dplyr::left_join(x = dtm_texts_tidy, y = sent,
  by = c("term" = "words"))

# Ergebnisse darstellen
sentiment_texts <- dtm_texts_tidy %>%
group_by(document, Einzelunternehmen, Hauptunternehmen) %>%
summarise(sentiment = mean(value, na.rm = T)) %>%
arrange(sentiment)
knitr::kable(sentiment_texts, format = "markdown")

library(ggrepel)
ggplot(sentiment_texts, aes(x = reorder(Hauptunternehmen, -sentiment), y = sentiment)) +
  theme_minimal(base_size = 28) + xlab("Unternehmensgruppe") + ylab("mean sentiment") +
   coord_flip() +
  geom_point(aes(x = Hauptunternehmen, y = sentiment), size = 8) +
  geom_label_repel(aes(label = Einzelunternehmen), box.padding = unit(0.35, "lines"),
    point.padding = unit(0.5, "lines"), size = 8)

dtm_texts_tidy %>% group_by(Hauptunternehmen) %>%
  top_n(5, value) %>%
  ggplot(aes(x = reorder(term, value), y = value, group = Hauptunternehmen)) +
  geom_col() + coord_flip() + theme_minimal(base_size = 22) + xlab("") +
   ylab("sentiment") +
  facet_wrap( ~ Hauptunternehmen, ncol = 4, scales = "free")

dtm_texts_tidy %>% group_by(Hauptunternehmen) %>%
  filter(value < 0.1) %>%
  top_n(-3, value) %>%
  ggplot(aes(x = reorder(term, value), y = value, group = Hauptunternehmen)) +
  geom_col() + coord_flip() + theme_minimal() + xlab("") + ylab("sentiment") +
  facet_wrap( ~ Hauptunternehmen, ncol = 4, scales = "free")

library(rvest)
library(urltools)

# Leere Zeilen entfernen
company_list <- company_list %>% filter(Archiv.URL != "")

# Loop zum Suchen und Sammeln der Bild-URLs
image_urls_raw <- list()

for(i in 1:nrow(company_list)) {
  name <- company_list[i, 4]
  html_content <- read_html(company_list[i, 5])
  img_urls <- html_content %>% html_nodes("img") %>% html_attr("src")
  image_urls_raw[[name]] <- img_urls
}

# Manuelles Bearbeiten der Bild-URLs, wo nötig
image_urls_raw[["Lidl"]] <- paste0("https://", domain(company_list[1, 5]), image_urls_raw[["Lidl"]])
image_urls_raw[["Aldi Nord"]] <- paste0("https://", domain(company_list[3, 5]), "/karriere/", image_urls_raw[["Aldi Nord"]])
image_urls_raw[["Aldi Süd"]] <- paste0("https://", domain(company_list[4, 5]), image_urls_raw[["Aldi Süd"]])
image_urls_raw[["METRO Cash & Carry"]] <- paste0("https://", domain(company_list[5, 5]), image_urls_raw[["METRO Cash & Carry"]])
image_urls_raw[["Real"]] <- paste0("https://", domain(company_list[6, 5]), "/",  image_urls_raw[["Real"]])
image_urls_raw[["Edeka-Verbund"]] <- paste0("https://", domain(company_list[7, 5]),  image_urls_raw[["Edeka-Verbund"]])
image_urls_raw[["Diska"]] <- gsub("..", "", paste0("https://", domain(company_list[8, 5]), image_urls_raw[["Diska"]]), fixed = T)
image_urls_raw[["NP.Discount"]] <- paste0("https://", domain(company_list[10, 5]), "/",  image_urls_raw[["NP.Discount"]])
image_urls_raw[["Rewe"]] <- paste0("https://", domain(company_list[12, 5]),  image_urls_raw[["Rewe"]])
image_urls_raw[["Penny"]] <- paste0("https://", domain(company_list[13, 5]),  image_urls_raw[["Penny"]])
image_urls_raw[["Nahkauf"]] <- paste0("https://", domain(company_list[14, 5]),  image_urls_raw[["Nahkauf"]])
image_urls_raw[["Toom"]] <- paste0("https://", domain(company_list[15, 5]),  image_urls_raw[["Toom"]])
image_urls_raw[["Frankonia"]] <- paste0("https://", domain(company_list[19, 5]),  image_urls_raw[["Frankonia"]])
image_urls_raw[["OBI"]] <- gsub("../", "", paste0("https://", domain(company_list[22, 5]), "/karriere/",  image_urls_raw[["OBI"]]), fixed = T)
image_urls_raw[["Globus"]] <- paste0("https://", domain(company_list[25, 5]),  image_urls_raw[["Globus"]])

# in Anlehnung an https://bigdataenthusiast.wordpress.com/2016/10/17/face-api-microsoft-cognitive-services/
# Unternehmensnamen und URLs zusammenführen
urls_df <- data.frame(company = NA, url = NA)
for (i in 1:length(image_urls_raw)) {
  for(j in 1:length(image_urls_raw[[i]])){
    df <- data.frame(company = names(image_urls_raw[i]), url = image_urls_raw[[i]][j])
    urls_df <- dplyr::bind_rows(urls_df, df)
  }
}
urls_df <- na.omit(urls_df)

# Azure Face API einbinden
library(httr)
face_api_url = "https://westeurope.api.cognitive.microsoft.com/face/v1.0/detect?returnFaceId=true&returnFaceLandmarks=false&returnFaceAttributes=age,gender,smile,emotion,makeup"
api_results_list <- list()

# Achtung!
# Hier muss der eigene "Ocp-Apim-Subscription-Key" Wert eingetragen werden.
# Dazu ist ein Azure-Konto anzulegen :-)

for (i in 1:nrow(urls_df)) {
  url <- urls_df[i,2]
  api_url_generator <- paste0("{'url':'", url, "'}")
  result = POST(face_api_url,
              body = api_url_generator,
              add_headers(.headers = c("Content-Type" = "application/json",
                                       "Ocp-Apim-Subscription-Key" = "DEIN_SCHLÜSSEL")))
  nameing <- paste0(urls_df[i,1], "##", i)
  res <- content(result)
  api_results_list[[nameing]] <- res
  Sys.sleep(0.3)
}

## In Datensatz konvertieren
api_results_df <- data.frame()

for (i in 1:length(api_results_list)) {
  for (j in length(api_results_list[[i]])) {
    df <- (as.data.frame(api_results_list[[i]][j]))
    if (length(df) > 0) {
      df$name <- names(api_results_list[i])
      api_results_df <- dplyr::bind_rows(api_results_df, df)
    }
  }
}

## In einem neuen Datensatz zusammenführen

face_data <- api_results_df %>% filter(!is.na(faceId))
face_data$company_name <- sub("##.*", "", face_data$name)

face_data$faceAttributes.gender_num[face_data$faceAttributes.gender == "male"] <- 0
face_data$faceAttributes.gender_num[face_data$faceAttributes.gender == "female"] <- 1


labels <- c(faceAttributes.age = "Alter",
            faceAttributes.smile = "Lächeln",
            faceAttributes.gender_num = "Geschlecht",
            faceAttributes.emotion.anger = "Wut",
            faceAttributes.emotion.contempt = "Verachtung",
            faceAttributes.emotion.disgust = "Ekel",
            faceAttributes.emotion.fear = "Angst",
            faceAttributes.emotion.happiness = "Zufriedenheit",
            faceAttributes.emotion.neutral = "neutral",
            faceAttributes.emotion.sadness = "Traurigkeit",
            faceAttributes.emotion.sadness = "Traurigkeit",
            faceAttributes.emotion.surprise = "Überraschung"
            )

face_data <- left_join(x = face_data, y = company_list,
  by = c("company_name" = "Einzelunternehmen"))

face_data %>%
  keep(is.numeric) %>%
  select(-faceRectangle.height, -faceRectangle.left, -faceRectangle.top,
     -faceRectangle.width, -starts_with("ID_Haupt")) %>% na.omit() %>%
  gather() %>%
  ggplot(aes(value)) +
    facet_wrap(~ key, scales = "free", nrow = 4, labeller = labeller(key = labels)) +
    geom_histogram() + theme_minimal(base_size = 18) + ylab("Anzahl") + xlab("Werte")

ggplot(data = face_data, aes(x = faceAttributes.gender, faceAttributes.age)) +
  geom_boxplot(outlier.shape = NA) +
  geom_jitter(aes(size = faceAttributes.emotion.happiness)) +
  theme_minimal(base_size = 28) +
  xlab("Geschlecht") + ylab("Alter") +
  scale_size_continuous(name="Wert Happiness", range = c(4, 8))

face_data %>% group_by(Hauptunternehmen) %>%
  ggplot(aes(x = Hauptunternehmen, y = faceAttributes.age,
     shape = as.factor(faceAttributes.gender_num))) +
  geom_jitter(height = 0.01, width = 0.3, size = 7) + coord_flip() +
   theme_minimal(base_size = 28) +
  xlab("") + ylab("Alter") + scale_shape_manual(name="Geschlecht", values = c(2, 6)) +
   theme(legend.position="bottom")