Repository: stan-dev/posteriordb (language: python)

Issue:
Guard reference access
Hello, 

I realized that access to the reference should be guarded since these references are not available for the entire database.
Here is a proposal using simple assertion (similar to what is used elsewhere in the project).

This is to avoid weird type error when the file is missing. E.g.:
```
TypeError: unsupported operand type(s) for +: 'NoneType' and 'str'
```

and get an assertion error on `assert reference_name is not None` instead.


Discussion / hints:

Fix patch:
diff --git a/posterior_database/bibliography/references.bib b/posterior_database/bibliography/references.bib
index 6d32b22e..691f05a3 100644
--- a/posterior_database/bibliography/references.bib
+++ b/posterior_database/bibliography/references.bib
@@ -204,3 +204,18 @@ @article{lampinen2001bayesian
   year={2001},
   publisher={Elsevier}
 }
+
+
+@misc{jonasson2021rapid,
+Author = {Johan Jonasson and Måns Magnusson},
+Title = {Rapid mixing in unimodal landscapes and efficient simulated annealing for multimodal distributions},
+Year = {2021},
+Eprint = {arXiv:2101.10004},
+}
+
+@misc{farkas2014threemen,
+Author = {Andras Farkas},
+Title = {Three Men and a Boat, translated by Andras Farkas to French and Swedish},
+url = {FarkasTranslations.com},
+Year = {2014},
+}
diff --git a/posterior_database/data/data-raw/three_docs/three_docs.R b/posterior_database/data/data-raw/three_docs/three_docs.R
new file mode 100644
index 00000000..3a848b1f
--- /dev/null
+++ b/posterior_database/data/data-raw/three_docs/three_docs.R
@@ -0,0 +1,12 @@
+
+m <- 400
+dat <- data.frame(doc = factor(c(rep("doc1", m), rep("doc2", m), rep("doc3", m))),
+                  type = factor(c(rep("word1", 0.9*m), c(rep("word2", 0.1*m), rep("word2", m), rep("word3", m)))))
+
+# To stan
+three_docs1200 <-
+  list(V = length(levels(dat$type)),
+       M = length(unique(dat$doc)),
+       N = length(dat$type),
+       w = as.integer(dat$type),
+       doc = as.integer(dat$doc))
diff --git a/posterior_database/data/data-raw/three_men/three_men.R b/posterior_database/data/data-raw/three_men/three_men.R
new file mode 100644
index 00000000..604a65c5
--- /dev/null
+++ b/posterior_database/data/data-raw/three_men/three_men.R
@@ -0,0 +1,144 @@
+library(stopwords)
+library(dplyr)
+library(tidytext)
+library(checkmate)
+
+
+#' @param txt a tidy text corpus with columns [line], [chapter], [text] and [lang]
+#' @param no_words_per_chapter a named vector with language as name and the number of tokens to choose per chapter
+#' @param rare_word_limit The minimal number of token per word type
+#' @param remove_stopwords Remove stopwords based on a stopwordlist
+create_corpus <- function(txt,
+                         no_words_per_chapter = c("en" = 700, "fr" = 210, "sv" = 300),
+                         rare_word_limit = 10,
+                         remove_stopwords = "nltk"){
+  checkmate::assert_names(names(txt), must.include = c("line", "chapter", "text", "lang"))
+  checkmate::assert_names(names(no_words_per_chapter), subset.of = unique(txt$lang))
+  checkmate::assert_int(rare_word_limit, lower = 0)
+  checkmate::assert_integerish(no_words_per_chapter, lower = 0)
+  checkmate::assert_choice(remove_stopwords, null.ok = TRUE, stopwords::stopwords_getsources())
+  checkmate::assert_subset(names(no_words_per_chapter),  stopwords::stopwords_getlanguages(remove_stopwords))
+
+  # Remove it and chapter titles
+  txt <- txt[c(TRUE, !(txt$chapter[-length(txt$chapter)] != txt$chapter[-1])),]
+  txt <- txt[txt$chapter >= 1,]
+
+  txt <- tidytext::unnest_tokens(txt, word, text, token = "words", to_lower = TRUE)
+
+  # Add chapter row no
+  txt <- dplyr::group_by(txt, lang, chapter)
+  txt <- dplyr::mutate(txt, chapter_word_no = dplyr::row_number())
+
+  res <- list()
+  for(i in seq_along(no_words_per_chapter)){
+    lang <- names(no_words_per_chapter)[i]
+    tmp <- txt[txt$lang == lang, ]
+    # Remove stopwords here
+    sw <- stopwords::stopwords(language = lang, source = remove_stopwords)
+    if(lang == "fr") sw <- c(sw, "qu’il", "a", "c’est", "d’un")
+    swt <- dplyr::tibble(word = sw)
+
+    tmp <- dplyr::anti_join(tmp, swt, by = "word")
+
+    # Extract top words
+    tmp <- get_first_n_words_from_chapter(tmp, lang, no_words_per_chapter[lang])
+    res[[i]] <- remove_rare_words(tmp, rare_word_limit)
+  }
+  res <- do.call(rbind, res)
+  res
+}
+
+#' @rdname create_corpus
+remove_rare_words <- function(txt, rare_word_limit){
+  checkmate::assert_names(names(txt), must.include = c("line", "chapter", "word", "lang"))
+  checkmate::assert_int(rare_word_limit, lower = 0)
+
+  txt <- dplyr::group_by(txt, word, lang)
+  wf <- dplyr::summarise(txt, n = dplyr::n(), .groups = "keep")
+  txt <- dplyr::left_join(txt, wf, by = c("word", "lang"))
+  txt <- txt[txt$n >= rare_word_limit,]
+  txt$n <- NULL
+  txt
+}
+
+#' @rdname create_corpus
+#' @param lang language to choose
+#' #' @param n extract chapter_word_no <= n
+get_first_n_words_from_chapter <- function(txt, lang, n){
+  checkmate::assert_names(names(txt), must.include = c("line", "chapter", "word", "lang", "chapter_word_no"))
+  checkmate::assert_choice(lang, choices = unique(txt$lang))
+  checkmate::assert_int(n, lower = 1)
+  tmp <- txt[txt$lang == lang,]
+  tmp <- tmp[tmp$chapter_word_no <= n,]
+  tmp
+}
+
+
+
+#txt <- readRDS(file = "three-men-all.rds")
+#write.csv(txt, file = "three_men_multilingual.csv", row.names = FALSE)
+#zip(files =  "three_men_multilingual.csv", zipfile =  "three_men_multilingual.csv.zip")
+unzip(zipfile = "posterior_database/data/data-raw/three_men_multilingual/three_men_multilingual.csv.zip",
+      exdir = "posterior_database/data/data-raw/three_men_multilingual/")
+txt <- read.csv(file = "posterior_database/data/data-raw/three_men_multilingual/three_men_multilingual.csv")
+
+crp <- create_corpus(txt,
+                     no_words_per_chapter = c("en" = 1200, "fr" = 360, "sv" = 420),
+                     rare_word_limit = 10,
+                     remove_stopwords = "nltk")
+names(crp)[4] <- "type"
+crp$type <- as.factor(crp$type)
+
+# tab <- table(crp$lang)
+# tab;sum(tab)
+# N <- nrow(crp)
+
+# Create corpus versions
+crp1 <- crp
+crp1$chapter[crp1$chapter <= 10] <- 1
+crp1$chapter[crp1$chapter > 10] <- 2
+crp1$doc <- as.factor(paste0(crp1$lang, crp1$chapter))
+table(crp1$doc)
+
+# To stan
+three_men1 <-
+  list(V = length(levels(crp1$type)),
+       M = length(unique(crp1$doc)),
+       N = length(crp1$type),
+       w = as.integer(crp1$type),
+       doc = as.integer(crp1$doc))
+
+
+# Corpus 2
+crp2 <- crp
+crp2$chapter[crp2$chapter <= 10 & crp2$lang != "en"] <- 1
+crp2$chapter[crp2$chapter > 10 & crp2$lang != "en"] <- 2
+docs <- c(1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,9,9,10)
+for(i in seq_along(docs)) crp2$chapter[crp2$lang == "en" & crp2$chapter == i ] <- docs[i]
+crp2$doc <- as.factor(paste0(crp2$lang, crp2$chapter))
+table(crp2$doc)
+
+# To stan
+three_men2 <-
+  list(V = length(levels(crp2$type)),
+       M = length(unique(crp2$doc)),
+       N = length(crp2$type),
+       w = as.integer(crp2$type),
+       doc = as.integer(crp2$doc))
+
+# Corpus 3
+crp3 <- crp
+crp3$chapter[crp3$chapter <= 10 & crp3$lang != "en"] <- 1
+crp3$chapter[crp3$chapter > 10 & crp3$lang != "en"] <- 2
+docs <- c(1,1,1,1,2,2,2,2,3,3,3,3,4,4,4,4,5,5,5)
+for(i in seq_along(docs)) crp3$chapter[crp3$lang == "en" & crp3$chapter == i ] <- docs[i]
+crp3$doc <- as.factor(paste0(crp3$lang, crp3$chapter))
+table(crp3$doc)
+
+# To stan
+three_men3 <-
+  list(V = length(levels(crp3$type)),
+       M = length(unique(crp3$doc)),
+       N = length(crp3$type),
+       w = as.integer(crp3$type),
+       doc = as.integer(crp3$doc))
diff --git a/posterior_database/data/data-raw/three_men/three_men_multilingual.csv.zip b/posterior_database/data/data-raw/three_men/three_men_multilingual.csv.zip
new file mode 100644
index 00000000..29eedad7
Binary files /dev/null and b/posterior_database/data/data-raw/three_men/three_men_multilingual.csv.zip differ
diff --git a/posterior_database/data/data/three_docs1200.json.zip b/posterior_database/data/data/three_docs1200.json.zip
new file mode 100644
index 00000000..9fc6ca65
Binary files /dev/null and b/posterior_database/data/data/three_docs1200.json.zip differ
diff --git a/posterior_database/data/data/three_men1.json.zip b/posterior_database/data/data/three_men1.json.zip
new file mode 100644
index 00000000..883f1b7c
Binary files /dev/null and b/posterior_database/data/data/three_men1.json.zip differ
diff --git a/posterior_database/data/data/three_men2.json.zip b/posterior_database/data/data/three_men2.json.zip
new file mode 100644
index 00000000..c2b0067e
Binary files /dev/null and b/posterior_database/data/data/three_men2.json.zip differ
diff --git a/posterior_database/data/data/three_men3.json.zip b/posterior_database/data/data/three_men3.json.zip
new file mode 100644
index 00000000..47058a3e
Binary files /dev/null and b/posterior_database/data/data/three_men3.json.zip differ
diff --git a/posterior_database/data/info/three_docs1200.info.json b/posterior_database/data/info/three_docs1200.info.json
new file mode 100644
index 00000000..9acdfb03
--- /dev/null
+++ b/posterior_database/data/info/three_docs1200.info.json
@@ -0,0 +1,10 @@
+{
+  "name": "three_docs1200",
+  "keywords": ["simulated data", "text"],
+  "title": "A simulated textual dataset",
+  "description": "The dataset contain three document, three words and 1200 tokens. The model is generated based on three topics, and if only two topics are used, the posterior become multimodal.",
+  "references": "jonasson2021rapid",
+  "added_date": "2021-02-26",
+  "added_by": "Mans Magnusson",
+  "data_file": "data/data/three_docs1200.json"
+}
diff --git a/posterior_database/data/info/three_men1.info.json b/posterior_database/data/info/three_men1.info.json
new file mode 100644
index 00000000..8d227974
--- /dev/null
+++ b/posterior_database/data/info/three_men1.info.json
@@ -0,0 +1,11 @@
+{
+  "name": "three_men1",
+  "keywords": "text",
+  "title": "Three Men and a Boat (corpus 1)",
+  "description": "The data consist of a pre-processed multilingual corpus of the book 'Three men and a boat' in English, French, and Swedish. The corpus has been processed so that a small proportion of the corpus is in Swedish and French (roughly 20%). The original translation was made by Andras Farka. Corpus 1 divide up the chapters som that there are in total 6 documents, 2 English, 2 French and 2 Swedish documents.",
+  "references": ["jonasson2021rapid", "farkas2014threemen"],
+  "urls": "https://farkastranslations.com/books/Jerome_Jerome_K-Three_Men_in_a_Boat-en-fr-it-sv.html",
+  "added_date": "2021-02-26",
+  "added_by": "Mans Magnusson",
+  "data_file": "data/data/three_men1.json"
+}
diff --git a/posterior_database/data/info/three_men2.info.json b/posterior_database/data/info/three_men2.info.json
new file mode 100644
index 00000000..be8ef098
--- /dev/null
+++ b/posterior_database/data/info/three_men2.info.json
@@ -0,0 +1,11 @@
+{
+  "name": "three_men2",
+  "keywords": "text",
+  "title": "Three Men and a Boat (corpus 2)",
+  "description": "The data consist of a pre-processed multilingual corpus of the book 'Three men and a boat' in English, French, and Swedish. The corpus has been processed so that a small proportion of the corpus is in Swedish and French (roughly 20%). The original translation was made by Andras Farka. Corpus 1 divide up the chapters som that there are in total 14 documents, 10 English, 2 French and 2 Swedish documents.",
+  "references": ["jonasson2021rapid", "farkas2014threemen"],
+  "urls": "https://farkastranslations.com/books/Jerome_Jerome_K-Three_Men_in_a_Boat-en-fr-it-sv.html",
+  "added_date": "2021-02-26",
+  "added_by": "Mans Magnusson",
+  "data_file": "data/data/three_men2.json"
+}
diff --git a/posterior_database/data/info/three_men3.info.json b/posterior_database/data/info/three_men3.info.json
new file mode 100644
index 00000000..c4e36e9a
--- /dev/null
+++ b/posterior_database/data/info/three_men3.info.json
@@ -0,0 +1,11 @@
+{
+  "name": "three_men3",
+  "keywords": "text",
+  "title": "Three Men and a Boat (corpus 3)",
+  "description": "The data consist of a pre-processed multilingual corpus of the book 'Three men and a boat' in English, French, and Swedish. The corpus has been processed so that a small proportion of the corpus is in Swedish and French (roughly 20%). The original translation was made by Andras Farka. Corpus 1 divide up the chapters som that there are in total 9 documents, 5 English, 2 French and 2 Swedish documents.",
+  "references": ["jonasson2021rapid", "farkas2014threemen"],
+  "urls": "https://farkastranslations.com/books/Jerome_Jerome_K-Three_Men_in_a_Boat-en-fr-it-sv.html",
+  "added_date": "2021-02-26",
+  "added_by": "Mans Magnusson",
+  "data_file": "data/data/three_men3.json"
+}
diff --git a/posterior_database/models/info/ldaK2.info.json b/posterior_database/models/info/ldaK2.info.json
new file mode 100644
index 00000000..4a568722
--- /dev/null
+++ b/posterior_database/models/info/ldaK2.info.json
@@ -0,0 +1,15 @@
+{
+  "name": "ldaK2",
+  "keywords": "mixed_membership",
+  "title": "LDA with 2 topics",
+  "description": "Latent Dirichlet Allocation (LDA) with token indicators marginalized out and 2 topics.",
+  "urls": "https://mc-stan.org/docs/2_18/stan-users-guide/latent-dirichlet-allocation.html",
+  "references": "blei2003latent",
+  "added_by": "Mans Magnusson",
+  "added_date": "2021-02-26",
+  "model_implementations": {
+    "stan": {
+      "model_code": "models/stan/ldaK2.stan"
+    }
+  }
+}
diff --git a/posterior_database/models/info/ldaK5.info.json b/posterior_database/models/info/ldaK5.info.json
index 0550fe73..43925110 100644
--- a/posterior_database/models/info/ldaK5.info.json
+++ b/posterior_database/models/info/ldaK5.info.json
@@ -1,8 +1,10 @@
 {
   "name": "ldaK5",
   "title": "LDA with 5 topics",
-  "description": "Latent Dirichlet Allocation (LDA) with token indicators summed out.",
-  "keywords": [],
+  "description": "Latent Dirichlet Allocation (LDA) with token indicators marginalized out.",
+  "keywords": [
+    "mixed_membership"
+  ],
   "references": [
     "blei2003latent"
   ],
diff --git a/posterior_database/models/stan/ldaK2.stan b/posterior_database/models/stan/ldaK2.stan
new file mode 100644
index 00000000..cb9e6d1b
--- /dev/null
+++ b/posterior_database/models/stan/ldaK2.stan
@@ -0,0 +1,32 @@
+data {
+  int<lower=2> V;               // num words
+  int<lower=1> M;               // num docs
+  int<lower=1> N;               // total word instances
+  int<lower=1,upper=V> w[N];    // word n
+  int<lower=1,upper=M> doc[N];  // doc ID for word n
+}
+transformed data {
+  int<lower=1> K = 2;
+  vector<lower=0>[K] alpha; // topic prior
+  vector<lower=0>[V] beta; // word prior
+  for (k in 1:K)
+    alpha[k] = 1;
+  for (v in 1:V)
+    beta[v] = 1;
+}
+parameters {
+  simplex[K] theta[M];   // topic dist for doc m
+  simplex[V] phi[K];     // word dist for topic k
+}
+model {
+  for (m in 1:M)
+    theta[m] ~ dirichlet(alpha);  // prior
+  for (k in 1:5)
+    phi[k] ~ dirichlet(beta);     // prior
+  for (n in 1:N) {
+    real gamma[K];
+    for (k in 1:K)
+      gamma[k] = log(theta[doc[n], k]) + log(phi[k, w[n]]);
+    target += log_sum_exp(gamma);  // likelihood;
+  }
+}
diff --git a/posterior_database/posteriors/three_docs1200-ldaK2.json b/posterior_database/posteriors/three_docs1200-ldaK2.json
new file mode 100644
index 00000000..e142b731
--- /dev/null
+++ b/posterior_database/posteriors/three_docs1200-ldaK2.json
@@ -0,0 +1,14 @@
+{
+  "keywords": ["multimodal", "text"],
+  "references": "jonasson2021rapid",
+  "dimensions": {
+    "theta": 6,
+    "phi": 6
+  },
+  "reference_posterior_name": null,
+  "added_date": "2021-02-26",
+  "added_by": "Mans Magnusson",
+  "name": "three_docs1200-ldaK2",
+  "model_name": "ldaK2",
+  "data_name": "three_docs1200"
+}
diff --git a/posterior_database/posteriors/three_men1-ldaK2.json b/posterior_database/posteriors/three_men1-ldaK2.json
new file mode 100644
index 00000000..047f6cf6
--- /dev/null
+++ b/posterior_database/posteriors/three_men1-ldaK2.json
@@ -0,0 +1,14 @@
+{
+  "keywords": ["multimodal", "text"],
+  "references": "jonasson2021rapid",
+  "dimensions": {
+    "theta": 12,
+    "phi": 498
+  },
+  "reference_posterior_name": null,
+  "added_date": "2021-02-26",
+  "added_by": "Mans Magnusson",
+  "name": "three_men1-ldaK2",
+  "model_name": "ldaK2",
+  "data_name": "three_men1"
+}
diff --git a/posterior_database/posteriors/three_men2-ldaK2.json b/posterior_database/posteriors/three_men2-ldaK2.json
new file mode 100644
index 00000000..d5569612
--- /dev/null
+++ b/posterior_database/posteriors/three_men2-ldaK2.json
@@ -0,0 +1,14 @@
+{
+  "keywords": ["multimodal", "text"],
+  "references": "jonasson2021rapid",
+  "dimensions": {
+    "theta": 28,
+    "phi": 498
+  },
+  "reference_posterior_name": null,
+  "added_date": "2021-02-26",
+  "added_by": "Mans Magnusson",
+  "name": "three_men2-ldaK2",
+  "model_name": "ldaK2",
+  "data_name": "three_men2"
+}
diff --git a/posterior_database/posteriors/three_men3-ldaK2.json b/posterior_database/posteriors/three_men3-ldaK2.json
new file mode 100644
index 00000000..96538b35
--- /dev/null
+++ b/posterior_database/posteriors/three_men3-ldaK2.json
@@ -0,0 +1,14 @@
+{
+  "keywords": ["multimodal", "text"],
+  "references": "jonasson2021rapid",
+  "dimensions": {
+    "theta": 18,
+    "phi": 498
+  },
+  "reference_posterior_name": null,
+  "added_date": "2021-02-26",
+  "added_by": "Mans Magnusson",
+  "name": "three_men3-ldaK2",
+  "model_name": "ldaK2",
+  "data_name": "three_men3"
+}
diff --git a/rpackage/man/pdb_local.Rd b/rpackage/man/pdb_local.Rd
index 6477fa22..1fbad93e 100644
--- a/rpackage/man/pdb_local.Rd
+++ b/rpackage/man/pdb_local.Rd
@@ -26,7 +26,7 @@ pdb_default(cache_path = tempdir())
 pdb_config(directory = getwd())
 
 pdb_github(
-  repo = getOption("pdb_repo", "MansMeg/posteriordb/posterior_database"),
+  repo = getOption("pdb_repo", "stan-dev/posteriordb/posterior_database"),
   cache_path = tempdir(),
   ref = github_ref(),
   subdir = NULL,


Test patch:
diff --git a/rpackage/tests/testthat/test-doc-README.R b/rpackage/tests/testthat/test-doc-README.R
index f0140808..f827017e 100644
--- a/rpackage/tests/testthat/test-doc-README.R
+++ b/rpackage/tests/testthat/test-doc-README.R
@@ -21,7 +21,7 @@ test_that("README.md works as stated", {
   # Please check that no code has been changed or update this test suite accordingly
   # Then change the hash to the md5 of the new updated file.
   md5_hash <- digest::digest(readLines(fp_to_README_md), algo = "md5")
-  expect_equal(md5_hash, "a4d6669ea9dbb5d95253e4d0342b215b")
+  expect_equal(md5_hash, "876b5e91919b555eab66fa75f8e11459")
 
   skip_if(is.null(github_pat()))