Repository: piskvorky/gensim

Issue:
Add script to verify that autogenerated docs are up to date
I see no real alternative either. Doesn't moving parts of our docs + testing elsewhere simply shift the problem around? That new place will still need testing, whenever things change. No difference.

Except code + docs in one place (=now) makes more sense conceptually IMO.

Yes, scripts / hooks that verify docs are in-sync, preferably at PR merge-time, would be nice.

_Originally posted by @piskvorky in https://github.com/RaRe-Technologies/gensim/pull/2907#issuecomment-671317949_


Discussion:
During each PR build, our CI scripts should check that the autogenerated files are in sync with their parents. If they are out of sync, the CI build should crash. The PR author should then bring the files up to date and add them to the PR.

Resolving patch:
diff --git a/docs/src/auto_examples/tutorials/run_lda.py b/docs/src/auto_examples/tutorials/run_lda.py
--- a/docs/src/auto_examples/tutorials/run_lda.py
+++ b/docs/src/auto_examples/tutorials/run_lda.py
@@ -58,6 +58,8 @@
 #     careful before applying the code to a large dataset.
 #
 
+import io
+import os.path
 import re
 import tarfile
 
@@ -120,7 +122,7 @@ def extract_documents(url='https://cs.nyu.edu/~roweis/data/nips12raw_str602.tgz'
 
 ###############################################################################
 # We use the WordNet lemmatizer from NLTK. A lemmatizer is preferred over a
-# stemmer in this case because it produces more readable words. An output that is
+# stemmer in this case because it produces more readable words. Output that is
 # easy to read is very desirable in topic modelling.
 #
 
@@ -149,7 +151,7 @@ def extract_documents(url='https://cs.nyu.edu/~roweis/data/nips12raw_str602.tgz'
 # Compute bigrams.
 from gensim.models import Phrases
 
-# Add bigrams to docs (only ones that appear 20 times or more).
+# Add bigrams and trigrams to docs (only ones that appear 20 times or more).
 bigram = Phrases(docs, min_count=20)
 for idx in range(len(docs)):
     for token in bigram[docs[idx]]:
@@ -195,20 +197,19 @@ def extract_documents(url='https://cs.nyu.edu/~roweis/data/nips12raw_str602.tgz'
 # We are ready to train the LDA model. We will first discuss how to set some of
 # the training parameters.
 #
-# First of all, the elephant in the room: how many topics do I need?
-# There is really no easy answer for this. It will depend on both your
-# data and your application. I have used 10 topics here because I wanted
-# to have a few topics that I could interpret and "label", and because that
-# turned out to give me reasonably good results. On the other hand, you might
-# not need to interpret all your topics, so you could use many topics,
-# for example, 100.
+# First of all, the elephant in the room: how many topics do I need? There is
+# really no easy answer for this, it will depend on both your data and your
+# application. I have used 10 topics here because I wanted to have a few topics
+# that I could interpret and "label", and because that turned out to give me
+# reasonably good results. You might not need to interpret all your topics, so
+# you could use a large number of topics, for example 100.
 #
 # ``chunksize`` controls how many documents are processed at a time in the
 # training algorithm. Increasing chunksize will speed up training, at least as
 # long as the chunk of documents easily fit into memory. I've set ``chunksize =
-# 2000``, which is more than the number of documents, so I process all the
-# data in one go. However, chunksize can influence the quality of the model, as
-# discussed in Hoffman and al. [2], but the difference was not
+# 2000``, which is more than the amount of documents, so I process all the
+# data in one go. Chunksize can however influence the quality of the model, as
+# discussed in Hoffman and co-authors [2], but the difference was not
 # substantial in this case.
 #
 # ``passes`` controls how often we train the model on the entire corpus.
@@ -218,9 +219,8 @@ def extract_documents(url='https://cs.nyu.edu/~roweis/data/nips12raw_str602.tgz'
 # "iterations" high enough.
 #
 # I suggest the following way to choose iterations and passes. First, enable
-# logging (``logging.basicConfig(level=logging.INFO, format='PID:%(process)d:%(threadName)s -
-# %(asctime)s - %(levelname)s - %(filename)s:%(lineno)s - %(message)s')``), and set ``eval_every = 1``
-# in ``LdaModel``. Then, when training the model, look for a line in the log that
+# logging (as described in many Gensim tutorials), and set ``eval_every = 1``
+# in ``LdaModel``. When training the model look for a line in the log that
 # looks something like this::
 #
 #    2016-06-21 15:40:06,753 - gensim.models.ldamodel - DEBUG - 68/1566 documents converged within 400 iterations
diff --git a/docs/src/check_gallery.py b/docs/src/check_gallery.py
new file mode 100644
--- /dev/null
+++ b/docs/src/check_gallery.py
@@ -0,0 +1,69 @@
+"""Check that the cached gallery files are up to date.
+
+If they are stale, then Sphinx will attempt to rebuild them from source.  When
+running the documentation build on CI, we want to avoid rebuilding the gallery,
+because that takes too long.  Instead, we use this script to warn the author of
+the PR that they need to rebuild the docs themselves.
+"""
+
+import hashlib
+import os
+import sys
+
+
+def different(path1, path2):
+    with open(path1) as fin:
+        f1 = fin.read()
+    with open(path2) as fin:
+        f2 = fin.read()
+    return f1 != f2
+
+
+curr_dir = os.path.dirname(__file__)
+stale = []
+for root, dirs, files in os.walk(os.path.join(curr_dir, 'gallery')):
+    for f in files:
+        if f.endswith('.py'):
+            source_path = os.path.join(root, f)
+            cache_path = source_path.replace('docs/src/gallery/', 'docs/src/auto_examples/')
+
+            #
+            # We check two things:
+            #
+            #   1) Actual file content
+            #   2) MD5 checksums
+            #
+            # We check 1) because that's the part that matters to the user -
+            # it's what will appear in the documentation.  We check 2) because
+            # that's what Sphinx Gallery relies on to decide what it needs to
+            # rebuild.  In practice, only one of these checks is necessary,
+            # but we run them both because it's trivial.
+            #
+            if different(source_path, cache_path):
+                stale.append(cache_path)
+                continue
+
+            actual_md5 = hashlib.md5()
+            with open(source_path, 'rb') as fin:
+                actual_md5.update(fin.read())
+
+            md5_path = cache_path + '.md5'
+            with open(md5_path) as fin:
+                expected_md5 = fin.read()
+
+            if actual_md5.hexdigest() != expected_md5:
+                stale.append(cache_path)
+
+if stale:
+    print(f"""The gallery cache appears stale.
+
+Rebuild the documentation using the following commands from the gensim root subdirectory:
+
+    pip install -e .[docs]
+    make -C docs/src html
+
+and then run `git add docs/src/auto_examples` to update the cache.
+
+Stale files: {stale}
+""", file=sys.stderr)
+    sys.exit(1)
diff --git a/docs/src/gallery/tutorials/run_lda.py b/docs/src/gallery/tutorials/run_lda.py
--- a/docs/src/gallery/tutorials/run_lda.py
+++ b/docs/src/gallery/tutorials/run_lda.py
@@ -245,7 +245,7 @@ def extract_documents(url='https://cs.nyu.edu/~roweis/data/nips12raw_str602.tgz'
 iterations = 400
 eval_every = None  # Don't evaluate model perplexity, takes too much time.
 
-# Make a index to word dictionary.
+# Make an index to word dictionary.
 temp = dictionary[0]  # This is only to "load" the dictionary.
 id2word = dictionary.id2token
 
@@ -278,7 +278,7 @@ def extract_documents(url='https://cs.nyu.edu/~roweis/data/nips12raw_str602.tgz'
 # methods on the blog at http://rare-technologies.com/lda-training-tips/ !
 #
 
-top_topics = model.top_topics(corpus) #, num_words=20)
+top_topics = model.top_topics(corpus)
 
 # Average topic coherence is the sum of topic coherences of all topics, divided by the number of topics.
 avg_topic_coherence = sum([t[1] for t in top_topics]) / num_topics


Test patch:
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -28,6 +28,9 @@ jobs:
       # - name: Run flake8 linter (documentation)
       #   run: flake8 --ignore E202,E402,E302,E305,F821 --max-line-length 120 --filename '*.py,*.rst' docs
 
+      - name: Check Sphinx Gallery cache
+        run: python docs/src/check_gallery.py
+
   docs:
     name: build documentation
     timeout-minutes: 10