Repository: bokeh/bokeh

Issue:
Improve Sampledata downloading
compare md5sums of files and only download when necessary


Discussion:
Just noting that there was a relatively large spike in sample data usage the last few days to 5-6 USD  per day. Not a problem every once in a while but not sustainable if it persists. I would actually consider having `bokeh sampledata` prompt to ask permission to send more detailed telemetry (python version, OS, bokeh version, etc) in a user agent as well.
Additionally: If we move the list of files to download to an external JSON file we can hash that file and utilize GH CI caching to not actually fetch it often or ever. 

Resolving patch:
diff --git a/bokeh/util/sampledata.py b/bokeh/util/sampledata.py
--- a/bokeh/util/sampledata.py
+++ b/bokeh/util/sampledata.py
@@ -22,6 +22,8 @@
 # imports are generally deferrered in this module
 
 # Standard library imports
+import hashlib
+import json
 from os import mkdir, remove
 from os.path import abspath, dirname, exists, expanduser, isdir, isfile, join, splitext
 from sys import stdout
@@ -52,31 +54,26 @@ def download(progress: bool = True) -> None:
 
     # HTTP requests are cheaper for us, and there is nothing private to protect
     s3 = 'http://sampledata.bokeh.org'
-    files = [
-        (s3, 'CGM.csv'),
-        (s3, 'US_Counties.zip'),
-        (s3, 'us_cities.json'),
-        (s3, 'unemployment09.csv'),
-        (s3, 'AAPL.csv'),
-        (s3, 'FB.csv'),
-        (s3, 'GOOG.csv'),
-        (s3, 'IBM.csv'),
-        (s3, 'MSFT.csv'),
-        (s3, 'WPP2012_SA_DB03_POPULATION_QUINQUENNIAL.zip'),
-        (s3, 'gapminder_fertility.csv'),
-        (s3, 'gapminder_population.csv'),
-        (s3, 'gapminder_life_expectancy.csv'),
-        (s3, 'gapminder_regions.csv'),
-        (s3, 'world_cities.zip'),
-        (s3, 'airports.json'),
-        (s3, 'movies.db.zip'),
-        (s3, 'airports.csv'),
-        (s3, 'routes.csv'),
-        (s3, 'haarcascade_frontalface_default.xml'),
-    ]
-
-    for base_url, filename in files:
-        _download_file(base_url, filename, data_dir, progress=progress)
+    files = json.load(open(join(dirname(__file__), "sampledata.json")))
+
+    for filename, md5 in files:
+        real_name, ext = splitext(filename)
+        if ext == '.zip':
+            if not splitext(real_name)[1]:
+                real_name += ".csv"
+        else:
+            real_name += ext
+        real_path = join(data_dir, real_name)
+
+        if exists(real_path):
+            local_md5 = hashlib.md5(open(real_path,'rb').read()).hexdigest()
+            if local_md5 == md5:
+                print(f"Skipping {filename!r} (checksum match)")
+                continue
+            else:
+                print(f"Re-fetching {filename!r} (checksum mismatch)")
+
+        _download_file(s3, filename, data_dir, progress=progress)
 
 #-----------------------------------------------------------------------------
 # Dev API
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -111,7 +111,7 @@
 package_path(join(ROOT, 'bokeh', 'sphinxext', '_templates'))
 package_path(join(ROOT, 'bokeh', 'server', 'views'), ('.html'))
 package_path(join(ROOT, 'bokeh', 'sampledata', '_data'))
-package_files('LICENSE.txt', 'themes/*.yaml', 'themes/*.json')
+package_files('_sri.json', 'LICENSE.txt', 'themes/*.yaml', 'themes/*.json', 'util/sampledata.json')
 
 setup(
     # basic package metadata


Test patch:
diff --git a/bokeh/_testing/plugins/examples_report.py b/bokeh/_testing/plugins/examples_report.py
--- a/bokeh/_testing/plugins/examples_report.py
+++ b/bokeh/_testing/plugins/examples_report.py
@@ -68,7 +68,7 @@ def get_all_examples(config):
     if _examples is None:
         base_dir = abspath(join(dirname(__file__), pardir, pardir, pardir))
 
-        _examples = collect_examples(join(base_dir, "examples.yaml"))
+        _examples = collect_examples(join(base_dir, "tests", "examples.yaml"))
 
         for example in _examples:
             example._diff_ref = config.option.diff_ref
diff --git a/bokeh/_testing/util/examples.py b/bokeh/_testing/util/examples.py
--- a/bokeh/_testing/util/examples.py
+++ b/bokeh/_testing/util/examples.py
@@ -30,6 +30,7 @@
     isfile,
     join,
     normpath,
+    pardir,
     relpath,
     splitext,
 )
@@ -345,7 +346,7 @@ def add_examples(list_of_examples, path, examples_dir, example_type=None, slow=N
 
 
 def collect_examples(config_path):
-    examples_dir = dirname(config_path)
+    examples_dir = join(dirname(config_path), pardir)
     list_of_examples = []
 
     with open(config_path, "r") as f:
diff --git a/examples.yaml b/tests/examples.yaml
similarity index 100%
rename from examples.yaml
rename to tests/examples.yaml