Repository: weecology/retriever

Issue:
Distribute and update scripts on install
We want the software to be immediately functional on install without having to run any extra commands. Currently following install it is necessary to run `retriever update` to download all of the scripts.

Future releases should do two things:
1. Auto-populate `.retriever/scripts` with the scripts at the state of the release being installed.
2. Automatically run `retriever update` (or something equivalent) at the end of installation to update any scripts that have changed since the release.

This combination makes sure that:
1. The retriever is ready to go immediately following the installation
2. The user has the most up-to-date scripts immediately following installation
3. Installation is as fast as possible because instead of separately downloading all of the scripts only scripts that have changed since the last release need to be updated.

This issue is a detailed treatment of one of the issues in #318.



Discussion:

Resolving patch:
diff --git a/lib/repository.py b/lib/repository.py
--- a/lib/repository.py
+++ b/lib/repository.py
@@ -1,18 +1,16 @@
 """Checks the repository for updates."""
 from __future__ import print_function
 from future import standard_library
-standard_library.install_aliases()
-
-
 import os
 import sys
-import urllib.request, urllib.parse, urllib.error
+import urllib.request
+import urllib.parse
+import urllib.error
 import imp
-from hashlib import md5
-from inspect import getsourcelines
-from threading import Thread
-from retriever import REPOSITORY, VERSION, MASTER_BRANCH, REPO_URL, SCRIPT_WRITE_PATH, HOME_DIR
+from pkg_resources import parse_version
+from retriever import REPOSITORY, SCRIPT_WRITE_PATH, HOME_DIR
 from retriever.lib.models import file_exists
+standard_library.install_aliases()
 
 global abort, executable_name
 abort = False
@@ -29,123 +27,79 @@ def download_from_repository(filepath, newpath, repo=REPOSITORY):
         pass
 
 
-def more_recent(latest, current):
-    """Given two version number strings, returns True if the first is more recent."""
-    latest_parts = latest.split('.')
-    current_parts = current.split('.')
-    for n in range(len(latest_parts)):
-        l = latest_parts[n]
-        if len(current_parts) < (n + 1):
-            return (l != "rc")
-        c = current_parts[n]
-        if l > c:
-            return True
-        elif c > l:
-            return False
-    return (len(current_parts) > (n + 1) and current_parts[n + 1] == "rc")
-
-
 def check_for_updates():
-    """Check for updates to scripts and executable."""
-    init = InitThread()
-    init.run()
+    """Check for updates to scripts.
 
+    This updates the HOME_DIR scripts directory with the latest script versions
+    """
+    try:
+        # open version.txt for current release branch and get script versions
+        version_file = urllib.request.urlopen(REPOSITORY + "version.txt")
+        version_file.readline()
+
+        # read scripts from the repository and the checksums from the version.txt
+        scripts = []
+        print("Downloading scripts...")
+        for line in version_file:
+            scripts.append(line.decode().strip('\n').split(','))
+
+        total_script_count = len(scripts)
+
+        # create script directory if not available
+        if not os.path.isdir(SCRIPT_WRITE_PATH):
+            os.makedirs(SCRIPT_WRITE_PATH)
+
+        update_progressbar(0.0 / float(total_script_count))
+        for index, script in enumerate(scripts):
+            script_name = script[0]
+            if len(script) > 1:
+                script_version = script[1]
+            else:
+                script_version = None
+
+            path_script_name = os.path.normpath(os.path.join(HOME_DIR, "scripts", script_name))
+            if not file_exists(path_script_name):
+                download_from_repository("scripts/" + script_name,
+                                         os.path.normpath(os.path.join(SCRIPT_WRITE_PATH, script_name)))
+
+            need_to_download = False
+            try:
+                file_object, pathname, desc = imp.find_module(''.join(script_name.split('.')[:-1]), [SCRIPT_WRITE_PATH])
+                new_module = imp.load_module(script_name, file_object, pathname, desc)
+                m = str(new_module.SCRIPT.version)
+                need_to_download = parse_version(str(script_version)) > parse_version(m)
+            except:
+                pass
+            if need_to_download:
+                try:
+                    os.remove(os.path.normpath(os.path.join(HOME_DIR, "scripts", script_name)))
+                    download_from_repository("scripts/" + script_name,
+                                             os.path.normpath(os.path.join(SCRIPT_WRITE_PATH, script_name)))
+                except Exception as e:
+                    print(e)
+                    pass
+            update_progressbar(float(index + 1) / float(total_script_count))
+    except:
+        raise
+        return
     print("\nThe retriever is up-to-date")
 
 
 def update_progressbar(progress):
     """Show progressbar
-
     Takes a number between 0 and 1 to indicate progress from 0 to 100%.
-
+    And set the bar_length according to the console size
     """
-    # Try to set the bar_length according to the console size
     try:
         rows, columns = os.popen('stty size', 'r').read().split()
         bar_length = int(columns) - 35
-        if(not (bar_length > 1)):
+        if not bar_length > 1:
             bar_length = 20
     except:
         # Default value if determination of console size fails
         bar_length = 20
-    block = int(round(bar_length*progress))
+    block = int(round(bar_length * progress))
     text = "\rDownload Progress: [{0}] {1:.2f}%".format(
-        "#"*block + "-"*(bar_length-block), progress*100)
+        "#" * block + "-" * (bar_length - block), progress * 100)
     sys.stdout.write(text)
     sys.stdout.flush()
-
-
-class InitThread(Thread):
-    """This thread performs all of the necessary updates while the splash screen
-    is shown.
-
-    1. Check master/version.txt to get the latest version (Windows only).
-    2. Prompt for update if necessary (Windows only).
-    3. Download latest versions of scripts from current branch."""
-
-    def run(self):
-        try:
-            running_from = os.path.basename(sys.argv[0])
-
-            # NOTE: exe auto-update functionality has been temporarily disabled
-            # since the binaries were moved to AWS.
-
-
-            # open version.txt for current release branch and get script
-            # versions
-            version_file = urllib.request.urlopen(REPOSITORY + "version.txt")
-            version_file.readline()
-
-            # read scripts from the repository and the checksums from the
-            # version.txt
-            scripts = []
-            print("Downloading scripts...")
-            for line in version_file:
-                scripts.append(line.decode().strip('\n').split(','))
-
-            total_script_count = len(scripts)
-
-            # create script directory if not available
-            if not os.path.isdir(SCRIPT_WRITE_PATH):
-                os.makedirs(SCRIPT_WRITE_PATH)
-
-            update_progressbar(0.0/float(total_script_count))
-            for index, script in enumerate(scripts):
-                script_name = script[0]
-                if len(script) > 1:
-                    script_version = script[1]
-                else:
-                    script_version = None
-
-                path_script_name = os.path.normpath(os.path.join(HOME_DIR, "scripts", script_name))
-
-                if not file_exists(path_script_name):
-                    download_from_repository("scripts/" + script_name,
-                                             os.path.normpath(os.path.join(SCRIPT_WRITE_PATH, script_name)))
-
-                # check MD5sum based on the script version to download the right scripts
-                # if the MD5sum doesn't match need_to_download is set to True
-                need_to_download = False
-
-                try:
-                    file, pathname, desc = imp.find_module(''.join(script_name.split('.')[:-1]), ["scripts"])
-                    new_module = imp.load_module(script_name, file, pathname, desc)
-                    m = md5()
-                    m.update(''.join(getsourcelines(new_module)[0]).replace("\r\n", "\n"))
-                    m = m.hexdigest()
-                    need_to_download = script_version != m
-                except:
-                    pass
-
-                if need_to_download:
-                    try:
-                        os.remove(os.path.normpath(os.path.join(HOME_DIR, "scripts", script_name)))
-                        download_from_repository("scripts/" + script_name,
-                                                 os.path.normpath(os.path.join(SCRIPT_WRITE_PATH, script_name)))
-                    except Exception as e:
-                        print(e)
-                        pass
-                update_progressbar(float(index + 1)/float(total_script_count))
-        except:
-            raise
-            return
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -2,9 +2,9 @@
 from __future__ import absolute_import
 
 from setuptools import setup
+from pkg_resources import parse_version
 import platform
-import sys
-import warnings
+
 
 current_platform = platform.system().lower()
 extra_includes = []
@@ -25,14 +25,13 @@
         "C:\\Windows\\winsxs\\x86_microsoft.vc90.crt_1fc8b3b9a1e18e3b_9.0.21022.8_none_bcb86ed6ac711f91")
 
 __version__ = 'v2.0.dev'
-with open("_version.py","w") as version_file:
+with open("_version.py", "w") as version_file:
     version_file.write("__version__ = " + "'" + __version__ + "'\n")
     version_file.close()
 
+
 def clean_version(v):
-    if v == 'master':
-        return '1.0.0'
-    return v.replace('v', '').replace('.rc', '').replace('.beta', '')
+    return parse_version(v).__repr__().lstrip("<Version('").rstrip("')>")
 
 packages = [
     'retriever.lib',
@@ -73,7 +72,7 @@ def clean_version(v):
       classifiers=['Intended Audience :: Science/Research',
                    'License :: OSI Approved :: MIT License',
                    'Programming Language :: Python',
-                   'Programming Language :: Python :: 2',],
+                   'Programming Language :: Python :: 2', ],
       packages=packages,
       package_dir={
           'retriever': ''
@@ -126,6 +125,8 @@ def clean_version(v):
 
 try:
     from retriever.compile import compile
+    from retriever.lib.repository import check_for_updates
     compile()
+    check_for_updates()
 except:
     pass


Test patch: