Repository: getredash/redash

Issue:
Set TTL on job keys in Redis
To make sure the memory footprint stays constant and manageable.



Discussion:

Resolving patch:
diff --git a/bin/latest_release.py b/bin/latest_release.py
new file mode 100755
--- /dev/null
+++ b/bin/latest_release.py
@@ -0,0 +1,25 @@
+#!/usr/bin/env python
+import sys
+import requests
+
+if __name__ == '__main__':
+    response = requests.get('https://api.github.com/repos/EverythingMe/redash/releases')
+
+    if response.status_code != 200:
+        exit("Failed getting releases (status code: %s)." % response.status_code)
+
+    sorted_releases = sorted(response.json(), key=lambda release: release['id'], reverse=True)
+
+    latest_release = sorted_releases[0]
+    asset_url = latest_release['assets'][0]['url']
+
+    if '--url-only' in sys.argv:
+        print asset_url
+    else:
+        print "Latest release: %s" % latest_release['tag_name']
+        print latest_release['body']
+
+        print "\nTarball URL: %s" % asset_url
+        print 'wget: wget --header="Accept: application/octet-stream" %s' % asset_url
+
+
diff --git a/bin/upload_version.py b/bin/upload_version.py
--- a/bin/upload_version.py
+++ b/bin/upload_version.py
@@ -3,30 +3,44 @@
 import sys
 import json
 import requests
+import subprocess
+
+
+def capture_output(command):
+    proc = subprocess.Popen(command, stdout=subprocess.PIPE)
+    return proc.stdout.read()
+
 
 if __name__ == '__main__':
-  version = sys.argv[1]
-  filepath = sys.argv[2]
-  filename = filepath.split('/')[-1]
-  github_token = os.environ['GITHUB_TOKEN']
-  auth = (github_token, 'x-oauth-basic')
-  commit_sha = os.environ['CIRCLE_SHA1']
-
-  params = json.dumps({
-    'tag_name': 'v{0}'.format(version),
-    'name': 're:dash v{0}'.format(version),
-    'target_commitish': commit_sha,
-    'prerelease': True
-  })
-
-  response = requests.post('https://api.github.com/repos/everythingme/redash/releases',
-                            data=params,
-                            auth=auth)
-
-  upload_url = response.json()['upload_url']
-  upload_url = upload_url.replace('{?name}', '')
-
-  with open(filepath) as file_content:
-    headers = {'Content-Type': 'application/gzip'}
-    response = requests.post(upload_url, file_content, params={'name': filename}, auth=auth, headers=headers, verify=False)
+    version = sys.argv[1]
+    filepath = sys.argv[2]
+    filename = filepath.split('/')[-1]
+    github_token = os.environ['GITHUB_TOKEN']
+    auth = (github_token, 'x-oauth-basic')
+    commit_sha = os.environ['CIRCLE_SHA1']
+
+    commit_body = capture_output(["git", "log", "--format=%b", "-n", "1", commit_sha])
+    file_md5_checksum = capture_output(["md5sum", filename]).split()[0]
+    file_sha256_checksum = capture_output(["sha256sum", filename]).split()[0]
+    version_body = "%s\n\nMD5: %s\nSHA256: %s" % (commit_body, file_md5_checksum, file_sha256_checksum)
+
+    params = json.dumps({
+        'tag_name': 'v{0}'.format(version),
+        'name': 're:dash v{0}'.format(version),
+        'body': version_body,
+        'target_commitish': commit_sha,
+        'prerelease': True
+    })
+
+    response = requests.post('https://api.github.com/repos/everythingme/redash/releases',
+                             data=params,
+                             auth=auth)
+
+    upload_url = response.json()['upload_url']
+    upload_url = upload_url.replace('{?name}', '')
+
+    with open(filepath) as file_content:
+        headers = {'Content-Type': 'application/gzip'}
+        response = requests.post(upload_url, file_content, params={'name': filename}, auth=auth,
+                                 headers=headers, verify=False)
 
diff --git a/redash/data/worker.py b/redash/data/worker.py
--- a/redash/data/worker.py
+++ b/redash/data/worker.py
@@ -157,9 +157,13 @@ def cancel(self):
             return
 
         if self.status == self.PROCESSING:
-            os.kill(self.process_id, signal.SIGINT)
-        else:
-            self.done(None, "Interrupted/Cancelled while running.")
+            try:
+                os.kill(self.process_id, signal.SIGINT)
+            except OSError as e:
+                logging.warning("[%s] Tried to cancel job but os.kill failed (pid=%d, error=%s)",
+                                self.id, self.process_id, e)
+
+        self.done(None, "Interrupted/Cancelled while running.")
 
     def save(self, pipe=None):
         if not pipe:
@@ -173,6 +177,9 @@ def save(self, pipe=None):
 
         super(Job, self).save(pipe)
 
+    def expire(self, expire_time):
+        self.redis_connection.expire(self._redis_key(self.id), expire_time)
+
     def processing(self, process_id):
         self.update(status=self.PROCESSING,
                     process_id=process_id,
@@ -279,6 +286,8 @@ def _fork_and_process(self, job_id):
                                  self.name, job_id)
                     job.done(None, "Interrupted/Cancelled while running.")
 
+            job.expire(24 * 3600)
+
             logging.info("[%s] Finished Processing %s (pid: %d status: %d)",
                          self.name, job_id, self.child_pid, status)
 


Test patch:
diff --git a/tests/test_job.py b/tests/test_job.py
--- a/tests/test_job.py
+++ b/tests/test_job.py
@@ -98,3 +98,14 @@ def test_unicode_serialization(self):
         loaded_job = Job.load(redis_connection, job.id)
         self.assertEquals(loaded_job.query, unicode_query)
 
+    def test_cancel_job_with_no_process(self):
+        job = Job(redis_connection, query=self.query, priority=self.priority)
+        job.status = Job.PROCESSING
+        job.process_id = 699999
+        job.save()
+
+        job.cancel()
+
+        job = Job.load(redis_connection, job.id)
+
+        self.assertEquals(job.status, Job.FAILED)