Repository: deis/deis

Issue:
Registry error on git push
After upgrading the registry to 0.6.8 we are now getting this trace on push:

```
Apr 18 21:42:21 deis sh[3573]: 2014-04-18 21:42:21,083 INFO: 10.212.0.6 - - [18/Apr/2014:21:42:21] "PUT /v1/images/fe6d61cb3ef5499a9fcceebd8f6aabcdfe6d61cb3ef5499a9fcceebd8f6aabcd/json HTTP/1.1" 200 4 "-" "python-requests/2.2.1 CPython/2.7.3 Linux/3.13.10+"
Apr 18 21:42:21 deis sh[3573]: 2014-04-18 21:42:21,089 ERROR: Exception on /v1/images/fe6d61cb3ef5499a9fcceebd8f6aabcdfe6d61cb3ef5499a9fcceebd8f6aabcd/layer [PUT]
Apr 18 21:42:21 deis sh[3573]: Traceback (most recent call last):
Apr 18 21:42:21 deis sh[3573]: File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1687, in wsgi_app
Apr 18 21:42:21 deis sh[3573]: response = self.full_dispatch_request()
Apr 18 21:42:21 deis sh[3573]: File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1360, in full_dispatch_request
Apr 18 21:42:21 deis sh[3573]: rv = self.handle_user_exception(e)
Apr 18 21:42:21 deis sh[3573]: File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1358, in full_dispatch_request
Apr 18 21:42:21 deis sh[3573]: rv = self.dispatch_request()
Apr 18 21:42:21 deis sh[3573]: File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1344, in dispatch_request
Apr 18 21:42:21 deis sh[3573]: return self.view_functions[rule.endpoint](**req.view_args)
Apr 18 21:42:21 deis sh[3573]: File "/docker-registry/registry/toolkit.py", line 237, in wrapper
Apr 18 21:42:21 deis sh[3573]: return f(*args, **kwargs)
Apr 18 21:42:21 deis sh[3573]: File "/docker-registry/registry/images.py", line 215, in put_image_layer
Apr 18 21:42:21 deis sh[3573]: if toolkit.DockerVersion() < '0.10':
Apr 18 21:42:21 deis sh[3573]: File "/docker-registry/registry/toolkit.py", line 30, in __init__
Apr 18 21:42:21 deis sh[3573]: raise RuntimeError('toolkit.DockerVersion: cannot parse version')
Apr 18 21:42:21 deis sh[3573]: RuntimeError: toolkit.DockerVersion: cannot parse version
Apr 18 21:42:21 deis sh[3573]: 2014-04-18 21:42:21,089 ERROR: Exception on /v1/images/fe6d61cb3ef5499a9fcceebd8f6aabcdfe6d61cb3ef5499a9fcceebd8f6aabcd/layer [PUT]
Apr 18 21:42:21 deis sh[3573]: Traceback (most recent call last):
Apr 18 21:42:21 deis sh[3573]: File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1687, in wsgi_app
Apr 18 21:42:21 deis sh[3573]: response = self.full_dispatch_request()
Apr 18 21:42:21 deis sh[3573]: File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1360, in full_dispatch_request
Apr 18 21:42:21 deis sh[3573]: rv = self.handle_user_exception(e)
Apr 18 21:42:21 deis sh[3573]: File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1358, in full_dispatch_request
Apr 18 21:42:21 deis sh[3573]: rv = self.dispatch_request()
Apr 18 21:42:21 deis sh[3573]: File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1344, in dispatch_request
Apr 18 21:42:21 deis sh[3573]: return self.view_functions[rule.endpoint](**req.view_args)
Apr 18 21:42:21 deis sh[3573]: File "/docker-registry/registry/toolkit.py", line 237, in wrapper
Apr 18 21:42:21 deis sh[3573]: return f(*args, **kwargs)
Apr 18 21:42:21 deis sh[3573]: File "/docker-registry/registry/images.py", line 215, in put_image_layer
Apr 18 21:42:21 deis sh[3573]: if toolkit.DockerVersion() < '0.10':
Apr 18 21:42:21 deis sh[3573]: File "/docker-registry/registry/toolkit.py", line 30, in __init__
Apr 18 21:42:21 deis sh[3573]: raise RuntimeError('toolkit.DockerVersion: cannot parse version')
Apr 18 21:42:21 deis sh[3573]: RuntimeError: toolkit.DockerVersion: cannot parse version
Apr 18 21:42:21 deis sh[3573]: 10.212.0.6 - - [18/Apr/2014:21:42:21] "PUT /v1/images/fe6d61cb3ef5499a9fcceebd8f6aabcdfe6d61cb3ef5499a9fcceebd8f6aabcd/layer HTTP/1.1" 500 291 "-" "python-requests/2.2.1 CPython/2.7.3 Linux/3.13.10+"
Apr 18 21:42:21 deis sh[3573]: 2014-04-18 21:42:21,107 INFO: 10.212.0.6 - - [18/Apr/2014:21:42:21] "PUT /v1/images/fe6d61cb3ef5499a9fcceebd8f6aabcdfe6d61cb3ef5499a9fcceebd8f6aabcd/layer HTTP/1.1" 500 291 "-" "python-requests/2.2.1 CPython/2.7.3 Linux/3.13.10+"
Apr 18 21:42:21 deis sh[4422]: ERROR Internal Server Error: /api/hooks/build
Apr 18 21:42:21 deis sh[4422]: Traceback (most recent call last):
Apr 18 21:42:21 deis sh[4422]: File "/usr/local/lib/python2.7/dist-packages/django/core/handlers/base.py", line 114, in get_response
Apr 18 21:42:21 deis sh[4422]: response = wrapped_callback(request, *callback_args, **callback_kwargs)
Apr 18 21:42:21 deis sh[4422]: File "/usr/local/lib/python2.7/dist-packages/rest_framework/viewsets.py", line 78, in view
Apr 18 21:42:21 deis sh[4422]: return self.dispatch(request, *args, **kwargs)
Apr 18 21:42:21 deis sh[4422]: File "/usr/local/lib/python2.7/dist-packages/django/views/decorators/csrf.py", line 57, in wrapped_view
Apr 18 21:42:21 deis sh[4422]: return view_func(*args, **kwargs)
Apr 18 21:42:21 deis sh[4422]: File "/usr/local/lib/python2.7/dist-packages/rest_framework/views.py", line 399, in dispatch
Apr 18 21:42:21 deis sh[4422]: response = self.handle_exception(exc)
Apr 18 21:42:21 deis sh[4422]: File "/usr/local/lib/python2.7/dist-packages/rest_framework/views.py", line 396, in dispatch
Apr 18 21:42:21 deis sh[4422]: response = handler(request, *args, **kwargs)
Apr 18 21:42:21 deis sh[4422]: File "/app/api/views.py", line 507, in create
Apr 18 21:42:21 deis sh[4422]: super(BuildHookViewSet, self).create(request, *args, **kwargs)
Apr 18 21:42:21 deis sh[4422]: File "/usr/local/lib/python2.7/dist-packages/rest_framework/mixins.py", line 55, in create
Apr 18 21:42:21 deis sh[4422]: self.post_save(self.object, created=True)
Apr 18 21:42:21 deis sh[4422]: File "/app/api/views.py", line 517, in post_save
Apr 18 21:42:21 deis sh[4422]: new_release = release.new(build.owner, build=build)
Apr 18 21:42:21 deis sh[4422]: File "/app/api/models.py", line 427, in new
Apr 18 21:42:21 deis sh[4422]: publish_release(repository_path, config.values, tag)
Apr 18 21:42:21 deis sh[4422]: File "/app/registry/private.py", line 35, in publish_release
Apr 18 21:42:21 deis sh[4422]: _commit(repository_path, image, _empty_tar_archive(), tag)
Apr 18 21:42:21 deis sh[4422]: File "/app/registry/private.py", line 43, in _commit
Apr 18 21:42:21 deis sh[4422]: cookies = _put_layer(image['id'], layer)
Apr 18 21:42:21 deis sh[4422]: File "/app/registry/private.py", line 95, in _put_layer
Apr 18 21:42:21 deis sh[4422]: raise RuntimeError("PUT Layer Error ({}: {})".format(r.status_code, r.text))
Apr 18 21:42:21 deis sh[4422]: RuntimeError: PUT Layer Error (500: <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">
Apr 18 21:42:21 deis sh[4422]: <title>500 Internal Server Error</title>
Apr 18 21:42:21 deis sh[4422]: <h1>Internal Server Error</h1>
Apr 18 21:42:21 deis sh[4422]: <p>The server encountered an internal error and was unable to complete your request.  Either the server is overloaded or there is an error in the application.</p>
Apr 18 21:42:21 deis sh[4422]: )
```

After deeper inspection, the issue is due to the registry code now requiring a docker version in the user-agent header.  See: dotcloud/docker-registry@64f4198c90



Discussion:
This appears to be the format of the user-agent string from a `docker push`:

```
2014-04-18 22:06:27,866 INFO: 10.0.2.15 - - [18/Apr/2014:22:06:27] "GET /v1/images/511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158/json HTTP/1.1" 200 483 "-" "docker/0.10.0 go/go1.2 git-commit/dc9c28f kernel/3.13.10+ os/linux arch/amd64"
```

So in our case, we should be sending `docker/0.10.0` in the user-agent string, but that may require another spot to update every time we update the builder's docker version. Maybe there's some way we can get this user agent string for free in the builder through reading docker's remote API?

I'm fixing this for the time being, but hopefully we can remove the registry module entirely in the future with https://github.com/dotcloud/docker-registry/pull/297


Resolving patch:
diff --git a/controller/registry/private.py b/controller/registry/private.py
--- a/controller/registry/private.py
+++ b/controller/registry/private.py
@@ -59,10 +59,26 @@ def _put_first_image(repository_path):
     _commit(repository_path, image, _empty_tar_archive(), 'v0')
 
 
+def _api_call(endpoint, data=None, headers={}, cookies=None, request_type='GET'):
+    # FIXME: update API calls for docker 0.10.0+
+    base_headers = {'user-agent': 'docker/0.9.0'}
+    r = None
+    if len(headers) > 0:
+        for header, value in headers.iteritems():
+            base_headers[header] = value
+    if request_type == 'GET':
+        r = requests.get(endpoint, headers=base_headers)
+    elif request_type == 'PUT':
+        r = requests.put(endpoint, data=data, headers=base_headers, cookies=cookies)
+    else:
+        raise AttributeError("request type not supported: {}".format(request_type))
+    return r
+
+
 def _get_tag(repository, tag):
     path = "/v1/repositories/{repository}/tags/{tag}".format(**locals())
     url = urlparse.urljoin(settings.REGISTRY_URL, path)
-    r = requests.get(url)
+    r = _api_call(url)
     if not r.status_code == 200:
         raise RuntimeError("GET Image Error ({}: {})".format(r.status_code, r.text))
     print r.text
@@ -72,7 +88,7 @@ def _get_tag(repository, tag):
 def _get_image(image_id):
     path = "/v1/images/{image_id}/json".format(**locals())
     url = urlparse.urljoin(settings.REGISTRY_URL, path)
-    r = requests.get(url)
+    r = _api_call(url)
     if not r.status_code == 200:
         raise RuntimeError("GET Image Error ({}: {})".format(r.status_code, r.text))
     return r.json()
@@ -81,7 +97,7 @@ def _get_image(image_id):
 def _put_image(image):
     path = "/v1/images/{id}/json".format(**image)
     url = urlparse.urljoin(settings.REGISTRY_URL, path)
-    r = requests.put(url, data=json.dumps(image))
+    r = _api_call(url, data=json.dumps(image), request_type='PUT')
     if not r.status_code == 200:
         raise RuntimeError("PUT Image Error ({}: {})".format(r.status_code, r.text))
     return r.json()
@@ -90,7 +106,7 @@ def _put_image(image):
 def _put_layer(image_id, layer_fileobj):
     path = "/v1/images/{image_id}/layer".format(**locals())
     url = urlparse.urljoin(settings.REGISTRY_URL, path)
-    r = requests.put(url, data=layer_fileobj.read())
+    r = _api_call(url, data=layer_fileobj.read(), request_type='PUT')
     if not r.status_code == 200:
         raise RuntimeError("PUT Layer Error ({}: {})".format(r.status_code, r.text))
     return r.cookies
@@ -101,7 +117,7 @@ def _put_checksum(image, cookies):
     url = urlparse.urljoin(settings.REGISTRY_URL, path)
     tarsum = TarSum(json.dumps(image)).compute()
     headers = {'X-Docker-Checksum': tarsum}
-    r = requests.put(url, headers=headers, cookies=cookies)
+    r = _api_call(url, headers=headers, cookies=cookies, request_type='PUT')
     if not r.status_code == 200:
         raise RuntimeError("PUT Checksum Error ({}: {})".format(r.status_code, r.text))
     print r.json()
@@ -110,7 +126,7 @@ def _put_checksum(image, cookies):
 def _put_tag(image_id, repository_path, tag):
     path = "/v1/repositories/{repository_path}/tags/{tag}".format(**locals())
     url = urlparse.urljoin(settings.REGISTRY_URL, path)
-    r = requests.put(url, data=json.dumps(image_id))
+    r = _api_call(url, data=json.dumps(image_id), request_type='PUT')
     if not r.status_code == 200:
         raise RuntimeError("PUT Tag Error ({}: {})".format(r.status_code, r.text))
     print r.json()


Test patch: