Repository: google/clusterfuzz

Issue:
run_server doesn't work with latest setuptools/gcloud sdk
looks like something broke:

```
| ERROR    2019-03-13 01:05:41,541 wsgi.py:263]
| Traceback (most recent call last):
|   File "/usr/lib/google-cloud-sdk/platform/google_appengine/google/appengine/runtime/wsgi.py", line 240, in Handle
|     handler = _config_handle.add_wsgi_middleware(self._LoadHandler())
|   File "/usr/lib/google-cloud-sdk/platform/google_appengine/google/appengine/runtime/wsgi.py", line 299, in _LoadHandler
|     handler, path, err = LoadObject(self._handler)
|   File "/usr/lib/google-cloud-sdk/platform/google_appengine/google/appengine/runtime/wsgi.py", line 85, in LoadObject
|     obj = __import__(path[0])
|   File "/usr/local/google/home/ochang/clusterfuzz/src/appengine/server.py", line 21, in <module>
|     from handlers import base_handler
|   File "/usr/local/google/home/ochang/clusterfuzz/src/appengine/handlers/base_handler.py", line 34, in <module>
|     from google_cloud_utils import storage
|   File "/usr/local/google/home/ochang/clusterfuzz/src/appengine/python/google_cloud_utils/storage.py", line 39, in <module>
|     from google.cloud import storage as gcs
|   File "/usr/local/google/home/ochang/clusterfuzz/src/appengine/third_party/google/cloud/storage/__init__.py", line 36, in <module>
|     __version__ = get_distribution("google-cloud-storage").version
|   File "/usr/local/google/home/ochang/clusterfuzz/src/appengine/third_party/pkg_resources/__init__.py", line 481, in get_distribution
|     dist = get_provider(dist)
| INFO     2019-03-13 01:05:41,549 module.py:861] default: "GET /_ah/start HTTP/1.1" 500 -
|   File "/usr/local/google/home/ochang/clusterfuzz/src/appengine/third_party/pkg_resources/__init__.py", line 357, in get_provider
|     return working_set.find(moduleOrReq) or require(str(moduleOrReq))[0]
|   File "/usr/local/google/home/ochang/clusterfuzz/src/appengine/third_party/pkg_resources/__init__.py", line 900, in require
|     needed = self.resolve(parse_requirements(requirements))
|   File "/usr/local/google/home/ochang/clusterfuzz/src/appengine/third_party/pkg_resources/__init__.py", line 786, in resolve
|     raise DistributionNotFound(req, requirers)
| DistributionNotFound: The 'google-cloud-storage' distribution was not found and is required by the application
| ERROR    2019-03-13 01:05:41,585 wsgi.py:263]
| Traceback (most recent call last):
|   File "/usr/lib/google-cloud-sdk/platform/google_appengine/google/appengine/runtime/wsgi.py", line 240, in Handle
|     handler = _config_handle.add_wsgi_middleware(self._LoadHandler())
|   File "/usr/lib/google-cloud-sdk/platform/google_appengine/google/appengine/runtime/wsgi.py", line 299, in _LoadHandler
|     handler, path, err = LoadObject(self._handler)
|   File "/usr/lib/google-cloud-sdk/platform/google_appengine/google/appengine/runtime/wsgi.py", line 85, in LoadObject
|     obj = __import__(path[0])
|   File "/usr/local/google/home/ochang/clusterfuzz/src/appengine/server.py", line 21, in <module>
|     from handlers import base_handler
|   File "/usr/local/google/home/ochang/clusterfuzz/src/appengine/handlers/base_handler.py", line 34, in <module>
|     from google_cloud_utils import storage
|   File "/usr/local/google/home/ochang/clusterfuzz/src/appengine/python/google_cloud_utils/storage.py", line 39, in <module>
|     from google.cloud import storage as gcs
|   File "/usr/local/google/home/ochang/clusterfuzz/src/appengine/third_party/google/cloud/storage/__init__.py", line 36, in <module>
|     __version__ = get_distribution("google-cloud-storage").version
|   File "/usr/local/google/home/ochang/clusterfuzz/src/appengine/third_party/pkg_resources/__init__.py", line 481, in get_distribution
|     dist = get_provider(dist)
|   File "/usr/local/google/home/ochang/clusterfuzz/src/appengine/third_party/pkg_resources/__init__.py", line 357, in get_provider
| INFO     2019-03-13 01:05:41,592 module.py:861] default: "GET / HTTP/1.1" 500 -
|     return working_set.find(moduleOrReq) or require(str(moduleOrReq))[0]
|   File "/usr/local/google/home/ochang/clusterfuzz/src/appengine/third_party/pkg_resources/__init__.py", line 900, in require
|     needed = self.resolve(parse_requirements(requirements))
|   File "/usr/local/google/home/ochang/clusterfuzz/src/appengine/third_party/pkg_resources/__init__.py", line 786, in resolve
|     raise DistributionNotFound(req, requirers)
| DistributionNotFound: The 'google-cloud-storage' distribution was not found and is required by the application
| ERROR    2019-03-13 01:05:42,708 wsgi.py:263]
| Traceback (most recent call last):
|   File "/usr/lib/google-cloud-sdk/platform/google_appengine/google/appengine/runtime/wsgi.py", line 240, in Handle
|     handler = _config_handle.add_wsgi_middleware(self._LoadHandler())
|   File "/usr/lib/google-cloud-sdk/platform/google_appengine/google/appengine/runtime/wsgi.py", line 299, in _LoadHandler
|     handler, path, err = LoadObject(self._handler)
|   File "/usr/lib/google-cloud-sdk/platform/google_appengine/google/appengine/runtime/wsgi.py", line 85, in LoadObject
|     obj = __import__(path[0])
|   File "/usr/local/google/home/ochang/clusterfuzz/src/appengine/server.py", line 21, in <module>
|     from handlers import base_handler
|   File "/usr/local/google/home/ochang/clusterfuzz/src/appengine/handlers/base_handler.py", line 34, in <module>
|     from google_cloud_utils import storage
|   File "/usr/local/google/home/ochang/clusterfuzz/src/appengine/python/google_cloud_utils/storage.py", line 39, in <module>
|     from google.cloud import storage as gcs
|   File "/usr/local/google/home/ochang/clusterfuzz/src/appengine/third_party/google/cloud/storage/__init__.py", line 36, in <module>
| INFO     2019-03-13 01:05:42,713 module.py:861] default: "GET / HTTP/1.1" 500 -
|     __version__ = get_distribution("google-cloud-storage").version
|   File "/usr/local/google/home/ochang/clusterfuzz/src/appengine/third_party/pkg_resources/__init__.py", line 481, in get_distribution
|     dist = get_provider(dist)
|   File "/usr/local/google/home/ochang/clusterfuzz/src/appengine/third_party/pkg_resources/__init__.py", line 357, in get_provider
|     return working_set.find(moduleOrReq) or require(str(moduleOrReq))[0]
|   File "/usr/local/google/home/ochang/clusterfuzz/src/appengine/third_party/pkg_resources/__init__.py", line 900, in require
|     needed = self.resolve(parse_requirements(requirements))
|   File "/usr/local/google/home/ochang/clusterfuzz/src/appengine/third_party/pkg_resources/__init__.py", line 786, in resolve
|     raise DistributionNotFound(req, requirers)
| DistributionNotFound: The 'google-cloud-storage' distribution was not found and is required by the application
| ERROR    2019-03-13 01:05:43,677 wsgi.py:263]
| Traceback (most recent call last):
|   File "/usr/lib/google-cloud-sdk/platform/google_appengine/google/appengine/runtime/wsgi.py", line 240, in Handle
|     handler = _config_handle.add_wsgi_middleware(self._LoadHandler())
|   File "/usr/lib/google-cloud-sdk/platform/google_appengine/google/appengine/runtime/wsgi.py", line 299, in _LoadHandler
|     handler, path, err = LoadObject(self._handler)
|   File "/usr/lib/google-cloud-sdk/platform/google_appengine/google/appengine/runtime/wsgi.py", line 85, in LoadObject
|     obj = __import__(path[0])
|   File "/usr/local/google/home/ochang/clusterfuzz/src/appengine/server.py", line 21, in <module>
|     from handlers import base_handler
|   File "/usr/local/google/home/ochang/clusterfuzz/src/appengine/handlers/base_handler.py", line 34, in <module>
|     from google_cloud_utils import storage
|   File "/usr/local/google/home/ochang/clusterfuzz/src/appengine/python/google_cloud_utils/storage.py", line 39, in <module>
|     from google.cloud import storage as gcs
|   File "/usr/local/google/home/ochang/clusterfuzz/src/appengine/third_party/google/cloud/storage/__init__.py", line 36, in <module>
|     __version__ = get_distribution("google-cloud-storage").version
|   File "/usr/local/google/home/ochang/clusterfuzz/src/appengine/third_party/pkg_resources/__init__.py", line 481, in get_distribution
|     dist = get_provider(dist)
|   File "/usr/local/google/home/ochang/clusterfuzz/src/appengine/third_party/pkg_resources/__init__.py", line 357, in get_provider
| INFO     2019-03-13 01:05:43,683 module.py:861] default: "GET / HTTP/1.1" 500 -
|     return working_set.find(moduleOrReq) or require(str(moduleOrReq))[0]
|   File "/usr/local/google/home/ochang/clusterfuzz/src/appengine/third_party/pkg_resources/__init__.py", line 900, in require
|     needed = self.resolve(parse_requirements(requirements))
|   File "/usr/local/google/home/ochang/clusterfuzz/src/appengine/third_party/pkg_resources/__init__.py", line 786, in resolve
|     raise DistributionNotFound(req, requirers)
| DistributionNotFound: The 'google-cloud-storage' distribution was not found and is required by the application
| ERROR    2019-03-13 01:05:43,901 wsgi.py:263]
| Traceback (most recent call last):
|   File "/usr/lib/google-cloud-sdk/platform/google_appengine/google/appengine/runtime/wsgi.py", line 240, in Handle
|     handler = _config_handle.add_wsgi_middleware(self._LoadHandler())
|   File "/usr/lib/google-cloud-sdk/platform/google_appengine/google/appengine/runtime/wsgi.py", line 299, in _LoadHandler
|     handler, path, err = LoadObject(self._handler)
|   File "/usr/lib/google-cloud-sdk/platform/google_appengine/google/appengine/runtime/wsgi.py", line 85, in LoadObject
|     obj = __import__(path[0])
|   File "/usr/local/google/home/ochang/clusterfuzz/src/appengine/server.py", line 21, in <module>
|     from handlers import base_handler
|   File "/usr/local/google/home/ochang/clusterfuzz/src/appengine/handlers/base_handler.py", line 34, in <module>
|     from google_cloud_utils import storage
|   File "/usr/local/google/home/ochang/clusterfuzz/src/appengine/python/google_cloud_utils/storage.py", line 39, in <module>
|     from google.cloud import storage as gcs
|   File "/usr/local/google/home/ochang/clusterfuzz/src/appengine/third_party/google/cloud/storage/__init__.py", line 36, in <module>
|     __version__ = get_distribution("google-cloud-storage").version
| INFO     2019-03-13 01:05:43,906 module.py:861] default: "GET / HTTP/1.1" 500 -
|   File "/usr/local/google/home/ochang/clusterfuzz/src/appengine/third_party/pkg_resources/__init__.py", line 481, in get_distribution
|     dist = get_provider(dist)
|   File "/usr/local/google/home/ochang/clusterfuzz/src/appengine/third_party/pkg_resources/__init__.py", line 357, in get_provider
|     return working_set.find(moduleOrReq) or require(str(moduleOrReq))[0]
|   File "/usr/local/google/home/ochang/clusterfuzz/src/appengine/third_party/pkg_resources/__init__.py", line 900, in require
|     needed = self.resolve(parse_requirements(requirements))
|   File "/usr/local/google/home/ochang/clusterfuzz/src/appengine/third_party/pkg_resources/__init__.py", line 786, in resolve
|     raise DistributionNotFound(req, requirers)
| DistributionNotFound: The 'google-cloud-storage' distribution was not found and is required by the application
| INFO     2019-03-13 01:06:13,787 instance.py:294] Instance PID: 204674
| ERROR    2019-03-13 01:06:14,449 wsgi.py:263]
| Traceback (most recent call last):
|   File "/usr/lib/google-cloud-sdk/platform/google_appengine/google/appengine/runtime/wsgi.py", line 240, in Handle
|     handler = _config_handle.add_wsgi_middleware(self._LoadHandler())
|   File "/usr/lib/google-cloud-sdk/platform/google_appengine/google/appengine/runtime/wsgi.py", line 299, in _LoadHandler
|     handler, path, err = LoadObject(self._handler)
|   File "/usr/lib/google-cloud-sdk/platform/google_appengine/google/appengine/runtime/wsgi.py", line 85, in LoadObject
|     obj = __import__(path[0])
|   File "/usr/local/google/home/ochang/clusterfuzz/src/appengine/server.py", line 21, in <module>
|     from handlers import base_handler
|   File "/usr/local/google/home/ochang/clusterfuzz/src/appengine/handlers/base_handler.py", line 34, in <module>
|     from google_cloud_utils import storage
|   File "/usr/local/google/home/ochang/clusterfuzz/src/appengine/python/google_cloud_utils/storage.py", line 39, in <module>
|     from google.cloud import storage as gcs
|   File "/usr/local/google/home/ochang/clusterfuzz/src/appengine/third_party/google/cloud/storage/__init__.py", line 36, in <module>
|     __version__ = get_distribution("google-cloud-storage").version
|   File "/usr/local/google/home/ochang/clusterfuzz/src/appengine/third_party/pkg_resources/__init__.py", line 481, in get_distribution
| INFO     2019-03-13 01:06:14,456 module.py:861] cron-service: "GET /_ah/start HTTP/1.1" 500 -
|     dist = get_provider(dist)
|   File "/usr/local/google/home/ochang/clusterfuzz/src/appengine/third_party/pkg_resources/__init__.py", line 357, in get_provider
|     return working_set.find(moduleOrReq) or require(str(moduleOrReq))[0]
|   File "/usr/local/google/home/ochang/clusterfuzz/src/appengine/third_party/pkg_resources/__init__.py", line 900, in require
|     needed = self.resolve(parse_requirements(requirements))
|   File "/usr/local/google/home/ochang/clusterfuzz/src/appengine/third_party/pkg_resources/__init__.py", line 786, in resolve
|     raise DistributionNotFound(req, requirers)
| DistributionNotFound: The 'google-cloud-storage' distribution was not found and is required by the application
| ERROR    2019-03-13 01:06:14,508 wsgi.py:263]
| Traceback (most recent call last):
|   File "/usr/lib/google-cloud-sdk/platform/google_appengine/google/appengine/runtime/wsgi.py", line 240, in Handle
|     handler = _config_handle.add_wsgi_middleware(self._LoadHandler())
|   File "/usr/lib/google-cloud-sdk/platform/google_appengine/google/appengine/runtime/wsgi.py", line 299, in _LoadHandler
|     handler, path, err = LoadObject(self._handler)
|   File "/usr/lib/google-cloud-sdk/platform/google_appengine/google/appengine/runtime/wsgi.py", line 85, in LoadObject
|     obj = __import__(path[0])
|   File "/usr/local/google/home/ochang/clusterfuzz/src/appengine/server.py", line 21, in <module>
|     from handlers import base_handler
|   File "/usr/local/google/home/ochang/clusterfuzz/src/appengine/handlers/base_handler.py", line 34, in <module>
|     from google_cloud_utils import storage
|   File "/usr/local/google/home/ochang/clusterfuzz/src/appengine/python/google_cloud_utils/storage.py", line 39, in <module>
|     from google.cloud import storage as gcs
|   File "/usr/local/google/home/ochang/clusterfuzz/src/appengine/third_party/google/cloud/storage/__init__.py", line 36, in <module>
|     __version__ = get_distribution("google-cloud-storage").version
|   File "/usr/local/google/home/ochang/clusterfuzz/src/appengine/third_party/pkg_resources/__init__.py", line 481, in get_distribution
|     dist = get_provider(dist)
| INFO     2019-03-13 01:06:14,513 module.py:861] cron-service: "GET /triage HTTP/1.1" 500 -
|   File "/usr/local/google/home/ochang/clusterfuzz/src/appengine/third_party/pkg_resources/__init__.py", line 357, in get_provider
|     return working_set.find(moduleOrReq) or require(str(moduleOrReq))[0]
|   File "/usr/local/google/home/ochang/clusterfuzz/src/appengine/third_party/pkg_resources/__init__.py", line 900, in require
|     needed = self.resolve(parse_requirements(requirements))
|   File "/usr/local/google/home/ochang/clusterfuzz/src/appengine/third_party/pkg_resources/__init__.py", line 786, in resolve
|     raise DistributionNotFound(req, requirers)
| DistributionNotFound: The 'google-cloud-storage' distribution was not found and is required by the application
| ERROR    2019-03-13 01:06:42,809 wsgi.py:263]
| Traceback (most recent call last):
|   File "/usr/lib/google-cloud-sdk/platform/google_appengine/google/appengine/runtime/wsgi.py", line 240, in Handle
|     handler = _config_handle.add_wsgi_middleware(self._LoadHandler())
|   File "/usr/lib/google-cloud-sdk/platform/google_appengine/google/appengine/runtime/wsgi.py", line 299, in _LoadHandler
|     handler, path, err = LoadObject(self._handler)
|   File "/usr/lib/google-cloud-sdk/platform/google_appengine/google/appengine/runtime/wsgi.py", line 85, in LoadObject
|     obj = __import__(path[0])
|   File "/usr/local/google/home/ochang/clusterfuzz/src/appengine/server.py", line 21, in <module>
|     from handlers import base_handler
|   File "/usr/local/google/home/ochang/clusterfuzz/src/appengine/handlers/base_handler.py", line 34, in <module>
|     from google_cloud_utils import storage
|   File "/usr/local/google/home/ochang/clusterfuzz/src/appengine/python/google_cloud_utils/storage.py", line 39, in <module>
|     from google.cloud import storage as gcs
|   File "/usr/local/google/home/ochang/clusterfuzz/src/appengine/third_party/google/cloud/storage/__init__.py", line 36, in <module>
|     __version__ = get_distribution("google-cloud-storage").version
|   File "/usr/local/google/home/ochang/clusterfuzz/src/appengine/third_party/pkg_resources/__init__.py", line 481, in get_distribution
|     dist = get_provider(dist)
| INFO     2019-03-13 01:06:42,815 module.py:861] cron-service: "GET /cleanup HTTP/1.1" 500 -
|   File "/usr/local/google/home/ochang/clusterfuzz/src/appengine/third_party/pkg_resources/__init__.py", line 357, in get_provider
|     return working_set.find(moduleOrReq) or require(str(moduleOrReq))[0]
|   File "/usr/local/google/home/ochang/clusterfuzz/src/appengine/third_party/pkg_resources/__init__.py", line 900, in require
|     needed = self.resolve(parse_requirements(requirements))
|   File "/usr/local/google/home/ochang/clusterfuzz/src/appengine/third_party/pkg_resources/__init__.py", line 786, in resolve
|     raise DistributionNotFound(req, requirers)
| DistributionNotFound: The 'google-cloud-storage' distribution was not found and is required by the application
```


Discussion:
looks like some sandboxing issue.

getting:

`[Errno 13] path not accessible: '/usr/local/google/home/ochang/clusterfuzz/src/third_party'`

on https://github.com/pypa/setuptools/blob/master/pkg_resources/__init__.py#L2092

```python
def safe_listdir(path):
    """
    Attempt to list contents of path, but suppress some exceptions.
    """
    try:
        return os.listdir(path)
    except (PermissionError, NotADirectoryError):
        pass
    except OSError as e:
        # Ignore the directory if does not exist, not a directory or
        # permission denied
        ignorable = (
            e.errno in (errno.ENOTDIR, errno.EACCES, errno.ENOENT)
            # Python 2 on Windows needs to be handled this way :(
            or getattr(e, "winerror", None) == 267
        )
        if not ignorable:
            raise
    return ()
```
if I copy the third_party directory instead of symlinking, this works. looks like some behaviour might have changed somewhere with following symlinks.

Resolving patch:
diff --git a/src/local/butler/appengine.py b/src/local/butler/appengine.py
--- a/src/local/butler/appengine.py
+++ b/src/local/butler/appengine.py
@@ -120,7 +120,11 @@ def symlink_dirs():
   common.symlink(
       src=os.path.join('src', 'python'),
       target=os.path.join(SRC_DIR_PY, 'python'))
-  common.symlink(
+  # While importing third party modules, we may call pkg_resources.
+  # pkg_resources normalizes paths by calling os.path.realpath on them, which is
+  # incompatible with the App Engine sandbox since the resulting path will no
+  # longer be under appengine/.
+  common.copy_dir(
       src=os.path.join('src', 'third_party'),
       target=os.path.join(SRC_DIR_PY, 'third_party'))
 
diff --git a/src/local/butler/common.py b/src/local/butler/common.py
--- a/src/local/butler/common.py
+++ b/src/local/butler/common.py
@@ -300,6 +300,17 @@ def symlink(src, target):
       src=src, target=target))
 
 
+def copy_dir(src, target):
+  """Copy directory."""
+  if os.path.islink(target):
+    os.remove(target)
+
+  if os.path.isdir(target):
+    shutil.rmtree(target, ignore_errors=True)
+
+  shutil.copytree(src, target)
+
+
 def has_file_in_path(filename):
   """Check to see if filename exists in the user's PATH."""
   path = os.getenv('PATH')


Test patch: