Repository: DataBiosphere/toil

Issue:
Update Google job store
It looks like there are some errors when trying to run a workflow using the Google job store:

```
Traceback (most recent call last):
  File "sort.py", line 259, in <module>
    main()
  File "sort.py", line 247, in main
    sortedFileID = workflow.start(Job.wrapJobFn(setup,
  File "/Users/jonathanxu/toil/src/toil/common.py", line 845, in start
    rootJobDescription = rootJob.saveAsRootJob(self._jobStore)
  File "/Users/jonathanxu/toil/src/toil/job.py", line 2186, in saveAsRootJob
    self._saveJobGraph(jobStore, saveSelf=True)
  File "/Users/jonathanxu/toil/src/toil/job.py", line 2169, in _saveJobGraph
    jobStore.create(job.description)
  File "/Users/jonathanxu/toil/src/toil/jobStores/googleJobStore.py", line 182, in create
    self._writeString(jobDescription.jobStoreID, pickle.dumps(jobDescription, protocol=pickle.HIGHEST_PROTOCOL))
  File "/Users/jonathanxu/toil/src/toil/jobStores/googleJobStore.py", line 456, in _writeString
    self._writeFile(jobStoreID, StringIO(stringToUpload), **kwarg)
TypeError: initial_value must be str or None, not bytes
```
**Steps to reproduce:**

Follow the instructions here:
    [https://toil.readthedocs.io/en/latest/running/cloud/gce.html](https://toil.readthedocs.io/en/latest/running/cloud/gce.html)

Create a new service account with make bucket permissions and a new ssh key.  ^
These are the environment variables needed for running tests and credentialing (modify GOOGLE_APPLICATION_CREDENTIALS to your own path):
```
export GOOGLE_APPLICATION_CREDENTIALS=path to your downloaded service credentials json
export TOIL_TEST_INTEGRATIVE=True
export TRAVIS=True
export TOIL_GOOGLE_KEYNAME=name of your key
```
```
git clone https://github.com/DataBiosphere/toil.git
cd toil/src/toil/test/sort
python sort.py google:toil-dev:some-unique-name-like-jim
```

From what I discussed with @adamnovak and @DailyDreaming, I think there may be some type inconsistencies between Strings and Bytes that are being caused by Python2/Python3 compatibility problems.

┆Issue is synchronized with this [Jira Task](https://ucsc-cgl.atlassian.net/browse/TOIL-763)
┆Issue Number: TOIL-763



Discussion:

Resolving patch:
diff --git a/src/toil/jobStores/googleJobStore.py b/src/toil/jobStores/googleJobStore.py
--- a/src/toil/jobStores/googleJobStore.py
+++ b/src/toil/jobStores/googleJobStore.py
@@ -18,7 +18,7 @@
 import uuid
 from contextlib import contextmanager
 from functools import wraps
-from io import StringIO
+from io import BytesIO
 
 from google.api_core.exceptions import (GoogleAPICallError,
                                         InternalServerError,
@@ -136,7 +136,7 @@ def initialize(self, config=None):
 
         # set up sever side encryption after we set up config in super
         if self.config.sseKey is not None:
-            with open(self.config.sseKey) as f:
+            with open(self.config.sseKey, 'rb') as f:
                 self.sseKey = compat_bytes(f.read())
                 assert len(self.sseKey) == 32
 
@@ -179,7 +179,7 @@ def assignID(self, jobDescription):
 
     def create(self, jobDescription):
         # TODO: we don't implement batching, but we probably should.
-        self._writeString(jobDescription.jobStoreID, pickle.dumps(jobDescription, protocol=pickle.HIGHEST_PROTOCOL))
+        self._writeBytes(jobDescription.jobStoreID, pickle.dumps(jobDescription, protocol=pickle.HIGHEST_PROTOCOL))
         return jobDescription
 
     @googleRetry
@@ -210,7 +210,7 @@ def load(self, jobStoreID):
         return job
 
     def update(self, job):
-        self._writeString(job.jobStoreID, pickle.dumps(job, protocol=pickle.HIGHEST_PROTOCOL), update=True)
+        self._writeBytes(job.jobStoreID, pickle.dumps(job, protocol=pickle.HIGHEST_PROTOCOL), update=True)
 
     @googleRetry
     def delete(self, jobStoreID):
@@ -244,7 +244,7 @@ def jobs(self):
 
     def writeFile(self, localFilePath, jobStoreID=None, cleanup=False):
         fileID = self._newID(isFile=True, jobStoreID=jobStoreID if cleanup else None)
-        with open(localFilePath) as f:
+        with open(localFilePath, 'rb') as f:
             self._writeFile(fileID, f)
         return fileID
 
@@ -256,7 +256,7 @@ def writeFileStream(self, jobStoreID=None, cleanup=False, basename=None):
 
     def getEmptyFileStoreID(self, jobStoreID=None, cleanup=False, basename=None):
         fileID = self._newID(isFile=True, jobStoreID=jobStoreID if cleanup else None)
-        self._writeFile(fileID, StringIO(""))
+        self._writeFile(fileID, BytesIO(b""))
         return fileID
 
     @googleRetry
@@ -266,7 +266,7 @@ def readFile(self, jobStoreFileID, localFilePath, symlink=False):
         if not self.fileExists(jobStoreFileID):
             raise NoSuchFileException(jobStoreFileID)
         with AtomicFileCreate(localFilePath) as tmpPath:
-            with open(tmpPath, 'w') as writeable:
+            with open(tmpPath, 'wb') as writeable:
                 blob = self.bucket.get_blob(compat_bytes(jobStoreFileID), encryption_key=self.sseKey)
                 blob.download_to_file(writeable)
 
@@ -289,7 +289,7 @@ def getFileSize(self, jobStoreFileID):
         return self.bucket.get_blob(compat_bytes(jobStoreFileID), encryption_key=self.sseKey).size
 
     def updateFile(self, jobStoreFileID, localFilePath):
-        with open(localFilePath) as f:
+        with open(localFilePath, 'rb') as f:
             self._writeFile(jobStoreFileID, f, update=True)
 
     @contextmanager
@@ -349,18 +349,18 @@ def getSize(cls, url):
     def _readFromUrl(cls, url, writable):
         blob = cls._getBlobFromURL(url, exists=True)
         blob.download_to_file(writable)
-        return blob.size
+        return blob.size, False
 
     @classmethod
     def _supportsUrl(cls, url, export=False):
         return url.scheme.lower() == 'gs'
 
     @classmethod
-    def _writeToUrl(cls, readable, url):
+    def _writeToUrl(cls, readable: bytes, url, executable=False) -> None:
         blob = cls._getBlobFromURL(url)
         blob.upload_from_file(readable)
 
-    def writeStatsAndLogging(self, statsAndLoggingString):
+    def writeStatsAndLogging(self, statsAndLoggingString: bytes) -> None:
         statsID = self.statsBaseID + str(uuid.uuid4())
         log.debug("Writing stats file: %s", statsID)
         with self._uploadStream(statsID, encrypt=False, update=False) as f:
@@ -442,7 +442,7 @@ def _readContents(self, jobStoreID):
         return job.download_as_string()
 
     @googleRetry
-    def _writeFile(self, jobStoreID, fileObj, update=False, encrypt=True):
+    def _writeFile(self, jobStoreID: str, fileObj: bytes, update=False, encrypt=True) -> None:
         blob = self.bucket.blob(compat_bytes(jobStoreID), encryption_key=self.sseKey if encrypt else None)
         if not update:
             # TODO: should probably raise a special exception and be added to all jobStores
@@ -452,8 +452,8 @@ def _writeFile(self, jobStoreID, fileObj, update=False, encrypt=True):
                 raise NoSuchFileException(jobStoreID)
         blob.upload_from_file(fileObj)
 
-    def _writeString(self, jobStoreID, stringToUpload, **kwarg):
-        self._writeFile(jobStoreID, StringIO(stringToUpload), **kwarg)
+    def _writeBytes(self, jobStoreID: str, stringToUpload: bytes, **kwarg) -> None:
+        self._writeFile(jobStoreID, BytesIO(stringToUpload), **kwarg)
 
     @contextmanager
     @googleRetry
diff --git a/src/toil/provisioners/__init__.py b/src/toil/provisioners/__init__.py
--- a/src/toil/provisioners/__init__.py
+++ b/src/toil/provisioners/__init__.py
@@ -93,7 +93,7 @@ def check_valid_node_types(provisioner, node_types):
             if nodeType and ':' in nodeType:
                 nodeType = nodeType.split(':')[0]
 
-            if nodeType not in E2Instances:
+            if nodeType in E2Instances:
                 raise RuntimeError(f"It looks like you've specified an AWS nodeType with the {provisioner} "
                                    f"provisioner.  Please specify a nodeType for {provisioner}.")
     else:
diff --git a/src/toil/statsAndLogging.py b/src/toil/statsAndLogging.py
--- a/src/toil/statsAndLogging.py
+++ b/src/toil/statsAndLogging.py
@@ -173,7 +173,7 @@ def callback(fileHandle):
         # Finish the stats file
         text = json.dumps(dict(total_time=str(time.time() - startTime),
                                total_clock=str(get_total_cpu_time() - startClock)), ensure_ascii=True)
-        jobStore.writeStatsAndLogging(text)
+        jobStore.writeStatsAndLogging(bytes(text, 'utf-8'))
 
     def check(self):
         """


Test patch:
diff --git a/src/toil/test/__init__.py b/src/toil/test/__init__.py
--- a/src/toil/test/__init__.py
+++ b/src/toil/test/__init__.py
@@ -316,8 +316,6 @@ def needs_google(test_item):
 
     if not os.getenv('TOIL_GOOGLE_PROJECTID'):
         return unittest.skip("Set TOIL_GOOGLE_PROJECTID to include this test.")(test_item)
-    elif not config.get('Credentials'):  # TODO: Deprecate this check?  Needed by only by the ancients.
-        return unittest.skip("Configure ~/.boto with Google Cloud credentials to include this test.")(test_item)
     return test_item
 
 
diff --git a/src/toil/test/jobStores/jobStoreTest.py b/src/toil/test/jobStores/jobStoreTest.py
--- a/src/toil/test/jobStores/jobStoreTest.py
+++ b/src/toil/test/jobStores/jobStoreTest.py
@@ -24,7 +24,7 @@
 import urllib.parse as urlparse
 import uuid
 from abc import ABCMeta, abstractmethod
-from io import StringIO
+from io import BytesIO
 from itertools import chain, islice
 from queue import Queue
 from threading import Thread
@@ -1168,8 +1168,8 @@ def _prepareTestFile(self, bucket, size=None):
             return url
         with open('/dev/urandom', 'rb') as readable:
             contents = str(readable.read(size))
-        GoogleJobStore._writeToUrl(StringIO(contents), urlparse.urlparse(url))
-        return url, hashlib.md5(contents).hexdigest()
+        GoogleJobStore._writeToUrl(BytesIO(bytes(contents, 'utf-8')), urlparse.urlparse(url))
+        return url, hashlib.md5(contents.encode()).hexdigest()
 
     def _hashTestFile(self, url):
         from toil.jobStores.googleJobStore import GoogleJobStore