Repository: DataBiosphere/toil

Issue:
Maximum Job Data Size on Azure Too Small
Upon trying to scale up my pipeline from one sample to hundreds of samples on Azure, I am met with errors like the following:

```
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:   Traceback (most recent call last):
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:     File "/usr/local/lib/python2.7/dist-packages/toil/worker.py", line 282, in main
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:       fileStore=fileStore)
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:     File "/usr/local/lib/python2.7/dist-packages/toil/job.py", line 1282, in _execute
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:       self._serialiseJobGraph(jobWrapper, jobStore, returnValues, False)
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:     File "/usr/local/lib/python2.7/dist-packages/toil/job.py", line 1233, in _serialiseJobGraph
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:       job._serialiseJob(jobStore, jobsToJobWrappers, jobWrapper)
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:     File "/usr/local/lib/python2.7/dist-packages/toil/job.py", line 1196, in _serialiseJob
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:       jobStore.update(jobsToJobWrappers[self])
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:     File "/usr/local/lib/python2.7/dist-packages/toil/jobStores/azureJobStore.py", line 150, in update
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:       entity=job.toItem(chunkSize=self.jobChunkSize))
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:     File "/usr/local/lib/python2.7/dist-packages/toil/jobStores/azureJobStore.py", line 517, in f
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:       return function(**kwargs)
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:     File "/usr/local/lib/python2.7/dist-packages/azure/storage/tableservice.py", line 476, in update_entity
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:       response = self._perform_request(request)
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:     File "/usr/local/lib/python2.7/dist-packages/azure/storage/storageclient.py", line 179, in _perform_request
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:       _storage_error_handler(ex)
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:     File "/usr/local/lib/python2.7/dist-packages/azure/storage/__init__.py", line 1174, in _storage_error_handler
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:       return _general_error_handler(http_error)
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:     File "/usr/local/lib/python2.7/dist-packages/azure/__init__.py", line 551, in _general_error_handler
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:       http_error.respbody.decode('utf-8-sig'))
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:   WindowsAzureError: Unknown error (Bad Request)
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:   <?xml version="1.0" encoding="utf-8"?><error xmlns="http://schemas.microsoft.com/ado/2007/08/dataservices/metadata"><code>PropertyValueTooLarge</code><message xml:lang="en-US">The property value exceeds the maximum allowed size (64KB). If the property value is a string, it is UTF-16 encoded and the maximum number of characters should be 32K or less.
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:   RequestId:bc75b149-0002-0131-4274-260c9f000000
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:   Time:2015-11-24T04:54:04.2292362Z</message></error>
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:   Exiting the worker because of a failed jobWrapper on host c1agent34
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:   ERROR:toil.worker:Exiting the worker because of a failed jobWrapper on host c1agent34
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:   WARNING:toil.jobWrapper:Due to failure we are reducing the remaining retry count of job 4f782575_20d9_48bb_bc1f_94cc8b207479 to 0
```

I think, since I'm passing around lists of hundreds of sample names, and since I'm adding hundreds of grandchild jobs to one of my child jobs, that I might have jobs that are too big.

I think the offending code may be here: https://github.com/BD2KGenomics/toil/blob/8efce29fe68b8356f8c4fca9b52c2649c782dca0/src/toil/jobStores/azureJobStore.py#L585

It looks like it's cutting strings to 64k, but the error message says Azure does its own UTF-16 encoding, and that for strings we really need to cut at 32k.

Maximum Job Data Size on Azure Too Small
Upon trying to scale up my pipeline from one sample to hundreds of samples on Azure, I am met with errors like the following:

```
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:   Traceback (most recent call last):
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:     File "/usr/local/lib/python2.7/dist-packages/toil/worker.py", line 282, in main
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:       fileStore=fileStore)
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:     File "/usr/local/lib/python2.7/dist-packages/toil/job.py", line 1282, in _execute
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:       self._serialiseJobGraph(jobWrapper, jobStore, returnValues, False)
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:     File "/usr/local/lib/python2.7/dist-packages/toil/job.py", line 1233, in _serialiseJobGraph
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:       job._serialiseJob(jobStore, jobsToJobWrappers, jobWrapper)
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:     File "/usr/local/lib/python2.7/dist-packages/toil/job.py", line 1196, in _serialiseJob
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:       jobStore.update(jobsToJobWrappers[self])
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:     File "/usr/local/lib/python2.7/dist-packages/toil/jobStores/azureJobStore.py", line 150, in update
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:       entity=job.toItem(chunkSize=self.jobChunkSize))
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:     File "/usr/local/lib/python2.7/dist-packages/toil/jobStores/azureJobStore.py", line 517, in f
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:       return function(**kwargs)
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:     File "/usr/local/lib/python2.7/dist-packages/azure/storage/tableservice.py", line 476, in update_entity
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:       response = self._perform_request(request)
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:     File "/usr/local/lib/python2.7/dist-packages/azure/storage/storageclient.py", line 179, in _perform_request
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:       _storage_error_handler(ex)
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:     File "/usr/local/lib/python2.7/dist-packages/azure/storage/__init__.py", line 1174, in _storage_error_handler
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:       return _general_error_handler(http_error)
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:     File "/usr/local/lib/python2.7/dist-packages/azure/__init__.py", line 551, in _general_error_handler
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:       http_error.respbody.decode('utf-8-sig'))
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:   WindowsAzureError: Unknown error (Bad Request)
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:   <?xml version="1.0" encoding="utf-8"?><error xmlns="http://schemas.microsoft.com/ado/2007/08/dataservices/metadata"><code>PropertyValueTooLarge</code><message xml:lang="en-US">The property value exceeds the maximum allowed size (64KB). If the property value is a string, it is UTF-16 encoded and the maximum number of characters should be 32K or less.
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:   RequestId:bc75b149-0002-0131-4274-260c9f000000
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:   Time:2015-11-24T04:54:04.2292362Z</message></error>
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:   Exiting the worker because of a failed jobWrapper on host c1agent34
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:   ERROR:toil.worker:Exiting the worker because of a failed jobWrapper on host c1agent34
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:   WARNING:toil.jobWrapper:Due to failure we are reducing the remaining retry count of job 4f782575_20d9_48bb_bc1f_94cc8b207479 to 0
```

I think, since I'm passing around lists of hundreds of sample names, and since I'm adding hundreds of grandchild jobs to one of my child jobs, that I might have jobs that are too big.

I think the offending code may be here: https://github.com/BD2KGenomics/toil/blob/8efce29fe68b8356f8c4fca9b52c2649c782dca0/src/toil/jobStores/azureJobStore.py#L585

It looks like it's cutting strings to 64k, but the error message says Azure does its own UTF-16 encoding, and that for strings we really need to cut at 32k.



Discussion:

Resolving patch:
diff --git a/src/toil/jobStores/azureJobStore.py b/src/toil/jobStores/azureJobStore.py
--- a/src/toil/jobStores/azureJobStore.py
+++ b/src/toil/jobStores/azureJobStore.py
@@ -19,7 +19,6 @@
 import inspect
 import bz2
 import cPickle
-import base64
 import socket
 import httplib
 from datetime import datetime, timedelta
@@ -29,7 +28,7 @@
 from azure import WindowsAzureMissingResourceError, WindowsAzureError
 
 from azure.storage import (TableService, BlobService, SharedAccessPolicy, AccessPolicy,
-                           BlobSharedAccessPermissions)
+                           BlobSharedAccessPermissions, EntityProperty)
 from bd2k.util import strict_bool
 
 from bd2k.util.threading import ExceptionalThread
@@ -68,12 +67,14 @@ def _fetchAzureAccountKey(accountName):
                            credential_file_path % accountName)
 
 
+maxAzureTablePropertySize = 64 * 1024
+
 class AzureJobStore(AbstractJobStore):
     """
     A job store that uses Azure's blob store for file storage and
     Table Service to store job info with strong consistency."""
 
-    def __init__(self, accountName, namePrefix, config=None, jobChunkSize=65535):
+    def __init__(self, accountName, namePrefix, config=None, jobChunkSize=maxAzureTablePropertySize):
         self.jobChunkSize = jobChunkSize
         self.keyPath = None
 
@@ -576,22 +577,24 @@ def fromItem(cls, item):
         chunkedJob.sort()
         if len(chunkedJob) == 1:
             # First element of list = tuple, second element of tuple = serialized job
-            wholeJobString = chunkedJob[0][1]
+            wholeJobString = chunkedJob[0][1].value
         else:
-            wholeJobString = ''.join(item[1] for item in chunkedJob)
-        return cPickle.loads(bz2.decompress(base64.b64decode(wholeJobString)))
+            wholeJobString = ''.join(item[1].value for item in chunkedJob)
+        return cPickle.loads(bz2.decompress(wholeJobString))
 
-    # Max size of a string value in Azure is 64K
-    def toItem(self, chunkSize=65535):
+    def toItem(self, chunkSize=maxAzureTablePropertySize):
         """
+        :param chunkSize: the size of a chunk for splitting up the serialized job into chunks
+        that each fit into a property value of the an Azure table entity
         :rtype: dict
         """
+        assert chunkSize <= maxAzureTablePropertySize
         item = {}
-        serializedAndEncodedJob = base64.b64encode(bz2.compress(cPickle.dumps(self)))
+        serializedAndEncodedJob = bz2.compress(cPickle.dumps(self))
         jobChunks = [serializedAndEncodedJob[i:i + chunkSize]
                      for i in range(0, len(serializedAndEncodedJob), chunkSize)]
         for attributeOrder, chunk in enumerate(jobChunks):
-            item['_' + str(attributeOrder).zfill(3)] = chunk
+            item['_' + str(attributeOrder).zfill(3)] = EntityProperty('Edm.Binary', chunk)
         return item
 
 


Test patch:
diff --git a/src/toil/test/jobStores/jobStoreTest.py b/src/toil/test/jobStores/jobStoreTest.py
--- a/src/toil/test/jobStores/jobStoreTest.py
+++ b/src/toil/test/jobStores/jobStoreTest.py
@@ -24,7 +24,6 @@
 import tempfile
 import uuid
 import shutil
-
 from toil.common import Config
 from toil.jobStores.abstractJobStore import (AbstractJobStore, NoSuchJobException,
                                              NoSuchFileException)
@@ -526,12 +525,21 @@ def testInvalidJobStoreName(self):
 class AzureJobStoreTest(hidden.AbstractJobStoreTest):
     def _createJobStore(self, config=None):
         from toil.jobStores.azureJobStore import AzureJobStore
-        return AzureJobStore('toiltest', self.namePrefix, config=config, jobChunkSize=128)
+        return AzureJobStore('toiltest', self.namePrefix, config=config)
 
     def _partSize(self):
         from toil.jobStores.azureJobStore import AzureJobStore
         return AzureJobStore._maxAzureBlockBytes
 
+    def testLargeJob(self):
+        from toil.jobStores.azureJobStore import maxAzureTablePropertySize
+        command = os.urandom(maxAzureTablePropertySize * 2)
+        job1 = self.master.create(command=command, memory=0, cores=0, disk=0)
+        self.assertEqual(job1.command, command)
+        job2 = self.master.load(job1.jobStoreID)
+        self.assertIsNot(job1, job2)
+        self.assertEqual(job2.command, command)
+
 
 class EncryptedFileJobStoreTest(FileJobStoreTest, hidden.AbstractEncryptedJobStoreTest):
     pass