Repository: databiosphere/toil
Issue: Maximum Job Data Size on Azure Too Small

Upon trying to scale up my pipeline from one sample to hundreds of samples on Azure, I am met with errors like the following:

```
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:   Traceback (most recent call last):
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:     File "/usr/local/lib/python2.7/dist-packages/toil/worker.py", line 282, in main
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:       fileStore=fileStore)
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:     File "/usr/local/lib/python2.7/dist-packages/toil/job.py", line 1282, in _execute
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:       self._serialiseJobGraph(jobWrapper, jobStore, returnValues, False)
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:     File "/usr/local/lib/python2.7/dist-packages/toil/job.py", line 1233, in _serialiseJobGraph
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:       job._serialiseJob(jobStore, jobsToJobWrappers, jobWrapper)
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:     File "/usr/local/lib/python2.7/dist-packages/toil/job.py", line 1196, in _serialiseJob
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:       jobStore.update(jobsToJobWrappers[self])
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:     File "/usr/local/lib/python2.7/dist-packages/toil/jobStores/azureJobStore.py", line 150, in update
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:       entity=job.toItem(chunkSize=self.jobChunkSize))
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:     File "/usr/local/lib/python2.7/dist-packages/toil/jobStores/azureJobStore.py", line 517, in f
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:       return function(**kwargs)
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:     File "/usr/local/lib/python2.7/dist-packages/azure/storage/tableservice.py", line 476, in update_entity
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:       response = self._perform_request(request)
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:     File "/usr/local/lib/python2.7/dist-packages/azure/storage/storageclient.py", line 179, in _perform_request
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:       _storage_error_handler(ex)
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:     File "/usr/local/lib/python2.7/dist-packages/azure/storage/__init__.py", line 1174, in _storage_error_handler
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:       return _general_error_handler(http_error)
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:     File "/usr/local/lib/python2.7/dist-packages/azure/__init__.py", line 551, in _general_error_handler
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:       http_error.respbody.decode('utf-8-sig'))
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:   WindowsAzureError: Unknown error (Bad Request)
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:   <?xml version="1.0" encoding="utf-8"?><error xmlns="http://schemas.microsoft.com/ado/2007/08/dataservices/metadata"><code>PropertyValueTooLarge</code><message xml:lang="en-US">The property value exceeds the maximum allowed size (64KB). If the property value is a string, it is UTF-16 encoded and the maximum number of characters should be 32K or less.
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:   RequestId:bc75b149-0002-0131-4274-260c9f000000
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:   Time:2015-11-24T04:54:04.2292362Z</message></error>
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:   Exiting the worker because of a failed jobWrapper on host c1agent34
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:   ERROR:toil.worker:Exiting the worker because of a failed jobWrapper on host c1agent34
WARNING:toil.leader:4f782575_20d9_48bb_bc1f_94cc8b207479:   WARNING:toil.jobWrapper:Due to failure we are reducing the remaining retry count of job 4f782575_20d9_48bb_bc1f_94cc8b207479 to 0
```

I think, since I'm passing around lists of hundreds of sample names, and since I'm adding hundreds of grandchild jobs to one of my child jobs, that I might have jobs that are too big.

I think the offending code may be here: https://github.com/BD2KGenomics/toil/blob/8efce29fe68b8356f8c4fca9b52c2649c782dca0/src/toil/jobStores/azureJobStore.py#L585

It looks like it's cutting strings to 64k, but the error message says Azure does its own UTF-16 encoding, and that for strings we really need to cut at 32k.


Changed files:
src/toil/jobStores/azureJobStore.py, src/toil/test/jobStores/jobStoreTest.py

diff --git a/src/toil/jobStores/azureJobStore.py b/src/toil/jobStores/azureJobStore.py
index bde3d534..ddc522c4 100644
--- a/src/toil/jobStores/azureJobStore.py
+++ b/src/toil/jobStores/azureJobStore.py
@@ -19,7 +19,6 @@ from contextlib import contextmanager
 import inspect
 import bz2
 import cPickle
-import base64
 import socket
 import httplib
 from datetime import datetime, timedelta
@@ -29,7 +28,7 @@ from ConfigParser import RawConfigParser, NoOptionError
 from azure import WindowsAzureMissingResourceError, WindowsAzureError
 
 from azure.storage import (TableService, BlobService, SharedAccessPolicy, AccessPolicy,
-                           BlobSharedAccessPermissions)
+                           BlobSharedAccessPermissions, EntityProperty)
 from bd2k.util import strict_bool
 
 from bd2k.util.threading import ExceptionalThread
@@ -68,12 +67,14 @@ def _fetchAzureAccountKey(accountName):
                            credential_file_path % accountName)
 
 
+maxAzureTablePropertySize = 64 * 1024
+
 class AzureJobStore(AbstractJobStore):
     """
     A job store that uses Azure's blob store for file storage and
     Table Service to store job info with strong consistency."""
 
-    def __init__(self, accountName, namePrefix, config=None, jobChunkSize=65535):
+    def __init__(self, accountName, namePrefix, config=None, jobChunkSize=maxAzureTablePropertySize):
         self.jobChunkSize = jobChunkSize
         self.keyPath = None
 
@@ -576,22 +577,24 @@ class AzureJob(JobWrapper):
         chunkedJob.sort()
         if len(chunkedJob) == 1:
             # First element of list = tuple, second element of tuple = serialized job
-            wholeJobString = chunkedJob[0][1]
+            wholeJobString = chunkedJob[0][1].value
         else:
-            wholeJobString = ''.join(item[1] for item in chunkedJob)
-        return cPickle.loads(bz2.decompress(base64.b64decode(wholeJobString)))
+            wholeJobString = ''.join(item[1].value for item in chunkedJob)
+        return cPickle.loads(bz2.decompress(wholeJobString))
 
-    # Max size of a string value in Azure is 64K
-    def toItem(self, chunkSize=65535):
+    def toItem(self, chunkSize=maxAzureTablePropertySize):
         """
+        :param chunkSize: the size of a chunk for splitting up the serialized job into chunks
+        that each fit into a property value of the an Azure table entity
         :rtype: dict
         """
+        assert chunkSize <= maxAzureTablePropertySize
         item = {}
-        serializedAndEncodedJob = base64.b64encode(bz2.compress(cPickle.dumps(self)))
+        serializedAndEncodedJob = bz2.compress(cPickle.dumps(self))
         jobChunks = [serializedAndEncodedJob[i:i + chunkSize]
                      for i in range(0, len(serializedAndEncodedJob), chunkSize)]
         for attributeOrder, chunk in enumerate(jobChunks):
-            item['_' + str(attributeOrder).zfill(3)] = chunk
+            item['_' + str(attributeOrder).zfill(3)] = EntityProperty('Edm.Binary', chunk)
         return item
 
 
diff --git a/src/toil/test/jobStores/jobStoreTest.py b/src/toil/test/jobStores/jobStoreTest.py
index 781f26b3..b06a8329 100644
--- a/src/toil/test/jobStores/jobStoreTest.py
+++ b/src/toil/test/jobStores/jobStoreTest.py
@@ -24,7 +24,6 @@ from threading import Thread
 import tempfile
 import uuid
 import shutil
-
 from toil.common import Config
 from toil.jobStores.abstractJobStore import (AbstractJobStore, NoSuchJobException,
                                              NoSuchFileException)
@@ -526,12 +525,21 @@ class InvalidAWSJobStoreTest(ToilTest):
 class AzureJobStoreTest(hidden.AbstractJobStoreTest):
     def _createJobStore(self, config=None):
         from toil.jobStores.azureJobStore import AzureJobStore
-        return AzureJobStore('toiltest', self.namePrefix, config=config, jobChunkSize=128)
+        return AzureJobStore('toiltest', self.namePrefix, config=config)
 
     def _partSize(self):
         from toil.jobStores.azureJobStore import AzureJobStore
         return AzureJobStore._maxAzureBlockBytes
 
+    def testLargeJob(self):
+        from toil.jobStores.azureJobStore import maxAzureTablePropertySize
+        command = os.urandom(maxAzureTablePropertySize * 2)
+        job1 = self.master.create(command=command, memory=0, cores=0, disk=0)
+        self.assertEqual(job1.command, command)
+        job2 = self.master.load(job1.jobStoreID)
+        self.assertIsNot(job1, job2)
+        self.assertEqual(job2.command, command)
+
 
 class EncryptedFileJobStoreTest(FileJobStoreTest, hidden.AbstractEncryptedJobStoreTest):
     pass