# Buggy code
def _send_to_s3(self, force=False):
    """Copy in-memory batches to s3"""
    for table_name, batches in self._batches.items():
        if not force and len(batches) <= CACHE_SIZE:
            continue
        if table_name == SITE_VISITS_INDEX:
            out_str = "\n".join([json.dumps(x) for x in batches])
            if not isinstance(out_str, six.binary_type):
                out_str = out_str.encode("utf-8")
            fname = "%s/site_index/instance-%s-%s.json.gz" % (
                self.dir,
                self._instance_id,
                hashlib.md5(out_str).hexdigest(),
            )
            self._write_str_to_s3(out_str, fname)
        else:
            if len(batches) == 0:
                continue
            try:
                table = pa.Table.from_batches(batches)
                pq.write_to_dataset(
                    table,
                    self._s3_bucket_uri % table_name,
                    filesystem=self._fs,
                    preserve_index=False,
                    partition_cols=["instance_id"],
                    compression="snappy",
                    flavor="spark",
                )
            except (pa.lib.ArrowInvalid, EndpointConnectionError):
                self.logger.error(
                    "Error while sending records for: %s" % table_name, exc_info=True
                )
                pass
        self._batches[table_name] = list()


# Traceback (TypeError)
multiprocess_utils   - ERROR    - Exception in child process.

Traceback (most recent call last):

File "/home/travis/build/mozilla/OpenWPM/automation/utilities/multiprocess_utils.py", line 42, in run

mp.Process.run(self)

File "/home/travis/virtualenv/python2.7.15/lib/python2.7/site-packages/multiprocess/process.py", line 114, in run

self._target(*self._args, **self._kwargs)

File "/home/travis/build/mozilla/OpenWPM/automation/DataAggregator/S3Aggregator.py", line 56, in listener_process_runner

listener.drain_queue()

File "/home/travis/build/mozilla/OpenWPM/automation/DataAggregator/S3Aggregator.py", line 320, in drain_queue

self._send_to_s3(force=True)

File "/home/travis/build/mozilla/OpenWPM/automation/DataAggregator/S3Aggregator.py", line 227, in _send_to_s3

flavor='spark'

File "/home/travis/virtualenv/python2.7.15/lib/python2.7/site-packages/pyarrow/parquet.py", line 1450, in write_to_dataset

write_table(subtable, f, **kwargs)

File "/home/travis/virtualenv/python2.7.15/lib/python2.7/site-packages/pyarrow/parquet.py", line 1343, in write_table

**kwargs) as writer:

File "/home/travis/virtualenv/python2.7.15/lib/python2.7/site-packages/pyarrow/parquet.py", line 448, in __init__

**options)

File "pyarrow/_parquet.pyx", line 1220, in pyarrow._parquet.ParquetWriter.__cinit__

def __cinit__(self, where, Schema schema, use_dictionary=None,

TypeError: __cinit__() got an unexpected keyword argument 'preserve_index'

Process Process-17:

Traceback (most recent call last):

File "/home/travis/virtualenv/python2.7.15/lib/python2.7/site-packages/multiprocess/process.py", line 267, in _bootstrap

self.run()

File "/home/travis/build/mozilla/OpenWPM/automation/utilities/multiprocess_utils.py", line 50, in run

raise e

TypeError: __cinit__() got an unexpected keyword argument 'preserve_index'

# Fixed code
def _send_to_s3(self, force=False):
    """Copy in-memory batches to s3"""
    for table_name, batches in self._batches.items():
        if not force and len(batches) <= CACHE_SIZE:
            continue
        if table_name == SITE_VISITS_INDEX:
            out_str = "\n".join([json.dumps(x) for x in batches])
            if not isinstance(out_str, six.binary_type):
                out_str = out_str.encode("utf-8")
            fname = "%s/site_index/instance-%s-%s.json.gz" % (
                self.dir,
                self._instance_id,
                hashlib.md5(out_str).hexdigest(),
            )
            self._write_str_to_s3(out_str, fname)
        else:
            if len(batches) == 0:
                continue
            try:
                table = pa.Table.from_batches(batches)
                pq.write_to_dataset(
                    table,
                    self._s3_bucket_uri % table_name,
                    filesystem=self._fs,
                    partition_cols=["instance_id"],
                    compression="snappy",
                    flavor="spark",
                )
            except (pa.lib.ArrowInvalid, EndpointConnectionError):
                self.logger.error(
                    "Error while sending records for: %s" % table_name, exc_info=True
                )
                pass
        self._batches[table_name] = list()