PyTables/PyTables — Issue #129: Concurrent Reads of StringColumn Returning Zero-Length Strings
Author: JohnVinyard
Created: 2012-02-08 01:33:43+00:00
State: open

I was under the impression that concurrent reads were ok, but I've run into some strange behavior using PyTables and the multiprocessing module. Here's some code that reproduces the problem almost every time I run it on my machine:

import numpy as np
from tables import *
from uuid import uuid4
from multiprocessing import Pool
import os

if __name__ == '__main__':

class Data(IsDescription):
n = StringCol(itemsize=10,pos=0)

filename = '%s.h5' % str(uuid4())

# create a new file
hdf5 = openFile(filename,'w')
hdf5.createTable('/','data',Data)
table = hdf5.root.data
newrow = table.row

nrows = 500000

# populate it with data
for i in xrange(nrows):
newrow['n'] = str(i)
newrow.append()
hdf5.flush()
hdf5.close()
print 'created file with %i rows' % nrows

# re-open the file in read mode
hdf5 = openFile(filename,'r')
table = hdf5.root.data

# define a "worker" to check values from the file
def read(row):
n = table[row]['n']
val = str(n)
try:
assert val
except AssertionError,e:
print 'Bad val: %s' % val
print 'Length: %i' % len(n)
print 'Type: %s' % str(type(n))
print '--------------------------'
raise e
return n

# First, do the work in a single process to
# prove that everything works as expected
print 'trying with a single process'
try:
results = map(read,xrange(nrows))
except AssertionError:
print 'Value was missing in single process example'
hdf5.close()
os.remove(filename)
exit()

print 'single process example had no errors'

# Next, try a multiprocess version
print 'trying with multiple processes'
try:
# spawn some processes
p = Pool(processes=2)
results = p.map(read,xrange(nrows),chunksize=1000)
except AssertionError:
print 'Value was missing in multiprocess example'

hdf5.close()
os.remove(filename)