astropy/astropy — Issue #106: vo tests all fail in Mac OS X 10.6
Author: eteq
Created: 2011-12-08 08:32:21+00:00
State: open

When I run the current master (baeb755edc89ecb78ca942ed1d1efa98d7d4f85d) test suite under OS X with python 2.7, all except one (test_too_many_columns is the only passing one) of the tests in astropy/io/vo/tests/vo_test.py fail or error. The failures are in pastebin at http://paste.pocoo.org/show/517880/ through http://paste.pocoo.org/show/517888/ . There are 96 errors, so I'm not going to include them all here, but they all seem fairly similar in where they lead to the one I've pasted below (mainly in that they all end at _convert_to_fd_or_read_function with an IOError).

Also, I did a ``git bisect`` to figure out where the error was introduced - it appears to be baeb755edc89ecb78ca942ed1d1efa98d7d4f85d , the xml refactoring commit (not necesarilly surprisingly).

```
________________ ERROR at setup of TestFixups.test_implicit_id _________________

self =

def setup_class(self):
self.table = parse(
> join(ROOT_DIR, "regression.xml"), pedantic=False).get_first_table()

astropy/io/vo/tests/vo_test.py:179:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

source = '/Users/erik/src/astropy/astropy/io/vo/tests/data/regression.xml'
columns = None, invalid = 'exception', pedantic = False, chunk_size = 256
table_number = None, filename = None, _debug_python_based_parser = False

def parse(source, columns=None, invalid='exception', pedantic=True,
chunk_size=tree.DEFAULT_CHUNK_SIZE, table_number=None,
filename=None,
_debug_python_based_parser=False):
"""
Parses a VOTABLE_ xml file (or file-like object), and returns a
`~astropy.io.vo.tree.VOTable` object, with nested
`~astropy.io.vo.tree.Resource` instances and
`~astropy.io.vo.tree.Table` instances.

Parameters
----------
source : str or readable file-like object
Path or file object containing a VOTABLE_ xml file.

columns : sequence of str, optional
List of field names to include in the output. The default is
to include all fields.

invalid : str, optional
One of the following values:

- 'exception': throw an exception when an invalid value is
encountered (default)

- 'mask': mask out invalid values

pedantic : bool, optional
When `True`, raise an error when the file violates the spec,
otherwise issue a warning. Warnings may be controlled using
the standard Python mechanisms. See the `warnings`
module in the Python standard library for more information.

chunk_size : int, optional
The number of rows to read before converting to an array.
Higher numbers are likely to be faster, but will consume more
memory.

table_number : int, optional
The number of table in the file to read in. If `None`, all
tables will be read. If a number, 0 refers to the first table
in the file, and only that numbered table will be parsed and
read in.

filename : str, optional
A filename, URL or other identifier to use in error messages.
If *filename* is None and *source* is a string (i.e. a path),
then *source* will be used as a filename for error messages.
Therefore, *filename* is only required when source is a
file-like object.

Returns
-------
votable : `astropy.io.vo.tree.VOTableFile` object

See also
--------
astropy.io.vo.exceptions : The exceptions this function may raise.
"""
invalid = invalid.lower()
assert invalid in ('exception', 'mask')

config = {
'columns' : columns,
'invalid' : invalid,
'pedantic' : pedantic,
'chunk_size' : chunk_size,
'table_number' : table_number,
'filename' : filename}

if filename is None and isinstance(source, basestring):
config['filename'] = source

with iterparser.get_xml_iterator(
source,
_debug_python_based_parser=_debug_python_based_parser) as iterator:
return tree.VOTableFile(
> config=config, pos=(1, 1)).parse(iterator, config)

def parse_single_table(source, **kwargs):

astropy/io/vo/table.py:100:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = , type = None
value = None, traceback = None

def __exit__(self, type, value, traceback):
if type is None:
try:
> self.gen.next()

/Library/Frameworks/Python.framework/Versions/7.0/lib/python2.7/contextlib.py:24:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

source = '/Users/erik/src/astropy/astropy/io/vo/tests/data/regression.xml'
_debug_python_based_parser = False

@contextlib.contextmanager
def get_xml_iterator(source, _debug_python_based_parser=False):
"""
Returns an iterator over the elements of an XML file.

The iterator doesn't ever build a tree, so it is much more memory
and time efficient than the alternative in `cElementTree`.

Parameters
----------
fd : readable file-like object or read function

Returns
-------
parts : iterator

The iterator returns 4-tuples (*start*, *tag*, *data*, *pos*):

- *start*: when `True` is a start element event, otherwise
an end element event.

- *tag*: The name of the element

- *data*: Depends on the value of *event*:

- if *start* == `True`, data is a dictionary of
attributes

- if *start* == `False`, data is a string containing
the text content of the element

- *pos*: Tuple (*line*, *col*) indicating the source of the
event.
"""
with _convert_to_fd_or_read_function(source) as fd:
if _debug_python_based_parser:
context = _slow_iterparse(fd)
else:
context = _fast_iterparse(fd)
> yield iter(context)

def get_xml_encoding(source):

astropy/utils/xml/iterparser.py:195:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = , type = None
value = None, traceback = None

def __exit__(self, type, value, traceback):
if type is None:
try:
> self.gen.next()

/Library/Frameworks/Python.framework/Versions/7.0/lib/python2.7/contextlib.py:24:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

fd = '/Users/erik/src/astropy/astropy/io/vo/tests/data/regression.xml'

@contextlib.contextmanager
def _convert_to_fd_or_read_function(fd):
"""
Returns a function suitable for streaming input, or a file object.

This function is only useful if passing off to C code where:

- If it's a real file object, we want to use it as a real
C file object to avoid the Python overhead.

- If it's not a real file object, it's much handier to just
have a Python function to call.

Parameters
----------
fd : object
May be:

- a file object, in which case it is returned verbatim.

- a function that reads from a stream, in which case it is
returned verbatim.

- a file path, in which case it is opened. If it ends in
`.gz`, it is assumed to be a gzipped file, and the
:meth:`read` method on the file object is returned.
Otherwise, the raw file object is returned.

- an object with a :meth:`read` method, in which case that
method is returned.

Returns
-------
fd : context-dependent
See above.
"""
if IS_PY3K:
if isinstance(fd, io.IOBase):
yield fd
return
else:
if isinstance(fd, file):
yield fd
return
if is_callable(fd):
yield fd
return
elif isinstance(fd, basestring):
if fd.endswith('.gz'):
from ...utils.compat import gzip
with gzip.GzipFile(fd, 'rb') as real_fd:
yield real_fd.read
real_fd.flush()
return
else:
with open(fd, 'rb') as real_fd:
yield real_fd
> real_fd.flush()
E IOError: [Errno 9] Bad file descriptor

astropy/utils/xml/iterparser.py:94: IOError
________________ ERROR at setup of TestReferences.test_fieldref ________________

self =

def setup_class(self):
> self.votable = parse(join(ROOT_DIR, "regression.xml"), pedantic=False)

astropy/io/vo/tests/vo_test.py:190:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

source = '/Users/erik/src/astropy/astropy/io/vo/tests/data/regression.xml'
columns = None, invalid = 'exception', pedantic = False, chunk_size = 256
table_number = None, filename = None, _debug_python_based_parser = False

def parse(source, columns=None, invalid='exception', pedantic=True,
chunk_size=tree.DEFAULT_CHUNK_SIZE, table_number=None,
filename=None,
_debug_python_based_parser=False):
"""
Parses a VOTABLE_ xml file (or file-like object), and returns a
`~astropy.io.vo.tree.VOTable` object, with nested
`~astropy.io.vo.tree.Resource` instances and
`~astropy.io.vo.tree.Table` instances.

Parameters
----------
source : str or readable file-like object
Path or file object containing a VOTABLE_ xml file.

columns : sequence of str, optional
List of field names to include in the output. The default is
to include all fields.

invalid : str, optional
One of the following values:

- 'exception': throw an exception when an invalid value is
encountered (default)

- 'mask': mask out invalid values

pedantic : bool, optional
When `True`, raise an error when the file violates the spec,
otherwise issue a warning. Warnings may be controlled using
the standard Python mechanisms. See the `warnings`
module in the Python standard library for more information.

chunk_size : int, optional
The number of rows to read before converting to an array.
Higher numbers are likely to be faster, but will consume more
memory.

table_number : int, optional
The number of table in the file to read in. If `None`, all
tables will be read. If a number, 0 refers to the first table
in the file, and only that numbered table will be parsed and
read in.

filename : str, optional
A filename, URL or other identifier to use in error messages.
If *filename* is None and *source* is a string (i.e. a path),
then *source* will be used as a filename for error messages.
Therefore, *filename* is only required when source is a
file-like object.

Returns
-------
votable : `astropy.io.vo.tree.VOTableFile` object

See also
--------
astropy.io.vo.exceptions : The exceptions this function may raise.
"""
invalid = invalid.lower()
assert invalid in ('exception', 'mask')

config = {
'columns' : columns,
'invalid' : invalid,
'pedantic' : pedantic,
'chunk_size' : chunk_size,
'table_number' : table_number,
'filename' : filename}

if filename is None and isinstance(source, basestring):
config['filename'] = source

with iterparser.get_xml_iterator(
source,
_debug_python_based_parser=_debug_python_based_parser) as iterator:
return tree.VOTableFile(
> config=config, pos=(1, 1)).parse(iterator, config)

def parse_single_table(source, **kwargs):

astropy/io/vo/table.py:100:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = , type = None
value = None, traceback = None

def __exit__(self, type, value, traceback):
if type is None:
try:
> self.gen.next()

/Library/Frameworks/Python.framework/Versions/7.0/lib/python2.7/contextlib.py:24:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

source = '/Users/erik/src/astropy/astropy/io/vo/tests/data/regression.xml'
_debug_python_based_parser = False

@contextlib.contextmanager
def get_xml_iterator(source, _debug_python_based_parser=False):
"""
Returns an iterator over the elements of an XML file.

The iterator doesn't ever build a tree, so it is much more memory
and time efficient than the alternative in `cElementTree`.

Parameters
----------
fd : readable file-like object or read function

Returns
-------
parts : iterator

The iterator returns 4-tuples (*start*, *tag*, *data*, *pos*):

- *start*: when `True` is a start element event, otherwise
an end element event.

- *tag*: The name of the element

- *data*: Depends on the value of *event*:

- if *start* == `True`, data is a dictionary of
attributes

- if *start* == `False`, data is a string containing
the text content of the element

- *pos*: Tuple (*line*, *col*) indicating the source of the
event.
"""
with _convert_to_fd_or_read_function(source) as fd:
if _debug_python_based_parser:
context = _slow_iterparse(fd)
else:
context = _fast_iterparse(fd)
> yield iter(context)

def get_xml_encoding(source):

astropy/utils/xml/iterparser.py:195:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = , type = None
value = None, traceback = None

def __exit__(self, type, value, traceback):
if type is None:
try:
> self.gen.next()

/Library/Frameworks/Python.framework/Versions/7.0/lib/python2.7/contextlib.py:24:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

fd = '/Users/erik/src/astropy/astropy/io/vo/tests/data/regression.xml'

@contextlib.contextmanager
def _convert_to_fd_or_read_function(fd):
"""
Returns a function suitable for streaming input, or a file object.

This function is only useful if passing off to C code where:

- If it's a real file object, we want to use it as a real
C file object to avoid the Python overhead.

- If it's not a real file object, it's much handier to just
have a Python function to call.

Parameters
----------
fd : object
May be:

- a file object, in which case it is returned verbatim.

- a function that reads from a stream, in which case it is
returned verbatim.

- a file path, in which case it is opened. If it ends in
`.gz`, it is assumed to be a gzipped file, and the
:meth:`read` method on the file object is returned.
Otherwise, the raw file object is returned.

- an object with a :meth:`read` method, in which case that
method is returned.

Returns
-------
fd : context-dependent
See above.
"""
if IS_PY3K:
if isinstance(fd, io.IOBase):
yield fd
return
else:
if isinstance(fd, file):
yield fd
return
if is_callable(fd):
yield fd
return
elif isinstance(fd, basestring):
if fd.endswith('.gz'):
from ...utils.compat import gzip
with gzip.GzipFile(fd, 'rb') as real_fd:
yield real_fd.read
real_fd.flush()
return
else:
with open(fd, 'rb') as real_fd:
yield real_fd
> real_fd.flush()
E IOError: [Errno 9] Bad file descriptor

astropy/utils/xml/iterparser.py:94: IOError
```