Repository: pydap/pydap

Issue:
Check byte use
I noticed in some places of the code byte is signed, while in others it's unsigned. I should verify how the DAP spec defines it (unsigned, I think) and ensure the proper type in the code.


Discussion:

Patch:
diff --git a/src/pydap/handlers/dap.py b/src/pydap/handlers/dap.py
index 8e8f6c5..72c44f1 100644
--- a/src/pydap/handlers/dap.py
+++ b/src/pydap/handlers/dap.py
@@ -26,11 +26,13 @@ from pydap.model import (BaseType,
 from ..net import GET, raise_for_status
 from ..lib import (
     encode, combine_slices, fix_slice, hyperslab,
-    START_OF_SEQUENCE, walk, StreamReader, BytesReader)
+    START_OF_SEQUENCE, walk, StreamReader, BytesReader,
+    DAP2_ARRAY_LENGTH_NUMPY_TYPE)
 from .lib import ConstraintExpression, BaseHandler, IterData
 from ..parsers.dds import build_dataset
 from ..parsers.das import parse_das, add_attributes
 from ..parsers import parse_ce
+from ..responses.dods import DAP2_response_dtypemap
 logger = logging.getLogger('pydap')
 logger.addHandler(logging.NullHandler())
 
@@ -354,15 +356,19 @@ def unpack_children(stream, template):
     return out
 
 
-def convert_stream_to_list(stream, dtype, shape, id):
+def convert_stream_to_list(stream, parser_dtype, shape, id):
     out = []
+    response_dtype = DAP2_response_dtypemap(parser_dtype)
     if shape:
-        n = np.fromstring(stream.read(4), ">I")[0]
-        count = dtype.itemsize * n
-        if dtype.char in "SU":
+        n = np.fromstring(stream.read(4), DAP2_ARRAY_LENGTH_NUMPY_TYPE)[0]
+        count = response_dtype.itemsize * n
+        if response_dtype.char in 'S':
+            # Consider on 'S' and not 'SU' because
+            # response_dtype.char should never be
             data = []
             for _ in range(n):
-                k = np.fromstring(stream.read(4), ">I")[0]
+                k = np.fromstring(stream.read(4),
+                                  DAP2_ARRAY_LENGTH_NUMPY_TYPE)[0]
                 data.append(stream.read(k))
                 stream.read(-k % 4)
             out.append(np.array([text_type(x.decode('ascii'))
@@ -372,7 +378,8 @@ def convert_stream_to_list(stream, dtype, shape, id):
             try:
                 out.append(
                     np.fromstring(
-                        stream.read(count), dtype).reshape(shape))
+                        stream.read(count), response_dtype)
+                    .astype(parser_dtype).reshape(shape))
             except ValueError as e:
                 if str(e) == 'total size of new array must be unchanged':
                     # server-side failure.
@@ -384,22 +391,26 @@ def convert_stream_to_list(stream, dtype, shape, id):
                                  'output_grid=False).').format(quote(id)))
                 else:
                     raise
-            if dtype.char == "B":
+            if response_dtype.char == "B":
+                # Unsigned Byte type is packed to multiples of 4 bytes:
                 stream.read(-n % 4)
 
     # special types: strings and bytes
-    elif dtype.char in 'SU':
-        k = np.fromstring(stream.read(4), '>I')[0]
+    elif response_dtype.char in 'S':
+        # Consider on 'S' and not 'SU' because
+        # response_dtype.char should never be
+        # 'U'
+        k = np.fromstring(stream.read(4), DAP2_ARRAY_LENGTH_NUMPY_TYPE)[0]
         out.append(text_type(stream.read(k).decode('ascii')))
         stream.read(-k % 4)
-    elif dtype.char == 'B':
-        data = np.fromstring(stream.read(1), dtype)[0]
-        stream.read(3)
-        out.append(data)
     # usual data
     else:
         out.append(
-            np.fromstring(stream.read(dtype.itemsize), dtype)[0])
+            np.fromstring(stream.read(response_dtype.itemsize), response_dtype)
+            .astype(parser_dtype)[0])
+        if response_dtype.char == "B":
+            # Unsigned Byte type is packed to multiples of 4 bytes:
+            stream.read(3)
     return out
 
 
@@ -427,9 +438,9 @@ def dump():  # pragma: no cover
     """
     dods = sys.stdin.read()
     dds, xdrdata = dods.split(b'\nData:\n', 1)
-    xdr_stream = io.BytesIO(xdrdata)
-    dds = dds.decode('ascii')
     dataset = build_dataset(dds)
+    xdr_stream = io.BytesIO(xdrdata)
     data = unpack_data(xdr_stream, dataset)
+    data = dataset.data
 
     pprint.pprint(data)
diff --git a/src/pydap/lib.py b/src/pydap/lib.py
index 6321a7d..916b9aa 100644
--- a/src/pydap/lib.py
+++ b/src/pydap/lib.py
@@ -17,6 +17,95 @@ START_OF_SEQUENCE = b'\x5a\x00\x00\x00'
 END_OF_SEQUENCE = b'\xa5\x00\x00\x00'
 STRING = '|S128'
 
+NUMPY_TO_DAP2_TYPEMAP = {
+    'd': 'Float64',
+    'f': 'Float32',
+    'h': 'Int16',
+    'H': 'UInt16',
+    'i': 'Int32', 'l': 'Int32', 'q': 'Int32',
+    'I': 'UInt32', 'L': 'UInt32', 'Q': 'UInt32',
+    # DAP2 does not support signed bytes.
+    # Its Byte type is unsigned and thus corresponds
+    # to numpy's 'B'.
+    # The consequence is that there is no natural way
+    # in DAP2 to represent numpy's 'b' type.
+    # Ideally, DAP2 would have a signed Byte type
+    # and an unsigned UByte type and we would have the
+    # following mapping: {'b': 'Byte', 'B': 'UByte'}
+    # but this not how the protocol has been defined.
+    # This means that numpy's 'b' must be mapped to Int16
+    # and data must be upconverted in the DODS response.
+    'b': 'Int16',
+    'B': 'Byte',
+    # There are no boolean types in DAP2. Upconvert to
+    # Byte:
+    '?': 'Byte',
+    'S': 'String',
+    # Map numpy's 'U' to String b/c
+    # DAP2 does not explicitly support unicode.
+    'U': 'String'
+}
+
+# DAP2 demands big-endian 32 bytes signed integers
+# www.opendap.org/pdf/dap_2_data_model.pdf
+# Before pydap 3.2.2, length was
+# big-endian 32 bytes UNSIGNED integers:
+# DAP2_ARRAY_LENGTH_NUMPY_TYPE = '>I'
+# Since pydap 3.2.2, the length type is accurate:
+DAP2_ARRAY_LENGTH_NUMPY_TYPE = '>i'
+
+DAP2_TO_NUMPY_RESPONSE_TYPEMAP = {
+    'Float64': '>d',
+    'Float32': '>f',
+    # This is a weird aspect of the DAP2 specification.
+    # For backward-compatibility, Int16 and UInt16 are
+    # encoded as 32 bits integers in the response,
+    # respectively:
+    'Int16': '>i',
+    'UInt16': '>I',
+    'Int32': '>i',
+    'UInt32': '>I',
+    # DAP2 does not support signed bytes.
+    # It's Byte type is unsigned and thus corresponds
+    # to numpy 'B'.
+    # The consequence is that there is no natural way
+    # in DAP2 to represent numpy's 'b' type.
+    # Ideally, DAP2 would have a signed Byte type
+    # and a usigned UByte type and we would have the
+    # following mapping: {'Byte': 'b', 'UByte': 'B'}
+    # but this not how the protocol has been defined.
+    # This means that DAP2 Byte is unsigned and must be
+    # mapped to numpy's 'B' type, usigned byte.
+    'Byte': 'B',
+    # Map String to numpy's string type 'S' b/c
+    # DAP2 does not explicitly support unicode.
+    'String': 'S',
+    'URL': 'S',
+    #
+    # These two types are not DAP2 but it is useful
+    # to include them for compatiblity with other
+    # data sources:
+    'Int': '>i',
+    'UInt': '>I',
+}
+
+# Typemap from lower case DAP2 types to
+# numpy dtype string with specified endiannes.
+# Here, the endianness is very important:
+LOWER_DAP2_TO_NUMPY_PARSER_TYPEMAP = {
+    'float64': '>d',
+    'float32': '>f',
+    'int16': '>h',
+    'uint16': '>H',
+    'int32': '>i',
+    'uint32': '>I',
+    'byte': 'B',
+    'string': STRING,
+    'url': STRING,
+    'int': '>i',
+    'uint': '>I',
+}
+
 
 def quote(name):
     """Return quoted name according to the DAP specification.
diff --git a/src/pydap/parsers/das.py b/src/pydap/parsers/das.py
index ff3910d..1618e89 100644
--- a/src/pydap/parsers/das.py
+++ b/src/pydap/parsers/das.py
@@ -16,10 +16,6 @@ from . import SimpleParser
 from ..lib import walk
 
 
-atomic = ('byte', 'int', 'uint', 'int16', 'uint16', 'int32', 'uint32',
-          'float32', 'float64', 'string', 'url')
-
-
 class DASParser(SimpleParser):
 
     """A parser for the Dataset Attribute Structure response."""
diff --git a/src/pydap/parsers/dds.py b/src/pydap/parsers/dds.py
index 981cb7f..b0cb56a 100644
--- a/src/pydap/parsers/dds.py
+++ b/src/pydap/parsers/dds.py
@@ -8,25 +8,22 @@ from . import SimpleParser
 from ..model import (DatasetType, BaseType,
                      SequenceType, StructureType,
                      GridType)
-from ..lib import quote, STRING
-
-typemap = {
-    'byte':    np.dtype("B"),
-    'int':     np.dtype(">i4"),
-    'uint':    np.dtype(">u4"),
-    'int16':   np.dtype(">i4"),
-    'uint16':  np.dtype(">u4"),
-    'int32':   np.dtype(">i4"),
-    'uint32':  np.dtype(">u4"),
-    'float32': np.dtype(">f4"),
-    'float64': np.dtype(">f8"),
-    'string':  np.dtype(STRING),  # default is "|S128"
-    'url':     np.dtype(STRING),
-    }
+from ..lib import (quote, LOWER_DAP2_TO_NUMPY_PARSER_TYPEMAP)
+
 constructors = ('grid', 'sequence', 'structure')
 name_regexp = r'[\w%!~"\'\*-]+'
 
 
+def DAP2_parser_typemap(type_string):
+    """
+    This function takes a numpy dtype object
+    and returns a dtype object that is compatible with
+    the DAP2 specification.
+    """
+    dtype_str = LOWER_DAP2_TO_NUMPY_PARSER_TYPEMAP[type_string.lower()]
+    return np.dtype(dtype_str)
+
+
 class DDSParser(SimpleParser):
 
     """A parser for the DDS."""
@@ -72,14 +69,15 @@ class DDSParser(SimpleParser):
 
     def base(self):
         """Parse a base variable, returning a ``BaseType``."""
-        type = self.consume(r'\w+')
+        data_type_string = self.consume('\w+')
+
+        parser_dtype = DAP2_parser_typemap(data_type_string)
+        name = quote(self.consume('[^;\[]+'))
 
-        dtype = typemap[type.lower()]
-        name = quote(self.consume(r'[^;\[]+'))
         shape, dimensions = self.dimensions()
         self.consume(';')
 
-        data = DummyData(dtype, shape)
+        data = DummyData(parser_dtype, shape)
         var = BaseType(name, data, dimensions=dimensions)
 
         return var
diff --git a/src/pydap/responses/das.py b/src/pydap/responses/das.py
index 5364734..1b18516 100644
--- a/src/pydap/responses/das.py
+++ b/src/pydap/responses/das.py
@@ -19,9 +19,8 @@ from six.moves import map
 from ..model import (DatasetType, BaseType,
                      StructureType, SequenceType,
                      GridType)
-from ..lib import encode, quote, __version__
+from ..lib import encode, quote, __version__, NUMPY_TO_DAP2_TYPEMAP
 from .lib import BaseResponse
-from .dds import typemap
 
 
 INDENT = ' ' * 4
@@ -130,13 +129,13 @@ def get_type(values):
 
     """
     if hasattr(values, 'dtype'):
-        return typemap[values.dtype.char]
+        return NUMPY_TO_DAP2_TYPEMAP[values.dtype.char]
     elif isinstance(values, string_types) or not isinstance(values, Iterable):
         return type_convert(values)
     else:
         # if there are several values, they may have different types, so we
         # need to convert all of them and use a precedence table
-        types = list(map(type_convert, values))
+        types = [type_convert(val) for val in values]
         precedence = ['String', 'Float64', 'Int32']
         types.sort(key=precedence.index)
         return types[0]
diff --git a/src/pydap/responses/dds.py b/src/pydap/responses/dds.py
index aa38c02..44f9bf8 100644
--- a/src/pydap/responses/dds.py
+++ b/src/pydap/responses/dds.py
@@ -19,24 +19,11 @@ from ..model import (DatasetType, BaseType,
                      SequenceType, StructureType,
                      GridType)
 from .lib import BaseResponse
-from ..lib import __version__
+from ..lib import __version__, NUMPY_TO_DAP2_TYPEMAP
 
 
 INDENT = ' ' * 4
 
-typemap = {
-    'd': 'Float64',
-    'f': 'Float32',
-    'h': 'Int16',
-    'i': 'Int32', 'l': 'Int32', 'q': 'Int32',
-    'b': 'Byte',
-    'H': 'UInt16',
-    'I': 'UInt32', 'L': 'UInt32', 'Q': 'UInt32',
-    'B': 'Byte',
-    'S': 'String',
-    'U': 'String',
-}
-
 
 class DDSResponse(BaseResponse):
 
@@ -119,6 +106,6 @@ def _basetype(var, level=0, sequence=0):
 
     yield '{indent}{type} {name}{shape};\n'.format(
         indent=level*INDENT,
-        type=typemap[var.dtype.char],
+        type=NUMPY_TO_DAP2_TYPEMAP[var.dtype.char],
         name=var.name,
         shape=shape)
diff --git a/src/pydap/responses/dods.py b/src/pydap/responses/dods.py
index a9d10ab..16cd795 100644
--- a/src/pydap/responses/dods.py
+++ b/src/pydap/responses/dods.py
@@ -16,7 +16,10 @@ from six import string_types
 
 from ..model import (BaseType,
                      SequenceType, StructureType)
-from ..lib import walk, START_OF_SEQUENCE, END_OF_SEQUENCE, __version__
+from ..lib import (walk, START_OF_SEQUENCE, END_OF_SEQUENCE, __version__,
+                   NUMPY_TO_DAP2_TYPEMAP,
+                   DAP2_TO_NUMPY_RESPONSE_TYPEMAP,
+                   DAP2_ARRAY_LENGTH_NUMPY_TYPE)
 from .lib import BaseResponse
 from .dds import dds
 
@@ -25,16 +28,24 @@ try:
 except ImportError:
     from singledispatch import singledispatch
 
-typemap = {
-    'd': '>d',
-    'f': '>f',
-    'i': '>i', 'l': '>i', 'q': '>i', 'h': '>i',
-    'b': 'B',
-    'I': '>I', 'L': '>I', 'Q': '>I', 'H': '>I',
-    'B': 'B',
-    'U': 'S'  # Map unicode to string type b/c
-              # DAP doesn't explicitly support it
-}
+
+def DAP2_response_dtypemap(dtype):
+    """
+    This function takes a numpy dtype object
+    and returns a dtype object that is compatible with
+    the DAP2 specification.
+    """
+    dtype_str = DAP2_TO_NUMPY_RESPONSE_TYPEMAP[
+                    NUMPY_TO_DAP2_TYPEMAP[
+                        dtype.char]]
+    return np.dtype(dtype_str)
+
+
+def tostring_with_byteorder(x, dtype):
+    return (x
+            .astype(dtype.str)
+            .newbyteorder(dtype.byteorder)
+            .tostring())
 
 
 class DODSResponse(BaseResponse):
@@ -79,16 +90,19 @@ def _structuretype(var):
 def _sequencetype(var):
     # a flat array can be processed one record (or more?) at a time
     if all(isinstance(child, BaseType) for child in var.children()):
-        types = []
+        DAP2_types = []
         position = 0
         for child in var.children():
-            if child.dtype.char in 'SU':
-                types.append('>I')                  # string length as int
-                types.append('|S{%s}' % position)   # string padded to 4n
+            if DAP2_response_dtypemap(child.dtype).char == 'S':
+                (DAP2_types
+                 .append(DAP2_ARRAY_LENGTH_NUMPY_TYPE))  # string length
+                DAP2_types.append('|S{%s}' % position)   # string padded to 4n
                 position += 1
             else:
-                types.append(typemap[child.dtype.char])
-        dtype = ','.join(types)
+                # Convert any numpy dtypes to numpy dtypes compatible
+                # with the DAP2 standard:
+                DAP2_types.append(DAP2_response_dtypemap(child.dtype).str)
+        DAP2_dtype_str = ','.join(DAP2_types)
         strings = position > 0
 
         # array initializations is costy, so we keep a cache here; this will
@@ -108,12 +122,21 @@ def _sequencetype(var):
                         padded.append(length + (-length % 4))
                     out.append(value)
                 record = out
-                dtype = ','.join(types).format(*padded)
-
-            if dtype not in cache:
-                cache[dtype] = np.zeros((1,), dtype=dtype)
-            cache[dtype][:] = tuple(record)
-            yield cache[dtype].tostring()
+                DAP2_dtype_str = ','.join(DAP2_types).format(*padded)
+
+            if DAP2_dtype_str not in cache:
+                # Remember that DAP2_dtype is a (possibly composite)
+                # numpy dtype that is compatible with the DAP2
+                # data model. This means that all dtypes in
+                # DAP2_dtype are representable in DAP2 -- AND --
+                # the data in var can all be upconverted
+                # in a lossless manner to the dtypes in DAP2_dtype.
+                cache[DAP2_dtype_str] = np.zeros((1,), dtype=DAP2_dtype_str)
+            # By assigning record to ``cache`` the upconversion
+            # occurs naturally:
+            cache[DAP2_dtype_str][:] = tuple(record)
+            # byteorder was taken care of during the upconversion:
+            yield cache[DAP2_dtype_str].tostring()
 
         yield END_OF_SEQUENCE
 
@@ -139,35 +162,52 @@ def _basetype(var):
     if not hasattr(data, "shape"):
         data = np.asarray(data)
 
+    # Convert any numpy dtypes to numpy dtypes compatible
+    # with the DAP2 standard:
+    DAP2_dtype = DAP2_response_dtypemap(data.dtype)
+
     if data.shape:
         # pack length for arrays
-        length = np.prod(data.shape).astype('I')
-        if data.dtype.char in 'SU':
-            yield length.newbyteorder('>').tostring()
-        else:
-            yield length.newbyteorder('>').tostring() * 2
-
-    # bytes are padded up to 4n
-    if data.dtype == np.byte:
-        length = np.prod(data.shape)
+        length = np.prod(data.shape).astype(np.int)
+
+        # send length twice at the begining of an array...
+        factor = 2
+        if DAP2_dtype.char == 'S':
+            # ... expcept for strings:
+            factor = 1
+        yield tostring_with_byteorder(
+                length,
+                np.dtype(DAP2_ARRAY_LENGTH_NUMPY_TYPE)) * factor
+
+    # make data iterable; 1D arrays must be converted to 2D, since
+    # iteration over 1D yields scalars which are not properly cast to big
+    # endian
+    # This line was removed because endianness is now treated explicitly
+    # in tostring_with_byteorder()
+    # if len(data.shape) < 2:
+    #     data = data.reshape(1, -1)
+    # Only ensure that 0d arrays are iterable:
+    if len(data.shape) == 0:
+        data = data[np.newaxis]
+
+    # unsigned bytes are padded up to 4n
+    if DAP2_dtype == np.ubyte:
+        length = np.prod(data.shape).astype(np.int)
         for block in data:
-            yield block.tostring()
+            yield tostring_with_byteorder(block, DAP2_dtype)
         yield (-length % 4) * b'\0'
 
     # regular data
     else:
-        # make data iterable; 1D arrays must be converted to 2D, since
-        # iteration over 1D yields scalars which are not properly cast to big
-        # endian
-        if len(data.shape) < 2:
-            data = data.reshape(1, -1)
-
         # strings are also zero padded and preceeded by their length
-        if data.dtype.char in 'SU':
+        if DAP2_dtype.char == 'S':
             for block in data:
                 for word in block.flat:
                     length = len(word)
-                    yield np.array(length).astype('>I').tostring()
+                    yield tostring_with_byteorder(
+                                np.array(length),
+                                np.dtype(DAP2_ARRAY_LENGTH_NUMPY_TYPE))
+                    # byteorder is not important for strings:
                     if hasattr(word, 'encode'):
                         yield word.encode('ascii')
                     elif hasattr(word, 'tostring'):
@@ -178,7 +218,13 @@ def _basetype(var):
                     yield (-length % 4) * b'\0'
         else:
             for block in data:
-                yield block.astype(typemap[data.dtype.char]).tostring()
+                # Remember that DAP2_dtype is a
+                # numpy dtype that is compatible with the DAP2
+                # data model. This means that the dtype in
+                # DAP2_dtype is representable in DAP2 -- AND --
+                # the data in var can all be upconverted
+                # in a lossless manner to the dtype in DAP2_dtype.
+                yield tostring_with_byteorder(block, DAP2_dtype)
 
 
 def calculate_size(dataset):
@@ -191,22 +237,26 @@ def calculate_size(dataset):
         # individually, so it's not possible to get their size unless we read
         # everything.
         if (isinstance(var, SequenceType) or
-                (isinstance(var, BaseType) and var.dtype.char in 'SU')):
+            (isinstance(var, BaseType) and
+             DAP2_response_dtypemap(var.dtype).char == 'S')):
             return None
         elif isinstance(var, BaseType):
             if var.shape:
                 length += 8  # account for array size marker
 
             size = int(np.prod(var.shape))
-            if var.data.dtype == np.byte:
+
+            # Convert any numpy dtype to numpy dtype compatible
+            # with the DAP2 standard:
+            DAP2_dtype = DAP2_response_dtypemap(var.data.dtype)
+            if DAP2_dtype == np.ubyte:
                 length += size + (-size % 4)
-            elif var.data.dtype == np.short:
-                length += size * 4
             else:
-                opendap_size = np.dtype(typemap[var.data.dtype.char]).itemsize
-                length += size * opendap_size
+                # Remember that numpy dtypes are upconverted to
+                # DAP2_dtype when sent in the response.
+                # Their length must thus be modified accordingly:
+                length += size * DAP2_dtype.itemsize
 
     # account for DDS
     length += len(''.join(dds(dataset))) + len(b'Data:\n')
-
     return length


Test patch:
diff --git a/src/pydap/tests/datasets.py b/src/pydap/tests/datasets.py
index 9c52498..4e9be1e 100644
--- a/src/pydap/tests/datasets.py
+++ b/src/pydap/tests/datasets.py
@@ -15,6 +15,7 @@ from pydap.client import open_file
 
 from collections import OrderedDict
 
+# Note that DAP2 does not support signed bytes (signed 8bits integers).
 
 # A very simple sequence: flat and with no strings. This sequence can be mapped
 # directly to a Numpy structured array, and can be easily encoded and decoded
@@ -33,7 +34,7 @@ VerySimpleSequence["sequence"].data = np.array([
     (5, 6, 60.),
     (6, 7, 70.),
     (7, 8, 80.),
-    ], dtype=[('byte', 'b'), ('int', 'i4'), ('float', 'f4')])
+    ], dtype=[('byte', np.ubyte), ('int', 'i4'), ('float', 'f4')])
 
 
 # A nested sequence.
@@ -57,7 +58,7 @@ NestedSequence["location"].data = IterData([
 # A simple array with bytes, strings and shorts. These types require special
 # encoding for the DODS response.
 SimpleArray = DatasetType("SimpleArray")
-SimpleArray["byte"] = BaseType("byte", np.arange(5, dtype="b"))
+SimpleArray["byte"] = BaseType("byte", np.arange(5, dtype=np.ubyte))
 SimpleArray["string"] = BaseType("string", np.array(["one", "two"]))
 SimpleArray["short"] = BaseType("short", np.array(1, dtype="h"))
 
@@ -95,16 +96,19 @@ SimpleStructure['types'] = StructureType(
         ("array", np.array(1)),
         ("float", 1000.0),
     ]))
-SimpleStructure['types']['b'] = BaseType('b', np.array(0, np.byte))
-SimpleStructure['types']['i32'] = BaseType('i32', np.array(1, np.int32))
-SimpleStructure['types']['ui32'] = BaseType('ui32', np.array(0, np.uint32))
-SimpleStructure['types']['i16'] = BaseType('i16', np.array(0, np.int16))
-SimpleStructure['types']['ui16'] = BaseType('ui16', np.array(0, np.uint16))
-SimpleStructure['types']['f32'] = BaseType('f32', np.array(0.0, np.float32))
+SimpleStructure['types']['b'] = BaseType('b', np.array(-10, np.byte))
+SimpleStructure['types']['ub'] = BaseType('ub', np.array(10, np.ubyte))
+SimpleStructure['types']['i32'] = BaseType('i32', np.array(-10, np.int32))
+SimpleStructure['types']['ui32'] = BaseType('ui32', np.array(10, np.uint32))
+SimpleStructure['types']['i16'] = BaseType('i16', np.array(-10, np.int16))
+SimpleStructure['types']['ui16'] = BaseType('ui16', np.array(10, np.uint16))
+SimpleStructure['types']['f32'] = BaseType('f32', np.array(100.0, np.float32))
 SimpleStructure['types']['f64'] = BaseType('f64', np.array(1000., np.float64))
 SimpleStructure['types']['s'] = BaseType(
     's', np.array("This is a data test string (pass 0)."))
 SimpleStructure['types']['u'] = BaseType('u', np.array("http://www.dods.org"))
+SimpleStructure['types']['U'] = BaseType('U', np.array(u"test unicode",
+                                                       np.unicode))
 
 
 # test grid
diff --git a/src/pydap/tests/test_client.py b/src/pydap/tests/test_client.py
index 1f03425..3fa6f68 100644
--- a/src/pydap/tests/test_client.py
+++ b/src/pydap/tests/test_client.py
@@ -189,7 +189,6 @@ class TestFunctions(unittest.TestCase):
             np.array(1.0))
 
 
-@unittest.skip("Unsure about this behavior")
 class Test16Bits(unittest.TestCase):
 
     """Test that 16-bit values are represented correctly.
@@ -204,11 +203,11 @@ class Test16Bits(unittest.TestCase):
         self.app = BaseHandler(SimpleStructure)
 
     def test_int16(self):
-        """Load an int16."""
+        """Load an int16 -> should yield '>i2' type."""
         dataset = open_url("http://localhost:8001/", self.app)
         self.assertEqual(dataset.types.i16.dtype, np.dtype(">i2"))
 
     def test_uint16(self):
-        """Load an uint16."""
+        """Load an uint16 -> should yield '>u2' type."""
         dataset = open_url("http://localhost:8001/", self.app)
         self.assertEqual(dataset.types.ui16.dtype, np.dtype(">u2"))
diff --git a/src/pydap/tests/test_handlers_dap.py b/src/pydap/tests/test_handlers_dap.py
index 5c563ec..6f77f08 100644
--- a/src/pydap/tests/test_handlers_dap.py
+++ b/src/pydap/tests/test_handlers_dap.py
@@ -215,7 +215,7 @@ class TestBaseProxy(unittest.TestCase):
 
         self.data = BaseProxy(
                               "http://localhost:8001/", "byte",
-                              np.dtype("b"), (5,),
+                              np.dtype("B"), (5,),
                               application=self.app)
 
     def test_repr(self):
@@ -223,7 +223,7 @@ class TestBaseProxy(unittest.TestCase):
         self.assertEqual(
                          repr(self.data),
                          "BaseProxy('http://localhost:8001/', 'byte', "
-                         "dtype('int8'), (5,), "
+                         "dtype('uint8'), (5,), "
                          "(slice(None, None, None),))")
 
     def test_getitem(self):
@@ -270,12 +270,13 @@ class TestBaseProxyShort(unittest.TestCase):
 
         self.data = BaseProxy(
                               "http://localhost:8001/", "short",
-                              np.dtype(">i"), (),
+                              np.dtype(">h"), (),
                               application=self.app)
 
     def test_getitem(self):
         """Test the ``__getitem__`` method."""
         np.testing.assert_array_equal(self.data[:], np.array(1))
+        assert self.data[:].dtype.char == 'h'
 
 
 class TestBaseProxyString(unittest.TestCase):
diff --git a/src/pydap/tests/test_parsers_dds.py b/src/pydap/tests/test_parsers_dds.py
index 9726f44..0e707e5 100644
--- a/src/pydap/tests/test_parsers_dds.py
+++ b/src/pydap/tests/test_parsers_dds.py
@@ -70,13 +70,13 @@ class TestBuildDataset(unittest.TestCase):
     def test_integer_16(self):
         """Test integer 16 parsing."""
         self.assertIsInstance(self.dataset.structure.i16, BaseType)
-        self.assertEqual(self.dataset.structure.i16.dtype, np.dtype(">i"))
+        self.assertEqual(self.dataset.structure.i16.dtype, np.dtype(">h"))
         self.assertEqual(self.dataset.structure.i16.shape, ())
 
     def test_unsigned_integer_16(self):
         """Test unsigned integer 16 parsing."""
         self.assertIsInstance(self.dataset.structure.ui16, BaseType)
-        self.assertEqual(self.dataset.structure.ui16.dtype, np.dtype(">I"))
+        self.assertEqual(self.dataset.structure.ui16.dtype, np.dtype(">H"))
         self.assertEqual(self.dataset.structure.ui16.shape, ())
 
     def test_integer_32(self):
diff --git a/src/pydap/tests/test_responses_das.py b/src/pydap/tests/test_responses_das.py
index c7935e9..95440fc 100644
--- a/src/pydap/tests/test_responses_das.py
+++ b/src/pydap/tests/test_responses_das.py
@@ -126,6 +126,8 @@ class TestDASResponseStructure(unittest.TestCase):
         }
         b {
         }
+        ub {
+        }
         i32 {
         }
         ui32 {
@@ -142,6 +144,8 @@ class TestDASResponseStructure(unittest.TestCase):
         }
         u {
         }
+        U {
+        }
     }
 }
 """)
diff --git a/src/pydap/tests/test_responses_dds.py b/src/pydap/tests/test_responses_dds.py
index 0073b00..2d4d341 100644
--- a/src/pydap/tests/test_responses_dds.py
+++ b/src/pydap/tests/test_responses_dds.py
@@ -96,7 +96,8 @@ class TestDDSResponseStructure(unittest.TestCase):
         res = app.get('/.dds')
         self.assertEqual(res.text, """Dataset {
     Structure {
-        Byte b;
+        Int16 b;
+        Byte ub;
         Int32 i32;
         UInt32 ui32;
         Int16 i16;
@@ -105,6 +106,7 @@ class TestDDSResponseStructure(unittest.TestCase):
         Float64 f64;
         String s;
         String u;
+        String U;
     } types;
 } SimpleStructure;
 """)
diff --git a/src/pydap/tests/test_responses_dods.py b/src/pydap/tests/test_responses_dods.py
index 595ccd0..fdfd4ef 100644
--- a/src/pydap/tests/test_responses_dods.py
+++ b/src/pydap/tests/test_responses_dods.py
@@ -5,12 +5,13 @@ import copy
 import numpy as np
 from webtest import TestApp as App
 from webob.headers import ResponseHeaders
+from collections import OrderedDict
 
 from pydap.lib import START_OF_SEQUENCE, END_OF_SEQUENCE, __version__
 from pydap.handlers.lib import BaseHandler
 from pydap.tests.datasets import (
     VerySimpleSequence, SimpleSequence, SimpleGrid,
-    SimpleArray, NestedSequence)
+    SimpleArray, NestedSequence, SimpleStructure)
 from pydap.responses.dods import dods, DODSResponse
 import unittest
 
@@ -147,6 +148,72 @@ class TestDODSResponseGrid(unittest.TestCase):
             b"\x00\x00\x00\x01")
 
 
+class TestDODSResponseStructure(unittest.TestCase):
+
+    """Test the DODS response with a sequence that has a string."""
+
+    def setUp(self):
+        """Create a simple WSGI app."""
+        app = App(BaseHandler(SimpleStructure))
+        self.res = app.get('/.dods')
+
+    def test_body(self):
+        """Test response body."""
+        dds, xdrdata = self.res.body.split(b'\nData:\n', 1)
+        dds = dds.decode('ascii')
+        self.assertEqual(dds, """Dataset {
+    Structure {
+        Int16 b;
+        Byte ub;
+        Int32 i32;
+        UInt32 ui32;
+        Int16 i16;
+        UInt16 ui16;
+        Float32 f32;
+        Float64 f64;
+        String s;
+        String u;
+        String U;
+    } types;
+} SimpleStructure;""")
+
+        expected = OrderedDict([
+            # -10 signed byte upconv. to Int32 for transfer
+            ('b', (np.array(-10, dtype=np.byte)
+                   .astype('>i')
+                   .tostring())),
+            # 10 usigned byte padded to multiple of 4 bytes for transfer
+            ('ub', (np.array(10, dtype=np.ubyte)
+                    .tostring()) + b'\x00\x00\x00'),
+            ('i32', (np.array(-10, dtype=np.int32)
+                     .astype('>i').tostring())),
+            ('ui32', (np.array(10, dtype=np.uint32)
+                      .astype('>I').tostring())),
+            # -10 int16 upconv. to int32 for transfer
+            ('i16', (np.array(-10, dtype=np.int16)
+                     .astype('>i')
+                     .newbyteorder('>').tostring())),
+            # 10 uint16 upconv. to uint32 for transfer
+            ('ui16', (np.array(10, dtype=np.uint16)
+                      .astype('>I')
+                      .tostring())),
+            ('f32', (np.array(100, dtype=np.float32)
+                     .astype('>f')
+                     .tostring())),
+            ('f64', (np.array(1000, dtype=np.float64)
+                     .astype('>d')
+                     .tostring())),
+            ('s', b'\x00\x00\x00$This is a data test string (pass 0).\x00'),
+            ('u', b'\x00\x00\x13http://www.dods.org\x00'),
+            ('U', b'\x00\x00\x00\x0ctest unicode')])
+
+        for key in expected:
+            string = expected[key]
+            assert xdrdata.startswith(string)
+            xdrdata = xdrdata[len(string):]
+        assert xdrdata == b''
+
+
 class TestDODSResponseSequence(unittest.TestCase):
 
     """Test the DODS response with a sequence that has a string."""