Repository: jubatus/jubatus-python-client

Issue:
Support python3
Now jubatus doesn't work in py3 environment

Discussion:

Patch:
diff --git a/jubatus/common/__init__.py b/jubatus/common/__init__.py
index 8775c9c..04ebbc0 100644
--- a/jubatus/common/__init__.py
+++ b/jubatus/common/__init__.py
@@ -1,7 +1,7 @@
-from message_string_generator import MessageStringGenerator
-from datum import Datum
-from types import TInt, TFloat, TBool, TString, TRaw, TNullable, TList, TMap, TTuple, TUserDef, TObject
-from client import Client, ClientBase, TypeMismatch, UnknownMethod
+from .message_string_generator import MessageStringGenerator
+from .datum import Datum
+from .types import TInt, TFloat, TBool, TString, TRaw, TNullable, TList, TMap, TTuple, TUserDef, TObject
+from .client import Client, ClientBase, TypeMismatch, UnknownMethod
 
 from contextlib import contextmanager
 
diff --git a/jubatus/common/client.py b/jubatus/common/client.py
index 29b197c..f965fe3 100644
--- a/jubatus/common/client.py
+++ b/jubatus/common/client.py
@@ -1,5 +1,5 @@
 import msgpackrpc
-from types import *
+from .types import *
 
 class InterfaceMismatch(Exception):
     pass
@@ -43,7 +43,7 @@ class Client(object):
 class ClientBase(object):
     def __init__(self, host, port, name, timeout=10):
         address = msgpackrpc.Address(host, port)
-        self.client = msgpackrpc.Client(address, timeout=timeout)
+        self.client = msgpackrpc.Client(address, timeout=timeout, unpack_encoding='utf-8')
         self.jubatus_client = Client(self.client, name)
 
     def get_client(self):
diff --git a/jubatus/common/compat.py b/jubatus/common/compat.py
new file mode 100644
index 0000000..1ee57fa
--- /dev/null
+++ b/jubatus/common/compat.py
@@ -0,0 +1,23 @@
+import sys
+
+if sys.version_info >= (3, 0):
+    int_types = (int, )
+    string_types = (str, )
+    binary_types = (bytes, )
+
+    def u(s):
+        return s
+
+    def b(s):
+        return s.encode()
+
+else:
+    int_types = (int, long)
+    string_types = (str, unicode)
+    binary_types = (str, )
+
+    def u(s):
+        return unicode(s)
+
+    def b(s):
+        return s
diff --git a/jubatus/common/datum.py b/jubatus/common/datum.py
index 16aacd3..b2aafbd 100644
--- a/jubatus/common/datum.py
+++ b/jubatus/common/datum.py
@@ -1,5 +1,6 @@
-from message_string_generator import MessageStringGenerator
-from types import *
+from .message_string_generator import MessageStringGenerator
+from .types import *
+from .compat import int_types, string_types, binary_types
 
 class Datum:
     TYPE = TTuple(TList(TTuple(TString(), TString())),
@@ -11,41 +12,41 @@ class Datum:
         self.num_values = []
         self.binary_values = []
 
-        for (k, v) in values.iteritems():
-            if not isinstance(k, (str, unicode)):
+        for (k, v) in values.items():
+            if not isinstance(k, string_types):
                 raise TypeError
 
-            if isinstance(v, (str, unicode)):
+            if isinstance(v, string_types):
                 self.string_values.append([k, v])
             elif isinstance(v, float):
                 self.num_values.append([k, v])
-            elif isinstance(v, int):
+            elif isinstance(v, int_types):
                 self.num_values.append([k, float(v)])
             else:
                 raise TypeError
 
     def add_string(self, key, value):
-        if not isinstance(key, (str, unicode)):
+        if not isinstance(key, string_types):
             raise TypeError
-        if isinstance(value, (str, unicode)):
+        if isinstance(value, string_types):
             self.string_values.append([key, value])
         else:
             raise TypeError
 
     def add_number(self, key, value):
-        if not isinstance(key, (str, unicode)):
+        if not isinstance(key, string_types):
             raise TypeError
         if isinstance(value, float):
             self.num_values.append([key, value])
-        elif isinstance(value, int):
+        elif isinstance(value, int_types):
             self.num_values.append([key, float(value)])
         else:
             raise TypeError
 
     def add_binary(self, key, value):
-        if not isinstance(key, (str, unicode)):
+        if not isinstance(key, string_types):
             raise TypeError
-        if isinstance(value, str):
+        if isinstance(value, binary_types):
             self.binary_values.append([key, value])
         else:
             raise TypeError
diff --git a/jubatus/common/types.py b/jubatus/common/types.py
index 9c909c4..91fa5bd 100644
--- a/jubatus/common/types.py
+++ b/jubatus/common/types.py
@@ -1,3 +1,5 @@
+from .compat import int_types, string_types, binary_types
+
 def check_type(value, typ):
     if not isinstance(value, typ):
         raise TypeError('type %s is expected, but %s is given' % (typ, type(value)))
@@ -23,20 +25,20 @@ class TPrimitive(object):
 class TInt(object):
     def __init__(self, signed, byts):
         if signed:
-            self.max = (1L << (8 * byts - 1)) - 1
-            self.min = - (1L << (8 * byts - 1))
+            self.max = (1 << (8 * byts - 1)) - 1
+            self.min = - (1 << (8 * byts - 1))
         else:
-            self.max = (1L << 8 * byts) - 1
+            self.max = (1 << 8 * byts) - 1
             self.min = 0
 
     def from_msgpack(self, m):
-        check_types(m, (int, long))
+        check_types(m, int_types)
         if not (self.min <= m and m <= self.max):
             raise ValueError('int value must be in (%d, %d), but %d is given' % (self.min, self.max, m))
         return m
 
     def to_msgpack(self, m):
-        check_types(m, (int, long))
+        check_types(m, int_types)
         if not (self.min <= m and m <= self.max):
             raise ValueError('int value must be in (%d, %d), but %d is given' % (self.min, self.max, m))
         return m
@@ -49,23 +51,32 @@ class TBool(TPrimitive):
     def __init__(self):
         super(TBool, self).__init__((bool,))
 
-class TString(TPrimitive):
-    def __init__(self):
-        super(TString, self).__init__((str, unicode))
+class TString(object):
+    def to_msgpack(self, m):
+        check_types(m, string_types)
+        return m
 
+    def from_msgpack(self, m):
+        check_types(m, string_types)
+        return m
+        # if isinstance(m, str):
+        #     return m
+        # elif isinstance(m, bytes):
+        #     return m.decode()
+        
 class TDatum(object):
     def from_msgpack(self, m):
-        from datum import Datum
+        from .datum import Datum
         return Datum.from_msgpack(m)
 
     def to_msgpack(self, m):
-        from datum import Datum
+        from .datum import Datum
         check_type(m, Datum)
         return m.to_msgpack()
 
 class TRaw(TPrimitive):
     def __init__(self):
-        super(TRaw, self).__init__((str,))
+        super(TRaw, self).__init__(binary_types)
 
 class TNullable(object):
     def __init__(self, type):
@@ -89,11 +100,11 @@ class TList(object):
 
     def from_msgpack(self, m):
         check_types(m, (list, tuple))
-        return map(self.type.from_msgpack, m)
+        return list(map(self.type.from_msgpack, m))
 
     def to_msgpack(self, m):
         check_types(m, (list, tuple))
-        return map(self.type.to_msgpack, m)
+        return list(map(self.type.to_msgpack, m))
 
 class TMap(object):
     def __init__(self, key, value):
@@ -103,7 +114,7 @@ class TMap(object):
     def from_msgpack(self, m):
         check_type(m, dict)
         dic = {}
-        for k, v in m.iteritems():
+        for k, v in m.items():
             dic[self.key.from_msgpack(k)] = self.value.from_msgpack(v)
         return dic
 
@@ -165,13 +176,13 @@ class TEnum(object):
         self.values = values
 
     def from_msgpack(self, m):
-        check_types(m, (int, long))
+        check_types(m, int_types)
         if m not in self.values:
             raise ValueError
         return m
 
     def to_msgpack(self, m):
-        check_types(m, (int, long))
+        check_types(m, int_types)
         if m not in self.values:
             raise ValueError
         return m
diff --git a/setup.py b/setup.py
index 83a96f8..9d09aa8 100644
--- a/setup.py
+++ b/setup.py
@@ -43,5 +43,5 @@ setup(name='jubatus',
           'Topic :: Scientific/Engineering :: Information Analysis'
       ],
 
-      test_suite='jubatus_test',
+      test_suite='jubatus_test.suite',
 )


Test patch:
diff --git a/test/__init__.py b/test/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/test/jubatus_test/__init__.py b/test/jubatus_test/__init__.py
index e69de29..d28802b 100644
--- a/test/jubatus_test/__init__.py
+++ b/test/jubatus_test/__init__.py
@@ -0,0 +1,4 @@
+import unittest
+
+def suite():
+    return unittest.defaultTestLoader.discover('.')
diff --git a/test/jubatus_test/common/client_test.py b/test/jubatus_test/common/test_client.py
similarity index 96%
rename from test/jubatus_test/common/client_test.py
rename to test/jubatus_test/common/test_client.py
index b390e29..0e929f8 100644
--- a/test/jubatus_test/common/client_test.py
+++ b/test/jubatus_test/common/test_client.py
@@ -64,7 +64,7 @@ class ClientTest(unittest.TestCase):
 
     def test_wrong_number_of_arguments(self):
         c = jubatus.common.Client(Echo(), "name")
-        self.assertEquals("test", c.call("test", [], AnyType(), []))
+        self.assertEqual("test", c.call("test", [], AnyType(), []))
         self.assertRaises(TypeError, c.call, "test", [1], AnyType(), [])
 
 if __name__ == '__main__':
diff --git a/test/jubatus_test/common/datum_test.py b/test/jubatus_test/common/test_datum.py
similarity index 63%
rename from test/jubatus_test/common/datum_test.py
rename to test/jubatus_test/common/test_datum.py
index c17d077..ef2d8c2 100644
--- a/test/jubatus_test/common/datum_test.py
+++ b/test/jubatus_test/common/test_datum.py
@@ -1,27 +1,28 @@
 from jubatus.common import Datum
 import unittest
 import msgpack
+from jubatus.common.compat import b, u
 
 class DatumTest(unittest.TestCase):
     def test_pack(self):
-        self.assertEquals(
+        self.assertEqual(
             msgpack.packb(([['name', 'Taro']], [['age', 20.0]], [])),
             msgpack.packb(Datum({'name': 'Taro', 'age': 20}).to_msgpack()))
 
     def test_unpack(self):
-        d = Datum.from_msgpack(([['name', 'Taro']], [['age', 20.0]], [['img', '0101']]))
-        self.assertEquals(
+        d = Datum.from_msgpack(([['name', 'Taro']], [['age', 20.0]], [['img', b('0101')]]))
+        self.assertEqual(
             [('name', 'Taro')],
             d.string_values)
-        self.assertEquals(
+        self.assertEqual(
             [('age', 20.0)],
             d.num_values)
-        self.assertEquals(
-            [('img', '0101')],
+        self.assertEqual(
+            [('img', b('0101'))],
             d.binary_values)
 
     def test_empty(self):
-        self.assertEquals(
+        self.assertEqual(
             msgpack.packb(([], [], [])),
             msgpack.packb(Datum().to_msgpack()))
 
@@ -35,13 +36,13 @@ class DatumTest(unittest.TestCase):
     def test_add_string(self):
         d = Datum()
         d.add_string('key', 'value')
-        self.assertEquals(Datum({'key': 'value'}).to_msgpack(),
-                          d.to_msgpack())
+        self.assertEqual(Datum({'key': 'value'}).to_msgpack(),
+                         d.to_msgpack())
 
         d = Datum()
-        d.add_string(u'key', u'value')
-        self.assertEquals(Datum({'key': 'value'}).to_msgpack(),
-                          d.to_msgpack())
+        d.add_string(u('key'), u('value'))
+        self.assertEqual(Datum({'key': 'value'}).to_msgpack(),
+                         d.to_msgpack())
 
     def test_invalid_add_string(self):
         d = Datum()
@@ -51,14 +52,14 @@ class DatumTest(unittest.TestCase):
     def test_add_number(self):
         d = Datum()
         d.add_number('key', 1.0)
-        self.assertEquals(Datum({'key': 1.0}).to_msgpack(),
-                          d.to_msgpack())
+        self.assertEqual(Datum({'key': 1.0}).to_msgpack(),
+                         d.to_msgpack())
 
     def test_add_int(self):
         d = Datum()
         d.add_number('key', 1)
-        self.assertEquals(Datum({'key': 1.0}).to_msgpack(),
-                          d.to_msgpack())
+        self.assertEqual(Datum({'key': 1.0}).to_msgpack(),
+                         d.to_msgpack())
 
     def test_invalid_add_number(self):
         d = Datum()
@@ -67,9 +68,9 @@ class DatumTest(unittest.TestCase):
 
     def test_add_binary(self):
         d = Datum()
-        d.add_binary('key', 'value')
-        self.assertEquals(
-            ([], [], [['key', 'value']]),
+        d.add_binary('key', b('value'))
+        self.assertEqual(
+            ([], [], [['key', b('value')]]),
             d.to_msgpack())
 
     def test_invalid_add_binary(self):
@@ -81,9 +82,9 @@ class DatumTest(unittest.TestCase):
         d = Datum()
         d.add_string('name', 'john')
         d.add_number('age', 20)
-        d.add_binary('image', '0101')
-        self.assertEquals('datum{string_values: [[\'name\', \'john\']], num_values: [[\'age\', 20.0]], binary_values: [[\'image\', \'0101\']]}',
-                          str(d))
+        d.add_binary('image', b('0101'))
+        s = str(d)
+        self.assertTrue('datum{string_values: [[\'name\', \'john\']], num_values: [[\'age\', 20.0]], binary_values: [[\'image\', \'0101\']]}' == s or 'datum{string_values: [[\'name\', \'john\']], num_values: [[\'age\', 20.0]], binary_values: [[\'image\', b\'0101\']]}' == s)
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/test/jubatus_test/common/message_string_generator_test.py b/test/jubatus_test/common/test_message_string_generator.py
similarity index 75%
rename from test/jubatus_test/common/message_string_generator_test.py
rename to test/jubatus_test/common/test_message_string_generator.py
index 567e7ab..a096917 100644
--- a/test/jubatus_test/common/message_string_generator_test.py
+++ b/test/jubatus_test/common/test_message_string_generator.py
@@ -8,14 +8,14 @@ class MessageStringGeneratorTest(unittest.TestCase):
         gen = MessageStringGenerator()
         gen.open("test")
         gen.close()
-        self.assertEquals("test{}", gen.to_string())
+        self.assertEqual("test{}", gen.to_string())
 
     def testOne(self):
         gen = MessageStringGenerator()
         gen.open("test")
         gen.add("k1", "v1")
         gen.close()
-        self.assertEquals("test{k1: v1}", gen.to_string())
+        self.assertEqual("test{k1: v1}", gen.to_string())
 
     def testTwo(self):
         gen = MessageStringGenerator()
@@ -23,14 +23,14 @@ class MessageStringGeneratorTest(unittest.TestCase):
         gen.add("k1", "v1")
         gen.add("k2", "v2")
         gen.close()
-        self.assertEquals("test{k1: v1, k2: v2}", gen.to_string())
+        self.assertEqual("test{k1: v1, k2: v2}", gen.to_string())
 
     def testNumber(self):
         gen = MessageStringGenerator()
         gen.open("test")
         gen.add("k1", 1)
         gen.close()
-        self.assertEquals("test{k1: 1}", gen.to_string())
+        self.assertEqual("test{k1: 1}", gen.to_string())
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/test/jubatus_test/common/types_test.py b/test/jubatus_test/common/test_types.py
similarity index 83%
rename from test/jubatus_test/common/types_test.py
rename to test/jubatus_test/common/test_types.py
index 41d6d6a..39a2f54 100644
--- a/test/jubatus_test/common/types_test.py
+++ b/test/jubatus_test/common/test_types.py
@@ -1,10 +1,11 @@
 from jubatus.common import *
+from jubatus.common.compat import u, b
 import unittest
 
 class TypeCheckTest(unittest.TestCase):
     def assertTypeOf(self, type, value):
-        self.assertEquals(value, type.from_msgpack(value))
-        self.assertEquals(value, type.to_msgpack(value))
+        self.assertEqual(value, type.from_msgpack(value))
+        self.assertEqual(value, type.to_msgpack(value))
 
     def assertTypeError(self, type, value):
         self.assertRaises(TypeError, lambda: type.from_msgpack(value))
@@ -19,14 +20,20 @@ class TypeCheckTest(unittest.TestCase):
         self.assertTypeError(TInt(True, 1), None)
         self.assertTypeError(TInt(True, 1), "")
         self.assertValueError(TInt(True, 1), 128)
+        self.assertValueError(TInt(True, 1), 1 << 40)
         self.assertValueError(TInt(True, 1), -129)
         self.assertValueError(TInt(False, 1), 256)
         self.assertValueError(TInt(False, 1), -1)
 
+    def testLong(self):
+        self.assertTypeOf(TInt(True, 8), 1)
+        self.assertTypeOf(TInt(True, 8), 1 << 40)
+
     def testFloat(self):
         self.assertTypeOf(TFloat(), 1.3)
         self.assertTypeError(TFloat(), None)
         self.assertTypeError(TFloat(), 1)
+        self.assertTypeError(TFloat(), 1 << 40)
 
     def testBool(self):
         self.assertTypeOf(TBool(), True)
@@ -34,13 +41,13 @@ class TypeCheckTest(unittest.TestCase):
         self.assertTypeError(TBool(), 1)
 
     def testString(self):
+        #self.assertTypeOf(TString(), b("test"))
         self.assertTypeOf(TString(), "test")
-        self.assertTypeOf(TString(), u"test")
         self.assertTypeError(TString(), 1)
 
     def testRaw(self):
-        self.assertTypeOf(TRaw(), "test")
-        self.assertTypeError(TRaw(), u"test")
+        self.assertTypeOf(TRaw(), b("test"))
+        self.assertTypeError(TRaw(), u("test"))
         self.assertTypeError(TRaw(), 1)
 
     def testNullable(self):
@@ -61,10 +68,10 @@ class TypeCheckTest(unittest.TestCase):
 
     def testTuple(self):
         typ = TTuple(TInt(True, 8), TTuple(TString(), TInt(True, 8)))
-        self.assertEquals(
+        self.assertEqual(
             [1, ["test", 1]],
             typ.to_msgpack((1, ("test", 1))))
-        self.assertEquals(
+        self.assertEqual(
             (1, ("test", 1)),
             typ.from_msgpack((1, ("test", 1))))
         self.assertTypeError(TTuple(TInt(True, 8)), ("test", ))
@@ -90,7 +97,7 @@ class TypeCheckTest(unittest.TestCase):
         typ = TUserDef(MyType)
         obj = typ.from_msgpack(("hoge", 1.0))
         self.assertTrue(isinstance(obj, MyType))
-        self.assertEquals(["hoge", 1.0], typ.to_msgpack(obj))
+        self.assertEqual(["hoge", 1.0], typ.to_msgpack(obj))
 
         self.assertTypeError(typ, 1)
         self.assertTypeError(typ, [])
diff --git a/test/jubatus_test/test_util.py b/test/jubatus_test/test_util.py
index 7de706d..8009a22 100644
--- a/test/jubatus_test/test_util.py
+++ b/test/jubatus_test/test_util.py
@@ -19,7 +19,7 @@ class TestUtil:
         try:
             cli.call("dummy")
             raise Exception("dummy rpc succeeded")
-        except RPCError, e:
+        except RPCError as e:
             if e.args[0] == 1: # "no such method"
                 return True                 # ... means server is fully up
         return False
@@ -34,7 +34,7 @@ class TestUtil:
                     return
                 if proc.poll():
                     stderr = proc.stderr.read()
-                    raise Exception('Cannot run server process: \n' + stderr)
+                    raise Exception('Cannot run server process: \n{0}'.format(stderr))
                 sleep_time *= 2;
         raise Exception("cannot connect")
 
@@ -53,7 +53,7 @@ class TestUtil:
                 raise Exception('Cannot run server process: \n' + stderr)
             return proc
         except OSError as error:
-            print 'Unable to fork. Error: %d (%s)' % (error.errno, error.strerror)
+            print('Unable to fork. Error: {0} ({1})'.format(error.errno, error.strerror))
             raise error
 
     @staticmethod