text: # Copyright 2016 The TensorFlow Authors. All Rights Reserved.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  # ==============================================================================
  """Tests for string_split_op."""
  
  from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  
  from absl.testing import parameterized
  import numpy as np
  
  from tensorflow.python.framework import constant_op
  from tensorflow.python.framework import dtypes
  from tensorflow.python.framework import errors_impl
  from tensorflow.python.framework import test_util
  from tensorflow.python.ops import array_ops
  from tensorflow.python.ops import string_ops
  from tensorflow.python.ops.ragged import ragged_factory_ops
  from tensorflow.python.ops.ragged import ragged_string_ops
  from tensorflow.python.platform import test
  from tensorflow.python.util import compat
  
  
  class StringSplitOpTest(test.TestCase):
  
    def testStringSplit(self):
      strings = ["pigs on the wing", "animals"]
  
      with self.cached_session():
        tokens = string_ops.string_split(strings)
        indices, values, shape = self.evaluate(tokens)
        self.assertAllEqual(indices, [[0, 0], [0, 1], [0, 2], [0, 3], [1, 0]])
        self.assertAllEqual(values, [b"pigs", b"on", b"the", b"wing", b"animals"])
        self.assertAllEqual(shape, [2, 4])
  
    @test_util.run_deprecated_v1
    def testStringSplitEmptyDelimiter(self):
      strings = ["hello", "hola", b"\xF0\x9F\x98\x8E"]  # Last string is U+1F60E
  
      with self.cached_session():
        tokens = string_ops.string_split(strings, delimiter="")
        indices, values, shape = self.evaluate(tokens)
        self.assertAllEqual(indices, [[0, 0], [0, 1], [0, 2], [0, 3], [0, 4],
                                      [1, 0], [1, 1], [1, 2], [1, 3], [2, 0],
                                      [2, 1], [2, 2], [2, 3]])
        expected = np.array(
            [
                "h", "e", "l", "l", "o", "h", "o", "l", "a", b"\xf0", b"\x9f",
                b"\x98", b"\x8e"
            ],
            dtype="|S1")
        self.assertAllEqual(values.tolist(), expected)
        self.assertAllEqual(shape, [3, 5])
  
    def testStringSplitEmptyToken(self):
      strings = ["", " a", "b ", " c", " ", " d ", "  e", "f  ", "  g  ", "  "]
  
      with self.cached_session():
        tokens = string_ops.string_split(strings)
        indices, values, shape = self.evaluate(tokens)
        self.assertAllEqual(
            indices,
            [[1, 0], [2, 0], [3, 0], [5, 0], [6, 0], [7, 0], [8, 0]])
        self.assertAllEqual(values, [b"a", b"b", b"c", b"d", b"e", b"f", b"g"])
        self.assertAllEqual(shape, [10, 1])
  
    def testStringSplitOnSetEmptyToken(self):
      strings = ["", " a", "b ", " c", " ", " d ", ". e", "f .", " .g. ", " ."]
  
      with self.cached_session():
        tokens = string_ops.string_split(strings, delimiter=" .")
        indices, values, shape = self.evaluate(tokens)
        self.assertAllEqual(
            indices,
            [[1, 0], [2, 0], [3, 0], [5, 0], [6, 0], [7, 0], [8, 0]])
        self.assertAllEqual(values, [b"a", b"b", b"c", b"d", b"e", b"f", b"g"])
        self.assertAllEqual(shape, [10, 1])
  
    @test_util.run_deprecated_v1
    def testStringSplitWithDelimiter(self):
      strings = ["hello|world", "hello world"]
  
      with self.cached_session():
        self.assertRaises(
            ValueError, string_ops.string_split, strings, delimiter=["|", ""])
  
        self.assertRaises(
            ValueError, string_ops.string_split, strings, delimiter=["a"])
  
        tokens = string_ops.string_split(strings, delimiter="|")
        indices, values, shape = self.evaluate(tokens)
        self.assertAllEqual(indices, [[0, 0], [0, 1], [1, 0]])
        self.assertAllEqual(values, [b"hello", b"world", b"hello world"])
        self.assertAllEqual(shape, [2, 2])
  
        tokens = string_ops.string_split(strings, delimiter="| ")
        indices, values, shape = self.evaluate(tokens)
        self.assertAllEqual(indices, [[0, 0], [0, 1], [1, 0], [1, 1]])
        self.assertAllEqual(values, [b"hello", b"world", b"hello", b"world"])
        self.assertAllEqual(shape, [2, 2])
  
    @test_util.run_deprecated_v1
    def testStringSplitWithDelimiterTensor(self):
      strings = ["hello|world", "hello world"]
  
      with self.cached_session() as sess:
        delimiter = array_ops.placeholder(dtypes.string)
  
        tokens = string_ops.string_split(strings, delimiter=delimiter)
  
        with self.assertRaises(errors_impl.InvalidArgumentError):
          sess.run(tokens, feed_dict={delimiter: ["a", "b"]})
        with self.assertRaises(errors_impl.InvalidArgumentError):
          sess.run(tokens, feed_dict={delimiter: ["a"]})
        indices, values, shape = sess.run(tokens, feed_dict={delimiter: "|"})
  
        self.assertAllEqual(indices, [[0, 0], [0, 1], [1, 0]])
        self.assertAllEqual(values, [b"hello", b"world", b"hello world"])
        self.assertAllEqual(shape, [2, 2])
  
    @test_util.run_deprecated_v1
    def testStringSplitWithDelimitersTensor(self):
      strings = ["hello.cruel,world", "hello cruel world"]
  
      with self.cached_session() as sess:
        delimiter = array_ops.placeholder(dtypes.string)
  
        tokens = string_ops.string_split(strings, delimiter=delimiter)
  
        with self.assertRaises(errors_impl.InvalidArgumentError):
          sess.run(tokens, feed_dict={delimiter: ["a", "b"]})
        with self.assertRaises(errors_impl.InvalidArgumentError):
          sess.run(tokens, feed_dict={delimiter: ["a"]})
        indices, values, shape = sess.run(tokens, feed_dict={delimiter: ".,"})
  
        self.assertAllEqual(indices, [[0, 0], [0, 1], [0, 2], [1, 0]])
        self.assertAllEqual(values,
                            [b"hello", b"cruel", b"world", b"hello cruel world"])
        self.assertAllEqual(shape, [2, 3])
  
    def testStringSplitWithNoSkipEmpty(self):
      strings = ["#a", "b#", "#c#"]
  
      with self.cached_session():
        tokens = string_ops.string_split(strings, "#", skip_empty=False)
        indices, values, shape = self.evaluate(tokens)
        self.assertAllEqual(indices, [[0, 0], [0, 1],
                                      [1, 0], [1, 1],
                                      [2, 0], [2, 1], [2, 2]])
        self.assertAllEqual(values, [b"", b"a", b"b", b"", b"", b"c", b""])
        self.assertAllEqual(shape, [3, 3])
  
      with self.cached_session():
        tokens = string_ops.string_split(strings, "#")
        indices, values, shape = self.evaluate(tokens)
        self.assertAllEqual(values, [b"a", b"b", b"c"])
        self.assertAllEqual(indices, [[0, 0], [1, 0], [2, 0]])
        self.assertAllEqual(shape, [3, 1])
  
  
  class StringSplitV2OpTest(test_util.TensorFlowTestCase, parameterized.TestCase):
  
    @parameterized.named_parameters([
        {"testcase_name": "Simple",
         "input": [b"pigs on the wing", b"animals"],
         "expected": [[b"pigs", b"on", b"the", b"wing"], [b"animals"]]},
  
        {"testcase_name": "MultiCharSeparator",
         "input": [b"1<>2<>3", b"<><>4<>5<><>6<>"],
         "sep": b"<>",
         "expected": [[b"1", b"2", b"3"],
                      [b"", b"", b"4", b"5", b"", b"6", b""]]},
  
        {"testcase_name": "SimpleSeparator",
         "input": [b"1,2,3", b"4,5,,6,"],
         "sep": b",",
         "expected": [[b"1", b"2", b"3"], [b"4", b"5", b"", b"6", b""]]},
  
        {"testcase_name": "EmptySeparator",
         "input": [b"1 2 3", b"  4  5    6  "],
         "expected": [[b"1", b"2", b"3"], [b"4", b"5", b"6"]]},
  
        {"testcase_name": "EmptySeparatorEmptyInputString",
         "input": [b""],
         "expected": [[]]},
  
        {"testcase_name": "EmptyInputVector",
         "input": [],
         "expected": []},
  
        {"testcase_name": "SimpleSeparatorMaxSplit",
         "input": [b"1,2,3", b"4,5,,6,"],
         "sep": b",",
         "maxsplit": 1,
         "expected": [[b"1", b"2,3"], [b"4", b"5,,6,"]]},
  
        {"testcase_name": "EmptySeparatorMaxSplit",
         "input": [b"1 2 3", b"  4  5    6  "],
         "maxsplit": 1,
         "expected": [[b"1", b"2 3"], [b"4", b"5    6  "]]},
  
        {"testcase_name": "ScalarInput",
         "input": b"1,2,3",
         "sep": b",",
         "expected": [b"1", b"2", b"3"]},
  
        {"testcase_name": "Dense2DInput",
         "input": [[b"1,2,3", b"4"], [b"5,6", b"7,8,9"]],
         "sep": b",",
         "expected": [[[b"1", b"2", b"3"], [b"4"]],
                      [[b"5", b"6"], [b"7", b"8", b"9"]]]},
  
        {"testcase_name": "Ragged2DInput",
         "input": [[b"1,2,3", b"4"], [b"5,6"]],
         "input_is_ragged": True,
         "sep": b",",
         "expected": [[[b"1", b"2", b"3"], [b"4"]], [[b"5", b"6"]]]},
  
        {"testcase_name": "Ragged3DInput",
         "input": [[[b"1,2,3", b"4"], [b"5,6"]], [[b"7,8,9"]]],
         "input_is_ragged": True,
         "sep": b",",
         "expected": [[[[b"1", b"2", b"3"], [b"4"]], [[b"5", b"6"]]],
                      [[[b"7", b"8", b"9"]]]]},
  
        {"testcase_name": "Ragged4DInput",
         "input": [[[[b"1,2,3", b"4"], [b"5,6"]], [[b"7,8,9"]]], [[[b""]]]],
         "input_is_ragged": True,
         "sep": b",",
         "expected": [[[[[b"1", b"2", b"3"], [b"4"]], [[b"5", b"6"]]],
                       [[[b"7", b"8", b"9"]]]], [[[[b""]]]]]},
  
        {"testcase_name": "Ragged4DInputEmptySeparator",
         "input": [[[[b"1 2 3", b"4"], [b"5 6"]], [[b"7 8 9"]]], [[[b""]]]],
         "input_is_ragged": True,
         "expected": [[[[[b"1", b"2", b"3"], [b"4"]], [[b"5", b"6"]]],
                       [[[b"7", b"8", b"9"]]]], [[[[]]]]]},
  
        ])  # pyformat: disable
    def testSplitV2(self,
                    input,
                    expected,
                    input_is_ragged=False,
                    **kwargs):  # pylint: disable=redefined-builtin
      # Check that we are matching the behavior of Python's str.split:
      self.assertEqual(expected, self._py_split(input, **kwargs))
  
      # Prepare the input tensor.
      if input_is_ragged:
        input = ragged_factory_ops.constant(input, dtype=dtypes.string)
      else:
        input = constant_op.constant(input, dtype=dtypes.string)
  
      # Check that the public version (which returns a RaggedTensor) works
      # correctly.
      expected_ragged = ragged_factory_ops.constant(
          expected, ragged_rank=input.shape.ndims)
      actual_ragged_v1 = ragged_string_ops.strings_split_v1(
          input, result_type="RaggedTensor", **kwargs)
      actual_ragged_v1_input_kwarg = ragged_string_ops.strings_split_v1(
          input=input, result_type="RaggedTensor", **kwargs)
      actual_ragged_v1_source_kwarg = ragged_string_ops.strings_split_v1(
          source=input, result_type="RaggedTensor", **kwargs)
      actual_ragged_v2 = ragged_string_ops.string_split_v2(input, **kwargs)
      actual_ragged_v2_input_kwarg = ragged_string_ops.string_split_v2(
          input=input, **kwargs)
      self.assertAllEqual(expected_ragged, actual_ragged_v1)
      self.assertAllEqual(expected_ragged, actual_ragged_v1_input_kwarg)
      self.assertAllEqual(expected_ragged, actual_ragged_v1_source_kwarg)
      self.assertAllEqual(expected_ragged, actual_ragged_v2)
      self.assertAllEqual(expected_ragged, actual_ragged_v2_input_kwarg)
  
      # Check that the internal version (which returns a SparseTensor) works
      # correctly.  Note: the internal version oly supports vector inputs.
      if input.shape.ndims == 1:
        expected_sparse = self.evaluate(expected_ragged.to_sparse())
        actual_sparse_v1 = ragged_string_ops.strings_split_v1(
            input, result_type="SparseTensor", **kwargs)
        actual_sparse_v2 = string_ops.string_split_v2(input, **kwargs)
        for actual_sparse in [actual_sparse_v1, actual_sparse_v2]:
          self.assertEqual(expected_sparse.indices.tolist(),
                           self.evaluate(actual_sparse.indices).tolist())
          self.assertEqual(expected_sparse.values.tolist(),
                           self.evaluate(actual_sparse.values).tolist())
          self.assertEqual(expected_sparse.dense_shape.tolist(),
                           self.evaluate(actual_sparse.dense_shape).tolist())
  
    def _py_split(self, strings, **kwargs):
      if isinstance(strings, compat.bytes_or_text_types):
        # Note: str.split doesn't accept keyword args.
        if "maxsplit" in kwargs:
          return strings.split(kwargs.get("sep", None), kwargs["maxsplit"])
        else:
          return strings.split(kwargs.get("sep", None))
      else:
        return [self._py_split(s, **kwargs) for s in strings]
  
  
  if __name__ == "__main__":
    test.main()
  