Repository: jpsim/Yams
Pull request #145 (2018-12-30 01:05:26)

Enhance encoding support
- [x] Switch `Parser`'s preferred encoding on runtime
- [ ] ~~Switch `Emitter`'s preferred encoding on runtime~~
- [x] Optimize performance on Swift 5.0


Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0138eec..74ff658 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,11 +2,17 @@
 
 ##### Breaking
 
-* None.
+* Change `byteOffset` to `offset` in `YamlError.reader`.  
+  [Norio Nomura](https://github.com/norio-nomura)
 
 ##### Enhancements
 
-* None.
+* Add `encoding` option to `Parser` as `Parser.Encoding` type to specify
+  which encoding to pass to libYAML. Along with that change, add `encoding`
+  options to `load()`, `load_all()`, `compose()`, `compose_all()` and 
+  `YAMLDecoder`. The default encoding will be determined at run time based on
+  the String type's native encoding.  
+  [Norio Nomura](https://github.com/norio-nomura)
 
 ##### Bug Fixes
 
diff --git a/Sources/Yams/Decoder.swift b/Sources/Yams/Decoder.swift
index 19fba00..3c4d318 100644
--- a/Sources/Yams/Decoder.swift
+++ b/Sources/Yams/Decoder.swift
@@ -12,7 +12,11 @@ import Foundation
 /// user info mapping. Similar to `Foundation.JSONDecoder`.
 public class YAMLDecoder {
     /// Creates a `YAMLDecoder` instance.
-    public init() {}
+    ///
+    /// - parameter encoding: Encoding, `.default` if omitted.
+    public init(encoding: Parser.Encoding = .default) {
+        self.encoding = encoding
+    }
 
     /// Decode a `Decodable` type from a given `String` and optional user info mapping.
     ///
@@ -27,7 +31,7 @@ public class YAMLDecoder {
                           from yaml: String,
                           userInfo: [CodingUserInfoKey: Any] = [:]) throws -> T where T: Swift.Decodable {
         do {
-            let node = try Yams.compose(yaml: yaml, .basic) ?? ""
+            let node = try Parser(yaml: yaml, resolver: .basic, encoding: encoding).singleRoot() ?? ""
             let decoder = _Decoder(referencing: node, userInfo: userInfo)
             let container = try decoder.singleValueContainer()
             return try container.decode(type)
@@ -39,6 +43,9 @@ public class YAMLDecoder {
                                                     underlyingError: error))
         }
     }
+
+    /// Encoding
+    public var encoding: Parser.Encoding
 }
 
 private struct _Decoder: Decoder {
diff --git a/Sources/Yams/Emitter.swift b/Sources/Yams/Emitter.swift
index 2dbd3b9..736b900 100644
--- a/Sources/Yams/Emitter.swift
+++ b/Sources/Yams/Emitter.swift
@@ -143,11 +143,7 @@ public func serialize<Nodes>(
     try emitter.open()
     try nodes.forEach(emitter.serialize)
     try emitter.close()
-#if USE_UTF8
     return String(data: emitter.data, encoding: .utf8)!
-#else
-    return String(data: emitter.data, encoding: .utf16)!
-#endif
 }
 
 /// Produce a YAML string from a `Node`.
@@ -278,11 +274,7 @@ public final class Emitter {
 
         applyOptionsToEmitter()
 
-#if USE_UTF8
         yaml_emitter_set_encoding(&emitter, YAML_UTF8_ENCODING)
-#else
-        yaml_emitter_set_encoding(&emitter, isLittleEndian ? YAML_UTF16LE_ENCODING : YAML_UTF16BE_ENCODING)
-#endif
     }
 
     deinit {
@@ -296,12 +288,7 @@ public final class Emitter {
         switch state {
         case .initialized:
             var event = yaml_event_t()
-#if USE_UTF8
             yaml_stream_start_event_initialize(&event, YAML_UTF8_ENCODING)
-#else
-            let encoding = isLittleEndian ? YAML_UTF16LE_ENCODING : YAML_UTF16BE_ENCODING
-            yaml_stream_start_event_initialize(&event, encoding)
-#endif
             try emit(&event)
             state = .opened
         case .opened:
diff --git a/Sources/Yams/Parser.swift b/Sources/Yams/Parser.swift
index de6e008..72ac0c1 100644
--- a/Sources/Yams/Parser.swift
+++ b/Sources/Yams/Parser.swift
@@ -17,14 +17,16 @@ import Foundation
 /// - parameter yaml: String
 /// - parameter resolver: Resolver
 /// - parameter constructor: Constructor
+/// - parameter encoding: Parser.Encoding
 ///
 /// - returns: YamlSequence<Any>
 ///
 /// - throws: YamlError
 public func load_all(yaml: String,
                      _ resolver: Resolver = .default,
-                     _ constructor: Constructor = .default) throws -> YamlSequence<Any> {
-    let parser = try Parser(yaml: yaml, resolver: resolver, constructor: constructor)
+                     _ constructor: Constructor = .default,
+                     _ encoding: Parser.Encoding = .default) throws -> YamlSequence<Any> {
+    let parser = try Parser(yaml: yaml, resolver: resolver, constructor: constructor, encoding: encoding)
     return YamlSequence { try parser.nextRoot()?.any }
 }
 
@@ -34,14 +36,16 @@ public func load_all(yaml: String,
 /// - parameter yaml: String
 /// - parameter resolver: Resolver
 /// - parameter constructor: Constructor
+/// - parameter encoding: Parser.Encoding
 ///
 /// - returns: Any?
 ///
 /// - throws: YamlError
 public func load(yaml: String,
                  _ resolver: Resolver = .default,
-                 _ constructor: Constructor = .default) throws -> Any? {
-    return try Parser(yaml: yaml, resolver: resolver, constructor: constructor).singleRoot()?.any
+                 _ constructor: Constructor = .default,
+                 _ encoding: Parser.Encoding = .default) throws -> Any? {
+    return try Parser(yaml: yaml, resolver: resolver, constructor: constructor, encoding: encoding).singleRoot()?.any
 }
 
 /// Parse all YAML documents in a String
@@ -50,14 +54,16 @@ public func load(yaml: String,
 /// - parameter yaml: String
 /// - parameter resolver: Resolver
 /// - parameter constructor: Constructor
+/// - parameter encoding: Parser.Encoding
 ///
 /// - returns: YamlSequence<Node>
 ///
 /// - throws: YamlError
 public func compose_all(yaml: String,
                         _ resolver: Resolver = .default,
-                        _ constructor: Constructor = .default) throws -> YamlSequence<Node> {
-    let parser = try Parser(yaml: yaml, resolver: resolver, constructor: constructor)
+                        _ constructor: Constructor = .default,
+                        _ encoding: Parser.Encoding = .default) throws -> YamlSequence<Node> {
+    let parser = try Parser(yaml: yaml, resolver: resolver, constructor: constructor, encoding: encoding)
     return YamlSequence(parser.nextRoot)
 }
 
@@ -67,14 +73,16 @@ public func compose_all(yaml: String,
 /// - parameter yaml: String
 /// - parameter resolver: Resolver
 /// - parameter constructor: Constructor
+/// - parameter encoding: Parser.Encoding
 ///
 /// - returns: Node?
 ///
 /// - throws: YamlError
 public func compose(yaml: String,
                     _ resolver: Resolver = .default,
-                    _ constructor: Constructor = .default) throws -> Node? {
-    return try Parser(yaml: yaml, resolver: resolver, constructor: constructor).singleRoot()
+                    _ constructor: Constructor = .default,
+                    _ encoding: Parser.Encoding = .default) throws -> Node? {
+    return try Parser(yaml: yaml, resolver: resolver, constructor: constructor, encoding: encoding).singleRoot()
 }
 
 /// Sequence that holds an error.
@@ -108,33 +116,70 @@ public final class Parser {
     /// Constructor.
     public let constructor: Constructor
 
+    /// Encoding
+    public enum Encoding: String {
+        /// Use `YAML_UTF8_ENCODING`
+        case utf8
+        /// Use `YAML_UTF16(BE|LE)_ENCODING`
+        case utf16
+        /// The default encoding, determined at run time based on the String type's native encoding.
+        /// This can be overridden by setting `YAMS_DEFAULT_ENCODING` to either `UTF8` or `UTF16`.
+        /// This value is case insensitive.
+        public static var `default`: Encoding = {
+            let key = "YAMS_DEFAULT_ENCODING"
+            if let yamsEncoding = ProcessInfo.processInfo.environment[key],
+                let encoding = Encoding(rawValue: yamsEncoding.lowercased()) {
+                print("""
+                    `Parser.Encoding.default` was set to `\(encoding)` by the `\(key)` environment variable.
+                    """)
+                return encoding
+            }
+            return key.utf8.withContiguousStorageIfAvailable({ _ in true }) != nil ? .utf8 : .utf16
+        }()
+    }
+    /// Encoding
+    public let encoding: Encoding
+
     /// Set up Parser.
     ///
     /// - parameter string: YAML string.
     /// - parameter resolver: Resolver, `.default` if omitted.
     /// - parameter constructor: Constructor, `.default` if omitted.
+    /// - parameter encoding: Encoding, `.default` if omitted.
     ///
     /// - throws: `YamlError`.
     public init(yaml string: String,
                 resolver: Resolver = .default,
-                constructor: Constructor = .default) throws {
+                constructor: Constructor = .default,
+                encoding: Encoding = .default) throws {
         yaml = string
         self.resolver = resolver
         self.constructor = constructor
+        self.encoding = encoding
 
         yaml_parser_initialize(&parser)
-#if USE_UTF8
-        yaml_parser_set_encoding(&parser, YAML_UTF8_ENCODING)
-        utf8Slice = string.utf8CString.dropLast()
-        try utf8Slice.withUnsafeBytes(startParse(with:))
-#else
-        // use native endian
-        let isLittleEndian = 1 == 1.littleEndian
-        yaml_parser_set_encoding(&parser, isLittleEndian ? YAML_UTF16LE_ENCODING : YAML_UTF16BE_ENCODING)
-        let encoding: String.Encoding = isLittleEndian ? .utf16LittleEndian : .utf16BigEndian
-        data = yaml.data(using: encoding)!
-        try data.withUnsafeBytes(startParse(with:))
-#endif
+        switch encoding {
+        case .utf8:
+            yaml_parser_set_encoding(&parser, YAML_UTF8_ENCODING)
+            let utf8View = yaml.utf8
+            buffer = .utf8View(utf8View)
+            if try utf8View.withContiguousStorageIfAvailable(startParse(with:)) != nil {
+                // Attempt to parse with underlying UTF8 String encoding was successful, nothing further to do
+            } else {
+                // Fall back to using UTF8 slice
+                let utf8Slice = string.utf8CString.dropLast()
+                buffer = .utf8Slice(utf8Slice)
+                try utf8Slice.withUnsafeBytes(startParse(with:))
+            }
+        case .utf16:
+            // use native endianness
+            let isLittleEndian = 1 == 1.littleEndian
+            yaml_parser_set_encoding(&parser, isLittleEndian ? YAML_UTF16LE_ENCODING : YAML_UTF16BE_ENCODING)
+            let encoding: String.Encoding = isLittleEndian ? .utf16LittleEndian : .utf16BigEndian
+            let data = yaml.data(using: encoding)!
+            buffer = .utf16(data)
+            try data.withUnsafeBytes(startParse(with:))
+        }
     }
 
     deinit {
@@ -175,16 +220,18 @@ public final class Parser {
 
     private var anchors = [String: Node]()
     private var parser = yaml_parser_t()
-#if USE_UTF8
-    private let utf8Slice: ArraySlice<CChar>
-#else
-    private let data: Data
-#endif
+
+    private enum Buffer {
+        case utf8View(String.UTF8View)
+        case utf8Slice(ArraySlice<CChar>)
+        case utf16(Data)
+    }
+    private var buffer: Buffer
 }
 
 // MARK: Implementation Details
 private extension Parser {
-    private var streamEndProduced: Bool {
+    var streamEndProduced: Bool {
         return parser.stream_end_produced != 0
     }
 
@@ -194,7 +241,7 @@ private extension Parser {
         return node
     }
 
-    private func loadNode(from event: Event) throws -> Node {
+    func loadNode(from event: Event) throws -> Node {
         switch event.type {
         case YAML_ALIAS_EVENT:
             return try loadAlias(from: event)
@@ -214,6 +261,11 @@ private extension Parser {
         try parse() // Drop YAML_STREAM_START_EVENT
     }
 
+    func startParse(with buffer: UnsafeBufferPointer<UInt8>) throws {
+        yaml_parser_set_input_string(&parser, buffer.baseAddress, buffer.count)
+        try parse() // Drop YAML_STREAM_START_EVENT
+    }
+
     @discardableResult
     func parse() throws -> Event {
         let event = Event()
@@ -223,7 +275,7 @@ private extension Parser {
         return event
     }
 
-    private func loadAlias(from event: Event) throws -> Node {
+    func loadAlias(from event: Event) throws -> Node {
         guard let alias = event.aliasAnchor else {
             fatalError("unreachable")
         }
@@ -235,7 +287,7 @@ private extension Parser {
         return node
     }
 
-    private func loadScalar(from event: Event) throws -> Node {
+    func loadScalar(from event: Event) throws -> Node {
         let node = Node.scalar(.init(event.scalarValue, tag(event.scalarTag), event.scalarStyle, event.startMark))
         if let anchor = event.scalarAnchor {
             anchors[anchor] = node
@@ -243,7 +295,7 @@ private extension Parser {
         return node
     }
 
-    private func loadSequence(from firstEvent: Event) throws -> Node {
+    func loadSequence(from firstEvent: Event) throws -> Node {
         var array = [Node]()
         var event = try parse()
         while event.type != YAML_SEQUENCE_END_EVENT {
@@ -257,7 +309,7 @@ private extension Parser {
         return node
     }
 
-    private func loadMapping(from firstEvent: Event) throws -> Node {
+    func loadMapping(from firstEvent: Event) throws -> Node {
         var pairs = [(Node, Node)]()
         var event = try parse()
         while event.type != YAML_MAPPING_END_EVENT {
@@ -274,7 +326,7 @@ private extension Parser {
         return node
     }
 
-    private func tag(_ string: String?) -> Tag {
+    func tag(_ string: String?) -> Tag {
         let tagName = string.map(Tag.Name.init(rawValue:)) ?? .implicit
         return Tag(tagName, resolver, constructor)
     }
@@ -371,3 +423,19 @@ private extension Data {
     }
 }
 #endif
+
+#if swift(>=4.2)
+#if !compiler(>=5)
+private extension String.UTF8View {
+    func withContiguousStorageIfAvailable<R>(_ body: (UnsafeBufferPointer<Element>) throws -> R) rethrows -> R? {
+        return nil
+    }
+}
+#endif
+#else
+private extension String.UTF8View {
+    func withContiguousStorageIfAvailable<R>(_ body: (UnsafeBufferPointer<Element>) throws -> R) rethrows -> R? {
+        return nil
+    }
+}
+#endif
diff --git a/Sources/Yams/String+Yams.swift b/Sources/Yams/String+Yams.swift
index 8043614..0f44b2d 100644
--- a/Sources/Yams/String+Yams.swift
+++ b/Sources/Yams/String+Yams.swift
@@ -11,32 +11,15 @@ import Foundation
 extension String {
     typealias LineNumberColumnAndContents = (lineNumber: Int, column: Int, contents: String)
 
-    /// line number, column and contents at utf8 offset.
+    /// line number, column and contents at offset.
     ///
     /// - parameter offset: Int
     ///
     /// - returns: lineNumber: line number start from 0,
     ///            column: utf16 column start from 0,
     ///            contents: substring of line
-    func utf8LineNumberColumnAndContents(at offset: Int) -> LineNumberColumnAndContents? {
-        guard let index = utf8
-            .index(utf8.startIndex, offsetBy: offset, limitedBy: utf8.endIndex)?
-            .samePosition(in: self) else { return nil }
-        return lineNumberColumnAndContents(at: index)
-    }
-
-    /// line number, column and contents at utf16 offset.
-    ///
-    /// - parameter offset: Int
-    ///
-    /// - returns: lineNumber: line number start from 0,
-    ///            column: utf16 column start from 0,
-    ///            contents: substring of line
-    func utf16LineNumberColumnAndContents(at offset: Int) -> LineNumberColumnAndContents? {
-        guard let index = utf16
-            .index(utf16.startIndex, offsetBy: offset, limitedBy: utf16.endIndex)?
-            .samePosition(in: self) else { return nil }
-        return lineNumberColumnAndContents(at: index)
+    func lineNumberColumnAndContents(at offset: Int) -> LineNumberColumnAndContents? {
+        return index(startIndex, offsetBy: offset, limitedBy: endIndex).flatMap(lineNumberColumnAndContents)
     }
 
     /// line number, column and contents at Index.
diff --git a/Sources/Yams/YamlError.swift b/Sources/Yams/YamlError.swift
index 4a3a059..65b474d 100644
--- a/Sources/Yams/YamlError.swift
+++ b/Sources/Yams/YamlError.swift
@@ -23,11 +23,11 @@ public enum YamlError: Swift.Error {
     // Used in `yaml_parser_t`
     /// `YAML_READER_ERROR`. Cannot read or decode the input stream.
     ///
-    /// - parameter problem:    Error description.
-    /// - parameter byteOffset: The byte about which the problem occured.
-    /// - parameter value:      The problematic value (-1 is none).
-    /// - parameter yaml:       YAML String which the problem occured while reading.
-    case reader(problem: String, byteOffset: Int, value: Int32, yaml: String)
+    /// - parameter problem: Error description.
+    /// - parameter offset:  The offset from `yaml.startIndex` at which the problem occured.
+    /// - parameter value:   The problematic value (-1 is none).
+    /// - parameter yaml:    YAML String which the problem occured while reading.
+    case reader(problem: String, offset: Int?, value: Int32, yaml: String)
 
     // line and column start from 1, column is counted by unicodeScalars
     /// `YAML_SCANNER_ERROR`. Cannot scan the input stream.
@@ -81,6 +81,13 @@ public enum YamlError: Swift.Error {
             return text + " in line \(mark.line), column \(mark.column)\n"
         }
     }
+
+#if swift(>=4.1.50)
+    @available(*, unavailable, renamed: "reader(problem:offset:value:yaml:)")
+    public static func reader(problem: String, byteOffset: Int, value: Int32, yaml: String) {
+        fatalError("unavailable")
+    }
+#endif
 }
 
 extension YamlError {
@@ -101,8 +108,19 @@ extension YamlError {
         case YAML_MEMORY_ERROR:
             self = .memory
         case YAML_READER_ERROR:
+            let index: String.Index?
+            if parser.encoding == YAML_UTF8_ENCODING {
+                index = yaml.utf8
+                    .index(yaml.utf8.startIndex, offsetBy: parser.problem_offset, limitedBy: yaml.utf8.endIndex)?
+                    .samePosition(in: yaml)
+            } else {
+                index = yaml.utf16
+                    .index(yaml.utf16.startIndex, offsetBy: parser.problem_offset / 2, limitedBy: yaml.utf16.endIndex)?
+                    .samePosition(in: yaml)
+            }
+            let offset = index.map { yaml.distance(from: yaml.startIndex, to: $0) }
             self = .reader(problem: String(cString: parser.problem),
-                           byteOffset: parser.problem_offset,
+                           offset: offset,
                            value: parser.problem_value,
                            yaml: yaml)
         case YAML_SCANNER_ERROR:
@@ -142,11 +160,13 @@ extension YamlError: CustomStringConvertible {
             return "No error is produced"
         case .memory:
             return "Memory error"
-        case let .reader(problem, byteOffset, value, yaml):
-            guard let (mark, contents) = markAndSnippet(from: yaml, byteOffset)
-                else { return "\(problem) at byte offset: \(byteOffset), value: \(value)" }
+        case let .reader(problem, offset, value, yaml):
+            guard let (line, column, contents) = offset.flatMap(yaml.lineNumberColumnAndContents(at:)) else {
+                return "\(problem) at offset: \(String(describing: offset)), value: \(value)"
+            }
+            let mark = Mark(line: line + 1, column: column + 1)
             return "\(mark): error: reader: \(problem):\n" + contents.endingWithNewLine
-                + String(repeating: " ", count: mark.column - 1) + "^"
+                + String(repeating: " ", count: column) + "^"
         case let .scanner(context, problem, mark, yaml):
             return "\(mark): error: scanner: \(context?.description ?? "")\(problem):\n" + mark.snippet(from: yaml)
         case let .parser(context, problem, mark, yaml):
@@ -158,16 +178,3 @@ extension YamlError: CustomStringConvertible {
         }
     }
 }
-
-extension YamlError {
-    private func markAndSnippet(from yaml: String, _ byteOffset: Int) -> (Mark, String)? {
-#if USE_UTF8
-        guard let (line, column, contents) = yaml.utf8LineNumberColumnAndContents(at: byteOffset)
-            else { return nil }
-#else
-        guard let (line, column, contents) = yaml.utf16LineNumberColumnAndContents(at: byteOffset / 2)
-            else { return nil }
-#endif
-        return (Mark(line: line + 1, column: column + 1), contents)
-    }
-}
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 809931e..4c5786b 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -50,9 +50,13 @@ jobs:
   container: $[ variables['containerResource'] ]
   steps:
     - script: swift test --parallel
-      displayName: swift test
-    - script: swift test --parallel -Xswiftc -DUSE_UTF8
+      displayName: swift test with UTF16
+      env:
+        YAMS_DEFAULT_ENCODING: UTF16
+    - script: swift test --parallel
       displayName: swift test with UTF8
+      env:
+        YAMS_DEFAULT_ENCODING: UTF8
     - script: |
         set -o pipefail
         mkdir -p build/reports/
@@ -69,9 +73,13 @@ jobs:
   container: '5.0'
   steps:
     - script: swift test --parallel
-      displayName: swift test
-    - script: swift test --parallel -Xswiftc -DUSE_UTF8
+      displayName: swift test with UTF16
+      env:
+        YAMS_DEFAULT_ENCODING: UTF16
+    - script: swift test --parallel
       displayName: swift test with UTF8
+      env:
+        YAMS_DEFAULT_ENCODING: UTF8
 
 - job: macOS
   pool:
@@ -104,14 +112,18 @@ jobs:
         set -o pipefail &&
         xcodebuild $XCODE_FLAGS test |
         xcpretty -r junit -o build/reports/xcodebuild-macos.xml
-      displayName: tests on macOS
+      displayName: tests on macOS with UTF16
       condition: succeededOrFailed()
+      env:
+        YAMS_DEFAULT_ENCODING: UTF16
     - script: >
         set -o pipefail &&
-        xcodebuild $XCODE_FLAGS test OTHER_SWIFT_FLAGS=-DUSE_UTF8 |
+        xcodebuild $XCODE_FLAGS test |
         xcpretty -r junit -o build/reports/xcodebuild-macos-utf8.xml
       displayName: tests on macOS with UTF8
       condition: succeededOrFailed()
+      env:
+        YAMS_DEFAULT_ENCODING: UTF8
     - script: >
         set -o pipefail &&
         xcodebuild $XCODE_FLAGS test -sdk iphonesimulator -destination "name=iPhone 8" |
@@ -132,8 +144,17 @@ jobs:
     - script: >
         set -o pipefail &&
         swift test --parallel
-      displayName: swift test
+      displayName: swift test with UTF16
+      condition: succeededOrFailed()
+      env:
+        YAMS_DEFAULT_ENCODING: UTF16
+    - script: >
+        set -o pipefail &&
+        swift test --parallel
+      displayName: swift test with UTF8
       condition: succeededOrFailed()
+      env:
+        YAMS_DEFAULT_ENCODING: UTF8
     - task: PublishTestResults@2
       inputs:
         testRunner: JUnit


Test patch:
diff --git a/Tests/YamsTests/PerformanceTests.swift b/Tests/YamsTests/PerformanceTests.swift
index 180d8f0..3408cd7 100644
--- a/Tests/YamsTests/PerformanceTests.swift
+++ b/Tests/YamsTests/PerformanceTests.swift
@@ -57,80 +57,109 @@ class PerformanceTests: XCTestCase {
         "/SourceKitten/Source/SourceKittenFramework/Xcode.swift"
     ]
 
-    func testSourceKittenIssue289UsingLoad() {
-        guard let yamlString = try? String(contentsOfFile: filename, encoding: .utf8) else {
-            XCTFail("Can't load \(filename)")
-            return
-        }
+    func loadYAML() throws -> String {
+        let data = try Data(contentsOf: URL(fileURLWithPath: filename))
+        return String(data: data, encoding: .utf8)!
+    }
+
+    func parseSourceKittenIssue289UsingLoad(yaml: String, encoding: Parser.Encoding) {
         let spmName = "SourceKittenFramework"
-        self.measure {
-            do {
-                guard let yaml = try Yams.load(yaml: yamlString) as? [String: Any],
-                    let commands = (yaml["commands"] as? [String: [String: Any]])?.values,
-                    let moduleCommand = commands.first(where: { ($0["module-name"] as? String ?? "") == spmName }),
-                    let imports = moduleCommand["import-paths"] as? [String],
-                    let otherArguments = moduleCommand["other-args"] as? [String],
-                    let sources = moduleCommand["sources"] as? [String] else {
+        do {
+            guard let object = try Yams.load(yaml: yaml, .default, .default, encoding) as? [String: Any],
+                let commands = (object["commands"] as? [String: [String: Any]])?.values,
+                let moduleCommand = commands.first(where: { ($0["module-name"] as? String ?? "") == spmName }),
+                let imports = moduleCommand["import-paths"] as? [String],
+                let otherArguments = moduleCommand["other-args"] as? [String],
+                let sources = moduleCommand["sources"] as? [String] else {
                     XCTFail("Invalid result form Yams.load()")
                     return
-                }
-                XCTAssertEqual(imports, self.expectedImports)
-                XCTAssertEqual(otherArguments, self.expectedOtherArguments)
-                XCTAssertEqual(sources, self.expectedSources)
-            } catch {
-                XCTFail("\(error)")
             }
+            XCTAssertEqual(imports, self.expectedImports)
+            XCTAssertEqual(otherArguments, self.expectedOtherArguments)
+            XCTAssertEqual(sources, self.expectedSources)
+        } catch {
+            XCTFail("\(error)")
         }
     }
 
-    func testSourceKittenIssue289UsingCompose() {
-        guard let yamlString = try? String(contentsOfFile: filename, encoding: .utf8) else {
-            XCTFail("Can't load \(filename)")
-            return
+    func testUsingLoadWithUTF16() throws {
+        let yaml = try loadYAML()
+        self.measure {
+            parseSourceKittenIssue289UsingLoad(yaml: yaml, encoding: .utf16)
         }
-        let spmName = "SourceKittenFramework"
+    }
+
+    func testUsingLoadWithUTF8() throws {
+        let yaml = try loadYAML()
         self.measure {
-            do {
-                guard let yaml = try Yams.compose(yaml: yamlString),
-                    let commands = yaml["commands"]?.mapping?.values,
-                    let moduleCommand = commands.first(where: { $0["module-name"]?.string == spmName }),
-                    let imports = moduleCommand["import-paths"]?.array(of: String.self),
-                    let otherArguments = moduleCommand["other-args"]?.array(of: String.self),
-                    let sources = moduleCommand["sources"]?.array(of: String.self) else {
-                        XCTFail("Invalid result form Yams.load()")
-                        return
-                }
-                XCTAssertEqual(imports, self.expectedImports)
-                XCTAssertEqual(otherArguments, self.expectedOtherArguments)
-                XCTAssertEqual(sources, self.expectedSources)
-            } catch {
-                XCTFail("\(error)")
+            parseSourceKittenIssue289UsingLoad(yaml: yaml, encoding: .utf8)
+        }
+    }
+
+    func parseSourceKittenIssue289UsingCompose(yaml: String, encoding: Parser.Encoding) {
+        let spmName = "SourceKittenFramework"
+        do {
+            guard let node = try Yams.compose(yaml: yaml, .default, .default, encoding),
+                let commands = node["commands"]?.mapping?.values,
+                let moduleCommand = commands.first(where: { $0["module-name"]?.string == spmName }),
+                let imports = moduleCommand["import-paths"]?.array(of: String.self),
+                let otherArguments = moduleCommand["other-args"]?.array(of: String.self),
+                let sources = moduleCommand["sources"]?.array(of: String.self) else {
+                    XCTFail("Invalid result form Yams.load()")
+                    return
             }
+            XCTAssertEqual(imports, self.expectedImports)
+            XCTAssertEqual(otherArguments, self.expectedOtherArguments)
+            XCTAssertEqual(sources, self.expectedSources)
+        } catch {
+            XCTFail("\(error)")
         }
     }
 
-    func testSourceKittenIssue289UsingSwiftDecodable() {
-        guard let yamlString = try? String(contentsOfFile: filename, encoding: .utf8) else {
-            XCTFail("Can't load \(filename)")
-            return
+    func testUsingComposeWithUTF16() throws {
+        let yaml = try loadYAML()
+        self.measure {
+            parseSourceKittenIssue289UsingCompose(yaml: yaml, encoding: .utf16)
         }
-        let spmName = "SourceKittenFramework"
+    }
+
+    func testUsingComposeWithUTF8() throws {
+        let yaml = try loadYAML()
         self.measure {
-            do {
-                guard let manifest: Manifest = try YAMLDecoder().decode(from: yamlString),
-                    let command = manifest.commands.values.first(where: { $0.moduleName == spmName }),
-                    let imports = command.importPaths,
-                    let otherArguments = command.otherArguments,
-                    let sources = command.sources else {
-                        XCTFail("Invalid result form Yams.load()")
-                        return
-                }
-                XCTAssertEqual(imports, self.expectedImports)
-                XCTAssertEqual(otherArguments, self.expectedOtherArguments)
-                XCTAssertEqual(sources, self.expectedSources)
-            } catch {
-                XCTFail("\(error)")
+            parseSourceKittenIssue289UsingCompose(yaml: yaml, encoding: .utf8)
+        }
+    }
+
+    func parseSourceKittenIssue289UsingSwiftDecodable(yaml: String, encoding: Parser.Encoding) {
+        let spmName = "SourceKittenFramework"
+        do {
+            guard let manifest: Manifest = try YAMLDecoder(encoding: encoding).decode(from: yaml),
+                let command = manifest.commands.values.first(where: { $0.moduleName == spmName }),
+                let imports = command.importPaths,
+                let otherArguments = command.otherArguments,
+                let sources = command.sources else {
+                    XCTFail("Invalid result form Yams.load()")
+                    return
             }
+            XCTAssertEqual(imports, self.expectedImports)
+            XCTAssertEqual(otherArguments, self.expectedOtherArguments)
+            XCTAssertEqual(sources, self.expectedSources)
+        } catch {
+            XCTFail("\(error)")
+        }
+    }
+
+    func testUsingSwiftDecodableWithUTF16() throws {
+        let yaml = try loadYAML()
+        self.measure {
+            parseSourceKittenIssue289UsingSwiftDecodable(yaml: yaml, encoding: .utf16)
+        }
+    }
+
+    func testUsingSwiftDecodableWithUTF8() throws {
+        let yaml = try loadYAML()
+        self.measure {
+            parseSourceKittenIssue289UsingSwiftDecodable(yaml: yaml, encoding: .utf8)
         }
     }
 }
@@ -138,9 +167,12 @@ class PerformanceTests: XCTestCase {
 extension PerformanceTests {
     static var allTests: [(String, (PerformanceTests) -> () throws -> Void)] {
         return [
-            ("testSourceKittenIssue289Load", testSourceKittenIssue289UsingLoad),
-            ("testSourceKittenIssue289Compose", testSourceKittenIssue289UsingCompose),
-            ("testSourceKittenIssue289UsingSwiftDecodable", testSourceKittenIssue289UsingSwiftDecodable)
+            ("testUsingLoadWithUTF16", testUsingLoadWithUTF16),
+            ("testUsingLoadWithUTF8", testUsingLoadWithUTF8),
+            ("testUsingComposeWithUTF16", testUsingComposeWithUTF16),
+            ("testUsingComposeWithUTF8", testUsingComposeWithUTF8),
+            ("testUsingSwiftDecodableWithUTF16", testUsingSwiftDecodableWithUTF16),
+            ("testUsingSwiftDecodableWithUTF8", testUsingSwiftDecodableWithUTF8)
         ]
     }
 }
diff --git a/Tests/YamsTests/StringTests.swift b/Tests/YamsTests/StringTests.swift
index f1605f5..c62eafc 100644
--- a/Tests/YamsTests/StringTests.swift
+++ b/Tests/YamsTests/StringTests.swift
@@ -25,36 +25,8 @@ class StringTests: XCTestCase {
         XCTAssertEqual(firstLine, "LINE1_67あ\n")
     }
 
-    // `String.utf8LineNumberColumnAndContents(at:)`
-    func testUTF8LineNumberColumnAndContentsAtOffset() {
-        guard !doesSR9454Affect() else { return }
-
-        // offset     0 1 2 3 4 5 6 7 8 9 10 11
-        // line 1     L I N E 1 _ 6 7 あ     \n
-        // line 2     L I N E 2 _ 7 8 9 0 \n
-        // line 3     L I N E 3 _ 8 9 0 1 \n
-        do {
-            let (number, column, content) = string.utf8LineNumberColumnAndContents(at: 0)!
-            XCTAssertEqual(number, 0)
-            XCTAssertEqual(column, 0)
-            XCTAssertEqual(content, "LINE1_67あ\n")
-        }
-        do {
-            let (number, column, content) = string.utf8LineNumberColumnAndContents(at: 11)!
-            XCTAssertEqual(number, 0)
-            XCTAssertEqual(column, 9)
-            XCTAssertEqual(content, "LINE1_67あ\n")
-        }
-        do {
-            let (number, column, content) = string.utf8LineNumberColumnAndContents(at: 12)!
-            XCTAssertEqual(number, 1)
-            XCTAssertEqual(column, 0)
-            XCTAssertEqual(content, "LINE2_7890\n")
-        }
-    }
-
-    // `String.utf16LineNumberColumnAndContents(at:)`
-    func testUTF16LineNumberColumnAndContentsAtOffset() {
+    // `String.lineNumberColumnAndContents(at:)`
+    func testLineNumberColumnAndContentsAtOffset() {
         guard !doesSR9454Affect() else { return }
 
         // offset     0 1 2 3 4 5 6 7 8 9 10
@@ -62,19 +34,19 @@ class StringTests: XCTestCase {
         // line 2     L I N E 2 _ 7 8 9 0 \n
         // line 3     L I N E 3 _ 8 9 0 1 \n
         do {
-            let (number, column, content) = string.utf16LineNumberColumnAndContents(at: 0)!
+            let (number, column, content) = string.lineNumberColumnAndContents(at: 0)!
             XCTAssertEqual(number, 0)
             XCTAssertEqual(column, 0)
             XCTAssertEqual(content, "LINE1_67あ\n")
         }
         do {
-            let (number, column, content) = string.utf16LineNumberColumnAndContents(at: 9)!
+            let (number, column, content) = string.lineNumberColumnAndContents(at: 9)!
             XCTAssertEqual(number, 0)
             XCTAssertEqual(column, 9)
             XCTAssertEqual(content, "LINE1_67あ\n")
         }
         do {
-            let (number, column, content) = string.utf16LineNumberColumnAndContents(at: 10)!
+            let (number, column, content) = string.lineNumberColumnAndContents(at: 10)!
             XCTAssertEqual(number, 1)
             XCTAssertEqual(column, 0)
             XCTAssertEqual(content, "LINE2_7890\n")
@@ -94,8 +66,7 @@ extension StringTests {
     static var allTests: [(String, (StringTests) -> () throws -> Void)] {
         return [
             ("testConfirmBehaviorOfStandardLibraryAPI", testConfirmBehaviorOfStandardLibraryAPI),
-            ("testUTF8LineNumberColumnAndContentsAtOffset", testUTF8LineNumberColumnAndContentsAtOffset),
-            ("testUTF16LineNumberColumnAndContentsAtOffset", testUTF16LineNumberColumnAndContentsAtOffset),
+            ("testLineNumberColumnAndContentsAtOffset", testLineNumberColumnAndContentsAtOffset),
             ("testSubstringAtLine", testSubstringAtLine)
         ]
     }