Repository: dbcli/litecli

Issue:
TabularOutputFormatter chokes on values that can't be converted to UTF-8 if it's a text column
I don't know how to solve this.  If there is a single record with a non-convertible column it won't print anything.

`Could not decode to UTF-8 column 'verifier' with text '`'���U'`
sqlite3 prints
```
2bho15FMrSQhKAYnjBqRQ1x4LS3zcHANsRjKMJyiSwT9|GnyZktv2SaCehfNCGjoYdAgAirxpCjvBCUXH6MiEHEH7                                                                                        
`'ŜU|`'ŜU 
```

Seeing this was actually helpful because it notified me that I had garbage data, but I still would've thought the other rows would print.

Discussion:

Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index b8c8122..0ccb53a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,7 +1,5 @@
 ## Unreleased - TBD
 
-<!-- add upcoming change notes here -->
-
 ### Features
 
 - Add verbose feature to `favorite_query` command. (Thanks: [Zhaolong Zhu])
@@ -12,6 +10,7 @@
 ### Bug Fixes
 
 - Fix compatibility with sqlparse >= 0.4.0. (Thanks: [chocolateboy])
+- Fix invalid utf-8 exception. (Thanks: [Amjith])
 
 ## 1.4.1 - 2020-07-27
 
diff --git a/litecli/sqlexecute.py b/litecli/sqlexecute.py
index 7ef103c..93acd91 100644
--- a/litecli/sqlexecute.py
+++ b/litecli/sqlexecute.py
@@ -17,6 +17,13 @@ _logger = logging.getLogger(__name__)
 # })
 
 
+def utf8_resilient_decoder(s):
+    try:
+        return s.decode("utf-8")
+    except UnicodeDecodeError:
+        return s.decode("latin-1")
+
+
 class SQLExecute(object):
 
     databases_query = """
@@ -61,6 +68,7 @@ class SQLExecute(object):
             raise Exception("Path does not exist: {}".format(db_dir_name))
 
         conn = sqlite3.connect(database=db_name, isolation_level=None)
+        conn.text_factory = utf8_resilient_decoder
         if self.conn:
             self.conn.close()
 


Test patch:
diff --git a/tests/test_sqlexecute.py b/tests/test_sqlexecute.py
index 03e7f19..0ddfb8f 100644
--- a/tests/test_sqlexecute.py
+++ b/tests/test_sqlexecute.py
@@ -101,6 +101,17 @@ def test_unicode_support_in_output(executor):
     assert_result_equal(results, headers=["t"], rows=[(u"é",)])
 
 
+@dbtest
+def test_invalid_unicode_values_dont_choke(executor):
+    run(executor, "create table unicodechars(t text)")
+    # \xc3 is not a valid utf-8 char. But we can insert it into the database
+    # which can break querying if not handled correctly.
+    run(executor, u"insert into unicodechars (t) values (cast(x'c3' as text))")
+
+    results = run(executor, u"select * from unicodechars")
+    assert_result_equal(results, headers=["t"], rows=[(u"Ã",)])
+
+
 @dbtest
 def test_multiple_queries_same_line(executor):
     results = run(executor, "select 'foo'; select 'bar'")
@@ -199,11 +210,7 @@ def test_verbose_feature_of_favorite_query(executor):
 
     results = run(executor, "\\f sh_param 1")
     assert_result_equal(
-        results,
-        title=None,
-        headers=["a", "id"],
-        rows=[("abc", 1)],
-        auto_status=False,
+        results, title=None, headers=["a", "id"], rows=[("abc", 1)], auto_status=False,
     )
 
     results = run(executor, "\\f+ sh_param 1")