Repository: RocketMap/RocketMap

Issue:
incorrect string value 1366 error using gym details with cyrillic names
so i am getting this every time i change my DB. (it mostly affects Easter Europe and other Cyrillic using locations) 

the solution is this: 

ALTER TABLE `gymdetails` COLLATE='utf8_general_ci', CONVERT TO CHARSET utf8;

if someone adds it to the guide it will save time for others.(ot devs change the gymdetails charset to utf8 by default :)





Discussion:

Resolving patch:
diff --git a/pogom/models.py b/pogom/models.py
--- a/pogom/models.py
+++ b/pogom/models.py
@@ -49,6 +49,13 @@ class MyRetryDB(RetryOperationalError, PooledMySQLDatabase):
     pass
 
 
+# Reduction of CharField to fit max length inside 767 bytes for utf8mb4 charset
+class Utf8mb4CharField(CharField):
+    def __init__(self, max_length=191, *args, **kwargs):
+        self.max_length = max_length
+        super(CharField, self).__init__(*args, **kwargs)
+
+
 def init_database(app):
     if args.db_type == 'mysql':
         log.info('Connecting to MySQL database on %s:%i...',
@@ -63,7 +70,8 @@ def init_database(app):
             host=args.db_host,
             port=args.db_port,
             max_connections=connections,
-            stale_timeout=300)
+            stale_timeout=300,
+            charset='utf8mb4')
     else:
         log.info('Connecting to local SQLite database')
         db = SqliteExtDatabase(args.db,
@@ -95,8 +103,8 @@ def get_all(cls):
 class Pokemon(BaseModel):
     # We are base64 encoding the ids delivered by the api
     # because they are too big for sqlite to handle.
-    encounter_id = CharField(primary_key=True, max_length=50)
-    spawnpoint_id = CharField(index=True)
+    encounter_id = Utf8mb4CharField(primary_key=True, max_length=50)
+    spawnpoint_id = Utf8mb4CharField(index=True)
     pokemon_id = SmallIntegerField(index=True)
     latitude = DoubleField()
     longitude = DoubleField()
@@ -435,13 +443,14 @@ def get_spawnpoints_in_hex(cls, center, steps):
 
 
 class Pokestop(BaseModel):
-    pokestop_id = CharField(primary_key=True, max_length=50)
+    pokestop_id = Utf8mb4CharField(primary_key=True, max_length=50)
     enabled = BooleanField()
     latitude = DoubleField()
     longitude = DoubleField()
     last_modified = DateTimeField(index=True)
     lure_expiration = DateTimeField(null=True, index=True)
-    active_fort_modifier = CharField(max_length=50, null=True, index=True)
+    active_fort_modifier = Utf8mb4CharField(max_length=50,
+                                            null=True, index=True)
     last_updated = DateTimeField(
         null=True, index=True, default=datetime.utcnow)
 
@@ -533,7 +542,7 @@ def get_stops(swLat, swLng, neLat, neLng, timestamp=0, oSwLat=None,
 
 class Gym(BaseModel):
 
-    gym_id = CharField(primary_key=True, max_length=50)
+    gym_id = Utf8mb4CharField(primary_key=True, max_length=50)
     team_id = SmallIntegerField()
     guard_pokemon_id = SmallIntegerField()
     gym_points = IntegerField()
@@ -701,7 +710,7 @@ def get_gym(id):
 
 
 class LocationAltitude(BaseModel):
-    cellid = CharField(primary_key=True, max_length=50)
+    cellid = Utf8mb4CharField(primary_key=True, max_length=50)
     latitude = DoubleField()
     longitude = DoubleField()
     last_modified = DateTimeField(index=True, default=datetime.utcnow,
@@ -746,7 +755,7 @@ def save_altitude(cls, loc, altitude):
 
 
 class ScannedLocation(BaseModel):
-    cellid = CharField(primary_key=True, max_length=50)
+    cellid = Utf8mb4CharField(primary_key=True, max_length=50)
     latitude = DoubleField()
     longitude = DoubleField()
     last_modified = DateTimeField(
@@ -1083,9 +1092,9 @@ def select_in_hex(cls, locs):
 
 
 class MainWorker(BaseModel):
-    worker_name = CharField(primary_key=True, max_length=50)
+    worker_name = Utf8mb4CharField(primary_key=True, max_length=50)
     message = TextField(null=True, default="")
-    method = CharField(max_length=50)
+    method = Utf8mb4CharField(max_length=50)
     last_modified = DateTimeField(index=True)
     accounts_working = IntegerField()
     accounts_captcha = IntegerField()
@@ -1108,15 +1117,15 @@ def get_account_stats():
 
 
 class WorkerStatus(BaseModel):
-    username = CharField(primary_key=True, max_length=50)
-    worker_name = CharField(index=True, max_length=50)
+    username = Utf8mb4CharField(primary_key=True, max_length=50)
+    worker_name = Utf8mb4CharField(index=True, max_length=50)
     success = IntegerField()
     fail = IntegerField()
     no_items = IntegerField()
     skip = IntegerField()
     captcha = IntegerField()
     last_modified = DateTimeField(index=True)
-    message = CharField(max_length=255)
+    message = Utf8mb4CharField(max_length=191)
     last_scan_date = DateTimeField(index=True)
     latitude = DoubleField(null=True)
     longitude = DoubleField(null=True)
@@ -1187,13 +1196,13 @@ def get_worker(username, loc=False):
 
 
 class SpawnPoint(BaseModel):
-    id = CharField(primary_key=True, max_length=50)
+    id = Utf8mb4CharField(primary_key=True, max_length=50)
     latitude = DoubleField()
     longitude = DoubleField()
     last_scanned = DateTimeField(index=True)
     # kind gives the four quartiles of the spawn, as 's' for seen
     # or 'h' for hidden.  For example, a 30 minute spawn is 'hhss'.
-    kind = CharField(max_length=4, default='hhhs')
+    kind = Utf8mb4CharField(max_length=4, default='hhhs')
 
     # links shows whether a Pokemon encounter id changes between quartiles or
     # stays the same.  Both 1x45 and 1x60h3 have the kind of 'sssh', but the
@@ -1205,7 +1214,7 @@ class SpawnPoint(BaseModel):
     # Note index is shifted by a half. links[0] is the link between
     # kind[0] and kind[1] and so on. links[3] is the link between
     # kind[3] and kind[0]
-    links = CharField(max_length=4, default='????')
+    links = Utf8mb4CharField(max_length=4, default='????')
 
     # Count consecutive times spawn should have been seen, but wasn't.
     # If too high, will not be scheduled for review, and treated as inactive.
@@ -1421,11 +1430,11 @@ class Meta:
 
 
 class SpawnpointDetectionData(BaseModel):
-    id = CharField(primary_key=True, max_length=54)
+    id = Utf8mb4CharField(primary_key=True, max_length=54)
     # Removed ForeignKeyField since it caused MySQL issues.
-    encounter_id = CharField(max_length=54)
+    encounter_id = Utf8mb4CharField(max_length=54)
     # Removed ForeignKeyField since it caused MySQL issues.
-    spawnpoint_id = CharField(max_length=54, index=True)
+    spawnpoint_id = Utf8mb4CharField(max_length=54, index=True)
     scan_time = DateTimeField()
     tth_secs = SmallIntegerField(null=True)
 
@@ -1620,7 +1629,7 @@ def unseen(cls, sp, now_secs):
 
 
 class Versions(flaskDb.Model):
-    key = CharField()
+    key = Utf8mb4CharField()
     val = SmallIntegerField()
 
     class Meta:
@@ -1628,8 +1637,8 @@ class Meta:
 
 
 class GymMember(BaseModel):
-    gym_id = CharField(index=True)
-    pokemon_uid = CharField(index=True)
+    gym_id = Utf8mb4CharField(index=True)
+    pokemon_uid = Utf8mb4CharField(index=True)
     last_scanned = DateTimeField(default=datetime.utcnow, index=True)
 
     class Meta:
@@ -1637,10 +1646,10 @@ class Meta:
 
 
 class GymPokemon(BaseModel):
-    pokemon_uid = CharField(primary_key=True, max_length=50)
+    pokemon_uid = Utf8mb4CharField(primary_key=True, max_length=50)
     pokemon_id = SmallIntegerField()
     cp = SmallIntegerField()
-    trainer_name = CharField(index=True)
+    trainer_name = Utf8mb4CharField(index=True)
     num_upgrades = SmallIntegerField(null=True)
     move_1 = SmallIntegerField(null=True)
     move_2 = SmallIntegerField(null=True)
@@ -1657,17 +1666,17 @@ class GymPokemon(BaseModel):
 
 
 class Trainer(BaseModel):
-    name = CharField(primary_key=True, max_length=50)
+    name = Utf8mb4CharField(primary_key=True, max_length=50)
     team = SmallIntegerField()
     level = SmallIntegerField()
     last_seen = DateTimeField(default=datetime.utcnow)
 
 
 class GymDetails(BaseModel):
-    gym_id = CharField(primary_key=True, max_length=50)
-    name = CharField()
+    gym_id = Utf8mb4CharField(primary_key=True, max_length=50)
+    name = Utf8mb4CharField()
     description = TextField(null=True, default="")
-    url = CharField()
+    url = Utf8mb4CharField()
     last_scanned = DateTimeField(default=datetime.utcnow)
 
 
@@ -1705,7 +1714,7 @@ def get_valid(limit=15):
 
 
 class HashKeys(BaseModel):
-    key = CharField(primary_key=True, max_length=20)
+    key = Utf8mb4CharField(primary_key=True, max_length=20)
     maximum = SmallIntegerField(default=0)
     remaining = SmallIntegerField(default=0)
     peak = SmallIntegerField(default=0)
@@ -2614,7 +2623,6 @@ def bulk_upsert(cls, data, db):
 
 def create_tables(db):
     db.connect()
-    verify_database_schema(db)
     tables = [Pokemon, Pokestop, Gym, ScannedLocation, GymDetails,
               GymMember, GymPokemon, Trainer, MainWorker, WorkerStatus,
               SpawnPoint, ScanSpawnPoint, SpawnpointDetectionData,
@@ -2625,7 +2633,6 @@ def create_tables(db):
             db.create_tables([table], safe=True)
         else:
             log.debug('Skipping table %s, it already exists.', table.__name__)
-
     db.close()
 
 
@@ -2646,7 +2653,40 @@ def drop_tables(db):
     db.close()
 
 
+def verify_table_encoding(db):
+    if args.db_type == 'mysql':
+        db.connect()
+
+        cmd_sql = '''
+            SELECT table_name FROM information_schema.tables WHERE
+            table_collation != "utf8mb4_unicode_ci" AND table_schema = "%s";
+            ''' % args.db_name
+        change_tables = db.execute_sql(cmd_sql)
+
+        cmd_sql = "SHOW tables;"
+        tables = db.execute_sql(cmd_sql)
+
+        if change_tables.rowcount > 0:
+            log.info('Changing collation and charset on %s tables.',
+                     change_tables.rowcount)
+
+            if change_tables.rowcount == tables.rowcount:
+                log.info('Changing whole database, this might a take while.')
+
+            with db.atomic():
+                db.execute_sql('SET FOREIGN_KEY_CHECKS=0;')
+                for table in change_tables:
+                    log.debug('Changing collation and charset on table %s.',
+                              table[0])
+                    cmd_sql = '''ALTER TABLE %s CONVERT TO CHARACTER SET utf8mb4
+                                COLLATE utf8mb4_unicode_ci;''' % str(table[0])
+                    db.execute_sql(cmd_sql)
+                db.execute_sql('SET FOREIGN_KEY_CHECKS=1;')
+        db.close()
+
+
 def verify_database_schema(db):
+    db.connect()
     if not Versions.table_exists():
         db.create_tables([Versions])
 
@@ -2673,6 +2713,7 @@ def verify_database_schema(db):
             log.error('Please upgrade your code base or drop all tables in '
                       'your database.')
             sys.exit(1)
+    db.close()
 
 
 def database_migrate(db, old_ver):
@@ -2692,12 +2733,13 @@ def database_migrate(db, old_ver):
 
     if old_ver < 2:
         migrate(migrator.add_column('pokestop', 'encounter_id',
-                                    CharField(max_length=50, null=True)))
+                                    Utf8mb4CharField(max_length=50,
+                                                     null=True)))
 
     if old_ver < 3:
         migrate(
             migrator.add_column('pokestop', 'active_fort_modifier',
-                                CharField(max_length=50, null=True)),
+                                Utf8mb4CharField(max_length=50, null=True)),
             migrator.drop_column('pokestop', 'encounter_id'),
             migrator.drop_column('pokestop', 'active_pokemon_id')
         )
diff --git a/runserver.py b/runserver.py
--- a/runserver.py
+++ b/runserver.py
@@ -23,7 +23,8 @@
 
 from pogom.search import search_overseer_thread
 from pogom.models import (init_database, create_tables, drop_tables,
-                          Pokemon, db_updater, clean_db_loop)
+                          Pokemon, db_updater, clean_db_loop,
+                          verify_table_encoding, verify_database_schema)
 from pogom.webhook import wh_updater
 
 from pogom.proxy import check_proxies, proxies_refresher
@@ -221,7 +222,14 @@ def main():
             drop_tables(db)
         elif os.path.isfile(args.db):
             os.remove(args.db)
+
+    verify_database_schema(db)
+
     create_tables(db)
+
+    # fixing encoding on present and future tables
+    verify_table_encoding(db)
+
     if args.clear_db:
         log.info("Drop and recreate is complete. Now remove -cd and restart.")
         sys.exit()


Test patch: