Repository: fonttools/fonttools

Issue:
pyftmerge error when merging some Not fonts
I’m trying to merge Noto Serif into Noto Naskh Arabic, but `pyftmerge NotoNaskhArabic-Regular.ttf NotoSerif-Regular.ttf` fails with:

```
Traceback (most recent call last):
  File "pyftmerge", line 9, in <module>
    load_entry_point('fonttools==2.5', 'console_scripts', 'pyftmerge')()
  File "/fonttools/Lib/fontTools/merge.py", line 940, in main
    font = merger.merge(args)
  File "/fonttools/Lib/fontTools/merge.py", line 770, in merge
    table = clazz(tag).merge(self, tables)
  File "/fonttools/Lib/fontTools/merge.py", line 374, in merge
    assert m.duplicateGlyphsPerFont[fontIdx].get(oldgid, gid) == gid
AssertionError
```

But if I reverse the order, i.e. `pyftmerge NotoSerif-Regular.ttf NotoNaskhArabic-Regular.ttf`, it works just fine.



Discussion:
I also found this when working with a myanmar/Latin pair, in
github.com/khmertype/Gantgaw
On Jul 21, 2015 9:55 PM, "Khaled Hosny" notifications@github.com wrote:

> I’m trying to merge Noto Serif into Noto Naskh Arabic, but pyftmerge
> NotoNaskhArabic-Regular.ttf NotoSerif-Regular.ttf fails with:
> 
> Traceback (most recent call last):
>   File "pyftmerge", line 9, in <module>
>     load_entry_point('fonttools==2.5', 'console_scripts', 'pyftmerge')()
>   File "/fonttools/Lib/fontTools/merge.py", line 940, in main
>     font = merger.merge(args)
>   File "/fonttools/Lib/fontTools/merge.py", line 770, in merge
>     table = clazz(tag).merge(self, tables)
>   File "/fonttools/Lib/fontTools/merge.py", line 374, in merge
>     assert m.duplicateGlyphsPerFont[fontIdx].get(oldgid, gid) == gid
> AssertionError
> 
> But if I reverse the order, i.e. pyftmerge NotoSerif-Regular.ttf
> NotoNaskhArabic-Regular.ttf, it works just fine.
> 
> —
> Reply to this email directly or view it on GitHub
> https://github.com/behdad/fonttools/issues/322.


Resolving patch:
diff --git a/Lib/fontTools/merge.py b/Lib/fontTools/merge.py
--- a/Lib/fontTools/merge.py
+++ b/Lib/fontTools/merge.py
@@ -359,20 +359,27 @@ def merge(self, m, tables):
 @_add_method(ttLib.getTableClass('cmap'))
 def merge(self, m, tables):
 	# TODO Handle format=14.
-	cmapTables = [(t,fontIdx) for fontIdx,table in enumerate(tables) for t in table.tables if t.isUnicode()]
-	# TODO Better handle format-4 and format-12 coexisting in same font.
-	# TODO Insert both a format-4 and format-12 if needed.
-	module = ttLib.getTableModule('cmap')
-	assert all(t.format in [4, 12] for t,_ in cmapTables)
-	format = max(t.format for t,_ in cmapTables)
-	cmapTable = module.cmap_classes[format](format)
-	cmapTable.cmap = {}
-	cmapTable.platformID = 3
-	cmapTable.platEncID = max(t.platEncID for t,_ in cmapTables)
-	cmapTable.language = 0
-	cmap = cmapTable.cmap
+	# Only merges 4/3/1 and 12/3/10 subtables, ignores all other subtables
+	# If there is a format 12 table for the same font, ignore the format 4 table
+	cmapTables = []
+	for fontIdx,table in enumerate(tables):
+		format4 = None
+		format12 = None
+		for subtable in table.tables:
+			properties = (subtable.format, subtable.platformID, subtable.platEncID)
+			if properties == (4,3,1):
+				format4 = subtable
+			elif properties == (12,3,10):
+				format12 = subtable
+		if format12 is not None:
+			cmapTables.append((format12, fontIdx))
+		elif format4 is not None:
+			cmapTables.append((format4, fontIdx))
+
+	# Build a unicode mapping, then decide which format is needed to store it.
+	cmap = {}
 	for table,fontIdx in cmapTables:
-		# TODO handle duplicates.
+		# handle duplicates
 		for uni,gid in table.cmap.items():
 			oldgid = cmap.get(uni, None)
 			if oldgid is None:
@@ -380,10 +387,34 @@ def merge(self, m, tables):
 			elif oldgid != gid:
 				# Char previously mapped to oldgid, now to gid.
 				# Record, to fix up in GSUB 'locl' later.
-				assert m.duplicateGlyphsPerFont[fontIdx].get(oldgid, gid) == gid
-				m.duplicateGlyphsPerFont[fontIdx][oldgid] = gid
+				if m.duplicateGlyphsPerFont[fontIdx].get(oldgid, gid) == gid:
+					m.duplicateGlyphsPerFont[fontIdx][oldgid] = gid
+				else:
+					# char previously mapped to oldgid but already remapped to a different gid,
+					# save new gid as an alternate
+					# TODO: try harder to save these
+					log.warn("Dropped mapping from codepoint %#06X to glyphId '%s'", uni, gid)
+
+	cmapBmpOnly = {uni: gid for uni,gid in cmap.items() if uni <= 0xFFFF}
+	self.tables = []
+	module = ttLib.getTableModule('cmap')
+	if len(cmapBmpOnly) != len(cmap):
+		# format-12 required.
+		cmapTable = module.cmap_classes[12](12)
+		cmapTable.platformID = 3
+		cmapTable.platEncID = 10
+		cmapTable.language = 0
+		cmapTable.cmap = cmap
+		self.tables.append(cmapTable)
+	# always create format-4
+	cmapTable = module.cmap_classes[4](4)
+	cmapTable.platformID = 3
+	cmapTable.platEncID = 1
+	cmapTable.language = 0
+	cmapTable.cmap = cmapBmpOnly
+	# ordered by platform then encoding
+	self.tables.insert(0, cmapTable)
 	self.tableVersion = 0
-	self.tables = [cmapTable]
 	self.numSubTables = len(self.tables)
 	return self
 
@@ -777,6 +808,7 @@ def merge(self, fontfiles):
 			with timer("merge '%s'" % tag):
 				tables = [font.get(tag, NotImplemented) for font in fonts]
 
+				log.info("Merging '%s'.", tag)
 				clazz = ttLib.getTableClass(tag)
 				table = clazz(tag).merge(self, tables)
 				# XXX Clean this up and use:  table = mergeObjects(tables)


Test patch:
diff --git a/Lib/fontTools/merge_test.py b/Lib/fontTools/merge_test.py
new file mode 100644
--- /dev/null
+++ b/Lib/fontTools/merge_test.py
@@ -0,0 +1,82 @@
+from fontTools.misc.py23 import *
+from fontTools import ttLib
+from fontTools.merge import *
+import unittest
+
+
+class MergeIntegrationTest(unittest.TestCase):
+	# TODO
+	pass
+
+class CmapMergeUnitTest(unittest.TestCase):
+	def setUp(self):
+		self.merger = Merger()
+		self.table1 = ttLib.newTable('cmap')
+		self.table2 = ttLib.newTable('cmap')
+		self.mergedTable = ttLib.newTable('cmap')
+		pass
+
+	def tearDown(self):
+		pass
+
+
+	def makeSubtable(self, format, platformID, platEncID, cmap):
+		module = ttLib.getTableModule('cmap')
+		subtable = module.cmap_classes[format](format)
+		(subtable.platformID,
+			subtable.platEncID,
+			subtable.language,
+			subtable.cmap) = (platformID, platEncID, 0, cmap)
+		return subtable
+
+	# 4-3-1 table merged with 12-3-10 table with no dupes with codepoints outside BMP
+	def test_cmap_merge_no_dupes(self):
+		table1 = self.table1
+		table2 = self.table2
+		mergedTable = self.mergedTable
+
+		cmap1 = {0x2603: 'SNOWMAN'}
+		table1.tables = [self.makeSubtable(4,3,1, cmap1)]
+
+		cmap2 = {0x26C4: 'SNOWMAN WITHOUT SNOW'}
+		cmap2Extended = {0x1F93C: 'WRESTLERS'}
+		cmap2Extended.update(cmap2)
+		table2.tables = [self.makeSubtable(4,3,1, cmap2), self.makeSubtable(12,3,10, cmap2Extended)]
+
+		self.merger.alternateGlyphsPerFont = [{},{}]
+		mergedTable.merge(self.merger, [table1, table2])
+
+		expectedCmap = cmap2.copy()
+		expectedCmap.update(cmap1)
+		expectedCmapExtended = cmap2Extended.copy()
+		expectedCmapExtended.update(cmap1)
+		self.assertEqual(mergedTable.numSubTables, 2)
+		self.assertEqual([(table.format, table.platformID, table.platEncID, table.language) for table in mergedTable.tables],
+			[(4,3,1,0),(12,3,10,0)])
+		self.assertEqual(mergedTable.tables[0].cmap, expectedCmap)
+		self.assertEqual(mergedTable.tables[1].cmap, expectedCmapExtended)
+
+	# Tests Issue #322
+	def test_cmap_merge_three_dupes(self):
+		table1 = self.table1
+		table2 = self.table2
+		mergedTable = self.mergedTable
+
+		cmap1 = {0x20: 'space#0', 0xA0: 'space#0'}
+		table1.tables = [self.makeSubtable(4,3,1,cmap1)]
+		cmap2 = {0x20: 'space#1', 0xA0: 'uni00A0#1'}
+		table2.tables = [self.makeSubtable(4,3,1,cmap2)]
+
+		self.merger.duplicateGlyphsPerFont = [{},{}]
+		mergedTable.merge(self.merger, [table1, table2])
+
+		expectedCmap = cmap1.copy()
+		self.assertEqual(mergedTable.numSubTables, 1)
+		table = mergedTable.tables[0]
+		self.assertEqual((table.format, table.platformID, table.platEncID, table.language), (4,3,1,0))
+		self.assertEqual(table.cmap, expectedCmap)
+		self.assertEqual(self.merger.duplicateGlyphsPerFont, [{}, {'space#0': 'space#1'}])
+
+
+if __name__ == "__main__":
+	unittest.main()