Repository: mozilla/bugbug

Issue:
When cleaning up DLL names, differentiate between third-party DLLs and Firefox ones
We're currently replacing all DLL names with a _DLL_NAME_ token, but we should differentiate between third-party DLLs and Firefox DLLs (XUL.dll, and so on).


Discussion:
At https://github.com/mozilla/bugbug/blob/008c34b83a3d73dc8c9b007a98367dbacbf78233/bugbug/bug_features.py#L302, we should only cleaup third-party DLLs, and not Firefox ones.

So, for example xul.dll should still be xul.dll and not replaced with __DLL_NAME__. Same with mozglue.dll, libxul.so, and so on.

To get all Firefox DLLs, you can either download the archives containing the Firefox binaries from a URL such as https://ftp.mozilla.org/pub/firefox/nightly/2019/02/2019-02-27-10-44-26-mozilla-central/ (firefox-67.0a1.en-US.win64.zip, firefox-67.0a1.en-US.mac.dmg, firefox-67.0a1.en-US.linux-x86_64.tar.bz2), or install Firefox on the different OSes, go in the installation directory and check which DLLs are present.

I would like to take this up. Could I be assigned?
@LakshyAAAgrawal feel free to work on any open issue. See https://github.com/mozilla/bugbug#contributing, all issues are unassigned until there is a PR open to fix them.
Could I create a file "firefox_DLLs.json" which contains a list of all firefox library files including *.so and *.dll files, and then load this file when the function "cleanup_dll" is called, and then do the needful for differentiating?
You can just hardcode the DLL names in a constant that you define right before cleanup_dll, a separate file is overkill.

Resolving patch:
diff --git a/bugbug/bug_features.py b/bugbug/bug_features.py
--- a/bugbug/bug_features.py
+++ b/bugbug/bug_features.py
@@ -299,8 +299,31 @@ def cleanup_hex(text):
     return re.sub(r'\b0[xX][0-9a-fA-F]+\b', '__HEX_NUMBER__', text)
 
 
+FIREFOX_DLLS_MATCH = '|'.join([
+    'libmozwayland.so', 'libssl3.so', 'libnssdbm3.so', 'liblgpllibs.so', 'libmozavutil.so', 'libxul.so', 'libmozgtk.so', 'libnssckbi.so', 'libclearkey.dylib',
+    'libmozsqlite3.so', 'libplc4.so', 'libsmime3.so', 'libclearkey.so', 'libnssutil3.so', 'libnss3.so', 'libplds4.so', 'libfreeblpriv3.so',
+    'libsoftokn3.so', 'libmozgtk.so', 'libmozavcodec.so', 'libnspr4.so', 'IA2Marshal.dll', 'lgpllibs.dll', 'libEGL.dll', 'libGLESv2.dll',
+    'libmozsandbox.so', 'AccessibleHandler.dll', 'AccessibleMarshal.dll', 'api-ms-win-core-console-l1-1-0.dll',
+    'api-ms-win-core-datetime-l1-1-0.dll', 'api-ms-win-core-debug-l1-1-0.dll', 'api-ms-win-core-errorhandling-l1-1-0.dll', 'api-ms-win-core-file-l1-1-0.dll',
+    'api-ms-win-core-file-l1-2-0.dll', 'api-ms-win-core-file-l2-1-0.dll', 'api-ms-win-core-handle-l1-1-0.dll', 'api-ms-win-core-heap-l1-1-0.dll',
+    'api-ms-win-core-interlocked-l1-1-0.dll', 'api-ms-win-core-libraryloader-l1-1-0.dll', 'api-ms-win-core-localization-l1-2-0.dll', 'api-ms-win-core-memory-l1-1-0.dll',
+    'api-ms-win-core-namedpipe-l1-1-0.dll', 'api-ms-win-core-processenvironment-l1-1-0.dll', 'api-ms-win-core-processthreads-l1-1-0.dll',
+    'api-ms-win-core-processthreads-l1-1-1.dll', 'api-ms-win-core-profile-l1-1-0.dll',
+    'api-ms-win-core-rtlsupport-l1-1-0.dll', 'api-ms-win-core-string-l1-1-0.dll', 'api-ms-win-core-synch-l1-1-0.dll', 'api-ms-win-core-synch-l1-2-0.dll',
+    'api-ms-win-core-sysinfo-l1-1-0.dll', 'api-ms-win-core-timezone-l1-1-0.dll', 'api-ms-win-core-util-l1-1-0.dll', 'api-ms-win-crt-conio-l1-1-0.dll',
+    'api-ms-win-crt-convert-l1-1-0.dll', 'api-ms-win-crt-environment-l1-1-0.dll', 'api-ms-win-crt-filesystem-l1-1-0.dll', 'api-ms-win-crt-heap-l1-1-0.dll',
+    'api-ms-win-crt-locale-l1-1-0.dll', 'api-ms-win-crt-math-l1-1-0.dll', 'api-ms-win-crt-multibyte-l1-1-0.dll', 'api-ms-win-crt-private-l1-1-0.dll',
+    'api-ms-win-crt-process-l1-1-0.dll', 'api-ms-win-crt-runtime-l1-1-0.dll', 'api-ms-win-crt-stdio-l1-1-0.dll', 'api-ms-win-crt-string-l1-1-0.dll',
+    'api-ms-win-crt-time-l1-1-0.dll', 'api-ms-win-crt-utility-l1-1-0.dll', 'd3dcompiler_47.dll', 'freebl3.dll',
+    'mozavcodec.dll', 'mozavutil.dll', 'mozglue.dll', 'msvcp140.dll', 'nss3.dll', 'nssckbi.dll', 'nssdbm3.dll', 'qipcap64.dll',
+    'softokn3.dll', 'ucrtbase.dll', 'vcruntime140.dll', 'xul.dll', 'clearkey.dll', 'libfreebl3.dylib', 'liblgpllibs.dylib', 'libmozavcodec.dylib',
+    'libmozavutil.dylib', 'libmozglue.dylib', 'libnss3.dylib', 'libnssckbi.dylib', 'libnssdbm3.dylib', 'libplugin_child_interpose.dylib', 'libsoftokn3.dylib'
+]).replace('.', r'\.')
+
+
 def cleanup_dll(text):
-    return re.sub(r'\w+(\.dll|\.so|\.dylib)\b', '__DLL_NAME__', text)
+    regex = fr'\b(?!{FIREFOX_DLLS_MATCH})\w+(\.dll|\.so|\.dylib)\b'
+    return re.sub(regex, '__DLL_NAME__', text)
 
 
 def cleanup_synonyms(text):


Test patch:
diff --git a/tests/test_cleanup_functions.py b/tests/test_cleanup_functions.py
--- a/tests/test_cleanup_functions.py
+++ b/tests/test_cleanup_functions.py
@@ -48,8 +48,11 @@ def test_cleanup_hex():
 def test_cleanup_dll():
     tests = [
         ('Crashing thread: 0 scdetour.dll scdetour.dll@0x2dd77', 'Crashing thread: 0 __DLL_NAME__ __DLL_NAME__@0x2dd77'),
-        ('Crash in libxul.so@0x287ad36 | libxul.so@0x270c062', 'Crash in __DLL_NAME__@0x287ad36 | __DLL_NAME__@0x270c062'),
-        ('Crash in libsystem_pthread.dylib@0x14fc', 'Crash in __DLL_NAME__@0x14fc')
+        ('Crash in libxul.so@0x287ad36 | libxul.so@0x270c062', 'Crash in libxul.so@0x287ad36 | libxul.so@0x270c062'),
+        ('Crash in libsystem_pthread.dylib@0x14fc', 'Crash in __DLL_NAME__@0x14fc'),
+        ('Crash in liblgpllibs.so@0x14fc exmpl.so@0xask ', 'Crash in liblgpllibs.so@0x14fc __DLL_NAME__@0xask '),
+        ('Crash in lgpllibs.dll@0x14fc exmpl.dll@0xask ', 'Crash in lgpllibs.dll@0x14fc __DLL_NAME__@0xask '),
+        ('Crash in libmozglue.dylib@0x14fc exmpl.dylib@0xask ', 'Crash in libmozglue.dylib@0x14fc __DLL_NAME__@0xask ')
     ]
     for orig_text, cleaned_text in tests:
         assert bug_features.cleanup_dll(orig_text) == cleaned_text