Repository: playcanvas/engine
Pull request #7226 (2024-12-20 16:00:49)

Updated preprocessor to optionally remove defines from source
We use preprocessor to process hash directives in the GLSL shader. The processor would leave lines like this in the code:
```
#define VALUE
...
// the value could be used in the code
float count = VALUE;
...
#undef VALUE
```

but as WGSL language does not support defines at all, the preprocessor can optionally strip those.

This is not used currently, but will be used to process WGSL shaders.


Patch:
diff --git a/src/core/preprocessor.js b/src/core/preprocessor.js
index bd8871a36..d36cad1a4 100644
--- a/src/core/preprocessor.js
+++ b/src/core/preprocessor.js
@@ -50,10 +50,12 @@ class Preprocessor {
      * @param {string} source - The source code to work on.
      * @param {Map<string, string>} [includes] - A map containing key-value pairs of include names
      * and their content. These are used for resolving #include directives in the source.
-     * @param {boolean} [stripUnusedColorAttachments] - If true, strips unused color attachments.
+     * @param {object} [options] - Optional parameters.
+     * @param {boolean} [options.stripUnusedColorAttachments] - If true, strips unused color attachments.
+     * @param {boolean} [options.stripDefines] - If true, strips all defines from the source.
      * @returns {string|null} Returns preprocessed source code, or null in case of error.
      */
-    static run(source, includes = new Map(), stripUnusedColorAttachments = false) {
+    static run(source, includes = new Map(), options = {}) {
 
         // strips comments, handles // and many cases of /*
         source = this.stripComments(source);
@@ -65,7 +67,7 @@ class Preprocessor {
 
         // generate defines to remove unused color attachments
         const defines = new Map();
-        if (stripUnusedColorAttachments) {
+        if (options.stripUnusedColorAttachments) {
 
             // find out how many times pcFragColorX is used (see gles3.js)
             const counts = new Map();
@@ -85,7 +87,7 @@ class Preprocessor {
         }
 
         // preprocess defines / ifdefs ..
-        source = this._preprocess(source, defines, includes);
+        source = this._preprocess(source, defines, includes, options.stripDefines);
 
         // extract defines that evaluate to an integer number
         const intDefines = new Map();
@@ -149,9 +151,10 @@ class Preprocessor {
      * by the function.
      * @param {Map<string, string>} [includes] - An object containing key-value pairs of include names and their
      * content.
+     * @param {boolean} [stripDefines] - If true, strips all defines from the source.
      * @returns {string} Returns preprocessed source code.
      */
-    static _preprocess(source, defines = new Map(), includes) {
+    static _preprocess(source, defines = new Map(), includes, stripDefines) {
 
         const originalSource = source;
 
@@ -187,12 +190,22 @@ class Preprocessor {
 
                     if (keep) {
                         defines.set(identifier, value);
+
+                        if (stripDefines) {
+                            // cut out the define line
+                            source = source.substring(0, define.index - 1) + source.substring(DEFINE.lastIndex);
+
+                            // continue processing on the next symbol
+                            KEYWORD.lastIndex = define.index;
+                        }
                     }
 
                     Debug.trace(TRACEID, `${keyword}: [${identifier}] ${value} ${keep ? '' : 'IGNORED'}`);
 
                     // continue on the next line
-                    KEYWORD.lastIndex = define.index + define[0].length;
+                    if (!stripDefines) {
+                        KEYWORD.lastIndex = define.index + define[0].length;
+                    }
                     break;
                 }
 
@@ -209,12 +222,22 @@ class Preprocessor {
                     // remove it from defines
                     if (keep) {
                         defines.delete(identifier);
+
+                        if (stripDefines) {
+                            // cut out the undef line
+                            source = source.substring(0, undef.index - 1) + source.substring(UNDEF.lastIndex);
+
+                            // continue processing on the next symbol
+                            KEYWORD.lastIndex = undef.index;
+                        }
                     }
 
                     Debug.trace(TRACEID, `${keyword}: [${identifier}] ${keep ? '' : 'IGNORED'}`);
 
                     // continue on the next line
-                    KEYWORD.lastIndex = undef.index + undef[0].length;
+                    if (!stripDefines) {
+                        KEYWORD.lastIndex = undef.index + undef[0].length;
+                    }
                     break;
                 }
 
diff --git a/src/platform/graphics/shader.js b/src/platform/graphics/shader.js
index d3b5bab01..7ad036504 100644
--- a/src/platform/graphics/shader.js
+++ b/src/platform/graphics/shader.js
@@ -127,7 +127,9 @@ class Shader {
             // Note: this is only needed for iOS 15 on WebGL2 where there seems to be a bug where color attachments that are not
             // written to generate metal linking errors. This is fixed on iOS 16, and iOS 14 does not support WebGL2.
             const stripUnusedColorAttachments = graphicsDevice.isWebGL2 && (platform.name === 'osx' || platform.name === 'ios');
-            definition.fshader = Preprocessor.run(definition.fshader, definition.fincludes, stripUnusedColorAttachments);
+            definition.fshader = Preprocessor.run(definition.fshader, definition.fincludes, {
+                stripUnusedColorAttachments
+            });
         }
 
         this.impl = graphicsDevice.createShaderImpl(this);


Test patch:
diff --git a/test/core/preprocessor.test.mjs b/test/core/preprocessor.test.mjs
index d722980cc..c56b7a9f5 100644
--- a/test/core/preprocessor.test.mjs
+++ b/test/core/preprocessor.test.mjs
@@ -197,4 +197,8 @@ describe('Preprocessor', function () {
         expect(Preprocessor.run(srcData, includes).includes('CMP5')).to.equal(false);
     });
 
+    it('returns false for any leftover hash symbols', function () {
+        expect(Preprocessor.run(srcData, includes, { stripDefines: true }).includes('#')).to.equal(false);
+    });
+
 });