Repository: sveltejs/svelte (JavaScript)
Date: 2019-01-14 00:30:07+00:00
Task type: Feature

Issue:
Proposal: Preprocessor composition
It would be nice to allow `svelte.preprocess` to be passed an array of preprocessors. Each of these would be an object containing `markup`/`style`/`script` functions. All `markup` preprocessors would be called first in order, and then all `style`/`script` ones would be called.

See https://github.com/nsivertsen/svelte-compose-preprocessors for an existing userland implementation of this.

If we do this, Svelte should also eventually support composing the source maps from these preprocessors - but perhaps that need not be implemented until the compiler itself supports consuming source maps.


Issue discussion:

Solution patch:
diff --git a/src/preprocess/index.ts b/src/preprocess/index.ts
--- a/src/preprocess/index.ts
+++ b/src/preprocess/index.ts
@@ -1,25 +1,25 @@
 import { SourceMap } from 'magic-string';
 import replaceAsync from '../utils/replaceAsync';
 
-export interface PreprocessOptions {
+export interface PreprocessorGroup {
 	markup?: (options: {
 		content: string,
 		filename: string
-	}) => { code: string, map?: SourceMap | string };
+	}) => { code: string, map?: SourceMap | string, dependencies?: string[] };
 	style?: Preprocessor;
 	script?: Preprocessor;
-	filename?: string
 }
 
 export type Preprocessor = (options: {
 	content: string,
 	attributes: Record<string, string | boolean>,
 	filename?: string
-}) => { code: string, map?: SourceMap | string };
+}) => { code: string, map?: SourceMap | string, dependencies?: string[] };
 
 interface Processed {
 	code: string;
 	map?: SourceMap | string;
+	dependencies?: string[];
 }
 
 function parseAttributeValue(value: string) {
@@ -39,28 +39,55 @@ function parseAttributes(str: string) {
 
 export default async function preprocess(
 	source: string,
-	options: PreprocessOptions
+	preprocessor: PreprocessorGroup | PreprocessorGroup[],
+	options?: { filename?: string }
 ) {
-	if (options.markup) {
-		const processed: Processed = await options.markup({
+	const filename = (options && options.filename) || preprocessor.filename; // legacy
+	const dependencies = [];
+
+	const preprocessors = Array.isArray(preprocessor) ? preprocessor : [preprocessor];
+
+	const markup = preprocessors.map(p => p.markup).filter(Boolean);
+	const script = preprocessors.map(p => p.script).filter(Boolean);
+	const style = preprocessors.map(p => p.style).filter(Boolean);
+
+	for (const fn of markup) {
+		const processed: Processed = await fn({
 			content: source,
-			filename: options.filename,
+			filename
 		});
+		if (processed && processed.dependencies) dependencies.push(...processed.dependencies);
 		source = processed ? processed.code : source;
 	}
-	if (options.style || options.script) {
+
+	for (const fn of script) {
+		source = await replaceAsync(
+			source,
+			/<script([^]*?)>([^]*?)<\/script>/gi,
+			async (match, attributes, content) => {
+				const processed: Processed = await fn({
+					content,
+					attributes: parseAttributes(attributes),
+					filename
+				});
+				if (processed && processed.dependencies) dependencies.push(...processed.dependencies);
+				return processed ? `<script${attributes}>${processed.code}</script>` : match;
+			}
+		);
+	}
+
+	for (const fn of style) {
 		source = await replaceAsync(
 			source,
-			/<(script|style)([^]*?)>([^]*?)<\/\1>/gi,
-			async (match, type, attributes, content) => {
-				const processed: Processed =
-					type in options &&
-					(await options[type]({
-						content,
-						attributes: parseAttributes(attributes),
-						filename: options.filename,
-					}));
-				return processed ? `<${type}${attributes}>${processed.code}</${type}>` : match;
+			/<style([^]*?)>([^]*?)<\/style>/gi,
+			async (match, attributes, content) => {
+				const processed: Processed = await fn({
+					content,
+					attributes: parseAttributes(attributes),
+					filename
+				});
+				if (processed && processed.dependencies) dependencies.push(...processed.dependencies);
+				return processed ? `<style${attributes}>${processed.code}</style>` : match;
 			}
 		);
 	}
@@ -71,6 +98,9 @@ export default async function preprocess(
 		// script { code: scriptCode, map: scriptMap },
 		// markup { code: markupCode, map: markupMap },
 
+		code: source,
+		dependencies: [...new Set(dependencies)],
+
 		toString() {
 			return source;
 		}


Test patch:
diff --git a/test/preprocess/index.js b/test/preprocess/index.js
--- a/test/preprocess/index.js
+++ b/test/preprocess/index.js
@@ -1,226 +1,27 @@
+import * as fs from 'fs';
 import * as assert from 'assert';
-import { svelte } from '../helpers.js';
+import { loadConfig, svelte } from '../helpers.js';
 
 describe('preprocess', () => {
-	it('preprocesses entire component', () => {
-		const source = `
-			<h1>Hello __NAME__!</h1>
-		`;
+	fs.readdirSync('test/preprocess/samples').forEach(dir => {
+		if (dir[0] === '.') return;
 
-		const expected = `
-			<h1>Hello world!</h1>
-		`;
+		const config = loadConfig(`./preprocess/samples/${dir}/_config.js`);
 
-		return svelte.preprocess(source, {
-			markup: ({ content }) => {
-				return {
-					code: content.replace('__NAME__', 'world')
-				};
-			}
-		}).then(processed => {
-			assert.equal(processed.toString(), expected);
-		});
-	});
-
-	it('preprocesses style', () => {
-		const source = `
-			<div class='brand-color'>$brand</div>
-
-			<style>
-				.brand-color {
-					color: $brand;
-				}
-			</style>
-		`;
-
-		const expected = `
-			<div class='brand-color'>$brand</div>
-
-			<style>
-				.brand-color {
-					color: purple;
-				}
-			</style>
-		`;
-
-		return svelte.preprocess(source, {
-			style: ({ content }) => {
-				return {
-					code: content.replace('$brand', 'purple')
-				};
-			}
-		}).then(processed => {
-			assert.equal(processed.toString(), expected);
-		});
-	});
-
-	it('preprocesses style asynchronously', () => {
-		const source = `
-			<div class='brand-color'>$brand</div>
-
-			<style>
-				.brand-color {
-					color: $brand;
-				}
-			</style>
-		`;
-
-		const expected = `
-			<div class='brand-color'>$brand</div>
-
-			<style>
-				.brand-color {
-					color: purple;
-				}
-			</style>
-		`;
-
-		return svelte.preprocess(source, {
-			style: ({ content }) => {
-				return Promise.resolve({
-					code: content.replace('$brand', 'purple')
-				});
-			}
-		}).then(processed => {
-			assert.equal(processed.toString(), expected);
-		});
-	});
-
-	it('preprocesses script', () => {
-		const source = `
-			<script>
-				console.log(__THE_ANSWER__);
-			</script>
-		`;
-
-		const expected = `
-			<script>
-				console.log(42);
-			</script>
-		`;
+		if (config.solo && process.env.CI) {
+			throw new Error('Forgot to remove `solo: true` from test');
+		}
 
-		return svelte.preprocess(source, {
-			script: ({ content }) => {
-				return {
-					code: content.replace('__THE_ANSWER__', '42')
-				};
-			}
-		}).then(processed => {
-			assert.equal(processed.toString(), expected);
-		});
-	});
-
-	it('preprocesses multiple matching tags', () => {
-		const source = `
-			<script>
-				REPLACEME
-			</script>
-			<style>
-				SHOULD NOT BE REPLACED
-			</style>
-			<script>
-				REPLACEMETOO
-			</script>
-		`;
-
-		const expected = `
-			<script>
-				replaceme
-			</script>
-			<style>
-				SHOULD NOT BE REPLACED
-			</style>
-			<script>
-				replacemetoo
-			</script>
-		`;
-
-		return svelte.preprocess(source, {
-			script: ({ content }) => {
-				return {
-					code: content.toLowerCase()
-				};
-			}
-		}).then(processed => {
-			assert.equal(processed.toString(), expected);
-		});
-	});
-
-	it('parses attributes', () => {
-		const source = `
-			<style type='text/scss' data-foo="bar" bool></style>
-		`;
-
-		const expected = `
-			<style type='text/scss' data-foo="bar" bool>PROCESSED</style>
-		`;
-
-		return svelte.preprocess(source, {
-			style: ({ attributes }) => {
-				assert.deepEqual(attributes, {
-					type: 'text/scss',
-					'data-foo': 'bar',
-					bool: true
-				});
-				return { code: 'PROCESSED' };
-			}
-		}).then(processed => {
-			assert.equal(processed.toString(), expected);
-		});
-	});
-
-	it('provides filename to processing hooks', () => {
-		const source = `
-			<h1>Hello __MARKUP_FILENAME__!</h1>
-			<style>.red { color: __STYLE_FILENAME__; }</style>
-			<script>console.log('__SCRIPT_FILENAME__');</script>
-		`;
-
-		const expected = `
-			<h1>Hello file.html!</h1>
-			<style>.red { color: file.html; }</style>
-			<script>console.log('file.html');</script>
-		`;
-
-		return svelte.preprocess(source, {
-			filename: 'file.html',
-			markup: ({ content, filename }) => {
-				return {
-					code: content.replace('__MARKUP_FILENAME__', filename)
-				};
-			},
-			style: ({ content, filename }) => {
-				return {
-					code: content.replace('__STYLE_FILENAME__', filename)
-				};
-			},
-			script: ({ content, filename }) => {
-				return {
-					code: content.replace('__SCRIPT_FILENAME__', filename)
-				};
-			}
-		}).then(processed => {
-			assert.equal(processed.toString(), expected);
-		});
-	});
+		(config.skip ? it.skip : config.solo ? it.only : it)(dir, async () => {
+			const input = fs.readFileSync(`test/preprocess/samples/${dir}/input.html`, 'utf-8');
+			const expected = fs.readFileSync(`test/preprocess/samples/${dir}/output.html`, 'utf-8');
 
-	it('ignores null/undefined returned from preprocessor', () => {
-		const source = `
-			<script>
-				console.log('ignore me');
-			</script>
-		`;
+			const result = await svelte.preprocess(input, config.preprocess);
+			fs.writeFileSync(`test/preprocess/samples/${dir}/_actual.html`, result.code);
 
-		const expected = `
-			<script>
-				console.log('ignore me');
-			</script>
-		`;
+			assert.equal(result.code, expected);
 
-		return svelte.preprocess(source, {
-			script: () => null
-		}).then(processed => {
-			assert.equal(processed.toString(), expected);
+			assert.deepEqual(result.dependencies, config.dependencies || []);
 		});
 	});
 });
diff --git a/test/preprocess/samples/dependencies/_config.js b/test/preprocess/samples/dependencies/_config.js
new file mode 100644
--- /dev/null
+++ b/test/preprocess/samples/dependencies/_config.js
@@ -0,0 +1,15 @@
+export default {
+	preprocess: {
+		style: ({ content }) => {
+			const dependencies = [];
+			const code = content.replace(/@import '(.+)';/g, (match, $1) => {
+				dependencies.push($1);
+				return '/* removed */';
+			});
+
+			return { code, dependencies };
+		}
+	},
+
+	dependencies: ['./foo.css']
+};
\ No newline at end of file
diff --git a/test/preprocess/samples/dependencies/input.html b/test/preprocess/samples/dependencies/input.html
new file mode 100644
--- /dev/null
+++ b/test/preprocess/samples/dependencies/input.html
@@ -0,0 +1,3 @@
+<style>
+	@import './foo.css';
+</style>
\ No newline at end of file
diff --git a/test/preprocess/samples/dependencies/output.html b/test/preprocess/samples/dependencies/output.html
new file mode 100644
--- /dev/null
+++ b/test/preprocess/samples/dependencies/output.html
@@ -0,0 +1,3 @@
+<style>
+	/* removed */
+</style>
\ No newline at end of file
diff --git a/test/preprocess/samples/filename/_config.js b/test/preprocess/samples/filename/_config.js
new file mode 100644
--- /dev/null
+++ b/test/preprocess/samples/filename/_config.js
@@ -0,0 +1,20 @@
+export default {
+	preprocess: {
+		filename: 'file.html',
+		markup: ({ content, filename }) => {
+			return {
+				code: content.replace('__MARKUP_FILENAME__', filename)
+			};
+		},
+		style: ({ content, filename }) => {
+			return {
+				code: content.replace('__STYLE_FILENAME__', filename)
+			};
+		},
+		script: ({ content, filename }) => {
+			return {
+				code: content.replace('__SCRIPT_FILENAME__', filename)
+			};
+		}
+	}
+};
\ No newline at end of file
diff --git a/test/preprocess/samples/filename/input.html b/test/preprocess/samples/filename/input.html
new file mode 100644
--- /dev/null
+++ b/test/preprocess/samples/filename/input.html
@@ -0,0 +1,3 @@
+<h1>Hello __MARKUP_FILENAME__!</h1>
+<style>.red { color: __STYLE_FILENAME__; }</style>
+<script>console.log('__SCRIPT_FILENAME__');</script>
\ No newline at end of file
diff --git a/test/preprocess/samples/filename/output.html b/test/preprocess/samples/filename/output.html
new file mode 100644
--- /dev/null
+++ b/test/preprocess/samples/filename/output.html
@@ -0,0 +1,3 @@
+<h1>Hello file.html!</h1>
+<style>.red { color: file.html; }</style>
+<script>console.log('file.html');</script>
\ No newline at end of file
diff --git a/test/preprocess/samples/ignores-null/_config.js b/test/preprocess/samples/ignores-null/_config.js
new file mode 100644
--- /dev/null
+++ b/test/preprocess/samples/ignores-null/_config.js
@@ -0,0 +1,5 @@
+export default {
+	preprocess: {
+		script: () => null
+	}
+};
\ No newline at end of file
diff --git a/test/preprocess/samples/ignores-null/input.html b/test/preprocess/samples/ignores-null/input.html
new file mode 100644
--- /dev/null
+++ b/test/preprocess/samples/ignores-null/input.html
@@ -0,0 +1,3 @@
+<script>
+	console.log('ignore me');
+</script>
\ No newline at end of file
diff --git a/test/preprocess/samples/ignores-null/output.html b/test/preprocess/samples/ignores-null/output.html
new file mode 100644
--- /dev/null
+++ b/test/preprocess/samples/ignores-null/output.html
@@ -0,0 +1,3 @@
+<script>
+	console.log('ignore me');
+</script>
\ No newline at end of file
diff --git a/test/preprocess/samples/markup/_config.js b/test/preprocess/samples/markup/_config.js
new file mode 100644
--- /dev/null
+++ b/test/preprocess/samples/markup/_config.js
@@ -0,0 +1,9 @@
+export default {
+	preprocess: {
+		markup: ({ content }) => {
+			return {
+				code: content.replace('__NAME__', 'world')
+			};
+		}
+	}
+};
\ No newline at end of file
diff --git a/test/preprocess/samples/markup/input.html b/test/preprocess/samples/markup/input.html
new file mode 100644
--- /dev/null
+++ b/test/preprocess/samples/markup/input.html
@@ -0,0 +1 @@
+<h1>Hello __NAME__!</h1>
\ No newline at end of file
diff --git a/test/preprocess/samples/markup/output.html b/test/preprocess/samples/markup/output.html
new file mode 100644
--- /dev/null
+++ b/test/preprocess/samples/markup/output.html
@@ -0,0 +1 @@
+<h1>Hello world!</h1>
\ No newline at end of file
diff --git a/test/preprocess/samples/multiple-preprocessors/_config.js b/test/preprocess/samples/multiple-preprocessors/_config.js
new file mode 100644
--- /dev/null
+++ b/test/preprocess/samples/multiple-preprocessors/_config.js
@@ -0,0 +1,14 @@
+export default {
+	preprocess: [
+		{
+			markup: ({ content }) => ({ code: content.replace(/one/g, 'two') }),
+			script: ({ content }) => ({ code: content.replace(/three/g, 'four') }),
+			style:  ({ content }) => ({ code: content.replace(/four/g, 'five') })
+		},
+		{
+			markup: ({ content }) => ({ code: content.replace(/two/g, 'three') }),
+			script: ({ content }) => ({ code: content.replace(/four/g, 'five') }),
+			style:  ({ content }) => ({ code: content.replace(/three/g, 'four') })
+		}
+	]
+};
\ No newline at end of file
diff --git a/test/preprocess/samples/multiple-preprocessors/input.html b/test/preprocess/samples/multiple-preprocessors/input.html
new file mode 100644
--- /dev/null
+++ b/test/preprocess/samples/multiple-preprocessors/input.html
@@ -0,0 +1,11 @@
+<p>one</p>
+
+<style>
+	.one {
+		color: red;
+	}
+</style>
+
+<script>
+	console.log('one');
+</script>
\ No newline at end of file
diff --git a/test/preprocess/samples/multiple-preprocessors/output.html b/test/preprocess/samples/multiple-preprocessors/output.html
new file mode 100644
--- /dev/null
+++ b/test/preprocess/samples/multiple-preprocessors/output.html
@@ -0,0 +1,11 @@
+<p>three</p>
+
+<style>
+	.four {
+		color: red;
+	}
+</style>
+
+<script>
+	console.log('five');
+</script>
\ No newline at end of file
diff --git a/test/preprocess/samples/script-multiple/_config.js b/test/preprocess/samples/script-multiple/_config.js
new file mode 100644
--- /dev/null
+++ b/test/preprocess/samples/script-multiple/_config.js
@@ -0,0 +1,9 @@
+export default {
+	preprocess: {
+		script: ({ content }) => {
+			return {
+				code: content.toLowerCase()
+			};
+		}
+	}
+};
\ No newline at end of file
diff --git a/test/preprocess/samples/script-multiple/input.html b/test/preprocess/samples/script-multiple/input.html
new file mode 100644
--- /dev/null
+++ b/test/preprocess/samples/script-multiple/input.html
@@ -0,0 +1,9 @@
+<script>
+	REPLACEME
+</script>
+<style>
+	SHOULD NOT BE REPLACED
+</style>
+<script>
+	REPLACEMETOO
+</script>
\ No newline at end of file
diff --git a/test/preprocess/samples/script-multiple/output.html b/test/preprocess/samples/script-multiple/output.html
new file mode 100644
--- /dev/null
+++ b/test/preprocess/samples/script-multiple/output.html
@@ -0,0 +1,9 @@
+<script>
+	replaceme
+</script>
+<style>
+	SHOULD NOT BE REPLACED
+</style>
+<script>
+	replacemetoo
+</script>
\ No newline at end of file
diff --git a/test/preprocess/samples/script/_config.js b/test/preprocess/samples/script/_config.js
new file mode 100644
--- /dev/null
+++ b/test/preprocess/samples/script/_config.js
@@ -0,0 +1,9 @@
+export default {
+	preprocess: {
+		script: ({ content }) => {
+			return {
+				code: content.replace('__THE_ANSWER__', '42')
+			};
+		}
+	}
+};
\ No newline at end of file
diff --git a/test/preprocess/samples/script/input.html b/test/preprocess/samples/script/input.html
new file mode 100644
--- /dev/null
+++ b/test/preprocess/samples/script/input.html
@@ -0,0 +1,3 @@
+<script>
+	console.log(__THE_ANSWER__);
+</script>
\ No newline at end of file
diff --git a/test/preprocess/samples/script/output.html b/test/preprocess/samples/script/output.html
new file mode 100644
--- /dev/null
+++ b/test/preprocess/samples/script/output.html
@@ -0,0 +1,3 @@
+<script>
+	console.log(42);
+</script>
\ No newline at end of file
diff --git a/test/preprocess/samples/style-async/_config.js b/test/preprocess/samples/style-async/_config.js
new file mode 100644
--- /dev/null
+++ b/test/preprocess/samples/style-async/_config.js
@@ -0,0 +1,9 @@
+export default {
+	preprocess: {
+		style: ({ content }) => {
+			return Promise.resolve({
+				code: content.replace('$brand', 'purple')
+			});
+		}
+	}
+};
\ No newline at end of file
diff --git a/test/preprocess/samples/style-async/input.html b/test/preprocess/samples/style-async/input.html
new file mode 100644
--- /dev/null
+++ b/test/preprocess/samples/style-async/input.html
@@ -0,0 +1,7 @@
+<div class='brand-color'>$brand</div>
+
+<style>
+	.brand-color {
+		color: $brand;
+	}
+</style>
\ No newline at end of file
diff --git a/test/preprocess/samples/style-async/output.html b/test/preprocess/samples/style-async/output.html
new file mode 100644
--- /dev/null
+++ b/test/preprocess/samples/style-async/output.html
@@ -0,0 +1,7 @@
+<div class='brand-color'>$brand</div>
+
+<style>
+	.brand-color {
+		color: purple;
+	}
+</style>
\ No newline at end of file
diff --git a/test/preprocess/samples/style-attributes/_config.js b/test/preprocess/samples/style-attributes/_config.js
new file mode 100644
--- /dev/null
+++ b/test/preprocess/samples/style-attributes/_config.js
@@ -0,0 +1,14 @@
+import * as assert from 'assert';
+
+export default {
+	preprocess: {
+		style: ({ attributes }) => {
+			assert.deepEqual(attributes, {
+				type: 'text/scss',
+				'data-foo': 'bar',
+				bool: true
+			});
+			return { code: 'PROCESSED' };
+		}
+	}
+};
\ No newline at end of file
diff --git a/test/preprocess/samples/style-attributes/input.html b/test/preprocess/samples/style-attributes/input.html
new file mode 100644
--- /dev/null
+++ b/test/preprocess/samples/style-attributes/input.html
@@ -0,0 +1 @@
+<style type='text/scss' data-foo="bar" bool></style>
\ No newline at end of file
diff --git a/test/preprocess/samples/style-attributes/output.html b/test/preprocess/samples/style-attributes/output.html
new file mode 100644
--- /dev/null
+++ b/test/preprocess/samples/style-attributes/output.html
@@ -0,0 +1 @@
+<style type='text/scss' data-foo="bar" bool>PROCESSED</style>
\ No newline at end of file
diff --git a/test/preprocess/samples/style/_config.js b/test/preprocess/samples/style/_config.js
new file mode 100644
--- /dev/null
+++ b/test/preprocess/samples/style/_config.js
@@ -0,0 +1,9 @@
+export default {
+	preprocess: {
+		style: ({ content }) => {
+			return {
+				code: content.replace('$brand', 'purple')
+			};
+		}
+	}
+};
\ No newline at end of file
diff --git a/test/preprocess/samples/style/input.html b/test/preprocess/samples/style/input.html
new file mode 100644
--- /dev/null
+++ b/test/preprocess/samples/style/input.html
@@ -0,0 +1,7 @@
+<div class='brand-color'>$brand</div>
+
+<style>
+	.brand-color {
+		color: $brand;
+	}
+</style>
\ No newline at end of file
diff --git a/test/preprocess/samples/style/output.html b/test/preprocess/samples/style/output.html
new file mode 100644
--- /dev/null
+++ b/test/preprocess/samples/style/output.html
@@ -0,0 +1,7 @@
+<div class='brand-color'>$brand</div>
+
+<style>
+	.brand-color {
+		color: purple;
+	}
+</style>
\ No newline at end of file