Change:
<!-- Do Not Delete This! featurerequesttemplate --<!-- Please read our Rules of Conduct: --<!-- Please search existing issues to avoid creating duplicates. --<!-- Describe the feature you'd like. --This request is to allow semantic tokens to not just pick the first most provider in the list of providers. But instead pick the provider that actually has tokens to return. The use case for this is to support jupyter notebook token colorization. Right now there are at least 2 language servers returning tokens for python notebooks: 1) Python extension has a language server registered for all 'python' files. There's no way to 'exclude' files from a DocumentFilter so it has a piece that excludes all notebook cells. 2) Jupyter extension has one ore more language servers registered for all notebook cells. One or more is because the language server in use can only support one python interpreter at a time, so one language server is created per python interpreter in use for any notebook (multiple interpreters can be in use in one VS code session). When semantic tokens are requested the middleware piece in the juptyer extension will skip sending token requests to if they're not for the appropriate document. This creates a problem with the current implementation because the first registered language server wins. This can randomly be python's (where no tokens are returned) or one of the jupyter one's (where it might return tokens if the interpreter for the document matches). I believe this could all be accomplished by making a compositing token provider in the VS code source. Something like so:
I'm going to try out the composite idea and submit a PR if it works.
Verified it works with a notebook !
hey, uhh, I opened an feature-request about semantic tokens the other day, just right after 4 hours from you.... I was trying to make a SemanticTokensProvider that overrides or works on top of already registered another one.... 't it be better if there could be multiple SemanticTokensProvider to work alongside, maybe override one another by User-Specified priority levels? I was struggling to override some of SemanticTokens that the C/C++ extension provides, but then I found out its not really possible to have multiple running Provider for SemanticTokens....
This is a different use case than what I had envisioned. My use case was there were multiple providers but only one was actually returning anything. Combining them sounds like an even better way to handle it. It would handle it for my case and your case. I upvoted your request.
Thank you.... 19 More people and I really dont have any idea how to get there tho, haha.

Discussion:
import { VSBuffer } from 'vs/base/common/buffer'; import { encodeSemanticTokensDto } from 'vs/editor/common/services/semanticTokensDto'; import { Range } from 'vs/editor/common/core/range'; <ins> import { Emitter } from 'vs/base/common/event'; import { DisposableStore } from 'vs/base/common/lifecycle'; </ins> export function isSemanticTokens(v: SemanticTokens | SemanticTokensEdits): v is SemanticTokens { return v && !!((<SemanticTokens>v).data);

} export function getDocumentSemanticTokens(model: ITextModel, lastResultId: string | null, token: CancellationToken): IDocumentSemanticTokensResult | null { <del> const provider = _getDocumentSemanticTokensProvider(model); if (!provider) { </del> <ins> const providerGroup = _getDocumentSemanticTokensProviderHighestGroup(model); if (!providerGroup) { </ins> return null; } <ins> const compositeProvider = new CompositeDocumentSemanticTokensProvider(model, providerGroup); </ins> return { <del> provider: provider, request: Promise.resolve(provider.provideDocumentSemanticTokens(model, lastResultId, token)) </del> <ins> provider: compositeProvider, request: Promise.resolve(compositeProvider.provideDocumentSemanticTokens(model, lastResultId, token)) </ins> }; } <del> function _getDocumentSemanticTokensProvider(model: ITextModel): DocumentSemanticTokensProvider | null { const result = DocumentSemanticTokensProviderRegistry.ordered(model); </del> <ins> class CompositeDocumentSemanticTokensProvider implements DocumentSemanticTokensProvider { private readonly disposables = new DisposableStore(); private readonly didChangeEmitter = this.disposables.add(new Emitter<void>()); public readonly onDidChange = this.didChangeEmitter.event; private lastUsedProvider: DocumentSemanticTokensProvider | undefined = undefined; private static providerToLastResult = new WeakMap<DocumentSemanticTokensProvider, string>(); constructor(model: ITextModel, private readonly providerGroup: DocumentSemanticTokensProvider[]) { // Lifetime of this provider is tied to the text model model.onWillDispose(() => this.disposables.clear()); // Mirror did change events providerGroup.forEach(p => { if (p.onDidChange) { p.onDidChange(() => this.didChangeEmitter.fire(), this, this.disposables); } }); } public async provideDocumentSemanticTokens(model: ITextModel, lastResultId: string | null, token: CancellationToken): Promise<SemanticTokens | SemanticTokensEdits | null | undefined> { // Get tokens from the group all at the same time. Return the first // that actually returned tokens const list = await Promise.all(this.providerGroup.map(async provider => { try { // If result id is passed in, make sure it's for this provider const localLastResultId = lastResultId && CompositeDocumentSemanticTokensProvider.providerToLastResult.get(provider) === lastResultId ? lastResultId : null; // Get the result for this provider const result = await provider.provideDocumentSemanticTokens(model, localLastResultId, token); // Save result id for this provider if (result?.resultId) { CompositeDocumentSemanticTokensProvider.providerToLastResult.set(provider, result.resultId); } return result; } catch (err) { onUnexpectedExternalError(err); } return undefined; })); const hasTokensIndex = list.findIndex(l => l); // Save last used provider. Use it for the legend if called this.lastUsedProvider = this.providerGroup[hasTokensIndex]; return list[hasTokensIndex]; } public getLegend(): SemanticTokensLegend { return this.lastUsedProvider?.getLegend() || this.providerGroup[0].getLegend(); } public releaseDocumentSemanticTokens(resultId: string | undefined): void { this.providerGroup.forEach(p => { // If this result is for this provider, release it if (resultId) { if (CompositeDocumentSemanticTokensProvider.providerToLastResult.get(p) === resultId) { p.releaseDocumentSemanticTokens(resultId); CompositeDocumentSemanticTokensProvider.providerToLastResult.delete(p); } // Else if the result is empty, release for all providers that aren't waiting for a result id } else if (CompositeDocumentSemanticTokensProvider.providerToLastResult.get(p) === undefined) { p.releaseDocumentSemanticTokens(undefined); } }); } } class CompositeDocumentRangeSemanticTokensProvider implements DocumentRangeSemanticTokensProvider { private lastUsedProvider: DocumentRangeSemanticTokensProvider | undefined = undefined; constructor( private readonly providerGroup: DocumentRangeSemanticTokensProvider[] ) { } public async provideDocumentRangeSemanticTokens(model: ITextModel, range: Range, token: CancellationToken): Promise<SemanticTokens | null | undefined> { // Get tokens from the group all at the same time. Return the first // that actually returned tokens const list = await Promise.all(this.providerGroup.map(async provider => { try { return await provider.provideDocumentRangeSemanticTokens(model, range, token); } catch (err) { onUnexpectedExternalError(err); } return undefined; })); const hasTokensIndex = list.findIndex(l => l); // Save last used provider. Use it for the legend if called this.lastUsedProvider = this.providerGroup[hasTokensIndex]; return list[hasTokensIndex]; } getLegend(): SemanticTokensLegend { return this.lastUsedProvider?.getLegend() || this.providerGroup[0].getLegend(); } } function _getDocumentSemanticTokensProviderHighestGroup(model: ITextModel): DocumentSemanticTokensProvider[] | null { const result = DocumentSemanticTokensProviderRegistry.orderedGroups(model); </ins> return (result.length > 0 ? result[0] : null); } export function getDocumentRangeSemanticTokensProvider(model: ITextModel): DocumentRangeSemanticTokensProvider | null { <del> const result = DocumentRangeSemanticTokensProviderRegistry.ordered(model); return (result.length > 0 ? result[0] : null); </del> <ins> const groups = DocumentRangeSemanticTokensProviderRegistry.orderedGroups(model); const highestGroup = (groups.length > 0 ? groups[0] : []); if (highestGroup.length === 0) { // there are no providers return null; } if (highestGroup.length === 1) { // there is a single provider return highestGroup[0]; } return new CompositeDocumentRangeSemanticTokensProvider(highestGroup); </ins> } CommandsRegistry.registerCommand('_provideDocumentSemanticTokensLegend', async (accessor, ...args): Promise<SemanticTokensLegend | undefined> => {

return undefined; } <del> const provider = _getDocumentSemanticTokensProvider(model); if (!provider) { </del> <ins> const providers = _getDocumentSemanticTokensProviderHighestGroup(model); if (!providers) { </ins> // there is no provider => fall back to a document range semantic tokens provider return accessor.get(ICommandService).executeCommand('_provideDocumentRangeSemanticTokensLegend', uri); } <del> return provider.getLegend(); </del> <ins> return providers[0].getLegend(); </ins> }); CommandsRegistry.registerCommand('_provideDocumentSemanticTokens', async (accessor, ...args): Promise<VSBuffer | undefined> => {

pendingChanges.push(e); }); <del> const styling = this._semanticStyling.get(provider); </del> request.then((res) => { this._currentDocumentRequestCancellationTokenSource = null; <ins> const styling = this._semanticStyling.get(provider); // Do this after the provider gets results to ensure legend matches </ins> contentChangeListener.dispose(); this._setDocumentSemanticTokens(provider, res || null, styling, pendingChanges); }, (err) => {

import { IModeService } from 'vs/editor/common/services/modeService'; import { TestTextResourcePropertiesService } from 'vs/editor/test/common/services/testTextResourcePropertiesService'; import { TestLanguageConfigurationService } from 'vs/editor/test/common/modes/testLanguageConfigurationService'; <ins> import { getDocumentSemanticTokens } from 'vs/editor/common/services/getSemanticTokens'; </ins> const GENERATE_TESTS = false;

// assert that it got called twice assert.strictEqual(callCount, 2); }); <ins> test('DocumentSemanticTokens should be pick the token provider with actual items', async () => { let calledBoth = false; disposables.add(ModesRegistry.registerLanguage({ id: 'testMode2' })); disposables.add(DocumentSemanticTokensProviderRegistry.register('testMode2', new class implements DocumentSemanticTokensProvider { getLegend(): SemanticTokensLegend { return { tokenTypes: ['class1'], tokenModifiers: [] }; } async provideDocumentSemanticTokens(model: ITextModel, lastResultId: string | null, token: CancellationToken): Promise<SemanticTokens | SemanticTokensEdits | null> { // For a secondary request return a different value if (lastResultId) { return { data: new Uint32Array([2, 1, 1, 1, 1, 0, 2, 1, 1, 1]) }; } return { resultId: '1', data: new Uint32Array([0, 1, 1, 1, 1, 0, 2, 1, 1, 1]) }; } releaseDocumentSemanticTokens(resultId: string | undefined): void { } })); disposables.add(DocumentSemanticTokensProviderRegistry.register('testMode2', new class implements DocumentSemanticTokensProvider { getLegend(): SemanticTokensLegend { return { tokenTypes: ['class2'], tokenModifiers: [] }; } async provideDocumentSemanticTokens(model: ITextModel, lastResultId: string | null, token: CancellationToken): Promise<SemanticTokens | SemanticTokensEdits | null> { calledBoth = true; return null; } releaseDocumentSemanticTokens(resultId: string | undefined): void { } })); const textModel = modelService.createModel('Hello world 2', modeService.create('testMode2')); try { let request = getDocumentSemanticTokens(textModel, null, CancellationToken.None); assert.deepStrictEqual(request?.provider.getLegend(), { tokenTypes: ['class2'], tokenModifiers: [] }, `Legend does not match prior to request`); let tokens = await request?.request; // We should have tokens assert.ok(tokens, `Tokens not found from multiple providers`); assert.ok(tokens.resultId, `Token result id not found from multiple providers`); assert.deepStrictEqual([...(tokens as any).data], [0, 1, 1, 1, 1, 0, 2, 1, 1, 1], `Token data not returned for multiple providers`); assert.ok(calledBoth, `Did not actually call both token providers`); assert.deepStrictEqual(request?.provider.getLegend(), { tokenTypes: ['class1'], tokenModifiers: [] }, `Legend did not update after match`); // Make a second request. Make sure we get the secondary value request = getDocumentSemanticTokens(textModel, tokens!.resultId!, CancellationToken.None); tokens = await request?.request; assert.deepStrictEqual([...(tokens as any).data], [2, 1, 1, 1, 1, 0, 2, 1, 1, 1], `Token data not returned for second request for multiple providers`); } finally { disposables.clear(); // Wait for scheduler to finish await timeout(0); // Now dispose the text model textModel.dispose(); } }); </ins> }); function assertComputeEdits(lines1: string[], lines2: string[]): void {