Expose scope comments in IToken

src/grammar/grammar.ts

@@ -433,12 +433,12 @@ export class AttributedScopeStack {
 	}
 
 	public static createRoot(scopeName: ScopeName, tokenAttributes: EncodedTokenAttributes): AttributedScopeStack {
-		return new AttributedScopeStack(null, new ScopeStack(null, scopeName), tokenAttributes, null);
+		return new AttributedScopeStack(null, new ScopeStack(null, scopeName, null), tokenAttributes, null);
 	}
 
 	public static createRootAndLookUpScopeName(scopeName: ScopeName, tokenAttributes: EncodedTokenAttributes, grammar: Grammar): AttributedScopeStack {
 		const rawRootMetadata = grammar.getMetadataForScope(scopeName);
-		const scopePath = new ScopeStack(null, scopeName);
+		const scopePath = new ScopeStack(null, scopeName, null);
 		const rootStyle = grammar.themeProvider.themeMatch(scopePath);
 
 		const resolvedTokenAttributes = AttributedScopeStack.mergeAttributes(
@@ -531,21 +531,22 @@ export class AttributedScopeStack {
 		);
 	}
 
-	public pushAttributed(scopePath: ScopePath | null, grammar: Grammar): AttributedScopeStack {
+	public pushAttributed(scopePath: ScopePath | null, grammar: Grammar, scopeComment?: string | null): AttributedScopeStack {
 		if (scopePath === null) {
 			return this;
 		}
 
 		if (scopePath.indexOf(' ') === -1) {
 			// This is the common case and much faster
 
-			return AttributedScopeStack._pushAttributed(this, scopePath, grammar);
+			return AttributedScopeStack._pushAttributed(this, scopePath, grammar, scopeComment);
 		}
 
 		const scopes = scopePath.split(/ /g);
 		let result: AttributedScopeStack = this;
 		for (const scope of scopes) {
-			result = AttributedScopeStack._pushAttributed(result, scope, grammar);
+			// For multi-scope pushes, only the first scope gets the comment
+			result = AttributedScopeStack._pushAttributed(result, scope, grammar, result === this ? scopeComment : null);
 		}
 		return result;
 
@@ -555,10 +556,11 @@ export class AttributedScopeStack {
 		target: AttributedScopeStack,
 		scopeName: ScopeName,
 		grammar: Grammar,
+		scopeComment?: string | null
 	): AttributedScopeStack {
 		const rawMetadata = grammar.getMetadataForScope(scopeName);
 
-		const newPath = target.scopePath.push(scopeName);
+		const newPath = target.scopePath.push(scopeName, scopeComment || null);
 		const scopeThemeMatchResult =
 			grammar.themeProvider.themeMatch(newPath);
 		const metadata = AttributedScopeStack.mergeAttributes(
@@ -573,6 +575,10 @@ export class AttributedScopeStack {
 		return this.scopePath.getSegments();
 	}
 
+	public getScopeComments(): (string | null)[] {
+		return this.scopePath.getComments();
+	}
+
 	public getExtensionIfDefined(base: AttributedScopeStack | null): AttributedScopeStackFrame[] | undefined {
 		const result: AttributedScopeStackFrame[] = [];
 		let self: AttributedScopeStack | null = this;
@@ -1063,6 +1069,7 @@ export class LineTokens {
 		}
 
 		const scopes = scopesList?.getScopeNames() ?? [];
+		const scopeComments = scopesList?.getScopeComments() ?? [];
 
 		if (DebugFlags.InDebugMode) {
 			console.log('  token: |' + this._lineText!.substring(this._lastTokenEndIndex, endIndex).replace(/\n$/, '\\n') + '|');
@@ -1075,7 +1082,8 @@ export class LineTokens {
 			startIndex: this._lastTokenEndIndex,
 			endIndex: endIndex,
 			// value: lineText.substring(lastTokenEndIndex, endIndex),
-			scopes: scopes
+			scopes: scopes,
+			scopeComments: scopeComments
 		});
 
 		this._lastTokenEndIndex = endIndex;

src/grammar/tokenizeString.ts

@@ -170,9 +170,11 @@ class TokenizeStringResult {
 			const beforePush = stack;
 			// push it on the stack rule
 			const scopeName = _rule.getName(lineText.content, captureIndices);
+			const scopeComment = _rule.getComment();
 			const nameScopesList = stack.contentNameScopesList!.pushAttributed(
 				scopeName,
-				grammar
+				grammar,
+				scopeComment
 			);
 			stack = stack.push(
 				matchedRuleId,
@@ -214,7 +216,8 @@ class TokenizeStringResult {
 				);
 				const contentNameScopesList = nameScopesList.pushAttributed(
 					contentName,
-					grammar
+					grammar,
+					null  // contentName doesn't have a separate comment
 				);
 				stack = stack.withContentNameScopesList(contentNameScopesList);
 
@@ -263,7 +266,8 @@ class TokenizeStringResult {
 				);
 				const contentNameScopesList = nameScopesList.pushAttributed(
 					contentName,
-					grammar
+					grammar,
+					null  // contentName doesn't have a separate comment
 				);
 				stack = stack.withContentNameScopesList(contentNameScopesList);
 
@@ -626,9 +630,10 @@ function handleCaptures(grammar: Grammar, lineText: OnigString, isFirstLine: boo
 		if (captureRule.retokenizeCapturedWithRuleId) {
 			// the capture requires additional matching
 			const scopeName = captureRule.getName(lineTextContent, captureIndices);
-			const nameScopesList = stack.contentNameScopesList!.pushAttributed(scopeName, grammar);
+			const scopeComment = captureRule.getComment();
+			const nameScopesList = stack.contentNameScopesList!.pushAttributed(scopeName, grammar, scopeComment);
 			const contentName = captureRule.getContentName(lineTextContent, captureIndices);
-			const contentNameScopesList = nameScopesList.pushAttributed(contentName, grammar);
+			const contentNameScopesList = nameScopesList.pushAttributed(contentName, grammar, null);
 
 			const stackClone = stack.push(captureRule.retokenizeCapturedWithRuleId, captureIndex.start, -1, false, null, nameScopesList, contentNameScopesList);
 			const onigSubStr = grammar.createOnigString(lineTextContent.substring(0, captureIndex.end));
@@ -641,7 +646,8 @@ function handleCaptures(grammar: Grammar, lineText: OnigString, isFirstLine: boo
 		if (captureRuleScopeName !== null) {
 			// push
 			const base = localStack.length > 0 ? localStack[localStack.length - 1].scopes : stack.contentNameScopesList;
-			const captureRuleScopesList = base!.pushAttributed(captureRuleScopeName, grammar);
+			const captureRuleScopeComment = captureRule.getComment();
+			const captureRuleScopesList = base!.pushAttributed(captureRuleScopeName, grammar, captureRuleScopeComment);
 			localStack.push(new LocalStackElement(captureRuleScopesList, captureIndex.end));
 		}
 	}

src/main.ts

@@ -285,6 +285,7 @@ export interface IToken {
 	startIndex: number;
 	readonly endIndex: number;
 	readonly scopes: string[];
+	readonly scopeComments: (string | null)[];
 }
 
 /**

src/rawGrammar.ts

@@ -43,6 +43,7 @@ export interface IRawRule extends ILocatable {
 
 	readonly name?: ScopeName;
 	readonly contentName?: ScopeName;
+	readonly comment?: string;
 
 	readonly match?: RegExpString;
 	readonly captures?: IRawCaptures;

src/rule.ts

@@ -51,13 +51,16 @@ export abstract class Rule {
 	private readonly _contentNameIsCapturing: boolean;
 	private readonly _contentName: string | null;
 
-	constructor($location: ILocation | undefined, id: RuleId, name: string | null | undefined, contentName: string | null | undefined) {
+	private readonly _comment: string | null;
+
+	constructor($location: ILocation | undefined, id: RuleId, name: string | null | undefined, contentName: string | null | undefined, comment: string | null | undefined) {
 		this.$location = $location;
 		this.id = id;
 		this._name = name || null;
 		this._nameIsCapturing = RegexSource.hasCaptures(this._name);
 		this._contentName = contentName || null;
 		this._contentNameIsCapturing = RegexSource.hasCaptures(this._contentName);
+		this._comment = comment || null;
 	}
 
 	public abstract dispose(): void;
@@ -81,6 +84,10 @@ export abstract class Rule {
 		return RegexSource.replaceCaptures(this._contentName, lineText, captureIndices);
 	}
 
+	public getComment(): string | null {
+		return this._comment;
+	}
+
 	public abstract collectPatterns(grammar: IRuleRegistry, out: RegExpSourceList): void;
 
 	public abstract compile(grammar: IRuleRegistry & IOnigLib, endRegexSource: string | null): CompiledRule;
@@ -97,8 +104,8 @@ export class CaptureRule extends Rule {
 
 	public readonly retokenizeCapturedWithRuleId: RuleId | 0;
 
-	constructor($location: ILocation | undefined, id: RuleId, name: string | null | undefined, contentName: string | null | undefined, retokenizeCapturedWithRuleId: RuleId | 0) {
-		super($location, id, name, contentName);
+	constructor($location: ILocation | undefined, id: RuleId, name: string | null | undefined, contentName: string | null | undefined, comment: string | null | undefined, retokenizeCapturedWithRuleId: RuleId | 0) {
+		super($location, id, name, contentName, comment);
 		this.retokenizeCapturedWithRuleId = retokenizeCapturedWithRuleId;
 	}
 
@@ -124,8 +131,8 @@ export class MatchRule extends Rule {
 	public readonly captures: (CaptureRule | null)[];
 	private _cachedCompiledPatterns: RegExpSourceList | null;
 
-	constructor($location: ILocation | undefined, id: RuleId, name: string | undefined, match: string, captures: (CaptureRule | null)[]) {
-		super($location, id, name, null);
+	constructor($location: ILocation | undefined, id: RuleId, name: string | undefined, comment: string | undefined, match: string, captures: (CaptureRule | null)[]) {
+		super($location, id, name, null, comment);
 		this._match = new RegExpSource(match, this.id);
 		this.captures = captures;
 		this._cachedCompiledPatterns = null;
@@ -168,8 +175,8 @@ export class IncludeOnlyRule extends Rule {
 	public readonly patterns: RuleId[];
 	private _cachedCompiledPatterns: RegExpSourceList | null;
 
-	constructor($location: ILocation | undefined, id: RuleId, name: string | null | undefined, contentName: string | null | undefined, patterns: ICompilePatternsResult) {
-		super($location, id, name, contentName);
+	constructor($location: ILocation | undefined, id: RuleId, name: string | null | undefined, contentName: string | null | undefined, comment: string | null | undefined, patterns: ICompilePatternsResult) {
+		super($location, id, name, contentName, comment);
 		this.patterns = patterns.patterns;
 		this.hasMissingPatterns = patterns.hasMissingPatterns;
 		this._cachedCompiledPatterns = null;
@@ -217,8 +224,8 @@ export class BeginEndRule extends Rule {
 	public readonly patterns: RuleId[];
 	private _cachedCompiledPatterns: RegExpSourceList | null;
 
-	constructor($location: ILocation | undefined, id: RuleId, name: string | null | undefined, contentName: string | null | undefined, begin: string, beginCaptures: (CaptureRule | null)[], end: string | undefined, endCaptures: (CaptureRule | null)[], applyEndPatternLast: boolean | undefined, patterns: ICompilePatternsResult) {
-		super($location, id, name, contentName);
+	constructor($location: ILocation | undefined, id: RuleId, name: string | null | undefined, contentName: string | null | undefined, comment: string | null | undefined, begin: string, beginCaptures: (CaptureRule | null)[], end: string | undefined, endCaptures: (CaptureRule | null)[], applyEndPatternLast: boolean | undefined, patterns: ICompilePatternsResult) {
+		super($location, id, name, contentName, comment);
 		this._begin = new RegExpSource(begin, this.id);
 		this.beginCaptures = beginCaptures;
 		this._end = new RegExpSource(end ? end : '\uFFFF', -1);
@@ -298,8 +305,8 @@ export class BeginWhileRule extends Rule {
 	private _cachedCompiledPatterns: RegExpSourceList | null;
 	private _cachedCompiledWhilePatterns: RegExpSourceList<RuleId | typeof whileRuleId> | null;
 
-	constructor($location: ILocation | undefined, id: RuleId, name: string | null | undefined, contentName: string | null | undefined, begin: string, beginCaptures: (CaptureRule | null)[], _while: string, whileCaptures: (CaptureRule | null)[], patterns: ICompilePatternsResult) {
-		super($location, id, name, contentName);
+	constructor($location: ILocation | undefined, id: RuleId, name: string | null | undefined, contentName: string | null | undefined, comment: string | null | undefined, begin: string, beginCaptures: (CaptureRule | null)[], _while: string, whileCaptures: (CaptureRule | null)[], patterns: ICompilePatternsResult) {
+		super($location, id, name, contentName, comment);
 		this._begin = new RegExpSource(begin, this.id);
 		this.beginCaptures = beginCaptures;
 		this.whileCaptures = whileCaptures;
@@ -380,9 +387,9 @@ export class BeginWhileRule extends Rule {
 
 export class RuleFactory {
 
-	public static createCaptureRule(helper: IRuleFactoryHelper, $location: ILocation | undefined, name: string | null | undefined, contentName: string | null | undefined, retokenizeCapturedWithRuleId: RuleId | 0): CaptureRule {
+	public static createCaptureRule(helper: IRuleFactoryHelper, $location: ILocation | undefined, name: string | null | undefined, contentName: string | null | undefined, comment: string | null | undefined, retokenizeCapturedWithRuleId: RuleId | 0): CaptureRule {
 		return helper.registerRule((id) => {
-			return new CaptureRule($location, id, name, contentName, retokenizeCapturedWithRuleId);
+			return new CaptureRule($location, id, name, contentName, comment, retokenizeCapturedWithRuleId);
 		});
 	}
 
@@ -396,6 +403,7 @@ export class RuleFactory {
 						desc.$vscodeTextmateLocation,
 						desc.id,
 						desc.name,
+						desc.comment,
 						desc.match,
 						RuleFactory._compileCaptures(desc.captures, helper, repository)
 					);
@@ -414,6 +422,7 @@ export class RuleFactory {
 						desc.id,
 						desc.name,
 						desc.contentName,
+						desc.comment,
 						RuleFactory._compilePatterns(patterns, helper, repository)
 					);
 				}
@@ -424,6 +433,7 @@ export class RuleFactory {
 						desc.id,
 						desc.name,
 						desc.contentName,
+						desc.comment,
 						desc.begin, RuleFactory._compileCaptures(desc.beginCaptures || desc.captures, helper, repository),
 						desc.while, RuleFactory._compileCaptures(desc.whileCaptures || desc.captures, helper, repository),
 						RuleFactory._compilePatterns(desc.patterns, helper, repository)
@@ -435,6 +445,7 @@ export class RuleFactory {
 					desc.id,
 					desc.name,
 					desc.contentName,
+					desc.comment,
 					desc.begin, RuleFactory._compileCaptures(desc.beginCaptures || desc.captures, helper, repository),
 					desc.end, RuleFactory._compileCaptures(desc.endCaptures || desc.captures, helper, repository),
 					desc.applyEndPatternLast,
@@ -477,7 +488,7 @@ export class RuleFactory {
 				if (captures[captureId].patterns) {
 					retokenizeCapturedWithRuleId = RuleFactory.getCompiledRuleId(captures[captureId], helper, repository);
 				}
-				r[numericCaptureId] = RuleFactory.createCaptureRule(helper, captures[captureId].$vscodeTextmateLocation, captures[captureId].name, captures[captureId].contentName, retokenizeCapturedWithRuleId);
+				r[numericCaptureId] = RuleFactory.createCaptureRule(helper, captures[captureId].$vscodeTextmateLocation, captures[captureId].name, captures[captureId].contentName, captures[captureId].comment, retokenizeCapturedWithRuleId);
 			}
 		}
 

src/tests/all.test.ts

@@ -5,5 +5,6 @@
 import './grammar.test';
 import './json.test';
 import './matcher.test';
+import './scopeComments.test';
 import './themes.test';
 import './tokenization.test';

src/tests/scopeComments.test.ts

@@ -0,0 +1,64 @@
+/*---------------------------------------------------------
+ * Copyright (C) Microsoft Corporation. All rights reserved.
+ *--------------------------------------------------------*/
+
+import * as path from 'path';
+import * as assert from 'assert';
+import { Registry, IGrammar, RegistryOptions, parseRawGrammar } from '../main';
+import { getOniguruma } from './onigLibs';
+import * as fs from 'fs';
+
+const REPO_ROOT = path.join(__dirname, '../../');
+
+suite('Scope Comments', () => {
+	test('should expose scope comments in tokens', async () => {
+		const grammarPath = path.join(REPO_ROOT, 'test-cases/scope-comments/test.grammar.json');
+		const grammarContent = fs.readFileSync(grammarPath).toString();
+		const rawGrammar = parseRawGrammar(grammarContent, grammarPath);
+
+		const options: RegistryOptions = {
+			onigLib: getOniguruma(),
+			loadGrammar: () => Promise.resolve(rawGrammar)
+		};
+
+		const registry = new Registry(options);
+		const grammar: IGrammar | null = await registry.loadGrammar('source.test');
+
+		assert.ok(grammar, 'Grammar should be loaded');
+
+		// Test line with comment
+		const result1 = grammar.tokenizeLine('hello world test', null);
+
+		// Filter out whitespace tokens for easier testing
+		const nonWhitespaceTokens = result1.tokens.filter(t => {
+			const text = 'hello world test'.substring(t.startIndex, t.endIndex);
+			return text.trim().length > 0;
+		});
+
+		assert.strictEqual(nonWhitespaceTokens.length, 3, 'Should have 3 non-whitespace tokens');
+
+		// First token: "hello" - should have comment
+		assert.strictEqual(nonWhitespaceTokens[0].scopes.length, 2);
+		assert.strictEqual(nonWhitespaceTokens[0].scopes[0], 'source.test');
+		assert.strictEqual(nonWhitespaceTokens[0].scopes[1], 'keyword.test');
+		assert.strictEqual(nonWhitespaceTokens[0].scopeComments.length, 2);
+		assert.strictEqual(nonWhitespaceTokens[0].scopeComments[0], null); // root scope has no comment
+		assert.strictEqual(nonWhitespaceTokens[0].scopeComments[1], 'Matches the hello keyword');
+
+		// Second token: "world" - should have comment
+		assert.strictEqual(nonWhitespaceTokens[1].scopes.length, 2);
+		assert.strictEqual(nonWhitespaceTokens[1].scopes[0], 'source.test');
+		assert.strictEqual(nonWhitespaceTokens[1].scopes[1], 'string.test');
+		assert.strictEqual(nonWhitespaceTokens[1].scopeComments.length, 2);
+		assert.strictEqual(nonWhitespaceTokens[1].scopeComments[0], null); // root scope has no comment
+		assert.strictEqual(nonWhitespaceTokens[1].scopeComments[1], 'Matches the world keyword');
+
+		// Third token: "test" - should have no comment (null)
+		assert.strictEqual(nonWhitespaceTokens[2].scopes.length, 2);
+		assert.strictEqual(nonWhitespaceTokens[2].scopes[0], 'source.test');
+		assert.strictEqual(nonWhitespaceTokens[2].scopes[1], 'variable.test');
+		assert.strictEqual(nonWhitespaceTokens[2].scopeComments.length, 2);
+		assert.strictEqual(nonWhitespaceTokens[2].scopeComments[0], null); // root scope has no comment
+		assert.strictEqual(nonWhitespaceTokens[2].scopeComments[1], null); // this scope has no comment
+	});
+});