diff --git a/src/astUtils/creators.ts b/src/astUtils/creators.ts index 97906e2bc..a08f81a19 100644 --- a/src/astUtils/creators.ts +++ b/src/astUtils/creators.ts @@ -92,7 +92,8 @@ export function createToken(kind: T, text?: string, range?: text: text ?? tokenDefaults[kind as string] ?? kind.toString().toLowerCase(), isReserved: !text || text === kind.toString(), range: range, - leadingWhitespace: '' + leadingWhitespace: '', + leadingTrivia: [] }; } @@ -102,7 +103,8 @@ export function createIdentifier(name: string, range?: Range): Identifier { text: name, isReserved: false, range: range, - leadingWhitespace: '' + leadingWhitespace: '', + leadingTrivia: [] }; } @@ -172,7 +174,8 @@ export function createCall(callee: Expression, args?: Expression[]) { callee, createToken(TokenKind.LeftParen, '('), createToken(TokenKind.RightParen, ')'), - args || [] + args || [], + [] ); } diff --git a/src/astUtils/reflection.spec.ts b/src/astUtils/reflection.spec.ts index c47b849e1..b2819b37a 100644 --- a/src/astUtils/reflection.spec.ts +++ b/src/astUtils/reflection.spec.ts @@ -47,8 +47,8 @@ describe('reflection', () => { const fors = new ForStatement(token, assignment, token, expr, block, token, token, expr); const foreach = new ForEachStatement({ forEach: token, in: token, endFor: token }, token, expr, block); const whiles = new WhileStatement({ while: token, endWhile: token }, expr, block); - const dottedSet = new DottedSetStatement(expr, ident, expr); - const indexedSet = new IndexedSetStatement(expr, expr, expr, token, token); + const dottedSet = new DottedSetStatement(expr, ident, expr, token, token); + const indexedSet = new IndexedSetStatement(expr, expr, expr, token, token, [], token); const library = new LibraryStatement({ library: token, filePath: token }); const namespace = new NamespaceStatement(token, new NamespacedVariableNameExpression(createVariableExpression('a')), body, token); const cls = new ClassStatement(token, ident, [], token); @@ -193,11 +193,12 @@ describe('reflection', () => { range: undefined, isReserved: false, charCode: 0, - leadingWhitespace: '' + leadingWhitespace: '', + leadingTrivia: [] }; const nsVar = new NamespacedVariableNameExpression(createVariableExpression('a')); const binary = new BinaryExpression(expr, token, expr); - const call = new CallExpression(expr, token, token, []); + const call = new CallExpression(expr, token, token, [], []); const fun = new FunctionExpression([], block, token, token, token, token); const dottedGet = new DottedGetExpression(expr, ident, token); const xmlAttrGet = new XmlAttributeGetExpression(expr, ident, token); @@ -205,16 +206,16 @@ describe('reflection', () => { const grouping = new GroupingExpression({ left: token, right: token }, expr); const literal = createStringLiteral('test'); const escapedCarCode = new EscapedCharCodeLiteralExpression(charCode); - const arrayLit = new ArrayLiteralExpression([], token, token); + const arrayLit = new ArrayLiteralExpression([], token, token, []); const aaLit = new AALiteralExpression([], token, token); const unary = new UnaryExpression(token, expr); const variable = new VariableExpression(ident); const sourceLit = new SourceLiteralExpression(token); const newx = new NewExpression(token, call); - const callfunc = new CallfuncExpression(expr, token, ident, token, [], token); + const callfunc = new CallfuncExpression(expr, token, ident, token, [], token, []); const tplQuasi = new TemplateStringQuasiExpression([expr]); - const tplString = new TemplateStringExpression(token, [tplQuasi], [], token); - const taggedTpl = new TaggedTemplateStringExpression(ident, token, [tplQuasi], [], token); + const tplString = new TemplateStringExpression(token, [tplQuasi], [], token, [], []); + const taggedTpl = new TaggedTemplateStringExpression(ident, token, [tplQuasi], [], token, [], []); const annotation = new AnnotationExpression(token, token); it('isExpression', () => { diff --git a/src/astUtils/reflection.ts b/src/astUtils/reflection.ts index 8496c1e8f..1575b261a 100644 --- a/src/astUtils/reflection.ts +++ b/src/astUtils/reflection.ts @@ -20,6 +20,7 @@ import { DynamicType } from '../types/DynamicType'; import type { InterfaceType } from '../types/InterfaceType'; import type { ObjectType } from '../types/ObjectType'; import type { AstNode, Expression, Statement } from '../parser/AstNode'; +import type { Token } from '../lexer/Token'; // File reflection @@ -198,9 +199,9 @@ export function isAliasStatement(element: AstNode | undefined): element is Alias * this will work for StringLiteralExpression -> Expression, * but will not work CustomStringLiteralExpression -> StringLiteralExpression -> Expression */ -export function isExpression(element: AstNode | undefined): element is Expression { +export function isExpression(element: AstNode | Token | undefined): element is Expression { // eslint-disable-next-line no-bitwise - return !!(element && element.visitMode & InternalWalkMode.visitExpressions); + return !!(element && (element as any).visitMode & InternalWalkMode.visitExpressions); } export function isBinaryExpression(element: AstNode | undefined): element is BinaryExpression { diff --git a/src/lexer/Lexer.spec.ts b/src/lexer/Lexer.spec.ts index 6dfaf2c3f..b9d9c4783 100644 --- a/src/lexer/Lexer.spec.ts +++ b/src/lexer/Lexer.spec.ts @@ -1,8 +1,8 @@ /* eslint no-template-curly-in-string: 0 */ import { expect } from '../chai-config.spec'; - import { TokenKind } from './TokenKind'; -import { Lexer } from './Lexer'; +import { Lexer, triviaKinds } from './Lexer'; +import type { Token } from './Token'; import { isToken } from './Token'; import { rangeToArray } from '../parser/Parser.spec'; import { Range } from 'vscode-languageserver'; @@ -1452,6 +1452,63 @@ describe('lexer', () => { TokenKind.Eof ]); }); + + describe('trivia', () => { + function stringify(tokens: Token[]) { + return tokens + //exclude the explicit triva tokens since they'll be included in the leading/trailing arrays + .filter(x => !triviaKinds.includes(x.kind)) + .flatMap(x => [...x.leadingTrivia, x]) + .map(x => x.text) + .join(''); + } + + it('combining token text and trivia can reproduce full input', () => { + const input = ` + function test( ) + 'comment + print alpha ' blabla + end function 'trailing + 'trailing2 + `; + expect( + stringify( + Lexer.scan(input).tokens + ) + ).to.eql(input); + }); + + function expectTrivia(text: string, expected: Array<{ text: string; leadingTrivia?: string[]; trailingTrivia?: string[] }>) { + const tokens = Lexer.scan(text).tokens.filter(x => !triviaKinds.includes(x.kind)); + expect( + tokens.map(x => { + return { + text: x.text, + leadingTrivia: x.leadingTrivia.map(x => x.text) + }; + }) + ).to.eql( + expected.map(x => ({ + leadingTrivia: [], + ...x + })) + ); + } + + it('associates trailing items on same line with the preceeding token', () => { + expectTrivia( + `'leading\n` + + `alpha = true 'trueComment\n` + + `'eof` + , [ + { leadingTrivia: [`'leading`, `\n`], text: `alpha` }, + { leadingTrivia: [` `], text: `=` }, + { leadingTrivia: [` `], text: `true` }, + //EOF + { leadingTrivia: [` `, `'trueComment`, `\n`, `'eof`], text: `` } + ]); + }); + }); }); function expectKinds(text: string, tokenKinds: TokenKind[]) { diff --git a/src/lexer/Lexer.ts b/src/lexer/Lexer.ts index 6f45df185..bb9845a81 100644 --- a/src/lexer/Lexer.ts +++ b/src/lexer/Lexer.ts @@ -6,6 +6,13 @@ import type { Range, Diagnostic } from 'vscode-languageserver'; import { DiagnosticMessages } from '../DiagnosticMessages'; import util from '../util'; +export const triviaKinds: ReadonlyArray = [ + TokenKind.Newline, + TokenKind.Whitespace, + TokenKind.Comment, + TokenKind.Colon +]; + /** * Numeric type designators can only be one of these characters */ @@ -110,12 +117,22 @@ export class Lexer { range: this.options.trackLocations ? util.createRange(this.lineBegin, this.columnBegin, this.lineEnd, this.columnEnd + 1) : undefined, - leadingWhitespace: this.leadingWhitespace + leadingWhitespace: this.leadingWhitespace, + leadingTrivia: this.leadingTrivia }); this.leadingWhitespace = ''; return this; } + private leadingTrivia: Token[] = []; + + /** + * Pushes a token into the leadingTrivia list + */ + private pushTrivia(token: Token) { + this.leadingTrivia.push(token); + } + /** * Fill in missing/invalid options with defaults */ @@ -1050,6 +1067,13 @@ export class Lexer { return false; } + /** + * Determine if this token is a trivia token + */ + private isTrivia(token: Token) { + return triviaKinds.includes(token.kind); + } + /** * Creates a `Token` and adds it to the `tokens` array. * @param kind the type of token to produce. @@ -1061,8 +1085,15 @@ export class Lexer { text: text, isReserved: ReservedWords.has(text.toLowerCase()), range: this.rangeOf(), - leadingWhitespace: this.leadingWhitespace + leadingWhitespace: this.leadingWhitespace, + leadingTrivia: [] }; + if (this.isTrivia(token)) { + this.pushTrivia(token); + } else { + token.leadingTrivia.push(...this.leadingTrivia); + this.leadingTrivia = []; + } this.leadingWhitespace = ''; this.tokens.push(token); this.sync(); diff --git a/src/lexer/Token.ts b/src/lexer/Token.ts index 94647d1e0..921f1457d 100644 --- a/src/lexer/Token.ts +++ b/src/lexer/Token.ts @@ -17,6 +17,10 @@ export interface Token { * Any leading whitespace found prior to this token. Excludes newline characters. */ leadingWhitespace?: string; + /** + * Any tokens starting on the next line of the previous token, up to the start of this token + */ + leadingTrivia: Token[]; } /** diff --git a/src/parser/AstNode.spec.ts b/src/parser/AstNode.spec.ts index 9b1ecb13a..6a25af4f7 100644 --- a/src/parser/AstNode.spec.ts +++ b/src/parser/AstNode.spec.ts @@ -6,10 +6,10 @@ import { expect } from '../chai-config.spec'; import type { AALiteralExpression, AAMemberExpression, ArrayLiteralExpression, BinaryExpression, CallExpression, CallfuncExpression, DottedGetExpression, FunctionExpression, GroupingExpression, IndexedGetExpression, NewExpression, NullCoalescingExpression, TaggedTemplateStringExpression, TemplateStringExpression, TemplateStringQuasiExpression, TernaryExpression, TypeCastExpression, UnaryExpression, XmlAttributeGetExpression } from './Expression'; import { expectZeroDiagnostics } from '../testHelpers.spec'; import { tempDir, rootDir, stagingDir } from '../testHelpers.spec'; +import { ParseMode, Parser } from './Parser'; import { isAALiteralExpression, isAAMemberExpression, isAnnotationExpression, isArrayLiteralExpression, isAssignmentStatement, isBinaryExpression, isBlock, isCallExpression, isCallfuncExpression, isCatchStatement, isClassStatement, isCommentStatement, isConstStatement, isDimStatement, isDottedGetExpression, isDottedSetStatement, isEnumMemberStatement, isEnumStatement, isExpressionStatement, isForEachStatement, isForStatement, isFunctionExpression, isFunctionStatement, isGroupingExpression, isIfStatement, isIncrementStatement, isIndexedGetExpression, isIndexedSetStatement, isInterfaceFieldStatement, isInterfaceMethodStatement, isInterfaceStatement, isLibraryStatement, isMethodStatement, isNamespaceStatement, isNewExpression, isNullCoalescingExpression, isPrintStatement, isReturnStatement, isTaggedTemplateStringExpression, isTemplateStringExpression, isTemplateStringQuasiExpression, isTernaryExpression, isThrowStatement, isTryCatchStatement, isTypeCastExpression, isUnaryExpression, isWhileStatement, isXmlAttributeGetExpression } from '../astUtils/reflection'; import type { ClassStatement, FunctionStatement, InterfaceFieldStatement, InterfaceMethodStatement, MethodStatement, InterfaceStatement, CatchStatement, ThrowStatement, EnumStatement, EnumMemberStatement, ConstStatement, Block, CommentStatement, PrintStatement, DimStatement, ForStatement, WhileStatement, IndexedSetStatement, LibraryStatement, NamespaceStatement, TryCatchStatement, DottedSetStatement } from './Statement'; import { AssignmentStatement, EmptyStatement } from './Statement'; -import { ParseMode, Parser } from './Parser'; import type { AstNode } from './AstNode'; type DeepWriteable = { -readonly [P in keyof T]: DeepWriteable }; @@ -1670,4 +1670,45 @@ describe('AstNode', () => { testClone(original); }); }); + + describe('toString', () => { + function testToString(text: string) { + expect( + Parser.parse(text).ast.toString() + ).to.eql( + text + ); + } + it('retains full fidelity', () => { + testToString(` + thing = true + + if true + thing = true + end if + + if true + thing = true + else + thing = true + end if + + if true + thing = true + else if true + thing = true + else + thing = true + end if + + for i = 0 to 10 step 1 + print true,false;3 + end for + + for each item in thing + print 1 + end for + `); + }); + }); }); diff --git a/src/parser/AstNode.ts b/src/parser/AstNode.ts index 17c63731e..ddff4513d 100644 --- a/src/parser/AstNode.ts +++ b/src/parser/AstNode.ts @@ -8,6 +8,8 @@ import type { BrsTranspileState } from './BrsTranspileState'; import type { TranspileResult } from '../interfaces'; import type { AnnotationExpression } from './Expression'; import util from '../util'; +import type { SourceNode } from 'source-map'; +import { TranspileState } from './TranspileState'; /** * A BrightScript AST node @@ -129,6 +131,20 @@ export abstract class AstNode { }); } + /** + * Return the string value of this AstNode + */ + public toString() { + return this + .toSourceNode(new TranspileState('', {})) + .toString(); + } + + /** + * Generate a SourceNode that represents the stringified value of this node (used to generate sourcemaps and transpile the code + */ + public abstract toSourceNode(state: TranspileState): SourceNode; + /** * Clone this node and all of its children. This creates a completely detached and identical copy of the AST. * All tokens, statements, expressions, range, and location are cloned. diff --git a/src/parser/Expression.ts b/src/parser/Expression.ts index 4388989cf..4f39af16b 100644 --- a/src/parser/Expression.ts +++ b/src/parser/Expression.ts @@ -19,6 +19,7 @@ import { FunctionType } from '../types/FunctionType'; import type { AstNode } from './AstNode'; import { Expression } from './AstNode'; import { SymbolTable } from '../SymbolTable'; +import type { TranspileState } from './TranspileState'; import { SourceNode } from 'source-map'; export type ExpressionVisitor = (expression: Expression, parent: Expression) => void; @@ -45,6 +46,14 @@ export class BinaryExpression extends Expression { ]; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + this.left?.toSourceNode(state), + state.tokenToSourceNodeWithTrivia(this.operator), + this.right?.toSourceNode(state) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { if (options.walkMode & InternalWalkMode.walkExpressions) { walk(this, 'left', visitor, options); @@ -81,7 +90,7 @@ export class CallExpression extends Expression { readonly openingParen: Token, readonly closingParen: Token, readonly args: Expression[], - unused?: any + readonly argCommas: Token[] ) { super(); this.range = util.createBoundingRange(this.callee, this.openingParen, ...args ?? [], this.closingParen); @@ -126,6 +135,18 @@ export class CallExpression extends Expression { return result; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + this.callee?.toSourceNode(state), + state.tokenToSourceNodeWithTrivia(this.openingParen), + ...this.args.map((x, i) => ([ + x.toSourceNode(state), + state.tokenToSourceNodeWithTrivia(this.argCommas[i]) + ])).flat(), + state.tokenToSourceNodeWithTrivia(this.closingParen) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { if (options.walkMode & InternalWalkMode.walkExpressions) { walk(this, 'callee', visitor, options); @@ -139,7 +160,8 @@ export class CallExpression extends Expression { this.callee?.clone(), util.cloneToken(this.openingParen), util.cloneToken(this.closingParen), - this.args?.map(e => e?.clone()) + this.args?.map(e => e?.clone()), + this.argCommas?.map(x => util.cloneToken(x)) ), ['callee', 'args'] ); @@ -155,7 +177,8 @@ export class FunctionExpression extends Expression implements TypedefProvider { readonly leftParen: Token, readonly rightParen: Token, readonly asToken?: Token, - readonly returnTypeToken?: Token + readonly returnTypeToken?: Token, + readonly paramCommas?: Token[] ) { super(); this.setReturnType(); // set the initial return type that we parse @@ -293,7 +316,25 @@ export class FunctionExpression extends Expression implements TypedefProvider { return results; } - getTypedef(state: BrsTranspileState) { + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.tokenToSourceNodeWithTrivia(this.functionType), + //include the name (if we have a parent FunctionStatement) + isFunctionStatement(this.parent) ? state.tokenToSourceNodeWithTrivia(this.parent.name) : undefined, + state.tokenToSourceNodeWithTrivia(this.leftParen), + ...this.parameters?.map((x, i) => ([ + x.toSourceNode(state), + state.tokenToSourceNodeWithTrivia(this.paramCommas?.[i]) + ])).flat() ?? [], + state.tokenToSourceNodeWithTrivia(this.rightParen), + state.tokenToSourceNodeWithTrivia(this.asToken), + state.tokenToSourceNodeWithTrivia(this.returnTypeToken), + this.body?.toSourceNode(state), + state.tokenToSourceNodeWithTrivia(this.end) + ); + } + + getTypedef(state: BrsTranspileState, name?: Identifier) { let results = [ new SourceNode(1, 0, null, [ //'function'|'sub' @@ -394,7 +435,8 @@ export class FunctionExpression extends Expression implements TypedefProvider { util.cloneToken(this.leftParen), util.cloneToken(this.rightParen), util.cloneToken(this.asToken), - util.cloneToken(this.returnTypeToken) + util.cloneToken(this.returnTypeToken), + this.paramCommas?.map(x => util.cloneToken(x)) ), ['body'] ); @@ -414,7 +456,8 @@ export class FunctionParameterExpression extends Expression { public name: Identifier, public typeToken?: Token, public defaultValue?: Expression, - public asToken?: Token + public asToken?: Token, + readonly equalsToken?: Token ) { super(); if (typeToken) { @@ -474,6 +517,16 @@ export class FunctionParameterExpression extends Expression { return results; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.tokenToSourceNodeWithTrivia(this.name), + state.tokenToSourceNodeWithTrivia(this.equalsToken), + this.defaultValue?.toSourceNode(state), + state.tokenToSourceNodeWithTrivia(this.asToken), + state.tokenToSourceNodeWithTrivia(this.typeToken) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { // eslint-disable-next-line no-bitwise if (this.defaultValue && options.walkMode & InternalWalkMode.walkExpressions) { @@ -487,7 +540,8 @@ export class FunctionParameterExpression extends Expression { util.cloneToken(this.name), util.cloneToken(this.typeToken), this.defaultValue?.clone(), - util.cloneToken(this.asToken) + util.cloneToken(this.asToken), + util.cloneToken(this.equalsToken) ), ['defaultValue'] ); @@ -510,6 +564,10 @@ export class NamespacedVariableNameExpression extends Expression { ]; } + public toSourceNode(state: TranspileState): SourceNode { + return this.expression?.toSourceNode(state); + } + public getNameParts() { let parts = [] as string[]; if (isVariableExpression(this.expression)) { @@ -588,6 +646,14 @@ export class DottedGetExpression extends Expression { ]; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + this.obj?.toSourceNode(state), + state.tokenToSourceNodeWithTrivia(this.dot), + state.tokenToSourceNodeWithTrivia(this.name) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { if (options.walkMode & InternalWalkMode.walkExpressions) { walk(this, 'obj', visitor, options); @@ -629,6 +695,14 @@ export class XmlAttributeGetExpression extends Expression { ]; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + this.obj?.toSourceNode(state), + state.tokenToSourceNodeWithTrivia(this.at), + state.tokenToSourceNodeWithTrivia(this.name) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { if (options.walkMode & InternalWalkMode.walkExpressions) { walk(this, 'obj', visitor, options); @@ -693,6 +767,16 @@ export class IndexedGetExpression extends Expression { return result; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + this.obj?.toSourceNode(state), + state.tokenToSourceNodeWithTrivia(this.questionDotToken), + state.tokenToSourceNodeWithTrivia(this.openingSquare), + this.index?.toSourceNode(state), + state.tokenToSourceNodeWithTrivia(this.closingSquare) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { if (options.walkMode & InternalWalkMode.walkExpressions) { walk(this, 'obj', visitor, options); @@ -741,6 +825,14 @@ export class GroupingExpression extends Expression { ]; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.tokenToSourceNodeWithTrivia(this.tokens.left), + this.expression?.toSourceNode(state), + state.tokenToSourceNodeWithTrivia(this.tokens.right) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { if (options.walkMode & InternalWalkMode.walkExpressions) { walk(this, 'expression', visitor, options); @@ -799,6 +891,12 @@ export class LiteralExpression extends Expression { ]; } + public toSourceNode(state: TranspileState): SourceNode { + return new SourceNode(0, 0, null, [ + state.tokenToSourceNodeWithTrivia(this.token) + ]); + } + walk(visitor: WalkVisitor, options: WalkOptions) { //nothing to walk } @@ -831,6 +929,10 @@ export class EscapedCharCodeLiteralExpression extends Expression { ]; } + public toSourceNode(state: TranspileState): SourceNode { + return state.tokenToSourceNodeWithTrivia(this.token); + } + walk(visitor: WalkVisitor, options: WalkOptions) { //nothing to walk } @@ -849,10 +951,19 @@ export class ArrayLiteralExpression extends Expression { readonly elements: Array, readonly open: Token, readonly close: Token, - readonly hasSpread = false + /** + * An array of commas used to separate elements. + * Since commas are optional, but we need indexes to be aligned, + * there should be an entry in here for every element, even if `undefined` + */ + readonly commas: Array ) { super(); this.range = util.createBoundingRange(this.open, ...this.elements ?? [], this.close); + //this param used to be `hasSpread as bool`. guard against non-arrays to avoid old code breaking new code + if (!Array.isArray(this.commas)) { + this.commas = undefined; + } } public readonly range: Range | undefined; @@ -905,6 +1016,17 @@ export class ArrayLiteralExpression extends Expression { return result; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.tokenToSourceNodeWithTrivia(this.open), + ...this.elements?.map((x, i) => ([ + x?.toSourceNode(state), + state.tokenToSourceNodeWithTrivia(this.commas[i]) + ])).flat() ?? [], + state.tokenToSourceNodeWithTrivia(this.close) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { if (options.walkMode & InternalWalkMode.walkExpressions) { walkArray(this.elements, visitor, options, this); @@ -917,7 +1039,7 @@ export class ArrayLiteralExpression extends Expression { this.elements?.map(e => e?.clone()), util.cloneToken(this.open), util.cloneToken(this.close), - this.hasSpread + this.commas?.map(x => util.cloneToken(x)) ), ['elements'] ); @@ -943,6 +1065,15 @@ export class AAMemberExpression extends Expression { return []; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.tokenToSourceNodeWithTrivia(this.keyToken), + //colon tokens are technically trivia, so we don't need to include that here + this.value?.toSourceNode(state), + state.tokenToSourceNodeWithTrivia(this.commaToken) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { walk(this, 'value', visitor, options); } @@ -1042,6 +1173,14 @@ export class AALiteralExpression extends Expression { return result; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.tokenToSourceNodeWithTrivia(this.open), + state.arrayToSourceNodeWithTrivia(this.elements, isCommentStatement), + state.tokenToSourceNodeWithTrivia(this.close) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { if (options.walkMode & InternalWalkMode.walkExpressions) { walkArray(this.elements, visitor, options, this); @@ -1086,6 +1225,13 @@ export class UnaryExpression extends Expression { ]; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.tokenToSourceNodeWithTrivia(this.operator), + this.right?.toSourceNode(state) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { if (options.walkMode & InternalWalkMode.walkExpressions) { walk(this, 'right', visitor, options); @@ -1145,6 +1291,10 @@ export class VariableExpression extends Expression { ]; } + public toSourceNode(state: TranspileState): SourceNode { + return state.tokenToSourceNodeWithTrivia(this.name); + } + walk(visitor: WalkVisitor, options: WalkOptions) { //nothing to walk } @@ -1258,6 +1408,10 @@ export class SourceLiteralExpression extends Expression { ]; } + public toSourceNode(state: TranspileState): SourceNode { + return state.tokenToSourceNodeWithTrivia(this.token); + } + walk(visitor: WalkVisitor, options: WalkOptions) { //nothing to walk } @@ -1307,6 +1461,13 @@ export class NewExpression extends Expression { return this.call.transpile(state, cls?.getName(ParseMode.BrightScript)); } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.tokenToSourceNodeWithTrivia(this.newKeyword), + this.call?.toSourceNode(state) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { if (options.walkMode & InternalWalkMode.walkExpressions) { walk(this, 'call', visitor, options); @@ -1331,7 +1492,8 @@ export class CallfuncExpression extends Expression { readonly methodName: Identifier, readonly openingParen: Token, readonly args: Expression[], - readonly closingParen: Token + readonly closingParen: Token, + readonly argCommas: Token[] ) { super(); this.range = util.createBoundingRange( @@ -1384,6 +1546,20 @@ export class CallfuncExpression extends Expression { return result; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + this.callee?.toSourceNode(state), + state.tokenToSourceNodeWithTrivia(this.operator), + state.tokenToSourceNodeWithTrivia(this.methodName), + state.tokenToSourceNodeWithTrivia(this.openingParen), + ...this.args.map((x, i) => ([ + x.toSourceNode(state), + state.tokenToSourceNodeWithTrivia(this.argCommas[i]) + ]))?.flat() ?? [], + state.tokenToSourceNodeWithTrivia(this.closingParen) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { if (options.walkMode & InternalWalkMode.walkExpressions) { walk(this, 'callee', visitor, options); @@ -1399,7 +1575,8 @@ export class CallfuncExpression extends Expression { util.cloneToken(this.methodName), util.cloneToken(this.openingParen), this.args?.map(e => e?.clone()), - util.cloneToken(this.closingParen) + util.cloneToken(this.closingParen), + this.argCommas?.map(x => util.cloneToken(x)) ), ['callee', 'args'] ); @@ -1439,6 +1616,12 @@ export class TemplateStringQuasiExpression extends Expression { return result; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + ...this.expressions.map(x => x.toSourceNode(state)) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { if (options.walkMode & InternalWalkMode.walkExpressions) { walkArray(this.expressions, visitor, options, this); @@ -1460,7 +1643,9 @@ export class TemplateStringExpression extends Expression { readonly openingBacktick: Token, readonly quasis: TemplateStringQuasiExpression[], readonly expressions: Expression[], - readonly closingBacktick: Token + readonly closingBacktick: Token, + readonly expressionBeginTokens: Token[], + readonly expressionEndTokens: Token[] ) { super(); this.range = util.createBoundingRange( @@ -1527,6 +1712,21 @@ export class TemplateStringExpression extends Expression { return result; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.tokenToSourceNodeWithTrivia(this.openingBacktick), + ...this.quasis?.map((x, i) => ([ + this.quasis[i]?.toSourceNode(state), + ...(this.expressions?.[i] ? [ + state.tokenToSourceNodeWithTrivia(this.expressionBeginTokens[i]), + this.expressions[i]?.toSourceNode(state), + state.tokenToSourceNodeWithTrivia(this.expressionEndTokens[i]) + ] : []) + ])).flat() ?? [], + state.tokenToSourceNodeWithTrivia(this.closingBacktick) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { if (options.walkMode & InternalWalkMode.walkExpressions) { //walk the quasis and expressions in left-to-right order @@ -1547,7 +1747,9 @@ export class TemplateStringExpression extends Expression { util.cloneToken(this.openingBacktick), this.quasis?.map(e => e?.clone()), this.expressions?.map(e => e?.clone()), - util.cloneToken(this.closingBacktick) + util.cloneToken(this.closingBacktick), + this.expressionBeginTokens?.map(x => util.cloneToken(x)), + this.expressionEndTokens?.map(x => util.cloneToken(x)) ), ['quasis', 'expressions'] ); @@ -1560,7 +1762,9 @@ export class TaggedTemplateStringExpression extends Expression { readonly openingBacktick: Token, readonly quasis: TemplateStringQuasiExpression[], readonly expressions: Expression[], - readonly closingBacktick: Token + readonly closingBacktick: Token, + readonly expressionBeginTokens: Token[], + readonly expressionEndTokens: Token[] ) { super(); this.range = util.createBoundingRange( @@ -1616,6 +1820,22 @@ export class TaggedTemplateStringExpression extends Expression { return result; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.tokenToSourceNodeWithTrivia(this.tagName), + state.tokenToSourceNodeWithTrivia(this.openingBacktick), + ...this.quasis?.map((x, i) => ([ + this.quasis[i]?.toSourceNode(state), + ...(this.expressions?.[i] ? [ + state.tokenToSourceNodeWithTrivia(this.expressionBeginTokens[i]), + this.expressions[i]?.toSourceNode(state), + state.tokenToSourceNodeWithTrivia(this.expressionEndTokens[i]) + ] : []) + ])).flat() ?? [], + state.tokenToSourceNodeWithTrivia(this.closingBacktick) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { if (options.walkMode & InternalWalkMode.walkExpressions) { //walk the quasis and expressions in left-to-right order @@ -1637,7 +1857,9 @@ export class TaggedTemplateStringExpression extends Expression { util.cloneToken(this.openingBacktick), this.quasis?.map(e => e?.clone()), this.expressions?.map(e => e?.clone()), - util.cloneToken(this.closingBacktick) + util.cloneToken(this.closingBacktick), + this.expressionBeginTokens?.map(x => util.cloneToken(x)), + this.expressionEndTokens?.map(x => util.cloneToken(x)) ), ['quasis', 'expressions'] ); @@ -1679,6 +1901,13 @@ export class AnnotationExpression extends Expression { return []; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.tokenToSourceNodeWithTrivia(this.atToken), + this.call.toSourceNode(state) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { //nothing to walk } @@ -1786,6 +2015,17 @@ export class TernaryExpression extends Expression { return result; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + this.test?.toSourceNode(state), + state.tokenToSourceNodeWithTrivia(this.questionMarkToken), + this.consequent?.toSourceNode(state), + // don't add the colon because colons are included in leadingTrivia + // state.tokenToSourceNodeWithTrivia(this.colonToken), + this.alternate?.toSourceNode(state) + ); + } + public walk(visitor: WalkVisitor, options: WalkOptions) { if (options.walkMode & InternalWalkMode.walkExpressions) { walk(this, 'test', visitor, options); @@ -1888,6 +2128,14 @@ export class NullCoalescingExpression extends Expression { return result; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + this.consequent?.toSourceNode(state), + state.tokenToSourceNodeWithTrivia(this.questionQuestionToken), + this.alternate?.toSourceNode(state) + ); + } + public walk(visitor: WalkVisitor, options: WalkOptions) { if (options.walkMode & InternalWalkMode.walkExpressions) { walk(this, 'consequent', visitor, options); @@ -1945,6 +2193,12 @@ export class RegexLiteralExpression extends Expression { ]; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.tokenToSourceNodeWithTrivia(this.tokens.regexLiteral) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { //nothing to walk } @@ -1978,6 +2232,15 @@ export class TypeCastExpression extends Expression { public transpile(state: BrsTranspileState): TranspileResult { return this.obj.transpile(state); } + + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + this.obj?.toSourceNode(state), + state.tokenToSourceNodeWithTrivia(this.asToken), + state.tokenToSourceNodeWithTrivia(this.typeToken) + ); + } + public walk(visitor: WalkVisitor, options: WalkOptions) { if (options.walkMode & InternalWalkMode.walkExpressions) { walk(this, 'obj', visitor, options); diff --git a/src/parser/Parser.ts b/src/parser/Parser.ts index d75765acb..adcd2ca93 100644 --- a/src/parser/Parser.ts +++ b/src/parser/Parser.ts @@ -237,6 +237,8 @@ export class Parser { this.pendingAnnotations = []; this.ast = this.body(); + //assign the eofToken to body + this.ast.eofToken = this.tokens[this.tokens.length - 1]; //now that we've built the AST, link every node to its parent this.ast.link(); @@ -840,7 +842,8 @@ export class Parser { start: this.peek().range.start, end: this.peek().range.start }, - leadingWhitespace: '' + leadingWhitespace: '', + leadingTrivia: [] }; } let isSub = functionType?.kind === TokenKind.Sub; @@ -886,10 +889,16 @@ export class Parser { let params = [] as FunctionParameterExpression[]; let asToken: Token; let typeToken: Token; + const paramCommas: Token[] = []; if (!this.check(TokenKind.RightParen)) { - do { + while (true) { params.push(this.functionParameter()); - } while (this.match(TokenKind.Comma)); + if (this.check(TokenKind.Comma)) { + paramCommas.push(this.advance()); + } else { + break; + } + } } let rightParen = this.advance(); @@ -927,7 +936,8 @@ export class Parser { leftParen, rightParen, asToken, - typeToken + typeToken, + paramCommas ); // add the function to the relevant symbol tables @@ -1000,9 +1010,10 @@ export class Parser { let typeToken: Token | undefined; let defaultValue; - + let equalsToken: Token; // parse argument default value if (this.match(TokenKind.Equal)) { + equalsToken = this.previous(); // it seems any expression is allowed here -- including ones that operate on other arguments! defaultValue = this.expression(false); } @@ -1024,7 +1035,8 @@ export class Parser { name, typeToken, defaultValue, - asToken + asToken, + equalsToken ); } @@ -1057,7 +1069,7 @@ export class Parser { } else { const nameExpression = new VariableExpression(name); result = new AssignmentStatement( - { kind: TokenKind.Equal, text: '=', range: operator.range }, + { kind: TokenKind.Equal, text: '=', range: operator.range, leadingTrivia: [] }, name, new BinaryExpression(nameExpression, operator, value) ); @@ -1676,6 +1688,10 @@ export class Parser { let openingBacktick = this.peek(); this.advance(); let currentQuasiExpressionParts = []; + + let expressionBeginTokens: Token[] = []; + let expressionEndTokens: Token[] = []; + while (!this.isAtEnd() && !this.check(TokenKind.BackTick)) { let next = this.peek(); if (next.kind === TokenKind.TemplateStringQuasi) { @@ -1697,13 +1713,17 @@ export class Parser { currentQuasiExpressionParts = []; if (next.kind === TokenKind.TemplateStringExpressionBegin) { - this.advance(); + expressionBeginTokens.push( + this.advance() + ); } //now keep this expression expressions.push(this.expression()); if (!this.isAtEnd() && this.check(TokenKind.TemplateStringExpressionEnd)) { //TODO is it an error if this is not present? - this.advance(); + expressionEndTokens.push( + this.advance() + ); } else { this.diagnostics.push({ ...DiagnosticMessages.unterminatedTemplateExpression(), @@ -1730,9 +1750,9 @@ export class Parser { } else { let closingBacktick = this.advance(); if (isTagged) { - return new TaggedTemplateStringExpression(tagName, openingBacktick, quasis, expressions, closingBacktick); + return new TaggedTemplateStringExpression(tagName, openingBacktick, quasis, expressions, closingBacktick, expressionBeginTokens, expressionEndTokens); } else { - return new TemplateStringExpression(openingBacktick, quasis, expressions, closingBacktick); + return new TemplateStringExpression(openingBacktick, quasis, expressions, closingBacktick, expressionBeginTokens, expressionEndTokens); } } } @@ -2149,7 +2169,7 @@ export class Parser { left.additionalIndexes, operator.kind === TokenKind.Equal ? operator - : { kind: TokenKind.Equal, text: '=', range: operator.range } + : { kind: TokenKind.Equal, text: '=', range: operator.range, leadingTrivia: [] } ); } else if (isDottedGetExpression(left)) { return new DottedSetStatement( @@ -2161,7 +2181,7 @@ export class Parser { left.dot, operator.kind === TokenKind.Equal ? operator - : { kind: TokenKind.Equal, text: '=', range: operator.range } + : { kind: TokenKind.Equal, text: '=', range: operator.range, leadingTrivia: [] } ); } } @@ -2598,7 +2618,7 @@ export class Parser { let openParen = this.consume(DiagnosticMessages.expectedOpenParenToFollowCallfuncIdentifier(), TokenKind.LeftParen); let call = this.finishCall(openParen, callee, false); - return new CallfuncExpression(callee, operator, methodName as Identifier, openParen, call.args, call.closingParen); + return new CallfuncExpression(callee, operator, methodName as Identifier, openParen, call.args, call.closingParen, call.argCommas); } private call(): Expression { @@ -2674,9 +2694,10 @@ export class Parser { private finishCall(openingParen: Token, callee: Expression, addToCallExpressionList = true) { let args = [] as Expression[]; while (this.match(TokenKind.Newline)) { } + const commas: Token[] = []; if (!this.check(TokenKind.RightParen)) { - do { + while (true) { while (this.match(TokenKind.Newline)) { } if (args.length >= CallExpression.MaximumArguments) { @@ -2693,7 +2714,12 @@ export class Parser { // we were unable to get an expression, so don't continue break; } - } while (this.match(TokenKind.Comma)); + if (this.check(TokenKind.Comma)) { + commas.push(this.advance()); + } else { + break; + } + } } while (this.match(TokenKind.Newline)) { } @@ -2703,7 +2729,7 @@ export class Parser { TokenKind.RightParen ); - let expression = new CallExpression(callee, openingParen, closingParen, args); + let expression = new CallExpression(callee, openingParen, closingParen, args, commas); if (addToCallExpressionList) { this.callExpressions.push(expression); } @@ -2718,6 +2744,7 @@ export class Parser { */ private typeToken(ignoreDiagnostics = false): Token { let typeToken: Token; + const leadingTrivia = this.peek()?.leadingTrivia; let lookForUnions = true; let isAUnion = false; let resultToken; @@ -2758,6 +2785,9 @@ export class Parser { } } } + if (typeToken) { + typeToken.leadingTrivia = leadingTrivia; + } if (isAUnion) { resultToken = createToken(TokenKind.Dynamic, null, util.createBoundingRange(resultToken, typeToken)); } @@ -2841,6 +2871,7 @@ export class Parser { private arrayLiteral() { let elements: Array = []; + let commas: Array = []; let openingSquare = this.previous(); //add any comment found right after the opening square @@ -2857,19 +2888,22 @@ export class Parser { elements.push(this.expression()); while (this.matchAny(TokenKind.Comma, TokenKind.Newline, TokenKind.Comment)) { + const previous = this.previous(); if (this.checkPrevious(TokenKind.Comment) || this.check(TokenKind.Comment)) { let comment = this.check(TokenKind.Comment) ? this.advance() : this.previous(); elements.push(new CommentStatement([comment])); } - while (this.match(TokenKind.Newline)) { - - } + while (this.match(TokenKind.Newline)) { } if (this.check(TokenKind.RightSquareBracket)) { break; } elements.push(this.expression()); + commas.push( + previous?.kind === TokenKind.Comma ? previous : undefined + ); + } } catch (error: any) { this.rethrowNonDiagnosticError(error); @@ -2884,7 +2918,7 @@ export class Parser { } //this.consume("Expected newline or ':' after array literal", TokenKind.Newline, TokenKind.Colon, TokenKind.Eof); - return new ArrayLiteralExpression(elements, openingSquare, closingSquare); + return new ArrayLiteralExpression(elements, openingSquare, closingSquare, commas); } private aaLiteral() { diff --git a/src/parser/Statement.ts b/src/parser/Statement.ts index ea8fee117..16c2e8262 100644 --- a/src/parser/Statement.ts +++ b/src/parser/Statement.ts @@ -15,6 +15,7 @@ import type { TranspileResult, TypedefProvider } from '../interfaces'; import { createIdentifier, createInvalidLiteral, createMethodStatement, createToken } from '../astUtils/creators'; import { DynamicType } from '../types/DynamicType'; import type { BscType } from '../types/BscType'; +import { SourceNode } from 'source-map'; import type { TranspileState } from './TranspileState'; import { SymbolTable } from '../SymbolTable'; import type { AstNode, Expression } from './AstNode'; @@ -33,6 +34,10 @@ export class EmptyStatement extends Statement { transpile(state: BrsTranspileState) { return []; } + + toSourceNode() { + return new SourceNode(); + } walk(visitor: WalkVisitor, options: WalkOptions) { //nothing to walk } @@ -51,7 +56,11 @@ export class EmptyStatement extends Statement { */ export class Body extends Statement implements TypedefProvider { constructor( - public statements: Statement[] = [] + public statements: Statement[] = [], + /** + * If this is the top-level program body, it will have the EOF token attached + */ + public eofToken?: Token ) { super(); } @@ -98,6 +107,13 @@ export class Body extends Statement implements TypedefProvider { return result; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.arrayToSourceNodeWithTrivia(this.statements, isCommentStatement), + this.eofToken ? state.tokenToSourceNodeWithTrivia(this.eofToken) : '' + ); + } + getTypedef(state: BrsTranspileState): TranspileResult { let result = [] as TranspileResult; for (const statement of this.statements) { @@ -122,7 +138,8 @@ export class Body extends Statement implements TypedefProvider { public clone() { return this.finalizeClone( new Body( - this.statements?.map(s => s?.clone()) + this.statements?.map(s => s?.clone()), + util.cloneToken(this.eofToken) ), ['statements'] ); @@ -164,6 +181,14 @@ export class AssignmentStatement extends Statement { } } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.tokenToSourceNodeWithTrivia(this.name), + state.tokenToSourceNodeWithTrivia(this.equals), + this.value.toSourceNode(state) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { if (options.walkMode & InternalWalkMode.walkExpressions) { walk(this, 'value', visitor, options); @@ -229,6 +254,10 @@ export class Block extends Statement { return results; } + public toSourceNode(state: TranspileState): SourceNode { + return state.arrayToSourceNodeWithTrivia(this.statements, isCommentStatement); + } + walk(visitor: WalkVisitor, options: WalkOptions) { if (options.walkMode & InternalWalkMode.walkStatements) { walkArray(this.statements, visitor, options, this); @@ -260,6 +289,10 @@ export class ExpressionStatement extends Statement { return this.expression.transpile(state); } + public toSourceNode(state: TranspileState): SourceNode { + return this.expression.toSourceNode(state); + } + walk(visitor: WalkVisitor, options: WalkOptions) { if (options.walkMode & InternalWalkMode.walkExpressions) { walk(this, 'expression', visitor, options); @@ -317,6 +350,12 @@ export class CommentStatement extends Statement implements Expression, TypedefPr return this.transpile(state as BrsTranspileState); } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + ...this.comments.map(x => state.tokenToSourceNodeWithTrivia(x)) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { //nothing to walk } @@ -349,6 +388,10 @@ export class ExitForStatement extends Statement { ]; } + public toSourceNode(state: TranspileState): SourceNode { + return state.tokenToSourceNodeWithTrivia(this.tokens.exitFor); + } + walk(visitor: WalkVisitor, options: WalkOptions) { //nothing to walk } @@ -380,6 +423,10 @@ export class ExitWhileStatement extends Statement { ]; } + public toSourceNode(state: TranspileState): SourceNode { + return state.tokenToSourceNodeWithTrivia(this.tokens.exitWhile); + } + walk(visitor: WalkVisitor, options: WalkOptions) { //nothing to walk } @@ -436,6 +483,14 @@ export class FunctionStatement extends Statement implements TypedefProvider { return this.func.transpile(state, nameToken); } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.arrayToSourceNodeWithTrivia(this.annotations), + //the FunctionExpression looks upwards for its name, so we don't need to include it here + this.func.toSourceNode(state) + ); + } + getTypedef(state: BrsTranspileState) { let result = [] as TranspileResult; for (let annotation of this.annotations ?? []) { @@ -571,6 +626,25 @@ export class IfStatement extends Statement { return results; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + //if + state.tokenToSourceNodeWithTrivia(this.tokens.if), + //conditions + this.condition.toSourceNode(state), + //then + state.tokenToSourceNodeWithTrivia(this.tokens.then), + //then branch + this.thenBranch?.toSourceNode(state), + //else + state.tokenToSourceNodeWithTrivia(this.tokens.else), + //else branch + this.elseBranch?.toSourceNode(state), + //end if + state.tokenToSourceNodeWithTrivia(this.tokens.endIf) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { if (options.walkMode & InternalWalkMode.walkExpressions) { walk(this, 'condition', visitor, options); @@ -623,6 +697,13 @@ export class IncrementStatement extends Statement { ]; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + this.value.toSourceNode(state), + state.tokenToSourceNodeWithTrivia(this.operator) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { if (options.walkMode & InternalWalkMode.walkExpressions) { walk(this, 'value', visitor, options); @@ -697,6 +778,24 @@ export class PrintStatement extends Statement { return result; } + public toSourceNode(state: TranspileState): SourceNode { + let result = [ + state.tokenToSourceNodeWithTrivia(this.tokens.print) + ]; + for (const expression of this.expressions) { + if (isExpression(expression)) { + result.push( + expression.toSourceNode(state) + ); + } else { + result.push( + state.tokenToSourceNodeWithTrivia(expression) + ); + } + } + return state.toSourceNode(...result); + } + walk(visitor: WalkVisitor, options: WalkOptions) { if (options.walkMode & InternalWalkMode.walkExpressions) { //sometimes we have semicolon Tokens in the expressions list (should probably fix that...), so only walk the actual expressions @@ -761,6 +860,27 @@ export class DimStatement extends Statement { return result; } + public toSourceNode(state: TranspileState): SourceNode { + const chunks: Array = [ + state.tokenToSourceNodeWithTrivia(this.dimToken), + state.tokenToSourceNodeWithTrivia(this.identifier), + state.tokenToSourceNodeWithTrivia(this.openingSquare) + ]; + for (let i = 0; i < this.dimensions.length; i++) { + if (i > 0) { + chunks.push(', '); + } + chunks.push( + this.dimensions[i].toSourceNode(state) + ); + } + chunks.push( + state.tokenToSourceNodeWithTrivia(this.closingSquare) + ); + return state.toSourceNode(...chunks); + } + + public walk(visitor: WalkVisitor, options: WalkOptions) { if (this.dimensions?.length !== undefined && this.dimensions?.length > 0 && options.walkMode & InternalWalkMode.walkExpressions) { walkArray(this.dimensions, visitor, options, this); @@ -806,6 +926,13 @@ export class GotoStatement extends Statement { ]; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.tokenToSourceNodeWithTrivia(this.tokens.goto), + state.tokenToSourceNodeWithTrivia(this.tokens.label) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { //nothing to walk } @@ -844,6 +971,13 @@ export class LabelStatement extends Statement { ]; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.tokenToSourceNodeWithTrivia(this.tokens.identifier), + state.tokenToSourceNodeWithTrivia(this.tokens.colon) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { //nothing to walk } @@ -886,6 +1020,13 @@ export class ReturnStatement extends Statement { return result; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.tokenToSourceNodeWithTrivia(this.tokens.return), + this.value?.toSourceNode(state) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { if (options.walkMode & InternalWalkMode.walkExpressions) { walk(this, 'value', visitor, options); @@ -923,6 +1064,12 @@ export class EndStatement extends Statement { ]; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.tokenToSourceNodeWithTrivia(this.tokens.end) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { //nothing to walk } @@ -954,6 +1101,12 @@ export class StopStatement extends Statement { ]; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.tokenToSourceNodeWithTrivia(this.tokens.stop) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { //nothing to walk } @@ -1038,6 +1191,27 @@ export class ForStatement extends Statement { return result; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + //for + state.tokenToSourceNodeWithTrivia(this.forToken), + //i=1 + this.counterDeclaration?.toSourceNode(state), + //to + state.tokenToSourceNodeWithTrivia(this.toToken), + //finalValue + this.finalValue?.toSourceNode(state), + //step + state.tokenToSourceNodeWithTrivia(this.stepToken), + //stepValue + this.increment?.toSourceNode(state), + //body + this.body?.toSourceNode(state), + //end for + state.tokenToSourceNodeWithTrivia(this.endForToken) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { if (options.walkMode & InternalWalkMode.walkStatements) { walk(this, 'counterDeclaration', visitor, options); @@ -1127,6 +1301,23 @@ export class ForEachStatement extends Statement { return result; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + //for each + state.tokenToSourceNodeWithTrivia(this.tokens.forEach), + //item + state.tokenToSourceNodeWithTrivia(this.item), + //in + state.tokenToSourceNodeWithTrivia(this.tokens.in), + //target + this.target?.toSourceNode(state), + //body + this.body?.toSourceNode(state), + //end for + state.tokenToSourceNodeWithTrivia(this.tokens.endFor) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { if (options.walkMode & InternalWalkMode.walkExpressions) { walk(this, 'target', visitor, options); @@ -1201,6 +1392,19 @@ export class WhileStatement extends Statement { return result; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + //while + state.tokenToSourceNodeWithTrivia(this.tokens.while), + //condition + this.condition?.toSourceNode(state), + //body + this.body?.toSourceNode(state), + //end while + state.tokenToSourceNodeWithTrivia(this.tokens.endWhile) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { if (options.walkMode & InternalWalkMode.walkExpressions) { walk(this, 'condition', visitor, options); @@ -1231,20 +1435,31 @@ export class DottedSetStatement extends Statement { readonly name: Identifier, readonly value: Expression, readonly dot?: Token, - readonly equals?: Token + readonly operator?: Token ) { super(); this.range = util.createBoundingRange( obj, dot, name, - equals, + operator, value ); } public readonly range: Range | undefined; + /** + * @deprecated use `this.operator` instead + */ + public get equals() { + //back-compat fix for this token. TODO remove in + return this.operator; + } + public set equals(value: Token | undefined) { + (this as any).operator = value; + } + transpile(state: BrsTranspileState) { //if the value is a compound assignment, don't add the obj, dot, name, or operator...the expression will handle that if (CompoundAssignmentOperators.includes((this.value as BinaryExpression)?.operator?.kind)) { @@ -1265,6 +1480,26 @@ export class DottedSetStatement extends Statement { } } + public toSourceNode(state: TranspileState): SourceNode { + //if the value is a compound assignment, don't add the obj, dot, name, or operator...the expression will handle that + if (CompoundAssignmentOperators.includes((this.value as BinaryExpression)?.operator?.kind)) { + return this.value.toSourceNode(state); + } else { + return state.toSourceNode( + // object + this.obj.toSourceNode(state), + // . + state.tokenToSourceNodeWithTrivia(this.dot), + // name + state.tokenToSourceNodeWithTrivia(this.name), + // = + state.tokenToSourceNodeWithTrivia(this.operator), + //right-hand-side of assignment + this.value?.toSourceNode(state) + ); + } + } + walk(visitor: WalkVisitor, options: WalkOptions) { if (options.walkMode & InternalWalkMode.walkExpressions) { walk(this, 'obj', visitor, options); @@ -1279,7 +1514,7 @@ export class DottedSetStatement extends Statement { util.cloneToken(this.name), this.value?.clone(), util.cloneToken(this.dot), - util.cloneToken(this.equals) + util.cloneToken(this.operator) ), ['obj', 'value'] ); @@ -1294,7 +1529,7 @@ export class IndexedSetStatement extends Statement { readonly openingSquare: Token, readonly closingSquare: Token, readonly additionalIndexes?: Expression[], - readonly equals?: Token + readonly operator?: Token ) { super(); this.additionalIndexes ??= []; @@ -1303,7 +1538,7 @@ export class IndexedSetStatement extends Statement { openingSquare, index, closingSquare, - equals, + operator, value, ...this.additionalIndexes ); @@ -1311,6 +1546,17 @@ export class IndexedSetStatement extends Statement { public readonly range: Range | undefined; + /** + * @deprecated use `this.operator` instead + */ + public get equals() { + //back-compat fix for this token. TODO remove in + return this.operator; + } + public set equals(value: Token | undefined) { + (this as any).operator = value; + } + transpile(state: BrsTranspileState) { //if the value is a component assignment, don't add the obj, index or operator...the expression will handle that if (CompoundAssignmentOperators.includes((this.value as BinaryExpression)?.operator?.kind)) { @@ -1345,6 +1591,27 @@ export class IndexedSetStatement extends Statement { } } + public toSourceNode(state: TranspileState): SourceNode { + //if the value is a component assignment, don't add the obj, index or operator...the expression will handle that + if (CompoundAssignmentOperators.includes((this.value as BinaryExpression)?.operator?.kind)) { + return this.value.toSourceNode(state); + } else { + return state.toSourceNode( + //obj + this.obj?.toSourceNode(state), + // [ + state.tokenToSourceNodeWithTrivia(this.openingSquare), + // index + this.index?.toSourceNode(state), + // ] + state.tokenToSourceNodeWithTrivia(this.closingSquare), + // = + state.tokenToSourceNodeWithTrivia(this.operator), + // value + this.value.toSourceNode(state) + ); + } + } walk(visitor: WalkVisitor, options: WalkOptions) { if (options.walkMode & InternalWalkMode.walkExpressions) { walk(this, 'obj', visitor, options); @@ -1363,7 +1630,7 @@ export class IndexedSetStatement extends Statement { util.cloneToken(this.openingSquare), util.cloneToken(this.closingSquare), this.additionalIndexes?.map(e => e?.clone()), - util.cloneToken(this.equals) + util.cloneToken(this.operator) ), ['obj', 'index', 'value', 'additionalIndexes'] ); @@ -1405,6 +1672,13 @@ export class LibraryStatement extends Statement implements TypedefProvider { return this.transpile(state); } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.tokenToSourceNodeWithTrivia(this.tokens.library), + state.tokenToSourceNodeWithTrivia(this.tokens.filePath) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { //nothing to walk } @@ -1477,6 +1751,15 @@ export class NamespaceStatement extends Statement implements TypedefProvider { return this.body.transpile(state); } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.tokenToSourceNodeWithTrivia(this.keyword), + this.nameExpression?.toSourceNode(state), + this.body?.toSourceNode(state), + state.tokenToSourceNodeWithTrivia(this.endKeyword) + ); + } + getTypedef(state: BrsTranspileState): TranspileResult { let result = [ 'namespace ', @@ -1559,6 +1842,13 @@ export class ImportStatement extends Statement implements TypedefProvider { ]; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.tokenToSourceNodeWithTrivia(this.importToken), + state.tokenToSourceNodeWithTrivia(this.filePathToken) + ); + } + /** * Get the typedef for this statement */ @@ -1679,6 +1969,24 @@ export class InterfaceStatement extends Statement implements TypedefProvider { return []; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.arrayToSourceNodeWithTrivia(this.annotations), + // interface + state.tokenToSourceNodeWithTrivia(this.tokens.interface), + // SomeInterface + state.tokenToSourceNodeWithTrivia(this.tokens.name), + // extends + state.tokenToSourceNodeWithTrivia(this.tokens.extends), + // SomeParentInterface + this.parentInterfaceName?.toSourceNode(state), + //interface body + state.arrayToSourceNodeWithTrivia(this.body, isCommentStatement), + //end interface + state.tokenToSourceNodeWithTrivia(this.tokens.endInterface) + ); + } + getTypedef(state: BrsTranspileState) { const result = [] as TranspileResult; for (let annotation of this.annotations ?? []) { @@ -1792,6 +2100,18 @@ export class InterfaceFieldStatement extends Statement implements TypedefProvide return this.tokens.name.text; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.arrayToSourceNodeWithTrivia(this.annotations), + //name + state.tokenToSourceNodeWithTrivia(this.tokens.name), + //as + state.tokenToSourceNodeWithTrivia(this.tokens.as), + //type + state.tokenToSourceNodeWithTrivia(this.tokens.type) + ); + } + public get isOptional() { return !!this.tokens.optional; } @@ -1948,6 +2268,26 @@ export class InterfaceMethodStatement extends Statement implements TypedefProvid return result; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.arrayToSourceNodeWithTrivia(this.annotations), + // function|sub + state.tokenToSourceNodeWithTrivia(this.tokens.functionType), + // name + state.tokenToSourceNodeWithTrivia(this.tokens.name), + // ( + state.tokenToSourceNodeWithTrivia(this.tokens.leftParen), + // params + state.arrayToSourceNodeWithTrivia(this.params), + // ) + state.tokenToSourceNodeWithTrivia(this.tokens.rightParen), + // as + state.tokenToSourceNodeWithTrivia(this.tokens.as), + // + state.tokenToSourceNodeWithTrivia(this.tokens.returnType) + ); + } + public clone() { return this.finalizeClone( new InterfaceMethodStatement( @@ -2052,6 +2392,18 @@ export class ClassStatement extends Statement implements TypedefProvider { return result; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.arrayToSourceNodeWithTrivia(this.annotations), + state.tokenToSourceNodeWithTrivia(this.classKeyword), + state.tokenToSourceNodeWithTrivia(this.name), + state.tokenToSourceNodeWithTrivia(this.extendsKeyword), + this.parentClassName?.toSourceNode(state), + state.arrayToSourceNodeWithTrivia(this.body, isCommentStatement), + state.tokenToSourceNodeWithTrivia(this.end) + ); + } + getTypedef(state: BrsTranspileState) { const result = [] as TranspileResult; for (let annotation of this.annotations ?? []) { @@ -2306,7 +2658,8 @@ export class ClassStatement extends Statement implements TypedefProvider { new VariableExpression(createToken(TokenKind.Identifier, 'super')), createToken(TokenKind.LeftParen), createToken(TokenKind.RightParen), - params.map(x => new VariableExpression(x.name)) + params.map(x => new VariableExpression(x.name)), + []//TODO should we include the number of commas here? ) ); body.unshift( @@ -2517,6 +2870,13 @@ export class MethodStatement extends FunctionStatement { return this.func.transpile(state, name); } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.tokenToSourceNodeWithTrivia(this.accessModifier), + super.toSourceNode(state) + ); + } + getTypedef(state: BrsTranspileState) { const result = [] as TranspileResult; for (let annotation of this.annotations ?? []) { @@ -2574,7 +2934,8 @@ export class MethodStatement extends FunctionStatement { text: 'super', isReserved: false, range: state.classStatement!.name.range, - leadingWhitespace: '' + leadingWhitespace: '', + leadingTrivia: [] } ), { @@ -2582,15 +2943,18 @@ export class MethodStatement extends FunctionStatement { text: '(', isReserved: false, range: state.classStatement!.name.range, - leadingWhitespace: '' + leadingWhitespace: '', + leadingTrivia: [] }, { kind: TokenKind.RightParen, text: ')', isReserved: false, range: state.classStatement!.name.range, - leadingWhitespace: '' + leadingWhitespace: '', + leadingTrivia: [] }, + [], [] ) ); @@ -2695,6 +3059,17 @@ export class FieldStatement extends Statement implements TypedefProvider { throw new Error('transpile not implemented for ' + Object.getPrototypeOf(this).constructor.name); } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.tokenToSourceNodeWithTrivia(this.accessModifier), + state.tokenToSourceNodeWithTrivia(this.name), + state.tokenToSourceNodeWithTrivia(this.as), + state.tokenToSourceNodeWithTrivia(this.type), + state.tokenToSourceNodeWithTrivia(this.equal), + this.initialValue?.toSourceNode(state) + ); + } + getTypedef(state: BrsTranspileState) { const result = [] as TranspileResult; if (this.name) { @@ -2793,6 +3168,15 @@ export class TryCatchStatement extends Statement { ] as TranspileResult; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.tokenToSourceNodeWithTrivia(this.tokens.try), + this.tryBranch?.toSourceNode(state), + this.catchStatement?.toSourceNode(state), + state.tokenToSourceNodeWithTrivia(this.tokens.endTry) + ); + } + public walk(visitor: WalkVisitor, options: WalkOptions) { if (this.tryBranch && options.walkMode & InternalWalkMode.walkStatements) { walk(this, 'tryBranch', visitor, options); @@ -2842,6 +3226,14 @@ export class CatchStatement extends Statement { ]; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.tokenToSourceNodeWithTrivia(this.tokens.catch), + state.tokenToSourceNodeWithTrivia(this.exceptionVariable), + this.catchBranch?.toSourceNode(state) + ); + } + public walk(visitor: WalkVisitor, options: WalkOptions) { if (this.catchBranch && options.walkMode & InternalWalkMode.walkStatements) { walk(this, 'catchBranch', visitor, options); @@ -2894,6 +3286,13 @@ export class ThrowStatement extends Statement { return result; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.tokenToSourceNodeWithTrivia(this.throwToken), + this.expression?.toSourceNode(state) + ); + } + public walk(visitor: WalkVisitor, options: WalkOptions) { if (this.expression && options.walkMode & InternalWalkMode.walkExpressions) { walk(this, 'expression', visitor, options); @@ -3058,6 +3457,16 @@ export class EnumStatement extends Statement implements TypedefProvider { return result; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.arrayToSourceNodeWithTrivia(this.annotations), + state.tokenToSourceNodeWithTrivia(this.tokens.enum), + state.tokenToSourceNodeWithTrivia(this.tokens.name), + state.arrayToSourceNodeWithTrivia(this.body, isCommentStatement), + state.tokenToSourceNodeWithTrivia(this.tokens.endEnum) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { if (options.walkMode & InternalWalkMode.walkStatements) { walkArray(this.body, visitor, options, this); @@ -3133,6 +3542,15 @@ export class EnumMemberStatement extends Statement implements TypedefProvider { return result; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.arrayToSourceNodeWithTrivia(this.annotations), + state.tokenToSourceNodeWithTrivia(this.tokens.name), + state.tokenToSourceNodeWithTrivia(this.tokens.equal), + this.value?.toSourceNode(state) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { if (this.value && options.walkMode & InternalWalkMode.walkExpressions) { walk(this, 'value', visitor, options); @@ -3209,6 +3627,16 @@ export class ConstStatement extends Statement implements TypedefProvider { ]; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.arrayToSourceNodeWithTrivia(this.annotations), + state.tokenToSourceNodeWithTrivia(this.tokens.const), + state.tokenToSourceNodeWithTrivia(this.tokens.name), + state.tokenToSourceNodeWithTrivia(this.tokens.equals), + this.value?.toSourceNode(state) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { if (this.value && options.walkMode & InternalWalkMode.walkExpressions) { walk(this, 'value', visitor, options); @@ -3254,6 +3682,13 @@ export class ContinueStatement extends Statement { ]; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.tokenToSourceNodeWithTrivia(this.tokens.continue), + state.tokenToSourceNodeWithTrivia(this.tokens.loopType) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { //nothing to walk } @@ -3306,6 +3741,16 @@ export class TypecastStatement extends Statement { return []; } + public toSourceNode(state: TranspileState): SourceNode { + // Provide a source node for the typecast statement for tools that consume the AST. + return state.toSourceNode( + state.tokenToSourceNodeWithTrivia(this.tokens.typecast), + this.typecastExpression?.toSourceNode(state), + state.tokenToSourceNodeWithTrivia(this.tokens.as), + state.tokenToSourceNodeWithTrivia(this.tokens.type) + ); + } + walk(visitor: WalkVisitor, options: WalkOptions) { //nothing to walk } @@ -3358,6 +3803,16 @@ export class AliasStatement extends Statement { return []; } + public toSourceNode(state: TranspileState): SourceNode { + return state.toSourceNode( + state.tokenToSourceNodeWithTrivia(this.tokens.alias), + state.tokenToSourceNodeWithTrivia(this.tokens.name), + state.tokenToSourceNodeWithTrivia(this.tokens.equals), + state.tokenToSourceNodeWithTrivia(this.tokens.value) + ); + } + + walk(visitor: WalkVisitor, options: WalkOptions) { //nothing to walk } @@ -3374,4 +3829,3 @@ export class AliasStatement extends Statement { ); } } - diff --git a/src/parser/TranspileState.ts b/src/parser/TranspileState.ts index 49ea74d13..6c0e8d2c3 100644 --- a/src/parser/TranspileState.ts +++ b/src/parser/TranspileState.ts @@ -1,6 +1,7 @@ import { SourceNode } from 'source-map'; import type { Range } from 'vscode-languageserver'; import type { BsConfig } from '../BsConfig'; +import type { Token } from '../lexer/Token'; import type { TranspileResult } from '../interfaces'; import util from '../util'; @@ -67,6 +68,20 @@ export class TranspileState { ); } + /** + * Shorthand for creating a container SourceNode (one that doesn't have a location) + */ + public toSourceNode(...chunks: Array) { + return new SourceNode( + //convert 0-based range line to 1-based SourceNode line + 1, + //range and SourceNode character are both 0-based, so no conversion necessary + 0, + null, + chunks.filter(x => !!x) + ); + } + /** * Create a SourceNode from a token. This is more efficient than the above `sourceNode` function * because the entire token is passed by reference, instead of the raw string being copied to the parameter, @@ -83,6 +98,28 @@ export class TranspileState { ); } + /** + * Create a SourceNode from a token. This is more efficient than the above `sourceNode` function + * because the entire token is passed by reference, instead of the raw string being copied to the parameter, + * only to then be copied again for the SourceNode constructor + */ + public tokenToSourceNodeWithTrivia(token: { range?: Range; text: string; leadingTrivia: Token[] }) { + const result: SourceNode[] = []; + for (const item of [...token?.leadingTrivia ?? [], token]) { + if (token) { + result.push(new SourceNode( + //convert 0-based range line to 1-based SourceNode line + item.range.start.line + 1, + //range and SourceNode character are both 0-based, so no conversion necessary + item.range.start.character, + this.srcPath, + item.text + )); + } + } + return new SourceNode(1, 0, null, result); + } + /** * Create a SourceNode from a token, accounting for missing range and multi-line text */ @@ -118,4 +155,23 @@ export class TranspileState { return this.tokenToSourceNode(token); } } + + /** + * Convert an array of `toSourceNode`-enabled objects to a single SourceNode, applying an optional filter as they are iterated + * @param elements the items that have a `toSourceNode()` function + * @param filter a function to call to EXCLUDE an item. It's an exclude filter so you can pass in things like `isCommentStatement` directly + */ + public arrayToSourceNodeWithTrivia SourceNode }>(elements: Array, filter?: (element: TElement) => boolean) { + const nodes: Array = []; + if (Array.isArray(elements)) { + for (const element of elements) { + if (element && !filter?.(element)) { + nodes.push( + element?.toSourceNode(this) ?? '' + ); + } + } + } + return new SourceNode(1, 0, null, nodes); + } } diff --git a/src/parser/tests/Parser.spec.ts b/src/parser/tests/Parser.spec.ts index ca503dcf7..25d7224c3 100644 --- a/src/parser/tests/Parser.spec.ts +++ b/src/parser/tests/Parser.spec.ts @@ -10,10 +10,11 @@ import type { Range } from 'vscode-languageserver'; export function token(kind: TokenKind, text?: string): Token { return { kind: kind, - text: text!, - isReserved: ReservedWords.has((text ?? '').toLowerCase()), - range: null, - leadingWhitespace: '' + text: text, + isReserved: ReservedWords.has((text || '').toLowerCase()), + range: null as any, + leadingWhitespace: '', + leadingTrivia: [] }; } diff --git a/src/parser/tests/controlFlow/For.spec.ts b/src/parser/tests/controlFlow/For.spec.ts index 0130b76d9..1829bf73e 100644 --- a/src/parser/tests/controlFlow/For.spec.ts +++ b/src/parser/tests/controlFlow/For.spec.ts @@ -103,25 +103,29 @@ describe('parser for loops', () => { kind: TokenKind.For, text: 'for', isReserved: true, - range: Range.create(0, 0, 0, 3) + range: Range.create(0, 0, 0, 3), + leadingTrivia: [] }, { kind: TokenKind.Identifier, text: 'i', isReserved: false, - range: Range.create(0, 4, 0, 5) + range: Range.create(0, 4, 0, 5), + leadingTrivia: [] }, { kind: TokenKind.Equal, text: '=', isReserved: false, - range: Range.create(0, 6, 0, 7) + range: Range.create(0, 6, 0, 7), + leadingTrivia: [] }, { kind: TokenKind.IntegerLiteral, text: '0', isReserved: false, - range: Range.create(0, 8, 0, 9) + range: Range.create(0, 8, 0, 9), + leadingTrivia: [] }, { kind: TokenKind.To, @@ -130,19 +134,22 @@ describe('parser for loops', () => { range: { start: { line: 0, character: 10 }, end: { line: 0, character: 12 } - } + }, + leadingTrivia: [] }, { kind: TokenKind.IntegerLiteral, text: '10', isReserved: false, - range: Range.create(0, 13, 0, 15) + range: Range.create(0, 13, 0, 15), + leadingTrivia: [] }, { kind: TokenKind.Newline, text: '\n', isReserved: false, - range: Range.create(0, 15, 0, 16) + range: Range.create(0, 15, 0, 16), + leadingTrivia: [] }, // loop body isn't significant for location tracking, so helper functions are safe identifier('Rnd'), @@ -154,7 +161,8 @@ describe('parser for loops', () => { kind: TokenKind.EndFor, text: 'end for', isReserved: false, - range: Range.create(2, 0, 2, 8) + range: Range.create(2, 0, 2, 8), + leadingTrivia: [] }, EOF ]); diff --git a/src/parser/tests/controlFlow/ForEach.spec.ts b/src/parser/tests/controlFlow/ForEach.spec.ts index a340992fa..f143659ad 100644 --- a/src/parser/tests/controlFlow/ForEach.spec.ts +++ b/src/parser/tests/controlFlow/ForEach.spec.ts @@ -66,31 +66,36 @@ describe('parser foreach loops', () => { kind: TokenKind.ForEach, text: 'for each', isReserved: true, - range: Range.create(0, 0, 0, 8) + range: Range.create(0, 0, 0, 8), + leadingTrivia: [] }, { kind: TokenKind.Identifier, text: 'a', isReserved: false, - range: Range.create(0, 9, 0, 10) + range: Range.create(0, 9, 0, 10), + leadingTrivia: [] }, { kind: TokenKind.Identifier, text: 'in', isReserved: true, - range: Range.create(0, 11, 0, 13) + range: Range.create(0, 11, 0, 13), + leadingTrivia: [] }, { kind: TokenKind.Identifier, text: 'b', isReserved: false, - range: Range.create(0, 14, 0, 15) + range: Range.create(0, 14, 0, 15), + leadingTrivia: [] }, { kind: TokenKind.Newline, text: '\n', isReserved: false, - range: Range.create(0, 15, 0, 16) + range: Range.create(0, 15, 0, 16), + leadingTrivia: [] }, // loop body isn't significant for location tracking, so helper functions are safe identifier('Rnd'), @@ -102,7 +107,8 @@ describe('parser foreach loops', () => { kind: TokenKind.EndFor, text: 'end for', isReserved: false, - range: Range.create(2, 0, 2, 7) + range: Range.create(2, 0, 2, 7), + leadingTrivia: [] }, EOF ]); diff --git a/src/parser/tests/controlFlow/While.spec.ts b/src/parser/tests/controlFlow/While.spec.ts index c85930281..c9d00247a 100644 --- a/src/parser/tests/controlFlow/While.spec.ts +++ b/src/parser/tests/controlFlow/While.spec.ts @@ -76,19 +76,22 @@ describe('parser while statements', () => { kind: TokenKind.While, text: 'while', isReserved: true, - range: Range.create(0, 0, 0, 5) + range: Range.create(0, 0, 0, 5), + leadingTrivia: [] }, { kind: TokenKind.True, text: 'true', isReserved: true, - range: Range.create(0, 6, 0, 10) + range: Range.create(0, 6, 0, 10), + leadingTrivia: [] }, { kind: TokenKind.Newline, text: '\n', isReserved: false, - range: Range.create(0, 10, 0, 11) + range: Range.create(0, 10, 0, 11), + leadingTrivia: [] }, // loop body isn't significant for location tracking, so helper functions are safe identifier('Rnd'), @@ -100,7 +103,8 @@ describe('parser while statements', () => { kind: TokenKind.EndWhile, text: 'end while', isReserved: false, - range: Range.create(2, 0, 2, 9) + range: Range.create(2, 0, 2, 9), + leadingTrivia: [] }, EOF ]); diff --git a/src/parser/tests/expression/Call.spec.ts b/src/parser/tests/expression/Call.spec.ts index a92e82090..1505ab061 100644 --- a/src/parser/tests/expression/Call.spec.ts +++ b/src/parser/tests/expression/Call.spec.ts @@ -14,7 +14,7 @@ describe('parser call expressions', () => { it('parses named function calls', () => { const { statements, diagnostics } = Parser.parse([ identifier('RebootSystem'), - { kind: TokenKind.LeftParen, text: '(', range: null as any }, + { kind: TokenKind.LeftParen, text: '(', range: null as any, leadingTrivia: [] }, token(TokenKind.RightParen, ')'), EOF ]); @@ -65,7 +65,7 @@ describe('parser call expressions', () => { it('allows closing parentheses on separate line', () => { const { statements, diagnostics } = Parser.parse([ identifier('RebootSystem'), - { kind: TokenKind.LeftParen, text: '(', range: null as any }, + { kind: TokenKind.LeftParen, text: '(', range: null, leadingTrivia: [] }, token(TokenKind.Newline, '\\n'), token(TokenKind.Newline, '\\n'), token(TokenKind.RightParen, ')'), @@ -128,9 +128,9 @@ describe('parser call expressions', () => { it('accepts arguments', () => { const { statements, diagnostics } = Parser.parse([ identifier('add'), - { kind: TokenKind.LeftParen, text: '(', range: null as any }, + { kind: TokenKind.LeftParen, text: '(', range: null as any, leadingTrivia: [] }, token(TokenKind.IntegerLiteral, '1'), - { kind: TokenKind.Comma, text: ',', range: null as any }, + { kind: TokenKind.Comma, text: ',', range: null as any, leadingTrivia: [] }, token(TokenKind.IntegerLiteral, '2'), token(TokenKind.RightParen, ')'), EOF diff --git a/src/parser/tests/statement/PrintStatement.spec.ts b/src/parser/tests/statement/PrintStatement.spec.ts index 4e8a172cc..4bae099e2 100644 --- a/src/parser/tests/statement/PrintStatement.spec.ts +++ b/src/parser/tests/statement/PrintStatement.spec.ts @@ -80,21 +80,24 @@ describe('parser print statements', () => { text: 'print', isReserved: true, range: Range.create(0, 0, 1, 5), - leadingWhitespace: '' + leadingWhitespace: '', + leadingTrivia: [] }, { kind: TokenKind.StringLiteral, text: `"foo"`, isReserved: false, range: Range.create(0, 6, 0, 11), - leadingWhitespace: '' + leadingWhitespace: '', + leadingTrivia: [] }, { kind: TokenKind.Eof, text: '\0', isReserved: false, range: Range.create(0, 11, 0, 12), - leadingWhitespace: '' + leadingWhitespace: '', + leadingTrivia: [] } ]); diff --git a/src/parser/tests/statement/ReturnStatement.spec.ts b/src/parser/tests/statement/ReturnStatement.spec.ts index 0c65b1ea6..9e0f29111 100644 --- a/src/parser/tests/statement/ReturnStatement.spec.ts +++ b/src/parser/tests/statement/ReturnStatement.spec.ts @@ -51,7 +51,7 @@ describe('parser return statements', () => { token(TokenKind.Newline, '\\n'), token(TokenKind.Return, 'return'), identifier('RebootSystem'), - { kind: TokenKind.LeftParen, text: '(', range: null as any }, + { kind: TokenKind.LeftParen, text: '(', range: null, leadingTrivia: [] }, token(TokenKind.RightParen, ')'), token(TokenKind.Newline, '\\n'), token(TokenKind.EndFunction, 'end function'), @@ -81,13 +81,15 @@ describe('parser return statements', () => { kind: TokenKind.Return, text: 'return', isReserved: true, - range: Range.create(1, 2, 1, 8) + range: Range.create(1, 2, 1, 8), + leadingTrivia: [] }, { kind: TokenKind.IntegerLiteral, text: '5', isReserved: false, - range: Range.create(1, 9, 1, 10) + range: Range.create(1, 9, 1, 10), + leadingTrivia: [] }, token(TokenKind.Newline, '\\n'), token(TokenKind.EndFunction, 'end function'), diff --git a/src/parser/tests/statement/Set.spec.ts b/src/parser/tests/statement/Set.spec.ts index e52797d95..cdf2737d7 100644 --- a/src/parser/tests/statement/Set.spec.ts +++ b/src/parser/tests/statement/Set.spec.ts @@ -218,7 +218,7 @@ describe('parser indexed assignment', () => { range: Range.create(1, 10, 1, 11), leadingWhitespace: '' } - ]); + ] as any[]); expect(diagnostics).to.be.empty; expect(statements).to.be.lengthOf(2); diff --git a/src/util.ts b/src/util.ts index db38cfd28..9a1325aeb 100644 --- a/src/util.ts +++ b/src/util.ts @@ -1066,7 +1066,8 @@ export class Util { range: this.cloneRange(token.range), text: token.text, isReserved: token.isReserved, - leadingWhitespace: token.leadingWhitespace + leadingWhitespace: token.leadingWhitespace, + leadingTrivia: token.leadingTrivia?.map(x => this.cloneToken(x)) } as T; //handle those tokens that have charCode if ('charCode' in token) {