From 4a169fe33b29b1afb4b7876d89ff125b347fe6c3 Mon Sep 17 00:00:00 2001 From: duckdoom4 <60387522+duckdoom4@users.noreply.github.com> Date: Tue, 13 Jun 2023 22:13:55 +0200 Subject: [PATCH] Fix infinite tokenizer loop (#341) * Major code cleanup * Fix infinite loop in tokenizer * Fix rgb color not accepting `.0` float format --- .eslintrc.json | 3 +- .vscode/launch.json | 9 +- TODO.md | 1 - package.json | 2 +- src/color.ts | 111 ++++++--- src/completion.ts | 22 +- src/configuration.ts | 56 +++++ src/definition.ts | 20 +- src/diagnostics.ts | 2 - src/displayable.ts | 1 - src/extension.ts | 322 ++++++------------------- src/hover.ts | 16 +- src/logger.ts | 22 +- src/outline.ts | 16 +- src/references.ts | 22 +- src/semantics.ts | 32 ++- src/signature.ts | 24 +- src/tokenizer/debug-decorator.ts | 26 +- src/tokenizer/tokenizer.ts | 391 +++++++++++++++++-------------- src/utilities/utils.ts | 17 ++ 20 files changed, 610 insertions(+), 505 deletions(-) create mode 100644 src/configuration.ts diff --git a/.eslintrc.json b/.eslintrc.json index 5df43ca..02070a4 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -22,6 +22,7 @@ "require-await": "warn", "camelcase": "error", "@typescript-eslint/no-var-requires": "warn", - "no-param-reassign": "warn" // Disable reassign, since this basically means you override the reference from the caller function with a new local version. (It doesn't do what you expect) + "no-param-reassign": "warn", // Disable reassign, since this basically means you override the reference from the caller function with a new local version. (It doesn't do what you expect) + "@typescript-eslint/no-namespace": "off" } } diff --git a/.vscode/launch.json b/.vscode/launch.json index 24bea41..c6210f6 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -15,7 +15,8 @@ "${workspaceFolder}/examples" // open examples directory ], "outFiles": ["${workspaceFolder}/dist/**/*.js"], - "preLaunchTask": "npm: watch" + "preLaunchTask": "npm: watch", + "skipFiles": ["/**", "**/extensions/git*/**", "**/node_modules/prettier/**", "**/node/extensionHostProcess.js"] }, { "name": "Launch Extension (Release)", @@ -27,7 +28,8 @@ "${workspaceFolder}/examples" // open examples directory ], "outFiles": ["${workspaceFolder}/dist/**/*.js"], - "preLaunchTask": "npm: watch-release" + "preLaunchTask": "npm: watch-release", + "skipFiles": ["/**", "**/extensions/git*/**", "**/node_modules/prettier/**"] }, { "name": "Extension Tests", @@ -36,7 +38,8 @@ "runtimeExecutable": "${execPath}", "args": ["--extensionDevelopmentPath=${workspaceFolder}", "--extensionTestsPath=${workspaceFolder}/out/test"], "outFiles": ["${workspaceFolder}/out/test/**/*.js"], - "preLaunchTask": "npm: test-compile" + "preLaunchTask": "npm: test-compile", + "skipFiles": ["/**", "**/extensions/git*/**", "**/node_modules/prettier/**"] } ] } diff --git a/TODO.md b/TODO.md index 96ddf9b..86afdf7 100644 --- a/TODO.md +++ b/TODO.md @@ -3,7 +3,6 @@ Renpy Features List - Support launching the project through VSCode - Bugs to fix: - * Fix error on restart * Show color editor in tags (also check https://www.renpy.org/doc/html/color_class.html) * % can be escaped in strings * if line contains unclosed ( [ or { line is continued (see https://www.renpy.org/doc/html/language_basics.html#logical-lines) diff --git a/package.json b/package.json index 6de502a..51ad98e 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "languague-renpy", "displayName": "Ren'Py Language", "description": "Adds rich support for the Ren'Py programming language to Visual Studio Code.", - "version": "2.3.3", + "version": "2.3.4", "publisher": "LuqueDaniel", "license": "MIT", "homepage": "https://github.com/LuqueDaniel/vscode-language-renpy", diff --git a/src/color.ts b/src/color.ts index 4f21263..a8f3427 100644 --- a/src/color.ts +++ b/src/color.ts @@ -1,30 +1,44 @@ // Color conversion methods for Color provider -import { CancellationToken, Color, ColorInformation, ColorPresentation, DocumentColorProvider, Range, TextDocument, TextEdit } from "vscode"; +import { CancellationToken, Color, ColorInformation, ColorPresentation, ProviderResult, Range, TextDocument, TextEdit, languages } from "vscode"; import { ValueEqualsSet } from "./utilities/hashset"; -import { tokenizeDocument } from "./tokenizer/tokenizer"; +import { Tokenizer } from "./tokenizer/tokenizer"; import { LiteralTokenType } from "./tokenizer/renpy-tokens"; import { TextMateRule, injectCustomTextmateTokens } from "./decorator"; -/*import { tokenizeDocument } from "./tokenizer/tokenizer"; -import { injectCustomTextmateTokens, TextMateRule } from "./decorator"; -import { LiteralTokenType } from "./tokenizer/renpy-tokens"; -import { ValueEqualsSet } from "./utilities/hashset";*/ -export class RenpyColorProvider implements DocumentColorProvider { - public provideDocumentColors(document: TextDocument, token: CancellationToken): Thenable { - return getColorInformation(document); - } - public provideColorPresentations(color: Color, context: { document: TextDocument; range: Range }, token: CancellationToken): Thenable { - return getColorPresentations(color, context.document, context.range); - } -} +export type DocumentColorContext = { + document: TextDocument; + range: Range; +}; + +export const colorProvider = languages.registerColorProvider("renpy", { + provideDocumentColors(document: TextDocument, token: CancellationToken): ProviderResult { + if (token.isCancellationRequested) { + return; + } + + return new Promise((resolve) => { + resolve(getColorInformation(document)); + }); + }, + + provideColorPresentations(color: Color, context: DocumentColorContext, token: CancellationToken): ProviderResult { + if (token.isCancellationRequested) { + return; + } + + return new Promise((resolve) => { + resolve(getColorPresentations(color, context)); + }); + }, +}); /** * Finds all colors in the given document and returns their ranges and color * @param document - the TextDocument to search - * @returns - Thenable - an array that provides a range and color for each match + * @returns - ColorInformation[] - an array that provides a range and color for each match */ -export function getColorInformation(document: TextDocument): Thenable { - injectCustomColorStyles(document); +export async function getColorInformation(document: TextDocument) { + await injectCustomColorStyles(document); // find all colors in the document const colors: ColorInformation[] = []; @@ -67,7 +81,7 @@ export function getColorInformation(document: TextDocument): Thenable { +export function getColorPresentations(color: Color, context: DocumentColorContext): ColorPresentation[] { // user hovered/tapped the color block/return the color they picked const colors: ColorPresentation[] = []; - const line = document.lineAt(range.start.line).text; - const text = line.substring(range.start.character, range.end.character); - const oldRange = new Range(range.start.line, range.start.character, range.start.line, range.start.character + text.length); + const range = context.range; + const text = context.document.getText(range); + const oldRange = new Range(range.start, range.end); const colR = Math.round(color.red * 255); const colG = Math.round(color.green * 255); @@ -112,12 +126,12 @@ export function getColorPresentations(color: Color, document: TextDocument, rang colors.push(rgbColorPres); } - return Promise.resolve(colors); + return colors; } -export function injectCustomColorStyles(document: TextDocument) { +export async function injectCustomColorStyles(document: TextDocument) { // Disabled until filter is added to the tree class - const documentTokens = tokenizeDocument(document); + const documentTokens = await Tokenizer.tokenizeDocument(document); // TODO: Should probably make sure this constant is actually part of a tag, but for now this is fine. const colorTags = documentTokens.filter((x) => x.token?.tokenType === LiteralTokenType.Color); const colorRules = new ValueEqualsSet(); @@ -221,32 +235,55 @@ export function convertHtmlToColor(htmlHex: string): Color | null { */ export function convertRenpyColorToColor(renpy: string): Color | null { try { - const colorTuple = renpy.replace("Color(", "").replace("color", "").replace("=", "").replace(" ", "").replace("(", "[").replace(")", "]"); - const result = JSON.parse(colorTuple); - if (result.length === 3) { - return new Color(parseInt(result[0], 16) / 255, parseInt(result[1], 16) / 255, parseInt(result[2], 16) / 255, 1.0); - } else if (result.length === 4) { - return new Color(parseInt(result[0], 16) / 255, parseInt(result[1], 16) / 255, parseInt(result[2], 16) / 255, parseInt(result[3], 16) / 255); + const colorTuple = renpy + .replaceAll(" ", "") + .replace(/[Cc]olor=?\(/g, "") + .replace(")", ""); + + const result = colorTuple.split(","); + if (result.length < 3) { + return null; } - return null; + + const r = parseInt(result[0], 16) / 255; + const g = parseInt(result[1], 16) / 255; + const b = parseInt(result[2], 16) / 255; + const a = result.length === 4 ? parseInt(result[3], 16) / 255 : 1.0; + return new Color(r, g, b, a); } catch (error) { return null; } } +/** + * Returns a float value based on the given Ren'Py float string value + * @remarks Values starting with a dot (e.g., `.5`) are forced to be parsed as `0.5` due to javascript's `parseFloat` behavior. + * @param value The renpy float value to parse + * @returns The parsed float value + */ +function parseRenpyFloat(value: string): number { + if (value.startsWith(".")) { + return parseFloat("0" + value); + } + return parseFloat(value); +} + /** * Returns a Color provider object based on the given Ren'Py rgb tuple - * @remarks - * The rgb tuple values should be numeric values between 0.0 and 1.0 (e.g., `rgb=(1.0, 0.0, 0.0)`) + * @remarks The rgb tuple values should be numeric values between 0.0 and 1.0 (e.g., `rgb=(1.0, 0.0, 0.0)`). + * Values starting with a dot (e.g., `.5`) are forced to be parsed as `0.5` due to javascript's `parseFloat` behavior. * @param renpyColor - Renpy `rgb` color tuple (e.g., `rgb=(r, g, b)`) * @returns The `Color` provider object */ export function convertRgbColorToColor(renpyColor: string): Color | null { try { - const colorTuple = renpyColor.replace("rgb", "").replace("=", "").replace(" ", "").replace("(", "[").replace(")", "]"); - const result = JSON.parse(colorTuple); + const colorTuple = renpyColor + .replaceAll(" ", "") + .replace(/rgb=\(/g, "") + .replace(")", ""); + const result = colorTuple.split(","); if (result.length === 3) { - return new Color(parseFloat(result[0]), parseFloat(result[1]), parseFloat(result[2]), 1.0); + return new Color(parseRenpyFloat(result[0]), parseRenpyFloat(result[1]), parseRenpyFloat(result[2]), 1.0); } return null; } catch (error) { diff --git a/src/completion.ts b/src/completion.ts index 008bd26..0f89047 100644 --- a/src/completion.ts +++ b/src/completion.ts @@ -1,10 +1,30 @@ // Completion Provider -import { TextDocument, Position, CompletionContext, CompletionItem, CompletionTriggerKind, CompletionItemKind, workspace } from "vscode"; +import { TextDocument, Position, CompletionContext, CompletionItem, CompletionTriggerKind, CompletionItemKind, workspace, languages, CancellationToken, ProviderResult } from "vscode"; import { Displayable } from "./displayable"; import { getDefinitionFromFile } from "./hover"; import { getCurrentContext } from "./navigation"; import { NavigationData } from "./navigation-data"; +export const completionProvider = languages.registerCompletionItemProvider( + "renpy", + { + provideCompletionItems(document: TextDocument, position: Position, token: CancellationToken, context: CompletionContext): ProviderResult { + if (token.isCancellationRequested) { + return; + } + + return new Promise((resolve) => { + resolve(getCompletionList(document, position, context)); + }); + }, + }, + ".", + " ", + "@", + "-", + "(" +); + /** * Returns an array of auto-complete items related to the keyword at the given document/position * @param document - The current TextDocument diff --git a/src/configuration.ts b/src/configuration.ts new file mode 100644 index 0000000..c053958 --- /dev/null +++ b/src/configuration.ts @@ -0,0 +1,56 @@ +import { ConfigurationTarget, ExtensionContext, WorkspaceConfiguration, workspace } from "vscode"; + +export class Configuration { + public static initialize(context: ExtensionContext) { + // hide rpyc files if the setting is enabled + const config = workspace.getConfiguration("renpy"); + if (config?.excludeCompiledFilesFromWorkspace) { + this.excludeCompiledFilesConfig(); + } + + // Listen to configuration changes + context.subscriptions.push( + workspace.onDidChangeConfiguration((e) => { + if (e.affectsConfiguration("renpy.excludeCompiledFilesFromWorkspace")) { + if (workspace.getConfiguration("renpy").get("excludeCompiledFilesFromWorkspace")) { + this.excludeCompiledFilesConfig(); + } + } + }) + ); + } + + public static isAutoSaveDisabled(): boolean { + const config = workspace.getConfiguration("files"); + const autoSave = config.get("autoSave"); + return autoSave === "off"; + } + + public static compileOnDocumentSave(): boolean { + const config = workspace.getConfiguration("renpy"); + return config.get("compileOnDocumentSave") === true; + } + + public static shouldWatchFoldersForChanges(): boolean { + const config = workspace.getConfiguration("renpy"); + return config.get("watchFoldersForChanges") === true; + } + + public static getRenpyExecutablePath(): string { + const config = workspace.getConfiguration("renpy"); + return config.get("renpyExecutableLocation") || ""; + } + + private static excludeCompiledFilesConfig() { + const renpyExclude = ["**/*.rpyc", "**/*.rpa", "**/*.rpymc", "**/cache/"]; + const config = workspace.getConfiguration("files"); + const workspaceExclude = config.inspect("exclude"); + const exclude = { ...workspaceExclude?.workspaceValue }; + renpyExclude.forEach((element) => { + if (!(element in exclude)) { + Object.assign(exclude, { [element]: true }); + } + }); + config.update("exclude", exclude, ConfigurationTarget.Workspace); + } +} diff --git a/src/definition.ts b/src/definition.ts index cacf127..d8648e8 100644 --- a/src/definition.ts +++ b/src/definition.ts @@ -1,16 +1,26 @@ -// Definition Provider -"use strict"; - -import { Definition, Location, Position, TextDocument, Uri } from "vscode"; +// Provider for Go To Definition +import { CancellationToken, Definition, Location, Position, ProviderResult, TextDocument, Uri, languages } from "vscode"; import { getKeywordPrefix } from "./extension"; import { rangeAsString } from "./navigation"; import { NavigationData } from "./navigation-data"; import { getFileWithPath, stripWorkspaceFromFile } from "./workspace"; +export const definitionProvider = languages.registerDefinitionProvider("renpy", { + provideDefinition(document: TextDocument, position: Position, token: CancellationToken): ProviderResult { + if (token.isCancellationRequested) { + return; + } + + return new Promise((resolve) => { + resolve(getDefinition(document, position)); + }); + }, +}); + export function getDefinition(document: TextDocument, position: Position): Definition | undefined { const range = document.getWordRangeAtPosition(position); if (!range) { - return; + return undefined; } // check if this range is a semantic token diff --git a/src/diagnostics.ts b/src/diagnostics.ts index 6388810..f8417aa 100644 --- a/src/diagnostics.ts +++ b/src/diagnostics.ts @@ -1,6 +1,4 @@ // Diagnostics (warnings and errors) -"use strict"; - import { Diagnostic, DiagnosticCollection, DiagnosticSeverity, ExtensionContext, Range, TextDocument, window, workspace } from "vscode"; import { NavigationData } from "./navigation-data"; import { extractFilename } from "./workspace"; diff --git a/src/displayable.ts b/src/displayable.ts index 70b076e..3766bd9 100644 --- a/src/displayable.ts +++ b/src/displayable.ts @@ -1,5 +1,4 @@ // Displayable Class -"use strict"; export class Displayable { name: string; diff --git a/src/extension.ts b/src/extension.ts index 2c6f7fb..a0de162 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -4,184 +4,43 @@ import * as cp from "child_process"; import * as fs from "fs"; -import { - ExtensionContext, - languages, - commands, - window, - TextDocument, - Position, - CancellationToken, - CompletionContext, - CompletionItem, - CompletionItemProvider, - ConfigurationTarget, - Definition, - DefinitionProvider, - DocumentSemanticTokensProvider, - DocumentSymbol, - debug, - DocumentSymbolProvider, - Hover, - HoverProvider, - Location, - ProviderResult, - Range, - ReferenceContext, - ReferenceProvider, - SemanticTokens, - SemanticTokensLegend, - DocumentSelector, - StatusBarItem, - workspace, - WorkspaceConfiguration, - SignatureHelpProvider, - SignatureHelp, - SignatureHelpContext, - StatusBarAlignment, - Uri, -} from "vscode"; -import { RenpyColorProvider } from "./color"; +import { ExtensionContext, languages, commands, window, TextDocument, Position, debug, Range, workspace, Uri } from "vscode"; +import { colorProvider } from "./color"; import { getStatusBarText, NavigationData } from "./navigation-data"; import { cleanUpPath, getAudioFolder, getImagesFolder, getNavigationJsonFilepath, getWorkspaceFolder, stripWorkspaceFromFile } from "./workspace"; import { refreshDiagnostics, subscribeToDocumentChanges } from "./diagnostics"; -import { getSemanticTokens } from "./semantics"; -import { getHover } from "./hover"; -import { getCompletionList } from "./completion"; -import { getDefinition } from "./definition"; -import { getDocumentSymbols } from "./outline"; -import { findAllReferences } from "./references"; +import { semanticTokensProvider } from "./semantics"; +import { hoverProvider } from "./hover"; +import { completionProvider } from "./completion"; +import { definitionProvider } from "./definition"; +import { symbolProvider } from "./outline"; +import { referencesProvider } from "./references"; import { registerDebugDecorator, unregisterDebugDecorator } from "./tokenizer/debug-decorator"; -import { clearTokenCache } from "./tokenizer/tokenizer"; -import { getSignatureHelp } from "./signature"; -import { LogCategory, LogLevel, logCatMessage, logMessage, logToast } from "./logger"; - -const selector: DocumentSelector = { scheme: "file", language: "renpy" }; -let myStatusBarItem: StatusBarItem; +import { Tokenizer } from "./tokenizer/tokenizer"; +import { signatureProvider } from "./signature"; +import { LogLevel, intializeLoggingSystems, logMessage, logToast, updateStatusBar } from "./logger"; +import { Configuration } from "./configuration"; export async function activate(context: ExtensionContext): Promise { - logMessage(LogLevel.Info, "Ren'Py extension activated"); - - const filepath = getNavigationJsonFilepath(); - const jsonFileExists = fs.existsSync(filepath); - if (!jsonFileExists) { - logMessage(LogLevel.Warning, "Navigation.json file is missing."); - } + intializeLoggingSystems(context); + updateStatusBar("$(sync~spin) Loading Ren'Py extension..."); - // hide rpyc files if the setting is enabled - const config = workspace.getConfiguration("renpy"); - if (config?.excludeCompiledFilesFromWorkspace) { - excludeCompiledFilesConfig(); - } + Configuration.initialize(context); - // Listen to configuration changes - context.subscriptions.push( - workspace.onDidChangeConfiguration((e) => { - if (e.affectsConfiguration("renpy.excludeCompiledFilesFromWorkspace")) { - if (workspace.getConfiguration("renpy").get("excludeCompiledFilesFromWorkspace")) { - excludeCompiledFilesConfig(); - } - } - }) - ); - - // hover provider for code tooltip - const hoverProvider = languages.registerHoverProvider( - selector, - new (class implements HoverProvider { - async provideHover(document: TextDocument, position: Position, token: CancellationToken): Promise { - return getHover(document, position); - } - })() - ); + // Subscribe to supported language features context.subscriptions.push(hoverProvider); - - // provider for Go To Definition - const definitionProvider = languages.registerDefinitionProvider( - selector, - new (class implements DefinitionProvider { - provideDefinition(document: TextDocument, position: Position, token: CancellationToken): ProviderResult { - return getDefinition(document, position); - } - })() - ); context.subscriptions.push(definitionProvider); - - // provider for Outline view - const symbolProvider = languages.registerDocumentSymbolProvider( - selector, - new (class implements DocumentSymbolProvider { - provideDocumentSymbols(document: TextDocument, token: CancellationToken): ProviderResult { - return getDocumentSymbols(document); - } - })() - ); context.subscriptions.push(symbolProvider); - - // provider for Method Signature Help - const signatureProvider = languages.registerSignatureHelpProvider( - selector, - new (class implements SignatureHelpProvider { - provideSignatureHelp(document: TextDocument, position: Position, token: CancellationToken, context: SignatureHelpContext): ProviderResult { - return getSignatureHelp(document, position, context); - } - })(), - "(", - ",", - "=" - ); context.subscriptions.push(signatureProvider); - - // Completion provider - const completionProvider = languages.registerCompletionItemProvider( - selector, - new (class implements CompletionItemProvider { - provideCompletionItems(document: TextDocument, position: Position, token: CancellationToken, context: CompletionContext): ProviderResult { - return getCompletionList(document, position, context); - } - })(), - ".", - " ", - "@", - "-", - "(" - ); context.subscriptions.push(completionProvider); - - // Color Provider - const colorProvider = languages.registerColorProvider("renpy", new RenpyColorProvider()); context.subscriptions.push(colorProvider); + context.subscriptions.push(referencesProvider); + context.subscriptions.push(semanticTokensProvider); - // Find All References provider - const references = languages.registerReferenceProvider( - selector, - new (class implements ReferenceProvider { - async provideReferences(document: TextDocument, position: Position, context: ReferenceContext, token: CancellationToken): Promise { - return await findAllReferences(document, position, context); - } - })() - ); - context.subscriptions.push(references); - - const tokenTypes = ["class", "parameter", "variable", "keyword"]; - const tokenModifiers = ["declaration", "defaultLibrary"]; - const legend = new SemanticTokensLegend(tokenTypes, tokenModifiers); - - // Semantic Token Provider - const semanticTokens = languages.registerDocumentSemanticTokensProvider( - selector, - new (class implements DocumentSemanticTokensProvider { - provideDocumentSemanticTokens(document: TextDocument, token: CancellationToken): ProviderResult { - if (document.languageId !== "renpy") { - return; - } else { - return getSemanticTokens(document, legend); - } - } - })(), - legend - ); - context.subscriptions.push(semanticTokens); + // diagnostics (errors and warnings) + const diagnostics = languages.createDiagnosticCollection("renpy"); + context.subscriptions.push(diagnostics); + subscribeToDocumentChanges(context, diagnostics); // A TextDocument was saved context.subscriptions.push( @@ -190,14 +49,12 @@ export async function activate(context: ExtensionContext): Promise { return; } - const filesConfig = workspace.getConfiguration("files"); - if (filesConfig.get("autoSave") === undefined || filesConfig.get("autoSave") !== "off") { + if (Configuration.isAutoSaveDisabled()) { // only trigger document refreshes if file autoSave is off return; } - const config = workspace.getConfiguration("renpy"); - if (config && config.compileOnDocumentSave) { + if (Configuration.compileOnDocumentSave()) { if (!NavigationData.isCompiling) { ExecuteRenpyCompile(); } @@ -214,11 +71,6 @@ export async function activate(context: ExtensionContext): Promise { }) ); - // diagnostics (errors and warnings) - const diagnostics = languages.createDiagnosticCollection("renpy"); - context.subscriptions.push(diagnostics); - subscribeToDocumentChanges(context, diagnostics); - // custom command - refresh data const refreshCommand = commands.registerCommand("renpy.refreshNavigationData", async () => { updateStatusBar("$(sync~spin) Refreshing Ren'Py navigation data..."); @@ -264,7 +116,6 @@ export async function activate(context: ExtensionContext): Promise { }); } }); - context.subscriptions.push(migrateOldFilesCommand); // custom command - refresh diagnostics @@ -277,11 +128,13 @@ export async function activate(context: ExtensionContext): Promise { // custom command - toggle token debug view let isShowingTokenDebugView = false; - const toggleTokenDebugViewCommand = commands.registerCommand("renpy.toggleTokenDebugView", () => { + const toggleTokenDebugViewCommand = commands.registerCommand("renpy.toggleTokenDebugView", async () => { if (!isShowingTokenDebugView) { - clearTokenCache(); - registerDebugDecorator(context); + logToast(LogLevel.Info, "Enabled token debug view"); + Tokenizer.clearTokenCache(); + await registerDebugDecorator(context); } else { + logToast(LogLevel.Info, "Disabled token debug view"); unregisterDebugDecorator(); } isShowingTokenDebugView = !isShowingTokenDebugView; @@ -291,26 +144,28 @@ export async function activate(context: ExtensionContext): Promise { // custom command - call renpy to run workspace const runCommand = commands.registerCommand("renpy.runCommand", () => { //EsLint recommends config be removed as it has already been declared in a previous scope - if (!config || !isValidExecutable(config.renpyExecutableLocation)) { + const rpyPath = Configuration.getRenpyExecutablePath(); + + if (!isValidExecutable(rpyPath)) { logToast(LogLevel.Error, "Ren'Py executable location not configured or is invalid."); - } else { - //this is kinda a hob botched together attempt that I'm like 30% certain has a chance of working - debug.startDebugging( - undefined, - { - type: "cmd", - name: "Run File", - request: "launch", - program: config.renpyExecutableLocation, - }, - { noDebug: true } - ); - - //call renpy - const result = RunWorkspaceFolder(); - if (result) { - logToast(LogLevel.Info, "Ren'Py is running successfully"); - } + return; + } + + debug.startDebugging( + undefined, + { + type: "cmd", + name: "Run File", + request: "launch", + program: rpyPath, + }, + { noDebug: true } + ); + + //call renpy + const result = RunWorkspaceFolder(); + if (result) { + logToast(LogLevel.Info, "Ren'Py is running successfully"); } }); context.subscriptions.push(runCommand); @@ -336,11 +191,11 @@ export async function activate(context: ExtensionContext): Promise { }); context.subscriptions.push(compileCommand); - // Custom status bar - myStatusBarItem = window.createStatusBarItem(StatusBarAlignment.Right, 100); - context.subscriptions.push(myStatusBarItem); - myStatusBarItem.text = "$(sync~spin) Initializing Ren'Py static data..."; - myStatusBarItem.show(); + const filepath = getNavigationJsonFilepath(); + const jsonFileExists = fs.existsSync(filepath); + if (!jsonFileExists) { + logMessage(LogLevel.Warning, "Navigation.json file is missing."); + } // Detect file system change to the navigation.json file and trigger a refresh updateStatusBar("$(sync~spin) Initializing Ren'Py static data..."); @@ -349,23 +204,25 @@ export async function activate(context: ExtensionContext): Promise { try { fs.watch(getNavigationJsonFilepath(), async (event, filename) => { - if (filename) { - logMessage(LogLevel.Debug, `${filename} changed`); - updateStatusBar("$(sync~spin) Refreshing Ren'Py navigation data..."); - try { - await NavigationData.refresh(); - } catch (error) { - logMessage(LogLevel.Error, `${Date()}: error refreshing NavigationData: ${error}`); - } finally { - updateStatusBar(getStatusBarText()); - } + if (!filename) { + return; + } + + logMessage(LogLevel.Debug, `${filename} changed`); + updateStatusBar("$(sync~spin) Refreshing Ren'Py navigation data..."); + try { + await NavigationData.refresh(); + } catch (error) { + logMessage(LogLevel.Error, `${Date()}: error refreshing NavigationData: ${error}`); + } finally { + updateStatusBar(getStatusBarText()); } }); } catch (error) { logMessage(LogLevel.Error, `Watch navigation.json file error: ${error}`); } - if (config && config.watchFoldersForChanges) { + if (Configuration.shouldWatchFoldersForChanges()) { logMessage(LogLevel.Info, "Starting Watcher for images folder."); try { fs.watch(getImagesFolder(), { recursive: true }, async (event, filename) => { @@ -390,6 +247,8 @@ export async function activate(context: ExtensionContext): Promise { logMessage(LogLevel.Error, `Watch audio folder error: ${error}`); } } + + logMessage(LogLevel.Info, "Ren'Py extension activated!"); } export function deactivate() { @@ -423,29 +282,6 @@ export function getKeywordPrefix(document: TextDocument, position: Position, ran return; } -function updateStatusBar(text: string) { - if (text === "") { - myStatusBarItem.hide(); - } else { - logCatMessage(LogLevel.Info, LogCategory.Status, text); - myStatusBarItem.text = text; - myStatusBarItem.show(); - } -} - -function excludeCompiledFilesConfig() { - const renpyExclude = ["**/*.rpyc", "**/*.rpa", "**/*.rpymc", "**/cache/"]; - const config = workspace.getConfiguration("files"); - const workspaceExclude = config.inspect("exclude"); - const exclude = { ...workspaceExclude?.workspaceValue }; - renpyExclude.forEach((element) => { - if (!(element in exclude)) { - Object.assign(exclude, { [element]: true }); - } - }); - config.update("exclude", exclude, ConfigurationTarget.Workspace); -} - function isValidExecutable(renpyExecutableLocation: string): boolean { if (!renpyExecutableLocation || renpyExecutableLocation === "") { return false; @@ -454,18 +290,17 @@ function isValidExecutable(renpyExecutableLocation: string): boolean { } // Attempts to run renpy executable through console commands. function RunWorkspaceFolder(): boolean { - const config = workspace.getConfiguration("renpy"); + const rpyPath = Configuration.getRenpyExecutablePath(); - if (config && isValidExecutable(config.renpyExecutableLocation)) { - const renpy = config.renpyExecutableLocation; - const renpyPath = cleanUpPath(Uri.file(renpy).path); + if (isValidExecutable(rpyPath)) { + const renpyPath = cleanUpPath(Uri.file(rpyPath).path); const cwd = renpyPath.substring(0, renpyPath.lastIndexOf("/")); const workfolder = getWorkspaceFolder(); const args: string[] = [`${workfolder}`, "run"]; if (workfolder.endsWith("/game")) { try { updateStatusBar("$(sync~spin) Running Ren'Py..."); - const result = cp.spawnSync(renpy, args, { + const result = cp.spawnSync(rpyPath, args, { cwd: `${cwd}`, env: { PATH: process.env.PATH }, }); @@ -493,10 +328,9 @@ function RunWorkspaceFolder(): boolean { } function ExecuteRenpyCompile(): boolean { - const config = workspace.getConfiguration("renpy"); - const renpy = config.renpyExecutableLocation; - if (isValidExecutable(renpy)) { - const renpyPath = cleanUpPath(Uri.file(renpy).path); + const rpyPath = Configuration.getRenpyExecutablePath(); + if (isValidExecutable(rpyPath)) { + const renpyPath = cleanUpPath(Uri.file(rpyPath).path); const cwd = renpyPath.substring(0, renpyPath.lastIndexOf("/")); let wf = getWorkspaceFolder(); @@ -509,7 +343,7 @@ function ExecuteRenpyCompile(): boolean { try { NavigationData.isCompiling = true; updateStatusBar("$(sync~spin) Compiling Ren'Py navigation data..."); - const result = cp.spawnSync(renpy, args, { + const result = cp.spawnSync(rpyPath, args, { cwd: `${cwd}`, env: { PATH: process.env.PATH }, encoding: "utf-8", diff --git a/src/hover.ts b/src/hover.ts index 8d2d1b8..46e33b8 100644 --- a/src/hover.ts +++ b/src/hover.ts @@ -1,14 +1,26 @@ // Hover Provider "use strict"; -import { Hover, MarkdownString, Position, Range, TextDocument, Uri } from "vscode"; +import { CancellationToken, Hover, MarkdownString, Position, ProviderResult, Range, TextDocument, Uri, languages } from "vscode"; import { getKeywordPrefix } from "./extension"; import { rangeAsString, Navigation, getPyDocsAtLine, formatDocumentationAsMarkdown } from "./navigation"; import { NavigationData } from "./navigation-data"; import { stripWorkspaceFromFile, extractFilename, getFileWithPath } from "./workspace"; import * as fs from "fs"; -export function getHover(document: TextDocument, position: Position): Hover | null | undefined { +export const hoverProvider = languages.registerHoverProvider("renpy", { + provideHover(document: TextDocument, position: Position, token: CancellationToken): ProviderResult { + if (token.isCancellationRequested) { + return; + } + + return new Promise((resolve) => { + resolve(getHoverContent(document, position)); + }); + }, +}); + +export function getHoverContent(document: TextDocument, position: Position): Hover | null | undefined { let range = document.getWordRangeAtPosition(position); if (!range) { return undefined; diff --git a/src/logger.ts b/src/logger.ts index 605db81..dc69c1f 100644 --- a/src/logger.ts +++ b/src/logger.ts @@ -1,6 +1,26 @@ -import { window } from "vscode"; +import { ExtensionContext, StatusBarAlignment, window } from "vscode"; const outputChannel = window.createOutputChannel("Ren'Py Language Extension", "renpy-log"); +const statusBar = window.createStatusBarItem(StatusBarAlignment.Right, 100); + +export function intializeLoggingSystems(context: ExtensionContext) { + context.subscriptions.push(outputChannel); + + statusBar.name = "Ren'Py Language Extension Status"; + statusBar.tooltip = "Ren'Py Language Extension Status"; + context.subscriptions.push(statusBar); +} + +export function updateStatusBar(text: string) { + if (text === "") { + statusBar.hide(); + return; + } + + logCatMessage(LogLevel.Info, LogCategory.Status, text); + statusBar.text = text; + statusBar.show(); +} // eslint-disable-next-line no-shadow export const enum LogLevel { diff --git a/src/outline.ts b/src/outline.ts index e87eca3..baefbd0 100644 --- a/src/outline.ts +++ b/src/outline.ts @@ -1,11 +1,21 @@ // Document Symbol (Outline) Provider -"use strict"; - -import { TextDocument, DocumentSymbol, Uri, Range, SymbolKind } from "vscode"; +import { TextDocument, DocumentSymbol, Uri, Range, SymbolKind, languages, CancellationToken, ProviderResult } from "vscode"; import { Navigation } from "./navigation"; import { NavigationData } from "./navigation-data"; import { stripWorkspaceFromFile } from "./workspace"; +export const symbolProvider = languages.registerDocumentSymbolProvider("renpy", { + provideDocumentSymbols(document: TextDocument, token: CancellationToken): ProviderResult { + if (token.isCancellationRequested) { + return; + } + + return new Promise((resolve) => { + resolve(getDocumentSymbols(document)); + }); + }, +}); + /** * Gets an array of Document Symbols for the given TextDocument used to populate the editor's Outline view * @param document - The current TextDocument diff --git a/src/references.ts b/src/references.ts index 7270ba3..84d6534 100644 --- a/src/references.ts +++ b/src/references.ts @@ -1,10 +1,18 @@ -// References Provider -"use strict"; - -import { TextDocument, Position, ReferenceContext, Location, workspace } from "vscode"; +// Find all References Provider +import { TextDocument, Position, ReferenceContext, Location, workspace, languages, CancellationToken } from "vscode"; import { getKeywordPrefix } from "./extension"; import { NavigationData } from "./navigation-data"; +export const referencesProvider = languages.registerReferenceProvider("renpy", { + async provideReferences(document: TextDocument, position: Position, context: ReferenceContext, token: CancellationToken) { + if (token.isCancellationRequested) { + return; + } + + return await findAllReferences(document, position, context); + }, +}); + /** * Returns an array of Locations that describe all matches for the keyword at the current position * @param document - The current text document @@ -16,7 +24,7 @@ export async function findAllReferences(document: TextDocument, position: Positi const range = document.getWordRangeAtPosition(position); let keyword = document.getText(range); if (!keyword) { - return; + return undefined; } if (range) { @@ -30,8 +38,8 @@ export async function findAllReferences(document: TextDocument, position: Positi const files = await workspace.findFiles("**/*.rpy"); if (files && files.length > 0) { for (const file of files) { - document = await workspace.openTextDocument(file); - const locations = findReferenceMatches(keyword, document); + const textDocument = await workspace.openTextDocument(file); + const locations = findReferenceMatches(keyword, textDocument); if (locations) { for (const l of locations) { references.push(l); diff --git a/src/semantics.ts b/src/semantics.ts index 60f9f59..24bcce5 100644 --- a/src/semantics.ts +++ b/src/semantics.ts @@ -1,13 +1,35 @@ -// Semantic Tokens -"use strict"; - -import { Position, Range, SemanticTokens, SemanticTokensBuilder, SemanticTokensLegend, TextDocument } from "vscode"; +// Semantic Token Provider +import { CancellationToken, Position, ProviderResult, Range, SemanticTokens, SemanticTokensBuilder, SemanticTokensLegend, TextDocument, languages } from "vscode"; import { Navigation, splitParameters, rangeAsString, getCurrentContext, DataType } from "./navigation"; import { NavigationData, updateNavigationData } from "./navigation-data"; import { stripWorkspaceFromFile } from "./workspace"; import { LogLevel, logMessage } from "./logger"; -export function getSemanticTokens(document: TextDocument, legend: SemanticTokensLegend): SemanticTokens { +const tokenTypes = ["class", "parameter", "variable", "keyword"]; +const tokenModifiers = ["declaration", "defaultLibrary"]; +const legend = new SemanticTokensLegend(tokenTypes, tokenModifiers); + +export const semanticTokensProvider = languages.registerDocumentSemanticTokensProvider( + "renpy", + { + provideDocumentSemanticTokens(document: TextDocument, token: CancellationToken): ProviderResult { + if (token.isCancellationRequested) { + return; + } + + if (document.languageId !== "renpy") { + return; + } + + return new Promise((resolve) => { + resolve(getSemanticTokens(document)); + }); + }, + }, + legend +); + +export function getSemanticTokens(document: TextDocument): SemanticTokens { const tokensBuilder = new SemanticTokensBuilder(legend); const rxKeywordList = /\s*(screen|label|transform|def|class)\s+/; const rxParameterList = diff --git a/src/signature.ts b/src/signature.ts index 1c40a38..b903c7f 100644 --- a/src/signature.ts +++ b/src/signature.ts @@ -1,11 +1,27 @@ -// Signature Provider -"use strict"; - -import { TextDocument, Position, SignatureHelp, SignatureHelpContext } from "vscode"; +// Signature Provider for Method Signature Help +import { TextDocument, Position, SignatureHelp, SignatureHelpContext, languages, CancellationToken, ProviderResult } from "vscode"; import { getKeywordPrefix } from "./extension"; import { getArgumentParameterInfo } from "./navigation"; import { NavigationData } from "./navigation-data"; +export const signatureProvider = languages.registerSignatureHelpProvider( + "renpy", + { + provideSignatureHelp(document: TextDocument, position: Position, token: CancellationToken, context: SignatureHelpContext): ProviderResult { + if (token.isCancellationRequested) { + return; + } + + return new Promise((resolve) => { + resolve(getSignatureHelp(document, position, context)); + }); + }, + }, + "(", + ",", + "=" +); + /** * Gets method signature help for the keyword at the given position in the given document * @param document - The current TextDocument diff --git a/src/tokenizer/debug-decorator.ts b/src/tokenizer/debug-decorator.ts index 0421e8c..f0ca499 100644 --- a/src/tokenizer/debug-decorator.ts +++ b/src/tokenizer/debug-decorator.ts @@ -2,7 +2,7 @@ import { performance } from "perf_hooks"; import { DecorationOptions, Disposable, ExtensionContext, MarkdownString, Uri, window, workspace } from "vscode"; import { CharacterTokenType, LiteralTokenType, EntityTokenType, EscapedCharacterTokenType, KeywordTokenType, MetaTokenType, OperatorTokenType } from "./renpy-tokens"; import { TokenTree, tokenTypeToStringMap } from "./token-definitions"; -import { tokenizeDocument } from "./tokenizer"; +import { Tokenizer } from "./tokenizer"; import { LogLevel, logMessage, logToast } from "../logger"; let timeout: NodeJS.Timer | undefined = undefined; @@ -150,24 +150,24 @@ let documentUri: Uri | null = null; let textChangedEvent: Disposable | null = null; let activeEditorChangedEvent: Disposable | null = null; -export function registerDebugDecorator(context: ExtensionContext) { - triggerUpdateDecorations(); +export async function registerDebugDecorator(context: ExtensionContext) { + await triggerUpdateDecorations(); // A TextDocument was changed context.subscriptions.push( - (textChangedEvent = workspace.onDidChangeTextDocument((event) => { + (textChangedEvent = workspace.onDidChangeTextDocument(async (event) => { const activeEditor = window.activeTextEditor; if (activeEditor && event.document === activeEditor.document) { - triggerUpdateDecorations(true); + await triggerUpdateDecorations(true); } })) ); // The active text editor was changed context.subscriptions.push( - (activeEditorChangedEvent = window.onDidChangeActiveTextEditor(() => { - triggerUpdateDecorations(); + (activeEditorChangedEvent = window.onDidChangeActiveTextEditor(async () => { + await triggerUpdateDecorations(); })) ); } @@ -185,7 +185,7 @@ export function unregisterDebugDecorator() { allDecorationTypes.forEach((x) => activeEditor.setDecorations(x, [])); } -function updateDecorations() { +async function updateDecorations() { const activeEditor = window.activeTextEditor; if (!activeEditor) { return; @@ -197,7 +197,7 @@ function updateDecorations() { // Update tokens only if document has changed const t0 = performance.now(); - tokenCache = tokenizeDocument(activeEditor.document); + tokenCache = await Tokenizer.tokenizeDocument(activeEditor.document); const t1 = performance.now(); logToast(LogLevel.Info, `DocumentTokenizer took ${(t1 - t0).toFixed(2)} milliseconds to complete.`); @@ -669,14 +669,16 @@ ${(decoration.hoverMessage as MarkdownString).value}` activeEditor.setDecorations(deprecatedDecorationType, deprecated); } -function triggerUpdateDecorations(throttle = false) { +async function triggerUpdateDecorations(throttle = false) { if (timeout) { clearTimeout(timeout); timeout = undefined; } if (throttle) { - timeout = setTimeout(updateDecorations, 500); + timeout = setTimeout(async () => { + await updateDecorations(); + }, 500); } else { - updateDecorations(); + await updateDecorations(); } } diff --git a/src/tokenizer/tokenizer.ts b/src/tokenizer/tokenizer.ts index 0c7b521..8973356 100644 --- a/src/tokenizer/tokenizer.ts +++ b/src/tokenizer/tokenizer.ts @@ -8,6 +8,7 @@ import { Stack } from "../utilities/stack"; import { Vector } from "../utilities/vector"; import { TokenPatternCapture, TokenCapturePattern, TokenRepoPattern, TokenRangePattern, TokenMatchPattern } from "./token-pattern-types"; import { LogCategory, LogLevel, logCatMessage } from "../logger"; +import { escapeRegExpCharacters } from "../utilities/utils"; const cloneScanResult = (obj: ScanResult | undefined): ScanResult | undefined => { if (obj === undefined) return undefined; @@ -42,186 +43,213 @@ interface RangeScanResult { type ScanResult = MatchScanResult | RangeScanResult | null; type TokenCache = { readonly documentVersion: number; readonly tokens: TokenTree }; -const tokenCache = new Map(); -const runBenchmark = false; -let uniquePatternCount = -1; +const RUN_BENCHMARKS = false; +//const TOKENIZER_TIMEOUT = 5_000; -export function tokenizeDocument(document: TextDocument): TokenTree { - setupAndValidatePatterns(); +export class Tokenizer { + private static _uniquePatternCount = -1; + private static _tokenCache = new Map(); - if (runBenchmark) { - benchmark(document); + public static get UNIQUE_PATTERN_COUNT() { + return this._uniquePatternCount; } - const cachedTokens = tokenCache.get(document.uri); - if (cachedTokens?.documentVersion === document.version) { - return cachedTokens.tokens; - } - - logCatMessage(LogLevel.Info, LogCategory.Tokenizer, `Running tokenizer on document: ${document.fileName}`); - const tokenizer = new DocumentTokenizer(document); - logCatMessage(LogLevel.Info, LogCategory.Tokenizer, `Tokenizer completed!`); - tokenCache.set(document.uri, { documentVersion: document.version, tokens: tokenizer.tokens }); - return tokenizer.tokens; -} + public static async tokenizeDocument(document: TextDocument) { + this.setupAndValidatePatterns(); -export function clearTokenCache() { - tokenCache.clear(); -} + if (RUN_BENCHMARKS) { + this.benchmark(document); + } -// eslint-disable-next-line @typescript-eslint/no-unused-vars -function benchmark(document: TextDocument) { - // screens.rpy, 10000 loops; 19.69293530988693 avg. + const cachedTokens = this._tokenCache.get(document.uri); + if (cachedTokens?.documentVersion === document.version) { + return cachedTokens.tokens; + } - const loops = 10000; - const reportEveryXPercent = 1; + return await this.runTokenizer(document); + } - const onePercent = loops / 100; - const everyX = onePercent * reportEveryXPercent; + public static clearTokenCache() { + this._tokenCache.clear(); + } - logCatMessage(LogLevel.Info, LogCategory.Tokenizer, `Running tokenizer benchmark for ${loops} loops...`); + private static async runTokenizer(document: TextDocument) { + logCatMessage(LogLevel.Info, LogCategory.Tokenizer, `Running tokenizer on document: ${document.fileName}`); + const tokenizer = new DocumentTokenizer(document); + await Promise.resolve(tokenizer.tokenize()); + + // TODO: Need to mark all these functions async for this to work properly + /*await withTimeout(, TOKENIZER_TIMEOUT, () => { + // If the tokenizer times out, we still want to cache the document so that we don't try to tokenize it again + this._tokenCache.set(document.uri, { documentVersion: document.version, tokens: new TokenTree() }); + logToast(LogLevel.Info, `Tokenizer timed out after ${TOKENIZER_TIMEOUT}ms, while attempting to tokenize the document at: ${document.uri}`); + });*/ + + logCatMessage(LogLevel.Info, LogCategory.Tokenizer, `Tokenizer completed!`); + this._tokenCache.set(document.uri, { documentVersion: document.version, tokens: tokenizer.tokens }); + return tokenizer.tokens; + } - let avg = 0; - for (let i = 0; i < loops; ++i) { - const t0 = performance.now(); - const tokenizer: DocumentTokenizer = new DocumentTokenizer(document); - const t1 = performance.now(); + private static benchmark(document: TextDocument) { + // screens.rpy, 10000 loops; 19.69293530988693 avg. - avg += t1 - t0; + const loops = 10000; + const reportEveryXPercent = 1; - // This is really just here to prevent the unused variable error - if (tokenizer.tokens.isEmpty()) { - logCatMessage(LogLevel.Error, LogCategory.Tokenizer, "No tokens were found."); - } + const onePercent = loops / 100; + const everyX = onePercent * reportEveryXPercent; - // Show timer - const msLoop = avg / (i + 1); - const sRemaining = msLoop * (loops - i + 1) * 0.001; - const h = Math.floor(sRemaining / 3600); - const m = Math.floor((sRemaining / 60) % 60); - const s = Math.ceil(sRemaining % 60); - const timeString = [h ? h + "h" : false, m ? m + "m" : false, s ? s + "s" : false] - .filter(Boolean) - .join(":") - .replace(/\b(\d)(?=[hms])/g, "0$1"); + logCatMessage(LogLevel.Info, LogCategory.Tokenizer, `Running tokenizer benchmark for ${loops} loops...`); - if (i % everyX === 0) { - logCatMessage(LogLevel.Info, LogCategory.Tokenizer, `${i / onePercent}% complete... (avg.: ${msLoop.toFixed(2)}ms, approx. ${timeString} remaining)`); - } - } - avg /= loops; + let avg = 0; + for (let i = 0; i < loops; ++i) { + const t0 = performance.now(); + const tokenizer: DocumentTokenizer = new DocumentTokenizer(document); + const t1 = performance.now(); - logCatMessage(LogLevel.Info, LogCategory.Tokenizer, `DocumentTokenizer took ${avg} avg. milliseconds to complete.`); -} + avg += t1 - t0; -function setupAndValidatePatterns() { - if (uniquePatternCount !== -1) return; + // This is really just here to prevent the unused variable error + if (tokenizer.tokens.isEmpty()) { + logCatMessage(LogLevel.Error, LogCategory.Tokenizer, "No tokens were found."); + } - uniquePatternCount = 0; - const stack = new Stack(32); - stack.push(RenpyPatterns.basePatterns as ExTokenRepoPattern); + // Show timer + const msLoop = avg / (i + 1); + const sRemaining = msLoop * (loops - i + 1) * 0.001; + const h = Math.floor(sRemaining / 3600); + const m = Math.floor((sRemaining / 60) % 60); + const s = Math.ceil(sRemaining % 60); + const timeString = [h ? h + "h" : false, m ? m + "m" : false, s ? s + "s" : false] + .filter(Boolean) + .join(":") + .replace(/\b(\d)(?=[hms])/g, "0$1"); + + if (i % everyX === 0) { + logCatMessage(LogLevel.Info, LogCategory.Tokenizer, `${i / onePercent}% complete... (avg.: ${msLoop.toFixed(2)}ms, approx. ${timeString} remaining)`); + } + } + avg /= loops; - const mFlagRe = /(?(32); + stack.push(RenpyPatterns.basePatterns as ExTokenRepoPattern); - if (isRepoPattern(p)) { - p._patternType = TokenPatternType.RepoPattern; - for (let i = 0; i < p.patterns.length; ++i) stack.push(p.patterns[i]); - } else if (isRangePattern(p)) { - p._patternType = TokenPatternType.RangePattern; - let reBeginSource = p.begin.source; + const mFlagRe = /(? { - if (v.patterns) stack.push(v as ExTokenRepoPattern); - }); - } else { - assert(!p.begin.hasIndices, "This pattern should not have the 'd' flag set!"); - } - if (p.endCaptures) { - assert(p.end.hasIndices, "To match this end pattern the 'd' flag is required!"); + if (reBeginSource.match(mFlagRe)) { + assert(p.begin.multiline, "To match this pattern the 'm' flag is required on the begin RegExp!"); + } - Object.entries(p.endCaptures).forEach(([, v]) => { - if (v.patterns) stack.push(v as ExTokenRepoPattern); - }); - } else { - assert(!p.end.hasIndices, "This pattern should not have the 'd' flag set!"); - } + if (gAnchorRe.test(reBeginSource)) { + assert("Can't use the \\G anchor, please update the regex!"); + reBeginSource = reBeginSource.replace(gAnchorRe, ""); + } - if (p.patterns) { - p._patternsRepo = { - patterns: p.patterns as ExTokenPatternArray, - _patternId: -1, - _patternType: TokenPatternType.RepoPattern, - }; - stack.push(p._patternsRepo!); - } + assert(p.begin.global && p.end.global, "To match this pattern the 'g' flag is required on the begin and end RegExp!"); + if (p.beginCaptures) { + assert(p.begin.hasIndices, "To match this begin pattern the 'd' flag is required!"); - let reEndSource = p.end.source; + Object.entries(p.beginCaptures).forEach(([, v]) => { + if (v.patterns) stack.push(v as ExTokenRepoPattern); + }); + } else { + assert(!p.begin.hasIndices, "This pattern should not have the 'd' flag set!"); + } + if (p.endCaptures) { + assert(p.end.hasIndices, "To match this end pattern the 'd' flag is required!"); + + Object.entries(p.endCaptures).forEach(([, v]) => { + if (v.patterns) stack.push(v as ExTokenRepoPattern); + }); + } else { + assert(!p.end.hasIndices, "This pattern should not have the 'd' flag set!"); + } - if (mFlagRe.test(reEndSource)) { - assert(p.end.multiline, "To match this pattern the 'm' flag is required on the end RegExp!"); - } + if (p.patterns) { + p._patternsRepo = { + patterns: p.patterns as ExTokenPatternArray, + _patternId: -1, + _patternType: TokenPatternType.RepoPattern, + }; + stack.push(p._patternsRepo!); + } - if (gAnchorRe.test(reEndSource)) { - logCatMessage(LogLevel.Warning, LogCategory.Tokenizer, "\\G anchor is not supported and will be ignored!", true); - reEndSource = reEndSource.replace(gAnchorRe, ""); - } + let reEndSource = p.end.source; - p._hasBackref = /\\\d+/.test(reEndSource); - //reEndSource = reEndSource.replaceAll("\\A", "¨0"); - reEndSource = reEndSource.replaceAll("\\Z", "$(?!\r\n|\r|\n)"); - reEndSource = reEndSource.replaceAll("\\R", "(?!\r\n|\r|\n)"); + if (mFlagRe.test(reEndSource)) { + assert(p.end.multiline, "To match this pattern the 'm' flag is required on the end RegExp!"); + } - p.begin = new RegExp(reBeginSource, p.begin.flags); - p.end = new RegExp(reEndSource, p.end.flags); - } else if (isMatchPattern(p)) { - p._patternType = TokenPatternType.MatchPattern; + p._endNotG = false; + if (gAnchorRe.test(reEndSource)) { + if (reEndSource.startsWith("(?!\\G)")) { + p._endNotG = true; + } else { + assert("The end patterns only supports (?!\\G) at the start of the regex. Please update the regex!"); + reEndSource = reEndSource.replace(gAnchorRe, ""); + } + } - if (p.match.source.match(mFlagRe)) { - assert(p.match.multiline, "To match this pattern the 'm' flag is required!"); - } + p._hasBackref = /\\\d+/.test(reEndSource); + //reEndSource = reEndSource.replaceAll("\\A", "¨0"); + reEndSource = reEndSource.replaceAll("\\Z", "$(?!\r\n|\r|\n)"); + reEndSource = reEndSource.replaceAll("\\R", "(?!\r\n|\r|\n)"); - assert(p.match.global, "To match this pattern the 'g' flag is required!"); - if (p.captures) { - assert(p.match.hasIndices, "To match this pattern the 'd' flag is required!"); + p.begin = new RegExp(reBeginSource, p.begin.flags); + p.end = new RegExp(reEndSource, p.end.flags); + } else if (isMatchPattern(p)) { + p._patternType = TokenPatternType.MatchPattern; - Object.entries(p.captures).forEach(([, v]) => { - if (v.patterns) stack.push(v as ExTokenRepoPattern); - }); + if (p.match.source.match(mFlagRe)) { + assert(p.match.multiline, "To match this pattern the 'm' flag is required!"); + } + + assert(p.match.global, "To match this pattern the 'g' flag is required!"); + if (p.captures) { + assert(p.match.hasIndices, "To match this pattern the 'd' flag is required!"); + + Object.entries(p.captures).forEach(([, v]) => { + if (v.patterns) stack.push(v as ExTokenRepoPattern); + }); + } else { + assert(!p.match.hasIndices, "This pattern should not have the 'd' flag set!"); + } } else { - assert(!p.match.hasIndices, "This pattern should not have the 'd' flag set!"); + assert(false, "Should not get here!"); } - } else { - assert(false, "Should not get here!"); } } } -export function escapeRegExpCharacters(value: string): string { - return value.replace(/[-\\{}*+?|^$.,[\]()#\s]/g, "\\$&"); -} - class DocumentTokenizer { private readonly backrefReplaceRe = /\\(\d+)/g; public readonly tokens: TokenTree = new TokenTree(); @@ -229,7 +257,10 @@ class DocumentTokenizer { constructor(document: TextDocument) { this.document = document; - const text = document.getText(); + } + + public tokenize() { + const text = this.document.getText(); this.executePattern(RenpyPatterns.basePatterns as ExTokenRepoPattern, text, new Range(0, text.length), this.tokens.root); } @@ -341,12 +372,11 @@ class DocumentTokenizer { const re = pattern.match; re.lastIndex = sourceStartOffset; const match = re.exec(source); - if (match) { - const result = { pattern, matchBegin: match }; - return result; + if (!match) { + return null; } - return null; + return { pattern, matchBegin: match }; } private scanRangePattern(pattern: ExTokenRangePattern, source: string, sourceStartOffset: number): RangeScanResult | null { @@ -354,11 +384,11 @@ class DocumentTokenizer { reBegin.lastIndex = sourceStartOffset; const matchBegin = reBegin.exec(source); - if (matchBegin) { - return { pattern, matchBegin: matchBegin, matchEnd: null, expanded: false, contentMatches: null, source }; + if (!matchBegin) { + return null; } - return null; + return { pattern, matchBegin: matchBegin, matchEnd: null, expanded: false, contentMatches: null, source }; } private expandRangeScanResult(result: RangeScanResult, cache: Array) { @@ -385,42 +415,53 @@ class DocumentTokenizer { // Start end pattern after the last matched character in the begin pattern reEnd.lastIndex = matchBegin.index + matchBegin[0].length; + if (p._endNotG) { + ++reEnd.lastIndex; + } let matchEnd = reEnd.exec(result.source); const contentMatches = new Stack(); if (matchEnd) { // Check if any child pattern has content that would extend the currently determined end match if (p._patternsRepo) { - const contentStartIndex = matchBegin.index + matchBegin[0].length; - const contentEndIndex = matchEnd.index; + const sourceRange = new Range(matchBegin.index + matchBegin[0].length, matchEnd.index); const lastMatchedChar = matchEnd.index + matchEnd[0].length; // Scan the content for any matches that would extend beyond the current end match const tempCache = cloneCache(cache); //const tempCache = new Array(uniquePatternCount).fill(undefined); - for (let lastMatchIndex = contentStartIndex; lastMatchIndex < contentEndIndex; ) { - const bestChildMatch = this.scanPattern(p._patternsRepo, result.source, lastMatchIndex, tempCache); - if (!bestChildMatch) { - break; // No more matches + const lastCharIndex = sourceRange.end; + let lastMatchIndex = sourceRange.start; + while (lastMatchIndex < lastCharIndex) { + const bestMatch = this.scanPattern(p._patternsRepo, result.source, lastMatchIndex, tempCache); + + if (!bestMatch || bestMatch.matchBegin.index >= lastCharIndex) { + break; // No valid match was found in the remaining text. Break the loop } + const failSafeIndex = lastMatchIndex; // Debug index to break in case of an infinite loop + // Update the last match index to the end of the child match, so the next scan starts after it - const childMatchBegin = bestChildMatch.matchBegin; - if (bestChildMatch.pattern._patternType === TokenPatternType.RangePattern) { - const childMatchEnd = bestChildMatch.matchEnd!; + const childMatchBegin = bestMatch.matchBegin; + if (bestMatch.pattern._patternType === TokenPatternType.RangePattern) { + if (bestMatch?.pattern._patternType === TokenPatternType.RangePattern && !bestMatch.expanded) { + this.expandRangeScanResult(bestMatch as RangeScanResult, cache); + } + + const childMatchEnd = bestMatch.matchEnd!; lastMatchIndex = childMatchEnd.index + childMatchEnd[0].length; } else { lastMatchIndex = childMatchBegin.index + childMatchBegin[0].length; } - // Check if the match starts after the currently determined end match start, if so we ignore it - if (childMatchBegin.index >= contentEndIndex) { - continue; + if (failSafeIndex === lastMatchIndex) { + logCatMessage(LogLevel.Error, LogCategory.Tokenizer, "The range expand loop has not advanced since the last cycle. This indicates a programming error. Breaking the loop!", true); + break; } // To speed up the search, we can add any tokens that are within the content range - contentMatches.push(bestChildMatch); + contentMatches.push(bestMatch); // If the child match last char doesn't extend the current range, we can also ignore it if (lastMatchIndex <= lastMatchedChar) { @@ -518,7 +559,9 @@ class DocumentTokenizer { cache[next._patternId] = scanResult; } - if (!scanResult) continue; + if (!scanResult) { + continue; + } const matchRating = scanResult.matchBegin.index; if (matchRating >= bestMatchRating) { @@ -537,10 +580,6 @@ class DocumentTokenizer { cache[p._patternId] = bestResult; - if (bestResult?.pattern._patternType === TokenPatternType.RangePattern && !bestResult.expanded) { - this.expandRangeScanResult(bestResult as RangeScanResult, cache); - } - return bestResult; } @@ -579,12 +618,12 @@ class DocumentTokenizer { // Patterns are only applied on 'content' (see p.contentToken above) if (p._patternsRepo) { - /*while (!bestMatch.contentMatches!.isEmpty()) { + while (!bestMatch.contentMatches!.isEmpty()) { const contentScanResult = bestMatch.contentMatches!.pop()!; this.applyScanResult(contentScanResult, source, contentNode); - }*/ + } - this.executePattern(p._patternsRepo, source, new Range(contentStart, matchEnd!.index), contentNode); + //this.executePattern(p._patternsRepo, source, new Range(contentStart, matchEnd!.index), contentNode); } if (!contentNode.isEmpty()) { @@ -673,23 +712,24 @@ class DocumentTokenizer { return; } - const cache = new Array(uniquePatternCount).fill(undefined); + const cache = new Array(Tokenizer.UNIQUE_PATTERN_COUNT).fill(undefined); const lastCharIndex = sourceRange.end; let lastMatchIndex = sourceRange.start; while (lastMatchIndex < lastCharIndex) { const bestMatch = this.scanPattern(pattern, source, lastMatchIndex, cache); - if (bestMatch === null || bestMatch.matchBegin.index >= lastCharIndex) { - // No match was found in the remaining text. Break the loop - lastMatchIndex = lastCharIndex; - continue; + if (!bestMatch || bestMatch.matchBegin.index >= lastCharIndex) { + break; // No valid match was found in the remaining text. Break the loop } const failSafeIndex = lastMatchIndex; // Debug index to break in case of an infinite loop if (bestMatch.pattern._patternType === TokenPatternType.RangePattern) { - assert(bestMatch.expanded, "A RangePattern must be expanded!"); + if (bestMatch?.pattern._patternType === TokenPatternType.RangePattern && !bestMatch.expanded) { + this.expandRangeScanResult(bestMatch as RangeScanResult, cache); + } + const matchEnd = bestMatch.matchEnd!; lastMatchIndex = matchEnd.index + matchEnd[0].length; } else { @@ -697,12 +737,12 @@ class DocumentTokenizer { lastMatchIndex = matchBegin.index + matchBegin[0].length; } - this.applyScanResult(bestMatch, source, parentNode); - if (failSafeIndex === lastMatchIndex) { logCatMessage(LogLevel.Error, LogCategory.Tokenizer, "The loop has not advanced since the last cycle. This indicates a programming error. Breaking the loop!", true); break; } + + this.applyScanResult(bestMatch, source, parentNode); } } @@ -744,6 +784,7 @@ interface ExTokenRepoPattern extends TokenRepoPattern, PatternStateProperties { interface ExTokenRangePattern extends TokenRangePattern, PatternStateProperties { _patternType: TokenPatternType.RangePattern; _hasBackref: boolean; + _endNotG: boolean; _patternsRepo?: ExTokenRepoPattern; readonly beginCaptures?: ExTokenPatternCapture; diff --git a/src/utilities/utils.ts b/src/utilities/utils.ts index 1351c00..c3f8435 100644 --- a/src/utilities/utils.ts +++ b/src/utilities/utils.ts @@ -1,3 +1,20 @@ export type EnumToString = { [P in keyof Type]: { name: P; value: Type[P] }; }; + +export function escapeRegExpCharacters(value: string): string { + return value.replace(/[-\\{}*+?|^$.,[\]()#\s]/g, "\\$&"); +} + +export function withTimeout(promise: Promise, timeout: number, onTimeout?: () => void, timeoutMessage?: string): Promise { + const timer = new Promise((resolve, reject) => { + setTimeout(() => { + if (onTimeout) { + onTimeout(); + } + const message = timeoutMessage || `Promise timed out after ${timeout}ms.`; + reject(message); + }, timeout); + }); + return Promise.race([promise, timer]); +}