From 37ac0fa8fc6ba372df2e28d3a9e4f7037b697f61 Mon Sep 17 00:00:00 2001 From: Darian Benam Date: Sun, 13 Aug 2023 01:08:00 -0400 Subject: [PATCH] Implemented basic IntelliSense auto completion support (#4) --- .eslintrc.json | 7 ++ README.md | 1 + src/Core/Analysis.ts | 20 ++--- src/Core/AutoCompletion.ts | 173 +++++++++++++++++++++++++++++++++++++ src/Core/Format.ts | 2 +- src/Core/Tokenization.ts | 36 ++++---- src/Main.ts | 10 +++ 7 files changed, 219 insertions(+), 30 deletions(-) create mode 100644 src/Core/AutoCompletion.ts diff --git a/.eslintrc.json b/.eslintrc.json index 32d77ae..a4dbd87 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -38,6 +38,13 @@ } ], "curly": "error", + "indent": [ + "error", + "tab", + { + "SwitchCase": 1 + } + ], "quotes": [ "error", "double" diff --git a/README.md b/README.md index ef585cf..72d8420 100644 --- a/README.md +++ b/README.md @@ -13,3 +13,4 @@ An extension for Visual Studio Code that enables support for `robots.txt` files. * Formatter * Snippets * Real-time Syntax Analysis +* Basic IntelliSense diff --git a/src/Core/Analysis.ts b/src/Core/Analysis.ts index e514e48..4279c21 100644 --- a/src/Core/Analysis.ts +++ b/src/Core/Analysis.ts @@ -5,6 +5,7 @@ */ import { isRobotsDotTextSyntaxAnalysisEnabled } from "../Config/ExtensionConfig"; +import { VALID_ROBOTS_TEXT_DIRECTIVES } from "./AutoCompletion"; import { RobotsDotTextToken, RobotsDotTextTokenType, tokenizeRobotsDotTextConfig } from "./Tokenization"; import { Diagnostic, @@ -15,17 +16,14 @@ import { TextDocument } from "vscode"; -const VALID_ROBOTS_TXT_DIRECTIVES: string[] = [ - "allow", - "crawl-delay", - "disallow", - "host", - "sitemap", - "user-agent" -]; - const isValidRobotsDotTextDirective = function(directive: string): boolean { - return VALID_ROBOTS_TXT_DIRECTIVES.includes(directive.toLowerCase()); + for (let i = 0; i < VALID_ROBOTS_TEXT_DIRECTIVES.length; i++) { + if (VALID_ROBOTS_TEXT_DIRECTIVES[i].name.toLowerCase() === directive.toLowerCase()) { + return true; + } + } + + return false; } const createDiagnosticIssue = function( @@ -63,7 +61,7 @@ export const analyzeRobotsDotTextConfig = function( return; } - const configTokens: RobotsDotTextToken[] = tokenizeRobotsDotTextConfig(document); + const configTokens: RobotsDotTextToken[] = tokenizeRobotsDotTextConfig(document.getText()); const diagnosticList: Diagnostic[] = []; let userAgentDirectiveFound: boolean = false; diff --git a/src/Core/AutoCompletion.ts b/src/Core/AutoCompletion.ts new file mode 100644 index 0000000..eb69b80 --- /dev/null +++ b/src/Core/AutoCompletion.ts @@ -0,0 +1,173 @@ +/** + * @fileoverview Handles basic auto completion IntelliSense for Robots.txt directives. + * @author Darian Benam + */ + +import { ROBOTS_DOT_TXT_DIRECTIVE_SEPARATOR, RobotsDotTextToken, RobotsDotTextTokenType, tokenizeRobotsDotTextConfig } from "./Tokenization"; +import { + CompletionItem, + CompletionItemKind, + CompletionItemProvider, + CompletionList, + Position, + ProviderResult, + TextDocument, + TextLine +} from "vscode"; + +export type RobotsDotTextAutoCompletionScope = { + [directive: string]: CompletionItem[]; +} + +export type RobotsDotTextDirective = { + name: string; + description: string; +} + +export const VALID_ROBOTS_TEXT_DIRECTIVES: RobotsDotTextDirective[] = [ + { name: "Allow", description: "Allows indexing site sections or individual pages." }, + { name: "Crawl-delay", description: "Specifies the minimum interval (in seconds) for the search robot to wait after loading one page before starting to load another." }, + { name: "Disallow", description: "Prohibits indexing site sections or individual pages." }, + { name: "Host", description: "Specifies the main mirror of the site." }, + { name: "Sitemap", description: "Specifies the absolute path to the `sitemap.xml` file that represents the website's sitemap." }, + { name: "User-agent", description: "Indicates the robot or crawler to which a list of directives apply too." } +]; + +const createAutoCompletionValueItem = function(label: string): CompletionItem { + return new CompletionItem(label, CompletionItemKind.Value); +} + +const DIRECTIVE_COMPLETION_SCOPE_LOOKUP_TABLE: RobotsDotTextAutoCompletionScope = { + "allow": [ + createAutoCompletionValueItem("/") + ], + "disallow": [ + createAutoCompletionValueItem("/") + ], + "user-agent": [ + createAutoCompletionValueItem("*"), + createAutoCompletionValueItem("AhrefsBot"), + createAutoCompletionValueItem("Applebot"), + createAutoCompletionValueItem("Baiduspider"), + createAutoCompletionValueItem("Bingbot"), + createAutoCompletionValueItem("Bingbot-Image"), + createAutoCompletionValueItem("Bingbot-Media"), + createAutoCompletionValueItem("Bingbot-News"), + createAutoCompletionValueItem("Bingbot-Video"), + createAutoCompletionValueItem("BingPreview"), + createAutoCompletionValueItem("BlexBot"), + createAutoCompletionValueItem("Chrome-Lighthouse"), + createAutoCompletionValueItem("Dataprovider"), + createAutoCompletionValueItem("Discordbot"), + createAutoCompletionValueItem("DuckDuckBot"), + createAutoCompletionValueItem("EtaoSpider"), + createAutoCompletionValueItem("Exabot"), + createAutoCompletionValueItem("Facebot"), + createAutoCompletionValueItem("FacebookExternalHit"), + createAutoCompletionValueItem("Google-InspectionTool"), + createAutoCompletionValueItem("Googlebot"), + createAutoCompletionValueItem("Googlebot-Image"), + createAutoCompletionValueItem("Googlebot-News"), + createAutoCompletionValueItem("Googlebot-Video"), + createAutoCompletionValueItem("GoogleOther"), + createAutoCompletionValueItem("Googlebot-Video"), + createAutoCompletionValueItem("Gort"), + createAutoCompletionValueItem("LinkedInBot"), + createAutoCompletionValueItem("MJ12bot"), + createAutoCompletionValueItem("PiplBot"), + createAutoCompletionValueItem("SemrushBot"), + createAutoCompletionValueItem("Slurp"), + createAutoCompletionValueItem("Storebot-Google"), + createAutoCompletionValueItem("TelegramBot"), + createAutoCompletionValueItem("Twitterbot"), + createAutoCompletionValueItem("UptimeRobot"), + createAutoCompletionValueItem("YandexAccessibilityBot"), + createAutoCompletionValueItem("YandexAdNet"), + createAutoCompletionValueItem("YandexBlogs"), + createAutoCompletionValueItem("YandexCalendar"), + createAutoCompletionValueItem("YandexDirect"), + createAutoCompletionValueItem("YandexDirectDyn"), + createAutoCompletionValueItem("YandexFavicons"), + createAutoCompletionValueItem("YaDirectFetcher"), + createAutoCompletionValueItem("YandexForDomain"), + createAutoCompletionValueItem("YandexImages"), + createAutoCompletionValueItem("YandexImageResizer"), + createAutoCompletionValueItem("YandexMarket"), + createAutoCompletionValueItem("YandexMedia"), + createAutoCompletionValueItem("YandexMetrika"), + createAutoCompletionValueItem("YandexMobileBot"), + createAutoCompletionValueItem("YandexMobileScreenShotBot"), + createAutoCompletionValueItem("YandexNews"), + createAutoCompletionValueItem("YandexOntoDB"), + createAutoCompletionValueItem("YandexOntoDBAPI"), + createAutoCompletionValueItem("YandexPagechecker"), + createAutoCompletionValueItem("YandexPartner"), + createAutoCompletionValueItem("YandexRCA"), + createAutoCompletionValueItem("YandexSearchShop"), + createAutoCompletionValueItem("YandexSitelinks"), + createAutoCompletionValueItem("YandexScreenshotBot"), + createAutoCompletionValueItem("YandexTracker"), + createAutoCompletionValueItem("YandexTracker"), + createAutoCompletionValueItem("YandexVertis"), + createAutoCompletionValueItem("YandexVerticals"), + createAutoCompletionValueItem("YandexVideo"), + createAutoCompletionValueItem("YandexVideoParser"), + createAutoCompletionValueItem("YandexWebmaster") + ] +} + +export const globalDirectiveAutoCompletionHandler: CompletionItemProvider = { + provideCompletionItems(document: TextDocument, position: Position) { + const currentLine: TextLine = document.lineAt(position.line); + + if (currentLine.text.indexOf(ROBOTS_DOT_TXT_DIRECTIVE_SEPARATOR) !== -1) { + return undefined; + } + + const globalDirectiveCompletionItems: CompletionItem[] = []; + + for (const directive of VALID_ROBOTS_TEXT_DIRECTIVES) { + const directiveCompletionItem: CompletionItem = new CompletionItem(directive.name, CompletionItemKind.Property); + + directiveCompletionItem.insertText = `${directive.name}: `; + directiveCompletionItem.detail = directive.description; + directiveCompletionItem.command = { + title: "Re-trigger auto completion suggestions", + command: "editor.action.triggerSuggest" + }; + + globalDirectiveCompletionItems.push(directiveCompletionItem) + } + + return globalDirectiveCompletionItems; + } +} + +export const directiveValueAutoCompletionHandler: CompletionItemProvider = { + provideCompletionItems( + document: TextDocument, + position: Position + ): ProviderResult | null | undefined> { + const currentLine: TextLine = document.lineAt(position.line); + + if (currentLine.isEmptyOrWhitespace) { + return undefined; + } + + const currentLineToken: RobotsDotTextToken = tokenizeRobotsDotTextConfig(currentLine.text)[0]; + + if (currentLineToken.type !== RobotsDotTextTokenType.Directive) { + return undefined; + } + + if (currentLineToken.directive?.value.length === 0) { + const directive: string = currentLineToken.directive?.name.toLowerCase(); + + if (directive in DIRECTIVE_COMPLETION_SCOPE_LOOKUP_TABLE) { + return DIRECTIVE_COMPLETION_SCOPE_LOOKUP_TABLE[directive]; + } + } + + return undefined; + } +} diff --git a/src/Core/Format.ts b/src/Core/Format.ts index 5636e5b..32178df 100644 --- a/src/Core/Format.ts +++ b/src/Core/Format.ts @@ -7,7 +7,7 @@ import { RobotsDotTextToken, tokenizeRobotsDotTextConfig } from "./Tokenization" import { TextDocument, TextEdit } from "vscode"; export const formatRobotsDotTextDocument = function(document: TextDocument): TextEdit[] { - const robotsDotTextTokens: RobotsDotTextToken[] = tokenizeRobotsDotTextConfig(document); + const robotsDotTextTokens: RobotsDotTextToken[] = tokenizeRobotsDotTextConfig(document.getText()); const formatTextEditList: TextEdit[] = []; for (const token of robotsDotTextTokens) { diff --git a/src/Core/Tokenization.ts b/src/Core/Tokenization.ts index 55910c6..5821ad9 100644 --- a/src/Core/Tokenization.ts +++ b/src/Core/Tokenization.ts @@ -3,11 +3,11 @@ * @author Darian Benam */ -import { Position, Range, TextDocument } from "vscode"; +import { Position, Range } from "vscode"; const ROBOTS_DOT_TXT_COMMENT_PREFIX: string = "#"; -const ROBOTS_DOT_TXT_DIRECTIVE_SEPARATOR: string = ":"; +export const ROBOTS_DOT_TXT_DIRECTIVE_SEPARATOR: string = ":"; export enum RobotsDotTextTokenType { BlankLine, @@ -58,16 +58,16 @@ export class RobotsDotTextToken { } } -export const tokenizeRobotsDotTextConfig = function(document: TextDocument | undefined): RobotsDotTextToken[] { - if (document === undefined) { +export const tokenizeRobotsDotTextConfig = function(configRawText: string | undefined): RobotsDotTextToken[] { + if (configRawText === undefined) { return []; } const robotsDotTextTokens: RobotsDotTextToken[] = []; - const configLineList: string[] = document.getText().split("\n"); + const configLineList: string[] = configRawText.split("\n"); let currentLineIndex: number = -1; - for (const rawLine of configLineList) { + for (const rawLine of configLineList) { ++currentLineIndex; const sanitizedLine: string = rawLine.trim(); @@ -77,13 +77,13 @@ export const tokenizeRobotsDotTextConfig = function(document: TextDocument | und raw: rawLine, rawRange: new Range( new Position(currentLineIndex, 0), - new Position(currentLineIndex, rawLine.length) + new Position(currentLineIndex, rawLine.length) ), sanitized: sanitizedLine, sanitizedRange: new Range( - new Position(currentLineIndex, sanitizedLineFirstCharacter), - new Position(currentLineIndex, sanitizedLineFirstCharacter + sanitizedLine.length) - ) + new Position(currentLineIndex, sanitizedLineFirstCharacter), + new Position(currentLineIndex, sanitizedLineFirstCharacter + sanitizedLine.length) + ) }; if (sanitizedLine.length === 0) { @@ -117,15 +117,15 @@ export const tokenizeRobotsDotTextConfig = function(document: TextDocument | und const directiveNameIndex: number = rawLine.indexOf(directiveName); const directiveValueIndex: number = rawLine.indexOf(directiveValue, directiveNameIndex + directiveName.length); - const nameRange: Range = new Range( - new Position(currentLineIndex, directiveNameIndex), - new Position(currentLineIndex, directiveNameIndex + directiveName.length) - ); + const nameRange: Range = new Range( + new Position(currentLineIndex, directiveNameIndex), + new Position(currentLineIndex, directiveNameIndex + directiveName.length) + ); - const valueRange: Range = new Range( - new Position(currentLineIndex, directiveValueIndex), - new Position(currentLineIndex, directiveValueIndex + directiveValue.length) - ); + const valueRange: Range = new Range( + new Position(currentLineIndex, directiveValueIndex), + new Position(currentLineIndex, directiveValueIndex + directiveValue.length) + ); robotsDotTextTokens.push(new RobotsDotTextToken( RobotsDotTextTokenType.Directive, diff --git a/src/Main.ts b/src/Main.ts index 0376a24..9600b05 100644 --- a/src/Main.ts +++ b/src/Main.ts @@ -5,6 +5,7 @@ import { isRobotsDotTextSyntaxAnalysisEnabled } from "./Config/ExtensionConfig"; import { analyzeRobotsDotTextConfig, clearRobotsDotTextConfigDiagnosticIssues } from "./Core/Analysis"; +import { directiveValueAutoCompletionHandler, globalDirectiveAutoCompletionHandler } from "./Core/AutoCompletion"; import { formatRobotsDotTextDocument } from "./Core/Format"; import { DiagnosticCollection, @@ -23,6 +24,15 @@ const DIAGNOSTIC_COLLECTION: DiagnosticCollection = languages.createDiagnosticCo export function activate(context: ExtensionContext): void { const extensionEventHandlers: Disposable[] = [ + languages.registerCompletionItemProvider( + ROBOTS_DOT_TXT_LANGUAGE_ID, + globalDirectiveAutoCompletionHandler + ), + languages.registerCompletionItemProvider( + ROBOTS_DOT_TXT_LANGUAGE_ID, + directiveValueAutoCompletionHandler, + " " + ), languages.registerDocumentFormattingEditProvider(ROBOTS_DOT_TXT_LANGUAGE_ID, { provideDocumentFormattingEdits(document: TextDocument): TextEdit[] { return formatRobotsDotTextDocument(document);