Implemented basic IntelliSense auto completion support (#4)

.eslintrc.json

@@ -38,6 +38,13 @@
 			}
 		],
 		"curly": "error",
+		"indent": [
+			"error",
+			"tab",
+			{
+				"SwitchCase": 1
+			}
+		],
 		"quotes": [
 			"error",
 			"double"

README.md

@@ -13,3 +13,4 @@ An extension for Visual Studio Code that enables support for `robots.txt` files.
 * Formatter
 * Snippets
 * Real-time Syntax Analysis
+* Basic IntelliSense

src/Core/Analysis.ts

@@ -5,6 +5,7 @@
  */
 
 import { isRobotsDotTextSyntaxAnalysisEnabled } from "../Config/ExtensionConfig";
+import { VALID_ROBOTS_TEXT_DIRECTIVES } from "./AutoCompletion";
 import { RobotsDotTextToken, RobotsDotTextTokenType, tokenizeRobotsDotTextConfig } from "./Tokenization";
 import {
 	Diagnostic,
@@ -15,17 +16,14 @@ import {
 	TextDocument
 } from "vscode";
 
-const VALID_ROBOTS_TXT_DIRECTIVES: string[] = [
-	"allow",
-	"crawl-delay",
-	"disallow",
-	"host",
-	"sitemap",
-	"user-agent"
-];
-
 const isValidRobotsDotTextDirective = function(directive: string): boolean {
-	return VALID_ROBOTS_TXT_DIRECTIVES.includes(directive.toLowerCase());
+	for (let i = 0; i < VALID_ROBOTS_TEXT_DIRECTIVES.length; i++) {
+		if (VALID_ROBOTS_TEXT_DIRECTIVES[i].name.toLowerCase() === directive.toLowerCase()) {
+			return true;
+		}
+	}
+
+	return false;
 }
 
 const createDiagnosticIssue = function(
@@ -63,7 +61,7 @@ export const analyzeRobotsDotTextConfig = function(
 		return;
 	}
 
-	const configTokens: RobotsDotTextToken[] = tokenizeRobotsDotTextConfig(document);
+	const configTokens: RobotsDotTextToken[] = tokenizeRobotsDotTextConfig(document.getText());
 	const diagnosticList: Diagnostic[] = [];
 
 	let userAgentDirectiveFound: boolean = false;

src/Core/AutoCompletion.ts

@@ -0,0 +1,173 @@
+/**
+ *  @fileoverview    Handles basic auto completion IntelliSense for Robots.txt directives.
+ *  @author          Darian Benam <darian@darianbenam.com>
+ */
+
+import { ROBOTS_DOT_TXT_DIRECTIVE_SEPARATOR, RobotsDotTextToken, RobotsDotTextTokenType, tokenizeRobotsDotTextConfig } from "./Tokenization";
+import {
+	CompletionItem,
+	CompletionItemKind,
+	CompletionItemProvider,
+	CompletionList,
+	Position,
+	ProviderResult,
+	TextDocument,
+	TextLine
+} from "vscode";
+
+export type RobotsDotTextAutoCompletionScope = {
+	[directive: string]: CompletionItem[];
+}
+
+export type RobotsDotTextDirective = {
+	name: string;
+	description: string;
+}
+
+export const VALID_ROBOTS_TEXT_DIRECTIVES: RobotsDotTextDirective[] = [
+	{ name: "Allow", description: "Allows indexing site sections or individual pages." },
+	{ name: "Crawl-delay", description: "Specifies the minimum interval (in seconds) for the search robot to wait after loading one page before starting to load another." },
+	{ name: "Disallow", description: "Prohibits indexing site sections or individual pages." },
+	{ name: "Host", description: "Specifies the main mirror of the site." },
+	{ name: "Sitemap", description: "Specifies the absolute path to the `sitemap.xml` file that represents the website's sitemap." },
+	{ name: "User-agent", description: "Indicates the robot or crawler to which a list of directives apply too." }
+];
+
+const createAutoCompletionValueItem = function(label: string): CompletionItem {
+	return new CompletionItem(label, CompletionItemKind.Value);
+}
+
+const DIRECTIVE_COMPLETION_SCOPE_LOOKUP_TABLE: RobotsDotTextAutoCompletionScope = {
+	"allow": [
+		createAutoCompletionValueItem("/")
+	],
+	"disallow": [
+		createAutoCompletionValueItem("/")
+	],
+	"user-agent": [
+		createAutoCompletionValueItem("*"),
+		createAutoCompletionValueItem("AhrefsBot"),
+		createAutoCompletionValueItem("Applebot"),
+		createAutoCompletionValueItem("Baiduspider"),
+		createAutoCompletionValueItem("Bingbot"),
+		createAutoCompletionValueItem("Bingbot-Image"),
+		createAutoCompletionValueItem("Bingbot-Media"),
+		createAutoCompletionValueItem("Bingbot-News"),
+		createAutoCompletionValueItem("Bingbot-Video"),
+		createAutoCompletionValueItem("BingPreview"),
+		createAutoCompletionValueItem("BlexBot"),
+		createAutoCompletionValueItem("Chrome-Lighthouse"),
+		createAutoCompletionValueItem("Dataprovider"),
+		createAutoCompletionValueItem("Discordbot"),
+		createAutoCompletionValueItem("DuckDuckBot"),
+		createAutoCompletionValueItem("EtaoSpider"),
+		createAutoCompletionValueItem("Exabot"),
+		createAutoCompletionValueItem("Facebot"),
+		createAutoCompletionValueItem("FacebookExternalHit"),
+		createAutoCompletionValueItem("Google-InspectionTool"),
+		createAutoCompletionValueItem("Googlebot"),
+		createAutoCompletionValueItem("Googlebot-Image"),
+		createAutoCompletionValueItem("Googlebot-News"),
+		createAutoCompletionValueItem("Googlebot-Video"),
+		createAutoCompletionValueItem("GoogleOther"),
+		createAutoCompletionValueItem("Googlebot-Video"),
+		createAutoCompletionValueItem("Gort"),
+		createAutoCompletionValueItem("LinkedInBot"),
+		createAutoCompletionValueItem("MJ12bot"),
+		createAutoCompletionValueItem("PiplBot"),
+		createAutoCompletionValueItem("SemrushBot"),
+		createAutoCompletionValueItem("Slurp"),
+		createAutoCompletionValueItem("Storebot-Google"),
+		createAutoCompletionValueItem("TelegramBot"),
+		createAutoCompletionValueItem("Twitterbot"),
+		createAutoCompletionValueItem("UptimeRobot"),
+		createAutoCompletionValueItem("YandexAccessibilityBot"),
+		createAutoCompletionValueItem("YandexAdNet"),
+		createAutoCompletionValueItem("YandexBlogs"),
+		createAutoCompletionValueItem("YandexCalendar"),
+		createAutoCompletionValueItem("YandexDirect"),
+		createAutoCompletionValueItem("YandexDirectDyn"),
+		createAutoCompletionValueItem("YandexFavicons"),
+		createAutoCompletionValueItem("YaDirectFetcher"),
+		createAutoCompletionValueItem("YandexForDomain"),
+		createAutoCompletionValueItem("YandexImages"),
+		createAutoCompletionValueItem("YandexImageResizer"),
+		createAutoCompletionValueItem("YandexMarket"),
+		createAutoCompletionValueItem("YandexMedia"),
+		createAutoCompletionValueItem("YandexMetrika"),
+		createAutoCompletionValueItem("YandexMobileBot"),
+		createAutoCompletionValueItem("YandexMobileScreenShotBot"),
+		createAutoCompletionValueItem("YandexNews"),
+		createAutoCompletionValueItem("YandexOntoDB"),
+		createAutoCompletionValueItem("YandexOntoDBAPI"),
+		createAutoCompletionValueItem("YandexPagechecker"),
+		createAutoCompletionValueItem("YandexPartner"),
+		createAutoCompletionValueItem("YandexRCA"),
+		createAutoCompletionValueItem("YandexSearchShop"),
+		createAutoCompletionValueItem("YandexSitelinks"),
+		createAutoCompletionValueItem("YandexScreenshotBot"),
+		createAutoCompletionValueItem("YandexTracker"),
+		createAutoCompletionValueItem("YandexTracker"),
+		createAutoCompletionValueItem("YandexVertis"),
+		createAutoCompletionValueItem("YandexVerticals"),
+		createAutoCompletionValueItem("YandexVideo"),
+		createAutoCompletionValueItem("YandexVideoParser"),
+		createAutoCompletionValueItem("YandexWebmaster")
+	]
+}
+
+export const globalDirectiveAutoCompletionHandler: CompletionItemProvider<CompletionItem> = {
+	provideCompletionItems(document: TextDocument, position: Position) {
+		const currentLine: TextLine = document.lineAt(position.line);
+
+		if (currentLine.text.indexOf(ROBOTS_DOT_TXT_DIRECTIVE_SEPARATOR) !== -1) {
+			return undefined;
+		}
+
+		const globalDirectiveCompletionItems: CompletionItem[] = [];
+
+		for (const directive of VALID_ROBOTS_TEXT_DIRECTIVES) {
+			const directiveCompletionItem: CompletionItem = new CompletionItem(directive.name, CompletionItemKind.Property);
+
+			directiveCompletionItem.insertText = `${directive.name}: `;
+			directiveCompletionItem.detail = directive.description;
+			directiveCompletionItem.command = {
+				title: "Re-trigger auto completion suggestions",
+				command: "editor.action.triggerSuggest"
+			};
+
+			globalDirectiveCompletionItems.push(directiveCompletionItem)
+		}
+
+		return globalDirectiveCompletionItems;
+	}
+}
+
+export const directiveValueAutoCompletionHandler: CompletionItemProvider<CompletionItem> = {
+	provideCompletionItems(
+		document: TextDocument,
+		position: Position
+	): ProviderResult<CompletionItem[] | CompletionList<CompletionItem> | null | undefined> {
+		const currentLine: TextLine = document.lineAt(position.line);
+
+		if (currentLine.isEmptyOrWhitespace) {
+			return undefined;
+		}
+
+		const currentLineToken: RobotsDotTextToken = tokenizeRobotsDotTextConfig(currentLine.text)[0];
+
+		if (currentLineToken.type !== RobotsDotTextTokenType.Directive) {
+			return undefined;
+		}
+
+		if (currentLineToken.directive?.value.length === 0) {
+			const directive: string = currentLineToken.directive?.name.toLowerCase();
+
+			if (directive in DIRECTIVE_COMPLETION_SCOPE_LOOKUP_TABLE) {
+				return DIRECTIVE_COMPLETION_SCOPE_LOOKUP_TABLE[directive];
+			}
+		}
+
+		return undefined;
+	}
+}

src/Core/Format.ts

@@ -7,7 +7,7 @@ import { RobotsDotTextToken, tokenizeRobotsDotTextConfig } from "./Tokenization"
 import { TextDocument, TextEdit } from "vscode";
 
 export const formatRobotsDotTextDocument = function(document: TextDocument): TextEdit[] {
-	const robotsDotTextTokens: RobotsDotTextToken[] = tokenizeRobotsDotTextConfig(document);
+	const robotsDotTextTokens: RobotsDotTextToken[] = tokenizeRobotsDotTextConfig(document.getText());
 	const formatTextEditList: TextEdit[] = [];
 
 	for (const token of robotsDotTextTokens) {

src/Core/Tokenization.ts

@@ -3,11 +3,11 @@
  *  @author          Darian Benam <darian@darianbenam.com>
  */
 
-import { Position, Range, TextDocument } from "vscode";
+import { Position, Range } from "vscode";
 
 const ROBOTS_DOT_TXT_COMMENT_PREFIX: string = "#";
 
-const ROBOTS_DOT_TXT_DIRECTIVE_SEPARATOR: string = ":";
+export const ROBOTS_DOT_TXT_DIRECTIVE_SEPARATOR: string = ":";
 
 export enum RobotsDotTextTokenType {
 	BlankLine,
@@ -58,16 +58,16 @@ export class RobotsDotTextToken {
 	}
 }
 
-export const tokenizeRobotsDotTextConfig = function(document: TextDocument | undefined): RobotsDotTextToken[] {
-	if (document === undefined) {
+export const tokenizeRobotsDotTextConfig = function(configRawText: string | undefined): RobotsDotTextToken[] {
+	if (configRawText === undefined) {
 		return [];
 	}
 
 	const robotsDotTextTokens: RobotsDotTextToken[] = [];
-	const configLineList: string[] = document.getText().split("\n");
+	const configLineList: string[] = configRawText.split("\n");
 	let currentLineIndex: number = -1;
 
-    for (const rawLine of configLineList) {
+	for (const rawLine of configLineList) {
 		++currentLineIndex;
 
 		const sanitizedLine: string = rawLine.trim();
@@ -77,13 +77,13 @@ export const tokenizeRobotsDotTextConfig = function(document: TextDocument | und
 			raw: rawLine,
 			rawRange: new Range(
 				new Position(currentLineIndex, 0),
-                new Position(currentLineIndex, rawLine.length)
+				new Position(currentLineIndex, rawLine.length)
 			),
 			sanitized: sanitizedLine,
 			sanitizedRange: new Range(
-                new Position(currentLineIndex, sanitizedLineFirstCharacter),
-                new Position(currentLineIndex, sanitizedLineFirstCharacter + sanitizedLine.length)
-            )
+				new Position(currentLineIndex, sanitizedLineFirstCharacter),
+				new Position(currentLineIndex, sanitizedLineFirstCharacter + sanitizedLine.length)
+			)
 		};
 
 		if (sanitizedLine.length === 0) {
@@ -117,15 +117,15 @@ export const tokenizeRobotsDotTextConfig = function(document: TextDocument | und
 			const directiveNameIndex: number = rawLine.indexOf(directiveName);
 			const directiveValueIndex: number = rawLine.indexOf(directiveValue, directiveNameIndex + directiveName.length);
 
-            const nameRange: Range = new Range(
-                new Position(currentLineIndex, directiveNameIndex),
-                new Position(currentLineIndex, directiveNameIndex + directiveName.length)
-            );
+			const nameRange: Range = new Range(
+				new Position(currentLineIndex, directiveNameIndex),
+				new Position(currentLineIndex, directiveNameIndex + directiveName.length)
+			);
 
-            const valueRange: Range = new Range(
-                new Position(currentLineIndex, directiveValueIndex),
-                new Position(currentLineIndex, directiveValueIndex + directiveValue.length)
-            );
+			const valueRange: Range = new Range(
+				new Position(currentLineIndex, directiveValueIndex),
+				new Position(currentLineIndex, directiveValueIndex + directiveValue.length)
+			);
 
 			robotsDotTextTokens.push(new RobotsDotTextToken(
 				RobotsDotTextTokenType.Directive,

src/Main.ts

@@ -5,6 +5,7 @@
 
 import { isRobotsDotTextSyntaxAnalysisEnabled } from "./Config/ExtensionConfig";
 import { analyzeRobotsDotTextConfig, clearRobotsDotTextConfigDiagnosticIssues } from "./Core/Analysis";
+import { directiveValueAutoCompletionHandler, globalDirectiveAutoCompletionHandler } from "./Core/AutoCompletion";
 import { formatRobotsDotTextDocument } from "./Core/Format";
 import {
 	DiagnosticCollection,
@@ -23,6 +24,15 @@ const DIAGNOSTIC_COLLECTION: DiagnosticCollection = languages.createDiagnosticCo
 
 export function activate(context: ExtensionContext): void {
 	const extensionEventHandlers: Disposable[] = [
+		languages.registerCompletionItemProvider(
+			ROBOTS_DOT_TXT_LANGUAGE_ID,
+			globalDirectiveAutoCompletionHandler
+		),
+		languages.registerCompletionItemProvider(
+			ROBOTS_DOT_TXT_LANGUAGE_ID,
+			directiveValueAutoCompletionHandler,
+			" "
+		),
 		languages.registerDocumentFormattingEditProvider(ROBOTS_DOT_TXT_LANGUAGE_ID, {
 			provideDocumentFormattingEdits(document: TextDocument): TextEdit[] {
 				return formatRobotsDotTextDocument(document);