Use bundle for configuration and not project.json (#924)

## Changes
* Select bundle target from configuration menu. If not selected, a
default target will be selected (based on mode and `default=true`
flags).
* Load the configuration from the bundle for the selected target.
* Users can override 
  * **auth type**:  triggers login flow with the selected auth type.
  * **cluster**: triggers cluster selector is logged in
  *  workspace folder

This does NOT use bundle to sync and run code. This is still handled by
`databricks sync` and custom logic for running.

## Tests
<!-- How is this tested? -->

packages/databricks-vscode-types/index.ts

@@ -12,7 +12,6 @@ export interface PublicApi {
     connectionManager: {
         onDidChangeState: Event<ConnectionState>;
 
-        login(interactive?: boolean, force?: boolean): Promise<void>;
         waitForConnect(): Promise<void>;
 
         get state(): ConnectionState;

packages/databricks-vscode/package.json

@@ -211,9 +211,13 @@
             {
                 "command": "databricks.call",
                 "title": "Call",
-                "enablement": "databricks.context.activated && databricks.context.loggedIn",
+                "enablement": "databricks.context.activated",
                 "category": "Databricks"
             },
+            {
+                "command": "databricks.internal.clearOverrides",
+                "title": "Clear workspace overrides"
+            },
             {
                 "command": "databricks.notebookInitScript.verify",
                 "title": "Verify Databricks notebook init scripts",

packages/databricks-vscode/src/bundle/types.ts

@@ -1,3 +1,4 @@
-import {BundleSchema} from "./BundleSchema";
+export {type BundleSchema} from "./BundleSchema";
 
+import {BundleSchema} from "./BundleSchema";
 export type BundleTarget = Required<BundleSchema>["targets"][string];

packages/databricks-vscode/src/cluster/ClusterManager.ts

@@ -16,8 +16,13 @@ export class ClusterManager implements Disposable {
 
     private setInterval() {
         this.refreshTimer = setInterval(async () => {
+            const oldState = this.cluster.state;
             await this.cluster.refresh();
-            this.onChange(this.cluster.state);
+            if (
+                JSON.stringify(oldState) !== JSON.stringify(this.cluster.state)
+            ) {
+                this.onChange(this.cluster.state);
+            }
         }, this.refreshTimeout.toMillSeconds().value);
     }
 

packages/databricks-vscode/src/configuration/ConfigurationDataProvider.test.ts

@@ -14,14 +14,16 @@ import {StateStorage} from "../vscode-objs/StateStorage";
 import {WorkspaceFsAccessVerifier} from "../workspace-fs";
 import {FeatureManager} from "../feature-manager/FeatureManager";
 import {Telemetry} from "../telemetry";
+import {ConfigModel} from "./ConfigModel";
+import {expect} from "chai";
 
 describe(__filename, () => {
     let connectionManagerMock: ConnectionManager;
     let disposables: Array<Disposable>;
     let onChangeClusterListener: (e: Cluster) => void;
     let onChangeSyncDestinationListener: (e: SyncDestinationMapper) => void;
     let sync: CodeSynchronizer;
-
+    let mockConfigModel: ConfigModel;
     beforeEach(() => {
         disposables = [];
         connectionManagerMock = mock(ConnectionManager);
@@ -56,6 +58,13 @@ describe(__filename, () => {
             dispose() {},
         });
         sync = instance(syncMock);
+
+        mockConfigModel = mock(ConfigModel);
+        mockConfigModel.onDidChangeAny = () => {
+            return {
+                dispose() {},
+            };
+        };
     });
 
     afterEach(() => {
@@ -74,7 +83,8 @@ describe(__filename, () => {
             instance(mock(StateStorage)),
             instance(mock(WorkspaceFsAccessVerifier)),
             instance(mock(FeatureManager<"debugging.dbconnect">)),
-            instance(mock(Telemetry))
+            instance(mock(Telemetry)),
+            mockConfigModel
         );
         disposables.push(provider);
 
@@ -98,7 +108,8 @@ describe(__filename, () => {
             instance(mock(StateStorage)),
             instance(mock(WorkspaceFsAccessVerifier)),
             instance(mock(FeatureManager<"debugging.dbconnect">)),
-            instance(mock(Telemetry))
+            instance(mock(Telemetry)),
+            mockConfigModel
         );
         disposables.push(provider);
 
@@ -122,16 +133,17 @@ describe(__filename, () => {
             instance(mock(StateStorage)),
             instance(mock(WorkspaceFsAccessVerifier)),
             instance(mock(FeatureManager<"debugging.dbconnect">)),
-            instance(mock(Telemetry))
+            instance(mock(Telemetry)),
+            mockConfigModel
         );
         disposables.push(provider);
 
         const children = await resolveProviderResult(provider.getChildren());
         assert(children);
-        assert.equal(children.length, 0);
+        assert.equal(children.length, 4);
     });
 
-    it("should return cluster children", async () => {
+    it("should return children", async () => {
         const mockApiClient = mock(ApiClient);
         when(mockApiClient.host).thenResolve(
             new URL("https://www.example.com")
@@ -155,12 +167,13 @@ describe(__filename, () => {
             instance(mock(StateStorage)),
             instance(mock(WorkspaceFsAccessVerifier)),
             instance(mock(FeatureManager<"debugging.dbconnect">)),
-            instance(mock(Telemetry))
+            instance(mock(Telemetry)),
+            mockConfigModel
         );
         disposables.push(provider);
 
         const children = await resolveProviderResult(provider.getChildren());
-        assert.deepEqual(children, [
+        expect(children).to.include.deep.members([
             {
                 collapsibleState: 2,
                 contextValue: "workspace",

packages/databricks-vscode/src/configuration/ConfigurationDataProvider.ts

@@ -21,6 +21,7 @@ import {
 } from "../workspace-fs";
 import {FeatureManager} from "../feature-manager/FeatureManager";
 import {Telemetry} from "../telemetry";
+import {ConfigModel} from "./ConfigModel";
 
 export type ConfigurationTreeItem = TreeItem & {
     url?: string;
@@ -47,7 +48,8 @@ export class ConfigurationDataProvider
         private readonly stateStorage: StateStorage,
         private readonly wsfsAccessVerifier: WorkspaceFsAccessVerifier,
         private readonly featureManager: FeatureManager,
-        private readonly telemetry: Telemetry
+        private readonly telemetry: Telemetry,
+        private readonly configModel: ConfigModel
     ) {
         this.disposables.push(
             this.connectionManager.onDidChangeState(() => {
@@ -64,6 +66,9 @@ export class ConfigurationDataProvider
             }),
             this.wsfsAccessVerifier.onDidChangeState(() => {
                 this._onDidChangeTreeData.fire();
+            }),
+            this.configModel.onDidChangeAny(() => {
+                this._onDidChangeTreeData.fire();
             })
         );
 
@@ -82,28 +87,65 @@ export class ConfigurationDataProvider
         element?: ConfigurationTreeItem | undefined
     ): Promise<Array<ConfigurationTreeItem>> {
         switch (this.connectionManager.state) {
+            case "DISCONNECTED":
             case "CONNECTED":
                 break;
             case "CONNECTING":
                 await this.connectionManager.waitForConnect();
                 break;
-            case "DISCONNECTED":
-                return [];
         }
 
         const cluster = this.connectionManager.cluster;
         const syncDestination = this.connectionManager.syncDestinationMapper;
 
         if (!element) {
             const children: Array<ConfigurationTreeItem> = [];
-            children.push({
-                label: `Workspace`,
-                iconPath: new ThemeIcon("account"),
-                id: "WORKSPACE",
-                collapsibleState: TreeItemCollapsibleState.Expanded,
-                contextValue: "workspace",
-                url: this.connectionManager.databricksWorkspace?.host?.toString(),
-            });
+            children.push(
+                {
+                    label:
+                        this.configModel.target !== undefined
+                            ? `Bundle Target - ${this.configModel.target}`
+                            : `Bundle Target - "None selected"`,
+                    id: "BUNDLE-TARGET",
+                    collapsibleState: TreeItemCollapsibleState.Expanded,
+                    contextValue: "bundleTarget",
+                    command: {
+                        title: "Call",
+                        command: "databricks.call",
+                        arguments: [
+                            async () => {
+                                const targets = await this.configModel
+                                    .bundleConfigReaderWriter.targets;
+                                if (targets === undefined) {
+                                    return;
+                                }
+
+                                const selectedTarget =
+                                    await window.showQuickPick(
+                                        Object.keys(targets),
+                                        {title: "Select bundle target"}
+                                    );
+                                if (selectedTarget === undefined) {
+                                    return;
+                                }
+                                const currentTarget = this.configModel.target;
+                                if (currentTarget !== selectedTarget) {
+                                    this._onDidChangeTreeData.fire();
+                                }
+                                this.configModel.setTarget(selectedTarget);
+                            },
+                        ],
+                    },
+                },
+                {
+                    label: `Workspace`,
+                    iconPath: new ThemeIcon("account"),
+                    id: "WORKSPACE",
+                    collapsibleState: TreeItemCollapsibleState.Expanded,
+                    contextValue: "workspace",
+                    url: this.connectionManager.databricksWorkspace?.host?.toString(),
+                }
+            );
 
             if (cluster) {
                 let contextValue:
@@ -362,6 +404,23 @@ export class ConfigurationDataProvider
             return children;
         }
 
+        if (element.id === "BUNDLE-TARGET") {
+            if (this.configModel.target === undefined) {
+                return [];
+            } else {
+                return [
+                    {
+                        label: "Host",
+                        description: await this.configModel.get("host"),
+                    },
+                    {
+                        label: "Mode",
+                        description: await this.configModel.get("mode"),
+                    },
+                ];
+            }
+        }
+
         return [];
     }
 

packages/databricks-vscode/src/configuration/ConnectionCommands.ts

@@ -104,7 +104,7 @@ export class ConnectionCommands implements Disposable {
      */
     attachClusterCommand() {
         return async (cluster: Cluster) => {
-            await this.connectionManager.attachCluster(cluster);
+            await this.connectionManager.attachCluster(cluster.id);
         };
     }
 
@@ -168,7 +168,7 @@ export class ConnectionCommands implements Disposable {
                 const selectedItem = quickPick.selectedItems[0];
                 if ("cluster" in selectedItem) {
                     const cluster = selectedItem.cluster;
-                    await this.connectionManager.attachCluster(cluster);
+                    await this.connectionManager.attachCluster(cluster.id);
                 } else {
                     await UrlUtils.openExternal(
                         `${