From cb0d7a33016d72d763d75c16105ffd471812dca3 Mon Sep 17 00:00:00 2001 From: Hassan Shah Date: Fri, 25 Oct 2024 10:43:53 +0200 Subject: [PATCH 1/5] [Feature] intial commit for key constraints --- catalog/resource_sql_table.go | 80 +++++++++++++++++++++++++++++------ 1 file changed, 66 insertions(+), 14 deletions(-) diff --git a/catalog/resource_sql_table.go b/catalog/resource_sql_table.go index ce9d4dbd7a..6d43cbf098 100644 --- a/catalog/resource_sql_table.go +++ b/catalog/resource_sql_table.go @@ -41,21 +41,69 @@ const IdentityColumnNone IdentityColumn = "" const IdentityColumnAlways IdentityColumn = "always" const IdentityColumnDefault IdentityColumn = "default" +type SqlKeyConstraintInfo struct { + SqlKeyConstraint SqlKeyConstraint `json:"key_constraint" tf:"alias:key_constraint,computed"` + TypeJson string `json:"type_json,omitempty" tf:"computed"` +} + +type SqlKeyConstraint interface { + getConstraint() string +} + +type SqlPrimaryKeyConstraint struct { + PrimaryKey string `json:"primary_key" tf:"alias:key,computed"` +} + +type SqlForeignKeyConstraint struct { + ReferencedKey string `json:"referenced_key" tf:"alias:referenced_key,computed"` + ReferencedCatalog string `json:"referenced_catalog" tf:"alias:referenced_catalog,computed"` + ReferencedSchema string `json:"referenced_schema" tf:"alias:referenced_schema,computed"` + ReferencedTable string `json:"referenced_table" tf:"alias:referenced_table,computed"` + ReferencedForeignKey string `json:"referenced_foreign_key" tf:"alias:referenced_foreign_key,computed"` +} + +func (sqlKeyConstraint SqlPrimaryKeyConstraint) getConstraint() string { + return fmt.Sprintf("PRIMARY KEY (%s)", sqlKeyConstraint.PrimaryKey) +} + +func (sqlKeyConstraint SqlForeignKeyConstraint) getConstraint() string { + return fmt.Sprintf( + "FOREIGN KEY (%s) REFERENCES %s.%s.%s(%s)", + sqlKeyConstraint.ReferencedKey, + sqlKeyConstraint.ReferencedCatalog, + sqlKeyConstraint.ReferencedSchema, + sqlKeyConstraint.ReferencedTable, + sqlKeyConstraint.ReferencedForeignKey) +} + +func (ti *SqlTableInfo) serializeSqlKeyConstraintInfo(keyConstraint SqlKeyConstraintInfo) string { + return keyConstraint.SqlKeyConstraint.getConstraint() +} + +func (ti *SqlTableInfo) serializeSqlKeyConstraintInfos() string { + keyConstraintFragments := make([]string, len(ti.KeyConstraintInfos)) + for i, keyConstraint := range ti.KeyConstraintInfos { + keyConstraintFragments[i] = ti.serializeSqlKeyConstraintInfo(keyConstraint) + } + return strings.Join(keyConstraintFragments[:], ", ") +} + type SqlTableInfo struct { - Name string `json:"name"` - CatalogName string `json:"catalog_name" tf:"force_new"` - SchemaName string `json:"schema_name" tf:"force_new"` - TableType string `json:"table_type" tf:"force_new"` - DataSourceFormat string `json:"data_source_format,omitempty" tf:"force_new"` - ColumnInfos []SqlColumnInfo `json:"columns,omitempty" tf:"alias:column,computed"` - Partitions []string `json:"partitions,omitempty" tf:"force_new"` - ClusterKeys []string `json:"cluster_keys,omitempty"` - StorageLocation string `json:"storage_location,omitempty" tf:"suppress_diff"` - StorageCredentialName string `json:"storage_credential_name,omitempty" tf:"force_new"` - ViewDefinition string `json:"view_definition,omitempty"` - Comment string `json:"comment,omitempty"` - Properties map[string]string `json:"properties,omitempty"` - Options map[string]string `json:"options,omitempty" tf:"force_new"` + Name string `json:"name"` + CatalogName string `json:"catalog_name" tf:"force_new"` + SchemaName string `json:"schema_name" tf:"force_new"` + TableType string `json:"table_type" tf:"force_new"` + DataSourceFormat string `json:"data_source_format,omitempty" tf:"force_new"` + ColumnInfos []SqlColumnInfo `json:"columns,omitempty" tf:"alias:column,computed"` + KeyConstraintInfos []SqlKeyConstraintInfo `json:"key_constraints,omitempty" tf:"alias:key_constraints,computed"` + Partitions []string `json:"partitions,omitempty" tf:"force_new"` + ClusterKeys []string `json:"cluster_keys,omitempty"` + StorageLocation string `json:"storage_location,omitempty" tf:"suppress_diff"` + StorageCredentialName string `json:"storage_credential_name,omitempty" tf:"force_new"` + ViewDefinition string `json:"view_definition,omitempty"` + Comment string `json:"comment,omitempty"` + Properties map[string]string `json:"properties,omitempty"` + Options map[string]string `json:"options,omitempty" tf:"force_new"` // EffectiveProperties includes both properties and options. Options are prefixed with `option.`. EffectiveProperties map[string]string `json:"effective_properties" tf:"computed"` ClusterID string `json:"cluster_id,omitempty" tf:"computed"` @@ -293,6 +341,10 @@ func (ti *SqlTableInfo) buildTableCreateStatement() string { statements = append(statements, fmt.Sprintf(" (%s)", ti.serializeColumnInfos())) } + if len(ti.KeyConstraintInfos) > 0 { + statements = append(statements, fmt.Sprintf(" (%s)", ti.serializeSqlKeyConstraintInfos())) + } + if !isView { if ti.DataSourceFormat != "" { statements = append(statements, fmt.Sprintf("\nUSING %s", ti.DataSourceFormat)) // USING CSV From 021c2061f65b7f1e8b9cd6d235265542d3b5adac Mon Sep 17 00:00:00 2001 From: Hassan Shah Date: Fri, 25 Oct 2024 11:23:26 +0200 Subject: [PATCH 2/5] [Feature] added tests --- catalog/resource_sql_table_test.go | 76 ++++++++++++++++++++++ internal/acceptance/sql_table_test.go | 90 +++++++++++++++++++++++++++ 2 files changed, 166 insertions(+) diff --git a/catalog/resource_sql_table_test.go b/catalog/resource_sql_table_test.go index f2f0a6c5e2..d9b09b5951 100644 --- a/catalog/resource_sql_table_test.go +++ b/catalog/resource_sql_table_test.go @@ -65,6 +65,82 @@ func TestResourceSqlTableCreateStatement_IdentityColumn(t *testing.T) { assert.Contains(t, stmt, "COMMENT 'terraform managed'") } +func TestResourceSqlTableCreateStatement_PrimaryKeyConstraint(t *testing.T) { + ti := &SqlTableInfo{ + Name: "bar", + CatalogName: "main", + SchemaName: "foo", + TableType: "EXTERNAL", + DataSourceFormat: "DELTA", + StorageLocation: "s3://ext-main/foo/bar1", + StorageCredentialName: "somecred", + Comment: "terraform managed", + ColumnInfos: []SqlColumnInfo{ + { + Name: "id", + }, + { + Name: "name", + Comment: "a comment", + }, + }, + KeyConstraintInfos: []SqlKeyConstraintInfo{ + { + SqlKeyConstraint: SqlPrimaryKeyConstraint{ + PrimaryKey: "id", + }, + }, + }, + } + stmt := ti.buildTableCreateStatement() + assert.Contains(t, stmt, "CREATE EXTERNAL TABLE `main`.`foo`.`bar`") + assert.Contains(t, stmt, "USING DELTA") + assert.Contains(t, stmt, "(`id` NOT NULL, `name` NOT NULL COMMENT 'a comment')") + assert.Contains(t, stmt, "(PRIMARY KEY (id))") + assert.Contains(t, stmt, "LOCATION 's3://ext-main/foo/bar1' WITH (CREDENTIAL `somecred`)") + assert.Contains(t, stmt, "COMMENT 'terraform managed'") +} + +func TestResourceSqlTableCreateStatement_ForeignKeyConstraint(t *testing.T) { + ti := &SqlTableInfo{ + Name: "bar", + CatalogName: "main", + SchemaName: "foo", + TableType: "EXTERNAL", + DataSourceFormat: "DELTA", + StorageLocation: "s3://ext-main/foo/bar1", + StorageCredentialName: "somecred", + Comment: "terraform managed", + ColumnInfos: []SqlColumnInfo{ + { + Name: "id", + }, + { + Name: "name", + Comment: "a comment", + }, + }, + KeyConstraintInfos: []SqlKeyConstraintInfo{ + { + SqlKeyConstraint: SqlForeignKeyConstraint{ + ReferencedKey: "id", + ReferencedCatalog: "bronze", + ReferencedSchema: "biz", + ReferencedTable: "transactions", + ReferencedForeignKey: "transactionId", + }, + }, + }, + } + stmt := ti.buildTableCreateStatement() + assert.Contains(t, stmt, "CREATE EXTERNAL TABLE `main`.`foo`.`bar`") + assert.Contains(t, stmt, "USING DELTA") + assert.Contains(t, stmt, "(`id` NOT NULL, `name` NOT NULL COMMENT 'a comment')") + assert.Contains(t, stmt, "(FOREIGN KEY (id) REFERENCES bronze.biz.transactions(transactionId)") + assert.Contains(t, stmt, "LOCATION 's3://ext-main/foo/bar1' WITH (CREDENTIAL `somecred`)") + assert.Contains(t, stmt, "COMMENT 'terraform managed'") +} + func TestResourceSqlTableCreateStatement_View(t *testing.T) { ti := &SqlTableInfo{ Name: "bar", diff --git a/internal/acceptance/sql_table_test.go b/internal/acceptance/sql_table_test.go index 6ba5a83714..a32932789d 100644 --- a/internal/acceptance/sql_table_test.go +++ b/internal/acceptance/sql_table_test.go @@ -136,6 +136,96 @@ func TestUcAccResourceSqlTableWithIdentityColumn_Managed(t *testing.T) { }) } +func TestUcAccResourceSqlTableWithPrimaryAndForeignKeyConstraints_Managed(t *testing.T) { + if os.Getenv("GOOGLE_CREDENTIALS") != "" { + skipf(t)("databricks_sql_table resource not available on GCP") + } + UnityWorkspaceLevel(t, Step{ + Template: ` + resource "databricks_schema" "this" { + name = "{var.STICKY_RANDOM}" + catalog_name = "main" + } + + resource "databricks_sql_table" "this" { + name = "bar" + catalog_name = "main" + schema_name = databricks_schema.this.name + table_type = "MANAGED" + properties = { + this = "that" + something = "else" + } + + column { + name = "id" + type = "bigint" + } + column { + name = "name" + type = "string" + } + column { + name = "externalId" + type = "string" + } + key_constraint { + primary_key = "id" + } + key_constraint { + referenced_key = "external_id" + referenced_catalog = "bronze" + referenced_schema = "biz" + referenced_table = "transactions" + referenced_foreign_key = "transactionId" + } + comment = "this table is managed by terraform" + owner = "account users" + }`, + }, Step{ + Template: ` + resource "databricks_schema" "this" { + name = "{var.STICKY_RANDOM}" + catalog_name = "main" + } + + resource "databricks_sql_table" "this" { + name = "bar" + catalog_name = "main" + schema_name = databricks_schema.this.name + table_type = "MANAGED" + properties = { + that = "this" + something = "else2" + } + + column { + name = "id" + type = "bigint" + } + column { + name = "name" + type = "string" + } + column { + name = "externalId" + type = "string" + } + key_constraint { + primary_key = "id" + } + key_constraint { + referenced_key = "external_id" + referenced_catalog = "bronze" + referenced_schema = "biz" + referenced_table = "transactions" + referenced_foreign_key = "transactionId" + } + comment = "this table is managed by terraform..." + }`, + }) +} + func TestUcAccResourceSqlTable_External(t *testing.T) { UnityWorkspaceLevel(t, Step{ Template: ` From eb8f4ca4ee491eafd3b6cfd7feb6769a427f92d3 Mon Sep 17 00:00:00 2001 From: Hassan Shah Date: Fri, 25 Oct 2024 13:47:24 +0200 Subject: [PATCH 3/5] added support for primary key rely --- catalog/resource_sql_table.go | 7 ++++++- catalog/resource_sql_table_test.go | 3 ++- internal/acceptance/sql_table_test.go | 2 ++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/catalog/resource_sql_table.go b/catalog/resource_sql_table.go index 6d43cbf098..353925045b 100644 --- a/catalog/resource_sql_table.go +++ b/catalog/resource_sql_table.go @@ -52,6 +52,7 @@ type SqlKeyConstraint interface { type SqlPrimaryKeyConstraint struct { PrimaryKey string `json:"primary_key" tf:"alias:key,computed"` + Rely bool `json:"rely,omitempty" tf:"default:false,alias:key,computed"` } type SqlForeignKeyConstraint struct { @@ -63,7 +64,11 @@ type SqlForeignKeyConstraint struct { } func (sqlKeyConstraint SqlPrimaryKeyConstraint) getConstraint() string { - return fmt.Sprintf("PRIMARY KEY (%s)", sqlKeyConstraint.PrimaryKey) + var constraint = fmt.Sprintf("PRIMARY KEY (%s)", sqlKeyConstraint.PrimaryKey) + if sqlKeyConstraint.Rely == true { + constraint += " RELY" + } + return constraint } func (sqlKeyConstraint SqlForeignKeyConstraint) getConstraint() string { diff --git a/catalog/resource_sql_table_test.go b/catalog/resource_sql_table_test.go index d9b09b5951..e6f4b0b1f0 100644 --- a/catalog/resource_sql_table_test.go +++ b/catalog/resource_sql_table_test.go @@ -88,6 +88,7 @@ func TestResourceSqlTableCreateStatement_PrimaryKeyConstraint(t *testing.T) { { SqlKeyConstraint: SqlPrimaryKeyConstraint{ PrimaryKey: "id", + Rely: true, }, }, }, @@ -96,7 +97,7 @@ func TestResourceSqlTableCreateStatement_PrimaryKeyConstraint(t *testing.T) { assert.Contains(t, stmt, "CREATE EXTERNAL TABLE `main`.`foo`.`bar`") assert.Contains(t, stmt, "USING DELTA") assert.Contains(t, stmt, "(`id` NOT NULL, `name` NOT NULL COMMENT 'a comment')") - assert.Contains(t, stmt, "(PRIMARY KEY (id))") + assert.Contains(t, stmt, "(PRIMARY KEY (id) RELY)") assert.Contains(t, stmt, "LOCATION 's3://ext-main/foo/bar1' WITH (CREDENTIAL `somecred`)") assert.Contains(t, stmt, "COMMENT 'terraform managed'") } diff --git a/internal/acceptance/sql_table_test.go b/internal/acceptance/sql_table_test.go index a32932789d..69edccf6be 100644 --- a/internal/acceptance/sql_table_test.go +++ b/internal/acceptance/sql_table_test.go @@ -171,6 +171,7 @@ func TestUcAccResourceSqlTableWithPrimaryAndForeignKeyConstraints_Managed(t *tes } key_constraint { primary_key = "id" + rely = "true" } key_constraint { referenced_key = "external_id" @@ -213,6 +214,7 @@ func TestUcAccResourceSqlTableWithPrimaryAndForeignKeyConstraints_Managed(t *tes } key_constraint { primary_key = "id" + rely = "true" } key_constraint { referenced_key = "external_id" From 63ecd6e2be5e2a75362cffda4f55ba894a85a623 Mon Sep 17 00:00:00 2001 From: Hassan Shah Date: Fri, 25 Oct 2024 15:57:32 +0200 Subject: [PATCH 4/5] fixed tests --- catalog/resource_sql_table.go | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/catalog/resource_sql_table.go b/catalog/resource_sql_table.go index 353925045b..89865c3012 100644 --- a/catalog/resource_sql_table.go +++ b/catalog/resource_sql_table.go @@ -42,8 +42,7 @@ const IdentityColumnAlways IdentityColumn = "always" const IdentityColumnDefault IdentityColumn = "default" type SqlKeyConstraintInfo struct { - SqlKeyConstraint SqlKeyConstraint `json:"key_constraint" tf:"alias:key_constraint,computed"` - TypeJson string `json:"type_json,omitempty" tf:"computed"` + SqlKeyConstraint SqlKeyConstraint } type SqlKeyConstraint interface { @@ -51,21 +50,21 @@ type SqlKeyConstraint interface { } type SqlPrimaryKeyConstraint struct { - PrimaryKey string `json:"primary_key" tf:"alias:key,computed"` - Rely bool `json:"rely,omitempty" tf:"default:false,alias:key,computed"` + PrimaryKey string `json:"primary_key"` + Rely bool `json:"rely,omitempty" tf:"default:false"` } type SqlForeignKeyConstraint struct { - ReferencedKey string `json:"referenced_key" tf:"alias:referenced_key,computed"` - ReferencedCatalog string `json:"referenced_catalog" tf:"alias:referenced_catalog,computed"` - ReferencedSchema string `json:"referenced_schema" tf:"alias:referenced_schema,computed"` - ReferencedTable string `json:"referenced_table" tf:"alias:referenced_table,computed"` - ReferencedForeignKey string `json:"referenced_foreign_key" tf:"alias:referenced_foreign_key,computed"` + ReferencedKey string `json:"referenced_key"` + ReferencedCatalog string `json:"referenced_catalog"` + ReferencedSchema string `json:"referenced_schema"` + ReferencedTable string `json:"referenced_table"` + ReferencedForeignKey string `json:"referenced_foreign_key"` } func (sqlKeyConstraint SqlPrimaryKeyConstraint) getConstraint() string { var constraint = fmt.Sprintf("PRIMARY KEY (%s)", sqlKeyConstraint.PrimaryKey) - if sqlKeyConstraint.Rely == true { + if sqlKeyConstraint.Rely { constraint += " RELY" } return constraint @@ -100,7 +99,7 @@ type SqlTableInfo struct { TableType string `json:"table_type" tf:"force_new"` DataSourceFormat string `json:"data_source_format,omitempty" tf:"force_new"` ColumnInfos []SqlColumnInfo `json:"columns,omitempty" tf:"alias:column,computed"` - KeyConstraintInfos []SqlKeyConstraintInfo `json:"key_constraints,omitempty" tf:"alias:key_constraints,computed"` + KeyConstraintInfos []SqlKeyConstraintInfo `json:"key_constraints,omitempty" tf:"alias:key_constraint"` Partitions []string `json:"partitions,omitempty" tf:"force_new"` ClusterKeys []string `json:"cluster_keys,omitempty"` StorageLocation string `json:"storage_location,omitempty" tf:"suppress_diff"` From c8d3075d7aa9348d4aea6273dda46222372a16a4 Mon Sep 17 00:00:00 2001 From: Hassan Shah Date: Fri, 25 Oct 2024 16:07:44 +0200 Subject: [PATCH 5/5] added documentation --- docs/resources/sql_table.md | 65 +++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/docs/resources/sql_table.md b/docs/resources/sql_table.md index 87b2d11533..786abfd6ec 100644 --- a/docs/resources/sql_table.md +++ b/docs/resources/sql_table.md @@ -149,6 +149,57 @@ resource "databricks_sql_table" "thing" { } ``` +## Use Primary and Foreign Keys + +```hcl +resource "databricks_catalog" "sandbox" { + name = "sandbox" + comment = "this catalog is managed by terraform" + properties = { + purpose = "testing" + } +} +resource "databricks_schema" "things" { + catalog_name = databricks_catalog.sandbox.id + name = "things" + comment = "this database is managed by terraform" + properties = { + kind = "various" + } +} +resource "databricks_sql_table" "thing" { + provider = databricks.workspace + name = "quickstart_table" + catalog_name = databricks_catalog.sandbox.name + schema_name = databricks_schema.things.name + table_type = "MANAGED" + data_source_format = "DELTA" + storage_location = "" + column { + name = "id" + type = "bigint" + } + column { + name = "name" + type = "string" + comment = "name of thing" + } + key_constraint { + primary_key = "id" + rely = "true" + } + key_constraint { + referenced_key = "external_id" + referenced_catalog = "bronze" + referenced_schema = "biz" + referenced_table = "transactions" + referenced_foreign_key = "transactionId" + } + comment = "this table is managed by terraform" +} +``` + + ## Argument Reference The following arguments are supported: @@ -163,6 +214,7 @@ The following arguments are supported: * `cluster_id` - (Optional) All table CRUD operations must be executed on a running cluster or SQL warehouse. If a cluster_id is specified, it will be used to execute SQL commands to manage this table. If empty, a cluster will be created automatically with the name `terraform-sql-table`. * `warehouse_id` - (Optional) All table CRUD operations must be executed on a running cluster or SQL warehouse. If a `warehouse_id` is specified, that SQL warehouse will be used to execute SQL commands to manage this table. Conflicts with `cluster_id`. * `cluster_keys` - (Optional) a subset of columns to liquid cluster the table by. Conflicts with `partitions`. +* `key_constraint` - (Optional) Constraints for Primary and Foreign Keys. * `storage_credential_name` - (Optional) For EXTERNAL Tables only: the name of storage credential to use. Change forces creation of a new resource. * `owner` - (Optional) Username/groupname/sp application_id of the schema owner. * `comment` - (Optional) User-supplied free-form text. Changing comment is not currently supported on `VIEW` table_type. @@ -181,6 +233,19 @@ Currently, changing the column definitions for a table will require dropping and * `comment` - (Optional) User-supplied free-form text. * `nullable` - (Optional) Whether field is nullable (Default: `true`) +### `key_constraint` configuration block + +For Primary Keys +* `primary_key` - Column that will get the constraint. +* `rely` - (Optional) Whether utilizing rely optimization. Can be `true` or `false`. Default to `false`. + +For Foreign Keys +* `referenced_key` - Column that will get the constraint. +* `referenced_catalog` - Catalog name of the remote table. +* `referenced_schema` - Schema name of the remote table. +* `referenced_table` - Remote table name. +* `referenced_foreign_key` - Column name of the foreign key in the remote table that will get referenced by the Foreign Key. + ## Attribute Reference In addition to all arguments above, the following attributes are exported: