Merge pull request #716 from basedosdados/staging/br_rf_cno

[dbt] br_rf_cno

dbt_project.yml

@@ -318,6 +318,9 @@ models:
     br_rf_cafir:
       +materialized: table
       +schema: br_rf_cafir
+    br_rf_cno:
+      +materialized: table
+      +schema: br_rf_cno
     br_rj_isp_estatisticas_seguranca:
       +materialized: table
       +schema: br_rj_isp_estatisticas_seguranca

macros/custom_get_where_subquery.sql

@@ -44,6 +44,25 @@
             {% endif %}
         {% endif %}
 
+        {% if "__most_recent_date_cno__" in where %}
+            {% set max_date_query = (
+                "select max(data_extracao) as max_date from " ~ relation
+            ) %}
+            {% set max_date_result = run_query(max_date_query) %}
+            {% if execute and max_date_result.rows[0][0] %}
+                {% set max_date = max_date_result.rows[0][0] %}
+                {% set where = where | replace(
+                    "__most_recent_date_cno__",
+                    "data_extracao = '" ~ max_date ~ "'",
+                ) %}
+                {% do log(
+                    "The test will filter by the most recent date: "
+                    ~ max_date,
+                    info=True,
+                ) %}
+            {% endif %}
+        {% endif %}
+
         {# This block looks for __most_recent_year_month__  placeholder #}
         {% if "__most_recent_year_month__" in where %}
             {% set max_date_query = (

models/br_rf_cno/br_rf_cno__areas.sql

@@ -0,0 +1,23 @@
+{{
+    config(
+        alias="areas",
+        schema="br_rf_cno",
+        materialized="incremental",
+        partition_by={
+            "field": "data_extracao",
+            "data_type": "date",
+        },
+    )
+}}
+
+select
+    safe_cast(data as date) data_extracao,
+    safe_cast(id_cno as string) id_cno,
+    safe_cast(categoria as string) categoria,
+    safe_cast(destinacao as string) destinacao,
+    safe_cast(tipo_obra as string) tipo_obra,
+    safe_cast(tipo_area as string) tipo_area,
+    safe_cast(tipo_area_complementar as string) tipo_area_complementar,
+    safe_cast(metragem as float64) metragem,
+from `basedosdados-staging.br_rf_cno_staging.areas` as t
+{% if is_incremental() %} where data > (select max(data) from {{ this }}) {% endif %}

models/br_rf_cno/br_rf_cno__cnaes.sql

@@ -0,0 +1,18 @@
+{{
+    config(
+        alias="cnaes",
+        schema="br_rf_cno",
+        materialized="incremental",
+        partition_by={
+            "field": "data_extracao",
+            "data_type": "date",
+        },
+    )
+}}
+select
+    safe_cast(data as date) data_extracao,
+    safe_cast(data_registro as date) data_registro,
+    safe_cast(id_cno as string) id_cno,
+    safe_cast(cnae_2_subclasse as string) cnae_2_subclasse,
+from `basedosdados-staging.br_rf_cno_staging.cnaes` as t
+{% if is_incremental() %} where data > (select max(data) from {{ this }}) {% endif %}

models/br_rf_cno/br_rf_cno__dicionario.sql

@@ -0,0 +1,8 @@
+{{ config(alias="dicionario", schema="br_rf_cno", materialized="table") }}
+select
+    safe_cast(id_tabela as string) id_tabela,
+    safe_cast(nome_coluna as string) nome_coluna,
+    safe_cast(chave as string) chave,
+    safe_cast(cobertura_temporal as string) cobertura_temporal,
+    safe_cast(valor as string) valor,
+from `basedosdados-staging.br_rf_cno_staging.dicionario` as t

models/br_rf_cno/br_rf_cno__microdados.sql

@@ -0,0 +1,48 @@
+{{
+    config(
+        alias="microdados",
+        schema="br_rf_cno",
+        materialized="incremental",
+        partition_by={
+            "field": "data_extracao",
+            "data_type": "date",
+        },
+        pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}",
+    )
+}}
+
+select
+    safe_cast(data as date) data_extracao,
+    safe_cast(data_situacao as date) data_situacao,
+    safe_cast(data_registro as date) data_registro,
+    safe_cast(data_inicio as date) data_inicio,
+    safe_cast(data_inicio_responsabilidade as date) data_inicio_responsabilidade,
+    safe_cast(id_pais as string) id_pais,
+    safe_cast(nome_pais as string) nome_pais,
+    safe_cast(sigla_uf as string) sigla_uf,
+    safe_cast(b.id_municipio as string) id_municipio,
+    safe_cast(id_cno as string) id_cno,
+    safe_cast(id_cno_vinculado as string) id_cno_vinculado,
+    safe_cast(ltrim(situacao, '0') as string) situacao,
+    safe_cast(id_responsavel as string) id_responsavel,
+    safe_cast(nome_responsavel as string) nome_responsavel,
+    safe_cast(ltrim(qualificacao_responsavel, '0') as string) qualificacao_responsavel,
+    safe_cast(nome_empresarial as string) nome_empresarial,
+    safe_cast(area as float64) area,
+    safe_cast(unidade_medida as string) unidade_medida,
+    safe_cast(bairro as string) bairro,
+    safe_cast(cep as string) cep,
+    safe_cast(logradouro as string) logradouro,
+    safe_cast(tipo_logradouro as string) tipo_logradouro,
+    safe_cast(numero_logradouro as string) numero_logradouro,
+    safe_cast(complemento as string) complemento,
+    safe_cast(caixa_postal as string) caixa_postal,
+from `basedosdados-staging.br_rf_cno_staging.microdados` microdados
+left join
+    (
+        select id_municipio, id_municipio_rf
+        from `basedosdados.br_bd_diretorios_brasil.municipio`
+    ) b
+    on ltrim(microdados.id_municipio_rf, '0') = b.id_municipio_rf
+
+{% if is_incremental() %} where data > (select max(data) from {{ this }}) {% endif %}

models/br_rf_cno/br_rf_cno__vinculos.sql

@@ -0,0 +1,27 @@
+{{
+    config(
+        alias="vinculos",
+        schema="br_rf_cno",
+        materialized="incremental",
+        partition_by={
+            "field": "data_extracao",
+            "data_type": "date",
+        },
+        pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}",
+    )
+}}
+
+
+select
+    safe_cast(data as date) data_extracao,
+    safe_cast(data_registro as date) data_registro,
+    safe_cast(data_inicio as date) data_inicio,
+    safe_cast(data_fim as date) data_fim,
+    safe_cast(id_cno as string) id_cno,
+    safe_cast(id_responsavel as string) id_responsavel,
+    safe_cast(
+        ltrim(qualificacao_contribuinte, '0') as string
+    ) qualificacao_contribuinte,
+from `basedosdados-staging.br_rf_cno_staging.vinculos` as t
+
+{% if is_incremental() %} where data > (select max(data) from {{ this }}) {% endif %}