Merge pull request #592 from basedosdados/staging/fix_cnes_test

[dbt] br_ms_cnes
basedosdados · May 16, 2024 · e83da3b · e83da3b
2 parents 3133b3a + 7d7bc4c
commit e83da3b
Show file tree

Hide file tree

Showing 17 changed files with 544 additions and 177 deletions.
diff --git a/models/br_ms_cnes/br_ms_cnes__dados_complementares.sql b/models/br_ms_cnes/br_ms_cnes__dados_complementares.sql
@@ -8,11 +8,6 @@
             "data_type": "int64",
             "range": {"start": 2005, "end": 2024, "interval": 1},
         },
-        pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}",
-        post_hook=[
-            'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter                     ON {{this}}                     GRANT TO ("allUsers")                     FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
-            'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter        ON  {{this}}                     GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org")                     FILTER USING (True)',
-        ],
     )
 }}
 with

diff --git a/models/br_ms_cnes/br_ms_cnes__dicionario.sql b/models/br_ms_cnes/br_ms_cnes__dicionario.sql
@@ -5,6 +5,6 @@ select
     safe_cast(id_tabela as string) id_tabela,
     safe_cast(nome_coluna as string) nome_coluna,
     safe_cast(chave as string) chave,
-    safe_cast(cobertura_temporal as string) cobertura_temporal,
+    safe_cast(replace(cobertura_temporal, '-1', '(1)') as string) cobertura_temporal,
     safe_cast(valor as string) valor,
 from `basedosdados-staging.br_ms_cnes_staging.dicionario` as t
diff --git a/models/br_ms_cnes/br_ms_cnes__equipamento.sql b/models/br_ms_cnes/br_ms_cnes__equipamento.sql
@@ -8,11 +8,6 @@
             "data_type": "int64",
             "range": {"start": 2005, "end": 2024, "interval": 1},
         },
-        pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}",
-        post_hook=[
-            'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter                     ON {{this}}                     GRANT TO ("allUsers")                     FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
-            'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter        ON  {{this}}                     GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org")                     FILTER USING (True)',
-        ],
     )
 }}
 
@@ -45,7 +40,7 @@ select
     safe_cast(sigla_uf as string) as sigla_uf,
     safe_cast(id_municipio as string) as id_municipio,
     safe_cast(cnes as string) as id_estabelecimento_cnes,
-    safe_cast(codequip as string) as id_equipamento,
+    ltrim(safe_cast(codequip as string), '0') as id_equipamento,
     safe_cast(tipequip as string) as tipo_equipamento,
     safe_cast(qt_exist as string) as quantidade_equipamentos,
     safe_cast(qt_uso as string) as quantidade_equipamentos_ativos,

diff --git a/models/br_ms_cnes/br_ms_cnes__equipe.sql b/models/br_ms_cnes/br_ms_cnes__equipe.sql
@@ -8,11 +8,6 @@
             "data_type": "int64",
             "range": {"start": 2005, "end": 2024, "interval": 1},
         },
-        pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}",
-        post_hook=[
-            'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter                     ON {{this}}                     GRANT TO ("allUsers")                     FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
-            'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter        ON  {{this}}                     GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org")                     FILTER USING (True)',
-        ],
     )
 }}
 
@@ -46,17 +41,41 @@ select
     safe_cast(id_municipio as string) as id_municipio,
     safe_cast(cnes as string) as id_estabelecimento_cnes,
     safe_cast(id_equipe as string) as id_equipe,
-    safe_cast(tipo_eqp as string) as tipo_equipe,
+    safe_cast(
+        case
+            when regexp_replace(tipo_eqp, '^0+', '') = ''
+            then '0'
+            else regexp_replace(tipo_eqp, '^0+', '')
+        end as string
+    ) as tipo_equipe,
     safe_cast(nome_eqp as string) as equipe,
     safe_cast(nomearea as string) as area,
     safe_cast(id_segm as string) as id_segmento,
-    safe_cast(tiposegm as string) as tipo_segmento,
+    safe_cast(
+        case
+            when regexp_replace(tiposegm, '^0+', '') = ''
+            then '0'
+            else regexp_replace(tiposegm, '^0+', '')
+        end as string
+    ) as tipo_segmento,
     safe_cast(descsegm as string) as descricao_segmento,
     -- - inserir subsrt para criar ano e mes
     safe_cast(substr(dt_ativa, 1, 4) as int64) as ano_ativacao_equipe,
     safe_cast(substr(dt_ativa, 5, 6) as int64) as mes_ativacao_equipe,
-    safe_cast(motdesat as string) as motivo_desativacao_equipe,
-    safe_cast(tp_desat as string) as tipo_desativacao_equipe,
+    safe_cast(
+        case
+            when regexp_replace(motdesat, '^0+', '') = ''
+            then '0'
+            else regexp_replace(motdesat, '^0+', '')
+        end as string
+    ) as motivo_desativacao_equipe,
+    safe_cast(
+        case
+            when regexp_replace(tp_desat, '^0+', '') = ''
+            then '0'
+            else regexp_replace(tp_desat, '^0+', '')
+        end as string
+    ) as tipo_desativacao_equipe,
     safe_cast(substr(dt_desat, 1, 4) as int64) as ano_desativacao_equipe,
     safe_cast(substr(dt_desat, 5, 6) as int64) as mes_desativacao_equipe,
     safe_cast(quilombo as int64) as indicador_atende_populacao_assistida_quilombolas,

diff --git a/models/br_ms_cnes/br_ms_cnes__estabelecimento.sql b/models/br_ms_cnes/br_ms_cnes__estabelecimento.sql
@@ -8,11 +8,6 @@
             "data_type": "int64",
             "range": {"start": 2005, "end": 2024, "interval": 1},
         },
-        pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}",
-        post_hook=[
-            'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter                     ON {{this}}                     GRANT TO ("allUsers")                     FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
-            'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter        ON  {{this}}                     GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org")                     FILTER USING (True)',
-        ],
     )
 }}
 with
@@ -61,17 +56,51 @@ select
     -- fazer replace em valores de linha com 14 zeros para null. 14 zeros é o tamanho
     -- de um cpf/cnpj nulo
     safe_cast(regexp_replace(cnpj_man, '0{14}', '') as string) cnpj_mantenedora,
-    safe_cast({{ clean_cols("COD_IR") }} as string) tipo_retencao_tributos_mantenedora,
+    replace(
+        safe_cast({{ clean_cols("COD_IR") }} as string), 'nan', null
+    ) tipo_retencao_tributos_mantenedora,
     safe_cast(vinc_sus as int64) indicador_vinculo_sus,
     safe_cast(tpgestao as string) tipo_gestao,
-    safe_cast({{ clean_cols("ESFERA_A") }} as string) tipo_esfera_administrativa,
-    safe_cast(retencao as string) tipo_retencao_tributos,
+    replace(
+        safe_cast({{ clean_cols("ESFERA_A") }} as string), 'nan', null
+    ) tipo_esfera_administrativa,
+    replace(
+        safe_cast(
+            case
+                when regexp_replace(safe_cast(retencao as string), '^0+', '') = ''
+                then '0'
+                else regexp_replace(safe_cast(retencao as string), '^0+', '')
+            end as string
+        ),
+        'nan',
+        null
+    ) as tipo_retencao_tributos,
     safe_cast({{ clean_cols("ATIVIDAD") }} as string) tipo_atividade_ensino_pesquisa,
-    safe_cast(natureza as string) tipo_natureza_administrativa,
-    safe_cast(nat_jur as string) id_natureza_juridica,
-    safe_cast(clientel as string) tipo_fluxo_atendimento,
+    replace(
+        safe_cast(
+            case
+                when regexp_replace(safe_cast(natureza as string), '^0+', '') = ''
+                then '0'
+                else regexp_replace(safe_cast(natureza as string), '^0+', '')
+            end as string
+        ),
+        'nan',
+        null
+    ) as tipo_natureza_administrativa,
+    nullif(safe_cast(nat_jur as string), '') id_natureza_juridica,
+    replace(
+        safe_cast(
+            case
+                when regexp_replace(safe_cast(clientel as string), '^0+', '') = ''
+                then '0'
+                else regexp_replace(safe_cast(clientel as string), '^0+', '')
+            end as string
+        ),
+        'nan',
+        null
+    ) as tipo_fluxo_atendimento,
     safe_cast({{ clean_cols("TP_UNID") }} as string) tipo_unidade,
-    safe_cast({{ clean_cols("TURNO_AT") }} as string) tipo_turno,
+    replace(safe_cast({{ clean_cols("TURNO_AT") }} as string), 'nan', null) tipo_turno,
     safe_cast({{ clean_cols("NIV_HIER") }} as string) tipo_nivel_hierarquia,
     safe_cast({{ clean_cols("TP_PREST") }} as string) tipo_prestador,
     safe_cast(co_banco as string) banco,
@@ -89,11 +118,23 @@ select
     safe_cast(
         safe.parse_date('%Y%m%d', cast(dt_exped as string)) as date
     ) data_expedicao_alvara,
-    safe_cast({{ clean_cols("ORGEXPED") }} as string) tipo_orgao_expedidor,
-    safe_cast(
-        {{ clean_cols("AV_ACRED") }} as string
+    replace(
+        safe_cast({{ clean_cols("ORGEXPED") }} as string), 'nan', null
+    ) tipo_orgao_expedidor,
+    replace(
+        safe_cast({{ clean_cols("AV_ACRED") }} as string), 'nan', null
     ) tipo_avaliacao_acreditacao_hospitalar,
-    safe_cast(clasaval as string) tipo_classificacao_acreditacao_hospitalar,
+    replace(
+        safe_cast(
+            case
+                when regexp_replace(safe_cast(clasaval as string), '^0+', '') = ''
+                then '0'
+                else regexp_replace(safe_cast(clasaval as string), '^0+', '')
+            end as string
+        ),
+        'nan',
+        null
+    ) as tipo_classificacao_acreditacao_hospitalar,
     safe_cast(substr(cast(dt_acred as string), 1, 4) as int64) as ano_acreditacao,
     safe_cast(substr(cast(dt_acred as string), 5, 2) as int64) as mes_acreditacao,
     safe_cast(

diff --git a/models/br_ms_cnes/br_ms_cnes__estabelecimento_ensino.sql b/models/br_ms_cnes/br_ms_cnes__estabelecimento_ensino.sql
@@ -8,11 +8,6 @@
             "data_type": "int64",
             "range": {"start": 2005, "end": 2024, "interval": 1},
         },
-        pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}",
-        post_hook=[
-            'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter                     ON {{this}}                     GRANT TO ("allUsers")                     FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
-            'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter        ON  {{this}}                     GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org")                     FILTER USING (True)',
-        ],
     )
 }}
 with
@@ -48,7 +43,7 @@ select
     cast(substr(cmpt_ini, 5, 2) as int64) as mes_competencia_inicial,
     cast(substr(cmpt_fim, 1, 4) as int64) as ano_competencia_final,
     cast(substr(cmpt_fim, 5, 2) as int64) as mes_competencia_final,
-    safe_cast(sgruphab as string) tipo_habilitacao,
+    ltrim(safe_cast(sgruphab as string), '0') as tipo_habilitacao,
     safe_cast(portaria as string) portaria,
     cast(
         concat(

diff --git a/models/br_ms_cnes/br_ms_cnes__estabelecimento_filantropico.sql b/models/br_ms_cnes/br_ms_cnes__estabelecimento_filantropico.sql
@@ -8,11 +8,6 @@
             "data_type": "int64",
             "range": {"start": 2005, "end": 2024, "interval": 1},
         },
-        pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}",
-        post_hook=[
-            'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter                     ON {{this}}                     GRANT TO ("allUsers")                     FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
-            'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter        ON  {{this}}                     GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org")                     FILTER USING (True)',
-        ],
     )
 }}
 with
@@ -49,7 +44,7 @@ select
     cast(substr(cmpt_ini, 5, 2) as int64) as mes_competencia_inicial,
     cast(substr(cmpt_fim, 1, 4) as int64) as ano_competencia_final,
     cast(substr(cmpt_fim, 5, 2) as int64) as mes_competencia_final,
-    safe_cast(sgruphab as string) tipo_habilitacao,
+    ltrim(safe_cast(sgruphab as string), '0') as tipo_habilitacao,
     safe_cast(portaria as string) portaria,
     cast(
         concat(
@@ -61,8 +56,9 @@ select
         ) as date
     ) data_portaria,
     cast(substr(maportar, 1, 4) as int64) as ano_portaria,
-    cast(substr(maportar, 5, 2) as int64) as mes_portaria,
+    cast(substr(maportar, 5, 2) as int64) as mes_portaria
 from cnes_add_muni
+
 {% if is_incremental() %}
     where
 

diff --git a/models/br_ms_cnes/br_ms_cnes__gestao_metas.sql b/models/br_ms_cnes/br_ms_cnes__gestao_metas.sql
@@ -8,11 +8,6 @@
             "data_type": "int64",
             "range": {"start": 2005, "end": 2024, "interval": 1},
         },
-        pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}",
-        post_hook=[
-            'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter                     ON {{this}}                     GRANT TO ("allUsers")                     FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
-            'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter        ON  {{this}}                     GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org")                     FILTER USING (True)',
-        ],
     )
 }}
 with
@@ -48,7 +43,7 @@ select
     cast(substr(cmpt_ini, 5, 2) as int64) as mes_competencia_inicial,
     cast(substr(cmpt_fim, 1, 4) as int64) as ano_competencia_final,
     cast(substr(cmpt_fim, 5, 2) as int64) as mes_competencia_final,
-    safe_cast(sgruphab as string) tipo_habilitacao,
+    ltrim(safe_cast(sgruphab as string), '0') tipo_habilitacao,
     case
         when safe_cast(sgruphab as string) in ("7003", "7004", "7005", "7006")
         then '1'

diff --git a/models/br_ms_cnes/br_ms_cnes__habilitacao.sql b/models/br_ms_cnes/br_ms_cnes__habilitacao.sql
@@ -8,11 +8,6 @@
             "data_type": "int64",
             "range": {"start": 2005, "end": 2024, "interval": 1},
         },
-        pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}",
-        post_hook=[
-            'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter                     ON {{this}}                     GRANT TO ("allUsers")                     FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
-            'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter        ON  {{this}}                     GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org")                     FILTER USING (True)',
-        ],
     )
 }}
 with
@@ -33,7 +28,7 @@ with
         left join
             (
                 select id_municipio, id_municipio_6,
-                from `basedosdados-dev.br_bd_diretorios_brasil.municipio`
+                from `basedosdados.br_bd_diretorios_brasil.municipio`
             ) as mun
             on raw_cnes_habilitacao_without_duplicates.codufmun = mun.id_municipio_6
     )
@@ -49,7 +44,7 @@ select
     safe_cast(substr(cmpt_ini, 5, 2) as int64) as mes_competencia_inicial,
     safe_cast(substr(cmpt_fim, 1, 4) as int64) as ano_competencia_final,
     safe_cast(substr(cmpt_fim, 5, 2) as int64) as mes_competencia_final,
-    safe_cast(sgruphab as string) tipo_habilitacao,
+    ltrim(safe_cast(sgruphab as string), '0') as tipo_habilitacao,
     case
         when
             safe_cast(sgruphab as string) in (

diff --git a/models/br_ms_cnes/br_ms_cnes__incentivos.sql b/models/br_ms_cnes/br_ms_cnes__incentivos.sql
@@ -8,11 +8,6 @@
             "data_type": "int64",
             "range": {"start": 2005, "end": 2024, "interval": 1},
         },
-        pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}",
-        post_hook=[
-            'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter                     ON {{this}}                     GRANT TO ("allUsers")                     FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
-            'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter        ON  {{this}}                     GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org")                     FILTER USING (True)',
-        ],
     )
 }}
 with
@@ -48,7 +43,7 @@ select
     cast(substr(cmpt_ini, 5, 2) as int64) as mes_competencia_inicial,
     cast(substr(cmpt_fim, 1, 4) as int64) as ano_competencia_final,
     cast(substr(cmpt_fim, 5, 2) as int64) as mes_competencia_final,
-    safe_cast(sgruphab as string) tipo_habilitacao,
+    ltrim(safe_cast(sgruphab as string), '0') as tipo_habilitacao,
     case
         when safe_cast(sgruphab as string) in ("8105", "8106", "8107") then '2' else '1'
     end as tipo_incentivo,

diff --git a/models/br_ms_cnes/br_ms_cnes__leito.sql b/models/br_ms_cnes/br_ms_cnes__leito.sql
@@ -8,11 +8,6 @@
             "data_type": "int64",
             "range": {"start": 2007, "end": 2024, "interval": 1},
         },
-        pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}",
-        post_hook=[
-            'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter                     ON {{this}}                     GRANT TO ("allUsers")                     FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
-            'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter        ON  {{this}}                     GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org")                     FILTER USING (True)',
-        ],
     )
 }}
 
@@ -52,8 +47,8 @@ select
     safe_cast(sigla_uf as string) as sigla_uf,
     safe_cast(id_municipio as string) as id_municipio,
     safe_cast(cnes as string) as id_estabelecimento_cnes,
-    safe_cast(codleito as string) as tipo_especialidade_leito,
-    safe_cast(tp_leito as string) as tipo_leito,
+    ltrim(safe_cast(codleito as string), '0') as tipo_especialidade_leito,
+    ltrim(safe_cast(tp_leito as string), '0') as tipo_leito,
     safe_cast(qt_exist as int64) as quantidade_total,
     safe_cast(qt_contr as int64) as quantidade_contratado,
     safe_cast(qt_sus as int64) as quantidade_sus

diff --git a/models/br_ms_cnes/br_ms_cnes__profissional.sql b/models/br_ms_cnes/br_ms_cnes__profissional.sql
@@ -8,11 +8,6 @@
             "data_type": "int64",
             "range": {"start": 2005, "end": 2024, "interval": 1},
         },
-        pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}",
-        post_hook=[
-            'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter                     ON {{this}}                     GRANT TO ("allUsers")                     FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
-            'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter        ON  {{this}}                     GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org")                     FILTER USING (True)',
-        ],
     )
 }}
 with
@@ -48,13 +43,27 @@ select
     -- UFMUNRES que indica null
     safe_cast(regexp_replace(ufmunres, '0{6}', '') as string) id_municipio_6_residencia,
     safe_cast(nomeprof as string) nome,
-    safe_cast(vinculac as string) id_vinculo,
-    safe_cast(registro as string) id_registro_conselho,
-    safe_cast(conselho as string) id_conselho,
+    safe_cast(replace(vinculac, 'nan', null) as string) tipo_vinculo,
+    safe_cast(replace(registro, 'nan', null) as string) id_registro_conselho,
+    safe_cast(replace(conselho, 'nan', null) as string) tipo_conselho,
     -- replace de valores de linha com 15 zeros para null. 15 zeros é valor do campo
     -- CNS_PROF que indica null
-    safe_cast(regexp_replace(cns_prof, '0{15}', '') as string) cartao_nacional_saude,
-    safe_cast(cbo as string) cbo_2002,
+    safe_cast(regexp_replace(cns_prof, '0{15}', null) as string) cartao_nacional_saude,
+    safe_cast(cbo as string) cbo_2002_original,
+    safe_cast(
+        case
+            when length(cbo) = 6 and regexp_contains(cbo, r'^[0-9]{6}$')
+            then cbo
+            else null
+        end as string
+    ) cbo_2002,
+    safe_cast(
+        case
+            when length(cbo) = 5 and regexp_contains(cbo, r'^[0-9]{5}$')
+            then cbo
+            else null
+        end as string
+    ) cbo_1994,
     safe_cast(terceiro as int64) indicador_estabelecimento_terceiro,
     safe_cast(vincul_c as int64) indicador_vinculo_contratado_sus,
     safe_cast(vincul_a as int64) indicador_vinculo_autonomo_sus,