-
Notifications
You must be signed in to change notification settings - Fork 4
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #554 from basedosdados/staging/add-ncm-sh
[dbt] novos diretórios
- Loading branch information
Showing
6 changed files
with
256 additions
and
32 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,38 +1,34 @@ | ||
| --- | ||
| name: Check BQ project name | ||
| on: | ||
| workflow_dispatch: | ||
| pull_request: | ||
| paths: | ||
| - '**/*.sql' | ||
|
|
||
| paths: ['**/*.sql'] | ||
| jobs: | ||
| check_bucket_name: | ||
| runs-on: ubuntu-latest | ||
|
|
||
| steps: | ||
| - name: Checkout code | ||
| uses: actions/checkout@v2 | ||
|
|
||
| - name: Get changed files | ||
| id: get_files | ||
| uses: dorny/paths-filter@v2 | ||
| with: | ||
| list-files: shell | ||
| filters: | | ||
| pr: | ||
| - added|deleted|modified: '**' | ||
| - name: Install Python | ||
| uses: actions/setup-python@v2 | ||
| with: | ||
| python-version: '3.x' | ||
|
|
||
| - name: Run Python script | ||
| run: | | ||
| for file in ${{ steps.get_files.outputs.pr_files }}; do | ||
| if [[ $file == *.sql ]]; then | ||
| echo "SQL file detected: $file" | ||
| python .github/workflows/scripts/check_sql_files.py $file | ||
| else | ||
| echo "Não é um arquivo SQL: $file" | ||
| fi | ||
| done | ||
| - name: Checkout code | ||
| uses: actions/checkout@v2 | ||
| - name: Get changed files | ||
| id: get_files | ||
| uses: dorny/paths-filter@v2 | ||
| with: | ||
| list-files: shell | ||
| filters: | | ||
| pr: | ||
| - added|deleted|modified: '**' | ||
| - name: Install Python | ||
| uses: actions/setup-python@v2 | ||
| with: | ||
| python-version: 3.x | ||
| - name: Run Python script | ||
| run: |- | ||
| for file in ${{ steps.get_files.outputs.pr_files }}; do | ||
| if [[ $file == *.sql ]]; then | ||
| echo "SQL file detected: $file" | ||
| python .github/workflows/scripts/check_sql_files.py $file | ||
| else | ||
| echo "Não é um arquivo SQL: $file" | ||
| fi | ||
| done |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
52 changes: 52 additions & 0 deletions
52
models/br_bd_diretorios_mundo/br_bd_diretorios_mundo__nomenclatura_comum_mercosul.sql
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,52 @@ | ||
| {{ config(alias="nomenclatura_comum_mercosul", schema="br_bd_diretorios_mundo") }} | ||
| select | ||
| safe_cast(co_ncm as string) id_ncm, | ||
| safe_cast(co_unid as string) id_unidade, | ||
| safe_cast(co_sh6 as string) id_sh6, | ||
| safe_cast(co_ppe as string) id_ppe, | ||
| safe_cast(co_ppi as string) id_ppi, | ||
| safe_cast(co_fat_agreg as string) id_fator_agregado_ncm, | ||
| safe_cast(co_cuci_item as string) id_cgce_n3, | ||
| safe_cast(co_cgce_n3 as string) id_isic_classe, | ||
| safe_cast(co_siit as string) id_exp_subset, | ||
| safe_cast( | ||
| case | ||
| when co_unid = '10' | ||
| then lower('QUILOGRAMA LIQUIDO') | ||
| when co_unid = '11' | ||
| then lower('NUMERO (UNIDADE)') | ||
| when co_unid = '12' | ||
| then lower('MILHEIRO') | ||
| when co_unid = '13' | ||
| then lower('PARES') | ||
| when co_unid = '14' | ||
| then lower('METRO') | ||
| when co_unid = '15' | ||
| then lower('METRO QUADRADO') | ||
| when co_unid = '16' | ||
| then lower('METRO CUBICO') | ||
| when co_unid = '17' | ||
| then lower('LITRO') | ||
| when co_unid = '18' | ||
| then lower('MIL QUILOWATT HORA') | ||
| when co_unid = '19' | ||
| then lower('QUILATE') | ||
| when co_unid = '20' | ||
| then lower('DUZIA') | ||
| when co_unid = '21' | ||
| then lower('TONELADA METRICA LIQUIDA') | ||
| when co_unid = '22' | ||
| then lower('GRAMA LIQUIDO') | ||
| when co_unid = '23' | ||
| then lower('BILHOES DE UNIDADES INTERNACIONAIS') | ||
| when co_unid = '24' | ||
| then lower('QUILOGRAMA BRUTO') | ||
| else lower(co_unid) | ||
| end as string | ||
| ) as nome_unidade, | ||
| safe_cast(co_isic_classe as string) nome_ncm_portugues, | ||
| safe_cast(co_exp_subset as string) nome_ncm_espanhol, | ||
| safe_cast(no_ncm_por as string) nome_ncm_ingles, | ||
| from | ||
| `basedosdados-staging.br_bd_diretorios_mundo_staging.nomenclatura_comum_mercosul` | ||
| as t |
19 changes: 19 additions & 0 deletions
19
models/br_bd_diretorios_mundo/br_bd_diretorios_mundo__sistema_harmonizado.sql
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,19 @@ | ||
| {{ config(alias="sistema_harmonizado", schema="br_bd_diretorios_mundo") }} | ||
| select | ||
| safe_cast(co_sh6 as string) id_sh6, | ||
| safe_cast(co_sh4 as string) id_sh4, | ||
| safe_cast(co_sh2 as string) id_sh2, | ||
| safe_cast(co_ncm_secrom as string) id_ncm_secrom, | ||
| safe_cast(no_sh6_por as string) nome_sh6_portugues, | ||
| safe_cast(no_sh4_por as string) nome_sh4_portugues, | ||
| safe_cast(no_sh2_por as string) nome_sh2_portugues, | ||
| safe_cast(no_sec_por as string) nome_sec_portugues, | ||
| safe_cast(no_sh6_esp as string) nome_sh6_espanhol, | ||
| safe_cast(no_sh4_esp as string) nome_sh4_espanhol, | ||
| safe_cast(no_sh2_esp as string) nome_sh2_espanhol, | ||
| safe_cast(no_sec_esp as string) nome_sec_espanhol, | ||
| safe_cast(no_sh6_ing as string) nome_sh6_ingles, | ||
| safe_cast(no_sh4_ing as string) nome_sh4_ingles, | ||
| safe_cast(no_sh2_ing as string) nome_sh2_ingles, | ||
| safe_cast(no_sec_ing as string) nome_sec_ingles, | ||
| from `basedosdados-staging.br_bd_diretorios_mundo_staging.sistema_harmonizado` as t |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,67 @@ | ||
| { | ||
| "cells": [ | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": 5, | ||
| "metadata": {}, | ||
| "outputs": [], | ||
| "source": [ | ||
| "import pandas as pd\n", | ||
| "import requests\n", | ||
| "from io import BytesIO\n" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": null, | ||
| "metadata": {}, | ||
| "outputs": [], | ||
| "source": [ | ||
| "urls = {\n", | ||
| " 'ncm' : 'https://balanca.economia.gov.br/balanca/bd/tabelas/NCM.csv',\n", | ||
| " 'sh4': 'https://balanca.economia.gov.br/balanca/bd/tabelas/NCM_SH.csv'\n", | ||
| " }\n", | ||
| "\n", | ||
| "for k in urls:\n", | ||
| " print(f'Baixando tabela {k}')\n", | ||
| " \n", | ||
| " response = requests.get(urls[k], verify=False)\n", | ||
| " if response.status_code == 200:\n", | ||
| " \n", | ||
| " data = BytesIO(response.content)\n", | ||
| " df = pd.read_csv(data, encoding='latin-1', sep=';', dtype = str)\n", | ||
| " df.to_csv(\n", | ||
| " f'{k}.csv',\n", | ||
| " index=False,\n", | ||
| " encoding = 'utf-8',\n", | ||
| " sep=',', \n", | ||
| " )\n", | ||
| " else:\n", | ||
| " print(f'Erro! O status da requisição foi {response.status_code}')\n", | ||
| "\n", | ||
| " " | ||
| ] | ||
| } | ||
| ], | ||
| "metadata": { | ||
| "kernelspec": { | ||
| "display_name": "bd_pipelines", | ||
| "language": "python", | ||
| "name": "python3" | ||
| }, | ||
| "language_info": { | ||
| "codemirror_mode": { | ||
| "name": "ipython", | ||
| "version": 3 | ||
| }, | ||
| "file_extension": ".py", | ||
| "mimetype": "text/x-python", | ||
| "name": "python", | ||
| "nbconvert_exporter": "python", | ||
| "pygments_lexer": "ipython3", | ||
| "version": "3.9.15" | ||
| } | ||
| }, | ||
| "nbformat": 4, | ||
| "nbformat_minor": 2 | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters