Merge pull request #75 from ThiagoPanini/2.0.x

Included test cases for aux functions on the application side
ThiagoPanini · Apr 29, 2023 · 2d4e42f · 2d4e42f
2 parents c326ced + 3a48b6f
commit 2d4e42f
Show file tree

Hide file tree

Showing 17 changed files with 763 additions and 1 deletion.
diff --git a/.github/codecov.yml b/.github/codecov.yml
@@ -0,0 +1,14 @@
+ignore:
+  - setup.py
+coverage:
+  status:
+    patch:
+      default:
+        target: 0%
+        threshold: 5%
+        if_ci_failed: error
+    project:
+      default:
+        target: 0%
+        threshold: 5%
+        if_ci_failed: error
diff --git a/.github/workflows/ci-feature.yml b/.github/workflows/ci-feature.yml
@@ -23,11 +23,43 @@ jobs:
       - name: Terraform Validate
         uses: dflook/terraform-validate@v1
 
+  ci-python:
+    name: ci-python
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Python Install
+        uses: actions/setup-python@v3
+        with:
+          python-version: '3.10'
+
+      - name: Project Requirements
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -r ./requirements/dev.txt
+      
+      - name: Linter with flake8
+        run:
+          flake8 . --ignore E501 
+
+      - name: Unit Test with pytest
+        run: |
+          python3 -m pytest -vv --color=yes --cov=./ --cov-report=xml
+
+      - name: Test Coverage with codecov
+        uses: codecov/codecov-action@v3
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+
   open-pr-to-main:
     name: open-pr-to-main
     runs-on: ubuntu-latest
     needs:
       - ci-terraform
+      - ci-python
 
     steps:
       - name: Checkout

diff --git a/.github/workflows/ci-main.yml b/.github/workflows/ci-main.yml
@@ -20,3 +20,34 @@ jobs:
 
       - name: Terraform Validate
         uses: dflook/terraform-validate@v1
+
+  ci-python:
+    name: ci-python
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Python Install
+        uses: actions/setup-python@v3
+        with:
+          python-version: '3.10'
+
+      - name: Project Requirements
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -r ./requirements/dev.txt
+      
+      - name: Linter with flake8
+        run:
+          flake8 . --ignore E501 
+
+      - name: Unit Test with pytest
+        run: |
+          python3 -m pytest -vv --color=yes --cov=./ --cov-report=xml
+
+      - name: Test Coverage with codecov
+        uses: codecov/codecov-action@v3
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
diff --git a/app/src/transformers.py b/app/src/transformers.py
@@ -85,6 +85,25 @@ def transform_orders(df: DataFrame) -> DataFrame:
             weekofyear=True
         )
 
+        # Selecting attributes
+        df_orders_prep = df_orders_prep.selectExpr(
+            "order_id",
+            "customer_id",
+            "order_status",
+            "order_approved_at",
+            "order_deliv_carrier_dt",
+            "order_deliv_customer_dt",
+            "order_estim_deliv_dt",
+            "order_purchase_ts",
+            "year_order_purchase_ts",
+            "quarter_order_purchase_ts",
+            "month_order_purchase_ts",
+            "dayofmonth_order_purchase_ts",
+            "dayofweek_order_purchase_ts",
+            "dayofyear_order_purchase_ts",
+            "weekofyear_order_purchase_ts"
+        )
+
         return df_orders_prep
 
     except Exception as e:

diff --git a/app/tests/__init__.py b/app/tests/__init__.py
diff --git a/app/tests/configs/expected_schemas.json b/app/tests/configs/expected_schemas.json
@@ -0,0 +1,14 @@
+{
+    "expected": [
+        {
+            "dataframe_reference": "df_orders_prep",
+            "schema": [
+                {
+                    "attribute": "idx",
+                    "dtype": "int",
+                    "nullable": false
+                }
+            ]
+        }
+    ]
+}
diff --git a/app/tests/configs/source_schemas.json b/app/tests/configs/source_schemas.json
@@ -0,0 +1,69 @@
+{
+    "source": [
+        {
+            "name": "tbl_brecommerce_orders",
+            "dataframe_reference": "df_orders",
+            "schema": [
+                {
+                    "attribute": "idx",
+                    "dtype": "int",
+                    "nullable": true
+                },
+                {
+                    "attribute": "order_id",
+                    "dtype": "string",
+                    "nullable": true
+                },
+                {
+                    "attribute": "customer_id",
+                    "dtype": "string",
+                    "nullable": true
+                },
+                {
+                    "attribute": "order_status",
+                    "dtype": "string",
+                    "nullable": true
+                },
+                {
+                    "attribute": "order_purchase_ts",
+                    "dtype": "string",
+                    "nullable": true
+                },
+                {
+                    "attribute": "order_approved_at",
+                    "dtype": "string",
+                    "nullable": true
+                },
+                {
+                    "attribute": "order_deliv_carrier_dt",
+                    "dtype": "string",
+                    "nullable": true
+                },
+                {
+                    "attribute": "order_deliv_customer_dt",
+                    "dtype": "string",
+                    "nullable": true
+                },
+                {
+                    "attribute": "order_estim_deliv_dt",
+                    "dtype": "string",
+                    "nullable": true
+                }
+            ],
+            "empty": false,
+            "fake_data": false,
+            "data": [
+                [1, "e481f51cbdc54678b7cc49136f2d6af7", "9ef432eb6251297304e76186b10a928d", "delivered", "02/10/2017 10:56", "02/10/2017 11:07", "04/10/2017 19:55", "10/10/2017 21:25", "18/10/2017 00:00"],
+                [2, "53cdb2fc8bc7dce0b6741e2150273451", "b0830fb4747a6c6d20dea0b8c802d7ef", "delivered", "24/07/2018 20:41", "26/07/2018 03:24", "26/07/2018 14:31", "07/08/2018 15:27", "13/08/2018 00:00"],
+                [3, "47770eb9100c2d0c44946d9cf07ec65d", "41ce2a54c0b03bf3443c3d931a367089", "delivered", "08/08/2018 08:38", "08/08/2018 08:55", "08/08/2018 13:50", "17/08/2018 18:06", "04/09/2018 00:00"],
+                [4, "949d5b44dbf5de918fe9c16f97b45f8a", "f88197465ea7920adcdbec7375364d82", "delivered", "18/11/2017 19:28", "18/11/2017 19:45", "22/11/2017 13:39", "02/12/2017 00:28", "15/12/2017 00:00"],
+                [5, "ad21c59c0840e6cb83a9ceb5573f8159", "8ab97904e6daea8866dbdbc4fb7aad2c", "delivered", "13/02/2018 21:18", "13/02/2018 22:20", "14/02/2018 19:46", "16/02/2018 18:17", "26/02/2018 00:00"],
+                [6, "a4591c265e18cb1dcee52889e2d8acc3", "503740e9ca751ccdda7ba28e9ab8f608", "delivered", "09/07/2017 21:57", "09/07/2017 22:10", "11/07/2017 14:58", "26/07/2017 10:57", "01/08/2017 00:00"],
+                [7, "136cce7faa42fdb2cefd53fdc79a6098", "ed0271e0b7da060a393796590e7b737a", "invoiced", "11/04/2017 12:22", "13/04/2017 13:25", "09/05/2017 00:00", "", ""],
+                [8, "6514b8ad8028c9f2cc2374ded245783f", "9bdf08b4b3b52b5526ff42d37d47f222", "delivered", "16/05/2017 13:10", "16/05/2017 13:22", "22/05/2017 10:07", "26/05/2017 12:55", "07/06/2017 00:00"],
+                [9, "76c6e866289321a7c93b82b54852dc33", "f54a9f0e6b351c431402b8461ea51999", "delivered", "23/01/2017 18:29", "25/01/2017 02:50", "26/01/2017 14:16", "02/02/2017 14:08", "06/03/2017 00:00"],
+                [10, "e69bfb5eb88e0ed6a785585b27e16dbf", "31ad1d1b63eb9962463f764d4e6e0c9d", "delivered", "29/07/2017 11:55", "29/07/2017 12:05", "10/08/2017 19:45", "16/08/2017 17:14", "23/08/2017 00:00"]
+            ]
+        }
+    ]
+}
diff --git a/app/tests/conftest.py b/app/tests/conftest.py
@@ -0,0 +1,57 @@
+"""Confest file for managing pytest fixtures and other components.
+
+This file will handle essential components and elements to be used on test
+scripts along the project, like features and other things.
+
+___
+"""
+
+# Importing libraries
+import pytest
+import os
+import json
+
+from pyspark.sql import SparkSession, DataFrame
+
+from tests.helpers.dataframes import create_spark_dataframe_from_json_info
+
+# from src.transformers import transform_orders
+
+
+# Creating a SparkSession object
+spark = SparkSession.builder.getOrCreate()
+
+# Defining paths for JSON files with infos to create Spark DataFrames
+SOURCE_JSON_SCHEMAS_PATH = os.path.join(
+    os.getcwd(), "app/tests/configs/source_schemas.json"
+)
+
+
+# A JSON file loaded with source schema definition
+@pytest.fixture()
+def json_data_info():
+    with open(SOURCE_JSON_SCHEMAS_PATH, "r") as f:
+        return json.load(f)["source"]
+
+
+# A dictionary with all source DataFrames to be used on the Glue job
+@pytest.fixture()
+def source_dataframes_dict() -> dict:
+    return create_spark_dataframe_from_json_info(
+        json_path=SOURCE_JSON_SCHEMAS_PATH,
+        spark=spark
+    )
+
+
+# A df_orders sample DataFrame
+@pytest.fixture()
+def df_orders(source_dataframes_dict: dict) -> DataFrame:
+    return source_dataframes_dict["df_orders"]
+
+
+# A df_orders_prep generated running the transform_orders function
+"""
+@pytest.fixture()
+def df_orders_prep(df_orders) -> DataFrame:
+    return transform_orders(df=df_orders)
+"""
diff --git a/app/tests/helpers/__init__.py b/app/tests/helpers/__init__.py