diff --git a/404.html b/404.html index 61339fe..e0239b2 100644 --- a/404.html +++ b/404.html @@ -600,6 +600,39 @@ + +
  • + 12 - CI/CD + +
    • + + diff --git a/about/index.html b/about/index.html index 4c9d3f2..b644ce0 100644 --- a/about/index.html +++ b/about/index.html @@ -628,6 +628,39 @@ + +
  • + 12 - CI/CD + +
    • + + diff --git a/classes/01-intro/aps01_part_1/index.html b/classes/01-intro/aps01_part_1/index.html index 72591bb..560ae33 100644 --- a/classes/01-intro/aps01_part_1/index.html +++ b/classes/01-intro/aps01_part_1/index.html @@ -648,6 +648,39 @@ + +
  • + 12 - CI/CD + +
    • + + diff --git a/classes/01-intro/aps01_part_2/index.html b/classes/01-intro/aps01_part_2/index.html index 2b0d01d..2c139f4 100644 --- a/classes/01-intro/aps01_part_2/index.html +++ b/classes/01-intro/aps01_part_2/index.html @@ -624,6 +624,39 @@ + +
  • + 12 - CI/CD + +
    • + + diff --git a/classes/01-intro/intro/index.html b/classes/01-intro/intro/index.html index bbd51f8..b06eafc 100644 --- a/classes/01-intro/intro/index.html +++ b/classes/01-intro/intro/index.html @@ -620,6 +620,39 @@ + +
  • + 12 - CI/CD + +
    • + + diff --git a/classes/02-api/api_deploy/index.html b/classes/02-api/api_deploy/index.html index 36c66a5..c5181ed 100644 --- a/classes/02-api/api_deploy/index.html +++ b/classes/02-api/api_deploy/index.html @@ -632,6 +632,39 @@ + +
  • + 12 - CI/CD + +
    • + + diff --git a/classes/03-batch/aps02_sql/index.html b/classes/03-batch/aps02_sql/index.html index 893d992..485a69e 100644 --- a/classes/03-batch/aps02_sql/index.html +++ b/classes/03-batch/aps02_sql/index.html @@ -632,6 +632,39 @@ + +
  • + 12 - CI/CD + +
    • + + diff --git a/classes/03-batch/data_formats/index.html b/classes/03-batch/data_formats/index.html index 4bb49c4..25003c7 100644 --- a/classes/03-batch/data_formats/index.html +++ b/classes/03-batch/data_formats/index.html @@ -632,6 +632,39 @@ + +
  • + 12 - CI/CD + +
    • + + diff --git a/classes/03-batch/db_tool/index.html b/classes/03-batch/db_tool/index.html index 984aa49..e1f4a4c 100644 --- a/classes/03-batch/db_tool/index.html +++ b/classes/03-batch/db_tool/index.html @@ -620,6 +620,39 @@ + +
  • + 12 - CI/CD + +
    • + + diff --git a/classes/03-batch/dot_env/index.html b/classes/03-batch/dot_env/index.html index c50daaa..e540109 100644 --- a/classes/03-batch/dot_env/index.html +++ b/classes/03-batch/dot_env/index.html @@ -616,6 +616,39 @@ + +
  • + 12 - CI/CD + +
    • + + diff --git a/classes/03-batch/intro/index.html b/classes/03-batch/intro/index.html index 954df02..9a5480f 100644 --- a/classes/03-batch/intro/index.html +++ b/classes/03-batch/intro/index.html @@ -620,6 +620,39 @@ + +
  • + 12 - CI/CD + +
    • + + diff --git a/classes/03-batch/practicing/index.html b/classes/03-batch/practicing/index.html index 0a55009..61ed162 100644 --- a/classes/03-batch/practicing/index.html +++ b/classes/03-batch/practicing/index.html @@ -640,6 +640,39 @@ + +
  • + 12 - CI/CD + +
    • + + diff --git a/classes/03-batch/sql/index.html b/classes/03-batch/sql/index.html index 3e19ad5..e1a1c79 100644 --- a/classes/03-batch/sql/index.html +++ b/classes/03-batch/sql/index.html @@ -608,6 +608,39 @@ + +
  • + 12 - CI/CD + +
    • + + diff --git a/classes/04-int01/faq/index.html b/classes/04-int01/faq/index.html index 45a9a43..eb83995 100644 --- a/classes/04-int01/faq/index.html +++ b/classes/04-int01/faq/index.html @@ -616,6 +616,39 @@ + +
  • + 12 - CI/CD + +
    • + + diff --git a/classes/04-int01/int01/index.html b/classes/04-int01/int01/index.html index d48aff5..4911d88 100644 --- a/classes/04-int01/int01/index.html +++ b/classes/04-int01/int01/index.html @@ -608,6 +608,39 @@ + +
  • + 12 - CI/CD + +
    • + + diff --git a/classes/05-docker/docker/index.html b/classes/05-docker/docker/index.html index 4058536..f7b5ce8 100644 --- a/classes/05-docker/docker/index.html +++ b/classes/05-docker/docker/index.html @@ -624,6 +624,39 @@ + +
  • + 12 - CI/CD + +
    • + + diff --git a/classes/05-docker/intro/index.html b/classes/05-docker/intro/index.html index 8de12e9..e783793 100644 --- a/classes/05-docker/intro/index.html +++ b/classes/05-docker/intro/index.html @@ -620,6 +620,39 @@ + +
  • + 12 - CI/CD + +
    • + + diff --git a/classes/05-docker/s3/index.html b/classes/05-docker/s3/index.html index 42e6cbf..e526939 100644 --- a/classes/05-docker/s3/index.html +++ b/classes/05-docker/s3/index.html @@ -624,6 +624,39 @@ + +
  • + 12 - CI/CD + +
    • + + diff --git a/classes/06-message-broker/celery/index.html b/classes/06-message-broker/celery/index.html index 4aac1b7..3de9990 100644 --- a/classes/06-message-broker/celery/index.html +++ b/classes/06-message-broker/celery/index.html @@ -636,6 +636,39 @@ + +
  • + 12 - CI/CD + +
    • + + diff --git a/classes/06-message-broker/intro/index.html b/classes/06-message-broker/intro/index.html index 0e45a16..0a25386 100644 --- a/classes/06-message-broker/intro/index.html +++ b/classes/06-message-broker/intro/index.html @@ -624,6 +624,39 @@ + +
  • + 12 - CI/CD + +
    • + + diff --git a/classes/06-message-broker/rabbitmq/index.html b/classes/06-message-broker/rabbitmq/index.html index 7abb9f0..b8dd65a 100644 --- a/classes/06-message-broker/rabbitmq/index.html +++ b/classes/06-message-broker/rabbitmq/index.html @@ -624,6 +624,39 @@ + +
  • + 12 - CI/CD + +
    • + + diff --git a/classes/07-lambda/api_gateway/index.html b/classes/07-lambda/api_gateway/index.html index c505fce..c2e776e 100644 --- a/classes/07-lambda/api_gateway/index.html +++ b/classes/07-lambda/api_gateway/index.html @@ -624,6 +624,39 @@ + +
  • + 12 - CI/CD + +
    • + + diff --git a/classes/07-lambda/aps03_lambda/index.html b/classes/07-lambda/aps03_lambda/index.html index c8a0e97..ab5d6be 100644 --- a/classes/07-lambda/aps03_lambda/index.html +++ b/classes/07-lambda/aps03_lambda/index.html @@ -620,6 +620,39 @@ + +
  • + 12 - CI/CD + +
    • + + diff --git a/classes/07-lambda/aws_lambda/index.html b/classes/07-lambda/aws_lambda/index.html index b55e267..77430af 100644 --- a/classes/07-lambda/aws_lambda/index.html +++ b/classes/07-lambda/aws_lambda/index.html @@ -628,6 +628,39 @@ + +
  • + 12 - CI/CD + +
    • + + diff --git a/classes/07-lambda/lambda_and_docker/index.html b/classes/07-lambda/lambda_and_docker/index.html index 6bea6fa..aa4116b 100644 --- a/classes/07-lambda/lambda_and_docker/index.html +++ b/classes/07-lambda/lambda_and_docker/index.html @@ -640,6 +640,39 @@ + +
  • + 12 - CI/CD + +
    • + + diff --git a/classes/07-lambda/lambda_layer/index.html b/classes/07-lambda/lambda_layer/index.html index e06f54d..6b8eaa0 100644 --- a/classes/07-lambda/lambda_layer/index.html +++ b/classes/07-lambda/lambda_layer/index.html @@ -640,6 +640,39 @@ + +
  • + 12 - CI/CD + +
    • + + diff --git a/classes/07-lambda/lambda_practicing/index.html b/classes/07-lambda/lambda_practicing/index.html index ea40f9c..2d70338 100644 --- a/classes/07-lambda/lambda_practicing/index.html +++ b/classes/07-lambda/lambda_practicing/index.html @@ -636,6 +636,39 @@ + +
  • + 12 - CI/CD + +
    • + + diff --git a/classes/07-lambda/lambda_quotas/index.html b/classes/07-lambda/lambda_quotas/index.html index e880932..9b63c8b 100644 --- a/classes/07-lambda/lambda_quotas/index.html +++ b/classes/07-lambda/lambda_quotas/index.html @@ -608,6 +608,39 @@ + +
  • + 12 - CI/CD + +
    • + + diff --git a/classes/07-lambda/sa_lambda_function/index.html b/classes/07-lambda/sa_lambda_function/index.html index 6e9587f..397b6c3 100644 --- a/classes/07-lambda/sa_lambda_function/index.html +++ b/classes/07-lambda/sa_lambda_function/index.html @@ -628,6 +628,39 @@ + +
  • + 12 - CI/CD + +
    • + + diff --git a/classes/07-lambda/sentiment_analysis/index.html b/classes/07-lambda/sentiment_analysis/index.html index 51dccbd..154c826 100644 --- a/classes/07-lambda/sentiment_analysis/index.html +++ b/classes/07-lambda/sentiment_analysis/index.html @@ -620,6 +620,39 @@ + +
  • + 12 - CI/CD + +
    • + + diff --git a/classes/08-queue/concurrent_execution/index.html b/classes/08-queue/concurrent_execution/index.html index ca3df47..a9a89e3 100644 --- a/classes/08-queue/concurrent_execution/index.html +++ b/classes/08-queue/concurrent_execution/index.html @@ -632,6 +632,39 @@ + +
  • + 12 - CI/CD + +
    • + + diff --git a/classes/08-queue/intro/index.html b/classes/08-queue/intro/index.html index dd54b49..f753135 100644 --- a/classes/08-queue/intro/index.html +++ b/classes/08-queue/intro/index.html @@ -616,6 +616,39 @@ + +
  • + 12 - CI/CD + +
    • + + diff --git a/classes/08-queue/sqs_and_lambda/index.html b/classes/08-queue/sqs_and_lambda/index.html index cb05681..759800a 100644 --- a/classes/08-queue/sqs_and_lambda/index.html +++ b/classes/08-queue/sqs_and_lambda/index.html @@ -624,6 +624,39 @@ + +
  • + 12 - CI/CD + +
    • + + diff --git a/classes/08-queue/sqs_practicing/index.html b/classes/08-queue/sqs_practicing/index.html index e615d64..e33fece 100644 --- a/classes/08-queue/sqs_practicing/index.html +++ b/classes/08-queue/sqs_practicing/index.html @@ -628,6 +628,39 @@ + +
  • + 12 - CI/CD + +
    • + + diff --git a/classes/09-int02/int02/index.html b/classes/09-int02/int02/index.html index 4eb3aef..851edcc 100644 --- a/classes/09-int02/int02/index.html +++ b/classes/09-int02/int02/index.html @@ -608,6 +608,39 @@ + +
  • + 12 - CI/CD + +
    • + + diff --git a/classes/10-documentation/code_documentation/index.html b/classes/10-documentation/code_documentation/index.html index e1aaa58..8693972 100644 --- a/classes/10-documentation/code_documentation/index.html +++ b/classes/10-documentation/code_documentation/index.html @@ -620,6 +620,39 @@ + +
  • + 12 - CI/CD + +
    • + + diff --git a/classes/10-documentation/fast_api_docs/index.html b/classes/10-documentation/fast_api_docs/index.html index 7773077..c667b77 100644 --- a/classes/10-documentation/fast_api_docs/index.html +++ b/classes/10-documentation/fast_api_docs/index.html @@ -608,6 +608,39 @@ + +
  • + 12 - CI/CD + +
    • + + diff --git a/classes/10-documentation/intro/index.html b/classes/10-documentation/intro/index.html index 5e9a005..22bbcf9 100644 --- a/classes/10-documentation/intro/index.html +++ b/classes/10-documentation/intro/index.html @@ -624,6 +624,39 @@ + +
  • + 12 - CI/CD + +
    • + + diff --git a/classes/10-documentation/ml_canvas/index.html b/classes/10-documentation/ml_canvas/index.html index 0f4949f..563edee 100644 --- a/classes/10-documentation/ml_canvas/index.html +++ b/classes/10-documentation/ml_canvas/index.html @@ -620,6 +620,39 @@ + +
  • + 12 - CI/CD + +
    • + + diff --git a/classes/11-logging/intro/index.html b/classes/11-logging/intro/index.html index 717cad3..a98b9b9 100644 --- a/classes/11-logging/intro/index.html +++ b/classes/11-logging/intro/index.html @@ -620,6 +620,39 @@ + +
  • + 12 - CI/CD + +
    • + + diff --git a/classes/11-logging/python_logging/index.html b/classes/11-logging/python_logging/index.html index cfa8a0f..2556477 100644 --- a/classes/11-logging/python_logging/index.html +++ b/classes/11-logging/python_logging/index.html @@ -620,6 +620,39 @@ + +
  • + 12 - CI/CD + +
    • + + diff --git a/classes/11-logging/read_aws_logs/index.html b/classes/11-logging/read_aws_logs/index.html index 300b709..0ee7406 100644 --- a/classes/11-logging/read_aws_logs/index.html +++ b/classes/11-logging/read_aws_logs/index.html @@ -608,6 +608,39 @@ + +
  • + 12 - CI/CD + +
    • + + diff --git a/classes/11-logging/write_logs_to_s3/index.html b/classes/11-logging/write_logs_to_s3/index.html index 914b16b..cc82ae3 100644 --- a/classes/11-logging/write_logs_to_s3/index.html +++ b/classes/11-logging/write_logs_to_s3/index.html @@ -624,6 +624,39 @@ + +
  • + 12 - CI/CD + +
    • + + @@ -792,6 +825,13 @@

    Task

    + + Next + Intro + + diff --git a/classes/12-ci-cd/auto_deploy/index.html b/classes/12-ci-cd/auto_deploy/index.html new file mode 100644 index 0000000..9a63029 --- /dev/null +++ b/classes/12-ci-cd/auto_deploy/index.html @@ -0,0 +1,1043 @@ + + + + + + + + + Auto deploy - MLOps & Interviews + + + + + + + + + + + + + + + + +
    +
    + + +
    + + + + + + + +
    +
    + +
    + +
    +
      + + + +
    • Classes
      • + + + +
    • 12 - CI/CD
      • + + +
    • +
    + + +
    + +
    +

    Automate Deploy

    +

    Let's automate the deployment of the lambda function on AWS.

    +

    +

    Whenever a commit occurs in main, the action will be triggered and will run two jobs:

    +
      +
    • The first will run the automatic tests.
      • +
    • If the first job is successful, the second job will deploy the function to AWS.
      • +
    +

    Create Deploy Python Script

    +

    At some point in the action on github, we will need to communicate with AWS for the function to be deployed. This can be done in several ways, such as:

    +
      +
    • Python script (like we have been doing in class)
      • +
    • AWS CLI
      • +
    • AWS SAM
      • +
    +

    Among others.

    +

    Let's do a first example using Python Script.

    +
    +

    Question 1

    +
    +

    Create a deploy folder at root directory.

    +

    This directory will be used to store deploy scripts, like the one we will use to create the lambda function.

    +
    + + +
    +
    +
    +
    +

    Question 2

    +
    +

    Create the file deploy/requirements_deploy.txt with the content:

    +
    boto3
+
    +

    This file will contain the dependencies of the action, not the lambda function.

    +
    + + +
    +
    +
    +
    +

    Question 3

    +
    +

    Create the file deploy/create_function.py with the content:

    +
    import boto3
+import os
+
+# Provide function name: wc_INSPER_USERNAME
+function_name = ""
+
+lambda_client = boto3.client(
+    "lambda",
+    aws_access_key_id=os.getenv("AWS_ACCESS_KEY_ID"),
+    aws_secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY"),
+    region_name=os.getenv("AWS_REGION"),
+)
+
+lambda_role_arn = os.getenv("AWS_LAMBDA_ROLE_ARN")
+
+with open("word_count.zip", "rb") as f:
+    zip_to_deploy = f.read()
+
+lambda_response = lambda_client.create_function(
+    FunctionName=function_name,
+    Runtime="python3.10",
+    Role=lambda_role_arn,
+    Handler="word_count.word_count_handler",
+    Code={"ZipFile": zip_to_deploy},
+)
+
    +

    Now the file tree will be:

    +
    .
+├── deploy
+│   ├── create_function.py
+│   └── requirements_deploy.txt
+├── .github
+│   └── workflows
+│       └── test_workflow.yaml
+├── .gitignore
+├── pytest.ini
+├── README.md
+├── requirements.txt
+├── src
+│   └── word_count.py
+└── tests
+    └── test_word_count.py
+
    +
    + + +
    +
    +
    +

    Create Job: Github Action

    +

    Let's create a new job in the github action workflow.

    +
    +

    Attention!

    +

    Check if your branch name is main.

    +

    Change if necessary.

    +
    +
    +

    Question 4

    +
    +

    To do this, update the .github/workflows/test_workflow.yaml file:

    +
    name: An example of action with two jobs - one for testing, one for aws lambda deploy
+on:
+  push:
+    branches:
+      - main
+jobs:
+  build-and-test:
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+
+      - name: Install dependencies
+        run: pip install -r requirements.txt
+
+      - name: Run tests
+        run: pytest
+
+  deploy-to-aws:
+    needs: build-and-test
+    runs-on: ubuntu-latest
+    env:
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_REGION: ${{ secrets.AWS_REGION }}
+      AWS_LAMBDA_ROLE_ARN: ${{ secrets.AWS_LAMBDA_ROLE_ARN }}
+    steps:
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Zip Python file
+        run: |
+          zip -j word_count.zip src/word_count.py
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+
+      - name: Install dependencies
+        run: pip install -r deploy/requirements_deploy.txt
+
+      - name: Run deploy file
+        run: python deploy/create_function.py
+
    +
    + + +
    +
    +
    +
    +

    Tip! 1

    +

    Notice the new deploy-to-aws job!

    +
    +

    The deploy-to-aws job is very similar to the previous one (repository checkout, install python, install dependencies). The main differences are:

    +
      +
    • +

      Concurrency: Note that the job has needs: build-and-test. This will define that jobs must run in sequence and never in parallel. So, if the test fails there will be no deploy!

      +
      • +
    • +

      Variables: check the env section, where we define some secret variables. We will explore this topic further in the next subsection.

      +
      • +
    • +

      ZIP: we will deploy the lambda function using ZIP. Thus, this file will be created inside the container and used immediately afterwards.

      +
      • +
    • +

      Run deploy file: The python script that creates the lambda function is called at the end of the job.

      +
      • +
    +

    Secrets: Github Action

    +

    Secrets allow you to store sensitive information in your organization, repository, or repository environments. They are variables that you create in an organization, repository, or repository environment.

    +

    The secrets that you create are available to use in GitHub Actions workflows. We will use them to store AWS credentials, as it wouldn't make sense to keep a file in the repository with such information!

    +
    +

    Tip! 2

    +

    GitHub Actions can only read a secret if you explicitly include the secret in a workflow.

    +

    This is why the env section exists in the workflow!

    +
    +

    See more Here and Here.

    +

    In order to find the repository secrets configuration, go to the repository site on github / settings / Secrets and variables / Actions.

    +

    +
    +

    Question 5

    +
    +

    In our workflow, which information could be variables and which needs to be secret?

    +
    +
    + + +
    +
    +
    +

    Answer

    +

    Both AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY are very sensitive and definitely secrets.

    +

    The AWS_LAMBDA_ROLE_ARN has our account id. It's not the end of the world for this information to leak, but it's best not to share it and leave it as a secret.

    +

    The AWS_REGION could be a variable (but let's use secrets for ease).

    +

    Other information such as the function name could be variables instead of being hard coded.

    +
    +
    +
    +

    Question 6

    +
    +

    Go to the secrets configuration sections and set all secrets used in the workflow:

    +
      +
    • AWS_ACCESS_KEY_ID
      • +
    • AWS_SECRET_ACCESS_KEY
      • +
    • AWS_REGION
      • +
    • AWS_LAMBDA_ROLE_ARN
      • +
    +
    + + +
    +
    +
    +

    Commit and Deploy!

    +
    +

    Question 7

    +
    +

    Commit and push and check if the action executes successfully.

    +

    +
    + + +
    +
    +
    +
    +

    Question 8

    +
    +

    Test the function to check if it works!

    +

    You can use the code:

    +
    +Code to Test Lambda Function: Click to expand! +
    import io
+import boto3
+import json
+
+# Lambda function name: wc_INSPER_USERNAME
+function_name = ""
+
+# Create a Boto3 client for AWS Lambda
+lambda_client = boto3.client("lambda")
+
+msg = {"body": "hello from mars"}
+
+try:
+    print(f"Message:\n{msg}")
+
+    response = lambda_client.invoke(
+        FunctionName=function_name,
+        InvocationType="RequestResponse",
+        Payload=json.dumps(msg),
+    )
+
+    payload = response["Payload"]
+
+    txt = io.BytesIO(payload.read()).read().decode("utf-8")
+    print(f"\nResponse:\n{txt}")
+except Exception as e:
+    print(e)
+
    +
    +
    + + +
    +
    +
    +

    Task

    +
    +

    Question 9

    +
    +

    Make any changes to the function handler. For example, return one more field with the number of spaces.

    +

    Test whether automatic deployment occurs when pushing to main.

    +
    +

    Attention!

    +

    Remember that our script creates the function. You will have to remove the function for the deploy to work!

    +
    +
    + + +
    +
    +
    +
    +

    Question 10

    +
    +

    Change the deploy Python script to check if the function already exists.

    +
      +
    • +

      If so, just update the function code.

      +
      • +
    • +

      If not, then the function is created (as we already do).

      +
      • +
    +
    + + +
    +
    +
    +

    References

    + + + +
    + + +
    + +
    + + + + + \ No newline at end of file diff --git a/classes/12-ci-cd/ci-cd-flow-desktop.png b/classes/12-ci-cd/ci-cd-flow-desktop.png new file mode 100644 index 0000000..8ac5337 Binary files /dev/null and b/classes/12-ci-cd/ci-cd-flow-desktop.png differ diff --git a/classes/12-ci-cd/deploy.png b/classes/12-ci-cd/deploy.png new file mode 100644 index 0000000..7a251de Binary files /dev/null and b/classes/12-ci-cd/deploy.png differ diff --git a/classes/12-ci-cd/deploy_aws_python_success.png b/classes/12-ci-cd/deploy_aws_python_success.png new file mode 100644 index 0000000..735a7bd Binary files /dev/null and b/classes/12-ci-cd/deploy_aws_python_success.png differ diff --git a/classes/12-ci-cd/github_actions_click.png b/classes/12-ci-cd/github_actions_click.png new file mode 100644 index 0000000..9f44852 Binary files /dev/null and b/classes/12-ci-cd/github_actions_click.png differ diff --git a/classes/12-ci-cd/github_actions_details.png b/classes/12-ci-cd/github_actions_details.png new file mode 100644 index 0000000..14ca122 Binary files /dev/null and b/classes/12-ci-cd/github_actions_details.png differ diff --git a/classes/12-ci-cd/github_actions_fail.png b/classes/12-ci-cd/github_actions_fail.png new file mode 100644 index 0000000..2d2093d Binary files /dev/null and b/classes/12-ci-cd/github_actions_fail.png differ diff --git a/classes/12-ci-cd/github_actions_finished.png b/classes/12-ci-cd/github_actions_finished.png new file mode 100644 index 0000000..0b0c6af Binary files /dev/null and b/classes/12-ci-cd/github_actions_finished.png differ diff --git a/classes/12-ci-cd/github_actions_running.png b/classes/12-ci-cd/github_actions_running.png new file mode 100644 index 0000000..88d2754 Binary files /dev/null and b/classes/12-ci-cd/github_actions_running.png differ diff --git a/classes/12-ci-cd/intro/index.html b/classes/12-ci-cd/intro/index.html new file mode 100644 index 0000000..22e3196 --- /dev/null +++ b/classes/12-ci-cd/intro/index.html @@ -0,0 +1,803 @@ + + + + + + + + + Intro - MLOps & Interviews + + + + + + + + + + + + + + + + +
    +
    + + +
    + + + + + + + +
    +
    + +
    + +
    +
      + + + +
    • Classes
      • + + + +
    • 12 - CI/CD
      • + + +
    • +
    + + +
    + +
    +

    Continuous Integration and Continuous Delivery

    +

    Introduction

    +

    CI/CD is a short for Continuous Integration and Continuous Delivery (or Deployment) and represents a modern approach to agile software development. It encompasses a set of practices and tools aimed at releasing applications more frequently and efficiently, while maintaining high quality and minimizing risks.

    +

    +
    +

    CI/CD

    +

    By embracing CI/CD, organizations can streamline their development processes, enabling faster and more frequent releases while ensuring stringent quality control measures.

    +
    +

    Continuous Integration

    +

    In modern software development, it is common to have multiple developers or data scientists working simultaneously on different features within the same ML application.

    +
    +

    Question 1

    +
    +

    How do you think it would be if an organization has to consolidate all the branch's source code, with project modifications made by data scientists, in a single day?

    +
    +
    + + +
    +
    +
    +

    Answer

    +

    This, known as merge day or consolidation day, is a process that can be tedious, manual, and time-consuming, with a high chance of conflicts with changes made by colleagues at the same time.

    +
    +
    +

    With Continuous Integration (CI), the team regularly consolidates code changes back into a shared branch, often on a daily basis. The changes are consolidated and then validated through automated application builds. Multiple automated tests, typically including unit and integration tests, are performed to ensure that the changes do not introduce any issues or break the application.

    +

    Continuous Delivery

    +

    Continuous Delivery builds on Continuous Integration by automating the release process for builds that pass validation. With Continuous Delivery, code changes can be released to production with the click of a button after passing automated tests.

    +

    Some key aspects of Continuous Delivery:

    +
      +
    • Automated release process that can deploy code to production environments
      • +
    • Shortens the release cycle and provides faster feedback
      • +
    • Allows for more incremental updates rather than big bang releases
      • +
    • Releases still require manual approval before going live
      • +
    +

    Continuous Deployment

    +

    Continuous Deployment takes automation one step further than Continuous Delivery. With Continuous Deployment, validated code changes are automatically released to production without any manual intervention.

    +

    Key aspects of Continuous Deployment:

    +
      +
    • Fully automated process from commit to production with no manual steps
      • +
    • New changes are immediately tested and deployed if they pass
      • +
    • Enables much faster release cycles
      • +
    • Requires comprehensive test automation and continuous monitoring
      • +
    • Well suited for web services and cloud infrastructure
      • +
    • Not suitable for every application - depends on comfort level
      • +
    +
    +

    Question 2

    +
    +

    Could you give an example of an ML project where continuous deployment is not suitable?

    +
    +
    + + +
    +
    +
    +

    Answer

    +

    Some projects in medical diagnosis and financial transactions, where safety and sensitive data risks, besides regulations, are a concern.

    +

    Also, early research projects and unstable models with poor performance may need further intervention and experimentation before being ready for automation.

    +
    +
    +
    +

    Tip! 1

    +

    We need less manual steps and more automation!

    +
    +

    Some tools

    +

    Some tools that could be used for CI/CD of Machine Learning (ML) projects:

    +
      +
    • +

      Jenkins: Jenkins is a widely used open-source automation server that can be configured to support ML workflows.

      +
      • +
    • +

      GitLab CI/CD: GitLab provides built-in CI/CD capabilities that support ML workflows.

      +
      • +
    • +

      Github Actions: Github Actions provides a flexible and customizable platform to automate your ML pipelines directly from your GitHub repositories.

      +
      • +
    +

    We will work with Github Actions. Advance to the next section!

    + + +
    + + +
    + +
    + + + + + \ No newline at end of file diff --git a/classes/12-ci-cd/repo_secrets.png b/classes/12-ci-cd/repo_secrets.png new file mode 100644 index 0000000..1797b28 Binary files /dev/null and b/classes/12-ci-cd/repo_secrets.png differ diff --git a/classes/12-ci-cd/testing/index.html b/classes/12-ci-cd/testing/index.html new file mode 100644 index 0000000..6493c94 --- /dev/null +++ b/classes/12-ci-cd/testing/index.html @@ -0,0 +1,1142 @@ + + + + + + + + + Testing - MLOps & Interviews + + + + + + + + + + + + + + + + +
    +
    + + +
    + + + + + + + +
    +
    + +
    + +
    +
      + + + +
    • Classes
      • + + + +
    • 12 - CI/CD
      • + + +
    • +
    + + +
    + +
    +

    Testing

    +

    Let's build an action on github that runs automatic tests whenever a push is made to the main branch.

    +

    First, we will need to build the testing infrastructure in a repository.

    +

    Create Repository

    +
    +

    Question 1

    +
    +

    Create a private repository for the class and clone it on your machine.

    +
    +

    Info!

    +

    This activity is just for practice! It isn't part of mandatory APSs!

    +
    +
    + + +
    +
    +
    +
    +

    Question 2

    +
    +

    Create an .gitignore file. Ignore, at least:

    +
      +
    • .env
      • +
    • __pycache__
      • +
    +
    + + +
    +
    +
    +

    Create Folders

    +
    +

    Question 3

    +
    +

    Create the files and folder tree at the root of the repository.

    +
    ├── .gitignore
+├── README.md
+├── requirements.txt
+├── src/
+└── tests/
+
    +
    + + +
    +
    +
    +

    Let's create a lambda function similar to those created in previous classes. It will be a simple word processing function.

    +
    +

    Question 4

    +
    +

    Create the file src/word_count.py with the content:

    +
    def word_count_handler(event, context):
+    """
+    A simple function that counts
+    the number of words in a string
+    """
+    msg = event["body"]
+
+    n_words = len(msg.split())
+
+    return {
+        "len": len(msg),
+        "words": n_words,
+    }
+
    +

    Now the file tree will be:

    +
    ├── .gitignore
+├── README.md
+├── requirements.txt
+├── src
+│   └── word_count.py
+└── tests/
+
    +
    + + +
    +
    +
    +

    Add Testing

    +

    Let's add unit tests to our project. Initially we will run the tests locally. Then, we will automate this task to be executed in a github action.

    +
    +

    Question 5

    +
    +

    Add pytest to requirements.txt.

    +
    + + +
    +
    +
    +
    +

    Question 6

    +
    +

    Create the file pytest.ini (at root) with the content:

    +
    [pytest]
+pythonpath = src
+
    +

    This will enable module importing during test running.

    +
    + + +
    +
    +
    +
    +

    Question 7

    +
    +

    Create the file tests/test_word_count.py with the content:

    +
    import pytest
+import sys
+import word_count as wc
+
+
+def test_simple_text_count():
+    """Tests the word len and word count of a simple text"""
+    event = {"body": "Hello World"}
+    expected = {
+        "len": 11,
+        "words": 2,
+    }
+
+    # Test if return is as expected
+    assert wc.word_count_handler(event, None) == expected
+
    +

    Now the file tree will be:

    +
    ├── .gitignore
+├── pytest.ini
+├── README.md
+├── requirements.txt
+├── src
+│   └── word_count.py
+└── tests
+    └── test_word_count.py
+
    +
    + + +
    +
    +
    +
    +

    Question 8

    +
    +

    To check if the tests are passing, run:

    +

    +

    +
    $ pytest
+== test session starts ===
+platform linux -- Python 3.10.12, pytest-7.4.0, pluggy-1.2.0
+rootdir: /path/to/directory
+configfile: pytest.ini
+plugins: anyio-3.7.1
+collected 1 item                                                                                                     
+
+tests/test_word_count.py .                                                                                     [100%]
+
+=== 1 passed in 0.00s ====
+
    +

    +

    +
    + + +
    +
    +
    +
    +

    Question 9

    +
    +

    To finalize the edits, add basic project information to README.md!

    +
    + + +
    +
    +
    +

    Automating

    +

    In an environment where several Data Scientists work on the data team's various projects, it is desirable that there are automatic verification routines whenever modifications are sent to the repository.

    +

    Let's see how to automate the execution of tests using github actions.

    +
    +

    Tip! 1

    +

    Failing tests is a good indication that the code should not have been submitted and should not be deployed!

    +
    +

    Create Github Action

    +

    Let's create a directed acyclic graph (DAG) or pipeline with the sequence of steps to verify the tests in our repository.

    +

    +

    The actions to be performed are stored in the .github/workflows folder in the repository root and are represented in YAML format.

    +
    +

    Info!

    +

    YAML is a human-readable data serialization standard used for configuration files!

    +
    +
    +

    Question 10

    +
    +

    Create the file .github/workflows/test_workflow.yaml with the content:

    +
    name: An example of an automatic testing action
+on:
+  push:
+    branches:
+      - main
+jobs:
+  build-and-test:
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+
+      - name: Install dependencies
+        run: pip install -r requirements.txt
+
+      - name: Run tests
+        run: pytest
+
    +

    Now the file tree will be:

    +
    ├── .github
+│   └── workflows
+│       └── test_workflow.yaml
+├── .gitignore
+├── pytest.ini
+├── README.md
+├── requirements.txt
+├── src
+│   └── word_count.py
+└── tests
+    └── test_word_count.py
+
    +
    + + +
    +
    +
    +

    Explaining workflow YAML:

    +
      +
    • First, we provide a description for the workflow with: +
      name: An example of an automatic testing action
+
      • +
    • We set the action to be run whenever there is a push on the main branch when doing: +
      on:
+push:
+    branches:
+    - main
+
      • +
    • To define a job, which is a group of steps that are executed together as part of a workflow run, we do: +
      jobs:
+  build-and-test:
+    runs-on: ubuntu-latest
+    steps:
+
      +Note that we defined that the task must be executed in an ubuntu container.
      • +
    +
    +

    Github will deal with container creation and management.

    +

    You have a free quota, see more Here

    +
    +
      +
    • Then we define a sequence of steps for our job:
        +
      • To bring code from the repository into the container: +
        - name: Checkout code
+  uses: actions/checkout@v4
+
        • +
      • To set up Python: +
        - name: Set up Python
+  uses: actions/setup-python@v4
+  with:
+    python-version: '3.10'
+
        • +
      • To install dependencies +
        - name: Install dependencies
+  run: pip install -r requirements.txt
+
        • +
      • To run tests: +
        - name: Run tests
+  run: pytest
+
        • +
      +
      • +
    +

    Check if it worked

    +

    Let's check if the action is executed correctly! Before everything:

    +
    +

    Question 11

    +
    +

    Go to the repository site at https://github.com and access the Actions tab.

    +

    You don't need to do anything there for now, just leave it open to facilitate our check!

    +
    + + +
    +
    +
    +
    +

    Question 12

    +
    +

    Run git status and check if all folders (including workflow) are listed and ready to be sent to github!

    +
    + + +
    +
    +
    +
    +

    Question 13

    +
    +

    Then commit and push and go to the repository site at https://github.com.

    +

    +

    +
    $ git add .
+$ git commit -m "Try testing with gh actions"
+$ git push
+
    +

    +

    +

    Refresh the Actions tab. You should see the action being runned.

    +

    +

    After some seconds, refresh the page and see the action in finished status, with success!

    +

    +
    + + +
    +
    +
    +
    +

    Question 14

    +
    +

    Click on the workflow name (which has the commit message).

    +

    You will see all the jobs of this workflow. In this case, just one: build-and-test.

    +

    +

    Click on build-and-test to see more details on each step of this job:

    +

    +
    + + +
    +
    +
    +
    +

    Question 15

    +
    +

    Will the commits be ignored on action fail?

    +
    +
    + + +
    +
    +
    +

    Answer

    +

    No! Although it is possible to configure this functionality, we did not do this.

    +
    +
    +

    Tasks

    +
    +

    Question 16

    +
    +

    Add the following to the final of tests/test_word_count.py:

    +
    def test_no_body():
+    event = {}
+    expected = {"error": "no body"}
+    assert wc.word_count_handler(event, None) == expected
+
    +
    + + +
    +
    +
    +
    +

    Question 17

    +
    +

    Check locally if the tests pass.

    +

    Since the body key is not being sent but is being read in the function, the test should fail (KEYERROR).

    +

    +

    +
    $ pytest
+
    +

    +

    +
    + + +
    +
    +
    +
    +

    Question 18

    +
    +

    Commit the changes and check if the action also fails.

    +

    +
    + + +
    +
    +
    +
    +

    Question 19

    +
    +

    Make corrections to the source code until the tests passes locally.

    +
    +

    Commit and verify that the action was also executed successfully!

    +
    +
    + + +
    +
    +
    + + +
    + + +
    + +
    + + + + + \ No newline at end of file diff --git a/classes/12-ci-cd/workflow-graph.webp b/classes/12-ci-cd/workflow-graph.webp new file mode 100644 index 0000000..d1d8e73 Binary files /dev/null and b/classes/12-ci-cd/workflow-graph.webp differ diff --git a/contributions/index.html b/contributions/index.html index 455a0f5..a819bd7 100644 --- a/contributions/index.html +++ b/contributions/index.html @@ -608,6 +608,39 @@ + +
  • + 12 - CI/CD + +
    • + + diff --git a/deadlines/index.html b/deadlines/index.html index 1b9952e..e7791f4 100644 --- a/deadlines/index.html +++ b/deadlines/index.html @@ -608,6 +608,39 @@ + +
  • + 12 - CI/CD + +
    • + + diff --git a/index.html b/index.html index dec671c..14ed1c3 100644 --- a/index.html +++ b/index.html @@ -612,6 +612,39 @@ + +
  • + 12 - CI/CD + +
    • + + diff --git a/sitemap.xml.gz b/sitemap.xml.gz index 8d17df7..1e215a7 100644 Binary files a/sitemap.xml.gz and b/sitemap.xml.gz differ