From 00b2e39daaba02451812ca70379ee158eeae0cab Mon Sep 17 00:00:00 2001 From: Peter Muriuki Date: Thu, 4 Jul 2024 17:38:13 +0300 Subject: [PATCH 01/18] Update gitignore --- importer/.gitignore | 165 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 165 insertions(+) diff --git a/importer/.gitignore b/importer/.gitignore index 4acd06b1..21deed30 100644 --- a/importer/.gitignore +++ b/importer/.gitignore @@ -1 +1,166 @@ config.py + +sampleData +.idea +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/latest/usage/project/#working-with-version-control +.pdm.toml +.pdm-python +.pdm-build/ + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ From 533c944dbe03ec37160f1b0558caa41f6b4f6670 Mon Sep 17 00:00:00 2001 From: Peter Muriuki Date: Thu, 4 Jul 2024 17:38:53 +0300 Subject: [PATCH 02/18] move config.py to own config subpackage --- importer/README.md | 2 +- importer/{config.py => config/__init__.py} | 0 importer/config/sample_config.py | 17 ++++++++++ importer/config/settings.py | 36 ++++++++++++++++++++++ 4 files changed, 54 insertions(+), 1 deletion(-) rename importer/{config.py => config/__init__.py} (100%) create mode 100644 importer/config/sample_config.py create mode 100644 importer/config/settings.py diff --git a/importer/README.md b/importer/README.md index c1dc4b4c..d8bbdd58 100644 --- a/importer/README.md +++ b/importer/README.md @@ -17,7 +17,7 @@ This script is used to setup keycloak roles and groups. It takes in a csv file w ### To run script 1. Create virtualenv 2. Install requirements.txt - `pip install -r requirements.txt` -3. Create a `config.py` file. The `sample_config.py` is an example of what this should look like. Populate it with the right credentials, you can either provide an access token or client credentials. Ensure that the user whose details you provide in this config file has the necessary permissions/privilleges. +3. Create a `config/config.py` file. The `config/sample_config.py` is an example of what this should look like. Populate it with the right credentials, you can either provide an access token or client credentials. Ensure that the user whose details you provide in this config file has the necessary permissions/privilleges. 4. Run script - `python3 main.py --setup roles --csv_file csv/setup/roles.csv --group Supervisor` 5. If you are running the script without `https` setup e.g locally or a server without https setup, you will need to set the `OAUTHLIB_INSECURE_TRANSPORT` environment variable to 1. For example `export OAUTHLIB_INSECURE_TRANSPORT=1 && python3 main.py --setup roles --csv_file csv/setup/roles.csv --group OpenSRP_Provider --log_level debug` 6. You can turn on logging by passing a `--log_level` to the command line as `info`, `debug` or `error`. For example `python3 main.py --setup roles --csv_file csv/setup/roles.csv --group Supervisor --log_level debug` diff --git a/importer/config.py b/importer/config/__init__.py similarity index 100% rename from importer/config.py rename to importer/config/__init__.py diff --git a/importer/config/sample_config.py b/importer/config/sample_config.py new file mode 100644 index 00000000..3adbe8f2 --- /dev/null +++ b/importer/config/sample_config.py @@ -0,0 +1,17 @@ +client_id = 'example-client-id' +client_secret = 'example-client-secret' +fhir_base_url = 'https://example.smartregister.org/fhir' +keycloak_url = 'https://keycloak.smartregister.org/auth' + +# access token for access to where product images are remotely stored +product_access_token = 'example-product-access-token' + +# if using resource owner credentials (i.e importer handles getting authentication by itself) +# This has greater precedence over the access and refresh tokens if supplied +username = 'example-username' +password = 'example-password' + +# if embedding importer into a service that already does the authentication +access_token = 'example-access-token' +refresh_token = 'example-refresh-token' + diff --git a/importer/config/settings.py b/importer/config/settings.py new file mode 100644 index 00000000..c3474a64 --- /dev/null +++ b/importer/config/settings.py @@ -0,0 +1,36 @@ +from services.fhir_keycloak_api import FhirKeycloakApi, FhirKeycloakApiOptions, ExternalAuthenticationOptions, \ + InternalAuthenticationOptions +from config.config import client_id, client_secret, fhir_base_url, keycloak_url, realm +import importlib +import sys + +def dynamic_import(variable_name): + try: + config_module = importlib.import_module("temporal.config") + value = getattr(config_module, variable_name, None) + return value + except ModuleNotFoundError: + sys.exit(1) + + +username = dynamic_import("username") +password = dynamic_import("password") + +# TODO - retrieving at and rt as args via the command line as well. +access_token = dynamic_import("access_token") +refresh_token = dynamic_import("refresh_token") + +authentication_options = None +if username is not None and password is not None: + authentication_options = InternalAuthenticationOptions(client_id=client_id, client_secret=client_secret, keycloak_base_uri=keycloak_url, realm=realm, user_username=username, user_password=password) +elif access_token is not None and refresh_token is not None: + authentication_options = ExternalAuthenticationOptions(client_id=client_id, client_secret=client_secret, keycloak_base_uri=keycloak_url, realm=realm, access_token=access_token, refresh_token=refresh_token) +else: + sys.exit(1) + +api_service_options = FhirKeycloakApiOptions(fhir_base_uri=fhir_base_url, authentication_options=authentication_options) +api_service = FhirKeycloakApi(api_service_options) + + + + From 09d3f9f172dd0250bb0043edb7a00ff1e1b93e0c Mon Sep 17 00:00:00 2001 From: Peter Muriuki Date: Thu, 4 Jul 2024 17:39:22 +0300 Subject: [PATCH 03/18] Add jwt as a dependence --- importer/requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/importer/requirements.txt b/importer/requirements.txt index 4c748647..a72ac03b 100644 --- a/importer/requirements.txt +++ b/importer/requirements.txt @@ -7,4 +7,5 @@ backoff==2.2.1 pytest==7.4.2 jsonschema==4.21.1 mock==5.1.0 -python-magic==0.4.27 \ No newline at end of file +python-magic==0.4.27 +jwt From 52c818245273b8b66dd4d8a391f535655769da60 Mon Sep 17 00:00:00 2001 From: Peter Muriuki Date: Thu, 4 Jul 2024 17:39:47 +0300 Subject: [PATCH 04/18] Add services subpackage --- importer/services/__init__.py | 0 importer/services/fhir_keycloak_api.py | 186 +++++++++++++++++++++++++ 2 files changed, 186 insertions(+) create mode 100644 importer/services/__init__.py create mode 100644 importer/services/fhir_keycloak_api.py diff --git a/importer/services/__init__.py b/importer/services/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/importer/services/fhir_keycloak_api.py b/importer/services/fhir_keycloak_api.py new file mode 100644 index 00000000..9e51ed24 --- /dev/null +++ b/importer/services/fhir_keycloak_api.py @@ -0,0 +1,186 @@ +""" +Class that provides utility to: + 1. get access and refresh tokens from keycloak + 2. Update the tokens when expired. + 3. upload/update keycloak resources + 4. upload/update fhir resources +""" + +from dataclasses import dataclass, fields, field +from typing import Union + +import requests +# from keycloak import KeycloakOpenID, KeycloakOpenIDConnection +from requests_oauthlib import OAuth2Session +from oauthlib.oauth2 import BackendApplicationClient, LegacyApplicationClient +from urllib.parse import urljoin +import time +import jwt +import backoff +from functools import wraps + +def is_readable_string(s): + """ + Check if a variable is not an empty string. + + Args: + s (str): The variable to check. + + Returns: + bool: True if s is not an empty string, False otherwise. + """ + return isinstance(s, str) and s.strip() != "" + + +@dataclass +class IamUri: + """Keycloak authentication uris""" + keycloak_base_uri: str + realm: str + client_id: str + client_secret: str + token_uri: str = field(init=False) + + def __post_init__(self): + for field in fields(self): + if field.init and not is_readable_string(getattr(self, field.name)): + raise ValueError(f"{self.__class__.__name__} can only be initialized with str values") + self.token_uri = self.keycloak_base_uri + "/realms/" + self.realm + "/protocol/openid-connect/token" + self.keycloak_realm_uri = self.keycloak_base_uri + "/admin/realms/" + self.realm + + +@dataclass +class InternalAuthenticationOptions(IamUri): + """Describes config options for authentication that we have to handle ourselves""" + user_username: str + user_password: str + +@dataclass +class ExternalAuthenticationOptions(IamUri): + """Describes config options for authentication that we have to handle ourselves""" + access_token: str + refresh_token: str + + +@dataclass +class FhirKeycloakApiOptions: + authentication_options: Union[InternalAuthenticationOptions, ExternalAuthenticationOptions] + fhir_base_uri: str + +class FhirKeycloakApi(): + def __init__(self, options: FhirKeycloakApiOptions): + auth_options = options.authentication_options + if isinstance(auth_options, ExternalAuthenticationOptions): + self.authentication_Side = "external" + self.api_service = ExternalAuthenticationService(auth_options) + if isinstance(auth_options, InternalAuthenticationOptions): + self.authentication_Side = "internal" + self.api_service = InternalAuthenticationService(auth_options) + self.auth_options = auth_options + self.fhir_base_uri = options.fhir_base_uri + + @backoff.on_exception(backoff.expo, requests.exceptions.RequestException, max_time=180) + def request(self, **kwargs): + # TODO - spread headers into kwargs. + headers = { + "content-type": "application/json", + "accept": "application/json" + } + response = self.api_service.oauth.request(headers=headers, **kwargs) + if response.status_code == 401: + self.api_service.refresh_token() + return self.api_service.oauth.request(headers=headers, **kwargs) + return response + + +class InternalAuthenticationService: + + def __init__(self, option: InternalAuthenticationOptions): + self.options = option + client = LegacyApplicationClient(client_id=self.options.client_id) + oauth = OAuth2Session(client=client, auto_refresh_url=self.options.token_uri) + self.client = client + self.oauth = oauth + + def get_token(self): + """ + Oauth 2 does not work without an ssl layer to test this locally see https://stackoverflow.com/a/27785830/14564571 + :return: + """ + + token = self.oauth.fetch_token(token_url=self.options.token_uri, client_id=self.options.client_id, + client_secret=self.options.client_secret, username=self.options.user_username, password=self.options.user_password) + return token + + def refresh_token(self,): + return self.get_token() + + + def _is_refresh_required(self): + # TODO some defensive programming would be nice. + return time.time() > self.oauth.token.get("expires_at") + + def decode_token(self): + # full_jwt = jwt.JWT(jwt=token, **kwargs) + # full_jwt.token.objects["valid"] = True + # return json.loads(full_jwt.token.payload.decode("utf-8")) + pass + +class ExternalAuthenticationService: + + def __init__(self, option: ExternalAuthenticationOptions): + self.options = option + client = LegacyApplicationClient(client_id=self.options.client_id) + oauth = OAuth2Session(client=client, + token={"access_token": self.options.access_token, "refresh_token": self.options.refresh_token, 'token_type': "Bearer", "expires_in": 18000}, + auto_refresh_url=self.options.token_uri, + ) + self.client = client + self.oauth = oauth + + def get_token(self): + """ + Oauth 2 does not work without an ssl layer to test this locally see https://stackoverflow.com/a/27785830/14564571 + :return: + """ + # return the current token, not if its expired or invalid raise an irrecoverable show stopper error. + if self._is_refresh_required(): + # if expired + self.oauth.refresh_token(token_url=self.options.token_uri, client_id=self.options.client_id, client_secret=self.options.client_secret) + + token = self.oauth.fetch_token(token_url=self.options.token_uri, client_id=self.options.client_id, + client_secret=self.options.client_secret) + return token + else: + return self.oauth.token + + def refresh_token(self,): + return self.oauth.refresh_token(self.options.token_uri, client_id=self.options.client_id, client_secret=self.options.client_secret) + + + def _is_refresh_required(self): + # TODO some defensive programming would be nice. + at = self.oauth.token.get("access_token") + try: + decoded_at = self.decode_token(at) + return time.time() > decoded_at.get("exp") + except: + return False + + def decode_token(self, token: str): + # TODO - verify JWT + _algorithms = "HS256" + _do_verify=False + cert_uri = "http://localhost:8080/auth/realms/fhir/protocol/openid-connect/certs" + res = self.oauth.get(cert_uri).json().get("keys") + # tired + first_key = res[0] + jwk = jwt.jwk_from_dict(first_key) + _algorithms = first_key.get("alg") + instance = jwt.JWT() + return instance.decode(token, algorithms=_algorithms, key=jwk, do_verify=True, do_time_check=True) + + def handleRequest(self, **kwargs): + # self.oauth. + pass + From 83733ed8159d1ac5c507168a97301a0cb68d78e3 Mon Sep 17 00:00:00 2001 From: Peter Muriuki Date: Thu, 4 Jul 2024 17:40:49 +0300 Subject: [PATCH 05/18] Initial attempt at creating a script to generate mock data --- importer/stub_data/orgs-stup-gen.py | 35 +++++++++++++++++++++++ importer/stub_data/users-stub-gen.py | 42 ++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+) create mode 100644 importer/stub_data/orgs-stup-gen.py create mode 100644 importer/stub_data/users-stub-gen.py diff --git a/importer/stub_data/orgs-stup-gen.py b/importer/stub_data/orgs-stup-gen.py new file mode 100644 index 00000000..92c02bc0 --- /dev/null +++ b/importer/stub_data/orgs-stup-gen.py @@ -0,0 +1,35 @@ +import csv +import uuid +from faker import Faker + +# Initialize Faker +fake = Faker() + +# Template data (header and sample row) +header = [ + "orgName", "orgActive", "method", "orgId", "identifier" +] + +# Function to generate random row data +def generate_random_row(): + org_name = fake.name() + active = fake.random_element(["true", "false", ""]) + method = "create" + id = fake.uuid4() + identifier = fake.uuid4() + + return [org_name, active, method, id, identifier] + +# Generate 100 rows of data +rows = [] +for _ in range(100): + rows.append(generate_random_row()) + +# Write to CSV file +filename = f"localCsvs/orgs.csv" +with open(filename, mode='w', newline='') as file: + writer = csv.writer(file) + writer.writerow(header) + writer.writerows(rows) + +print(f"CSV file '{filename}' with 100 rows has been created.") diff --git a/importer/stub_data/users-stub-gen.py b/importer/stub_data/users-stub-gen.py new file mode 100644 index 00000000..3dd1a322 --- /dev/null +++ b/importer/stub_data/users-stub-gen.py @@ -0,0 +1,42 @@ +import csv +import uuid +from faker import Faker + +# Initialize Faker +fake = Faker() + +# Template data (header and sample row) +header = [ + "firstName", "lastName", "username", "email", "userId", + "userType", "enableUser", "keycloakGroupId", "keycloakGroupName", + "appId", "password" +] + +# Function to generate random row data +def generate_random_row(): + f_name = fake.first_name() + l_name = fake.last_name() + u_name = fake.user_name() + email = fake.email() + user_id = "" + user_type = fake.random_element(["Practitioner", "Supervisor", ""]) + enable_user = fake.random_element(["true", "false", ""]) + group_ids = "" + group_names = "" + app_id = "quest" + password = fake.password() + return [f_name, l_name, u_name, email, user_id, user_type, enable_user, group_ids, group_names, app_id, password] + +# Generate 100 rows of data +rows = [] +for _ in range(100): + rows.append(generate_random_row()) + +# Write to CSV file +filename = f"./users.csv" +with open(filename, mode='w', newline='') as file: + writer = csv.writer(file) + writer.writerow(header) + writer.writerows(rows) + +print(f"CSV file '{filename}' with 100 rows has been created.") From 7e0db15d2d108d3b6136ce385b88754cf32924ee Mon Sep 17 00:00:00 2001 From: Peter Muriuki Date: Thu, 4 Jul 2024 17:41:19 +0300 Subject: [PATCH 06/18] move sample config to config folder --- importer/sample_config.py | 9 --------- 1 file changed, 9 deletions(-) delete mode 100644 importer/sample_config.py diff --git a/importer/sample_config.py b/importer/sample_config.py deleted file mode 100644 index b8432523..00000000 --- a/importer/sample_config.py +++ /dev/null @@ -1,9 +0,0 @@ -client_id = 'example-client-id' -client_secret = 'example-client-secret' -username = 'example-username' -password = 'example-password' -access_token_url = 'https://keycloak.smartregister.org/auth/realms/example-realm/protocol/openid-connect/token' -fhir_base_url = 'https://example.smartregister.org/fhir' -keycloak_url = 'https://keycloak.smartregister.org/auth/admin/realms/example-realm' -access_token = 'example-access-token' -product_access_token = 'example-product-access-token' From 7be8e64b5d3d0edfbc675a7cc039bbca4fac4212 Mon Sep 17 00:00:00 2001 From: Peter Muriuki Date: Thu, 4 Jul 2024 17:41:51 +0300 Subject: [PATCH 07/18] Group tests under single folder --- importer/{ => tests}/test_main.py | 209 ++++-------------------------- 1 file changed, 22 insertions(+), 187 deletions(-) rename importer/{ => tests}/test_main.py (86%) diff --git a/importer/test_main.py b/importer/tests/test_main.py similarity index 86% rename from importer/test_main.py rename to importer/tests/test_main.py index b82dac74..9f7d8572 100644 --- a/importer/test_main.py +++ b/importer/tests/test_main.py @@ -22,7 +22,7 @@ class TestMain(unittest.TestCase): def test_read_csv(self): - csv_file = "csv/users.csv" + csv_file = "../csv/users.csv" records = read_csv(csv_file) self.assertIsInstance(records, list) self.assertEqual(len(records), 3) @@ -56,10 +56,10 @@ def test_write_csv(self): def test_build_payload_organizations(self, mock_get_resource): mock_get_resource.return_value = "1" - csv_file = "csv/organizations/organizations_full.csv" + csv_file = "../csv/organizations/organizations_full.csv" resource_list = read_csv(csv_file) payload = build_payload( - "organizations", resource_list, "json_payloads/organizations_payload.json" + "organizations", resource_list, "../json_payloads/organizations_payload.json" ) payload_obj = json.loads(payload) self.assertIsInstance(payload_obj, dict) @@ -90,10 +90,10 @@ def test_build_payload_organizations(self, mock_get_resource): validate(payload_obj["entry"][2]["request"], request_schema) # TestCase organizations_min.csv - csv_file = "csv/organizations/organizations_min.csv" + csv_file = "../csv/organizations/organizations_min.csv" resource_list = read_csv(csv_file) payload = build_payload( - "organizations", resource_list, "json_payloads/organizations_payload.json" + "organizations", resource_list, "../json_payloads/organizations_payload.json" ) payload_obj = json.loads(payload) self.assertIsInstance(payload_obj, dict) @@ -131,10 +131,10 @@ def test_build_payload_locations( mock_get_resource.return_value = "1" mock_check_parent_admin_level.return_value = "3" - csv_file = "csv/locations/locations_full.csv" + csv_file = "../csv/locations/locations_full.csv" resource_list = read_csv(csv_file) payload = build_payload( - "locations", resource_list, "json_payloads/locations_payload.json" + "locations", resource_list, "../json_payloads/locations_payload.json" ) payload_obj = json.loads(payload) self.assertIsInstance(payload_obj, dict) @@ -220,10 +220,10 @@ def test_build_payload_locations( validate(payload_obj["entry"][0]["request"], request_schema) # TestCase locations_min.csv - csv_file = "csv/locations/locations_min.csv" + csv_file = "../csv/locations/locations_min.csv" resource_list = read_csv(csv_file) payload = build_payload( - "locations", resource_list, "json_payloads/locations_payload.json" + "locations", resource_list, "../json_payloads/locations_payload.json" ) payload_obj = json.loads(payload) self.assertIsInstance(payload_obj, dict) @@ -287,10 +287,10 @@ def test_check_parent_admin_level(self, mock_get_base_url, mock_handle_request): def test_build_payload_care_teams(self, mock_get_resource): mock_get_resource.return_value = "1" - csv_file = "csv/careteams/careteam_full.csv" + csv_file = "../csv/careteams/careteam_full.csv" resource_list = read_csv(csv_file) payload = build_payload( - "careTeams", resource_list, "json_payloads/careteams_payload.json" + "careTeams", resource_list, "../json_payloads/careteams_payload.json" ) payload_obj = json.loads(payload) self.assertIsInstance(payload_obj, dict) @@ -386,10 +386,10 @@ def test_build_payload_group(self, mock_get_resource, mock_save_image): mock_get_resource.return_value = "1" mock_save_image.return_value = "f374a23a-3c6a-4167-9970-b10c16a91bbd" - csv_file = "csv/import/product.csv" + csv_file = "../csv/import/product.csv" resource_list = read_csv(csv_file) payload = build_payload( - "Group", resource_list, "json_payloads/product_group_payload.json" + "Group", resource_list, "../json_payloads/product_group_payload.json" ) payload_obj = json.loads(payload) @@ -436,7 +436,7 @@ def test_build_payload_group(self, mock_get_resource, mock_save_image): validate(payload_obj["entry"][0]["request"], request_schema) def test_extract_matches(self): - csv_file = "csv/organizations/organizations_locations.csv" + csv_file = "../csv/organizations/organizations_locations.csv" resource_list = read_csv(csv_file) resources = extract_matches(resource_list) expected_matches = { @@ -450,7 +450,7 @@ def test_extract_matches(self): self.assertEqual(resources, expected_matches) def test_build_org_affiliation(self): - csv_file = "csv/organizations/organizations_locations.csv" + csv_file = "../csv/organizations/organizations_locations.csv" resource_list = read_csv(csv_file) resources = extract_matches(resource_list) payload = build_org_affiliation(resources, resource_list) @@ -577,7 +577,7 @@ def test_uuid_generated_for_locations_is_unique_and_repeatable(self): ] payload = build_payload( - "locations", resources, "json_payloads/locations_payload.json" + "locations", resources, "../json_payloads/locations_payload.json" ) payload_obj = json.loads(payload) location1 = payload_obj["entry"][0]["resource"]["id"] @@ -628,7 +628,7 @@ def test_update_resource_with_no_id_fails(self): ] with self.assertRaises(ValueError) as raised_error: build_payload( - "locations", resources, "json_payloads/locations_payload.json" + "locations", resources, "../json_payloads/locations_payload.json" ) self.assertEqual( "The id is required to update a resource", str(raised_error.exception) @@ -654,7 +654,7 @@ def test_update_resource_with_non_existing_id_fails(self, mock_get_resource): ] with self.assertRaises(ValueError) as raised_error: build_payload( - "locations", resources, "json_payloads/locations_payload.json" + "locations", resources, "../json_payloads/locations_payload.json" ) self.assertEqual( "Trying to update a Non-existent resource", str(raised_error.exception) @@ -766,9 +766,7 @@ def test_build_assign_payload_update_assigned_org( "98199caa-4455-4b2f-a5cf-cb9c89b6bbdc", ] ] - payload = build_assign_payload( - resource_list, "PractitionerRole", "practitioner=Practitioner/" - ) + payload = build_assign_payload(resource_list, "PractitionerRole") payload_obj = json.loads(payload) self.assertIsInstance(payload_obj, dict) @@ -826,9 +824,7 @@ def test_build_assign_payload_create_org_assignment( "98199caa-4455-4b2f-a5cf-cb9c89b6bbdc", ] ] - payload = build_assign_payload( - resource_list, "PractitionerRole", "practitioner=Practitioner/" - ) + payload = build_assign_payload(resource_list, "PractitionerRole") payload_obj = json.loads(payload) self.assertIsInstance(payload_obj, dict) @@ -866,9 +862,7 @@ def test_build_assign_payload_create_new_practitioner_role( "98199caa-4455-4b2f-a5cf-cb9c89b6bbdc", ] ] - payload = build_assign_payload( - resource_list, "PractitionerRole", "practitioner=Practitioner/" - ) + payload = build_assign_payload(resource_list, "PractitionerRole") payload_obj = json.loads(payload) self.assertIsInstance(payload_obj, dict) @@ -890,165 +884,6 @@ def test_build_assign_payload_create_new_practitioner_role( payload_obj["entry"][0]["resource"]["organization"]["display"], "New Org" ) - @patch("main.handle_request") - @patch("main.get_base_url") - def test_build_assign_payload_create_new_link_location_to_inventory_list( - self, mock_get_base_url, mock_handle_request - ): - mock_get_base_url.return_value = "https://example.smartregister.org/fhir" - mock_response_data = {"resourceType": "Bundle", "total": 0} - string_response = json.dumps(mock_response_data) - mock_response = (string_response, 200) - mock_handle_request.return_value = mock_response - - resource_list = [ - [ - "Nairobi Inventory Items", - "e62a049f-8d48-456c-a387-f52e72c39c74", - "2024-06-01T10:40:10.111Z", - "3af23539-850a-44ed-8fb1-d4999e2145ff", - ] - ] - payload = build_assign_payload(resource_list, "List", "subject=List/") - payload_obj = json.loads(payload) - - self.assertIsInstance(payload_obj, dict) - self.assertEqual(payload_obj["resourceType"], "Bundle") - self.assertEqual(len(payload_obj["entry"]), 1) - - self.assertEqual( - payload_obj["entry"][0]["resource"]["title"], "Nairobi Inventory Items" - ) - self.assertEqual( - payload_obj["entry"][0]["resource"]["entry"][0]["item"]["reference"], - "Group/e62a049f-8d48-456c-a387-f52e72c39c74", - ) - self.assertEqual( - payload_obj["entry"][0]["resource"]["entry"][0]["date"], - "2024-06-01T10:40:10.111Z", - ) - self.assertEqual( - payload_obj["entry"][0]["resource"]["subject"]["reference"], - "Location/3af23539-850a-44ed-8fb1-d4999e2145ff", - ) - - @patch("main.handle_request") - @patch("main.get_base_url") - def test_build_assign_payload_update_location_with_new_inventory( - self, mock_get_base_url, mock_handle_request - ): - mock_get_base_url.return_value = "https://example.smartregister.org/fhir" - mock_response_data = { - "resourceType": "Bundle", - "total": 1, - "entry": [ - { - "resource": { - "resourceType": "List", - "id": "6d7d2e70-1c90-11db-861d-0242ac120002", - "meta": {"versionId": "2"}, - "subject": { - "reference": "Location/46bb8a3f-cf50-4cc2-b421-fe4f77c3e75d" - }, - "entry": [ - { - "item": { - "reference": "Group/f2734756-a6bb-4e90-bbc6-1c34f51d3d5c" - } - } - ], - } - } - ], - } - string_response = json.dumps(mock_response_data) - mock_response = (string_response, 200) - mock_handle_request.return_value = mock_response - - resource_list = [ - [ - "Nairobi Inventory Items", - "e62a049f-8d48-456c-a387-f52e72c39c74", - "2024-06-01T10:40:10.111Z", - "3af23539-850a-44ed-8fb1-d4999e2145ff", - ] - ] - - payload = build_assign_payload(resource_list, "List", "subject=List/") - payload_obj = json.loads(payload) - - self.assertIsInstance(payload_obj, dict) - self.assertEqual(payload_obj["resourceType"], "Bundle") - self.assertEqual(len(payload_obj["entry"]), 1) - - self.assertEqual( - payload_obj["entry"][0]["resource"]["entry"][0]["item"]["reference"], - "Group/f2734756-a6bb-4e90-bbc6-1c34f51d3d5c", - ) - self.assertEqual( - payload_obj["entry"][0]["resource"]["entry"][1]["item"]["reference"], - "Group/e62a049f-8d48-456c-a387-f52e72c39c74", - ) - - @patch("main.handle_request") - @patch("main.get_base_url") - def test_build_assign_payload_create_new_link_location_to_inventory_list_with_multiples( - self, mock_get_base_url, mock_handle_request - ): - mock_get_base_url.return_value = "https://example.smartregister.org/fhir" - mock_response_data = {"resourceType": "Bundle", "total": 0} - string_response = json.dumps(mock_response_data) - mock_response = (string_response, 200) - mock_handle_request.return_value = mock_response - - resource_list = [ - [ - "Nairobi Inventory Items", - "e62a049f-8d48-456c-a387-f52e72c39c74", - "2024-06-01T10:40:10.111Z", - "3af23539-850a-44ed-8fb1-d4999e2145ff", - ], - [ - "Nairobi Inventory Items", - "a36b595c-68a7-4244-91d5-c64be23b1ebd", - "2024-06-05T30:30:30.264Z", - "3af23539-850a-44ed-8fb1-d4999e2145ff", - ], - [ - "Mombasa Inventory Items", - "c0666a5a-00f6-488c-9001-8630560b5810", - "2024-06-06T55:23:19.492Z", - "3cd687a4-a169-45b3-a939-0418470c29db", - ], - ] - payload = build_assign_payload(resource_list, "List", "subject=List/") - payload_obj = json.loads(payload) - - self.assertIsInstance(payload_obj, dict) - self.assertEqual(payload_obj["resourceType"], "Bundle") - self.assertEqual(len(payload_obj["entry"]), 2) - self.assertEqual(len(payload_obj["entry"][0]["resource"]["entry"]), 2) - self.assertEqual(len(payload_obj["entry"][1]["resource"]["entry"]), 1) - - self.assertEqual( - payload_obj["entry"][0]["resource"]["title"], "Nairobi Inventory Items" - ) - self.assertEqual( - payload_obj["entry"][1]["resource"]["title"], "Mombasa Inventory Items" - ) - self.assertEqual( - payload_obj["entry"][0]["resource"]["entry"][0]["item"]["reference"], - "Group/e62a049f-8d48-456c-a387-f52e72c39c74", - ) - self.assertEqual( - payload_obj["entry"][0]["resource"]["entry"][1]["item"]["reference"], - "Group/a36b595c-68a7-4244-91d5-c64be23b1ebd", - ) - self.assertEqual( - payload_obj["entry"][1]["resource"]["entry"][0]["item"]["reference"], - "Group/c0666a5a-00f6-488c-9001-8630560b5810", - ) - @patch("main.logging") @patch("main.handle_request") @patch("main.get_keycloak_url") @@ -1475,7 +1310,7 @@ def test_split_chunk_sort_sync_first_chunk_less_than_size( mock_build_resource_type_map.assert_called_once_with(chunk_list, {}, 0) def test_build_resource_type_map(self): - json_file = "tests/json/sample.json" + json_file = "json/sample.json" mapping = read_file_in_chunks(json_file, 300, "sort") mapped_resources = { "Patient": [0], From 3881353bdf50b9f9a55a66d24acfcf857843e683 Mon Sep 17 00:00:00 2001 From: Peter Muriuki Date: Thu, 4 Jul 2024 17:42:10 +0300 Subject: [PATCH 08/18] Add utils function that groups locations by admin level --- importer/utils/__init__.py | 0 importer/utils/location_process.py | 91 ++++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+) create mode 100644 importer/utils/__init__.py create mode 100644 importer/utils/location_process.py diff --git a/importer/utils/__init__.py b/importer/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/importer/utils/location_process.py b/importer/utils/location_process.py new file mode 100644 index 00000000..9d1e589f --- /dev/null +++ b/importer/utils/location_process.py @@ -0,0 +1,91 @@ +""" +preprocess a csv of location records, group them by their admin levels +""" + + +class LocationNode: + def __init__(self, raw_record=None, parent_node=None, children=None): + if children is None: + children = [] + self.parent = parent_node + self.children = children + self.raw_record = raw_record + + @property + def admin_level(self): + try: + return self.raw_record[8] + except: + return None + + @property + def location_id(self): + try: + return self.raw_record[3] + except: + return None + + def __repr__(self): + return f"" + + +def group_by_admin_level(csv_records): + location_node_store = {} + for record in csv_records: + location_id, parent_id = record[3], record[5] + this_record_node = location_node_store.get(location_id) + if this_record_node: + # assume tombstone + this_record_node.raw_record = record + else: + this_record_node = LocationNode(record) + location_node_store[location_id] = this_record_node + # see if this parentNode exists in the nodeStore + if parent_id: + this_location_parent_node = location_node_store.get(parent_id) + if this_location_parent_node is not None: + pass + else: + # create a tombstone + this_location_parent_node = LocationNode() + location_node_store[parent_id] = this_location_parent_node + this_location_parent_node.children.append(this_record_node) + this_record_node.parent = this_location_parent_node + return location_node_store + + +def get_node_children(parents): + children = [] + for node in parents: + children.extend(node.children) + return children + + +def get_next_admin_level(node_map_store: dict): + """generator function that yields the next group of admin level locations""" + # start by getting the parent locations. i.e. locations that do not have parent + parent_nodes = [] + for node in node_map_store.values(): + if node.parent is None and node.raw_record or node.parent and node.parent.raw_record is None: + parent_nodes.append(node) + yield parent_nodes + + fully_traversed = False + while not fully_traversed: + children_at_this_level = get_node_children(parent_nodes) + if len(children_at_this_level) == 0: + fully_traversed = True + else: + parent_nodes = children_at_this_level + yield children_at_this_level + + +def process_locations(csv_records): + nodes_map = group_by_admin_level(csv_records) + for batch in get_next_admin_level(nodes_map): + batch_raw_records = [] + for node in batch: + batch_raw_records.append(node.raw_record) + yield batch_raw_records + +# TODO - validate based on adminlevel and the generated groupings based on the parentIds \ No newline at end of file From 9524c0d6e7fb1fd9ccbfb39e6986869152cbac1e Mon Sep 17 00:00:00 2001 From: Peter Muriuki Date: Thu, 4 Jul 2024 17:45:36 +0300 Subject: [PATCH 09/18] de-dedupe physical type in location type payload --- importer/json_payloads/locations_payload.json | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/importer/json_payloads/locations_payload.json b/importer/json_payloads/locations_payload.json index d06dbe46..12bd5a97 100644 --- a/importer/json_payloads/locations_payload.json +++ b/importer/json_payloads/locations_payload.json @@ -37,6 +37,11 @@ "display": "Level $adminLevelCode" } ] + }, + { + "system": "http://terminology.hl7.org/CodeSystem/location-physical-type", + "code": "$pt_code", + "display": "$pt_display" } ], "physicalType": { From ce9fb07234ed728c24e810a5ab824c6e9d952b41 Mon Sep 17 00:00:00 2001 From: Peter Muriuki Date: Thu, 4 Jul 2024 18:23:13 +0300 Subject: [PATCH 10/18] First iteration update of main py --- importer/main.py | 214 ++++++++++++++++++++--------------------------- 1 file changed, 91 insertions(+), 123 deletions(-) diff --git a/importer/main.py b/importer/main.py index 0923336c..9e17b226 100644 --- a/importer/main.py +++ b/importer/main.py @@ -6,18 +6,18 @@ import requests import logging import logging.config -import backoff import base64 import magic from datetime import datetime -from oauthlib.oauth2 import LegacyApplicationClient -from requests_oauthlib import OAuth2Session -try: - import config -except ModuleNotFoundError: - logging.error("The config.py file is missing!") - exit() +from config.settings import api_service, keycloak_url, fhir_base_url, product_access_token +from utils.location_process import process_locations + +# try: +# import config +# except ModuleNotFoundError: +# logging.error("The config.py file is missing!") +# exit() global_access_token = "" @@ -46,62 +46,47 @@ def read_csv(csv_file): logging.error("Stop iteration on empty file") -def get_access_token(): - access_token = "" - if global_access_token: - return global_access_token - - try: - if config.access_token: - # get access token from config file - access_token = config.access_token - except AttributeError: - logging.debug("No access token provided, trying to use client credentials") - - if not access_token: - # get client credentials from config file - client_id = config.client_id - client_secret = config.client_secret - username = config.username - password = config.password - access_token_url = config.access_token_url - - oauth = OAuth2Session(client=LegacyApplicationClient(client_id=client_id)) - token = oauth.fetch_token( - token_url=access_token_url, - username=username, - password=password, - client_id=client_id, - client_secret=client_secret, - ) - access_token = token["access_token"] +# def get_access_token(): +# access_token = "" +# if global_access_token: +# return global_access_token +# +# try: +# if config.access_token: +# # get access token from config file +# access_token = config.access_token +# except AttributeError: +# logging.debug("No access token provided, trying to use client credentials") +# +# if not access_token: +# # get client credentials from config file +# client_id = config.client_id +# client_secret = config.client_secret +# username = config.username +# password = config.password +# access_token_url = config.access_token_url +# +# oauth = OAuth2Session(client=LegacyApplicationClient(client_id=client_id)) +# token = oauth.fetch_token( +# token_url=access_token_url, +# username=username, +# password=password, +# client_id=client_id, +# client_secret=client_secret, +# ) +# access_token = token["access_token"] +# +# return access_token - return access_token -# This function makes the request to the provided url -# to create resources -@backoff.on_exception(backoff.expo, requests.exceptions.RequestException, max_time=180) def post_request(request_type, payload, url, json_payload): logging.info("Posting request") logging.info("Request type: " + request_type) logging.info("Url: " + url) logging.debug("Payload: " + payload) - - access_token = "Bearer " + get_access_token() - headers = {"Content-type": "application/json", "Authorization": access_token} - - if request_type == "POST": - return requests.post(url, data=payload, json=json_payload, headers=headers) - elif request_type == "PUT": - return requests.put(url, data=payload, json=json_payload, headers=headers) - elif request_type == "GET": - return requests.get(url, headers=headers) - elif request_type == "DELETE": - return requests.delete(url, headers=headers) - else: - logging.error("Unsupported request type!") - + return api_service.request(method=request_type, url=url, + data=payload, json=json_payload) def handle_request(request_type, payload, url, json_payload=None): try: @@ -118,7 +103,7 @@ def handle_request(request_type, payload, url, json_payload=None): def get_keycloak_url(): - return config.keycloak_url + return api_service.auth_options.keycloak_realm_uri # This function builds the user payload and posts it to @@ -140,6 +125,7 @@ def create_user(user): password, ) = user + # TODO - move this out so that its not recreated for every user. with open("json_payloads/keycloak_user_payload.json") as json_file: payload_string = json_file.read() @@ -983,7 +969,7 @@ def get_valid_resource_type(resource_type): def get_resource(resource_id, resource_type): if resource_type != "Group": resource_type = get_valid_resource_type(resource_type) - resource_url = "/".join([config.fhir_base_url, resource_type, resource_id]) + resource_url = "/".join([fhir_base_url, resource_type, resource_id]) response = handle_request("GET", "", resource_url) return json.loads(response[0])["meta"]["versionId"] if response[1] == 200 else "0" @@ -1074,48 +1060,25 @@ def build_payload(resource_type, resources, resource_payload_file): final_string = final_string + ps + "," - final_string = initial_string + final_string[:-1] + " ] } " + final_string = json.dumps(json.loads(initial_string + final_string[:-1] + " ] } ")) return final_string -def link_to_location(resource_list): - arr = [] - with click.progressbar( - resource_list, label="Progress::Linking inventory to location" - ) as link_locations_progress: - for resource in link_locations_progress: - try: - if resource[14]: - # name, inventory_id, supply_date, location_id - resource_link = [ - resource[0], - resource[3], - resource[9], - resource[14], - ] - arr.append(resource_link) - except IndexError: - logging.info("No location provided for " + resource[0]) - - if len(arr) > 0: - return build_assign_payload(arr, "List", "subject=Location/") - else: - return "" - - def confirm_keycloak_user(user): # Confirm that the keycloak user details are as expected user_username = str(user[2]).strip() user_email = str(user[3]).strip() keycloak_url = get_keycloak_url() response = handle_request( - "GET", "", keycloak_url + "/users?exact=true&username=" + user_username + "GET", "", api_service.auth_options.keycloak_realm_uri + "/users?exact=true&username=" + user_username ) logging.debug(response) json_response = json.loads(response[0]) try: - response_email = json_response[0]["email"] + # TODO - apparently not all user uploads will have an email + print("============>", json_response) + response_email = json_response[0].get("email", "") except IndexError: response_email = "" @@ -1181,7 +1144,7 @@ def confirm_practitioner(user, user_id): return True except Exception as err: - logging.error("Error occured trying to find Practitioner: " + str(err)) + logging.error("Error occurred trying to find Practitioner: " + str(err)) return True @@ -1192,7 +1155,7 @@ def create_roles(role_list, roles_max): # check if role already exists role_response = handle_request( - "GET", "", config.keycloak_url + "/roles/" + current_role + "GET", "", keycloak_url + "/roles/" + current_role ) logging.debug(role_response) if current_role in role_response[0]: @@ -1200,7 +1163,7 @@ def create_roles(role_list, roles_max): else: role_payload = '{"name": "' + current_role + '"}' create_role = handle_request( - "POST", role_payload, config.keycloak_url + "/roles" + "POST", role_payload, keycloak_url + "/roles" ) if create_role.status_code == 201: logging.info("Successfully created role: " + current_role) @@ -1211,7 +1174,7 @@ def create_roles(role_list, roles_max): logging.debug("Role has composite roles") # get roled id full_role = handle_request( - "GET", "", config.keycloak_url + "/roles/" + current_role + "GET", "", keycloak_url + "/roles/" + current_role ) json_resp = json.loads(full_role[0]) role_id = json_resp["id"] @@ -1221,7 +1184,7 @@ def create_roles(role_list, roles_max): available_roles = handle_request( "GET", "", - config.keycloak_url + keycloak_url + "/admin-ui-available-roles/roles/" + role_id + "?first=0&max=" @@ -1256,7 +1219,7 @@ def create_roles(role_list, roles_max): handle_request( "POST", payload_arr, - config.keycloak_url + "/roles-by-id/" + role_id + "/composites", + keycloak_url + "/roles-by-id/" + role_id + "/composites", ) except IndexError: @@ -1265,7 +1228,7 @@ def create_roles(role_list, roles_max): def get_group_id(group): # check if group exists - all_groups = handle_request("GET", "", config.keycloak_url + "/groups") + all_groups = handle_request("GET", "", keycloak_url + "/groups") json_groups = json.loads(all_groups[0]) group_obj = {} @@ -1281,7 +1244,7 @@ def get_group_id(group): logging.info("Group does not exists, lets create it") # create the group create_group_payload = '{"name":"' + group + '"}' - handle_request("POST", create_group_payload, config.keycloak_url + "/groups") + handle_request("POST", create_group_payload, keycloak_url + "/groups") return get_group_id(group) @@ -1293,7 +1256,7 @@ def assign_group_roles(role_list, group, roles_max): available_roles_for_group = handle_request( "GET", "", - config.keycloak_url + keycloak_url + "/groups/" + group_id + "/role-mappings/realm/available?first=0&max=" @@ -1314,7 +1277,7 @@ def assign_group_roles(role_list, group, roles_max): handle_request( "POST", json_assign_payload, - config.keycloak_url + "/groups/" + group_id + "/role-mappings/realm", + keycloak_url + "/groups/" + group_id + "/role-mappings/realm", ) @@ -1325,7 +1288,7 @@ def delete_resource(resource_type, resource_id, cascade): cascade = "" resource_url = "/".join( - [config.fhir_base_url, resource_type, resource_id + cascade] + [fhir_base_url, resource_type, resource_id + cascade] ) r = handle_request("DELETE", "", resource_url) logging.info(r.text) @@ -1336,7 +1299,7 @@ def clean_duplicates(users, cascade_delete): # get keycloak user uuid username = str(user[2].strip()) user_details = handle_request( - "GET", "", config.keycloak_url + "/users?exact=true&username=" + username + "GET", "", keycloak_url + "/users?exact=true&username=" + username ) obj = json.loads(user_details[0]) keycloak_uuid = obj[0]["id"] @@ -1345,7 +1308,7 @@ def clean_duplicates(users, cascade_delete): r = handle_request( "GET", "", - config.fhir_base_url + "/Practitioner?identifier=" + keycloak_uuid, + fhir_base_url + "/Practitioner?identifier=" + keycloak_uuid, ) practitioner_details = json.loads(r[0]) count = practitioner_details["total"] @@ -1406,7 +1369,7 @@ def write_csv(data, resource_type, fieldnames): def get_base_url(): - return config.fhir_base_url + return api_service.fhir_base_uri # This function exports resources from the API to a csv file @@ -1547,8 +1510,9 @@ def encode_image(image_file): # and saves it as a Binary resource. It returns the id of the Binary resource if # successful and 0 if failed def save_image(image_source_url): + try: - headers = {"Authorization": "Bearer " + config.product_access_token} + headers = {"Authorization": "Bearer " + product_access_token} except AttributeError: headers = {} @@ -1627,7 +1591,7 @@ def process_chunk(resources_array: list, resource_type: str): json_payload = {"resourceType": "Bundle", "type": "transaction", "entry": new_arr} - r = handle_request("POST", "", config.fhir_base_url, json_payload) + r = handle_request("POST", "", fhir_base_url, json_payload) logging.info(r.text) # TODO handle failures @@ -1820,6 +1784,7 @@ def main( ) logging.getLogger().addHandler(logging.StreamHandler()) + # TODO - should be an empty flag that does not need a value. if only_response: logging.config.dictConfig(LOGGING) @@ -1870,15 +1835,20 @@ def main( if not practitioner_exists: payload = create_user_resources(user_id, user) final_response = handle_request( - "POST", payload, config.fhir_base_url + "POST", payload, fhir_base_url ) logging.info("Processing complete!") elif resource_type == "locations": logging.info("Processing locations") - json_payload = build_payload( - "locations", resource_list, "json_payloads/locations_payload.json" - ) - final_response = handle_request("POST", json_payload, config.fhir_base_url) + batch_generator = process_locations(resource_list) + final_response = [] + for batch in batch_generator: + json_payload = build_payload( + "locations", batch, "json_payloads/locations_payload.json" + ) + response = handle_request("POST", json_payload, fhir_base_url) + final_response.append(response.text) + final_response = ",\n".join(final_response) logging.info("Processing complete!") elif resource_type == "organizations": logging.info("Processing organizations") @@ -1887,27 +1857,25 @@ def main( resource_list, "json_payloads/organizations_payload.json", ) - final_response = handle_request("POST", json_payload, config.fhir_base_url) + final_response = handle_request("POST", json_payload, fhir_base_url) logging.info("Processing complete!") elif resource_type == "careTeams": logging.info("Processing CareTeams") json_payload = build_payload( "careTeams", resource_list, "json_payloads/careteams_payload.json" ) - final_response = handle_request("POST", json_payload, config.fhir_base_url) + final_response = handle_request("POST", json_payload, fhir_base_url) logging.info("Processing complete!") elif assign == "organizations-Locations": logging.info("Assigning Organizations to Locations") matches = extract_matches(resource_list) json_payload = build_org_affiliation(matches, resource_list) - final_response = handle_request("POST", json_payload, config.fhir_base_url) + final_response = handle_request("POST", json_payload, fhir_base_url) logging.info("Processing complete!") elif assign == "users-organizations": logging.info("Assigning practitioner to Organization") - json_payload = build_assign_payload( - resource_list, "PractitionerRole", "practitioner=Practitioner/" - ) - final_response = handle_request("POST", json_payload, config.fhir_base_url) + json_payload = build_assign_payload(resource_list, "PractitionerRole") + final_response = handle_request("POST", json_payload, fhir_base_url) logging.info("Processing complete!") elif setup == "roles": logging.info("Setting up keycloak roles") @@ -1927,25 +1895,25 @@ def main( json_payload = build_payload( "Group", resource_list, "json_payloads/product_group_payload.json" ) - final_response = handle_request("POST", json_payload, config.fhir_base_url) + final_response = handle_request("POST", json_payload, fhir_base_url) elif setup == "inventories": logging.info("Importing inventories as FHIR Group resources") json_payload = build_payload( "Group", resource_list, "json_payloads/inventory_group_payload.json" ) - final_response = handle_request("POST", json_payload, config.fhir_base_url) - link_payload = link_to_location(resource_list) - if len(link_payload) > 0: - link_response = handle_request( - "POST", link_payload, config.fhir_base_url - ) - logging.info(link_response.text) + final_response = handle_request("POST", json_payload, fhir_base_url) else: logging.error("Unsupported request!") else: logging.error("Empty csv file!") - logging.info('{ "final-response": ' + final_response.text + "}") + + # TODO - final_response does not have text - trial uploading users that have already been uploaded + try: + final_response = final_response.text + except: + pass + logging.info('{ "final-response": ' + final_response + "}") end_time = datetime.now() logging.info("End time: " + end_time.strftime("%H:%M:%S")) From b7ea61925f31265b51a64b9207977983deabac92 Mon Sep 17 00:00:00 2001 From: Peter Muriuki Date: Thu, 4 Jul 2024 18:23:37 +0300 Subject: [PATCH 11/18] Add product access token to settings file --- importer/config/settings.py | 1 + 1 file changed, 1 insertion(+) diff --git a/importer/config/settings.py b/importer/config/settings.py index c3474a64..3f4d8053 100644 --- a/importer/config/settings.py +++ b/importer/config/settings.py @@ -19,6 +19,7 @@ def dynamic_import(variable_name): # TODO - retrieving at and rt as args via the command line as well. access_token = dynamic_import("access_token") refresh_token = dynamic_import("refresh_token") +product_access_token = dynamic_import("product_access_token") authentication_options = None if username is not None and password is not None: From 982a56c5b19ea0fcbacecab3e671f3c051a18788 Mon Sep 17 00:00:00 2001 From: Peter Muriuki Date: Thu, 4 Jul 2024 18:31:40 +0300 Subject: [PATCH 12/18] Fix import in setting fil --- importer/config/settings.py | 12 ++++++++---- importer/main.py | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/importer/config/settings.py b/importer/config/settings.py index 3f4d8053..7a87112b 100644 --- a/importer/config/settings.py +++ b/importer/config/settings.py @@ -1,15 +1,19 @@ +import importlib +import sys +import logging.config + from services.fhir_keycloak_api import FhirKeycloakApi, FhirKeycloakApiOptions, ExternalAuthenticationOptions, \ InternalAuthenticationOptions from config.config import client_id, client_secret, fhir_base_url, keycloak_url, realm -import importlib -import sys + def dynamic_import(variable_name): try: - config_module = importlib.import_module("temporal.config") + config_module = importlib.import_module("config.config") value = getattr(config_module, variable_name, None) return value - except ModuleNotFoundError: + except: + logging.error("Unable to import the configuration!") sys.exit(1) diff --git a/importer/main.py b/importer/main.py index 9e17b226..1c0208f9 100644 --- a/importer/main.py +++ b/importer/main.py @@ -19,7 +19,7 @@ # logging.error("The config.py file is missing!") # exit() -global_access_token = "" +# global_access_token = "" # This function takes in a csv file From 2500df16d75ad997fc8e6cc40e2ad0fdd3cda42a Mon Sep 17 00:00:00 2001 From: Peter Muriuki Date: Thu, 4 Jul 2024 18:32:13 +0300 Subject: [PATCH 13/18] Rename stub data generatr --- importer/{stub_data => stub_data_gen}/orgs-stup-gen.py | 0 importer/{stub_data => stub_data_gen}/users-stub-gen.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename importer/{stub_data => stub_data_gen}/orgs-stup-gen.py (100%) rename importer/{stub_data => stub_data_gen}/users-stub-gen.py (100%) diff --git a/importer/stub_data/orgs-stup-gen.py b/importer/stub_data_gen/orgs-stup-gen.py similarity index 100% rename from importer/stub_data/orgs-stup-gen.py rename to importer/stub_data_gen/orgs-stup-gen.py diff --git a/importer/stub_data/users-stub-gen.py b/importer/stub_data_gen/users-stub-gen.py similarity index 100% rename from importer/stub_data/users-stub-gen.py rename to importer/stub_data_gen/users-stub-gen.py From 308289ed9bae1f718442d3c9dba00aaad93236e4 Mon Sep 17 00:00:00 2001 From: Peter Muriuki Date: Thu, 4 Jul 2024 18:50:25 +0300 Subject: [PATCH 14/18] Showstopper fail if authentication is not found --- importer/config/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/importer/config/settings.py b/importer/config/settings.py index 7a87112b..13839313 100644 --- a/importer/config/settings.py +++ b/importer/config/settings.py @@ -31,7 +31,7 @@ def dynamic_import(variable_name): elif access_token is not None and refresh_token is not None: authentication_options = ExternalAuthenticationOptions(client_id=client_id, client_secret=client_secret, keycloak_base_uri=keycloak_url, realm=realm, access_token=access_token, refresh_token=refresh_token) else: - sys.exit(1) + raise ValueError("Unable to get authentication parameters!") api_service_options = FhirKeycloakApiOptions(fhir_base_uri=fhir_base_url, authentication_options=authentication_options) api_service = FhirKeycloakApi(api_service_options) From 4294cccbd2902d70ce3e29cca01f5cd5b01e7638 Mon Sep 17 00:00:00 2001 From: Peter Muriuki Date: Fri, 5 Jul 2024 13:18:07 +0300 Subject: [PATCH 15/18] Some code cleanup --- importer/.gitignore | 125 +------------------------ importer/main.py | 30 ++++++ importer/services/fhir_keycloak_api.py | 38 ++++---- 3 files changed, 51 insertions(+), 142 deletions(-) diff --git a/importer/.gitignore b/importer/.gitignore index 21deed30..b9c85fa2 100644 --- a/importer/.gitignore +++ b/importer/.gitignore @@ -1,41 +1,11 @@ config.py +importer.log -sampleData -.idea # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - # Installer logs pip-log.txt pip-delete-this-directory.txt @@ -55,76 +25,11 @@ coverage.xml .pytest_cache/ cover/ -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -.pybuilder/ -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -# For a library or package, you might want to ignore these files since the code is -# intended to run in multiple environments; otherwise, check them in: -# .python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock - -# poetry -# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. -# This is especially recommended for binary packages to ensure reproducibility, and is more -# commonly ignored for libraries. -# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control -#poetry.lock - -# pdm -# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. -#pdm.lock -# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it -# in version control. -# https://pdm.fming.dev/latest/usage/project/#working-with-version-control -.pdm.toml -.pdm-python -.pdm-build/ +.python-version # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm __pypackages__/ -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - # Environments .env .venv @@ -134,33 +39,9 @@ ENV/ env.bak/ venv.bak/ -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ - -# pytype static type analyzer -.pytype/ - -# Cython debug symbols -cython_debug/ - # PyCharm # JetBrains specific template is maintained in a separate JetBrains.gitignore that can # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ +.idea/ diff --git a/importer/main.py b/importer/main.py index 1c0208f9..36baba9e 100644 --- a/importer/main.py +++ b/importer/main.py @@ -1064,6 +1064,30 @@ def build_payload(resource_type, resources, resource_payload_file): return final_string +def link_to_location(resource_list): + arr = [] + with click.progressbar( + resource_list, label="Progress::Linking inventory to location" + ) as link_locations_progress: + for resource in link_locations_progress: + try: + if resource[14]: + # name, inventory_id, supply_date, location_id + resource_link = [ + resource[0], + resource[3], + resource[9], + resource[14], + ] + arr.append(resource_link) + except IndexError: + logging.info("No location provided for " + resource[0]) + + if len(arr) > 0: + return build_assign_payload(arr, "List", "subject=Location/") + else: + return "" + def confirm_keycloak_user(user): # Confirm that the keycloak user details are as expected user_username = str(user[2]).strip() @@ -1902,6 +1926,12 @@ def main( "Group", resource_list, "json_payloads/inventory_group_payload.json" ) final_response = handle_request("POST", json_payload, fhir_base_url) + link_payload = link_to_location(resource_list) + if len(link_payload) > 0: + link_response = handle_request( + "POST", link_payload, fhir_base_url + ) + logging.info(link_response.text) else: logging.error("Unsupported request!") else: diff --git a/importer/services/fhir_keycloak_api.py b/importer/services/fhir_keycloak_api.py index 9e51ed24..55bd173f 100644 --- a/importer/services/fhir_keycloak_api.py +++ b/importer/services/fhir_keycloak_api.py @@ -2,22 +2,22 @@ Class that provides utility to: 1. get access and refresh tokens from keycloak 2. Update the tokens when expired. - 3. upload/update keycloak resources - 4. upload/update fhir resources + # 3. upload/update keycloak resources + # 4. upload/update fhir resources """ +import time from dataclasses import dataclass, fields, field from typing import Union +import backoff +import jwt import requests -# from keycloak import KeycloakOpenID, KeycloakOpenIDConnection +from oauthlib.oauth2 import LegacyApplicationClient from requests_oauthlib import OAuth2Session -from oauthlib.oauth2 import BackendApplicationClient, LegacyApplicationClient -from urllib.parse import urljoin -import time -import jwt -import backoff -from functools import wraps + +from config import config + def is_readable_string(s): """ @@ -104,7 +104,7 @@ def __init__(self, option: InternalAuthenticationOptions): def get_token(self): """ - Oauth 2 does not work without an ssl layer to test this locally see https://stackoverflow.com/a/27785830/14564571 + Oauth 2 does not work without a ssl layer to test this locally see https://stackoverflow.com/a/27785830/14564571 :return: """ @@ -126,13 +126,16 @@ def decode_token(self): # return json.loads(full_jwt.token.payload.decode("utf-8")) pass + class ExternalAuthenticationService: def __init__(self, option: ExternalAuthenticationOptions): self.options = option client = LegacyApplicationClient(client_id=self.options.client_id) oauth = OAuth2Session(client=client, - token={"access_token": self.options.access_token, "refresh_token": self.options.refresh_token, 'token_type': "Bearer", "expires_in": 18000}, + token={"access_token": self.options.access_token, + "refresh_token": self.options.refresh_token, + 'token_type': "Bearer", "expires_in": 18000}, auto_refresh_url=self.options.token_uri, ) self.client = client @@ -143,10 +146,11 @@ def get_token(self): Oauth 2 does not work without an ssl layer to test this locally see https://stackoverflow.com/a/27785830/14564571 :return: """ - # return the current token, not if its expired or invalid raise an irrecoverable show stopper error. + # return the current token, not if its expired or invalid raise an irrecoverable showstopper error. if self._is_refresh_required(): # if expired - self.oauth.refresh_token(token_url=self.options.token_uri, client_id=self.options.client_id, client_secret=self.options.client_secret) + self.oauth.refresh_token(token_url=self.options.token_uri, client_id=self.options.client_id, + client_secret=self.options.client_secret) token = self.oauth.fetch_token(token_url=self.options.token_uri, client_id=self.options.client_id, client_secret=self.options.client_secret) @@ -157,7 +161,6 @@ def get_token(self): def refresh_token(self,): return self.oauth.refresh_token(self.options.token_uri, client_id=self.options.client_id, client_secret=self.options.client_secret) - def _is_refresh_required(self): # TODO some defensive programming would be nice. at = self.oauth.token.get("access_token") @@ -171,7 +174,7 @@ def decode_token(self, token: str): # TODO - verify JWT _algorithms = "HS256" _do_verify=False - cert_uri = "http://localhost:8080/auth/realms/fhir/protocol/openid-connect/certs" + cert_uri = f"{config.keycloak_url}/realms/fhir/protocol/openid-connect/certs" res = self.oauth.get(cert_uri).json().get("keys") # tired first_key = res[0] @@ -179,8 +182,3 @@ def decode_token(self, token: str): _algorithms = first_key.get("alg") instance = jwt.JWT() return instance.decode(token, algorithms=_algorithms, key=jwk, do_verify=True, do_time_check=True) - - def handleRequest(self, **kwargs): - # self.oauth. - pass - From fd9ccb7bac92f13f11170cb9b7bfb63b6007cc5e Mon Sep 17 00:00:00 2001 From: Peter Muriuki Date: Fri, 5 Jul 2024 16:17:24 +0300 Subject: [PATCH 16/18] Add a few general notes --- importer/notes.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 importer/notes.md diff --git a/importer/notes.md b/importer/notes.md new file mode 100644 index 00000000..06f26419 --- /dev/null +++ b/importer/notes.md @@ -0,0 +1,5 @@ +**TODO** + +- When we need to make a lot of requests when updating a resource can lead to dropped connecttions + 1. We can fetch the records in a single or fewer paginated requests + 2. Add a sleep intervals to requests that might fire too fast. \ No newline at end of file From acaec4a7c9214e1d17c5ff680bddc78755e2e67e Mon Sep 17 00:00:00 2001 From: Peter Muriuki Date: Fri, 5 Jul 2024 16:18:06 +0300 Subject: [PATCH 17/18] Update how partOf is added to a location --- importer/main.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/importer/main.py b/importer/main.py index 36baba9e..3c6a88d2 100644 --- a/importer/main.py +++ b/importer/main.py @@ -318,10 +318,16 @@ def location_extras(resource, payload_string): longitude = "longitude" try: - if locationParentName and locationParentName != "parentName": - payload_string = payload_string.replace( - "$parentName", locationParentName - ).replace("$parentID", locationParentId) + if locationParentId and locationParentId != "parentId": + payload_string = payload_string.replace("$parentID", locationParentId) + if not locationParentName or locationParentName == "parentName": + obj = json.loads(payload_string) + del obj["resource"]["partOf"]['display'] + payload_string = json.dumps(obj, indent=4) + else: + payload_string = payload_string.replace( + "$parentName", locationParentName + ) else: obj = json.loads(payload_string) del obj["resource"]["partOf"] From a7967966d2cb8a3f95eb1b72a6c9eb94de1d58ee Mon Sep 17 00:00:00 2001 From: Peter Muriuki Date: Fri, 5 Jul 2024 16:18:31 +0300 Subject: [PATCH 18/18] Fix how location physical type is added to location resources --- importer/json_payloads/locations_payload.json | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/importer/json_payloads/locations_payload.json b/importer/json_payloads/locations_payload.json index 12bd5a97..43019b60 100644 --- a/importer/json_payloads/locations_payload.json +++ b/importer/json_payloads/locations_payload.json @@ -39,10 +39,15 @@ ] }, { + "coding": [ + { "system": "http://terminology.hl7.org/CodeSystem/location-physical-type", "code": "$pt_code", "display": "$pt_display" } + ] + } + ], "physicalType": { "coding": [