diff --git a/importer/.gitignore b/importer/.gitignore index 4acd06b1..b9c85fa2 100644 --- a/importer/.gitignore +++ b/importer/.gitignore @@ -1 +1,47 @@ config.py +importer.log + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +.python-version + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +.idea/ diff --git a/importer/README.md b/importer/README.md index c1dc4b4c..d8bbdd58 100644 --- a/importer/README.md +++ b/importer/README.md @@ -17,7 +17,7 @@ This script is used to setup keycloak roles and groups. It takes in a csv file w ### To run script 1. Create virtualenv 2. Install requirements.txt - `pip install -r requirements.txt` -3. Create a `config.py` file. The `sample_config.py` is an example of what this should look like. Populate it with the right credentials, you can either provide an access token or client credentials. Ensure that the user whose details you provide in this config file has the necessary permissions/privilleges. +3. Create a `config/config.py` file. The `config/sample_config.py` is an example of what this should look like. Populate it with the right credentials, you can either provide an access token or client credentials. Ensure that the user whose details you provide in this config file has the necessary permissions/privilleges. 4. Run script - `python3 main.py --setup roles --csv_file csv/setup/roles.csv --group Supervisor` 5. If you are running the script without `https` setup e.g locally or a server without https setup, you will need to set the `OAUTHLIB_INSECURE_TRANSPORT` environment variable to 1. For example `export OAUTHLIB_INSECURE_TRANSPORT=1 && python3 main.py --setup roles --csv_file csv/setup/roles.csv --group OpenSRP_Provider --log_level debug` 6. You can turn on logging by passing a `--log_level` to the command line as `info`, `debug` or `error`. For example `python3 main.py --setup roles --csv_file csv/setup/roles.csv --group Supervisor --log_level debug` diff --git a/importer/config.py b/importer/config/__init__.py similarity index 100% rename from importer/config.py rename to importer/config/__init__.py diff --git a/importer/config/sample_config.py b/importer/config/sample_config.py new file mode 100644 index 00000000..3adbe8f2 --- /dev/null +++ b/importer/config/sample_config.py @@ -0,0 +1,17 @@ +client_id = 'example-client-id' +client_secret = 'example-client-secret' +fhir_base_url = 'https://example.smartregister.org/fhir' +keycloak_url = 'https://keycloak.smartregister.org/auth' + +# access token for access to where product images are remotely stored +product_access_token = 'example-product-access-token' + +# if using resource owner credentials (i.e importer handles getting authentication by itself) +# This has greater precedence over the access and refresh tokens if supplied +username = 'example-username' +password = 'example-password' + +# if embedding importer into a service that already does the authentication +access_token = 'example-access-token' +refresh_token = 'example-refresh-token' + diff --git a/importer/config/settings.py b/importer/config/settings.py new file mode 100644 index 00000000..13839313 --- /dev/null +++ b/importer/config/settings.py @@ -0,0 +1,41 @@ +import importlib +import sys +import logging.config + +from services.fhir_keycloak_api import FhirKeycloakApi, FhirKeycloakApiOptions, ExternalAuthenticationOptions, \ + InternalAuthenticationOptions +from config.config import client_id, client_secret, fhir_base_url, keycloak_url, realm + + +def dynamic_import(variable_name): + try: + config_module = importlib.import_module("config.config") + value = getattr(config_module, variable_name, None) + return value + except: + logging.error("Unable to import the configuration!") + sys.exit(1) + + +username = dynamic_import("username") +password = dynamic_import("password") + +# TODO - retrieving at and rt as args via the command line as well. +access_token = dynamic_import("access_token") +refresh_token = dynamic_import("refresh_token") +product_access_token = dynamic_import("product_access_token") + +authentication_options = None +if username is not None and password is not None: + authentication_options = InternalAuthenticationOptions(client_id=client_id, client_secret=client_secret, keycloak_base_uri=keycloak_url, realm=realm, user_username=username, user_password=password) +elif access_token is not None and refresh_token is not None: + authentication_options = ExternalAuthenticationOptions(client_id=client_id, client_secret=client_secret, keycloak_base_uri=keycloak_url, realm=realm, access_token=access_token, refresh_token=refresh_token) +else: + raise ValueError("Unable to get authentication parameters!") + +api_service_options = FhirKeycloakApiOptions(fhir_base_uri=fhir_base_url, authentication_options=authentication_options) +api_service = FhirKeycloakApi(api_service_options) + + + + diff --git a/importer/json_payloads/locations_payload.json b/importer/json_payloads/locations_payload.json index d06dbe46..43019b60 100644 --- a/importer/json_payloads/locations_payload.json +++ b/importer/json_payloads/locations_payload.json @@ -37,7 +37,17 @@ "display": "Level $adminLevelCode" } ] + }, + { + "coding": [ + { + "system": "http://terminology.hl7.org/CodeSystem/location-physical-type", + "code": "$pt_code", + "display": "$pt_display" } + ] + } + ], "physicalType": { "coding": [ diff --git a/importer/main.py b/importer/main.py index 8c9a0185..2cdcbc8c 100644 --- a/importer/main.py +++ b/importer/main.py @@ -6,20 +6,20 @@ import requests import logging import logging.config -import backoff import base64 import magic from datetime import datetime -from oauthlib.oauth2 import LegacyApplicationClient -from requests_oauthlib import OAuth2Session -try: - import config -except ModuleNotFoundError: - logging.error("The config.py file is missing!") - exit() +from config.settings import api_service, keycloak_url, fhir_base_url, product_access_token +from utils.location_process import process_locations -global_access_token = "" +# try: +# import config +# except ModuleNotFoundError: +# logging.error("The config.py file is missing!") +# exit() + +# global_access_token = "" # This function takes in a csv file @@ -46,62 +46,47 @@ def read_csv(csv_file): logging.error("Stop iteration on empty file") -def get_access_token(): - access_token = "" - if global_access_token: - return global_access_token - - try: - if config.access_token: - # get access token from config file - access_token = config.access_token - except AttributeError: - logging.debug("No access token provided, trying to use client credentials") - - if not access_token: - # get client credentials from config file - client_id = config.client_id - client_secret = config.client_secret - username = config.username - password = config.password - access_token_url = config.access_token_url - - oauth = OAuth2Session(client=LegacyApplicationClient(client_id=client_id)) - token = oauth.fetch_token( - token_url=access_token_url, - username=username, - password=password, - client_id=client_id, - client_secret=client_secret, - ) - access_token = token["access_token"] +# def get_access_token(): +# access_token = "" +# if global_access_token: +# return global_access_token +# +# try: +# if config.access_token: +# # get access token from config file +# access_token = config.access_token +# except AttributeError: +# logging.debug("No access token provided, trying to use client credentials") +# +# if not access_token: +# # get client credentials from config file +# client_id = config.client_id +# client_secret = config.client_secret +# username = config.username +# password = config.password +# access_token_url = config.access_token_url +# +# oauth = OAuth2Session(client=LegacyApplicationClient(client_id=client_id)) +# token = oauth.fetch_token( +# token_url=access_token_url, +# username=username, +# password=password, +# client_id=client_id, +# client_secret=client_secret, +# ) +# access_token = token["access_token"] +# +# return access_token - return access_token -# This function makes the request to the provided url -# to create resources -@backoff.on_exception(backoff.expo, requests.exceptions.RequestException, max_time=180) def post_request(request_type, payload, url, json_payload): logging.info("Posting request") logging.info("Request type: " + request_type) logging.info("Url: " + url) logging.debug("Payload: " + payload) - - access_token = "Bearer " + get_access_token() - headers = {"Content-type": "application/json", "Authorization": access_token} - - if request_type == "POST": - return requests.post(url, data=payload, json=json_payload, headers=headers) - elif request_type == "PUT": - return requests.put(url, data=payload, json=json_payload, headers=headers) - elif request_type == "GET": - return requests.get(url, headers=headers) - elif request_type == "DELETE": - return requests.delete(url, headers=headers) - else: - logging.error("Unsupported request type!") - + return api_service.request(method=request_type, url=url, + data=payload, json=json_payload) def handle_request(request_type, payload, url, json_payload=None): try: @@ -118,7 +103,7 @@ def handle_request(request_type, payload, url, json_payload=None): def get_keycloak_url(): - return config.keycloak_url + return api_service.auth_options.keycloak_realm_uri # This function builds the user payload and posts it to @@ -140,6 +125,7 @@ def create_user(user): password, ) = user + # TODO - move this out so that its not recreated for every user. with open("json_payloads/keycloak_user_payload.json") as json_file: payload_string = json_file.read() @@ -332,10 +318,16 @@ def location_extras(resource, payload_string): longitude = "longitude" try: - if locationParentName and locationParentName != "parentName": - payload_string = payload_string.replace( - "$parentName", locationParentName - ).replace("$parentID", locationParentId) + if locationParentId and locationParentId != "parentId": + payload_string = payload_string.replace("$parentID", locationParentId) + if not locationParentName or locationParentName == "parentName": + obj = json.loads(payload_string) + del obj["resource"]["partOf"]['display'] + payload_string = json.dumps(obj, indent=4) + else: + payload_string = payload_string.replace( + "$parentName", locationParentName + ) else: obj = json.loads(payload_string) del obj["resource"]["partOf"] @@ -984,7 +976,7 @@ def get_valid_resource_type(resource_type): def get_resource(resource_id, resource_type): if resource_type != "Group": resource_type = get_valid_resource_type(resource_type) - resource_url = "/".join([config.fhir_base_url, resource_type, resource_id]) + resource_url = "/".join([fhir_base_url, resource_type, resource_id]) response = handle_request("GET", "", resource_url) return json.loads(response[0])["meta"]["versionId"] if response[1] == 200 else "0" @@ -1075,7 +1067,7 @@ def build_payload(resource_type, resources, resource_payload_file): final_string = final_string + ps + "," - final_string = initial_string + final_string[:-1] + " ] } " + final_string = json.dumps(json.loads(initial_string + final_string[:-1] + " ] } ")) return final_string @@ -1103,20 +1095,21 @@ def link_to_location(resource_list): else: return "" - def confirm_keycloak_user(user): # Confirm that the keycloak user details are as expected user_username = str(user[2]).strip() user_email = str(user[3]).strip() keycloak_url = get_keycloak_url() response = handle_request( - "GET", "", keycloak_url + "/users?exact=true&username=" + user_username + "GET", "", api_service.auth_options.keycloak_realm_uri + "/users?exact=true&username=" + user_username ) logging.debug(response) json_response = json.loads(response[0]) try: - response_email = json_response[0]["email"] + # TODO - apparently not all user uploads will have an email + print("============>", json_response) + response_email = json_response[0].get("email", "") except IndexError: response_email = "" @@ -1182,7 +1175,7 @@ def confirm_practitioner(user, user_id): return True except Exception as err: - logging.error("Error occured trying to find Practitioner: " + str(err)) + logging.error("Error occurred trying to find Practitioner: " + str(err)) return True @@ -1193,7 +1186,7 @@ def create_roles(role_list, roles_max): # check if role already exists role_response = handle_request( - "GET", "", config.keycloak_url + "/roles/" + current_role + "GET", "", keycloak_url + "/roles/" + current_role ) logging.debug(role_response) if current_role in role_response[0]: @@ -1201,7 +1194,7 @@ def create_roles(role_list, roles_max): else: role_payload = '{"name": "' + current_role + '"}' create_role = handle_request( - "POST", role_payload, config.keycloak_url + "/roles" + "POST", role_payload, keycloak_url + "/roles" ) if create_role.status_code == 201: logging.info("Successfully created role: " + current_role) @@ -1212,7 +1205,7 @@ def create_roles(role_list, roles_max): logging.debug("Role has composite roles") # get roled id full_role = handle_request( - "GET", "", config.keycloak_url + "/roles/" + current_role + "GET", "", keycloak_url + "/roles/" + current_role ) json_resp = json.loads(full_role[0]) role_id = json_resp["id"] @@ -1222,7 +1215,7 @@ def create_roles(role_list, roles_max): available_roles = handle_request( "GET", "", - config.keycloak_url + keycloak_url + "/admin-ui-available-roles/roles/" + role_id + "?first=0&max=" @@ -1257,7 +1250,7 @@ def create_roles(role_list, roles_max): handle_request( "POST", payload_arr, - config.keycloak_url + "/roles-by-id/" + role_id + "/composites", + keycloak_url + "/roles-by-id/" + role_id + "/composites", ) except IndexError: @@ -1266,7 +1259,7 @@ def create_roles(role_list, roles_max): def get_group_id(group): # check if group exists - all_groups = handle_request("GET", "", config.keycloak_url + "/groups") + all_groups = handle_request("GET", "", keycloak_url + "/groups") json_groups = json.loads(all_groups[0]) group_obj = {} @@ -1282,7 +1275,7 @@ def get_group_id(group): logging.info("Group does not exists, lets create it") # create the group create_group_payload = '{"name":"' + group + '"}' - handle_request("POST", create_group_payload, config.keycloak_url + "/groups") + handle_request("POST", create_group_payload, keycloak_url + "/groups") return get_group_id(group) @@ -1294,7 +1287,7 @@ def assign_group_roles(role_list, group, roles_max): available_roles_for_group = handle_request( "GET", "", - config.keycloak_url + keycloak_url + "/groups/" + group_id + "/role-mappings/realm/available?first=0&max=" @@ -1315,7 +1308,7 @@ def assign_group_roles(role_list, group, roles_max): handle_request( "POST", json_assign_payload, - config.keycloak_url + "/groups/" + group_id + "/role-mappings/realm", + keycloak_url + "/groups/" + group_id + "/role-mappings/realm", ) @@ -1326,7 +1319,7 @@ def delete_resource(resource_type, resource_id, cascade): cascade = "" resource_url = "/".join( - [config.fhir_base_url, resource_type, resource_id + cascade] + [fhir_base_url, resource_type, resource_id + cascade] ) r = handle_request("DELETE", "", resource_url) logging.info(r.text) @@ -1337,7 +1330,7 @@ def clean_duplicates(users, cascade_delete): # get keycloak user uuid username = str(user[2].strip()) user_details = handle_request( - "GET", "", config.keycloak_url + "/users?exact=true&username=" + username + "GET", "", keycloak_url + "/users?exact=true&username=" + username ) obj = json.loads(user_details[0]) keycloak_uuid = obj[0]["id"] @@ -1346,7 +1339,7 @@ def clean_duplicates(users, cascade_delete): r = handle_request( "GET", "", - config.fhir_base_url + "/Practitioner?identifier=" + keycloak_uuid, + fhir_base_url + "/Practitioner?identifier=" + keycloak_uuid, ) practitioner_details = json.loads(r[0]) count = practitioner_details["total"] @@ -1407,7 +1400,7 @@ def write_csv(data, resource_type, fieldnames): def get_base_url(): - return config.fhir_base_url + return api_service.fhir_base_uri # This function exports resources from the API to a csv file @@ -1548,8 +1541,9 @@ def encode_image(image_file): # and saves it as a Binary resource. It returns the id of the Binary resource if # successful and 0 if failed def save_image(image_source_url): + try: - headers = {"Authorization": "Bearer " + config.product_access_token} + headers = {"Authorization": "Bearer " + product_access_token} except AttributeError: headers = {} @@ -1628,7 +1622,7 @@ def process_chunk(resources_array: list, resource_type: str): json_payload = {"resourceType": "Bundle", "type": "transaction", "entry": new_arr} - r = handle_request("POST", "", config.fhir_base_url, json_payload) + r = handle_request("POST", "", fhir_base_url, json_payload) logging.info(r.text) # TODO handle failures @@ -1821,6 +1815,7 @@ def main( ) logging.getLogger().addHandler(logging.StreamHandler()) + # TODO - should be an empty flag that does not need a value. if only_response: logging.config.dictConfig(LOGGING) @@ -1871,15 +1866,20 @@ def main( if not practitioner_exists: payload = create_user_resources(user_id, user) final_response = handle_request( - "POST", payload, config.fhir_base_url + "POST", payload, fhir_base_url ) logging.info("Processing complete!") elif resource_type == "locations": logging.info("Processing locations") - json_payload = build_payload( - "locations", resource_list, "json_payloads/locations_payload.json" - ) - final_response = handle_request("POST", json_payload, config.fhir_base_url) + batch_generator = process_locations(resource_list) + final_response = [] + for batch in batch_generator: + json_payload = build_payload( + "locations", batch, "json_payloads/locations_payload.json" + ) + response = handle_request("POST", json_payload, fhir_base_url) + final_response.append(response.text) + final_response = ",\n".join(final_response) logging.info("Processing complete!") elif resource_type == "organizations": logging.info("Processing organizations") @@ -1888,27 +1888,25 @@ def main( resource_list, "json_payloads/organizations_payload.json", ) - final_response = handle_request("POST", json_payload, config.fhir_base_url) + final_response = handle_request("POST", json_payload, fhir_base_url) logging.info("Processing complete!") elif resource_type == "careTeams": logging.info("Processing CareTeams") json_payload = build_payload( "careTeams", resource_list, "json_payloads/careteams_payload.json" ) - final_response = handle_request("POST", json_payload, config.fhir_base_url) + final_response = handle_request("POST", json_payload, fhir_base_url) logging.info("Processing complete!") elif assign == "organizations-Locations": logging.info("Assigning Organizations to Locations") matches = extract_matches(resource_list) json_payload = build_org_affiliation(matches, resource_list) - final_response = handle_request("POST", json_payload, config.fhir_base_url) + final_response = handle_request("POST", json_payload, fhir_base_url) logging.info("Processing complete!") elif assign == "users-organizations": logging.info("Assigning practitioner to Organization") - json_payload = build_assign_payload( - resource_list, "PractitionerRole", "practitioner=Practitioner/" - ) - final_response = handle_request("POST", json_payload, config.fhir_base_url) + json_payload = build_assign_payload(resource_list, "PractitionerRole") + final_response = handle_request("POST", json_payload, fhir_base_url) logging.info("Processing complete!") elif setup == "roles": logging.info("Setting up keycloak roles") @@ -1928,17 +1926,17 @@ def main( json_payload = build_payload( "Group", resource_list, "json_payloads/product_group_payload.json" ) - final_response = handle_request("POST", json_payload, config.fhir_base_url) + final_response = handle_request("POST", json_payload, fhir_base_url) elif setup == "inventories": logging.info("Importing inventories as FHIR Group resources") json_payload = build_payload( "Group", resource_list, "json_payloads/inventory_group_payload.json" ) - final_response = handle_request("POST", json_payload, config.fhir_base_url) + final_response = handle_request("POST", json_payload, fhir_base_url) link_payload = link_to_location(resource_list) if len(link_payload) > 0: link_response = handle_request( - "POST", link_payload, config.fhir_base_url + "POST", link_payload, fhir_base_url ) logging.info(link_response.text) else: @@ -1946,7 +1944,13 @@ def main( else: logging.error("Empty csv file!") - logging.info('{ "final-response": ' + final_response.text + "}") + + # TODO - final_response does not have text - trial uploading users that have already been uploaded + try: + final_response = final_response.text + except: + pass + logging.info('{ "final-response": ' + final_response + "}") end_time = datetime.now() logging.info("End time: " + end_time.strftime("%H:%M:%S")) diff --git a/importer/notes.md b/importer/notes.md new file mode 100644 index 00000000..06f26419 --- /dev/null +++ b/importer/notes.md @@ -0,0 +1,5 @@ +**TODO** + +- When we need to make a lot of requests when updating a resource can lead to dropped connecttions + 1. We can fetch the records in a single or fewer paginated requests + 2. Add a sleep intervals to requests that might fire too fast. \ No newline at end of file diff --git a/importer/requirements.txt b/importer/requirements.txt index 4c748647..a72ac03b 100644 --- a/importer/requirements.txt +++ b/importer/requirements.txt @@ -7,4 +7,5 @@ backoff==2.2.1 pytest==7.4.2 jsonschema==4.21.1 mock==5.1.0 -python-magic==0.4.27 \ No newline at end of file +python-magic==0.4.27 +jwt diff --git a/importer/sample_config.py b/importer/sample_config.py deleted file mode 100644 index b8432523..00000000 --- a/importer/sample_config.py +++ /dev/null @@ -1,9 +0,0 @@ -client_id = 'example-client-id' -client_secret = 'example-client-secret' -username = 'example-username' -password = 'example-password' -access_token_url = 'https://keycloak.smartregister.org/auth/realms/example-realm/protocol/openid-connect/token' -fhir_base_url = 'https://example.smartregister.org/fhir' -keycloak_url = 'https://keycloak.smartregister.org/auth/admin/realms/example-realm' -access_token = 'example-access-token' -product_access_token = 'example-product-access-token' diff --git a/importer/services/__init__.py b/importer/services/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/importer/services/fhir_keycloak_api.py b/importer/services/fhir_keycloak_api.py new file mode 100644 index 00000000..55bd173f --- /dev/null +++ b/importer/services/fhir_keycloak_api.py @@ -0,0 +1,184 @@ +""" +Class that provides utility to: + 1. get access and refresh tokens from keycloak + 2. Update the tokens when expired. + # 3. upload/update keycloak resources + # 4. upload/update fhir resources +""" + +import time +from dataclasses import dataclass, fields, field +from typing import Union + +import backoff +import jwt +import requests +from oauthlib.oauth2 import LegacyApplicationClient +from requests_oauthlib import OAuth2Session + +from config import config + + +def is_readable_string(s): + """ + Check if a variable is not an empty string. + + Args: + s (str): The variable to check. + + Returns: + bool: True if s is not an empty string, False otherwise. + """ + return isinstance(s, str) and s.strip() != "" + + +@dataclass +class IamUri: + """Keycloak authentication uris""" + keycloak_base_uri: str + realm: str + client_id: str + client_secret: str + token_uri: str = field(init=False) + + def __post_init__(self): + for field in fields(self): + if field.init and not is_readable_string(getattr(self, field.name)): + raise ValueError(f"{self.__class__.__name__} can only be initialized with str values") + self.token_uri = self.keycloak_base_uri + "/realms/" + self.realm + "/protocol/openid-connect/token" + self.keycloak_realm_uri = self.keycloak_base_uri + "/admin/realms/" + self.realm + + +@dataclass +class InternalAuthenticationOptions(IamUri): + """Describes config options for authentication that we have to handle ourselves""" + user_username: str + user_password: str + +@dataclass +class ExternalAuthenticationOptions(IamUri): + """Describes config options for authentication that we have to handle ourselves""" + access_token: str + refresh_token: str + + +@dataclass +class FhirKeycloakApiOptions: + authentication_options: Union[InternalAuthenticationOptions, ExternalAuthenticationOptions] + fhir_base_uri: str + +class FhirKeycloakApi(): + def __init__(self, options: FhirKeycloakApiOptions): + auth_options = options.authentication_options + if isinstance(auth_options, ExternalAuthenticationOptions): + self.authentication_Side = "external" + self.api_service = ExternalAuthenticationService(auth_options) + if isinstance(auth_options, InternalAuthenticationOptions): + self.authentication_Side = "internal" + self.api_service = InternalAuthenticationService(auth_options) + self.auth_options = auth_options + self.fhir_base_uri = options.fhir_base_uri + + @backoff.on_exception(backoff.expo, requests.exceptions.RequestException, max_time=180) + def request(self, **kwargs): + # TODO - spread headers into kwargs. + headers = { + "content-type": "application/json", + "accept": "application/json" + } + response = self.api_service.oauth.request(headers=headers, **kwargs) + if response.status_code == 401: + self.api_service.refresh_token() + return self.api_service.oauth.request(headers=headers, **kwargs) + return response + + +class InternalAuthenticationService: + + def __init__(self, option: InternalAuthenticationOptions): + self.options = option + client = LegacyApplicationClient(client_id=self.options.client_id) + oauth = OAuth2Session(client=client, auto_refresh_url=self.options.token_uri) + self.client = client + self.oauth = oauth + + def get_token(self): + """ + Oauth 2 does not work without a ssl layer to test this locally see https://stackoverflow.com/a/27785830/14564571 + :return: + """ + + token = self.oauth.fetch_token(token_url=self.options.token_uri, client_id=self.options.client_id, + client_secret=self.options.client_secret, username=self.options.user_username, password=self.options.user_password) + return token + + def refresh_token(self,): + return self.get_token() + + + def _is_refresh_required(self): + # TODO some defensive programming would be nice. + return time.time() > self.oauth.token.get("expires_at") + + def decode_token(self): + # full_jwt = jwt.JWT(jwt=token, **kwargs) + # full_jwt.token.objects["valid"] = True + # return json.loads(full_jwt.token.payload.decode("utf-8")) + pass + + +class ExternalAuthenticationService: + + def __init__(self, option: ExternalAuthenticationOptions): + self.options = option + client = LegacyApplicationClient(client_id=self.options.client_id) + oauth = OAuth2Session(client=client, + token={"access_token": self.options.access_token, + "refresh_token": self.options.refresh_token, + 'token_type': "Bearer", "expires_in": 18000}, + auto_refresh_url=self.options.token_uri, + ) + self.client = client + self.oauth = oauth + + def get_token(self): + """ + Oauth 2 does not work without an ssl layer to test this locally see https://stackoverflow.com/a/27785830/14564571 + :return: + """ + # return the current token, not if its expired or invalid raise an irrecoverable showstopper error. + if self._is_refresh_required(): + # if expired + self.oauth.refresh_token(token_url=self.options.token_uri, client_id=self.options.client_id, + client_secret=self.options.client_secret) + + token = self.oauth.fetch_token(token_url=self.options.token_uri, client_id=self.options.client_id, + client_secret=self.options.client_secret) + return token + else: + return self.oauth.token + + def refresh_token(self,): + return self.oauth.refresh_token(self.options.token_uri, client_id=self.options.client_id, client_secret=self.options.client_secret) + + def _is_refresh_required(self): + # TODO some defensive programming would be nice. + at = self.oauth.token.get("access_token") + try: + decoded_at = self.decode_token(at) + return time.time() > decoded_at.get("exp") + except: + return False + + def decode_token(self, token: str): + # TODO - verify JWT + _algorithms = "HS256" + _do_verify=False + cert_uri = f"{config.keycloak_url}/realms/fhir/protocol/openid-connect/certs" + res = self.oauth.get(cert_uri).json().get("keys") + # tired + first_key = res[0] + jwk = jwt.jwk_from_dict(first_key) + _algorithms = first_key.get("alg") + instance = jwt.JWT() + return instance.decode(token, algorithms=_algorithms, key=jwk, do_verify=True, do_time_check=True) diff --git a/importer/stub_data_gen/orgs-stup-gen.py b/importer/stub_data_gen/orgs-stup-gen.py new file mode 100644 index 00000000..92c02bc0 --- /dev/null +++ b/importer/stub_data_gen/orgs-stup-gen.py @@ -0,0 +1,35 @@ +import csv +import uuid +from faker import Faker + +# Initialize Faker +fake = Faker() + +# Template data (header and sample row) +header = [ + "orgName", "orgActive", "method", "orgId", "identifier" +] + +# Function to generate random row data +def generate_random_row(): + org_name = fake.name() + active = fake.random_element(["true", "false", ""]) + method = "create" + id = fake.uuid4() + identifier = fake.uuid4() + + return [org_name, active, method, id, identifier] + +# Generate 100 rows of data +rows = [] +for _ in range(100): + rows.append(generate_random_row()) + +# Write to CSV file +filename = f"localCsvs/orgs.csv" +with open(filename, mode='w', newline='') as file: + writer = csv.writer(file) + writer.writerow(header) + writer.writerows(rows) + +print(f"CSV file '{filename}' with 100 rows has been created.") diff --git a/importer/stub_data_gen/users-stub-gen.py b/importer/stub_data_gen/users-stub-gen.py new file mode 100644 index 00000000..3dd1a322 --- /dev/null +++ b/importer/stub_data_gen/users-stub-gen.py @@ -0,0 +1,42 @@ +import csv +import uuid +from faker import Faker + +# Initialize Faker +fake = Faker() + +# Template data (header and sample row) +header = [ + "firstName", "lastName", "username", "email", "userId", + "userType", "enableUser", "keycloakGroupId", "keycloakGroupName", + "appId", "password" +] + +# Function to generate random row data +def generate_random_row(): + f_name = fake.first_name() + l_name = fake.last_name() + u_name = fake.user_name() + email = fake.email() + user_id = "" + user_type = fake.random_element(["Practitioner", "Supervisor", ""]) + enable_user = fake.random_element(["true", "false", ""]) + group_ids = "" + group_names = "" + app_id = "quest" + password = fake.password() + return [f_name, l_name, u_name, email, user_id, user_type, enable_user, group_ids, group_names, app_id, password] + +# Generate 100 rows of data +rows = [] +for _ in range(100): + rows.append(generate_random_row()) + +# Write to CSV file +filename = f"./users.csv" +with open(filename, mode='w', newline='') as file: + writer = csv.writer(file) + writer.writerow(header) + writer.writerows(rows) + +print(f"CSV file '{filename}' with 100 rows has been created.") diff --git a/importer/test_main.py b/importer/tests/test_main.py similarity index 86% rename from importer/test_main.py rename to importer/tests/test_main.py index b82dac74..9f7d8572 100644 --- a/importer/test_main.py +++ b/importer/tests/test_main.py @@ -22,7 +22,7 @@ class TestMain(unittest.TestCase): def test_read_csv(self): - csv_file = "csv/users.csv" + csv_file = "../csv/users.csv" records = read_csv(csv_file) self.assertIsInstance(records, list) self.assertEqual(len(records), 3) @@ -56,10 +56,10 @@ def test_write_csv(self): def test_build_payload_organizations(self, mock_get_resource): mock_get_resource.return_value = "1" - csv_file = "csv/organizations/organizations_full.csv" + csv_file = "../csv/organizations/organizations_full.csv" resource_list = read_csv(csv_file) payload = build_payload( - "organizations", resource_list, "json_payloads/organizations_payload.json" + "organizations", resource_list, "../json_payloads/organizations_payload.json" ) payload_obj = json.loads(payload) self.assertIsInstance(payload_obj, dict) @@ -90,10 +90,10 @@ def test_build_payload_organizations(self, mock_get_resource): validate(payload_obj["entry"][2]["request"], request_schema) # TestCase organizations_min.csv - csv_file = "csv/organizations/organizations_min.csv" + csv_file = "../csv/organizations/organizations_min.csv" resource_list = read_csv(csv_file) payload = build_payload( - "organizations", resource_list, "json_payloads/organizations_payload.json" + "organizations", resource_list, "../json_payloads/organizations_payload.json" ) payload_obj = json.loads(payload) self.assertIsInstance(payload_obj, dict) @@ -131,10 +131,10 @@ def test_build_payload_locations( mock_get_resource.return_value = "1" mock_check_parent_admin_level.return_value = "3" - csv_file = "csv/locations/locations_full.csv" + csv_file = "../csv/locations/locations_full.csv" resource_list = read_csv(csv_file) payload = build_payload( - "locations", resource_list, "json_payloads/locations_payload.json" + "locations", resource_list, "../json_payloads/locations_payload.json" ) payload_obj = json.loads(payload) self.assertIsInstance(payload_obj, dict) @@ -220,10 +220,10 @@ def test_build_payload_locations( validate(payload_obj["entry"][0]["request"], request_schema) # TestCase locations_min.csv - csv_file = "csv/locations/locations_min.csv" + csv_file = "../csv/locations/locations_min.csv" resource_list = read_csv(csv_file) payload = build_payload( - "locations", resource_list, "json_payloads/locations_payload.json" + "locations", resource_list, "../json_payloads/locations_payload.json" ) payload_obj = json.loads(payload) self.assertIsInstance(payload_obj, dict) @@ -287,10 +287,10 @@ def test_check_parent_admin_level(self, mock_get_base_url, mock_handle_request): def test_build_payload_care_teams(self, mock_get_resource): mock_get_resource.return_value = "1" - csv_file = "csv/careteams/careteam_full.csv" + csv_file = "../csv/careteams/careteam_full.csv" resource_list = read_csv(csv_file) payload = build_payload( - "careTeams", resource_list, "json_payloads/careteams_payload.json" + "careTeams", resource_list, "../json_payloads/careteams_payload.json" ) payload_obj = json.loads(payload) self.assertIsInstance(payload_obj, dict) @@ -386,10 +386,10 @@ def test_build_payload_group(self, mock_get_resource, mock_save_image): mock_get_resource.return_value = "1" mock_save_image.return_value = "f374a23a-3c6a-4167-9970-b10c16a91bbd" - csv_file = "csv/import/product.csv" + csv_file = "../csv/import/product.csv" resource_list = read_csv(csv_file) payload = build_payload( - "Group", resource_list, "json_payloads/product_group_payload.json" + "Group", resource_list, "../json_payloads/product_group_payload.json" ) payload_obj = json.loads(payload) @@ -436,7 +436,7 @@ def test_build_payload_group(self, mock_get_resource, mock_save_image): validate(payload_obj["entry"][0]["request"], request_schema) def test_extract_matches(self): - csv_file = "csv/organizations/organizations_locations.csv" + csv_file = "../csv/organizations/organizations_locations.csv" resource_list = read_csv(csv_file) resources = extract_matches(resource_list) expected_matches = { @@ -450,7 +450,7 @@ def test_extract_matches(self): self.assertEqual(resources, expected_matches) def test_build_org_affiliation(self): - csv_file = "csv/organizations/organizations_locations.csv" + csv_file = "../csv/organizations/organizations_locations.csv" resource_list = read_csv(csv_file) resources = extract_matches(resource_list) payload = build_org_affiliation(resources, resource_list) @@ -577,7 +577,7 @@ def test_uuid_generated_for_locations_is_unique_and_repeatable(self): ] payload = build_payload( - "locations", resources, "json_payloads/locations_payload.json" + "locations", resources, "../json_payloads/locations_payload.json" ) payload_obj = json.loads(payload) location1 = payload_obj["entry"][0]["resource"]["id"] @@ -628,7 +628,7 @@ def test_update_resource_with_no_id_fails(self): ] with self.assertRaises(ValueError) as raised_error: build_payload( - "locations", resources, "json_payloads/locations_payload.json" + "locations", resources, "../json_payloads/locations_payload.json" ) self.assertEqual( "The id is required to update a resource", str(raised_error.exception) @@ -654,7 +654,7 @@ def test_update_resource_with_non_existing_id_fails(self, mock_get_resource): ] with self.assertRaises(ValueError) as raised_error: build_payload( - "locations", resources, "json_payloads/locations_payload.json" + "locations", resources, "../json_payloads/locations_payload.json" ) self.assertEqual( "Trying to update a Non-existent resource", str(raised_error.exception) @@ -766,9 +766,7 @@ def test_build_assign_payload_update_assigned_org( "98199caa-4455-4b2f-a5cf-cb9c89b6bbdc", ] ] - payload = build_assign_payload( - resource_list, "PractitionerRole", "practitioner=Practitioner/" - ) + payload = build_assign_payload(resource_list, "PractitionerRole") payload_obj = json.loads(payload) self.assertIsInstance(payload_obj, dict) @@ -826,9 +824,7 @@ def test_build_assign_payload_create_org_assignment( "98199caa-4455-4b2f-a5cf-cb9c89b6bbdc", ] ] - payload = build_assign_payload( - resource_list, "PractitionerRole", "practitioner=Practitioner/" - ) + payload = build_assign_payload(resource_list, "PractitionerRole") payload_obj = json.loads(payload) self.assertIsInstance(payload_obj, dict) @@ -866,9 +862,7 @@ def test_build_assign_payload_create_new_practitioner_role( "98199caa-4455-4b2f-a5cf-cb9c89b6bbdc", ] ] - payload = build_assign_payload( - resource_list, "PractitionerRole", "practitioner=Practitioner/" - ) + payload = build_assign_payload(resource_list, "PractitionerRole") payload_obj = json.loads(payload) self.assertIsInstance(payload_obj, dict) @@ -890,165 +884,6 @@ def test_build_assign_payload_create_new_practitioner_role( payload_obj["entry"][0]["resource"]["organization"]["display"], "New Org" ) - @patch("main.handle_request") - @patch("main.get_base_url") - def test_build_assign_payload_create_new_link_location_to_inventory_list( - self, mock_get_base_url, mock_handle_request - ): - mock_get_base_url.return_value = "https://example.smartregister.org/fhir" - mock_response_data = {"resourceType": "Bundle", "total": 0} - string_response = json.dumps(mock_response_data) - mock_response = (string_response, 200) - mock_handle_request.return_value = mock_response - - resource_list = [ - [ - "Nairobi Inventory Items", - "e62a049f-8d48-456c-a387-f52e72c39c74", - "2024-06-01T10:40:10.111Z", - "3af23539-850a-44ed-8fb1-d4999e2145ff", - ] - ] - payload = build_assign_payload(resource_list, "List", "subject=List/") - payload_obj = json.loads(payload) - - self.assertIsInstance(payload_obj, dict) - self.assertEqual(payload_obj["resourceType"], "Bundle") - self.assertEqual(len(payload_obj["entry"]), 1) - - self.assertEqual( - payload_obj["entry"][0]["resource"]["title"], "Nairobi Inventory Items" - ) - self.assertEqual( - payload_obj["entry"][0]["resource"]["entry"][0]["item"]["reference"], - "Group/e62a049f-8d48-456c-a387-f52e72c39c74", - ) - self.assertEqual( - payload_obj["entry"][0]["resource"]["entry"][0]["date"], - "2024-06-01T10:40:10.111Z", - ) - self.assertEqual( - payload_obj["entry"][0]["resource"]["subject"]["reference"], - "Location/3af23539-850a-44ed-8fb1-d4999e2145ff", - ) - - @patch("main.handle_request") - @patch("main.get_base_url") - def test_build_assign_payload_update_location_with_new_inventory( - self, mock_get_base_url, mock_handle_request - ): - mock_get_base_url.return_value = "https://example.smartregister.org/fhir" - mock_response_data = { - "resourceType": "Bundle", - "total": 1, - "entry": [ - { - "resource": { - "resourceType": "List", - "id": "6d7d2e70-1c90-11db-861d-0242ac120002", - "meta": {"versionId": "2"}, - "subject": { - "reference": "Location/46bb8a3f-cf50-4cc2-b421-fe4f77c3e75d" - }, - "entry": [ - { - "item": { - "reference": "Group/f2734756-a6bb-4e90-bbc6-1c34f51d3d5c" - } - } - ], - } - } - ], - } - string_response = json.dumps(mock_response_data) - mock_response = (string_response, 200) - mock_handle_request.return_value = mock_response - - resource_list = [ - [ - "Nairobi Inventory Items", - "e62a049f-8d48-456c-a387-f52e72c39c74", - "2024-06-01T10:40:10.111Z", - "3af23539-850a-44ed-8fb1-d4999e2145ff", - ] - ] - - payload = build_assign_payload(resource_list, "List", "subject=List/") - payload_obj = json.loads(payload) - - self.assertIsInstance(payload_obj, dict) - self.assertEqual(payload_obj["resourceType"], "Bundle") - self.assertEqual(len(payload_obj["entry"]), 1) - - self.assertEqual( - payload_obj["entry"][0]["resource"]["entry"][0]["item"]["reference"], - "Group/f2734756-a6bb-4e90-bbc6-1c34f51d3d5c", - ) - self.assertEqual( - payload_obj["entry"][0]["resource"]["entry"][1]["item"]["reference"], - "Group/e62a049f-8d48-456c-a387-f52e72c39c74", - ) - - @patch("main.handle_request") - @patch("main.get_base_url") - def test_build_assign_payload_create_new_link_location_to_inventory_list_with_multiples( - self, mock_get_base_url, mock_handle_request - ): - mock_get_base_url.return_value = "https://example.smartregister.org/fhir" - mock_response_data = {"resourceType": "Bundle", "total": 0} - string_response = json.dumps(mock_response_data) - mock_response = (string_response, 200) - mock_handle_request.return_value = mock_response - - resource_list = [ - [ - "Nairobi Inventory Items", - "e62a049f-8d48-456c-a387-f52e72c39c74", - "2024-06-01T10:40:10.111Z", - "3af23539-850a-44ed-8fb1-d4999e2145ff", - ], - [ - "Nairobi Inventory Items", - "a36b595c-68a7-4244-91d5-c64be23b1ebd", - "2024-06-05T30:30:30.264Z", - "3af23539-850a-44ed-8fb1-d4999e2145ff", - ], - [ - "Mombasa Inventory Items", - "c0666a5a-00f6-488c-9001-8630560b5810", - "2024-06-06T55:23:19.492Z", - "3cd687a4-a169-45b3-a939-0418470c29db", - ], - ] - payload = build_assign_payload(resource_list, "List", "subject=List/") - payload_obj = json.loads(payload) - - self.assertIsInstance(payload_obj, dict) - self.assertEqual(payload_obj["resourceType"], "Bundle") - self.assertEqual(len(payload_obj["entry"]), 2) - self.assertEqual(len(payload_obj["entry"][0]["resource"]["entry"]), 2) - self.assertEqual(len(payload_obj["entry"][1]["resource"]["entry"]), 1) - - self.assertEqual( - payload_obj["entry"][0]["resource"]["title"], "Nairobi Inventory Items" - ) - self.assertEqual( - payload_obj["entry"][1]["resource"]["title"], "Mombasa Inventory Items" - ) - self.assertEqual( - payload_obj["entry"][0]["resource"]["entry"][0]["item"]["reference"], - "Group/e62a049f-8d48-456c-a387-f52e72c39c74", - ) - self.assertEqual( - payload_obj["entry"][0]["resource"]["entry"][1]["item"]["reference"], - "Group/a36b595c-68a7-4244-91d5-c64be23b1ebd", - ) - self.assertEqual( - payload_obj["entry"][1]["resource"]["entry"][0]["item"]["reference"], - "Group/c0666a5a-00f6-488c-9001-8630560b5810", - ) - @patch("main.logging") @patch("main.handle_request") @patch("main.get_keycloak_url") @@ -1475,7 +1310,7 @@ def test_split_chunk_sort_sync_first_chunk_less_than_size( mock_build_resource_type_map.assert_called_once_with(chunk_list, {}, 0) def test_build_resource_type_map(self): - json_file = "tests/json/sample.json" + json_file = "json/sample.json" mapping = read_file_in_chunks(json_file, 300, "sort") mapped_resources = { "Patient": [0], diff --git a/importer/utils/__init__.py b/importer/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/importer/utils/location_process.py b/importer/utils/location_process.py new file mode 100644 index 00000000..9d1e589f --- /dev/null +++ b/importer/utils/location_process.py @@ -0,0 +1,91 @@ +""" +preprocess a csv of location records, group them by their admin levels +""" + + +class LocationNode: + def __init__(self, raw_record=None, parent_node=None, children=None): + if children is None: + children = [] + self.parent = parent_node + self.children = children + self.raw_record = raw_record + + @property + def admin_level(self): + try: + return self.raw_record[8] + except: + return None + + @property + def location_id(self): + try: + return self.raw_record[3] + except: + return None + + def __repr__(self): + return f"" + + +def group_by_admin_level(csv_records): + location_node_store = {} + for record in csv_records: + location_id, parent_id = record[3], record[5] + this_record_node = location_node_store.get(location_id) + if this_record_node: + # assume tombstone + this_record_node.raw_record = record + else: + this_record_node = LocationNode(record) + location_node_store[location_id] = this_record_node + # see if this parentNode exists in the nodeStore + if parent_id: + this_location_parent_node = location_node_store.get(parent_id) + if this_location_parent_node is not None: + pass + else: + # create a tombstone + this_location_parent_node = LocationNode() + location_node_store[parent_id] = this_location_parent_node + this_location_parent_node.children.append(this_record_node) + this_record_node.parent = this_location_parent_node + return location_node_store + + +def get_node_children(parents): + children = [] + for node in parents: + children.extend(node.children) + return children + + +def get_next_admin_level(node_map_store: dict): + """generator function that yields the next group of admin level locations""" + # start by getting the parent locations. i.e. locations that do not have parent + parent_nodes = [] + for node in node_map_store.values(): + if node.parent is None and node.raw_record or node.parent and node.parent.raw_record is None: + parent_nodes.append(node) + yield parent_nodes + + fully_traversed = False + while not fully_traversed: + children_at_this_level = get_node_children(parent_nodes) + if len(children_at_this_level) == 0: + fully_traversed = True + else: + parent_nodes = children_at_this_level + yield children_at_this_level + + +def process_locations(csv_records): + nodes_map = group_by_admin_level(csv_records) + for batch in get_next_admin_level(nodes_map): + batch_raw_records = [] + for node in batch: + batch_raw_records.append(node.raw_record) + yield batch_raw_records + +# TODO - validate based on adminlevel and the generated groupings based on the parentIds \ No newline at end of file