Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add progress bar #151

Merged
merged 1 commit into from
Feb 26, 2024
Merged

Add progress bar #151

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions importer/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@ and then posts them to the API for creation
3. Create a `config.py` file. The `sample_config.py` is an example of what this should look like. Populate it with the right credentials
4. Run script - `python3 main.py --csv_file csv/locations.csv --resource_type locations`
5. You can turn on logging by passing a `--log_level` to the command line as `info`, `debug` or `error`. For example `python3 main.py --csv_file csv/locations.csv --resource_type locations --log_level info`
6. There is a progress bar that shows the read_csv and build_payload progress as it is going on


See example csvs in the csv folder

Expand Down
136 changes: 70 additions & 66 deletions importer/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,9 @@ def read_csv(csv_file):
next(records)
all_records = []

for record in records:
all_records.append(record)
with click.progressbar(records, label='Progress::Reading csv ') as read_csv_progress:
for record in read_csv_progress:
all_records.append(record)

logging.info("Returning records from csv file")
return all_records
Expand Down Expand Up @@ -75,7 +76,7 @@ def get_access_token():
# to create resources
@backoff.on_exception(backoff.expo, requests.exceptions.RequestException, max_time=180)
def post_request(request_type, payload, url):
logging.info("Posting request-----------------")
logging.info("Posting request")
logging.info("Request type: " + request_type)
logging.info("Url: " + url)
logging.debug("Payload: " + payload)
Expand Down Expand Up @@ -514,63 +515,65 @@ def build_payload(resource_type, resources, resource_payload_file):
with open(resource_payload_file) as json_file:
payload_string = json_file.read()

for resource in resources:
name, status, method, id, *_ = resource
try:
if method == "create":
version = "1"
if len(id.strip()) > 0:
# use the provided id
unique_uuid = id.strip()
identifier_uuid = id.strip()
else:
# generate a new uuid
unique_uuid = str(uuid.uuid5(uuid.NAMESPACE_DNS, name))
identifier_uuid = unique_uuid
except IndexError:
# default if method is not provided
unique_uuid = str(uuid.uuid5(uuid.NAMESPACE_DNS, name))
identifier_uuid = unique_uuid
version = "1"

try:
if method == "update":
if len(id.strip()) > 0:
version = get_resource(id, resource_type)
if version != "0":
with click.progressbar(resources, label='Progress::Building payload ') as build_payload_progress:
for resource in build_payload_progress:
logging.info("\t")
name, status, method, id, *_ = resource
try:
if method == "create":
version = "1"
if len(id.strip()) > 0:
# use the provided id
unique_uuid = id.strip()
identifier_uuid = id.strip()
else:
logging.info("Failed to get resource!")
raise ValueError("Trying to update a Non-existent resource")
else:
logging.info("The id is required!")
raise ValueError("The id is required to update a resource")
except IndexError:
raise ValueError("The id is required to update a resource")

# ps = payload_string
ps = (
payload_string.replace("$name", name)
.replace("$unique_uuid", unique_uuid)
.replace("$identifier_uuid", identifier_uuid)
.replace("$version", version)
)
# generate a new uuid
unique_uuid = str(uuid.uuid5(uuid.NAMESPACE_DNS, name))
identifier_uuid = unique_uuid
except IndexError:
# default if method is not provided
unique_uuid = str(uuid.uuid5(uuid.NAMESPACE_DNS, name))
identifier_uuid = unique_uuid
version = "1"

try:
ps = ps.replace("$status", status)
except IndexError:
ps = ps.replace("$status", "active")
try:
if method == "update":
if len(id.strip()) > 0:
version = get_resource(id, resource_type)
if version != "0":
# use the provided id
unique_uuid = id.strip()
identifier_uuid = id.strip()
else:
logging.info("Failed to get resource!")
raise ValueError("Trying to update a Non-existent resource")
else:
logging.info("The id is required!")
raise ValueError("The id is required to update a resource")
except IndexError:
raise ValueError("The id is required to update a resource")

# ps = payload_string
ps = (
payload_string.replace("$name", name)
.replace("$unique_uuid", unique_uuid)
.replace("$identifier_uuid", identifier_uuid)
.replace("$version", version)
)

if resource_type == "organizations":
ps = organization_extras(resource, ps)
elif resource_type == "locations":
ps = location_extras(resource, ps)
elif resource_type == "careTeams":
ps = care_team_extras(resource, ps, "min", [], [], "orgs & users")
try:
ps = ps.replace("$status", status)
except IndexError:
ps = ps.replace("$status", "active")

if resource_type == "organizations":
ps = organization_extras(resource, ps)
elif resource_type == "locations":
ps = location_extras(resource, ps)
elif resource_type == "careTeams":
ps = care_team_extras(resource, ps, "min", [], [], "orgs & users")

final_string = final_string + ps + ","
final_string = final_string + ps + ","

final_string = initial_string + final_string[:-1] + " ] } "
return final_string
Expand Down Expand Up @@ -897,19 +900,20 @@ def main(
if resource_list:
if resource_type == "users":
logging.info("Processing users")
for user in resource_list:
user_id = create_user(user)
if user_id == 0:
# user was not created above, check if it already exists
user_id = confirm_keycloak_user(user)
if user_id != 0:
# user_id has been retrieved
# check practitioner
practitioner_exists = confirm_practitioner(user, user_id)
if not practitioner_exists:
payload = create_user_resources(user_id, user)
handle_request("POST", payload, config.fhir_base_url)
logging.info("Processing complete!")
with click.progressbar(resource_list, label="Progress:Processing users ") as process_user_progress:
for user in process_user_progress:
user_id = create_user(user)
if user_id == 0:
# user was not created above, check if it already exists
user_id = confirm_keycloak_user(user)
if user_id != 0:
# user_id has been retrieved
# check practitioner
practitioner_exists = confirm_practitioner(user, user_id)
if not practitioner_exists:
payload = create_user_resources(user_id, user)
handle_request("POST", payload, config.fhir_base_url)
logging.info("Processing complete!")
elif resource_type == "locations":
logging.info("Processing locations")
json_payload = build_payload(
Expand Down
Loading