From c7dac8127f567ab97c3a20ddb5910c9e777073f5 Mon Sep 17 00:00:00 2001 From: Sinisa Ivkovic Date: Tue, 17 May 2022 10:51:11 -0400 Subject: [PATCH 01/11] Save new refresh token --- beaglecli | 1 + 1 file changed, 1 insertion(+) diff --git a/beaglecli b/beaglecli index 69692fe..a675474 100755 --- a/beaglecli +++ b/beaglecli @@ -272,6 +272,7 @@ def _check_is_authenticated(config): response = requests.post(urljoin(BEAGLE_ENDPOINT, API['refresh']), {'refresh': config.refresh}) if response.status_code == 200: config.set('token', response.json()['access']) + config.set('refresh', response.json()['refresh']) return True return False From a6d40f6c438b09ec793364b91b12a63952d09ae7 Mon Sep 17 00:00:00 2001 From: buehlere Date: Fri, 5 Aug 2022 15:52:57 -0400 Subject: [PATCH 02/11] adding features for latest-info --- README.md | 2 +- beaglecli | 14 +++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 02e8e3b..206d719 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ To access other endpoints, export the environment variable `BEAGLE_ENDPOINT`. beaglecli file-group list [--page-size=] beaglecli etl delete --job-id=... beaglecli run list [--page-size=] [--request-id=]... [--tags=]... [--job-groups=]... [--jira-ids=]... - beaglecli run latest-info [--request-id=] [--job-group=] [--jira-id=] [--output-file=] [--completed] + beaglecli run latest-info [--request-id=] [--job-group=] [--jira-id=] [--output-file=] [--completed] [--page-size=] beaglecli run get beaglecli run submit-request --pipeline= [--request-ids=] [--job-group-id=] [--for-each=] beaglecli run submit-runs --pipelines=... --versions=...[--run-file=] [--run-ids=]... [--job-group-id=] [--for-each=] diff --git a/beaglecli b/beaglecli index de0e647..8d5b2fe 100755 --- a/beaglecli +++ b/beaglecli @@ -83,7 +83,7 @@ Usage: beaglecli file-group list [--page-size=] beaglecli etl delete --job-id=... beaglecli run list [--page-size=] [--request-id=]... [--tags=]... [--job-groups=]... [--jira-ids=]... - beaglecli run latest-info [--request-id=] [--job-group=] [--jira-id=] [--output-file=] [--completed] + beaglecli run latest-info [--request-id=] [--job-group=] [--jira-id=] [--output-file=] [--completed][--page-size=] beaglecli run get beaglecli run submit-request --pipeline= [--request-ids=] [--job-group-id=] [--for-each=] beaglecli run submit-runs --pipelines=... --versions=...[--run-file=] [--run-ids=]... [--job-group-id=] [--for-each=] @@ -353,20 +353,24 @@ def _get_latest_run_info_command(arguments, config): requestId = arguments.get('--request-id') completed = arguments.get('--completed') output_file = arguments.get('--output-file') + page_size = arguments.get('--page-size') info_keys = ['id', 'status', 'name', 'tags', 'message', 'app', - 'operator_run', 'created_date', 'finished_date', 'execution_id'] + 'operator_run', 'created_date', 'finished_date', 'execution_id', 'output_metadata'] file_keys = ['name', 'status', 'tags', 'message', 'id', 'execution_id'] params = dict() + params['page_size'] = 1000 if requestId: - params['requestId'] = requestId + params['request_ids'] = requestId if job_group: params['job_groups'] = job_group if jira_id: params['jira_id'] = jira_id - params['page_size'] = 1000000 + if page_size: + params['page_size'] = page_size params['full'] = True if completed: params['status'] = "COMPLETED" + # params['values_run'] = ['id', 'status', 'name', 'tags', 'message', # 'app', 'operator_run', 'created_date', 'finished_date'] url = urljoin(BEAGLE_ENDPOINT, API['run']) @@ -414,7 +418,7 @@ def _get_runs_command(arguments, config): params = dict() if requestId: - params['requestId'] = requestId + params['request_ids'] = requestId if tags: params['tags'] = tags if job_groups: From 493da601078383799bdc55eb2aa2027745975978 Mon Sep 17 00:00:00 2001 From: buehlere Date: Tue, 23 Aug 2022 15:35:07 -0400 Subject: [PATCH 03/11] adding more features :heavy_check_mark: adding option to return only meta-data :heavy_check_mark: adding option to return max number of pages for request id :heavy_check_mark: a little additional formatting --- README.md | 2 +- beaglecli | 22 +++++++++++++++++++--- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 206d719..9c19b75 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ To access other endpoints, export the environment variable `BEAGLE_ENDPOINT`. beaglecli file-group list [--page-size=] beaglecli etl delete --job-id=... beaglecli run list [--page-size=] [--request-id=]... [--tags=]... [--job-groups=]... [--jira-ids=]... - beaglecli run latest-info [--request-id=] [--job-group=] [--jira-id=] [--output-file=] [--completed] [--page-size=] + beaglecli run latest-info [--request-id=] [--job-group=] [--jira-id=] [--output-file=] [--completed] [--page-size=] [--metadata-only] [--max-pages] beaglecli run get beaglecli run submit-request --pipeline= [--request-ids=] [--job-group-id=] [--for-each=] beaglecli run submit-runs --pipelines=... --versions=...[--run-file=] [--run-ids=]... [--job-group-id=] [--for-each=] diff --git a/beaglecli b/beaglecli index 8d5b2fe..f8c54ad 100755 --- a/beaglecli +++ b/beaglecli @@ -83,7 +83,7 @@ Usage: beaglecli file-group list [--page-size=] beaglecli etl delete --job-id=... beaglecli run list [--page-size=] [--request-id=]... [--tags=]... [--job-groups=]... [--jira-ids=]... - beaglecli run latest-info [--request-id=] [--job-group=] [--jira-id=] [--output-file=] [--completed][--page-size=] + beaglecli run latest-info [--request-id=] [--job-group=] [--jira-id=] [--output-file=] [--completed][--page-size=] [--metadata-only] [--max-pages] beaglecli run get beaglecli run submit-request --pipeline= [--request-ids=] [--job-group-id=] [--for-each=] beaglecli run submit-runs --pipelines=... --versions=...[--run-file=] [--run-ids=]... [--job-group-id=] [--for-each=] @@ -348,17 +348,26 @@ def _get_latest_runs(run_dict): def _get_latest_run_info_command(arguments, config): + # getting params job_group = arguments.get('--job-group') jira_id = arguments.get('--jira-id') requestId = arguments.get('--request-id') completed = arguments.get('--completed') output_file = arguments.get('--output-file') page_size = arguments.get('--page-size') + metadata_only = arguments.get('--metadata-only') + max_pages = arguments.get('--max-pages') info_keys = ['id', 'status', 'name', 'tags', 'message', 'app', 'operator_run', 'created_date', 'finished_date', 'execution_id', 'output_metadata'] file_keys = ['name', 'status', 'tags', 'message', 'id', 'execution_id'] params = dict() + + # setting url + url = urljoin(BEAGLE_ENDPOINT, API['run']) + + # setting / adjusting parameters params['page_size'] = 1000 + params['full'] = True if requestId: params['request_ids'] = requestId if job_group: @@ -367,13 +376,15 @@ def _get_latest_run_info_command(arguments, config): params['jira_id'] = jira_id if page_size: params['page_size'] = page_size - params['full'] = True + if max_pages and not page_size: + count_params = {'request_ids': params['request_ids'], 'count':True} + params['page_size'] = requests.get(url, + headers={'Authorization': 'Bearer %s' % config.token}, params=count_params) if completed: params['status'] = "COMPLETED" # params['values_run'] = ['id', 'status', 'name', 'tags', 'message', # 'app', 'operator_run', 'created_date', 'finished_date'] - url = urljoin(BEAGLE_ENDPOINT, API['run']) response = requests.get(url, headers={'Authorization': 'Bearer %s' % config.token}, params=params) response_json = response.json() @@ -393,6 +404,10 @@ def _get_latest_run_info_command(arguments, config): latest_runs = [] for single_app in run_list: latest_runs += _get_latest_runs(run_list[single_app]) + # only return metadata + if metadata_only: + for idx, single_run in enumerate(latest_runs): + latest_runs[idx] = single_run['output_metadata'] response_json = json.dumps(latest_runs, indent=4) if output_file: output_str = "redact(y/n)\t" + "\t".join(file_keys) + "\n" @@ -406,6 +421,7 @@ def _get_latest_run_info_command(arguments, config): with open(output_file_path, "w") as output_file_obj: output_file_obj.write(output_str) response_json = "Done! Output location: " + str(output_file_path) + return response_json From 1969052cbf5698e5db4042770d83d354443d9ff6 Mon Sep 17 00:00:00 2001 From: buehlere Date: Wed, 24 Aug 2022 13:25:55 -0400 Subject: [PATCH 04/11] Update beaglecli :heavy_check_mark: adding support for multiple request ids --- beaglecli | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/beaglecli b/beaglecli index f8c54ad..349be0f 100755 --- a/beaglecli +++ b/beaglecli @@ -25,6 +25,7 @@ from os.path import expanduser from urllib.parse import urljoin from datetime import datetime import traceback +import csv from apps.access import access_commands from apps.cmoch import cmoch_commands @@ -368,6 +369,19 @@ def _get_latest_run_info_command(arguments, config): # setting / adjusting parameters params['page_size'] = 1000 params['full'] = True + # if a requestID is a txt file of multiple requests + if requestId[0].endswith(".txt"): + results = '' + # open csv + with open(requestId[0], newline='') as f: + reader = csv.reader(f, skipinitialspace=True) + # iterate over individual requests + for r in reader: + arguments['--request-id'] = [r[0]] + # recursive call for individual request + results = results + _get_latest_run_info_command(arguments, config)[1:-1] + ',' + # final format and return + return '[' + results[:-1] + ']' if requestId: params['request_ids'] = requestId if job_group: @@ -383,8 +397,6 @@ def _get_latest_run_info_command(arguments, config): if completed: params['status'] = "COMPLETED" - # params['values_run'] = ['id', 'status', 'name', 'tags', 'message', - # 'app', 'operator_run', 'created_date', 'finished_date'] response = requests.get(url, headers={'Authorization': 'Bearer %s' % config.token}, params=params) response_json = response.json() From 1e5503c6929b76e3a912b5680120f2ca16bedb13 Mon Sep 17 00:00:00 2001 From: buehlere Date: Wed, 24 Aug 2022 14:00:42 -0400 Subject: [PATCH 05/11] add multiple request ids for latest-info :heavy_check_mark: allow users to provide a csv file of request ids when getting the latest info from the run api :heavy_check_mark: add examples to README :heavy_check_mark: add example file for inputting multiple request ids --- README.md | 11 ++++++++++- beaglecli | 2 +- requests.txt | 3 +++ 3 files changed, 14 insertions(+), 2 deletions(-) create mode 100644 requests.txt diff --git a/README.md b/README.md index 9c19b75..1d4e3e6 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ To access other endpoints, export the environment variable `BEAGLE_ENDPOINT`. beaglecli file-group list [--page-size=] beaglecli etl delete --job-id=... beaglecli run list [--page-size=] [--request-id=]... [--tags=]... [--job-groups=]... [--jira-ids=]... - beaglecli run latest-info [--request-id=] [--job-group=] [--jira-id=] [--output-file=] [--completed] [--page-size=] [--metadata-only] [--max-pages] + beaglecli run latest-info [--request-id=] [--job-group=] [--jira-id=] [--output-file=] [--completed] [--page-size=] [--metadata-only] [--max-pages] beaglecli run get beaglecli run submit-request --pipeline= [--request-ids=] [--job-group-id=] [--for-each=] beaglecli run submit-runs --pipelines=... --versions=...[--run-file=] [--run-ids=]... [--job-group-id=] [--for-each=] @@ -65,6 +65,15 @@ To access other endpoints, export the environment variable `BEAGLE_ENDPOINT`. ``` beaglecli run submit-request --pipeline=argos --request-ids=ABCDE_1,ABCDE_2 --job-group-id=FGHIJK-LMNOP-QRSTUV-WXY --job-group-id=FGHIJK-LMNOP-QRSTUV-WXYZ ``` +- Return only the metadata info for a given request id from the run api + ``` + run latest-info --request-id 10101_A --completed --metadata-only --max-pages + ``` +- Return only the metadata info for multiple request ids + ``` + beaglecli run latest-info --request-id requests.txt --completed --metadata-only --max-pages + ``` +Note: Use `requests.txt` as a template for providing a multiple request ids #### Troubleshooting diff --git a/beaglecli b/beaglecli index 349be0f..47f2ae0 100755 --- a/beaglecli +++ b/beaglecli @@ -84,7 +84,7 @@ Usage: beaglecli file-group list [--page-size=] beaglecli etl delete --job-id=... beaglecli run list [--page-size=] [--request-id=]... [--tags=]... [--job-groups=]... [--jira-ids=]... - beaglecli run latest-info [--request-id=] [--job-group=] [--jira-id=] [--output-file=] [--completed][--page-size=] [--metadata-only] [--max-pages] + beaglecli run latest-info [--request-id= ] [--job-group=] [--jira-id=] [--output-file=] [--completed][--page-size=] [--metadata-only] [--max-pages] beaglecli run get beaglecli run submit-request --pipeline= [--request-ids=] [--job-group-id=] [--for-each=] beaglecli run submit-runs --pipelines=... --versions=...[--run-file=] [--run-ids=]... [--job-group-id=] [--for-each=] diff --git a/requests.txt b/requests.txt new file mode 100644 index 0000000..08ccb4a --- /dev/null +++ b/requests.txt @@ -0,0 +1,3 @@ +10101_A, +10101_B, +10101_C From acd83b906e7a87de6cbe03ae766b9f6c1caa7eb4 Mon Sep 17 00:00:00 2001 From: buehlere Date: Wed, 24 Aug 2022 14:49:04 -0400 Subject: [PATCH 06/11] edits to docs :heavy_check_mark: adding edits to documentation --- README.md | 8 ++++---- beaglecli | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 1d4e3e6..1d8afe2 100644 --- a/README.md +++ b/README.md @@ -65,13 +65,13 @@ To access other endpoints, export the environment variable `BEAGLE_ENDPOINT`. ``` beaglecli run submit-request --pipeline=argos --request-ids=ABCDE_1,ABCDE_2 --job-group-id=FGHIJK-LMNOP-QRSTUV-WXY --job-group-id=FGHIJK-LMNOP-QRSTUV-WXYZ ``` -- Return only the metadata info for a given request id from the run api +- Return only the output metadata info for a given request id from the run api ``` - run latest-info --request-id 10101_A --completed --metadata-only --max-pages + run latest-info --request-id 10101_A --completed --output-metadata-only --max-pages ``` -- Return only the metadata info for multiple request ids +- Return only the output metadata info for multiple request ids ``` - beaglecli run latest-info --request-id requests.txt --completed --metadata-only --max-pages + beaglecli run latest-info --request-id requests.txt --completed --output-metadata-only --max-pages ``` Note: Use `requests.txt` as a template for providing a multiple request ids diff --git a/beaglecli b/beaglecli index 47f2ae0..45bb7f6 100755 --- a/beaglecli +++ b/beaglecli @@ -84,7 +84,7 @@ Usage: beaglecli file-group list [--page-size=] beaglecli etl delete --job-id=... beaglecli run list [--page-size=] [--request-id=]... [--tags=]... [--job-groups=]... [--jira-ids=]... - beaglecli run latest-info [--request-id= ] [--job-group=] [--jira-id=] [--output-file=] [--completed][--page-size=] [--metadata-only] [--max-pages] + beaglecli run latest-info [--request-id= ] [--job-group=] [--jira-id=] [--output-file=] [--completed][--page-size=] [--output-metadata-only] [--max-pages] beaglecli run get beaglecli run submit-request --pipeline= [--request-ids=] [--job-group-id=] [--for-each=] beaglecli run submit-runs --pipelines=... --versions=...[--run-file=] [--run-ids=]... [--job-group-id=] [--for-each=] @@ -356,7 +356,7 @@ def _get_latest_run_info_command(arguments, config): completed = arguments.get('--completed') output_file = arguments.get('--output-file') page_size = arguments.get('--page-size') - metadata_only = arguments.get('--metadata-only') + metadata_only = arguments.get('--output-metadata-only') max_pages = arguments.get('--max-pages') info_keys = ['id', 'status', 'name', 'tags', 'message', 'app', 'operator_run', 'created_date', 'finished_date', 'execution_id', 'output_metadata'] From 9481aa99b4ef3a80c38816b10381880bb48e03ff Mon Sep 17 00:00:00 2001 From: Nikhil Kumar Date: Tue, 6 Sep 2022 16:48:31 -0400 Subject: [PATCH 07/11] Added a few fixes - treat request-id as optional input for latest run info = indicate that output_file will not be used when metadata_only is specified --- beaglecli | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/beaglecli b/beaglecli index b260942..fdebbb0 100755 --- a/beaglecli +++ b/beaglecli @@ -371,9 +371,9 @@ def _get_latest_run_info_command(arguments, config): params['page_size'] = 1000 params['full'] = True # if a requestID is a txt file of multiple requests - if requestId[0].endswith(".txt"): results = '' # open csv + if requestId and requestId[0].endswith(".txt"): with open(requestId[0], newline='') as f: reader = csv.reader(f, skipinitialspace=True) # iterate over individual requests @@ -422,7 +422,10 @@ def _get_latest_run_info_command(arguments, config): for idx, single_run in enumerate(latest_runs): latest_runs[idx] = single_run['output_metadata'] response_json = json.dumps(latest_runs, indent=4) - if output_file: + if metadata_only and output_file: + print("Not writing to " + str(output_file) + + " as metadata only has been specified") + elif output_file: output_str = "redact(y/n)\t" + "\t".join(file_keys) + "\n" for single_run in latest_runs: output_str += "n" From 2fce3e71c525d812c4bdd9982353cf38c9b43fa1 Mon Sep 17 00:00:00 2001 From: buehlere Date: Fri, 9 Sep 2022 11:05:12 -0400 Subject: [PATCH 08/11] Update beaglecli :heavy_check_mark: adding filetypes argument --- beaglecli | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/beaglecli b/beaglecli index fdebbb0..4edf28a 100755 --- a/beaglecli +++ b/beaglecli @@ -71,7 +71,7 @@ Usage: beaglecli files create [--metadata-path=] [--size=] beaglecli files update [--file-path=] [--file-type=] [--file-group=] [--metadata-path=] [--size=] beaglecli files patch [--file-path=] [--file-type=] [--file-group=] [--metadata=]... [--size=] - beaglecli files list [--page-size=] [--path=]... [--metadata=]... [--file-group=]... [--file-name=]... [--filename-regex=] + beaglecli files list [--page-size=] [--path=]... [--metadata=]... [--file-group=]... [--file-name=]... [--filename-regex=] [--file-type=]... beaglecli files delete --file-id=... beaglecli sample create beaglecli sample list [--sample-id=] @@ -371,7 +371,7 @@ def _get_latest_run_info_command(arguments, config): params['page_size'] = 1000 params['full'] = True # if a requestID is a txt file of multiple requests - results = '' + results = '' # open csv if requestId and requestId[0].endswith(".txt"): with open(requestId[0], newline='') as f: @@ -512,12 +512,14 @@ def _list_files(arguments, config): file_name = arguments.get('--file-name') filename_regex = arguments.get('--filename-regex') page_size = arguments.get('--page-size') + file_type = arguments.get('--file-type') params = dict() params['path'] = paths params['metadata'] = metadata params['file_group'] = file_group params['file_name'] = file_name params['filename_regex'] = filename_regex + params['file_type'] = file_type if page_size: params['page_size'] = page_size response = requests.get(urljoin(BEAGLE_ENDPOINT, API['files']), headers={ From e73616c9c37ade87599bdf545348bf7b4479c58d Mon Sep 17 00:00:00 2001 From: Nikhil Kumar Date: Tue, 13 Sep 2022 10:45:48 -0400 Subject: [PATCH 09/11] Added method to get runs by apps - Added a bug fix to show latest for runs of the same requests --- beaglecli | 144 ++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 107 insertions(+), 37 deletions(-) diff --git a/beaglecli b/beaglecli index fdebbb0..5a1cde6 100755 --- a/beaglecli +++ b/beaglecli @@ -14,6 +14,7 @@ $ beaglecli files list --metadata=igoRequestId:09324_C """ from cmath import sin +from ensurepip import version import os import sys from urllib import response @@ -25,7 +26,7 @@ from os.path import expanduser from urllib.parse import urljoin from datetime import datetime import traceback -import csv +import csv from apps.access import access_commands from apps.cmoch import cmoch_commands @@ -46,10 +47,10 @@ CONFIG_TEMPLATE = { API = { - "auth": "api-token-auth/", - "verify": "api-token-verify/", - "refresh": "api-token-refresh/", - "storage": "v0/fs/storage/", + "auth": 'api-token-auth/', + "verify": 'api-token-verify/', + "refresh": 'api-token-refresh/', + "storage": 'v0/fs/storage/', "file-types": 'v0/fs/file-types/', "pipelines": 'v0/run/pipelines/', "files": '/v0/fs/files/', @@ -83,8 +84,8 @@ Usage: beaglecli file-group create beaglecli file-group list [--page-size=] beaglecli etl delete --job-id=... - beaglecli run list [--page-size=] [--request-id=]... [--tags=]... [--job-groups=]... [--jira-ids=]... - beaglecli run latest-info [--request-id= ] [--job-group=] [--jira-id=] [--output-file=] [--completed][--page-size=] [--output-metadata-only] [--max-pages] + beaglecli run list [--page-size=] [--request-id=]... [--tags=]... [--apps=]... [--job-groups=]... [--jira-ids=]... + beaglecli run latest-info [--request-id= ] [--job-group=] [--apps=]... [--jira-id=] [--output-file=] [--completed][--page-size=] [--output-metadata-only] [--max-pages] beaglecli run get beaglecli run submit-request --pipeline= [--request-ids=] [--job-group-id=] [--for-each=] beaglecli run submit-runs --pipelines=... --versions=...[--run-file=] [--run-ids=]... [--job-group-id=] [--for-each=] @@ -349,10 +350,73 @@ def _get_latest_runs(run_dict): return run_list +def _get_apps_dict(): + url = urljoin(BEAGLE_ENDPOINT, API['pipelines']) + params = dict() + params['page_size'] = 1000000 + response = requests.get(url, headers={ + 'Authorization': 'Bearer %s' % config.token, 'Content-Type': 'application/json'}, params=params) + app_dict = {} + if response.ok: + response_json = response.json() + if "results" in response_json: + result_list = response_json["results"] + name_set = set() + duplicate_set = set() + for single_pipeline in result_list: + name = single_pipeline["name"] + if name in name_set: + duplicate_set.add(name) + name_set.add(name) + for single_pipeline in result_list: + name = single_pipeline["name"] + version = single_pipeline["version"] + id = single_pipeline["id"] + key_name = "" + if name in duplicate_set: + key_name = str(name) + ":"+str(version) + else: + key_name = str(name) + app_dict[key_name] = id + return app_dict + else: + print("Error: beagle returned an empty") + exit(1) + else: + print("ERROR: Could not retrieve app list") + exit(1) + + +def _get_app_uuid(app_names): + app_dict = _get_apps_dict() + keys = app_dict.keys() + if not keys: + print("Error: Could not retrieve pipeline info") + exit(1) + uuid_list = [] + for single_name in app_names: + if single_name not in keys: + matches = ( + single_key for single_key in keys if single_name in single_key) + print("Could not find the app " + str(single_name) + " in beagle") + if matches: + print("Here are possible matches:") + print(list(matches)) + exit(1) + else: + uuid_list.append(app_dict[single_name]) + return uuid_list + + +def _get_request_Id(run_data): + return run_data.get("tags", {}).get("igoRequestId", "None") + + def _get_latest_run_info_command(arguments, config): # getting params job_group = arguments.get('--job-group') jira_id = arguments.get('--jira-id') + apps = arguments.get('--apps') requestId = arguments.get('--request-id') completed = arguments.get('--completed') output_file = arguments.get('--output-file') @@ -363,41 +427,42 @@ def _get_latest_run_info_command(arguments, config): 'operator_run', 'created_date', 'finished_date', 'execution_id', 'output_metadata'] file_keys = ['name', 'status', 'tags', 'message', 'id', 'execution_id'] params = dict() - - # setting url + + # setting url url = urljoin(BEAGLE_ENDPOINT, API['run']) - # setting / adjusting parameters + # setting / adjusting parameters params['page_size'] = 1000 params['full'] = True - # if a requestID is a txt file of multiple requests - results = '' - # open csv - if requestId and requestId[0].endswith(".txt"): - with open(requestId[0], newline='') as f: - reader = csv.reader(f, skipinitialspace=True) - # iterate over individual requests - for r in reader: - arguments['--request-id'] = [r[0]] - # recursive call for individual request - results = results + _get_latest_run_info_command(arguments, config)[1:-1] + ',' - # final format and return - return '[' + results[:-1] + ']' + # open csv if requestId: - params['request_ids'] = requestId + if requestId[0].endswith(".txt"): + with open(requestId[0], newline='') as f: + reader = csv.reader(f, skipinitialspace=True) + requestIdList = [] + # iterate over individual requests + for r in reader: + if r: + requestIdList.append(r[0]) + params['request_ids'] = requestIdList + else: + params['request_ids'] = requestId + if apps: + uuid_list = _get_app_uuid(apps) + params['apps'] = uuid_list if job_group: params['job_groups'] = job_group if jira_id: params['jira_id'] = jira_id - if page_size: + if page_size: params['page_size'] = page_size if max_pages and not page_size: - count_params = {'request_ids': params['request_ids'], 'count':True} + count_params = {'request_ids': params['request_ids'], 'count': True} params['page_size'] = requests.get(url, - headers={'Authorization': 'Bearer %s' % config.token}, params=count_params) + headers={'Authorization': 'Bearer %s' % config.token}, params=count_params) if completed: params['status'] = "COMPLETED" - + response = requests.get(url, headers={'Authorization': 'Bearer %s' % config.token}, params=params) response_json = response.json() @@ -408,18 +473,20 @@ def _get_latest_run_info_command(arguments, config): if single_key in single_run: run_data[single_key] = single_run[single_key] app_name = run_data['app'] + request_id = _get_request_Id(run_data) operator_run = run_data['operator_run'] - if app_name not in run_list: - run_list[app_name] = {} - if operator_run not in run_list[app_name]: - run_list[app_name][operator_run] = [] - run_list[app_name][operator_run].append(run_data) + run_type = "{}:{}".format(app_name, request_id) + if run_type not in run_list: + run_list[run_type] = {} + if operator_run not in run_list[run_type]: + run_list[run_type][operator_run] = [] + run_list[run_type][operator_run].append(run_data) latest_runs = [] for single_app in run_list: latest_runs += _get_latest_runs(run_list[single_app]) - # only return metadata + # only return metadata if metadata_only: - for idx, single_run in enumerate(latest_runs): + for idx, single_run in enumerate(latest_runs): latest_runs[idx] = single_run['output_metadata'] response_json = json.dumps(latest_runs, indent=4) if metadata_only and output_file: @@ -437,7 +504,7 @@ def _get_latest_run_info_command(arguments, config): with open(output_file_path, "w") as output_file_obj: output_file_obj.write(output_str) response_json = "Done! Output location: " + str(output_file_path) - + return response_json @@ -447,10 +514,14 @@ def _get_runs_command(arguments, config): tags = arguments.get('--tags') job_groups = arguments.get('--job-groups') jira_ids = arguments.get('--jira-ids') + apps = arguments.get('--apps') params = dict() if requestId: params['request_ids'] = requestId + if apps: + uuid_list = _get_app_uuid(apps) + params['apps'] = uuid_list if tags: params['tags'] = tags if job_groups: @@ -851,7 +922,6 @@ def _submit_operator_runs(arguments, config): if for_each: body['for_each'] = for_each url = urljoin(BEAGLE_ENDPOINT, API['run-operator-runs']) - print(body) response = requests.post(url, data=json.dumps(body), headers={'Authorization': 'Bearer %s' % config.token, 'Content-Type': 'application/json'}) response_json = json.dumps(response.json(), indent=4) From c02638b06cc65d2a2eb4244db5b5eaea4e934e9e Mon Sep 17 00:00:00 2001 From: Nikhil Kumar Date: Tue, 13 Sep 2022 11:28:56 -0400 Subject: [PATCH 10/11] Updated version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b8b13f9..ad90de0 100755 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setuptools.setup( name='beaglecli', - version='0.2.0', + version='0.3.0', scripts=['beaglecli'] , description="Beagle API command line tool", url="https://github.com/mskcc/beagle_cli", From b4650a758302aab52fe1405ec4c040b2485b2da0 Mon Sep 17 00:00:00 2001 From: Nikhil Kumar Date: Tue, 13 Sep 2022 16:42:06 -0400 Subject: [PATCH 11/11] Update beaglecli --- beaglecli | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beaglecli b/beaglecli index 528f4b6..c37b282 100755 --- a/beaglecli +++ b/beaglecli @@ -84,8 +84,8 @@ Usage: beaglecli file-group create beaglecli file-group list [--page-size=] beaglecli etl delete --job-id=... - beaglecli run list [--page-size=] [--request-id=]... [--tags=]... [--apps=]... [--job-groups=]... [--jira-ids=]... - beaglecli run latest-info [--request-id= ] [--job-group=] [--apps=]... [--jira-id=] [--output-file=] [--completed][--page-size=] [--output-metadata-only] [--max-pages] + beaglecli run list [--page-size=] [--request-id=]... [--tags=]... [--apps="apps"]... [--job-groups=]... [--jira-ids=]... + beaglecli run latest-info [--request-id= ] [--job-group=] [--apps="apps"]... [--jira-id=] [--output-file=] [--completed][--page-size=] [--output-metadata-only] [--max-pages] beaglecli run get beaglecli run submit-request --pipeline= [--request-ids=] [--job-group-id=] [--for-each=] beaglecli run submit-runs --pipelines=... --versions=...[--run-file=] [--run-ids=]... [--job-group-id=] [--for-each=]