From 196ae3855a9c25766da74f2f1b6c8ba1e2fc4952 Mon Sep 17 00:00:00 2001 From: SooLee Date: Tue, 4 Feb 2020 15:40:57 +0000 Subject: [PATCH 1/2] step function handling lambda service error --- tibanna/ec2_utils.py | 6 +++--- tibanna/stepfunction.py | 12 ++++++++++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/tibanna/ec2_utils.py b/tibanna/ec2_utils.py index df01e7b3c..be0d5c5cc 100644 --- a/tibanna/ec2_utils.py +++ b/tibanna/ec2_utils.py @@ -626,9 +626,9 @@ def upload_run_json(self, runjson): except Exception as e: raise Exception("boto3 client error: Failed to connect to s3 : %s" % str(e)) try: - res = s3.put_object(Body=jsonbody.encode('utf-8'), Bucket=self.cfg.json_bucket, Key=jsonkey) - except Exception: - raise Exception("boto3 client error: Failed to upload run.json %s to s3: %s" % (jsonkey, str(res))) + s3.put_object(Body=jsonbody.encode('utf-8'), Bucket=self.cfg.json_bucket, Key=jsonkey) + except Exception as e: + raise Exception("boto3 client error: Failed to upload run.json %s to s3: %s" % (jsonkey, str(e))) def create_userdata(self, profile=None): """Create a userdata script to pass to the instance. The userdata script is run_workflow.$JOBID.sh. diff --git a/tibanna/stepfunction.py b/tibanna/stepfunction.py index 29f1242fb..0428553d5 100644 --- a/tibanna/stepfunction.py +++ b/tibanna/stepfunction.py @@ -15,6 +15,12 @@ class StepFunctionUnicorn(object): "IntervalSeconds": 600, "MaxAttempts": 1008, # 1 wk "BackoffRate": 1.0 + }, + { + "ErrorEquals": [ "Lambda.ServiceException", "Lambda.AWSLambdaException", "Lambda.SdkClientException"], + "IntervalSeconds": 60, + "MaxAttempts": 6, + "BackoffRate": 2 } ] sfn_check_task_retry_conditions = [ @@ -29,6 +35,12 @@ class StepFunctionUnicorn(object): "IntervalSeconds": 300, "MaxAttempts": 100000, "BackoffRate": 1.0 + }, + { + "ErrorEquals": [ "Lambda.ServiceException", "Lambda.AWSLambdaException", "Lambda.SdkClientException"], + "IntervalSeconds": 60, + "MaxAttempts": 6, + "BackoffRate": 2 } ] From a45d750122dd982c02c4f979cd20ec582dd2fc43 Mon Sep 17 00:00:00 2001 From: SooLee Date: Tue, 4 Feb 2020 15:48:13 +0000 Subject: [PATCH 2/2] updated version number --- tibanna/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tibanna/_version.py b/tibanna/_version.py index 6160ca5ef..e6454ea4a 100644 --- a/tibanna/_version.py +++ b/tibanna/_version.py @@ -1,4 +1,4 @@ """Version information.""" # The following line *must* be the last in the module, exactly as formatted: -__version__ = "0.12.1" +__version__ = "0.13.1"