Merge pull request #265 from 4dn-dcic/lambda_err

step function handling lambda service error

tibanna/_version.py

@@ -1,4 +1,4 @@
 """Version information."""
 
 # The following line *must* be the last in the module, exactly as formatted:
-__version__ = "0.12.1"
+__version__ = "0.13.1"

tibanna/ec2_utils.py

@@ -626,9 +626,9 @@ def upload_run_json(self, runjson):
         except Exception as e:
             raise Exception("boto3 client error: Failed to connect to s3 : %s" % str(e))
         try:
-            res = s3.put_object(Body=jsonbody.encode('utf-8'), Bucket=self.cfg.json_bucket, Key=jsonkey)
-        except Exception:
-            raise Exception("boto3 client error: Failed to upload run.json %s to s3: %s" % (jsonkey, str(res)))
+            s3.put_object(Body=jsonbody.encode('utf-8'), Bucket=self.cfg.json_bucket, Key=jsonkey)
+        except Exception as e:
+            raise Exception("boto3 client error: Failed to upload run.json %s to s3: %s" % (jsonkey, str(e)))
 
     def create_userdata(self, profile=None):
         """Create a userdata script to pass to the instance. The userdata script is run_workflow.$JOBID.sh.

tibanna/stepfunction.py

@@ -15,6 +15,12 @@ class StepFunctionUnicorn(object):
             "IntervalSeconds": 600,
             "MaxAttempts": 1008,  # 1 wk
             "BackoffRate": 1.0
+        },
+        {
+            "ErrorEquals": [ "Lambda.ServiceException", "Lambda.AWSLambdaException", "Lambda.SdkClientException"],
+            "IntervalSeconds": 60,
+            "MaxAttempts": 6,
+            "BackoffRate": 2
         }
     ]
     sfn_check_task_retry_conditions = [
@@ -29,6 +35,12 @@ class StepFunctionUnicorn(object):
             "IntervalSeconds": 300,
             "MaxAttempts": 100000,
             "BackoffRate": 1.0
+        },
+        {
+            "ErrorEquals": [ "Lambda.ServiceException", "Lambda.AWSLambdaException", "Lambda.SdkClientException"],
+            "IntervalSeconds": 60,
+            "MaxAttempts": 6,
+            "BackoffRate": 2
         }
     ]