Merge pull request #111 from 4dn-dcic/iam2

stepfunction permission issue fixed

core/iam_utils.py

@@ -93,8 +93,27 @@ def generate_policy_iam_passrole_s3(account_id, tibanna_policy_prefix):
     return policy_iam_passrole_s3
 
 
+def generate_lambdainvoke_policy(account_id, region, tibanna_policy_prefix):
+    function_arn_prefix = 'arn:aws:lambda:' + region + ':' + account_id + ':function/'
+    resource = [function_arn_prefix + 'run_task_awsem' + '_' + tibanna_policy_prefix,
+                function_arn_prefix + 'check_task_awsem' + '_' + tibanna_policy_prefix]
+    policy = {
+        "Version": "2012-10-17",
+        "Statement": [
+            {
+                "Effect": "Allow",
+                "Action": [
+                    "lambda:InvokeFunction"
+                ],
+                "Resource": resource
+            }
+        ]
+    }
+    return policy
+
+
 def generate_assume_role_policy_document(service):
-    '''service: 'ec2' or 'lambda' '''
+    '''service: 'ec2', 'lambda' or 'states' '''
     AssumeRolePolicyDocument = {
         "Version": "2012-10-17",
         "Statement": [
@@ -118,6 +137,10 @@ def get_lambda_role_name(tibanna_policy_prefix, lambda_name):
     return tibanna_policy_prefix + '_' + lambda_name
 
 
+def get_stepfunction_role_name(tibanna_policy_prefix):
+    return tibanna_policy_prefix + '_states'
+
+
 def create_empty_role_for_lambda(iam, verbose=False):
     client = iam.meta.client
     role_policy_doc_lambda = generate_assume_role_policy_document('lambda')
@@ -217,6 +240,26 @@ def create_role_for_check_task_awsem(iam, tibanna_policy_prefix, account_id,
         print(response)
 
 
+def create_role_for_stepfunction(iam, tibanna_policy_prefix, account_id,
+                                 lambdainvoke_policy_name, verbose=False):
+    client = iam.meta.client
+    stepfunction_role_name = get_stepfunction_role_name(tibanna_policy_prefix)
+    role_policy_doc = generate_assume_role_policy_document('states')
+    response = client.create_role(
+        RoleName=stepfunction_role_name,
+        AssumeRolePolicyDocument=json.dumps(role_policy_doc)
+    )
+    if verbose:
+        print(response)
+    role_stepfunction = iam.Role(stepfunction_role_name)
+    response = role_stepfunction.attach_policy(
+        PolicyArn='arn:aws:iam::aws:policy/service-role/AWSLambdaRole'
+        # PolicyArn='arn:aws:iam::' + account_id + ':policy/' + lambdainvoke_policy_name
+    )
+    if verbose:
+        print(response)
+
+
 def create_user_group(iam, group_name, bucket_policy_name, account_id, verbose=False):
     client = iam.meta.client
     response = client.create_group(
@@ -247,7 +290,7 @@ def create_user_group(iam, group_name, bucket_policy_name, account_id, verbose=F
         print(response)
 
 
-def create_tibanna_iam(account_id, bucket_names, user_group_name, verbose=False):
+def create_tibanna_iam(account_id, bucket_names, user_group_name, region, verbose=False):
     """creates IAM policies and roles and a user group for tibanna
     returns prefix of all IAM policies, roles and group.
     Total 4 policies, 3 roles and 1 group is generated that is associated with a single user group
@@ -290,16 +333,29 @@ def create_tibanna_iam(account_id, bucket_names, user_group_name, verbose=False)
         PolicyName=passrole_policy_name,
         PolicyDocument=json.dumps(policy_iam_ps3),
     )
+    if verbose:
+        print(response)
+    # lambdainvoke policy for step function
+    lambdainvoke_policy_name = tibanna_policy_prefix + '_lambdainvoke'
+    policy_lambdainvoke = generate_lambdainvoke_policy(account_id, region, tibanna_policy_prefix)
+    response = client.create_policy(
+        PolicyName=lambdainvoke_policy_name,
+        PolicyDocument=json.dumps(policy_lambdainvoke),
+    )
     if verbose:
         print(response)
     # roles
+    # role for bucket
     create_role_for_bucket(iam, tibanna_policy_prefix, account_id, bucket_policy_name)
+    # role for lambda
     create_role_for_run_task_awsem(iam, tibanna_policy_prefix, account_id,
                                    cloudwatch_policy_name, bucket_policy_name,
                                    list_policy_name, passrole_policy_name)
     create_role_for_check_task_awsem(iam, tibanna_policy_prefix, account_id,
                                      cloudwatch_policy_name, bucket_policy_name)
     create_empty_role_for_lambda(iam)
+    # role for step function
+    create_role_for_stepfunction(iam, tibanna_policy_prefix, account_id, lambdainvoke_policy_name)
     # instance profile
     instance_profile_name = get_bucket_role_name(tibanna_policy_prefix)
     client.create_instance_profile(

core/utils.py

@@ -11,6 +11,7 @@
     generate_rand_accession
 )
 from dcicutils.s3_utils import s3Utils
+from core.iam_utils import get_stepfunction_role_name
 import logging
 import traceback
 
@@ -353,14 +354,25 @@ def run_workflow(input_json, accession='', workflow='tibanna_pony',
 def create_stepfunction(dev_suffix=None,
                         sfn_type='pony',  # vs 'unicorn'
                         region_name=AWS_REGION,
-                        aws_acc=AWS_ACCOUNT_NUMBER):
-    if dev_suffix:
-        lambda_suffix = '_' + dev_suffix
+                        aws_acc=AWS_ACCOUNT_NUMBER,
+                        usergroup=None):
+    if usergroup:
+        if dev_suffix:
+            lambda_suffix = '_' + usergroup + '_' + dev_suffix
+        else:
+            lambda_suffix = '_' + usergroup
     else:
-        lambda_suffix = ''
+        if dev_suffix:
+            lambda_suffix = '_' + dev_suffix
+        else:
+            lambda_suffix = ''
     sfn_name = 'tibanna_' + sfn_type + lambda_suffix
     lambda_arn_prefix = "arn:aws:lambda:" + region_name + ":" + aws_acc + ":function:"
-    sfn_role_arn = "arn:aws:iam::" + aws_acc + ":role/service-role/StatesExecutionRole-" + region_name
+    if sfn_type == 'pony':  # 4dn
+        sfn_role_arn = "arn:aws:iam::" + aws_acc + ":role/service-role/StatesExecutionRole-" + region_name
+    else:
+        sfn_role_arn = "arn:aws:iam::" + aws_acc + ":role/" + \
+            get_stepfunction_role_name('tibanna_' + usergroup)
     sfn_check_task_retry_conditions = [
         {
             "ErrorEquals": ["EC2StartingException"],

tasks.py

@@ -547,7 +547,8 @@ def setup_tibanna_env(ctx, buckets='', usergroup_tag='default'):
     '''The very first function to run as admin to set up environment on AWS'''
     print("setting up tibanna environment on AWS...")
     bucket_names = buckets.split(',')
-    tibanna_policy_prefix = create_tibanna_iam(AWS_ACCOUNT_NUMBER, bucket_names, usergroup_tag)
+    tibanna_policy_prefix = create_tibanna_iam(AWS_ACCOUNT_NUMBER, bucket_names,
+                                               usergroup_tag, AWS_REGION)
     tibanna_usergroup = tibanna_policy_prefix.replace("tibanna_", "")
     print("Tibanna usergroup %s has been created on AWS." % tibanna_usergroup)
 
@@ -557,14 +558,7 @@ def deploy_tibanna(ctx, suffix=None, sfn_type='pony', usergroup=None, version=No
     print("creating a new workflow..")
     if sfn_type not in ['pony', 'unicorn']:
         raise Exception("Invalid sfn_type : it must be either pony or unicorn.")
-    if usergroup:
-        if suffix:
-            sfn_suffix = usergroup + suffix
-        else:
-            sfn_suffix = usergroup
-    else:
-        sfn_suffix = suffix
-    res = _create_stepfunction(sfn_suffix, sfn_type)
+    res = _create_stepfunction(suffix, sfn_type, usergroup=usergroup)
     print(res)
     print("deploying lambdas..")
     if sfn_type == 'pony':

test_json/unicorn/my_test_tibanna_bucket2.json

@@ -1,8 +1,4 @@
 {
-  "_tibanna": {
-    "env": "fourfront-webdev",
-    "run_type": "md5"
-  },
   "args": {
     "app_name": "md5",
     "input_parameters": {},
@@ -14,12 +10,12 @@
     "secondary_output_target": {},
     "cwl_main_filename": "md5.cwl",
     "secondary_files": {},
-    "output_S3_bucket": "soo2-tibanna-log",
+    "output_S3_bucket": "soo2-tibanna-data",
     "app_version": "0.0.4",
     "cwl_directory_url": "https://raw.githubusercontent.com/4dn-dcic/pipelines-cwl/0.0.4/cwl_awsem/",
     "input_files": {
       "input_file": {
-        "bucket_name": "my-tibanna-test-input-bucket",
+        "bucket_name": "soo2-tibanna-data",
         "object_key": "somefastqfile.fastq.gz"
       }
     }

test_json/unicorn/my_test_tibanna_bucket3.json

@@ -0,0 +1,37 @@
+{
+  "args": {
+    "app_name": "md5",
+    "input_parameters": {},
+    "cwl_child_filenames": [],
+    "cwl_version": "draft-3",
+    "output_target": {
+      "report": "my_outdir/report"
+    },
+    "secondary_output_target": {},
+    "cwl_main_filename": "md5.cwl",
+    "secondary_files": {},
+    "output_S3_bucket": "4dntest-tibanna-data",
+    "app_version": "0.0.4",
+    "cwl_directory_url": "https://raw.githubusercontent.com/4dn-dcic/pipelines-cwl/0.0.4/cwl_awsem/",
+    "input_files": {
+      "input_file": {
+        "bucket_name": "4dntest-tibanna-data",
+        "object_key": "somefastqfile.fastq.gz"
+      }
+    }
+  },
+  "config": {
+    "ebs_size": 0,
+    "ebs_type": "io1",
+    "json_bucket": "4dntest-tibanna-log",
+    "EBS_optimized": "",
+    "ebs_iops": 500,
+    "shutdown_min": 120,
+    "instance_type": "",
+    "copy_to_s3": true,
+    "launch_instance": true,
+    "password": "dragonfly",
+    "log_bucket": "4dntest-tibanna-log",
+    "key_name": ""
+  }
+}