Skip to content

Commit

Permalink
Merge pull request #66 from 4dn-dcic/ffmeta_for_qc
Browse files Browse the repository at this point in the history 
Ffmeta for qc
  • Loading branch information
@SooLee
SooLee authored Jan 17, 2018
2 parents 4bca65a + be44252 commit 199c077
Show file tree
Hide file tree
Showing 16 changed files with 646 additions and 167 deletions.
15 changes: 12 additions & 3 deletions core/ec2_utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -300,12 +300,13 @@ def update_config(config, app_name, input_files, parameters):

class WorkflowFile(object):

def __init__(self, bucket, key, runner, accession=None):
def __init__(self, bucket, key, runner, accession=None, output_type=None):
self.bucket = bucket
self.key = key
self.s3 = utils.s3Utils(self.bucket, self.bucket, self.bucket)
self.runner = runner
self.accession = accession
self.output_type = output_type

@property
def status(self):
Expand All @@ -328,11 +329,19 @@ def __init__(self, json):
self.config = json['config']
self.output_s3 = self.args['output_S3_bucket']
self.app_name = self.args['app_name']
self.output_files_meta = json['ff_meta']['output_files']

def output_files(self):
files = dict()
output_types = dict()
for x in self.output_files_meta:
output_types[x['workflow_argument_name']] = x['type']
for k, v in self.args.get('output_target').iteritems():
wff = {k: WorkflowFile(self.output_s3, v, self)}
if k in output_types:
out_type = output_types[k]
else:
out_type = None
wff = {k: WorkflowFile(self.output_s3, v, self, output_type=out_type)}
files.update(wff)
return files

Expand All @@ -357,4 +366,4 @@ def input_files(self):

@property
def inputfile_accessions(self):
return [v.accession for k, v in self.input_files().iteritems()]
return {k: v.accession for k, v in self.input_files().iteritems()}
6 changes: 5 additions & 1 deletion core/fastqc_utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,11 @@ def parse_qc_table(data_list, url, qc_schema):
if a[0] in qc_schema and qc_schema.get(a[0]).get('type') == 'string':
qc_json.update({a[0]: str(a[1])})
elif a[0] in qc_schema and qc_schema.get(a[0]).get('type') == 'number':
qc_json.update({a[0]: number(a[1])})
qc_json.update({a[0]: number(a[1].replace(',', ''))})
if a[1] in qc_schema and qc_schema.get(a[1]).get('type') == 'string':
qc_json.update({a[1]: str(a[0])})
elif a[1] in qc_schema and qc_schema.get(a[1]).get('type') == 'number':
qc_json.update({a[1]: number(a[0].replace(',', ''))})
except IndexError: # pragma: no cover
# maybe a blank line or something
pass
Expand Down
6 changes: 3 additions & 3 deletions core/run_task_awsf/event.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,13 +24,13 @@
}
},
"config": {
"ebs_size": 10,
"ebs_size": 0,
"ebs_type": "io1",
"json_bucket": "4dn-aws-pipeline-run-json",
"EBS_optimized": false,
"EBS_optimized": "",
"ebs_iops": 500,
"shutdown_min": 120,
"instance_type": "t2.micro",
"instance_type": "",
"s3_access_arn": "arn:aws:iam::643366669028:instance-profile/S3_access",
"ami_id": "ami-cfb14bb5",
"copy_to_s3": true,
Expand Down
98 changes: 98 additions & 0 deletions core/update_ffmeta_awsf/event_fastqc.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
{
"ff_meta": {
"run_platform": "AWSEM",
"uuid": "18564368-1299-4ce0-82d1-c8917450d834",
"parameters": [],
"workflow": "2324ad76-ff37-4157-8bcc-3ce72b7dace9",
"title": "fastqc-0-11-4-1 hahaha run 2018-01-16 23:08:21.335210",
"award": "1U01CA200059-01",
"awsem_job_id": "",
"run_url": "https://console.aws.amazon.com/states/home?region=us-east-1#/executions/details/arn:aws:states:us-east-1:643366669028:execution:run_awsem_new_pony-dev:fastqc_613a0edc-f130-49a0-b97c-b812a6a42d84",
"awsem_app_name": "fastqc-0-11-4-1",
"lab": "4dn-dcic-lab",
"run_status": "started",
"output_files": [
{
"type": "Output QC file",
"workflow_argument_name": "report_zip"
}
],
"input_files": [
{
"ordinal": 1,
"workflow_argument_name": "input_fastq",
"value": "f4864029-a8ad-4bb8-93e7-5108f462ccaa"
}
]
},
"instance_ip": "54.164.70.164",
"app_name": "fastqc-0-11-4-1",
"_tibanna": {
"env": "fourfront-webdev",
"settings": {
"url": "https://console.aws.amazon.com/states/home?region=us-east-1#/executions/details/arn:aws:states:us-east-1:643366669028:execution:run_awsem_new_pony-dev:fastqc_613a0edc-f130-49a0-b97c-b812a6a42d84",
"run_type": "fastqc",
"run_id": "613a0edc-f130-49a0-b97c-b812a6a42d84",
"env": "fourfront-webdev",
"run_name": "fastqc_613a0edc-f130-49a0-b97c-b812a6a42d84"
}
},
"start_time": "20180116-23:08:29-UTC",
"args": {
"secondary_output_target": {},
"app_name": "fastqc-0-11-4-1",
"input_parameters": {},
"cwl_child_filenames": [
"fastqc-0-11-4.cwl"
],
"output_target": {
"report_zip": "18564368-1299-4ce0-82d1-c8917450d834/report_zip"
},
"cwl_main_filename": "fastqc-0-11-4-1.cwl",
"secondary_files": {},
"output_S3_bucket": "elasticbeanstalk-fourfront-webdev-wfoutput",
"app_version": "0.2.0",
"cwl_directory_url": "https://raw.githubusercontent.com/4dn-dcic/pipelines-cwl/0.2.0/cwl_awsem/",
"input_files": {
"input_fastq": {
"bucket_name": "elasticbeanstalk-fourfront-webdev-files",
"object_key": "f4864029-a8ad-4bb8-93e7-5108f462ccaa/4DNFIRSRJH45.fastq.gz"
}
}
},
"output_bucket": "elasticbeanstalk-fourfront-webdev-wfoutput",
"jobid": "eUJZ9nH7Rsz2",
"instance_id": "i-04df1c1ca7a06ef66",
"tag": "hahaha",
"pf_meta": [],
"parameters": {},
"config": {
"ebs_size": 10,
"password": "",
"ebs_type": "io1",
"json_bucket": "4dn-aws-pipeline-run-json",
"json_dir": "/tmp/json",
"EBS_optimized": false,
"ebs_iops": 500,
"userdata_dir": "/tmp/userdata",
"shutdown_min": "300",
"instance_type": "t2.micro",
"s3_access_arn": "arn:aws:iam::643366669028:instance-profile/S3_access",
"job_tag": "fastqc-0-11-4-1",
"copy_to_s3": true,
"script_url": "https://raw.githubusercontent.com/4dn-dcic/tibanna/master/awsf/",
"key_name": "4dn-encode",
"launch_instance": true,
"ami_id": "ami-cfb14bb5",
"log_bucket": "tibanna-output"
},
"workflow_uuid": "2324ad76-ff37-4157-8bcc-3ce72b7dace9",
"input_files": [
{
"workflow_argument_name": "input_fastq",
"bucket_name": "elasticbeanstalk-fourfront-webdev-files",
"uuid": "f4864029-a8ad-4bb8-93e7-5108f462ccaa",
"object_key": "4DNFIRSRJH45.fastq.gz"
}
]
}
130 changes: 130 additions & 0 deletions core/update_ffmeta_awsf/event_pairsqc.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
{
"ff_meta": {
"run_platform": "AWSEM",
"uuid": "3d71f45f-6a77-47b4-9839-eb02204c6dae",
"parameters": [
{
"workflow_argument_name": "enzyme",
"value": "6"
},
{
"workflow_argument_name": "sample_name",
"value": "4DNFI1ZLO9D7"
}
],
"workflow": "ae3a87cb-3fa2-469e-97c7-540fc2d0a117",
"title": "pairsqc-single run 2018-01-16 23:08:25.250148",
"award": "1U01CA200059-01",
"awsem_job_id": "",
"run_url": "https://console.aws.amazon.com/states/home?region=us-east-1#/executions/details/arn:aws:states:us-east-1:643366669028:execution:run_awsem_new_pony-dev:pairsqc-single_649e1149-68b5-43b3-bc92-65ca077dc983",
"awsem_app_name": "pairsqc-single",
"lab": "4dn-dcic-lab",
"run_status": "started",
"output_files": [
{
"type": "Output QC file",
"workflow_argument_name": "report"
}
],
"input_files": [
{
"ordinal": 1,
"workflow_argument_name": "input_pairs",
"value": "817f3faa-0573-45c0-8230-02ec19de6544"
},
{
"ordinal": 1,
"workflow_argument_name": "chromsizes",
"value": "4a6d10ee-2edb-4402-a98f-0edb1d58f5e9"
}
]
},
"instance_ip": "52.87.161.131",
"app_name": "pairsqc-single",
"_tibanna": {
"env": "fourfront-webdev",
"settings": {
"url": "https://console.aws.amazon.com/states/home?region=us-east-1#/executions/details/arn:aws:states:us-east-1:643366669028:execution:run_awsem_new_pony-dev:pairsqc-single_649e1149-68b5-43b3-bc92-65ca077dc983",
"run_type": "pairsqc-single",
"run_id": "649e1149-68b5-43b3-bc92-65ca077dc983",
"env": "fourfront-webdev",
"run_name": "pairsqc-single_649e1149-68b5-43b3-bc92-65ca077dc983"
}
},
"start_time": "20180116-23:08:32-UTC",
"args": {
"secondary_output_target": {},
"app_name": "pairsqc-single",
"input_parameters": {
"enzyme": "6",
"sample_name": "4DNFI1ZLO9D7"
},
"cwl_child_filenames": [],
"output_target": {
"report": "3d71f45f-6a77-47b4-9839-eb02204c6dae/report"
},
"cwl_main_filename": "pairsqc-single.cwl",
"secondary_files": {
"input_pairs": {
"bucket_name": "elasticbeanstalk-fourfront-webdev-wfoutput",
"object_key": "817f3faa-0573-45c0-8230-02ec19de6544/4DNFI1ZLO9D7.pairs.gz.px2"
}
},
"output_S3_bucket": "elasticbeanstalk-fourfront-webdev-wfoutput",
"app_version": "dev",
"cwl_directory_url": "https://raw.githubusercontent.com/4dn-dcic/pipelines-cwl/dev/cwl_awsem/",
"input_files": {
"chromsizes": {
"bucket_name": "elasticbeanstalk-fourfront-webdev-files",
"object_key": "4a6d10ee-2edb-4402-a98f-0edb1d58f5e9/4DNFI823LSII.chrom.sizes"
},
"input_pairs": {
"bucket_name": "elasticbeanstalk-fourfront-webdev-wfoutput",
"object_key": "817f3faa-0573-45c0-8230-02ec19de6544/4DNFI1ZLO9D7.pairs.gz"
}
}
},
"output_bucket": "elasticbeanstalk-fourfront-webdev-wfoutput",
"jobid": "11Jz5nf2xJGt",
"instance_id": "i-0901c30deadd480e4",
"pf_meta": [],
"parameters": {
"enzyme": "6",
"sample_name": "4DNFI1ZLO9D7"
},
"config": {
"ebs_size": 13,
"password": "dragonfly",
"ebs_type": "io1",
"json_bucket": "4dn-aws-pipeline-run-json",
"json_dir": "/tmp/json",
"EBS_optimized": false,
"ebs_iops": 500,
"userdata_dir": "/tmp/userdata",
"shutdown_min": 120,
"instance_type": "t2.micro",
"s3_access_arn": "arn:aws:iam::643366669028:instance-profile/S3_access",
"job_tag": "pairsqc-single",
"copy_to_s3": true,
"script_url": "https://raw.githubusercontent.com/4dn-dcic/tibanna/master/awsf/",
"key_name": "4dn-encode",
"launch_instance": true,
"ami_id": "ami-cfb14bb5",
"log_bucket": "tibanna-output"
},
"workflow_uuid": "ae3a87cb-3fa2-469e-97c7-540fc2d0a117",
"input_files": [
{
"workflow_argument_name": "input_pairs",
"bucket_name": "elasticbeanstalk-fourfront-webdev-wfoutput",
"uuid": "817f3faa-0573-45c0-8230-02ec19de6544",
"object_key": "4DNFI1ZLO9D7.pairs.gz"
},
{
"object_key": "4DNFI823LSII.chrom.sizes",
"bucket_name": "elasticbeanstalk-fourfront-webdev-files",
"workflow_argument_name": "chromsizes",
"uuid": "4a6d10ee-2edb-4402-a98f-0edb1d58f5e9"
}
]
}
Loading

0 comments on commit 199c077

Please sign in to comment.