Add full custom metadata handling

.travis.yml

@@ -33,7 +33,7 @@ install:
 
 script:
     - ./test/test.sh
-    - ./test/lint.sh reaper
+    - ./test/lint.sh
 
 after_success:
     - ./docker/build-trigger.sh Tag "${TRAVIS_TAG}" "${BUILD_TRIGGER_URL}"

reaper/dcm.py

@@ -1,9 +1,11 @@
 """SciTran Reaper DICOM utility functions"""
 
-import os
+import datetime
 import hashlib
 import logging
-import datetime
+import os
+import shlex
+import subprocess
 
 import dicom
 
@@ -16,7 +18,15 @@
 GEMS_TYPE_VXTL = ['DERIVED', 'SECONDARY', 'VXTL STATE']
 
 
-def pkg_series(_id, path, map_key, opt_key=None, anonymize=False, timezone=None):
+def __external_metadata(command, filepath):
+    try:
+        return subprocess.check_output(shlex.split(command), filepath)
+    except subprocess.CalledProcessError as ex:
+        log.error('Error running external command. Exit %d', ex.returncode)
+        return None
+
+
+def pkg_series(_id, path, map_key, opt_key=None, anonymize=False, timezone=None, additional_metadata=None):
     # pylint: disable=missing-docstring
     dcm_dict = {}
     log.info('inspecting   %s', _id)
@@ -39,7 +49,15 @@ def pkg_series(_id, path, map_key, opt_key=None, anonymize=False, timezone=None)
             os.utime(filepath, (file_time, file_time))  # correct timestamps
             os.rename(filepath, '%s.dcm' % os.path.join(arcdir_path, filename))
         arc_path = util.create_archive(arcdir_path, dir_name)
-        metadata = util.object_metadata(dcm, timezone, os.path.basename(arc_path))
+        for md_group_info in (additional_metadata or {}).itervalues():
+            for md_field, md_value in md_group_info.iteritems():
+                if md_value.startswith('^'):    # DICOM header value
+                    md_group_info[md_field] = dcm.raw_header.get(md_value[1:], None)
+                elif md_value.startswith('@'):  # external command
+                    md_group_info[md_field] = __external_metadata(md_value[1:], filepath)
+                else:                           # verbatim value
+                    md_group_info[md_field] = md_value[1:]
+        metadata = util.object_metadata(dcm, timezone, os.path.basename(arc_path), additional_metadata)
         util.set_archive_metadata(arc_path, metadata)
         metadata_map[arc_path] = metadata
     return metadata_map
@@ -57,6 +75,7 @@ def __init__(self, filepath, map_key, opt_key=None, parse=False, anonymize=False
         if not parse and anonymize:
             raise Exception('Cannot anonymize DICOM file without parsing')
         dcm = dicom.read_file(filepath, stop_before_pixels=(not anonymize))
+        self.raw_header = dcm
         self._id = dcm.get(map_key, '')
         self.opt = dcm.get(opt_key, '') if opt_key else None
         self.acq_no = str(dcm.get('AcquisitionNumber', '')) or None if dcm.get('Manufacturer').upper() != 'SIEMENS' else None

reaper/dicom_reaper.py

@@ -3,8 +3,6 @@
 import os
 import logging
 import datetime
-import re
-from subprocess import Popen, PIPE, STDOUT
 
 from . import dcm
 from . import scu
@@ -21,7 +19,8 @@ def __init__(self, options):
         self.scu = scu.SCU(options.get('host'), options.get('port'), options.get('return_port'), options.get('aet'), options.get('aec'))
         super(DicomReaper, self).__init__(self.scu.aec, options)
         self.anonymize = options.get('anonymize')
-        self.uid_ext_command = options.get('uid_ext_command')
+        self.additional_metadata = {group: {field: value} for group, field, value in options.get('metadata')}
+        print self.additional_metadata
 
         self.query_tags = {self.map_key: ''}
         if self.opt_key is not None:
@@ -76,33 +75,11 @@ def reap(self, _id, item, tempdir):
                 log.info('ignoring     %s (non-matching opt-%s)', _id, self.opt)
                 return None, {}
         if success and reap_cnt == item['state']['images']:
-            metadata_map = dcm.pkg_series(_id, reapdir, self.map_key, self.opt_key, self.anonymize, self.timezone)
-            self._custom_acq_uid(metadata_map)
+            metadata_map = dcm.pkg_series(_id, reapdir, self.map_key, self.opt_key, self.anonymize, self.timezone, self.additional_metadata)
             return True, metadata_map
         else:
             return False, {}
 
-    def _custom_acq_uid(self, metadata_map):
-        if self.uid_ext_command is None:
-            return
-        for filepath, metadata in metadata_map.iteritems():
-            dcm_dir = re.sub(r'\.zip$', '', filepath)
-
-            unzipped_file = [os.path.join(dcm_dir, filename) for filename in os.listdir(dcm_dir)][0]
-
-            formatted_string = self.uid_ext_command.format(unzipped_file)
-            arg_list = formatted_string.split()
-
-            proc = Popen(arg_list,
-                         stdout=PIPE,
-                         stderr=STDOUT)
-            out, _ = proc.communicate()
-
-            if proc.returncode and proc.returncode != 0:
-                log.error('Error with command. Return code = %d', proc.returncode)
-                raise RuntimeError(out)
-            metadata['acquisition']['uid'] = out.rstrip()
-
 
 def update_arg_parser(ap):
     # pylint: disable=missing-docstring
@@ -113,10 +90,7 @@ def update_arg_parser(ap):
     ap.add_argument('aec', help='remote AE title')
 
     ap.add_argument('-A', '--no-anonymize', dest='anonymize', action='store_false', help='do not anonymize patient name and birthdate')
-    ap.add_argument('--uid-ext-command',
-                    dest='uid_ext_command',
-                    help='Command to execute against single source file to generate acquisition uid',
-                    default=None)
+    ap.add_argument('--metadata', nargs=3, default=[], action='append', help='Additional metadata to package')
 
     return ap
 

reaper/util.py

@@ -61,7 +61,7 @@ def hrsize(size):
     return '%.0f%sB' % (size, 'Y')
 
 
-def object_metadata(obj, timezone, filename):
+def object_metadata(obj, timezone, filename, additional_metadata=None):
     # pylint: disable=missing-docstring
     metadata = {
         'session': {'timezone': timezone},
@@ -75,6 +75,10 @@ def object_metadata(obj, timezone, filename):
     metadata['file']['name'] = filename
     metadata['session']['subject'] = metadata.pop('subject', {})
     metadata['acquisition']['files'] = [metadata.pop('file', {})]
+    for md_group, md_group_info in additional_metadata.iteritems():
+        metadata.setdefault(md_group, {})
+        metadata[md_group].setdefault('metadata', {})
+        metadata[md_group]['metadata'].update(md_group_info)
     return metadata
 
 

test/lint.sh

@@ -6,9 +6,9 @@ unset CDPATH
 cd "$( dirname "${BASH_SOURCE[0]}" )/.."
 
 echo "Running pylint ..."
-pylint --reports=no "$@"
+pylint --reports=no reaper
 
 echo
 
 echo "Running pep8 ..."
-pep8 --max-line-length=150 --ignore=E402 "$@"
+pep8 --max-line-length=150 --ignore=E402 reaper