Merge branch 'master' into numpy-extension

dace/builtin_hooks.py

@@ -15,7 +15,12 @@
 
 
 @contextmanager
-def profile(repetitions: int = 100, warmup: int = 0):
+def profile(
+    repetitions: int = 100,
+    warmup: int = 0,
+    tqdm_leave: bool = True,
+    print_results: bool = True,
+):
     """
     Context manager that enables profiling of each called DaCe program. If repetitions is greater than 1, the
     program is run multiple times and the average execution time is reported.
@@ -35,6 +40,10 @@ def profile(repetitions: int = 100, warmup: int = 0):
 
     :param repetitions: The number of times to run each DaCe program.
     :param warmup: Number of additional repetitions to run the program without measuring time.
+    :param tqdm_leave: Sets the ``leave`` parameter of the ``tqdm`` progress bar (useful
+        for nested progress bars). Ignored if tqdm progress bar is not used.
+    :param print_results: Whether or not to print the median execution time after
+        all repetitions.
     :note: Running functions multiple times may affect the results of the program.
     """
     from dace.frontend.operations import CompiledSDFGProfiler  # Avoid circular import
@@ -51,7 +60,7 @@ def profile(repetitions: int = 100, warmup: int = 0):
             yield hook
             return
 
-    profiler = CompiledSDFGProfiler(repetitions, warmup)
+    profiler = CompiledSDFGProfiler(repetitions, warmup, tqdm_leave, print_results)
 
     with on_compiled_sdfg_call(context_manager=profiler):
         yield profiler

dace/codegen/targets/cpu.py

@@ -19,6 +19,7 @@
 from dace.sdfg import (ScopeSubgraphView, SDFG, scope_contains_scope, is_array_stream_view, NodeNotExpandedError,
                        dynamic_map_inputs, local_transients)
 from dace.sdfg.scope import is_devicelevel_gpu, is_devicelevel_fpga, is_in_scope
+from dace.sdfg.validation import validate_memlet_data
 from typing import Union
 from dace.codegen.targets import fpga
 
@@ -40,7 +41,7 @@ def _visit_structure(struct: data.Structure, args: dict, prefix: str = ''):
                     _visit_structure(v, args, f'{prefix}->{k}')
                 elif isinstance(v, data.ContainerArray):
                     _visit_structure(v.stype, args, f'{prefix}->{k}')
-                elif isinstance(v, data.Data):
+                if isinstance(v, data.Data):
                     args[f'{prefix}->{k}'] = v
 
         # Keeps track of generated connectors, so we know how to access them in nested scopes
@@ -620,6 +621,7 @@ def copy_memory(
             callsite_stream,
         )
 
+
     def _emit_copy(
         self,
         sdfg,
@@ -637,9 +639,9 @@ def _emit_copy(
         orig_vconn = vconn
 
         # Determine memlet directionality
-        if isinstance(src_node, nodes.AccessNode) and memlet.data == src_node.data:
+        if isinstance(src_node, nodes.AccessNode) and validate_memlet_data(memlet.data, src_node.data):
             write = True
-        elif isinstance(dst_node, nodes.AccessNode) and memlet.data == dst_node.data:
+        elif isinstance(dst_node, nodes.AccessNode) and validate_memlet_data(memlet.data, dst_node.data):
             write = False
         elif isinstance(src_node, nodes.CodeNode) and isinstance(dst_node, nodes.CodeNode):
             # Code->Code copy (not read nor write)

dace/codegen/targets/framecode.py

@@ -155,6 +155,8 @@ def generate_fileheader(self, sdfg: SDFG, global_stream: CodeIOStream, backend:
             if arr is not None:
                 datatypes.add(arr.dtype)
 
+        emitted = set()
+
         def _emit_definitions(dtype: dtypes.typeclass, wrote_something: bool) -> bool:
             if isinstance(dtype, dtypes.pointer):
                 wrote_something = _emit_definitions(dtype._typeclass, wrote_something)
@@ -164,7 +166,10 @@ def _emit_definitions(dtype: dtypes.typeclass, wrote_something: bool) -> bool:
             if hasattr(dtype, 'emit_definition'):
                 if not wrote_something:
                     global_stream.write("", sdfg)
-                global_stream.write(dtype.emit_definition(), sdfg)
+                if dtype not in emitted:
+                    global_stream.write(dtype.emit_definition(), sdfg)
+                    wrote_something = True
+                    emitted.add(dtype)
             return wrote_something
 
         # Emit unique definitions

dace/dtypes.py

@@ -1449,8 +1449,10 @@ def validate_name(name):
         return False
     if name in {'True', 'False', 'None'}:
         return False
-    if namere.match(name) is None:
-        return False
+    tokens = name.split('.')
+    for token in tokens:
+        if namere.match(token) is None:
+            return False
     return True
 
 

dace/frontend/operations.py

@@ -1,6 +1,7 @@
 # Copyright 2019-2021 ETH Zurich and the DaCe authors. All rights reserved.
 from __future__ import print_function
 from functools import partial
+from itertools import chain, repeat
 
 from contextlib import contextmanager
 from timeit import default_timer as timer
@@ -10,6 +11,7 @@
 import sympy
 import os
 import sys
+import warnings
 
 from dace import dtypes
 from dace.config import Config
@@ -28,12 +30,20 @@ class CompiledSDFGProfiler:
 
     times: List[Tuple['SDFG', List[float]]]  #: The list of SDFGs and times for each SDFG called within the context.
 
-    def __init__(self, repetitions: int = 0, warmup: int = 0) -> None:
+    def __init__(
+        self,
+        repetitions: int = 0,
+        warmup: int = 0,
+        tqdm_leave: bool = True,
+        print_results: bool = True,
+    ) -> None:
         # Avoid import loop
         from dace.codegen.instrumentation import report
 
         self.repetitions = repetitions or int(Config.get('treps'))
         self.warmup = warmup
+        self.tqdm_leave = tqdm_leave
+        self.print_results = print_results
         if self.repetitions < 1:
             raise ValueError('Number of repetitions must be at least 1')
         if self.warmup < 0:
@@ -47,34 +57,45 @@ def __init__(self, repetitions: int = 0, warmup: int = 0) -> None:
     def __call__(self, compiled_sdfg: 'CompiledSDFG', args: Tuple[Any, ...]):
         from dace.codegen.instrumentation import report  # Avoid import loop
 
-        start = timer()
+        # zeros to overwrite start time, followed by indices for each repetition
+        iterator = chain(repeat(0, self.warmup), range(1, self.repetitions + 1))
 
-        times = [start] * (self.repetitions + 1)
-        ret = None
-        print('\nProfiling...')
-
-        iterator = range(self.warmup + self.repetitions)
         if Config.get_bool('profiling_status'):
             try:
                 from tqdm import tqdm
-                iterator = tqdm(iterator, desc="Profiling", file=sys.stdout)
+
+                iterator = tqdm(
+                    iterator,
+                    desc='Profiling',
+                    total=(self.warmup + self.repetitions),
+                    file=sys.stdout,
+                    leave=self.tqdm_leave,
+                )
             except ImportError:
-                print('WARNING: Cannot show profiling progress, missing optional '
-                      'dependency tqdm...\n\tTo see a live progress bar please install '
-                      'tqdm (`pip install tqdm`)\n\tTo disable this feature (and '
-                      'this warning) set `profiling_status` to false in the dace '
-                      'config (~/.dace.conf).')
+                warnings.warn(
+                    'Cannot show profiling progress, missing optional dependency '
+                    'tqdm...\n\tTo see a live progress bar please install tqdm '
+                    '(`pip install tqdm`)\n\tTo disable this feature (and this '
+                    'warning) set `profiling_status` to false in the dace config '
+                    '(~/.dace.conf).'
+                )
+                print('\nProfiling...')
+        else:
+            print('\nProfiling...')
 
-        offset = 1 - self.warmup
         start_time = int(time.time())
+
+        times = np.ndarray(self.repetitions + 1, dtype=np.float64)
         times[0] = timer()
+
         for i in iterator:
             # Call function
             compiled_sdfg._cfunc(compiled_sdfg._libhandle, *args)
-            if i >= self.warmup:
-                times[i + offset] = timer()
 
-        diffs = np.array([(times[i] - times[i - 1])*1e3 for i in range(1, self.repetitions + 1)])
+            times[i] = timer()
+
+        # compute pairwise differences and convert to milliseconds
+        diffs = np.diff(times) * 1e3
 
         # Add entries to the instrumentation report
         self.report.name = self.report.name or start_time
@@ -88,8 +109,9 @@ def __call__(self, compiled_sdfg: 'CompiledSDFG', args: Tuple[Any, ...]):
         self.report.durations[(0, -1, -1)][f'Python call to {compiled_sdfg.sdfg.name}'][-1].extend(diffs)
 
         # Print profiling results
-        time_msecs = np.median(diffs)
-        print(compiled_sdfg.sdfg.name, time_msecs, 'ms')
+        if self.print_results:
+            time_msecs = np.median(diffs)
+            print(compiled_sdfg.sdfg.name, time_msecs, 'ms')
 
         # Save every call separately
         self.times.append((compiled_sdfg.sdfg, diffs))
@@ -105,7 +127,7 @@ def __call__(self, compiled_sdfg: 'CompiledSDFG', args: Tuple[Any, ...]):
         # Restore state after skipping contents
         compiled_sdfg.do_not_execute = old_dne
 
-        return ret
+        return None
 
 
 def detect_reduction_type(wcr_str, openmp=False):

dace/frontend/python/parser.py

@@ -21,6 +21,12 @@
 except ImportError:
     from typing_compat import get_origin, get_args
 
+try:
+    import mpi4py
+    from dace.sdfg.utils import distributed_compile
+except ImportError:
+    mpi4py = None
+
 ArgTypes = Dict[str, Data]
 
 
@@ -443,9 +449,12 @@ def __call__(self, *args, **kwargs):
                 sdfg.simplify()
 
         with hooks.invoke_sdfg_call_hooks(sdfg) as sdfg:
-            # Compile SDFG (note: this is done after symbol inference due to shape
-            # altering transformations such as Vectorization)
-            binaryobj = sdfg.compile(validate=self.validate)
+            if not mpi4py:
+                # Compile SDFG (note: this is done after symbol inference due to shape
+                # altering transformations such as Vectorization)
+                binaryobj = sdfg.compile(validate=self.validate)
+            else:
+                binaryobj = distributed_compile(sdfg, mpi4py.MPI.COMM_WORLD, validate=self.validate)
 
             # Recreate key and add to cache
             cachekey = self._cache.make_key(argtypes, specified, self.closure_array_keys, self.closure_constant_keys,

dace/properties.py

@@ -24,35 +24,6 @@
 ###############################################################################
 
 
-def set_property_from_string(prop, obj, string, sdfg=None, from_json=False):
-    """ Interface function that guarantees that a property will always be
-    correctly set, if possible, by accepting all possible input arguments to
-    from_string. """
-
-    # If the property is a string (property name), obtain it from the object
-    if isinstance(prop, str):
-        prop = type(obj).__properties__[prop]
-
-    if isinstance(prop, CodeProperty):
-        if from_json:
-            val = prop.from_json(string)
-        else:
-            val = prop.from_string(string, obj.language)
-    elif isinstance(prop, (ReferenceProperty, DataProperty)):
-        if sdfg is None:
-            raise ValueError("You cannot pass sdfg=None when editing a ReferenceProperty!")
-        if from_json:
-            val = prop.from_json(string, sdfg)
-        else:
-            val = prop.from_string(string, sdfg)
-    else:
-        if from_json:
-            val = prop.from_json(string, sdfg)
-        else:
-            val = prop.from_string(string)
-    setattr(obj, prop.attr_name, val)
-
-
 ###############################################################################
 # Property base implementation
 ###############################################################################
@@ -74,8 +45,6 @@ def __init__(
             setter=None,
             dtype: Type[T] = None,
             default=None,
-            from_string=None,
-            to_string=None,
             from_json=None,
             to_json=None,
             meta_to_json=None,
@@ -114,35 +83,8 @@ def __init__(
                 if not isinstance(choice, dtype):
                     raise TypeError("All choices must be an instance of dtype")
 
-        if from_string is not None:
-            self._from_string = from_string
-        elif choices is not None:
-            self._from_string = lambda s: choices[s]
-        else:
-            self._from_string = self.dtype
-
-        if to_string is not None:
-            self._to_string = to_string
-        elif choices is not None:
-            self._to_string = lambda val: val.__name__
-        else:
-            self._to_string = str
-
         if from_json is None:
-            if self._from_string is not None:
-
-                def fs(obj, *args, **kwargs):
-                    if isinstance(obj, str):
-                        # The serializer does not know about this property, so if
-                        # we can convert using our to_string method, do that here
-                        return self._from_string(obj)
-                    # Otherwise ship off to the serializer, telling it which type
-                    # it's dealing with as a sanity check
-                    return dace.serialize.from_json(obj, *args, known_type=dtype, **kwargs)
-
-                self._from_json = fs
-            else:
-                self._from_json = lambda *args, **kwargs: dace.serialize.from_json(*args, known_type=dtype, **kwargs)
+            self._from_json = lambda *args, **kwargs: dace.serialize.from_json(*args, known_type=dtype, **kwargs)
         else:
             self._from_json = from_json
             if self.from_json != from_json:
@@ -226,7 +168,6 @@ def __set__(self, obj, val):
         if (self.dtype is not None and not isinstance(val, self.dtype) and not (val is None and self.allow_none)):
             if isinstance(val, str):
                 raise TypeError("Received str for property {} of type {}. Use "
-                                "dace.properties.set_property_from_string or the "
                                 "from_string method of the property.".format(self.attr_name, self.dtype))
             raise TypeError("Invalid type \"{}\" for property {}: expected {}".format(
                 type(val).__name__, self.attr_name, self.dtype.__name__))
@@ -296,14 +237,6 @@ def allow_none(self):
     def desc(self):
         return self._desc
 
-    @property
-    def from_string(self):
-        return self._from_string
-
-    @property
-    def to_string(self):
-        return self._to_string
-
     @property
     def from_json(self):
         return self._from_json
@@ -853,8 +786,6 @@ def __init__(
             getter=None,
             setter=None,
             default=None,
-            from_string=None,
-            to_string=None,
             from_json=None,
             to_json=None,
             unmapped=False,  # Don't enforce 1:1 mapping with a member variable
@@ -867,8 +798,6 @@ def __init__(
                                           setter=setter,
                                           dtype=set,
                                           default=default,
-                                          from_string=from_string,
-                                          to_string=to_string,
                                           from_json=from_json,
                                           to_json=to_json,
                                           choices=None,

dace/sdfg/graph.py

@@ -526,10 +526,10 @@ def edges(self):
         return [DiGraph._from_nx(e) for e in self._nx.edges()]
 
     def in_edges(self, node):
-        return [DiGraph._from_nx(e) for e in self._nx.in_edges()]
+        return [DiGraph._from_nx(e) for e in self._nx.in_edges(node, True)]
 
     def out_edges(self, node):
-        return [DiGraph._from_nx(e) for e in self._nx.out_edges()]
+        return [DiGraph._from_nx(e) for e in self._nx.out_edges(node, True)]
 
     def add_node(self, node):
         return self._nx.add_node(node)