Merge remote-tracking branch 'nipy/master'

Author: tsalo

.zenodo.json

@@ -38,6 +38,10 @@
         "name": "Clark, Dav",
         "orcid": "0000-0002-3982-4416"
       },
+      {
+        "affiliation": "Shattuck Lab, UCLA Brain Mapping Center",
+        "name": "Wong, Jason"
+      },
       {
         "affiliation": "Mayo Clinic, Neurology, Rochester, MN, USA",
         "name": "Dayan, Michael",
@@ -302,8 +306,8 @@
       },
       {
         "affiliation": "MIT",
-        "name": "Goncalves, Mathias", 
-      }
+        "name": "Goncalves, Mathias"
+      },
       {
         "affiliation": "MIT, HMS",
         "name": "Ghosh, Satrajit",

nipype/interfaces/utility/base.py

@@ -99,13 +99,29 @@ def _list_outputs(self):
 class MergeInputSpec(DynamicTraitedSpec, BaseInterfaceInputSpec):
     axis = traits.Enum('vstack', 'hstack', usedefault=True,
                        desc='direction in which to merge, hstack requires same number of elements in each input')
-    no_flatten = traits.Bool(False, usedefault=True, desc='append to outlist instead of extending in vstack mode')
+    no_flatten = traits.Bool(False, usedefault=True,
+                             desc='append to outlist instead of extending in vstack mode')
+    ravel_inputs = traits.Bool(False, usedefault=True,
+                               desc='ravel inputs when no_flatten is False')
 
 
 class MergeOutputSpec(TraitedSpec):
     out = traits.List(desc='Merged output')
 
 
+def _ravel(in_val):
+    if not isinstance(in_val, list):
+        return in_val
+    flat_list = []
+    for val in in_val:
+        raveled_val = _ravel(val)
+        if isinstance(raveled_val, list):
+            flat_list.extend(raveled_val)
+        else:
+            flat_list.append(raveled_val)
+    return flat_list
+
+
 class Merge(IOBase):
     """Basic interface class to merge inputs into a single list
 
@@ -123,23 +139,34 @@ class Merge(IOBase):
     >>> out.outputs.out
     [1, 2, 5, 3]
 
-    >>> merge = Merge()   # Or Merge(1)
-    >>> merge.inputs.in_lists = [1, [2, 5], 3]
+    >>> merge = Merge(1)
+    >>> merge.inputs.in1 = [1, [2, 5], 3]
+    >>> out = merge.run()
+    >>> out.outputs.out
+    [1, [2, 5], 3]
+
+    >>> merge = Merge(1)
+    >>> merge.inputs.in1 = [1, [2, 5], 3]
+    >>> merge.inputs.ravel_inputs = True
     >>> out = merge.run()
     >>> out.outputs.out
     [1, 2, 5, 3]
 
+    >>> merge = Merge(1)
+    >>> merge.inputs.in1 = [1, [2, 5], 3]
+    >>> merge.inputs.no_flatten = True
+    >>> out = merge.run()
+    >>> out.outputs.out
+    [[1, [2, 5], 3]]
     """
     input_spec = MergeInputSpec
     output_spec = MergeOutputSpec
 
-    def __init__(self, numinputs=1, **inputs):
+    def __init__(self, numinputs=0, **inputs):
         super(Merge, self).__init__(**inputs)
         self._numinputs = numinputs
-        if numinputs > 1:
+        if numinputs >= 1:
             input_names = ['in%d' % (i + 1) for i in range(numinputs)]
-        elif numinputs == 1:
-            input_names = ['in_lists']
         else:
             input_names = []
         add_traits(self.inputs, input_names)
@@ -150,8 +177,6 @@ def _list_outputs(self):
 
         if self._numinputs < 1:
             return outputs
-        elif self._numinputs == 1:
-            values = self.inputs.in_lists
         else:
             getval = lambda idx: getattr(self.inputs, 'in%d' % (idx + 1))
             values = [getval(idx) for idx in range(self._numinputs)
@@ -160,7 +185,8 @@ def _list_outputs(self):
         if self.inputs.axis == 'vstack':
             for value in values:
                 if isinstance(value, list) and not self.inputs.no_flatten:
-                    out.extend(value)
+                    out.extend(_ravel(value) if self.inputs.ravel_inputs else
+                               value)
                 else:
                     out.append(value)
         else:

nipype/interfaces/utility/tests/test_base.py

@@ -56,25 +56,20 @@ def test_split(tmpdir, args, expected):
         ([3], {}, [0, [1, 2], [3, 4, 5]], [0, 1, 2, 3, 4, 5]),
         ([0], {}, None, None),
         ([], {}, [], []),
-        ([], {}, [0, [1, 2], [3, 4, 5]], [0, 1, 2, 3, 4, 5]),
+        ([], {}, [0, [1, 2], [3, 4, 5]], [0, [1, 2], [3, 4, 5]]),
         ([3], {'axis': 'hstack'}, [[0], [1, 2], [3, 4, 5]], [[0, 1, 3]]),
         ([3], {'axis': 'hstack'}, [[0, 1], [2, 3], [4, 5]],
          [[0, 2, 4], [1, 3, 5]]),
         ([3], {'axis': 'hstack'}, [[0, 1], [2, 3], [4, 5]],
          [[0, 2, 4], [1, 3, 5]]),
-        ([1], {'axis': 'hstack'}, [[0], [1, 2], [3, 4, 5]], [[0, 1, 3]]),
-        ([1], {'axis': 'hstack'}, [[0, 1], [2, 3], [4, 5]],
-         [[0, 2, 4], [1, 3, 5]]),
         ])
 def test_merge(tmpdir, args, kwargs, in_lists, expected):
     os.chdir(str(tmpdir))
 
     node = pe.Node(utility.Merge(*args, **kwargs), name='merge')
 
-    numinputs = args[0] if args else 1
-    if numinputs == 1:
-        node.inputs.in_lists = in_lists
-    elif numinputs > 1:
+    numinputs = args[0] if args else 0
+    if numinputs >= 1:
         for i in range(1, numinputs + 1):
             setattr(node.inputs, 'in{:d}'.format(i), in_lists[i - 1])
 

nipype/utils/filemanip.py

@@ -12,6 +12,7 @@
 
 import sys
 import pickle
+import subprocess
 import gzip
 import hashlib
 from hashlib import md5
@@ -237,6 +238,54 @@ def hash_timestamp(afile):
     return md5hex
 
 
+def _generate_cifs_table():
+    """Construct a reverse-length-ordered list of mount points that
+    fall under a CIFS mount.
+
+    This precomputation allows efficient checking for whether a given path
+    would be on a CIFS filesystem.
+
+    On systems without a ``mount`` command, or with no CIFS mounts, returns an
+    empty list.
+    """
+    exit_code, output = subprocess.getstatusoutput("mount")
+    # Not POSIX
+    if exit_code != 0:
+        return []
+
+    # (path, fstype) tuples, sorted by path length (longest first)
+    mount_info = sorted((line.split()[2:5:2] for line in output.splitlines()),
+                        key=lambda x: len(x[0]),
+                        reverse=True)
+    cifs_paths = [path for path, fstype in mount_info if fstype == 'cifs']
+
+    return [mount for mount in mount_info
+            if any(mount[0].startswith(path) for path in cifs_paths)]
+
+
+_cifs_table = _generate_cifs_table()
+
+
+def on_cifs(fname):
+    """ Checks whether a file path is on a CIFS filesystem mounted in a POSIX
+    host (i.e., has the ``mount`` command).
+
+    On Windows, Docker mounts host directories into containers through CIFS
+    shares, which has support for Minshall+French symlinks, or text files that
+    the CIFS driver exposes to the OS as symlinks.
+    We have found that under concurrent access to the filesystem, this feature
+    can result in failures to create or read recently-created symlinks,
+    leading to inconsistent behavior and ``FileNotFoundError``s.
+
+    This check is written to support disabling symlinks on CIFS shares.
+    """
+    # Only the first match (most recent parent) counts
+    for fspath, fstype in _cifs_table:
+        if fname.startswith(fspath):
+            return fstype == 'cifs'
+    return False
+
+
 def copyfile(originalfile, newfile, copy=False, create_new=False,
              hashmethod=None, use_hardlink=False,
              copy_related_files=True):
@@ -288,6 +337,10 @@ def copyfile(originalfile, newfile, copy=False, create_new=False,
     if hashmethod is None:
         hashmethod = config.get('execution', 'hash_method').lower()
 
+    # Don't try creating symlinks on CIFS
+    if copy is False and on_cifs(newfile):
+        copy = True
+
     # Existing file
     # -------------
     # Options:

nipype/utils/tests/test_filemanip.py

@@ -15,6 +15,7 @@
 from ...utils.filemanip import (save_json, load_json,
                                 fname_presuffix, fnames_presuffix,
                                 hash_rename, check_forhash,
+                                _cifs_table, on_cifs,
                                 copyfile, copyfiles,
                                 filename_to_list, list_to_filename,
                                 check_depends,
@@ -334,3 +335,28 @@ def test_related_files(file, length, expected_files):
     for ef in expected_files:
         assert ef in related_files
 
+
+def test_cifs_check():
+    assert isinstance(_cifs_table, list)
+    assert isinstance(on_cifs('/'), bool)
+    fake_table = [('/scratch/tmp', 'ext4'), ('/scratch', 'cifs')]
+    cifs_targets = [('/scratch/tmp/x/y', False),
+                    ('/scratch/tmp/x', False),
+                    ('/scratch/x/y', True),
+                    ('/scratch/x', True),
+                    ('/x/y', False),
+                    ('/x', False),
+                    ('/', False)]
+
+    orig_table = _cifs_table[:]
+    _cifs_table[:] = []
+
+    for target, _ in cifs_targets:
+        assert on_cifs(target) is False
+
+    _cifs_table.extend(fake_table)
+    for target, expected in cifs_targets:
+        assert on_cifs(target) is expected
+
+    _cifs_table[:] = []
+    _cifs_table.extend(orig_table)