Merge remote-tracking branch 'upstream/master' into fix/configparser

* upstream/master:
  Closes #2026
  fix: use new link with fsaverage files for mris_expand
  Fix 3dUnifize outputs.
  updated freesurfer URL and added ANTS as well
  [Docker] Install minimized version of FS6
  [ENH] Revise CircleCI settings
  FIX: ?h.white is actually created in the -pial step
  FIX: config can't be passed directly to Node
  TEST: Test attribute copying during MapNode expansion
  FIX: Pass parameters to all MapNode-generated Nodes
  update specs
  [ENH] Enable num_threads for antsAffineInitializer

Author: satra

circle.yml

@@ -27,22 +27,35 @@ dependencies:
     - mkdir -p $WORKDIR && sudo setfacl -d -m group:ubuntu:rwx $WORKDIR && sudo setfacl -m group:ubuntu:rwx $WORKDIR
     - mkdir -p $HOME/docker $HOME/examples $WORKDIR/tests $WORKDIR/logs $WORKDIR/crashfiles ${CIRCLE_TEST_REPORTS}/tests/
     - if [[ ! -e "$HOME/bin/codecov" ]]; then mkdir -p $HOME/bin; curl -so $HOME/bin/codecov https://codecov.io/bash && chmod 755 $HOME/bin/codecov; fi
-  override:
-    - if [[ -e "$HOME/docker/cache.tar" ]]; then docker load --input $HOME/docker/cache.tar; fi :
+    - docker load --input $HOME/docker/cache.tar || true :
         timeout: 6000
-    - docker images
-    - docker pull nipype/base:latest
+  override:
+    # Get data
     - if [[ ! -d ~/examples/nipype-tutorial ]]; then wget --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q -O nipype-tutorial.tar.bz2 "${DATA_NIPYPE_TUTORIAL_URL}" && tar xjf nipype-tutorial.tar.bz2 -C ~/examples/; fi
     - if [[ ! -d ~/examples/nipype-fsl_course_data ]]; then wget --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q -O nipype-fsl_course_data.tar.gz "${DATA_NIPYPE_FSL_COURSE}" && tar xzf nipype-fsl_course_data.tar.gz -C ~/examples/; fi
     - if [[ ! -d ~/examples/feeds ]]; then wget --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q -O fsl-5.0.9-feeds.tar.gz "${DATA_NIPYPE_FSL_FEEDS}" && tar xzf fsl-5.0.9-feeds.tar.gz -C ~/examples/; fi
     - if [ "$CIRCLE_TAG" != "" ]; then sed -i -E "s/(__version__ = )'[A-Za-z0-9.-]+'/\1'$CIRCLE_TAG'/" nipype/info.py; fi
-#     - e=1 && for i in {1..5}; do docker build -f docker/base.Dockerfile --rm=false -t nipype/base:latest . && e=0 && break || sleep 15; done && [ "$e" -eq "0" ] :
-#         timeout: 21600
-    - e=1 && for i in {1..5}; do docker build --rm=false -t nipype/nipype:latest -t nipype/nipype:py36 --build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` --build-arg VCS_REF=`git rev-parse --short HEAD` --build-arg VERSION=$CIRCLE_TAG . && e=0 && break || sleep 15; done && [ "$e" -eq "0" ] :
+    # Docker
+    - docker images
+    - ? |
+        e=1 && for i in {1..5}; do
+            docker build --rm=false -f docker/base.Dockerfile -t nipype/base:latest . && e=0 && break || sleep 15;
+        done && [ "$e" -eq "0" ]
+      :
+        timeout: 21600
+    - ? |
+        e=1 && for i in {1..5}; do
+            docker build --rm=false -t nipype/nipype:latest -t nipype/nipype:py36 --build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` --build-arg VCS_REF=`git rev-parse --short HEAD` --build-arg VERSION=$CIRCLE_TAG . && e=0 && break || sleep 15;
+        done && [ "$e" -eq "0" ]
+      :
         timeout: 6000
-    - e=1 && for i in {1..5}; do docker build --rm=false -t nipype/nipype:py27 --build-arg PYTHON_VERSION_MAJOR=2 --build-arg PYTHON_VERSION_MINOR=7 --build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` --build-arg VCS_REF=`git rev-parse --short HEAD` --build-arg VERSION=$CIRCLE_TAG-py27 . && e=0 && break || sleep 15; done && [ "$e" -eq "0" ] :
+    - ? |
+        e=1 && for i in {1..5}; do
+            docker build --rm=false -t nipype/nipype:py27 --build-arg PYTHON_VERSION_MAJOR=2 --build-arg PYTHON_VERSION_MINOR=7 --build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` --build-arg VCS_REF=`git rev-parse --short HEAD` --build-arg VERSION=$CIRCLE_TAG-py27 . && e=0 && break || sleep 15;
+        done && [ "$e" -eq "0" ]
+      :
         timeout: 6000
-    - docker save -o $HOME/docker/cache.tar nipype/base:latest nipype/nipype:py36 nipype/nipype:py27 :
+    - docker save -o $HOME/docker/cache.tar ubuntu:xenial-20161213 nipype/base:latest nipype/nipype:py36 nipype/nipype:py27 :
         timeout: 6000
 
 test:

docker/base.Dockerfile

@@ -44,39 +44,25 @@ RUN apt-get update && \
 
 WORKDIR /opt
 # Installing freesurfer -- do it first so that it is cached early
-RUN curl -sSL https://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/6.0.0/freesurfer-Linux-centos6_x86_64-stable-pub-v6.0.0.tar.gz | tar zxv -C /opt \
-    --exclude='freesurfer/trctrain' \
-    --exclude='freesurfer/subjects/fsaverage_sym' \
-    --exclude='freesurfer/subjects/fsaverage3' \
-    --exclude='freesurfer/subjects/fsaverage4' \
-    --exclude='freesurfer/subjects/fsaverage5' \
-    --exclude='freesurfer/subjects/fsaverage6' \
-    --exclude='freesurfer/subjects/cvs_avg35' \
-    --exclude='freesurfer/subjects/cvs_avg35_inMNI152' \
-    --exclude='freesurfer/subjects/bert' \
-    --exclude='freesurfer/subjects/V1_average' \
-    --exclude='freesurfer/average/mult-comp-cor' \
-    --exclude='freesurfer/lib/cuda' \
-    --exclude='freesurfer/lib/qt'
-
-ENV FSL_DIR=/usr/share/fsl/5.0 \
+#-----------------------------------------------------------------------------
+# 3. Install FreeSurfer v6.0 (minimized with reprozip):
+#    https://github.com/freesurfer/freesurfer/issues/70
+#-----------------------------------------------------------------------------
+RUN curl -sSL https://dl.dropbox.com/s/pbaisn6m5qpi9uu/recon-all-freesurfer6-2.min.tgz?dl=0 | tar zx -C /opt
+ENV FS_OVERRIDE=0 \
     OS=Linux \
-    FS_OVERRIDE=0 \
-    FIX_VERTEX_AREA= \
     FSF_OUTPUT_FORMAT=nii.gz \
+    FIX_VERTEX_AREA=\
     FREESURFER_HOME=/opt/freesurfer
-ENV SUBJECTS_DIR=$FREESURFER_HOME/subjects \
-    FUNCTIONALS_DIR=$FREESURFER_HOME/sessions \
-    MNI_DIR=$FREESURFER_HOME/mni \
-    LOCAL_DIR=$FREESURFER_HOME/local \
-    FSFAST_HOME=$FREESURFER_HOME/fsfast \
-    MINC_BIN_DIR=$FREESURFER_HOME/mni/bin \
-    MINC_LIB_DIR=$FREESURFER_HOME/mni/lib \
-    MNI_DATAPATH=$FREESURFER_HOME/mni/data \
-    FMRI_ANALYSIS_DIR=$FREESURFER_HOME/fsfast
-ENV PERL5LIB=$MINC_LIB_DIR/perl5/5.8.5 \
-    MNI_PERL5LIB=$MINC_LIB_DIR/perl5/5.8.5 \
-    PATH=$FREESURFER_HOME/bin:$FSFAST_HOME/bin:$FREESURFER_HOME/tktools:$MINC_BIN_DIR:$PATH
+ENV MNI_DIR=$FREESURFER_HOME/mni \
+    SUBJECTS_DIR=$FREESURFER_HOME/subjects
+ENV PERL5LIB=$MNI_DIR/share/perl5 \
+    MNI_PERL5LIB=$MNI_DIR/share/perl5 \
+    MINC_BIN_DIR=$MNI_DIR/bin \
+    MINC_LIB_DIR=$MNI_DIR/lib \
+    MNI_DATAPATH=$MNI_DIR/data
+ENV PATH=$FREESURFER_HOME/bin:$FREESURFER_HOME/tktools:$MINC_BIN_DIR:$PATH
+ENV FSL_DIR=/usr/share/fsl/5.0
 RUN echo "cHJpbnRmICJrcnp5c3p0b2YuZ29yZ29sZXdza2lAZ21haWwuY29tXG41MTcyXG4gKkN2dW12RVYzelRmZ1xuRlM1Si8yYzFhZ2c0RVxuIiA+IC9vcHQvZnJlZXN1cmZlci9saWNlbnNlLnR4dAo=" | base64 -d | sh
 
 # Enable neurodebian
@@ -122,8 +108,8 @@ ENV FSLDIR=/usr/share/fsl/5.0 \
 
 # Installing and setting up ANTs
 RUN mkdir -p /opt/ants && \
-    curl -sSL "https://github.com/stnava/ANTs/releases/download/v2.1.0/Linux_Ubuntu14.04.tar.bz2" \
-    | tar -xjC /opt/ants --strip-components 1
+    curl -sSL "https://dl.dropbox.com/s/2f4sui1z6lcgyek/ANTs-Linux-centos5_x86_64-v2.2.0-0740f91.tar.gz?dl=0" \
+    | tar -zx -C /opt
 
 ENV ANTSPATH=/opt/ants \
     PATH=$ANTSPATH:$PATH

nipype/interfaces/afni/utils.py

@@ -1247,7 +1247,8 @@ class UnifizeInputSpec(AFNICommandInputSpec):
         copyfile=False)
     out_file = File(
         desc='output image file name',
-        argstr='-prefix %s')
+        argstr='-prefix %s',
+        name_source='in_file')
     t2 = traits.Bool(
         desc='Treat the input as if it were T2-weighted, rather than '
              'T1-weighted. This processing is done simply by inverting '

nipype/interfaces/ants/tests/test_auto_AffineInitializer.py

@@ -28,6 +28,9 @@ def test_AffineInitializer_inputs():
     mandatory=True,
     position=2,
     ),
+    num_threads=dict(nohash=True,
+    usedefault=True,
+    ),
     out_file=dict(argstr='%s',
     position=3,
     usedefault=True,

nipype/interfaces/ants/utils.py

@@ -181,7 +181,7 @@ def _list_outputs(self):
         return outputs
 
 
-class AffineInitializerInputSpec(CommandLineInputSpec):
+class AffineInitializerInputSpec(ANTSCommandInputSpec):
     dimension = traits.Enum(3, 2, usedefault=True, position=0, argstr='%s',
                             desc='dimension')
     fixed_image = File(exists=True, mandatory=True, position=1, argstr='%s',
@@ -207,7 +207,7 @@ class AffineInitializerOutputSpec(TraitedSpec):
     out_file = File(desc='output transform file')
 
 
-class AffineInitializer(CommandLine):
+class AffineInitializer(ANTSCommand):
     """
     Initialize an affine transform (as in antsBrainExtraction.sh)
 

nipype/interfaces/freesurfer/preprocess.py

@@ -886,7 +886,8 @@ class ReconAll(CommandLine):
             ('avgcurv', ['surf/lh.avg_curv'], []),
             ('cortparc', ['label/lh.aparc.annot'], []),
             ('pial', ['surf/lh.pial', 'surf/lh.curv.pial',
-                      'surf/lh.area.pial', 'surf/lh.thickness'], []),
+                      'surf/lh.area.pial', 'surf/lh.thickness',
+                      'surf/lh.white'], []),
             ('parcstats', ['stats/lh.aparc.stats'], []),
             ('cortparc2', ['label/lh.aparc.a2009s.annot'], []),
             ('parcstats2', ['stats/lh.aparc.a2009s.stats'], []),

nipype/interfaces/fsl/ICA_AROMA.py

@@ -107,6 +107,7 @@ class ICA_AROMA(CommandLine):
     output_spec = ICA_AROMAOutputSpec
 
     def _list_outputs(self):
+        outputs = self.output_spec().get()
         out_dir = os.path.abspath(self.inputs.out_dir)
         outputs['out_dir'] = out_dir
 

nipype/pipeline/engine/nodes.py

@@ -1112,9 +1112,14 @@ def _make_nodes(self, cwd=None):
             nitems = len(filename_to_list(getattr(self.inputs, self.iterfield[0])))
         for i in range(nitems):
             nodename = '_' + self.name + str(i)
-            node = Node(deepcopy(self._interface), name=nodename)
-            node.overwrite = self.overwrite
-            node.run_without_submitting = self.run_without_submitting
+            node = Node(deepcopy(self._interface),
+                        n_procs=self._interface.num_threads,
+                        mem_gb=self._interface.estimated_memory_gb,
+                        overwrite=self.overwrite,
+                        needed_outputs=self.needed_outputs,
+                        run_without_submitting=self.run_without_submitting,
+                        base_dir=op.join(cwd, 'mapflow'),
+                        name=nodename)
             node.plugin_args = self.plugin_args
             node._interface.inputs.set(
                 **deepcopy(self._interface.inputs.get()))
@@ -1126,7 +1131,6 @@ def _make_nodes(self, cwd=None):
                 logger.debug('setting input %d %s %s', i, field, fieldvals[i])
                 setattr(node.inputs, field, fieldvals[i])
             node.config = self.config
-            node.base_dir = op.join(cwd, 'mapflow')
             yield i, node
 
     def _node_runner(self, nodes, updatehash=False):

nipype/pipeline/engine/tests/test_engine.py

@@ -486,6 +486,28 @@ def func1(in1):
     assert "can only concatenate list" in str(excinfo.value)
 
 
+def test_mapnode_expansion(tmpdir):
+    os.chdir(str(tmpdir))
+    from nipype import MapNode, Function
+
+    def func1(in1):
+        return in1 + 1
+
+    mapnode = MapNode(Function(function=func1),
+                      iterfield='in1',
+                      name='mapnode')
+    mapnode.inputs.in1 = [1, 2]
+    mapnode.interface.num_threads = 2
+    mapnode.interface.estimated_memory_gb = 2
+
+    for idx, node in mapnode._make_nodes():
+        for attr in ('overwrite', 'run_without_submitting', 'plugin_args'):
+            assert getattr(node, attr) == getattr(mapnode, attr)
+        for attr in ('num_threads', 'estimated_memory_gb'):
+            assert (getattr(node._interface, attr) ==
+                    getattr(mapnode._interface, attr))
+
+
 def test_node_hash(tmpdir):
     wd = str(tmpdir)
     os.chdir(wd)