diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index 826a62fe9f..6f762e919b 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -1,3 +1,7 @@ +# Fri Feb 21 14:06:53 2025 -0500 - markiewicz@stanford.edu - sty: black [ignore-rev] +8ed2b2306aeb7d89de4958b5293223ffe27a4f34 +# Tue Apr 13 10:16:17 2021 -0400 - markiewicz@stanford.edu - STY: black +b1690d5beb391e08c1e5463f1e3c641cf1e9f58e # Thu Oct 31 10:01:38 2024 -0400 - effigies@gmail.com - STY: black [ignore-rev] bd0d5856d183ba3918eda31f80db3b1d4387c55c # Thu Mar 21 13:34:09 2024 -0400 - effigies@gmail.com - STY: black [ignore-rev] diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5b0943c4ca..d789ec9061 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -34,7 +34,7 @@ jobs: with: fetch-depth: 0 - name: Install the latest version of uv - uses: astral-sh/setup-uv@v4 + uses: astral-sh/setup-uv@v5 - run: uv build - run: uvx twine check dist/* - uses: actions/upload-artifact@v4 @@ -102,7 +102,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Install the latest version of uv - uses: astral-sh/setup-uv@v4 + uses: astral-sh/setup-uv@v5 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: @@ -152,7 +152,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Install the latest version of uv - uses: astral-sh/setup-uv@v4 + uses: astral-sh/setup-uv@v5 - name: Show tox config run: uvx tox c - name: Show tox config (this call) diff --git a/.wci.yml b/.wci.yml new file mode 100644 index 0000000000..2adbae9fcc --- /dev/null +++ b/.wci.yml @@ -0,0 +1,30 @@ +# Project available at https://github.com/nipy/nipype + +name: nipype + +headline: "Neuroimaging in Python: Pipelines and Interfaces" + +description: | + Nipype, an open-source, community-developed initiative under the umbrella of NiPy, is a Python project that + provides a uniform interface to existing neuroimaging software and facilitates interaction between these + packages within a single workflow. Nipype provides an environment that encourages interactive exploration of + algorithms from different packages (e.g., SPM, FSL, FreeSurfer, AFNI, Slicer, ANTS), eases the design of + workflows within and between packages, and reduces the learning curve necessary to use different packages. + +language: Python3 + +documentation: + general: https://nipype.readthedocs.io/en/latest/ + installation: https://nipype.readthedocs.io/en/latest/users/install.html + tutorial: https://miykael.github.io/nipype_tutorial/ + +execution_environment: + resource_managers: + - SLURM + - Condor + - DAGMan + - LSF + - OAR + - PBS + - SGE + - Soma-workflow diff --git a/doc/changelog/1.X.X-changelog.rst b/doc/changelog/1.X.X-changelog.rst index a51ef7f13e..e31e508edf 100644 --- a/doc/changelog/1.X.X-changelog.rst +++ b/doc/changelog/1.X.X-changelog.rst @@ -1,3 +1,24 @@ +1.10.0 (March 19, 2025) +======================= + +New feature release in the 1.10.x series. + +This release adds GPUs to multiprocess resource management. +In general, no changes to existing code should be required if the GPU-enabled +interface has a ``use_gpu`` input. +The ``n_gpu_procs`` can be used to set the number of GPU processes that may +be run in parallel, which will override the default of GPUs identified by +``nvidia-smi``, or 1 if no GPUs are detected. + + * FIX: Reimplement ``gpu_count()`` (https://github.com/nipy/nipype/pull/3718) + * FIX: Avoid 0D array in ``algorithms.misc.merge_rois`` (https://github.com/nipy/nipype/pull/3713) + * FIX: Allow nipype.sphinx.ext.apidoc Config to work with Sphinx 8.2.1+ (https://github.com/nipy/nipype/pull/3716) + * FIX: Resolve crashes when running workflows with updatehash=True (https://github.com/nipy/nipype/pull/3709) + * ENH: Support for gpu queue (https://github.com/nipy/nipype/pull/3642) + * ENH: Update to .wci.yml (https://github.com/nipy/nipype/pull/3708) + * ENH: Add Workflow Community Initiative (WCI) descriptor (https://github.com/nipy/nipype/pull/3608) + + 1.9.2 (December 17, 2024) ========================= diff --git a/doc/interfaces.rst b/doc/interfaces.rst index da817fa163..795574a5e6 100644 --- a/doc/interfaces.rst +++ b/doc/interfaces.rst @@ -8,7 +8,7 @@ Interfaces and Workflows :Release: |version| :Date: |today| -Previous versions: `1.9.1 `_ `1.9.0 `_ +Previous versions: `1.9.2 `_ `1.9.1 `_ Workflows --------- diff --git a/nipype/algorithms/misc.py b/nipype/algorithms/misc.py index e1a67f0b08..fe27b877a2 100644 --- a/nipype/algorithms/misc.py +++ b/nipype/algorithms/misc.py @@ -1490,14 +1490,13 @@ def merge_rois(in_files, in_idxs, in_ref, dtype=None, out_file=None): for cname, iname in zip(in_files, in_idxs): f = np.load(iname) - idxs = np.squeeze(f["arr_0"]) + idxs = np.atleast_1d(np.squeeze(f["arr_0"])) + nels = len(idxs) for d, fname in enumerate(nii): data = np.asanyarray(nb.load(fname).dataobj).reshape(-1) cdata = nb.load(cname).dataobj[..., d].reshape(-1) - nels = len(idxs) - idata = (idxs,) - data[idata] = cdata[0:nels] + data[idxs] = cdata[:nels] nb.Nifti1Image(data.reshape(rsh[:3]), aff, hdr).to_filename(fname) imgs = [nb.load(im) for im in nii] diff --git a/nipype/caching/tests/test_memory.py b/nipype/caching/tests/test_memory.py index 5bd9fad528..cd5b8f8075 100644 --- a/nipype/caching/tests/test_memory.py +++ b/nipype/caching/tests/test_memory.py @@ -1,5 +1,4 @@ -""" Test the nipype interface caching mechanism -""" +"""Test the nipype interface caching mechanism""" from .. import Memory from ...pipeline.engine.tests.test_engine import EngineTestInterface diff --git a/nipype/external/cloghandler.py b/nipype/external/cloghandler.py index 289c8dfa2f..680ba30e2e 100644 --- a/nipype/external/cloghandler.py +++ b/nipype/external/cloghandler.py @@ -9,7 +9,7 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. -""" cloghandler.py: A smart replacement for the standard RotatingFileHandler +"""cloghandler.py: A smart replacement for the standard RotatingFileHandler ConcurrentRotatingFileHandler: This class is a log handler which is a drop-in replacement for the python standard log handler 'RotateFileHandler', the primary diff --git a/nipype/info.py b/nipype/info.py index 3fd328e995..c546e4c2fc 100644 --- a/nipype/info.py +++ b/nipype/info.py @@ -1,11 +1,11 @@ -""" This file contains defines parameters for nipy that we use to fill +"""This file contains defines parameters for nipy that we use to fill settings in setup.py, the nipy top-level docstring, and for building the docs. In setup.py in particular, we exec this file, so it cannot import nipy """ # nipype version information # Remove .dev0 for release -__version__ = "1.9.2" +__version__ = "1.10.0" def get_nipype_gitversion(): diff --git a/nipype/interfaces/ants/registration.py b/nipype/interfaces/ants/registration.py index 91b131bbf3..55e9738170 100644 --- a/nipype/interfaces/ants/registration.py +++ b/nipype/interfaces/ants/registration.py @@ -1,5 +1,5 @@ """The ants module provides basic functions for interfacing with ants - functions. +functions. """ import os diff --git a/nipype/interfaces/ants/resampling.py b/nipype/interfaces/ants/resampling.py index 95f29d5982..883eff1de3 100644 --- a/nipype/interfaces/ants/resampling.py +++ b/nipype/interfaces/ants/resampling.py @@ -1,5 +1,4 @@ -"""ANTS Apply Transforms interface -""" +"""ANTS Apply Transforms interface""" import os diff --git a/nipype/interfaces/ants/visualization.py b/nipype/interfaces/ants/visualization.py index c73b64c632..cdfa3529a7 100644 --- a/nipype/interfaces/ants/visualization.py +++ b/nipype/interfaces/ants/visualization.py @@ -1,5 +1,4 @@ -"""The ants visualisation module provides basic functions based on ITK. -""" +"""The ants visualisation module provides basic functions based on ITK.""" import os diff --git a/nipype/interfaces/bru2nii.py b/nipype/interfaces/bru2nii.py index 746af18f1a..b07f6a58d3 100644 --- a/nipype/interfaces/bru2nii.py +++ b/nipype/interfaces/bru2nii.py @@ -1,5 +1,4 @@ -"""The bru2nii module provides basic functions for dicom conversion -""" +"""The bru2nii module provides basic functions for dicom conversion""" import os from .base import ( diff --git a/nipype/interfaces/camino/__init__.py b/nipype/interfaces/camino/__init__.py index 766fa9c906..67e973df66 100644 --- a/nipype/interfaces/camino/__init__.py +++ b/nipype/interfaces/camino/__init__.py @@ -1,7 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -"""Camino top level namespace -""" +"""Camino top level namespace""" from .connectivity import Conmat from .convert import ( diff --git a/nipype/interfaces/cmtk/base.py b/nipype/interfaces/cmtk/base.py index d0c226dc49..c4c997288b 100644 --- a/nipype/interfaces/cmtk/base.py +++ b/nipype/interfaces/cmtk/base.py @@ -1,6 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -""" Base interface for cmtk """ +"""Base interface for cmtk""" from ..base import LibraryBaseInterface from ...utils.misc import package_check diff --git a/nipype/interfaces/diffusion_toolkit/dti.py b/nipype/interfaces/diffusion_toolkit/dti.py index fa031799e3..bf6336c96d 100644 --- a/nipype/interfaces/diffusion_toolkit/dti.py +++ b/nipype/interfaces/diffusion_toolkit/dti.py @@ -1,7 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -"""Provides interfaces to various commands provided by diffusion toolkit -""" +"""Provides interfaces to various commands provided by diffusion toolkit""" import os import re diff --git a/nipype/interfaces/diffusion_toolkit/odf.py b/nipype/interfaces/diffusion_toolkit/odf.py index 00f86a322c..daadffc200 100644 --- a/nipype/interfaces/diffusion_toolkit/odf.py +++ b/nipype/interfaces/diffusion_toolkit/odf.py @@ -1,7 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -"""Provides interfaces to various commands provided by diffusion toolkit -""" +"""Provides interfaces to various commands provided by diffusion toolkit""" import os import re diff --git a/nipype/interfaces/diffusion_toolkit/postproc.py b/nipype/interfaces/diffusion_toolkit/postproc.py index 5190843875..d05cfadff6 100644 --- a/nipype/interfaces/diffusion_toolkit/postproc.py +++ b/nipype/interfaces/diffusion_toolkit/postproc.py @@ -1,7 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -"""Provides interfaces to various commands provided by diffusion toolkit -""" +"""Provides interfaces to various commands provided by diffusion toolkit""" import os from ..base import ( diff --git a/nipype/interfaces/dipy/base.py b/nipype/interfaces/dipy/base.py index 1b9bdea6d5..44290cd1d7 100644 --- a/nipype/interfaces/dipy/base.py +++ b/nipype/interfaces/dipy/base.py @@ -1,4 +1,4 @@ -""" Base interfaces for dipy """ +"""Base interfaces for dipy""" import os.path as op import inspect diff --git a/nipype/interfaces/freesurfer/longitudinal.py b/nipype/interfaces/freesurfer/longitudinal.py index 227ea76775..41e95c091b 100644 --- a/nipype/interfaces/freesurfer/longitudinal.py +++ b/nipype/interfaces/freesurfer/longitudinal.py @@ -1,7 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -"""Provides interfaces to various longitudinal commands provided by freesurfer -""" +"""Provides interfaces to various longitudinal commands provided by freesurfer""" import os diff --git a/nipype/interfaces/freesurfer/model.py b/nipype/interfaces/freesurfer/model.py index 6376c1b971..5e245a9a85 100644 --- a/nipype/interfaces/freesurfer/model.py +++ b/nipype/interfaces/freesurfer/model.py @@ -1,7 +1,7 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: """The freesurfer module provides basic functions for interfacing with - freesurfer tools. +freesurfer tools. """ import os diff --git a/nipype/interfaces/freesurfer/petsurfer.py b/nipype/interfaces/freesurfer/petsurfer.py index 4505985127..28aa763b06 100644 --- a/nipype/interfaces/freesurfer/petsurfer.py +++ b/nipype/interfaces/freesurfer/petsurfer.py @@ -1,7 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -"""Provides interfaces to various commands for running PET analyses provided by FreeSurfer -""" +"""Provides interfaces to various commands for running PET analyses provided by FreeSurfer""" import os diff --git a/nipype/interfaces/freesurfer/preprocess.py b/nipype/interfaces/freesurfer/preprocess.py index 5b2fd19a0b..89c218f969 100644 --- a/nipype/interfaces/freesurfer/preprocess.py +++ b/nipype/interfaces/freesurfer/preprocess.py @@ -1,7 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -"""Provides interfaces to various commands provided by FreeSurfer -""" +"""Provides interfaces to various commands provided by FreeSurfer""" import os import os.path as op from glob import glob diff --git a/nipype/interfaces/freesurfer/registration.py b/nipype/interfaces/freesurfer/registration.py index bc70fc44a6..790066d0ec 100644 --- a/nipype/interfaces/freesurfer/registration.py +++ b/nipype/interfaces/freesurfer/registration.py @@ -1,7 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -"""Provides interfaces to various longitudinal commands provided by freesurfer -""" +"""Provides interfaces to various longitudinal commands provided by freesurfer""" import os import os.path diff --git a/nipype/interfaces/freesurfer/utils.py b/nipype/interfaces/freesurfer/utils.py index 777f42f019..2c1cdbcc94 100644 --- a/nipype/interfaces/freesurfer/utils.py +++ b/nipype/interfaces/freesurfer/utils.py @@ -1,7 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -"""Interfaces to assorted Freesurfer utility programs. -""" +"""Interfaces to assorted Freesurfer utility programs.""" import os import re import shutil diff --git a/nipype/interfaces/io.py b/nipype/interfaces/io.py index 46cdfb44f2..d6af1ba073 100644 --- a/nipype/interfaces/io.py +++ b/nipype/interfaces/io.py @@ -1,14 +1,14 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -""" Set of interfaces that allow interaction with data. Currently - available interfaces are: +"""Set of interfaces that allow interaction with data. Currently +available interfaces are: - DataSource: Generic nifti to named Nifti interface - DataSink: Generic named output from interfaces to data store - XNATSource: preliminary interface to XNAT +DataSource: Generic nifti to named Nifti interface +DataSink: Generic named output from interfaces to data store +XNATSource: preliminary interface to XNAT - To come : - XNATSink +To come : +XNATSink """ import glob import fnmatch diff --git a/nipype/interfaces/mixins/reporting.py b/nipype/interfaces/mixins/reporting.py index 90ca804618..a836cfa3fa 100644 --- a/nipype/interfaces/mixins/reporting.py +++ b/nipype/interfaces/mixins/reporting.py @@ -1,6 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -""" class mixin and utilities for enabling reports for nipype interfaces """ +"""class mixin and utilities for enabling reports for nipype interfaces""" import os from abc import abstractmethod diff --git a/nipype/interfaces/nipy/base.py b/nipype/interfaces/nipy/base.py index 25aef8b873..1f8f1e4657 100644 --- a/nipype/interfaces/nipy/base.py +++ b/nipype/interfaces/nipy/base.py @@ -1,6 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -""" Base interface for nipy """ +"""Base interface for nipy""" from ..base import LibraryBaseInterface from ...utils.misc import package_check diff --git a/nipype/interfaces/nitime/base.py b/nipype/interfaces/nitime/base.py index 7e434f1d3e..4109bc3a74 100644 --- a/nipype/interfaces/nitime/base.py +++ b/nipype/interfaces/nitime/base.py @@ -1,6 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -""" Base interface for nitime """ +"""Base interface for nitime""" from ..base import LibraryBaseInterface diff --git a/nipype/interfaces/spm/preprocess.py b/nipype/interfaces/spm/preprocess.py index c7f69785ff..8d931a72ba 100644 --- a/nipype/interfaces/spm/preprocess.py +++ b/nipype/interfaces/spm/preprocess.py @@ -1,7 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -"""SPM wrappers for preprocessing data -""" +"""SPM wrappers for preprocessing data""" import os from copy import deepcopy diff --git a/nipype/interfaces/utility/base.py b/nipype/interfaces/utility/base.py index 564966cb5b..ecc1bf7935 100644 --- a/nipype/interfaces/utility/base.py +++ b/nipype/interfaces/utility/base.py @@ -1,9 +1,9 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: """ - # changing to temporary directories - >>> tmp = getfixture('tmpdir') - >>> old = tmp.chdir() +# changing to temporary directories +>>> tmp = getfixture('tmpdir') +>>> old = tmp.chdir() """ import os import re diff --git a/nipype/interfaces/utility/csv.py b/nipype/interfaces/utility/csv.py index 979e328bb6..7470eecbfe 100644 --- a/nipype/interfaces/utility/csv.py +++ b/nipype/interfaces/utility/csv.py @@ -1,7 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -"""CSV Handling utilities -""" +"""CSV Handling utilities""" import csv from ..base import traits, TraitedSpec, DynamicTraitedSpec, File, BaseInterface from ..io import add_traits diff --git a/nipype/pipeline/engine/nodes.py b/nipype/pipeline/engine/nodes.py index 31ee29e04d..e29b56718b 100644 --- a/nipype/pipeline/engine/nodes.py +++ b/nipype/pipeline/engine/nodes.py @@ -452,7 +452,7 @@ def run(self, updatehash=False): cached, updated = self.is_cached() # If the node is cached, check on pklz files and finish - if not force_run and (updated or (not updated and updatehash)): + if cached and not force_run and (updated or updatehash): logger.debug("Only updating node hashes or skipping execution") inputs_file = op.join(outdir, "_inputs.pklz") if not op.exists(inputs_file): @@ -820,6 +820,11 @@ def update(self, **opts): """Update inputs""" self.inputs.update(**opts) + def is_gpu_node(self): + return bool(getattr(self.inputs, 'use_cuda', False)) or bool( + getattr(self.inputs, 'use_gpu', False) + ) + class JoinNode(Node): """Wraps interface objects that join inputs into a list. diff --git a/nipype/pipeline/engine/tests/test_engine.py b/nipype/pipeline/engine/tests/test_engine.py index f1b6817e74..7650be1cd3 100644 --- a/nipype/pipeline/engine/tests/test_engine.py +++ b/nipype/pipeline/engine/tests/test_engine.py @@ -1,7 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -"""Tests for the engine module -""" +"""Tests for the engine module""" from copy import deepcopy from glob import glob import os diff --git a/nipype/pipeline/engine/tests/test_join.py b/nipype/pipeline/engine/tests/test_join.py index 2fe5f70564..c177ad24d3 100644 --- a/nipype/pipeline/engine/tests/test_join.py +++ b/nipype/pipeline/engine/tests/test_join.py @@ -1,7 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -"""Tests for join expansion -""" +"""Tests for join expansion""" import pytest from .... import config diff --git a/nipype/pipeline/engine/tests/test_utils.py b/nipype/pipeline/engine/tests/test_utils.py index 78483b6923..7ae8ce5b33 100644 --- a/nipype/pipeline/engine/tests/test_utils.py +++ b/nipype/pipeline/engine/tests/test_utils.py @@ -1,7 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -"""Tests for the engine utils module -""" +"""Tests for the engine utils module""" import os from copy import deepcopy import pytest diff --git a/nipype/pipeline/engine/tests/test_workflows.py b/nipype/pipeline/engine/tests/test_workflows.py index 12d56de285..980b54fa28 100644 --- a/nipype/pipeline/engine/tests/test_workflows.py +++ b/nipype/pipeline/engine/tests/test_workflows.py @@ -1,7 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -"""Tests for the engine workflows module -""" +"""Tests for the engine workflows module""" from glob import glob import os from shutil import rmtree diff --git a/nipype/pipeline/plugins/condor.py b/nipype/pipeline/plugins/condor.py index 0fff477377..789eaecfab 100644 --- a/nipype/pipeline/plugins/condor.py +++ b/nipype/pipeline/plugins/condor.py @@ -1,5 +1,4 @@ -"""Parallel workflow execution via Condor -""" +"""Parallel workflow execution via Condor""" import os from time import sleep diff --git a/nipype/pipeline/plugins/dagman.py b/nipype/pipeline/plugins/dagman.py index 55f3f03bee..1c424c24ef 100644 --- a/nipype/pipeline/plugins/dagman.py +++ b/nipype/pipeline/plugins/dagman.py @@ -1,5 +1,4 @@ -"""Parallel workflow execution via Condor DAGMan -""" +"""Parallel workflow execution via Condor DAGMan""" import os import sys diff --git a/nipype/pipeline/plugins/debug.py b/nipype/pipeline/plugins/debug.py index 1dac35cf8f..4798e083bd 100644 --- a/nipype/pipeline/plugins/debug.py +++ b/nipype/pipeline/plugins/debug.py @@ -1,7 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -"""Debug plugin -""" +"""Debug plugin""" import networkx as nx from .base import PluginBase, logger diff --git a/nipype/pipeline/plugins/ipython.py b/nipype/pipeline/plugins/ipython.py index f52b3e6282..2c80eb4655 100644 --- a/nipype/pipeline/plugins/ipython.py +++ b/nipype/pipeline/plugins/ipython.py @@ -1,7 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -"""Parallel workflow execution via IPython controller -""" +"""Parallel workflow execution via IPython controller""" from pickle import dumps import sys diff --git a/nipype/pipeline/plugins/linear.py b/nipype/pipeline/plugins/linear.py index 93029ee1b9..aa29a5951b 100644 --- a/nipype/pipeline/plugins/linear.py +++ b/nipype/pipeline/plugins/linear.py @@ -1,7 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -"""Local serial workflow execution -""" +"""Local serial workflow execution""" import os from .base import PluginBase, logger, report_crash, report_nodes_not_run, str2bool diff --git a/nipype/pipeline/plugins/lsf.py b/nipype/pipeline/plugins/lsf.py index cf334be051..4ca380dfaa 100644 --- a/nipype/pipeline/plugins/lsf.py +++ b/nipype/pipeline/plugins/lsf.py @@ -1,5 +1,4 @@ -"""Parallel workflow execution via LSF -""" +"""Parallel workflow execution via LSF""" import os import re diff --git a/nipype/pipeline/plugins/multiproc.py b/nipype/pipeline/plugins/multiproc.py index 401b01b388..be0e006229 100644 --- a/nipype/pipeline/plugins/multiproc.py +++ b/nipype/pipeline/plugins/multiproc.py @@ -21,6 +21,7 @@ from ...utils.profiler import get_system_total_memory_gb from ..engine import MapNode from .base import DistributedPluginBase +from ...utils.gpu_count import gpu_count try: from textwrap import indent @@ -100,6 +101,7 @@ class MultiProcPlugin(DistributedPluginBase): - non_daemon: boolean flag to execute as non-daemon processes - n_procs: maximum number of threads to be executed in parallel + - n_gpu_procs: maximum number of GPU threads to be executed in parallel - memory_gb: maximum memory (in GB) that can be used at once. - raise_insufficient: raise error if the requested resources for a node over the maximum `n_procs` and/or `memory_gb` @@ -130,10 +132,24 @@ def __init__(self, plugin_args=None): ) self.raise_insufficient = self.plugin_args.get("raise_insufficient", True) + # GPU found on system + self.n_gpus_visible = gpu_count() + # proc per GPU set by user + self.n_gpu_procs = self.plugin_args.get('n_gpu_procs', self.n_gpus_visible) + + # total no. of processes allowed on all gpus + if self.n_gpu_procs > self.n_gpus_visible: + logger.info( + 'Total number of GPUs proc requested (%d) exceeds the available number of GPUs (%d) on the system. Using requested GPU slots at your own risk!', + self.n_gpu_procs, + self.n_gpus_visible, + ) + # Instantiate different thread pools for non-daemon processes logger.debug( - "[MultiProc] Starting (n_procs=%d, mem_gb=%0.2f, cwd=%s)", + "[MultiProc] Starting (n_procs=%d, n_gpu_procs=%d, mem_gb=%0.2f, cwd=%s)", self.processors, + self.n_gpu_procs, self.memory_gb, self._cwd, ) @@ -184,9 +200,12 @@ def _prerun_check(self, graph): """Check if any node exceeds the available resources""" tasks_mem_gb = [] tasks_num_th = [] + tasks_gpu_th = [] for node in graph.nodes(): tasks_mem_gb.append(node.mem_gb) tasks_num_th.append(node.n_procs) + if node.is_gpu_node(): + tasks_gpu_th.append(node.n_procs) if np.any(np.array(tasks_mem_gb) > self.memory_gb): logger.warning( @@ -203,6 +222,10 @@ def _prerun_check(self, graph): ) if self.raise_insufficient: raise RuntimeError("Insufficient resources available for job") + if np.any(np.array(tasks_gpu_th) > self.n_gpu_procs): + logger.warning('Nodes demand more GPU than allowed (%d).', self.n_gpu_procs) + if self.raise_insufficient: + raise RuntimeError('Insufficient GPU resources available for job') def _postrun_check(self): self.pool.shutdown() @@ -213,11 +236,14 @@ def _check_resources(self, running_tasks): """ free_memory_gb = self.memory_gb free_processors = self.processors + free_gpu_slots = self.n_gpu_procs for _, jobid in running_tasks: free_memory_gb -= min(self.procs[jobid].mem_gb, free_memory_gb) free_processors -= min(self.procs[jobid].n_procs, free_processors) + if self.procs[jobid].is_gpu_node(): + free_gpu_slots -= min(self.procs[jobid].n_procs, free_gpu_slots) - return free_memory_gb, free_processors + return free_memory_gb, free_processors, free_gpu_slots def _send_procs_to_workers(self, updatehash=False, graph=None): """ @@ -232,7 +258,9 @@ def _send_procs_to_workers(self, updatehash=False, graph=None): ) # Check available resources by summing all threads and memory used - free_memory_gb, free_processors = self._check_resources(self.pending_tasks) + free_memory_gb, free_processors, free_gpu_slots = self._check_resources( + self.pending_tasks + ) stats = ( len(self.pending_tasks), @@ -241,6 +269,8 @@ def _send_procs_to_workers(self, updatehash=False, graph=None): self.memory_gb, free_processors, self.processors, + free_gpu_slots, + self.n_gpu_procs, ) if self._stats != stats: tasks_list_msg = "" @@ -256,13 +286,15 @@ def _send_procs_to_workers(self, updatehash=False, graph=None): tasks_list_msg = indent(tasks_list_msg, " " * 21) logger.info( "[MultiProc] Running %d tasks, and %d jobs ready. Free " - "memory (GB): %0.2f/%0.2f, Free processors: %d/%d.%s", + "memory (GB): %0.2f/%0.2f, Free processors: %d/%d, Free GPU slot:%d/%d.%s", len(self.pending_tasks), len(jobids), free_memory_gb, self.memory_gb, free_processors, self.processors, + free_gpu_slots, + self.n_gpu_procs, tasks_list_msg, ) self._stats = stats @@ -304,28 +336,39 @@ def _send_procs_to_workers(self, updatehash=False, graph=None): # Check requirements of this job next_job_gb = min(self.procs[jobid].mem_gb, self.memory_gb) next_job_th = min(self.procs[jobid].n_procs, self.processors) + next_job_gpu_th = min(self.procs[jobid].n_procs, self.n_gpu_procs) + + is_gpu_node = self.procs[jobid].is_gpu_node() # If node does not fit, skip at this moment - if next_job_th > free_processors or next_job_gb > free_memory_gb: + if ( + next_job_th > free_processors + or next_job_gb > free_memory_gb + or (is_gpu_node and next_job_gpu_th > free_gpu_slots) + ): logger.debug( - "Cannot allocate job %d (%0.2fGB, %d threads).", + "Cannot allocate job %d (%0.2fGB, %d threads, %d GPU slots).", jobid, next_job_gb, next_job_th, + next_job_gpu_th, ) continue free_memory_gb -= next_job_gb free_processors -= next_job_th + if is_gpu_node: + free_gpu_slots -= next_job_gpu_th logger.debug( "Allocating %s ID=%d (%0.2fGB, %d threads). Free: " - "%0.2fGB, %d threads.", + "%0.2fGB, %d threads, %d GPU slots.", self.procs[jobid].fullname, jobid, next_job_gb, next_job_th, free_memory_gb, free_processors, + free_gpu_slots, ) # change job status in appropriate queues @@ -336,8 +379,11 @@ def _send_procs_to_workers(self, updatehash=False, graph=None): if self._local_hash_check(jobid, graph): continue + cached, updated = self.procs[jobid].is_cached() # updatehash and run_without_submitting are also run locally - if updatehash or self.procs[jobid].run_without_submitting: + if (cached and updatehash and not updated) or self.procs[ + jobid + ].run_without_submitting: logger.debug("Running node %s on master thread", self.procs[jobid]) try: self.procs[jobid].run(updatehash=updatehash) @@ -352,6 +398,8 @@ def _send_procs_to_workers(self, updatehash=False, graph=None): self._remove_node_dirs() free_memory_gb += next_job_gb free_processors += next_job_th + if is_gpu_node: + free_gpu_slots += next_job_gpu_th # Display stats next loop self._stats = None diff --git a/nipype/pipeline/plugins/oar.py b/nipype/pipeline/plugins/oar.py index df56391bae..b9c4a050ab 100644 --- a/nipype/pipeline/plugins/oar.py +++ b/nipype/pipeline/plugins/oar.py @@ -1,5 +1,4 @@ -"""Parallel workflow execution via OAR http://oar.imag.fr -""" +"""Parallel workflow execution via OAR http://oar.imag.fr""" import os import stat diff --git a/nipype/pipeline/plugins/pbs.py b/nipype/pipeline/plugins/pbs.py index d967af0bed..01c80efc5a 100644 --- a/nipype/pipeline/plugins/pbs.py +++ b/nipype/pipeline/plugins/pbs.py @@ -1,5 +1,4 @@ -"""Parallel workflow execution via PBS/Torque -""" +"""Parallel workflow execution via PBS/Torque""" import os from time import sleep diff --git a/nipype/pipeline/plugins/pbsgraph.py b/nipype/pipeline/plugins/pbsgraph.py index 4b245dedb7..0cb925af38 100644 --- a/nipype/pipeline/plugins/pbsgraph.py +++ b/nipype/pipeline/plugins/pbsgraph.py @@ -1,5 +1,4 @@ -"""Parallel workflow execution via PBS/Torque -""" +"""Parallel workflow execution via PBS/Torque""" import os import sys diff --git a/nipype/pipeline/plugins/sge.py b/nipype/pipeline/plugins/sge.py index 38079e947d..ce8e046f01 100644 --- a/nipype/pipeline/plugins/sge.py +++ b/nipype/pipeline/plugins/sge.py @@ -1,5 +1,4 @@ -"""Parallel workflow execution via SGE -""" +"""Parallel workflow execution via SGE""" import os import pwd diff --git a/nipype/pipeline/plugins/sgegraph.py b/nipype/pipeline/plugins/sgegraph.py index 5cd1c7bfb7..3b33b73dee 100644 --- a/nipype/pipeline/plugins/sgegraph.py +++ b/nipype/pipeline/plugins/sgegraph.py @@ -1,5 +1,4 @@ -"""Parallel workflow execution via SGE -""" +"""Parallel workflow execution via SGE""" import os import sys diff --git a/nipype/pipeline/plugins/slurmgraph.py b/nipype/pipeline/plugins/slurmgraph.py index c74ab05a87..05824b016b 100644 --- a/nipype/pipeline/plugins/slurmgraph.py +++ b/nipype/pipeline/plugins/slurmgraph.py @@ -1,5 +1,4 @@ -"""Parallel workflow execution via SLURM -""" +"""Parallel workflow execution via SLURM""" import os import sys diff --git a/nipype/pipeline/plugins/somaflow.py b/nipype/pipeline/plugins/somaflow.py index 2105204979..16bedaab23 100644 --- a/nipype/pipeline/plugins/somaflow.py +++ b/nipype/pipeline/plugins/somaflow.py @@ -1,5 +1,4 @@ -"""Parallel workflow execution via PBS/Torque -""" +"""Parallel workflow execution via PBS/Torque""" import os import sys diff --git a/nipype/pipeline/plugins/tests/test_base.py b/nipype/pipeline/plugins/tests/test_base.py index 43471a7d64..11acb369e9 100644 --- a/nipype/pipeline/plugins/tests/test_base.py +++ b/nipype/pipeline/plugins/tests/test_base.py @@ -1,7 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -"""Tests for the engine module -""" +"""Tests for the engine module""" import numpy as np import scipy.sparse as ssp diff --git a/nipype/pipeline/plugins/tests/test_legacymultiproc_nondaemon.py b/nipype/pipeline/plugins/tests/test_legacymultiproc_nondaemon.py index 2f35579a40..cd79fbe31c 100644 --- a/nipype/pipeline/plugins/tests/test_legacymultiproc_nondaemon.py +++ b/nipype/pipeline/plugins/tests/test_legacymultiproc_nondaemon.py @@ -1,7 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -"""Testing module for functions and classes from multiproc.py -""" +"""Testing module for functions and classes from multiproc.py""" # Import packages import os import sys diff --git a/nipype/pipeline/plugins/tests/test_multiproc.py b/nipype/pipeline/plugins/tests/test_multiproc.py index 938e1aab9e..484c0d07bc 100644 --- a/nipype/pipeline/plugins/tests/test_multiproc.py +++ b/nipype/pipeline/plugins/tests/test_multiproc.py @@ -56,6 +56,7 @@ def test_run_multiproc(tmpdir): class InputSpecSingleNode(nib.TraitedSpec): input1 = nib.traits.Int(desc="a random int") input2 = nib.traits.Int(desc="a random int") + use_gpu = nib.traits.Bool(False, mandatory=False, desc="boolean for GPU nodes") class OutputSpecSingleNode(nib.TraitedSpec): @@ -117,6 +118,24 @@ def test_no_more_threads_than_specified(tmpdir): pipe.run(plugin="MultiProc", plugin_args={"n_procs": max_threads}) +def test_no_more_gpu_threads_than_specified(tmpdir): + tmpdir.chdir() + + pipe = pe.Workflow(name="pipe") + n1 = pe.Node(SingleNodeTestInterface(), name="n1", n_procs=2) + n1.inputs.use_gpu = True + n1.inputs.input1 = 4 + pipe.add_nodes([n1]) + + max_threads = 2 + max_gpu = 1 + with pytest.raises(RuntimeError): + pipe.run( + plugin="MultiProc", + plugin_args={"n_procs": max_threads, 'n_gpu_procs': max_gpu}, + ) + + @pytest.mark.skipif( sys.version_info >= (3, 8), reason="multiprocessing issues in Python 3.8" ) diff --git a/nipype/pipeline/plugins/tests/test_tools.py b/nipype/pipeline/plugins/tests/test_tools.py index e21ef42072..e352253dbe 100644 --- a/nipype/pipeline/plugins/tests/test_tools.py +++ b/nipype/pipeline/plugins/tests/test_tools.py @@ -1,7 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -"""Tests for the engine module -""" +"""Tests for the engine module""" import re from unittest import mock diff --git a/nipype/pipeline/plugins/tools.py b/nipype/pipeline/plugins/tools.py index bce3eb82da..7e066b0ea3 100644 --- a/nipype/pipeline/plugins/tools.py +++ b/nipype/pipeline/plugins/tools.py @@ -1,7 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -"""Common graph operations for execution -""" +"""Common graph operations for execution""" import os import getpass from socket import gethostname diff --git a/nipype/sphinxext/apidoc/__init__.py b/nipype/sphinxext/apidoc/__init__.py index 151011bdfc..429848d2f5 100644 --- a/nipype/sphinxext/apidoc/__init__.py +++ b/nipype/sphinxext/apidoc/__init__.py @@ -2,6 +2,9 @@ # vi: set ft=python sts=4 ts=4 sw=4 et: """Settings for sphinxext.interfaces and connection to sphinx-apidoc.""" import re +from packaging.version import Version + +import sphinx from sphinx.ext.napoleon import ( Config as NapoleonConfig, _patch_python_domain, @@ -39,13 +42,24 @@ class Config(NapoleonConfig): """ - _config_values = { - "nipype_skip_classes": ( - ["Tester", "InputSpec", "OutputSpec", "Numpy", "NipypeTester"], - "env", - ), - **NapoleonConfig._config_values, - } + if Version(sphinx.__version__) >= Version("8.2.1"): + _config_values = ( + ( + "nipype_skip_classes", + ["Tester", "InputSpec", "OutputSpec", "Numpy", "NipypeTester"], + "env", + frozenset({list[str]}), + ), + *NapoleonConfig._config_values, + ) + else: + _config_values = { + "nipype_skip_classes": ( + ["Tester", "InputSpec", "OutputSpec", "Numpy", "NipypeTester"], + "env", + ), + **NapoleonConfig._config_values, + } def setup(app): @@ -82,8 +96,12 @@ def setup(app): app.connect("autodoc-process-docstring", _process_docstring) app.connect("autodoc-skip-member", _skip_member) - for name, (default, rebuild) in Config._config_values.items(): - app.add_config_value(name, default, rebuild) + if Version(sphinx.__version__) >= Version("8.2.1"): + for name, default, rebuild, types in Config._config_values: + app.add_config_value(name, default, rebuild, types=types) + else: + for name, (default, rebuild) in Config._config_values.items(): + app.add_config_value(name, default, rebuild) return {"version": __version__, "parallel_read_safe": True} diff --git a/nipype/testing/tests/test_utils.py b/nipype/testing/tests/test_utils.py index 9217d54694..c3b1cae638 100644 --- a/nipype/testing/tests/test_utils.py +++ b/nipype/testing/tests/test_utils.py @@ -1,7 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -"""Test testing utilities -""" +"""Test testing utilities""" import os import subprocess diff --git a/nipype/testing/utils.py b/nipype/testing/utils.py index 71a75a41c7..96a94d6564 100644 --- a/nipype/testing/utils.py +++ b/nipype/testing/utils.py @@ -1,7 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -"""Additional handy utilities for testing -""" +"""Additional handy utilities for testing""" import os import time import shutil diff --git a/nipype/utils/filemanip.py b/nipype/utils/filemanip.py index 52558f59f0..4916cbacef 100644 --- a/nipype/utils/filemanip.py +++ b/nipype/utils/filemanip.py @@ -1,7 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -"""Miscellaneous file manipulation functions -""" +"""Miscellaneous file manipulation functions""" import sys import pickle import errno diff --git a/nipype/utils/gpu_count.py b/nipype/utils/gpu_count.py new file mode 100644 index 0000000000..70eb6d724e --- /dev/null +++ b/nipype/utils/gpu_count.py @@ -0,0 +1,46 @@ +# -*- DISCLAIMER: this file contains code derived from gputil (https://github.com/anderskm/gputil) +# and therefore is distributed under to the following license: +# +# MIT License +# +# Copyright (c) 2017 anderskm +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import platform +import shutil +import subprocess +import os + + +def gpu_count(): + nvidia_smi = shutil.which('nvidia-smi') + if nvidia_smi is None and platform.system() == "Windows": + nvidia_smi = f'{os.environ["systemdrive"]}\\Program Files\\NVIDIA Corporation\\NVSMI\\nvidia-smi.exe' + if nvidia_smi is None: + return 0 + try: + p = subprocess.run( + [nvidia_smi, "--query-gpu=name", "--format=csv,noheader,nounits"], + stdout=subprocess.PIPE, + text=True, + ) + except (OSError, UnicodeDecodeError): + return 0 + return len(p.stdout.splitlines()) diff --git a/nipype/utils/matlabtools.py b/nipype/utils/matlabtools.py index ea06cd4126..d871885c06 100644 --- a/nipype/utils/matlabtools.py +++ b/nipype/utils/matlabtools.py @@ -1,6 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -""" Useful Functions for working with matlab""" +"""Useful Functions for working with matlab""" # Stdlib imports import os diff --git a/nipype/utils/misc.py b/nipype/utils/misc.py index ed8a539e66..3f76fbab3c 100644 --- a/nipype/utils/misc.py +++ b/nipype/utils/misc.py @@ -1,7 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -"""Miscellaneous utility functions -""" +"""Miscellaneous utility functions""" import os import sys import re diff --git a/nipype/utils/subprocess.py b/nipype/utils/subprocess.py index acd6b63256..2fa9e52c3b 100644 --- a/nipype/utils/subprocess.py +++ b/nipype/utils/subprocess.py @@ -1,7 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -"""Miscellaneous utility functions -""" +"""Miscellaneous utility functions""" import os import sys import gc diff --git a/tox.ini b/tox.ini index 9704158bec..571b93628b 100644 --- a/tox.ini +++ b/tox.ini @@ -68,8 +68,6 @@ pass_env = CLICOLOR CLICOLOR_FORCE PYTHON_GIL -deps = - py313: traits @ git+https://github.com/enthought/traits.git@10954eb extras = tests full: doc