diff --git a/nipype/utils/filemanip.py b/nipype/utils/filemanip.py index 97a7538780..4b4942227e 100644 --- a/nipype/utils/filemanip.py +++ b/nipype/utils/filemanip.py @@ -275,27 +275,28 @@ def hash_timestamp(afile): return md5hex -def _generate_cifs_table(): - """Construct a reverse-length-ordered list of mount points that - fall under a CIFS mount. - - This precomputation allows efficient checking for whether a given path - would be on a CIFS filesystem. +def _parse_mount_table(exit_code, output): + """Parses the output of ``mount`` to produce (path, fs_type) pairs - On systems without a ``mount`` command, or with no CIFS mounts, returns an - empty list. + Separated from _generate_cifs_table to enable testing logic with real + outputs """ - exit_code, output = sp.getstatusoutput("mount") # Not POSIX if exit_code != 0: return [] + # Linux mount example: sysfs on /sys type sysfs (rw,nosuid,nodev,noexec) + # ^^^^ ^^^^^ + # OSX mount example: /dev/disk2 on / (hfs, local, journaled) + # ^ ^^^ + pattern = re.compile(r'.*? on (/.*?) (?:type |\()([^\s,]+)(?:, |\)| )') + # (path, fstype) tuples, sorted by path length (longest first) - mount_info = sorted( - (line.split()[2:5:2] for line in output.splitlines()), - key=lambda x: len(x[0]), - reverse=True) - cifs_paths = [path for path, fstype in mount_info if fstype == 'cifs'] + mount_info = sorted((pattern.match(l).groups() + for l in output.splitlines()), + key=lambda x: len(x[0]), reverse=True) + cifs_paths = [path for path, fstype in mount_info + if fstype.lower() == 'cifs'] return [ mount for mount in mount_info @@ -303,6 +304,20 @@ def _generate_cifs_table(): ] +def _generate_cifs_table(): + """Construct a reverse-length-ordered list of mount points that + fall under a CIFS mount. + + This precomputation allows efficient checking for whether a given path + would be on a CIFS filesystem. + + On systems without a ``mount`` command, or with no CIFS mounts, returns an + empty list. + """ + exit_code, output = sp.getstatusoutput("mount") + return _parse_mount_table(exit_code, output) + + _cifs_table = _generate_cifs_table() @@ -763,8 +778,8 @@ def emptydirs(path, noexist_ok=False): elcont = os.listdir(path) if ex.errno == errno.ENOTEMPTY and not elcont: fmlogger.warning( - 'An exception was raised trying to remove old %s, but the path ' - 'seems empty. Is it an NFS mount?. Passing the exception.', + 'An exception was raised trying to remove old %s, but the path' + ' seems empty. Is it an NFS mount?. Passing the exception.', path) elif ex.errno == errno.ENOTEMPTY and elcont: fmlogger.debug('Folder %s contents (%d items).', path, len(elcont)) diff --git a/nipype/utils/tests/test_filemanip.py b/nipype/utils/tests/test_filemanip.py index acaa261d72..e8c0f7da30 100644 --- a/nipype/utils/tests/test_filemanip.py +++ b/nipype/utils/tests/test_filemanip.py @@ -12,10 +12,9 @@ from ...testing import TempFATFS from ...utils.filemanip import ( save_json, load_json, fname_presuffix, fnames_presuffix, hash_rename, - check_forhash, _cifs_table, on_cifs, copyfile, copyfiles, filename_to_list, - list_to_filename, check_depends, split_filename, get_related_files) - -import numpy as np + check_forhash, _parse_mount_table, _cifs_table, on_cifs, copyfile, + copyfiles, filename_to_list, list_to_filename, check_depends, + split_filename, get_related_files) def _ignore_atime(stat): @@ -123,7 +122,7 @@ def test_copyfiles(_temp_analyze_files, _temp_analyze_files_prime): pth, fname = os.path.split(orig_img2) new_img2 = os.path.join(pth, 'secondfile.img') new_hdr2 = os.path.join(pth, 'secondfile.hdr') - newfiles = copyfiles([orig_img1, orig_img2], [new_img1, new_img2]) + copyfiles([orig_img1, orig_img2], [new_img1, new_img2]) assert os.path.exists(new_img1) assert os.path.exists(new_hdr1) assert os.path.exists(new_img2) @@ -335,6 +334,132 @@ def test_related_files(file, length, expected_files): assert ef in related_files +MOUNT_OUTPUTS = ( +# Linux, no CIFS +(r'''sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) +proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) +udev on /dev type devtmpfs (rw,nosuid,relatime,size=8121732k,nr_inodes=2030433,mode=755) +devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) +tmpfs on /run type tmpfs (rw,nosuid,noexec,relatime,size=1628440k,mode=755) +/dev/nvme0n1p2 on / type ext4 (rw,relatime,errors=remount-ro,data=ordered) +securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) +tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) +tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) +cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/lib/systemd/systemd-cgroups-agent,name=systemd) +pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) +efivarfs on /sys/firmware/efi/efivars type efivarfs (rw,nosuid,nodev,noexec,relatime) +cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpu,cpuacct) +cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) +cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) +cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) +systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct) +hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) +debugfs on /sys/kernel/debug type debugfs (rw,relatime) +mqueue on /dev/mqueue type mqueue (rw,relatime) +fusectl on /sys/fs/fuse/connections type fusectl (rw,relatime) +/dev/nvme0n1p1 on /boot/efi type vfat (rw,relatime,fmask=0077,dmask=0077,codepage=437,iocharset=iso8859-1,shortname=mixed,errors=remount-ro) +/dev/nvme0n1p2 on /var/lib/docker/aufs type ext4 (rw,relatime,errors=remount-ro,data=ordered) +gvfsd-fuse on /run/user/1002/gvfs type fuse.gvfsd-fuse (rw,nosuid,nodev,relatime,user_id=1002,group_id=1002) +''', 0, []), +# OS X, no CIFS +(r'''/dev/disk2 on / (hfs, local, journaled) +devfs on /dev (devfs, local, nobrowse) +map -hosts on /net (autofs, nosuid, automounted, nobrowse) +map auto_home on /home (autofs, automounted, nobrowse) +map -fstab on /Network/Servers (autofs, automounted, nobrowse) +/dev/disk3s2 on /Volumes/MyBookData (hfs, local, nodev, nosuid, journaled) +afni:/elrond0 on /Volumes/afni (nfs) +afni:/var/www/INCOMING on /Volumes/INCOMING (nfs) +afni:/fraid on /Volumes/afni (nfs, asynchronous) +boromir:/raid.bot on /Volumes/raid.bot (nfs) +elros:/volume2/AFNI_SHARE on /Volumes/AFNI_SHARE (nfs) +map -static on /Volumes/safni (autofs, automounted, nobrowse) +map -static on /Volumes/raid.top (autofs, automounted, nobrowse) +/dev/disk1s3 on /Volumes/Boot OS X (hfs, local, journaled, nobrowse) +''', 0, []), +# Non-zero exit code +('', 1, []), +# Variant of Linux example with CIFS added manually +(r'''sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime) +proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) +udev on /dev type devtmpfs (rw,nosuid,relatime,size=8121732k,nr_inodes=2030433,mode=755) +devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000) +tmpfs on /run type tmpfs (rw,nosuid,noexec,relatime,size=1628440k,mode=755) +/dev/nvme0n1p2 on / type ext4 (rw,relatime,errors=remount-ro,data=ordered) +securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime) +tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev) +tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755) +cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/lib/systemd/systemd-cgroups-agent,name=systemd) +pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime) +efivarfs on /sys/firmware/efi/efivars type efivarfs (rw,nosuid,nodev,noexec,relatime) +cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpu,cpuacct) +cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer) +cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids) +cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset) +systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=26,pgrp=1,timeout=0,minproto=5,maxproto=5,direct) +hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime) +debugfs on /sys/kernel/debug type debugfs (rw,relatime) +mqueue on /dev/mqueue type mqueue (rw,relatime) +fusectl on /sys/fs/fuse/connections type fusectl (rw,relatime) +/dev/nvme0n1p1 on /boot/efi type vfat (rw,relatime,fmask=0077,dmask=0077,codepage=437,iocharset=iso8859-1,shortname=mixed,errors=remount-ro) +/dev/nvme0n1p2 on /var/lib/docker/aufs type ext4 (rw,relatime,errors=remount-ro,data=ordered) +gvfsd-fuse on /run/user/1002/gvfs type fuse.gvfsd-fuse (rw,nosuid,nodev,relatime,user_id=1002,group_id=1002) +''', 0, []), +# Variant of OS X example with CIFS added manually +(r'''/dev/disk2 on / (hfs, local, journaled) +devfs on /dev (devfs, local, nobrowse) +afni:/elrond0 on /Volumes/afni (cifs) +afni:/var/www/INCOMING on /Volumes/INCOMING (nfs) +afni:/fraid on /Volumes/afni/fraid (nfs, asynchronous) +boromir:/raid.bot on /Volumes/raid.bot (nfs) +elros:/volume2/AFNI_SHARE on /Volumes/AFNI_SHARE (nfs) +''', 0, [('/Volumes/afni/fraid', 'nfs'), ('/Volumes/afni', 'cifs')]), +# From Windows: docker run --rm -it -v C:\:/data busybox mount +(r'''overlay on / type overlay (rw,relatime,lowerdir=/var/lib/docker/overlay2/l/26UTYITLF24YE7KEGTMHUNHPPG:/var/lib/docker/overlay2/l/SWGNP3T2EEB4CNBJFN3SDZLXHP,upperdir=/var/lib/docker/overlay2/a4c54ab1aa031bb5a14a424abd655510521e183ee4fa4158672e8376c89df394/diff,workdir=/var/lib/docker/overlay2/a4c54ab1aa031bb5a14a424abd655510521e183ee4fa4158672e8376c89df394/work) +proc on /proc type proc (rw,nosuid,nodev,noexec,relatime) +tmpfs on /dev type tmpfs (rw,nosuid,size=65536k,mode=755) +devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=666) +sysfs on /sys type sysfs (ro,nosuid,nodev,noexec,relatime) +tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,relatime,mode=755) +cpuset on /sys/fs/cgroup/cpuset type cgroup (ro,nosuid,nodev,noexec,relatime,cpuset) +cpu on /sys/fs/cgroup/cpu type cgroup (ro,nosuid,nodev,noexec,relatime,cpu) +cpuacct on /sys/fs/cgroup/cpuacct type cgroup (ro,nosuid,nodev,noexec,relatime,cpuacct) +blkio on /sys/fs/cgroup/blkio type cgroup (ro,nosuid,nodev,noexec,relatime,blkio) +memory on /sys/fs/cgroup/memory type cgroup (ro,nosuid,nodev,noexec,relatime,memory) +devices on /sys/fs/cgroup/devices type cgroup (ro,nosuid,nodev,noexec,relatime,devices) +freezer on /sys/fs/cgroup/freezer type cgroup (ro,nosuid,nodev,noexec,relatime,freezer) +net_cls on /sys/fs/cgroup/net_cls type cgroup (ro,nosuid,nodev,noexec,relatime,net_cls) +perf_event on /sys/fs/cgroup/perf_event type cgroup (ro,nosuid,nodev,noexec,relatime,perf_event) +net_prio on /sys/fs/cgroup/net_prio type cgroup (ro,nosuid,nodev,noexec,relatime,net_prio) +hugetlb on /sys/fs/cgroup/hugetlb type cgroup (ro,nosuid,nodev,noexec,relatime,hugetlb) +pids on /sys/fs/cgroup/pids type cgroup (ro,nosuid,nodev,noexec,relatime,pids) +cgroup on /sys/fs/cgroup/systemd type cgroup (ro,nosuid,nodev,noexec,relatime,name=systemd) +mqueue on /dev/mqueue type mqueue (rw,nosuid,nodev,noexec,relatime) +//10.0.75.1/C on /data type cifs (rw,relatime,vers=3.02,sec=ntlmsspi,cache=strict,username=filo,domain=MSI,uid=0,noforceuid,gid=0,noforcegid,addr=10.0.75.1,file_mode=0755,dir_mode=0755,iocharset=utf8,nounix,serverino,mapposix,nobrl,mfsymlinks,noperm,rsize=1048576,wsize=1048576,echo_interval=60,actimeo=1) +/dev/sda1 on /etc/resolv.conf type ext4 (rw,relatime,data=ordered) +/dev/sda1 on /etc/hostname type ext4 (rw,relatime,data=ordered) +/dev/sda1 on /etc/hosts type ext4 (rw,relatime,data=ordered) +shm on /dev/shm type tmpfs (rw,nosuid,nodev,noexec,relatime,size=65536k) +devpts on /dev/console type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=666) +proc on /proc/bus type proc (ro,relatime) +proc on /proc/fs type proc (ro,relatime) +proc on /proc/irq type proc (ro,relatime) +proc on /proc/sys type proc (ro,relatime) +proc on /proc/sysrq-trigger type proc (ro,relatime) +tmpfs on /proc/kcore type tmpfs (rw,nosuid,size=65536k,mode=755) +tmpfs on /proc/timer_list type tmpfs (rw,nosuid,size=65536k,mode=755) +tmpfs on /proc/sched_debug type tmpfs (rw,nosuid,size=65536k,mode=755) +tmpfs on /proc/scsi type tmpfs (ro,relatime) +tmpfs on /sys/firmware type tmpfs (ro,relatime) +''', 0, [('/data', 'cifs')]) +) + + +@pytest.mark.parametrize("output, exit_code, expected", MOUNT_OUTPUTS) +def test_parse_mount_table(output, exit_code, expected): + assert _parse_mount_table(exit_code, output) == expected + + def test_cifs_check(): assert isinstance(_cifs_table, list) assert isinstance(on_cifs('/'), bool)