Repository: openvstorage/framework

Issue:
Extending vpool failed due to an existing directory
Version: Fargo RC2

Create vpool worked fine, extending failed with the following error:
```
Jan 13 14:19:21 stor-08.be-g8-4 celery[36183]: 2017-01-13 14:19:21 44700 +0100 - stor-08.be-g8-4 - 36268/139817925715712 - lib/storagerouter - 444 - INFO - Add vPool vmstor started
Jan 13 14:19:26 stor-08.be-g8-4 celery[36183]: 2017-01-13 14:19:26 52900 +0100 - stor-08.be-g8-4 - 36268/139817925715712 - extensions/sshclient - 445 - ERROR - Command "'mkdir' '-p' '/mnt/vmstor'" failed with output "" and error "mkdir: cannot create directory '/mnt/vmst
Jan 13 14:19:26 stor-08.be-g8-4 celery[36183]: 2017-01-13 14:19:26 74400 +0100 - stor-08.be-g8-4 - 36183/139817925715712 - celery/celery.worker.job - 822 - ERROR - Task ovs.storagerouter.add_vpool[9c3d9b8b-57cc-445f-a366-53938a39363d] raised unexpected: CalledProcessErro
Jan 13 14:19:26 stor-08.be-g8-4 celery[36183]: Traceback (most recent call last):
Jan 13 14:19:26 stor-08.be-g8-4 celery[36183]:   File "/usr/lib/python2.7/dist-packages/celery/app/trace.py", line 240, in trace_task
Jan 13 14:19:26 stor-08.be-g8-4 celery[36183]:     R = retval = fun(*args, **kwargs)
Jan 13 14:19:26 stor-08.be-g8-4 celery[36183]:   File "/usr/lib/python2.7/dist-packages/celery/app/trace.py", line 438, in __protected_call__
Jan 13 14:19:26 stor-08.be-g8-4 celery[36183]:     return self.run(*args, **kwargs)
Jan 13 14:19:26 stor-08.be-g8-4 celery[36183]:   File "/opt/OpenvStorage/ovs/lib/storagerouter.py", line 614, in add_vpool
Jan 13 14:19:26 stor-08.be-g8-4 celery[36183]:     root_client.dir_create(dirs2create)
Jan 13 14:19:26 stor-08.be-g8-4 celery[36183]:   File "/opt/OpenvStorage/ovs/extensions/generic/sshclient.py", line 319, in dir_create
Jan 13 14:19:26 stor-08.be-g8-4 celery[36183]:     self.run(['mkdir', '-p', directory])
Jan 13 14:19:26 stor-08.be-g8-4 celery[36183]:   File "/opt/OpenvStorage/ovs/extensions/generic/sshclient.py", line 60, in inner_function
Jan 13 14:19:26 stor-08.be-g8-4 celery[36183]:     return outer_function(self, *args, **kwargs)
Jan 13 14:19:26 stor-08.be-g8-4 celery[36183]:   File "/opt/OpenvStorage/ovs/extensions/generic/sshclient.py", line 301, in run
Jan 13 14:19:26 stor-08.be-g8-4 celery[36183]:     raise CalledProcessError(exit_code, command, output)
Jan 13 14:19:26 stor-08.be-g8-4 celery[36183]: CalledProcessError
Jan 13 14:19:27 stor-08.be-g8-4 celery[36183]: 2017-01-13 14:19:27 24900 +0100 - stor-08.be-g8-4 - 36183/139817925715712 - celery/celery.worker.strategy - 823 - INFO - Received task: ovs.storagerouter.add_vpool[a7892684-7807-4f32-88ab-e3105026cce7]
Jan 13 14:19:27 stor-08.be-g8-4 celery[36183]: 2017-01-13 14:19:27 25000 +0100 - stor-08.be-g8-4 - 36183/139817925715712 - celery/celery.pool - 824 - DEBUG - TaskPool: Apply <function _fast_trace_task at 0x7f29e19112a8> (args:('ovs.storagerouter.add_vpool', 'a7892684-780
Jan 13 14:19:27 stor-08.be-g8-4 celery[36183]: 2017-01-13 14:19:27 25100 +0100 - stor-08.be-g8-4 - 36183/139817925715712 - celery/celery.worker.job - 825 - DEBUG - Task accepted: ovs.storagerouter.add_vpool[a7892684-7807-4f32-88ab-e3105026cce7] pid:36269
```


Discussion:
Root cause:
```
root@stor-06:~# ls -al /mnt
ls: cannot access '/mnt/vmstor': Transport endpoint is not connected
total 0
drwxr-xr-x 1 root root  12 Jan 13 15:10 .
drwxr-xr-x 1 root root 246 Jan 11 08:33 ..
d????????? ? ?    ?      ?            ? vmstor
root@stor-06:~# 
root@stor-06:~# 
root@stor-06:~# 
root@stor-06:~# 
root@stor-06:~# cat /proc/mounts 
sysfs /sys sysfs rw,relatime 0 0
proc /proc proc rw,relatime 0 0
udev /dev devtmpfs rw,relatime,size=131999948k,nr_inodes=32999987,mode=755 0 0
devpts /dev/pts devpts rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0
tmpfs /run tmpfs rw,nosuid,noexec,relatime,size=26403964k,mode=755 0 0
/dev/md1 / btrfs rw,relatime,compress=lzo,space_cache,autodefrag,subvolid=257,subvol=/@ 0 0
securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0
tmpfs /dev/shm tmpfs rw,nosuid,nodev 0 0
tmpfs /run/lock tmpfs rw,nosuid,nodev,noexec,relatime,size=5120k 0 0
tmpfs /sys/fs/cgroup tmpfs rw,mode=755 0 0
cgroup /sys/fs/cgroup/systemd cgroup rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/lib/systemd/systemd-cgroups-agent,name=systemd 0 0
pstore /sys/fs/pstore pstore rw,nosuid,nodev,noexec,relatime 0 0
cgroup /sys/fs/cgroup/freezer cgroup rw,nosuid,nodev,noexec,relatime,freezer 0 0
cgroup /sys/fs/cgroup/cpuset cgroup rw,nosuid,nodev,noexec,relatime,cpuset,clone_children 0 0
cgroup /sys/fs/cgroup/blkio cgroup rw,nosuid,nodev,noexec,relatime,blkio 0 0
cgroup /sys/fs/cgroup/hugetlb cgroup rw,nosuid,nodev,noexec,relatime,hugetlb,release_agent=/run/cgmanager/agents/cgm-release-agent.hugetlb 0 0
cgroup /sys/fs/cgroup/net_cls,net_prio cgroup rw,nosuid,nodev,noexec,relatime,net_cls,net_prio 0 0
cgroup /sys/fs/cgroup/memory cgroup rw,nosuid,nodev,noexec,relatime,memory 0 0
cgroup /sys/fs/cgroup/perf_event cgroup rw,nosuid,nodev,noexec,relatime,perf_event,release_agent=/run/cgmanager/agents/cgm-release-agent.perf_event 0 0
cgroup /sys/fs/cgroup/pids cgroup rw,nosuid,nodev,noexec,relatime,pids,release_agent=/run/cgmanager/agents/cgm-release-agent.pids 0 0
cgroup /sys/fs/cgroup/devices cgroup rw,nosuid,nodev,noexec,relatime,devices 0 0
cgroup /sys/fs/cgroup/cpu,cpuacct cgroup rw,nosuid,nodev,noexec,relatime,cpu,cpuacct 0 0
systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=33,pgrp=1,timeout=0,minproto=5,maxproto=5,direct 0 0
mqueue /dev/mqueue mqueue rw,relatime 0 0
debugfs /sys/kernel/debug debugfs rw,relatime 0 0
hugetlbfs /dev/hugepages hugetlbfs rw,relatime 0 0
configfs /sys/kernel/config configfs rw,relatime 0 0
fusectl /sys/fs/fuse/connections fusectl rw,relatime 0 0
/dev/md0 /boot ext4 rw,relatime,data=ordered 0 0
/dev/md1 /home btrfs rw,relatime,compress=lzo,space_cache,autodefrag,subvolid=258,subvol=/@home 0 0
tmpfs /run/user/0 tmpfs rw,nosuid,nodev,relatime,size=26403964k,mode=700 0 0
32880cb6-2d5b-410e-bc1f-8d94f6882105 /mnt/vmstor fuse.32880cb6-2d5b-410e-bc1f-8d94f6882105 rw,nosuid,nodev,relatime,user_id=0,group_id=0,default_permissions,allow_other 0 0
cgmfs /run/cgmanager/fs tmpfs rw,relatime,size=100k,mode=755 0 0
binfmt_misc /proc/sys/fs/binfmt_misc binfmt_misc rw,relatime 0 0
root@stor-06:~# umount /mnt/vmstor
umount: /mnt/vmstor: target is busy
        (In some cases useful info about processes that
         use the device is found by lsof(8) or fuser(1).)
root@stor-06:~# umount -l /mnt/vmstor
root@stor-06:~# 
root@stor-06:~# 
root@stor-06:~# ls -al /mnt
total 0
drwxr-xr-x 1 root root  12 Jan 13 15:10 .
drwxr-xr-x 1 root root 246 Jan 11 08:33 ..
drwxr-xr-x 1 root root   0 Jan  9 18:35 vmstor
root@stor-06:~# 
```

Resolving patch:
diff --git a/ovs/lib/storagerouter.py b/ovs/lib/storagerouter.py
--- a/ovs/lib/storagerouter.py
+++ b/ovs/lib/storagerouter.py
@@ -918,6 +918,10 @@ def remove_storagedriver(storagedriver_guid, offline_storage_router_guids=None):
                 temp_client = SSHClient(sr, username='root')
                 if sr in storage_routers_offline:
                     raise Exception('Storage Router "{0}" passed as "offline Storage Router" appears to be reachable'.format(sr.name))
+                if sr == storage_router:
+                    mtpt_pids = temp_client.run("lsof -t +D '/mnt/{0}' || true".format(vpool.name.replace(r"'", r"'\''")), allow_insecure=True).splitlines()
+                    if len(mtpt_pids) > 0:
+                        raise RuntimeError('vPool cannot be deleted. Following processes keep the vPool mountpoint occupied: {0}'.format(', '.join(mtpt_pids)))
                 with remote(temp_client.ip, [LocalStorageRouterClient]) as rem:
                     sd_key = '/ovs/vpools/{0}/hosts/{1}/config'.format(vpool.guid, sd.storagedriver_id)
                     if Configuration.exists(sd_key) is True:


Test patch: