Support for images with multiple volumes, addressing inconsistent behavior with CreatUsers plugin

starcluster/cluster.py

@@ -1496,12 +1496,19 @@ def attach_volumes_to_master(self):
         for vol in wait_for_volumes:
             self.ec2.wait_for_volume(vol, state='attached')
 
-    def detach_volumes(self):
+    def detach_volumes(self,external=False):
         """
         Detach all volumes from all nodes
         """
-        for node in self.nodes:
-            node.detach_external_volumes()
+        if external:
+            log.info("Detaching all external volumes")
+            for node in self.nodes:
+                node.detach_external_volumes()
+        else:
+            log.info("Detaching shared volumes only")
+            for node in self.nodes:
+                node.detach_shared_volumes()
+
 
     @print_timing('Restarting cluster')
     def restart_cluster(self, reboot_only=False):
@@ -1553,7 +1560,7 @@ def stop_cluster(self, terminate_unstoppable=False, force=False):
                 log.warn("Cannot run plugins: %s" % e)
             else:
                 raise
-        self.detach_volumes()
+        self.detach_volumes(external=False)
         for node in nodes:
             node.shutdown()
 
@@ -1570,7 +1577,7 @@ def terminate_cluster(self, force=False):
                 log.warn("Cannot run plugins: %s" % e)
             else:
                 raise
-        self.detach_volumes()
+        self.detach_volumes(external=True)
         nodes = self.nodes
         for node in nodes:
             node.terminate()

starcluster/node.py

@@ -891,6 +891,19 @@ def detach_external_volumes(self):
             if vol.status not in ['available', 'detaching']:
                 vol.detach()
 
+    def detach_shared_volumes(self):
+        """
+        Detaches all shared volumes (i.e. from the master) returned by self.get_volumes
+        """
+        if self.is_master():
+            block_devs = self.get_volumes().values()
+            for dev in block_devs:
+                vol_id = dev.volume_id
+                vol = self.ec2.get_volume(vol_id)
+                log.info("Detaching volume %s from %s" % (vol.id, self.alias))
+                if vol.status not in ['available', 'detaching']:
+                    vol.detach()
+
     def delete_root_volume(self):
         """
         Detach and destroy EBS root volume (EBS-backed node only)

starcluster/plugins/users.py

@@ -17,6 +17,7 @@
 
 import os
 import posixpath
+from datetime import datetime
 
 from starcluster import utils
 from starcluster import static
@@ -66,8 +67,10 @@ def run(self, nodes, master, user, user_shell, volumes):
         self._user_shell = user_shell
         self._volumes = volumes
         log.info("Creating %d cluster users" % self._num_users)
+        log.info("Usernames listing %s " % self._usernames)
         newusers = self._get_newusers_batch_file(master, self._usernames,
                                                  user_shell)
+        log.info("newuser_names listing %s " % newusers)
         for node in nodes:
             self.pool.simple_job(node.ssh.execute,
                                  ("echo -n '%s' | newusers" % newusers),
@@ -111,20 +114,25 @@ def _download_user_keys(self, master, usernames):
 
     def _get_newusers_batch_file(self, master, usernames, shell,
                                  batch_file=None):
+        bfilecontents = ''
         batch_file = batch_file or self.BATCH_USER_FILE
         if master.ssh.isfile(batch_file):
             bfile = master.ssh.remote_file(batch_file, 'r')
             bfilecontents = bfile.read()
             bfile.close()
-            return bfilecontents
-        bfilecontents = ''
+            # return bfilecontents
         tmpl = "%(username)s:%(password)s:%(uid)d:%(gid)d:"
         tmpl += "Cluster user account %(username)s:"
         tmpl += "/home/%(username)s:%(shell)s\n"
         shpath = master.ssh.which(shell)[0]
         ctx = dict(shell=shpath)
         base_uid, base_gid = self._get_max_unused_user_id()
+        found_newuser = False
         for user in usernames:
+            if user in bfilecontents:
+	        log.info ("Skipping %s user previously created" % user)
+                continue	
+            found_newuser = True
             home_folder = '/home/%s' % user
             if master.ssh.path_exists(home_folder):
                 s = master.ssh.stat(home_folder)
@@ -141,9 +149,13 @@ def _get_newusers_batch_file(self, master, usernames, shell,
         pardir = posixpath.dirname(batch_file)
         if not master.ssh.isdir(pardir):
             master.ssh.makedirs(pardir)
-        bfile = master.ssh.remote_file(batch_file, 'w')
-        bfile.write(bfilecontents)
-        bfile.close()
+        if master.ssh.isfile(batch_file) and found_newuser:
+            d=datetime.now().isoformat('T')
+            master.ssh.rename(batch_file, batch_file+"."+d+".bk")
+        if not master.ssh.isfile(batch_file):
+            bfile = master.ssh.remote_file(batch_file, 'w')
+            bfile.write(bfilecontents)
+            bfile.close()
         return bfilecontents
 
     def on_add_node(self, node, nodes, master, user, user_shell, volumes):

starcluster/sshutils.py

@@ -271,6 +271,16 @@ def mkdir(self, path, mode=0755, ignore_failure=False):
             if not ignore_failure:
                 raise
 
+    def rename(self, oldpath, newpath, ignore_failure=False):
+        """
+        Rename a file on the remote machine
+        """
+        try:
+            return self.sftp.rename(oldpath,newpath)
+        except IOError:
+            if not ignore_failure:
+                raise
+
     def get_remote_file_lines(self, remote_file, regex=None, matching=True):
         """
         Returns list of lines in a remote_file