Merge pull request #139 from jyejare/gce_cleanup

Develop: Rebase GCE cleanup

.pre-commit-config.yaml

@@ -15,6 +15,6 @@ repos:
     hooks:
       - id: black
   - repo: https://github.com/pycqa/flake8
-    rev: 3.9.2
+    rev: 7.1.0
     hooks:
       - id: flake8

cloudwash/constants.py

@@ -1,2 +1,3 @@
 aws_data = ['VMS', 'NICS', 'DISCS', 'PIPS', 'RESOURCES', 'STACKS']
 azure_data = ['VMS', 'NICS', 'DISCS', 'IMAGES', 'PIPS', 'RESOURCES']
+gce_data = ['VMS', 'NICS', 'DISCS']

cloudwash/entities/providers.py

@@ -9,6 +9,7 @@
 from cloudwash.entities.resources.stacks import CleanAWSStacks
 from cloudwash.entities.resources.vms import CleanAWSVms
 from cloudwash.entities.resources.vms import CleanAzureVMs
+from cloudwash.entities.resources.vms import CleanGCEVMs
 
 
 class providerCleanup:
@@ -22,6 +23,8 @@ def vms(self):
             return CleanAzureVMs(client=self.client)
         elif 'AWS' in providerclass:
             return CleanAWSVms(client=self.client)
+        elif 'GCE' in providerclass:
+            return CleanGCEVMs(client=self.client)
 
     @property
     def discs(self):
@@ -72,3 +75,9 @@ class AWSCleanup(providerCleanup):
     def __init__(self, client):
         self.client = client
         super().__init__(client)
+
+
+class GCECleanup(providerCleanup):
+    def __init__(self, client):
+        self.client = client
+        super().__init__(client)

cloudwash/entities/resources/vms.py

@@ -23,14 +23,14 @@ def _set_dry(self):
     def list(self):
         pass
 
-    def remove(self):
+    def remove(self, **kwargs):
         for vm_name in self._delete:
-            self.client.get_vm(vm_name).delete()
+            self.client.get_vm(vm_name, **kwargs).delete()
         logger.info(f"Removed VMs: \n{self._delete}")
 
-    def stop(self):
+    def stop(self, **kwargs):
         for vm_name in self._stop:
-            self.client.get_vm(vm_name).stop()
+            self.client.get_vm(vm_name, **kwargs).stop()
         logger.info(f"Stopped VMs: \n{self._stop}")
 
     def skip(self):
@@ -90,3 +90,27 @@ def list(self):
                 elif vm.name.startswith(settings.azure.criteria.vm.delete_vm):
                     self._delete.append(vm.name)
         self._set_dry()
+
+
+class CleanGCEVMs(CleanVMs):
+    def list(self):
+
+        allvms = self.client.list_vms(zones=[self.client.cleaning_zone])
+
+        for vm in allvms:
+            if vm.name in settings.gce.exceptions.vm.vm_list:
+                self._skip.append(vm.name)
+                continue
+            elif total_running_time(vm).minutes >= settings.gce.criteria.vm.sla_minutes:
+                if vm.name in settings.gce.exceptions.vm.stop_list:
+                    self._stop.append(vm.name)
+                    continue
+                elif vm.name.startswith(settings.gce.criteria.vm.delete_vm):
+                    self._delete.append(vm.name)
+        self._set_dry()
+
+    def remove(self):
+        super().remove(zone=self.client.cleaning_zone)
+
+    def stop(self):
+        super().stop(zone=self.client.cleaning_zone)

cloudwash/providers/gce.py

@@ -1,53 +1,31 @@
 """GCE CR Cleanup Utilities"""
 from cloudwash.client import compute_client
 from cloudwash.config import settings
+from cloudwash.constants import gce_data as data
+from cloudwash.entities.providers import GCECleanup
 from cloudwash.logger import logger
 from cloudwash.utils import dry_data
 from cloudwash.utils import echo_dry
 from cloudwash.utils import gce_zones
-from cloudwash.utils import total_running_time
 
 
 def cleanup(**kwargs):
-    is_dry_run = kwargs["dry_run"]
-
+    is_dry_run = kwargs.get("dry_run", False)
+    zones = settings.gce.auth.get('zones', ['all'])
+    if "all" in zones:
+        zones = gce_zones()
+    if kwargs["nics"] or kwargs["_all"]:
+        logger.warning("Cloudwash does not supports NICs operation for GCE yet!")
+    if kwargs["discs"] or kwargs["_all"]:
+        logger.warning("Cloudwash does not supports DISCs operation for GCE yet!")
     with compute_client("gce") as gce_client:
-        if kwargs["vms"] or kwargs["_all"]:
-            allvms = gce_client.list_vms(zones=gce_zones())
-            for vm in allvms:
-                if vm.name in settings.gce.exceptions.vm.vm_list:
-                    dry_data["VMS"]["skip"].append(vm.name)
-                    continue
-                elif total_running_time(vm).minutes >= settings.gce.criteria.vm.sla_minutes:
-                    if vm.name in settings.gce.exceptions.vm.stop_list:
-                        dry_data["VMS"]["stop"].append(vm.name)
-                        if not is_dry_run:
-                            try:
-                                vm.stop()
-                            except TypeError:
-                                logger.info(f"Stopped VM: {vm.name}")
-                            except Exception:
-                                logger.exception(f"Error stopping VM {vm.name}")
-                        continue
-                    elif vm.name.startswith(settings.gce.criteria.vm.delete_vm):
-                        dry_data["VMS"]["delete"].append(vm.name)
-                        if not is_dry_run:
-                            try:
-                                vm.delete()
-                            # There as an issue with GCE API while deleting/stopping the VM
-                            # That it throws TypeError, hence catching that error here
-                            # Remove it once its fixed
-                            except TypeError:
-                                logger.info(f"Deleted VM: {vm.name}")
-                            except Exception:
-                                logger.exception(f"Error deleting VM {vm.name}")
-        if kwargs["nics"] or kwargs["_all"]:
-            logger.warning(
-                "Cloudwash dependency 'WrapanAPI' does not supports NICs operation for GCE yet!"
-            )
-        if kwargs["discs"] or kwargs["_all"]:
-            logger.warning(
-                "Cloudwash dependency 'WrapanAPI' does not supports DISCs operation for GCE yet!"
-            )
-        if is_dry_run:
-            echo_dry(dry_data)
+        for zone in zones:
+            for items in data:
+                dry_data[items]['delete'] = []
+            gce_client.cleaning_zone = zone
+            gcecleanup = GCECleanup(client=gce_client)
+            logger.info(f"\nResources from the zone: {zone}")
+            if kwargs["vms"] or kwargs["_all"]:
+                gcecleanup.vms.cleanup()
+            if is_dry_run:
+                echo_dry(dry_data)

settings.yaml.template

@@ -6,6 +6,8 @@ GCE:
         # Multiline json contents from Service Account key
         SERVICE_ACCOUNT: >
             {}
+        # Multiple zones can be added like ["eastus2"] or ["northeurope", "eastus2", "westus"] or ["all"] for all regions
+        ZONES: ['all']
     CRITERIA:
         VM:
             # The VM to be deleted with prepend string, e.g VM name that starts with 'test'