From 33bf21c2bb160b28f5cd0da17eda855c4b4ece45 Mon Sep 17 00:00:00 2001 From: brehboyy Date: Fri, 10 Nov 2023 17:48:52 +0100 Subject: [PATCH 01/31] backroll quick onboarding --- .gitattributes | 4 +++ .gitignore | 5 ++-- common/config/core/env | 16 +++++------ common/config/sso/Dockerfile | 9 ++++++ common/config/sso/env | 2 ++ common/config/sso/realm.json | 29 +++++++++++++++++++ common/config/ui/env | 8 +++--- docker-compose.yml | 54 ++++++++++++++++++------------------ 8 files changed, 86 insertions(+), 41 deletions(-) create mode 100644 .gitattributes create mode 100644 common/config/sso/Dockerfile create mode 100644 common/config/sso/env create mode 100644 common/config/sso/realm.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..0620874 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,4 @@ +# Set line endings for shell scripts to LF +*.sh text eol=lf +*.env text eol=lf +*.yml text eol=lf diff --git a/.gitignore b/.gitignore index ccd628e..a0fa2cf 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # Avoid compromising secret information -common/** flower/** mariadb/** -keycloak/** \ No newline at end of file +keycloak/** +common/secret/** +install.sh \ No newline at end of file diff --git a/common/config/core/env b/common/config/core/env index 53c7704..4665e3b 100644 --- a/common/config/core/env +++ b/common/config/core/env @@ -1,15 +1,15 @@ ### DATABASE CONFIGURATION [MANDATORY] ### -DB_IP= -DB_PORT= -DB_USER_NAME= -DB_USER_PASSWORD= -DB_BASE= +DB_IP=database +DB_PORT=3306 +DB_USER_NAME=backroll +DB_USER_PASSWORD=backroll +DB_BASE=backroll ### FLOWER AUTH CONFIGURATION [OPTIONAL] ### FLOWER_USER= FLOWER_PASSWORD= ### OPENID [MANDATORY] ### -OPENID_ISSUER= -OPENID_CLIENTID= -OPENID_CLIENTSECRET= +OPENID_CLIENTID=backroll-api +OPENID_CLIENTSECRET=exvxRJEgwNjUXM5e8dETHP3wwkZeRGS8 +OPENID_ISSUER=http://localhost:8081/realms/backroll diff --git a/common/config/sso/Dockerfile b/common/config/sso/Dockerfile new file mode 100644 index 0000000..5bcbbec --- /dev/null +++ b/common/config/sso/Dockerfile @@ -0,0 +1,9 @@ +FROM quay.io/keycloak/keycloak:20.0.0 + +# Fixes logout "invalid parameter: redirect_uri" error but the user name doesn’t show up in the backroll navigation bar. +#FROM quay.io/keycloak/keycloak:17.0 + +COPY realm.json /tmp/ +RUN /opt/keycloak/bin/kc.sh import --file /tmp/realm.json + +CMD [ "start-dev" ] diff --git a/common/config/sso/env b/common/config/sso/env new file mode 100644 index 0000000..a87eea7 --- /dev/null +++ b/common/config/sso/env @@ -0,0 +1,2 @@ +KEYCLOAK_ADMIN=admin +KEYCLOAK_ADMIN_PASSWORD=admin \ No newline at end of file diff --git a/common/config/sso/realm.json b/common/config/sso/realm.json new file mode 100644 index 0000000..18f6b29 --- /dev/null +++ b/common/config/sso/realm.json @@ -0,0 +1,29 @@ +{ + "realm": "backroll", + "enabled": true, + "users": [ + { + "username": "developer", + "enabled": true, + "credentials": [{ "type": "password", "value": "developer" }] + } + ], + "clients": [ + { + "clientId": "backroll-front", + "enabled": true, + "publicClient": true, + "baseUrl": "http://localhost:8080/admin/dashboard", + "redirectUris": ["http://localhost:8080/*"], + "webOrigins": ["*"] + }, + { + "clientId": "backroll-api", + "enabled": true, + "clientAuthenticatorType": "client-secret", + "redirectUris": ["http://localhost:5050/*"], + "webOrigins": ["*"], + "serviceAccountsEnabled": true + } + ] + } \ No newline at end of file diff --git a/common/config/ui/env b/common/config/ui/env index 865cffd..b367268 100644 --- a/common/config/ui/env +++ b/common/config/ui/env @@ -1,5 +1,5 @@ ### BACKROLL UI CONFIGURATION [MANDATORY] ### -API_ENDPOINT_URL= -OPENID_ISSUER= -OPENID_CLIENTID= -OPENID_REALM= \ No newline at end of file +API_ENDPOINT_URL=http://localhost:5050 +OPENID_ISSUER=http://localhost:8081 +OPENID_REALM=backroll +OPENID_CLIENTID=backroll-front \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 259c652..01eaf31 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,38 +3,38 @@ version: "3.9" services: # ## Optional if you already have a working db server - # database: - # image: 'mariadb:10.3' - # container_name: database - # networks: - # - backroll-network - # restart: always - # ports: - # - 3306:3306 - # volumes: - # - ${PWD}/mariadb/:/var/lib/mysql/ - # env_file: - # - ${PWD}/common/config/database/env + database: + image: 'mariadb:10.3' + container_name: database + networks: + - backroll-network + restart: always + ports: + - 3306:3306 + volumes: + - ${PWD}/mariadb/:/var/lib/mysql/ + env_file: + - ${PWD}/common/config/database/env ## Optional if you already have a working keycloak environment - #sso: - # restart: always - # container_name: sso - # image: jboss/keycloak:16.1.1 - # networks: - # - backroll-network - # expose: - # - 8081 - # - 9990 - # ports: - # - 8081:8080 - # - 9990:9990 - # env_file: - # - ${PWD}/common/config/sso/env + sso: + restart: always + container_name: sso + #image: jboss/keycloak:16.1.1 + networks: + - backroll-network + expose: + - 8081 + - 9990 + ports: + - 8081:8080 + - 9990:9990 + env_file: + - ${PWD}/common/config/sso/env redis: restart: always - image: redis:7.0.2 + image: redis:7.2 container_name: redis command: "redis-server" networks: From 0561e7d8c472d53e2ac8833c08bdbdfeeb30d2b0 Mon Sep 17 00:00:00 2001 From: brehboyy Date: Fri, 10 Nov 2023 17:56:18 +0100 Subject: [PATCH 02/31] change sso --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 01eaf31..35c72fd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,7 +19,7 @@ services: ## Optional if you already have a working keycloak environment sso: restart: always - container_name: sso + build: ${PWD}/common/config/sso #image: jboss/keycloak:16.1.1 networks: - backroll-network From 73ed4005016d53bfda7e6de4722a0459e308896e Mon Sep 17 00:00:00 2001 From: brehboyy Date: Fri, 10 Nov 2023 17:58:28 +0100 Subject: [PATCH 03/31] add env file for database --- common/config/database/env | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 common/config/database/env diff --git a/common/config/database/env b/common/config/database/env new file mode 100644 index 0000000..46aa12e --- /dev/null +++ b/common/config/database/env @@ -0,0 +1,5 @@ +MARIADB_ROOT_PASSWORD=root + +MARIADB_DATABASE=backroll +MARIADB_USER=backroll +MARIADB_PASSWORD=backroll \ No newline at end of file From cf175c92f23e9484063b0e61f540b111c5014747 Mon Sep 17 00:00:00 2001 From: brehboyy Date: Fri, 10 Nov 2023 18:41:45 +0100 Subject: [PATCH 04/31] static client secret --- common/config/core/env | 2 +- common/config/sso/realm.json | 1 + docker-compose.yml | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/common/config/core/env b/common/config/core/env index 4665e3b..a2ba4b0 100644 --- a/common/config/core/env +++ b/common/config/core/env @@ -11,5 +11,5 @@ FLOWER_PASSWORD= ### OPENID [MANDATORY] ### OPENID_CLIENTID=backroll-api -OPENID_CLIENTSECRET=exvxRJEgwNjUXM5e8dETHP3wwkZeRGS8 +OPENID_CLIENTSECRET=7UjRzIcCRQBPOwl1HxI7YbMDTuipFjht OPENID_ISSUER=http://localhost:8081/realms/backroll diff --git a/common/config/sso/realm.json b/common/config/sso/realm.json index 18f6b29..1740119 100644 --- a/common/config/sso/realm.json +++ b/common/config/sso/realm.json @@ -21,6 +21,7 @@ "clientId": "backroll-api", "enabled": true, "clientAuthenticatorType": "client-secret", + "secret": "7UjRzIcCRQBPOwl1HxI7YbMDTuipFjht", "redirectUris": ["http://localhost:5050/*"], "webOrigins": ["*"], "serviceAccountsEnabled": true diff --git a/docker-compose.yml b/docker-compose.yml index 35c72fd..1d10ea0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,6 +19,7 @@ services: ## Optional if you already have a working keycloak environment sso: restart: always + image: dimsicloud/backroll:${BACKROLL_VERSION:-latest} build: ${PWD}/common/config/sso #image: jboss/keycloak:16.1.1 networks: From 98c0d0a0f027818d7fa24aef862c938651f13bec Mon Sep 17 00:00:00 2001 From: Pierre Date: Tue, 14 Nov 2023 16:29:32 +0100 Subject: [PATCH 05/31] Fix for Backroll + add new feature --- .gitignore | 7 +- development/docker-compose.yml | 4 + docker-compose.yml | 2 +- src/core/app/cloudstack/virtual_machine.py | 27 +++- src/core/app/kvm/kvm_manage_vm.py | 15 +- src/core/app/main.py | 3 + src/core/app/restore.py | 137 +++++++++++++++++- src/core/app/routes/job.py | 5 + src/core/app/routes/storage.py | 15 ++ src/core/app/routes/task.py | 26 +++- src/core/app/routes/virtual_machine.py | 97 ++++++++++--- src/core/app/task_handler.py | 15 ++ .../virtualmachines/BackupSelector.vue | 50 +++++-- .../virtualmachines/StorageSelector.vue | 84 +++++++++++ .../VirtualMachineSelector.vue | 74 ++++++++++ src/ui/src/pages/admin/tasks/Kickstart.vue | 126 ++++++++++++++-- .../src/pages/admin/tasks/backup/Backup.vue | 17 ++- .../src/pages/admin/tasks/restore/Restore.vue | 17 ++- 18 files changed, 656 insertions(+), 65 deletions(-) create mode 100644 src/ui/src/components/virtualmachines/StorageSelector.vue create mode 100644 src/ui/src/components/virtualmachines/VirtualMachineSelector.vue diff --git a/.gitignore b/.gitignore index ccd628e..9de66e8 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,9 @@ common/** flower/** mariadb/** -keycloak/** \ No newline at end of file +keycloak/** +src/core/.env.dev +src/core/.env.dev.tmpl +src/core/.env.production.tmpl +src/core/dev.entrypoint.sh +src/core/dev.Dockerfile diff --git a/development/docker-compose.yml b/development/docker-compose.yml index d044056..a1a1ed9 100644 --- a/development/docker-compose.yml +++ b/development/docker-compose.yml @@ -35,11 +35,13 @@ services: command: uvicorn app:app --host 0.0.0.0 --port 5050 --reload environment: DEBUG: 1 + PYTHONUNBUFFERED: 1 networks: - backroll-network volumes: - ./common/secret/:/root/.ssh:ro - ../src/core:/usr/src/app + - /mnt:/mnt expose: - "5050" ports: @@ -48,6 +50,8 @@ services: - "redis" env_file: - ./common/config/core/env + logging: + driver: local worker_primary: diff --git a/docker-compose.yml b/docker-compose.yml index 259c652..980034c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,4 @@ -version: "3.9" +version: "3.3" services: diff --git a/src/core/app/cloudstack/virtual_machine.py b/src/core/app/cloudstack/virtual_machine.py index 3e80b87..d512183 100644 --- a/src/core/app/cloudstack/virtual_machine.py +++ b/src/core/app/cloudstack/virtual_machine.py @@ -57,9 +57,6 @@ def stop_vm(connector, identity): raise ValueError(jobquery) except Exception as e: - ddata = e.response.json() - k,val = ddata.popitem() - status_code = e.error error_message = "HTTP {0} response from CloudStack.\nErrorcode {1}: {2}" fmt = error_message.format(e.error['errorcode'], e.error['cserrorcode'], e.error['errortext']) errorcode = str(Exception(fmt)).split(':')[1] @@ -88,6 +85,30 @@ def listPoweredOffVms(connector): vm["connector_id"] = str(connector.id) for e in ['nic', 'details', 'guestosid', 'ostypeid', 'zoneid', 'userid', 'serviceofferingid', 'serviceofferingname', 'osdisplayname', 'pooltype']: vm.pop(e) + print + return cloudstack_vm_list['virtualmachine'] + else: return [] + except Exception as e: + print(e) + print("Unable to connect to Cloudstack. Likely is a timeout issue or wrong url/credentials.") + return [] + +def listAllVms(connector): + try: + cs = endpoint.cloudstack_connector(connector) + cloudstack_vm_list = cs.listVirtualMachines(listall=True) + if "virtualmachine" in cloudstack_vm_list: + for vm in cloudstack_vm_list['virtualmachine']: + vm["uuid"] = vm["id"] + vm["id"] = -1 + vm["cpus"] = vm["cpunumber"] + vm["mem"] = vm["memory"] * 1024 + vm["name"] = vm["instancename"] + vm["pool_id"] = str(connector.pool_id) + vm["connector_id"] = str(connector.id) + for e in ['nic', 'details', 'guestosid', 'ostypeid', 'zoneid', 'userid', 'serviceofferingid', 'serviceofferingname', 'osdisplayname', 'pooltype']: + vm.pop(e) + print return cloudstack_vm_list['virtualmachine'] else: return [] except Exception as e: diff --git a/src/core/app/kvm/kvm_manage_vm.py b/src/core/app/kvm/kvm_manage_vm.py index 961f574..9b39183 100644 --- a/src/core/app/kvm/kvm_manage_vm.py +++ b/src/core/app/kvm/kvm_manage_vm.py @@ -37,13 +37,14 @@ def retrieve_virtualmachine(host): is_cloudstack_instance = False # Check that VM is managed by CloudStack - raw_xml = domain.XMLDesc(0) + raw_xml = domain.XMLDesc(0) xml = minidom.parseString(raw_xml) sysbios_xml = xml.getElementsByTagName('system') - smbiosEntries = sysbios_xml[0].getElementsByTagName('entry') - for smbiosEntry in smbiosEntries: - if "cloudstack" in str(smbiosEntry.firstChild.nodeValue).lower(): - is_cloudstack_instance = True + if len(sysbios_xml) > 0: + smbiosEntries = sysbios_xml[0].getElementsByTagName('entry') + for smbiosEntry in smbiosEntries: + if "cloudstack" in str(smbiosEntry.firstChild.nodeValue).lower(): + is_cloudstack_instance = True # Ignoring VMs managed by CloudStack and name starting with r-/s- as these are VR or SystemVM if is_cloudstack_instance and not re.search("^((?!^r-)(?!^v-)(?!^s-).)*$", domain.name()): continue @@ -84,8 +85,8 @@ def retrieve_virtualmachine(host): domain_list.append(instance) - conn.close() - return domain_list + conn.close() + return domain_list def stop_vm(virtual_machine, hypervisor): try: diff --git a/src/core/app/main.py b/src/core/app/main.py index 9e0ed3b..1b46795 100644 --- a/src/core/app/main.py +++ b/src/core/app/main.py @@ -27,6 +27,8 @@ # Celery Imports from app import celery +import logging, sys + # Set this variable to "threading", "eventlet" or "gevent" to select the # different async modes, or leave it set to None for the application to choose # the best option based on installed packages. @@ -35,6 +37,7 @@ def main(): app.run(app, debug=True) + logging.basicConfig(stream=sys.stdout, level=logging.INFO) celery.conf.beat_schedule = { 'daily_routine_cleaning_backups': { diff --git a/src/core/app/restore.py b/src/core/app/restore.py index 6640ade..ea30af7 100644 --- a/src/core/app/restore.py +++ b/src/core/app/restore.py @@ -38,7 +38,16 @@ from app.cloudstack import virtual_machine as cs_manage_vm @celery.task(name='VM_Restore_Disk', bind=True, max_retries=3, base=QueueOnce) -def restore_disk_vm(self, info, backup_name): +def restore_disk_vm(self, info, backup_name, storage, mode): +#def restore_disk_vm(self, info, backup_name): + + if not info: + raise Exception("Virtual machine not found to process restore") + #mode = "single" + #storage = "test" + print("DEBUG $$$ vm uuid: " + info["uuid"]) + for x in info: + print(x) try: redis_instance = Redis(host='redis', port=6379) unique_task_key = f'''vmlock-{info}''' @@ -52,19 +61,30 @@ def restore_disk_vm(self, info, backup_name): host_info = jsonable_encoder(host.filter_host_by_id(info['host'])) vm_storage_info = kvm_list_disk.getDisk(info, host_info) else: + print("DEBUG POOL ID") host_info = None connector = connectors.filter_connector_by_id(pool.filter_pool_by_id(info["pool_id"]).connector_id) vm_storage_info = cs_manage_vm.getDisk(connector, info) - try: - restore_task(self, info, host_info, vm_storage_info, backup_name) - except Exception: - self.retry(countdown=3**self.request.retries) + + if mode == "mounted": + try: + print("Debug - go to restore_to_path_task") + restore_to_path_task(self, info, host_info, storage, backup_name) + except Exception: + self.retry(countdown=3**self.request.retries) + else: + try: + print("Debug - go to restore_task") + restore_task(self, info, host_info, vm_storage_info, backup_name) + except Exception: + self.retry(countdown=3**self.request.retries) except: raise else: #Duplicated key found in redis - target IS locked right now raise ValueError("This task is already running / scheduled") redis_instance.delete(unique_task_key) + print("Debug - restore_disk_vm - start") except Exception as e: redis_instance.delete(unique_task_key) # potentially log error? @@ -72,9 +92,10 @@ def restore_disk_vm(self, info, backup_name): # def restore_task(self, info, hypervisor, disk_list, backup): def restore_task(self, virtual_machine_info, hypervisor, vm_storage_info, backup_name): - + print("Debug - restore_task - start") vm_storage = storage.retrieveStoragePathFromHostBackupPolicy(virtual_machine_info) borg_repository = vm_storage["path"] + print("DEBUG borg_repository " + borg_repository) try: @@ -175,6 +196,110 @@ def restore_task(self, virtual_machine_info, hypervisor, vm_storage_info, backup command = f"rm -rf {borg_repository}restore/{virtual_machine_info['name']}" request = subprocess.run(command.split()) raise e + print("Debug - restore_task - end") + except Exception as e: + + # Remove restore artifacts + try: + command = f"rm -rf {borg_repository}restore/{virtual_machine_info['name']}" + request = subprocess.run(command.split()) + except Exception as err: + print(err) + + raise e + +# def restore_task_mounted(self, info, hypervisor, disk_list, backup): +def restore_to_path_task(self, virtual_machine_info, hypervisor, storage_path, backup_name): + + print("restore_to_path_task") + + borg_repository = storage_path + "/" + + print("borg repo: " + borg_repository) + + try: + + disk_device = backup_name.split('_')[0] + + print("disk-device: " + disk_device) + + print("vm name: " + virtual_machine_info['name']) + + # Remove existing files inside restore folder + command = f"rm -rf {borg_repository}restore/{virtual_machine_info['name']}" + subprocess.run(command.split()) + + # Create temporary folder to extract borg archive + command = f"mkdir -p {borg_repository}restore/{virtual_machine_info['name']}" + subprocess.run(command.split()) + + # Go into directory + os.chdir(f"{borg_repository}restore/{virtual_machine_info['name']}") + + connector = None + pool_id = None + + if "pool_id" in virtual_machine_info: + pool_id = virtual_machine_info["pool_id"] + else: + pool_id = hypervisor["pool_id"] + + print("Pool id : " + pool_id) + + connector_id = pool.filter_pool_by_id(pool_id).connector_id + if connector_id: + connector = connectors.filter_connector_by_id(connector_id) + + try: + # Extract selected borg archive + cmd = f"""borg extract --sparse --strip-components=2 {borg_repository}{virtual_machine_info['name']}::{backup_name}""" + processBorgExtract = subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT) + while True: + processBorgExtract.stdout.flush() + output = processBorgExtract.stdout.readline() + if output == '' and processBorgExtract.poll() is not None: + break + elif not output and processBorgExtract.poll() is not None: + break + + # Process each file in extraction directory + # For each file exec qemu-img info + # if backing file faire suite algo sinon rien + rootPath = f"""{borg_repository}{virtual_machine_info['name']}""" + print(f"rootPath: {rootPath}") + + for filename in os.listdir(borg_repository): + diskPath = f"""{rootPath}/{filename}""" + print(f"diskPath: {diskPath}") + + getDiskInfoCmd = f"""qemu-img info {diskPath}""" + diskInfoData = subprocess.run(getDiskInfoCmd.split(), capture_output=True, shell=True, text=True) + diskInfo = diskInfoData.stdout + print(f"diskInfo: {diskInfo}") + + indexBackingFile = diskInfo.find("backing file: ") + print(f"indexBackingFile: {indexBackingFile}") + indexFormatSpecificInformation = diskInfo.find("Format specific information") + print(f"indexFormatSpecificInformation: {indexFormatSpecificInformation}") + + if indexBackingFile > 0: + backingFileValue = diskInfo[indexBackingFile:indexFormatSpecificInformation] + print(f"backingFileValue: {backingFileValue}") + + if backingFileValue.len() > 0: + # Copy backing file in new folder + #subprocess.run(['cp', backingFileValue, "/mnt/eu-backup-cs/restore/"], check = True) + + cmd = f"""quemu-img rebase \ -f qcow2 \ -u \ -b $new_backing_file_location \ {diskPath}""" + #subprocess.run(cmd.split()) + + cmd = f"""quemu-img info {diskPath}""" + #subprocess.run(cmd.split()) + + #subprocess.run(['rm', diskPath], check = True) + + except: + raise except Exception as e: diff --git a/src/core/app/routes/job.py b/src/core/app/routes/job.py index 52a125c..fd5b61f 100644 --- a/src/core/app/routes/job.py +++ b/src/core/app/routes/job.py @@ -44,6 +44,11 @@ def retrieve_job(): 'mode': 'single', 'type': 'restore' }, + { + 'name': 'Disk Restore To Specified Path', + 'mode': 'mounted', + 'type': 'restore' + }, ] except Exception as e: raise ValueError(e) diff --git a/src/core/app/routes/storage.py b/src/core/app/routes/storage.py index c97e35d..b4df3f9 100644 --- a/src/core/app/routes/storage.py +++ b/src/core/app/routes/storage.py @@ -17,6 +17,7 @@ #!/usr/bin/env python import shutil +import os import uuid as uuid_pkg from fastapi import HTTPException, Depends from pydantic import BaseModel, Json @@ -45,6 +46,20 @@ class Config: } } +def retrieveStoragePathsFromDb(): + try: + engine = database.init_db_connection() + except Exception as e: + raise ValueError(e) + storagePaths = [] + # Get all storage define in storage bdd + with Session(engine) as session: + storagePathsRequest = select(Storage) + storagePathsFromDb = session.exec(storagePathsRequest) + for path in storagePathsFromDb: + storagePaths.append(path) + return storagePaths + def retrieveStoragePathFromHostBackupPolicy(virtual_machine_info): try: engine = database.init_db_connection() diff --git a/src/core/app/routes/task.py b/src/core/app/routes/task.py index 0f43d40..62978e4 100644 --- a/src/core/app/routes/task.py +++ b/src/core/app/routes/task.py @@ -17,6 +17,8 @@ #!/usr/bin/env python import os +import sys +import logging import uuid as uuid_pkg from fastapi import HTTPException, Depends from pydantic import BaseModel, Json @@ -42,11 +44,15 @@ class restorebackup_start(BaseModel): virtual_machine_id: str backup_name: str + storage: str + mode: str class Config: schema_extra = { "example": { "virtual_machine_id": "3414b922-a39f-11ec-b909-0242ac120002", "backup_name": "vda_VMDiskName_01092842912", + "storage": "path", + "mode": "simple" } } @@ -151,8 +157,26 @@ def start_vm_restore(virtual_machine_id, item: restorebackup_start, identity: Js except ValueError: raise HTTPException(status_code=404, detail='Given uuid is not valid') virtual_machine_id = item.virtual_machine_id + print("DEBUG ID VM : " + virtual_machine_id) + #print("virtual_machine_id: " + virtual_machine_id) backup_name = item.backup_name - res = chain(host.retrieve_host.s(), virtual_machine.dmap.s(virtual_machine.parse_host.s()), virtual_machine.handle_results.s(), virtual_machine.filter_virtual_machine_list.s(virtual_machine_id), restore.restore_disk_vm.s(backup_name)).apply_async() + #print("backup_name: " + backup_name) + storage = item.storage + #print("storage: " + storage) + mode = item.mode + #print("mode: " + mode) + res = chain(host.retrieve_host.s(), virtual_machine.dmap.s(virtual_machine.parse_host.s()), virtual_machine.handle_results.s(), virtual_machine.filter_virtual_machine_list.s(virtual_machine_id), restore.restore_disk_vm.s(backup_name, storage, mode)).apply_async() + #res = chain(host.retrieve_host.s(), virtual_machine.dmap.s(virtual_machine.parse_host.s()), virtual_machine.handle_results.s(), virtual_machine.filter_virtual_machine_list.s(virtual_machine_id), restore.restore_disk_vm.s(backup_name)).apply_async() + return {'Location': app.url_path_for('retrieve_task_status', task_id=res.id)} + +@app.post('/api/v1/tasks/restorespecificpath', status_code=202) +def start_vm_restoreSpecificPath(item: restorebackup_start, identity: Json = Depends(auth.valid_token)): + virtual_machine_id = item.virtual_machine_id + backup_name = item.backup_name + storage = item.storage + mode = item.mode + res = chain(host.retrieve_host.s(), virtual_machine.dmap.s(virtual_machine.parse_host.s()), virtual_machine.handle_results.s(), virtual_machine.filter_virtual_machine_list.s(virtual_machine_id), restore.restore_disk_vm.s(backup_name, storage, mode)).apply_async() + #res = chain(host.retrieve_host.s(), virtual_machine.dmap.s(virtual_machine.parse_host.s()), virtual_machine.handle_results.s(), virtual_machine.filter_virtual_machine_list.s(virtual_machine_id), restore.restore_disk_vm.s(backup_name)).apply_async() return {'Location': app.url_path_for('retrieve_task_status', task_id=res.id)} @app.get('/api/v1/tasks/backup', status_code=200) diff --git a/src/core/app/routes/virtual_machine.py b/src/core/app/routes/virtual_machine.py index 180dacc..e79d6cf 100644 --- a/src/core/app/routes/virtual_machine.py +++ b/src/core/app/routes/virtual_machine.py @@ -18,11 +18,13 @@ #!/usr/bin/env python from fastapi import HTTPException, Depends from fastapi.encoders import jsonable_encoder -from pydantic import Json +from pydantic import BaseModel, Json from celery.result import allow_join_result from celery import subtask, group, chain +import logging + import json from app import app @@ -47,15 +49,29 @@ from app.kvm import kvm_manage_vm from app.kvm import kvm_list_disk +import os + class connectorObject(object): pass +class VirtualMachineStorage: + def __init__(self, name, path): + self.name = name + self.path = path + +class VirtualMachineBackupsRequest(BaseModel): + virtualMachineName: str + storagePath: str + @celery_app.task(name='Filter VMs list') def filter_virtual_machine_list(virtual_machine_list, virtual_machine_id): + print(virtual_machine_id) + vmToFind = {} for vm in virtual_machine_list: + print(vm['uuid']) if vm['uuid'] == virtual_machine_id: - break - return vm + vmToFind = vm + return vmToFind @celery_app.task(name='Parse Host instance(s)') def parse_host(host): @@ -78,6 +94,7 @@ def handle_results(group_id): pool_set = set() connector_list = [] cs_vm_list = [] + # This part merge all hypervisors results into one global virtual machines list with allow_join_result(): restored_group_result = celery_app.GroupResult.restore(group_id) @@ -89,21 +106,30 @@ def handle_results(group_id): global_instance_list += instance_list["virtualmachines"] # This part is dedicated to retrieve VM using host related connectors (AKA Cloudstack discovery) for pool_id in pool_set: - connector = connectors.filter_connector_by_id(pool.filter_pool_by_id(pool_id).connector_id) - if connector not in connector_list: - connector_obj = connectorObject() - # Duplicate object connector to add pool_id property - connector_obj.id = connector.id - connector_obj.name = connector.name - connector_obj.url = connector.url - connector_obj.login = connector.login - connector_obj.password = connector.password - connector_obj.pool_id = pool_id - connector_list.append(connector_obj) + connector_id = pool.filter_pool_by_id(pool_id).connector_id + # if connector_id is not null, we need to retrieve connector object + if connector_id: + connector = connectors.filter_connector_by_id(connector_id) + if connector not in connector_list: + connector_obj = connectorObject() + # Duplicate object connector to add pool_id property + connector_obj.id = connector.id + connector_obj.name = connector.name + connector_obj.url = connector.url + connector_obj.login = connector.login + connector_obj.password = connector.password + connector_obj.pool_id = pool_id + connector_list.append(connector_obj) + for connector in connector_list: - cs_vm_list += cs_manage_vm.listPoweredOffVms(connector) + cs_vm_list += cs_manage_vm.listAllVms(connector) + # Merge KVM and CS discovered virtual machines to a single array - global_instance_list += cs_vm_list + + for vm in cs_vm_list: + if not any(x['uuid'] == vm['uuid'] for x in global_instance_list): + global_instance_list.append(vm) + return global_instance_list @celery_app.task(name='List VMs backups', bind=True, max_retries=3) @@ -176,6 +202,34 @@ def retrieve_virtual_machine_disk(self, virtual_machine_list, virtual_machine_id return virtual_machine except Exception as e: raise ValueError(e) + +@celery_app.task(name='List virtual machines folders', bind=True, max_retries=3) +def retrieve_virtual_machine_paths(self): + try: + storagePaths = [] + storagePathsFromDb = storage.retrieveStoragePathsFromDb() + print(storagePathsFromDb) + for path in storagePathsFromDb: + subFolders = os.scandir(path.path) + for subFolder in subFolders: + configFilePath = subFolder.path + "/config" + print("configFilePath _______ " + configFilePath) + if os.path.exists(configFilePath): + storagePaths.append(VirtualMachineStorage(subFolder.name, subFolder.path)) + return storagePaths + + except Exception as e: + self.retry(countdown=1) + raise ValueError(e) + +@celery_app.task(name='List virtual machine backups', bind=True) +def retrieve_virtual_machine_backups_from_path(self, virtualMachineName:str, storagePath:str): + try: + backup_list = json.loads(borg_core.borg_list_backup(virtualMachineName, storagePath)) + return backup_list + + except Exception as e: + raise ValueError(e) @app.get('/api/v1/virtualmachines/{virtual_machine_id}/breaklock', status_code=202) def break_virtual_machine_borg_lock(virtual_machine_id, identity: Json = Depends(auth.valid_token)): @@ -217,4 +271,13 @@ def list_virtual_machine_repository(virtual_machine_id, identity: Json = Depends def get_virtual_machine_backup_stats(virtual_machine_id, backup_name, identity: Json = Depends(auth.valid_token)): if not virtual_machine_id: raise HTTPException(status_code=404, detail='Virtual machine not found') res = chain(host.retrieve_host.s(), dmap.s(parse_host.s()), handle_results.s(), retrieve_virtual_machine_backup_stats.s(virtual_machine_id, backup_name)).apply_async() - return {'Location': app.url_path_for('retrieve_task_status', task_id=res.id)} \ No newline at end of file + return {'Location': app.url_path_for('retrieve_task_status', task_id=res.id)} + +@app.get('/api/v1/virtualmachinespaths', status_code=202) +def get_virtual_machine_paths(identity: Json = Depends(auth.valid_token)): + return {'paths': retrieve_virtual_machine_paths()} + +@app.post('/api/v1/virtualmachinebackupsfrompath', status_code=202) +def get_virtual_machine_backups_from_path(virtualMachineBackupsRequest: VirtualMachineBackupsRequest, identity: Json = Depends(auth.valid_token)): + res = chain(retrieve_virtual_machine_backups_from_path.s(virtualMachineBackupsRequest.virtualMachineName, virtualMachineBackupsRequest.storagePath)).apply_async() + return {'Location': app.url_path_for('retrieve_task_status', task_id=res.id)} diff --git a/src/core/app/task_handler.py b/src/core/app/task_handler.py index 64c47b5..3ce78eb 100644 --- a/src/core/app/task_handler.py +++ b/src/core/app/task_handler.py @@ -46,6 +46,7 @@ def retrieve_task_info(task_id): def cleanArgs(args): argument = str(args) + if ", ...,)" in argument: argument = argument[len('('):-len(', ...,)')] argument = argument + "}" @@ -61,6 +62,19 @@ def cleanArgs(args): argument = argument.replace("False", 'false') argument = argument.replace("(", '') argument = argument.replace(")", '') + + if"{...}" in argument: + argument = argument.replace("{...}", "") + + if '...", ...}' in argument: + argument = argument.replace('...", ...}', '": "" }') + + if "None" in argument: + argument = argument.replace("None", '"None"') + + if", ...}" in argument: + argument = argument.replace(", ...}", ': ""') + argument.split("}", 1)[0] return argument @@ -161,6 +175,7 @@ def handle_task_failure(task_id, msg): task_result = retrieve_task_info(task_id).decode('ascii') text = json.loads(task_result)['args'] + print("DEBUG TASK Failure: text: " + text) cleanedtext = cleanArgs(text) task_args = json.loads(cleanedtext) diff --git a/src/ui/src/components/virtualmachines/BackupSelector.vue b/src/ui/src/components/virtualmachines/BackupSelector.vue index 9301ae3..ddc274d 100644 --- a/src/ui/src/components/virtualmachines/BackupSelector.vue +++ b/src/ui/src/components/virtualmachines/BackupSelector.vue @@ -2,7 +2,7 @@ - + - + + + + + + + + + + + + + +