From 3702f8f9f2f709ff7d234b0ef159c2736952681b Mon Sep 17 00:00:00 2001 From: sgnanase Date: Thu, 12 Oct 2023 19:33:52 +0530 Subject: [PATCH] Improve compilation speed for android build FFTF python script used for flashfiles.zip in IVI. It uses ZipWrite python api to create it. This method is time consuming and adds several unwanted build steps that adds overhead. Like, flash.json generation, unpacking of target files, installer.cmd generation. This fix, changes FFTF python to a normal FFTF bash script, which uses tar + pigz combo for faster flashfiles generation. NOTE: We deprecate flash.json as its used only by PFT & installer.cmd will be maintained in product config repo. Tracked-On: OAM-112083 Signed-off-by: sgnanase --- releasetools/flashfiles_from_target_files.sh | 38 +++++++++++++++++ tasks/flashfiles.mk | 43 ++++++++++---------- 2 files changed, 60 insertions(+), 21 deletions(-) create mode 100755 releasetools/flashfiles_from_target_files.sh diff --git a/releasetools/flashfiles_from_target_files.sh b/releasetools/flashfiles_from_target_files.sh new file mode 100755 index 0000000..02b9e26 --- /dev/null +++ b/releasetools/flashfiles_from_target_files.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +echo "Preparing flashfiles - New Method" +echo "No more Zip. We use tar + pigz" + +echo "========================" +echo "Images / Files to be packed" +echo "========================" +echo "acpio.img" +echo "boot.img" +echo "bootloader.img" +echo "config.img" +echo "flash.json" +echo "gpt.bin" +echo "installer.cmd" +echo "installer.efi " +echo "odm.img " +echo "startup.nsh " +echo "super_empty.img " +echo "super.img " +echo "vbmeta.img " +echo "vendor_boot.img " +echo "========================" + + +#tar +flashfile=`basename $1` +flashfile_dir=`echo $flashfile | cut -d '.' -f1` +PRODUCT_OUT=`dirname $1` + +cd $PRODUCT_OUT +mkdir $flashfile_dir +cp acpio.img boot.img bootloader.img config.img gpt.bin installer.cmd efi/installer.efi odm.img efi/startup.nsh super_empty.img ./obj/PACKAGING/super.img_intermediates/super.img vbmeta.img vendor_boot.img $flashfile_dir/. +tar -cvf - $flashfile_dir/ | /usr/bin/pigz > $flashfile + +echo "========================" +echo "Flashfiles Tar created" +echo "========================" diff --git a/tasks/flashfiles.mk b/tasks/flashfiles.mk index ce2c179..1452b2a 100755 --- a/tasks/flashfiles.mk +++ b/tasks/flashfiles.mk @@ -81,7 +81,8 @@ endif endif ifeq ($(USE_INTEL_FLASHFILES),true) -fftf := $(INTEL_PATH_BUILD)/releasetools/flashfiles_from_target_files +fftf := $(INTEL_PATH_BUILD)/releasetools/flashfiles_from_target_files.sh +#fftf := $(INTEL_PATH_BUILD)/releasetools/flashfiles_from_target_files odf := $(INTEL_PATH_BUILD)/releasetools/ota_deployment_fixup ifneq ($(FLASHFILE_VARIANTS),) @@ -92,7 +93,7 @@ ifneq ($(FLASHFILE_VARIANTS),) $(info Adding $(var)) \ $(eval fn_prefix := $(PRODUCT_OUT)/$(TARGET_PRODUCT)) \ $(eval fn_suffix := $(var)-$(FILE_NAME_TAG)) \ - $(eval ff_zip := $(fn_prefix)-flashfiles-$(fn_suffix).zip) \ + $(eval ff_zip := $(fn_prefix)-flashfiles-$(fn_suffix).tar.gz) \ $(eval INTEL_FACTORY_FLASHFILES_TARGET += $(ff_zip)) \ $(call dist-for-goals,droidcore,$(ff_zip):$(notdir $(ff_zip)))) @@ -134,7 +135,7 @@ ifneq ($(FLASHFILE_VARIANTS),) FLASHFILES_ADD_ARGS := '--unified-variants' endif -INTEL_FACTORY_FLASHFILES_TARGET := $(PRODUCT_OUT)/$(name).zip +INTEL_FACTORY_FLASHFILES_TARGET := $(PRODUCT_OUT)/$(name).tar.gz ifneq ($(SOFIA_FIRMWARE_VARIANTS),) mvcfg_default_arg = $(MV_CONFIG_DEFAULT_TYPE.$(firstword $(SOFIA_FIRMWARE_VARIANTS))) @@ -142,12 +143,15 @@ else mvcfg_default_arg = $(MV_CONFIG_DEFAULT_TYPE) endif +########################################## +########################################## ifeq ($(SUPER_IMG_IN_FLASHZIP),true) $(INTEL_FACTORY_FLASHFILES_TARGET): $(BUILT_TARGET_FILES_PACKAGE) $(fftf) $(UEFI_ADDITIONAL_TOOLS) $(INTERNAL_SUPERIMAGE_DIST_TARGET) $(hide) mkdir -p $(dir $@) - $(fftf) $(FLASHFILES_ADD_ARGS) --mv_config_default=$(notdir $(mvcfg_default_arg)) --add_image=$(INTERNAL_SUPERIMAGE_DIST_TARGET) $(BUILT_TARGET_FILES_PACKAGE) $@ + #$(fftf) $(FLASHFILES_ADD_ARGS) --mv_config_default=$(notdir $(mvcfg_default_arg)) --add_image=$(INTERNAL_SUPERIMAGE_DIST_TARGET) $(BUILT_TARGET_FILES_PACKAGE) $@ + $(fftf) $@ #remove system.img vendor.img product.img from flashfiles.zip - $(hide)zip -d $@ "system.img" "product.img" "vendor.img"; + #$(hide)zip -d $@ "system.img" "product.img" "vendor.img"; else $(INTEL_FACTORY_FLASHFILES_TARGET): $(BUILT_TARGET_FILES_PACKAGE) $(fftf) $(UEFI_ADDITIONAL_TOOLS) $(hide) mkdir -p $(dir $@) @@ -286,7 +290,7 @@ ISO_INSTALL_IMG = $(PRODUCT_OUT)/$(TARGET_PRODUCT)-sign-flashfile-$(FILE_NAME_TA else ISO_INSTALL_IMG = $(PRODUCT_OUT)/$(TARGET_PRODUCT)-flashfile-$(FILE_NAME_TAG).iso endif -ISO_INSTALL_IMG_ZIP = $(ISO_INSTALL_IMG).zip +ISO_INSTALL_IMG_ZIP = $(ISO_INSTALL_IMG).tar.gz ISO_RELEASE_TAR = $(PRODUCT_OUT)/$(TARGET_PRODUCT)-releasefile-$(TARGET_BUILD_VARIANT).iso.tar.gz ISO_EFI = $(PRODUCT_OUT)/iso_tmp.efi @@ -300,7 +304,7 @@ flashfiles: $(INTEL_FACTORY_FLASHFILES_TARGET) $(BUILT_RELEASE_FLASH_FILES_PACKA ifeq (,$(filter apollo_ivi blizzard_ivi celadon_ivi,$(TARGET_PRODUCT))) @echo "Publishing Release files started ..." $(hide) mkdir -p $(TOP)/pub/$(TARGET_PRODUCT)/$(TARGET_BUILD_VARIANT)/Release_Files - $(hide) cp -r $(PRODUCT_OUT)/*-flashfiles-*.zip $(TOP)/pub/$(TARGET_PRODUCT)/$(TARGET_BUILD_VARIANT)/Release_Files + $(hide) cp -r $(PRODUCT_OUT)/*-flashfiles-*.tar.gz $(TOP)/pub/$(TARGET_PRODUCT)/$(TARGET_BUILD_VARIANT)/Release_Files $(hide) cp -r $(PRODUCT_OUT)/scripts $(TOP)/pub/$(TARGET_PRODUCT)/$(TARGET_BUILD_VARIANT)/Release_Files $(hide) cp -r vendor/intel/utils/host $(TOP)/pub/$(TARGET_PRODUCT)/$(TARGET_BUILD_VARIANT)/Release_Files $(hide) mv $(TOP)/pub/$(TARGET_PRODUCT)/$(TARGET_BUILD_VARIANT)/Release_Files/host $(TOP)/pub/$(TARGET_PRODUCT)/$(TARGET_BUILD_VARIANT)/Release_Files/patches @@ -349,7 +353,7 @@ flashfiles: $(INTEL_FACTORY_FLASHFILES_TARGET) publish_gptimage_var publish_mkdi ifeq (,$(filter apollo_ivi blizzard_ivi celadon_ivi,$(TARGET_PRODUCT))) @echo "Publishing Release files started" $(hide) mkdir -p $(TOP)/pub/$(TARGET_PRODUCT)/$(TARGET_BUILD_VARIANT)/Release_Files - $(hide) cp -r $(PRODUCT_OUT)/*-flashfiles-*.zip $(TOP)/pub/$(TARGET_PRODUCT)/$(TARGET_BUILD_VARIANT)/Release_Files + $(hide) cp -r $(PRODUCT_OUT)/*-flashfiles-*.tar.gz $(TOP)/pub/$(TARGET_PRODUCT)/$(TARGET_BUILD_VARIANT)/Release_Files $(hide) cp -r $(PRODUCT_OUT)/scripts $(TOP)/pub/$(TARGET_PRODUCT)/$(TARGET_BUILD_VARIANT)/Release_Files $(hide) cp -r vendor/intel/utils/host $(TOP)/pub/$(TARGET_PRODUCT)/$(TARGET_BUILD_VARIANT)/Release_Files $(hide) mv $(TOP)/pub/$(TARGET_PRODUCT)/$(TARGET_BUILD_VARIANT)/Release_Files/host $(TOP)/pub/$(TARGET_PRODUCT)/$(TARGET_BUILD_VARIANT)/Release_Files/patches @@ -357,12 +361,12 @@ ifeq (,$(filter apollo_ivi blizzard_ivi celadon_ivi,$(TARGET_PRODUCT))) ifneq (,$(wildcard vendor/intel/utils_vertical)) ifneq (,$(wildcard vendor/intel/fw/keybox_provisioning)) @echo "vertical_keybox_provisioning included" - $(hide) tar --exclude=*.git -czf $(TARGET_PRODUCT)-releasefiles-$(TARGET_BUILD_VARIANT).tar.gz scripts *patches $(TARGET_PRODUCT)-flashfiles-*.zip *provisioning + $(hide) tar --exclude=*.git -cvf - scripts *patches $(TARGET_PRODUCT)-flashfiles-*.zip *provisioning | /usr/bin/pigz > $(TARGET_PRODUCT)-releasefiles-$(TARGET_BUILD_VARIANT).tar.gz else - $(hide) tar --exclude=*.git -czf $(TARGET_PRODUCT)-releasefiles-$(TARGET_BUILD_VARIANT).tar.gz scripts *patches $(TARGET_PRODUCT)-flashfiles-*.zip + $(hide) tar --exclude=*.git -cvf - scripts *patches $(TARGET_PRODUCT)-flashfiles-*.zip | /usr/bin/pigz > $(TARGET_PRODUCT)-releasefiles-$(TARGET_BUILD_VARIANT).tar.gz endif else - $(hide) tar --exclude=*.git -czf $(TARGET_PRODUCT)-releasefiles-$(TARGET_BUILD_VARIANT).tar.gz scripts *patches $(TARGET_PRODUCT)-flashfiles-*.zip + $(hide) tar --exclude=*.git -cvf - scripts *patches $(TARGET_PRODUCT)-flashfiles-*.zip | /usr/bin/pigz > $(TARGET_PRODUCT)-releasefiles-$(TARGET_BUILD_VARIANT).tar.gz endif $(hide) cp -r $(TOP)/$(TARGET_PRODUCT)-releasefiles-$(TARGET_BUILD_VARIANT).tar.gz $(TOP)/pub/$(TARGET_PRODUCT)/$(TARGET_BUILD_VARIANT) $(hide) cp -r $(TOP)/$(TARGET_PRODUCT)-releasefiles-$(TARGET_BUILD_VARIANT).tar.gz $(PRODUCT_OUT) @@ -405,13 +409,9 @@ ifneq ($(iso_image),false) $(hide)rm -rf $(ISO_INSTALL_IMG) $(ISO_INSTALL_IMG_ZIP) $(hide)mkdir -p $(PRODUCT_OUT)/efi_images_tmp; -ifeq ($(RELEASE_BUILD),true) - $(hide)unzip $(BUILT_RELEASE_FLASH_FILES_PACKAGE) -d $(PRODUCT_OUT)/efi_images_tmp/ > /dev/null; -else - - $(hide)unzip $(INTEL_FACTORY_FLASHFILES_TARGET) -d $(PRODUCT_OUT)/efi_images_tmp/ > /dev/null; -endif G_size=`echo "$$((1 << 32))"`; \ + flashfile_dir=`echo $(INTEL_FACTORY_FLASHFILES_TARGET) | cut -d '.' -f1`; \ + cp -r $(PRODUCT_OUT)/$(flashfile_dir)/* $(PRODUCT_OUT)/efi_images_tmp/. ; \ for img in `ls $(PRODUCT_OUT)/efi_images_tmp/`;do \ size=`stat -c %s $(PRODUCT_OUT)/efi_images_tmp/$${img}`; \ if [[ $${size} -gt $${G_size} ]]; then \ @@ -441,6 +441,7 @@ endif @echo "Zipping ISO image $(ISO_INSTALL_IMG_ZIP) ..." $(hide)zip -r -j $(ISO_INSTALL_IMG_ZIP) $(ISO_INSTALL_IMG) + $(hide) tar -cvf - $(ISO_INSTALL_IMG) | /usr/bin/pigz > $(ISO_INSTALL_IMG_ZIP) ifeq (,$(filter apollo_ivi blizzard_ivi celadon_ivi,$(TARGET_PRODUCT))) @echo "Zipping ISO release image $(ISO_RELEASE_TAR) ..." $(hide)rm -rf $(ISO_RELEASE_TAR) @@ -448,12 +449,12 @@ ifeq (,$(filter apollo_ivi blizzard_ivi celadon_ivi,$(TARGET_PRODUCT))) ifneq (,$(wildcard vendor/intel/utils_vertical)) ifneq (,$(wildcard vendor/intel/fw/keybox_provisioning)) @echo "vertical_keybox_provisioning included" - $(hide) tar --exclude=*.git -czf $(ISO_RELEASE_TAR) scripts *patches *-flashfile-*.iso *provisioning + $(hide) tar --exclude=*.git -cvf - scripts *patches *-flashfile-*.iso *provisioning | /usr/bin/pigz > $(ISO_RELEASE_TAR) else - $(hide) tar --exclude=*.git -czf $(ISO_RELEASE_TAR) scripts *patches *-flashfile-*.iso + $(hide) tar --exclude=*.git -cvf - scripts *patches *-flashfile-*.iso | /usr/bin/pigz > $(ISO_RELEASE_TAR) endif else - $(hide) tar --exclude=*.git -czf $(ISO_RELEASE_TAR) scripts *patches *-flashfile-*.iso + $(hide) tar --exclude=*.git -cvf - scripts *patches *-flashfile-*.iso | /usr/bin/pigz > $(ISO_RELEASE_TAR) endif endif @echo "make ISO image done ---" @@ -466,4 +467,4 @@ endif @echo "ISO Release files are published" endif - $(hide)rm -rf $(TOP)/$(TARGET_PRODUCT)-releasefiles-$(TARGET_BUILD_VARIANT).tar.gz && rm -rf $(TOP)/Release_Files && rm -rf $(TOP)/$(TARGET_PRODUCT)-flashfiles-*.zip && rm -rf $(TOP)/scripts && rm -rf $(TOP)/*patches && rm -rf $(TOP)/*provisioning && rm -rf $(TOP)/pub/$(TARGET_PRODUCT)/$(TARGET_BUILD_VARIANT)/Release_Files $(TOP)/*-flashfile-*.iso + $(hide)rm -rf $(TOP)/$(TARGET_PRODUCT)-releasefiles-$(TARGET_BUILD_VARIANT).tar.gz && rm -rf $(TOP)/Release_Files && rm -rf $(TOP)/$(TARGET_PRODUCT)-flashfiles-*.tar.gz && rm -rf $(TOP)/scripts && rm -rf $(TOP)/*patches && rm -rf $(TOP)/*provisioning && rm -rf $(TOP)/pub/$(TARGET_PRODUCT)/$(TARGET_BUILD_VARIANT)/Release_Files $(TOP)/*-flashfile-*.iso