Merge pull request #2595 from OSInside/support_file_includes

Support file includes

build-tests/x86/tumbleweed/test-image-disk-simple/a_custom_file

@@ -0,0 +1 @@
+This is a custom file

build-tests/x86/tumbleweed/test-image-disk-simple/appliance.kiwi

@@ -25,6 +25,7 @@
     </preferences>
     <users>
         <user password="$1$wYJUgpM5$RXMMeASDc035eX.NbYWFl0" home="/root" name="root" groups="root"/>
+        <user password="$1$wYJUgpM5$RXMMeASDc035eX.NbYWFl0" home="/home/bob" name="bob" groups="users"/>
     </users>
     <repository type="rpm-md">
         <source path="obsrepositories:/"/>
@@ -56,6 +57,7 @@
         <package name="kernel-default"/>
         <package name="shim"/>
         <package name="timezone"/>
+        <file name="a_custom_file" target="/home/bob/some_file" owner="bob:users" permissions="444"/>
     </packages>
     <packages type="bootstrap">
         <package name="gawk"/>

doc/source/image_description/elements.rst

@@ -1536,6 +1536,37 @@ elements:
 * The `enable` attribute is mandatory because it should be an explicit
   setting if a module is effectively used or not.
 
+<packages><file>
+~~~~~~~~~~~~~~~~~~~
+.. code:: xml
+
+   <packages type="image"/>
+     <file name="name"/>
+   </packages>
+
+The file element takes the `name` attribute and looks up the
+given name as file on the system. If specified relative {kiwi}
+looks up the name in the image description directory. The file
+is installed using the `rsync` program. The file element has the
+following optional attributes:
+
+owner="user:group"
+  The `owner` attribute can be specified to make the file
+  belonging to the specified owner and group. The ownership of
+  the original file is meaningless in this case. The provided
+  value is passed along to the `chown` program.
+
+permissions="perms"
+  The `permissions` attribute can be specified to store the file
+  with the provided permission. The permission bits of the original
+  file are meaningless in this case. The provided value is passed
+  along to the `chmod` program.
+
+target="some/path"
+  The `target` attribute can be used to specify a target path to
+  install the file to the specified directory and name. Eventually
+  missing parent directories will be created.
+
 <packages><archive>
 ~~~~~~~~~~~~~~~~~~~
 .. code:: xml

kiwi/schema/kiwi.rnc

@@ -273,8 +273,20 @@ div {
 div {
     k.file.name.attribute = k.name.attribute
     k.file.arch.attribute = k.arch.attribute
+    k.file.target.attribute =
+        # include file as target to the root(/) of the image.
+        attribute target { text }
+    k.file.owner.attribute =
+        # set ownership to the file. The value is passed to chown
+        attribute owner { text }
+    k.file.permissions.attribute =
+        # set file permissions. The value is passed to chmod
+        attribute permissions { text }
     k.file.attlist =
         k.file.name.attribute &
+        k.file.target.attribute? &
+        k.file.owner.attribute? &
+        k.file.permissions.attribute? &
         k.file.arch.attribute?
     k.file =
         ## A Pointer to a File
@@ -3560,6 +3572,7 @@ div {
         element packages {
             k.packages.attlist &
             k.archive* &
+            k.file* &
             k.ignore* &
             k.namedCollection* &
             k.collectionModule* &

kiwi/schema/kiwi.rng

@@ -505,9 +505,30 @@ file was fetched</a:documentation>
     <define name="k.file.arch.attribute">
       <ref name="k.arch.attribute"/>
     </define>
+    <define name="k.file.target.attribute">
+      <!-- include file as target to the root(/) of the image. -->
+      <attribute name="target"/>
+    </define>
+    <define name="k.file.owner.attribute">
+      <!-- set ownership to the file. The value is passed to chown -->
+      <attribute name="owner"/>
+    </define>
+    <define name="k.file.permissions.attribute">
+      <!-- set file permissions. The value is passed to chmod -->
+      <attribute name="permissions"/>
+    </define>
     <define name="k.file.attlist">
       <interleave>
         <ref name="k.file.name.attribute"/>
+        <optional>
+          <ref name="k.file.target.attribute"/>
+        </optional>
+        <optional>
+          <ref name="k.file.owner.attribute"/>
+        </optional>
+        <optional>
+          <ref name="k.file.permissions.attribute"/>
+        </optional>
         <optional>
           <ref name="k.file.arch.attribute"/>
         </optional>
@@ -5367,6 +5388,9 @@ alternative bootstrap method for debootstrap</a:documentation>
           <zeroOrMore>
             <ref name="k.archive"/>
           </zeroOrMore>
+          <zeroOrMore>
+            <ref name="k.file"/>
+          </zeroOrMore>
           <zeroOrMore>
             <ref name="k.ignore"/>
           </zeroOrMore>

kiwi/system/prepare.py

@@ -576,9 +576,10 @@ def _install_archives(self, archive_list, archive_target_dir_dict):
                 )
             target_dir = self.root_bind.root_dir
             if archive_target_dir_dict.get(archive):
-                target_dir = os.path.join(
-                    target_dir,
-                    archive_target_dir_dict.get(archive)
+                target_dir = os.path.normpath(
+                    os.sep.join(
+                        [target_dir, archive_target_dir_dict.get(archive)]
+                    )
                 )
             log.info('--> target dir: %s', target_dir)
             tar = ArchiveTar(archive_file)

kiwi/system/setup.py

@@ -23,15 +23,17 @@
 from collections import OrderedDict
 from collections import namedtuple
 from typing import (
-    Any, List, Optional
+    Any, List, Optional, Dict
 )
 
 # project
 import kiwi.defaults as defaults
 
 from kiwi.xml_parse import repository
 from kiwi.utils.fstab import Fstab
-from kiwi.xml_state import XMLState
+from kiwi.xml_state import (
+    XMLState, FileT
+)
 from kiwi.runtime_config import RuntimeConfig
 from kiwi.mount_manager import MountManager
 from kiwi.system.uri import Uri
@@ -106,6 +108,7 @@ def import_description(self) -> None:
 
         self._import_custom_scripts()
         self._import_custom_archives()
+        self._import_custom_files()
         self._import_cdroot_archive()
 
     def script_exists(self, name: str) -> bool:
@@ -205,6 +208,14 @@ def import_cdroot_files(self, target_dir: str) -> None:
             archive.extract(target_dir)
             break
 
+    def import_files(self) -> None:
+        system_files = self.xml_state.get_system_files()
+        bootstrap_files = self.xml_state.get_bootstrap_files()
+        if system_files:
+            self._sync_files(system_files)
+        if bootstrap_files:
+            self._sync_files(bootstrap_files)
+
     def import_overlay_files(
         self, follow_links: bool = False, preserve_owner_group: bool = False
     ) -> None:
@@ -976,6 +987,50 @@ def _import_cdroot_archive(self):
             )
             break
 
+    def _import_custom_files(self):
+        """
+        Import custom file files
+        """
+        file_list = []
+        system_files = self.xml_state.get_system_files()
+        bootstrap_files = self.xml_state.get_bootstrap_files()
+        if system_files:
+            file_list += system_files.keys()
+        if bootstrap_files:
+            file_list += bootstrap_files.keys()
+
+        file_target_dir = os.path.join(
+            self.root_dir, defaults.IMAGE_METADATA_DIR
+        ) + os.sep
+
+        for file in file_list:
+            file_is_absolute = file.startswith(os.sep)
+            if file_is_absolute:
+                file_file = file
+            else:
+                file_file = os.path.join(self.description_dir, file)
+
+            file_exists = os.path.exists(file_file)
+
+            if not file_exists:
+                if self.derived_description_dir and not file_is_absolute:
+                    file_file = self.derived_description_dir + '/' + file
+                    file_exists = os.path.exists(file_file)
+
+            if file_exists:
+                log.info(
+                    '--> Importing {0} file to {1}'.format(
+                        file_file, file_target_dir
+                    )
+                )
+                Command.run(
+                    ['cp', file_file, file_target_dir]
+                )
+            else:
+                raise KiwiImportDescriptionError(
+                    f'Specified file {file_file} does not exist'
+                )
+
     def _import_custom_archives(self):
         """
         Import custom tar archive files
@@ -1358,6 +1413,45 @@ def _get_rpm_database_location(self):
             shared_mount.umount_lazy()
         return dbpath
 
+    def _sync_files(self, file_list: Dict[str, FileT]) -> None:
+        log.info("Installing files")
+        ordered_files = OrderedDict(sorted(file_list.items()))
+        for filename, file_t in list(ordered_files.items()):
+            target = file_t.target
+            target_owner = file_t.owner
+            target_permissions = file_t.permissions
+            file_file = '/'.join(
+                [self.root_dir, 'image', filename]
+            )
+            target_name = self.root_dir
+            if target:
+                target_name = os.path.normpath(
+                    os.sep.join([target_name, target])
+                )
+            log.info(f'--> file: {file_file} -> {target_name}')
+            if target_owner:
+                Command.run(
+                    [
+                        'chroot', self.root_dir,
+                        'chown', target_owner,
+                        file_file.replace(self.root_dir, '')
+                    ]
+                )
+            if target_permissions:
+                Command.run(
+                    [
+                        'chroot', self.root_dir,
+                        'chmod', target_permissions,
+                        file_file.replace(self.root_dir, '')
+                    ]
+                )
+            if os.path.dirname(target_name):
+                Path.create(os.path.dirname(target_name))
+            data = DataSync(file_file, target_name)
+            data.sync_data(
+                options=Defaults.get_sync_options()
+            )
+
     def _sync_overlay_files(
         self, overlay_directory, follow_links=False,
         preserve_owner_group=False, profile=None

kiwi/tasks/system_build.py

@@ -306,6 +306,7 @@ def process(self):
                 setup.setup_plymouth_splash()
                 setup.setup_timezone()
                 setup.setup_permissions()
+                setup.import_files()
 
                 # setup permanent image repositories after cleanup
                 setup.import_repositories_marked_as_imageinclude()

kiwi/tasks/system_prepare.py

@@ -315,6 +315,7 @@ def process(self):
                 setup.setup_plymouth_splash()
                 setup.setup_timezone()
                 setup.setup_permissions()
+                setup.import_files()
 
                 # setup permanent image repositories after cleanup
                 setup.import_repositories_marked_as_imageinclude()