diff --git a/pkg/overlay/overlay.go b/pkg/overlay/overlay.go index 289eac9c..3feec586 100644 --- a/pkg/overlay/overlay.go +++ b/pkg/overlay/overlay.go @@ -14,8 +14,6 @@ import ( ispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "golang.org/x/sys/unix" - "stackerbuild.io/stacker/pkg/mount" - "stackerbuild.io/stacker/pkg/squashfs" "stackerbuild.io/stacker/pkg/types" ) @@ -168,23 +166,10 @@ func (o *overlay) snapshot(source string, target string) error { for _, m := range ovl.Manifests { manifest = m } - cacheDir := path.Join(o.config.StackerDir, "layer-bases", "oci") + ociDir := path.Join(o.config.StackerDir, "layer-bases", "oci") for _, layer := range manifest.Layers { - if !squashfs.IsSquashfsMediaType(layer.MediaType) { - continue - } - digest := layer.Digest - contents := overlayPath(o.config.RootFSDir, digest, "overlay") - mounted, err := mount.IsMountpoint(contents) - if err == nil && mounted { - // We have already mounted this atom - continue - } - if hasDirEntries(contents) { - // We have done an unsquashfs of this atom - continue - } - if err := unpackOne(cacheDir, contents, digest, true); err != nil { + err := unpackOne(layer, ociDir, overlayPath(o.config.RootFSDir, layer.Digest, "overlay")) + if err != nil { return errors.Wrapf(err, "Failed mounting %#v", layer) } } diff --git a/pkg/overlay/pack.go b/pkg/overlay/pack.go index 2653b132..0d342a42 100644 --- a/pkg/overlay/pack.go +++ b/pkg/overlay/pack.go @@ -10,6 +10,7 @@ import ( "path/filepath" "runtime" "strings" + "sync" "time" "github.com/klauspost/pgzip" @@ -29,6 +30,8 @@ import ( "stackerbuild.io/stacker/pkg/types" ) +var tarEx sync.Mutex + func safeOverlayName(d digest.Digest) string { // dirs used in overlay lowerdir args can't have : in them, so lets // sanitize it @@ -57,36 +60,19 @@ func (o *overlay) Unpack(tag, name string) error { pool := NewThreadPool(runtime.NumCPU()) - for _, layer := range manifest.Layers { - digest := layer.Digest - contents := overlayPath(o.config.RootFSDir, digest, "overlay") - if squashfs.IsSquashfsMediaType(layer.MediaType) { - // don't really need to do this in parallel, but what - // the hell. - pool.Add(func(ctx context.Context) error { - return unpackOne(cacheDir, contents, digest, true) - }) - } else { - switch layer.MediaType { - case ispec.MediaTypeImageLayer: - fallthrough - case ispec.MediaTypeImageLayerGzip: - // don't extract things that have already been - // extracted - if _, err := os.Stat(contents); err == nil { - continue - } - - // TODO: when the umoci API grows support for uid - // shifting, we can use the fancier features of context - // cancelling in the thread pool... - pool.Add(func(ctx context.Context) error { - return unpackOne(cacheDir, contents, digest, false) - }) - default: - return errors.Errorf("unknown media type %s", layer.MediaType) - } + seen := map[digest.Digest]bool{} + for _, curLayer := range manifest.Layers { + // avoid calling unpackOne twice for the same digest + if seen[curLayer.Digest] { + continue } + seen[curLayer.Digest] = true + + // copy layer to avoid race on pool access. + l := curLayer + pool.Add(func(ctx context.Context) error { + return unpackOne(l, cacheDir, overlayPath(o.config.RootFSDir, l.Digest, "overlay")) + }) } pool.DoneAddingJobs() @@ -154,6 +140,9 @@ func ConvertAndOutput(config types.StackerConfig, tag, name string, layerType ty return err } + log.Debugf("new oci layer %s [%s] created from path %s as part of %s:%s", + desc.Digest, layerType, overlayPath(config.RootFSDir, theLayer.Digest), name, tag) + // slight hack, but this is much faster than a cp, and the // layers are the same, just in different formats err = os.Symlink(overlayPath(config.RootFSDir, theLayer.Digest), overlayPath(config.RootFSDir, desc.Digest)) @@ -232,7 +221,8 @@ func (o *overlay) initializeBasesInOutput(name string, layerTypes []types.LayerT return err } } else { - log.Debugf("converting between %v and %v", sourceLayerType, layerType) + log.Debugf("creating oci image %s (type=%s) by converting %s (type=%s)", + layerType.LayerName(name), layerType, sourceLayerType.LayerName(name), sourceLayerType) err = ConvertAndOutput(o.config, cacheTag, name, layerType) if err != nil { return err @@ -447,6 +437,7 @@ func generateLayer(config types.StackerConfig, oci casext.Engine, mutators []*mu return false, errors.Wrapf(err, "couldn't make new layer overlay dir") } + log.Debugf("renaming %s -> %s", dir, path.Join(target, "overlay")) err = os.Rename(dir, path.Join(target, "overlay")) if err != nil { if !os.IsExist(err) { @@ -477,6 +468,7 @@ func generateLayer(config types.StackerConfig, oci casext.Engine, mutators []*mu for _, desc := range descs[1:] { linkPath := overlayPath(config.RootFSDir, desc.Digest) + log.Debugf("link %s -> %s", linkPath, target) err = os.Symlink(target, linkPath) if err != nil { // as above, this symlink may already exist; if it does, we can @@ -548,7 +540,7 @@ func repackOverlay(config types.StackerConfig, name string, layerTypes []types.L mutators = append(mutators, mutator) } - log.Debugf("Generating overlay_dirs layers") + log.Debugf("Generating overlay_dirs layers for %s", name) mutated := false for i, layerType := range layerTypes { ods, ok := ovl.OverlayDirLayers[layerType] @@ -655,29 +647,57 @@ func repackOverlay(config types.StackerConfig, name string, layerTypes []types.L return ovl.write(config, name) } -func unpackOne(ociDir string, bundlePath string, digest digest.Digest, isSquashfs bool) error { - if isSquashfs { - return squashfs.ExtractSingleSquash( - path.Join(ociDir, "blobs", "sha256", digest.Encoded()), - bundlePath, "overlay") +// unpackOne - unpack a single layer (Descriptor) found in ociDir to extractDir +// +// The result of calling unpackOne is either error or the contents available +// at the provided extractDir. The extractDir should be either empty or +// fully populated with this layer. +func unpackOne(l ispec.Descriptor, ociDir string, extractDir string) error { + // population of a dir is not atomic, at least for tar extraction. + // As a result, we could hasDirEntries(extractDir) at the same time that + // something is un-populating that dir due to a failed extraction (like + // os.RemoveAll below). + // There needs to be a lock on the extract dir (scoped to the overlay storage backend). + // A sync.RWMutex would work well here since it is safe to check as long + // as no one is populating or unpopulating. + if hasDirEntries(extractDir) { + // the directory was already populated. + return nil } - oci, err := umoci.OpenLayout(ociDir) - if err != nil { - return err + if squashfs.IsSquashfsMediaType(l.MediaType) { + return squashfs.ExtractSingleSquash( + path.Join(ociDir, "blobs", "sha256", l.Digest.Encoded()), extractDir) } - defer oci.Close() + switch l.MediaType { + case ispec.MediaTypeImageLayer, ispec.MediaTypeImageLayerGzip: + tarEx.Lock() + defer tarEx.Unlock() - compressed, err := oci.GetBlob(context.Background(), digest) - if err != nil { - return err - } - defer compressed.Close() + oci, err := umoci.OpenLayout(ociDir) + if err != nil { + return err + } + defer oci.Close() - uncompressed, err := pgzip.NewReader(compressed) - if err != nil { + compressed, err := oci.GetBlob(context.Background(), l.Digest) + if err != nil { + return err + } + defer compressed.Close() + + uncompressed, err := pgzip.NewReader(compressed) + if err != nil { + return err + } + + err = layer.UnpackLayer(extractDir, uncompressed, nil) + if err != nil { + if rmErr := os.RemoveAll(extractDir); rmErr != nil { + log.Errorf("Failed to remove dir '%s' after failed extraction: %v", extractDir, rmErr) + } + } return err } - - return layer.UnpackLayer(bundlePath, uncompressed, nil) + return errors.Errorf("unknown media type %s", l.MediaType) } diff --git a/pkg/squashfs/squashfs.go b/pkg/squashfs/squashfs.go index 667e5c21..d08ddd0c 100644 --- a/pkg/squashfs/squashfs.go +++ b/pkg/squashfs/squashfs.go @@ -18,13 +18,17 @@ import ( "github.com/pkg/errors" "golang.org/x/sys/unix" "stackerbuild.io/stacker/pkg/log" + "stackerbuild.io/stacker/pkg/mount" ) var checkZstdSupported sync.Once var zstdIsSuspported bool -var tryKernelMountSquash bool = true -var kernelSquashMountFailed error = errors.New("kernel squash mount failed") +var exPolInfo struct { + once sync.Once + err error + policy *ExtractPolicy +} // ExcludePaths represents a list of paths to exclude in a squashfs listing. // Users should do something like filepath.Walk() over the whole filesystem, @@ -168,52 +172,30 @@ func MakeSquashfs(tempdir string, rootfs string, eps *ExcludePaths, verity Verit return blob, GenerateSquashfsMediaType(compression, verity), rootHash, nil } -// maybeKernelSquashMount - try to mount squashfile with kernel mount -// -// if global tryKernelMountSquash is false, do not try -// if environment variable STACKER_ALLOW_SQUASHFS_KERNEL_MOUNTS is "false", do not try. -// try. If it fails, log message and set tryKernelMountSquash=false. -func maybeKernelSquashMount(squashFile, extractDir string) (bool, error) { - if !tryKernelMountSquash { +func isMountedAtDir(src, dest string) (bool, error) { + dstat, err := os.Stat(dest) + if os.IsNotExist(err) { return false, nil } - - const strTrue, strFalse = "true", "false" - const envName = "STACKER_ALLOW_SQUASHFS_KERNEL_MOUNTS" - envVal := os.Getenv(envName) - if envVal == strFalse { - log.Debugf("Not trying kernel mounts per %s=%s", envName, envVal) - tryKernelMountSquash = false + if !dstat.IsDir() { return false, nil - } else if envVal != strTrue && envVal != "" { - return false, errors.Errorf("%s must be '%s' or '%s', found '%s'", envName, strTrue, strFalse, envVal) - } - - ecmd := []string{"mount", "-tsquashfs", "-oloop,ro", squashFile, extractDir} - var output bytes.Buffer - cmd := exec.Command(ecmd[0], ecmd[1:]...) - cmd.Stdin = nil - cmd.Stdout = &output - cmd.Stderr = cmd.Stdout - err := cmd.Run() - if err == nil { - return true, nil } - exitError, ok := err.(*exec.ExitError) - if !ok { - tryKernelMountSquash = false - return false, errors.Errorf("Unexpected error (no-rc), in exec (%v): %v", ecmd, err) + mounts, err := mount.ParseMounts("/proc/self/mountinfo") + if err != nil { + return false, err } - status, ok := exitError.Sys().(syscall.WaitStatus) - if !ok { - tryKernelMountSquash = false - return false, errors.Errorf("Unexpected error (no-status) in exec (%v): %v", ecmd, err) + fdest, err := filepath.Abs(dest) + if err != nil { + return false, err + } + for _, m := range mounts { + if m.Target == fdest { + return true, nil + } } - // we can't really tell why the mount failed. mount(8) does not give a lot specific rc exits. - log.Debugf("maybeKernelSquashMount(%s) exited %d: %s", squashFile, status.ExitStatus(), strings.TrimRight(output.String(), "\n")) - return false, kernelSquashMountFailed + return false, nil } func findSquashfusePath() string { @@ -300,36 +282,295 @@ func squashFuse(squashFile, extractDir string) (*exec.Cmd, error) { return cmd, nil } -func ExtractSingleSquash(squashFile string, extractDir string, storageType string) error { - err := os.MkdirAll(extractDir, 0755) +type ExtractPolicy struct { + Extractors []SquashExtractor + Extractor SquashExtractor + Excuses map[string]error + initialized bool + mutex sync.Mutex +} + +type SquashExtractor interface { + Name() string + IsAvailable() error + // Mount - Mount or extract path to dest. + // Return nil on "already extracted" + // Return error on failure. + Mount(path, dest string) error +} + +func NewExtractPolicy(args ...string) (*ExtractPolicy, error) { + p := &ExtractPolicy{ + Extractors: []SquashExtractor{}, + Excuses: map[string]error{}, + } + + allEx := []SquashExtractor{ + &KernelExtractor{}, + &SquashFuseExtractor{}, + &UnsquashfsExtractor{}, + } + byName := map[string]SquashExtractor{} + for _, i := range allEx { + byName[i.Name()] = i + } + + for _, i := range args { + extractor, ok := byName[i] + if !ok { + return nil, errors.Errorf("Unknown extractor: '%s'", i) + } + excuse := extractor.IsAvailable() + if excuse != nil { + p.Excuses[i] = excuse + continue + } + p.Extractors = append(p.Extractors, extractor) + } + return p, nil +} + +type UnsquashfsExtractor struct { + mutex sync.Mutex +} + +func (k *UnsquashfsExtractor) Name() string { + return "unsquashfs" +} + +func (k *UnsquashfsExtractor) IsAvailable() error { + if which("unsquashfs") == "" { + return errors.Errorf("no 'unsquashfs' in PATH") + } + return nil +} + +func (k *UnsquashfsExtractor) Mount(squashFile, extractDir string) error { + k.mutex.Lock() + defer k.mutex.Unlock() + + // check if already extracted + empty, err := isEmptyDir(extractDir) + if err != nil { + return errors.Wrapf(err, "Error checking for empty dir") + } + if !empty { + return nil + } + + log.Debugf("unsquashfs %s -> %s", squashFile, extractDir) + cmd := exec.Command("unsquashfs", "-f", "-d", extractDir, squashFile) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Stdin = nil + err = cmd.Run() + + // on failure, remove the directory if err != nil { + if rmErr := os.RemoveAll(extractDir); rmErr != nil { + log.Errorf("Failed to remove %s after failed extraction of %s: %v", extractDir, squashFile, rmErr) + } return err } - if mounted, err := maybeKernelSquashMount(squashFile, extractDir); err == nil && mounted { + // assert that extraction must create files. This way we can assume non-empty dir above + // was populated by unsquashfs. + empty, err = isEmptyDir(extractDir) + if err != nil { + return errors.Errorf("Failed to read %s after successful extraction of %s: %v", + extractDir, squashFile, err) + } + if empty { + return errors.Errorf("%s was an empty fs image", squashFile) + } + + return nil +} + +type KernelExtractor struct { + mutex sync.Mutex +} + +func (k *KernelExtractor) Name() string { + return "kmount" +} + +func (k *KernelExtractor) IsAvailable() error { + if !amHostRoot() { + return errors.Errorf("not host root") + } + return nil +} + +func (k *KernelExtractor) Mount(squashFile, extractDir string) error { + k.mutex.Lock() + defer k.mutex.Unlock() + + if mounted, err := isMountedAtDir(squashFile, extractDir); err != nil { + return err + } else if mounted { return nil - } else if err != kernelSquashMountFailed { + } + + ecmd := []string{"mount", "-tsquashfs", "-oloop,ro", squashFile, extractDir} + var output bytes.Buffer + cmd := exec.Command(ecmd[0], ecmd[1:]...) + cmd.Stdin = nil + cmd.Stdout = &output + cmd.Stderr = cmd.Stdout + err := cmd.Run() + if err == nil { + return nil + } + + var retErr error + + exitError, ok := err.(*exec.ExitError) + if !ok { + retErr = errors.Errorf("kmount(%s) had unexpected error (no-rc), in exec (%v): %v", + squashFile, ecmd, err) + } else if status, ok := exitError.Sys().(syscall.WaitStatus); !ok { + retErr = errors.Errorf("kmount(%s) had unexpected error (no-status), in exec (%v): %v", + squashFile, ecmd, err) + } else { + retErr = errors.Errorf("kmount(%s) exited %d: %v", squashFile, status.ExitStatus(), output.String()) + } + + return retErr +} + +type SquashFuseExtractor struct { + mutex sync.Mutex +} + +func (k *SquashFuseExtractor) Name() string { + return "squashfuse" +} + +func (k *SquashFuseExtractor) IsAvailable() error { + if findSquashfusePath() == "" { + return errors.Errorf("no 'squashfuse' in PATH") + } + return nil +} + +func (k *SquashFuseExtractor) Mount(squashFile, extractDir string) error { + k.mutex.Lock() + defer k.mutex.Unlock() + + if mounted, err := isMountedAtDir(squashFile, extractDir); mounted && err == nil { + log.Debugf("[%s] %s already mounted -> %s", k.Name(), squashFile, extractDir) + return nil + } else if err != nil { return err } cmd, err := squashFuse(squashFile, extractDir) - if err == nil { - if err := cmd.Process.Release(); err != nil { - return errors.Errorf("Failed to release process %s: %v", cmd, err) + if err != nil { + return err + } + + log.Debugf("squashFuse mounted (%d) %s -> %s", cmd.Process.Pid, squashFile, extractDir) + if err := cmd.Process.Release(); err != nil { + return errors.Errorf("Failed to release process %s: %v", cmd, err) + } + return nil +} + +// ExtractSingleSquashPolicy - extract squashfile to extractDir +func ExtractSingleSquashPolicy(squashFile, extractDir string, policy *ExtractPolicy) error { + const initName = "init" + if policy == nil { + return errors.Errorf("policy cannot be nil") + } + + // avoid taking a lock if already initialized (possibly premature optimization) + if !policy.initialized { + policy.mutex.Lock() + // We may have been waiting on the initializer. If so, then the policy will now be initialized. + // if not, then we are the initializer. + if !policy.initialized { + defer policy.mutex.Unlock() + defer func() { + policy.initialized = true + }() + } else { + policy.mutex.Unlock() } - return nil - } else if err != squashNotFound { + } + + err := os.MkdirAll(extractDir, 0755) + if err != nil { return err } - if p := which("unsquashfs"); p != "" { - log.Debugf("Extracting %s -> %s with unsquashfs -f -d %s %s", extractDir, squashFile, extractDir, squashFile) - cmd := exec.Command("unsquashfs", "-f", "-d", extractDir, squashFile) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - cmd.Stdin = nil - return cmd.Run() + + fdest, err := filepath.Abs(extractDir) + if err != nil { + return err + } + + if policy.initialized { + if err, ok := policy.Excuses[initName]; ok { + return err + } + return policy.Extractor.Mount(squashFile, fdest) + } + + // At this point we are the initialzer + if policy.Excuses == nil { + policy.Excuses = map[string]error{} + } + + if policy.Extractors == nil || len(policy.Extractors) == 0 { + policy.Excuses[initName] = errors.Errorf("policy had no extractors") + return policy.Excuses[initName] } - return errors.Errorf("Unable to extract squash archive %s", squashFile) + + var extractor SquashExtractor + allExcuses := []string{} + for _, extractor = range policy.Extractors { + err = extractor.Mount(squashFile, fdest) + if err == nil { + policy.Extractor = extractor + log.Debugf("Selected squashfs extractor %s", extractor.Name()) + return nil + } + policy.Excuses[extractor.Name()] = err + } + + for n, exc := range policy.Excuses { + allExcuses = append(allExcuses, fmt.Sprintf("%s: %v", n, exc)) + } + + // nothing worked. populate Excuses[initName] + policy.Excuses[initName] = errors.Errorf("No suitable extractor found:\n " + strings.Join(allExcuses, "\n ")) + return policy.Excuses[initName] +} + +// ExtractSingleSquash - extract the squashFile to extractDir +// Initialize a extractPolicy struct and then call ExtractSingleSquashPolicy +// wik()th that. +func ExtractSingleSquash(squashFile string, extractDir string) error { + exPolInfo.once.Do(func() { + const envName = "STACKER_SQUASHFS_EXTRACT_POLICY" + const defPolicy = "kmount squashfuse unsquashfs" + val := os.Getenv(envName) + if val == "" { + val = defPolicy + } + exPolInfo.policy, exPolInfo.err = NewExtractPolicy(strings.Fields(val)...) + if exPolInfo.err == nil { + for k, v := range exPolInfo.policy.Excuses { + log.Debugf(" squashfs extractor %s is not available: %v", k, v) + } + } + }) + + if exPolInfo.err != nil { + return exPolInfo.err + } + + return ExtractSingleSquashPolicy(squashFile, extractDir, exPolInfo.policy) } func mksquashfsSupportsZstd() bool { @@ -353,6 +594,19 @@ func mksquashfsSupportsZstd() bool { return zstdIsSuspported } +func isEmptyDir(path string) (bool, error) { + fh, err := os.Open(path) + if err != nil { + return false, err + } + + _, err = fh.ReadDir(1) + if err == io.EOF { + return true, nil + } + return false, err +} + // which - like the unix utility, return empty string for not-found. // this might fit well in lib/, but currently lib's test imports // squashfs creating a import loop. diff --git a/pkg/types/layer_type.go b/pkg/types/layer_type.go index 6d1a1353..652b2749 100644 --- a/pkg/types/layer_type.go +++ b/pkg/types/layer_type.go @@ -15,6 +15,13 @@ type LayerType struct { Verity squashfs.VerityMetadata } +func (lt LayerType) String() string { + if lt.Verity { + return fmt.Sprintf(lt.Type + "+verity") + } + return lt.Type +} + func (lt LayerType) MarshalText() ([]byte, error) { return []byte(fmt.Sprintf("%s+%v", lt.Type, lt.Verity)), nil }