add tests for image unpacking

test important details like file permissions, modified time, and removal
in intermediate layers

Dockerfile

@@ -15,4 +15,11 @@ RUN apk add --no-cache bash tzdata ca-certificates unzip zip gzip tar
 COPY --from=builder assets/ /opt/resource/
 RUN chmod +x /opt/resource/*
 
+FROM resource AS tests
+COPY --from=builder /tests /tests
+ADD . /docker-image-resource
+RUN set -e; for test in /tests/*.test; do \
+		$test -ginkgo.v; \
+	done
+
 FROM resource

cmd/in/main.go

@@ -19,8 +19,8 @@ type InRequest struct {
 }
 
 type InResponse struct {
-	Version  resource.Version `json:"version"`
-	Metadata []MetadataField  `json:"metadata"`
+	Version  resource.Version         `json:"version"`
+	Metadata []resource.MetadataField `json:"metadata"`
 }
 
 type ImageMetadata struct {
@@ -28,11 +28,6 @@ type ImageMetadata struct {
 	User string   `json:"user"`
 }
 
-type MetadataField struct {
-	Name  string `json:"name"`
-	Value string `json:"value"`
-}
-
 func main() {
 	logrus.SetOutput(os.Stderr)
 	logrus.SetFormatter(&logrus.TextFormatter{
@@ -49,6 +44,10 @@ func main() {
 		return
 	}
 
+	if req.Source.Debug {
+		logrus.SetLevel(logrus.DebugLevel)
+	}
+
 	if len(os.Args) < 2 {
 		logrus.Errorf("destination path not specified")
 		os.Exit(1)
@@ -75,7 +74,7 @@ func main() {
 		return
 	}
 
-	err = unpackImage(filepath.Join(dest, "rootfs"), image)
+	err = unpackImage(filepath.Join(dest, "rootfs"), image, req.Source.Debug)
 	if err != nil {
 		logrus.Errorf("failed to extract image: %s", err)
 		os.Exit(1)
@@ -115,6 +114,6 @@ func main() {
 
 	json.NewEncoder(os.Stdout).Encode(InResponse{
 		Version:  req.Version,
-		Metadata: []MetadataField{},
+		Metadata: []resource.MetadataField{},
 	})
 }

cmd/in/unpack.go

@@ -4,6 +4,7 @@ import (
 	"archive/tar"
 	"compress/gzip"
 	"io"
+	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
@@ -18,7 +19,7 @@ import (
 
 const whiteoutPrefix = ".wh."
 
-func unpackImage(dest string, img v1.Image) error {
+func unpackImage(dest string, img v1.Image, debug bool) error {
 	layers, err := img.Layers()
 	if err != nil {
 		return err
@@ -29,7 +30,14 @@ func unpackImage(dest string, img v1.Image) error {
 
 	chown := os.Getuid() == 0
 
-	progress := mpb.New(mpb.WithOutput(os.Stderr))
+	var out io.Writer
+	if debug {
+		out = ioutil.Discard
+	} else {
+		out = os.Stderr
+	}
+
+	progress := mpb.New(mpb.WithOutput(out))
 
 	bars := make([]*mpb.Bar, len(layers))
 
@@ -54,6 +62,8 @@ func unpackImage(dest string, img v1.Image) error {
 	// iterate over layers in reverse order; no need to write things files that
 	// are modified by later layers anyway
 	for i := len(layers) - 1; i >= 0; i-- {
+		logrus.Debugf("extracting layer %d of %d", i+1, len(layers))
+
 		err := extractLayer(dest, layers[i], bars[i], written, removed, chown)
 		if err != nil {
 			return err
@@ -92,20 +102,26 @@ func extractLayer(dest string, layer v1.Layer, bar *mpb.Bar, written, removed ma
 		base := filepath.Base(path)
 		dir := filepath.Dir(path)
 
+		logrus.Debugf("unpacking %s", hdr.Name)
+
 		if strings.HasPrefix(base, whiteoutPrefix) {
 			// layer has marked a file as deleted
 			name := strings.TrimPrefix(base, whiteoutPrefix)
-			removed[filepath.Join(dir, name)] = struct{}{}
+			removedPath := filepath.Join(dir, name)
+			removed[removedPath] = struct{}{}
+			logrus.Debugf("whiting out %s", removedPath)
 			continue
 		}
 
 		if pathIsRemoved(path, removed) {
 			// path has been removed by lower layer
+			logrus.Debugf("skipping removed path %s", path)
 			continue
 		}
 
 		if _, ok := written[path]; ok {
 			// path has already been written by lower layer
+			logrus.Debugf("skipping already-written file %s", path)
 			continue
 		}
 

go.mod

@@ -13,8 +13,8 @@ require (
 	github.com/kr/pretty v0.1.0 // indirect
 	github.com/mattn/go-colorable v0.0.9 // indirect
 	github.com/mattn/go-isatty v0.0.3 // indirect
-	github.com/onsi/ginkgo v1.6.0 // indirect
-	github.com/onsi/gomega v1.4.1 // indirect
+	github.com/onsi/ginkgo v1.6.0
+	github.com/onsi/gomega v1.4.1
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/sirupsen/logrus v1.0.6
 	github.com/stretchr/testify v1.2.2 // indirect

in_test.go

@@ -0,0 +1,153 @@
+package resource_test
+
+import (
+	"bytes"
+	"encoding/json"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"syscall"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	resource "github.com/concourse/registry-image-resource"
+)
+
+var _ = Describe("In", func() {
+	var destDir string
+
+	var req struct {
+		Source  resource.Source
+		Version resource.Version
+	}
+
+	var res struct {
+		Version  resource.Version
+		Metadata []resource.MetadataField
+	}
+
+	rootfsPath := func(path ...string) string {
+		return filepath.Join(append([]string{destDir, "rootfs"}, path...)...)
+	}
+
+	BeforeEach(func() {
+		var err error
+		destDir, err = ioutil.TempDir("", "docker-image-in-dir")
+		Expect(err).ToNot(HaveOccurred())
+	})
+
+	AfterEach(func() {
+		Expect(os.RemoveAll(destDir)).To(Succeed())
+	})
+
+	JustBeforeEach(func() {
+		cmd := exec.Command(bins.In, destDir)
+
+		payload, err := json.Marshal(req)
+		Expect(err).ToNot(HaveOccurred())
+
+		outBuf := new(bytes.Buffer)
+
+		cmd.Stdin = bytes.NewBuffer(payload)
+		cmd.Stdout = outBuf
+		cmd.Stderr = GinkgoWriter
+
+		err = cmd.Run()
+		Expect(err).ToNot(HaveOccurred())
+
+		err = json.Unmarshal(outBuf.Bytes(), &res)
+		Expect(err).ToNot(HaveOccurred())
+	})
+
+	Describe("image metadata", func() {
+		BeforeEach(func() {
+			req.Source.Repository = "concourse/test-image-metadata"
+			req.Version.Digest = latestDigest(req.Source.Repository)
+		})
+
+		It("captures the env and user", func() {
+			var meta struct {
+				User string   `json:"user"`
+				Env  []string `json:"env"`
+			}
+
+			md, err := os.Open(filepath.Join(destDir, "metadata.json"))
+			Expect(err).ToNot(HaveOccurred())
+
+			defer md.Close()
+
+			json.NewDecoder(md).Decode(&meta)
+			Expect(meta.User).To(Equal("someuser"))
+			Expect(meta.Env).To(Equal([]string{
+				"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
+				"FOO=1",
+			}))
+		})
+	})
+
+	Describe("file attributes", func() {
+		BeforeEach(func() {
+			req.Source.Repository = "concourse/test-image-file-perms-mtime"
+			req.Version.Digest = latestDigest(req.Source.Repository)
+		})
+
+		It("keeps file ownership, permissions, and modified times", func() {
+			stat, err := os.Stat(rootfsPath("home", "alex", "birthday"))
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(stat.Mode()).To(Equal(os.FileMode(0603)))
+			Expect(stat.ModTime()).To(BeTemporally("==", time.Date(1991, 06, 03, 05, 30, 30, 0, time.UTC)))
+
+			sys, ok := stat.Sys().(*syscall.Stat_t)
+			Expect(ok).To(BeTrue())
+			Expect(sys.Uid).To(Equal(uint32(1000)))
+			Expect(sys.Gid).To(Equal(uint32(1000)))
+		})
+	})
+
+	Describe("removed files in layers", func() {
+		BeforeEach(func() {
+			req.Source.Repository = "concourse/test-image-whiteout"
+			req.Version.Digest = latestDigest(req.Source.Repository)
+		})
+
+		It("does not restore files that were removed in later layers", func() {
+			infos, err := ioutil.ReadDir(rootfsPath("top-dir-1"))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(infos).To(HaveLen(2))
+
+			stat, err := os.Stat(rootfsPath("top-dir-1", "nested-file"))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(stat.IsDir()).To(BeFalse())
+
+			stat, err = os.Stat(rootfsPath("top-dir-1", "nested-dir"))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(stat.IsDir()).To(BeTrue())
+
+			infos, err = ioutil.ReadDir(rootfsPath("top-dir-1", "nested-dir"))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(infos).To(HaveLen(3))
+
+			stat, err = os.Stat(rootfsPath("top-dir-1", "nested-dir", "file-gone"))
+			Expect(err).To(HaveOccurred())
+
+			stat, err = os.Stat(rootfsPath("top-dir-1", "nested-dir", "file-here"))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(stat.IsDir()).To(BeFalse())
+
+			stat, err = os.Stat(rootfsPath("top-dir-1", "nested-dir", "file-recreated"))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(stat.IsDir()).To(BeFalse())
+
+			stat, err = os.Stat(rootfsPath("top-dir-1", "nested-dir", "file-then-dir"))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(stat.IsDir()).To(BeTrue())
+
+			stat, err = os.Stat(rootfsPath("top-dir-2"))
+			Expect(err).To(HaveOccurred())
+		})
+	})
+})

suite_test.go

@@ -0,0 +1,76 @@
+package resource_test
+
+import (
+	"encoding/json"
+	"os"
+	"testing"
+
+	"github.com/google/go-containerregistry/pkg/name"
+	"github.com/google/go-containerregistry/pkg/v1/remote"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/onsi/gomega/gexec"
+)
+
+var bins struct {
+	In    string `json:"in"`
+	Out   string `json:"out"`
+	Check string `json:"check"`
+}
+
+var _ = SynchronizedBeforeSuite(func() []byte {
+	var err error
+
+	b := bins
+
+	if _, err := os.Stat("/opt/resource/in"); err == nil {
+		b.In = "/opt/resource/in"
+	} else {
+		b.In, err = gexec.Build("github.com/concourse/registry-image-resource/cmd/in")
+		Expect(err).ToNot(HaveOccurred())
+	}
+
+	if _, err := os.Stat("/opt/resource/out"); err == nil {
+		b.Out = "/opt/resource/out"
+	} else {
+		b.Out, err = gexec.Build("github.com/concourse/registry-image-resource/cmd/out")
+		Expect(err).ToNot(HaveOccurred())
+	}
+
+	if _, err := os.Stat("/opt/resource/check"); err == nil {
+		b.Check = "/opt/resource/check"
+	} else {
+		b.Check, err = gexec.Build("github.com/concourse/registry-image-resource/cmd/check")
+		Expect(err).ToNot(HaveOccurred())
+	}
+
+	j, err := json.Marshal(b)
+	Expect(err).ToNot(HaveOccurred())
+
+	return j
+}, func(bp []byte) {
+	err := json.Unmarshal(bp, &bins)
+	Expect(err).ToNot(HaveOccurred())
+})
+
+var _ = AfterSuite(func() {
+	gexec.CleanupBuildArtifacts()
+})
+
+func TestRegistryImageResource(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "RegistryImageResource Suite")
+}
+
+func latestDigest(ref string) string {
+	n, err := name.ParseReference(ref, name.WeakValidation)
+	Expect(err).ToNot(HaveOccurred())
+
+	image, err := remote.Image(n)
+	Expect(err).ToNot(HaveOccurred())
+
+	digest, err := image.Digest()
+	Expect(err).ToNot(HaveOccurred())
+
+	return digest.String()
+}

testdata/file-perms-mtime/Dockerfile

@@ -0,0 +1,6 @@
+FROM alpine
+RUN adduser -D alex
+USER alex
+RUN touch ~/birthday
+RUN chmod 0603 ~/birthday
+RUN touch -t 9106030530.30 ~/birthday

testdata/metadata/Dockerfile

@@ -0,0 +1,4 @@
+FROM alpine
+ENV FOO=1
+RUN adduser -D someuser
+USER someuser

testdata/setup-images

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -e -u
+
+cd $(dirname $0)
+
+for df in $(find . -name Dockerfile); do
+  dir=$(dirname $df)
+  name=concourse/test-image-$(basename $dir)
+  echo "building $dir as $name"
+  docker build -t $name $dir
+  echo "pushing"
+  docker push $name
+done

testdata/whiteout/Dockerfile

@@ -0,0 +1,23 @@
+FROM alpine
+RUN mkdir top-dir-1
+RUN touch top-dir-1/nested-file
+RUN mkdir top-dir-1/nested-dir
+RUN touch top-dir-1/nested-dir/file-gone
+RUN touch top-dir-1/nested-dir/file-recreated
+RUN touch top-dir-1/nested-dir/file-then-dir
+RUN rm -rf top-dir-1/nested-dir
+RUN mkdir top-dir-1/nested-dir
+RUN touch top-dir-1/nested-dir/file-here
+RUN touch top-dir-1/nested-dir/file-recreated
+RUN mkdir top-dir-1/nested-dir/file-then-dir
+RUN mkdir top-dir-2
+RUN touch top-dir-2/file-gone
+RUN mkdir top-dir-2/nested-dir-gone
+RUN touch top-dir-2/nested-dir-gone/nested-file-gone
+RUN rm -rf top-dir-2
+
+# resulting file tree should be:
+# /top-dir-1/nested-file
+# /top-dir-1/nested-dir/file-here
+# /top-dir-1/nested-dir/file-recreated
+# /top-dir-1/nested-dir/file-then-dir

types.go

@@ -3,8 +3,15 @@ package resource
 type Source struct {
 	Repository string `json:"repository"`
 	Tag        string `json:"tag"`
+
+	Debug bool `json:"debug"`
 }
 
 type Version struct {
 	Digest string `json:"digest"`
 }
+
+type MetadataField struct {
+	Name  string `json:"name"`
+	Value string `json:"value"`
+}