Fix lazy-write of tars

Author: eandre

pkg/dockerbuild/tarcopy.go

@@ -265,7 +265,7 @@ func (tc *tarCopier) MkdirAll(dstPath ImagePath, mode fs.FileMode) (err error) {
 			header := &tar.Header{
 				Typeflag: tar.TypeDir,
 				ModTime:  modTime,
-				Name:     (dstPath + "/").String(), // from [archive/tar.FileInfoHeader]
+				Name:     tarHeaderName(dstPath, true),
 				Mode:     int64(mode.Perm()),
 			}
 			tc.entries = append(tc.entries, &tarEntry{
@@ -297,10 +297,8 @@ func (tc *tarCopier) CopyFile(dstPath ImagePath, srcPath HostPath, fi fs.FileInf
 		header.Mode = 0755
 	}
 
-	header.Name = filepath.ToSlash(dstPath.String())
-	entry := &tarEntry{
-		header: header,
-	}
+	header.Name = tarHeaderName(dstPath, fi.IsDir())
+	entry := &tarEntry{header: header}
 	tc.entries = append(tc.entries, entry)
 
 	if fi.IsDir() {
@@ -318,7 +316,7 @@ func (tc *tarCopier) CopyFile(dstPath ImagePath, srcPath HostPath, fi fs.FileInf
 
 func (tc *tarCopier) WriteFile(dstPath ImagePath, mode fs.FileMode, data []byte) (err error) {
 	header := &tar.Header{
-		Name:     filepath.ToSlash(dstPath.String()),
+		Name:     tarHeaderName(dstPath, false),
 		Typeflag: tar.TypeReg,
 		Mode:     int64(mode.Perm()),
 		Size:     int64(len(data)),
@@ -351,8 +349,9 @@ func (tc *tarCopier) Opener() tarball.Opener {
 		}
 	}
 
-	var tv tarstream.TarVec
+	var dvecs []tarstream.Datavec
 	for _, e := range tc.entries {
+		log.Info().Str("file", e.header.Name).Interface("header", e.header).Msg("processing file")
 		// create buffer to write tar header to
 		buf := new(bytes.Buffer)
 		tw := tar.NewWriter(buf)
@@ -367,8 +366,7 @@ func (tc *tarCopier) Opener() tarball.Opener {
 		}
 
 		// add the tar header mem buffer to the tarvec
-		tv.Dvecs = append(tv.Dvecs, memv)
-		tv.Size += memv.GetSize()
+		dvecs = append(dvecs, memv)
 
 		var dataEntry tarstream.Datavec
 		if hostPath, ok := e.hostPath.Get(); ok {
@@ -383,30 +381,31 @@ func (tc *tarCopier) Opener() tarball.Opener {
 
 		if dataEntry != nil {
 			// add the file path info to the tarvec
-			size := dataEntry.GetSize()
-			tv.Size += size
-			tv.Dvecs = append(tv.Dvecs, dataEntry)
-
-			// tar requires file entries to be padded out to
-			// 512 byte offset
-			// if needed, record how much padding is needed
-			// and add to the tarvec
-			if size%512 != 0 {
-				padv := tarstream.PadVec{
-					Size: 512 - (size % 512),
+			dvecs = append(dvecs, dataEntry)
+
+			// tar requires file entries to be padded out to 512 bytes.
+			if !e.header.FileInfo().IsDir() {
+				if size := dataEntry.GetSize(); size%512 != 0 {
+					padv := tarstream.PadVec{
+						Size: 512 - (size % 512),
+					}
+					dvecs = append(dvecs, padv)
 				}
-
-				tv.Dvecs = append(tv.Dvecs, padv)
-				tv.Size += padv.GetSize()
 			}
 		}
 	}
 
-	tv.ComputeSize()
-	tv.Pos = 0
-
+	tv := tarstream.NewTarVec(dvecs)
 	return func() (io.ReadCloser, error) {
 		tv2 := tv.Clone()
 		return tv2, nil
 	}
 }
+
+func tarHeaderName(p ImagePath, isDir bool) string {
+	name := strings.TrimPrefix(filepath.ToSlash(p.String()), "/")
+	if isDir {
+		name += "/"
+	}
+	return name
+}

pkg/tarstream/datavec.go

@@ -1,6 +1,7 @@
 package tarstream
 
 import (
+	"bytes"
 	"io"
 	"os"
 )
@@ -14,58 +15,47 @@ type MemVec struct {
 type PathVec struct {
 	Path string
 	Info os.FileInfo
-	file *os.File
 }
 
 // PadVec is a padding (0s) vec type
 type PadVec struct {
 	Size int64
 }
 
+type DataReader interface {
+	io.ReaderAt
+	io.Closer
+}
+
+func nopCloser(r io.ReaderAt) DataReader {
+	return noopCloser{r}
+}
+
+type noopCloser struct {
+	io.ReaderAt
+}
+
+func (n noopCloser) Close() error { return nil }
+
 // Datavec is an interface for all vector types
 type Datavec interface {
 	Clone() Datavec
 	GetSize() int64
-	Open() error
-	Close()
-	ReadAt(b []byte, off int64) (int, error)
+	Open() (DataReader, error)
 }
 
 // GetSize gets the size of the memory vec
 func (m MemVec) GetSize() int64 {
 	return int64(len(m.Data))
 }
 
-// Open opens a memory vec
-func (m MemVec) Open() error {
-	return nil
-}
-
-// Close closes the memory vec
-func (m MemVec) Close() {
-}
-
 func (m MemVec) Clone() Datavec {
 	return m
 }
 
-// ReadAt reads at an offset of a memory vec
-func (m MemVec) ReadAt(b []byte, off int64) (int, error) {
-	var end int64
-	if int64(len(m.Data))-off > int64(len(b)) {
-		end = off + int64(len(b))
-	} else {
-		end = off + int64(len(m.Data))
-	}
-	if end > int64(len(m.Data)) {
-		end = int64(len(m.Data))
-	}
-
-	n := copy(b, m.Data[off:end])
-	if n == 0 {
-		return n, io.EOF
-	}
-	return n, nil
+// Open opens a memory vec
+func (m MemVec) Open() (DataReader, error) {
+	return nopCloser(bytes.NewReader(m.Data)), nil
 }
 
 // GetSize gets the file size of the path vec
@@ -74,40 +64,12 @@ func (p PathVec) GetSize() int64 {
 }
 
 // Open opens a file represented by a path vec
-func (p *PathVec) Open() error {
-	var err error
-	p.file, err = os.Open(p.Path)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-// Close closes the file represented by the path vec
-func (p *PathVec) Close() {
-	p.file.Close()
-}
-
-// ReadAt reads the file represented by path vec at the given offset
-func (p *PathVec) ReadAt(b []byte, off int64) (int, error) {
-	n, err := p.file.ReadAt(b, off)
-	if err == io.EOF {
-		return n, nil
-	}
-	if n == 0 {
-		return n, io.EOF
-	}
-	return n, err
+func (p *PathVec) Open() (DataReader, error) {
+	return os.Open(p.Path)
 }
 
 func (p *PathVec) Clone() Datavec {
-	// Clone the path vec by creating a new instance with the same path and info
-	return &PathVec{
-		Path: p.Path,
-		Info: p.Info,
-		file: nil,
-	}
+	return p
 }
 
 // GetSize gets the size of the padding vec
@@ -116,27 +78,31 @@ func (p PadVec) GetSize() int64 {
 }
 
 // Open opens the padding vec
-func (p PadVec) Open() error {
-	return nil
+func (p PadVec) Open() (DataReader, error) {
+	return padReader{p.Size}, nil
 }
 
-// Close closes the padding vec
-func (p PadVec) Close() {
+func (p PadVec) Clone() Datavec {
+	return p
 }
 
-// ReadAt read the padding vec at a given offset (which is always 0s)
-func (p PadVec) ReadAt(b []byte, off int64) (int, error) {
-	n := min(int(p.Size-off), len(b))
+type padReader struct {
+	size int64
+}
 
-	if n == 0 {
+func (r padReader) ReadAt(b []byte, off int64) (int, error) {
+	rem := int(r.size - off)
+	if rem == 0 {
 		return 0, io.EOF
 	}
+
+	n := min(rem, len(b))
 	for i := range n {
 		b[i] = 0
 	}
 	return n, nil
 }
 
-func (p PadVec) Clone() Datavec {
-	return p
+func (r padReader) Close() error {
+	return nil
 }