util: Add new mkdirall variants for the copy functions

This adds versions that recursively `mkdir` and all don't error as
easily. This works around some bugs we were having with file copying.

etcd/fs/fs_test.go

@@ -231,15 +231,15 @@ func TestFs2(t *testing.T) {
 
 	var memFs = afero.NewMemMapFs()
 
-	if err := util.CopyFs(etcdFs, memFs, "/", "/", false); err != nil {
+	if err := util.CopyFs(etcdFs, memFs, "/", "/", false, false); err != nil {
 		t.Errorf("copyfs error: %+v", err)
 		return
 	}
-	if err := util.CopyFs(etcdFs, memFs, "/", "/", true); err != nil {
+	if err := util.CopyFs(etcdFs, memFs, "/", "/", true, false); err != nil {
 		t.Errorf("copyfs2 error: %+v", err)
 		return
 	}
-	if err := util.CopyFs(etcdFs, memFs, "/", "/tmp/d1/", false); err != nil {
+	if err := util.CopyFs(etcdFs, memFs, "/", "/tmp/d1/", false, false); err != nil {
 		t.Errorf("copyfs3 error: %+v", err)
 		return
 	}
@@ -300,11 +300,11 @@ func TestFs3(t *testing.T) {
 
 	var memFs = afero.NewMemMapFs()
 
-	if err := util.CopyFs(etcdFs, memFs, "/tmp/foo/bar", "/", false); err != nil {
+	if err := util.CopyFs(etcdFs, memFs, "/tmp/foo/bar", "/", false, false); err != nil {
 		t.Errorf("copyfs error: %+v", err)
 		return
 	}
-	if err := util.CopyFs(etcdFs, memFs, "/tmp/foo/bar", "/baz/", false); err != nil {
+	if err := util.CopyFs(etcdFs, memFs, "/tmp/foo/bar", "/baz/", false, false); err != nil {
 		t.Errorf("copyfs2 error: %+v", err)
 		return
 	}
@@ -419,7 +419,7 @@ func TestEtcdCopyFs0(t *testing.T) {
 		t.Logf("tree: \n%s", tree)
 
 		var memFs = afero.NewMemMapFs()
-		if err := util.CopyFs(etcdFs, memFs, tt.cpsrc, tt.cpdst, tt.force); err != nil {
+		if err := util.CopyFs(etcdFs, memFs, tt.cpsrc, tt.cpdst, tt.force, false); err != nil {
 			t.Errorf("copyfs error: %+v", err)
 			return
 		}

gapi/helpers.go

@@ -64,6 +64,14 @@ func CopyDirToFs(fs engine.Fs, src, dst string) error {
 	return util.CopyDiskToFs(fs, src, dst, false)
 }
 
+// CopyDirToFsForceAll copies a dir from src path on the local fs to a dst path
+// on fs, but it doesn't error when making a dir that already exists. It also
+// uses `MkdirAll` to prevent some issues.
+// FIXME: This is being added because of issues with CopyDirToFs. POSIX is hard.
+func CopyDirToFsForceAll(fs engine.Fs, src, dst string) error {
+	return util.CopyDiskToFsAll(fs, src, dst, true, true)
+}
+
 // CopyDirContentsToFs copies a dir contents from src path on the local fs to a
 // dst path on fs.
 func CopyDirContentsToFs(fs engine.Fs, src, dst string) error {

util/afero.go

@@ -87,7 +87,7 @@ func stringify(fs afero.Fs, name string, indent []bool) (string, error) {
 // dst has a different set of contents in the same location, the behaviour is
 // currently undefined.
 // TODO: this should be made more rsync like and robust!
-func CopyFs(srcFs, dstFs afero.Fs, src, dst string, force bool) error {
+func CopyFs(srcFs, dstFs afero.Fs, src, dst string, force bool, all bool) error {
 	src = path.Join("/", src)
 	dst = path.Join("/", dst)
 
@@ -103,7 +103,11 @@ func CopyFs(srcFs, dstFs afero.Fs, src, dst string, force bool) error {
 		perm := info.Mode() // get file permissions
 		p := path.Join(dst, name[srcFsLen:])
 		if info.IsDir() {
-			err := dstFs.Mkdir(p, perm)
+			if all {
+				err = dstFs.MkdirAll(p, perm)
+			} else {
+				err = dstFs.Mkdir(p, perm)
+			}
 			if os.IsExist(err) && (name == "/" || force) {
 				return nil
 			}
@@ -170,13 +174,21 @@ func CopyFsContents(srcFs afero.Fs, dstFs afero.Fs, src string, dst string, forc
 // CopyFsToDisk performs exactly as CopyFs, except that the dst fs is our local
 // disk os fs.
 func CopyFsToDisk(srcFs afero.Fs, src, dst string, force bool) error {
-	return CopyFs(srcFs, afero.NewOsFs(), src, dst, force)
+	return CopyFs(srcFs, afero.NewOsFs(), src, dst, force, false)
 }
 
 // CopyDiskToFs performs exactly as CopyFs, except that the src fs is our local
 // disk os fs.
 func CopyDiskToFs(dstFs afero.Fs, src, dst string, force bool) error {
-	return CopyFs(afero.NewOsFs(), dstFs, src, dst, force)
+	return CopyFs(afero.NewOsFs(), dstFs, src, dst, force, false)
+}
+
+// CopyDiskToFsAll performs exactly as CopyDiskToFs, except that it allows you
+// to specify the `all` argument which switches CopyFs to using MkdirAll instead
+// of the regular Mkdir.
+// TODO: This works around weird copying issues. POSIX is hard.
+func CopyDiskToFsAll(dstFs afero.Fs, src, dst string, force, all bool) error {
+	return CopyFs(afero.NewOsFs(), dstFs, src, dst, force, all)
 }
 
 // CopyFsContentsToDisk performs exactly as CopyFs, except that the dst fs is

util/afero_test.go

@@ -53,7 +53,7 @@ func TestCopyFs1(t *testing.T) {
 				t.Errorf("could not MkdirAll %+v", err)
 				return
 			}
-			err = CopyFs(src, dst, tt.srcCopyRoot, tt.dstCopyRoot, tt.force)
+			err = CopyFs(src, dst, tt.srcCopyRoot, tt.dstCopyRoot, tt.force, false)
 			if err != nil {
 				t.Errorf("error copying source %s to dest %s", tt.srcCopyRoot, tt.dstCopyRoot)
 				return
@@ -100,7 +100,7 @@ func TestCopyFs2(t *testing.T) {
 		}
 	}
 
-	if err = CopyFs(src, dst, "", "", false); err != nil {
+	if err = CopyFs(src, dst, "", "", false, false); err != nil {
 		t.Errorf("could not CopyFs: %+v", err)
 		return
 	}