diff --git a/examples/autoedges2.yaml b/examples/autoedges2.yaml new file mode 100644 index 00000000..7eb6b6c6 --- /dev/null +++ b/examples/autoedges2.yaml @@ -0,0 +1,24 @@ +--- +graph: mygraph +resources: + file: + - name: file1 + meta: + autoedge: true + path: "/etc/drbd.conf" + content: | + # this is an mgmt test + state: exists + - name: file2 + meta: + autoedge: true + path: "/tmp/foo/" + content: | + i am f2 + state: exists + pkg: + - name: drbd-utils + meta: + autoedge: true + state: installed +edges: [] diff --git a/file.go b/file.go index c8c6148a..1992c7bd 100644 --- a/file.go +++ b/file.go @@ -374,7 +374,6 @@ func (obj *FileRes) CheckApply(apply bool) (stateok bool, err error) { return false, nil // success } - type FileUUID struct { BaseUUID path string diff --git a/misc.go b/misc.go index 77b17844..092f9621 100644 --- a/misc.go +++ b/misc.go @@ -27,6 +27,39 @@ import ( "time" ) +// return true if a string exists inside a list, otherwise false +func StrInList(needle string, haystack []string) bool { + for _, x := range haystack { + if needle == x { + return true + } + } + return false +} + +// remove any duplicate values in the list +// possibly sub-optimal, O(n^2)? implementation +func StrRemoveDuplicatesInList(list []string) []string { + unique := []string{} + for _, x := range list { + if !StrInList(x, unique) { + unique = append(unique, x) + } + } + return unique +} + +// remove any of the elements in filter, if they exist in list +func StrFilterElementsInList(filter []string, list []string) []string { + result := []string{} + for _, x := range list { + if !StrInList(x, filter) { + result = append(result, x) + } + } + return result +} + // reverse a list of strings func ReverseStringList(in []string) []string { var out []string // empty list @@ -81,6 +114,46 @@ func HasPathPrefix(p, prefix string) bool { return true } +func StrInPathPrefixList(needle string, haystack []string) bool { + for _, x := range haystack { + if HasPathPrefix(x, needle) { + return true + } + } + return false +} + +// remove redundant file path prefixes that are under the tree of other files +func RemoveCommonFilePrefixes(paths []string) []string { + var result = make([]string, len(paths)) + for i := 0; i < len(paths); i++ { // copy, b/c append can modify the args!! + result[i] = paths[i] + } + // is there a string path which is common everywhere? + // if so, remove it, and iterate until nothing common is left + // return what's left over, that's the most common superset +loop: + for { + if len(result) <= 1 { + return result + } + for i := 0; i < len(result); i++ { + var copied = make([]string, len(result)) + for j := 0; j < len(result); j++ { // copy, b/c append can modify the args!! + copied[j] = result[j] + } + noi := append(copied[:i], copied[i+1:]...) // rm i + if StrInPathPrefixList(result[i], noi) { + // delete the element common to everyone + result = noi + continue loop + } + } + break + } + return result +} + // Delta of path prefix, tells you how many path tokens different the prefix is func PathPrefixDelta(p, prefix string) int { @@ -112,6 +185,31 @@ func PathSplitFullReversed(p string) []string { return ReverseStringList(result) } +// add trailing slashes to any likely dirs in a package manager fileList +// if removeDirs is true, instead, don't keep the dirs in our output +func DirifyFileList(fileList []string, removeDirs bool) []string { + dirs := []string{} + for _, file := range fileList { + dir, _ := path.Split(file) // dir + dir = path.Clean(dir) // clean so cmp is easier + if !StrInList(dir, dirs) { + dirs = append(dirs, dir) + } + } + + result := []string{} + for _, file := range fileList { + cleanFile := path.Clean(file) + if !StrInList(cleanFile, dirs) { // we're not a directory! + result = append(result, file) // pass through + } else if !removeDirs { + result = append(result, cleanFile+"/") + } + } + + return result +} + // encode an object as base 64, serialize and then base64 encode func ObjToB64(obj interface{}) (string, bool) { b := bytes.Buffer{} diff --git a/misc_test.go b/misc_test.go index e6fb18f5..1cc6cb04 100644 --- a/misc_test.go +++ b/misc_test.go @@ -20,6 +20,7 @@ package main import ( "fmt" "reflect" + "sort" "testing" ) @@ -33,6 +34,10 @@ func TestMiscT1(t *testing.T) { t.Errorf("Result is incorrect.") } + if Dirname("/foo/") != "/" { + t.Errorf("Result is incorrect.") + } + if Dirname("/") != "" { // TODO: should this equal "/" or "" ? t.Errorf("Result is incorrect.") } @@ -45,6 +50,10 @@ func TestMiscT1(t *testing.T) { t.Errorf("Result is incorrect.") } + if Basename("/foo/") != "foo/" { + t.Errorf("Result is incorrect.") + } + if Basename("/") != "/" { // TODO: should this equal "" or "/" ? t.Errorf("Result is incorrect.") } @@ -100,6 +109,10 @@ func TestMiscT3(t *testing.T) { if HasPathPrefix("/foo/bar/baz/", "/foo/bar/baz/dude") != false { t.Errorf("Result should be false.") } + + if HasPathPrefix("/foo/bar/baz/boo/", "/foo/") != true { + t.Errorf("Result should be true.") + } } func TestMiscT4(t *testing.T) { @@ -225,3 +238,422 @@ func TestMiscT8(t *testing.T) { } } + +func TestMiscT9(t *testing.T) { + fileListIn := []string{ // list taken from drbd-utils package + "/etc/drbd.conf", + "/etc/drbd.d/global_common.conf", + "/lib/drbd/drbd", + "/lib/drbd/drbdadm-83", + "/lib/drbd/drbdadm-84", + "/lib/drbd/drbdsetup-83", + "/lib/drbd/drbdsetup-84", + "/usr/lib/drbd/crm-fence-peer.sh", + "/usr/lib/drbd/crm-unfence-peer.sh", + "/usr/lib/drbd/notify-emergency-reboot.sh", + "/usr/lib/drbd/notify-emergency-shutdown.sh", + "/usr/lib/drbd/notify-io-error.sh", + "/usr/lib/drbd/notify-out-of-sync.sh", + "/usr/lib/drbd/notify-pri-lost-after-sb.sh", + "/usr/lib/drbd/notify-pri-lost.sh", + "/usr/lib/drbd/notify-pri-on-incon-degr.sh", + "/usr/lib/drbd/notify-split-brain.sh", + "/usr/lib/drbd/notify.sh", + "/usr/lib/drbd/outdate-peer.sh", + "/usr/lib/drbd/rhcs_fence", + "/usr/lib/drbd/snapshot-resync-target-lvm.sh", + "/usr/lib/drbd/stonith_admin-fence-peer.sh", + "/usr/lib/drbd/unsnapshot-resync-target-lvm.sh", + "/usr/lib/systemd/system/drbd.service", + "/usr/lib/tmpfiles.d/drbd.conf", + "/usr/sbin/drbd-overview", + "/usr/sbin/drbdadm", + "/usr/sbin/drbdmeta", + "/usr/sbin/drbdsetup", + "/usr/share/doc/drbd-utils/COPYING", + "/usr/share/doc/drbd-utils/ChangeLog", + "/usr/share/doc/drbd-utils/README", + "/usr/share/doc/drbd-utils/drbd.conf.example", + "/usr/share/man/man5/drbd.conf-8.3.5.gz", + "/usr/share/man/man5/drbd.conf-8.4.5.gz", + "/usr/share/man/man5/drbd.conf-9.0.5.gz", + "/usr/share/man/man5/drbd.conf.5.gz", + "/usr/share/man/man8/drbd-8.3.8.gz", + "/usr/share/man/man8/drbd-8.4.8.gz", + "/usr/share/man/man8/drbd-9.0.8.gz", + "/usr/share/man/man8/drbd-overview-9.0.8.gz", + "/usr/share/man/man8/drbd-overview.8.gz", + "/usr/share/man/man8/drbd.8.gz", + "/usr/share/man/man8/drbdadm-8.3.8.gz", + "/usr/share/man/man8/drbdadm-8.4.8.gz", + "/usr/share/man/man8/drbdadm-9.0.8.gz", + "/usr/share/man/man8/drbdadm.8.gz", + "/usr/share/man/man8/drbddisk-8.3.8.gz", + "/usr/share/man/man8/drbddisk-8.4.8.gz", + "/usr/share/man/man8/drbdmeta-8.3.8.gz", + "/usr/share/man/man8/drbdmeta-8.4.8.gz", + "/usr/share/man/man8/drbdmeta-9.0.8.gz", + "/usr/share/man/man8/drbdmeta.8.gz", + "/usr/share/man/man8/drbdsetup-8.3.8.gz", + "/usr/share/man/man8/drbdsetup-8.4.8.gz", + "/usr/share/man/man8/drbdsetup-9.0.8.gz", + "/usr/share/man/man8/drbdsetup.8.gz", + "/etc/drbd.d", + "/usr/share/doc/drbd-utils", + "/var/lib/drbd", + } + sort.Strings(fileListIn) + + fileListOut := []string{ // fixed up manually + "/etc/drbd.conf", + "/etc/drbd.d/global_common.conf", + "/lib/drbd/drbd", + "/lib/drbd/drbdadm-83", + "/lib/drbd/drbdadm-84", + "/lib/drbd/drbdsetup-83", + "/lib/drbd/drbdsetup-84", + "/usr/lib/drbd/crm-fence-peer.sh", + "/usr/lib/drbd/crm-unfence-peer.sh", + "/usr/lib/drbd/notify-emergency-reboot.sh", + "/usr/lib/drbd/notify-emergency-shutdown.sh", + "/usr/lib/drbd/notify-io-error.sh", + "/usr/lib/drbd/notify-out-of-sync.sh", + "/usr/lib/drbd/notify-pri-lost-after-sb.sh", + "/usr/lib/drbd/notify-pri-lost.sh", + "/usr/lib/drbd/notify-pri-on-incon-degr.sh", + "/usr/lib/drbd/notify-split-brain.sh", + "/usr/lib/drbd/notify.sh", + "/usr/lib/drbd/outdate-peer.sh", + "/usr/lib/drbd/rhcs_fence", + "/usr/lib/drbd/snapshot-resync-target-lvm.sh", + "/usr/lib/drbd/stonith_admin-fence-peer.sh", + "/usr/lib/drbd/unsnapshot-resync-target-lvm.sh", + "/usr/lib/systemd/system/drbd.service", + "/usr/lib/tmpfiles.d/drbd.conf", + "/usr/sbin/drbd-overview", + "/usr/sbin/drbdadm", + "/usr/sbin/drbdmeta", + "/usr/sbin/drbdsetup", + "/usr/share/doc/drbd-utils/COPYING", + "/usr/share/doc/drbd-utils/ChangeLog", + "/usr/share/doc/drbd-utils/README", + "/usr/share/doc/drbd-utils/drbd.conf.example", + "/usr/share/man/man5/drbd.conf-8.3.5.gz", + "/usr/share/man/man5/drbd.conf-8.4.5.gz", + "/usr/share/man/man5/drbd.conf-9.0.5.gz", + "/usr/share/man/man5/drbd.conf.5.gz", + "/usr/share/man/man8/drbd-8.3.8.gz", + "/usr/share/man/man8/drbd-8.4.8.gz", + "/usr/share/man/man8/drbd-9.0.8.gz", + "/usr/share/man/man8/drbd-overview-9.0.8.gz", + "/usr/share/man/man8/drbd-overview.8.gz", + "/usr/share/man/man8/drbd.8.gz", + "/usr/share/man/man8/drbdadm-8.3.8.gz", + "/usr/share/man/man8/drbdadm-8.4.8.gz", + "/usr/share/man/man8/drbdadm-9.0.8.gz", + "/usr/share/man/man8/drbdadm.8.gz", + "/usr/share/man/man8/drbddisk-8.3.8.gz", + "/usr/share/man/man8/drbddisk-8.4.8.gz", + "/usr/share/man/man8/drbdmeta-8.3.8.gz", + "/usr/share/man/man8/drbdmeta-8.4.8.gz", + "/usr/share/man/man8/drbdmeta-9.0.8.gz", + "/usr/share/man/man8/drbdmeta.8.gz", + "/usr/share/man/man8/drbdsetup-8.3.8.gz", + "/usr/share/man/man8/drbdsetup-8.4.8.gz", + "/usr/share/man/man8/drbdsetup-9.0.8.gz", + "/usr/share/man/man8/drbdsetup.8.gz", + "/etc/drbd.d/", // added trailing slash + "/usr/share/doc/drbd-utils/", // added trailing slash + "/var/lib/drbd", // can't be fixed :( + } + sort.Strings(fileListOut) + + dirify := DirifyFileList(fileListIn, false) // TODO: test with true + sort.Strings(dirify) + equals := reflect.DeepEqual(fileListOut, dirify) + if a, b := len(fileListOut), len(dirify); a != b { + t.Errorf("DirifyFileList counts didn't match: %d != %d", a, b) + } else if !equals { + t.Error("DirifyFileList did not match expected!") + for i := 0; i < len(dirify); i++ { + if fileListOut[i] != dirify[i] { + t.Errorf("# %d: %v <> %v", i, fileListOut[i], dirify[i]) + } + } + } +} + +func TestMiscT10(t *testing.T) { + fileListIn := []string{ // fake package list + "/etc/drbd.conf", + "/usr/share/man/man8/drbdsetup.8.gz", + "/etc/drbd.d", + "/etc/drbd.d/foo", + "/var/lib/drbd", + "/var/somedir/", + } + sort.Strings(fileListIn) + + fileListOut := []string{ // fixed up manually + "/etc/drbd.conf", + "/usr/share/man/man8/drbdsetup.8.gz", + "/etc/drbd.d/", // added trailing slash + "/etc/drbd.d/foo", + "/var/lib/drbd", // can't be fixed :( + "/var/somedir/", // stays the same + } + sort.Strings(fileListOut) + + dirify := DirifyFileList(fileListIn, false) // TODO: test with true + sort.Strings(dirify) + equals := reflect.DeepEqual(fileListOut, dirify) + if a, b := len(fileListOut), len(dirify); a != b { + t.Errorf("DirifyFileList counts didn't match: %d != %d", a, b) + } else if !equals { + t.Error("DirifyFileList did not match expected!") + for i := 0; i < len(dirify); i++ { + if fileListOut[i] != dirify[i] { + t.Errorf("# %d: %v <> %v", i, fileListOut[i], dirify[i]) + } + } + } +} + +func TestMiscT11(t *testing.T) { + in1 := []string{"/", "/usr/", "/usr/lib/", "/usr/share/"} // input + ex1 := []string{"/usr/lib/", "/usr/share/"} // expected + sort.Strings(ex1) + out1 := RemoveCommonFilePrefixes(in1) + sort.Strings(out1) + if !reflect.DeepEqual(ex1, out1) { + t.Errorf("RemoveCommonFilePrefixes expected: %v; got: %v.", ex1, out1) + } + + in2 := []string{"/", "/usr/"} + ex2 := []string{"/usr/"} + sort.Strings(ex2) + out2 := RemoveCommonFilePrefixes(in2) + sort.Strings(out2) + if !reflect.DeepEqual(ex2, out2) { + t.Errorf("RemoveCommonFilePrefixes expected: %v; got: %v.", ex2, out2) + } + + in3 := []string{"/"} + ex3 := []string{"/"} + out3 := RemoveCommonFilePrefixes(in3) + if !reflect.DeepEqual(ex3, out3) { + t.Errorf("RemoveCommonFilePrefixes expected: %v; got: %v.", ex3, out3) + } + + in4 := []string{"/usr/bin/foo", "/usr/bin/bar", "/usr/lib/", "/usr/share/"} + ex4 := []string{"/usr/bin/foo", "/usr/bin/bar", "/usr/lib/", "/usr/share/"} + sort.Strings(ex4) + out4 := RemoveCommonFilePrefixes(in4) + sort.Strings(out4) + if !reflect.DeepEqual(ex4, out4) { + t.Errorf("RemoveCommonFilePrefixes expected: %v; got: %v.", ex4, out4) + } + + in5 := []string{"/usr/bin/foo", "/usr/bin/bar", "/usr/lib/", "/usr/share/", "/usr/bin"} + ex5 := []string{"/usr/bin/foo", "/usr/bin/bar", "/usr/lib/", "/usr/share/"} + sort.Strings(ex5) + out5 := RemoveCommonFilePrefixes(in5) + sort.Strings(out5) + if !reflect.DeepEqual(ex5, out5) { + t.Errorf("RemoveCommonFilePrefixes expected: %v; got: %v.", ex5, out5) + } + + in6 := []string{"/etc/drbd.d/", "/lib/drbd/", "/usr/lib/drbd/", "/usr/lib/systemd/system/", "/usr/lib/tmpfiles.d/", "/usr/sbin/", "/usr/share/doc/drbd-utils/", "/usr/share/man/man5/", "/usr/share/man/man8/", "/usr/share/doc/", "/var/lib/"} + ex6 := []string{"/etc/drbd.d/", "/lib/drbd/", "/usr/lib/drbd/", "/usr/lib/systemd/system/", "/usr/lib/tmpfiles.d/", "/usr/sbin/", "/usr/share/doc/drbd-utils/", "/usr/share/man/man5/", "/usr/share/man/man8/", "/var/lib/"} + sort.Strings(ex6) + out6 := RemoveCommonFilePrefixes(in6) + sort.Strings(out6) + if !reflect.DeepEqual(ex6, out6) { + t.Errorf("RemoveCommonFilePrefixes expected: %v; got: %v.", ex6, out6) + } + + in7 := []string{"/etc/", "/lib/", "/usr/lib/", "/usr/lib/systemd/", "/usr/", "/usr/share/doc/", "/usr/share/man/", "/var/"} + ex7 := []string{"/etc/", "/lib/", "/usr/lib/systemd/", "/usr/share/doc/", "/usr/share/man/", "/var/"} + sort.Strings(ex7) + out7 := RemoveCommonFilePrefixes(in7) + sort.Strings(out7) + if !reflect.DeepEqual(ex7, out7) { + t.Errorf("RemoveCommonFilePrefixes expected: %v; got: %v.", ex7, out7) + } + + in8 := []string{ + "/etc/drbd.conf", + "/etc/drbd.d/global_common.conf", + "/lib/drbd/drbd", + "/lib/drbd/drbdadm-83", + "/lib/drbd/drbdadm-84", + "/lib/drbd/drbdsetup-83", + "/lib/drbd/drbdsetup-84", + "/usr/lib/drbd/crm-fence-peer.sh", + "/usr/lib/drbd/crm-unfence-peer.sh", + "/usr/lib/drbd/notify-emergency-reboot.sh", + "/usr/lib/drbd/notify-emergency-shutdown.sh", + "/usr/lib/drbd/notify-io-error.sh", + "/usr/lib/drbd/notify-out-of-sync.sh", + "/usr/lib/drbd/notify-pri-lost-after-sb.sh", + "/usr/lib/drbd/notify-pri-lost.sh", + "/usr/lib/drbd/notify-pri-on-incon-degr.sh", + "/usr/lib/drbd/notify-split-brain.sh", + "/usr/lib/drbd/notify.sh", + "/usr/lib/drbd/outdate-peer.sh", + "/usr/lib/drbd/rhcs_fence", + "/usr/lib/drbd/snapshot-resync-target-lvm.sh", + "/usr/lib/drbd/stonith_admin-fence-peer.sh", + "/usr/lib/drbd/unsnapshot-resync-target-lvm.sh", + "/usr/lib/systemd/system/drbd.service", + "/usr/lib/tmpfiles.d/drbd.conf", + "/usr/sbin/drbd-overview", + "/usr/sbin/drbdadm", + "/usr/sbin/drbdmeta", + "/usr/sbin/drbdsetup", + "/usr/share/doc/drbd-utils/COPYING", + "/usr/share/doc/drbd-utils/ChangeLog", + "/usr/share/doc/drbd-utils/README", + "/usr/share/doc/drbd-utils/drbd.conf.example", + "/usr/share/man/man5/drbd.conf-8.3.5.gz", + "/usr/share/man/man5/drbd.conf-8.4.5.gz", + "/usr/share/man/man5/drbd.conf-9.0.5.gz", + "/usr/share/man/man5/drbd.conf.5.gz", + "/usr/share/man/man8/drbd-8.3.8.gz", + "/usr/share/man/man8/drbd-8.4.8.gz", + "/usr/share/man/man8/drbd-9.0.8.gz", + "/usr/share/man/man8/drbd-overview-9.0.8.gz", + "/usr/share/man/man8/drbd-overview.8.gz", + "/usr/share/man/man8/drbd.8.gz", + "/usr/share/man/man8/drbdadm-8.3.8.gz", + "/usr/share/man/man8/drbdadm-8.4.8.gz", + "/usr/share/man/man8/drbdadm-9.0.8.gz", + "/usr/share/man/man8/drbdadm.8.gz", + "/usr/share/man/man8/drbddisk-8.3.8.gz", + "/usr/share/man/man8/drbddisk-8.4.8.gz", + "/usr/share/man/man8/drbdmeta-8.3.8.gz", + "/usr/share/man/man8/drbdmeta-8.4.8.gz", + "/usr/share/man/man8/drbdmeta-9.0.8.gz", + "/usr/share/man/man8/drbdmeta.8.gz", + "/usr/share/man/man8/drbdsetup-8.3.8.gz", + "/usr/share/man/man8/drbdsetup-8.4.8.gz", + "/usr/share/man/man8/drbdsetup-9.0.8.gz", + "/usr/share/man/man8/drbdsetup.8.gz", + "/etc/drbd.d/", + "/usr/share/doc/drbd-utils/", + "/var/lib/drbd", + } + ex8 := []string{ + "/etc/drbd.conf", + "/etc/drbd.d/global_common.conf", + "/lib/drbd/drbd", + "/lib/drbd/drbdadm-83", + "/lib/drbd/drbdadm-84", + "/lib/drbd/drbdsetup-83", + "/lib/drbd/drbdsetup-84", + "/usr/lib/drbd/crm-fence-peer.sh", + "/usr/lib/drbd/crm-unfence-peer.sh", + "/usr/lib/drbd/notify-emergency-reboot.sh", + "/usr/lib/drbd/notify-emergency-shutdown.sh", + "/usr/lib/drbd/notify-io-error.sh", + "/usr/lib/drbd/notify-out-of-sync.sh", + "/usr/lib/drbd/notify-pri-lost-after-sb.sh", + "/usr/lib/drbd/notify-pri-lost.sh", + "/usr/lib/drbd/notify-pri-on-incon-degr.sh", + "/usr/lib/drbd/notify-split-brain.sh", + "/usr/lib/drbd/notify.sh", + "/usr/lib/drbd/outdate-peer.sh", + "/usr/lib/drbd/rhcs_fence", + "/usr/lib/drbd/snapshot-resync-target-lvm.sh", + "/usr/lib/drbd/stonith_admin-fence-peer.sh", + "/usr/lib/drbd/unsnapshot-resync-target-lvm.sh", + "/usr/lib/systemd/system/drbd.service", + "/usr/lib/tmpfiles.d/drbd.conf", + "/usr/sbin/drbd-overview", + "/usr/sbin/drbdadm", + "/usr/sbin/drbdmeta", + "/usr/sbin/drbdsetup", + "/usr/share/doc/drbd-utils/COPYING", + "/usr/share/doc/drbd-utils/ChangeLog", + "/usr/share/doc/drbd-utils/README", + "/usr/share/doc/drbd-utils/drbd.conf.example", + "/usr/share/man/man5/drbd.conf-8.3.5.gz", + "/usr/share/man/man5/drbd.conf-8.4.5.gz", + "/usr/share/man/man5/drbd.conf-9.0.5.gz", + "/usr/share/man/man5/drbd.conf.5.gz", + "/usr/share/man/man8/drbd-8.3.8.gz", + "/usr/share/man/man8/drbd-8.4.8.gz", + "/usr/share/man/man8/drbd-9.0.8.gz", + "/usr/share/man/man8/drbd-overview-9.0.8.gz", + "/usr/share/man/man8/drbd-overview.8.gz", + "/usr/share/man/man8/drbd.8.gz", + "/usr/share/man/man8/drbdadm-8.3.8.gz", + "/usr/share/man/man8/drbdadm-8.4.8.gz", + "/usr/share/man/man8/drbdadm-9.0.8.gz", + "/usr/share/man/man8/drbdadm.8.gz", + "/usr/share/man/man8/drbddisk-8.3.8.gz", + "/usr/share/man/man8/drbddisk-8.4.8.gz", + "/usr/share/man/man8/drbdmeta-8.3.8.gz", + "/usr/share/man/man8/drbdmeta-8.4.8.gz", + "/usr/share/man/man8/drbdmeta-9.0.8.gz", + "/usr/share/man/man8/drbdmeta.8.gz", + "/usr/share/man/man8/drbdsetup-8.3.8.gz", + "/usr/share/man/man8/drbdsetup-8.4.8.gz", + "/usr/share/man/man8/drbdsetup-9.0.8.gz", + "/usr/share/man/man8/drbdsetup.8.gz", + "/var/lib/drbd", + } + sort.Strings(ex8) + out8 := RemoveCommonFilePrefixes(in8) + sort.Strings(out8) + if !reflect.DeepEqual(ex8, out8) { + t.Errorf("RemoveCommonFilePrefixes expected: %v; got: %v.", ex8, out8) + } + + in9 := []string{ + "/etc/drbd.conf", + "/etc/drbd.d/", + "/lib/drbd/drbd", + "/lib/drbd/", + "/lib/drbd/", + "/lib/drbd/", + "/usr/lib/drbd/", + "/usr/lib/drbd/", + "/usr/lib/drbd/", + "/usr/lib/drbd/", + "/usr/lib/drbd/", + "/usr/lib/systemd/system/", + "/usr/lib/tmpfiles.d/", + "/usr/sbin/", + "/usr/sbin/", + "/usr/share/doc/drbd-utils/", + "/usr/share/doc/drbd-utils/", + "/usr/share/man/man5/", + "/usr/share/man/man5/", + "/usr/share/man/man8/", + "/usr/share/man/man8/", + "/usr/share/man/man8/", + "/etc/drbd.d/", + "/usr/share/doc/drbd-utils/", + "/var/lib/drbd", + } + ex9 := []string{ + "/etc/drbd.conf", + "/etc/drbd.d/", + "/lib/drbd/drbd", + "/usr/lib/drbd/", + "/usr/lib/systemd/system/", + "/usr/lib/tmpfiles.d/", + "/usr/sbin/", + "/usr/share/doc/drbd-utils/", + "/usr/share/man/man5/", + "/usr/share/man/man8/", + "/var/lib/drbd", + } + sort.Strings(ex9) + out9 := RemoveCommonFilePrefixes(in9) + sort.Strings(out9) + if !reflect.DeepEqual(ex9, out9) { + t.Errorf("RemoveCommonFilePrefixes expected: %v; got: %v.", ex9, out9) + } +} diff --git a/packagekit.go b/packagekit.go index 9e0e7f05..fd86c051 100644 --- a/packagekit.go +++ b/packagekit.go @@ -36,17 +36,30 @@ const ( const ( // FIXME: if PkBufferSize is too low, install seems to drop signals - PkBufferSize = 1000 - PkPath = "/org/freedesktop/PackageKit" - PkIface = "org.freedesktop.PackageKit" - PkIfaceTransaction = PkIface + ".Transaction" - dbusAddMatch = "org.freedesktop.DBus.AddMatch" + PkBufferSize = 1000 + // TODO: the PkSignalTimeout value might be too low + PkSignalPackageTimeout = 60 // 60 seconds, arbitrary + PkSignalDestroyTimeout = 15 // 15 seconds, arbitrary + PkPath = "/org/freedesktop/PackageKit" + PkIface = "org.freedesktop.PackageKit" + PkIfaceTransaction = PkIface + ".Transaction" + dbusAddMatch = "org.freedesktop.DBus.AddMatch" ) var ( + // GOARCH's: 386, amd64, arm, arm64, mips64, mips64le, ppc64, ppc64le PkArchMap = map[string]string{ // map of PackageKit arch to GOARCH // TODO: add more values - "x86_64": "amd64", + // fedora + "x86_64": "amd64", + "aarch64": "arm64", + // debian, from: https://www.debian.org/ports/ + "amd64": "amd64", + "arm64": "arm64", + "i386": "386", + "i486": "386", + "i586": "386", + "i686": "386", } ) @@ -311,7 +324,7 @@ loop: // should already be broken break loop } else { - return []string{}, errors.New(fmt.Sprintf("PackageKit error: %v", signal.Body)) + return []string{}, errors.New(fmt.Sprintf("PackageKit: Error: %v", signal.Body)) } } } @@ -381,9 +394,10 @@ func (bus *Conn) InstallPackages(packageIds []string, transactionFlags uint64) e if call.Err != nil { return call.Err } + timeout := -1 // disabled initially + finished := false loop: for { - // FIXME: add a timeout option to error in case signals are dropped! select { case signal := <-ch: if signal.Path != interfacePath { @@ -392,22 +406,29 @@ loop: } if signal.Name == FmtTransactionMethod("ErrorCode") { - return errors.New(fmt.Sprintf("PackageKit error: %v", signal.Body)) + return errors.New(fmt.Sprintf("PackageKit: Error: %v", signal.Body)) } else if signal.Name == FmtTransactionMethod("Package") { // a package was installed... + // only start the timer once we're here... + timeout = PkSignalPackageTimeout continue loop } else if signal.Name == FmtTransactionMethod("Finished") { - // TODO: should we wait for the Destroy signal? - break loop + finished = true + timeout = PkSignalDestroyTimeout // wait a bit + continue loop } else if signal.Name == FmtTransactionMethod("Destroy") { - // should already be broken - break loop + return nil // success } else { - return errors.New(fmt.Sprintf("PackageKit error: %v", signal.Body)) + return errors.New(fmt.Sprintf("PackageKit: Error: %v", signal.Body)) } + case _ = <-TimeAfterOrBlock(timeout): + if finished { + log.Println("PackageKit: Timeout: InstallPackages: Waiting for 'Destroy'") + return nil // got tired of waiting for Destroy + } + return errors.New(fmt.Sprintf("PackageKit: Timeout: InstallPackages: %v", strings.Join(packageIds, ", "))) } } - return nil } // remove list of packages @@ -440,7 +461,7 @@ loop: } if signal.Name == FmtTransactionMethod("ErrorCode") { - return errors.New(fmt.Sprintf("PackageKit error: %v", signal.Body)) + return errors.New(fmt.Sprintf("PackageKit: Error: %v", signal.Body)) } else if signal.Name == FmtTransactionMethod("Package") { // a package was installed... continue loop @@ -451,7 +472,7 @@ loop: // should already be broken break loop } else { - return errors.New(fmt.Sprintf("PackageKit error: %v", signal.Body)) + return errors.New(fmt.Sprintf("PackageKit: Error: %v", signal.Body)) } } } @@ -485,7 +506,7 @@ loop: } if signal.Name == FmtTransactionMethod("ErrorCode") { - return errors.New(fmt.Sprintf("PackageKit error: %v", signal.Body)) + return errors.New(fmt.Sprintf("PackageKit: Error: %v", signal.Body)) } else if signal.Name == FmtTransactionMethod("Package") { // a package was installed... continue loop @@ -496,7 +517,7 @@ loop: // should already be broken break loop } else { - return errors.New(fmt.Sprintf("PackageKit error: %v", signal.Body)) + return errors.New(fmt.Sprintf("PackageKit: Error: %v", signal.Body)) } } } @@ -505,6 +526,8 @@ loop: // get the list of files that are contained inside a list of packageids func (bus *Conn) GetFilesByPackageId(packageIds []string) (files map[string][]string, err error) { + // NOTE: the maximum number of files in an RPM is 52116 in Fedora 23 + // https://gist.github.com/purpleidea/b98e60dcd449e1ac3b8a ch := make(chan *dbus.Signal, PkBufferSize) // we need to buffer :( interfacePath, err := bus.CreateTransaction() if err != nil { @@ -533,7 +556,7 @@ loop: } if signal.Name == FmtTransactionMethod("ErrorCode") { - err = errors.New(fmt.Sprintf("PackageKit error: %v", signal.Body)) + err = errors.New(fmt.Sprintf("PackageKit: Error: %v", signal.Body)) return // one signal returned per packageId found... @@ -560,7 +583,7 @@ loop: // should already be broken break loop } else { - err = errors.New(fmt.Sprintf("PackageKit error: %v", signal.Body)) + err = errors.New(fmt.Sprintf("PackageKit: Error: %v", signal.Body)) return } } @@ -596,7 +619,7 @@ loop: } if signal.Name == FmtTransactionMethod("ErrorCode") { - return nil, errors.New(fmt.Sprintf("PackageKit error: %v", signal.Body)) + return nil, errors.New(fmt.Sprintf("PackageKit: Error: %v", signal.Body)) } else if signal.Name == FmtTransactionMethod("Package") { //pkg_int, ok := signal.Body[0].(int) @@ -620,7 +643,7 @@ loop: // should already be broken break loop } else { - return nil, errors.New(fmt.Sprintf("PackageKit error: %v", signal.Body)) + return nil, errors.New(fmt.Sprintf("PackageKit: Error: %v", signal.Body)) } } } @@ -667,7 +690,8 @@ func (bus *Conn) PackagesToPackageIds(packageMap map[string]string, filter uint6 s := strings.Split(packageId, ";") //if len(s) != 4 { continue } // this would be a bug! pkg, ver, arch, data := s[0], s[1], s[2], s[3] - if goarch, ok := PkArchMap[arch]; !ok || goarch != runtime.GOARCH { + // we might need to allow some of this, eg: i386 .deb on amd64 + if !IsMyArch(arch) { continue } @@ -811,3 +835,12 @@ func FlagInData(flag, data string) bool { func FmtTransactionMethod(method string) string { return fmt.Sprintf("%s.%s", PkIfaceTransaction, method) } + +func IsMyArch(arch string) bool { + goarch, ok := PkArchMap[arch] + if !ok { + // if you get this error, please update the PkArchMap const + log.Fatalf("PackageKit: Arch '%v', not found!", arch) + } + return goarch == runtime.GOARCH +} diff --git a/pkg.go b/pkg.go index a55343fa..563e76a3 100644 --- a/pkg.go +++ b/pkg.go @@ -26,10 +26,11 @@ import ( type PkgRes struct { BaseRes `yaml:",inline"` - State string `yaml:"state"` // state: installed, uninstalled, newest, - AllowUntrusted bool `yaml:"allowuntrusted"` // allow untrusted packages to be installed? - AllowNonFree bool `yaml:"allownonfree"` // allow nonfree packages to be found? - AllowUnsupported bool `yaml:"allowunsupported"` // allow unsupported packages to be found? + State string `yaml:"state"` // state: installed, uninstalled, newest, + AllowUntrusted bool `yaml:"allowuntrusted"` // allow untrusted packages to be installed? + AllowNonFree bool `yaml:"allownonfree"` // allow nonfree packages to be found? + AllowUnsupported bool `yaml:"allowunsupported"` // allow unsupported packages to be found? + fileList []string // FIXME: update if pkg changes } // helper function for creating new pkg resources that calls Init() @@ -247,8 +248,115 @@ func (obj *PkgRes) CheckApply(apply bool) (stateok bool, err error) { return false, nil // success } +type PkgResAutoEdges struct { + fileList []string + testIsNext bool // safety + name string // saved data from PkgRes obj + kind string +} + +func (obj *PkgResAutoEdges) Next() []ResUUID { + if obj.testIsNext { + log.Fatal("Expecting a call to Test()") + } + obj.testIsNext = true // set after all the errors paths are past + var result []ResUUID + + // return UUID's for whatever is in obj.fileList + for _, x := range obj.fileList { + var reversed bool = false // cheat by passing a pointer + result = append(result, &FileUUID{ + BaseUUID: BaseUUID{ + name: obj.name, + kind: obj.kind, + reversed: &reversed, + }, + path: x, // what matters + }) // build list + } + return result +} + +func (obj *PkgResAutoEdges) Test(input []bool) bool { + if !obj.testIsNext { + log.Fatal("Expecting a call to Next()") + } + count := len(obj.fileList) + if count != len(input) { + log.Fatalf("Expecting %d value(s)!", count) + } + obj.testIsNext = false // set after all the errors paths are past + + // while i do believe this algorithm generates the *correct* result, i + // don't know if it does so in the optimal way. improvements welcome! + // the basic logic is: + // 0) Next() returns whatever is in fileList + // 1) Test() computes the dirname of each file, and removes duplicates + // and dirname's that have been in the path of an ack from input results + // 2) It then simplifies the list by removing the common path prefixes + // 3) Lastly, the remaining set of files (dirs) is used as new fileList + // 4) We then iterate in (0) until the fileList is empty! + var dirs = make([]string, count) + done := []string{} + for i := 0; i < count; i++ { + dir := Dirname(obj.fileList[i]) // dirname of /foo/ should be / + dirs[i] = dir + if input[i] { + done = append(done, dir) + } + } + nodupes := StrRemoveDuplicatesInList(dirs) // remove duplicates + nodones := StrFilterElementsInList(done, nodupes) // filter out done + noempty := StrFilterElementsInList([]string{""}, nodones) // remove the "" from / + obj.fileList = RemoveCommonFilePrefixes(noempty) // magic + + if len(obj.fileList) == 0 { // nothing more, don't continue + return false + } + return true // continue, there are more files! +} + +type PkgUUID struct { + BaseUUID +} + +// if and only if they are equivalent, return true +// if they are not equivalent, return false +func (obj *PkgUUID) IFF(uuid ResUUID) bool { + res, ok := uuid.(*PkgUUID) + if !ok { + return false + } + return obj.name == res.name +} + +// produce an object which generates a minimal pkg file optimization sequence func (obj *PkgRes) AutoEdges() AutoEdge { - return nil + // in contrast with the FileRes AutoEdges() function which contains + // more of the mechanics, most of the AutoEdge mechanics for the PkgRes + // is contained in the Test() method! This design is completely okay! + return &PkgResAutoEdges{ + fileList: obj.fileList, + testIsNext: false, // start with Next() call + name: obj.GetName(), // save data for PkgResAutoEdges obj + kind: obj.Kind(), + } +} + +// include all params to make a unique identification of this object +func (obj *PkgRes) GetUUIDs() []ResUUID { + x := &PkgUUID{ + BaseUUID: BaseUUID{name: obj.GetName(), kind: obj.Kind()}, + } + result := []ResUUID{x} + for _, path := range obj.fileList { + x := &FileUUID{ + BaseUUID: BaseUUID{name: obj.GetName(), kind: obj.Kind()}, + path: path, + } + result = append(result, x) + } + return result } func (obj *PkgRes) Compare(res Res) bool {