From f39551952f1075977284f00e797851faf2568a7c Mon Sep 17 00:00:00 2001 From: James Shubin Date: Mon, 7 Mar 2016 16:42:05 -0500 Subject: [PATCH] Add pkg auto edge basics with packagekit improvements This is a monster patch that finally gets the iterative pkg auto edges working the way they should. For each file, as soon as one matches, we don't want to keep add dependencies on other file objects under that tree structure. This reduces the number of necessary edges considerably, and allows the graph to run more concurrently. --- examples/autoedges2.yaml | 24 +++ file.go | 1 - misc.go | 98 +++++++++ misc_test.go | 432 +++++++++++++++++++++++++++++++++++++++ packagekit.go | 81 +++++--- pkg.go | 118 ++++++++++- 6 files changed, 724 insertions(+), 30 deletions(-) create mode 100644 examples/autoedges2.yaml diff --git a/examples/autoedges2.yaml b/examples/autoedges2.yaml new file mode 100644 index 00000000..7eb6b6c6 --- /dev/null +++ b/examples/autoedges2.yaml @@ -0,0 +1,24 @@ +--- +graph: mygraph +resources: + file: + - name: file1 + meta: + autoedge: true + path: "/etc/drbd.conf" + content: | + # this is an mgmt test + state: exists + - name: file2 + meta: + autoedge: true + path: "/tmp/foo/" + content: | + i am f2 + state: exists + pkg: + - name: drbd-utils + meta: + autoedge: true + state: installed +edges: [] diff --git a/file.go b/file.go index c8c6148a..1992c7bd 100644 --- a/file.go +++ b/file.go @@ -374,7 +374,6 @@ func (obj *FileRes) CheckApply(apply bool) (stateok bool, err error) { return false, nil // success } - type FileUUID struct { BaseUUID path string diff --git a/misc.go b/misc.go index 77b17844..092f9621 100644 --- a/misc.go +++ b/misc.go @@ -27,6 +27,39 @@ import ( "time" ) +// return true if a string exists inside a list, otherwise false +func StrInList(needle string, haystack []string) bool { + for _, x := range haystack { + if needle == x { + return true + } + } + return false +} + +// remove any duplicate values in the list +// possibly sub-optimal, O(n^2)? implementation +func StrRemoveDuplicatesInList(list []string) []string { + unique := []string{} + for _, x := range list { + if !StrInList(x, unique) { + unique = append(unique, x) + } + } + return unique +} + +// remove any of the elements in filter, if they exist in list +func StrFilterElementsInList(filter []string, list []string) []string { + result := []string{} + for _, x := range list { + if !StrInList(x, filter) { + result = append(result, x) + } + } + return result +} + // reverse a list of strings func ReverseStringList(in []string) []string { var out []string // empty list @@ -81,6 +114,46 @@ func HasPathPrefix(p, prefix string) bool { return true } +func StrInPathPrefixList(needle string, haystack []string) bool { + for _, x := range haystack { + if HasPathPrefix(x, needle) { + return true + } + } + return false +} + +// remove redundant file path prefixes that are under the tree of other files +func RemoveCommonFilePrefixes(paths []string) []string { + var result = make([]string, len(paths)) + for i := 0; i < len(paths); i++ { // copy, b/c append can modify the args!! + result[i] = paths[i] + } + // is there a string path which is common everywhere? + // if so, remove it, and iterate until nothing common is left + // return what's left over, that's the most common superset +loop: + for { + if len(result) <= 1 { + return result + } + for i := 0; i < len(result); i++ { + var copied = make([]string, len(result)) + for j := 0; j < len(result); j++ { // copy, b/c append can modify the args!! + copied[j] = result[j] + } + noi := append(copied[:i], copied[i+1:]...) // rm i + if StrInPathPrefixList(result[i], noi) { + // delete the element common to everyone + result = noi + continue loop + } + } + break + } + return result +} + // Delta of path prefix, tells you how many path tokens different the prefix is func PathPrefixDelta(p, prefix string) int { @@ -112,6 +185,31 @@ func PathSplitFullReversed(p string) []string { return ReverseStringList(result) } +// add trailing slashes to any likely dirs in a package manager fileList +// if removeDirs is true, instead, don't keep the dirs in our output +func DirifyFileList(fileList []string, removeDirs bool) []string { + dirs := []string{} + for _, file := range fileList { + dir, _ := path.Split(file) // dir + dir = path.Clean(dir) // clean so cmp is easier + if !StrInList(dir, dirs) { + dirs = append(dirs, dir) + } + } + + result := []string{} + for _, file := range fileList { + cleanFile := path.Clean(file) + if !StrInList(cleanFile, dirs) { // we're not a directory! + result = append(result, file) // pass through + } else if !removeDirs { + result = append(result, cleanFile+"/") + } + } + + return result +} + // encode an object as base 64, serialize and then base64 encode func ObjToB64(obj interface{}) (string, bool) { b := bytes.Buffer{} diff --git a/misc_test.go b/misc_test.go index e6fb18f5..1cc6cb04 100644 --- a/misc_test.go +++ b/misc_test.go @@ -20,6 +20,7 @@ package main import ( "fmt" "reflect" + "sort" "testing" ) @@ -33,6 +34,10 @@ func TestMiscT1(t *testing.T) { t.Errorf("Result is incorrect.") } + if Dirname("/foo/") != "/" { + t.Errorf("Result is incorrect.") + } + if Dirname("/") != "" { // TODO: should this equal "/" or "" ? t.Errorf("Result is incorrect.") } @@ -45,6 +50,10 @@ func TestMiscT1(t *testing.T) { t.Errorf("Result is incorrect.") } + if Basename("/foo/") != "foo/" { + t.Errorf("Result is incorrect.") + } + if Basename("/") != "/" { // TODO: should this equal "" or "/" ? t.Errorf("Result is incorrect.") } @@ -100,6 +109,10 @@ func TestMiscT3(t *testing.T) { if HasPathPrefix("/foo/bar/baz/", "/foo/bar/baz/dude") != false { t.Errorf("Result should be false.") } + + if HasPathPrefix("/foo/bar/baz/boo/", "/foo/") != true { + t.Errorf("Result should be true.") + } } func TestMiscT4(t *testing.T) { @@ -225,3 +238,422 @@ func TestMiscT8(t *testing.T) { } } + +func TestMiscT9(t *testing.T) { + fileListIn := []string{ // list taken from drbd-utils package + "/etc/drbd.conf", + "/etc/drbd.d/global_common.conf", + "/lib/drbd/drbd", + "/lib/drbd/drbdadm-83", + "/lib/drbd/drbdadm-84", + "/lib/drbd/drbdsetup-83", + "/lib/drbd/drbdsetup-84", + "/usr/lib/drbd/crm-fence-peer.sh", + "/usr/lib/drbd/crm-unfence-peer.sh", + "/usr/lib/drbd/notify-emergency-reboot.sh", + "/usr/lib/drbd/notify-emergency-shutdown.sh", + "/usr/lib/drbd/notify-io-error.sh", + "/usr/lib/drbd/notify-out-of-sync.sh", + "/usr/lib/drbd/notify-pri-lost-after-sb.sh", + "/usr/lib/drbd/notify-pri-lost.sh", + "/usr/lib/drbd/notify-pri-on-incon-degr.sh", + "/usr/lib/drbd/notify-split-brain.sh", + "/usr/lib/drbd/notify.sh", + "/usr/lib/drbd/outdate-peer.sh", + "/usr/lib/drbd/rhcs_fence", + "/usr/lib/drbd/snapshot-resync-target-lvm.sh", + "/usr/lib/drbd/stonith_admin-fence-peer.sh", + "/usr/lib/drbd/unsnapshot-resync-target-lvm.sh", + "/usr/lib/systemd/system/drbd.service", + "/usr/lib/tmpfiles.d/drbd.conf", + "/usr/sbin/drbd-overview", + "/usr/sbin/drbdadm", + "/usr/sbin/drbdmeta", + "/usr/sbin/drbdsetup", + "/usr/share/doc/drbd-utils/COPYING", + "/usr/share/doc/drbd-utils/ChangeLog", + "/usr/share/doc/drbd-utils/README", + "/usr/share/doc/drbd-utils/drbd.conf.example", + "/usr/share/man/man5/drbd.conf-8.3.5.gz", + "/usr/share/man/man5/drbd.conf-8.4.5.gz", + "/usr/share/man/man5/drbd.conf-9.0.5.gz", + "/usr/share/man/man5/drbd.conf.5.gz", + "/usr/share/man/man8/drbd-8.3.8.gz", + "/usr/share/man/man8/drbd-8.4.8.gz", + "/usr/share/man/man8/drbd-9.0.8.gz", + "/usr/share/man/man8/drbd-overview-9.0.8.gz", + "/usr/share/man/man8/drbd-overview.8.gz", + "/usr/share/man/man8/drbd.8.gz", + "/usr/share/man/man8/drbdadm-8.3.8.gz", + "/usr/share/man/man8/drbdadm-8.4.8.gz", + "/usr/share/man/man8/drbdadm-9.0.8.gz", + "/usr/share/man/man8/drbdadm.8.gz", + "/usr/share/man/man8/drbddisk-8.3.8.gz", + "/usr/share/man/man8/drbddisk-8.4.8.gz", + "/usr/share/man/man8/drbdmeta-8.3.8.gz", + "/usr/share/man/man8/drbdmeta-8.4.8.gz", + "/usr/share/man/man8/drbdmeta-9.0.8.gz", + "/usr/share/man/man8/drbdmeta.8.gz", + "/usr/share/man/man8/drbdsetup-8.3.8.gz", + "/usr/share/man/man8/drbdsetup-8.4.8.gz", + "/usr/share/man/man8/drbdsetup-9.0.8.gz", + "/usr/share/man/man8/drbdsetup.8.gz", + "/etc/drbd.d", + "/usr/share/doc/drbd-utils", + "/var/lib/drbd", + } + sort.Strings(fileListIn) + + fileListOut := []string{ // fixed up manually + "/etc/drbd.conf", + "/etc/drbd.d/global_common.conf", + "/lib/drbd/drbd", + "/lib/drbd/drbdadm-83", + "/lib/drbd/drbdadm-84", + "/lib/drbd/drbdsetup-83", + "/lib/drbd/drbdsetup-84", + "/usr/lib/drbd/crm-fence-peer.sh", + "/usr/lib/drbd/crm-unfence-peer.sh", + "/usr/lib/drbd/notify-emergency-reboot.sh", + "/usr/lib/drbd/notify-emergency-shutdown.sh", + "/usr/lib/drbd/notify-io-error.sh", + "/usr/lib/drbd/notify-out-of-sync.sh", + "/usr/lib/drbd/notify-pri-lost-after-sb.sh", + "/usr/lib/drbd/notify-pri-lost.sh", + "/usr/lib/drbd/notify-pri-on-incon-degr.sh", + "/usr/lib/drbd/notify-split-brain.sh", + "/usr/lib/drbd/notify.sh", + "/usr/lib/drbd/outdate-peer.sh", + "/usr/lib/drbd/rhcs_fence", + "/usr/lib/drbd/snapshot-resync-target-lvm.sh", + "/usr/lib/drbd/stonith_admin-fence-peer.sh", + "/usr/lib/drbd/unsnapshot-resync-target-lvm.sh", + "/usr/lib/systemd/system/drbd.service", + "/usr/lib/tmpfiles.d/drbd.conf", + "/usr/sbin/drbd-overview", + "/usr/sbin/drbdadm", + "/usr/sbin/drbdmeta", + "/usr/sbin/drbdsetup", + "/usr/share/doc/drbd-utils/COPYING", + "/usr/share/doc/drbd-utils/ChangeLog", + "/usr/share/doc/drbd-utils/README", + "/usr/share/doc/drbd-utils/drbd.conf.example", + "/usr/share/man/man5/drbd.conf-8.3.5.gz", + "/usr/share/man/man5/drbd.conf-8.4.5.gz", + "/usr/share/man/man5/drbd.conf-9.0.5.gz", + "/usr/share/man/man5/drbd.conf.5.gz", + "/usr/share/man/man8/drbd-8.3.8.gz", + "/usr/share/man/man8/drbd-8.4.8.gz", + "/usr/share/man/man8/drbd-9.0.8.gz", + "/usr/share/man/man8/drbd-overview-9.0.8.gz", + "/usr/share/man/man8/drbd-overview.8.gz", + "/usr/share/man/man8/drbd.8.gz", + "/usr/share/man/man8/drbdadm-8.3.8.gz", + "/usr/share/man/man8/drbdadm-8.4.8.gz", + "/usr/share/man/man8/drbdadm-9.0.8.gz", + "/usr/share/man/man8/drbdadm.8.gz", + "/usr/share/man/man8/drbddisk-8.3.8.gz", + "/usr/share/man/man8/drbddisk-8.4.8.gz", + "/usr/share/man/man8/drbdmeta-8.3.8.gz", + "/usr/share/man/man8/drbdmeta-8.4.8.gz", + "/usr/share/man/man8/drbdmeta-9.0.8.gz", + "/usr/share/man/man8/drbdmeta.8.gz", + "/usr/share/man/man8/drbdsetup-8.3.8.gz", + "/usr/share/man/man8/drbdsetup-8.4.8.gz", + "/usr/share/man/man8/drbdsetup-9.0.8.gz", + "/usr/share/man/man8/drbdsetup.8.gz", + "/etc/drbd.d/", // added trailing slash + "/usr/share/doc/drbd-utils/", // added trailing slash + "/var/lib/drbd", // can't be fixed :( + } + sort.Strings(fileListOut) + + dirify := DirifyFileList(fileListIn, false) // TODO: test with true + sort.Strings(dirify) + equals := reflect.DeepEqual(fileListOut, dirify) + if a, b := len(fileListOut), len(dirify); a != b { + t.Errorf("DirifyFileList counts didn't match: %d != %d", a, b) + } else if !equals { + t.Error("DirifyFileList did not match expected!") + for i := 0; i < len(dirify); i++ { + if fileListOut[i] != dirify[i] { + t.Errorf("# %d: %v <> %v", i, fileListOut[i], dirify[i]) + } + } + } +} + +func TestMiscT10(t *testing.T) { + fileListIn := []string{ // fake package list + "/etc/drbd.conf", + "/usr/share/man/man8/drbdsetup.8.gz", + "/etc/drbd.d", + "/etc/drbd.d/foo", + "/var/lib/drbd", + "/var/somedir/", + } + sort.Strings(fileListIn) + + fileListOut := []string{ // fixed up manually + "/etc/drbd.conf", + "/usr/share/man/man8/drbdsetup.8.gz", + "/etc/drbd.d/", // added trailing slash + "/etc/drbd.d/foo", + "/var/lib/drbd", // can't be fixed :( + "/var/somedir/", // stays the same + } + sort.Strings(fileListOut) + + dirify := DirifyFileList(fileListIn, false) // TODO: test with true + sort.Strings(dirify) + equals := reflect.DeepEqual(fileListOut, dirify) + if a, b := len(fileListOut), len(dirify); a != b { + t.Errorf("DirifyFileList counts didn't match: %d != %d", a, b) + } else if !equals { + t.Error("DirifyFileList did not match expected!") + for i := 0; i < len(dirify); i++ { + if fileListOut[i] != dirify[i] { + t.Errorf("# %d: %v <> %v", i, fileListOut[i], dirify[i]) + } + } + } +} + +func TestMiscT11(t *testing.T) { + in1 := []string{"/", "/usr/", "/usr/lib/", "/usr/share/"} // input + ex1 := []string{"/usr/lib/", "/usr/share/"} // expected + sort.Strings(ex1) + out1 := RemoveCommonFilePrefixes(in1) + sort.Strings(out1) + if !reflect.DeepEqual(ex1, out1) { + t.Errorf("RemoveCommonFilePrefixes expected: %v; got: %v.", ex1, out1) + } + + in2 := []string{"/", "/usr/"} + ex2 := []string{"/usr/"} + sort.Strings(ex2) + out2 := RemoveCommonFilePrefixes(in2) + sort.Strings(out2) + if !reflect.DeepEqual(ex2, out2) { + t.Errorf("RemoveCommonFilePrefixes expected: %v; got: %v.", ex2, out2) + } + + in3 := []string{"/"} + ex3 := []string{"/"} + out3 := RemoveCommonFilePrefixes(in3) + if !reflect.DeepEqual(ex3, out3) { + t.Errorf("RemoveCommonFilePrefixes expected: %v; got: %v.", ex3, out3) + } + + in4 := []string{"/usr/bin/foo", "/usr/bin/bar", "/usr/lib/", "/usr/share/"} + ex4 := []string{"/usr/bin/foo", "/usr/bin/bar", "/usr/lib/", "/usr/share/"} + sort.Strings(ex4) + out4 := RemoveCommonFilePrefixes(in4) + sort.Strings(out4) + if !reflect.DeepEqual(ex4, out4) { + t.Errorf("RemoveCommonFilePrefixes expected: %v; got: %v.", ex4, out4) + } + + in5 := []string{"/usr/bin/foo", "/usr/bin/bar", "/usr/lib/", "/usr/share/", "/usr/bin"} + ex5 := []string{"/usr/bin/foo", "/usr/bin/bar", "/usr/lib/", "/usr/share/"} + sort.Strings(ex5) + out5 := RemoveCommonFilePrefixes(in5) + sort.Strings(out5) + if !reflect.DeepEqual(ex5, out5) { + t.Errorf("RemoveCommonFilePrefixes expected: %v; got: %v.", ex5, out5) + } + + in6 := []string{"/etc/drbd.d/", "/lib/drbd/", "/usr/lib/drbd/", "/usr/lib/systemd/system/", "/usr/lib/tmpfiles.d/", "/usr/sbin/", "/usr/share/doc/drbd-utils/", "/usr/share/man/man5/", "/usr/share/man/man8/", "/usr/share/doc/", "/var/lib/"} + ex6 := []string{"/etc/drbd.d/", "/lib/drbd/", "/usr/lib/drbd/", "/usr/lib/systemd/system/", "/usr/lib/tmpfiles.d/", "/usr/sbin/", "/usr/share/doc/drbd-utils/", "/usr/share/man/man5/", "/usr/share/man/man8/", "/var/lib/"} + sort.Strings(ex6) + out6 := RemoveCommonFilePrefixes(in6) + sort.Strings(out6) + if !reflect.DeepEqual(ex6, out6) { + t.Errorf("RemoveCommonFilePrefixes expected: %v; got: %v.", ex6, out6) + } + + in7 := []string{"/etc/", "/lib/", "/usr/lib/", "/usr/lib/systemd/", "/usr/", "/usr/share/doc/", "/usr/share/man/", "/var/"} + ex7 := []string{"/etc/", "/lib/", "/usr/lib/systemd/", "/usr/share/doc/", "/usr/share/man/", "/var/"} + sort.Strings(ex7) + out7 := RemoveCommonFilePrefixes(in7) + sort.Strings(out7) + if !reflect.DeepEqual(ex7, out7) { + t.Errorf("RemoveCommonFilePrefixes expected: %v; got: %v.", ex7, out7) + } + + in8 := []string{ + "/etc/drbd.conf", + "/etc/drbd.d/global_common.conf", + "/lib/drbd/drbd", + "/lib/drbd/drbdadm-83", + "/lib/drbd/drbdadm-84", + "/lib/drbd/drbdsetup-83", + "/lib/drbd/drbdsetup-84", + "/usr/lib/drbd/crm-fence-peer.sh", + "/usr/lib/drbd/crm-unfence-peer.sh", + "/usr/lib/drbd/notify-emergency-reboot.sh", + "/usr/lib/drbd/notify-emergency-shutdown.sh", + "/usr/lib/drbd/notify-io-error.sh", + "/usr/lib/drbd/notify-out-of-sync.sh", + "/usr/lib/drbd/notify-pri-lost-after-sb.sh", + "/usr/lib/drbd/notify-pri-lost.sh", + "/usr/lib/drbd/notify-pri-on-incon-degr.sh", + "/usr/lib/drbd/notify-split-brain.sh", + "/usr/lib/drbd/notify.sh", + "/usr/lib/drbd/outdate-peer.sh", + "/usr/lib/drbd/rhcs_fence", + "/usr/lib/drbd/snapshot-resync-target-lvm.sh", + "/usr/lib/drbd/stonith_admin-fence-peer.sh", + "/usr/lib/drbd/unsnapshot-resync-target-lvm.sh", + "/usr/lib/systemd/system/drbd.service", + "/usr/lib/tmpfiles.d/drbd.conf", + "/usr/sbin/drbd-overview", + "/usr/sbin/drbdadm", + "/usr/sbin/drbdmeta", + "/usr/sbin/drbdsetup", + "/usr/share/doc/drbd-utils/COPYING", + "/usr/share/doc/drbd-utils/ChangeLog", + "/usr/share/doc/drbd-utils/README", + "/usr/share/doc/drbd-utils/drbd.conf.example", + "/usr/share/man/man5/drbd.conf-8.3.5.gz", + "/usr/share/man/man5/drbd.conf-8.4.5.gz", + "/usr/share/man/man5/drbd.conf-9.0.5.gz", + "/usr/share/man/man5/drbd.conf.5.gz", + "/usr/share/man/man8/drbd-8.3.8.gz", + "/usr/share/man/man8/drbd-8.4.8.gz", + "/usr/share/man/man8/drbd-9.0.8.gz", + "/usr/share/man/man8/drbd-overview-9.0.8.gz", + "/usr/share/man/man8/drbd-overview.8.gz", + "/usr/share/man/man8/drbd.8.gz", + "/usr/share/man/man8/drbdadm-8.3.8.gz", + "/usr/share/man/man8/drbdadm-8.4.8.gz", + "/usr/share/man/man8/drbdadm-9.0.8.gz", + "/usr/share/man/man8/drbdadm.8.gz", + "/usr/share/man/man8/drbddisk-8.3.8.gz", + "/usr/share/man/man8/drbddisk-8.4.8.gz", + "/usr/share/man/man8/drbdmeta-8.3.8.gz", + "/usr/share/man/man8/drbdmeta-8.4.8.gz", + "/usr/share/man/man8/drbdmeta-9.0.8.gz", + "/usr/share/man/man8/drbdmeta.8.gz", + "/usr/share/man/man8/drbdsetup-8.3.8.gz", + "/usr/share/man/man8/drbdsetup-8.4.8.gz", + "/usr/share/man/man8/drbdsetup-9.0.8.gz", + "/usr/share/man/man8/drbdsetup.8.gz", + "/etc/drbd.d/", + "/usr/share/doc/drbd-utils/", + "/var/lib/drbd", + } + ex8 := []string{ + "/etc/drbd.conf", + "/etc/drbd.d/global_common.conf", + "/lib/drbd/drbd", + "/lib/drbd/drbdadm-83", + "/lib/drbd/drbdadm-84", + "/lib/drbd/drbdsetup-83", + "/lib/drbd/drbdsetup-84", + "/usr/lib/drbd/crm-fence-peer.sh", + "/usr/lib/drbd/crm-unfence-peer.sh", + "/usr/lib/drbd/notify-emergency-reboot.sh", + "/usr/lib/drbd/notify-emergency-shutdown.sh", + "/usr/lib/drbd/notify-io-error.sh", + "/usr/lib/drbd/notify-out-of-sync.sh", + "/usr/lib/drbd/notify-pri-lost-after-sb.sh", + "/usr/lib/drbd/notify-pri-lost.sh", + "/usr/lib/drbd/notify-pri-on-incon-degr.sh", + "/usr/lib/drbd/notify-split-brain.sh", + "/usr/lib/drbd/notify.sh", + "/usr/lib/drbd/outdate-peer.sh", + "/usr/lib/drbd/rhcs_fence", + "/usr/lib/drbd/snapshot-resync-target-lvm.sh", + "/usr/lib/drbd/stonith_admin-fence-peer.sh", + "/usr/lib/drbd/unsnapshot-resync-target-lvm.sh", + "/usr/lib/systemd/system/drbd.service", + "/usr/lib/tmpfiles.d/drbd.conf", + "/usr/sbin/drbd-overview", + "/usr/sbin/drbdadm", + "/usr/sbin/drbdmeta", + "/usr/sbin/drbdsetup", + "/usr/share/doc/drbd-utils/COPYING", + "/usr/share/doc/drbd-utils/ChangeLog", + "/usr/share/doc/drbd-utils/README", + "/usr/share/doc/drbd-utils/drbd.conf.example", + "/usr/share/man/man5/drbd.conf-8.3.5.gz", + "/usr/share/man/man5/drbd.conf-8.4.5.gz", + "/usr/share/man/man5/drbd.conf-9.0.5.gz", + "/usr/share/man/man5/drbd.conf.5.gz", + "/usr/share/man/man8/drbd-8.3.8.gz", + "/usr/share/man/man8/drbd-8.4.8.gz", + "/usr/share/man/man8/drbd-9.0.8.gz", + "/usr/share/man/man8/drbd-overview-9.0.8.gz", + "/usr/share/man/man8/drbd-overview.8.gz", + "/usr/share/man/man8/drbd.8.gz", + "/usr/share/man/man8/drbdadm-8.3.8.gz", + "/usr/share/man/man8/drbdadm-8.4.8.gz", + "/usr/share/man/man8/drbdadm-9.0.8.gz", + "/usr/share/man/man8/drbdadm.8.gz", + "/usr/share/man/man8/drbddisk-8.3.8.gz", + "/usr/share/man/man8/drbddisk-8.4.8.gz", + "/usr/share/man/man8/drbdmeta-8.3.8.gz", + "/usr/share/man/man8/drbdmeta-8.4.8.gz", + "/usr/share/man/man8/drbdmeta-9.0.8.gz", + "/usr/share/man/man8/drbdmeta.8.gz", + "/usr/share/man/man8/drbdsetup-8.3.8.gz", + "/usr/share/man/man8/drbdsetup-8.4.8.gz", + "/usr/share/man/man8/drbdsetup-9.0.8.gz", + "/usr/share/man/man8/drbdsetup.8.gz", + "/var/lib/drbd", + } + sort.Strings(ex8) + out8 := RemoveCommonFilePrefixes(in8) + sort.Strings(out8) + if !reflect.DeepEqual(ex8, out8) { + t.Errorf("RemoveCommonFilePrefixes expected: %v; got: %v.", ex8, out8) + } + + in9 := []string{ + "/etc/drbd.conf", + "/etc/drbd.d/", + "/lib/drbd/drbd", + "/lib/drbd/", + "/lib/drbd/", + "/lib/drbd/", + "/usr/lib/drbd/", + "/usr/lib/drbd/", + "/usr/lib/drbd/", + "/usr/lib/drbd/", + "/usr/lib/drbd/", + "/usr/lib/systemd/system/", + "/usr/lib/tmpfiles.d/", + "/usr/sbin/", + "/usr/sbin/", + "/usr/share/doc/drbd-utils/", + "/usr/share/doc/drbd-utils/", + "/usr/share/man/man5/", + "/usr/share/man/man5/", + "/usr/share/man/man8/", + "/usr/share/man/man8/", + "/usr/share/man/man8/", + "/etc/drbd.d/", + "/usr/share/doc/drbd-utils/", + "/var/lib/drbd", + } + ex9 := []string{ + "/etc/drbd.conf", + "/etc/drbd.d/", + "/lib/drbd/drbd", + "/usr/lib/drbd/", + "/usr/lib/systemd/system/", + "/usr/lib/tmpfiles.d/", + "/usr/sbin/", + "/usr/share/doc/drbd-utils/", + "/usr/share/man/man5/", + "/usr/share/man/man8/", + "/var/lib/drbd", + } + sort.Strings(ex9) + out9 := RemoveCommonFilePrefixes(in9) + sort.Strings(out9) + if !reflect.DeepEqual(ex9, out9) { + t.Errorf("RemoveCommonFilePrefixes expected: %v; got: %v.", ex9, out9) + } +} diff --git a/packagekit.go b/packagekit.go index 9e0e7f05..fd86c051 100644 --- a/packagekit.go +++ b/packagekit.go @@ -36,17 +36,30 @@ const ( const ( // FIXME: if PkBufferSize is too low, install seems to drop signals - PkBufferSize = 1000 - PkPath = "/org/freedesktop/PackageKit" - PkIface = "org.freedesktop.PackageKit" - PkIfaceTransaction = PkIface + ".Transaction" - dbusAddMatch = "org.freedesktop.DBus.AddMatch" + PkBufferSize = 1000 + // TODO: the PkSignalTimeout value might be too low + PkSignalPackageTimeout = 60 // 60 seconds, arbitrary + PkSignalDestroyTimeout = 15 // 15 seconds, arbitrary + PkPath = "/org/freedesktop/PackageKit" + PkIface = "org.freedesktop.PackageKit" + PkIfaceTransaction = PkIface + ".Transaction" + dbusAddMatch = "org.freedesktop.DBus.AddMatch" ) var ( + // GOARCH's: 386, amd64, arm, arm64, mips64, mips64le, ppc64, ppc64le PkArchMap = map[string]string{ // map of PackageKit arch to GOARCH // TODO: add more values - "x86_64": "amd64", + // fedora + "x86_64": "amd64", + "aarch64": "arm64", + // debian, from: https://www.debian.org/ports/ + "amd64": "amd64", + "arm64": "arm64", + "i386": "386", + "i486": "386", + "i586": "386", + "i686": "386", } ) @@ -311,7 +324,7 @@ loop: // should already be broken break loop } else { - return []string{}, errors.New(fmt.Sprintf("PackageKit error: %v", signal.Body)) + return []string{}, errors.New(fmt.Sprintf("PackageKit: Error: %v", signal.Body)) } } } @@ -381,9 +394,10 @@ func (bus *Conn) InstallPackages(packageIds []string, transactionFlags uint64) e if call.Err != nil { return call.Err } + timeout := -1 // disabled initially + finished := false loop: for { - // FIXME: add a timeout option to error in case signals are dropped! select { case signal := <-ch: if signal.Path != interfacePath { @@ -392,22 +406,29 @@ loop: } if signal.Name == FmtTransactionMethod("ErrorCode") { - return errors.New(fmt.Sprintf("PackageKit error: %v", signal.Body)) + return errors.New(fmt.Sprintf("PackageKit: Error: %v", signal.Body)) } else if signal.Name == FmtTransactionMethod("Package") { // a package was installed... + // only start the timer once we're here... + timeout = PkSignalPackageTimeout continue loop } else if signal.Name == FmtTransactionMethod("Finished") { - // TODO: should we wait for the Destroy signal? - break loop + finished = true + timeout = PkSignalDestroyTimeout // wait a bit + continue loop } else if signal.Name == FmtTransactionMethod("Destroy") { - // should already be broken - break loop + return nil // success } else { - return errors.New(fmt.Sprintf("PackageKit error: %v", signal.Body)) + return errors.New(fmt.Sprintf("PackageKit: Error: %v", signal.Body)) } + case _ = <-TimeAfterOrBlock(timeout): + if finished { + log.Println("PackageKit: Timeout: InstallPackages: Waiting for 'Destroy'") + return nil // got tired of waiting for Destroy + } + return errors.New(fmt.Sprintf("PackageKit: Timeout: InstallPackages: %v", strings.Join(packageIds, ", "))) } } - return nil } // remove list of packages @@ -440,7 +461,7 @@ loop: } if signal.Name == FmtTransactionMethod("ErrorCode") { - return errors.New(fmt.Sprintf("PackageKit error: %v", signal.Body)) + return errors.New(fmt.Sprintf("PackageKit: Error: %v", signal.Body)) } else if signal.Name == FmtTransactionMethod("Package") { // a package was installed... continue loop @@ -451,7 +472,7 @@ loop: // should already be broken break loop } else { - return errors.New(fmt.Sprintf("PackageKit error: %v", signal.Body)) + return errors.New(fmt.Sprintf("PackageKit: Error: %v", signal.Body)) } } } @@ -485,7 +506,7 @@ loop: } if signal.Name == FmtTransactionMethod("ErrorCode") { - return errors.New(fmt.Sprintf("PackageKit error: %v", signal.Body)) + return errors.New(fmt.Sprintf("PackageKit: Error: %v", signal.Body)) } else if signal.Name == FmtTransactionMethod("Package") { // a package was installed... continue loop @@ -496,7 +517,7 @@ loop: // should already be broken break loop } else { - return errors.New(fmt.Sprintf("PackageKit error: %v", signal.Body)) + return errors.New(fmt.Sprintf("PackageKit: Error: %v", signal.Body)) } } } @@ -505,6 +526,8 @@ loop: // get the list of files that are contained inside a list of packageids func (bus *Conn) GetFilesByPackageId(packageIds []string) (files map[string][]string, err error) { + // NOTE: the maximum number of files in an RPM is 52116 in Fedora 23 + // https://gist.github.com/purpleidea/b98e60dcd449e1ac3b8a ch := make(chan *dbus.Signal, PkBufferSize) // we need to buffer :( interfacePath, err := bus.CreateTransaction() if err != nil { @@ -533,7 +556,7 @@ loop: } if signal.Name == FmtTransactionMethod("ErrorCode") { - err = errors.New(fmt.Sprintf("PackageKit error: %v", signal.Body)) + err = errors.New(fmt.Sprintf("PackageKit: Error: %v", signal.Body)) return // one signal returned per packageId found... @@ -560,7 +583,7 @@ loop: // should already be broken break loop } else { - err = errors.New(fmt.Sprintf("PackageKit error: %v", signal.Body)) + err = errors.New(fmt.Sprintf("PackageKit: Error: %v", signal.Body)) return } } @@ -596,7 +619,7 @@ loop: } if signal.Name == FmtTransactionMethod("ErrorCode") { - return nil, errors.New(fmt.Sprintf("PackageKit error: %v", signal.Body)) + return nil, errors.New(fmt.Sprintf("PackageKit: Error: %v", signal.Body)) } else if signal.Name == FmtTransactionMethod("Package") { //pkg_int, ok := signal.Body[0].(int) @@ -620,7 +643,7 @@ loop: // should already be broken break loop } else { - return nil, errors.New(fmt.Sprintf("PackageKit error: %v", signal.Body)) + return nil, errors.New(fmt.Sprintf("PackageKit: Error: %v", signal.Body)) } } } @@ -667,7 +690,8 @@ func (bus *Conn) PackagesToPackageIds(packageMap map[string]string, filter uint6 s := strings.Split(packageId, ";") //if len(s) != 4 { continue } // this would be a bug! pkg, ver, arch, data := s[0], s[1], s[2], s[3] - if goarch, ok := PkArchMap[arch]; !ok || goarch != runtime.GOARCH { + // we might need to allow some of this, eg: i386 .deb on amd64 + if !IsMyArch(arch) { continue } @@ -811,3 +835,12 @@ func FlagInData(flag, data string) bool { func FmtTransactionMethod(method string) string { return fmt.Sprintf("%s.%s", PkIfaceTransaction, method) } + +func IsMyArch(arch string) bool { + goarch, ok := PkArchMap[arch] + if !ok { + // if you get this error, please update the PkArchMap const + log.Fatalf("PackageKit: Arch '%v', not found!", arch) + } + return goarch == runtime.GOARCH +} diff --git a/pkg.go b/pkg.go index a55343fa..563e76a3 100644 --- a/pkg.go +++ b/pkg.go @@ -26,10 +26,11 @@ import ( type PkgRes struct { BaseRes `yaml:",inline"` - State string `yaml:"state"` // state: installed, uninstalled, newest, - AllowUntrusted bool `yaml:"allowuntrusted"` // allow untrusted packages to be installed? - AllowNonFree bool `yaml:"allownonfree"` // allow nonfree packages to be found? - AllowUnsupported bool `yaml:"allowunsupported"` // allow unsupported packages to be found? + State string `yaml:"state"` // state: installed, uninstalled, newest, + AllowUntrusted bool `yaml:"allowuntrusted"` // allow untrusted packages to be installed? + AllowNonFree bool `yaml:"allownonfree"` // allow nonfree packages to be found? + AllowUnsupported bool `yaml:"allowunsupported"` // allow unsupported packages to be found? + fileList []string // FIXME: update if pkg changes } // helper function for creating new pkg resources that calls Init() @@ -247,8 +248,115 @@ func (obj *PkgRes) CheckApply(apply bool) (stateok bool, err error) { return false, nil // success } +type PkgResAutoEdges struct { + fileList []string + testIsNext bool // safety + name string // saved data from PkgRes obj + kind string +} + +func (obj *PkgResAutoEdges) Next() []ResUUID { + if obj.testIsNext { + log.Fatal("Expecting a call to Test()") + } + obj.testIsNext = true // set after all the errors paths are past + var result []ResUUID + + // return UUID's for whatever is in obj.fileList + for _, x := range obj.fileList { + var reversed bool = false // cheat by passing a pointer + result = append(result, &FileUUID{ + BaseUUID: BaseUUID{ + name: obj.name, + kind: obj.kind, + reversed: &reversed, + }, + path: x, // what matters + }) // build list + } + return result +} + +func (obj *PkgResAutoEdges) Test(input []bool) bool { + if !obj.testIsNext { + log.Fatal("Expecting a call to Next()") + } + count := len(obj.fileList) + if count != len(input) { + log.Fatalf("Expecting %d value(s)!", count) + } + obj.testIsNext = false // set after all the errors paths are past + + // while i do believe this algorithm generates the *correct* result, i + // don't know if it does so in the optimal way. improvements welcome! + // the basic logic is: + // 0) Next() returns whatever is in fileList + // 1) Test() computes the dirname of each file, and removes duplicates + // and dirname's that have been in the path of an ack from input results + // 2) It then simplifies the list by removing the common path prefixes + // 3) Lastly, the remaining set of files (dirs) is used as new fileList + // 4) We then iterate in (0) until the fileList is empty! + var dirs = make([]string, count) + done := []string{} + for i := 0; i < count; i++ { + dir := Dirname(obj.fileList[i]) // dirname of /foo/ should be / + dirs[i] = dir + if input[i] { + done = append(done, dir) + } + } + nodupes := StrRemoveDuplicatesInList(dirs) // remove duplicates + nodones := StrFilterElementsInList(done, nodupes) // filter out done + noempty := StrFilterElementsInList([]string{""}, nodones) // remove the "" from / + obj.fileList = RemoveCommonFilePrefixes(noempty) // magic + + if len(obj.fileList) == 0 { // nothing more, don't continue + return false + } + return true // continue, there are more files! +} + +type PkgUUID struct { + BaseUUID +} + +// if and only if they are equivalent, return true +// if they are not equivalent, return false +func (obj *PkgUUID) IFF(uuid ResUUID) bool { + res, ok := uuid.(*PkgUUID) + if !ok { + return false + } + return obj.name == res.name +} + +// produce an object which generates a minimal pkg file optimization sequence func (obj *PkgRes) AutoEdges() AutoEdge { - return nil + // in contrast with the FileRes AutoEdges() function which contains + // more of the mechanics, most of the AutoEdge mechanics for the PkgRes + // is contained in the Test() method! This design is completely okay! + return &PkgResAutoEdges{ + fileList: obj.fileList, + testIsNext: false, // start with Next() call + name: obj.GetName(), // save data for PkgResAutoEdges obj + kind: obj.Kind(), + } +} + +// include all params to make a unique identification of this object +func (obj *PkgRes) GetUUIDs() []ResUUID { + x := &PkgUUID{ + BaseUUID: BaseUUID{name: obj.GetName(), kind: obj.Kind()}, + } + result := []ResUUID{x} + for _, path := range obj.fileList { + x := &FileUUID{ + BaseUUID: BaseUUID{name: obj.GetName(), kind: obj.Kind()}, + path: path, + } + result = append(result, x) + } + return result } func (obj *PkgRes) Compare(res Res) bool {