Add pkg auto edge basics with packagekit improvements

This is a monster patch that finally gets the iterative pkg auto edges
working the way they should. For each file, as soon as one matches, we
don't want to keep add dependencies on other file objects under that
tree structure. This reduces the number of necessary edges considerably,
and allows the graph to run more concurrently.
This commit is contained in:
James Shubin
2016-03-07 16:42:05 -05:00
parent a9538052bf
commit f39551952f
6 changed files with 724 additions and 30 deletions

24
examples/autoedges2.yaml Normal file
View File

@@ -0,0 +1,24 @@
---
graph: mygraph
resources:
file:
- name: file1
meta:
autoedge: true
path: "/etc/drbd.conf"
content: |
# this is an mgmt test
state: exists
- name: file2
meta:
autoedge: true
path: "/tmp/foo/"
content: |
i am f2
state: exists
pkg:
- name: drbd-utils
meta:
autoedge: true
state: installed
edges: []

View File

@@ -374,7 +374,6 @@ func (obj *FileRes) CheckApply(apply bool) (stateok bool, err error) {
return false, nil // success
}
type FileUUID struct {
BaseUUID
path string

98
misc.go
View File

@@ -27,6 +27,39 @@ import (
"time"
)
// return true if a string exists inside a list, otherwise false
func StrInList(needle string, haystack []string) bool {
for _, x := range haystack {
if needle == x {
return true
}
}
return false
}
// remove any duplicate values in the list
// possibly sub-optimal, O(n^2)? implementation
func StrRemoveDuplicatesInList(list []string) []string {
unique := []string{}
for _, x := range list {
if !StrInList(x, unique) {
unique = append(unique, x)
}
}
return unique
}
// remove any of the elements in filter, if they exist in list
func StrFilterElementsInList(filter []string, list []string) []string {
result := []string{}
for _, x := range list {
if !StrInList(x, filter) {
result = append(result, x)
}
}
return result
}
// reverse a list of strings
func ReverseStringList(in []string) []string {
var out []string // empty list
@@ -81,6 +114,46 @@ func HasPathPrefix(p, prefix string) bool {
return true
}
func StrInPathPrefixList(needle string, haystack []string) bool {
for _, x := range haystack {
if HasPathPrefix(x, needle) {
return true
}
}
return false
}
// remove redundant file path prefixes that are under the tree of other files
func RemoveCommonFilePrefixes(paths []string) []string {
var result = make([]string, len(paths))
for i := 0; i < len(paths); i++ { // copy, b/c append can modify the args!!
result[i] = paths[i]
}
// is there a string path which is common everywhere?
// if so, remove it, and iterate until nothing common is left
// return what's left over, that's the most common superset
loop:
for {
if len(result) <= 1 {
return result
}
for i := 0; i < len(result); i++ {
var copied = make([]string, len(result))
for j := 0; j < len(result); j++ { // copy, b/c append can modify the args!!
copied[j] = result[j]
}
noi := append(copied[:i], copied[i+1:]...) // rm i
if StrInPathPrefixList(result[i], noi) {
// delete the element common to everyone
result = noi
continue loop
}
}
break
}
return result
}
// Delta of path prefix, tells you how many path tokens different the prefix is
func PathPrefixDelta(p, prefix string) int {
@@ -112,6 +185,31 @@ func PathSplitFullReversed(p string) []string {
return ReverseStringList(result)
}
// add trailing slashes to any likely dirs in a package manager fileList
// if removeDirs is true, instead, don't keep the dirs in our output
func DirifyFileList(fileList []string, removeDirs bool) []string {
dirs := []string{}
for _, file := range fileList {
dir, _ := path.Split(file) // dir
dir = path.Clean(dir) // clean so cmp is easier
if !StrInList(dir, dirs) {
dirs = append(dirs, dir)
}
}
result := []string{}
for _, file := range fileList {
cleanFile := path.Clean(file)
if !StrInList(cleanFile, dirs) { // we're not a directory!
result = append(result, file) // pass through
} else if !removeDirs {
result = append(result, cleanFile+"/")
}
}
return result
}
// encode an object as base 64, serialize and then base64 encode
func ObjToB64(obj interface{}) (string, bool) {
b := bytes.Buffer{}

View File

@@ -20,6 +20,7 @@ package main
import (
"fmt"
"reflect"
"sort"
"testing"
)
@@ -33,6 +34,10 @@ func TestMiscT1(t *testing.T) {
t.Errorf("Result is incorrect.")
}
if Dirname("/foo/") != "/" {
t.Errorf("Result is incorrect.")
}
if Dirname("/") != "" { // TODO: should this equal "/" or "" ?
t.Errorf("Result is incorrect.")
}
@@ -45,6 +50,10 @@ func TestMiscT1(t *testing.T) {
t.Errorf("Result is incorrect.")
}
if Basename("/foo/") != "foo/" {
t.Errorf("Result is incorrect.")
}
if Basename("/") != "/" { // TODO: should this equal "" or "/" ?
t.Errorf("Result is incorrect.")
}
@@ -100,6 +109,10 @@ func TestMiscT3(t *testing.T) {
if HasPathPrefix("/foo/bar/baz/", "/foo/bar/baz/dude") != false {
t.Errorf("Result should be false.")
}
if HasPathPrefix("/foo/bar/baz/boo/", "/foo/") != true {
t.Errorf("Result should be true.")
}
}
func TestMiscT4(t *testing.T) {
@@ -225,3 +238,422 @@ func TestMiscT8(t *testing.T) {
}
}
func TestMiscT9(t *testing.T) {
fileListIn := []string{ // list taken from drbd-utils package
"/etc/drbd.conf",
"/etc/drbd.d/global_common.conf",
"/lib/drbd/drbd",
"/lib/drbd/drbdadm-83",
"/lib/drbd/drbdadm-84",
"/lib/drbd/drbdsetup-83",
"/lib/drbd/drbdsetup-84",
"/usr/lib/drbd/crm-fence-peer.sh",
"/usr/lib/drbd/crm-unfence-peer.sh",
"/usr/lib/drbd/notify-emergency-reboot.sh",
"/usr/lib/drbd/notify-emergency-shutdown.sh",
"/usr/lib/drbd/notify-io-error.sh",
"/usr/lib/drbd/notify-out-of-sync.sh",
"/usr/lib/drbd/notify-pri-lost-after-sb.sh",
"/usr/lib/drbd/notify-pri-lost.sh",
"/usr/lib/drbd/notify-pri-on-incon-degr.sh",
"/usr/lib/drbd/notify-split-brain.sh",
"/usr/lib/drbd/notify.sh",
"/usr/lib/drbd/outdate-peer.sh",
"/usr/lib/drbd/rhcs_fence",
"/usr/lib/drbd/snapshot-resync-target-lvm.sh",
"/usr/lib/drbd/stonith_admin-fence-peer.sh",
"/usr/lib/drbd/unsnapshot-resync-target-lvm.sh",
"/usr/lib/systemd/system/drbd.service",
"/usr/lib/tmpfiles.d/drbd.conf",
"/usr/sbin/drbd-overview",
"/usr/sbin/drbdadm",
"/usr/sbin/drbdmeta",
"/usr/sbin/drbdsetup",
"/usr/share/doc/drbd-utils/COPYING",
"/usr/share/doc/drbd-utils/ChangeLog",
"/usr/share/doc/drbd-utils/README",
"/usr/share/doc/drbd-utils/drbd.conf.example",
"/usr/share/man/man5/drbd.conf-8.3.5.gz",
"/usr/share/man/man5/drbd.conf-8.4.5.gz",
"/usr/share/man/man5/drbd.conf-9.0.5.gz",
"/usr/share/man/man5/drbd.conf.5.gz",
"/usr/share/man/man8/drbd-8.3.8.gz",
"/usr/share/man/man8/drbd-8.4.8.gz",
"/usr/share/man/man8/drbd-9.0.8.gz",
"/usr/share/man/man8/drbd-overview-9.0.8.gz",
"/usr/share/man/man8/drbd-overview.8.gz",
"/usr/share/man/man8/drbd.8.gz",
"/usr/share/man/man8/drbdadm-8.3.8.gz",
"/usr/share/man/man8/drbdadm-8.4.8.gz",
"/usr/share/man/man8/drbdadm-9.0.8.gz",
"/usr/share/man/man8/drbdadm.8.gz",
"/usr/share/man/man8/drbddisk-8.3.8.gz",
"/usr/share/man/man8/drbddisk-8.4.8.gz",
"/usr/share/man/man8/drbdmeta-8.3.8.gz",
"/usr/share/man/man8/drbdmeta-8.4.8.gz",
"/usr/share/man/man8/drbdmeta-9.0.8.gz",
"/usr/share/man/man8/drbdmeta.8.gz",
"/usr/share/man/man8/drbdsetup-8.3.8.gz",
"/usr/share/man/man8/drbdsetup-8.4.8.gz",
"/usr/share/man/man8/drbdsetup-9.0.8.gz",
"/usr/share/man/man8/drbdsetup.8.gz",
"/etc/drbd.d",
"/usr/share/doc/drbd-utils",
"/var/lib/drbd",
}
sort.Strings(fileListIn)
fileListOut := []string{ // fixed up manually
"/etc/drbd.conf",
"/etc/drbd.d/global_common.conf",
"/lib/drbd/drbd",
"/lib/drbd/drbdadm-83",
"/lib/drbd/drbdadm-84",
"/lib/drbd/drbdsetup-83",
"/lib/drbd/drbdsetup-84",
"/usr/lib/drbd/crm-fence-peer.sh",
"/usr/lib/drbd/crm-unfence-peer.sh",
"/usr/lib/drbd/notify-emergency-reboot.sh",
"/usr/lib/drbd/notify-emergency-shutdown.sh",
"/usr/lib/drbd/notify-io-error.sh",
"/usr/lib/drbd/notify-out-of-sync.sh",
"/usr/lib/drbd/notify-pri-lost-after-sb.sh",
"/usr/lib/drbd/notify-pri-lost.sh",
"/usr/lib/drbd/notify-pri-on-incon-degr.sh",
"/usr/lib/drbd/notify-split-brain.sh",
"/usr/lib/drbd/notify.sh",
"/usr/lib/drbd/outdate-peer.sh",
"/usr/lib/drbd/rhcs_fence",
"/usr/lib/drbd/snapshot-resync-target-lvm.sh",
"/usr/lib/drbd/stonith_admin-fence-peer.sh",
"/usr/lib/drbd/unsnapshot-resync-target-lvm.sh",
"/usr/lib/systemd/system/drbd.service",
"/usr/lib/tmpfiles.d/drbd.conf",
"/usr/sbin/drbd-overview",
"/usr/sbin/drbdadm",
"/usr/sbin/drbdmeta",
"/usr/sbin/drbdsetup",
"/usr/share/doc/drbd-utils/COPYING",
"/usr/share/doc/drbd-utils/ChangeLog",
"/usr/share/doc/drbd-utils/README",
"/usr/share/doc/drbd-utils/drbd.conf.example",
"/usr/share/man/man5/drbd.conf-8.3.5.gz",
"/usr/share/man/man5/drbd.conf-8.4.5.gz",
"/usr/share/man/man5/drbd.conf-9.0.5.gz",
"/usr/share/man/man5/drbd.conf.5.gz",
"/usr/share/man/man8/drbd-8.3.8.gz",
"/usr/share/man/man8/drbd-8.4.8.gz",
"/usr/share/man/man8/drbd-9.0.8.gz",
"/usr/share/man/man8/drbd-overview-9.0.8.gz",
"/usr/share/man/man8/drbd-overview.8.gz",
"/usr/share/man/man8/drbd.8.gz",
"/usr/share/man/man8/drbdadm-8.3.8.gz",
"/usr/share/man/man8/drbdadm-8.4.8.gz",
"/usr/share/man/man8/drbdadm-9.0.8.gz",
"/usr/share/man/man8/drbdadm.8.gz",
"/usr/share/man/man8/drbddisk-8.3.8.gz",
"/usr/share/man/man8/drbddisk-8.4.8.gz",
"/usr/share/man/man8/drbdmeta-8.3.8.gz",
"/usr/share/man/man8/drbdmeta-8.4.8.gz",
"/usr/share/man/man8/drbdmeta-9.0.8.gz",
"/usr/share/man/man8/drbdmeta.8.gz",
"/usr/share/man/man8/drbdsetup-8.3.8.gz",
"/usr/share/man/man8/drbdsetup-8.4.8.gz",
"/usr/share/man/man8/drbdsetup-9.0.8.gz",
"/usr/share/man/man8/drbdsetup.8.gz",
"/etc/drbd.d/", // added trailing slash
"/usr/share/doc/drbd-utils/", // added trailing slash
"/var/lib/drbd", // can't be fixed :(
}
sort.Strings(fileListOut)
dirify := DirifyFileList(fileListIn, false) // TODO: test with true
sort.Strings(dirify)
equals := reflect.DeepEqual(fileListOut, dirify)
if a, b := len(fileListOut), len(dirify); a != b {
t.Errorf("DirifyFileList counts didn't match: %d != %d", a, b)
} else if !equals {
t.Error("DirifyFileList did not match expected!")
for i := 0; i < len(dirify); i++ {
if fileListOut[i] != dirify[i] {
t.Errorf("# %d: %v <> %v", i, fileListOut[i], dirify[i])
}
}
}
}
func TestMiscT10(t *testing.T) {
fileListIn := []string{ // fake package list
"/etc/drbd.conf",
"/usr/share/man/man8/drbdsetup.8.gz",
"/etc/drbd.d",
"/etc/drbd.d/foo",
"/var/lib/drbd",
"/var/somedir/",
}
sort.Strings(fileListIn)
fileListOut := []string{ // fixed up manually
"/etc/drbd.conf",
"/usr/share/man/man8/drbdsetup.8.gz",
"/etc/drbd.d/", // added trailing slash
"/etc/drbd.d/foo",
"/var/lib/drbd", // can't be fixed :(
"/var/somedir/", // stays the same
}
sort.Strings(fileListOut)
dirify := DirifyFileList(fileListIn, false) // TODO: test with true
sort.Strings(dirify)
equals := reflect.DeepEqual(fileListOut, dirify)
if a, b := len(fileListOut), len(dirify); a != b {
t.Errorf("DirifyFileList counts didn't match: %d != %d", a, b)
} else if !equals {
t.Error("DirifyFileList did not match expected!")
for i := 0; i < len(dirify); i++ {
if fileListOut[i] != dirify[i] {
t.Errorf("# %d: %v <> %v", i, fileListOut[i], dirify[i])
}
}
}
}
func TestMiscT11(t *testing.T) {
in1 := []string{"/", "/usr/", "/usr/lib/", "/usr/share/"} // input
ex1 := []string{"/usr/lib/", "/usr/share/"} // expected
sort.Strings(ex1)
out1 := RemoveCommonFilePrefixes(in1)
sort.Strings(out1)
if !reflect.DeepEqual(ex1, out1) {
t.Errorf("RemoveCommonFilePrefixes expected: %v; got: %v.", ex1, out1)
}
in2 := []string{"/", "/usr/"}
ex2 := []string{"/usr/"}
sort.Strings(ex2)
out2 := RemoveCommonFilePrefixes(in2)
sort.Strings(out2)
if !reflect.DeepEqual(ex2, out2) {
t.Errorf("RemoveCommonFilePrefixes expected: %v; got: %v.", ex2, out2)
}
in3 := []string{"/"}
ex3 := []string{"/"}
out3 := RemoveCommonFilePrefixes(in3)
if !reflect.DeepEqual(ex3, out3) {
t.Errorf("RemoveCommonFilePrefixes expected: %v; got: %v.", ex3, out3)
}
in4 := []string{"/usr/bin/foo", "/usr/bin/bar", "/usr/lib/", "/usr/share/"}
ex4 := []string{"/usr/bin/foo", "/usr/bin/bar", "/usr/lib/", "/usr/share/"}
sort.Strings(ex4)
out4 := RemoveCommonFilePrefixes(in4)
sort.Strings(out4)
if !reflect.DeepEqual(ex4, out4) {
t.Errorf("RemoveCommonFilePrefixes expected: %v; got: %v.", ex4, out4)
}
in5 := []string{"/usr/bin/foo", "/usr/bin/bar", "/usr/lib/", "/usr/share/", "/usr/bin"}
ex5 := []string{"/usr/bin/foo", "/usr/bin/bar", "/usr/lib/", "/usr/share/"}
sort.Strings(ex5)
out5 := RemoveCommonFilePrefixes(in5)
sort.Strings(out5)
if !reflect.DeepEqual(ex5, out5) {
t.Errorf("RemoveCommonFilePrefixes expected: %v; got: %v.", ex5, out5)
}
in6 := []string{"/etc/drbd.d/", "/lib/drbd/", "/usr/lib/drbd/", "/usr/lib/systemd/system/", "/usr/lib/tmpfiles.d/", "/usr/sbin/", "/usr/share/doc/drbd-utils/", "/usr/share/man/man5/", "/usr/share/man/man8/", "/usr/share/doc/", "/var/lib/"}
ex6 := []string{"/etc/drbd.d/", "/lib/drbd/", "/usr/lib/drbd/", "/usr/lib/systemd/system/", "/usr/lib/tmpfiles.d/", "/usr/sbin/", "/usr/share/doc/drbd-utils/", "/usr/share/man/man5/", "/usr/share/man/man8/", "/var/lib/"}
sort.Strings(ex6)
out6 := RemoveCommonFilePrefixes(in6)
sort.Strings(out6)
if !reflect.DeepEqual(ex6, out6) {
t.Errorf("RemoveCommonFilePrefixes expected: %v; got: %v.", ex6, out6)
}
in7 := []string{"/etc/", "/lib/", "/usr/lib/", "/usr/lib/systemd/", "/usr/", "/usr/share/doc/", "/usr/share/man/", "/var/"}
ex7 := []string{"/etc/", "/lib/", "/usr/lib/systemd/", "/usr/share/doc/", "/usr/share/man/", "/var/"}
sort.Strings(ex7)
out7 := RemoveCommonFilePrefixes(in7)
sort.Strings(out7)
if !reflect.DeepEqual(ex7, out7) {
t.Errorf("RemoveCommonFilePrefixes expected: %v; got: %v.", ex7, out7)
}
in8 := []string{
"/etc/drbd.conf",
"/etc/drbd.d/global_common.conf",
"/lib/drbd/drbd",
"/lib/drbd/drbdadm-83",
"/lib/drbd/drbdadm-84",
"/lib/drbd/drbdsetup-83",
"/lib/drbd/drbdsetup-84",
"/usr/lib/drbd/crm-fence-peer.sh",
"/usr/lib/drbd/crm-unfence-peer.sh",
"/usr/lib/drbd/notify-emergency-reboot.sh",
"/usr/lib/drbd/notify-emergency-shutdown.sh",
"/usr/lib/drbd/notify-io-error.sh",
"/usr/lib/drbd/notify-out-of-sync.sh",
"/usr/lib/drbd/notify-pri-lost-after-sb.sh",
"/usr/lib/drbd/notify-pri-lost.sh",
"/usr/lib/drbd/notify-pri-on-incon-degr.sh",
"/usr/lib/drbd/notify-split-brain.sh",
"/usr/lib/drbd/notify.sh",
"/usr/lib/drbd/outdate-peer.sh",
"/usr/lib/drbd/rhcs_fence",
"/usr/lib/drbd/snapshot-resync-target-lvm.sh",
"/usr/lib/drbd/stonith_admin-fence-peer.sh",
"/usr/lib/drbd/unsnapshot-resync-target-lvm.sh",
"/usr/lib/systemd/system/drbd.service",
"/usr/lib/tmpfiles.d/drbd.conf",
"/usr/sbin/drbd-overview",
"/usr/sbin/drbdadm",
"/usr/sbin/drbdmeta",
"/usr/sbin/drbdsetup",
"/usr/share/doc/drbd-utils/COPYING",
"/usr/share/doc/drbd-utils/ChangeLog",
"/usr/share/doc/drbd-utils/README",
"/usr/share/doc/drbd-utils/drbd.conf.example",
"/usr/share/man/man5/drbd.conf-8.3.5.gz",
"/usr/share/man/man5/drbd.conf-8.4.5.gz",
"/usr/share/man/man5/drbd.conf-9.0.5.gz",
"/usr/share/man/man5/drbd.conf.5.gz",
"/usr/share/man/man8/drbd-8.3.8.gz",
"/usr/share/man/man8/drbd-8.4.8.gz",
"/usr/share/man/man8/drbd-9.0.8.gz",
"/usr/share/man/man8/drbd-overview-9.0.8.gz",
"/usr/share/man/man8/drbd-overview.8.gz",
"/usr/share/man/man8/drbd.8.gz",
"/usr/share/man/man8/drbdadm-8.3.8.gz",
"/usr/share/man/man8/drbdadm-8.4.8.gz",
"/usr/share/man/man8/drbdadm-9.0.8.gz",
"/usr/share/man/man8/drbdadm.8.gz",
"/usr/share/man/man8/drbddisk-8.3.8.gz",
"/usr/share/man/man8/drbddisk-8.4.8.gz",
"/usr/share/man/man8/drbdmeta-8.3.8.gz",
"/usr/share/man/man8/drbdmeta-8.4.8.gz",
"/usr/share/man/man8/drbdmeta-9.0.8.gz",
"/usr/share/man/man8/drbdmeta.8.gz",
"/usr/share/man/man8/drbdsetup-8.3.8.gz",
"/usr/share/man/man8/drbdsetup-8.4.8.gz",
"/usr/share/man/man8/drbdsetup-9.0.8.gz",
"/usr/share/man/man8/drbdsetup.8.gz",
"/etc/drbd.d/",
"/usr/share/doc/drbd-utils/",
"/var/lib/drbd",
}
ex8 := []string{
"/etc/drbd.conf",
"/etc/drbd.d/global_common.conf",
"/lib/drbd/drbd",
"/lib/drbd/drbdadm-83",
"/lib/drbd/drbdadm-84",
"/lib/drbd/drbdsetup-83",
"/lib/drbd/drbdsetup-84",
"/usr/lib/drbd/crm-fence-peer.sh",
"/usr/lib/drbd/crm-unfence-peer.sh",
"/usr/lib/drbd/notify-emergency-reboot.sh",
"/usr/lib/drbd/notify-emergency-shutdown.sh",
"/usr/lib/drbd/notify-io-error.sh",
"/usr/lib/drbd/notify-out-of-sync.sh",
"/usr/lib/drbd/notify-pri-lost-after-sb.sh",
"/usr/lib/drbd/notify-pri-lost.sh",
"/usr/lib/drbd/notify-pri-on-incon-degr.sh",
"/usr/lib/drbd/notify-split-brain.sh",
"/usr/lib/drbd/notify.sh",
"/usr/lib/drbd/outdate-peer.sh",
"/usr/lib/drbd/rhcs_fence",
"/usr/lib/drbd/snapshot-resync-target-lvm.sh",
"/usr/lib/drbd/stonith_admin-fence-peer.sh",
"/usr/lib/drbd/unsnapshot-resync-target-lvm.sh",
"/usr/lib/systemd/system/drbd.service",
"/usr/lib/tmpfiles.d/drbd.conf",
"/usr/sbin/drbd-overview",
"/usr/sbin/drbdadm",
"/usr/sbin/drbdmeta",
"/usr/sbin/drbdsetup",
"/usr/share/doc/drbd-utils/COPYING",
"/usr/share/doc/drbd-utils/ChangeLog",
"/usr/share/doc/drbd-utils/README",
"/usr/share/doc/drbd-utils/drbd.conf.example",
"/usr/share/man/man5/drbd.conf-8.3.5.gz",
"/usr/share/man/man5/drbd.conf-8.4.5.gz",
"/usr/share/man/man5/drbd.conf-9.0.5.gz",
"/usr/share/man/man5/drbd.conf.5.gz",
"/usr/share/man/man8/drbd-8.3.8.gz",
"/usr/share/man/man8/drbd-8.4.8.gz",
"/usr/share/man/man8/drbd-9.0.8.gz",
"/usr/share/man/man8/drbd-overview-9.0.8.gz",
"/usr/share/man/man8/drbd-overview.8.gz",
"/usr/share/man/man8/drbd.8.gz",
"/usr/share/man/man8/drbdadm-8.3.8.gz",
"/usr/share/man/man8/drbdadm-8.4.8.gz",
"/usr/share/man/man8/drbdadm-9.0.8.gz",
"/usr/share/man/man8/drbdadm.8.gz",
"/usr/share/man/man8/drbddisk-8.3.8.gz",
"/usr/share/man/man8/drbddisk-8.4.8.gz",
"/usr/share/man/man8/drbdmeta-8.3.8.gz",
"/usr/share/man/man8/drbdmeta-8.4.8.gz",
"/usr/share/man/man8/drbdmeta-9.0.8.gz",
"/usr/share/man/man8/drbdmeta.8.gz",
"/usr/share/man/man8/drbdsetup-8.3.8.gz",
"/usr/share/man/man8/drbdsetup-8.4.8.gz",
"/usr/share/man/man8/drbdsetup-9.0.8.gz",
"/usr/share/man/man8/drbdsetup.8.gz",
"/var/lib/drbd",
}
sort.Strings(ex8)
out8 := RemoveCommonFilePrefixes(in8)
sort.Strings(out8)
if !reflect.DeepEqual(ex8, out8) {
t.Errorf("RemoveCommonFilePrefixes expected: %v; got: %v.", ex8, out8)
}
in9 := []string{
"/etc/drbd.conf",
"/etc/drbd.d/",
"/lib/drbd/drbd",
"/lib/drbd/",
"/lib/drbd/",
"/lib/drbd/",
"/usr/lib/drbd/",
"/usr/lib/drbd/",
"/usr/lib/drbd/",
"/usr/lib/drbd/",
"/usr/lib/drbd/",
"/usr/lib/systemd/system/",
"/usr/lib/tmpfiles.d/",
"/usr/sbin/",
"/usr/sbin/",
"/usr/share/doc/drbd-utils/",
"/usr/share/doc/drbd-utils/",
"/usr/share/man/man5/",
"/usr/share/man/man5/",
"/usr/share/man/man8/",
"/usr/share/man/man8/",
"/usr/share/man/man8/",
"/etc/drbd.d/",
"/usr/share/doc/drbd-utils/",
"/var/lib/drbd",
}
ex9 := []string{
"/etc/drbd.conf",
"/etc/drbd.d/",
"/lib/drbd/drbd",
"/usr/lib/drbd/",
"/usr/lib/systemd/system/",
"/usr/lib/tmpfiles.d/",
"/usr/sbin/",
"/usr/share/doc/drbd-utils/",
"/usr/share/man/man5/",
"/usr/share/man/man8/",
"/var/lib/drbd",
}
sort.Strings(ex9)
out9 := RemoveCommonFilePrefixes(in9)
sort.Strings(out9)
if !reflect.DeepEqual(ex9, out9) {
t.Errorf("RemoveCommonFilePrefixes expected: %v; got: %v.", ex9, out9)
}
}

View File

@@ -36,17 +36,30 @@ const (
const (
// FIXME: if PkBufferSize is too low, install seems to drop signals
PkBufferSize = 1000
PkPath = "/org/freedesktop/PackageKit"
PkIface = "org.freedesktop.PackageKit"
PkIfaceTransaction = PkIface + ".Transaction"
dbusAddMatch = "org.freedesktop.DBus.AddMatch"
PkBufferSize = 1000
// TODO: the PkSignalTimeout value might be too low
PkSignalPackageTimeout = 60 // 60 seconds, arbitrary
PkSignalDestroyTimeout = 15 // 15 seconds, arbitrary
PkPath = "/org/freedesktop/PackageKit"
PkIface = "org.freedesktop.PackageKit"
PkIfaceTransaction = PkIface + ".Transaction"
dbusAddMatch = "org.freedesktop.DBus.AddMatch"
)
var (
// GOARCH's: 386, amd64, arm, arm64, mips64, mips64le, ppc64, ppc64le
PkArchMap = map[string]string{ // map of PackageKit arch to GOARCH
// TODO: add more values
"x86_64": "amd64",
// fedora
"x86_64": "amd64",
"aarch64": "arm64",
// debian, from: https://www.debian.org/ports/
"amd64": "amd64",
"arm64": "arm64",
"i386": "386",
"i486": "386",
"i586": "386",
"i686": "386",
}
)
@@ -311,7 +324,7 @@ loop:
// should already be broken
break loop
} else {
return []string{}, errors.New(fmt.Sprintf("PackageKit error: %v", signal.Body))
return []string{}, errors.New(fmt.Sprintf("PackageKit: Error: %v", signal.Body))
}
}
}
@@ -381,9 +394,10 @@ func (bus *Conn) InstallPackages(packageIds []string, transactionFlags uint64) e
if call.Err != nil {
return call.Err
}
timeout := -1 // disabled initially
finished := false
loop:
for {
// FIXME: add a timeout option to error in case signals are dropped!
select {
case signal := <-ch:
if signal.Path != interfacePath {
@@ -392,22 +406,29 @@ loop:
}
if signal.Name == FmtTransactionMethod("ErrorCode") {
return errors.New(fmt.Sprintf("PackageKit error: %v", signal.Body))
return errors.New(fmt.Sprintf("PackageKit: Error: %v", signal.Body))
} else if signal.Name == FmtTransactionMethod("Package") {
// a package was installed...
// only start the timer once we're here...
timeout = PkSignalPackageTimeout
continue loop
} else if signal.Name == FmtTransactionMethod("Finished") {
// TODO: should we wait for the Destroy signal?
break loop
finished = true
timeout = PkSignalDestroyTimeout // wait a bit
continue loop
} else if signal.Name == FmtTransactionMethod("Destroy") {
// should already be broken
break loop
return nil // success
} else {
return errors.New(fmt.Sprintf("PackageKit error: %v", signal.Body))
return errors.New(fmt.Sprintf("PackageKit: Error: %v", signal.Body))
}
case _ = <-TimeAfterOrBlock(timeout):
if finished {
log.Println("PackageKit: Timeout: InstallPackages: Waiting for 'Destroy'")
return nil // got tired of waiting for Destroy
}
return errors.New(fmt.Sprintf("PackageKit: Timeout: InstallPackages: %v", strings.Join(packageIds, ", ")))
}
}
return nil
}
// remove list of packages
@@ -440,7 +461,7 @@ loop:
}
if signal.Name == FmtTransactionMethod("ErrorCode") {
return errors.New(fmt.Sprintf("PackageKit error: %v", signal.Body))
return errors.New(fmt.Sprintf("PackageKit: Error: %v", signal.Body))
} else if signal.Name == FmtTransactionMethod("Package") {
// a package was installed...
continue loop
@@ -451,7 +472,7 @@ loop:
// should already be broken
break loop
} else {
return errors.New(fmt.Sprintf("PackageKit error: %v", signal.Body))
return errors.New(fmt.Sprintf("PackageKit: Error: %v", signal.Body))
}
}
}
@@ -485,7 +506,7 @@ loop:
}
if signal.Name == FmtTransactionMethod("ErrorCode") {
return errors.New(fmt.Sprintf("PackageKit error: %v", signal.Body))
return errors.New(fmt.Sprintf("PackageKit: Error: %v", signal.Body))
} else if signal.Name == FmtTransactionMethod("Package") {
// a package was installed...
continue loop
@@ -496,7 +517,7 @@ loop:
// should already be broken
break loop
} else {
return errors.New(fmt.Sprintf("PackageKit error: %v", signal.Body))
return errors.New(fmt.Sprintf("PackageKit: Error: %v", signal.Body))
}
}
}
@@ -505,6 +526,8 @@ loop:
// get the list of files that are contained inside a list of packageids
func (bus *Conn) GetFilesByPackageId(packageIds []string) (files map[string][]string, err error) {
// NOTE: the maximum number of files in an RPM is 52116 in Fedora 23
// https://gist.github.com/purpleidea/b98e60dcd449e1ac3b8a
ch := make(chan *dbus.Signal, PkBufferSize) // we need to buffer :(
interfacePath, err := bus.CreateTransaction()
if err != nil {
@@ -533,7 +556,7 @@ loop:
}
if signal.Name == FmtTransactionMethod("ErrorCode") {
err = errors.New(fmt.Sprintf("PackageKit error: %v", signal.Body))
err = errors.New(fmt.Sprintf("PackageKit: Error: %v", signal.Body))
return
// one signal returned per packageId found...
@@ -560,7 +583,7 @@ loop:
// should already be broken
break loop
} else {
err = errors.New(fmt.Sprintf("PackageKit error: %v", signal.Body))
err = errors.New(fmt.Sprintf("PackageKit: Error: %v", signal.Body))
return
}
}
@@ -596,7 +619,7 @@ loop:
}
if signal.Name == FmtTransactionMethod("ErrorCode") {
return nil, errors.New(fmt.Sprintf("PackageKit error: %v", signal.Body))
return nil, errors.New(fmt.Sprintf("PackageKit: Error: %v", signal.Body))
} else if signal.Name == FmtTransactionMethod("Package") {
//pkg_int, ok := signal.Body[0].(int)
@@ -620,7 +643,7 @@ loop:
// should already be broken
break loop
} else {
return nil, errors.New(fmt.Sprintf("PackageKit error: %v", signal.Body))
return nil, errors.New(fmt.Sprintf("PackageKit: Error: %v", signal.Body))
}
}
}
@@ -667,7 +690,8 @@ func (bus *Conn) PackagesToPackageIds(packageMap map[string]string, filter uint6
s := strings.Split(packageId, ";")
//if len(s) != 4 { continue } // this would be a bug!
pkg, ver, arch, data := s[0], s[1], s[2], s[3]
if goarch, ok := PkArchMap[arch]; !ok || goarch != runtime.GOARCH {
// we might need to allow some of this, eg: i386 .deb on amd64
if !IsMyArch(arch) {
continue
}
@@ -811,3 +835,12 @@ func FlagInData(flag, data string) bool {
func FmtTransactionMethod(method string) string {
return fmt.Sprintf("%s.%s", PkIfaceTransaction, method)
}
func IsMyArch(arch string) bool {
goarch, ok := PkArchMap[arch]
if !ok {
// if you get this error, please update the PkArchMap const
log.Fatalf("PackageKit: Arch '%v', not found!", arch)
}
return goarch == runtime.GOARCH
}

118
pkg.go
View File

@@ -26,10 +26,11 @@ import (
type PkgRes struct {
BaseRes `yaml:",inline"`
State string `yaml:"state"` // state: installed, uninstalled, newest, <version>
AllowUntrusted bool `yaml:"allowuntrusted"` // allow untrusted packages to be installed?
AllowNonFree bool `yaml:"allownonfree"` // allow nonfree packages to be found?
AllowUnsupported bool `yaml:"allowunsupported"` // allow unsupported packages to be found?
State string `yaml:"state"` // state: installed, uninstalled, newest, <version>
AllowUntrusted bool `yaml:"allowuntrusted"` // allow untrusted packages to be installed?
AllowNonFree bool `yaml:"allownonfree"` // allow nonfree packages to be found?
AllowUnsupported bool `yaml:"allowunsupported"` // allow unsupported packages to be found?
fileList []string // FIXME: update if pkg changes
}
// helper function for creating new pkg resources that calls Init()
@@ -247,8 +248,115 @@ func (obj *PkgRes) CheckApply(apply bool) (stateok bool, err error) {
return false, nil // success
}
type PkgResAutoEdges struct {
fileList []string
testIsNext bool // safety
name string // saved data from PkgRes obj
kind string
}
func (obj *PkgResAutoEdges) Next() []ResUUID {
if obj.testIsNext {
log.Fatal("Expecting a call to Test()")
}
obj.testIsNext = true // set after all the errors paths are past
var result []ResUUID
// return UUID's for whatever is in obj.fileList
for _, x := range obj.fileList {
var reversed bool = false // cheat by passing a pointer
result = append(result, &FileUUID{
BaseUUID: BaseUUID{
name: obj.name,
kind: obj.kind,
reversed: &reversed,
},
path: x, // what matters
}) // build list
}
return result
}
func (obj *PkgResAutoEdges) Test(input []bool) bool {
if !obj.testIsNext {
log.Fatal("Expecting a call to Next()")
}
count := len(obj.fileList)
if count != len(input) {
log.Fatalf("Expecting %d value(s)!", count)
}
obj.testIsNext = false // set after all the errors paths are past
// while i do believe this algorithm generates the *correct* result, i
// don't know if it does so in the optimal way. improvements welcome!
// the basic logic is:
// 0) Next() returns whatever is in fileList
// 1) Test() computes the dirname of each file, and removes duplicates
// and dirname's that have been in the path of an ack from input results
// 2) It then simplifies the list by removing the common path prefixes
// 3) Lastly, the remaining set of files (dirs) is used as new fileList
// 4) We then iterate in (0) until the fileList is empty!
var dirs = make([]string, count)
done := []string{}
for i := 0; i < count; i++ {
dir := Dirname(obj.fileList[i]) // dirname of /foo/ should be /
dirs[i] = dir
if input[i] {
done = append(done, dir)
}
}
nodupes := StrRemoveDuplicatesInList(dirs) // remove duplicates
nodones := StrFilterElementsInList(done, nodupes) // filter out done
noempty := StrFilterElementsInList([]string{""}, nodones) // remove the "" from /
obj.fileList = RemoveCommonFilePrefixes(noempty) // magic
if len(obj.fileList) == 0 { // nothing more, don't continue
return false
}
return true // continue, there are more files!
}
type PkgUUID struct {
BaseUUID
}
// if and only if they are equivalent, return true
// if they are not equivalent, return false
func (obj *PkgUUID) IFF(uuid ResUUID) bool {
res, ok := uuid.(*PkgUUID)
if !ok {
return false
}
return obj.name == res.name
}
// produce an object which generates a minimal pkg file optimization sequence
func (obj *PkgRes) AutoEdges() AutoEdge {
return nil
// in contrast with the FileRes AutoEdges() function which contains
// more of the mechanics, most of the AutoEdge mechanics for the PkgRes
// is contained in the Test() method! This design is completely okay!
return &PkgResAutoEdges{
fileList: obj.fileList,
testIsNext: false, // start with Next() call
name: obj.GetName(), // save data for PkgResAutoEdges obj
kind: obj.Kind(),
}
}
// include all params to make a unique identification of this object
func (obj *PkgRes) GetUUIDs() []ResUUID {
x := &PkgUUID{
BaseUUID: BaseUUID{name: obj.GetName(), kind: obj.Kind()},
}
result := []ResUUID{x}
for _, path := range obj.fileList {
x := &FileUUID{
BaseUUID: BaseUUID{name: obj.GetName(), kind: obj.Kind()},
path: path,
}
result = append(result, x)
}
return result
}
func (obj *PkgRes) Compare(res Res) bool {