Add initial plumbing for autogroups

This adds some of the API changes and improvements to the pkg resource
so that it can make use of this feature.

examples/autogroup1.yaml

@@ -0,0 +1,21 @@
+---
+graph: mygraph
+resources:
+  pkg:
+  - name: drbd-utils
+    meta:
+      autogroup: false
+    state: installed
+  - name: powertop
+    meta:
+      autogroup: true
+    state: installed
+  - name: sl
+    meta:
+      autogroup: true
+    state: installed
+  - name: cowsay
+    meta:
+      autogroup: true
+    state: installed
+edges: []

exec.go

@@ -372,6 +372,14 @@ func (obj *ExecRes) GetUUIDs() []ResUUID {
 	return []ResUUID{x}
 }
 
+func (obj *ExecRes) GroupCmp(r Res) bool {
+	_, ok := r.(*SvcRes)
+	if !ok {
+		return false
+	}
+	return false // not possible atm
+}
+
 func (obj *ExecRes) Compare(res Res) bool {
 	switch res.(type) {
 	case *ExecRes:

file.go

@@ -457,6 +457,16 @@ func (obj *FileRes) GetUUIDs() []ResUUID {
 	return []ResUUID{x}
 }
 
+func (obj *FileRes) GroupCmp(r Res) bool {
+	_, ok := r.(*FileRes)
+	if !ok {
+		return false
+	}
+	// TODO: we might be able to group directory children into a single
+	// recursive watcher in the future, thus saving fanotify watches
+	return false // not possible atm
+}
+
 func (obj *FileRes) Compare(res Res) bool {
 	switch res.(type) {
 	case *FileRes:

misc.go

@@ -23,6 +23,7 @@ import (
 	"encoding/gob"
 	"github.com/godbus/dbus"
 	"path"
+	"sort"
 	"strings"
 	"time"
 )
@@ -60,6 +61,18 @@ func StrFilterElementsInList(filter []string, list []string) []string {
 	return result
 }
 
+// remove any of the elements in filter, if they don't exist in list
+// this is an in order intersection of two lists
+func StrListIntersection(list1 []string, list2 []string) []string {
+	result := []string{}
+	for _, x := range list1 {
+		if StrInList(x, list2) {
+			result = append(result, x)
+		}
+	}
+	return result
+}
+
 // reverse a list of strings
 func ReverseStringList(in []string) []string {
 	var out []string // empty list
@@ -70,6 +83,48 @@ func ReverseStringList(in []string) []string {
 	return out
 }
 
+// return the sorted list of string keys in a map with string keys
+// NOTE: i thought it would be nice for this to use: map[string]interface{} but
+// it turns out that's not allowed. I know we don't have generics, but common!
+func StrMapKeys(m map[string]string) []string {
+	result := []string{}
+	for k, _ := range m {
+		result = append(result, k)
+	}
+	sort.Strings(result) // deterministic order
+	return result
+}
+
+// return the sorted list of bool values in a map with string values
+func BoolMapValues(m map[string]bool) []bool {
+	result := []bool{}
+	for _, v := range m {
+		result = append(result, v)
+	}
+	//sort.Bools(result) // TODO: deterministic order
+	return result
+}
+
+// return the sorted list of string values in a map with string values
+func StrMapValues(m map[string]string) []string {
+	result := []string{}
+	for _, v := range m {
+		result = append(result, v)
+	}
+	sort.Strings(result) // deterministic order
+	return result
+}
+
+// return true if everyone is true
+func BoolMapTrue(l []bool) bool {
+	for _, b := range l {
+		if !b {
+			return false
+		}
+	}
+	return true
+}
+
 // Similar to the GNU dirname command
 func Dirname(p string) string {
 	if p == "/" {

noop.go

@@ -109,6 +109,18 @@ func (obj *NoopRes) GetUUIDs() []ResUUID {
 	return []ResUUID{x}
 }
 
+func (obj *NoopRes) GroupCmp(r Res) bool {
+	_, ok := r.(*NoopRes)
+	if !ok {
+		// NOTE: technically we could group a noop into any other
+		// resource, if that resource knew how to handle it, although,
+		// since the mechanics of inter-kind resource grouping are
+		// tricky, avoid doing this until there's a good reason.
+		return false
+	}
+	return true // noop resources can always be grouped together!
+}
+
 func (obj *NoopRes) Compare(res Res) bool {
 	switch res.(type) {
 	// we can only compare NoopRes to others of the same resource

packagekit.go

@@ -814,6 +814,75 @@ func (bus *Conn) PackagesToPackageIDs(packageMap map[string]string, filter uint6
 	return result, nil
 }
 
+// returns a list of packageIDs which match the set of package names in packages
+func FilterPackageIDs(m map[string]*PkPackageIDActionData, packages []string) ([]string, error) {
+	result := []string{}
+	for _, k := range packages {
+		obj, ok := m[k] // lookup single package
+		// package doesn't exist, this is an error!
+		if !ok || !obj.Found || obj.PackageID == "" {
+			return nil, fmt.Errorf("Can't find package named '%s'.", k)
+		}
+		result = append(result, obj.PackageID)
+	}
+	return result, nil
+}
+
+func FilterState(m map[string]*PkPackageIDActionData, packages []string, state string) (result map[string]bool, err error) {
+	result = make(map[string]bool)
+	pkgs := []string{} // bad pkgs that don't have a bool state
+	for _, k := range packages {
+		obj, ok := m[k] // lookup single package
+		// package doesn't exist, this is an error!
+		if !ok || !obj.Found {
+			return nil, fmt.Errorf("Can't find package named '%s'.", k)
+		}
+		var b bool
+		if state == "installed" {
+			b = obj.Installed
+		} else if state == "uninstalled" {
+			b = !obj.Installed
+		} else if state == "newest" {
+			b = obj.Newest
+		} else {
+			// we can't filter "version" state in this function
+			pkgs = append(pkgs, k)
+			continue
+		}
+		result[k] = b // save
+	}
+	if len(pkgs) > 0 {
+		err = fmt.Errorf("Can't filter non-boolean state on: %v!", strings.Join(pkgs, ","))
+	}
+	return result, err
+}
+
+// return all packages that are in package and match the specific state
+func FilterPackageState(m map[string]*PkPackageIDActionData, packages []string, state string) (result []string, err error) {
+	result = []string{}
+	for _, k := range packages {
+		obj, ok := m[k] // lookup single package
+		// package doesn't exist, this is an error!
+		if !ok || !obj.Found {
+			return nil, fmt.Errorf("Can't find package named '%s'.", k)
+		}
+		b := false
+		if state == "installed" && obj.Installed {
+			b = true
+		} else if state == "uninstalled" && !obj.Installed {
+			b = true
+		} else if state == "newest" && obj.Newest {
+			b = true
+		} else if state == obj.Version {
+			b = true
+		}
+		if b {
+			result = append(result, k)
+		}
+	}
+	return result, err
+}
+
 // does flag exist inside data portion of packageID field?
 func FlagInData(flag, data string) bool {
 	flags := strings.Split(data, ":")

pgraph.go

@@ -162,7 +162,7 @@ func (g *Graph) GetVertex(name string) chan *Vertex {
 
 func (g *Graph) GetVertexMatch(obj Res) *Vertex {
 	for k := range g.Adjacency {
-		if k.Compare(obj) { // XXX test
+		if k.Res.Compare(obj) {
 			return k
 		}
 	}

pkg.go

@@ -63,10 +63,18 @@ func (obj *PkgRes) Init() {
 	}
 	defer bus.Close()
 
-	data, err := obj.PkgMappingHelper(bus)
+	result, err := obj.pkgMappingHelper(bus)
 	if err != nil {
 		// FIXME: return error?
-		log.Fatalf("The PkgMappingHelper failed with: %v.", err)
+		log.Fatalf("The pkgMappingHelper failed with: %v.", err)
+		return
+	}
+
+	data, ok := result[obj.Name] // lookup single package (init does just one)
+	// package doesn't exist, this is an error!
+	if !ok || !data.Found {
+		// FIXME: return error?
+		log.Fatalf("Can't find package named '%s'.", obj.Name)
 		return
 	}
 
@@ -82,11 +90,6 @@ func (obj *PkgRes) Init() {
 	}
 }
 
-// XXX: run this when resource exits
-func (obj *PkgRes) Close() {
-	//obj.bus.Close()
-}
-
 func (obj *PkgRes) Validate() bool {
 
 	if obj.State == "" {
@@ -123,7 +126,7 @@ func (obj *PkgRes) Watch() {
 
 	for {
 		if DEBUG {
-			log.Printf("Pkg[%v]: Watching...", obj.GetName())
+			log.Printf("%v: Watching...", obj.fmtNames(obj.getNames()))
 		}
 
 		obj.SetState(resStateWatching) // reset
@@ -131,7 +134,7 @@ func (obj *PkgRes) Watch() {
 		case event := <-ch:
 			// FIXME: ask packagekit for info on what packages changed
 			if DEBUG {
-				log.Printf("Pkg[%v]: Event: %v", obj.GetName(), event.Name)
+				log.Printf("%v: Event: %v", obj.fmtNames(obj.getNames()), event.Name)
 			}
 
 			// since the chan is buffered, remove any supplemental
@@ -170,13 +173,48 @@ func (obj *PkgRes) Watch() {
 	}
 }
 
-func (obj *PkgRes) PkgMappingHelper(bus *Conn) (*PkPackageIDActionData, error) {
-
-	var packageMap = map[string]string{
-		obj.Name: obj.State, // key is pkg name, value is pkg state
+// get list of names when grouped or not
+func (obj *PkgRes) getNames() []string {
+	if g := obj.GetGroup(); len(g) > 0 { // grouped elements
+		names := []string{obj.GetName()}
+		for _, x := range g {
+			pkg, ok := x.(*PkgRes) // convert from Res
+			if ok {
+				names = append(names, pkg.Name)
+			}
+		}
+		return names
 	}
-	var filter uint64             // initializes at the "zero" value of 0
-	filter += PK_FILTER_ENUM_ARCH // always search in our arch (optional!)
+	return []string{obj.GetName()}
+}
+
+// pretty print for header values
+func (obj *PkgRes) fmtNames(names []string) string {
+	if len(obj.GetGroup()) > 0 { // grouped elements
+		return fmt.Sprintf("%v[autogroup:(%v)]", obj.Kind(), strings.Join(names, ","))
+	}
+	return fmt.Sprintf("%v[%v]", obj.Kind(), obj.GetName())
+}
+
+func (obj *PkgRes) groupMappingHelper() map[string]string {
+	var result = make(map[string]string)
+	if g := obj.GetGroup(); len(g) > 0 { // add any grouped elements
+		for _, x := range g {
+			pkg, ok := x.(*PkgRes) // convert from Res
+			if !ok {
+				log.Fatalf("Grouped member %v is not a %v", x, obj.Kind())
+			}
+			result[pkg.Name] = pkg.State
+		}
+	}
+	return result
+}
+
+func (obj *PkgRes) pkgMappingHelper(bus *Conn) (map[string]*PkPackageIDActionData, error) {
+	packageMap := obj.groupMappingHelper() // get the grouped values
+	packageMap[obj.Name] = obj.State       // key is pkg name, value is pkg state
+	var filter uint64                      // initializes at the "zero" value of 0
+	filter += PK_FILTER_ENUM_ARCH          // always search in our arch (optional!)
 	// we're requesting latest version, or to narrow down install choices!
 	if obj.State == "newest" || obj.State == "installed" {
 		// if we add this, we'll still see older packages if installed
@@ -194,21 +232,14 @@ func (obj *PkgRes) PkgMappingHelper(bus *Conn) (*PkPackageIDActionData, error) {
 	if e != nil {
 		return nil, fmt.Errorf("Can't run PackagesToPackageIDs: %v", e)
 	}
-
-	data, ok := result[obj.Name] // lookup single package
-	// package doesn't exist, this is an error!
-	if !ok || !data.Found {
-		return nil, fmt.Errorf("Can't find package named '%s'.", obj.Name)
-	}
-
-	return data, nil
+	return result, nil
 }
 
 func (obj *PkgRes) CheckApply(apply bool) (stateok bool, err error) {
-	log.Printf("%v[%v]: CheckApply(%t)", obj.Kind(), obj.GetName(), apply)
+	log.Printf("%v: CheckApply(%t)", obj.fmtNames(obj.getNames()), apply)
 
 	if obj.State == "" { // TODO: Validate() should replace this check!
-		log.Fatalf("%v[%v]: Package state is undefined!", obj.Kind(), obj.GetName())
+		log.Fatalf("%v: Package state is undefined!", obj.fmtNames(obj.getNames()))
 	}
 
 	if obj.isStateOK { // cache the state
@@ -221,24 +252,35 @@ func (obj *PkgRes) CheckApply(apply bool) (stateok bool, err error) {
 	}
 	defer bus.Close()
 
-	data, err := obj.PkgMappingHelper(bus)
+	result, err := obj.pkgMappingHelper(bus)
 	if err != nil {
-		return false, fmt.Errorf("The PkgMappingHelper failed with: %v.", err)
+		return false, fmt.Errorf("The pkgMappingHelper failed with: %v.", err)
 	}
 
+	packageMap := obj.groupMappingHelper() // map[string]string
+	packageList := []string{obj.Name}
+	packageList = append(packageList, StrMapKeys(packageMap)...)
+	//stateList := []string{obj.State}
+	//stateList = append(stateList, StrMapValues(packageMap)...)
+
+	// TODO: at the moment, all the states are the same, but
+	// eventually we might be able to drop this constraint!
+	states, err := FilterState(result, packageList, obj.State)
+	if err != nil {
+		return false, fmt.Errorf("The FilterState method failed with: %v.", err)
+	}
+	data, _ := result[obj.Name] // if above didn't error, we won't either!
+	validState := BoolMapTrue(BoolMapValues(states))
+
 	// obj.State == "installed" || "uninstalled" || "newest" || "4.2-1.fc23"
 	switch obj.State {
 	case "installed":
-		if data.Installed {
-			return true, nil // state is correct, exit!
-		}
+		fallthrough
 	case "uninstalled":
-		if !data.Installed {
-			return true, nil
-		}
+		fallthrough
 	case "newest":
-		if data.Newest {
-			return true, nil
+		if validState {
+			return true, nil // state is correct, exit!
 		}
 	default: // version string
 		if obj.State == data.Version && data.Version != "" {
@@ -246,42 +288,45 @@ func (obj *PkgRes) CheckApply(apply bool) (stateok bool, err error) {
 		}
 	}
 
-	if data.PackageID == "" {
-		return false, errors.New("Can't find package id to use.")
-	}
-
 	// state is not okay, no work done, exit, but without error
 	if !apply {
 		return false, nil
 	}
 
 	// apply portion
-	log.Printf("%v[%v]: Apply", obj.Kind(), obj.GetName())
-	packageList := []string{data.PackageID}
+	log.Printf("%v: Apply", obj.fmtNames(obj.getNames()))
+	readyPackages, err := FilterPackageState(result, packageList, obj.State)
+	if err != nil {
+		return false, err // fail
+	}
+	// these are the packages that actually need their states applied!
+	applyPackages := StrFilterElementsInList(readyPackages, packageList)
+	packageIDs, _ := FilterPackageIDs(result, applyPackages) // would be same err as above
+
 	var transactionFlags uint64 // initializes at the "zero" value of 0
 	if !obj.AllowUntrusted {    // allow
 		transactionFlags += PK_TRANSACTION_FLAG_ENUM_ONLY_TRUSTED
 	}
 	// apply correct state!
-	log.Printf("%v[%v]: Set: %v...", obj.Kind(), obj.GetName(), obj.State)
+	log.Printf("%v: Set: %v...", obj.fmtNames(StrListIntersection(applyPackages, obj.getNames())), obj.State)
 	switch obj.State {
 	case "uninstalled": // run remove
 		// NOTE: packageID is different than when installed, because now
 		// it has the "installed" flag added to the data portion if it!!
-		err = bus.RemovePackages(packageList, transactionFlags)
+		err = bus.RemovePackages(packageIDs, transactionFlags)
 
 	case "newest": // TODO: isn't this the same operation as install, below?
-		err = bus.UpdatePackages(packageList, transactionFlags)
+		err = bus.UpdatePackages(packageIDs, transactionFlags)
 
 	case "installed":
 		fallthrough // same method as for "set specific version", below
 	default: // version string
-		err = bus.InstallPackages(packageList, transactionFlags)
+		err = bus.InstallPackages(packageIDs, transactionFlags)
 	}
 	if err != nil {
 		return false, err // fail
 	}
-	log.Printf("%v[%v]: Set: %v success!", obj.Kind(), obj.GetName(), obj.State)
+	log.Printf("%v: Set: %v success!", obj.fmtNames(StrListIntersection(applyPackages, obj.getNames())), obj.State)
 	return false, nil // success
 }
 
@@ -427,6 +472,27 @@ func (obj *PkgRes) GetUUIDs() []ResUUID {
 	return result
 }
 
+// can these two resources be merged ?
+// (aka does this resource support doing so?)
+// will resource allow itself to be grouped _into_ this obj?
+func (obj *PkgRes) GroupCmp(r Res) bool {
+	res, ok := r.(*PkgRes)
+	if !ok {
+		return false
+	}
+	objStateIsVersion := (obj.State != "installed" && obj.State != "uninstalled" && obj.State != "newest") // must be a ver. string
+	resStateIsVersion := (res.State != "installed" && res.State != "uninstalled" && res.State != "newest") // must be a ver. string
+	if objStateIsVersion || resStateIsVersion {
+		// can't merge specific version checks atm
+		return false
+	}
+	// FIXME: keep it simple for now, only merge same states
+	if obj.State != res.State {
+		return false
+	}
+	return true
+}
+
 func (obj *PkgRes) Compare(res Res) bool {
 	switch res.(type) {
 	case *PkgRes:

resources.go

@@ -18,6 +18,7 @@
 package main
 
 import (
+	"fmt"
 	"log"
 	"time"
 )
@@ -64,7 +65,8 @@ type AutoEdge interface {
 }
 
 type MetaParams struct {
-	AutoEdge bool `yaml:"autoedge"` // metaparam, should we generate auto edges? // XXX should default to true
+	AutoEdge  bool `yaml:"autoedge"`  // metaparam, should we generate auto edges? // XXX should default to true
+	AutoGroup bool `yaml:"autogroup"` // metaparam, should we auto group? // XXX should default to true
 }
 
 // this interface is everything that is common to all resources
@@ -87,6 +89,12 @@ type Base interface {
 	OKTimestamp() bool
 	Poke(bool)
 	BackPoke()
+	GroupCmp(Res) bool  // TODO: is there a better name for this?
+	GroupRes(Res) error // group resource (arg) into self
+	IsGrouped() bool    // am I grouped?
+	SetGrouped(bool)    // set grouped bool
+	GetGroup() []Res    // return everyone grouped inside me
+	SetGroup([]Res)
 }
 
 // this is the minimum interface you need to implement to make a new resource
@@ -113,7 +121,9 @@ type BaseRes struct {
 	watching       bool // is Watch() loop running ?
 	ctimeout       int  // converged timeout
 	converged      chan bool
-	isStateOK      bool // whether the state is okay based on events or not
+	isStateOK      bool  // whether the state is okay based on events or not
+	isGrouped      bool  // am i contained within a group?
+	grouped        []Res // list of any grouped resources
 }
 
 // wraps the IFF method when used with a list of UUID's
@@ -355,6 +365,35 @@ func (obj *BaseRes) ReadEvent(event *Event) (exit, poke bool) {
 	return true, false // required to keep the stupid go compiler happy
 }
 
+func (obj *BaseRes) GroupRes(res Res) error {
+	if l := len(res.GetGroup()); l > 0 {
+		return fmt.Errorf("Res: %v already contains %d grouped resources!", res, l)
+	}
+	if res.IsGrouped() {
+		return fmt.Errorf("Res: %v is already grouped!", res)
+	}
+
+	obj.grouped = append(obj.grouped, res)
+	res.SetGrouped(true) // i am contained _in_ a group
+	return nil
+}
+
+func (obj *BaseRes) IsGrouped() bool { // am I grouped?
+	return obj.isGrouped
+}
+
+func (obj *BaseRes) SetGrouped(b bool) {
+	obj.isGrouped = b
+}
+
+func (obj *BaseRes) GetGroup() []Res { // return everyone grouped inside me
+	return obj.grouped
+}
+
+func (obj *BaseRes) SetGroup(g []Res) {
+	obj.grouped = g
+}
+
 // XXX: rename this function
 func Process(obj Res) {
 	if DEBUG {

svc.go

@@ -398,6 +398,17 @@ func (obj *SvcRes) GetUUIDs() []ResUUID {
 	return []ResUUID{x}
 }
 
+func (obj *SvcRes) GroupCmp(r Res) bool {
+	_, ok := r.(*SvcRes)
+	if !ok {
+		return false
+	}
+	// TODO: depending on if the systemd service api allows batching, we
+	// might be able to build this, although not sure how useful it is...
+	// it might just eliminate parallelism be bunching up the graph
+	return false // not possible atm
+}
+
 func (obj *SvcRes) Compare(res Res) bool {
 	switch res.(type) {
 	case *SvcRes: