Sorry for the size of this patch, I was busy hacking and plumbing away and it got out of hand! I'm allowing this because there doesn't seem to be anyone hacking away on parts of the code that this would break, since the resource code is fairly stable in this change. In particular, it revisits and refreshes some areas of the code that didn't see anything new or innovative since the project first started. I've gotten rid of a lot of cruft, and in particular cleaned up some things that I didn't know how to do better before! Here's hoping I'll continue to learn and have more to improve upon in the future! (Well let's not hope _too_ hard though!) The logical goal of this patch was to make logical grouping of resources possible. For example, it might be more efficient to group three package installations into a single transaction, instead of having to run three separate transactions. This is because a package installation typically has an initial one-time per run cost which shouldn't need to be repeated. Another future goal would be to group file resources sharing a common base path under a common recursive fanotify watcher. Since this depends on fanotify capabilities first, this hasn't been implemented yet, but could be a useful method of reducing the number of separate watches needed, since there is a finite limit. It's worth mentioning that grouping resources typically _reduces_ the parallel execution capability of a particular graph, but depending on the cost/benefit tradeoff, this might be preferential. I'd submit it's almost universally beneficial for pkg resources. This monster patch includes: * the autogroup feature * the grouping interface * a placeholder algorithm * an extensive test case infrastructure to test grouping algorithms * a move of some base resource methods into pgraph refactoring * some config/compile clean ups to remove code duplication * b64 encoding/decoding improvements * a rename of the yaml "res" entries to "kind" (more logical) * some docs * small fixes * and more!
354 lines
9.3 KiB
Go
354 lines
9.3 KiB
Go
// Mgmt
|
|
// Copyright (C) 2013-2016+ James Shubin and the project contributors
|
|
// Written by James Shubin <james@shubin.ca> and the project contributors
|
|
//
|
|
// This program is free software: you can redistribute it and/or modify
|
|
// it under the terms of the GNU Affero General Public License as published by
|
|
// the Free Software Foundation, either version 3 of the License, or
|
|
// (at your option) any later version.
|
|
//
|
|
// This program is distributed in the hope that it will be useful,
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
// GNU Affero General Public License for more details.
|
|
//
|
|
// You should have received a copy of the GNU Affero General Public License
|
|
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/base64"
|
|
"encoding/gob"
|
|
"fmt"
|
|
"log"
|
|
)
|
|
|
|
//go:generate stringer -type=resState -output=resstate_stringer.go
|
|
type resState int
|
|
|
|
const (
|
|
resStateNil resState = iota
|
|
resStateWatching
|
|
resStateEvent // an event has happened, but we haven't poked yet
|
|
resStateCheckApply
|
|
resStatePoking
|
|
)
|
|
|
|
//go:generate stringer -type=resConvergedState -output=resconvergedstate_stringer.go
|
|
type resConvergedState int
|
|
|
|
const (
|
|
resConvergedNil resConvergedState = iota
|
|
//resConverged
|
|
resConvergedTimeout
|
|
)
|
|
|
|
// a unique identifier for a resource, namely it's name, and the kind ("type")
|
|
type ResUUID interface {
|
|
GetName() string
|
|
Kind() string
|
|
IFF(ResUUID) bool
|
|
|
|
Reversed() bool // true means this resource happens before the generator
|
|
}
|
|
|
|
type BaseUUID struct {
|
|
name string // name and kind are the values of where this is coming from
|
|
kind string
|
|
|
|
reversed *bool // piggyback edge information here
|
|
}
|
|
|
|
type AutoEdge interface {
|
|
Next() []ResUUID // call to get list of edges to add
|
|
Test([]bool) bool // call until false
|
|
}
|
|
|
|
type MetaParams struct {
|
|
AutoEdge bool `yaml:"autoedge"` // metaparam, should we generate auto edges? // XXX should default to true
|
|
AutoGroup bool `yaml:"autogroup"` // metaparam, should we auto group? // XXX should default to true
|
|
}
|
|
|
|
// this interface is everything that is common to all resources
|
|
// everything here only needs to be implemented once, in the BaseRes
|
|
type Base interface {
|
|
GetName() string // can't be named "Name()" because of struct field
|
|
SetName(string)
|
|
Kind() string
|
|
GetMeta() MetaParams
|
|
SetVertex(*Vertex)
|
|
SetConvergedCallback(ctimeout int, converged chan bool)
|
|
IsWatching() bool
|
|
SetWatching(bool)
|
|
GetConvergedState() resConvergedState
|
|
SetConvergedState(resConvergedState)
|
|
GetState() resState
|
|
SetState(resState)
|
|
SendEvent(eventName, bool, bool) bool
|
|
ReadEvent(*Event) (bool, bool) // TODO: optional here?
|
|
GroupCmp(Res) bool // TODO: is there a better name for this?
|
|
GroupRes(Res) error // group resource (arg) into self
|
|
IsGrouped() bool // am I grouped?
|
|
SetGrouped(bool) // set grouped bool
|
|
GetGroup() []Res // return everyone grouped inside me
|
|
SetGroup([]Res)
|
|
}
|
|
|
|
// this is the minimum interface you need to implement to make a new resource
|
|
type Res interface {
|
|
Base // include everything from the Base interface
|
|
Init()
|
|
//Validate() bool // TODO: this might one day be added
|
|
GetUUIDs() []ResUUID // most resources only return one
|
|
Watch(chan struct{}) // send on channel to signal process() events
|
|
CheckApply(bool) (bool, error)
|
|
AutoEdges() AutoEdge
|
|
Compare(Res) bool
|
|
CollectPattern(string) // XXX: temporary until Res collection is more advanced
|
|
}
|
|
|
|
type BaseRes struct {
|
|
Name string `yaml:"name"`
|
|
Meta MetaParams `yaml:"meta"` // struct of all the metaparams
|
|
kind string
|
|
events chan Event
|
|
vertex *Vertex
|
|
state resState
|
|
convergedState resConvergedState
|
|
watching bool // is Watch() loop running ?
|
|
ctimeout int // converged timeout
|
|
converged chan bool
|
|
isStateOK bool // whether the state is okay based on events or not
|
|
isGrouped bool // am i contained within a group?
|
|
grouped []Res // list of any grouped resources
|
|
}
|
|
|
|
// wraps the IFF method when used with a list of UUID's
|
|
func UUIDExistsInUUIDs(uuid ResUUID, uuids []ResUUID) bool {
|
|
for _, u := range uuids {
|
|
if uuid.IFF(u) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func (obj *BaseUUID) GetName() string {
|
|
return obj.name
|
|
}
|
|
|
|
func (obj *BaseUUID) Kind() string {
|
|
return obj.kind
|
|
}
|
|
|
|
// if and only if they are equivalent, return true
|
|
// if they are not equivalent, return false
|
|
// most resource will want to override this method, since it does the important
|
|
// work of actually discerning if two resources are identical in function
|
|
func (obj *BaseUUID) IFF(uuid ResUUID) bool {
|
|
res, ok := uuid.(*BaseUUID)
|
|
if !ok {
|
|
return false
|
|
}
|
|
return obj.name == res.name
|
|
}
|
|
|
|
func (obj *BaseUUID) Reversed() bool {
|
|
if obj.reversed == nil {
|
|
log.Fatal("Programming error!")
|
|
}
|
|
return *obj.reversed
|
|
}
|
|
|
|
// initialize structures like channels if created without New constructor
|
|
func (obj *BaseRes) Init() {
|
|
obj.events = make(chan Event) // unbuffered chan size to avoid stale events
|
|
}
|
|
|
|
// this method gets used by all the resources
|
|
func (obj *BaseRes) GetName() string {
|
|
return obj.Name
|
|
}
|
|
|
|
func (obj *BaseRes) SetName(name string) {
|
|
obj.Name = name
|
|
}
|
|
|
|
// return the kind of resource this is
|
|
func (obj *BaseRes) Kind() string {
|
|
return obj.kind
|
|
}
|
|
|
|
func (obj *BaseRes) GetMeta() MetaParams {
|
|
return obj.Meta
|
|
}
|
|
|
|
func (obj *BaseRes) GetVertex() *Vertex {
|
|
return obj.vertex
|
|
}
|
|
|
|
func (obj *BaseRes) SetVertex(v *Vertex) {
|
|
obj.vertex = v
|
|
}
|
|
|
|
func (obj *BaseRes) SetConvergedCallback(ctimeout int, converged chan bool) {
|
|
obj.ctimeout = ctimeout
|
|
obj.converged = converged
|
|
}
|
|
|
|
// is the Watch() function running?
|
|
func (obj *BaseRes) IsWatching() bool {
|
|
return obj.watching
|
|
}
|
|
|
|
// store status of if the Watch() function is running
|
|
func (obj *BaseRes) SetWatching(b bool) {
|
|
obj.watching = b
|
|
}
|
|
|
|
func (obj *BaseRes) GetConvergedState() resConvergedState {
|
|
return obj.convergedState
|
|
}
|
|
|
|
func (obj *BaseRes) SetConvergedState(state resConvergedState) {
|
|
obj.convergedState = state
|
|
}
|
|
|
|
func (obj *BaseRes) GetState() resState {
|
|
return obj.state
|
|
}
|
|
|
|
func (obj *BaseRes) SetState(state resState) {
|
|
if DEBUG {
|
|
log.Printf("%v[%v]: State: %v -> %v", obj.Kind(), obj.GetName(), obj.GetState(), state)
|
|
}
|
|
obj.state = state
|
|
}
|
|
|
|
// push an event into the message queue for a particular vertex
|
|
func (obj *BaseRes) SendEvent(event eventName, sync bool, activity bool) bool {
|
|
// TODO: isn't this race-y ?
|
|
if !obj.IsWatching() { // element has already exited
|
|
return false // if we don't return, we'll block on the send
|
|
}
|
|
if !sync {
|
|
obj.events <- Event{event, nil, "", activity}
|
|
return true
|
|
}
|
|
|
|
resp := make(chan bool)
|
|
obj.events <- Event{event, resp, "", activity}
|
|
for {
|
|
value := <-resp
|
|
// wait until true value
|
|
if value {
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
|
|
// process events when a select gets one, this handles the pause code too!
|
|
// the return values specify if we should exit and poke respectively
|
|
func (obj *BaseRes) ReadEvent(event *Event) (exit, poke bool) {
|
|
event.ACK()
|
|
switch event.Name {
|
|
case eventStart:
|
|
return false, true
|
|
|
|
case eventPoke:
|
|
return false, true
|
|
|
|
case eventBackPoke:
|
|
return false, true // forward poking in response to a back poke!
|
|
|
|
case eventExit:
|
|
return true, false
|
|
|
|
case eventPause:
|
|
// wait for next event to continue
|
|
select {
|
|
case e := <-obj.events:
|
|
e.ACK()
|
|
if e.Name == eventExit {
|
|
return true, false
|
|
} else if e.Name == eventStart { // eventContinue
|
|
return false, false // don't poke on unpause!
|
|
} else {
|
|
// if we get a poke event here, it's a bug!
|
|
log.Fatalf("%v[%v]: Unknown event: %v, while paused!", obj.Kind(), obj.GetName(), e)
|
|
}
|
|
}
|
|
|
|
default:
|
|
log.Fatal("Unknown event: ", event)
|
|
}
|
|
return true, false // required to keep the stupid go compiler happy
|
|
}
|
|
|
|
func (obj *BaseRes) GroupRes(res Res) error {
|
|
if l := len(res.GetGroup()); l > 0 {
|
|
return fmt.Errorf("Res: %v already contains %d grouped resources!", res, l)
|
|
}
|
|
if res.IsGrouped() {
|
|
return fmt.Errorf("Res: %v is already grouped!", res)
|
|
}
|
|
|
|
obj.grouped = append(obj.grouped, res)
|
|
res.SetGrouped(true) // i am contained _in_ a group
|
|
return nil
|
|
}
|
|
|
|
func (obj *BaseRes) IsGrouped() bool { // am I grouped?
|
|
return obj.isGrouped
|
|
}
|
|
|
|
func (obj *BaseRes) SetGrouped(b bool) {
|
|
obj.isGrouped = b
|
|
}
|
|
|
|
func (obj *BaseRes) GetGroup() []Res { // return everyone grouped inside me
|
|
return obj.grouped
|
|
}
|
|
|
|
func (obj *BaseRes) SetGroup(g []Res) {
|
|
obj.grouped = g
|
|
}
|
|
|
|
func (obj *BaseRes) CollectPattern(pattern string) {
|
|
// XXX: default method is empty
|
|
}
|
|
|
|
// ResToB64 encodes a resource to a base64 encoded string (after serialization)
|
|
func ResToB64(res Res) (string, error) {
|
|
b := bytes.Buffer{}
|
|
e := gob.NewEncoder(&b)
|
|
err := e.Encode(&res) // pass with &
|
|
if err != nil {
|
|
return "", fmt.Errorf("Gob failed to encode: %v", err)
|
|
}
|
|
return base64.StdEncoding.EncodeToString(b.Bytes()), nil
|
|
}
|
|
|
|
// B64ToRes decodes a resource from a base64 encoded string (after deserialization)
|
|
func B64ToRes(str string) (Res, error) {
|
|
var output interface{}
|
|
bb, err := base64.StdEncoding.DecodeString(str)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("Base64 failed to decode: %v", err)
|
|
}
|
|
b := bytes.NewBuffer(bb)
|
|
d := gob.NewDecoder(b)
|
|
err = d.Decode(&output) // pass with &
|
|
if err != nil {
|
|
return nil, fmt.Errorf("Gob failed to decode: %v", err)
|
|
}
|
|
res, ok := output.(Res)
|
|
if !ok {
|
|
return nil, fmt.Errorf("Output %v is not a Res", res)
|
|
|
|
}
|
|
return res, nil
|
|
}
|