lourenco/mgmt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

engine: Resources package rewrite

Browse Source 
This giant patch makes some much needed improvements to the code base.

* The engine has been rewritten and lives within engine/graph/
* All of the common interfaces and code now live in engine/
* All of the resources are in one package called engine/resources/
* The Res API can use different "traits" from engine/traits/
* The Res API has been simplified to hide many of the old internals
* The Watch & Process loops were previously inverted, but is now fixed
* The likelihood of package cycles has been reduced drastically
* And much, much more...

Unfortunately, some code had to be temporarily removed. The remote code
had to be taken out, as did the prometheus code. We hope to have these
back in new forms as soon as possible.
This commit is contained in:
James Shubin
2018-03-13 12:02:44 -04:00
parent ef49aa7e08
commit 9969286224
140 changed files with 9130 additions and 8764 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile
View File

@@ -122,7 +122,7 @@ lang: ## generates the lexer/parser for the language frontend
# build a `mgmt` binary for current host os/arch
$(PROGRAM): build/mgmt-${GOHOSTOS}-${GOHOSTARCH} ## build an mgmt binary for current host os/arch
cp $< $@
cp -a $< $@
$(PROGRAM).static: $(GO_FILES)
@echo "Building: $(PROGRAM).static, version: $(SVERSION)..."

102
converger/converger.go
View File

@@ -20,10 +20,13 @@ package converger
import (
"fmt"
"sort"
"sync"
"time"
"github.com/purpleidea/mgmt/util"
multierr "github.com/hashicorp/go-multierror"
)
// TODO: we could make a new function that masks out the state of certain
@@ -40,8 +43,9 @@ type Converger interface { // TODO: need a better name
Loop(bool)
ConvergedTimer(UID) <-chan time.Time
Status() map[uint64]bool
Timeout() int // returns the timeout that this was created with
SetStateFn(func(bool) error) // sets the stateFn
Timeout() int // returns the timeout that this was created with
AddStateFn(string, func(bool) error) error // adds a stateFn with a name
RemoveStateFn(string) error // remove a stateFn with a given name
}
// UID is the interface resources can use to notify with if converged. You'll
@@ -63,14 +67,15 @@ type UID interface {
// converger is an implementation of the Converger interface.
type converger struct {
timeout int // must be zero (instant) or greater seconds to run
stateFn func(bool) error // run on converged state changes with state bool
converged bool // did we converge (state changes of this run Fn)
channel chan struct{} // signal here to run an isConverged check
control chan bool // control channel for start/pause
mutex sync.RWMutex // used for controlling access to status and lastid
timeout int // must be zero (instant) or greater seconds to run
converged bool // did we converge (state changes of this run Fn)
channel chan struct{} // signal here to run an isConverged check
control chan bool // control channel for start/pause
mutex *sync.RWMutex // used for controlling access to status and lastid
lastid uint64
status map[uint64]bool
stateFns map[string]func(bool) error // run on converged state changes with state bool
smutex *sync.RWMutex // used for controlling access to stateFns
}
// cuid is an implementation of the UID interface.
@@ -78,21 +83,23 @@ type cuid struct {
converger Converger
id uint64
name string // user defined, friendly name
mutex sync.Mutex
mutex *sync.Mutex
timer chan struct{}
running bool // is the above timer running?
wg sync.WaitGroup
wg *sync.WaitGroup
}
// NewConverger builds a new converger struct.
func NewConverger(timeout int, stateFn func(bool) error) Converger {
func NewConverger(timeout int) Converger {
return &converger{
timeout: timeout,
stateFn: stateFn,
channel: make(chan struct{}),
control: make(chan bool),
lastid: 0,
status: make(map[uint64]bool),
timeout: timeout,
channel: make(chan struct{}),
control: make(chan bool),
mutex: &sync.RWMutex{},
lastid: 0,
status: make(map[uint64]bool),
stateFns: make(map[string]func(bool) error),
smutex: &sync.RWMutex{},
}
}
@@ -106,8 +113,10 @@ func (obj *converger) Register() UID {
converger: obj,
id: obj.lastid,
name: fmt.Sprintf("%d", obj.lastid), // some default
mutex: &sync.Mutex{},
timer: nil,
running: false,
wg: &sync.WaitGroup{},
}
}
@@ -216,11 +225,9 @@ func (obj *converger) Loop(startPaused bool) {
case <-obj.channel:
if !obj.isConverged() {
if obj.converged { // we're doing a state change
if obj.stateFn != nil {
// call an arbitrary function
if err := obj.stateFn(false); err != nil {
// FIXME: what to do on error ?
}
// call the arbitrary functions (takes a read lock!)
if err := obj.runStateFns(false); err != nil {
// FIXME: what to do on error ?
}
}
obj.converged = false
@@ -230,11 +237,9 @@ func (obj *converger) Loop(startPaused bool) {
// we have converged!
if obj.timeout >= 0 { // only run if timeout is valid
if !obj.converged { // we're doing a state change
if obj.stateFn != nil {
// call an arbitrary function
if err := obj.stateFn(true); err != nil {
// FIXME: what to do on error ?
}
// call the arbitrary functions (takes a read lock!)
if err := obj.runStateFns(true); err != nil {
// FIXME: what to do on error ?
}
}
}
@@ -275,9 +280,46 @@ func (obj *converger) Timeout() int {
return obj.timeout
}
// SetStateFn sets the state function to be run on change of converged state.
func (obj *converger) SetStateFn(stateFn func(bool) error) {
obj.stateFn = stateFn
// AddStateFn adds a state function to be run on change of converged state.
func (obj *converger) AddStateFn(name string, stateFn func(bool) error) error {
obj.smutex.Lock()
defer obj.smutex.Unlock()
if _, exists := obj.stateFns[name]; exists {
return fmt.Errorf("a stateFn with that name already exists")
}
obj.stateFns[name] = stateFn
return nil
}
// RemoveStateFn adds a state function to be run on change of converged state.
func (obj *converger) RemoveStateFn(name string) error {
obj.smutex.Lock()
defer obj.smutex.Unlock()
if _, exists := obj.stateFns[name]; !exists {
return fmt.Errorf("a stateFn with that name doesn't exist")
}
delete(obj.stateFns, name)
return nil
}
// runStateFns runs the listed of stored state functions.
func (obj *converger) runStateFns(converged bool) error {
obj.smutex.RLock()
defer obj.smutex.RUnlock()
var keys []string
for k := range obj.stateFns {
keys = append(keys, k)
}
sort.Strings(keys)
var err error
for _, name := range keys { // run in deterministic order
fn := obj.stateFns[name]
// call an arbitrary function
if e := fn(converged); e != nil {
err = multierr.Append(err, e) // list of errors
}
}
return err
}
// ID returns the unique id of this UID object.

545
docs/resource-guide.md
View File

@@ -19,17 +19,80 @@ on this design, please read the
[original article](https://purpleidea.com/blog/2016/01/18/next-generation-configuration-mgmt/)
on the subject.
## Resource Prerequisites
### Imports
You'll need to import a few packages to make writing your resource easier. Here
is the list:
```
"github.com/purpleidea/mgmt/engine"
"github.com/purpleidea/mgmt/engine/traits"
```
The `engine` package contains most of the interfaces and helper functions that
you'll need to use. The `traits` package contains some base functionality which
you can use to easily add functionality to your resource without needing to
implement it from scratch.
### Resource struct
Each resource will implement methods as pointer receivers on a resource struct.
The naming convention for resources is that they end with a `Res` suffix.
The resource struct should include an anonymous reference to the `Base` trait.
Other `traits` can be added to the resource to add additional functionality.
They are discussed below.
You'll most likely want to store a reference to the `*Init` struct type as
defined by the engine. This is data that the engine will provide to your
resource on Init.
Lastly you should define the public fields that make up your resource API, as
well as any private fields that you might want to use throughout your resource.
Do _not_ depend on global variables, since multiple copies of your resource
could get instantiated.
You'll want to add struct tags based on the different frontends that you want
your resources to be able to use. Some frontends can infer this information if
it is not specified, but others cannot, and some might poorly infer if the
struct name is ambiguous.
If you'd like your resource to be accessible by the `YAML` graph API (GAPI),
then you'll need to include the appropriate YAML fields as shown below. This is
used by the `Puppet` compiler as well, so make sure you include these struct
tags if you want existing `Puppet` code to be able to run using the `mgmt`
engine.
#### Example
```golang
type FooRes struct {
traits.Base // add the base methods without re-implementation
traits.Groupable
traits.Refreshable
init *engine.Init
Whatever string `lang:"whatever" yaml:"whatever"` // you pick!
Baz bool `lang:"baz" yaml:"baz"` // something else
something string // some private field
}
```
## Resource API
To implement a resource in `mgmt` it must satisfy the
[`Res`](https://github.com/purpleidea/mgmt/blob/master/resources/resources.go)
[`Res`](https://github.com/purpleidea/mgmt/blob/master/engine/resources.go)
interface. What follows are each of the method signatures and a description of
each.
### Default
```golang
Default() Res
Default() engine.Res
```
This returns a populated resource struct as a `Res`. It shouldn't populate any
@@ -55,9 +118,12 @@ Validate() error
This method is used to validate if the populated resource struct is a valid
representation of the resource kind. If it does not conform to the resource
specifications, it should generate an error. If you notice that this method is
specifications, it should return an error. If you notice that this method is
quite large, it might be an indication that you should reconsider the parameter
list and interface to this resource. This method is called _before_ `Init`.
list and interface to this resource. This method is called by the engine
_before_ `Init`. It can also be called occasionally after a Send/Recv operation
to verify that the newly populated parameters are valid. Remember not to expect
access to the outside world when using this.
#### Example
@@ -67,7 +133,7 @@ func (obj *FooRes) Validate() error {
if obj.Answer != 42 { // validate whatever you want
return fmt.Errorf("expected an answer of 42")
}
return obj.BaseRes.Validate() // remember to call the base method!
return nil
}
```
@@ -78,19 +144,28 @@ Init() error
```
This is called to initialize the resource. If something goes wrong, it should
return an error. It should do any resource specific work, and finish by calling
the `Init` method of the base resource.
return an error. It should do any resource specific work such as initializing
channels, sync primitives, or anything else that is relevant to your resource.
If it is not need throughout, it might be preferable to do some initialization
and tear down locally in either the Watch method or CheckApply method. The
choice depends on your particular resource and making the best decision requires
some experience with mgmt. If you are unsure, feel free to ask an existing
`mgmt` contributor. During `Init`, the engine will pass your resource a struct
containing some useful data and pointers. You should save a copy of this pointer
since you will need to use it in other parts of your resource.
#### Example
```golang
// Init initializes the Foo resource.
func (obj *FooRes) Init() error {
func (obj *FooRes) Init(init *engine.Init) error
obj.init = init // save for later
// run the resource specific initialization, and error if anything fails
if some_error {
return err // something went wrong!
}
return obj.BaseRes.Init() // call the base resource init
return nil
}
```
@@ -108,7 +183,9 @@ Close() error
This is called to cleanup after the resource. It is usually not necessary, but
can be useful if you'd like to properly close a persistent connection that you
opened in the `Init` method and were using throughout the resource.
opened in the `Init` method and were using throughout the resource. It is *not*
the shutdown signal that tells the resource to exit. That happens in the Watch
loop.
#### Example
@@ -116,21 +193,13 @@ opened in the `Init` method and were using throughout the resource.
// Close runs some cleanup code for this resource.
func (obj *FooRes) Close() error {
err := obj.conn.Close() // close some internal connection
// call base close, b/c we're overriding
if e := obj.BaseRes.Close(); err == nil {
err = e
} else if e != nil {
err = multierr.Append(err, e) // list of errors
}
obj.someMap = nil // free up some large data structure from memory
return err
}
```
You should probably check the return errors of your internal methods, and pass
on an error if something went wrong. Remember to always call the base `Close`
method! If you plan to return early if you hit an internal error, then at least
call it with a defer!
on an error if something went wrong.
### CheckApply
@@ -143,7 +212,8 @@ function should check if the state of this resource is correct, and if so, it
should return: `(true, nil)`. If the `apply` variable is set to `true`, then
this means that we should then proceed to run the changes required to bring the
resource into the correct state. If the `apply` variable is set to `false`, then
the resource is operating in _noop_ mode and _no operations_ should be executed!
the resource is operating in _noop_ mode and _no operational changes_ should be
made!
After having executed the necessary operations to bring the resource back into
the desired state, or after having detected that the state was incorrect, but
@@ -155,8 +225,8 @@ function. If you cannot, then you must return an error! The exception to this
rule is that if an external force changes the state of the resource while it is
being remedied, it is possible to return from this function even though the
resource isn't now converged. This is not a bug, as the resources `Watch`
facility will detect the change, ultimately resulting in a subsequent call to
`CheckApply`.
facility will detect the new change, ultimately resulting in a subsequent call
to `CheckApply`.
#### Example
@@ -165,11 +235,15 @@ facility will detect the change, ultimately resulting in a subsequent call to
func (obj *FooRes) CheckApply(apply bool) (bool, error) {
// check the state
if state_is_okay { return true, nil } // done early! :)
// state was bad
if !apply { return false, nil } // don't apply; !stateok, nil
if !apply { return false, nil } // don't apply, we're in noop mode
if any_error { return false, err } // anytime there's an err!
// do the apply!
return false, nil // after success applying
if any_error { return false, err } // anytime there's an err!
}
```
@@ -180,20 +254,6 @@ skipped. This is an engine optimization, and not a bug. It is mentioned here in
the documentation in case you are confused as to why a debug message you've
added to the code isn't always printed.
#### Refresh notifications
Some resources may choose to support receiving refresh notifications. In general
these should be avoided if possible, but nevertheless, they do make sense in
certain situations. Resources that support these need to verify if one was sent
during the CheckApply phase of execution. This is accomplished by calling the
`Refresh() bool` method of the resource, and inspecting the return value. This
is only necessary if you plan to perform a refresh action. Refresh actions
should still respect the `apply` variable, and no system changes should be made
if it is `false`. Refresh notifications are generated by any resource when an
action is applied by that resource and are transmitted through graph edges which
have enabled their propagation. Resources that currently perform some refresh
action include `svc`, `timer`, and `password`.
#### Paired execution
For many resources it is not uncommon to see `CheckApply` run twice in rapid
@@ -210,7 +270,7 @@ will likely find the state to now be correct.
* If the state is correct and no changes are needed, return `(true, nil)`.
* You should only make changes to the system if `apply` is set to `true`.
* After checking the state and possibly applying the fix, return `(false, nil)`.
* Returning `(true, err)` is a programming error and will cause a `Fatal`.
* Returning `(true, err)` is a programming error and can have a negative effect.
### Watch
@@ -223,7 +283,7 @@ state of the resource might have changed. To send a message you should write to
the input event channel using the `Event` helper method. The Watch function
should run continuously until a shutdown message is received. If at any time
something goes wrong, you should return an error, and the `mgmt` engine will
handle possibly restarting the main loop based on the `retry` meta parameters.
handle possibly restarting the main loop based on the `retry` meta parameter.
It is better to send an event notification which turns out to be spurious, than
to miss a possible event. Resources which can miss events are incorrect and need
@@ -248,17 +308,20 @@ The lifetime of most resources `Watch` method should be spent in an infinite
loop that is bounded by a `select` call. The `select` call is the point where
our method hands back control to the engine (and the kernel) so that we can
sleep until something of interest wakes us up. In this loop we must process
events from the engine via the `<-obj.Events()` call, and receive events for our
resource itself!
events from the engine via the `<-obj.init.Events` channel, and receive events
for our resource itself!
#### Events
If we receive an internal event from the `<-obj.Events()` method, we can read it
with the ReadEvent helper function. This function tells us if we should shutdown
our resource, and if we should generate an event. When we want to send an event,
we use the `Event` helper function. It is also important to mark the resource
state as `dirty` if we believe it might have changed. We do this with the
`StateOK(false)` function.
If we receive an internal event from the `<-obj.init.Events` channel, we should
read it with the `obj.init.Read` helper function. This function tells us if we
should shutdown our resource. It also handles pause functionality which blocks
our resource temporarily in this method. If this channel shuts down, then we
should treat that as an exit signal.
When we want to send an event, we use the `Event` helper function. It is also
important to mark the resource state as `dirty` if we believe it might have
changed. We do this by calling the `obj.init.Dirty` function.
#### Startup
@@ -266,22 +329,17 @@ Once the `Watch` function has finished starting up successfully, it is important
to generate one event to notify the `mgmt` engine that we're now listening
successfully, so that it can run an initial `CheckApply` to ensure we're safely
tracking a healthy state and that we didn't miss anything when `Watch` was down
or from before `mgmt` was running. It does this by calling the `Running` method.
or from before `mgmt` was running. You must do this by calling the
`obj.init.Running` method. If it returns an error, you must exit and return that
error.
#### Converged
The engine might be asked to shutdown when the entire state of the system has
not seen any changes for some duration of time. The engine can determine this
automatically, but each resource can block this if it is absolutely necessary.
To do this, the `Watch` method should get the `ConvergedUID` handle that has
been prepared for it by the engine. This is done by calling the `ConvergerUID`
method on the resource object. The result can be used to set the converged
status with `SetConverged`, and to notify when the particular timeout has been
reached by waiting on `ConvergedTimer`.
Instead of interacting with the `ConvergedUID` with these two methods, we can
instead use the `StartTimer` and `ResetTimer` methods which accomplish the same
thing, but provide a `select`-free interface for different coding situations.
If you need this functionality, please contact one of the maintainers and ask
about adding this feature and improving these docs right here.
This particular facility is most likely not required for most resources. It may
prove to be useful if a resource wants to start off a long operation, but avoid
@@ -297,28 +355,31 @@ func (obj *FooRes) Watch() error {
if err, obj.foo = OpenFoo(); err != nil {
return err // we couldn't startup
}
defer obj.whatever.CloseFoo() // shutdown our
defer obj.whatever.CloseFoo() // shutdown our Foo
// notify engine that we're running
if err := obj.Running(); err != nil {
return err // bubble up a NACK...
if err := obj.init.Running(); err != nil {
return err // exit if requested
}
var send = false // send event?
var exit *error
for {
select {
case event := <-obj.Events():
// we avoid sending events on unpause
if exit, send = obj.ReadEvent(event); exit != nil {
return *exit // exit
case event, ok := <-obj.init.Events:
if !ok {
// shutdown engine
// (it is okay if some `defer` code runs first)
return nil
}
if err := obj.init.Read(event); err != nil {
return err
}
// the actual events!
case event := <-obj.foo.Events:
if is_an_event {
send = true // used below
obj.StateOK(false) // dirty
send = true
obj.init.Dirty() // dirty
}
// event errors
@@ -329,7 +390,9 @@ func (obj *FooRes) Watch() error {
// do all our event sending all together to avoid duplicate msgs
if send {
send = false
obj.Event() // send the event!
if err := obj.init.Event(); err != nil {
return err // exit if requested
}
}
}
}
@@ -337,87 +400,259 @@ func (obj *FooRes) Watch() error {
#### Summary
* Remember to call the appropriate `converger` methods throughout the resource.
* Remember to call `Startup` when the `Watch` is running successfully.
* Remember to call `Running` when the `Watch` is running successfully.
* Remember to process internal events and shutdown promptly if asked to.
* Ensure the design of your resource is well thought out.
* Have a look at the existing resources for a rough idea of how this all works.
### Compare
### Cmp
```golang
Compare(Res) bool
Cmp(engine.Res) error
```
Each resource must have a `Compare` method. This takes as input another resource
and must return whether they are identical or not. This is used for identifying
if an existing resource can be used in place of a new one with a similar set of
parameters. In particular, when switching from one graph to a new (possibly
identical) graph, this avoids recomputing the state for resources which don't
change or that are sufficiently similar that they don't need to be swapped out.
Each resource must have a `Cmp` method. It is an abbreviation for `Compare`. It
takes as input another resource and must return whether they are identical or
not. This is used for identifying if an existing resource can be used in place
of a new one with a similar set of parameters. In particular, when switching
from one graph to a new (possibly identical) graph, this avoids recomputing the
state for resources which don't change or that are sufficiently similar that
they don't need to be swapped out.
In general if all the resource properties are identical, then they usually don't
need to be changed. On occasion, not all of them need to be compared, in
particular if they store some generated state, or if they aren't significant in
some way.
If the resource is identical, then you should return `nil`. If it is not, then
you should return a short error message which gives the reason it differs.
#### Example
```golang
// Compare two resources and return if they are equivalent.
func (obj *FooRes) Compare(r Res) bool {
// Cmp compares two resources and returns if they are equivalent.
func (obj *FooRes) Cmp(r engine.Res) error {
// we can only compare FooRes to others of the same resource kind
res, ok := r.(*FooRes)
if !ok {
return false
}
if !obj.BaseRes.Compare(res) { // call base Compare
return false
}
if obj.Name != res.Name {
return false
return fmt.Errorf("not a %s", obj.Kind())
}
if obj.whatever != res.whatever {
return false
if obj.Whatever != res.Whatever {
return fmt.Errorf("the Whatever param differs")
}
if obj.Flag != res.Flag {
return false
return fmt.Errorf("the Flag param differs")
}
return true // they must match!
return nil // they must match!
}
```
### UIDs
## Traits
Resources can have different `traits`, which means they can be extended to have
additional functionality or special properties. Those special properties are
usually added by extending your resource so that it is compatible with
additional interface that contain the `Res` interface. Each of these interfaces
represents the additional functionality. Since in most cases this requires some
common boilerplate, you can usually get some or most of the functionality by
embedding the correct trait struct anonymously in your struct. This is shown in
the struct example above. You'll always want to include the `Base` trait in all
resources. This provides some basics which you'll always need.
What follows are a list of available traits.
### Refreshable
Some resources may choose to support receiving refresh notifications. In general
these should be avoided if possible, but nevertheless, they do make sense in
certain situations. Resources that support these need to verify if one was sent
during the CheckApply phase of execution. This is accomplished by calling the
`obj.init.Refresh() bool` method, and inspecting the return value. This is only
necessary if you plan to perform a refresh action. Refresh actions should still
respect the `apply` variable, and no system changes should be made if it is
`false`. Refresh notifications are generated by any resource when an action is
applied by that resource and are transmitted through graph edges which have
enabled their propagation. Resources that currently perform some refresh action
include `svc`, `timer`, and `password`.
It is very important that you include the `traits.Refreshable` struct in your
resource. If you do not include this, then calling `obj.init.Refresh` may
trigger a panic. This is programmer error.
### Edgeable
Edgeable is a trait that allows your resource to automatically connect itself to
other resources that use this trait to add edge dependencies between the two. An
older blog post on this topic is
[available](https://purpleidea.com/blog/2016/03/14/automatic-edges-in-mgmt/).
After you've included this trait, you'll need to implement two methods on your
resource.
#### UIDs
```golang
UIDs() []ResUID
UIDs() []engine.ResUID
```
The `UIDs` method returns a list of `ResUID` interfaces that represent the
particular resource uniquely. This is used with the AutoEdges API to determine
if another resource can match a dependency to this one.
### AutoEdges
#### AutoEdges
```golang
AutoEdges() (AutoEdge, error)
AutoEdges() (engine.AutoEdge, error)
```
This returns a struct that implements the `AutoEdge` interface. This struct
is used to match other resources that might be relevant dependencies for this
resource.
### CollectPattern
### Groupable
```golang
CollectPattern() string
```
Groupable is a trait that can allow your resource automatically group itself to
other resources. Doing so can reduce the resource or runtime burden on the
engine, and improve performance in some scenarios. An older blog post on this
topic is
[available](https://purpleidea.com/blog/2016/03/30/automatic-grouping-in-mgmt/).
### Sendable
Sendable is a trait that allows your resource to send values through the graph
edges to another resource. These values are produced during `CheckApply`. They
can be sent to any resource that has an appropriate parameter and that has the
`Recvable` trait. You can read more about this in the Send/Recv section below.
### Recvable
Recvable is a trait that allows your resource to receive values through the
graph edges from another resource. These values are consumed during the
`CheckApply` phase, and can be detected there as well. They can be received from
any resource that has an appropriate value and that has the `Sendable` trait.
You can read more about this in the Send/Recv section below.
### Collectable
This is currently a stub and will be updated once the DSL is further along.
### UnmarshalYAML
## Resource Initialization
During the resource initialization in `Init`, the engine will pass in a struct
containing a bunch of data and methods. What follows is a description of each
one and how it is used.
### Program
Program is a string containing the name of the program. Very few resources need
this.
### Hostname
Hostname is the uuid for the host. It will be occasionally useful in some
resources. It is preferable if you can avoid depending on this. It is possible
that in the future this will be a channel which changes if the local hostname
changes.
### Running
Running must be called after your watches are all started and ready. It is only
called from within `Watch`. It is used to notify the engine that you're now
ready to detect changes.
### Event
Event sends an event notifying the engine of a possible state change. It is
only called from within `Watch`.
### Events
Events is a channel that we must watch for messages from the engine. When it
closes, this is a signal to shutdown. It is
only called from within `Watch`.
### Read
Read processes messages that come in from the `Events` channel. It is a helper
method that knows how to handle the pause mechanism correctly. It is
only called from within `Watch`.
### Dirty
Dirty marks the resource state as dirty. This signals to the engine that
CheckApply will have some work to do in order to converge it. It is
only called from within `Watch`.
### Refresh
Refresh returns whether the resource received a notification. This flag can be
used to tell a `svc` to reload, or to perform some state change that wouldn't
otherwise be noticed by inspection alone. You must implement the `Refreshable`
trait for this to work. It is only called from within `CheckApply`.
### Send
Send exposes some variables you wish to send via the `Send/Recv` mechanism. You
must implement the `Sendable` trait for this to work. It is only called from
within `CheckApply`.
### Recv
Recv provides a map of variables which were sent to this resource via the
`Send/Recv` mechanism. You must implement the `Recvable` trait for this to work.
It is only called from within `CheckApply`.
### World
World provides a connection to the outside world. This is most often used for
communicating with the distributed database. It can be used in `Init`,
`CheckApply` and `Watch`. Use with discretion and understanding of the internals
if needed in `Close`.
### VarDir
VarDir is a facility for local storage. It is used to return a path to a
directory which may be used for temporary storage. It should be cleaned up on
resource `Close` if the resource would like to delete the contents. The resource
should not assume that the initial directory is empty, and it should be cleaned
on `Init` if that is a requirement.
### Debug
Debug signals whether we are running in debugging mode. In this case, we might
want to log additional messages.
### Logf
Logf is a logging facility which will correctly namespace any messages which you
wish to pass on. You should use this instead of the log package directly for
production quality resources.
## Further considerations
There is some additional information that any resource writer will need to know.
Each issue is listed separately below!
### Resource registration
All resources must be registered with the engine so that they can be found. This
also ensures they can be encoded and decoded. Make sure to include the following
code snippet for this to work.
```golang
func init() { // special golang method that runs once
// set your resource kind and struct here (the kind must be lower case)
engine.RegisterResource("foo", func() engine.Res { return &FooRes{} })
}
```
### YAML Unmarshalling
To support YAML unmarshalling for your resource, you must implement an
additional method. It is recommended if you want to use your resource with the
`Puppet` compiler.
```golang
UnmarshalYAML(unmarshal func(interface{}) error) error // optional
@@ -455,105 +690,34 @@ func (obj *FooRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
}
```
## Further considerations
There is some additional information that any resource writer will need to know.
Each issue is listed separately below!
### Resource struct
Each resource will implement methods as pointer receivers on a resource struct.
The resource struct must include an anonymous reference to the `BaseRes` struct.
The naming convention for resources is that they end with a `Res` suffix. If
you'd like your resource to be accessible by the `YAML` graph API (GAPI), then
you'll need to include the appropriate YAML fields as shown below.
#### Example
```golang
type FooRes struct {
BaseRes `yaml:",inline"` // base properties
Whatever string `yaml:"whatever"` // you pick!
Bar int // no yaml, used as public output value for send/recv
Baz bool `yaml:"baz"` // something else
something string // some private field
}
```
### Resource registration
All resources must be registered with the engine so that they can be found. This
also ensures they can be encoded and decoded. Make sure to include the following
code snippet for this to work.
```golang
func init() { // special golang method that runs once
// set your resource kind and struct here (the kind must be lower case)
RegisterResource("foo", func() Res { return &FooRes{} })
}
```
## Automatic edges
Automatic edges in `mgmt` are well described in [this article](https://purpleidea.com/blog/2016/03/14/automatic-edges-in-mgmt/).
The best example of this technique can be seen in the `svc` resource.
Unfortunately no further documentation about this subject has been written. To
expand this section, please send a patch! Please contact us if you'd like to
work on a resource that uses this feature, or to add it to an existing one!
## Automatic grouping
Automatic grouping in `mgmt` is well described in [this article](https://purpleidea.com/blog/2016/03/30/automatic-grouping-in-mgmt/).
The best example of this technique can be seen in the `pkg` resource.
Unfortunately no further documentation about this subject has been written. To
expand this section, please send a patch! Please contact us if you'd like to
work on a resource that uses this feature, or to add it to an existing one!
## Send/Recv
In `mgmt` there is a novel concept called _Send/Recv_. For some background,
please [read the introductory article](https://purpleidea.com/blog/2016/12/07/sendrecv-in-mgmt/).
please read the [introductory article](https://purpleidea.com/blog/2016/12/07/sendrecv-in-mgmt/).
When using this feature, the engine will automatically send the user specified
value to the intended destination without requiring any resource specific code.
value to the intended destination without requiring much resource specific code.
Any time that one of the destination values is changed, the engine automatically
marks the resource state as `dirty`. To detect if a particular value was
received, and if it changed (during this invocation of CheckApply) from the
previous value, you can query the Recv parameter. It will contain a `map` of all
the keys which can be received on, and the value has a `Changed` property which
will indicate whether the value was updated on this particular `CheckApply`
invocation. The type of the sending key must match that of the receiving one.
This can _only_ be done inside of the `CheckApply` function!
received, and if it changed (during this invocation of `CheckApply`) from the
previous value, you can query the `obj.init.Recv()` method. It will contain a
`map` of all the keys which can be received on, and the value has a `Changed`
property which will indicate whether the value was updated on this particular
`CheckApply` invocation. The type of the sending key must match that of the
receiving one. This can _only_ be done inside of the `CheckApply` function!
```golang
// inside CheckApply, probably near the top
if val, exists := obj.Recv["SomeKey"]; exists {
log.Printf("SomeKey was sent to us from: %s.%s", val.Res, val.Key)
if val, exists := obj.init.Recv()["SomeKey"]; exists {
obj.init.Logf("the SomeKey param was sent to us from: %s.%s", val.Res, val.Key)
if val.Changed {
log.Printf("SomeKey was just updated!")
obj.init.Logf("the SomeKey param was just updated!")
// you may want to invalidate some local cache
}
}
```
Astute readers will note that there isn't anything that prevents a user from
sending an identically typed value to some arbitrary (public) key that the
resource author hadn't considered! While this is true, resources should probably
work within this problem space anyways. The rule of thumb is that any public
parameter which is normally used in a resource can be used safely.
One subtle scenario is that if a resource creates a local cache or stores a
computation that depends on the value of a public parameter and will require
invalidation should that public parameter change, then you must detect that
scenario and invalidate the cache when it occurs. This *must* be processed
before there is a possibility of failure in CheckApply, because if we fail (and
possibly run again) the subsequent send->recv transfer might not have a new
value to copy, and therefore we won't see this notification of change.
Therefore, it is important to process these promptly, if they must not be lost,
such as for cache invalidation.
Remember, `Send/Recv` only changes your resource code if you cache state.
The specifics of resource sending are not currently documented. Please send a
patch here!
## Composite resources
@@ -624,6 +788,15 @@ us know!
There are still many ideas for new resources that haven't been written yet. If
you'd like to contribute one, please contact us and tell us about your idea!
### Is the resource API stable? Does it ever change?
Since we are pre 1.0, the resource API is not guaranteed to be stable, however
it is not expected to change significantly. The last major change kept the
core functionality nearly identical, simplified the implementation of all the
resources, and took about five to ten minutes to port each resource to the new
API. The fundamental logic and behaviour behind the resource API has not changed
since it was initially introduced.
### Where can I find more information about mgmt?
Additional blog posts, videos and other material [is available!](https://github.com/purpleidea/mgmt/blob/master/docs/on-the-web.md).

51
resources/uid.go → engine/autoedge.go
View File

@@ -15,18 +15,61 @@
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package resources
package engine
import (
"fmt"
"log"
)
// EdgeableRes is the interface a resource must implement to support automatic
// edges. Both the vertices involved in an edge need to implement this for it to
// be able to work.
type EdgeableRes interface {
Res // implement everything in Res but add the additional requirements
// AutoEdgeMeta lets you get or set meta params for the automatic edges
// trait.
AutoEdgeMeta() *AutoEdgeMeta
// UIDs includes all params to make a unique identification of this
// object.
UIDs() []ResUID // most resources only return one
// AutoEdges returns a struct that implements the AutoEdge interface.
// This interface can be used to generate automatic edges to other
// resources.
AutoEdges() (AutoEdge, error)
}
// AutoEdgeMeta provides some parameters specific to automatic edges.
// TODO: currently this only supports disabling the feature per-resource, but in
// the future you could conceivably have some small pattern to control it better
type AutoEdgeMeta struct {
// Disabled specifies that automatic edges should be disabled for this
// resource.
Disabled bool
}
// Cmp compares two AutoEdgeMeta structs and determines if they're equivalent.
func (obj *AutoEdgeMeta) Cmp(aem *AutoEdgeMeta) error {
if obj.Disabled != aem.Disabled {
return fmt.Errorf("values for Disabled are different")
}
return nil
}
// The AutoEdge interface is used to implement the autoedges feature.
type AutoEdge interface {
Next() []ResUID // call to get list of edges to add
Test([]bool) bool // call until false
}
// ResUID is a unique identifier for a resource, namely it's name, and the kind ("type").
type ResUID interface {
fmt.Stringer // String() string
GetName() string
GetKind() string
fmt.Stringer // String() string
IFF(ResUID) bool
@@ -72,7 +115,7 @@ func (obj *BaseUID) IFF(uid ResUID) bool {
// happens before the generator.
func (obj *BaseUID) IsReversed() bool {
if obj.Reversed == nil {
log.Fatal("Programming error!")
panic("programming error!")
}
return *obj.Reversed
}

36
engine/autoedge_test.go Normal file
View File

@@ -0,0 +1,36 @@
// Mgmt
// Copyright (C) 2013-2018+ James Shubin and the project contributors
// Written by James Shubin <james@shubin.ca> and the project contributors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package engine
import (
"testing"
)
func TestIFF1(t *testing.T) {
uid := &BaseUID{Name: "/tmp/unit-test"}
same := &BaseUID{Name: "/tmp/unit-test"}
diff := &BaseUID{Name: "/tmp/other-file"}
if !uid.IFF(same) {
t.Errorf("basic resource UIDs with the same name should satisfy each other's IFF condition")
}
if uid.IFF(diff) {
t.Errorf("basic resource UIDs with different names should NOT satisfy each other's IFF condition")
}
}

84
engine/autogroup.go Normal file
View File

@@ -0,0 +1,84 @@
// Mgmt
// Copyright (C) 2013-2018+ James Shubin and the project contributors
// Written by James Shubin <james@shubin.ca> and the project contributors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package engine
import (
"fmt"
"github.com/purpleidea/mgmt/pgraph"
)
// GroupableRes is the interface a resource must implement to support automatic
// grouping. Default implementations for most of the methods declared in this
// interface can be obtained for your resource by anonymously adding the
// traits.Groupable struct to your resource implementation.
type GroupableRes interface {
Res // implement everything in Res but add the additional requirements
// AutoGroupMeta lets you get or set meta params for the automatic
// grouping trait.
AutoGroupMeta() *AutoGroupMeta
// GroupCmp compares two resources and decides if they're suitable for
//grouping. This usually needs to be unique to your resource.
GroupCmp(res GroupableRes) error
// GroupRes groups resource argument (res) into self.
GroupRes(res GroupableRes) error
// IsGrouped determines if we are grouped.
IsGrouped() bool // am I grouped?
// SetGrouped sets a flag to tell if we are grouped.
SetGrouped(bool)
// GetGroup returns everyone grouped inside me.
GetGroup() []GroupableRes // return everyone grouped inside me
// SetGroup sets the grouped resources into me.
SetGroup([]GroupableRes)
}
// AutoGroupMeta provides some parameters specific to automatic grouping.
// TODO: currently this only supports disabling the feature per-resource, but in
// the future you could conceivably have some small pattern to control it better
type AutoGroupMeta struct {
// Disabled specifies that automatic grouping should be disabled for
// this resource.
Disabled bool
}
// Cmp compares two AutoGroupMeta structs and determines if they're equivalent.
func (obj *AutoGroupMeta) Cmp(agm *AutoGroupMeta) error {
if obj.Disabled != agm.Disabled {
return fmt.Errorf("values for Disabled are different")
}
return nil
}
// AutoGrouper is the required interface to implement an autogrouping algorithm.
type AutoGrouper interface {
// listed in the order these are typically called in...
Name() string // friendly identifier
Init(*pgraph.Graph) error // only call once
VertexNext() (pgraph.Vertex, pgraph.Vertex, error) // mostly algorithmic
VertexCmp(pgraph.Vertex, pgraph.Vertex) error // can we merge these ?
VertexMerge(pgraph.Vertex, pgraph.Vertex) (pgraph.Vertex, error) // vertex merge fn to use
EdgeMerge(pgraph.Edge, pgraph.Edge) pgraph.Edge // edge merge fn to use
VertexTest(bool) (bool, error) // call until false
}

126
engine/cmp.go Normal file
View File

@@ -0,0 +1,126 @@
// Mgmt
// Copyright (C) 2013-2018+ James Shubin and the project contributors
// Written by James Shubin <james@shubin.ca> and the project contributors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package engine
import (
"fmt"
"github.com/purpleidea/mgmt/pgraph"
)
// ResCmp compares two resources by checking multiple aspects. This is the main
// entry point for running all the compare steps on two resource.
func ResCmp(r1, r2 Res) error {
if r1.Kind() != r2.Kind() {
return fmt.Errorf("kind differs")
}
if r1.Name() != r2.Name() {
return fmt.Errorf("name differs")
}
if err := r1.Cmp(r2); err != nil {
return err
}
// compare meta params for resources with auto edges
r1e, ok1 := r1.(EdgeableRes)
r2e, ok2 := r2.(EdgeableRes)
if ok1 != ok2 {
return fmt.Errorf("edgeable differs") // they must be different (optional)
}
if ok1 && ok2 {
if r1e.AutoEdgeMeta().Cmp(r2e.AutoEdgeMeta()) != nil {
return fmt.Errorf("autoedge differs")
}
}
// compare meta params for resources with auto grouping
r1g, ok1 := r1.(GroupableRes)
r2g, ok2 := r2.(GroupableRes)
if ok1 != ok2 {
return fmt.Errorf("groupable differs") // they must be different (optional)
}
if ok1 && ok2 {
if r1g.AutoGroupMeta().Cmp(r2g.AutoGroupMeta()) != nil {
return fmt.Errorf("autogroup differs")
}
// if resources are grouped, are the groups the same?
if i, j := r1g.GetGroup(), r2g.GetGroup(); len(i) != len(j) {
return fmt.Errorf("autogroup groups differ")
} else if len(i) > 0 { // trick the golinter
// Sort works with Res, so convert the lists to that
iRes := []Res{}
for _, r := range i {
res := r.(Res)
iRes = append(iRes, res)
}
jRes := []Res{}
for _, r := range j {
res := r.(Res)
jRes = append(jRes, res)
}
ix, jx := Sort(iRes), Sort(jRes) // now sort :)
for k := range ix {
// compare sub resources
if err := ResCmp(ix[k], jx[k]); err != nil {
return err
}
}
}
}
return nil
}
// VertexCmpFn returns if two vertices are equivalent. It errors if they can't
// be compared because one is not a vertex. This returns true if equal.
// TODO: shouldn't the first argument be an `error` instead?
func VertexCmpFn(v1, v2 pgraph.Vertex) (bool, error) {
r1, ok := v1.(Res)
if !ok {
return false, fmt.Errorf("v1 is not a Res")
}
r2, ok := v2.(Res)
if !ok {
return false, fmt.Errorf("v2 is not a Res")
}
if ResCmp(r1, r2) != nil {
return false, nil
}
return true, nil
}
// EdgeCmpFn returns if two edges are equivalent. It errors if they can't be
// compared because one is not an edge. This returns true if equal.
// TODO: shouldn't the first argument be an `error` instead?
func EdgeCmpFn(e1, e2 pgraph.Edge) (bool, error) {
edge1, ok := e1.(*Edge)
if !ok {
return false, fmt.Errorf("e1 is not an Edge")
}
edge2, ok := e2.(*Edge)
if !ok {
return false, fmt.Errorf("e2 is not an Edge")
}
return edge1.Cmp(edge2) == nil, nil
}

18
resources/edge.go → engine/edge.go
View File

@@ -15,7 +15,11 @@
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package resources
package engine
import (
"fmt"
)
// Edge is a struct that represents a graph's edge.
type Edge struct {
@@ -30,19 +34,19 @@ func (obj *Edge) String() string {
return obj.Name
}
// Compare returns true if two edges are equivalent. Otherwise it returns false.
func (obj *Edge) Compare(edge *Edge) bool {
// Cmp compares this edge to another. It returns nil if they are equivalent.
func (obj *Edge) Cmp(edge *Edge) error {
if obj.Name != edge.Name {
return false
return fmt.Errorf("edge names differ")
}
if obj.Notify != edge.Notify {
return false
return fmt.Errorf("notify values differ")
}
// FIXME: should we compare this as well?
//if obj.refresh != edge.refresh {
// return false
// return fmt.Errorf("refresh values differ")
//}
return true
return nil
}
// Refresh returns the pending refresh status of this edge.

32
engine/error.go Normal file
View File

@@ -0,0 +1,32 @@
// Mgmt
// Copyright (C) 2013-2018+ James Shubin and the project contributors
// Written by James Shubin <james@shubin.ca> and the project contributors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package engine
// Error is a constant error type that implements error.
type Error string
// Error fulfills the error interface of this type.
func (e Error) Error() string { return string(e) }
const (
// ErrWatchExit represents an exit from the Watch loop via chan closure.
ErrWatchExit = Error("watch exit")
// ErrSignalExit represents an exit from the Watch loop via exit signal.
ErrSignalExit = Error("signal exit")
)

33
engine/event/event.go Normal file
View File

@@ -0,0 +1,33 @@
// Mgmt
// Copyright (C) 2013-2018+ James Shubin and the project contributors
// Written by James Shubin <james@shubin.ca> and the project contributors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
// Package event provides some primitives that are used for message passing.
package event
//go:generate stringer -type=Kind -output=kind_stringer.go
// Kind represents the type of event being passed.
type Kind int
// The different event kinds are used in different contexts.
const (
EventNil Kind = iota
EventStart
EventPause
EventPoke
EventExit
)

30
resources/interfaces.go → engine/fs.go
View File

@@ -15,42 +15,14 @@
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package resources
package engine
import (
"os"
"github.com/purpleidea/mgmt/etcd/scheduler"
"github.com/spf13/afero"
)
// World is an interface to the rest of the different graph state. It allows
// the GAPI to store state and exchange information throughout the cluster. It
// is the interface each machine uses to communicate with the rest of the world.
type World interface { // TODO: is there a better name for this interface?
ResWatch() chan error
ResExport([]Res) error
// FIXME: should this method take a "filter" data struct instead of many args?
ResCollect(hostnameFilter, kindFilter []string) ([]Res, error)
StrWatch(namespace string) chan error
StrIsNotExist(error) bool
StrGet(namespace string) (string, error)
StrSet(namespace, value string) error
StrDel(namespace string) error
// XXX: add the exchange primitives in here directly?
StrMapWatch(namespace string) chan error
StrMapGet(namespace string) (map[string]string, error)
StrMapSet(namespace, value string) error
StrMapDel(namespace string) error
Scheduler(namespace string, opts ...scheduler.Option) (*scheduler.Result, error)
Fs(uri string) (Fs, error)
}
// from the ioutil package:
// NopCloser(r io.Reader) io.ReadCloser // not implemented here
// ReadAll(r io.Reader) ([]byte, error)

474
engine/graph/actions.go Normal file
View File

@@ -0,0 +1,474 @@
// Mgmt
// Copyright (C) 2013-2018+ James Shubin and the project contributors
// Written by James Shubin <james@shubin.ca> and the project contributors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package graph
import (
"fmt"
"strings"
"sync"
"time"
"github.com/purpleidea/mgmt/engine"
"github.com/purpleidea/mgmt/engine/event"
"github.com/purpleidea/mgmt/pgraph"
//multierr "github.com/hashicorp/go-multierror"
errwrap "github.com/pkg/errors"
"golang.org/x/time/rate"
)
// OKTimestamp returns true if this vertex can run right now.
func (obj *Engine) OKTimestamp(vertex pgraph.Vertex) bool {
return len(obj.BadTimestamps(vertex)) == 0
}
// BadTimestamps returns the list of vertices that are causing our timestamp to
// be bad.
func (obj *Engine) BadTimestamps(vertex pgraph.Vertex) []pgraph.Vertex {
vs := []pgraph.Vertex{}
ts := obj.state[vertex].timestamp
// these are all the vertices pointing TO vertex, eg: ??? -> vertex
for _, v := range obj.graph.IncomingGraphVertices(vertex) {
// If the vertex has a greater timestamp than any prerequisite,
// then we can't run right now. If they're equal (eg: initially
// with a value of 0) then we also can't run because we should
// let our pre-requisites go first.
t := obj.state[v].timestamp
if obj.Debug {
obj.Logf("OKTimestamp: %d >= %d (%s): !%t", ts, t, v.String(), ts >= t)
}
if ts >= t {
//return false
vs = append(vs, v)
}
}
return vs // formerly "true" if empty
}
// Process is the primary function to execute a particular vertex in the graph.
func (obj *Engine) Process(vertex pgraph.Vertex) error {
res, isRes := vertex.(engine.Res)
if !isRes {
return fmt.Errorf("vertex is not a Res")
}
// Engine Guarantee: Do not allow CheckApply to run while we are paused.
// This makes the resource able to know that synchronous channel sending
// to the main loop select in Watch from within CheckApply, will succeed
// without blocking because the resource went into a paused state. If we
// are using the Poll metaparam, then Watch will (of course) not be run.
// FIXME: should this lock be here, or wrapped right around CheckApply ?
obj.state[vertex].eventsLock.Lock() // this lock is taken within Event()
defer obj.state[vertex].eventsLock.Unlock()
// backpoke! (can be async)
if vs := obj.BadTimestamps(vertex); len(vs) > 0 {
// back poke in parallel (sync b/c of waitgroup)
for _, v := range obj.graph.IncomingGraphVertices(vertex) {
if !pgraph.VertexContains(v, vs) { // only poke what's needed
continue
}
go obj.state[v].Poke() // async
}
return nil // can't continue until timestamp is in sequence
}
// semaphores!
// These shouldn't ever block an exit, since the graph should eventually
// converge causing their them to unlock. More interestingly, since they
// run in a DAG alphabetically, there is no way to permanently deadlock,
// assuming that resources individually don't ever block from finishing!
// The exception is that semaphores with a zero count will always block!
// TODO: Add a close mechanism to close/unblock zero count semaphores...
semas := res.MetaParams().Sema
if obj.Debug && len(semas) > 0 {
obj.Logf("%s: Sema: P(%s)", res, strings.Join(semas, ", "))
}
if err := obj.semaLock(semas); err != nil { // lock
// NOTE: in practice, this might not ever be truly necessary...
return fmt.Errorf("shutdown of semaphores")
}
defer obj.semaUnlock(semas) // unlock
if obj.Debug && len(semas) > 0 {
defer obj.Logf("%s: Sema: V(%s)", res, strings.Join(semas, ", "))
}
// sendrecv!
// connect any senders to receivers and detect if values changed
if res, ok := vertex.(engine.RecvableRes); ok {
if updated, err := obj.SendRecv(res); err != nil {
return errwrap.Wrapf(err, "could not SendRecv")
} else if len(updated) > 0 {
for _, changed := range updated {
if changed { // at least one was updated
// invalidate cache, mark as dirty
obj.state[vertex].isStateOK = false
break
}
}
// re-validate after we change any values
if err := engine.Validate(res); err != nil {
return errwrap.Wrapf(err, "failed Validate after SendRecv")
}
}
}
var ok = true
var applied = false // did we run an apply?
var noop = res.MetaParams().Noop // lookup the noop value
var refresh bool
var checkOK bool
var err error
// lookup the refresh (notification) variable
refresh = obj.RefreshPending(vertex) // do i need to perform a refresh?
refreshableRes, isRefreshableRes := vertex.(engine.RefreshableRes)
if isRefreshableRes {
refreshableRes.SetRefresh(refresh) // tell the resource
}
// Check cached state, to skip CheckApply, but can't skip if refreshing!
// If the resource doesn't implement refresh, skip the refresh test.
// FIXME: if desired, check that we pass through refresh notifications!
if (!refresh || !isRefreshableRes) && obj.state[vertex].isStateOK {
checkOK, err = true, nil
} else if noop && (refresh && isRefreshableRes) { // had a refresh to do w/ noop!
checkOK, err = false, nil // therefore the state is wrong
// run the CheckApply!
} else {
obj.Logf("%s: CheckApply(%t)", res, !noop)
// if this fails, don't UpdateTimestamp()
checkOK, err = res.CheckApply(!noop)
obj.Logf("%s: CheckApply(%t): Return(%t, %+v)", res, !noop, checkOK, err)
}
if checkOK && err != nil { // should never return this way
return fmt.Errorf("%s: resource programming error: CheckApply(%t): %t, %+v", res, !noop, checkOK, err)
}
if !checkOK { // something changed, restart timer
obj.state[vertex].cuid.ResetTimer() // activity!
if obj.Debug {
obj.Logf("%s: converger: reset timer", res)
}
}
// if CheckApply ran without noop and without error, state should be good
if !noop && err == nil { // aka !noop || checkOK
obj.state[vertex].isStateOK = true // reset
if refresh {
obj.SetUpstreamRefresh(vertex, false) // refresh happened, clear the request
if isRefreshableRes {
refreshableRes.SetRefresh(false)
}
}
}
if !checkOK { // if state *was* not ok, we had to have apply'ed
if err != nil { // error during check or apply
ok = false
} else {
applied = true
}
}
// when noop is true we always want to update timestamp
if noop && err == nil {
ok = true
}
if ok {
// did we actually do work?
activity := applied
if noop {
activity = false // no we didn't do work...
}
if activity { // add refresh flag to downstream edges...
obj.SetDownstreamRefresh(vertex, true)
}
// poke! (should (must?) be sync)
wg := &sync.WaitGroup{}
// update this timestamp *before* we poke or the poked
// nodes might fail due to having a too old timestamp!
obj.state[vertex].timestamp = time.Now().UnixNano() // update timestamp
for _, v := range obj.graph.OutgoingGraphVertices(vertex) {
if !obj.OKTimestamp(v) {
// there is at least another one that will poke this...
continue
}
// If we're pausing (or exiting) then we can skip poking
// so that the graph doesn't go on running forever until
// it's completely done. This is an optional feature and
// we can select it via ^C on user exit or via the GAPI.
if obj.fastPause {
obj.Logf("%s: fast pausing, poke skipped", res)
continue
}
// poke each vertex individually, in parallel...
wg.Add(1)
go func(vv pgraph.Vertex) {
defer wg.Done()
obj.state[vv].Poke()
}(v)
}
wg.Wait()
}
return errwrap.Wrapf(err, "error during Process()")
}
// Worker is the common run frontend of the vertex. It handles all of the retry
// and retry delay common code, and ultimately returns the final status of this
// vertex execution.
func (obj *Engine) Worker(vertex pgraph.Vertex) error {
res, isRes := vertex.(engine.Res)
if !isRes {
return fmt.Errorf("vertex is not a resource")
}
defer close(obj.state[vertex].stopped) // done signal
obj.state[vertex].cuid = obj.Converger.Register()
// must wait for all users of the cuid to finish *before* we unregister!
// as a result, this defer happens *before* the below wait group Wait...
defer obj.state[vertex].cuid.Unregister()
defer obj.state[vertex].wg.Wait() // this Worker is the last to exit!
obj.state[vertex].wg.Add(1)
go func() {
defer obj.state[vertex].wg.Done()
defer close(obj.state[vertex].outputChan) // we close this on behalf of res
var err error
var retry = res.MetaParams().Retry // lookup the retry value
var delay uint64
for { // retry loop
// a retry-delay was requested, wait, but don't block events!
if delay > 0 {
errDelayExpired := engine.Error("delay exit")
err = func() error { // slim watch main loop
timer := time.NewTimer(time.Duration(delay) * time.Millisecond)
defer obj.state[vertex].init.Logf("the Watch delay expired!")
defer timer.Stop() // it's nice to cleanup
for {
select {
case <-timer.C: // the wait is over
return errDelayExpired // special
case event, ok := <-obj.state[vertex].init.Events:
if !ok {
return nil
}
if err := obj.state[vertex].init.Read(event); err != nil {
return err
}
}
}
}()
if err == errDelayExpired {
delay = 0 // reset
continue
}
} else if interval := res.MetaParams().Poll; interval > 0 { // poll instead of watching :(
obj.state[vertex].cuid.StartTimer()
err = obj.state[vertex].poll(interval)
obj.state[vertex].cuid.StopTimer() // clean up nicely
} else {
obj.state[vertex].cuid.StartTimer()
obj.Logf("Watch(%s)", vertex)
err = res.Watch() // run the watch normally
obj.Logf("Watch(%s): Exited(%+v)", vertex, err)
obj.state[vertex].cuid.StopTimer() // clean up nicely
}
if err == nil || err == engine.ErrWatchExit || err == engine.ErrSignalExit {
return // exited cleanly, we're done
}
// we've got an error...
delay = res.MetaParams().Delay
if retry < 0 { // infinite retries
obj.state[vertex].reset()
continue
}
if retry > 0 { // don't decrement past 0
retry--
obj.state[vertex].init.Logf("retrying Watch after %.4f seconds (%d left)", float64(delay)/1000, retry)
obj.state[vertex].reset()
continue
}
//if retry == 0 { // optional
// err = errwrap.Wrapf(err, "permanent watch error")
//}
break // break out of this and send the error
}
// this section sends an error...
// If the CheckApply loop exits and THEN the Watch fails with an
// error, then we'd be stuck here if exit signal didn't unblock!
select {
case obj.state[vertex].outputChan <- errwrap.Wrapf(err, "watch failed"):
// send
case <-obj.state[vertex].exit.Signal():
// pass
}
}()
// bonus safety check
if res.MetaParams().Burst == 0 && !(res.MetaParams().Limit == rate.Inf) { // blocked
return fmt.Errorf("permanently limited (rate != Inf, burst = 0)")
}
var limiter = rate.NewLimiter(res.MetaParams().Limit, res.MetaParams().Burst)
// It is important that we shutdown the Watch loop if this exits.
// Example, if Process errors permanently, we should ask Watch to exit.
defer obj.state[vertex].Event(event.EventExit) // signal an exit
for {
select {
case err, ok := <-obj.state[vertex].outputChan: // read from watch channel
if !ok {
return nil
}
if err != nil {
return err // permanent failure
}
// safe to go run the process...
case <-obj.state[vertex].exit.Signal(): // TODO: is this needed?
return nil
}
now := time.Now()
r := limiter.ReserveN(now, 1) // one event
// r.OK() seems to always be true here!
d := r.DelayFrom(now)
if d > 0 { // delay
obj.state[vertex].init.Logf("limited (rate: %v/sec, burst: %d, next: %v)", res.MetaParams().Limit, res.MetaParams().Burst, d)
var count int
timer := time.NewTimer(time.Duration(d) * time.Millisecond)
LimitWait:
for {
select {
case <-timer.C: // the wait is over
break LimitWait
// consume other events while we're waiting...
case e, ok := <-obj.state[vertex].outputChan: // read from watch channel
if !ok {
// FIXME: is this logic correct?
if count == 0 {
return nil
}
// loop, because we have
// the previous event to
// run process on first!
continue
}
if e != nil {
return e // permanent failure
}
count++ // count the events...
limiter.ReserveN(time.Now(), 1) // one event
}
}
timer.Stop() // it's nice to cleanup
obj.state[vertex].init.Logf("rate limiting expired!")
}
var err error
var retry = res.MetaParams().Retry // lookup the retry value
var delay uint64
Loop:
for { // retry loop
if delay > 0 {
var count int
timer := time.NewTimer(time.Duration(delay) * time.Millisecond)
RetryWait:
for {
select {
case <-timer.C: // the wait is over
break RetryWait
// consume other events while we're waiting...
case e, ok := <-obj.state[vertex].outputChan: // read from watch channel
if !ok {
// FIXME: is this logic correct?
if count == 0 {
// last process error
return err
}
// loop, because we have
// the previous event to
// run process on first!
continue
}
if e != nil {
return e // permanent failure
}
count++ // count the events...
limiter.ReserveN(time.Now(), 1) // one event
}
}
timer.Stop() // it's nice to cleanup
delay = 0 // reset
obj.state[vertex].init.Logf("the CheckApply delay expired!")
}
if obj.Debug {
obj.Logf("Process(%s)", vertex)
}
err = obj.Process(vertex)
if obj.Debug {
obj.Logf("Process(%s): Return(%+v)", vertex, err)
}
if err == nil {
break Loop
}
// we've got an error...
delay = res.MetaParams().Delay
if retry < 0 { // infinite retries
continue
}
if retry > 0 { // don't decrement past 0
retry--
obj.state[vertex].init.Logf("retrying CheckApply after %.4f seconds (%d left)", float64(delay)/1000, retry)
continue
}
//if retry == 0 { // optional
// err = errwrap.Wrapf(err, "permanent process error")
//}
// If this exits, defer calls Event(event.EventExit),
// which will cause the Watch loop to shutdown. Also,
// if the Watch loop shuts down, that will cause this
// Process loop to shut down. Also the graph sync can
// run an Event(event.EventExit) which causes this to
// shutdown as well. Lastly, it is possible that more
// that one of these scenarios happens simultaneously.
return err
}
}
//return nil // unreachable
}

15
resources/virt_disabled.go → engine/graph/autoedge.go
View File

@@ -15,11 +15,16 @@
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
// +build novirt
package graph
package resources
import (
"github.com/purpleidea/mgmt/engine/graph/autoedge"
)
// VirtRes represents the fields of the Virt resource. Since this file is
// only invoked with the tag "novirt", we do not need any fields here.
type VirtRes struct {
// AutoEdge adds the automatic edges to the graph.
func (obj *Engine) AutoEdge() error {
logf := func(format string, v ...interface{}) {
obj.Logf("autoedge: "+format, v...)
}
return autoedge.AutoEdge(obj.nextGraph, obj.Debug, logf)
}

177
resources/autoedge.go → engine/graph/autoedge/autoedge.go
View File

@@ -15,38 +15,88 @@
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package resources
package autoedge
import (
"fmt"
"log"
"github.com/purpleidea/mgmt/engine"
"github.com/purpleidea/mgmt/pgraph"
"github.com/purpleidea/mgmt/util"
multierr "github.com/hashicorp/go-multierror"
errwrap "github.com/pkg/errors"
)
// The AutoEdge interface is used to implement the autoedges feature.
type AutoEdge interface {
Next() []ResUID // call to get list of edges to add
Test([]bool) bool // call until false
}
// AutoEdge adds the automatic edges to the graph.
func AutoEdge(graph *pgraph.Graph, debug bool, logf func(format string, v ...interface{})) error {
logf("adding autoedges...")
// UIDExistsInUIDs wraps the IFF method when used with a list of UID's.
func UIDExistsInUIDs(uid ResUID, uids []ResUID) bool {
for _, u := range uids {
if uid.IFF(u) {
return true
// initially get all of the autoedges to seek out all possible errors
var err error
autoEdgeObjMap := make(map[engine.EdgeableRes]engine.AutoEdge)
sorted := []engine.EdgeableRes{}
for _, v := range graph.VerticesSorted() {
res, ok := v.(engine.EdgeableRes)
if !ok {
continue
}
if res.AutoEdgeMeta().Disabled { // skip if this res is disabled
continue
}
sorted = append(sorted, res)
}
for _, res := range sorted { // for each vertexes autoedges
autoEdgeObj, e := res.AutoEdges()
if e != nil {
err = multierr.Append(err, e) // collect all errors
continue
}
if autoEdgeObj == nil {
logf("no auto edges were found for: %s", res)
continue // next vertex
}
autoEdgeObjMap[res] = autoEdgeObj // save for next loop
}
if err != nil {
return errwrap.Wrapf(err, "the auto edges had errors")
}
// now that we're guaranteed error free, we can modify the graph safely
for _, res := range sorted { // stable sort order for determinism in logs
autoEdgeObj, exists := autoEdgeObjMap[res]
if !exists {
continue
}
for { // while the autoEdgeObj has more uids to add...
uids := autoEdgeObj.Next() // get some!
if uids == nil {
logf("the auto edge list is empty for: %s", res)
break // inner loop
}
if debug {
logf("autoedge: UIDS:")
for i, u := range uids {
logf("autoedge: UID%d: %v", i, u)
}
}
// match and add edges
result := addEdgesByMatchingUIDS(res, uids, graph, debug, logf)
// report back, and find out if we should continue
if !autoEdgeObj.Test(result) {
break
}
}
}
return false
return nil
}
// addEdgesByMatchingUIDS adds edges to the vertex in a graph based on if it
// matches a uid list.
func addEdgesByMatchingUIDS(g *pgraph.Graph, v pgraph.Vertex, uids []ResUID) []bool {
func addEdgesByMatchingUIDS(res engine.EdgeableRes, uids []engine.ResUID, graph *pgraph.Graph, debug bool, logf func(format string, v ...interface{})) []bool {
// search for edges and see what matches!
var result []bool
@@ -54,29 +104,36 @@ func addEdgesByMatchingUIDS(g *pgraph.Graph, v pgraph.Vertex, uids []ResUID) []b
for _, uid := range uids {
var found = false
// uid is a ResUID object
for _, vv := range g.Vertices() { // search
if v == vv { // skip self
for _, v := range graph.Vertices() { // search
r, ok := v.(engine.EdgeableRes)
if !ok {
continue
}
if b, ok := g.Value("debug"); ok && util.Bool(b) {
log.Printf("Compile: AutoEdge: Match: %s with UID: %s", vv, uid)
if r.AutoEdgeMeta().Disabled { // skip if this res is disabled
continue
}
if res == r { // skip self
continue
}
if debug {
logf("autoedge: Match: %s with UID: %s", r, uid)
}
// we must match to an effective UID for the resource,
// that is to say, the name value of a res is a helpful
// handle, but it is not necessarily a unique identity!
// remember, resources can return multiple UID's each!
if UIDExistsInUIDs(uid, VtoR(vv).UIDs()) {
// add edge from: vv -> v
if UIDExistsInUIDs(uid, r.UIDs()) {
// add edge from: r -> res
if uid.IsReversed() {
txt := fmt.Sprintf("AutoEdge: %s -> %s", vv, v)
log.Printf("Compile: Adding %s", txt)
edge := &Edge{Name: txt}
g.AddEdge(vv, v, edge)
txt := fmt.Sprintf("%s -> %s (autoedge)", r, res)
logf("autoedge: adding: %s", txt)
edge := &engine.Edge{Name: txt}
graph.AddEdge(r, res, edge)
} else { // edges go the "normal" way, eg: pkg resource
txt := fmt.Sprintf("AutoEdge: %s -> %s", v, vv)
log.Printf("Compile: Adding %s", txt)
edge := &Edge{Name: txt}
g.AddEdge(v, vv, edge)
txt := fmt.Sprintf("%s -> %s (autoedge)", res, r)
logf("autoedge: adding: %s", txt)
edge := &engine.Edge{Name: txt}
graph.AddEdge(res, r, edge)
}
found = true
break
@@ -87,62 +144,12 @@ func addEdgesByMatchingUIDS(g *pgraph.Graph, v pgraph.Vertex, uids []ResUID) []b
return result
}
// AutoEdges adds the automatic edges to the graph.
func AutoEdges(g *pgraph.Graph) error {
log.Println("Compile: Adding AutoEdges...")
// initially get all of the autoedges to seek out all possible errors
var err error
autoEdgeObjVertexMap := make(map[pgraph.Vertex]AutoEdge)
sorted := g.VerticesSorted()
for _, v := range sorted { // for each vertexes autoedges
if !VtoR(v).Meta().AutoEdge { // is the metaparam true?
continue
}
autoEdgeObj, e := VtoR(v).AutoEdges()
if e != nil {
err = multierr.Append(err, e) // collect all errors
continue
}
if autoEdgeObj == nil {
log.Printf("%s: No auto edges were found!", v)
continue // next vertex
}
autoEdgeObjVertexMap[v] = autoEdgeObj // save for next loop
}
if err != nil {
return errwrap.Wrapf(err, "the auto edges had errors")
}
// now that we're guaranteed error free, we can modify the graph safely
for _, v := range sorted { // stable sort order for determinism in logs
autoEdgeObj, exists := autoEdgeObjVertexMap[v]
if !exists {
continue
}
for { // while the autoEdgeObj has more uids to add...
uids := autoEdgeObj.Next() // get some!
if uids == nil {
log.Printf("%s: The auto edge list is empty!", v)
break // inner loop
}
if b, ok := g.Value("debug"); ok && util.Bool(b) {
log.Println("Compile: AutoEdge: UIDS:")
for i, u := range uids {
log.Printf("Compile: AutoEdge: UID%d: %v", i, u)
}
}
// match and add edges
result := addEdgesByMatchingUIDS(g, v, uids)
// report back, and find out if we should continue
if !autoEdgeObj.Test(result) {
break
}
// UIDExistsInUIDs wraps the IFF method when used with a list of UID's.
func UIDExistsInUIDs(uid engine.ResUID, uids []engine.ResUID) bool {
for _, u := range uids {
if uid.IFF(u) {
return true
}
}
return nil
return false
}

141
engine/graph/autogroup.go Normal file
View File

@@ -0,0 +1,141 @@
// Mgmt
// Copyright (C) 2013-2018+ James Shubin and the project contributors
// Written by James Shubin <james@shubin.ca> and the project contributors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package graph
import (
"fmt"
"github.com/purpleidea/mgmt/engine"
"github.com/purpleidea/mgmt/engine/graph/autogroup"
"github.com/purpleidea/mgmt/pgraph"
"github.com/purpleidea/mgmt/util"
errwrap "github.com/pkg/errors"
)
// AutoGroup runs the auto grouping on the loaded graph.
func (obj *Engine) AutoGroup(ag engine.AutoGrouper) error {
if obj.nextGraph == nil {
return fmt.Errorf("there is no active graph to autogroup")
}
logf := func(format string, v ...interface{}) {
obj.Logf("autogroup: "+format, v...)
}
// wrap ag with our own vertexCmp, vertexMerge and edgeMerge
wrapped := &wrappedGrouper{
AutoGrouper: ag, // pass in the existing autogrouper
}
if err := autogroup.AutoGroup(wrapped, obj.nextGraph, obj.Debug, logf); err != nil {
return errwrap.Wrapf(err, "autogrouping failed")
}
return nil
}
// wrappedGrouper is an autogrouper which adds our own Cmp and Merge functions
// on top of the desired AutoGrouper that was specified.
type wrappedGrouper struct {
engine.AutoGrouper // anonymous interface
}
func (obj *wrappedGrouper) Name() string {
return fmt.Sprintf("wrappedGrouper: %s", obj.AutoGrouper.Name())
}
func (obj *wrappedGrouper) VertexCmp(v1, v2 pgraph.Vertex) error {
// call existing vertexCmp first
if err := obj.AutoGrouper.VertexCmp(v1, v2); err != nil {
return err
}
r1, ok := v1.(engine.GroupableRes)
if !ok {
return fmt.Errorf("v1 is not a GroupableRes")
}
r2, ok := v2.(engine.GroupableRes)
if !ok {
return fmt.Errorf("v2 is not a GroupableRes")
}
if r1.Kind() != r2.Kind() { // we must group similar kinds
// TODO: maybe future resources won't need this limitation?
return fmt.Errorf("the two resources aren't the same kind")
}
// someone doesn't want to group!
if r1.AutoGroupMeta().Disabled || r2.AutoGroupMeta().Disabled {
return fmt.Errorf("one of the autogroup flags is false")
}
if r1.IsGrouped() { // already grouped!
return fmt.Errorf("already grouped")
}
if len(r2.GetGroup()) > 0 { // already has children grouped!
return fmt.Errorf("already has groups")
}
if err := r1.GroupCmp(r2); err != nil { // resource groupcmp failed!
return errwrap.Wrapf(err, "the GroupCmp failed")
}
return nil
}
func (obj *wrappedGrouper) VertexMerge(v1, v2 pgraph.Vertex) (v pgraph.Vertex, err error) {
r1, ok := v1.(engine.GroupableRes)
if !ok {
return nil, fmt.Errorf("v1 is not a GroupableRes")
}
r2, ok := v2.(engine.GroupableRes)
if !ok {
return nil, fmt.Errorf("v2 is not a GroupableRes")
}
if err = r1.GroupRes(r2); err != nil { // GroupRes skips stupid groupings
return // return early on error
}
// merging two resources into one should yield the sum of their semas
if semas := r2.MetaParams().Sema; len(semas) > 0 {
r1.MetaParams().Sema = append(r1.MetaParams().Sema, semas...)
r1.MetaParams().Sema = util.StrRemoveDuplicatesInList(r1.MetaParams().Sema)
}
return // success or fail, and no need to merge the actual vertices!
}
func (obj *wrappedGrouper) EdgeMerge(e1, e2 pgraph.Edge) pgraph.Edge {
e1x, ok := e1.(*engine.Edge)
if !ok {
return e2 // just return something to avoid needing to error
}
e2x, ok := e2.(*engine.Edge)
if !ok {
return e1 // just return something to avoid needing to error
}
// TODO: should we merge the edge.Notify or edge.refresh values?
edge := &engine.Edge{
Notify: e1x.Notify || e2x.Notify, // TODO: should we merge this?
}
refresh := e1x.Refresh() || e2x.Refresh() // TODO: should we merge this?
edge.SetRefresh(refresh)
return edge
}

71
engine/graph/autogroup/autogroup.go Normal file
View File

@@ -0,0 +1,71 @@
// Mgmt
// Copyright (C) 2013-2018+ James Shubin and the project contributors
// Written by James Shubin <james@shubin.ca> and the project contributors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package autogroup
import (
"fmt"
"github.com/purpleidea/mgmt/engine"
"github.com/purpleidea/mgmt/pgraph"
errwrap "github.com/pkg/errors"
)
// AutoGroup is the mechanical auto group "runner" that runs the interface spec.
// TODO: this algorithm may not be correct in all cases. replace if needed!
func AutoGroup(ag engine.AutoGrouper, g *pgraph.Graph, debug bool, logf func(format string, v ...interface{})) error {
logf("algorithm: %s...", ag.Name())
if err := ag.Init(g); err != nil {
return errwrap.Wrapf(err, "error running autoGroup(init)")
}
for {
var v, w pgraph.Vertex
v, w, err := ag.VertexNext() // get pair to compare
if err != nil {
return errwrap.Wrapf(err, "error running autoGroup(vertexNext)")
}
merged := false
// save names since they change during the runs
vStr := fmt.Sprintf("%v", v) // valid even if it is nil
wStr := fmt.Sprintf("%v", w)
if err := ag.VertexCmp(v, w); err != nil { // cmp ?
if debug {
logf("!GroupCmp for: %s into: %s", wStr, vStr)
}
// remove grouped vertex and merge edges (res is safe)
} else if err := VertexMerge(g, v, w, ag.VertexMerge, ag.EdgeMerge); err != nil { // merge...
logf("!VertexMerge for: %s into: %s", wStr, vStr)
} else { // success!
logf("success for: %s into: %s", wStr, vStr)
merged = true // woo
}
// did these get used?
if ok, err := ag.VertexTest(merged); err != nil {
return errwrap.Wrapf(err, "error running autoGroup(vertexTest)")
} else if !ok {
break // done!
}
}
return nil
}

336
resources/autogroup_test.go → engine/graph/autogroup/autogroup_test.go
View File

@@ -15,7 +15,7 @@
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package resources
package autogroup
import (
"fmt"
@@ -25,13 +25,114 @@ import (
"testing"
"time"
"github.com/purpleidea/mgmt/engine"
"github.com/purpleidea/mgmt/engine/traits"
"github.com/purpleidea/mgmt/pgraph"
"github.com/purpleidea/mgmt/util"
errwrap "github.com/pkg/errors"
)
func init() {
engine.RegisterResource("nooptest", func() engine.Res { return &NoopResTest{} })
}
// NoopResTest is a no-op resource that groups strangely.
type NoopResTest struct {
traits.Base // add the base methods without re-implementation
traits.Groupable
init *engine.Init
Comment string
}
func (obj *NoopResTest) Default() engine.Res {
return &NoopResTest{}
}
func (obj *NoopResTest) Validate() error {
return nil
}
func (obj *NoopResTest) Init(init *engine.Init) error {
obj.init = init // save for later
return nil
}
func (obj *NoopResTest) Close() error {
return nil
}
func (obj *NoopResTest) Watch() error {
return nil // not needed
}
func (obj *NoopResTest) CheckApply(apply bool) (checkOK bool, err error) {
return true, nil // state is always okay
}
func (obj *NoopResTest) Cmp(r engine.Res) error {
// we can only compare NoopRes to others of the same resource kind
res, ok := r.(*NoopResTest)
if !ok {
return fmt.Errorf("not a %s", obj.Kind())
}
if obj.Comment != res.Comment {
return fmt.Errorf("comment differs")
}
return nil
}
func (obj *NoopResTest) GroupCmp(r engine.GroupableRes) error {
res, ok := r.(*NoopResTest)
if !ok {
return fmt.Errorf("resource is not the same kind")
}
// TODO: implement this in vertexCmp for *testGrouper instead?
if strings.Contains(res.Name(), ",") { // HACK
return fmt.Errorf("already grouped") // element to be grouped is already grouped!
}
// group if they start with the same letter! (helpful hack for testing)
if obj.Name()[0] != res.Name()[0] {
return fmt.Errorf("different starting letter")
}
return nil
}
func NewNoopResTest(name string) *NoopResTest {
n, err := engine.NewNamedResource("nooptest", name)
if err != nil {
panic(fmt.Sprintf("unexpected error: %+v", err))
}
//x := n.(*resources.NoopRes)
g, ok := n.(engine.GroupableRes)
if !ok {
panic("not a GroupableRes")
}
g.AutoGroupMeta().Disabled = false // always autogroup
//x := g.(*NoopResTest)
x := n.(*NoopResTest)
return x
}
func NewNoopResTestSema(name string, semas []string) *NoopResTest {
n := NewNoopResTest(name)
n.MetaParams().Sema = semas
return n
}
// NE is a helper function to make testing easier. It creates a new noop edge.
func NE(s string) pgraph.Edge {
obj := &Edge{Name: s}
obj := &engine.Edge{Name: s}
return obj
}
@@ -40,41 +141,96 @@ type testGrouper struct {
NonReachabilityGrouper // "inherit" what we want, and reimplement the rest
}
func (ag *testGrouper) name() string {
func (obj *testGrouper) Name() string {
return "testGrouper"
}
func (ag *testGrouper) vertexMerge(v1, v2 pgraph.Vertex) (v pgraph.Vertex, err error) {
if err := VtoR(v1).GroupRes(VtoR(v2)); err != nil { // group them first
func (obj *testGrouper) VertexCmp(v1, v2 pgraph.Vertex) error {
// call existing vertexCmp first
if err := obj.NonReachabilityGrouper.VertexCmp(v1, v2); err != nil {
return err
}
r1, ok := v1.(engine.GroupableRes)
if !ok {
return fmt.Errorf("v1 is not a GroupableRes")
}
r2, ok := v2.(engine.GroupableRes)
if !ok {
return fmt.Errorf("v2 is not a GroupableRes")
}
if r1.Kind() != r2.Kind() { // we must group similar kinds
// TODO: maybe future resources won't need this limitation?
return fmt.Errorf("the two resources aren't the same kind")
}
// someone doesn't want to group!
if r1.AutoGroupMeta().Disabled || r2.AutoGroupMeta().Disabled {
return fmt.Errorf("one of the autogroup flags is false")
}
if r1.IsGrouped() { // already grouped!
return fmt.Errorf("already grouped")
}
if len(r2.GetGroup()) > 0 { // already has children grouped!
return fmt.Errorf("already has groups")
}
if err := r1.GroupCmp(r2); err != nil { // resource groupcmp failed!
return errwrap.Wrapf(err, "the GroupCmp failed")
}
return nil
}
func (obj *testGrouper) VertexMerge(v1, v2 pgraph.Vertex) (v pgraph.Vertex, err error) {
r1 := v1.(engine.GroupableRes)
r2 := v2.(engine.GroupableRes)
if err := r1.GroupRes(r2); err != nil { // group them first
return nil, err
}
// HACK: update the name so it matches full list of self+grouped
obj := VtoR(v1)
names := strings.Split(obj.GetName(), ",") // load in stored names
for _, n := range obj.GetGroup() {
names = append(names, n.GetName()) // add my contents
res := v1.(engine.GroupableRes)
names := strings.Split(res.Name(), ",") // load in stored names
for _, n := range res.GetGroup() {
names = append(names, n.Name()) // add my contents
}
names = util.StrRemoveDuplicatesInList(names) // remove duplicates
sort.Strings(names)
obj.SetName(strings.Join(names, ","))
res.SetName(strings.Join(names, ","))
// TODO: copied from autogroup.go, so try and build a better test...
// merging two resources into one should yield the sum of their semas
if semas := r2.MetaParams().Sema; len(semas) > 0 {
r1.MetaParams().Sema = append(r1.MetaParams().Sema, semas...)
r1.MetaParams().Sema = util.StrRemoveDuplicatesInList(r1.MetaParams().Sema)
}
return // success or fail, and no need to merge the actual vertices!
}
func (ag *testGrouper) edgeMerge(e1, e2 pgraph.Edge) pgraph.Edge {
edge1 := e1.(*Edge) // panic if wrong
edge2 := e2.(*Edge) // panic if wrong
func (obj *testGrouper) EdgeMerge(e1, e2 pgraph.Edge) pgraph.Edge {
edge1 := e1.(*engine.Edge) // panic if wrong
edge2 := e2.(*engine.Edge) // panic if wrong
// HACK: update the name so it makes a union of both names
n1 := strings.Split(edge1.Name, ",") // load
n2 := strings.Split(edge2.Name, ",") // load
names := append(n1, n2...)
names = util.StrRemoveDuplicatesInList(names) // remove duplicates
sort.Strings(names)
return &Edge{Name: strings.Join(names, ",")}
return &engine.Edge{Name: strings.Join(names, ",")}
}
// helper function
func runGraphCmp(t *testing.T, g1, g2 *pgraph.Graph) {
AutoGroup(g1, &testGrouper{}) // edits the graph
debug := true
logf := func(format string, v ...interface{}) {
t.Logf("test: "+format, v...)
}
if err := AutoGroup(&testGrouper{}, g1, debug, logf); err != nil { // edits the graph
t.Errorf("%v", err)
return
}
err := GraphCmp(g1, g2)
if err != nil {
t.Logf(" actual (g1): %v%v", g1, fullPrint(g1))
@@ -84,40 +240,6 @@ func runGraphCmp(t *testing.T, g1, g2 *pgraph.Graph) {
}
}
type NoopResTest struct {
NoopRes
}
func (obj *NoopResTest) GroupCmp(r Res) bool {
res, ok := r.(*NoopResTest)
if !ok {
return false
}
// TODO: implement this in vertexCmp for *testGrouper instead?
if strings.Contains(res.Name, ",") { // HACK
return false // element to be grouped is already grouped!
}
// group if they start with the same letter! (helpful hack for testing)
return obj.Name[0] == res.Name[0]
}
func NewNoopResTest(name string) *NoopResTest {
obj := &NoopResTest{
NoopRes: NoopRes{
BaseRes: BaseRes{
Name: name,
Kind: "noop",
MetaParams: MetaParams{
AutoGroup: true, // always autogroup
},
},
},
}
return obj
}
// GraphCmp compares the topology of two graphs and returns nil if they're
// equal. It also compares if grouped element groups are identical.
// TODO: port this to use the pgraph.GraphCmp function instead.
@@ -133,20 +255,20 @@ func GraphCmp(g1, g2 *pgraph.Graph) error {
Loop:
// check vertices
for v1 := range g1.Adjacency() { // for each vertex in g1
l1 := strings.Split(VtoR(v1).GetName(), ",") // make list of everyone's names...
for _, x1 := range VtoR(v1).GetGroup() {
l1 = append(l1, x1.GetName()) // add my contents
r1 := v1.(engine.GroupableRes)
l1 := strings.Split(r1.Name(), ",") // make list of everyone's names...
for _, x1 := range r1.GetGroup() {
l1 = append(l1, x1.Name()) // add my contents
}
l1 = util.StrRemoveDuplicatesInList(l1) // remove duplicates
sort.Strings(l1)
// inner loop
for v2 := range g2.Adjacency() { // does it match in g2 ?
l2 := strings.Split(VtoR(v2).GetName(), ",")
for _, x2 := range VtoR(v2).GetGroup() {
l2 = append(l2, x2.GetName())
r2 := v2.(engine.GroupableRes)
l2 := strings.Split(r2.Name(), ",")
for _, x2 := range r2.GetGroup() {
l2 = append(l2, x2.Name())
}
l2 = util.StrRemoveDuplicatesInList(l2) // remove duplicates
sort.Strings(l2)
@@ -157,7 +279,7 @@ Loop:
continue Loop
}
}
return fmt.Errorf("graph g1, has no match in g2 for: %v", VtoR(v1).GetName())
return fmt.Errorf("graph g1, has no match in g2 for: %v", r1.Name())
}
// vertices (and groups) match :)
@@ -166,35 +288,40 @@ Loop:
v2 := m[v1] // lookup in map to get correspondance
// g1.Adjacency()[v1] corresponds to g2.Adjacency()[v2]
if e1, e2 := len(g1.Adjacency()[v1]), len(g2.Adjacency()[v2]); e1 != e2 {
return fmt.Errorf("graph g1, vertex(%v) has %d edges, while g2, vertex(%v) has %d", VtoR(v1).GetName(), e1, VtoR(v2).GetName(), e2)
r1 := v1.(engine.Res)
r2 := v2.(engine.Res)
return fmt.Errorf("graph g1, vertex(%v) has %d edges, while g2, vertex(%v) has %d", r1.Name(), e1, r2.Name(), e2)
}
for vv1, ee1 := range g1.Adjacency()[v1] {
vv2 := m[vv1]
ee1 := ee1.(*Edge)
ee2 := g2.Adjacency()[v2][vv2].(*Edge)
ee1 := ee1.(*engine.Edge)
ee2 := g2.Adjacency()[v2][vv2].(*engine.Edge)
// these are edges from v1 -> vv1 via ee1 (graph 1)
// to cmp to edges from v2 -> vv2 via ee2 (graph 2)
// check: (1) vv1 == vv2 ? (we've already checked this!)
l1 := strings.Split(VtoR(vv1).GetName(), ",") // make list of everyone's names...
for _, x1 := range VtoR(vv1).GetGroup() {
l1 = append(l1, x1.GetName()) // add my contents
rr1 := vv1.(engine.GroupableRes)
rr2 := vv2.(engine.GroupableRes)
l1 := strings.Split(rr1.Name(), ",") // make list of everyone's names...
for _, x1 := range rr1.GetGroup() {
l1 = append(l1, x1.Name()) // add my contents
}
l1 = util.StrRemoveDuplicatesInList(l1) // remove duplicates
sort.Strings(l1)
l2 := strings.Split(VtoR(vv2).GetName(), ",")
for _, x2 := range VtoR(vv2).GetGroup() {
l2 = append(l2, x2.GetName())
l2 := strings.Split(rr2.Name(), ",")
for _, x2 := range rr2.GetGroup() {
l2 = append(l2, x2.Name())
}
l2 = util.StrRemoveDuplicatesInList(l2) // remove duplicates
sort.Strings(l2)
// does l1 match l2 ?
if !ListStrCmp(l1, l2) { // cmp!
return fmt.Errorf("graph g1 and g2 don't agree on: %v and %v", VtoR(vv1).GetName(), VtoR(vv2).GetName())
return fmt.Errorf("graph g1 and g2 don't agree on: %v and %v", rr1.Name(), rr2.Name())
}
// check: (2) ee1 == ee2
@@ -207,11 +334,13 @@ Loop:
// check meta parameters
for v1 := range g1.Adjacency() { // for each vertex in g1
for v2 := range g2.Adjacency() { // does it match in g2 ?
s1, s2 := VtoR(v1).Meta().Sema, VtoR(v2).Meta().Sema
r1 := v1.(engine.Res)
r2 := v2.(engine.Res)
s1, s2 := r1.MetaParams().Sema, r2.MetaParams().Sema
sort.Strings(s1)
sort.Strings(s2)
if !reflect.DeepEqual(s1, s2) {
return fmt.Errorf("vertex %s and vertex %s have different semaphores", VtoR(v1).GetName(), VtoR(v2).GetName())
return fmt.Errorf("vertex %s and vertex %s have different semaphores", r1.Name(), r2.Name())
}
}
}
@@ -242,17 +371,20 @@ func ListStrCmp(a, b []string) bool {
func fullPrint(g *pgraph.Graph) (str string) {
str += "\n"
for v := range g.Adjacency() {
if semas := VtoR(v).Meta().Sema; len(semas) > 0 {
str += fmt.Sprintf("* v: %v; sema: %v\n", VtoR(v).GetName(), semas)
r := v.(engine.Res)
if semas := r.MetaParams().Sema; len(semas) > 0 {
str += fmt.Sprintf("* v: %v; sema: %v\n", r.Name(), semas)
} else {
str += fmt.Sprintf("* v: %v\n", VtoR(v).GetName())
str += fmt.Sprintf("* v: %v\n", r.Name())
}
// TODO: add explicit grouping data?
}
for v1 := range g.Adjacency() {
for v2, e := range g.Adjacency()[v1] {
edge := e.(*Edge)
str += fmt.Sprintf("* e: %v -> %v # %v\n", VtoR(v1).GetName(), VtoR(v2).GetName(), edge.Name)
r1 := v1.(engine.Res)
r2 := v2.(engine.Res)
edge := e.(*engine.Edge)
str += fmt.Sprintf("* e: %v -> %v # %v\n", r1.Name(), r2.Name(), edge.Name)
}
}
return
@@ -731,3 +863,57 @@ func TestPgraphGroupingConnected1(t *testing.T) {
}
runGraphCmp(t, g1, g2)
}
func TestPgraphSemaphoreGrouping1(t *testing.T) {
g1, _ := pgraph.NewGraph("g1") // original graph
{
a1 := NewNoopResTestSema("a1", []string{"s:1"})
a2 := NewNoopResTestSema("a2", []string{"s:2"})
a3 := NewNoopResTestSema("a3", []string{"s:3"})
g1.AddVertex(a1)
g1.AddVertex(a2)
g1.AddVertex(a3)
}
g2, _ := pgraph.NewGraph("g2") // expected result
{
a123 := NewNoopResTestSema("a1,a2,a3", []string{"s:1", "s:2", "s:3"})
g2.AddVertex(a123)
}
runGraphCmp(t, g1, g2)
}
func TestPgraphSemaphoreGrouping2(t *testing.T) {
g1, _ := pgraph.NewGraph("g1") // original graph
{
a1 := NewNoopResTestSema("a1", []string{"s:10", "s:11"})
a2 := NewNoopResTestSema("a2", []string{"s:2"})
a3 := NewNoopResTestSema("a3", []string{"s:3"})
g1.AddVertex(a1)
g1.AddVertex(a2)
g1.AddVertex(a3)
}
g2, _ := pgraph.NewGraph("g2") // expected result
{
a123 := NewNoopResTestSema("a1,a2,a3", []string{"s:10", "s:11", "s:2", "s:3"})
g2.AddVertex(a123)
}
runGraphCmp(t, g1, g2)
}
func TestPgraphSemaphoreGrouping3(t *testing.T) {
g1, _ := pgraph.NewGraph("g1") // original graph
{
a1 := NewNoopResTestSema("a1", []string{"s:1", "s:2"})
a2 := NewNoopResTestSema("a2", []string{"s:2"})
a3 := NewNoopResTestSema("a3", []string{"s:3"})
g1.AddVertex(a1)
g1.AddVertex(a2)
g1.AddVertex(a3)
}
g2, _ := pgraph.NewGraph("g2") // expected result
{
a123 := NewNoopResTestSema("a1,a2,a3", []string{"s:1", "s:2", "s:3"})
g2.AddVertex(a123)
}
runGraphCmp(t, g1, g2)
}

127
engine/graph/autogroup/base.go Normal file
View File

@@ -0,0 +1,127 @@
// Mgmt
// Copyright (C) 2013-2018+ James Shubin and the project contributors
// Written by James Shubin <james@shubin.ca> and the project contributors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package autogroup
import (
"fmt"
"github.com/purpleidea/mgmt/pgraph"
)
// baseGrouper is the base type for implementing the AutoGrouper interface.
type baseGrouper struct {
graph *pgraph.Graph // store a pointer to the graph
vertices []pgraph.Vertex // cached list of vertices
i int
j int
done bool
}
// Name provides a friendly name for the logs to see.
func (ag *baseGrouper) Name() string {
return "baseGrouper"
}
// Init is called only once and before using other AutoGrouper interface methods
// the name method is the only exception: call it any time without side effects!
func (ag *baseGrouper) Init(g *pgraph.Graph) error {
if ag.graph != nil {
return fmt.Errorf("the init method has already been called")
}
ag.graph = g // pointer
ag.vertices = ag.graph.VerticesSorted() // cache in deterministic order!
ag.i = 0
ag.j = 0
if len(ag.vertices) == 0 { // empty graph
ag.done = true
return nil
}
return nil
}
// VertexNext is a simple iterator that loops through vertex (pair) combinations
// an intelligent algorithm would selectively offer only valid pairs of vertices
// these should satisfy logical grouping requirements for the autogroup designs!
// the desired algorithms can override, but keep this method as a base iterator!
func (ag *baseGrouper) VertexNext() (v1, v2 pgraph.Vertex, err error) {
// this does a for v... { for w... { return v, w }} but stepwise!
l := len(ag.vertices)
if ag.i < l {
v1 = ag.vertices[ag.i]
}
if ag.j < l {
v2 = ag.vertices[ag.j]
}
// in case the vertex was deleted
if !ag.graph.HasVertex(v1) {
v1 = nil
}
if !ag.graph.HasVertex(v2) {
v2 = nil
}
// two nested loops...
if ag.j < l {
ag.j++
}
if ag.j == l {
ag.j = 0
if ag.i < l {
ag.i++
}
if ag.i == l {
ag.done = true
}
}
return
}
// VertexCmp can be used in addition to an overridding implementation.
func (ag *baseGrouper) VertexCmp(v1, v2 pgraph.Vertex) error {
if v1 == nil || v2 == nil {
return fmt.Errorf("the vertex is nil")
}
if v1 == v2 { // skip yourself
return fmt.Errorf("the vertices are the same")
}
return nil // success
}
// VertexMerge needs to be overridden to add the actual merging functionality.
func (ag *baseGrouper) VertexMerge(v1, v2 pgraph.Vertex) (v pgraph.Vertex, err error) {
return nil, fmt.Errorf("vertexMerge needs to be overridden")
}
// EdgeMerge can be overridden, since it just simple returns the first edge.
func (ag *baseGrouper) EdgeMerge(e1, e2 pgraph.Edge) pgraph.Edge {
return e1 // noop
}
// VertexTest processes the results of the grouping for the algorithm to know
// return an error if something went horribly wrong, and bool false to stop.
func (ag *baseGrouper) VertexTest(b bool) (bool, error) {
// NOTE: this particular baseGrouper version doesn't track what happens
// because since we iterate over every pair, we don't care which merge!
if ag.done {
return false, nil
}
return true, nil
}

73
engine/graph/autogroup/nonreachability.go Normal file
View File

@@ -0,0 +1,73 @@
// Mgmt
// Copyright (C) 2013-2018+ James Shubin and the project contributors
// Written by James Shubin <james@shubin.ca> and the project contributors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package autogroup
import (
"github.com/purpleidea/mgmt/pgraph"
errwrap "github.com/pkg/errors"
)
// NonReachabilityGrouper is the most straight-forward algorithm for grouping.
// TODO: this algorithm may not be correct in all cases. replace if needed!
type NonReachabilityGrouper struct {
baseGrouper // "inherit" what we want, and reimplement the rest
}
// Name returns the name for the grouper algorithm.
func (ag *NonReachabilityGrouper) Name() string {
return "NonReachabilityGrouper"
}
// VertexNext iteratively finds vertex pairs with simple graph reachability...
// This algorithm relies on the observation that if there's a path from a to b,
// then they *can't* be merged (b/c of the existing dependency) so therefore we
// merge anything that *doesn't* satisfy this condition or that of the reverse!
func (ag *NonReachabilityGrouper) VertexNext() (v1, v2 pgraph.Vertex, err error) {
for {
v1, v2, err = ag.baseGrouper.VertexNext() // get all iterable pairs
if err != nil {
return nil, nil, errwrap.Wrapf(err, "error running autoGroup(vertexNext)")
}
// ignore self cmp early (perf optimization)
if v1 != v2 && v1 != nil && v2 != nil {
// if NOT reachable, they're viable...
out1, e1 := ag.graph.Reachability(v1, v2)
if e1 != nil {
return nil, nil, e1
}
out2, e2 := ag.graph.Reachability(v2, v1)
if e2 != nil {
return nil, nil, e2
}
if len(out1) == 0 && len(out2) == 0 {
return // return v1 and v2, they're viable
}
}
// if we got here, it means we're skipping over this candidate!
if ok, err := ag.baseGrouper.VertexTest(false); err != nil {
return nil, nil, errwrap.Wrapf(err, "error running autoGroup(vertexTest)")
} else if !ok {
return nil, nil, nil // done!
}
// the vertexTest passed, so loop and try with a new pair...
}
}

127
engine/graph/autogroup/util.go Normal file
View File

@@ -0,0 +1,127 @@
// Mgmt
// Copyright (C) 2013-2018+ James Shubin and the project contributors
// Written by James Shubin <james@shubin.ca> and the project contributors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package autogroup
import (
"github.com/purpleidea/mgmt/pgraph"
errwrap "github.com/pkg/errors"
)
// VertexMerge merges v2 into v1 by reattaching the edges where appropriate,
// and then by deleting v2 from the graph. Since more than one edge between two
// vertices is not allowed, duplicate edges are merged as well. an edge merge
// function can be provided if you'd like to control how you merge the edges!
func VertexMerge(g *pgraph.Graph, v1, v2 pgraph.Vertex, vertexMergeFn func(pgraph.Vertex, pgraph.Vertex) (pgraph.Vertex, error), edgeMergeFn func(pgraph.Edge, pgraph.Edge) pgraph.Edge) error {
// methodology
// 1) edges between v1 and v2 are removed
//Loop:
for k1 := range g.Adjacency() {
for k2 := range g.Adjacency()[k1] {
// v1 -> v2 || v2 -> v1
if (k1 == v1 && k2 == v2) || (k1 == v2 && k2 == v1) {
delete(g.Adjacency()[k1], k2) // delete map & edge
// NOTE: if we assume this is a DAG, then we can
// assume only v1 -> v2 OR v2 -> v1 exists, and
// we can break out of these loops immediately!
//break Loop
break
}
}
}
// 2) edges that point towards v2 from X now point to v1 from X (no dupes)
for _, x := range g.IncomingGraphVertices(v2) { // all to vertex v (??? -> v)
e := g.Adjacency()[x][v2] // previous edge
r, err := g.Reachability(x, v1)
if err != nil {
return err
}
// merge e with ex := g.Adjacency()[x][v1] if it exists!
if ex, exists := g.Adjacency()[x][v1]; exists && edgeMergeFn != nil && len(r) == 0 {
e = edgeMergeFn(e, ex)
}
if len(r) == 0 { // if not reachable, add it
g.AddEdge(x, v1, e) // overwrite edge
} else if edgeMergeFn != nil { // reachable, merge e through...
prev := x // initial condition
for i, next := range r {
if i == 0 {
// next == prev, therefore skip
continue
}
// this edge is from: prev, to: next
ex, _ := g.Adjacency()[prev][next] // get
ex = edgeMergeFn(ex, e)
g.Adjacency()[prev][next] = ex // set
prev = next
}
}
delete(g.Adjacency()[x], v2) // delete old edge
}
// 3) edges that point from v2 to X now point from v1 to X (no dupes)
for _, x := range g.OutgoingGraphVertices(v2) { // all from vertex v (v -> ???)
e := g.Adjacency()[v2][x] // previous edge
r, err := g.Reachability(v1, x)
if err != nil {
return err
}
// merge e with ex := g.Adjacency()[v1][x] if it exists!
if ex, exists := g.Adjacency()[v1][x]; exists && edgeMergeFn != nil && len(r) == 0 {
e = edgeMergeFn(e, ex)
}
if len(r) == 0 {
g.AddEdge(v1, x, e) // overwrite edge
} else if edgeMergeFn != nil { // reachable, merge e through...
prev := v1 // initial condition
for i, next := range r {
if i == 0 {
// next == prev, therefore skip
continue
}
// this edge is from: prev, to: next
ex, _ := g.Adjacency()[prev][next]
ex = edgeMergeFn(ex, e)
g.Adjacency()[prev][next] = ex
prev = next
}
}
delete(g.Adjacency()[v2], x)
}
// 4) merge and then remove the (now merged/grouped) vertex
if vertexMergeFn != nil { // run vertex merge function
if v, err := vertexMergeFn(v1, v2); err != nil {
return err
} else if v != nil { // replace v1 with the "merged" version...
// note: This branch isn't used if the vertexMergeFn
// decides to just merge logically on its own instead
// of actually returning something that we then merge.
v1 = v // TODO: ineffassign?
//*v1 = *v
}
}
g.DeleteVertex(v2) // remove grouped vertex
// 5) creation of a cyclic graph should throw an error
if _, err := g.TopologicalSort(); err != nil { // am i a dag or not?
return errwrap.Wrapf(err, "the TopologicalSort failed") // not a dag
}
return nil // success
}

336
engine/graph/engine.go Normal file
View File

@@ -0,0 +1,336 @@
// Mgmt
// Copyright (C) 2013-2018+ James Shubin and the project contributors
// Written by James Shubin <james@shubin.ca> and the project contributors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package graph
import (
"fmt"
"os"
"path"
"sync"
"github.com/purpleidea/mgmt/converger"
"github.com/purpleidea/mgmt/engine"
"github.com/purpleidea/mgmt/engine/event"
"github.com/purpleidea/mgmt/pgraph"
"github.com/purpleidea/mgmt/util/semaphore"
multierr "github.com/hashicorp/go-multierror"
errwrap "github.com/pkg/errors"
)
// Engine encapsulates a generic graph and manages its operations.
type Engine struct {
Program string
Hostname string
World engine.World
// Prefix is a unique directory prefix which can be used. It should be
// created if needed.
Prefix string
Converger converger.Converger
Debug bool
Logf func(format string, v ...interface{})
graph *pgraph.Graph
nextGraph *pgraph.Graph
state map[pgraph.Vertex]*State
waits map[pgraph.Vertex]*sync.WaitGroup
slock *sync.Mutex // semaphore lock
semas map[string]*semaphore.Semaphore
wg *sync.WaitGroup
fastPause bool
}
// Init initializes the internal structures and starts this the graph running.
// If the struct does not validate, or it cannot initialize, then this errors.
// Initially it will contain an empty graph.
func (obj *Engine) Init() error {
var err error
if obj.graph, err = pgraph.NewGraph("graph"); err != nil {
return err
}
if obj.Prefix == "" || obj.Prefix == "/" {
return fmt.Errorf("the prefix of `%s` is invalid", obj.Prefix)
}
if err := os.MkdirAll(obj.Prefix, 0770); err != nil {
return errwrap.Wrapf(err, "can't create prefix")
}
obj.state = make(map[pgraph.Vertex]*State)
obj.waits = make(map[pgraph.Vertex]*sync.WaitGroup)
obj.slock = &sync.Mutex{}
obj.semas = make(map[string]*semaphore.Semaphore)
obj.wg = &sync.WaitGroup{}
return nil
}
// Load a new graph into the engine. Offline graph operations will be performed
// on this graph. To switch it to the active graph, and run it, use Commit.
func (obj *Engine) Load(newGraph *pgraph.Graph) error {
if obj.nextGraph != nil {
return fmt.Errorf("can't overwrite pending graph, use abort")
}
obj.nextGraph = newGraph
return nil
}
// Abort the pending graph and any work in progress on it. After this call you
// may Load a new graph.
func (obj *Engine) Abort() error {
if obj.nextGraph == nil {
return fmt.Errorf("there is no pending graph to abort")
}
obj.nextGraph = nil
return nil
}
// Validate validates the pending graph to ensure it is appropriate for the
// engine. This should be called before Commit to avoid any surprises there!
// This prevents an error on Commit which could cause an engine shutdown.
func (obj *Engine) Validate() error {
for _, vertex := range obj.nextGraph.Vertices() {
res, ok := vertex.(engine.Res)
if !ok {
return fmt.Errorf("not a Res")
}
if err := engine.Validate(res); err != nil {
return errwrap.Wrapf(err, "the Res did not Validate")
}
}
return nil
}
// Apply a function to the pending graph. You must pass in a function which will
// receive this graph as input, and return an error if it something does not
// succeed.
func (obj *Engine) Apply(fn func(*pgraph.Graph) error) error {
return fn(obj.nextGraph)
}
// Commit runs a graph sync and swaps the loaded graph with the current one. If
// it errors, then the running graph wasn't changed. It is recommended that you
// pause the engine before running this, and resume it after you're done.
func (obj *Engine) Commit() error {
// TODO: Does this hurt performance or graph changes ?
vertexAddFn := func(vertex pgraph.Vertex) error {
// some of these validation steps happen before this Commit step
// in Validate() to avoid erroring here. These are redundant.
// FIXME: should we get rid of this redundant validation?
res, ok := vertex.(engine.Res)
if !ok { // should not happen, previously validated
return fmt.Errorf("not a Res")
}
if obj.Debug {
obj.Logf("loading resource `%s`", res)
}
if _, exists := obj.state[vertex]; exists {
return fmt.Errorf("the Res state already exists")
}
if obj.Debug {
obj.Logf("Validate(%s)", res)
}
err := engine.Validate(res)
if obj.Debug {
obj.Logf("Validate(%s): Return(%+v)", res, err)
}
if err != nil {
return errwrap.Wrapf(err, "the Res did not Validate")
}
// FIXME: is res.Name() sufficiently unique to use as a UID here?
pathUID := fmt.Sprintf("%s-%s", res.Kind(), res.Name())
statePrefix := fmt.Sprintf("%s/", path.Join(obj.Prefix, "state", pathUID))
// don't create this unless it *will* be used
//if err := os.MkdirAll(statePrefix, 0770); err != nil {
// return errwrap.Wrapf(err, "can't create state prefix")
//}
obj.waits[vertex] = &sync.WaitGroup{}
obj.state[vertex] = &State{
//Graph: obj.graph, // TODO: what happens if we swap the graph?
Vertex: vertex,
Program: obj.Program,
Hostname: obj.Hostname,
World: obj.World,
Prefix: statePrefix,
//Converger: obj.Converger,
Debug: obj.Debug,
Logf: func(format string, v ...interface{}) {
obj.Logf(res.String()+": "+format, v...)
},
}
if err := obj.state[vertex].Init(); err != nil {
return errwrap.Wrapf(err, "the Res did not Init")
}
return nil
}
vertexRemoveFn := func(vertex pgraph.Vertex) error {
// wait for exit before starting new graph!
obj.state[vertex].Event(event.EventExit) // signal an exit
obj.waits[vertex].Wait() // sync
// close the state and resource
// FIXME: will this mess up the sync and block the engine?
if err := obj.state[vertex].Close(); err != nil {
return errwrap.Wrapf(err, "the Res did not Close")
}
// delete to free up memory from old graphs
delete(obj.state, vertex)
delete(obj.waits, vertex)
return nil
}
// If GraphSync succeeds, it updates the receiver graph accordingly...
// Running the shutdown in vertexRemoveFn does not need to happen in a
// topologically sorted order because it already paused in that order.
obj.Logf("graph sync...")
if err := obj.graph.GraphSync(obj.nextGraph, engine.VertexCmpFn, vertexAddFn, vertexRemoveFn, engine.EdgeCmpFn); err != nil {
return errwrap.Wrapf(err, "error running graph sync")
}
obj.nextGraph = nil
// After this point, we must not error or we'd need to restore all of
// the changes that we'd made to the previously primary graph. This is
// because this function is meant to atomically swap the graphs safely.
// TODO: update all the `State` structs with the new Graph pointer
//for _, vertex := range obj.graph.Vertices() {
// state, exists := obj.state[vertex]
// if !exists {
// continue
// }
// state.Graph = obj.graph // update pointer to graph
//}
return nil
}
// Start runs the currently active graph. It also un-pauses the graph if it was
// paused.
func (obj *Engine) Start() error {
topoSort, err := obj.graph.TopologicalSort()
if err != nil {
return err
}
indegree := obj.graph.InDegree() // compute all of the indegree's
reversed := pgraph.Reverse(topoSort)
for _, vertex := range reversed {
state := obj.state[vertex]
state.starter = (indegree[vertex] == 0)
var unpause = true // assume true
if !state.working { // if not running...
state.working = true
unpause = false // doesn't need unpausing if starting
obj.wg.Add(1)
obj.waits[vertex].Add(1)
go func(v pgraph.Vertex) {
defer obj.wg.Done()
defer obj.waits[vertex].Done()
defer func() {
obj.state[v].working = false
}()
obj.Logf("Worker(%s)", v)
// contains the Watch and CheckApply loops
err := obj.Worker(v)
obj.Logf("Worker(%s): Exited(%+v)", v, err)
}(vertex)
}
select {
case <-state.started:
case <-state.stopped: // we failed on Watch start
}
if unpause { // unpause (if needed)
obj.state[vertex].Event(event.EventStart)
}
}
// we wait for everyone to start before exiting!
return nil
}
// SetFastPause puts the graph into fast pause mode. This is usually done via
// the argument to the Pause command, but this method can be used if a pause was
// already started, and you'd like subsequent parts to pause quickly. Once in
// fast pause mode for a given pause action, you cannot switch to regular pause.
// This is because once you've started a fast pause, some dependencies might
// have been skipped when fast pausing, and future resources might have missed a
// poke. In general this is only called when you're trying to hurry up the exit.
func (obj *Engine) SetFastPause() {
obj.fastPause = true
}
// Pause the active, running graph. At the moment this cannot error.
func (obj *Engine) Pause(fastPause bool) {
obj.fastPause = fastPause
topoSort, _ := obj.graph.TopologicalSort()
for _, vertex := range topoSort { // squeeze out the events...
// The Event is sent to an unbuffered channel, so this event is
// synchronous, and as a result it blocks until it is received.
obj.state[vertex].Event(event.EventPause)
}
// we are now completely paused...
obj.fastPause = false // reset
}
// Close triggers a shutdown. Engine must be already paused before this is run.
func (obj *Engine) Close() error {
var reterr error
emptyGraph, err := pgraph.NewGraph("empty")
if err != nil {
reterr = multierr.Append(reterr, err) // list of errors
}
// this is a graph switch (graph sync) that switches to an empty graph!
if err := obj.Load(emptyGraph); err != nil { // copy in empty graph
reterr = multierr.Append(reterr, err)
}
// the commit will cause the graph sync to shut things down cleverly...
if err := obj.Commit(); err != nil {
reterr = multierr.Append(reterr, err)
}
obj.wg.Wait() // for now, this doesn't need to be a separate Wait() method
return reterr
}
// Graph returns the running graph.
func (obj *Engine) Graph() *pgraph.Graph {
return obj.graph
}

59
engine/graph/refresh.go Normal file
View File

@@ -0,0 +1,59 @@
// Mgmt
// Copyright (C) 2013-2018+ James Shubin and the project contributors
// Written by James Shubin <james@shubin.ca> and the project contributors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package graph
import (
"github.com/purpleidea/mgmt/engine"
"github.com/purpleidea/mgmt/pgraph"
)
// RefreshPending determines if any previous nodes have a refresh pending here.
// If this is true, it means I am expected to apply a refresh when I next run.
func (obj *Engine) RefreshPending(vertex pgraph.Vertex) bool {
var refresh bool
for _, e := range obj.graph.IncomingGraphEdges(vertex) {
// if we asked for a notify *and* if one is pending!
edge := e.(*engine.Edge) // panic if wrong
if edge.Notify && edge.Refresh() {
refresh = true
break
}
}
return refresh
}
// SetUpstreamRefresh sets the refresh value to any upstream vertices.
func (obj *Engine) SetUpstreamRefresh(vertex pgraph.Vertex, b bool) {
for _, e := range obj.graph.IncomingGraphEdges(vertex) {
edge := e.(*engine.Edge) // panic if wrong
if edge.Notify {
edge.SetRefresh(b)
}
}
}
// SetDownstreamRefresh sets the refresh value to any downstream vertices.
func (obj *Engine) SetDownstreamRefresh(vertex pgraph.Vertex, b bool) {
for _, e := range obj.graph.OutgoingGraphEdges(vertex) {
edge := e.(*engine.Edge) // panic if wrong
// if we asked for a notify *and* if one is pending!
if edge.Notify {
edge.SetRefresh(b)
}
}
}

10
resources/semaphore.go → engine/graph/semaphore.go
View File

@@ -15,7 +15,7 @@
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package resources
package graph
import (
"fmt"
@@ -31,8 +31,8 @@ import (
// SemaSep is the trailing separator to split the semaphore id from the size.
const SemaSep = ":"
// SemaLock acquires the list of semaphores in the graph.
func (obj *MGraph) SemaLock(semas []string) error {
// semaLock acquires the list of semaphores in the graph.
func (obj *Engine) semaLock(semas []string) error {
var reterr error
sort.Strings(semas) // very important to avoid deadlock in the dag!
@@ -53,8 +53,8 @@ func (obj *MGraph) SemaLock(semas []string) error {
return reterr
}
// SemaUnlock releases the list of semaphores in the graph.
func (obj *MGraph) SemaUnlock(semas []string) error {
// semaUnlock releases the list of semaphores in the graph.
func (obj *Engine) semaUnlock(semas []string) error {
var reterr error
sort.Strings(semas) // unlock in the same order to remove partial locks

20
resources/augeas_disabled.go → engine/graph/semaphore_test.go
View File

@@ -15,11 +15,21 @@
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
// +build noaugeas
package graph
package resources
import (
"testing"
)
// AugeasRes represents the fields of the Augeas resource. Since this file is
// only invoked with the tag "noaugeas", we do not need any fields here.
type AugeasRes struct {
func TestSemaSize(t *testing.T) {
pairs := map[string]int{
"id:42": 42,
":13": 13,
"some_id": 1,
}
for id, size := range pairs {
if i := SemaSize(id); i != size {
t.Errorf("sema id `%s`, expected: `%d`, got: `%d`", id, size, i)
}
}
}

118
engine/graph/sendrecv.go Normal file
View File

@@ -0,0 +1,118 @@
// Mgmt
// Copyright (C) 2013-2018+ James Shubin and the project contributors
// Written by James Shubin <james@shubin.ca> and the project contributors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package graph
import (
"fmt"
"reflect"
"github.com/purpleidea/mgmt/engine"
multierr "github.com/hashicorp/go-multierror"
errwrap "github.com/pkg/errors"
)
// SendRecv pulls in the sent values into the receive slots. It is called by the
// receiver and must be given as input the full resource struct to receive on.
// It applies the loaded values to the resource.
func (obj *Engine) SendRecv(res engine.RecvableRes) (map[string]bool, error) {
recv := res.Recv()
if obj.Debug {
// NOTE: this could expose private resource data like passwords
obj.Logf("%s: SendRecv: %+v", res, recv)
}
var updated = make(map[string]bool) // list of updated keys
var err error
for k, v := range recv {
updated[k] = false // default
v.Changed = false // reset to the default
var st interface{} = v.Res // old style direct send/recv
if true { // new style send/recv API
st = v.Res.Sent()
}
// send
obj1 := reflect.Indirect(reflect.ValueOf(st))
type1 := obj1.Type()
value1 := obj1.FieldByName(v.Key)
kind1 := value1.Kind()
// recv
obj2 := reflect.Indirect(reflect.ValueOf(res)) // pass in full struct
type2 := obj2.Type()
value2 := obj2.FieldByName(k)
kind2 := value2.Kind()
if obj.Debug {
obj.Logf("Send(%s) has %v: %v", type1, kind1, value1)
obj.Logf("Recv(%s) has %v: %v", type2, kind2, value2)
}
// i think we probably want the same kind, at least for now...
if kind1 != kind2 {
e := fmt.Errorf("kind mismatch between %s: %s and %s: %s", v.Res, kind1, res, kind2)
err = multierr.Append(err, e) // list of errors
continue
}
// if the types don't match, we can't use send->recv
// FIXME: do we want to relax this for string -> *string ?
if e := TypeCmp(value1, value2); e != nil {
e := errwrap.Wrapf(e, "type mismatch between %s and %s", v.Res, res)
err = multierr.Append(err, e) // list of errors
continue
}
// if we can't set, then well this is pointless!
if !value2.CanSet() {
e := fmt.Errorf("can't set %s.%s", res, k)
err = multierr.Append(err, e) // list of errors
continue
}
// if we can't interface, we can't compare...
if !value1.CanInterface() || !value2.CanInterface() {
e := fmt.Errorf("can't interface %s.%s", res, k)
err = multierr.Append(err, e) // list of errors
continue
}
// if the values aren't equal, we're changing the receiver
if !reflect.DeepEqual(value1.Interface(), value2.Interface()) {
// TODO: can we catch the panics here in case they happen?
value2.Set(value1) // do it for all types that match
updated[k] = true // we updated this key!
v.Changed = true // tag this key as updated!
obj.Logf("SendRecv: %s.%s -> %s.%s", v.Res, v.Key, res, k)
}
}
return updated, err
}
// TypeCmp compares two reflect values to see if they are the same Kind. It can
// look into a ptr Kind to see if the underlying pair of ptr's can TypeCmp too!
func TypeCmp(a, b reflect.Value) error {
ta, tb := a.Type(), b.Type()
if ta != tb {
return fmt.Errorf("type mismatch: %s != %s", ta, tb)
}
// NOTE: it seems we don't need to recurse into pointers to sub check!
return nil // identical Type()'s
}

435
engine/graph/state.go Normal file
View File

@@ -0,0 +1,435 @@
// Mgmt
// Copyright (C) 2013-2018+ James Shubin and the project contributors
// Written by James Shubin <james@shubin.ca> and the project contributors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package graph
import (
"fmt"
"os"
"path"
"sync"
"time"
"github.com/purpleidea/mgmt/converger"
"github.com/purpleidea/mgmt/engine"
"github.com/purpleidea/mgmt/engine/event"
"github.com/purpleidea/mgmt/pgraph"
"github.com/purpleidea/mgmt/util"
errwrap "github.com/pkg/errors"
)
// State stores some state about the resource it is mapped to.
type State struct {
// Graph is a pointer to the graph that this vertex is part of.
//Graph pgraph.Graph
// Vertex is the pointer in the graph that this state corresponds to. It
// can be converted to a `Res` if necessary.
// TODO: should this be passed in on Init instead?
Vertex pgraph.Vertex
Program string
Hostname string
World engine.World
// Prefix is a unique directory prefix which can be used. It should be
// created if needed.
Prefix string
//Converger converger.Converger
// Debug turns on additional output and behaviours.
Debug bool
// Logf is the logging function that should be used to display messages.
Logf func(format string, v ...interface{})
timestamp int64 // last updated timestamp
isStateOK bool // is state OK or do we need to run CheckApply ?
// events is a channel of incoming events which is read by the Watch
// loop for that resource. It receives events like pause, start, and
// poke. The channel shuts down to signal for Watch to exit.
eventsChan chan event.Kind // incoming to resource
eventsLock *sync.Mutex // lock around sending and closing of events channel
eventsDone bool // is channel closed?
// output is the channel that the engine listens on for events from the
// Watch loop for that resource. The event is nil normally, except when
// events are sent on this channel from the engine. This only happens
// as a signaling mechanism when Watch has shutdown and we want to
// notify the Process loop which reads from this.
outputChan chan error // outgoing from resource
wg *sync.WaitGroup
exit *util.EasyExit
started chan struct{} // closes when it's started
stopped chan struct{} // closes when it's stopped
starter bool // do we have an indegree of 0 ?
working bool // is the Main() loop running ?
cuid converger.UID // primary converger
init *engine.Init // a copy of the init struct passed to res Init
}
// Init initializes structures like channels.
func (obj *State) Init() error {
obj.eventsChan = make(chan event.Kind)
obj.eventsLock = &sync.Mutex{}
obj.outputChan = make(chan error)
obj.wg = &sync.WaitGroup{}
obj.exit = util.NewEasyExit()
obj.started = make(chan struct{})
obj.stopped = make(chan struct{})
res, isRes := obj.Vertex.(engine.Res)
if !isRes {
return fmt.Errorf("vertex is not a Res")
}
if obj.Hostname == "" {
return fmt.Errorf("the Hostname is empty")
}
if obj.Prefix == "" {
return fmt.Errorf("the Prefix is empty")
}
if obj.Prefix == "/" {
return fmt.Errorf("the Prefix is root")
}
if obj.Logf == nil {
return fmt.Errorf("the Logf function is missing")
}
//obj.cuid = obj.Converger.Register() // gets registered in Worker()
obj.init = &engine.Init{
Program: obj.Program,
Hostname: obj.Hostname,
// Watch:
Running: func() error {
close(obj.started) // this is reset in the reset func
obj.isStateOK = false // assume we're initially dirty
// optimization: skip the initial send if not a starter
// because we'll get poked from a starter soon anyways!
if !obj.starter {
return nil
}
return obj.event()
},
Event: obj.event,
Events: obj.eventsChan,
Read: obj.read,
Dirty: func() { // TODO: should we rename this SetDirty?
obj.isStateOK = false
},
// CheckApply:
Refresh: func() bool {
res, ok := obj.Vertex.(engine.RefreshableRes)
if !ok {
panic("res does not support the Refreshable trait")
}
return res.Refresh()
},
Send: func(st interface{}) error {
res, ok := obj.Vertex.(engine.SendableRes)
if !ok {
panic("res does not support the Sendable trait")
}
// XXX: type check this
//expected := res.Sends()
//if err := XXX_TYPE_CHECK(expected, st); err != nil {
// return err
//}
return res.Send(st) // send the struct
},
Recv: func() map[string]*engine.Send { // TODO: change this API?
res, ok := obj.Vertex.(engine.RecvableRes)
if !ok {
panic("res does not support the Recvable trait")
}
return res.Recv()
},
World: obj.World,
VarDir: obj.varDir,
Debug: obj.Debug,
Logf: func(format string, v ...interface{}) {
obj.Logf("resource: "+format, v...)
},
}
// run the init
if obj.Debug {
obj.Logf("Init(%s)", res)
}
err := res.Init(obj.init)
if obj.Debug {
obj.Logf("Init(%s): Return(%+v)", res, err)
}
if err != nil {
return errwrap.Wrapf(err, "could not Init() resource")
}
return nil
}
// Close shuts down and performs any cleanup. This is most akin to a "post" or
// cleanup command as the initiator for closing a vertex happens in graph sync.
func (obj *State) Close() error {
res, isRes := obj.Vertex.(engine.Res)
if !isRes {
return fmt.Errorf("vertex is not a Res")
}
//if obj.cuid != nil {
// obj.cuid.Unregister() // gets unregistered in Worker()
//}
// redundant safety
obj.wg.Wait() // wait until all poke's and events on me have exited
// run the close
if obj.Debug {
obj.Logf("Close(%s)", res)
}
err := res.Close()
if obj.Debug {
obj.Logf("Close(%s): Return(%+v)", res, err)
}
return err
}
// reset is run to reset the state so that Watch can run a second time. Thus is
// needed for the Watch retry in particular.
func (obj *State) reset() {
obj.started = make(chan struct{})
obj.stopped = make(chan struct{})
}
// Poke sends a nil message on the outputChan. This channel is used by the
// resource to signal a possible change. This will cause the Process loop to
// run if it can.
func (obj *State) Poke() {
// add a wait group on the vertex we're poking!
obj.wg.Add(1)
defer obj.wg.Done()
select {
case obj.outputChan <- nil:
case <-obj.exit.Signal():
}
}
// Event sends a Pause or Start event to the resource. It can also be used to
// send Poke events, but it's much more efficient to send them directly instead
// of passing them through the resource.
func (obj *State) Event(kind event.Kind) {
// TODO: should these happen after the lock?
obj.wg.Add(1)
defer obj.wg.Done()
obj.eventsLock.Lock()
defer obj.eventsLock.Unlock()
if obj.eventsDone { // closing, skip events...
return
}
if kind == event.EventExit { // set this so future events don't deadlock
obj.Logf("exit event...")
obj.eventsDone = true
close(obj.eventsChan) // causes resource Watch loop to close
obj.exit.Done(nil) // trigger exit signal to unblock some cases
return
}
select {
case obj.eventsChan <- kind:
case <-obj.exit.Signal():
}
}
// read is a helper function used inside the main select statement of resources.
// If it returns an error, then this is a signal for the resource to exit.
func (obj *State) read(kind event.Kind) error {
switch kind {
case event.EventPoke:
return obj.event() // a poke needs to cause an event...
case event.EventStart:
return fmt.Errorf("unexpected start")
case event.EventPause:
// pass
case event.EventExit:
return engine.ErrSignalExit
default:
return fmt.Errorf("unhandled event: %+v", kind)
}
// we're paused now
select {
case kind, ok := <-obj.eventsChan:
if !ok {
return engine.ErrWatchExit
}
switch kind {
case event.EventPoke:
return fmt.Errorf("unexpected poke")
case event.EventPause:
return fmt.Errorf("unexpected pause")
case event.EventStart:
// resumed
return nil
case event.EventExit:
return engine.ErrSignalExit
default:
return fmt.Errorf("unhandled event: %+v", kind)
}
}
}
// event is a helper function to send an event from the resource Watch loop. It
// can be used for the initial `running` event, or any regular event. If it
// returns an error, then the Watch loop must return this error and shutdown.
func (obj *State) event() error {
// loop until we sent on obj.outputChan or exit with error
for {
select {
// send "activity" event
case obj.outputChan <- nil:
return nil // sent event!
// make sure to keep handling incoming
case kind, ok := <-obj.eventsChan:
if !ok {
return engine.ErrWatchExit
}
switch kind {
case event.EventPoke:
// we're trying to send an event, so swallow the
// poke: it's what we wanted to have happen here
case event.EventStart:
return fmt.Errorf("unexpected start")
case event.EventPause:
// pass
case event.EventExit:
return engine.ErrSignalExit
default:
return fmt.Errorf("unhandled event: %+v", kind)
}
}
// we're paused now
select {
case kind, ok := <-obj.eventsChan:
if !ok {
return engine.ErrWatchExit
}
switch kind {
case event.EventPoke:
return fmt.Errorf("unexpected poke")
case event.EventPause:
return fmt.Errorf("unexpected pause")
case event.EventStart:
// resumed
case event.EventExit:
return engine.ErrSignalExit
default:
return fmt.Errorf("unhandled event: %+v", kind)
}
}
}
}
// varDir returns the path to a working directory for the resource. It will try
// and create the directory first, and return an error if this failed. The dir
// should be cleaned up by the resource on Close if it wishes to discard the
// contents. If it does not, then a future resource with the same kind and name
// may see those contents in that directory. The resource should clean up the
// contents before use if it is important that nothing exist. It is always
// possible that contents could remain after an abrupt crash, so do not store
// overly sensitive data unless you're aware of the risks.
func (obj *State) varDir(extra string) (string, error) {
// Using extra adds additional dirs onto our namespace. An empty extra
// adds no additional directories.
if obj.Prefix == "" { // safety
return "", fmt.Errorf("the VarDir prefix is empty")
}
// an empty string at the end has no effect
p := fmt.Sprintf("%s/", path.Join(obj.Prefix, extra))
if err := os.MkdirAll(p, 0770); err != nil {
return "", errwrap.Wrapf(err, "can't create prefix in: %s", p)
}
// returns with a trailing slash as per the mgmt file res convention
return p, nil
}
// poll is a replacement for Watch when the Poll metaparameter is used.
func (obj *State) poll(interval uint32) error {
// create a time.Ticker for the given interval
ticker := time.NewTicker(time.Duration(interval) * time.Second)
defer ticker.Stop()
// notify engine that we're running
if err := obj.init.Running(); err != nil {
return err // exit if requested
}
var send = false // send event?
for {
select {
case <-ticker.C: // received the timer event
obj.init.Logf("polling...")
send = true
obj.init.Dirty() // dirty
case event, ok := <-obj.init.Events:
if !ok {
return nil
}
if err := obj.init.Read(event); err != nil {
return err
}
}
// do all our event sending all together to avoid duplicate msgs
if send {
send = false
if err := obj.init.Event(); err != nil {
return err // exit if requested
}
}
}
}

169
engine/metaparams.go Normal file
View File

@@ -0,0 +1,169 @@
// Mgmt
// Copyright (C) 2013-2018+ James Shubin and the project contributors
// Written by James Shubin <james@shubin.ca> and the project contributors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package engine
import (
"fmt"
"strconv"
"github.com/purpleidea/mgmt/util"
errwrap "github.com/pkg/errors"
"golang.org/x/time/rate"
)
// DefaultMetaParams are the defaults that are used for undefined metaparams.
// Don't modify this variable. Use .Copy() if you'd like some for yourself.
var DefaultMetaParams = &MetaParams{
Noop: false,
Retry: 0,
Delay: 0,
Poll: 0, // defaults to watching for events
Limit: rate.Inf, // defaults to no limit
Burst: 0, // no burst needed on an infinite rate
//Sema: []string{},
}
// MetaRes is the interface a resource must implement to support meta params.
// All resources must implement this.
type MetaRes interface {
// MetaParams lets you get or set meta params for the resource.
MetaParams() *MetaParams
}
// MetaParams provides some meta parameters that apply to every resource.
type MetaParams struct {
// Noop specifies that no changes should be made by the resource. It
// relies on the individual resource implementation, and can't protect
// you from a poorly or maliciously implemented resource.
Noop bool `yaml:"noop"`
// NOTE: there are separate Watch and CheckApply retry and delay values,
// but I've decided to use the same ones for both until there's a proper
// reason to want to do something differently for the Watch errors.
// Retry is the number of times to retry on error. Use -1 for infinite.
Retry int16 `yaml:"retry"`
// Delay is the number of milliseconds to wait between retries.
Delay uint64 `yaml:"delay"`
// Poll is the number of seconds between poll intervals. Use 0 to Watch.
Poll uint32 `yaml:"poll"`
// Limit is the number of events per second to allow through.
Limit rate.Limit `yaml:"limit"`
// Burst is the number of events to allow in a burst.
Burst int `yaml:"burst"`
// Sema is a list of semaphore ids in the form `id` or `id:count`. If
// you don't specify a count, then 1 is assumed. The sema of `foo` which
// has a count equal to 1, is different from a sema named `foo:1` which
// also has a count equal to 1, but is a different semaphore.
Sema []string `yaml:"sema"`
}
// Cmp compares two AutoGroupMeta structs and determines if they're equivalent.
func (obj *MetaParams) Cmp(meta *MetaParams) error {
if obj.Noop != meta.Noop {
return fmt.Errorf("values for Noop are different")
}
// XXX: add a one way cmp like we used to have ?
//if obj.Noop != meta.Noop {
// // obj is the existing res, res is the *new* resource
// // if we go from no-noop -> noop, we can re-use the obj
// // if we go from noop -> no-noop, we need to regenerate
// if obj.Noop { // asymmetrical
// return fmt.Errorf("values for Noop are different") // going from noop to no-noop!
// }
//}
if obj.Retry != meta.Retry {
return fmt.Errorf("values for Retry are different")
}
if obj.Delay != meta.Delay {
return fmt.Errorf("values for Delay are different")
}
if obj.Poll != meta.Poll {
return fmt.Errorf("values for Poll are different")
}
if obj.Limit != meta.Limit {
return fmt.Errorf("values for Limit are different")
}
if obj.Burst != meta.Burst {
return fmt.Errorf("values for Burst are different")
}
if err := util.SortedStrSliceCompare(obj.Sema, meta.Sema); err != nil {
return errwrap.Wrapf(err, "values for Sema are different")
}
return nil
}
// Validate runs some validation on the meta params.
func (obj *MetaParams) Validate() error {
if obj.Burst == 0 && !(obj.Limit == rate.Inf) { // blocked
return fmt.Errorf("permanently limited (rate != Inf, burst = 0)")
}
for _, s := range obj.Sema {
if s == "" {
return fmt.Errorf("semaphore is empty")
}
if _, err := strconv.Atoi(s); err == nil { // standalone int
return fmt.Errorf("semaphore format is invalid")
}
}
return nil
}
// Copy copies this struct and returns a new one.
func (obj *MetaParams) Copy() *MetaParams {
sema := []string{}
if obj.Sema != nil {
sema = make([]string, len(obj.Sema))
copy(sema, obj.Sema)
}
return &MetaParams{
Noop: obj.Noop,
Retry: obj.Retry,
Delay: obj.Delay,
Poll: obj.Poll,
Limit: obj.Limit, // FIXME: can we copy this type like this? test me!
Burst: obj.Burst,
Sema: sema,
}
}
// UnmarshalYAML is the custom unmarshal handler for the MetaParams struct. It
// is primarily useful for setting the defaults.
// TODO: this is untested
func (obj *MetaParams) UnmarshalYAML(unmarshal func(interface{}) error) error {
type rawMetaParams MetaParams // indirection to avoid infinite recursion
raw := rawMetaParams(*DefaultMetaParams) // convert; the defaults go here
if err := unmarshal(&raw); err != nil {
return err
}
*obj = MetaParams(raw) // restore from indirection with type conversion!
return nil
}

40
engine/metaparams_test.go Normal file
View File

@@ -0,0 +1,40 @@
// Mgmt
// Copyright (C) 2013-2018+ James Shubin and the project contributors
// Written by James Shubin <james@shubin.ca> and the project contributors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package engine
import (
"testing"
)
func TestMetaCmp1(t *testing.T) {
m1 := &MetaParams{
Noop: true,
}
m2 := &MetaParams{
Noop: false,
}
// TODO: should we allow this? Maybe only with the future Mutate API?
//if err := m2.Cmp(m1); err != nil { // going from noop(false) -> noop(true) is okay!
// t.Errorf("the two resources do not match")
//}
if m1.Cmp(m2) == nil { // going from noop(true) -> noop(false) is not okay!
t.Errorf("the two resources should not match")
}
}

32
engine/refresh.go Normal file
View File

@@ -0,0 +1,32 @@
// Mgmt
// Copyright (C) 2013-2018+ James Shubin and the project contributors
// Written by James Shubin <james@shubin.ca> and the project contributors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package engine
// RefreshableRes is the interface a resource must implement to support refresh
// notifications. Default implementations for all of the methods declared in
// this interface can be obtained for your resource by anonymously adding the
// traits.Refreshable struct to your resource implementation.
type RefreshableRes interface {
Res // implement everything in Res but add the additional requirements
// Refresh returns the refresh notification state.
Refresh() bool
// SetRefresh sets the refresh notification state.
SetRefresh(bool)
}

271
engine/resources.go Normal file
View File

@@ -0,0 +1,271 @@
// Mgmt
// Copyright (C) 2013-2018+ James Shubin and the project contributors
// Written by James Shubin <james@shubin.ca> and the project contributors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package engine
import (
"encoding/gob"
"fmt"
"github.com/purpleidea/mgmt/engine/event"
errwrap "github.com/pkg/errors"
"gopkg.in/yaml.v2"
)
// TODO: should each resource be a sub-package?
var registeredResources = map[string]func() Res{}
// RegisterResource registers a new resource by providing a constructor
// function that returns a resource object ready to be unmarshalled from YAML.
func RegisterResource(kind string, fn func() Res) {
f := fn()
if kind == "" {
panic("can't register a resource with an empty kind")
}
if _, ok := registeredResources[kind]; ok {
panic(fmt.Sprintf("a resource kind of %s is already registered", kind))
}
gob.Register(f)
registeredResources[kind] = fn
}
// RegisteredResourcesNames returns the kind of the registered resources.
func RegisteredResourcesNames() []string {
kinds := []string{}
for k := range registeredResources {
kinds = append(kinds, k)
}
return kinds
}
// NewResource returns an empty resource object from a registered kind. It
// errors if the resource kind doesn't exist.
func NewResource(kind string) (Res, error) {
fn, ok := registeredResources[kind]
if !ok {
return nil, fmt.Errorf("no resource kind `%s` available", kind)
}
res := fn().Default()
res.SetKind(kind)
return res, nil
}
// NewNamedResource returns an empty resource object from a registered kind. It
// also sets the name. It is a wrapper around NewResource. It also errors if the
// name is empty.
func NewNamedResource(kind, name string) (Res, error) {
if name == "" {
return nil, fmt.Errorf("resource name is empty")
}
res, err := NewResource(kind)
if err != nil {
return nil, err
}
res.SetName(name)
return res, nil
}
// Init is the structure of values and references which is passed into all
// resources on initialization. None of these are available in Validate, or
// before Init runs.
type Init struct {
// Program is the name of the program.
Program string
// Hostname is the uuid for the host.
Hostname string
// Called from within Watch:
// Running must be called after your watches are all started and ready.
Running func() error
// Event sends an event notifying the engine of a possible state change.
Event func() error
// Events returns a channel that we must watch for messages from the
// engine. When it closes, this is a signal to shutdown.
Events chan event.Kind
// Read processes messages that come in from the Events channel. It is a
// helper method that knows how to handle the pause mechanism correctly.
Read func(event.Kind) error
// Dirty marks the resource state as dirty. This signals to the engine
// that CheckApply will have some work to do in order to converge it.
Dirty func()
// Called from within CheckApply:
// Refresh returns whether the resource received a notification. This
// flag can be used to tell a svc to reload, or to perform some state
// change that wouldn't otherwise be noticed by inspection alone. You
// must implement the Refreshable trait for this to work.
Refresh func() bool
// Send exposes some variables you wish to send via the Send/Recv
// mechanism. You must implement the Sendable trait for this to work.
Send func(interface{}) error
// Recv provides a map of variables which were sent to this resource via
// the Send/Recv mechanism. You must implement the Recvable trait for
// this to work.
Recv func() map[string]*Send
// Other functionality:
// World provides a connection to the outside world. This is most often
// used for communicating with the distributed database.
World World
// VarDir is a facility for local storage. It is used to return a path
// to a directory which may be used for temporary storage. It should be
// cleaned up on resource Close if the resource would like to delete the
// contents. The resource should not assume that the initial directory
// is empty, and it should be cleaned on Init if that is a requirement.
VarDir func(string) (string, error)
// Debug signals whether we are running in debugging mode. In this case,
// we might want to log additional messages.
Debug bool
// Logf is a logging facility which will correctly namespace any
// messages which you wish to pass on. You should use this instead of
// the log package directly for production quality resources.
Logf func(format string, v ...interface{})
}
// KindedRes is an interface that is required for a resource to have a kind.
type KindedRes interface {
// Kind returns a string representing the kind of resource this is.
Kind() string
// SetKind sets the resource kind and should only be called by the
// engine.
SetKind(string)
}
// NamedRes is an interface that is used so a resource can have a unique name.
type NamedRes interface {
Name() string
SetName(string)
}
// Res is the minimum interface you need to implement to define a new resource.
type Res interface {
fmt.Stringer // String() string
KindedRes
NamedRes // TODO: consider making this optional in the future
MetaRes // All resources must have meta params.
// Default returns a struct with sane defaults for this resource.
Default() Res
// Validate determines if the struct has been defined in a valid state.
Validate() error
// Init initializes the resource and passes in some external information
// and data from the engine.
Init(*Init) error
// Close is run by the engine to clean up after the resource is done.
Close() error
// Watch is run by the engine to monitor for state changes. If it
// detects any, it notifies the engine which will usually run CheckApply
// in response.
Watch() error
// CheckApply determines if the state of the resource is connect and if
// asked to with the `apply` variable, applies the requested state.
CheckApply(apply bool) (checkOK bool, err error)
// Cmp compares itself to another resource and returns an error if they
// are not equivalent.
Cmp(Res) error
}
// Repr returns a representation of a resource from its kind and name. This is
// used as the definitive format so that it can be changed in one place.
func Repr(kind, name string) string {
return fmt.Sprintf("%s[%s]", kind, name)
}
// Stringer returns a consistent and unique string representation of a resource.
func Stringer(res Res) string {
return Repr(res.Kind(), res.Name())
}
// Validate validates a resource by checking multiple aspects. This is the main
// entry point for running all the validation steps on a resource.
func Validate(res Res) error {
if res.Kind() == "" { // shouldn't happen IIRC
return fmt.Errorf("the Res has an empty Kind")
}
if res.Name() == "" {
return fmt.Errorf("the Res has an empty Name")
}
if err := res.MetaParams().Validate(); err != nil {
return errwrap.Wrapf(err, "the Res has an invalid meta param")
}
return res.Validate()
}
// InterruptableRes is an interface that adds interrupt functionality to
// resources. If the resource implements this interface, the engine will call
// the Interrupt method to shutdown the resource quickly. Running this method
// may leave the resource in a partial state, however this may be desired if you
// want a faster exit or if you'd prefer a partial state over letting the
// resource complete in a situation where you made an error and you wish to
// exit quickly to avoid data loss. It is usually triggered after multiple ^C
// signals.
type InterruptableRes interface {
Res
// Ask the resource to shutdown quickly. This can be called at any point
// in the resource lifecycle after Init. Close will still be called. It
// will only get called after an exit or pause request has been made. It
// is designed to unblock any long running operation that is occurring
// in the CheckApply portion of the life cycle. If the resource has
// already exited, running this method should not block. (That is to say
// that you should not expect CheckApply or Watch to be able to alive
// and able to read from a channel to satisfy your request.) It is best
// to probably have this close a channel to multicast that signal around
// to anyone who can detect it in a select. If you are in a situation
// which cannot interrupt, then you can return an error.
// FIXME: implement, and check the above description is what we expect!
Interrupt() error
}
// CollectableRes is an interface for resources that support collection. It is
// currently temporary until a proper API for all resources is invented.
type CollectableRes interface {
Res
CollectPattern(string) // XXX: temporary until Res collection is more advanced
}
// YAMLRes is a resource that supports creation by unmarshalling.
type YAMLRes interface {
Res
yaml.Unmarshaler // UnmarshalYAML(unmarshal func(interface{}) error) error
}

151
resources/augeas.go → engine/resources/augeas.go
View File

@@ -21,10 +21,11 @@ package resources
import (
"fmt"
"log"
"os"
"strings"
"github.com/purpleidea/mgmt/engine"
"github.com/purpleidea/mgmt/engine/traits"
"github.com/purpleidea/mgmt/recwatch"
errwrap "github.com/pkg/errors"
@@ -39,13 +40,15 @@ const (
)
func init() {
RegisterResource("augeas", func() Res { return &AugeasRes{} })
engine.RegisterResource("augeas", func() engine.Res { return &AugeasRes{} })
}
// AugeasRes is a resource that enables you to use the augeas resource.
// Currently only allows you to change simple files (e.g sshd_config).
type AugeasRes struct {
BaseRes `yaml:",inline"`
traits.Base // add the base methods without re-implementation
init *engine.Init
// File is the path to the file targeted by this resource.
File string `yaml:"file"`
@@ -57,7 +60,7 @@ type AugeasRes struct {
// Sets is a list of changes that will be applied to the file, in the form of
// ["path", "value"]. mgmt will run augeas.Get() before augeas.Set(), to
// prevent changing the file when it is not needed.
Sets []AugeasSet `yaml:"sets"`
Sets []*AugeasSet `yaml:"sets"`
recWatcher *recwatch.RecWatcher // used to watch the changed files
}
@@ -68,13 +71,31 @@ type AugeasSet struct {
Value string `yaml:"value"` // The value to be set on the given Path.
}
// Default returns some sensible defaults for this resource.
func (obj *AugeasRes) Default() Res {
return &AugeasRes{
BaseRes: BaseRes{
MetaParams: DefaultMetaParams, // force a default
},
// Cmp compares this set with another one.
func (obj *AugeasSet) Cmp(set *AugeasSet) error {
if obj == nil && set == nil {
return nil
}
if obj == nil && set != nil {
return fmt.Errorf("can't compare nil set to set")
}
if obj != nil && set == nil {
return fmt.Errorf("can't compare set to nil set")
}
if obj.Path != set.Path {
return fmt.Errorf("the Path values differ")
}
if obj.Value != set.Value {
return fmt.Errorf("the Value values differ")
}
return nil
}
// Default returns some sensible defaults for this resource.
func (obj *AugeasRes) Default() engine.Res {
return &AugeasRes{}
}
// Validate if the params passed in are valid data.
@@ -88,12 +109,19 @@ func (obj *AugeasRes) Validate() error {
if (obj.Lens == "") != (obj.File == "") {
return fmt.Errorf("the File and Lens params must be specified together")
}
return obj.BaseRes.Validate()
return nil
}
// Init initiates the resource.
func (obj *AugeasRes) Init() error {
return obj.BaseRes.Init() // call base init, b/c we're overriding
// Init initializes the resource.
func (obj *AugeasRes) Init(init *engine.Init) error {
obj.init = init // save for later
return nil
}
// Close is run by the engine to clean up after the resource is done.
func (obj *AugeasRes) Close() error {
return nil
}
// Watch is the primary listener for this resource and it outputs events.
@@ -108,16 +136,14 @@ func (obj *AugeasRes) Watch() error {
defer obj.recWatcher.Close()
// notify engine that we're running
if err := obj.Running(); err != nil {
return err // bubble up a NACK...
if err := obj.init.Running(); err != nil {
return err // exit if requested
}
var send = false // send event?
var exit *error
for {
if obj.debug {
log.Printf("%s: Watching: %s", obj, obj.File) // attempting to watch...
if obj.init.Debug {
obj.init.Logf("Watching: %s", obj.File) // attempting to watch...
}
select {
@@ -128,29 +154,33 @@ func (obj *AugeasRes) Watch() error {
if err := event.Error; err != nil {
return errwrap.Wrapf(err, "Unknown %s watcher error", obj)
}
if obj.debug { // don't access event.Body if event.Error isn't nil
log.Printf("%s: Event(%s): %v", obj, event.Body.Name, event.Body.Op)
if obj.init.Debug { // don't access event.Body if event.Error isn't nil
obj.init.Logf("Event(%s): %v", event.Body.Name, event.Body.Op)
}
send = true
obj.StateOK(false) // dirty
obj.init.Dirty() // dirty
case event := <-obj.Events():
if exit, send = obj.ReadEvent(event); exit != nil {
return *exit // exit
case event, ok := <-obj.init.Events:
if !ok {
return nil
}
if err := obj.init.Read(event); err != nil {
return err
}
//obj.StateOK(false) // dirty // these events don't invalidate state
}
// do all our event sending all together to avoid duplicate msgs
if send {
send = false
obj.Event()
if err := obj.init.Event(); err != nil {
return err // exit if requested
}
}
}
}
// checkApplySet runs CheckApply for one element of the AugeasRes.Set
func (obj *AugeasRes) checkApplySet(apply bool, ag *augeas.Augeas, set AugeasSet) (bool, error) {
func (obj *AugeasRes) checkApplySet(apply bool, ag *augeas.Augeas, set *AugeasSet) (bool, error) {
fullpath := fmt.Sprintf("/files/%v/%v", obj.File, set.Path)
// We do not check for errors because errors are also thrown when
@@ -176,7 +206,7 @@ func (obj *AugeasRes) checkApplySet(apply bool, ag *augeas.Augeas, set AugeasSet
// CheckApply method for Augeas resource.
func (obj *AugeasRes) CheckApply(apply bool) (bool, error) {
log.Printf("%s: CheckApply: %s", obj, obj.File)
obj.init.Logf("CheckApply: %s", obj.File)
// By default we do not set any option to augeas, we use the defaults.
opts := augeas.None
if obj.Lens != "" {
@@ -224,7 +254,7 @@ func (obj *AugeasRes) CheckApply(apply bool) (bool, error) {
return checkOK, nil
}
log.Printf("%s: changes needed, saving", obj)
obj.init.Logf("changes needed, saving")
if err = ag.Save(); err != nil {
return false, errwrap.Wrapf(err, "augeas: error while saving augeas values")
}
@@ -240,41 +270,46 @@ func (obj *AugeasRes) CheckApply(apply bool) (bool, error) {
return false, nil
}
// Cmp compares two resources and returns an error if they are not equivalent.
func (obj *AugeasRes) Cmp(r engine.Res) error {
// we can only compare to others of the same resource kind
res, ok := r.(*AugeasRes)
if !ok {
return fmt.Errorf("resource is not the same kind")
}
if obj.File != res.File {
return fmt.Errorf("the File params differ")
}
if obj.Lens != res.Lens {
return fmt.Errorf("the Lens params differ")
}
if len(obj.Sets) != len(res.Sets) {
return fmt.Errorf("the length of the two Sets params differs")
}
for i := 0; i < len(obj.Sets); i++ {
if err := obj.Sets[i].Cmp(res.Sets[i]); err != nil {
return errwrap.Wrapf(err, "the Sets item at index %d differs", i)
}
}
return nil
}
// AugeasUID is the UID struct for AugeasRes.
type AugeasUID struct {
BaseUID
engine.BaseUID
name string
}
// UIDs includes all params to make a unique identification of this object.
func (obj *AugeasRes) UIDs() []ResUID {
func (obj *AugeasRes) UIDs() []engine.ResUID {
x := &AugeasUID{
BaseUID: BaseUID{Name: obj.GetName(), Kind: obj.GetKind()},
name: obj.Name,
BaseUID: engine.BaseUID{Name: obj.Name(), Kind: obj.Kind()},
name: obj.Name(),
}
return []ResUID{x}
}
// GroupCmp returns whether two resources can be grouped together or not.
func (obj *AugeasRes) GroupCmp(r Res) bool {
return false // Augeas commands can not be grouped together.
}
// Compare two resources and return if they are equivalent.
func (obj *AugeasRes) Compare(r Res) bool {
// we can only compare AugeasRes to others of the same resource kind
res, ok := r.(*AugeasRes)
if !ok {
return false
}
if !obj.BaseRes.Compare(res) { // call base Compare
return false
}
if obj.Name != res.Name {
return false
}
return true
return []engine.ResUID{x}
}
// UnmarshalYAML is the custom unmarshal handler for this struct.

540
resources/aws_ec2.go → engine/resources/aws_ec2.go
View File

@@ -25,7 +25,6 @@ import (
"encoding/pem"
"fmt"
"io/ioutil"
"log"
"net"
"net/http"
"regexp"
@@ -33,6 +32,9 @@ import (
"sync"
"time"
"github.com/purpleidea/mgmt/engine"
"github.com/purpleidea/mgmt/engine/traits"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/request"
@@ -45,7 +47,7 @@ import (
)
func init() {
RegisterResource("aws:ec2", func() Res { return &AwsEc2Res{} })
engine.RegisterResource("aws:ec2", func() engine.Res { return &AwsEc2Res{} })
}
const (
@@ -145,7 +147,10 @@ var AwsRegions = []string{
// AWS credentials must be present in ~/.aws - For detailed instructions see
// http://docs.aws.amazon.com/cli/latest/userguide/cli-config-files.html
type AwsEc2Res struct {
BaseRes `yaml:",inline"`
traits.Base // add the base methods without re-implementation
init *engine.Init
State string `yaml:"state"` // state: running, stopped, terminated
Region string `yaml:"region"` // region must match an element of AwsRegions
Type string `yaml:"type"` // type of ec2 instance, eg: t2.micro
@@ -250,12 +255,8 @@ type postMsg struct {
}
// Default returns some sensible defaults for this resource.
func (obj *AwsEc2Res) Default() Res {
return &AwsEc2Res{
BaseRes: BaseRes{
MetaParams: DefaultMetaParams, // force a default
},
}
func (obj *AwsEc2Res) Default() engine.Res {
return &AwsEc2Res{}
}
// Validate if the params passed in are valid data.
@@ -314,11 +315,13 @@ func (obj *AwsEc2Res) Validate() error {
return fmt.Errorf("you must set watchendpoint with watchlistenaddr to use http watch")
}
return obj.BaseRes.Validate()
return nil
}
// Init initializes the resource.
func (obj *AwsEc2Res) Init() error {
func (obj *AwsEc2Res) Init(init *engine.Init) error {
obj.init = init // save for later
// create a client session for the AWS API
sess, err := session.NewSession(&aws.Config{
Region: aws.String(obj.Region),
@@ -380,7 +383,30 @@ func (obj *AwsEc2Res) Init() error {
}
}
return obj.BaseRes.Init() // call base init, b/c we're overriding
return nil
}
// Close cleans up when we're done. This is needed to delete some of the AWS
// objects created for the SNS endpoint.
func (obj *AwsEc2Res) Close() error {
var errList error
// clean up sns objects created by Init/snsWatch
if obj.snsClient != nil {
// delete the topic and associated subscriptions
if err := obj.snsDeleteTopic(obj.snsTopicArn); err != nil {
errList = multierr.Append(errList, err)
}
// remove the target
if err := obj.cweRemoveTarget(CweTargetID, CweRuleName); err != nil {
errList = multierr.Append(errList, err)
}
// delete the cloudwatch rule
if err := obj.cweDeleteRule(CweRuleName); err != nil {
errList = multierr.Append(errList, err)
}
}
return errList
}
// Watch is the primary listener for this resource and it outputs events.
@@ -394,12 +420,11 @@ func (obj *AwsEc2Res) Watch() error {
// longpollWatch uses the ec2 api's built in methods to watch ec2 resource state.
func (obj *AwsEc2Res) longpollWatch() error {
send := false
var exit *error
// We tell the engine that we're running right away. This is not correct,
// but the api doesn't have a way to signal when the waiters are ready.
if err := obj.Running(); err != nil {
return err
if err := obj.init.Running(); err != nil {
return err // exit if requested
}
// cancellable context used for exiting cleanly
@@ -463,10 +488,14 @@ func (obj *AwsEc2Res) longpollWatch() error {
// process events from the goroutine
for {
select {
case event := <-obj.Events():
if exit, send = obj.ReadEvent(event); exit != nil {
return *exit
case event, ok := <-obj.init.Events:
if !ok {
return nil
}
if err := obj.init.Read(event); err != nil {
return err
}
case msg, ok := <-obj.awsChan:
if !ok {
return nil
@@ -479,105 +508,20 @@ func (obj *AwsEc2Res) longpollWatch() error {
case "", ec2.InstanceStateNamePending, ec2.InstanceStateNameStopping:
continue
default:
log.Printf("%s: State: %v", obj, msg.state)
obj.StateOK(false)
obj.init.Logf("State: %v", msg.state)
obj.init.Dirty() // dirty
send = true
}
}
if send {
send = false
obj.Event()
if err := obj.init.Event(); err != nil {
return err // exit if requested
}
}
}
}
// stateWaiter waits for an instance to change state and returns the new state.
func stateWaiter(ctx context.Context, instance *ec2.Instance, c *ec2.EC2) (string, error) {
var err error
var name string
// these cases are not permitted
if instance == nil {
return "", fmt.Errorf("nil instance")
}
if aws.StringValue(instance.State.Name) == "" {
return "", fmt.Errorf("nil or empty state")
}
// get the instance name
for _, tag := range instance.Tags {
if aws.StringValue(tag.Key) == nameKey {
name = aws.StringValue(tag.Value)
}
}
// error if we didn't find one
if name == "" {
return "", fmt.Errorf("name not found")
}
// build the input for the waiters
waitInput := &ec2.DescribeInstancesInput{
InstanceIds: []*string{instance.InstanceId},
Filters: []*ec2.Filter{
{
Name: aws.String(nameTag),
Values: []*string{aws.String(name)},
},
},
}
// When we are watching terminated instances and waiting for them to exist,
// we must exclude terminated instances from the waiter input. If we don't,
// the waiter will return even if it finds a terminated instance, which is
// not what we want.
existWaiterFilter := &ec2.Filter{
Name: aws.String("instance-state-name"),
Values: []*string{
aws.String(ec2.InstanceStateNameRunning),
aws.String(ec2.InstanceStateNameStopped),
},
}
// Select the appropriate waiter based on the instance state. There are
// five possible states and we will catch every pertinent state change
// (excluding transitional states) by waiting for the next state in the
// instance's lifecycle. For more information about the lifecycle, see:
// http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-lifecycle.html
switch aws.StringValue(instance.State.Name) {
case ec2.InstanceStateNameRunning, ec2.InstanceStateNameStopping:
err = c.WaitUntilInstanceStoppedWithContext(ctx, waitInput)
case ec2.InstanceStateNameStopped, ec2.InstanceStateNamePending:
err = c.WaitUntilInstanceRunningWithContext(ctx, waitInput)
case ec2.InstanceStateNameTerminated:
waitInput.Filters = append(waitInput.Filters, existWaiterFilter)
err = c.WaitUntilInstanceExistsWithContext(ctx, waitInput)
default:
return "", fmt.Errorf("unrecognized instance state: %s", aws.StringValue(instance.State.Name))
}
if err != nil {
aerr, ok := err.(awserr.Error)
if !ok {
return "", errwrap.Wrapf(err, "error casting awserr")
}
// ignore these errors
if aerr.Code() != request.CanceledErrorCode && aerr.Code() != request.WaiterResourceNotReadyErrorCode {
return "", errwrap.Wrapf(err, "internal waiter error")
}
// If the waiter returns, because it has exceeded the maximum number of
// attempts we return an empty state, which the event processing loop
// ignores, and the longpollWatch goroutine will loop and restart
// the waiter.
if aerr.Message() == AwsErrExceededWaitAttempts {
return "", nil
}
}
// return the instance state
instance, err = describeInstanceByName(c, name)
if err != nil {
return "", errwrap.Wrapf(err, "error describing instances")
}
return aws.StringValue(instance.State.Name), nil
}
// snsWatch uses amazon's SNS and CloudWatchEvents APIs to get instance state-
// change notifications pushed to the http endpoint (snsServer) set up below.
// In Init() a CloudWatch rule is created along with a corresponding SNS topic
@@ -585,7 +529,6 @@ func stateWaiter(ctx context.Context, instance *ec2.Instance, c *ec2.EC2) (strin
// messages published to the topic and processes them accordingly.
func (obj *AwsEc2Res) snsWatch() error {
send := false
var exit *error
defer obj.wg.Wait()
// create the sns listener
// closing is handled by http.Server.Shutdown in the defer func below
@@ -603,10 +546,10 @@ func (obj *AwsEc2Res) snsWatch() error {
defer cancel()
if err := snsServer.Shutdown(ctx); err != nil {
if err != context.Canceled {
log.Printf("%s: error stopping sns endpoint: %s", obj, err)
obj.init.Logf("error stopping sns endpoint: %s", err)
return
}
log.Printf("%s: sns server shutdown cancelled", obj)
obj.init.Logf("sns server shutdown cancelled")
}
}()
defer close(obj.closeChan)
@@ -618,7 +561,7 @@ func (obj *AwsEc2Res) snsWatch() error {
if err := snsServer.Serve(listener); err != nil {
// when we shut down
if err == http.ErrServerClosed {
log.Printf("%s: Stopped SNS Endpoint", obj)
obj.init.Logf("Stopped SNS Endpoint")
return
}
// any other error
@@ -630,7 +573,7 @@ func (obj *AwsEc2Res) snsWatch() error {
}
}
}()
log.Printf("%s: Started SNS Endpoint", obj)
obj.init.Logf("Started SNS Endpoint")
// Subscribing the endpoint to the topic needs to happen after starting
// the http server, so that the server can process the subscription
// confirmation. We won't drop incoming connections from aws by this
@@ -644,10 +587,14 @@ func (obj *AwsEc2Res) snsWatch() error {
// process events
for {
select {
case event := <-obj.Events():
if exit, send = obj.ReadEvent(event); exit != nil {
return *exit
case event, ok := <-obj.init.Events:
if !ok {
return nil
}
if err := obj.init.Read(event); err != nil {
return err
}
case msg, ok := <-obj.awsChan:
if !ok {
return nil
@@ -660,25 +607,27 @@ func (obj *AwsEc2Res) snsWatch() error {
// is confirmed, we are ready to receive events, so we
// can notify the engine that we're running.
if msg.event == awsEc2EventWatchReady {
if err := obj.Running(); err != nil {
return err
if err := obj.init.Running(); err != nil {
return err // exit if requested
}
continue
}
log.Printf("%s: State: %v", obj, msg.event)
obj.StateOK(false)
obj.init.Logf("State: %v", msg.event)
obj.init.Dirty() // dirty
send = true
}
if send {
send = false
obj.Event()
if err := obj.init.Event(); err != nil {
return err // exit if requested
}
}
}
}
// CheckApply method for AwsEc2 resource.
func (obj *AwsEc2Res) CheckApply(apply bool) (checkOK bool, err error) {
log.Printf("%s: CheckApply(%t)", obj, apply)
obj.init.Logf("CheckApply(%t)", apply)
// find the instance we need to check
instance, err := describeInstanceByName(obj.client, obj.prependName())
@@ -822,44 +771,22 @@ func (obj *AwsEc2Res) CheckApply(apply bool) (checkOK bool, err error) {
return false, nil
}
// AwsEc2UID is the UID struct for AwsEc2Res.
type AwsEc2UID struct {
BaseUID
name string
}
// UIDs includes all params to make a unique identification of this object.
// Most resources only return one, although some resources can return multiple.
func (obj *AwsEc2Res) UIDs() []ResUID {
x := &AwsEc2UID{
BaseUID: BaseUID{Name: obj.GetName(), Kind: obj.GetKind()},
name: obj.Name,
// Cmp compares two resources and returns an error if they are not equivalent.
func (obj *AwsEc2Res) Cmp(r engine.Res) error {
if !obj.Compare(r) {
return fmt.Errorf("did not compare")
}
return []ResUID{x}
}
// GroupCmp returns whether two resources can be grouped together or not.
func (obj *AwsEc2Res) GroupCmp(r Res) bool {
_, ok := r.(*AwsEc2Res)
if !ok {
return false
}
return false
return nil
}
// Compare two resources and return if they are equivalent.
func (obj *AwsEc2Res) Compare(r Res) bool {
func (obj *AwsEc2Res) Compare(r engine.Res) bool {
// we can only compare AwsEc2Res to others of the same resource kind
res, ok := r.(*AwsEc2Res)
if !ok {
return false
}
if !obj.BaseRes.Compare(res) { // call base Compare
return false
}
if obj.Name != res.Name {
return false
}
if obj.State != res.State {
return false
}
@@ -887,6 +814,27 @@ func (obj *AwsEc2Res) Compare(r Res) bool {
return true
}
func (obj *AwsEc2Res) prependName() string {
return AwsPrefix + obj.Name()
}
// AwsEc2UID is the UID struct for AwsEc2Res.
type AwsEc2UID struct {
engine.BaseUID
name string
}
// UIDs includes all params to make a unique identification of this object.
// Most resources only return one, although some resources can return multiple.
func (obj *AwsEc2Res) UIDs() []engine.ResUID {
x := &AwsEc2UID{
BaseUID: engine.BaseUID{Name: obj.Name(), Kind: obj.Kind()},
name: obj.Name(),
}
return []engine.ResUID{x}
}
// UnmarshalYAML is the custom unmarshal handler for this struct.
// It is primarily useful for setting the defaults.
func (obj *AwsEc2Res) UnmarshalYAML(unmarshal func(interface{}) error) error {
@@ -907,90 +855,6 @@ func (obj *AwsEc2Res) UnmarshalYAML(unmarshal func(interface{}) error) error {
return nil
}
func (obj *AwsEc2Res) prependName() string {
return AwsPrefix + obj.GetName()
}
// describeInstanceByName takes an ec2 client session and an instance name, and
// returns a *ec2.Instance or an error.
func describeInstanceByName(c *ec2.EC2, name string) (*ec2.Instance, error) {
// get any instance with the specified name, that isn't terminated.
diInput := &ec2.DescribeInstancesInput{
Filters: []*ec2.Filter{
{
Name: aws.String(nameTag),
Values: []*string{aws.String(name)},
},
{
Name: aws.String("instance-state-name"),
Values: []*string{
aws.String(ec2.InstanceStateNameRunning),
aws.String(ec2.InstanceStateNamePending),
aws.String(ec2.InstanceStateNameStopped),
aws.String(ec2.InstanceStateNameStopping),
},
},
},
}
diOutput, err := c.DescribeInstances(diInput)
if err != nil {
return nil, errwrap.Wrapf(err, "error describing instances")
}
// error if we get more than one reservation.
if len(diOutput.Reservations) > 1 {
return nil, fmt.Errorf("too many reservations")
}
// error if we got a reservation without exactly one instance.
if len(diOutput.Reservations) != 0 && len(diOutput.Reservations[0].Instances) != 1 {
return nil, fmt.Errorf("wrong number of instances")
}
// if we didn't find an instance, we consider it 'terminated'.
if len(diOutput.Reservations) == 0 {
return &ec2.Instance{
State: &ec2.InstanceState{
Name: aws.String(ec2.InstanceStateNameTerminated),
},
Tags: []*ec2.Tag{
{
Key: aws.String(nameKey),
Value: aws.String(name),
},
},
}, nil
}
return diOutput.Reservations[0].Instances[0], nil
}
// describeInstanceByID takes an ec2 client session and a pointer to an
// instanceID, and returns an *ec2.Instance or an error.
func describeInstanceByID(c *ec2.EC2, instanceID *string) (*ec2.Instance, error) {
if instanceID == nil {
return nil, fmt.Errorf("instanceID is nil")
}
// get any instance with the specified instanceID.
diInput := &ec2.DescribeInstancesInput{
InstanceIds: []*string{instanceID},
}
diOutput, err := c.DescribeInstances(diInput)
if err != nil {
return nil, errwrap.Wrapf(err, "error describing instances")
}
// error if we didn't find exactly one reservation with one instance.
if len(diOutput.Reservations) != 1 {
return nil, fmt.Errorf("wrong number of reservations")
}
if len(diOutput.Reservations[0].Instances) != 1 {
return nil, fmt.Errorf("wrong number of instances")
}
return diOutput.Reservations[0].Instances[0], nil
}
// snsListener returns a listener bound to listenAddr.
func (obj *AwsEc2Res) snsListener(listenAddr string) (net.Listener, error) {
addr := listenAddr
@@ -1017,7 +881,7 @@ func (obj *AwsEc2Res) snsPostHandler(w http.ResponseWriter, req *http.Request) {
decoder := json.NewDecoder(req.Body)
var post postData
if err := decoder.Decode(&post); err != nil {
log.Printf("%s: error decoding post: %s", obj, err)
obj.init.Logf("error decoding post: %s", err)
http.Error(w, "Bad request", http.StatusBadRequest)
if obj.ErrorOnMalformedPost {
select {
@@ -1032,7 +896,7 @@ func (obj *AwsEc2Res) snsPostHandler(w http.ResponseWriter, req *http.Request) {
// Verify the x509 signature. If there is an error verifying the
// signature, we print the error, ignore the event and return.
if err := obj.snsVerifySignature(post); err != nil {
log.Printf("%s: error verifying signature: %s", obj, err)
obj.init.Logf("error verifying signature: %s", err)
http.Error(w, "Bad request", http.StatusBadRequest)
return
}
@@ -1181,7 +1045,7 @@ func (obj *AwsEc2Res) snsMakeTopic() (string, error) {
if err != nil {
return "", err
}
log.Printf("%s: Created SNS Topic", obj)
obj.init.Logf("Created SNS Topic")
if topic.TopicArn == nil {
return "", fmt.Errorf("TopicArn is nil")
}
@@ -1197,7 +1061,7 @@ func (obj *AwsEc2Res) snsDeleteTopic(topicArn string) error {
if _, err := obj.snsClient.DeleteTopic(dtInput); err != nil {
return err
}
log.Printf("%s: Deleted SNS Topic", obj)
obj.init.Logf("Deleted SNS Topic")
return nil
}
@@ -1214,7 +1078,7 @@ func (obj *AwsEc2Res) snsSubscribe(endpoint string, topicArn string) error {
if err != nil {
return err
}
log.Printf("%s: Created Subscription", obj)
obj.init.Logf("Created Subscription")
return nil
}
@@ -1230,7 +1094,7 @@ func (obj *AwsEc2Res) snsConfirmSubscription(topicArn string, token string) erro
if err != nil {
return err
}
log.Printf("%s: Subscription Confirmed", obj)
obj.init.Logf("Subscription Confirmed")
return nil
}
@@ -1306,7 +1170,7 @@ func (obj *AwsEc2Res) snsAuthorizeCloudWatch(topicArn string) error {
for _, statement := range policy.Statement {
if statement == permission {
// if it's already there, we're done
log.Printf("%s: Target Already Authorized", obj)
obj.init.Logf("Target Already Authorized")
return nil
}
}
@@ -1328,7 +1192,7 @@ func (obj *AwsEc2Res) snsAuthorizeCloudWatch(topicArn string) error {
if err != nil {
return err
}
log.Printf("%s: Authorized Target", obj)
obj.init.Logf("Authorized Target")
return nil
}
@@ -1358,7 +1222,7 @@ func (obj *AwsEc2Res) cweMakeRule(name, eventPattern string) error {
if _, err := obj.cweClient.PutRule(putRuleInput); err != nil {
return err
}
log.Printf("%s: Created CloudWatch Rule", obj)
obj.init.Logf("Created CloudWatch Rule")
return nil
}
@@ -1368,7 +1232,7 @@ func (obj *AwsEc2Res) cweDeleteRule(name string) error {
drInput := &cwe.DeleteRuleInput{
Name: aws.String(name),
}
log.Printf("%s: Deleting CloudWatch Rule", obj)
obj.init.Logf("Deleting CloudWatch Rule")
if _, err := obj.cweClient.DeleteRule(drInput); err != nil {
return errwrap.Wrapf(err, "error deleting cloudwatch rule")
}
@@ -1391,7 +1255,7 @@ func (obj *AwsEc2Res) cweTargetRule(topicArn, targetID, inputPath, ruleName stri
if err != nil {
return errwrap.Wrapf(err, "error putting cloudwatch target")
}
log.Printf("%s: Targeted SNS Topic", obj)
obj.init.Logf("Targeted SNS Topic")
return nil
}
@@ -1402,34 +1266,176 @@ func (obj *AwsEc2Res) cweRemoveTarget(targetID, ruleName string) error {
Ids: []*string{aws.String(targetID)},
Rule: aws.String(ruleName),
}
log.Printf("%s: Removing Target", obj)
obj.init.Logf("Removing Target")
if _, err := obj.cweClient.RemoveTargets(rtInput); err != nil {
return errwrap.Wrapf(err, "error removing cloudwatch target")
}
return nil
}
// Close cleans up when we're done. This is needed to delete some of the AWS
// objects created for the SNS endpoint.
func (obj *AwsEc2Res) Close() error {
var errList error
// clean up sns objects created by Init/snsWatch
if obj.snsClient != nil {
// delete the topic and associated subscriptions
if err := obj.snsDeleteTopic(obj.snsTopicArn); err != nil {
errList = multierr.Append(errList, err)
}
// remove the target
if err := obj.cweRemoveTarget(CweTargetID, CweRuleName); err != nil {
errList = multierr.Append(errList, err)
}
// delete the cloudwatch rule
if err := obj.cweDeleteRule(CweRuleName); err != nil {
errList = multierr.Append(errList, err)
// stateWaiter waits for an instance to change state and returns the new state.
func stateWaiter(ctx context.Context, instance *ec2.Instance, c *ec2.EC2) (string, error) {
var err error
var name string
// these cases are not permitted
if instance == nil {
return "", fmt.Errorf("nil instance")
}
if aws.StringValue(instance.State.Name) == "" {
return "", fmt.Errorf("nil or empty state")
}
// get the instance name
for _, tag := range instance.Tags {
if aws.StringValue(tag.Key) == nameKey {
name = aws.StringValue(tag.Value)
}
}
if err := obj.BaseRes.Close(); err != nil {
errList = multierr.Append(errList, err) // list of errors
// error if we didn't find one
if name == "" {
return "", fmt.Errorf("name not found")
}
return errList
// build the input for the waiters
waitInput := &ec2.DescribeInstancesInput{
InstanceIds: []*string{instance.InstanceId},
Filters: []*ec2.Filter{
{
Name: aws.String(nameTag),
Values: []*string{aws.String(name)},
},
},
}
// When we are watching terminated instances and waiting for them to exist,
// we must exclude terminated instances from the waiter input. If we don't,
// the waiter will return even if it finds a terminated instance, which is
// not what we want.
existWaiterFilter := &ec2.Filter{
Name: aws.String("instance-state-name"),
Values: []*string{
aws.String(ec2.InstanceStateNameRunning),
aws.String(ec2.InstanceStateNameStopped),
},
}
// Select the appropriate waiter based on the instance state. There are
// five possible states and we will catch every pertinent state change
// (excluding transitional states) by waiting for the next state in the
// instance's lifecycle. For more information about the lifecycle, see:
// http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-lifecycle.html
switch aws.StringValue(instance.State.Name) {
case ec2.InstanceStateNameRunning, ec2.InstanceStateNameStopping:
err = c.WaitUntilInstanceStoppedWithContext(ctx, waitInput)
case ec2.InstanceStateNameStopped, ec2.InstanceStateNamePending:
err = c.WaitUntilInstanceRunningWithContext(ctx, waitInput)
case ec2.InstanceStateNameTerminated:
waitInput.Filters = append(waitInput.Filters, existWaiterFilter)
err = c.WaitUntilInstanceExistsWithContext(ctx, waitInput)
default:
return "", fmt.Errorf("unrecognized instance state: %s", aws.StringValue(instance.State.Name))
}
if err != nil {
aerr, ok := err.(awserr.Error)
if !ok {
return "", errwrap.Wrapf(err, "error casting awserr")
}
// ignore these errors
if aerr.Code() != request.CanceledErrorCode && aerr.Code() != request.WaiterResourceNotReadyErrorCode {
return "", errwrap.Wrapf(err, "internal waiter error")
}
// If the waiter returns, because it has exceeded the maximum number of
// attempts we return an empty state, which the event processing loop
// ignores, and the longpollWatch goroutine will loop and restart
// the waiter.
if aerr.Message() == AwsErrExceededWaitAttempts {
return "", nil
}
}
// return the instance state
instance, err = describeInstanceByName(c, name)
if err != nil {
return "", errwrap.Wrapf(err, "error describing instances")
}
return aws.StringValue(instance.State.Name), nil
}
// describeInstanceByName takes an ec2 client session and an instance name, and
// returns a *ec2.Instance or an error.
func describeInstanceByName(c *ec2.EC2, name string) (*ec2.Instance, error) {
// get any instance with the specified name, that isn't terminated.
diInput := &ec2.DescribeInstancesInput{
Filters: []*ec2.Filter{
{
Name: aws.String(nameTag),
Values: []*string{aws.String(name)},
},
{
Name: aws.String("instance-state-name"),
Values: []*string{
aws.String(ec2.InstanceStateNameRunning),
aws.String(ec2.InstanceStateNamePending),
aws.String(ec2.InstanceStateNameStopped),
aws.String(ec2.InstanceStateNameStopping),
},
},
},
}
diOutput, err := c.DescribeInstances(diInput)
if err != nil {
return nil, errwrap.Wrapf(err, "error describing instances")
}
// error if we get more than one reservation.
if len(diOutput.Reservations) > 1 {
return nil, fmt.Errorf("too many reservations")
}
// error if we got a reservation without exactly one instance.
if len(diOutput.Reservations) != 0 && len(diOutput.Reservations[0].Instances) != 1 {
return nil, fmt.Errorf("wrong number of instances")
}
// if we didn't find an instance, we consider it 'terminated'.
if len(diOutput.Reservations) == 0 {
return &ec2.Instance{
State: &ec2.InstanceState{
Name: aws.String(ec2.InstanceStateNameTerminated),
},
Tags: []*ec2.Tag{
{
Key: aws.String(nameKey),
Value: aws.String(name),
},
},
}, nil
}
return diOutput.Reservations[0].Instances[0], nil
}
// describeInstanceByID takes an ec2 client session and a pointer to an
// instanceID, and returns an *ec2.Instance or an error.
func describeInstanceByID(c *ec2.EC2, instanceID *string) (*ec2.Instance, error) {
if instanceID == nil {
return nil, fmt.Errorf("instanceID is nil")
}
// get any instance with the specified instanceID.
diInput := &ec2.DescribeInstancesInput{
InstanceIds: []*string{instanceID},
}
diOutput, err := c.DescribeInstances(diInput)
if err != nil {
return nil, errwrap.Wrapf(err, "error describing instances")
}
// error if we didn't find exactly one reservation with one instance.
if len(diOutput.Reservations) != 1 {
return nil, fmt.Errorf("wrong number of reservations")
}
if len(diOutput.Reservations[0].Instances) != 1 {
return nil, fmt.Errorf("wrong number of instances")
}
return diOutput.Reservations[0].Instances[0], nil
}

408
resources/exec.go → engine/resources/exec.go
View File

@@ -21,25 +21,31 @@ import (
"bufio"
"bytes"
"fmt"
"log"
"os/exec"
"os/user"
"strings"
"sync"
"syscall"
"github.com/purpleidea/mgmt/engine"
"github.com/purpleidea/mgmt/engine/traits"
engineUtil "github.com/purpleidea/mgmt/engine/util"
"github.com/purpleidea/mgmt/util"
errwrap "github.com/pkg/errors"
)
func init() {
RegisterResource("exec", func() Res { return &ExecRes{} })
engine.RegisterResource("exec", func() engine.Res { return &ExecRes{} })
}
// ExecRes is an exec resource for running commands.
type ExecRes struct {
BaseRes `yaml:",inline"`
traits.Base // add the base methods without re-implementation
traits.Edgeable
init *engine.Init
Cmd string `yaml:"cmd"` // the command to run
Shell string `yaml:"shell"` // the (optional) shell to use to run the cmd
Timeout int `yaml:"timeout"` // the cmd timeout in seconds
@@ -52,15 +58,13 @@ type ExecRes struct {
Output *string // all cmd output, read only, do not set!
Stdout *string // the cmd stdout, read only, do not set!
Stderr *string // the cmd stderr, read only, do not set!
wg *sync.WaitGroup
}
// Default returns some sensible defaults for this resource.
func (obj *ExecRes) Default() Res {
return &ExecRes{
BaseRes: BaseRes{
MetaParams: DefaultMetaParams, // force a default
},
}
func (obj *ExecRes) Default() engine.Res {
return &ExecRes{}
}
// Validate if the params passed in are valid data.
@@ -80,37 +84,27 @@ func (obj *ExecRes) Validate() error {
}
}
return obj.BaseRes.Validate()
return nil
}
// Init runs some startup code for this resource.
func (obj *ExecRes) Init() error {
return obj.BaseRes.Init() // call base init, b/c we're overriding
func (obj *ExecRes) Init(init *engine.Init) error {
obj.init = init // save for later
obj.wg = &sync.WaitGroup{}
return nil
}
// BufioChanScanner wraps the scanner output in a channel.
func (obj *ExecRes) BufioChanScanner(scanner *bufio.Scanner) (chan string, chan error) {
ch, errch := make(chan string), make(chan error)
go func() {
for scanner.Scan() {
ch <- scanner.Text() // blocks here ?
if e := scanner.Err(); e != nil {
errch <- e // send any misc errors we encounter
//break // TODO: ?
}
}
close(ch)
errch <- scanner.Err() // eof or some err
close(errch)
}()
return ch, errch
// Close is run by the engine to clean up after the resource is done.
func (obj *ExecRes) Close() error {
return nil
}
// Watch is the primary listener for this resource and it outputs events.
func (obj *ExecRes) Watch() error {
var send = false // send event?
var exit *error
bufioch, errch := make(chan string), make(chan error)
ioChan := make(chan *bufioOutput)
defer obj.wg.Wait()
if obj.WatchCmd != "" {
var cmdName string
@@ -157,43 +151,50 @@ func (obj *ExecRes) Watch() error {
return errwrap.Wrapf(err, "error starting Cmd")
}
bufioch, errch = obj.BufioChanScanner(scanner)
ioChan = obj.bufioChanScanner(scanner)
}
// notify engine that we're running
if err := obj.Running(); err != nil {
return err // bubble up a NACK...
if err := obj.init.Running(); err != nil {
return err // exit if requested
}
var send = false // send event?
for {
select {
case text := <-bufioch:
// each time we get a line of output, we loop!
log.Printf("%s: Watch output: %s", obj, text)
if text != "" {
send = true
obj.StateOK(false) // something made state dirty
}
case err := <-errch:
if err == nil { // EOF
case data, ok := <-ioChan:
if !ok { // EOF
// FIXME: add an "if watch command ends/crashes"
// restart or generate error option
return fmt.Errorf("reached EOF")
}
// error reading input?
return errwrap.Wrapf(err, "unknown error")
if err := data.err; err != nil {
// error reading input?
return errwrap.Wrapf(err, "unknown error")
}
case event := <-obj.Events():
if exit, send = obj.ReadEvent(event); exit != nil {
return *exit // exit
// each time we get a line of output, we loop!
obj.init.Logf("watch output: %s", data.text)
if data.text != "" {
send = true
obj.init.Dirty() // dirty
}
case event, ok := <-obj.init.Events:
if !ok {
return nil
}
if err := obj.init.Read(event); err != nil {
return err
}
}
// do all our event sending all together to avoid duplicate msgs
if send {
send = false
obj.Event()
if err := obj.init.Event(); err != nil {
return err // exit if requested
}
}
}
}
@@ -248,7 +249,7 @@ func (obj *ExecRes) CheckApply(apply bool) (bool, error) {
}
// apply portion
log.Printf("%s: Apply", obj)
obj.init.Logf("Apply")
var cmdName string
var cmdArgs []string
if obj.Shell == "" {
@@ -326,25 +327,23 @@ func (obj *ExecRes) CheckApply(apply bool) (bool, error) {
pStateSys := exitErr.Sys() // (*os.ProcessState) Sys
wStatus, ok := pStateSys.(syscall.WaitStatus)
if !ok {
e := errwrap.Wrapf(err, "error running cmd")
return false, e
return false, errwrap.Wrapf(err, "error running cmd")
}
return false, fmt.Errorf("cmd error, exit status: %d", wStatus.ExitStatus())
} else if err != nil {
e := errwrap.Wrapf(err, "general cmd error")
return false, e
return false, errwrap.Wrapf(err, "general cmd error")
}
// TODO: if we printed the stdout while the command is running, this
// would be nice, but it would require terminal log output that doesn't
// interleave all the parallel parts which would mix it all up...
if s := out.String(); s == "" {
log.Printf("%s: Command output is empty!", obj)
obj.init.Logf("Command output is empty!")
} else {
log.Printf("%s: Command output is:", obj)
log.Printf(out.String())
obj.init.Logf("Command output is:")
obj.init.Logf(out.String())
}
// The state tracking is for exec resources that can't "detect" their
@@ -355,83 +354,21 @@ func (obj *ExecRes) CheckApply(apply bool) (bool, error) {
return false, nil // success
}
// ExecUID is the UID struct for ExecRes.
type ExecUID struct {
BaseUID
Cmd string
IfCmd string
// TODO: add more elements here
}
// ExecResAutoEdges holds the state of the auto edge generator.
type ExecResAutoEdges struct {
edges []ResUID
}
// Next returns the next automatic edge.
func (obj *ExecResAutoEdges) Next() []ResUID {
return obj.edges
}
// Test gets results of the earlier Next() call, & returns if we should continue!
func (obj *ExecResAutoEdges) Test(input []bool) bool {
return false // Never keep going
// TODO: We could return false if we find as many edges as the number of different path in cmdFiles()
}
// AutoEdges returns the AutoEdge interface. In this case the systemd units.
func (obj *ExecRes) AutoEdges() (AutoEdge, error) {
var data []ResUID
for _, x := range obj.cmdFiles() {
var reversed = true
data = append(data, &PkgFileUID{
BaseUID: BaseUID{
Name: obj.GetName(),
Kind: obj.GetKind(),
Reversed: &reversed,
},
path: x, // what matters
})
// Cmp compares two resources and returns an error if they are not equivalent.
func (obj *ExecRes) Cmp(r engine.Res) error {
if !obj.Compare(r) {
return fmt.Errorf("did not compare")
}
return &ExecResAutoEdges{
edges: data,
}, nil
}
// UIDs includes all params to make a unique identification of this object.
// Most resources only return one, although some resources can return multiple.
func (obj *ExecRes) UIDs() []ResUID {
x := &ExecUID{
BaseUID: BaseUID{Name: obj.GetName(), Kind: obj.GetKind()},
Cmd: obj.Cmd,
IfCmd: obj.IfCmd,
// TODO: add more params here
}
return []ResUID{x}
}
// GroupCmp returns whether two resources can be grouped together or not.
func (obj *ExecRes) GroupCmp(r Res) bool {
_, ok := r.(*ExecRes)
if !ok {
return false
}
return false // not possible atm
return nil
}
// Compare two resources and return if they are equivalent.
func (obj *ExecRes) Compare(r Res) bool {
func (obj *ExecRes) Compare(r engine.Res) bool {
// we can only compare ExecRes to others of the same resource kind
res, ok := r.(*ExecRes)
if !ok {
return false
}
if !obj.BaseRes.Compare(res) { // call base Compare
return false
}
if obj.Name != res.Name {
return false
}
if obj.Cmd != res.Cmd {
return false
@@ -464,6 +401,61 @@ func (obj *ExecRes) Compare(r Res) bool {
return true
}
// ExecUID is the UID struct for ExecRes.
type ExecUID struct {
engine.BaseUID
Cmd string
IfCmd string
// TODO: add more elements here
}
// ExecResAutoEdges holds the state of the auto edge generator.
type ExecResAutoEdges struct {
edges []engine.ResUID
}
// Next returns the next automatic edge.
func (obj *ExecResAutoEdges) Next() []engine.ResUID {
return obj.edges
}
// Test gets results of the earlier Next() call, & returns if we should continue!
func (obj *ExecResAutoEdges) Test(input []bool) bool {
return false // never keep going
// TODO: we could return false if we find as many edges as the number of different path's in cmdFiles()
}
// AutoEdges returns the AutoEdge interface. In this case the systemd units.
func (obj *ExecRes) AutoEdges() (engine.AutoEdge, error) {
var data []engine.ResUID
for _, x := range obj.cmdFiles() {
var reversed = true
data = append(data, &PkgFileUID{
BaseUID: engine.BaseUID{
Name: obj.Name(),
Kind: obj.Kind(),
Reversed: &reversed,
},
path: x, // what matters
})
}
return &ExecResAutoEdges{
edges: data,
}, nil
}
// UIDs includes all params to make a unique identification of this object.
// Most resources only return one, although some resources can return multiple.
func (obj *ExecRes) UIDs() []engine.ResUID {
x := &ExecUID{
BaseUID: engine.BaseUID{Name: obj.Name(), Kind: obj.Kind()},
Cmd: obj.Cmd,
IfCmd: obj.IfCmd,
// TODO: add more params here
}
return []engine.ResUID{x}
}
// UnmarshalYAML is the custom unmarshal handler for this struct.
// It is primarily useful for setting the defaults.
func (obj *ExecRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
@@ -499,14 +491,14 @@ func (obj *ExecRes) getCredential() (*syscall.Credential, error) {
}
if obj.Group != "" {
gid, err = GetGID(obj.Group)
gid, err = engineUtil.GetGID(obj.Group)
if err != nil {
return nil, errwrap.Wrapf(err, "error looking up gid for %s", obj.Group)
}
}
if obj.User != "" {
uid, err = GetUID(obj.User)
uid, err = engineUtil.GetUID(obj.User)
if err != nil {
return nil, errwrap.Wrapf(err, "error looking up uid for %s", obj.User)
}
@@ -515,74 +507,6 @@ func (obj *ExecRes) getCredential() (*syscall.Credential, error) {
return &syscall.Credential{Uid: uint32(uid), Gid: uint32(gid)}, nil
}
// splitWriter mimics what the ssh.CombinedOutput command does, but stores the
// the stdout and stderr separately. This is slightly tricky because we don't
// want the combined output to be interleaved incorrectly. It creates sub writer
// structs which share the same lock and a shared output buffer.
type splitWriter struct {
Stdout *wrapWriter
Stderr *wrapWriter
stdout bytes.Buffer // just the stdout
stderr bytes.Buffer // just the stderr
output bytes.Buffer // combined output
mutex *sync.Mutex
initialized bool // is this initialized?
}
// Init initializes the splitWriter.
func (sw *splitWriter) Init() {
if sw.initialized {
panic("splitWriter is already initialized")
}
sw.mutex = &sync.Mutex{}
sw.Stdout = &wrapWriter{
Mutex: sw.mutex,
Buffer: &sw.stdout,
Output: &sw.output,
}
sw.Stderr = &wrapWriter{
Mutex: sw.mutex,
Buffer: &sw.stderr,
Output: &sw.output,
}
sw.initialized = true
}
// String returns the contents of the combined output buffer.
func (sw *splitWriter) String() string {
if !sw.initialized {
panic("splitWriter is not initialized")
}
return sw.output.String()
}
// wrapWriter is a simple writer which is used internally by splitWriter.
type wrapWriter struct {
Mutex *sync.Mutex
Buffer *bytes.Buffer // stdout or stderr
Output *bytes.Buffer // combined output
Activity bool // did we get any writes?
}
// Write writes to both bytes buffers with a parent lock to mix output safely.
func (w *wrapWriter) Write(p []byte) (int, error) {
// TODO: can we move the lock to only guard around the Output.Write ?
w.Mutex.Lock()
defer w.Mutex.Unlock()
w.Activity = true
i, err := w.Buffer.Write(p) // first write
if err != nil {
return i, err
}
return w.Output.Write(p) // shared write
}
// String returns the contents of the unshared buffer.
func (w *wrapWriter) String() string {
return w.Buffer.String()
}
// cmdFiles returns all the potential files/commands this command might need.
func (obj *ExecRes) cmdFiles() []string {
var paths []string
@@ -603,3 +527,95 @@ func (obj *ExecRes) cmdFiles() []string {
}
return paths
}
// bufioOutput is the output struct of the bufioChanScanner channel output.
type bufioOutput struct {
text string
err error
}
// bufioChanScanner wraps the scanner output in a channel.
func (obj *ExecRes) bufioChanScanner(scanner *bufio.Scanner) chan *bufioOutput {
ch := make(chan *bufioOutput)
obj.wg.Add(1)
go func() {
defer obj.wg.Done()
defer close(ch)
for scanner.Scan() {
ch <- &bufioOutput{text: scanner.Text()} // blocks here ?
}
// on EOF, scanner.Err() will be nil
if err := scanner.Err(); err != nil {
ch <- &bufioOutput{err: err} // send any misc errors we encounter
}
}()
return ch
}
// splitWriter mimics what the ssh.CombinedOutput command does, but stores the
// the stdout and stderr separately. This is slightly tricky because we don't
// want the combined output to be interleaved incorrectly. It creates sub writer
// structs which share the same lock and a shared output buffer.
type splitWriter struct {
Stdout *wrapWriter
Stderr *wrapWriter
stdout bytes.Buffer // just the stdout
stderr bytes.Buffer // just the stderr
output bytes.Buffer // combined output
mutex *sync.Mutex
initialized bool // is this initialized?
}
// Init initializes the splitWriter.
func (obj *splitWriter) Init() {
if obj.initialized {
panic("splitWriter is already initialized")
}
obj.mutex = &sync.Mutex{}
obj.Stdout = &wrapWriter{
Mutex: obj.mutex,
Buffer: &obj.stdout,
Output: &obj.output,
}
obj.Stderr = &wrapWriter{
Mutex: obj.mutex,
Buffer: &obj.stderr,
Output: &obj.output,
}
obj.initialized = true
}
// String returns the contents of the combined output buffer.
func (obj *splitWriter) String() string {
if !obj.initialized {
panic("splitWriter is not initialized")
}
return obj.output.String()
}
// wrapWriter is a simple writer which is used internally by splitWriter.
type wrapWriter struct {
Mutex *sync.Mutex
Buffer *bytes.Buffer // stdout or stderr
Output *bytes.Buffer // combined output
Activity bool // did we get any writes?
}
// Write writes to both bytes buffers with a parent lock to mix output safely.
func (obj *wrapWriter) Write(p []byte) (int, error) {
// TODO: can we move the lock to only guard around the Output.Write ?
obj.Mutex.Lock()
defer obj.Mutex.Unlock()
obj.Activity = true
i, err := obj.Buffer.Write(p) // first write
if err != nil {
return i, err
}
return obj.Output.Write(p) // shared write
}
// String returns the contents of the unshared buffer.
func (obj *wrapWriter) String() string {
return obj.Buffer.String()
}

40
resources/exec_test.go → engine/resources/exec_test.go
View File

@@ -19,20 +19,30 @@ package resources
import (
"testing"
"github.com/purpleidea/mgmt/engine"
)
func fakeInit(t *testing.T) *engine.Init {
debug := true
logf := func(format string, v ...interface{}) {
t.Logf("test: "+format, v...)
}
return &engine.Init{
Running: func() error {
return nil
},
Debug: debug,
Logf: logf,
}
}
func TestExecSendRecv1(t *testing.T) {
r1 := &ExecRes{
BaseRes: BaseRes{
Name: "exec1",
Kind: "exec",
MetaParams: DefaultMetaParams,
},
Cmd: "echo hello world",
Shell: "/bin/bash",
}
r1.Setup(nil, r1, r1)
if err := r1.Validate(); err != nil {
t.Errorf("validate failed with: %v", err)
}
@@ -41,7 +51,7 @@ func TestExecSendRecv1(t *testing.T) {
t.Errorf("close failed with: %v", err)
}
}()
if err := r1.Init(); err != nil {
if err := r1.Init(fakeInit(t)); err != nil {
t.Errorf("init failed with: %v", err)
}
// run artificially without the entire engine
@@ -73,16 +83,10 @@ func TestExecSendRecv1(t *testing.T) {
func TestExecSendRecv2(t *testing.T) {
r1 := &ExecRes{
BaseRes: BaseRes{
Name: "exec1",
Kind: "exec",
MetaParams: DefaultMetaParams,
},
Cmd: "echo hello world 1>&2", // to stderr
Shell: "/bin/bash",
}
r1.Setup(nil, r1, r1)
if err := r1.Validate(); err != nil {
t.Errorf("validate failed with: %v", err)
}
@@ -91,7 +95,7 @@ func TestExecSendRecv2(t *testing.T) {
t.Errorf("close failed with: %v", err)
}
}()
if err := r1.Init(); err != nil {
if err := r1.Init(fakeInit(t)); err != nil {
t.Errorf("init failed with: %v", err)
}
// run artificially without the entire engine
@@ -123,16 +127,10 @@ func TestExecSendRecv2(t *testing.T) {
func TestExecSendRecv3(t *testing.T) {
r1 := &ExecRes{
BaseRes: BaseRes{
Name: "exec1",
Kind: "exec",
MetaParams: DefaultMetaParams,
},
Cmd: "echo hello world && echo goodbye world 1>&2", // to stdout && stderr
Shell: "/bin/bash",
}
r1.Setup(nil, r1, r1)
if err := r1.Validate(); err != nil {
t.Errorf("validate failed with: %v", err)
}
@@ -141,7 +139,7 @@ func TestExecSendRecv3(t *testing.T) {
t.Errorf("close failed with: %v", err)
}
}()
if err := r1.Init(); err != nil {
if err := r1.Init(fakeInit(t)); err != nil {
t.Errorf("init failed with: %v", err)
}
// run artificially without the entire engine

1507
resources/file.go → engine/resources/file.go
View File

File diff suppressed because it is too large Load Diff

146
engine/resources/file_test.go Normal file
View File

@@ -0,0 +1,146 @@
// Mgmt
// Copyright (C) 2013-2018+ James Shubin and the project contributors
// Written by James Shubin <james@shubin.ca> and the project contributors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package resources
import (
"bytes"
"encoding/base64"
"encoding/gob"
"testing"
"github.com/purpleidea/mgmt/engine"
"github.com/purpleidea/mgmt/engine/graph/autoedge"
"github.com/purpleidea/mgmt/engine/util"
"github.com/purpleidea/mgmt/pgraph"
)
func TestFileAutoEdge1(t *testing.T) {
g, err := pgraph.NewGraph("TestGraph")
if err != nil {
t.Errorf("error creating graph: %v", err)
return
}
r1 := &FileRes{
Path: "/tmp/a/b/", // some dir
}
r2 := &FileRes{
Path: "/tmp/a/", // some parent dir
}
r3 := &FileRes{
Path: "/tmp/a/b/c", // some child file
}
g.AddVertex(r1, r2, r3)
if i := g.NumEdges(); i != 0 {
t.Errorf("should have 0 edges instead of: %d", i)
}
debug := true
logf := func(format string, v ...interface{}) {
t.Logf("test: "+format, v...)
}
// run artificially without the entire engine
if err := autoedge.AutoEdge(g, debug, logf); err != nil {
t.Errorf("error running autoedges: %v", err)
}
// two edges should have been added
if i := g.NumEdges(); i != 2 {
t.Errorf("should have 2 edges instead of: %d", i)
}
}
func TestMiscEncodeDecode1(t *testing.T) {
var err error
// encode
var input interface{} = &FileRes{}
b1 := bytes.Buffer{}
e := gob.NewEncoder(&b1)
err = e.Encode(&input) // pass with &
if err != nil {
t.Errorf("Gob failed to Encode: %v", err)
}
str := base64.StdEncoding.EncodeToString(b1.Bytes())
// decode
var output interface{}
bb, err := base64.StdEncoding.DecodeString(str)
if err != nil {
t.Errorf("Base64 failed to Decode: %v", err)
}
b2 := bytes.NewBuffer(bb)
d := gob.NewDecoder(b2)
err = d.Decode(&output) // pass with &
if err != nil {
t.Errorf("Gob failed to Decode: %v", err)
}
res1, ok := input.(engine.Res)
if !ok {
t.Errorf("Input %v is not a Res", res1)
return
}
res2, ok := output.(engine.Res)
if !ok {
t.Errorf("Output %v is not a Res", res2)
return
}
if err := res1.Cmp(res2); err != nil {
t.Errorf("The input and output Res values do not match: %+v", err)
}
}
func TestMiscEncodeDecode2(t *testing.T) {
var err error
// encode
input, err := engine.NewNamedResource("file", "file1")
if err != nil {
t.Errorf("Can't create: %v", err)
return
}
b64, err := util.ResToB64(input)
if err != nil {
t.Errorf("Can't encode: %v", err)
return
}
output, err := util.B64ToRes(b64)
if err != nil {
t.Errorf("Can't decode: %v", err)
return
}
res1, ok := input.(engine.Res)
if !ok {
t.Errorf("Input %v is not a Res", res1)
return
}
res2, ok := output.(engine.Res)
if !ok {
t.Errorf("Output %v is not a Res", res2)
return
}
if err := res1.Cmp(res2); err != nil {
t.Errorf("The input and output Res values do not match: %+v", err)
}
}

161
resources/group.go → engine/resources/group.go
View File

@@ -20,39 +20,39 @@ package resources
import (
"fmt"
"io/ioutil"
"log"
"os/exec"
"os/user"
"strconv"
"syscall"
"github.com/purpleidea/mgmt/engine"
"github.com/purpleidea/mgmt/engine/traits"
"github.com/purpleidea/mgmt/recwatch"
errwrap "github.com/pkg/errors"
)
func init() {
RegisterResource("group", func() Res { return &GroupRes{} })
engine.RegisterResource("group", func() engine.Res { return &GroupRes{} })
}
const groupFile = "/etc/group"
// GroupRes is a user group resource.
type GroupRes struct {
BaseRes `yaml:",inline"`
State string `yaml:"state"` // state: exists, absent
GID *uint32 `yaml:"gid"` // the group's gid
traits.Base // add the base methods without re-implementation
init *engine.Init
State string `yaml:"state"` // state: exists, absent
GID *uint32 `yaml:"gid"` // the group's gid
recWatcher *recwatch.RecWatcher
}
// Default returns some sensible defaults for this resource.
func (obj *GroupRes) Default() Res {
return &GroupRes{
BaseRes: BaseRes{
MetaParams: DefaultMetaParams, // force a default
},
}
func (obj *GroupRes) Default() engine.Res {
return &GroupRes{}
}
// Validate if the params passed in are valid data.
@@ -60,12 +60,19 @@ func (obj *GroupRes) Validate() error {
if obj.State != "exists" && obj.State != "absent" {
return fmt.Errorf("State must be 'exists' or 'absent'")
}
return obj.BaseRes.Validate()
return nil
}
// Init initializes the resource.
func (obj *GroupRes) Init() error {
return obj.BaseRes.Init() // call base init, b/c we're overriding
// Init runs some startup code for this resource.
func (obj *GroupRes) Init(init *engine.Init) error {
obj.init = init // save for later
return nil
}
// Close is run by the engine to clean up after the resource is done.
func (obj *GroupRes) Close() error {
return nil
}
// Watch is the primary listener for this resource and it outputs events.
@@ -78,16 +85,14 @@ func (obj *GroupRes) Watch() error {
defer obj.recWatcher.Close()
// notify engine that we're running
if err := obj.Running(); err != nil {
return err // bubble up a NACK...
if err := obj.init.Running(); err != nil {
return err // exit if requested
}
var send = false // send event?
var exit *error
for {
if obj.debug {
log.Printf("%s: Watching: %s", obj, groupFile) // attempting to watch...
if obj.init.Debug {
obj.init.Logf("Watching: %s", groupFile) // attempting to watch...
}
select {
@@ -98,34 +103,38 @@ func (obj *GroupRes) Watch() error {
if err := event.Error; err != nil {
return errwrap.Wrapf(err, "Unknown %s watcher error", obj)
}
if obj.debug { // don't access event.Body if event.Error isn't nil
log.Printf("%s: Event(%s): %v", obj, event.Body.Name, event.Body.Op)
if obj.init.Debug { // don't access event.Body if event.Error isn't nil
obj.init.Logf("Event(%s): %v", event.Body.Name, event.Body.Op)
}
send = true
obj.StateOK(false) // dirty
obj.init.Dirty() // dirty
case event := <-obj.Events():
if exit, send = obj.ReadEvent(event); exit != nil {
return *exit // exit
case event, ok := <-obj.init.Events:
if !ok {
return nil
}
if err := obj.init.Read(event); err != nil {
return err
}
//obj.StateOK(false) // dirty // these events don't invalidate state
}
// do all our event sending all together to avoid duplicate msgs
if send {
send = false
obj.Event()
if err := obj.init.Event(); err != nil {
return err // exit if requested
}
}
}
}
// CheckApply method for Group resource.
func (obj *GroupRes) CheckApply(apply bool) (checkOK bool, err error) {
log.Printf("%s: CheckApply(%t)", obj, apply)
obj.init.Logf("CheckApply(%t)", apply)
// check if the group exists
exists := true
group, err := user.LookupGroup(obj.GetName())
group, err := user.LookupGroup(obj.Name())
if err != nil {
if _, ok := err.(user.UnknownGroupError); !ok {
return false, errwrap.Wrapf(err, "error looking up group")
@@ -148,7 +157,7 @@ func (obj *GroupRes) CheckApply(apply bool) (checkOK bool, err error) {
return false, errwrap.Wrapf(err, "error looking up GID")
}
}
if lookupGID != nil && lookupGID.Name != obj.GetName() {
if lookupGID != nil && lookupGID.Name != obj.Name() {
return false, fmt.Errorf("the requested GID belongs to another group")
}
// get the existing group's GID
@@ -159,7 +168,7 @@ func (obj *GroupRes) CheckApply(apply bool) (checkOK bool, err error) {
// check if existing group has the wrong GID
// if it is wrong groupmod will change it to the desired value
if *obj.GID != uint32(existingGID) {
log.Printf("%s: Inconsistent GID: %s", obj, obj.GetName())
obj.init.Logf("Inconsistent GID: %s", obj.Name())
}
// if the group exists and has the correct GID, we are done
if obj.State == "exists" && *obj.GID == uint32(existingGID) {
@@ -172,14 +181,14 @@ func (obj *GroupRes) CheckApply(apply bool) (checkOK bool, err error) {
}
var cmdName string
args := []string{obj.GetName()}
args := []string{obj.Name()}
if obj.State == "exists" {
if exists {
log.Printf("%s: Modifying group: %s", obj, obj.GetName())
obj.init.Logf("Modifying group: %s", obj.Name())
cmdName = "groupmod"
} else {
log.Printf("%s: Adding group: %s", obj, obj.GetName())
obj.init.Logf("Adding group: %s", obj.Name())
cmdName = "groupadd"
}
if obj.GID != nil {
@@ -187,7 +196,7 @@ func (obj *GroupRes) CheckApply(apply bool) (checkOK bool, err error) {
}
}
if obj.State == "absent" && exists {
log.Printf("%s: Deleting group: %s", obj, obj.GetName())
obj.init.Logf("Deleting group: %s", obj.Name())
cmdName = "groupdel"
}
@@ -220,15 +229,45 @@ func (obj *GroupRes) CheckApply(apply bool) (checkOK bool, err error) {
return false, nil
}
// Cmp compares two resources and returns an error if they are not equivalent.
func (obj *GroupRes) Cmp(r engine.Res) error {
if !obj.Compare(r) {
return fmt.Errorf("did not compare")
}
return nil
}
// Compare two resources and return if they are equivalent.
func (obj *GroupRes) Compare(r engine.Res) bool {
// we can only compare GroupRes to others of the same resource kind
res, ok := r.(*GroupRes)
if !ok {
return false
}
if obj.State != res.State {
return false
}
if (obj.GID == nil) != (res.GID == nil) {
return false
}
if obj.GID != nil && res.GID != nil {
if *obj.GID != *res.GID {
return false
}
}
return true
}
// GroupUID is the UID struct for GroupRes.
type GroupUID struct {
BaseUID
engine.BaseUID
name string
gid *uint32
}
// IFF aka if and only if they are equivalent, return true. If not, false.
func (obj *GroupUID) IFF(uid ResUID) bool {
func (obj *GroupUID) IFF(uid engine.ResUID) bool {
res, ok := uid.(*GroupUID)
if !ok {
return false
@@ -248,49 +287,13 @@ func (obj *GroupUID) IFF(uid ResUID) bool {
// UIDs includes all params to make a unique identification of this object.
// Most resources only return one, although some resources can return multiple.
func (obj *GroupRes) UIDs() []ResUID {
func (obj *GroupRes) UIDs() []engine.ResUID {
x := &GroupUID{
BaseUID: BaseUID{Name: obj.GetName(), Kind: obj.GetKind()},
name: obj.Name,
BaseUID: engine.BaseUID{Name: obj.Name(), Kind: obj.Kind()},
name: obj.Name(),
gid: obj.GID,
}
return []ResUID{x}
}
// GroupCmp returns whether two resources can be grouped together or not.
func (obj *GroupRes) GroupCmp(r Res) bool {
_, ok := r.(*GroupRes)
if !ok {
return false
}
return false
}
// Compare two resources and return if they are equivalent.
func (obj *GroupRes) Compare(r Res) bool {
// we can only compare GroupRes to others of the same resource kind
res, ok := r.(*GroupRes)
if !ok {
return false
}
if !obj.BaseRes.Compare(res) { // call base Compare
return false
}
if obj.Name != res.Name {
return false
}
if obj.State != res.State {
return false
}
if (obj.GID == nil) != (res.GID == nil) {
return false
}
if obj.GID != nil && res.GID != nil {
if *obj.GID != *res.GID {
return false
}
}
return true
return []engine.ResUID{x}
}
// UnmarshalYAML is the custom unmarshal handler for this struct.

133
resources/hostname.go → engine/resources/hostname.go
View File

@@ -20,21 +20,17 @@ package resources
import (
"errors"
"fmt"
"log"
"github.com/purpleidea/mgmt/engine"
"github.com/purpleidea/mgmt/engine/traits"
"github.com/purpleidea/mgmt/util"
"github.com/godbus/dbus"
errwrap "github.com/pkg/errors"
)
// ErrResourceInsufficientParameters is returned when the configuration of the resource
// is insufficient for the resource to do any useful work.
var ErrResourceInsufficientParameters = errors.New(
"Insufficient parameters for this resource")
func init() {
RegisterResource("hostname", func() Res { return &HostnameRes{} })
engine.RegisterResource("hostname", func() engine.Res { return &HostnameRes{} })
}
const (
@@ -43,6 +39,10 @@ const (
dbusAddMatch = "org.freedesktop.DBus.AddMatch"
)
// ErrResourceInsufficientParameters is returned when the configuration of the
// resource is insufficient for the resource to do any useful work.
var ErrResourceInsufficientParameters = errors.New("insufficient parameters for this resource")
// HostnameRes is a resource that allows setting and watching the hostname.
//
// StaticHostname is the one configured in /etc/hostname or a similar file.
@@ -58,7 +58,10 @@ const (
// Hostname is the fallback value for all 3 fields above, if only Hostname is
// specified, it will set all 3 fields to this value.
type HostnameRes struct {
BaseRes `yaml:",inline"`
traits.Base // add the base methods without re-implementation
init *engine.Init
Hostname string `yaml:"hostname"`
PrettyHostname string `yaml:"pretty_hostname"`
StaticHostname string `yaml:"static_hostname"`
@@ -68,12 +71,8 @@ type HostnameRes struct {
}
// Default returns some sensible defaults for this resource.
func (obj *HostnameRes) Default() Res {
return &HostnameRes{
BaseRes: BaseRes{
MetaParams: DefaultMetaParams, // force a default
},
}
func (obj *HostnameRes) Default() engine.Res {
return &HostnameRes{}
}
// Validate if the params passed in are valid data.
@@ -81,11 +80,13 @@ func (obj *HostnameRes) Validate() error {
if obj.PrettyHostname == "" && obj.StaticHostname == "" && obj.TransientHostname == "" {
return ErrResourceInsufficientParameters
}
return obj.BaseRes.Validate()
return nil
}
// Init runs some startup code for this resource.
func (obj *HostnameRes) Init() error {
func (obj *HostnameRes) Init(init *engine.Init) error {
obj.init = init // save for later
if obj.PrettyHostname == "" {
obj.PrettyHostname = obj.Hostname
}
@@ -95,7 +96,12 @@ func (obj *HostnameRes) Init() error {
if obj.TransientHostname == "" {
obj.TransientHostname = obj.Hostname
}
return obj.BaseRes.Init() // call base init, b/c we're overriding
return nil
}
// Close is run by the engine to clean up after the resource is done.
func (obj *HostnameRes) Close() error {
return nil
}
// Watch is the primary listener for this resource and it outputs events.
@@ -117,36 +123,37 @@ func (obj *HostnameRes) Watch() error {
bus.Signal(signals)
// notify engine that we're running
if err := obj.Running(); err != nil {
return err // bubble up a NACK...
if err := obj.init.Running(); err != nil {
return err // exit if requested
}
var send = false // send event?
for {
select {
case <-signals:
send = true
obj.StateOK(false) // dirty
obj.init.Dirty() // dirty
case event := <-obj.Events():
// we avoid sending events on unpause
if exit, _ := obj.ReadEvent(event); exit != nil {
return *exit // exit
case event, ok := <-obj.init.Events:
if !ok {
return nil
}
if err := obj.init.Read(event); err != nil {
return err
}
send = true
obj.StateOK(false) // dirty
}
// do all our event sending all together to avoid duplicate msgs
if send {
send = false
obj.Event()
if err := obj.init.Event(); err != nil {
return err // exit if requested
}
}
}
}
func updateHostnameProperty(object dbus.BusObject, expectedValue, property, setterName string, apply bool) (checkOK bool, err error) {
func (obj *HostnameRes) updateHostnameProperty(object dbus.BusObject, expectedValue, property, setterName string, apply bool) (checkOK bool, err error) {
propertyObject, err := object.GetProperty("org.freedesktop.hostname1." + property)
if err != nil {
return false, errwrap.Wrapf(err, "failed to get org.freedesktop.hostname1.%s", property)
@@ -171,7 +178,7 @@ func updateHostnameProperty(object dbus.BusObject, expectedValue, property, sett
}
// attempting to apply the changes
log.Printf("Changing %s: %s => %s", property, propertyValue, expectedValue)
obj.init.Logf("Changing %s: %s => %s", property, propertyValue, expectedValue)
if err := object.Call("org.freedesktop.hostname1."+setterName, 0, expectedValue, false).Err; err != nil {
return false, errwrap.Wrapf(err, "failed to call org.freedesktop.hostname1.%s", setterName)
}
@@ -192,21 +199,21 @@ func (obj *HostnameRes) CheckApply(apply bool) (checkOK bool, err error) {
checkOK = true
if obj.PrettyHostname != "" {
propertyCheckOK, err := updateHostnameProperty(hostnameObject, obj.PrettyHostname, "PrettyHostname", "SetPrettyHostname", apply)
propertyCheckOK, err := obj.updateHostnameProperty(hostnameObject, obj.PrettyHostname, "PrettyHostname", "SetPrettyHostname", apply)
if err != nil {
return false, err
}
checkOK = checkOK && propertyCheckOK
}
if obj.StaticHostname != "" {
propertyCheckOK, err := updateHostnameProperty(hostnameObject, obj.StaticHostname, "StaticHostname", "SetStaticHostname", apply)
propertyCheckOK, err := obj.updateHostnameProperty(hostnameObject, obj.StaticHostname, "StaticHostname", "SetStaticHostname", apply)
if err != nil {
return false, err
}
checkOK = checkOK && propertyCheckOK
}
if obj.TransientHostname != "" {
propertyCheckOK, err := updateHostnameProperty(hostnameObject, obj.TransientHostname, "Hostname", "SetHostname", apply)
propertyCheckOK, err := obj.updateHostnameProperty(hostnameObject, obj.TransientHostname, "Hostname", "SetHostname", apply)
if err != nil {
return false, err
}
@@ -216,46 +223,21 @@ func (obj *HostnameRes) CheckApply(apply bool) (checkOK bool, err error) {
return checkOK, nil
}
// HostnameUID is the UID struct for HostnameRes.
type HostnameUID struct {
BaseUID
name string
prettyHostname string
staticHostname string
transientHostname string
}
// UIDs includes all params to make a unique identification of this object.
// Most resources only return one, although some resources can return multiple.
func (obj *HostnameRes) UIDs() []ResUID {
x := &HostnameUID{
BaseUID: BaseUID{Name: obj.GetName(), Kind: obj.GetKind()},
name: obj.Name,
prettyHostname: obj.PrettyHostname,
staticHostname: obj.StaticHostname,
transientHostname: obj.TransientHostname,
// Cmp compares two resources and returns an error if they are not equivalent.
func (obj *HostnameRes) Cmp(r engine.Res) error {
if !obj.Compare(r) {
return fmt.Errorf("did not compare")
}
return []ResUID{x}
}
// GroupCmp returns whether two resources can be grouped together or not.
func (obj *HostnameRes) GroupCmp(r Res) bool {
return false
return nil
}
// Compare two resources and return if they are equivalent.
func (obj *HostnameRes) Compare(r Res) bool {
func (obj *HostnameRes) Compare(r engine.Res) bool {
// we can only compare HostnameRes to others of the same resource kind
res, ok := r.(*HostnameRes)
if !ok {
return false
}
if !obj.BaseRes.Compare(res) { // call base Compare
return false
}
if obj.Name != res.Name {
return false
}
if obj.PrettyHostname != res.PrettyHostname {
return false
@@ -270,6 +252,29 @@ func (obj *HostnameRes) Compare(r Res) bool {
return true
}
// HostnameUID is the UID struct for HostnameRes.
type HostnameUID struct {
engine.BaseUID
name string
prettyHostname string
staticHostname string
transientHostname string
}
// UIDs includes all params to make a unique identification of this object.
// Most resources only return one, although some resources can return multiple.
func (obj *HostnameRes) UIDs() []engine.ResUID {
x := &HostnameUID{
BaseUID: engine.BaseUID{Name: obj.Name(), Kind: obj.Kind()},
name: obj.Name(),
prettyHostname: obj.PrettyHostname,
staticHostname: obj.StaticHostname,
transientHostname: obj.TransientHostname,
}
return []engine.ResUID{x}
}
// UnmarshalYAML is the custom unmarshal handler for this struct.
// It is primarily useful for setting the defaults.
func (obj *HostnameRes) UnmarshalYAML(unmarshal func(interface{}) error) error {

130
resources/kv.go → engine/resources/kv.go
View File

@@ -19,14 +19,16 @@ package resources
import (
"fmt"
"log"
"strconv"
"github.com/purpleidea/mgmt/engine"
"github.com/purpleidea/mgmt/engine/traits"
errwrap "github.com/pkg/errors"
)
func init() {
RegisterResource("kv", func() Res { return &KVRes{} })
engine.RegisterResource("kv", func() engine.Res { return &KVRes{} })
}
// KVResSkipCmpStyle represents the different styles of comparison when using SkipLessThan.
@@ -47,7 +49,13 @@ const (
// The one exception is that when this resource receives a refresh signal, then
// it will set the value to be the exact one if they are not identical already.
type KVRes struct {
BaseRes `yaml:",inline"`
traits.Base // add the base methods without re-implementation
//traits.Groupable // TODO: it could be useful to group our writes and watches!
traits.Refreshable
traits.Recvable
init *engine.Init
// XXX: shouldn't the name be the key?
Key string `yaml:"key"` // key to set
Value *string `yaml:"value"` // value to set (nil to delete)
@@ -57,17 +65,11 @@ type KVRes struct {
}
// Default returns some sensible defaults for this resource.
func (obj *KVRes) Default() Res {
return &KVRes{
BaseRes: BaseRes{
MetaParams: DefaultMetaParams, // force a default
},
}
func (obj *KVRes) Default() engine.Res {
return &KVRes{}
}
// Validate if the params passed in are valid data.
// FIXME: This will catch most issues unless data is passed in after Init with
// the Send/Recv mechanism. Should the engine re-call Validate after Send/Recv?
func (obj *KVRes) Validate() error {
if obj.Key == "" {
return fmt.Errorf("key must not be empty")
@@ -83,26 +85,32 @@ func (obj *KVRes) Validate() error {
}
}
}
return obj.BaseRes.Validate()
return nil
}
// Init initializes the resource.
func (obj *KVRes) Init() error {
return obj.BaseRes.Init() // call base init, b/c we're overriding
func (obj *KVRes) Init(init *engine.Init) error {
obj.init = init // save for later
return nil
}
// Close is run by the engine to clean up after the resource is done.
func (obj *KVRes) Close() error {
return nil
}
// Watch is the primary listener for this resource and it outputs events.
func (obj *KVRes) Watch() error {
// notify engine that we're running
if err := obj.Running(); err != nil {
return err // bubble up a NACK...
if err := obj.init.Running(); err != nil {
return err // exit if requested
}
ch := obj.Data().World.StrMapWatch(obj.Key) // get possible events!
ch := obj.init.World.StrMapWatch(obj.Key) // get possible events!
var send = false // send event?
var exit *error
for {
select {
// NOTE: this part is very similar to the file resource code
@@ -113,36 +121,39 @@ func (obj *KVRes) Watch() error {
if err != nil {
return errwrap.Wrapf(err, "unknown %s watcher error", obj)
}
if obj.Data().Debug {
log.Printf("%s: Event!", obj)
if obj.init.Debug {
obj.init.Logf("Event!")
}
send = true
obj.StateOK(false) // dirty
obj.init.Dirty() // dirty
case event := <-obj.Events():
// we avoid sending events on unpause
if exit, send = obj.ReadEvent(event); exit != nil {
return *exit // exit
case event, ok := <-obj.init.Events:
if !ok {
return nil
}
if err := obj.init.Read(event); err != nil {
return err
}
}
// do all our event sending all together to avoid duplicate msgs
if send {
send = false
obj.Event()
if err := obj.init.Event(); err != nil {
return err // exit if requested
}
}
}
}
// lessThanCheck checks for less than validity.
func (obj *KVRes) lessThanCheck(value string) (checkOK bool, err error) {
v := *obj.Value
if value == v { // redundant check for safety
return true, nil
}
var refresh = obj.Refresh() // do we have a pending reload to apply?
var refresh = obj.init.Refresh() // do we have a pending reload to apply?
if !obj.SkipLessThan || refresh { // update lessthan on refresh
return false, nil
}
@@ -175,15 +186,15 @@ func (obj *KVRes) lessThanCheck(value string) (checkOK bool, err error) {
// CheckApply method for Password resource. Does nothing, returns happy!
func (obj *KVRes) CheckApply(apply bool) (checkOK bool, err error) {
log.Printf("%s: CheckApply(%t)", obj, apply)
obj.init.Logf("CheckApply(%t)", apply)
if val, exists := obj.Recv["Value"]; exists && val.Changed {
if val, exists := obj.init.Recv()["Value"]; exists && val.Changed {
// if we received on Value, and it changed, wooo, nothing to do.
log.Printf("CheckApply: `Value` was updated!")
obj.init.Logf("CheckApply: `Value` was updated!")
}
hostname := obj.Data().Hostname // me
keyMap, err := obj.Data().World.StrMapGet(obj.Key)
hostname := obj.init.Hostname // me
keyMap, err := obj.init.World.StrMapGet(obj.Key)
if err != nil {
return false, errwrap.Wrapf(err, "check error during StrGet")
}
@@ -203,7 +214,7 @@ func (obj *KVRes) CheckApply(apply bool) (checkOK bool, err error) {
return true, nil // nothing to delete, we're good!
} else if ok && obj.Value == nil { // delete
err := obj.Data().World.StrMapDel(obj.Key)
err := obj.init.World.StrMapDel(obj.Key)
return false, errwrap.Wrapf(err, "apply error during StrDel")
}
@@ -211,49 +222,28 @@ func (obj *KVRes) CheckApply(apply bool) (checkOK bool, err error) {
return false, nil
}
if err := obj.Data().World.StrMapSet(obj.Key, *obj.Value); err != nil {
if err := obj.init.World.StrMapSet(obj.Key, *obj.Value); err != nil {
return false, errwrap.Wrapf(err, "apply error during StrSet")
}
return false, nil
}
// KVUID is the UID struct for KVRes.
type KVUID struct {
BaseUID
name string
}
// UIDs includes all params to make a unique identification of this object.
// Most resources only return one, although some resources can return multiple.
func (obj *KVRes) UIDs() []ResUID {
x := &KVUID{
BaseUID: BaseUID{Name: obj.GetName(), Kind: obj.GetKind()},
name: obj.Name,
// Cmp compares two resources and returns an error if they are not equivalent.
func (obj *KVRes) Cmp(r engine.Res) error {
if !obj.Compare(r) {
return fmt.Errorf("did not compare")
}
return []ResUID{x}
}
// GroupCmp returns whether two resources can be grouped together or not.
func (obj *KVRes) GroupCmp(r Res) bool {
_, ok := r.(*KVRes)
if !ok {
return false
}
return false // TODO: this is doable!
// TODO: it could be useful to group our writes and watches!
return nil
}
// Compare two resources and return if they are equivalent.
func (obj *KVRes) Compare(r Res) bool {
func (obj *KVRes) Compare(r engine.Res) bool {
// we can only compare KVRes to others of the same resource kind
res, ok := r.(*KVRes)
if !ok {
return false
}
if !obj.BaseRes.Compare(res) { // call base Compare
return false
}
if obj.Key != res.Key {
return false
@@ -276,6 +266,22 @@ func (obj *KVRes) Compare(r Res) bool {
return true
}
// KVUID is the UID struct for KVRes.
type KVUID struct {
engine.BaseUID
name string
}
// UIDs includes all params to make a unique identification of this object.
// Most resources only return one, although some resources can return multiple.
func (obj *KVRes) UIDs() []engine.ResUID {
x := &KVUID{
BaseUID: engine.BaseUID{Name: obj.Name(), Kind: obj.Kind()},
name: obj.Name(),
}
return []engine.ResUID{x}
}
// UnmarshalYAML is the custom unmarshal handler for this struct.
// It is primarily useful for setting the defaults.
func (obj *KVRes) UnmarshalYAML(unmarshal func(interface{}) error) error {

148
resources/msg.go → engine/resources/msg.go
View File

@@ -19,20 +19,26 @@ package resources
import (
"fmt"
"log"
"regexp"
"strings"
"github.com/purpleidea/mgmt/engine"
"github.com/purpleidea/mgmt/engine/traits"
"github.com/coreos/go-systemd/journal"
)
func init() {
RegisterResource("msg", func() Res { return &MsgRes{} })
engine.RegisterResource("msg", func() engine.Res { return &MsgRes{} })
}
// MsgRes is a resource that writes messages to logs.
type MsgRes struct {
BaseRes `yaml:",inline"`
traits.Base // add the base methods without re-implementation
traits.Refreshable
init *engine.Init
Body string `yaml:"body"`
Priority string `yaml:"priority"`
Fields map[string]string `yaml:"fields"`
@@ -43,19 +49,9 @@ type MsgRes struct {
syslogStateOK bool
}
// MsgUID is a unique representation for a MsgRes object.
type MsgUID struct {
BaseUID
body string
}
// Default returns some sensible defaults for this resource.
func (obj *MsgRes) Default() Res {
return &MsgRes{
BaseRes: BaseRes{
MetaParams: DefaultMetaParams, // force a default
},
}
func (obj *MsgRes) Default() engine.Res {
return &MsgRes{}
}
// Validate the params that are passed to MsgRes.
@@ -81,15 +77,52 @@ func (obj *MsgRes) Validate() error {
default:
return fmt.Errorf("invalid Priority '%s'", obj.Priority)
}
return obj.BaseRes.Validate()
return nil
}
// Init runs some startup code for this resource.
func (obj *MsgRes) Init() error {
return obj.BaseRes.Init() // call base init, b/c we're overrriding
func (obj *MsgRes) Init(init *engine.Init) error {
obj.init = init // save for later
return nil
}
// isAllStateOK derives a compound state from all internal cache flags that apply to this resource.
// Close is run by the engine to clean up after the resource is done.
func (obj *MsgRes) Close() error {
return nil
}
// Watch is the primary listener for this resource and it outputs events.
func (obj *MsgRes) Watch() error {
// notify engine that we're running
if err := obj.init.Running(); err != nil {
return err // exit if requested
}
var send = false // send event?
for {
select {
case event, ok := <-obj.init.Events:
if !ok {
return nil
}
if err := obj.init.Read(event); err != nil {
return err
}
}
// do all our event sending all together to avoid duplicate msgs
if send {
send = false
if err := obj.init.Event(); err != nil {
return err // exit if requested
}
}
}
}
// isAllStateOK derives a compound state from all internal cache flags that
// apply to this resource.
func (obj *MsgRes) isAllStateOK() bool {
if obj.Journal && !obj.journalStateOK {
return false
@@ -102,7 +135,10 @@ func (obj *MsgRes) isAllStateOK() bool {
// updateStateOK sets the global state so it can be read by the engine.
func (obj *MsgRes) updateStateOK() {
obj.StateOK(obj.isAllStateOK())
// XXX: this resource doesn't entirely make sense to me at the moment.
if !obj.isAllStateOK() {
obj.init.Dirty()
}
}
// JournalPriority converts a string description to a numeric priority.
@@ -128,42 +164,15 @@ func (obj *MsgRes) journalPriority() journal.Priority {
return journal.PriNotice
}
// Watch is the primary listener for this resource and it outputs events.
func (obj *MsgRes) Watch() error {
// notify engine that we're running
if err := obj.Running(); err != nil {
return err // bubble up a NACK...
}
var send = false // send event?
var exit *error
for {
select {
case event := <-obj.Events():
// we avoid sending events on unpause
if exit, send = obj.ReadEvent(event); exit != nil {
return *exit // exit
}
}
// do all our event sending all together to avoid duplicate msgs
if send {
send = false
obj.Event()
}
}
}
// CheckApply method for Msg resource.
// Every check leads to an apply, meaning that the message is flushed to the journal.
// CheckApply method for Msg resource. Every check leads to an apply, meaning
// that the message is flushed to the journal.
func (obj *MsgRes) CheckApply(apply bool) (bool, error) {
// isStateOK() done by engine, so we updateStateOK() to pass in value
//if obj.isAllStateOK() {
// return true, nil
//}
if obj.Refresh() { // if we were notified...
if obj.init.Refresh() { // if we were notified...
// invalidate cached state...
obj.logStateOK = false
if obj.Journal {
@@ -176,7 +185,7 @@ func (obj *MsgRes) CheckApply(apply bool) (bool, error) {
}
if !obj.logStateOK {
log.Printf("%s: Body: %s", obj, obj.Body)
obj.init.Logf("Body: %s", obj.Body)
obj.logStateOK = true
obj.updateStateOK()
}
@@ -199,29 +208,21 @@ func (obj *MsgRes) CheckApply(apply bool) (bool, error) {
return false, nil
}
// UIDs includes all params to make a unique identification of this object.
// Most resources only return one, although some resources can return multiple.
func (obj *MsgRes) UIDs() []ResUID {
x := &MsgUID{
BaseUID: BaseUID{
Name: obj.GetName(),
Kind: obj.GetKind(),
},
body: obj.Body,
// Cmp compares two resources and returns an error if they are not equivalent.
func (obj *MsgRes) Cmp(r engine.Res) error {
if !obj.Compare(r) {
return fmt.Errorf("did not compare")
}
return []ResUID{x}
return nil
}
// Compare two resources and return if they are equivalent.
func (obj *MsgRes) Compare(r Res) bool {
func (obj *MsgRes) Compare(r engine.Res) bool {
// we can only compare MsgRes to others of the same resource kind
res, ok := r.(*MsgRes)
if !ok {
return false
}
if !obj.BaseRes.Compare(res) {
return false
}
if obj.Body != res.Body {
return false
@@ -241,6 +242,23 @@ func (obj *MsgRes) Compare(r Res) bool {
return true
}
// MsgUID is a unique representation for a MsgRes object.
type MsgUID struct {
engine.BaseUID
body string
}
// UIDs includes all params to make a unique identification of this object.
// Most resources only return one, although some resources can return multiple.
func (obj *MsgRes) UIDs() []engine.ResUID {
x := &MsgUID{
BaseUID: engine.BaseUID{Name: obj.Name(), Kind: obj.Kind()},
body: obj.Body,
}
return []engine.ResUID{x}
}
// UnmarshalYAML is the custom unmarshal handler for this struct.
// It is primarily useful for setting the defaults.
func (obj *MsgRes) UnmarshalYAML(unmarshal func(interface{}) error) error {

12
resources/msg_test.go → engine/resources/msg_test.go
View File

@@ -23,15 +23,9 @@ import (
func TestMsgValidate1(t *testing.T) {
r1 := &MsgRes{
BaseRes: BaseRes{
Name: "msg1",
Kind: "msg",
MetaParams: DefaultMetaParams,
},
Priority: "Debug",
}
r1.Setup(nil, r1, r1)
if err := r1.Validate(); err != nil {
t.Errorf("validate failed with: %v", err)
}
@@ -39,15 +33,9 @@ func TestMsgValidate1(t *testing.T) {
func TestMsgValidate2(t *testing.T) {
r1 := &MsgRes{
BaseRes: BaseRes{
Name: "msg1",
Kind: "msg",
MetaParams: DefaultMetaParams,
},
Priority: "UnrealPriority",
}
r1.Setup(nil, r1, r1)
if err := r1.Validate(); err == nil {
t.Errorf("validation error is nil")
}

218
resources/net.go → engine/resources/net.go
View File

@@ -15,13 +15,14 @@
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
// +build !darwin
package resources
import (
"bytes"
"fmt"
"io/ioutil"
"log"
"net"
"os"
"path"
@@ -29,8 +30,8 @@ import (
"sync"
"syscall"
"golang.org/x/sys/unix"
"github.com/purpleidea/mgmt/engine"
"github.com/purpleidea/mgmt/engine/traits"
"github.com/purpleidea/mgmt/recwatch"
"github.com/purpleidea/mgmt/util"
@@ -40,10 +41,11 @@ import (
// do not clean up spawned goroutines. Should be replaced when a suitable
// alternative is available.
"github.com/vishvananda/netlink"
"golang.org/x/sys/unix"
)
func init() {
RegisterResource("net", func() Res { return &NetRes{} })
engine.RegisterResource("net", func() engine.Res { return &NetRes{} })
}
const (
@@ -75,21 +77,26 @@ const (
// https://github.com/torvalds/linux/blob/master/include/uapi/linux/rtnetlink.h
rtProtoKernel = 2 // kernel
rtProtoStatic = 4 // static
socketFile = "pipe.sock" // path in vardir to store our socket file
)
// NetRes is a network interface resource based on netlink. It manages the
// state of a network link. Configuration is also stored in a networkd
// configuration file, so the network is available upon reboot.
type NetRes struct {
BaseRes `yaml:",inline"`
traits.Base // add the base methods without re-implementation
init *engine.Init
State string `yaml:"state"` // up, down, or empty
Addrs []string `yaml:"addrs"` // list of addresses in cidr format
Gateway string `yaml:"gateway"` // gateway address
iface *iface // a struct containing the net.Interface and netlink.Link
unitFilePath string // the interface unit file path
// XXX: replace TempDir with VarDir
tempDir string // temporary directory for storing the pipe socket file
socketFile string // path for storing the pipe socket file
}
// nlChanStruct defines the channel used to send netlink messages and errors
@@ -100,12 +107,8 @@ type nlChanStruct struct {
}
// Default returns some sensible defaults for this resource.
func (obj *NetRes) Default() Res {
return &NetRes{
BaseRes: BaseRes{
MetaParams: DefaultMetaParams, // force a default
},
}
func (obj *NetRes) Default() engine.Res {
return &NetRes{}
}
// Validate if the params passed in are valid data.
@@ -133,37 +136,57 @@ func (obj *NetRes) Validate() error {
}
// validate the interface name
_, err := net.InterfaceByName(obj.GetName())
_, err := net.InterfaceByName(obj.Name())
if err != nil {
return errwrap.Wrapf(err, "error finding interface: %s", obj.GetName())
return errwrap.Wrapf(err, "error finding interface: %s", obj.Name())
}
return obj.BaseRes.Validate()
return nil
}
// Init runs some startup code for this resource.
func (obj *NetRes) Init() error {
func (obj *NetRes) Init(init *engine.Init) error {
obj.init = init // save for later
var err error
// tmp directory for pipe socket
// XXX: Replace with obj.VarDir
if obj.tempDir, err = ioutil.TempDir("", "pipe"); err != nil {
return errwrap.Wrapf(err, "could not get TempDir")
dir, err := obj.init.VarDir("")
if err != nil {
return errwrap.Wrapf(err, "could not get VarDir in Init()")
}
obj.socketFile = path.Join(dir, socketFile) // return a unique file
// store the network interface in the struct
obj.iface = &iface{}
if obj.iface.iface, err = net.InterfaceByName(obj.GetName()); err != nil {
return errwrap.Wrapf(err, "error finding interface: %s", obj.GetName())
if obj.iface.iface, err = net.InterfaceByName(obj.Name()); err != nil {
return errwrap.Wrapf(err, "error finding interface: %s", obj.Name())
}
// store the netlink link to use as interface input in netlink functions
if obj.iface.link, err = netlink.LinkByName(obj.GetName()); err != nil {
return errwrap.Wrapf(err, "error finding link: %s", obj.GetName())
if obj.iface.link, err = netlink.LinkByName(obj.Name()); err != nil {
return errwrap.Wrapf(err, "error finding link: %s", obj.Name())
}
// build the path to the networkd configuration file
obj.unitFilePath = networkdUnitFileDir + IfacePrefix + obj.GetName() + networkdUnitFileExt
obj.unitFilePath = networkdUnitFileDir + IfacePrefix + obj.Name() + networkdUnitFileExt
return obj.BaseRes.Init()
return nil
}
// Close cleans up when we're done.
func (obj *NetRes) Close() error {
var errList error
if obj.socketFile == "/" {
return fmt.Errorf("socket file should not be the root path")
}
if obj.socketFile != "" { // safety
if err := os.Remove(obj.socketFile); err != nil {
errList = multierr.Append(errList, err)
}
}
return errList
}
// Watch listens for events from the specified interface via a netlink socket.
@@ -175,7 +198,7 @@ func (obj *NetRes) Watch() error {
defer wg.Wait()
// create a netlink socket for receiving network interface events
conn, err := newSocketSet(rtmGrps, path.Join(obj.tempDir, "pipe.sock"))
conn, err := newSocketSet(rtmGrps, obj.socketFile)
if err != nil {
return errwrap.Wrapf(err, "error creating socket set")
}
@@ -224,14 +247,12 @@ func (obj *NetRes) Watch() error {
}()
// notify engine that we're running
if err := obj.Running(); err != nil {
return err // bubble up a NACK...
if err := obj.init.Running(); err != nil {
return err // exit if requested
}
var exit *error
var send bool
var send = false // send event?
var done bool
for {
select {
case s, ok := <-nlChan:
@@ -245,12 +266,12 @@ func (obj *NetRes) Watch() error {
if err := s.err; err != nil {
return errwrap.Wrapf(s.err, "unknown netlink error")
}
if obj.debug {
log.Printf("%s: Event: %+v", obj, s.msg)
if obj.init.Debug {
obj.init.Logf("Event: %+v", s.msg)
}
send = true
obj.StateOK(false)
obj.init.Dirty() // dirty
case event, ok := <-recWatcher.Events():
if !ok {
@@ -263,23 +284,28 @@ func (obj *NetRes) Watch() error {
if err := event.Error; err != nil {
return errwrap.Wrapf(err, "unknown recwatcher error")
}
if obj.debug {
log.Printf("%s: Event(%s): %v", obj, event.Body.Name, event.Body.Op)
if obj.init.Debug {
obj.init.Logf("Event(%s): %v", event.Body.Name, event.Body.Op)
}
send = true
obj.StateOK(false) // dirty
obj.init.Dirty() // dirty
case event := <-obj.Events():
if exit, send = obj.ReadEvent(event); exit != nil {
return *exit // exit
case event, ok := <-obj.init.Events:
if !ok {
return nil
}
if err := obj.init.Read(event); err != nil {
return err
}
}
// do all our event sending all together to avoid duplicate msgs
if send {
send = false
obj.Event()
if err := obj.init.Event(); err != nil {
return err // exit if requested
}
}
}
}
@@ -290,7 +316,7 @@ func (obj *NetRes) ifaceCheckApply(apply bool) (bool, error) {
// check the interface state
state, err := obj.iface.state()
if err != nil {
return false, errwrap.Wrapf(err, "error checking %s state", obj.GetName())
return false, errwrap.Wrapf(err, "error checking %s state", obj.Name())
}
// if the state is correct or unspecified, we're done
if obj.State == state || obj.State == "" {
@@ -301,11 +327,11 @@ func (obj *NetRes) ifaceCheckApply(apply bool) (bool, error) {
if !apply {
return false, nil
}
log.Printf("%s: ifaceCheckApply(%t)", obj, apply)
obj.init.Logf("ifaceCheckApply(%t)", apply)
// ip link set up/down
if err := obj.iface.linkUpDown(obj.State); err != nil {
return false, errwrap.Wrapf(err, "error setting %s up or down", obj.GetName())
return false, errwrap.Wrapf(err, "error setting %s up or down", obj.Name())
}
return false, nil
@@ -317,7 +343,7 @@ func (obj *NetRes) addrCheckApply(apply bool) (bool, error) {
// get the link's addresses
ifaceAddrs, err := obj.iface.getAddrs()
if err != nil {
return false, errwrap.Wrapf(err, "error getting addresses from %s", obj.GetName())
return false, errwrap.Wrapf(err, "error getting addresses from %s", obj.Name())
}
// if state is not defined
if obj.Addrs == nil {
@@ -341,7 +367,7 @@ func (obj *NetRes) addrCheckApply(apply bool) (bool, error) {
if !apply {
return false, nil
}
log.Printf("%s: addrCheckApply(%t)", obj, apply)
obj.init.Logf("addrCheckApply(%t)", apply)
// check each address and delete the ones that aren't in the definition
if err := obj.iface.addrApplyDelete(obj.Addrs); err != nil {
@@ -391,7 +417,7 @@ func (obj *NetRes) gatewayCheckApply(apply bool) (bool, error) {
if !apply {
return false, nil
}
log.Printf("%s: gatewayCheckApply(%t)", obj, apply)
obj.init.Logf("gatewayCheckApply(%t)", apply)
// delete all but one default route
for i := 1; i < len(defRoutes); i++ {
@@ -436,7 +462,7 @@ func (obj *NetRes) fileCheckApply(apply bool) (bool, error) {
if !apply {
return false, nil
}
log.Printf("%s: fileCheckApply(%t)", obj, apply)
obj.init.Logf("fileCheckApply(%t)", apply)
// write the file
if err := ioutil.WriteFile(obj.unitFilePath, contents, networkdUnitFileUmask); err != nil {
@@ -492,70 +518,22 @@ func (obj *NetRes) CheckApply(apply bool) (checkOK bool, err error) {
return checkOK, nil
}
// Close cleans up when we're done.
func (obj *NetRes) Close() error {
var errList error
// XXX: replace TempDir with VarDir
if err := os.RemoveAll(obj.tempDir); err != nil {
errList = multierr.Append(errList, err)
// Cmp compares two resources and returns an error if they are not equivalent.
func (obj *NetRes) Cmp(r engine.Res) error {
if !obj.Compare(r) {
return fmt.Errorf("did not compare")
}
if err := obj.BaseRes.Close(); err != nil {
errList = multierr.Append(errList, err)
}
return errList
}
// NetUID is a unique resource identifier.
type NetUID struct {
// NOTE: There is also a name variable in the BaseUID struct, this is
// information about where this UID came from, and is unrelated to the
// information about the resource we're matching. That data which is
// used in the IFF function, is what you see in the struct fields here.
BaseUID
name string // the network interface name
}
// IFF aka if and only if they are equivalent, return true. If not, false.
func (obj *NetUID) IFF(uid ResUID) bool {
res, ok := uid.(*NetUID)
if !ok {
return false
}
return obj.name == res.name
}
// UIDs includes all params to make a unique identification of this object.
// Most resources only return one although some resources can return multiple.
func (obj *NetRes) UIDs() []ResUID {
x := &NetUID{
BaseUID: BaseUID{Name: obj.GetName(), Kind: obj.GetKind()},
name: obj.Name,
}
return []ResUID{x}
}
// GroupCmp returns whether two resources can be grouped together or not.
func (obj *NetRes) GroupCmp(r Res) bool {
_, ok := r.(*NetRes)
if !ok {
return false
}
return false
return nil
}
// Compare two resources and return if they are equivalent.
func (obj *NetRes) Compare(r Res) bool {
func (obj *NetRes) Compare(r engine.Res) bool {
// we can only compare NetRes to others of the same resource kind
res, ok := r.(*NetRes)
if !ok {
return false
}
if !obj.BaseRes.Compare(res) {
return false
}
if obj.Name != res.Name {
return false
}
if obj.State != res.State {
return false
}
@@ -572,6 +550,36 @@ func (obj *NetRes) Compare(r Res) bool {
return true
}
// NetUID is a unique resource identifier.
type NetUID struct {
// NOTE: There is also a name variable in the BaseUID struct, this is
// information about where this UID came from, and is unrelated to the
// information about the resource we're matching. That data which is
// used in the IFF function, is what you see in the struct fields here.
engine.BaseUID
name string // the network interface name
}
// IFF aka if and only if they are equivalent, return true. If not, false.
func (obj *NetUID) IFF(uid engine.ResUID) bool {
res, ok := uid.(*NetUID)
if !ok {
return false
}
return obj.name == res.name
}
// UIDs includes all params to make a unique identification of this object.
// Most resources only return one although some resources can return multiple.
func (obj *NetRes) UIDs() []engine.ResUID {
x := &NetUID{
BaseUID: engine.BaseUID{Name: obj.Name(), Kind: obj.Kind()},
name: obj.Name(),
}
return []engine.ResUID{x}
}
// UnmarshalYAML is the custom unmarshal handler for this struct.
// It is primarily useful for setting the defaults.
func (obj *NetRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
@@ -596,7 +604,7 @@ func (obj *NetRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
func (obj *NetRes) unitFileContents() []byte {
// build the unit file contents
u := []string{"[Match]"}
u = append(u, fmt.Sprintf("Name=%s", obj.GetName()))
u = append(u, fmt.Sprintf("Name=%s", obj.Name()))
u = append(u, "[Network]")
for _, addr := range obj.Addrs {
u = append(u, fmt.Sprintf("Address=%s", addr))

106
resources/noop.go → engine/resources/noop.go
View File

@@ -19,117 +19,127 @@ package resources
import (
"fmt"
"log"
"github.com/purpleidea/mgmt/engine"
"github.com/purpleidea/mgmt/engine/traits"
)
func init() {
RegisterResource("noop", func() Res { return &NoopRes{} })
engine.RegisterResource("noop", func() engine.Res { return &NoopRes{} })
}
// NoopRes is a no-op resource that does nothing.
type NoopRes struct {
BaseRes `yaml:",inline"`
traits.Base // add the base methods without re-implementation
traits.Groupable
traits.Refreshable
init *engine.Init
Comment string `lang:"comment" yaml:"comment"` // extra field for example purposes
}
// Default returns some sensible defaults for this resource.
func (obj *NoopRes) Default() Res {
return &NoopRes{
BaseRes: BaseRes{
MetaParams: DefaultMetaParams, // force a default
},
}
func (obj *NoopRes) Default() engine.Res {
return &NoopRes{}
}
// Validate if the params passed in are valid data.
func (obj *NoopRes) Validate() error {
return obj.BaseRes.Validate()
return nil
}
// Init runs some startup code for this resource.
func (obj *NoopRes) Init() error {
return obj.BaseRes.Init() // call base init, b/c we're overriding
func (obj *NoopRes) Init(init *engine.Init) error {
obj.init = init // save for later
return nil
}
// Close is run by the engine to clean up after the resource is done.
func (obj *NoopRes) Close() error {
return nil
}
// Watch is the primary listener for this resource and it outputs events.
func (obj *NoopRes) Watch() error {
// notify engine that we're running
if err := obj.Running(); err != nil {
return err // bubble up a NACK...
if err := obj.init.Running(); err != nil {
return err // exit if requested
}
var send = false // send event?
var exit *error
for {
select {
case event := <-obj.Events():
// we avoid sending events on unpause
if exit, send = obj.ReadEvent(event); exit != nil {
return *exit // exit
case event, ok := <-obj.init.Events:
if !ok {
return nil
}
if err := obj.init.Read(event); err != nil {
return err
}
}
// do all our event sending all together to avoid duplicate msgs
if send {
send = false
obj.Event()
if err := obj.init.Event(); err != nil {
return err // exit if requested
}
}
}
}
// CheckApply method for Noop resource. Does nothing, returns happy!
func (obj *NoopRes) CheckApply(apply bool) (checkOK bool, err error) {
if obj.Refresh() {
log.Printf("%s: Received a notification!", obj)
if obj.init.Refresh() {
obj.init.Logf("received a notification!")
}
return true, nil // state is always okay
}
// Cmp compares two resources and returns an error if they are not equivalent.
func (obj *NoopRes) Cmp(r engine.Res) error {
// we can only compare NoopRes to others of the same resource kind
res, ok := r.(*NoopRes)
if !ok {
return fmt.Errorf("not a %s", obj.Kind())
}
if obj.Comment != res.Comment {
return fmt.Errorf("the Comment differs")
}
return nil
}
// NoopUID is the UID struct for NoopRes.
type NoopUID struct {
BaseUID
engine.BaseUID
name string
}
// UIDs includes all params to make a unique identification of this object.
// Most resources only return one, although some resources can return multiple.
func (obj *NoopRes) UIDs() []ResUID {
func (obj *NoopRes) UIDs() []engine.ResUID {
x := &NoopUID{
BaseUID: BaseUID{Name: obj.GetName(), Kind: obj.GetKind()},
name: obj.Name,
BaseUID: engine.BaseUID{Name: obj.Name(), Kind: obj.Kind()},
name: obj.Name(),
}
return []ResUID{x}
return []engine.ResUID{x}
}
// GroupCmp returns whether two resources can be grouped together or not.
func (obj *NoopRes) GroupCmp(r Res) bool {
func (obj *NoopRes) GroupCmp(r engine.GroupableRes) error {
_, ok := r.(*NoopRes)
if !ok {
// NOTE: technically we could group a noop into any other
// resource, if that resource knew how to handle it, although,
// since the mechanics of inter-kind resource grouping are
// tricky, avoid doing this until there's a good reason.
return false
return fmt.Errorf("resource is not the same kind")
}
return true // noop resources can always be grouped together!
}
// Compare two resources and return if they are equivalent.
func (obj *NoopRes) Compare(r Res) bool {
// we can only compare NoopRes to others of the same resource kind
res, ok := r.(*NoopRes)
if !ok {
return false
}
// calling base Compare is probably unneeded for the noop res, but do it
if !obj.BaseRes.Compare(res) { // call base Compare
return false
}
if obj.Name != res.Name {
return false
}
return true
return nil // noop resources can always be grouped together!
}
// UnmarshalYAML is the custom unmarshal handler for this struct.

102
engine/resources/noop_test.go Normal file
View File

@@ -0,0 +1,102 @@
// Mgmt
// Copyright (C) 2013-2018+ James Shubin and the project contributors
// Written by James Shubin <james@shubin.ca> and the project contributors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package resources
import (
"reflect"
"testing"
"github.com/purpleidea/mgmt/engine"
)
func TestCmp1(t *testing.T) {
r1, err := engine.NewResource("noop")
if err != nil {
t.Errorf("could not create resource: %+v", err)
}
r2, err := engine.NewResource("noop")
if err != nil {
t.Errorf("could not create resource: %+v", err)
}
r3, err := engine.NewResource("file")
if err != nil {
t.Errorf("could not create resource: %+v", err)
}
if err := r1.Cmp(r2); err != nil {
t.Errorf("the two resources do not match: %+v", err)
}
if err := r2.Cmp(r1); err != nil {
t.Errorf("the two resources do not match: %+v", err)
}
if r1.Cmp(r3) == nil {
t.Errorf("the two resources should not match")
}
if r3.Cmp(r1) == nil {
t.Errorf("the two resources should not match")
}
}
func TestSort0(t *testing.T) {
rs := []engine.Res{}
s := engine.Sort(rs)
if !reflect.DeepEqual(s, []engine.Res{}) {
t.Errorf("sort failed!")
if s == nil {
t.Logf("output is nil!")
} else {
str := "Got:"
for _, r := range s {
str += " " + r.String()
}
t.Errorf(str)
}
}
}
func TestSort1(t *testing.T) {
r1, _ := engine.NewNamedResource("noop", "noop1")
r2, _ := engine.NewNamedResource("noop", "noop2")
r3, _ := engine.NewNamedResource("noop", "noop3")
r4, _ := engine.NewNamedResource("noop", "noop4")
r5, _ := engine.NewNamedResource("noop", "noop5")
r6, _ := engine.NewNamedResource("noop", "noop6")
rs := []engine.Res{r3, r2, r6, r1, r5, r4}
s := engine.Sort(rs)
if !reflect.DeepEqual(s, []engine.Res{r1, r2, r3, r4, r5, r6}) {
t.Errorf("sort failed!")
str := "Got:"
for _, r := range s {
str += " " + r.String()
}
t.Errorf(str)
}
if !reflect.DeepEqual(rs, []engine.Res{r3, r2, r6, r1, r5, r4}) {
t.Errorf("sort modified input!")
str := "Got:"
for _, r := range rs {
str += " " + r.String()
}
t.Errorf(str)
}
}

555
resources/nspawn.go → engine/resources/nspawn.go
View File

@@ -20,10 +20,11 @@ package resources
import (
"errors"
"fmt"
"log"
"strconv"
"unicode"
"github.com/purpleidea/mgmt/engine"
"github.com/purpleidea/mgmt/engine/traits"
"github.com/purpleidea/mgmt/util"
systemdDbus "github.com/coreos/go-systemd/dbus"
@@ -43,13 +44,17 @@ const (
)
func init() {
RegisterResource("nspawn", func() Res { return &NspawnRes{} })
engine.RegisterResource("nspawn", func() engine.Res { return &NspawnRes{} })
}
// NspawnRes is an nspawn container resource.
type NspawnRes struct {
BaseRes `yaml:",inline"`
State string `yaml:"state"`
traits.Base // add the base methods without re-implementation
//traits.Groupable // TODO: this would be quite useful for this resource
init *engine.Init
State string `yaml:"state"`
// We're using the svc resource to start and stop the machine because
// that's what machinectl does. We're not using svc.Watch because then we
// would have two watches potentially racing each other and producing
@@ -59,11 +64,8 @@ type NspawnRes struct {
}
// Default returns some sensible defaults for this resource.
func (obj *NspawnRes) Default() Res {
func (obj *NspawnRes) Default() engine.Res {
return &NspawnRes{
BaseRes: BaseRes{
MetaParams: DefaultMetaParams, // force a default
},
State: running,
}
}
@@ -71,7 +73,7 @@ func (obj *NspawnRes) Default() Res {
// makeComposite creates a pointer to a SvcRes. The pointer is used to
// validate and initialize the nested svc.
func (obj *NspawnRes) makeComposite() (*SvcRes, error) {
res, err := NewNamedResource("svc", fmt.Sprintf(nspawnServiceTmpl, obj.GetName()))
res, err := engine.NewNamedResource("svc", fmt.Sprintf(nspawnServiceTmpl, obj.Name()))
if err != nil {
return nil, err
}
@@ -80,6 +82,278 @@ func (obj *NspawnRes) makeComposite() (*SvcRes, error) {
return svc, nil
}
// Validate if the params passed in are valid data.
func (obj *NspawnRes) Validate() error {
if len(obj.Name()) > 64 {
return fmt.Errorf("name must be 64 characters or less")
}
// check if systemd version is higher than 231 to allow non-alphanumeric
// machine names, as previous versions would error in such cases
ver, err := systemdVersion()
if err != nil {
return err
}
if ver < 231 {
for _, char := range obj.Name() {
if !unicode.IsLetter(char) && !unicode.IsNumber(char) {
return fmt.Errorf("name must only contain alphanumeric characters for systemd versions < 231")
}
}
}
if obj.State != running && obj.State != stopped {
return fmt.Errorf("invalid state: %s", obj.State)
}
svc, err := obj.makeComposite()
if err != nil {
return errwrap.Wrapf(err, "makeComposite failed in validate")
}
if err := svc.Validate(); err != nil { // composite resource
return errwrap.Wrapf(err, "validate failed for embedded svc: %s", obj.svc)
}
return nil
}
// Init runs some startup code for this resource.
func (obj *NspawnRes) Init(init *engine.Init) error {
obj.init = init // save for later
svc, err := obj.makeComposite()
if err != nil {
return errwrap.Wrapf(err, "makeComposite failed in init")
}
obj.svc = svc
// TODO: we could build a new init that adds a prefix to the logger...
if err := obj.svc.Init(init); err != nil {
return err
}
return nil
}
// Close is run by the engine to clean up after the resource is done.
func (obj *NspawnRes) Close() error {
if obj.svc != nil {
return obj.svc.Close()
}
return nil
}
// Watch for state changes and sends a message to the bus if there is a change.
func (obj *NspawnRes) Watch() error {
// this resource depends on systemd to ensure that it's running
if !systemdUtil.IsRunningSystemd() {
return fmt.Errorf("systemd is not running")
}
// create a private message bus
bus, err := util.SystemBusPrivateUsable()
if err != nil {
return errwrap.Wrapf(err, "failed to connect to bus")
}
// add a match rule to match messages going through the message bus
call := bus.BusObject().Call("org.freedesktop.DBus.AddMatch", 0,
fmt.Sprintf("type='signal',interface='%s',eavesdrop='true'",
dbusInterface))
// <-call.Done
if err := call.Err; err != nil {
return err
}
// TODO: verify that implementation doesn't deadlock if there are unread
// messages left in the channel
busChan := make(chan *dbus.Signal, 10)
bus.Signal(busChan)
// notify engine that we're running
if err := obj.init.Running(); err != nil {
return err // exit if requested
}
var send = false // send event?
defer close(busChan)
defer bus.Close()
defer bus.RemoveSignal(busChan)
for {
select {
case event := <-busChan:
// process org.freedesktop.machine1 events for this resource's name
if event.Body[0] == obj.Name() {
obj.init.Logf("Event received: %v", event.Name)
if event.Name == machineNew {
obj.init.Logf("Machine started")
} else if event.Name == machineRemoved {
obj.init.Logf("Machine stopped")
} else {
return fmt.Errorf("unknown event: %s", event.Name)
}
send = true
obj.init.Dirty() // dirty
}
case event, ok := <-obj.init.Events:
if !ok {
return nil
}
if err := obj.init.Read(event); err != nil {
return err
}
}
// do all our event sending all together to avoid duplicate msgs
if send {
send = false
if err := obj.init.Event(); err != nil {
return err // exit if requested
}
}
}
}
// CheckApply is run to check the state and, if apply is true, to apply the
// necessary changes to reach the desired state. This is run before Watch and
// again if Watch finds a change occurring to the state.
func (obj *NspawnRes) CheckApply(apply bool) (checkOK bool, err error) {
// this resource depends on systemd to ensure that it's running
if !systemdUtil.IsRunningSystemd() {
return false, errors.New("systemd is not running")
}
// connect to org.freedesktop.machine1.Manager
conn, err := machined.New()
if err != nil {
return false, errwrap.Wrapf(err, "failed to connect to dbus")
}
// compare the current state with the desired state and perform the
// appropriate action
var exists = true
properties, err := conn.DescribeMachine(obj.Name())
if err != nil {
if err, ok := err.(dbus.Error); ok && err.Name !=
"org.freedesktop.machine1.NoSuchMachine" {
return false, err
}
exists = false
// if we could not successfully get the properties because
// there's no such machine the machine is stopped
// error if we need the image ignore if we don't
if _, err = conn.GetImage(obj.Name()); err != nil && obj.State != stopped {
return false, fmt.Errorf(
"no machine nor image named '%s'",
obj.Name())
}
}
if obj.init.Debug {
obj.init.Logf("properties: %v", properties)
}
// if the machine doesn't exist and is supposed to
// be stopped or the state matches we're done
if !exists && obj.State == stopped || properties["State"] == obj.State {
if obj.init.Debug {
obj.init.Logf("CheckApply() in valid state")
}
return true, nil
}
// end of state checking. if we're here, checkOK is false
if !apply {
return false, nil
}
obj.init.Logf("CheckApply() applying '%s' state", obj.State)
// use the embedded svc to apply the correct state
if _, err := obj.svc.CheckApply(apply); err != nil {
return false, errwrap.Wrapf(err, "nested svc failed")
}
return false, nil
}
// Cmp compares two resources and returns an error if they are not equivalent.
func (obj *NspawnRes) Cmp(r engine.Res) error {
if !obj.Compare(r) {
return fmt.Errorf("did not compare")
}
return nil
}
// Compare two resources and return if they are equivalent.
func (obj *NspawnRes) Compare(r engine.Res) bool {
// we can only compare NspawnRes to others of the same resource kind
res, ok := r.(*NspawnRes)
if !ok {
return false
}
if obj.State != res.State {
return false
}
// TODO: why is res.svc ever nil?
if (obj.svc == nil) != (res.svc == nil) { // xor
return false
}
if obj.svc != nil && res.svc != nil {
if !obj.svc.Compare(res.svc) {
return false
}
}
return true
}
// NspawnUID is a unique resource identifier.
type NspawnUID struct {
// NOTE: There is also a name variable in the BaseUID struct, this is
// information about where this UID came from, and is unrelated to the
// information about the resource we're matching. That data which is
// used in the IFF function, is what you see in the struct fields here.
engine.BaseUID
name string // the machine name
}
// IFF aka if and only if they are equivalent, return true. If not, false.
func (obj *NspawnUID) IFF(uid engine.ResUID) bool {
res, ok := uid.(*NspawnUID)
if !ok {
return false
}
return obj.name == res.name
}
// UIDs includes all params to make a unique identification of this object.
// Most resources only return one although some resources can return multiple.
func (obj *NspawnRes) UIDs() []engine.ResUID {
x := &NspawnUID{
BaseUID: engine.BaseUID{Name: obj.Name(), Kind: obj.Kind()},
name: obj.Name(), // svc name
}
return append([]engine.ResUID{x}, obj.svc.UIDs()...)
}
// UnmarshalYAML is the custom unmarshal handler for this struct.
// It is primarily useful for setting the defaults.
func (obj *NspawnRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
type rawRes NspawnRes // indirection to avoid infinite recursion
def := obj.Default() // get the default
res, ok := def.(*NspawnRes) // put in the right format
if !ok {
return fmt.Errorf("could not convert to NspawnRes")
}
raw := rawRes(*res) // convert; the defaults go here
if err := unmarshal(&raw); err != nil {
return err
}
*obj = NspawnRes(raw) // restore from indirection with type conversion!
return nil
}
// systemdVersion uses dbus to check which version of systemd is installed.
func systemdVersion() (uint16, error) {
// check if systemd is running
@@ -108,266 +382,3 @@ func systemdVersion() (uint16, error) {
}
return uint16(ver), nil
}
// Validate if the params passed in are valid data.
func (obj *NspawnRes) Validate() error {
if len(obj.GetName()) > 64 {
return fmt.Errorf("name must be 64 characters or less")
}
// check if systemd version is higher than 231 to allow non-alphanumeric
// machine names, as previous versions would error in such cases
ver, err := systemdVersion()
if err != nil {
return err
}
if ver < 231 {
for _, char := range obj.GetName() {
if !unicode.IsLetter(char) && !unicode.IsNumber(char) {
return fmt.Errorf("name must only contain alphanumeric characters for systemd versions < 231")
}
}
}
if obj.State != running && obj.State != stopped {
return fmt.Errorf("invalid state: %s", obj.State)
}
svc, err := obj.makeComposite()
if err != nil {
return errwrap.Wrapf(err, "makeComposite failed in validate")
}
if err := svc.Validate(); err != nil { // composite resource
return errwrap.Wrapf(err, "validate failed for embedded svc: %s", obj.svc)
}
return obj.BaseRes.Validate()
}
// Init runs some startup code for this resource.
func (obj *NspawnRes) Init() error {
svc, err := obj.makeComposite()
if err != nil {
return errwrap.Wrapf(err, "makeComposite failed in init")
}
obj.svc = svc
if err := obj.svc.Init(); err != nil {
return err
}
return obj.BaseRes.Init()
}
// Watch for state changes and sends a message to the bus if there is a change.
func (obj *NspawnRes) Watch() error {
// this resource depends on systemd to ensure that it's running
if !systemdUtil.IsRunningSystemd() {
return fmt.Errorf("systemd is not running")
}
// create a private message bus
bus, err := util.SystemBusPrivateUsable()
if err != nil {
return errwrap.Wrapf(err, "failed to connect to bus")
}
// add a match rule to match messages going through the message bus
call := bus.BusObject().Call("org.freedesktop.DBus.AddMatch", 0,
fmt.Sprintf("type='signal',interface='%s',eavesdrop='true'",
dbusInterface))
// <-call.Done
if err := call.Err; err != nil {
return err
}
// TODO: verify that implementation doesn't deadlock if there are unread
// messages left in the channel
busChan := make(chan *dbus.Signal, 10)
bus.Signal(busChan)
// notify engine that we're running
if err := obj.Running(); err != nil {
return err // bubble up a NACK...
}
var send = false
var exit *error
defer close(busChan)
defer bus.Close()
defer bus.RemoveSignal(busChan)
for {
select {
case event := <-busChan:
// process org.freedesktop.machine1 events for this resource's name
if event.Body[0] == obj.GetName() {
log.Printf("%s: Event received: %v", obj, event.Name)
if event.Name == machineNew {
log.Printf("%s: Machine started", obj)
} else if event.Name == machineRemoved {
log.Printf("%s: Machine stopped", obj)
} else {
return fmt.Errorf("unknown event: %s", event.Name)
}
send = true
obj.StateOK(false) // dirty
}
case event := <-obj.Events():
if exit, send = obj.ReadEvent(event); exit != nil {
return *exit // exit
}
}
// do all our event sending all together to avoid duplicate msgs
if send {
send = false
obj.Event()
}
}
}
// CheckApply is run to check the state and, if apply is true, to apply the
// necessary changes to reach the desired state. This is run before Watch and
// again if Watch finds a change occurring to the state.
func (obj *NspawnRes) CheckApply(apply bool) (checkOK bool, err error) {
// this resource depends on systemd to ensure that it's running
if !systemdUtil.IsRunningSystemd() {
return false, errors.New("systemd is not running")
}
// connect to org.freedesktop.machine1.Manager
conn, err := machined.New()
if err != nil {
return false, errwrap.Wrapf(err, "failed to connect to dbus")
}
// compare the current state with the desired state and perform the
// appropriate action
var exists = true
properties, err := conn.DescribeMachine(obj.GetName())
if err != nil {
if err, ok := err.(dbus.Error); ok && err.Name !=
"org.freedesktop.machine1.NoSuchMachine" {
return false, err
}
exists = false
// if we could not successfully get the properties because
// there's no such machine the machine is stopped
// error if we need the image ignore if we don't
if _, err = conn.GetImage(obj.GetName()); err != nil && obj.State != stopped {
return false, fmt.Errorf(
"no machine nor image named '%s'",
obj.GetName())
}
}
if obj.debug {
log.Printf("%s: properties: %v", obj, properties)
}
// if the machine doesn't exist and is supposed to
// be stopped or the state matches we're done
if !exists && obj.State == stopped || properties["State"] == obj.State {
if obj.debug {
log.Printf("%s: CheckApply() in valid state", obj)
}
return true, nil
}
// end of state checking. if we're here, checkOK is false
if !apply {
return false, nil
}
log.Printf("%s: CheckApply() applying '%s' state", obj, obj.State)
// use the embedded svc to apply the correct state
if _, err := obj.svc.CheckApply(apply); err != nil {
return false, errwrap.Wrapf(err, "nested svc failed")
}
return false, nil
}
// NspawnUID is a unique resource identifier.
type NspawnUID struct {
// NOTE: There is also a name variable in the BaseUID struct, this is
// information about where this UID came from, and is unrelated to the
// information about the resource we're matching. That data which is
// used in the IFF function, is what you see in the struct fields here.
BaseUID
name string // the machine name
}
// IFF aka if and only if they are equivalent, return true. If not, false.
func (obj *NspawnUID) IFF(uid ResUID) bool {
res, ok := uid.(*NspawnUID)
if !ok {
return false
}
return obj.name == res.name
}
// UIDs includes all params to make a unique identification of this object.
// Most resources only return one although some resources can return multiple.
func (obj *NspawnRes) UIDs() []ResUID {
x := &NspawnUID{
BaseUID: BaseUID{Name: obj.GetName(), Kind: obj.GetKind()},
name: obj.Name, // svc name
}
return append([]ResUID{x}, obj.svc.UIDs()...)
}
// GroupCmp returns whether two resources can be grouped together or not.
func (obj *NspawnRes) GroupCmp(r Res) bool {
_, ok := r.(*NspawnRes)
if !ok {
return false
}
// TODO: this would be quite useful for this resource!
return false
}
// Compare two resources and return if they are equivalent.
func (obj *NspawnRes) Compare(r Res) bool {
// we can only compare NspawnRes to others of the same resource kind
res, ok := r.(*NspawnRes)
if !ok {
return false
}
if !obj.BaseRes.Compare(res) {
return false
}
if obj.Name != res.Name {
return false
}
if obj.State != res.State {
return false
}
// TODO: why is res.svc ever nil?
if (obj.svc == nil) != (res.svc == nil) { // xor
return false
}
if obj.svc != nil && res.svc != nil {
if !obj.svc.Compare(res.svc) {
return false
}
}
return true
}
// UnmarshalYAML is the custom unmarshal handler for this struct.
// It is primarily useful for setting the defaults.
func (obj *NspawnRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
type rawRes NspawnRes // indirection to avoid infinite recursion
def := obj.Default() // get the default
res, ok := def.(*NspawnRes) // put in the right format
if !ok {
return fmt.Errorf("could not convert to NspawnRes")
}
raw := rawRes(*res) // convert; the defaults go here
if err := unmarshal(&raw); err != nil {
return err
}
*obj = NspawnRes(raw) // restore from indirection with type conversion!
return nil
}

217
resources/packagekit/packagekit.go → engine/resources/packagekit/packagekit.go
View File

@@ -22,18 +22,17 @@ package packagekit
import (
"fmt"
"log"
"runtime"
"strings"
"github.com/purpleidea/mgmt/util"
"github.com/godbus/dbus"
errwrap "github.com/pkg/errors"
)
// global tweaks of verbosity and code path
const (
Debug = false
Paranoid = false // enable if you see any ghosts
)
@@ -149,6 +148,9 @@ const ( //typedef enum
// Conn is a wrapper struct so we can pass bus connection around in the struct.
type Conn struct {
conn *dbus.Conn
Debug bool
Logf func(format string, v ...interface{})
}
// PkPackageIDActionData is a struct that is returned by PackagesToPackageIDs in the map values.
@@ -173,31 +175,31 @@ func NewBus() *Conn {
}
// GetBus gets the dbus connection object.
func (bus *Conn) GetBus() *dbus.Conn {
return bus.conn
func (obj *Conn) GetBus() *dbus.Conn {
return obj.conn
}
// Close closes the dbus connection object.
func (bus *Conn) Close() error {
return bus.conn.Close()
func (obj *Conn) Close() error {
return obj.conn.Close()
}
// internal helper to add signal matches to the bus, should only be called once
func (bus *Conn) matchSignal(ch chan *dbus.Signal, path dbus.ObjectPath, iface string, signals []string) error {
if Debug {
log.Printf("PackageKit: matchSignal(%v, %v, %v, %v)", ch, path, iface, signals)
func (obj *Conn) matchSignal(ch chan *dbus.Signal, path dbus.ObjectPath, iface string, signals []string) error {
if obj.Debug {
obj.Logf("matchSignal(%v, %v, %s, %v)", ch, path, iface, signals)
}
// eg: gdbus monitor --system --dest org.freedesktop.PackageKit --object-path /org/freedesktop/PackageKit | grep <signal>
var call *dbus.Call
// TODO: if we make this call many times, we seem to receive signals
// that many times... Maybe this should be an object singleton?
obj := bus.GetBus().BusObject()
bus := obj.GetBus().BusObject()
pathStr := fmt.Sprintf("%s", path)
if len(signals) == 0 {
call = obj.Call(dbusAddMatch, 0, "type='signal',path='"+pathStr+"',interface='"+iface+"'")
call = bus.Call(dbusAddMatch, 0, "type='signal',path='"+pathStr+"',interface='"+iface+"'")
} else {
for _, signal := range signals {
call = obj.Call(dbusAddMatch, 0, "type='signal',path='"+pathStr+"',interface='"+iface+"',member='"+signal+"'")
call = bus.Call(dbusAddMatch, 0, "type='signal',path='"+pathStr+"',interface='"+iface+"',member='"+signal+"'")
if call.Err != nil {
break
}
@@ -209,18 +211,18 @@ func (bus *Conn) matchSignal(ch chan *dbus.Signal, path dbus.ObjectPath, iface s
// The caller has to make sure that ch is sufficiently buffered; if a
// message arrives when a write to c is not possible, it is discarded!
// This can be disastrous if we're waiting for a "Finished" signal!
bus.GetBus().Signal(ch)
obj.GetBus().Signal(ch)
return nil
}
// WatchChanges gets a signal anytime an event happens.
func (bus *Conn) WatchChanges() (chan *dbus.Signal, error) {
func (obj *Conn) WatchChanges() (chan *dbus.Signal, error) {
ch := make(chan *dbus.Signal, PkBufferSize)
// NOTE: the TransactionListChanged signal fires much more frequently,
// but with much less specificity. If we're missing events, report the
// issue upstream! The UpdatesChanged signal is what hughsie suggested
var signal = "UpdatesChanged"
err := bus.matchSignal(ch, PkPath, PkIface, []string{signal})
err := obj.matchSignal(ch, PkPath, PkIface, []string{signal})
if err != nil {
return nil, err
}
@@ -236,13 +238,13 @@ func (bus *Conn) WatchChanges() (chan *dbus.Signal, error) {
// zero value immediately": if i get nil here,
// it means the channel was closed by someone!!
if event == nil { // shared bus issue?
log.Println("PackageKit: Hrm, channel was closed!")
obj.Logf("Hrm, channel was closed!")
break loop // TODO: continue?
}
// i think this was caused by using the shared
// bus, but we might as well leave it in for now
if event.Path != PkPath || event.Name != fmt.Sprintf("%s.%s", PkIface, signal) {
log.Printf("PackageKit: Woops: Event: %+v", event)
obj.Logf("Woops: Event: %+v", event)
continue
}
rch <- event // forward...
@@ -256,41 +258,41 @@ func (bus *Conn) WatchChanges() (chan *dbus.Signal, error) {
}
// CreateTransaction creates and returns a transaction path.
func (bus *Conn) CreateTransaction() (dbus.ObjectPath, error) {
if Debug {
log.Println("PackageKit: CreateTransaction()")
func (obj *Conn) CreateTransaction() (dbus.ObjectPath, error) {
if obj.Debug {
obj.Logf("CreateTransaction()")
}
var interfacePath dbus.ObjectPath
obj := bus.GetBus().Object(PkIface, PkPath)
call := obj.Call(fmt.Sprintf("%s.CreateTransaction", PkIface), 0).Store(&interfacePath)
bus := obj.GetBus().Object(PkIface, PkPath)
call := bus.Call(fmt.Sprintf("%s.CreateTransaction", PkIface), 0).Store(&interfacePath)
if call != nil {
return "", call
}
if Debug {
log.Printf("PackageKit: CreateTransaction(): %v", interfacePath)
if obj.Debug {
obj.Logf("CreateTransaction(): %v", interfacePath)
}
return interfacePath, nil
}
// ResolvePackages runs the PackageKit Resolve method and returns the result.
func (bus *Conn) ResolvePackages(packages []string, filter uint64) ([]string, error) {
func (obj *Conn) ResolvePackages(packages []string, filter uint64) ([]string, error) {
packageIDs := []string{}
ch := make(chan *dbus.Signal, PkBufferSize) // we need to buffer :(
interfacePath, err := bus.CreateTransaction() // emits Destroy on close
interfacePath, err := obj.CreateTransaction() // emits Destroy on close
if err != nil {
return []string{}, err
}
// add signal matches for Package and Finished which will always be last
var signals = []string{"Package", "Finished", "Error", "Destroy"}
bus.matchSignal(ch, interfacePath, PkIfaceTransaction, signals)
if Debug {
log.Printf("PackageKit: ResolvePackages(): Object(%v, %v)", PkIface, interfacePath)
obj.matchSignal(ch, interfacePath, PkIfaceTransaction, signals)
if obj.Debug {
obj.Logf("ResolvePackages(): Object(%s, %v)", PkIface, interfacePath)
}
obj := bus.GetBus().Object(PkIface, interfacePath) // pass in found transaction path
call := obj.Call(FmtTransactionMethod("Resolve"), 0, filter, packages)
if Debug {
log.Println("PackageKit: ResolvePackages(): Call: Success!")
bus := obj.GetBus().Object(PkIface, interfacePath) // pass in found transaction path
call := bus.Call(FmtTransactionMethod("Resolve"), 0, filter, packages)
if obj.Debug {
obj.Logf("ResolvePackages(): Call: Success!")
}
if call.Err != nil {
return []string{}, call.Err
@@ -300,11 +302,11 @@ loop:
// FIXME: add a timeout option to error in case signals are dropped!
select {
case signal := <-ch:
if Debug {
log.Printf("PackageKit: ResolvePackages(): Signal: %+v", signal)
if obj.Debug {
obj.Logf("ResolvePackages(): Signal: %+v", signal)
}
if signal.Path != interfacePath {
log.Printf("PackageKit: Woops: Signal.Path: %+v", signal.Path)
obj.Logf("Woops: Signal.Path: %+v", signal.Path)
continue loop
}
@@ -337,10 +339,10 @@ loop:
}
// IsInstalledList queries a list of packages to see if they are installed.
func (bus *Conn) IsInstalledList(packages []string) ([]bool, error) {
func (obj *Conn) IsInstalledList(packages []string) ([]bool, error) {
var filter uint64 // initializes at the "zero" value of 0
filter += PkFilterEnumArch // always search in our arch
packageIDs, e := bus.ResolvePackages(packages, filter)
packageIDs, e := obj.ResolvePackages(packages, filter)
if e != nil {
return nil, fmt.Errorf("ResolvePackages error: %v", e)
}
@@ -375,8 +377,8 @@ func (bus *Conn) IsInstalledList(packages []string) ([]bool, error) {
// IsInstalled returns if a package is installed.
// TODO: this could be optimized by making the resolve call directly
func (bus *Conn) IsInstalled(pkg string) (bool, error) {
p, e := bus.IsInstalledList([]string{pkg})
func (obj *Conn) IsInstalled(pkg string) (bool, error) {
p, e := obj.IsInstalledList([]string{pkg})
if len(p) != 1 {
return false, e
}
@@ -384,23 +386,23 @@ func (bus *Conn) IsInstalled(pkg string) (bool, error) {
}
// InstallPackages installs a list of packages by packageID.
func (bus *Conn) InstallPackages(packageIDs []string, transactionFlags uint64) error {
func (obj *Conn) InstallPackages(packageIDs []string, transactionFlags uint64) error {
ch := make(chan *dbus.Signal, PkBufferSize) // we need to buffer :(
interfacePath, err := bus.CreateTransaction() // emits Destroy on close
interfacePath, err := obj.CreateTransaction() // emits Destroy on close
if err != nil {
return err
}
var signals = []string{"Package", "ErrorCode", "Finished", "Destroy"} // "ItemProgress", "Status" ?
bus.matchSignal(ch, interfacePath, PkIfaceTransaction, signals)
obj.matchSignal(ch, interfacePath, PkIfaceTransaction, signals)
obj := bus.GetBus().Object(PkIface, interfacePath) // pass in found transaction path
call := obj.Call(FmtTransactionMethod("RefreshCache"), 0, false)
bus := obj.GetBus().Object(PkIface, interfacePath) // pass in found transaction path
call := bus.Call(FmtTransactionMethod("RefreshCache"), 0, false)
if call.Err != nil {
return call.Err
}
call = obj.Call(FmtTransactionMethod("InstallPackages"), 0, transactionFlags, packageIDs)
call = bus.Call(FmtTransactionMethod("InstallPackages"), 0, transactionFlags, packageIDs)
if call.Err != nil {
return call.Err
}
@@ -411,7 +413,7 @@ loop:
select {
case signal := <-ch:
if signal.Path != interfacePath {
log.Printf("PackageKit: Woops: Signal.Path: %+v", signal.Path)
obj.Logf("Woops: Signal.Path: %+v", signal.Path)
continue loop
}
@@ -431,30 +433,30 @@ loop:
}
case <-util.TimeAfterOrBlock(timeout):
if finished {
log.Println("PackageKit: Timeout: InstallPackages: Waiting for 'Destroy'")
obj.Logf("Timeout: InstallPackages: Waiting for 'Destroy'")
return nil // got tired of waiting for Destroy
}
return fmt.Errorf("PackageKit: Timeout: InstallPackages: %v", strings.Join(packageIDs, ", "))
return fmt.Errorf("PackageKit: Timeout: InstallPackages: %s", strings.Join(packageIDs, ", "))
}
}
}
// RemovePackages removes a list of packages by packageID.
func (bus *Conn) RemovePackages(packageIDs []string, transactionFlags uint64) error {
func (obj *Conn) RemovePackages(packageIDs []string, transactionFlags uint64) error {
var allowDeps = true // TODO: configurable
var autoremove = false // unsupported on GNU/Linux
ch := make(chan *dbus.Signal, PkBufferSize) // we need to buffer :(
interfacePath, err := bus.CreateTransaction() // emits Destroy on close
interfacePath, err := obj.CreateTransaction() // emits Destroy on close
if err != nil {
return err
}
var signals = []string{"Package", "ErrorCode", "Finished", "Destroy"} // "ItemProgress", "Status" ?
bus.matchSignal(ch, interfacePath, PkIfaceTransaction, signals)
obj.matchSignal(ch, interfacePath, PkIfaceTransaction, signals)
obj := bus.GetBus().Object(PkIface, interfacePath) // pass in found transaction path
call := obj.Call(FmtTransactionMethod("RemovePackages"), 0, transactionFlags, packageIDs, allowDeps, autoremove)
bus := obj.GetBus().Object(PkIface, interfacePath) // pass in found transaction path
call := bus.Call(FmtTransactionMethod("RemovePackages"), 0, transactionFlags, packageIDs, allowDeps, autoremove)
if call.Err != nil {
return call.Err
}
@@ -464,7 +466,7 @@ loop:
select {
case signal := <-ch:
if signal.Path != interfacePath {
log.Printf("PackageKit: Woops: Signal.Path: %+v", signal.Path)
obj.Logf("Woops: Signal.Path: %+v", signal.Path)
continue loop
}
@@ -488,18 +490,18 @@ loop:
}
// UpdatePackages updates a list of packages to versions that are specified.
func (bus *Conn) UpdatePackages(packageIDs []string, transactionFlags uint64) error {
func (obj *Conn) UpdatePackages(packageIDs []string, transactionFlags uint64) error {
ch := make(chan *dbus.Signal, PkBufferSize) // we need to buffer :(
interfacePath, err := bus.CreateTransaction()
interfacePath, err := obj.CreateTransaction()
if err != nil {
return err
}
var signals = []string{"Package", "ErrorCode", "Finished", "Destroy"} // "ItemProgress", "Status" ?
bus.matchSignal(ch, interfacePath, PkIfaceTransaction, signals)
obj.matchSignal(ch, interfacePath, PkIfaceTransaction, signals)
obj := bus.GetBus().Object(PkIface, interfacePath) // pass in found transaction path
call := obj.Call(FmtTransactionMethod("UpdatePackages"), 0, transactionFlags, packageIDs)
bus := obj.GetBus().Object(PkIface, interfacePath) // pass in found transaction path
call := bus.Call(FmtTransactionMethod("UpdatePackages"), 0, transactionFlags, packageIDs)
if call.Err != nil {
return call.Err
}
@@ -509,7 +511,7 @@ loop:
select {
case signal := <-ch:
if signal.Path != interfacePath {
log.Printf("PackageKit: Woops: Signal.Path: %+v", signal.Path)
obj.Logf("Woops: Signal.Path: %+v", signal.Path)
continue loop
}
@@ -531,20 +533,20 @@ loop:
}
// GetFilesByPackageID gets the list of files that are contained inside a list of packageIDs.
func (bus *Conn) GetFilesByPackageID(packageIDs []string) (files map[string][]string, err error) {
func (obj *Conn) GetFilesByPackageID(packageIDs []string) (files map[string][]string, err error) {
// NOTE: the maximum number of files in an RPM is 52116 in Fedora 23
// https://gist.github.com/purpleidea/b98e60dcd449e1ac3b8a
ch := make(chan *dbus.Signal, PkBufferSize) // we need to buffer :(
interfacePath, err := bus.CreateTransaction()
interfacePath, err := obj.CreateTransaction()
if err != nil {
return
}
var signals = []string{"Files", "ErrorCode", "Finished", "Destroy"} // "ItemProgress", "Status" ?
bus.matchSignal(ch, interfacePath, PkIfaceTransaction, signals)
obj.matchSignal(ch, interfacePath, PkIfaceTransaction, signals)
obj := bus.GetBus().Object(PkIface, interfacePath) // pass in found transaction path
call := obj.Call(FmtTransactionMethod("GetFiles"), 0, packageIDs)
bus := obj.GetBus().Object(PkIface, interfacePath) // pass in found transaction path
call := bus.Call(FmtTransactionMethod("GetFiles"), 0, packageIDs)
if call.Err != nil {
err = call.Err
return
@@ -557,7 +559,7 @@ loop:
case signal := <-ch:
if signal.Path != interfacePath {
log.Printf("PackageKit: Woops: Signal.Path: %+v", signal.Path)
obj.Logf("Woops: Signal.Path: %+v", signal.Path)
continue loop
}
@@ -596,22 +598,22 @@ loop:
}
// GetUpdates gets a list of packages that are installed and which can be updated, mod filter.
func (bus *Conn) GetUpdates(filter uint64) ([]string, error) {
if Debug {
log.Println("PackageKit: GetUpdates()")
func (obj *Conn) GetUpdates(filter uint64) ([]string, error) {
if obj.Debug {
obj.Logf("GetUpdates()")
}
packageIDs := []string{}
ch := make(chan *dbus.Signal, PkBufferSize) // we need to buffer :(
interfacePath, err := bus.CreateTransaction()
interfacePath, err := obj.CreateTransaction()
if err != nil {
return nil, err
}
var signals = []string{"Package", "ErrorCode", "Finished", "Destroy"} // "ItemProgress" ?
bus.matchSignal(ch, interfacePath, PkIfaceTransaction, signals)
obj.matchSignal(ch, interfacePath, PkIfaceTransaction, signals)
obj := bus.GetBus().Object(PkIface, interfacePath) // pass in found transaction path
call := obj.Call(FmtTransactionMethod("GetUpdates"), 0, filter)
bus := obj.GetBus().Object(PkIface, interfacePath) // pass in found transaction path
call := bus.Call(FmtTransactionMethod("GetUpdates"), 0, filter)
if call.Err != nil {
return nil, call.Err
}
@@ -621,7 +623,7 @@ loop:
select {
case signal := <-ch:
if signal.Path != interfacePath {
log.Printf("PackageKit: Woops: Signal.Path: %+v", signal.Path)
obj.Logf("Woops: Signal.Path: %+v", signal.Path)
continue loop
}
@@ -660,7 +662,7 @@ loop:
// outside mgmt. The packageMap input has the package names as keys and
// requested states as values. These states can be: installed, uninstalled,
// newest or a requested version str.
func (bus *Conn) PackagesToPackageIDs(packageMap map[string]string, filter uint64) (map[string]*PkPackageIDActionData, error) {
func (obj *Conn) PackagesToPackageIDs(packageMap map[string]string, filter uint64) (map[string]*PkPackageIDActionData, error) {
count := 0
packages := make([]string, len(packageMap))
for k := range packageMap { // lol, golang has no hash.keys() function!
@@ -672,10 +674,10 @@ func (bus *Conn) PackagesToPackageIDs(packageMap map[string]string, filter uint6
filter += PkFilterEnumArch // always search in our arch
}
if Debug {
log.Printf("PackageKit: PackagesToPackageIDs(): %v", strings.Join(packages, ", "))
if obj.Debug {
obj.Logf("PackagesToPackageIDs(): %s", strings.Join(packages, ", "))
}
resolved, e := bus.ResolvePackages(packages, filter)
resolved, e := obj.ResolvePackages(packages, filter)
if e != nil {
return nil, fmt.Errorf("Resolve error: %v", e)
}
@@ -692,13 +694,16 @@ func (bus *Conn) PackagesToPackageIDs(packageMap map[string]string, filter uint6
for _, packageID := range resolved {
index = -1
//log.Printf("* %v", packageID)
//obj.Logf("* %v", packageID)
// format is: name;version;arch;data
s := strings.Split(packageID, ";")
//if len(s) != 4 { continue } // this would be a bug!
pkg, ver, arch, data := s[0], s[1], s[2], s[3]
// we might need to allow some of this, eg: i386 .deb on amd64
if !IsMyArch(arch) {
b, err := IsMyArch(arch)
if err != nil {
return nil, errwrap.Wrapf(err, "arch error")
} else if !b {
continue
}
@@ -748,12 +753,12 @@ func (bus *Conn) PackagesToPackageIDs(packageMap map[string]string, filter uint6
// to be done, and if so, anything that needs updating isn't newest!
// if something isn't installed, we can't verify it with this method
// FIXME: https://github.com/hughsie/PackageKit/issues/116
updates, e := bus.GetUpdates(filter)
updates, e := obj.GetUpdates(filter)
if e != nil {
return nil, fmt.Errorf("Updates error: %v", e)
}
for _, packageID := range updates {
//log.Printf("* %v", packageID)
//obj.Logf("* %v", packageID)
// format is: name;version;arch;data
s := strings.Split(packageID, ";")
//if len(s) != 4 { continue } // this would be a bug!
@@ -792,13 +797,13 @@ func (bus *Conn) PackagesToPackageIDs(packageMap map[string]string, filter uint6
}
// we _could_ do a second resolve and then parse like this...
//resolved, e := bus.ResolvePackages(..., filter+PkFilterEnumNewest)
//resolved, e := obj.ResolvePackages(..., filter+PkFilterEnumNewest)
// but that's basically what recursion here could do too!
if len(checkPackages) > 0 {
if Debug {
log.Printf("PackageKit: PackagesToPackageIDs(): Recurse: %v", strings.Join(checkPackages, ", "))
if obj.Debug {
obj.Logf("PackagesToPackageIDs(): Recurse: %s", strings.Join(checkPackages, ", "))
}
recursion, e = bus.PackagesToPackageIDs(filteredPackageMap, filter+PkFilterEnumNewest)
recursion, e = obj.PackagesToPackageIDs(filteredPackageMap, filter+PkFilterEnumNewest)
if e != nil {
return nil, fmt.Errorf("Recursion error: %v", e)
}
@@ -834,12 +839,12 @@ func (bus *Conn) PackagesToPackageIDs(packageMap map[string]string, filter uint6
func FilterPackageIDs(m map[string]*PkPackageIDActionData, packages []string) ([]string, error) {
result := []string{}
for _, k := range packages {
obj, ok := m[k] // lookup single package
p, ok := m[k] // lookup single package
// package doesn't exist, this is an error!
if !ok || !obj.Found || obj.PackageID == "" {
if !ok || !p.Found || p.PackageID == "" {
return nil, fmt.Errorf("can't find package named '%s'", k)
}
result = append(result, obj.PackageID)
result = append(result, p.PackageID)
}
return result, nil
}
@@ -849,18 +854,18 @@ func FilterState(m map[string]*PkPackageIDActionData, packages []string, state s
result = make(map[string]bool)
pkgs := []string{} // bad pkgs that don't have a bool state
for _, k := range packages {
obj, ok := m[k] // lookup single package
p, ok := m[k] // lookup single package
// package doesn't exist, this is an error!
if !ok || !obj.Found {
if !ok || !p.Found {
return nil, fmt.Errorf("can't find package named '%s'", k)
}
var b bool
if state == "installed" {
b = obj.Installed
b = p.Installed
} else if state == "uninstalled" {
b = !obj.Installed
b = !p.Installed
} else if state == "newest" {
b = obj.Newest
b = p.Newest
} else {
// we can't filter "version" state in this function
pkgs = append(pkgs, k)
@@ -869,7 +874,7 @@ func FilterState(m map[string]*PkPackageIDActionData, packages []string, state s
result[k] = b // save
}
if len(pkgs) > 0 {
err = fmt.Errorf("can't filter non-boolean state on: %v", strings.Join(pkgs, ","))
err = fmt.Errorf("can't filter non-boolean state on: %s", strings.Join(pkgs, ","))
}
return result, err
}
@@ -878,19 +883,19 @@ func FilterState(m map[string]*PkPackageIDActionData, packages []string, state s
func FilterPackageState(m map[string]*PkPackageIDActionData, packages []string, state string) (result []string, err error) {
result = []string{}
for _, k := range packages {
obj, ok := m[k] // lookup single package
p, ok := m[k] // lookup single package
// package doesn't exist, this is an error!
if !ok || !obj.Found {
if !ok || !p.Found {
return nil, fmt.Errorf("can't find package named '%s'", k)
}
b := false
if state == "installed" && obj.Installed {
if state == "installed" && p.Installed {
b = true
} else if state == "uninstalled" && !obj.Installed {
} else if state == "uninstalled" && !p.Installed {
b = true
} else if state == "newest" && obj.Newest {
} else if state == "newest" && p.Newest {
b = true
} else if state == obj.Version {
} else if state == p.Version {
b = true
}
if b {
@@ -917,14 +922,14 @@ func FmtTransactionMethod(method string) string {
}
// IsMyArch determines if a PackageKit architecture matches the current os arch.
func IsMyArch(arch string) bool {
func IsMyArch(arch string) (bool, error) {
goarch, ok := PkArchMap[arch]
if !ok {
// if you get this error, please update the PkArchMap const
log.Fatalf("PackageKit: Arch '%v', not found!", arch)
return false, fmt.Errorf("arch '%s', not found", arch)
}
if goarch == "ANY" { // special value that corresponds to noarch
return true
return true, nil
}
return goarch == runtime.GOARCH
return goarch == runtime.GOARCH, nil
}

163
resources/password.go → engine/resources/password.go
View File

@@ -21,19 +21,20 @@ import (
"crypto/rand"
"fmt"
"io/ioutil"
"log"
"math/big"
"os"
"path"
"strings"
"github.com/purpleidea/mgmt/engine"
"github.com/purpleidea/mgmt/engine/traits"
"github.com/purpleidea/mgmt/recwatch"
errwrap "github.com/pkg/errors"
)
func init() {
RegisterResource("password", func() Res { return &PasswordRes{} })
engine.RegisterResource("password", func() engine.Res { return &PasswordRes{} })
}
const (
@@ -43,43 +44,54 @@ const (
// PasswordRes is a no-op resource that returns a random password string.
type PasswordRes struct {
BaseRes `yaml:",inline"`
traits.Base // add the base methods without re-implementation
// TODO: it could be useful to group our tokens into a single write, and
// as a result, we save inotify watches too!
//traits.Groupable // TODO: this is doable, but probably not very useful
traits.Refreshable
traits.Sendable
init *engine.Init
// FIXME: is uint16 too big?
Length uint16 `yaml:"length"` // number of characters to return
Saved bool // this caches the password in the clear locally
CheckRecovery bool // recovery from integrity checks by re-generating
Password *string // the generated password, read only, do not set!
Length uint16 `yaml:"length"` // number of characters to return
Saved bool // this caches the password in the clear locally
CheckRecovery bool // recovery from integrity checks by re-generating
path string // the path to local storage
recWatcher *recwatch.RecWatcher
}
// Default returns some sensible defaults for this resource.
func (obj *PasswordRes) Default() Res {
func (obj *PasswordRes) Default() engine.Res {
return &PasswordRes{
BaseRes: BaseRes{
MetaParams: DefaultMetaParams, // force a default
},
Length: 64, // safe default
}
}
// Validate if the params passed in are valid data.
func (obj *PasswordRes) Validate() error {
return obj.BaseRes.Validate()
return nil
}
// Init generates a new password for this resource if one was not provided. It
// will save this into a local file. It will load it back in from previous runs.
func (obj *PasswordRes) Init() error {
// Init runs some startup code for this resource. It generates a new password
// for this resource if one was not provided. It will save this into a local
// file. It will load it back in from previous runs.
func (obj *PasswordRes) Init(init *engine.Init) error {
obj.init = init // save for later
dir, err := obj.VarDir("")
dir, err := obj.init.VarDir("")
if err != nil {
return errwrap.Wrapf(err, "could not get VarDir in Init()")
}
obj.path = path.Join(dir, "password") // return a unique file
return obj.BaseRes.Init() // call base init, b/c we're overriding
return nil
}
// Close is run by the engine to clean up after the resource is done.
func (obj *PasswordRes) Close() error {
return nil
}
func (obj *PasswordRes) read() (string, error) {
@@ -171,12 +183,11 @@ func (obj *PasswordRes) Watch() error {
defer obj.recWatcher.Close()
// notify engine that we're running
if err := obj.Running(); err != nil {
return err // bubble up a NACK...
if err := obj.init.Running(); err != nil {
return err // exit if requested
}
var send = false // send event?
var exit *error
for {
select {
// NOTE: this part is very similar to the file resource code
@@ -188,30 +199,33 @@ func (obj *PasswordRes) Watch() error {
return errwrap.Wrapf(err, "unknown %s watcher error", obj)
}
send = true
obj.StateOK(false) // dirty
obj.init.Dirty() // dirty
case event := <-obj.Events():
// we avoid sending events on unpause
if exit, send = obj.ReadEvent(event); exit != nil {
return *exit // exit
case event, ok := <-obj.init.Events:
if !ok {
return nil
}
if err := obj.init.Read(event); err != nil {
return err
}
}
// do all our event sending all together to avoid duplicate msgs
if send {
send = false
obj.Event()
if err := obj.init.Event(); err != nil {
return err // exit if requested
}
}
}
}
// CheckApply method for Password resource. Does nothing, returns happy!
func (obj *PasswordRes) CheckApply(apply bool) (checkOK bool, err error) {
var refresh = obj.Refresh() // do we have a pending reload to apply?
var exists = true // does the file (aka the token) exist?
var generate bool // do we need to generate a new password?
var write bool // do we need to write out to disk?
var refresh = obj.init.Refresh() // do we have a pending reload to apply?
var exists = true // does the file (aka the token) exist?
var generate bool // do we need to generate a new password?
var write bool // do we need to write out to disk?
password, err := obj.read() // password might be empty if just a token
if err != nil {
@@ -226,7 +240,7 @@ func (obj *PasswordRes) CheckApply(apply bool) (checkOK bool, err error) {
if !obj.CheckRecovery {
return false, errwrap.Wrapf(err, "check failed")
}
log.Printf("%s: Integrity check failed", obj)
obj.init.Logf("integrity check failed")
generate = true // okay to build a new one
write = true // make sure to write over the old one
}
@@ -240,9 +254,9 @@ func (obj *PasswordRes) CheckApply(apply bool) (checkOK bool, err error) {
}
// stored password isn't consistent with memory
if p := obj.Password; obj.Saved && (p != nil && *p != password) {
write = true
}
//if p := obj.Password; obj.Saved && (p != nil && *p != password) {
// write = true
//}
if !refresh && exists && !generate && !write { // nothing to do, done!
return true, nil
@@ -260,13 +274,18 @@ func (obj *PasswordRes) CheckApply(apply bool) (checkOK bool, err error) {
}
// generate the actual password
var err error
log.Printf("%s: Generating new password...", obj)
obj.init.Logf("generating new password...")
if password, err = obj.generate(); err != nil { // generate one!
return false, errwrap.Wrapf(err, "could not generate password")
}
}
obj.Password = &password // save in memory
// send
if err := obj.init.Send(&PasswordSends{
Password: &password,
}); err != nil {
return false, err
}
var output string // the string to write out
@@ -277,7 +296,7 @@ func (obj *PasswordRes) CheckApply(apply bool) (checkOK bool, err error) {
output = password
}
// write either an empty token, or the password
log.Printf("%s: Writing password token...", obj)
obj.init.Logf("writing password token...")
if _, err := obj.write(output); err != nil {
return false, errwrap.Wrapf(err, "can't write to file")
}
@@ -286,46 +305,21 @@ func (obj *PasswordRes) CheckApply(apply bool) (checkOK bool, err error) {
return false, nil
}
// PasswordUID is the UID struct for PasswordRes.
type PasswordUID struct {
BaseUID
name string
}
// UIDs includes all params to make a unique identification of this object.
// Most resources only return one, although some resources can return multiple.
func (obj *PasswordRes) UIDs() []ResUID {
x := &PasswordUID{
BaseUID: BaseUID{Name: obj.GetName(), Kind: obj.GetKind()},
name: obj.Name,
// Cmp compares two resources and returns an error if they are not equivalent.
func (obj *PasswordRes) Cmp(r engine.Res) error {
if !obj.Compare(r) {
return fmt.Errorf("did not compare")
}
return []ResUID{x}
}
// GroupCmp returns whether two resources can be grouped together or not.
func (obj *PasswordRes) GroupCmp(r Res) bool {
_, ok := r.(*PasswordRes)
if !ok {
return false
}
return false // TODO: this is doable, but probably not very useful
// TODO: it could be useful to group our tokens into a single write, and
// as a result, we save inotify watches too!
return nil
}
// Compare two resources and return if they are equivalent.
func (obj *PasswordRes) Compare(r Res) bool {
func (obj *PasswordRes) Compare(r engine.Res) bool {
// we can only compare PasswordRes to others of the same resource kind
res, ok := r.(*PasswordRes)
if !ok {
return false
}
if !obj.BaseRes.Compare(res) { // call base Compare
return false
}
if obj.Name != res.Name {
return false
}
if obj.Length != res.Length {
return false
@@ -342,6 +336,37 @@ func (obj *PasswordRes) Compare(r Res) bool {
return true
}
// PasswordUID is the UID struct for PasswordRes.
type PasswordUID struct {
engine.BaseUID
name string
}
// UIDs includes all params to make a unique identification of this object.
// Most resources only return one, although some resources can return multiple.
func (obj *PasswordRes) UIDs() []engine.ResUID {
x := &PasswordUID{
BaseUID: engine.BaseUID{Name: obj.Name(), Kind: obj.Kind()},
name: obj.Name(),
}
return []engine.ResUID{x}
}
// PasswordSends is the struct of data which is sent after a successful Apply.
type PasswordSends struct {
// Password is the generated password being sent.
Password *string
// Hashing is the algorithm used for this password. Empty is plain text.
Hashing string // TODO: implement me
}
// Sends represents the default struct of values we can send using Send/Recv.
func (obj *PasswordRes) Sends() interface{} {
return &PasswordSends{
Password: nil,
}
}
// UnmarshalYAML is the custom unmarshal handler for this struct.
// It is primarily useful for setting the defaults.
func (obj *PasswordRes) UnmarshalYAML(unmarshal func(interface{}) error) error {

235
resources/pkg.go → engine/resources/pkg.go
View File

@@ -19,23 +19,29 @@ package resources
import (
"fmt"
"log"
"path"
"strings"
"github.com/purpleidea/mgmt/resources/packagekit"
"github.com/purpleidea/mgmt/engine"
"github.com/purpleidea/mgmt/engine/resources/packagekit"
"github.com/purpleidea/mgmt/engine/traits"
"github.com/purpleidea/mgmt/util"
errwrap "github.com/pkg/errors"
)
func init() {
RegisterResource("pkg", func() Res { return &PkgRes{} })
engine.RegisterResource("pkg", func() engine.Res { return &PkgRes{} })
}
// PkgRes is a package resource for packagekit.
type PkgRes struct {
BaseRes `yaml:",inline"`
traits.Base // add the base methods without re-implementation
traits.Edgeable
traits.Groupable
init *engine.Init
State string `yaml:"state"` // state: installed, uninstalled, newest, <version>
AllowUntrusted bool `yaml:"allowuntrusted"` // allow untrusted packages to be installed?
AllowNonFree bool `yaml:"allownonfree"` // allow nonfree packages to be found?
@@ -45,11 +51,8 @@ type PkgRes struct {
}
// Default returns some sensible defaults for this resource.
func (obj *PkgRes) Default() Res {
func (obj *PkgRes) Default() engine.Res {
return &PkgRes{
BaseRes: BaseRes{
MetaParams: DefaultMetaParams, // force a default
},
State: "installed", // i think this is preferable to "latest"
}
}
@@ -60,14 +63,12 @@ func (obj *PkgRes) Validate() error {
return fmt.Errorf("state cannot be empty")
}
return obj.BaseRes.Validate()
return nil
}
// Init runs some startup code for this resource.
func (obj *PkgRes) Init() error {
if err := obj.BaseRes.Init(); err != nil { // call base init, b/c we're overriding
return err
}
func (obj *PkgRes) Init(init *engine.Init) error {
obj.init = init // save for later
if obj.fileList == nil {
if err := obj.populateFileList(); err != nil {
@@ -78,6 +79,11 @@ func (obj *PkgRes) Init() error {
return nil
}
// Close is run by the engine to clean up after the resource is done.
func (obj *PkgRes) Close() error {
return nil
}
// Watch is the primary listener for this resource and it outputs events.
// It uses the PackageKit UpdatesChanged signal to watch for changes.
// TODO: https://github.com/hughsie/PackageKit/issues/109
@@ -88,6 +94,10 @@ func (obj *PkgRes) Watch() error {
return fmt.Errorf("can't connect to PackageKit bus")
}
defer bus.Close()
bus.Debug = obj.init.Debug
bus.Logf = func(format string, v ...interface{}) {
obj.init.Logf("packagekit: "+format, v...)
}
ch, err := bus.WatchChanges()
if err != nil {
@@ -95,23 +105,21 @@ func (obj *PkgRes) Watch() error {
}
// notify engine that we're running
if err := obj.Running(); err != nil {
return err // bubble up a NACK...
if err := obj.init.Running(); err != nil {
return err // exit if requested
}
var send = false // send event?
var exit *error
for {
if obj.debug {
log.Printf("%s: Watching...", obj.fmtNames(obj.getNames()))
if obj.init.Debug {
obj.init.Logf("%s: Watching...", obj.fmtNames(obj.getNames()))
}
select {
case event := <-ch:
// FIXME: ask packagekit for info on what packages changed
if obj.debug {
log.Printf("%s: Event: %v", obj.fmtNames(obj.getNames()), event.Name)
if obj.init.Debug {
obj.init.Logf("Event(%s): %s", event.Name, obj.fmtNames(obj.getNames()))
}
// since the chan is buffered, remove any supplemental
@@ -121,20 +129,20 @@ func (obj *PkgRes) Watch() error {
}
send = true
obj.StateOK(false) // dirty
obj.init.Dirty() // dirty
case event := <-obj.Events():
if exit, send = obj.ReadEvent(event); exit != nil {
return *exit // exit
case event := <-obj.init.Events:
if err := obj.init.Read(event); err != nil {
return err
}
//obj.StateOK(false) // these events don't invalidate state
}
// do all our event sending all together to avoid duplicate msgs
if send {
send = false
obj.Event()
if err := obj.init.Event(); err != nil {
return err // exit if requested
}
}
}
}
@@ -142,22 +150,22 @@ func (obj *PkgRes) Watch() error {
// get list of names when grouped or not
func (obj *PkgRes) getNames() []string {
if g := obj.GetGroup(); len(g) > 0 { // grouped elements
names := []string{obj.GetName()}
names := []string{obj.Name()}
for _, x := range g {
pkg, ok := x.(*PkgRes) // convert from Res
if ok {
names = append(names, pkg.Name)
names = append(names, pkg.Name())
}
}
return names
}
return []string{obj.GetName()}
return []string{obj.Name()}
}
// pretty print for header values
func (obj *PkgRes) fmtNames(names []string) string {
if len(obj.GetGroup()) > 0 { // grouped elements
return fmt.Sprintf("%s[autogroup:(%s)]", obj.GetKind(), strings.Join(names, ","))
return fmt.Sprintf("%s[autogroup:(%s)]", obj.Kind(), strings.Join(names, ","))
}
return obj.String()
}
@@ -168,9 +176,9 @@ func (obj *PkgRes) groupMappingHelper() map[string]string {
for _, x := range g {
pkg, ok := x.(*PkgRes) // convert from Res
if !ok {
log.Fatalf("grouped member %v is not a %s", x, obj.GetKind())
panic(fmt.Sprintf("grouped member %v is not a %s", x, obj.Kind()))
}
result[pkg.Name] = pkg.State
result[pkg.Name()] = pkg.State
}
}
return result
@@ -178,7 +186,7 @@ func (obj *PkgRes) groupMappingHelper() map[string]string {
func (obj *PkgRes) pkgMappingHelper(bus *packagekit.Conn) (map[string]*packagekit.PkPackageIDActionData, error) {
packageMap := obj.groupMappingHelper() // get the grouped values
packageMap[obj.Name] = obj.State // key is pkg name, value is pkg state
packageMap[obj.Name()] = obj.State // key is pkg name, value is pkg state
var filter uint64 // initializes at the "zero" value of 0
filter += packagekit.PkFilterEnumArch // always search in our arch (optional!)
// we're requesting latest version, or to narrow down install choices!
@@ -210,16 +218,22 @@ func (obj *PkgRes) populateFileList() error {
return fmt.Errorf("can't connect to PackageKit bus")
}
defer bus.Close()
if obj.init != nil {
bus.Debug = obj.init.Debug
bus.Logf = func(format string, v ...interface{}) {
obj.init.Logf("packagekit: "+format, v...)
}
}
result, err := obj.pkgMappingHelper(bus)
if err != nil {
return errwrap.Wrapf(err, "the pkgMappingHelper failed")
}
data, ok := result[obj.Name] // lookup single package (init does just one)
data, ok := result[obj.Name()] // lookup single package (init does just one)
// package doesn't exist, this is an error!
if !ok || !data.Found {
return fmt.Errorf("can't find package named '%s'", obj.Name)
return fmt.Errorf("can't find package named '%s'", obj.Name())
}
packageIDs := []string{data.PackageID} // just one for now
@@ -237,13 +251,17 @@ func (obj *PkgRes) populateFileList() error {
// CheckApply checks the resource state and applies the resource if the bool
// input is true. It returns error info and if the state check passed or not.
func (obj *PkgRes) CheckApply(apply bool) (checkOK bool, err error) {
log.Printf("%s: Check", obj.fmtNames(obj.getNames()))
obj.init.Logf("Check: %s", obj.fmtNames(obj.getNames()))
bus := packagekit.NewBus()
if bus == nil {
return false, fmt.Errorf("can't connect to PackageKit bus")
}
defer bus.Close()
bus.Debug = obj.init.Debug
bus.Logf = func(format string, v ...interface{}) {
obj.init.Logf("packagekit: "+format, v...)
}
result, err := obj.pkgMappingHelper(bus)
if err != nil {
@@ -251,7 +269,7 @@ func (obj *PkgRes) CheckApply(apply bool) (checkOK bool, err error) {
}
packageMap := obj.groupMappingHelper() // map[string]string
packageList := []string{obj.Name}
packageList := []string{obj.Name()}
packageList = append(packageList, util.StrMapKeys(packageMap)...)
//stateList := []string{obj.State}
//stateList = append(stateList, util.StrMapValues(packageMap)...)
@@ -262,7 +280,7 @@ func (obj *PkgRes) CheckApply(apply bool) (checkOK bool, err error) {
if err != nil {
return false, errwrap.Wrapf(err, "the FilterState method failed")
}
data, _ := result[obj.Name] // if above didn't error, we won't either!
data, _ := result[obj.Name()] // if above didn't error, we won't either!
validState := util.BoolMapTrue(util.BoolMapValues(states))
// obj.State == "installed" || "uninstalled" || "newest" || "4.2-1.fc23"
@@ -287,7 +305,7 @@ func (obj *PkgRes) CheckApply(apply bool) (checkOK bool, err error) {
}
// apply portion
log.Printf("%s: Apply", obj.fmtNames(obj.getNames()))
obj.init.Logf("Apply: %s", obj.fmtNames(obj.getNames()))
readyPackages, err := packagekit.FilterPackageState(result, packageList, obj.State)
if err != nil {
return false, err // fail
@@ -301,7 +319,7 @@ func (obj *PkgRes) CheckApply(apply bool) (checkOK bool, err error) {
transactionFlags += packagekit.PkTransactionFlagEnumOnlyTrusted
}
// apply correct state!
log.Printf("%s: Set: %v...", obj.fmtNames(util.StrListIntersection(applyPackages, obj.getNames())), obj.State)
obj.init.Logf("Set(%s): %s...", obj.State, obj.fmtNames(util.StrListIntersection(applyPackages, obj.getNames())))
switch obj.State {
case "uninstalled": // run remove
// NOTE: packageID is different than when installed, because now
@@ -319,25 +337,61 @@ func (obj *PkgRes) CheckApply(apply bool) (checkOK bool, err error) {
if err != nil {
return false, err // fail
}
log.Printf("%s: Set: %v success!", obj.fmtNames(util.StrListIntersection(applyPackages, obj.getNames())), obj.State)
obj.init.Logf("Set(%s) success: %s", obj.State, obj.fmtNames(util.StrListIntersection(applyPackages, obj.getNames())))
return false, nil // success
}
// Cmp compares two resources and returns an error if they are not equivalent.
func (obj *PkgRes) Cmp(r engine.Res) error {
if !obj.Compare(r) {
return fmt.Errorf("did not compare")
}
return nil
}
// Compare two resources and return if they are equivalent.
func (obj *PkgRes) Compare(r engine.Res) bool {
// we can only compare PkgRes to others of the same resource kind
res, ok := r.(*PkgRes)
if !ok {
return false
}
// if obj.Name != res.Name {
// return false
// }
if obj.State != res.State {
return false
}
if obj.AllowUntrusted != res.AllowUntrusted {
return false
}
if obj.AllowNonFree != res.AllowNonFree {
return false
}
if obj.AllowUnsupported != res.AllowUnsupported {
return false
}
return true
}
// PkgUID is the main UID struct for PkgRes.
type PkgUID struct {
BaseUID
engine.BaseUID
name string // pkg name
state string // pkg state or "version"
}
// PkgFileUID is the UID struct for PkgRes files.
type PkgFileUID struct {
BaseUID
engine.BaseUID
path string // path of the file
}
// IFF aka if and only if they are equivalent, return true. If not, false.
func (obj *PkgUID) IFF(uid ResUID) bool {
func (obj *PkgUID) IFF(uid engine.ResUID) bool {
res, ok := uid.(*PkgUID)
if !ok {
return false
@@ -349,16 +403,16 @@ func (obj *PkgUID) IFF(uid ResUID) bool {
// PkgResAutoEdges holds the state of the auto edge generator.
type PkgResAutoEdges struct {
fileList []string
svcUIDs []ResUID
svcUIDs []engine.ResUID
testIsNext bool // safety
name string // saved data from PkgRes obj
kind string
}
// Next returns the next automatic edge.
func (obj *PkgResAutoEdges) Next() []ResUID {
func (obj *PkgResAutoEdges) Next() []engine.ResUID {
if obj.testIsNext {
log.Fatal("expecting a call to Test()")
panic("expecting a call to Test()")
}
obj.testIsNext = true // set after all the errors paths are past
@@ -367,12 +421,12 @@ func (obj *PkgResAutoEdges) Next() []ResUID {
return x
}
var result []ResUID
var result []engine.ResUID
// return UID's for whatever is in obj.fileList
for _, x := range obj.fileList {
var reversed = false // cheat by passing a pointer
result = append(result, &FileUID{
BaseUID: BaseUID{
BaseUID: engine.BaseUID{
Name: obj.name,
Kind: obj.kind,
Reversed: &reversed,
@@ -386,22 +440,22 @@ func (obj *PkgResAutoEdges) Next() []ResUID {
// Test gets results of the earlier Next() call, & returns if we should continue!
func (obj *PkgResAutoEdges) Test(input []bool) bool {
if !obj.testIsNext {
log.Fatal("expecting a call to Next()")
panic("expecting a call to Next()")
}
// ack the svcUID's...
if x := obj.svcUIDs; len(x) > 0 {
if y := len(x); y != len(input) {
log.Fatalf("expecting %d value(s)", y)
panic(fmt.Sprintf("expecting %d value(s)", y))
}
obj.svcUIDs = []ResUID{} // empty
obj.svcUIDs = []engine.ResUID{} // empty
obj.testIsNext = false
return true
}
count := len(obj.fileList)
if count != len(input) {
log.Fatalf("expecting %d value(s)", count)
panic(fmt.Sprintf("expecting %d value(s)", count))
}
obj.testIsNext = false // set after all the errors paths are past
@@ -436,7 +490,7 @@ func (obj *PkgResAutoEdges) Test(input []bool) bool {
// AutoEdges produces an object which generates a minimal pkg file optimization
// sequence of edges.
func (obj *PkgRes) AutoEdges() (AutoEdge, error) {
func (obj *PkgRes) AutoEdges() (engine.AutoEdge, error) {
// in contrast with the FileRes AutoEdges() function which contains
// more of the mechanics, most of the AutoEdge mechanics for the PkgRes
// are contained in the Test() method! This design is completely okay!
@@ -448,13 +502,13 @@ func (obj *PkgRes) AutoEdges() (AutoEdge, error) {
}
// add matches for any svc resources found in pkg definition!
var svcUIDs []ResUID
var svcUIDs []engine.ResUID
for _, x := range ReturnSvcInFileList(obj.fileList) {
var reversed = false
svcUIDs = append(svcUIDs, &SvcUID{
BaseUID: BaseUID{
Name: obj.GetName(),
Kind: obj.GetKind(),
BaseUID: engine.BaseUID{
Name: obj.Name(),
Kind: obj.Kind(),
Reversed: &reversed,
},
name: x, // the svc name itself in the SvcUID object!
@@ -464,25 +518,25 @@ func (obj *PkgRes) AutoEdges() (AutoEdge, error) {
return &PkgResAutoEdges{
fileList: util.RemoveCommonFilePrefixes(obj.fileList), // clean start!
svcUIDs: svcUIDs,
testIsNext: false, // start with Next() call
name: obj.GetName(), // save data for PkgResAutoEdges obj
kind: obj.GetKind(),
testIsNext: false, // start with Next() call
name: obj.Name(), // save data for PkgResAutoEdges obj
kind: obj.Kind(),
}, nil
}
// UIDs includes all params to make a unique identification of this object.
// Most resources only return one, although some resources can return multiple.
func (obj *PkgRes) UIDs() []ResUID {
func (obj *PkgRes) UIDs() []engine.ResUID {
x := &PkgUID{
BaseUID: BaseUID{Name: obj.GetName(), Kind: obj.GetKind()},
name: obj.Name,
BaseUID: engine.BaseUID{Name: obj.Name(), Kind: obj.Kind()},
name: obj.Name(),
state: obj.State,
}
result := []ResUID{x}
result := []engine.ResUID{x}
for _, y := range obj.fileList {
y := &PkgFileUID{
BaseUID: BaseUID{Name: obj.GetName(), Kind: obj.GetKind()},
BaseUID: engine.BaseUID{Name: obj.Name(), Kind: obj.Kind()},
path: y,
}
result = append(result, y)
@@ -491,55 +545,24 @@ func (obj *PkgRes) UIDs() []ResUID {
}
// GroupCmp returns whether two resources can be grouped together or not.
// can these two resources be merged ?
// (aka does this resource support doing so?)
// will resource allow itself to be grouped _into_ this obj?
func (obj *PkgRes) GroupCmp(r Res) bool {
// Can these two resources be merged, aka, does this resource support doing so?
// Will resource allow itself to be grouped _into_ this obj?
func (obj *PkgRes) GroupCmp(r engine.GroupableRes) error {
res, ok := r.(*PkgRes)
if !ok {
return false
return fmt.Errorf("resource is not the same kind")
}
objStateIsVersion := (obj.State != "installed" && obj.State != "uninstalled" && obj.State != "newest") // must be a ver. string
resStateIsVersion := (res.State != "installed" && res.State != "uninstalled" && res.State != "newest") // must be a ver. string
if objStateIsVersion || resStateIsVersion {
// can't merge specific version checks atm
return false
return fmt.Errorf("resource uses a version string")
}
// FIXME: keep it simple for now, only merge same states
if obj.State != res.State {
return false
return fmt.Errorf("resource is of a different state")
}
return true
}
// Compare two resources and return if they are equivalent.
func (obj *PkgRes) Compare(r Res) bool {
// we can only compare PkgRes to others of the same resource kind
res, ok := r.(*PkgRes)
if !ok {
return false
}
if !obj.BaseRes.Compare(res) { // call base Compare
return false
}
if obj.Name != res.Name {
return false
}
if obj.State != res.State {
return false
}
if obj.AllowUntrusted != res.AllowUntrusted {
return false
}
if obj.AllowNonFree != res.AllowNonFree {
return false
}
if obj.AllowUnsupported != res.AllowUnsupported {
return false
}
return true
return nil
}
// UnmarshalYAML is the custom unmarshal handler for this struct.

0
resources/pkg_test.go → engine/resources/pkg_test.go
View File

129
resources/print.go → engine/resources/print.go
View File

@@ -19,130 +19,118 @@ package resources
import (
"fmt"
"log"
"github.com/purpleidea/mgmt/engine"
"github.com/purpleidea/mgmt/engine/traits"
)
func init() {
RegisterResource("print", func() Res { return &PrintRes{} })
engine.RegisterResource("print", func() engine.Res { return &PrintRes{} })
}
// PrintRes is a resource that is useful for printing a message to the screen.
// It will also display a message when it receives a notification. It supports
// automatic grouping.
type PrintRes struct {
BaseRes `lang:"" yaml:",inline"`
traits.Base // add the base methods without re-implementation
traits.Groupable
traits.Refreshable
init *engine.Init
Msg string `lang:"msg" yaml:"msg"` // the message to display
}
// Default returns some sensible defaults for this resource.
func (obj *PrintRes) Default() Res {
return &PrintRes{
BaseRes: BaseRes{
MetaParams: DefaultMetaParams, // force a default
},
}
func (obj *PrintRes) Default() engine.Res {
return &PrintRes{}
}
// Validate if the params passed in are valid data.
func (obj *PrintRes) Validate() error {
return obj.BaseRes.Validate()
return nil
}
// Init runs some startup code for this resource.
func (obj *PrintRes) Init() error {
return obj.BaseRes.Init() // call base init, b/c we're overriding
func (obj *PrintRes) Init(init *engine.Init) error {
obj.init = init // save for later
return nil
}
// Close is run by the engine to clean up after the resource is done.
func (obj *PrintRes) Close() error {
return nil
}
// Watch is the primary listener for this resource and it outputs events.
func (obj *PrintRes) Watch() error {
// notify engine that we're running
if err := obj.Running(); err != nil {
return err // bubble up a NACK...
if err := obj.init.Running(); err != nil {
return err // exit if requested
}
var send = false // send event?
var exit *error
for {
select {
case event := <-obj.Events():
// we avoid sending events on unpause
if exit, send = obj.ReadEvent(event); exit != nil {
return *exit // exit
case event, ok := <-obj.init.Events:
if !ok {
return nil
}
if err := obj.init.Read(event); err != nil {
return err
}
}
// do all our event sending all together to avoid duplicate msgs
if send {
send = false
obj.Event()
if err := obj.init.Event(); err != nil {
return err // exit if requested
}
}
}
}
// CheckApply method for Print resource. Does nothing, returns happy!
func (obj *PrintRes) CheckApply(apply bool) (checkOK bool, err error) {
log.Printf("%s: CheckApply: %t", obj, apply)
if val, exists := obj.Recv["Msg"]; exists && val.Changed {
obj.init.Logf("CheckApply: %t", apply)
if val, exists := obj.init.Recv()["Msg"]; exists && val.Changed {
// if we received on Msg, and it changed, log message
log.Printf("CheckApply: Received `Msg` of: %s", obj.Msg)
obj.init.Logf("CheckApply: Received `Msg` of: %s", obj.Msg)
}
if obj.Refresh() {
log.Printf("%s: Received a notification!", obj)
if obj.init.Refresh() {
obj.init.Logf("Received a notification!")
}
log.Printf("%s: Msg: %s", obj, obj.Msg)
obj.init.Logf("Msg: %s", obj.Msg)
if g := obj.GetGroup(); len(g) > 0 { // add any grouped elements
for _, x := range g {
print, ok := x.(*PrintRes) // convert from Res
if !ok {
log.Fatalf("grouped member %v is not a %s", x, obj.GetKind())
panic(fmt.Sprintf("grouped member %v is not a %s", x, obj.Kind()))
}
log.Printf("%s: Msg: %s", print, print.Msg)
obj.init.Logf("%s: Msg: %s", print, print.Msg)
}
}
return true, nil // state is always okay
}
// PrintUID is the UID struct for PrintRes.
type PrintUID struct {
BaseUID
name string
}
// UIDs includes all params to make a unique identification of this object.
// Most resources only return one, although some resources can return multiple.
func (obj *PrintRes) UIDs() []ResUID {
x := &PrintUID{
BaseUID: BaseUID{Name: obj.GetName(), Kind: obj.GetKind()},
name: obj.Name,
// Cmp compares two resources and returns an error if they are not equivalent.
func (obj *PrintRes) Cmp(r engine.Res) error {
if !obj.Compare(r) {
return fmt.Errorf("did not compare")
}
return []ResUID{x}
}
// GroupCmp returns whether two resources can be grouped together or not.
func (obj *PrintRes) GroupCmp(r Res) bool {
_, ok := r.(*PrintRes)
if !ok {
return false
}
return true // grouped together if we were asked to
return nil
}
// Compare two resources and return if they are equivalent.
func (obj *PrintRes) Compare(r Res) bool {
func (obj *PrintRes) Compare(r engine.Res) bool {
// we can only compare PrintRes to others of the same resource kind
res, ok := r.(*PrintRes)
if !ok {
return false
}
// calling base Compare is probably unneeded for the print res, but do it
if !obj.BaseRes.Compare(res) { // call base Compare
return false
}
if obj.Name != res.Name {
return false
}
if obj.Msg != res.Msg {
return false
@@ -150,6 +138,31 @@ func (obj *PrintRes) Compare(r Res) bool {
return true
}
// PrintUID is the UID struct for PrintRes.
type PrintUID struct {
engine.BaseUID
name string
}
// UIDs includes all params to make a unique identification of this object.
// Most resources only return one, although some resources can return multiple.
func (obj *PrintRes) UIDs() []engine.ResUID {
x := &PrintUID{
BaseUID: engine.BaseUID{Name: obj.Name(), Kind: obj.Kind()},
name: obj.Name(),
}
return []engine.ResUID{x}
}
// GroupCmp returns whether two resources can be grouped together or not.
func (obj *PrintRes) GroupCmp(r engine.GroupableRes) error {
_, ok := r.(*PrintRes)
if !ok {
return fmt.Errorf("resource is not the same kind")
}
return nil // grouped together if we were asked to
}
// UnmarshalYAML is the custom unmarshal handler for this struct.
// It is primarily useful for setting the defaults.
func (obj *PrintRes) UnmarshalYAML(unmarshal func(interface{}) error) error {

331
resources/svc.go → engine/resources/svc.go
View File

@@ -21,8 +21,9 @@ package resources
import (
"fmt"
"log"
"github.com/purpleidea/mgmt/engine"
"github.com/purpleidea/mgmt/engine/traits"
"github.com/purpleidea/mgmt/util"
systemd "github.com/coreos/go-systemd/dbus" // change namespace
@@ -32,24 +33,26 @@ import (
)
func init() {
RegisterResource("svc", func() Res { return &SvcRes{} })
engine.RegisterResource("svc", func() engine.Res { return &SvcRes{} })
}
// SvcRes is a service resource for systemd units.
type SvcRes struct {
BaseRes `yaml:",inline"`
traits.Base // add the base methods without re-implementation
traits.Edgeable
traits.Groupable
traits.Refreshable
init *engine.Init
State string `yaml:"state"` // state: running, stopped, undefined
Startup string `yaml:"startup"` // enabled, disabled, undefined
Session bool `yaml:"session"` // user session (true) or system?
}
// Default returns some sensible defaults for this resource.
func (obj *SvcRes) Default() Res {
return &SvcRes{
BaseRes: BaseRes{
MetaParams: DefaultMetaParams, // force a default
},
}
func (obj *SvcRes) Default() engine.Res {
return &SvcRes{}
}
// Validate checks if the resource data structure was populated correctly.
@@ -60,12 +63,19 @@ func (obj *SvcRes) Validate() error {
if obj.Startup != "enabled" && obj.Startup != "disabled" && obj.Startup != "" {
return fmt.Errorf("startup must be either `enabled` or `disabled` or undefined")
}
return obj.BaseRes.Validate()
return nil
}
// Init runs some startup code for this resource.
func (obj *SvcRes) Init() error {
return obj.BaseRes.Init() // call base init, b/c we're overriding
func (obj *SvcRes) Init(init *engine.Init) error {
obj.init = init // save for later
return nil
}
// Close is run by the engine to clean up after the resource is done.
func (obj *SvcRes) Close() error {
return nil
}
// Watch is the primary listener for this resource and it outputs events.
@@ -101,16 +111,15 @@ func (obj *SvcRes) Watch() error {
bus.Signal(buschan)
// notify engine that we're running
if err := obj.Running(); err != nil {
return err // bubble up a NACK...
if err := obj.init.Running(); err != nil {
return err // exit if requested
}
var svc = fmt.Sprintf("%s.service", obj.Name) // systemd name
var send = false // send event?
var exit *error
var invalid = false // does the svc exist or not?
var previous bool // previous invalid value
set := conn.NewSubscriptionSet() // no error should be returned
var svc = fmt.Sprintf("%s.service", obj.Name()) // systemd name
var send = false // send event?
var invalid = false // does the svc exist or not?
var previous bool // previous invalid value
set := conn.NewSubscriptionSet() // no error should be returned
subChannel, subErrors := set.Subscribe()
var activeSet = false
@@ -124,25 +133,25 @@ func (obj *SvcRes) Watch() error {
// firstly, does svc even exist or not?
loadstate, err := conn.GetUnitProperty(svc, "LoadState")
if err != nil {
log.Printf("Failed to get property: %v", err)
obj.init.Logf("failed to get property: %+v", err)
invalid = true
}
if !invalid {
var notFound = (loadstate.Value == dbus.MakeVariant("not-found"))
if notFound { // XXX: in the loop we'll handle changes better...
log.Printf("Failed to find svc: %s", svc)
obj.init.Logf("failed to find svc")
invalid = true // XXX: ?
}
}
if previous != invalid { // if invalid changed, send signal
send = true
obj.StateOK(false) // dirty
obj.init.Dirty() // dirty
}
if invalid {
log.Printf("Waiting for: %s", svc) // waiting for svc to appear...
obj.init.Logf("waiting fo service") // waiting for svc to appear...
if activeSet {
activeSet = false
set.Remove(svc) // no return value should ever occur
@@ -151,11 +160,11 @@ func (obj *SvcRes) Watch() error {
select {
case <-buschan: // XXX: wait for new units event to unstick
// loop so that we can see the changed invalid signal
log.Printf("Svc[%s]->DaemonReload()", svc)
obj.init.Logf("daemon reload")
case event := <-obj.Events():
if exit, send = obj.ReadEvent(event); exit != nil {
return *exit // exit
case event := <-obj.init.Events:
if err := obj.init.Read(event); err != nil {
return err
}
}
} else {
@@ -164,51 +173,53 @@ func (obj *SvcRes) Watch() error {
set.Add(svc) // no return value should ever occur
}
log.Printf("Watching: %s", svc) // attempting to watch...
obj.init.Logf("watching...") // attempting to watch...
select {
case event := <-subChannel:
log.Printf("Svc event: %+v", event)
obj.init.Logf("event: %+v", event)
// NOTE: the value returned is a map for some reason...
if event[svc] != nil {
// event[svc].ActiveState is not nil
switch event[svc].ActiveState {
case "active":
log.Printf("Svc[%s]->Started", svc)
obj.init.Logf("started")
case "inactive":
log.Printf("Svc[%s]->Stopped", svc)
obj.init.Logf("stopped")
case "reloading":
log.Printf("Svc[%s]->Reloading", svc)
obj.init.Logf("reloading")
case "failed":
log.Printf("Svc[%s]->Failed", svc)
obj.init.Logf("failed")
case "activating":
log.Printf("Svc[%s]->Activating", svc)
obj.init.Logf("activating")
case "deactivating":
log.Printf("Svc[%s]->Deactivating", svc)
obj.init.Logf("deactivating")
default:
return fmt.Errorf("Unknown svc state: %s", event[svc].ActiveState)
return fmt.Errorf("unknown svc state: %s", event[svc].ActiveState)
}
} else {
// svc stopped (and ActiveState is nil...)
log.Printf("Svc[%s]->Stopped", svc)
obj.init.Logf("stopped")
}
send = true
obj.StateOK(false) // dirty
obj.init.Dirty() // dirty
case err := <-subErrors:
return errwrap.Wrapf(err, "unknown %s error", obj)
case event := <-obj.Events():
if exit, send = obj.ReadEvent(event); exit != nil {
return *exit // exit
case event := <-obj.init.Events:
if err := obj.init.Read(event); err != nil {
return err
}
}
}
if send {
send = false
obj.Event()
if err := obj.init.Event(); err != nil {
return err // exit if requested
}
}
}
}
@@ -232,7 +243,7 @@ func (obj *SvcRes) CheckApply(apply bool) (checkOK bool, err error) {
}
defer conn.Close()
var svc = fmt.Sprintf("%s.service", obj.Name) // systemd name
var svc = fmt.Sprintf("%s.service", obj.Name()) // systemd name
loadstate, err := conn.GetUnitProperty(svc, "LoadState")
if err != nil {
@@ -255,8 +266,8 @@ func (obj *SvcRes) CheckApply(apply bool) (checkOK bool, err error) {
var running = (activestate.Value == dbus.MakeVariant("active"))
var stateOK = ((obj.State == "") || (obj.State == "running" && running) || (obj.State == "stopped" && !running))
var startupOK = true // XXX: DETECT AND SET
var refresh = obj.Refresh() // do we have a pending reload to apply?
var startupOK = true // XXX: DETECT AND SET
var refresh = obj.init.Refresh() // do we have a pending reload to apply?
if stateOK && startupOK && !refresh {
return true, nil // we are in the correct state
@@ -268,7 +279,7 @@ func (obj *SvcRes) CheckApply(apply bool) (checkOK bool, err error) {
}
// apply portion
log.Printf("%s: Apply", obj)
obj.init.Logf("Apply")
var files = []string{svc} // the svc represented in a list
if obj.Startup == "enabled" {
_, _, err = conn.EnableUnitFiles(files, false, true)
@@ -290,7 +301,7 @@ func (obj *SvcRes) CheckApply(apply bool) (checkOK bool, err error) {
return false, errwrap.Wrapf(err, "failed to start unit")
}
if refresh {
log.Printf("%s: Skipping reload, due to pending start", obj)
obj.init.Logf("Skipping reload, due to pending start")
}
refresh = false // we did a start, so a reload is not needed
} else if obj.State == "stopped" {
@@ -299,7 +310,7 @@ func (obj *SvcRes) CheckApply(apply bool) (checkOK bool, err error) {
return false, errwrap.Wrapf(err, "failed to stop unit")
}
if refresh {
log.Printf("%s: Skipping reload, due to pending stop", obj)
obj.init.Logf("Skipping reload, due to pending stop")
}
refresh = false // we did a stop, so a reload is not needed
}
@@ -314,7 +325,7 @@ func (obj *SvcRes) CheckApply(apply bool) (checkOK bool, err error) {
if refresh { // we need to reload the service
// XXX: run a svc reload here!
log.Printf("%s: Reloading...", obj)
obj.init.Logf("Reloading...")
}
// XXX: also set enabled on boot
@@ -322,124 +333,21 @@ func (obj *SvcRes) CheckApply(apply bool) (checkOK bool, err error) {
return false, nil // success
}
// SvcUID is the UID struct for SvcRes.
type SvcUID struct {
// NOTE: there is also a name variable in the BaseUID struct, this is
// information about where this UID came from, and is unrelated to the
// information about the resource we're matching. That data which is
// used in the IFF function, is what you see in the struct fields here.
BaseUID
name string // the svc name
}
// IFF aka if and only if they are equivalent, return true. If not, false.
func (obj *SvcUID) IFF(uid ResUID) bool {
res, ok := uid.(*SvcUID)
if !ok {
return false
// Cmp compares two resources and returns an error if they are not equivalent.
func (obj *SvcRes) Cmp(r engine.Res) error {
if !obj.Compare(r) {
return fmt.Errorf("did not compare")
}
return obj.name == res.name
}
// SvcResAutoEdges holds the state of the auto edge generator.
type SvcResAutoEdges struct {
data []ResUID
pointer int
found bool
}
// Next returns the next automatic edge.
func (obj *SvcResAutoEdges) Next() []ResUID {
if obj.found {
log.Fatal("shouldn't be called anymore!")
}
if len(obj.data) == 0 { // check length for rare scenarios
return nil
}
value := obj.data[obj.pointer]
obj.pointer++
return []ResUID{value} // we return one, even though api supports N
}
// Test gets results of the earlier Next() call, & returns if we should continue!
func (obj *SvcResAutoEdges) Test(input []bool) bool {
// if there aren't any more remaining
if len(obj.data) <= obj.pointer {
return false
}
if obj.found { // already found, done!
return false
}
if len(input) != 1 { // in case we get given bad data
log.Fatal("expecting a single value")
}
if input[0] { // if a match is found, we're done!
obj.found = true // no more to find!
return false
}
return true // keep going
}
// AutoEdges returns the AutoEdge interface. In this case the systemd units.
func (obj *SvcRes) AutoEdges() (AutoEdge, error) {
var data []ResUID
svcFiles := []string{
fmt.Sprintf("/etc/systemd/system/%s.service", obj.Name), // takes precedence
fmt.Sprintf("/usr/lib/systemd/system/%s.service", obj.Name), // pkg default
}
for _, x := range svcFiles {
var reversed = true
data = append(data, &FileUID{
BaseUID: BaseUID{
Name: obj.GetName(),
Kind: obj.GetKind(),
Reversed: &reversed,
},
path: x, // what matters
})
}
return &FileResAutoEdges{
data: data,
pointer: 0,
found: false,
}, nil
}
// UIDs includes all params to make a unique identification of this object.
// Most resources only return one, although some resources can return multiple.
func (obj *SvcRes) UIDs() []ResUID {
x := &SvcUID{
BaseUID: BaseUID{Name: obj.GetName(), Kind: obj.GetKind()},
name: obj.Name, // svc name
}
return []ResUID{x}
}
// GroupCmp returns whether two resources can be grouped together or not.
func (obj *SvcRes) GroupCmp(r Res) bool {
_, ok := r.(*SvcRes)
if !ok {
return false
}
// TODO: depending on if the systemd service api allows batching, we
// might be able to build this, although not sure how useful it is...
// it might just eliminate parallelism be bunching up the graph
return false // not possible atm
return nil
}
// Compare two resources and return if they are equivalent.
func (obj *SvcRes) Compare(r Res) bool {
func (obj *SvcRes) Compare(r engine.Res) bool {
// we can only compare SvcRes to others of the same resource kind
res, ok := r.(*SvcRes)
if !ok {
return false
}
if !obj.BaseRes.Compare(res) { // call base Compare
return false
}
if obj.Name != res.Name {
return false
}
if obj.State != res.State {
return false
@@ -454,6 +362,111 @@ func (obj *SvcRes) Compare(r Res) bool {
return true
}
// SvcUID is the UID struct for SvcRes.
type SvcUID struct {
// NOTE: there is also a name variable in the BaseUID struct, this is
// information about where this UID came from, and is unrelated to the
// information about the resource we're matching. That data which is
// used in the IFF function, is what you see in the struct fields here.
engine.BaseUID
name string // the svc name
}
// IFF aka if and only if they are equivalent, return true. If not, false.
func (obj *SvcUID) IFF(uid engine.ResUID) bool {
res, ok := uid.(*SvcUID)
if !ok {
return false
}
return obj.name == res.name
}
// SvcResAutoEdges holds the state of the auto edge generator.
type SvcResAutoEdges struct {
data []engine.ResUID
pointer int
found bool
}
// Next returns the next automatic edge.
func (obj *SvcResAutoEdges) Next() []engine.ResUID {
if obj.found {
panic("shouldn't be called anymore!")
}
if len(obj.data) == 0 { // check length for rare scenarios
return nil
}
value := obj.data[obj.pointer]
obj.pointer++
return []engine.ResUID{value} // we return one, even though api supports N
}
// Test gets results of the earlier Next() call, & returns if we should continue!
func (obj *SvcResAutoEdges) Test(input []bool) bool {
// if there aren't any more remaining
if len(obj.data) <= obj.pointer {
return false
}
if obj.found { // already found, done!
return false
}
if len(input) != 1 { // in case we get given bad data
panic("expecting a single value")
}
if input[0] { // if a match is found, we're done!
obj.found = true // no more to find!
return false
}
return true // keep going
}
// AutoEdges returns the AutoEdge interface. In this case the systemd units.
func (obj *SvcRes) AutoEdges() (engine.AutoEdge, error) {
var data []engine.ResUID
svcFiles := []string{
fmt.Sprintf("/etc/systemd/system/%s.service", obj.Name()), // takes precedence
fmt.Sprintf("/usr/lib/systemd/system/%s.service", obj.Name()), // pkg default
}
for _, x := range svcFiles {
var reversed = true
data = append(data, &FileUID{
BaseUID: engine.BaseUID{
Name: obj.Name(),
Kind: obj.Kind(),
Reversed: &reversed,
},
path: x, // what matters
})
}
return &FileResAutoEdges{
data: data,
pointer: 0,
found: false,
}, nil
}
// UIDs includes all params to make a unique identification of this object.
// Most resources only return one, although some resources can return multiple.
func (obj *SvcRes) UIDs() []engine.ResUID {
x := &SvcUID{
BaseUID: engine.BaseUID{Name: obj.Name(), Kind: obj.Kind()},
name: obj.Name(), // svc name
}
return []engine.ResUID{x}
}
// GroupCmp returns whether two resources can be grouped together or not.
//func (obj *SvcRes) GroupCmp(r engine.GroupableRes) error {
// _, ok := r.(*SvcRes)
// if !ok {
// return fmt.Errorf("resource is not the same kind")
// }
// // TODO: depending on if the systemd service api allows batching, we
// // might be able to build this, although not sure how useful it is...
// // it might just eliminate parallelism by bunching up the graph
// return fmt.Errorf("not possible at the moment")
//}
// UnmarshalYAML is the custom unmarshal handler for this struct.
// It is primarily useful for setting the defaults.
func (obj *SvcRes) UnmarshalYAML(unmarshal func(interface{}) error) error {

219
resources/test.go → engine/resources/test.go
View File

@@ -19,17 +19,25 @@ package resources
import (
"fmt"
"log"
"reflect"
"github.com/purpleidea/mgmt/engine"
"github.com/purpleidea/mgmt/engine/traits"
)
func init() {
RegisterResource("test", func() Res { return &TestRes{} })
engine.RegisterResource("test", func() engine.Res { return &TestRes{} })
}
// TestRes is a resource that is mostly harmless and is used for internal tests.
type TestRes struct {
BaseRes `lang:"" yaml:",inline"`
traits.Base // add the base methods without re-implementation
traits.Groupable
traits.Refreshable
traits.Sendable
traits.Recvable
init *engine.Init
Bool bool `lang:"bool" yaml:"bool"`
Str string `lang:"str" yaml:"str"` // can't name it String because of String()
@@ -80,6 +88,7 @@ type TestRes struct {
ValidateError string `lang:"validateerror" yaml:"validate_error"` // set to cause a validate error
AlwaysGroup bool `lang:"alwaysgroup" yaml:"always_group"` // set to true to cause auto grouping
CompareFail bool `lang:"comparefail" yaml:"compare_fail"` // will compare fail?
SendValue string `lang:"sendvalue" yaml:"send_value"` // what value should we send?
// TODO: add more fun properties!
@@ -87,12 +96,8 @@ type TestRes struct {
}
// Default returns some sensible defaults for this resource.
func (obj *TestRes) Default() Res {
return &TestRes{
BaseRes: BaseRes{
MetaParams: DefaultMetaParams, // force a default
},
}
func (obj *TestRes) Default() engine.Res {
return &TestRes{}
}
// Validate if the params passed in are valid data.
@@ -103,128 +108,129 @@ func (obj *TestRes) Validate() error {
if s := obj.ValidateError; s != "" {
return fmt.Errorf("the validate error param was set to: %s", s)
}
return obj.BaseRes.Validate()
return nil
}
// Init runs some startup code for this resource.
func (obj *TestRes) Init() error {
return obj.BaseRes.Init() // call base init, b/c we're overriding
func (obj *TestRes) Init(init *engine.Init) error {
obj.init = init // save for later
return nil
}
// Close is run by the engine to clean up after the resource is done.
func (obj *TestRes) Close() error {
return nil
}
// Watch is the primary listener for this resource and it outputs events.
func (obj *TestRes) Watch() error {
// notify engine that we're running
if err := obj.Running(); err != nil {
return err // bubble up a NACK...
if err := obj.init.Running(); err != nil {
return err // exit if requested
}
var send = false // send event?
var exit *error
for {
select {
case event := <-obj.Events():
// we avoid sending events on unpause
if exit, send = obj.ReadEvent(event); exit != nil {
return *exit // exit
case event, ok := <-obj.init.Events:
if !ok {
return nil
}
if err := obj.init.Read(event); err != nil {
return err
}
}
// do all our event sending all together to avoid duplicate msgs
if send {
send = false
obj.Event()
if err := obj.init.Event(); err != nil {
return err // exit if requested
}
}
}
}
// CheckApply method for Test resource. Does nothing, returns happy!
func (obj *TestRes) CheckApply(apply bool) (checkOK bool, err error) {
log.Printf("%s: CheckApply: %t", obj, apply)
if obj.Refresh() {
log.Printf("%s: Received a notification!", obj)
for key, val := range obj.init.Recv() {
obj.init.Logf("CheckApply: Received `%s`, changed: %t", key, val.Changed)
}
log.Printf("%s: Bool: %v", obj, obj.Bool)
log.Printf("%s: Str: %v", obj, obj.Str)
if obj.init.Refresh() {
obj.init.Logf("Received a notification!")
}
log.Printf("%s: Int: %v", obj, obj.Int)
log.Printf("%s: Int8: %v", obj, obj.Int8)
log.Printf("%s: Int16: %v", obj, obj.Int16)
log.Printf("%s: Int32: %v", obj, obj.Int32)
log.Printf("%s: Int64: %v", obj, obj.Int64)
obj.init.Logf("%s: Bool: %v", obj, obj.Bool)
obj.init.Logf("%s: Str: %v", obj, obj.Str)
log.Printf("%s: Uint: %v", obj, obj.Uint)
log.Printf("%s: Uint8: %v", obj, obj.Uint)
log.Printf("%s: Uint16: %v", obj, obj.Uint)
log.Printf("%s: Uint32: %v", obj, obj.Uint)
log.Printf("%s: Uint64: %v", obj, obj.Uint)
obj.init.Logf("%s: Int: %v", obj, obj.Int)
obj.init.Logf("%s: Int8: %v", obj, obj.Int8)
obj.init.Logf("%s: Int16: %v", obj, obj.Int16)
obj.init.Logf("%s: Int32: %v", obj, obj.Int32)
obj.init.Logf("%s: Int64: %v", obj, obj.Int64)
//log.Printf("%s: Uintptr: %v", obj, obj.Uintptr)
log.Printf("%s: Byte: %v", obj, obj.Byte)
log.Printf("%s: Rune: %v", obj, obj.Rune)
obj.init.Logf("%s: Uint: %v", obj, obj.Uint)
obj.init.Logf("%s: Uint8: %v", obj, obj.Uint)
obj.init.Logf("%s: Uint16: %v", obj, obj.Uint)
obj.init.Logf("%s: Uint32: %v", obj, obj.Uint)
obj.init.Logf("%s: Uint64: %v", obj, obj.Uint)
log.Printf("%s: Float32: %v", obj, obj.Float32)
log.Printf("%s: Float64: %v", obj, obj.Float64)
log.Printf("%s: Complex64: %v", obj, obj.Complex64)
log.Printf("%s: Complex128: %v", obj, obj.Complex128)
//obj.init.Logf("%s: Uintptr: %v", obj, obj.Uintptr)
obj.init.Logf("%s: Byte: %v", obj, obj.Byte)
obj.init.Logf("%s: Rune: %v", obj, obj.Rune)
log.Printf("%s: BoolPtr: %v", obj, obj.BoolPtr)
log.Printf("%s: StringPtr: %v", obj, obj.StringPtr)
log.Printf("%s: Int64Ptr: %v", obj, obj.Int64Ptr)
log.Printf("%s: Int8Ptr: %v", obj, obj.Int8Ptr)
log.Printf("%s: Uint8Ptr: %v", obj, obj.Uint8Ptr)
obj.init.Logf("%s: Float32: %v", obj, obj.Float32)
obj.init.Logf("%s: Float64: %v", obj, obj.Float64)
obj.init.Logf("%s: Complex64: %v", obj, obj.Complex64)
obj.init.Logf("%s: Complex128: %v", obj, obj.Complex128)
log.Printf("%s: Int8PtrPtrPtr: %v", obj, obj.Int8PtrPtrPtr)
obj.init.Logf("%s: BoolPtr: %v", obj, obj.BoolPtr)
obj.init.Logf("%s: StringPtr: %v", obj, obj.StringPtr)
obj.init.Logf("%s: Int64Ptr: %v", obj, obj.Int64Ptr)
obj.init.Logf("%s: Int8Ptr: %v", obj, obj.Int8Ptr)
obj.init.Logf("%s: Uint8Ptr: %v", obj, obj.Uint8Ptr)
log.Printf("%s: SliceString: %v", obj, obj.SliceString)
log.Printf("%s: MapIntFloat: %v", obj, obj.MapIntFloat)
log.Printf("%s: MixedStruct: %v", obj, obj.MixedStruct)
log.Printf("%s: Interface: %v", obj, obj.Interface)
obj.init.Logf("%s: Int8PtrPtrPtr: %v", obj, obj.Int8PtrPtrPtr)
log.Printf("%s: AnotherStr: %v", obj, obj.AnotherStr)
obj.init.Logf("%s: SliceString: %v", obj, obj.SliceString)
obj.init.Logf("%s: MapIntFloat: %v", obj, obj.MapIntFloat)
obj.init.Logf("%s: MixedStruct: %v", obj, obj.MixedStruct)
obj.init.Logf("%s: Interface: %v", obj, obj.Interface)
obj.init.Logf("%s: AnotherStr: %v", obj, obj.AnotherStr)
// send
hello := obj.SendValue
if err := obj.init.Send(&TestSends{
Hello: &hello,
Answer: 42,
}); err != nil {
return false, err
}
return true, nil // state is always okay
}
// TestUID is the UID struct for TestRes.
type TestUID struct {
BaseUID
name string
}
// UIDs includes all params to make a unique identification of this object.
// Most resources only return one, although some resources can return multiple.
func (obj *TestRes) UIDs() []ResUID {
x := &TestUID{
BaseUID: BaseUID{Name: obj.GetName(), Kind: obj.GetKind()},
name: obj.Name,
// Cmp compares two resources and returns an error if they are not equivalent.
func (obj *TestRes) Cmp(r engine.Res) error {
if !obj.Compare(r) {
return fmt.Errorf("did not compare")
}
return []ResUID{x}
}
// GroupCmp returns whether two resources can be grouped together or not.
func (obj *TestRes) GroupCmp(r Res) bool {
_, ok := r.(*TestRes)
if !ok {
return false
}
return obj.AlwaysGroup // grouped together if we were asked to
return nil
}
// Compare two resources and return if they are equivalent.
func (obj *TestRes) Compare(r Res) bool {
func (obj *TestRes) Compare(r engine.Res) bool {
// we can only compare TestRes to others of the same resource kind
res, ok := r.(*TestRes)
if !ok {
return false
}
// calling base Compare is probably unneeded for the test res, but do it
if !obj.BaseRes.Compare(res) { // call base Compare
return false
}
if obj.Name != res.Name {
return false
}
//if obj.Name != res.Name {
// return false
//}
if obj.CompareFail || res.CompareFail {
return false
@@ -368,6 +374,9 @@ func (obj *TestRes) Compare(r Res) bool {
if obj.AlwaysGroup != res.AlwaysGroup {
return false
}
if obj.SendValue != res.SendValue {
return false
}
if obj.Comment != res.Comment {
return false
@@ -376,6 +385,50 @@ func (obj *TestRes) Compare(r Res) bool {
return true
}
// TestUID is the UID struct for TestRes.
type TestUID struct {
engine.BaseUID
name string
}
// UIDs includes all params to make a unique identification of this object.
// Most resources only return one, although some resources can return multiple.
func (obj *TestRes) UIDs() []engine.ResUID {
x := &TestUID{
BaseUID: engine.BaseUID{Name: obj.Name(), Kind: obj.Kind()},
name: obj.Name(),
}
return []engine.ResUID{x}
}
// GroupCmp returns whether two resources can be grouped together or not.
func (obj *TestRes) GroupCmp(r engine.GroupableRes) error {
_, ok := r.(*TestRes)
if !ok {
return fmt.Errorf("resource is not the same kind")
}
if !obj.AlwaysGroup { // grouped together if we were asked to
return fmt.Errorf("the AlwaysGroup param is false")
}
return nil
}
// TestSends is the struct of data which is sent after a successful Apply.
type TestSends struct {
// Hello is some value being sent.
Hello *string
Answer int // some other value being sent
}
// Sends represents the default struct of values we can send using Send/Recv.
func (obj *TestRes) Sends() interface{} {
return &TestSends{
Hello: nil,
Answer: -1,
}
}
// UnmarshalYAML is the custom unmarshal handler for this struct.
// It is primarily useful for setting the defaults.
func (obj *TestRes) UnmarshalYAML(unmarshal func(interface{}) error) error {

175
engine/resources/test_test.go Normal file
View File

@@ -0,0 +1,175 @@
// Mgmt
// Copyright (C) 2013-2018+ James Shubin and the project contributors
// Written by James Shubin <james@shubin.ca> and the project contributors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package resources
import (
"reflect"
"testing"
"github.com/purpleidea/mgmt/engine/util"
)
func TestStructTagToFieldName0(t *testing.T) {
type TestStruct struct {
TestRes // so that this struct implements `Res`
Alpha bool `lang:"alpha" yaml:"nope"`
Beta string `yaml:"beta"`
Gamma string
Delta int `lang:"surprise"`
}
mapping, err := util.StructTagToFieldName(&TestStruct{})
if err != nil {
t.Errorf("failed: %+v", err)
return
}
expected := map[string]string{
"alpha": "Alpha",
"surprise": "Delta",
}
if !reflect.DeepEqual(mapping, expected) {
t.Errorf("expected: %+v", expected)
t.Errorf("received: %+v", mapping)
}
}
func TestLowerStructFieldNameToFieldName0(t *testing.T) {
type TestStruct struct {
TestRes // so that this struct implements `Res`
Alpha bool
skipMe bool
Beta string
IAmACamel uint
pass *string
Gamma string
Delta int
}
mapping, err := util.LowerStructFieldNameToFieldName(&TestStruct{})
if err != nil {
t.Errorf("failed: %+v", err)
return
}
expected := map[string]string{
"testres": "TestRes", // hide by specifying `lang:""` on it
"alpha": "Alpha",
//"skipme": "skipMe",
"beta": "Beta",
"iamacamel": "IAmACamel",
//"pass": "pass",
"gamma": "Gamma",
"delta": "Delta",
}
if !reflect.DeepEqual(mapping, expected) {
t.Errorf("expected: %+v", expected)
t.Errorf("received: %+v", mapping)
}
}
func TestLowerStructFieldNameToFieldName1(t *testing.T) {
type TestStruct struct {
TestRes // so that this struct implements `Res`
Alpha bool
skipMe bool
Beta string
// these two should collide
DoubleWord bool
Doubleword string
IAmACamel uint
pass *string
Gamma string
Delta int
}
mapping, err := util.LowerStructFieldNameToFieldName(&TestStruct{})
if err == nil {
t.Errorf("expected failure, but passed with: %+v", mapping)
return
}
}
func TestLowerStructFieldNameToFieldName2(t *testing.T) {
mapping, err := util.LowerStructFieldNameToFieldName(&TestRes{})
if err != nil {
t.Errorf("failed: %+v", err)
return
}
expected := map[string]string{
"base": "Base", // all resources have this trait
"groupable": "Groupable", // the TestRes has this trait
"refreshable": "Refreshable", // the TestRes has this trait
"sendable": "Sendable",
"recvable": "Recvable",
"bool": "Bool",
"str": "Str",
"int": "Int",
"int8": "Int8",
"int16": "Int16",
"int32": "Int32",
"int64": "Int64",
"uint": "Uint",
"uint8": "Uint8",
"uint16": "Uint16",
"uint32": "Uint32",
"uint64": "Uint64",
"byte": "Byte",
"rune": "Rune",
"float32": "Float32",
"float64": "Float64",
"complex64": "Complex64",
"complex128": "Complex128",
"boolptr": "BoolPtr",
"stringptr": "StringPtr",
"int64ptr": "Int64Ptr",
"int8ptr": "Int8Ptr",
"uint8ptr": "Uint8Ptr",
"int8ptrptrptr": "Int8PtrPtrPtr",
"slicestring": "SliceString",
"mapintfloat": "MapIntFloat",
"mixedstruct": "MixedStruct",
"interface": "Interface",
"anotherstr": "AnotherStr",
"validatebool": "ValidateBool",
"validateerror": "ValidateError",
"alwaysgroup": "AlwaysGroup",
"comparefail": "CompareFail",
"sendvalue": "SendValue",
"comment": "Comment",
}
if !reflect.DeepEqual(mapping, expected) {
t.Errorf("expected: %+v", expected)
t.Errorf("received: %+v", mapping)
}
}

103
resources/timer.go → engine/resources/timer.go
View File

@@ -19,46 +19,49 @@ package resources
import (
"fmt"
"log"
"time"
"github.com/purpleidea/mgmt/engine"
"github.com/purpleidea/mgmt/engine/traits"
)
func init() {
RegisterResource("timer", func() Res { return &TimerRes{} })
engine.RegisterResource("timer", func() engine.Res { return &TimerRes{} })
}
// TimerRes is a timer resource for time based events. It outputs an event every
// interval seconds.
type TimerRes struct {
BaseRes `yaml:",inline"`
traits.Base // add the base methods without re-implementation
traits.Refreshable
init *engine.Init
Interval uint32 `yaml:"interval"` // interval between runs in seconds
ticker *time.Ticker
}
// TimerUID is the UID struct for TimerRes.
type TimerUID struct {
BaseUID
name string
}
// Default returns some sensible defaults for this resource.
func (obj *TimerRes) Default() Res {
return &TimerRes{
BaseRes: BaseRes{
MetaParams: DefaultMetaParams, // force a default
},
}
func (obj *TimerRes) Default() engine.Res {
return &TimerRes{}
}
// Validate the params that are passed to TimerRes.
func (obj *TimerRes) Validate() error {
return obj.BaseRes.Validate()
return nil
}
// Init runs some startup code for this resource.
func (obj *TimerRes) Init() error {
return obj.BaseRes.Init() // call base init, b/c we're overrriding
func (obj *TimerRes) Init(init *engine.Init) error {
obj.init = init // save for later
return nil
}
// Close is run by the engine to clean up after the resource is done.
func (obj *TimerRes) Close() error {
return nil
}
// newTicker creates a new ticker
@@ -73,27 +76,31 @@ func (obj *TimerRes) Watch() error {
defer obj.ticker.Stop()
// notify engine that we're running
if err := obj.Running(); err != nil {
return err // bubble up a NACK...
if err := obj.init.Running(); err != nil {
return err // exit if requested
}
var send = false
var send = false // send event?
for {
select {
case <-obj.ticker.C: // received the timer event
send = true
log.Printf("%s: received tick", obj)
obj.init.Logf("received tick")
case event := <-obj.Events():
if exit, _ := obj.ReadEvent(event); exit != nil {
return *exit // exit
case event, ok := <-obj.init.Events:
if !ok {
return nil
}
if err := obj.init.Read(event); err != nil {
return err
}
}
if send {
send = false
obj.Event()
if err := obj.init.Event(); err != nil {
return err // exit if requested
}
}
}
}
@@ -102,7 +109,7 @@ func (obj *TimerRes) Watch() error {
func (obj *TimerRes) CheckApply(apply bool) (bool, error) {
// because there are no checks to run, this resource has a less
// traditional pattern than what is seen in most resources...
if !obj.Refresh() { // this works for apply || !apply
if !obj.init.Refresh() { // this works for apply || !apply
return true, nil // state is always okay if no refresh to do
} else if !apply { // we had a refresh to do
return false, nil // therefore state is wrong
@@ -114,32 +121,21 @@ func (obj *TimerRes) CheckApply(apply bool) (bool, error) {
return false, nil
}
// UIDs includes all params to make a unique identification of this object.
// Most resources only return one, although some resources can return multiple.
func (obj *TimerRes) UIDs() []ResUID {
x := &TimerUID{
BaseUID: BaseUID{
Name: obj.GetName(),
Kind: obj.GetKind(),
},
name: obj.Name,
// Cmp compares two resources and returns an error if they are not equivalent.
func (obj *TimerRes) Cmp(r engine.Res) error {
if !obj.Compare(r) {
return fmt.Errorf("did not compare")
}
return []ResUID{x}
return nil
}
// Compare two resources and return if they are equivalent.
func (obj *TimerRes) Compare(r Res) bool {
func (obj *TimerRes) Compare(r engine.Res) bool {
// we can only compare TimerRes to others of the same resource kind
res, ok := r.(*TimerRes)
if !ok {
return false
}
if !obj.BaseRes.Compare(res) {
return false
}
if obj.Name != res.Name {
return false
}
if obj.Interval != res.Interval {
return false
@@ -148,6 +144,23 @@ func (obj *TimerRes) Compare(r Res) bool {
return true
}
// TimerUID is the UID struct for TimerRes.
type TimerUID struct {
engine.BaseUID
name string
}
// UIDs includes all params to make a unique identification of this object.
// Most resources only return one, although some resources can return multiple.
func (obj *TimerRes) UIDs() []engine.ResUID {
x := &TimerUID{
BaseUID: engine.BaseUID{Name: obj.Name(), Kind: obj.Kind()},
name: obj.Name(),
}
return []engine.ResUID{x}
}
// UnmarshalYAML is the custom unmarshal handler for this struct.
// It is primarily useful for setting the defaults.
func (obj *TimerRes) UnmarshalYAML(unmarshal func(interface{}) error) error {

268
resources/user.go → engine/resources/user.go
View File

@@ -20,7 +20,6 @@ package resources
import (
"fmt"
"io/ioutil"
"log"
"os/exec"
"os/user"
"sort"
@@ -28,20 +27,26 @@ import (
"strings"
"syscall"
"github.com/purpleidea/mgmt/engine"
"github.com/purpleidea/mgmt/engine/traits"
"github.com/purpleidea/mgmt/recwatch"
errwrap "github.com/pkg/errors"
)
func init() {
RegisterResource("user", func() Res { return &UserRes{} })
engine.RegisterResource("user", func() engine.Res { return &UserRes{} })
}
const passwdFile = "/etc/passwd"
// UserRes is a user account resource.
type UserRes struct {
BaseRes `yaml:",inline"`
traits.Base // add the base methods without re-implementation
traits.Edgeable
init *engine.Init
State string `yaml:"state"` // state: exists, absent
UID *uint32 `yaml:"uid"` // uid must be unique unless AllowDuplicateUID is true
GID *uint32 `yaml:"gid"` // gid of the user's primary group
@@ -54,12 +59,8 @@ type UserRes struct {
}
// Default returns some sensible defaults for this resource.
func (obj *UserRes) Default() Res {
return &UserRes{
BaseRes: BaseRes{
MetaParams: DefaultMetaParams, // force a default
},
}
func (obj *UserRes) Default() engine.Res {
return &UserRes{}
}
// Validate if the params passed in are valid data.
@@ -94,12 +95,19 @@ func (obj *UserRes) Validate() error {
}
}
}
return obj.BaseRes.Validate()
return nil
}
// Init initializes the resource.
func (obj *UserRes) Init() error {
return obj.BaseRes.Init() // call base init, b/c we're overriding
// Init runs some startup code for this resource.
func (obj *UserRes) Init(init *engine.Init) error {
obj.init = init // save for later
return nil
}
// Close is run by the engine to clean up after the resource is done.
func (obj *UserRes) Close() error {
return nil
}
// Watch is the primary listener for this resource and it outputs events.
@@ -112,16 +120,14 @@ func (obj *UserRes) Watch() error {
defer obj.recWatcher.Close()
// notify engine that we're running
if err := obj.Running(); err != nil {
return err // bubble up a NACK...
if err := obj.init.Running(); err != nil {
return err // exit if requested
}
var send = false // send event?
var exit *error
for {
if obj.debug {
log.Printf("%s: Watching: %s", obj, passwdFile) // attempting to watch...
if obj.init.Debug {
obj.init.Logf("Watching: %s", passwdFile) // attempting to watch...
}
select {
@@ -132,33 +138,37 @@ func (obj *UserRes) Watch() error {
if err := event.Error; err != nil {
return errwrap.Wrapf(err, "Unknown %s watcher error", obj)
}
if obj.debug { // don't access event.Body if event.Error isn't nil
log.Printf("%s: Event(%s): %v", obj, event.Body.Name, event.Body.Op)
if obj.init.Debug { // don't access event.Body if event.Error isn't nil
obj.init.Logf("Event(%s): %v", event.Body.Name, event.Body.Op)
}
send = true
obj.StateOK(false) // dirty
obj.init.Dirty() // dirty
case event := <-obj.Events():
if exit, send = obj.ReadEvent(event); exit != nil {
return *exit // exit
case event, ok := <-obj.init.Events:
if !ok {
return nil
}
if err := obj.init.Read(event); err != nil {
return err
}
//obj.StateOK(false) // dirty // these events don't invalidate state
}
// do all our event sending all together to avoid duplicate msgs
if send {
send = false
obj.Event()
if err := obj.init.Event(); err != nil {
return err // exit if requested
}
}
}
}
// CheckApply method for User resource.
func (obj *UserRes) CheckApply(apply bool) (checkOK bool, err error) {
log.Printf("%s: CheckApply(%t)", obj, apply)
obj.init.Logf("CheckApply(%t)", apply)
var exists = true
usr, err := user.Lookup(obj.GetName())
usr, err := user.Lookup(obj.Name())
if err != nil {
if _, ok := err.(user.UnknownUserError); !ok {
return false, errwrap.Wrapf(err, "error looking up user")
@@ -172,7 +182,7 @@ func (obj *UserRes) CheckApply(apply bool) (checkOK bool, err error) {
if _, ok := err.(user.UnknownUserIdError); !ok {
return false, errwrap.Wrapf(err, "error looking up UID")
}
} else if existingUID.Username != obj.GetName() {
} else if existingUID.Username != obj.Name() {
return false, fmt.Errorf("the requested UID is already taken")
}
}
@@ -213,10 +223,10 @@ func (obj *UserRes) CheckApply(apply bool) (checkOK bool, err error) {
if obj.State == "exists" {
if exists {
cmdName = "usermod"
log.Printf("%s: Modifying user: %s", obj, obj.GetName())
obj.init.Logf("Modifying user: %s", obj.Name())
} else {
cmdName = "useradd"
log.Printf("%s: Adding user: %s", obj, obj.GetName())
obj.init.Logf("Adding user: %s", obj.Name())
}
if obj.AllowDuplicateUID {
args = append(args, "--non-unique")
@@ -239,10 +249,10 @@ func (obj *UserRes) CheckApply(apply bool) (checkOK bool, err error) {
}
if obj.State == "absent" {
cmdName = "userdel"
log.Printf("%s: Deleting user: %s", obj, obj.GetName())
obj.init.Logf("Deleting user: %s", obj.Name())
}
args = append(args, obj.GetName())
args = append(args, obj.Name())
cmd := exec.Command(cmdName, args...)
cmd.SysProcAttr = &syscall.SysProcAttr{
@@ -273,113 +283,22 @@ func (obj *UserRes) CheckApply(apply bool) (checkOK bool, err error) {
return false, nil
}
// UserUID is the UID struct for UserRes.
type UserUID struct {
BaseUID
name string
}
// UserResAutoEdges holds the state of the auto edge generator.
type UserResAutoEdges struct {
UIDs []ResUID
pointer int
}
// AutoEdges returns edges from the user resource to each group found in
// its definition. The groups can be in any of the three applicable fields
// (GID, Group and Groups.) If the user exists, reversed ensures the edge
// goes from group to user, and if the user is absent the edge goes from
// user to group. This ensures that we don't add users to groups that
// don't exist or delete groups before we delete their members.
func (obj *UserRes) AutoEdges() (AutoEdge, error) {
var result []ResUID
var reversed bool
if obj.State == "exists" {
reversed = true
// Cmp compares two resources and returns an error if they are not equivalent.
func (obj *UserRes) Cmp(r engine.Res) error {
if !obj.Compare(r) {
return fmt.Errorf("did not compare")
}
if obj.GID != nil {
result = append(result, &GroupUID{
BaseUID: BaseUID{
Reversed: &reversed,
},
gid: obj.GID,
})
}
if obj.Group != nil {
result = append(result, &GroupUID{
BaseUID: BaseUID{
Reversed: &reversed,
},
name: *obj.Group,
})
}
for _, group := range obj.Groups {
result = append(result, &GroupUID{
BaseUID: BaseUID{
Reversed: &reversed,
},
name: group,
})
}
return &UserResAutoEdges{
UIDs: result,
pointer: 0,
}, nil
}
// Next returns the next automatic edge.
func (obj *UserResAutoEdges) Next() []ResUID {
if len(obj.UIDs) == 0 {
return nil
}
value := obj.UIDs[obj.pointer]
obj.pointer++
return []ResUID{value}
}
// Test gets results of the earlier Next() call, & returns if we should continue.
func (obj *UserResAutoEdges) Test(input []bool) bool {
if len(obj.UIDs) <= obj.pointer {
return false
}
if len(input) != 1 { // in case we get given bad data
log.Fatal("Expecting a single value!")
}
return true // keep going
}
// UIDs includes all params to make a unique identification of this object.
// Most resources only return one, although some resources can return multiple.
func (obj *UserRes) UIDs() []ResUID {
x := &UserUID{
BaseUID: BaseUID{Name: obj.GetName(), Kind: obj.GetKind()},
name: obj.Name,
}
return []ResUID{x}
}
// GroupCmp returns whether two resources can be grouped together or not.
func (obj *UserRes) GroupCmp(r Res) bool {
_, ok := r.(*UserRes)
if !ok {
return false
}
return false
return nil
}
// Compare two resources and return if they are equivalent.
func (obj *UserRes) Compare(r Res) bool {
func (obj *UserRes) Compare(r engine.Res) bool {
// we can only compare UserRes to others of the same resource kind
res, ok := r.(*UserRes)
if !ok {
return false
}
if !obj.BaseRes.Compare(res) { // call base Compare
return false
}
if obj.Name != res.Name {
return false
}
if obj.State != res.State {
return false
}
@@ -430,6 +349,91 @@ func (obj *UserRes) Compare(r Res) bool {
return true
}
// UserUID is the UID struct for UserRes.
type UserUID struct {
engine.BaseUID
name string
}
// UserResAutoEdges holds the state of the auto edge generator.
type UserResAutoEdges struct {
UIDs []engine.ResUID
pointer int
}
// AutoEdges returns edges from the user resource to each group found in
// its definition. The groups can be in any of the three applicable fields
// (GID, Group and Groups.) If the user exists, reversed ensures the edge
// goes from group to user, and if the user is absent the edge goes from
// user to group. This ensures that we don't add users to groups that
// don't exist or delete groups before we delete their members.
func (obj *UserRes) AutoEdges() (engine.AutoEdge, error) {
var result []engine.ResUID
var reversed bool
if obj.State == "exists" {
reversed = true
}
if obj.GID != nil {
result = append(result, &GroupUID{
BaseUID: engine.BaseUID{
Reversed: &reversed,
},
gid: obj.GID,
})
}
if obj.Group != nil {
result = append(result, &GroupUID{
BaseUID: engine.BaseUID{
Reversed: &reversed,
},
name: *obj.Group,
})
}
for _, group := range obj.Groups {
result = append(result, &GroupUID{
BaseUID: engine.BaseUID{
Reversed: &reversed,
},
name: group,
})
}
return &UserResAutoEdges{
UIDs: result,
pointer: 0,
}, nil
}
// Next returns the next automatic edge.
func (obj *UserResAutoEdges) Next() []engine.ResUID {
if len(obj.UIDs) == 0 {
return nil
}
value := obj.UIDs[obj.pointer]
obj.pointer++
return []engine.ResUID{value}
}
// Test gets results of the earlier Next() call, & returns if we should continue.
func (obj *UserResAutoEdges) Test(input []bool) bool {
if len(obj.UIDs) <= obj.pointer {
return false
}
if len(input) != 1 { // in case we get given bad data
panic(fmt.Sprintf("Expecting a single value!"))
}
return true // keep going
}
// UIDs includes all params to make a unique identification of this object.
// Most resources only return one, although some resources can return multiple.
func (obj *UserRes) UIDs() []engine.ResUID {
x := &UserUID{
BaseUID: engine.BaseUID{Name: obj.Name(), Kind: obj.Kind()},
name: obj.Name(),
}
return []engine.ResUID{x}
}
// UnmarshalYAML is the custom unmarshal handler for this struct.
// It is primarily useful for setting the defaults.
func (obj *UserRes) UnmarshalYAML(unmarshal func(interface{}) error) error {

227
resources/virt.go → engine/resources/virt.go
View File

@@ -21,26 +21,23 @@ package resources
import (
"fmt"
"log"
"math/rand"
"net/url"
"os/user"
"path"
"regexp"
"strings"
"sync"
"time"
"github.com/purpleidea/mgmt/engine"
"github.com/purpleidea/mgmt/engine/traits"
"github.com/purpleidea/mgmt/util"
multierr "github.com/hashicorp/go-multierror"
"github.com/libvirt/libvirt-go"
libvirtxml "github.com/libvirt/libvirt-go-xml"
errwrap "github.com/pkg/errors"
)
func init() {
RegisterResource("virt", func() Res { return &VirtRes{} })
engine.RegisterResource("virt", func() engine.Res { return &VirtRes{} })
}
const (
@@ -65,17 +62,15 @@ const (
lxcURI
)
// VirtAuth is used to pass credentials to libvirt.
type VirtAuth struct {
Username string `yaml:"username"`
Password string `yaml:"password"`
}
// VirtRes is a libvirt resource. A transient virt resource, which has its state
// set to `shutoff` is one which does not exist. The parallel equivalent is a
// file resource which removes a particular path.
type VirtRes struct {
BaseRes `yaml:",inline"`
traits.Base // add the base methods without re-implementation
traits.Refreshable
init *engine.Init
URI string `yaml:"uri"` // connection uri, eg: qemu:///session
State string `yaml:"state"` // running, paused, shutoff
Transient bool `yaml:"transient"` // defined (false) or undefined (true)
@@ -106,13 +101,15 @@ type VirtRes struct {
guestAgentConnected bool // our tracking of if guest agent is running
}
// Default returns some sensible defaults for this resource.
func (obj *VirtRes) Default() Res {
return &VirtRes{
BaseRes: BaseRes{
MetaParams: DefaultMetaParams, // force a default
},
// VirtAuth is used to pass credentials to libvirt.
type VirtAuth struct {
Username string `yaml:"username"`
Password string `yaml:"password"`
}
// Default returns some sensible defaults for this resource.
func (obj *VirtRes) Default() engine.Res {
return &VirtRes{
MaxCPUs: DefaultMaxCPUs,
HotCPUs: true, // we're a dynamic engine, be dynamic by default!
@@ -125,11 +122,13 @@ func (obj *VirtRes) Validate() error {
if obj.CPUs > obj.MaxCPUs {
return fmt.Errorf("the number of CPUs (%d) must not be greater than MaxCPUs (%d)", obj.CPUs, obj.MaxCPUs)
}
return obj.BaseRes.Validate()
return nil
}
// Init runs some startup code for this resource.
func (obj *VirtRes) Init() error {
func (obj *VirtRes) Init(init *engine.Init) error {
obj.init = init // save for later
if !libvirtInitialized {
if err := libvirt.EventRegisterDefaultImpl(); err != nil {
return errwrap.Wrapf(err, "method EventRegisterDefaultImpl failed")
@@ -154,7 +153,7 @@ func (obj *VirtRes) Init() error {
}
// check for hard to change properties
dom, err := obj.conn.LookupDomainByName(obj.GetName())
dom, err := obj.conn.LookupDomainByName(obj.Name())
if err == nil {
defer dom.Free()
} else if !isNotFound(err) {
@@ -194,7 +193,7 @@ func (obj *VirtRes) Init() error {
}
}
obj.wg = &sync.WaitGroup{}
return obj.BaseRes.Init() // call base init, b/c we're overriding
return nil
}
// Close runs some cleanup code for this resource.
@@ -209,12 +208,6 @@ func (obj *VirtRes) Close() error {
_, err := obj.conn.Close() // close libvirt conn that was opened in Init
obj.conn = nil // set to nil to help catch any nil ptr bugs!
// call base close, b/c we're overriding
if e := obj.BaseRes.Close(); err == nil {
err = e
} else if e != nil {
err = multierr.Append(err, e) // list of errors
}
return err
}
@@ -279,7 +272,7 @@ func (obj *VirtRes) Watch() error {
go func() {
defer obj.wg.Done()
defer wg.Done()
defer log.Printf("EventRunDefaultImpl exited!")
defer obj.init.Logf("EventRunDefaultImpl exited!")
for {
// TODO: can we merge this into our main for loop below?
select {
@@ -287,7 +280,7 @@ func (obj *VirtRes) Watch() error {
return
default:
}
//log.Printf("EventRunDefaultImpl started!")
//obj.init.Logf("EventRunDefaultImpl started!")
if err := libvirt.EventRunDefaultImpl(); err != nil {
select {
case errorChan <- errwrap.Wrapf(err, "EventRunDefaultImpl failed"):
@@ -296,14 +289,14 @@ func (obj *VirtRes) Watch() error {
}
return
}
//log.Printf("EventRunDefaultImpl looped!")
//obj.init.Logf("EventRunDefaultImpl looped!")
}
}()
// domain events callback
domCallback := func(c *libvirt.Connect, d *libvirt.Domain, ev *libvirt.DomainEventLifecycle) {
domName, _ := d.GetName()
if domName == obj.GetName() {
if domName == obj.Name() {
select {
case domChan <- ev.Event: // send
case <-exitChan:
@@ -320,7 +313,7 @@ func (obj *VirtRes) Watch() error {
// guest agent events callback
gaCallback := func(c *libvirt.Connect, d *libvirt.Domain, eva *libvirt.DomainEventAgentLifecycle) {
domName, _ := d.GetName()
if domName == obj.GetName() {
if domName == obj.Name() {
select {
case gaChan <- eva: // send
case <-exitChan:
@@ -334,13 +327,11 @@ func (obj *VirtRes) Watch() error {
defer obj.conn.DomainEventDeregister(gaCallbackID)
// notify engine that we're running
if err := obj.Running(); err != nil {
return err // bubble up a NACK...
if err := obj.init.Running(); err != nil {
return err // exit if requested
}
var send = false
var exit *error // if ptr exists, that is the exit error to return
var send = false // send event?
for {
processExited := false // did the process exit fully (shutdown)?
select {
@@ -349,31 +340,31 @@ func (obj *VirtRes) Watch() error {
switch event {
case libvirt.DOMAIN_EVENT_DEFINED:
if obj.Transient {
obj.StateOK(false) // dirty
obj.init.Dirty() // dirty
send = true
}
case libvirt.DOMAIN_EVENT_UNDEFINED:
if !obj.Transient {
obj.StateOK(false) // dirty
obj.init.Dirty() // dirty
send = true
}
case libvirt.DOMAIN_EVENT_STARTED:
fallthrough
case libvirt.DOMAIN_EVENT_RESUMED:
if obj.State != "running" {
obj.StateOK(false) // dirty
obj.init.Dirty() // dirty
send = true
}
case libvirt.DOMAIN_EVENT_SUSPENDED:
if obj.State != "paused" {
obj.StateOK(false) // dirty
obj.init.Dirty() // dirty
send = true
}
case libvirt.DOMAIN_EVENT_STOPPED:
fallthrough
case libvirt.DOMAIN_EVENT_SHUTDOWN:
if obj.State != "shutoff" {
obj.StateOK(false) // dirty
obj.init.Dirty() // dirty
send = true
}
processExited = true
@@ -384,7 +375,7 @@ func (obj *VirtRes) Watch() error {
// verify, detect and patch appropriately!
fallthrough
case libvirt.DOMAIN_EVENT_CRASHED:
obj.StateOK(false) // dirty
obj.init.Dirty() // dirty
send = true
processExited = true // FIXME: is this okay for PMSUSPENDED ?
}
@@ -399,16 +390,16 @@ func (obj *VirtRes) Watch() error {
if state == libvirt.CONNECT_DOMAIN_EVENT_AGENT_LIFECYCLE_STATE_CONNECTED {
obj.guestAgentConnected = true
obj.StateOK(false) // dirty
obj.init.Dirty() // dirty
send = true
log.Printf("%s: Guest agent connected", obj)
obj.init.Logf("Guest agent connected")
} else if state == libvirt.CONNECT_DOMAIN_EVENT_AGENT_LIFECYCLE_STATE_DISCONNECTED {
obj.guestAgentConnected = false
// ignore CONNECT_DOMAIN_EVENT_AGENT_LIFECYCLE_REASON_DOMAIN_STARTED
// events because they just tell you that guest agent channel was added
if reason == libvirt.CONNECT_DOMAIN_EVENT_AGENT_LIFECYCLE_REASON_CHANNEL {
log.Printf("%s: Guest agent disconnected", obj)
obj.init.Logf("Guest agent disconnected")
}
} else {
@@ -418,15 +409,21 @@ func (obj *VirtRes) Watch() error {
case err := <-errorChan:
return fmt.Errorf("unknown %s libvirt error: %s", obj, err)
case event := <-obj.Events():
if exit, send = obj.ReadEvent(event); exit != nil {
return *exit // exit
case event, ok := <-obj.init.Events:
if !ok {
return nil
}
if err := obj.init.Read(event); err != nil {
return err
}
}
// do all our event sending all together to avoid duplicate msgs
if send {
send = false
obj.Event()
if err := obj.init.Event(); err != nil {
return err // exit if requested
}
}
}
}
@@ -454,7 +451,7 @@ func (obj *VirtRes) domainCreate() (*libvirt.Domain, bool, error) {
if err != nil {
return dom, false, err // returned dom is invalid
}
log.Printf("%s: Domain transient %s", state, obj)
obj.init.Logf("Domain transient %s", state)
return dom, false, nil
}
@@ -462,20 +459,20 @@ func (obj *VirtRes) domainCreate() (*libvirt.Domain, bool, error) {
if err != nil {
return dom, false, err // returned dom is invalid
}
log.Printf("%s: Domain defined", obj)
obj.init.Logf("Domain defined")
if obj.State == "running" {
if err := dom.Create(); err != nil {
return dom, false, err
}
log.Printf("%s: Domain started", obj)
obj.init.Logf("Domain started")
}
if obj.State == "paused" {
if err := dom.CreateWithFlags(libvirt.DOMAIN_START_PAUSED); err != nil {
return dom, false, err
}
log.Printf("%s: Domain created paused", obj)
obj.init.Logf("Domain created paused")
}
return dom, false, nil
@@ -503,7 +500,7 @@ func (obj *VirtRes) stateCheckApply(apply bool, dom *libvirt.Domain) (bool, erro
}
if domInfo.State == libvirt.DOMAIN_BLOCKED {
// TODO: what should happen?
return false, fmt.Errorf("domain %s is blocked", obj.GetName())
return false, fmt.Errorf("domain %s is blocked", obj.Name())
}
if !apply {
return false, nil
@@ -513,14 +510,14 @@ func (obj *VirtRes) stateCheckApply(apply bool, dom *libvirt.Domain) (bool, erro
return false, errwrap.Wrapf(err, "domain.Resume failed")
}
checkOK = false
log.Printf("%s: Domain resumed", obj)
obj.init.Logf("Domain resumed")
break
}
if err := dom.Create(); err != nil {
return false, errwrap.Wrapf(err, "domain.Create failed")
}
checkOK = false
log.Printf("%s: Domain created", obj)
obj.init.Logf("Domain created")
case "paused":
if domInfo.State == libvirt.DOMAIN_PAUSED {
@@ -534,14 +531,14 @@ func (obj *VirtRes) stateCheckApply(apply bool, dom *libvirt.Domain) (bool, erro
return false, errwrap.Wrapf(err, "domain.Suspend failed")
}
checkOK = false
log.Printf("%s: Domain paused", obj)
obj.init.Logf("Domain paused")
break
}
if err := dom.CreateWithFlags(libvirt.DOMAIN_START_PAUSED); err != nil {
return false, errwrap.Wrapf(err, "domain.CreateWithFlags failed")
}
checkOK = false
log.Printf("%s: Domain created paused", obj)
obj.init.Logf("Domain created paused")
case "shutoff":
if domInfo.State == libvirt.DOMAIN_SHUTOFF || domInfo.State == libvirt.DOMAIN_SHUTDOWN {
@@ -555,7 +552,7 @@ func (obj *VirtRes) stateCheckApply(apply bool, dom *libvirt.Domain) (bool, erro
return false, errwrap.Wrapf(err, "domain.Destroy failed")
}
checkOK = false
log.Printf("%s: Domain destroyed", obj)
obj.init.Logf("Domain destroyed")
}
return checkOK, nil
@@ -581,7 +578,7 @@ func (obj *VirtRes) attrCheckApply(apply bool, dom *libvirt.Domain) (bool, error
if err := dom.SetMemory(obj.Memory); err != nil {
return false, errwrap.Wrapf(err, "domain.SetMemory failed")
}
log.Printf("%s: Memory changed to %d", obj, obj.Memory)
obj.init.Logf("Memory changed to %d", obj.Memory)
}
// check cpus
@@ -620,7 +617,7 @@ func (obj *VirtRes) attrCheckApply(apply bool, dom *libvirt.Domain) (bool, error
return false, errwrap.Wrapf(err, "domain.SetVcpus failed")
}
checkOK = false
log.Printf("%s: CPUs (hot) changed to %d", obj, obj.CPUs)
obj.init.Logf("CPUs (hot) changed to %d", obj.CPUs)
case libvirt.DOMAIN_SHUTOFF, libvirt.DOMAIN_SHUTDOWN:
if !obj.Transient {
@@ -632,7 +629,7 @@ func (obj *VirtRes) attrCheckApply(apply bool, dom *libvirt.Domain) (bool, error
return false, errwrap.Wrapf(err, "domain.SetVcpus failed")
}
checkOK = false
log.Printf("%s: CPUs (cold) changed to %d", obj, obj.CPUs)
obj.init.Logf("CPUs (cold) changed to %d", obj.CPUs)
}
default:
@@ -663,7 +660,7 @@ func (obj *VirtRes) attrCheckApply(apply bool, dom *libvirt.Domain) (bool, error
return false, errwrap.Wrapf(err, "domain.SetVcpus failed")
}
checkOK = false
log.Printf("%s: CPUs (guest) changed to %d", obj, obj.CPUs)
obj.init.Logf("CPUs (guest) changed to %d", obj.CPUs)
}
}
@@ -687,7 +684,7 @@ func (obj *VirtRes) domainShutdownSync(apply bool, dom *libvirt.Domain) (bool, e
return false, errwrap.Wrapf(err, "domain.GetInfo failed")
}
if domInfo.State == libvirt.DOMAIN_SHUTOFF || domInfo.State == libvirt.DOMAIN_SHUTDOWN {
log.Printf("%s: Shutdown", obj)
obj.init.Logf("Shutdown")
break
}
@@ -699,7 +696,7 @@ func (obj *VirtRes) domainShutdownSync(apply bool, dom *libvirt.Domain) (bool, e
obj.processExitChan = make(chan struct{})
// if machine shuts down before we call this, we error;
// this isn't ideal, but it happened due to user error!
log.Printf("%s: Running shutdown", obj)
obj.init.Logf("Running shutdown")
if err := dom.Shutdown(); err != nil {
// FIXME: if machine is already shutdown completely, return early
return false, errwrap.Wrapf(err, "domain.Shutdown failed")
@@ -734,8 +731,8 @@ func (obj *VirtRes) CheckApply(apply bool) (bool, error) {
panic("virt: CheckApply is being called with nil connection")
}
// if we do the restart, we must flip the flag back to false as evidence
var restart bool // do we need to do a restart?
if obj.RestartOnRefresh && obj.Refresh() { // a refresh is a restart ask
var restart bool // do we need to do a restart?
if obj.RestartOnRefresh && obj.init.Refresh() { // a refresh is a restart ask
restart = true
}
@@ -750,7 +747,7 @@ func (obj *VirtRes) CheckApply(apply bool) (bool, error) {
var checkOK = true
dom, err := obj.conn.LookupDomainByName(obj.GetName())
dom, err := obj.conn.LookupDomainByName(obj.Name())
if err == nil {
// pass
} else if isNotFound(err) {
@@ -792,7 +789,7 @@ func (obj *VirtRes) CheckApply(apply bool) (bool, error) {
if err := dom.Undefine(); err != nil {
return false, errwrap.Wrapf(err, "domain.Undefine failed")
}
log.Printf("%s: Domain undefined", obj)
obj.init.Logf("Domain undefined")
} else {
domXML, err := dom.GetXMLDesc(libvirt.DOMAIN_XML_INACTIVE)
if err != nil {
@@ -801,7 +798,7 @@ func (obj *VirtRes) CheckApply(apply bool) (bool, error) {
if _, err = obj.conn.DomainDefineXML(domXML); err != nil {
return false, errwrap.Wrapf(err, "conn.DomainDefineXML failed")
}
log.Printf("%s: Domain defined", obj)
obj.init.Logf("Domain defined")
}
checkOK = false
}
@@ -890,7 +887,7 @@ func (obj *VirtRes) getDomainXML() string {
var b string
b += obj.getDomainType() // start domain
b += fmt.Sprintf("<name>%s</name>", obj.GetName())
b += fmt.Sprintf("<name>%s</name>", obj.Name())
b += fmt.Sprintf("<memory unit='KiB'>%d</memory>", obj.Memory)
if obj.HotCPUs {
@@ -996,7 +993,7 @@ type filesystemDevice struct {
}
func (d *diskDevice) GetXML(idx int) string {
source, _ := expandHome(d.Source) // TODO: should we handle errors?
source, _ := util.ExpandHome(d.Source) // TODO: should we handle errors?
var b string
b += "<disk type='file' device='disk'>"
b += fmt.Sprintf("<driver name='qemu' type='%s'/>", d.Type)
@@ -1007,7 +1004,7 @@ func (d *diskDevice) GetXML(idx int) string {
}
func (d *cdRomDevice) GetXML(idx int) string {
source, _ := expandHome(d.Source) // TODO: should we handle errors?
source, _ := util.ExpandHome(d.Source) // TODO: should we handle errors?
var b string
b += "<disk type='file' device='cdrom'>"
b += fmt.Sprintf("<driver name='qemu' type='%s'/>", d.Type)
@@ -1031,7 +1028,7 @@ func (d *networkDevice) GetXML(idx int) string {
}
func (d *filesystemDevice) GetXML(idx int) string {
source, _ := expandHome(d.Source) // TODO: should we handle errors?
source, _ := util.ExpandHome(d.Source) // TODO: should we handle errors?
var b string
b += "<filesystem" // open
if d.Access != "" {
@@ -1047,43 +1044,21 @@ func (d *filesystemDevice) GetXML(idx int) string {
return b
}
// VirtUID is the UID struct for FileRes.
type VirtUID struct {
BaseUID
}
// UIDs includes all params to make a unique identification of this object.
// Most resources only return one, although some resources can return multiple.
func (obj *VirtRes) UIDs() []ResUID {
x := &VirtUID{
BaseUID: BaseUID{Name: obj.GetName(), Kind: obj.GetKind()},
// TODO: add more properties here so we can link to vm dependencies
// Cmp compares two resources and returns an error if they are not equivalent.
func (obj *VirtRes) Cmp(r engine.Res) error {
if !obj.Compare(r) {
return fmt.Errorf("did not compare")
}
return []ResUID{x}
}
// GroupCmp returns whether two resources can be grouped together or not.
func (obj *VirtRes) GroupCmp(r Res) bool {
_, ok := r.(*VirtRes)
if !ok {
return false
}
return false // not possible atm
return nil
}
// Compare two resources and return if they are equivalent.
func (obj *VirtRes) Compare(r Res) bool {
func (obj *VirtRes) Compare(r engine.Res) bool {
// we can only compare VirtRes to others of the same resource kind
res, ok := r.(*VirtRes)
if !ok {
return false
}
if !obj.BaseRes.Compare(res) { // call base Compare
return false
}
if obj.Name != res.Name {
return false
}
if obj.URI != res.URI {
return false
@@ -1131,6 +1106,21 @@ func (obj *VirtRes) Compare(r Res) bool {
return true
}
// VirtUID is the UID struct for FileRes.
type VirtUID struct {
engine.BaseUID
}
// UIDs includes all params to make a unique identification of this object.
// Most resources only return one, although some resources can return multiple.
func (obj *VirtRes) UIDs() []engine.ResUID {
x := &VirtUID{
BaseUID: engine.BaseUID{Name: obj.Name(), Kind: obj.Kind()},
// TODO: add more properties here so we can link to vm dependencies
}
return []engine.ResUID{x}
}
// UnmarshalYAML is the custom unmarshal handler for this struct.
// It is primarily useful for setting the defaults.
func (obj *VirtRes) UnmarshalYAML(unmarshal func(interface{}) error) error {
@@ -1171,30 +1161,3 @@ func isNotFound(err error) bool {
}
return false // some other error
}
// expandHome does an expansion of ~/ or ~james/ into user's home dir value.
func expandHome(p string) (string, error) {
if strings.HasPrefix(p, "~/") {
usr, err := user.Current()
if err != nil {
return p, fmt.Errorf("can't expand ~ into home directory")
}
return path.Join(usr.HomeDir, p[len("~/"):]), nil
}
// check if provided path is in format ~username and keep track of provided username
r, err := regexp.Compile("~([^/]+)/")
if err != nil {
return p, errwrap.Wrapf(err, "can't compile regexp")
}
if match := r.FindStringSubmatch(p); match != nil {
username := match[len(match)-1]
usr, err := user.Lookup(username)
if err != nil {
return p, fmt.Errorf("can't expand %s into home directory", match[0])
}
return path.Join(usr.HomeDir, p[len(match[0]):]), nil
}
return p, nil
}

67
engine/sendrecv.go Normal file
View File

@@ -0,0 +1,67 @@
// Mgmt
// Copyright (C) 2013-2018+ James Shubin and the project contributors
// Written by James Shubin <james@shubin.ca> and the project contributors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package engine
// SendableRes is the interface a resource must implement to support sending
// named parameters. You must specify to the engine what kind of values (and
// with their types) you will be sending. This is used for static type checking.
// Formerly, you had to make sure not to overwrite omitted parameters, otherwise
// it will be as if you've now declared a fixed state for that param. For that
// example, if a parameter `Foo string` had the zero value to mean that it was
// undefined, and you learned that the value is actually `up`, then sending on
// that param would cause that state to be managed, when it was previously not.
// This new interface actually provides a different namespace for sending keys.
type SendableRes interface {
Res // implement everything in Res but add the additional requirements
// Sends returns a struct containing the defaults of the type we send.
Sends() interface{}
// Send is used in CheckApply to send the desired data. It returns an
// error if the data is malformed or doesn't type check.
Send(st interface{}) error
// Sent returns the most recently sent data. This is used by the engine.
Sent() interface{}
}
// RecvableRes is the interface a resource must implement to support receiving
// on public parameters. The resource only has to include the correct trait for
// this interface to be fulfilled, as no additional methods need to be added. To
// get information about received changes, you can use the Recv method from the
// input API that comes in via Init.
type RecvableRes interface {
Res
// SetRecv stores the map of sendable data which should arrive here. It
// is called by the GAPI when building the resource.
SetRecv(recv map[string]*Send)
// Recv is used by the resource to get information on changes. This data
// can be used to invalidate caches, restart watches, or it can be
// ignored entirely.
Recv() map[string]*Send
}
// Send points to a value that a resource will send.
type Send struct {
Res SendableRes // a handle to the resource which is sending a value
Key string // the key in the resource that we're sending
Changed bool // set to true if this key was updated, read only!
}

42
engine/traits/autoedge.go Normal file
View File

@@ -0,0 +1,42 @@
// Mgmt
// Copyright (C) 2013-2018+ James Shubin and the project contributors
// Written by James Shubin <james@shubin.ca> and the project contributors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package traits
import (
"github.com/purpleidea/mgmt/engine"
)
// Edgeable contains a general implementation with some of the properties and
// methods needed to support autoedges on resources. It may be used as a start
// point to avoid re-implementing the straightforward methods.
type Edgeable struct {
meta *engine.AutoEdgeMeta
// Bug5819 works around issue https://github.com/golang/go/issues/5819
Bug5819 interface{} // XXX: workaround
}
// AutoEdgeMeta lets you get or set meta params for the automatic edges trait.
func (obj *Edgeable) AutoEdgeMeta() *engine.AutoEdgeMeta {
if obj.meta == nil { // set the defaults if previously empty
obj.meta = &engine.AutoEdgeMeta{
Disabled: false,
}
}
return obj.meta
}

89
engine/traits/autogroup.go Normal file
View File

@@ -0,0 +1,89 @@
// Mgmt
// Copyright (C) 2013-2018+ James Shubin and the project contributors
// Written by James Shubin <james@shubin.ca> and the project contributors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package traits
import (
"fmt"
"github.com/purpleidea/mgmt/engine"
)
// Groupable contains a general implementation with most of the properties and
// methods needed to support autogrouping on resources. It may be used as a
// starting point to avoid re-implementing the straightforward methods.
type Groupable struct {
meta *engine.AutoGroupMeta
isGrouped bool // am i contained within a group?
grouped []engine.GroupableRes // list of any grouped resources
// Bug5819 works around issue https://github.com/golang/go/issues/5819
Bug5819 interface{} // XXX: workaround
}
// AutoGroupMeta lets you get or set meta params for the automatic grouping
// trait.
func (obj *Groupable) AutoGroupMeta() *engine.AutoGroupMeta {
if obj.meta == nil { // set the defaults if previously empty
obj.meta = &engine.AutoGroupMeta{
Disabled: false,
}
}
return obj.meta
}
// GroupCmp compares two resources and decides if they're suitable for grouping.
// You'll probably want to override this method when implementing a resource...
// This base implementation assumes not, so override me!
func (obj *Groupable) GroupCmp(res engine.GroupableRes) error {
return fmt.Errorf("the default grouping compare is not nil")
}
// GroupRes groups resource argument (res) into self.
func (obj *Groupable) GroupRes(res engine.GroupableRes) error {
if l := len(res.GetGroup()); l > 0 {
return fmt.Errorf("the `%s` resource already contains %d grouped resources", res, l)
}
if res.IsGrouped() {
return fmt.Errorf("the `%s` resource is already grouped", res)
}
obj.grouped = append(obj.grouped, res)
res.SetGrouped(true) // i am contained _in_ a group
return nil
}
// IsGrouped determines if we are grouped.
func (obj *Groupable) IsGrouped() bool { // am I grouped?
return obj.isGrouped
}
// SetGrouped sets a flag to tell if we are grouped.
func (obj *Groupable) SetGrouped(b bool) {
obj.isGrouped = b
}
// GetGroup returns everyone grouped inside me.
func (obj *Groupable) GetGroup() []engine.GroupableRes {
return obj.grouped
}
// SetGroup sets the grouped resources into me.
func (obj *Groupable) SetGroup(grouped []engine.GroupableRes) {
obj.grouped = grouped
}

36
engine/traits/base.go Normal file
View File

@@ -0,0 +1,36 @@
// Mgmt
// Copyright (C) 2013-2018+ James Shubin and the project contributors
// Written by James Shubin <james@shubin.ca> and the project contributors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package traits
import (
"github.com/purpleidea/mgmt/engine"
)
// Base contains all the minimum necessary structs to build a resource. It
// should be used as a starting point to avoid re-implementing the
// straightforward methods.
type Base struct {
Kinded
Named
Meta
}
// String returns a string representation of a resource.
func (obj *Base) String() string {
return engine.Repr(obj.Kind(), obj.Name())
}

39
engine/traits/kind.go Normal file
View File

@@ -0,0 +1,39 @@
// Mgmt
// Copyright (C) 2013-2018+ James Shubin and the project contributors
// Written by James Shubin <james@shubin.ca> and the project contributors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package traits
// Kinded contains a general implementation of the properties and methods needed
// to support the resource kind. It should be used as a starting point to avoid
// re-implementing the straightforward kind methods.
type Kinded struct {
kind string
// Bug5819 works around issue https://github.com/golang/go/issues/5819
Bug5819 interface{} // XXX: workaround
}
// Kind returns the string representation for the kind this resource is.
func (obj *Kinded) Kind() string {
return obj.kind
}
// SetKind sets the kind string for this resource. It must only be set by the
// engine.
func (obj *Kinded) SetKind(kind string) {
obj.kind = kind
}

40
engine/traits/meta.go Normal file
View File

@@ -0,0 +1,40 @@
// Mgmt
// Copyright (C) 2013-2018+ James Shubin and the project contributors
// Written by James Shubin <james@shubin.ca> and the project contributors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package traits
import (
"github.com/purpleidea/mgmt/engine"
)
// Meta contains a general implementation of the properties and methods needed
// to support meta parameters. It should be used as a starting point to avoid
// re-implementing the straightforward meta methods.
type Meta struct {
meta *engine.MetaParams
// Bug5819 works around issue https://github.com/golang/go/issues/5819
Bug5819 interface{} // XXX: workaround
}
// MetaParams lets you get or set meta params for this trait.
func (obj *Meta) MetaParams() *engine.MetaParams {
if obj.meta == nil { // set the defaults if previously empty
obj.meta = engine.DefaultMetaParams.Copy()
}
return obj.meta
}

40
engine/traits/named.go Normal file
View File

@@ -0,0 +1,40 @@
// Mgmt
// Copyright (C) 2013-2018+ James Shubin and the project contributors
// Written by James Shubin <james@shubin.ca> and the project contributors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package traits
// Named contains a general implementation of the properties and methods needed
// to support named resources. It should be used as a starting point to avoid
// re-implementing the straightforward name methods.
type Named struct {
name string
// Bug5819 works around issue https://github.com/golang/go/issues/5819
Bug5819 interface{} // XXX: workaround
}
// Name returns the unique name this resource has. It is only unique within its
// own kind.
func (obj *Named) Name() string {
return obj.name
}
// SetName sets the unique name for this resource. It must only be unique within
// its own kind.
func (obj *Named) SetName(name string) {
obj.name = name
}

40
engine/traits/refresh.go Normal file
View File

@@ -0,0 +1,40 @@
// Mgmt
// Copyright (C) 2013-2018+ James Shubin and the project contributors
// Written by James Shubin <james@shubin.ca> and the project contributors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package traits
// Refreshable functions as flag storage for resources to signal that they
// support receiving refresh notifications, and what that value is. These are
// commonly used to send information that some aspect of the state is invalid
// due to an unlinked change. The canonical example is a svc resource that needs
// reloading after a configuration file changes.
type Refreshable struct {
refresh bool
// Bug5819 works around issue https://github.com/golang/go/issues/5819
Bug5819 interface{} // XXX: workaround
}
// Refresh returns the refresh notification state.
func (obj *Refreshable) Refresh() bool {
return obj.refresh
}
// SetRefresh sets the refresh notification state.
func (obj *Refreshable) SetRefresh(b bool) {
obj.refresh = b
}

75
engine/traits/sendrecv.go Normal file
View File

@@ -0,0 +1,75 @@
// Mgmt
// Copyright (C) 2013-2018+ James Shubin and the project contributors
// Written by James Shubin <james@shubin.ca> and the project contributors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package traits
import (
"github.com/purpleidea/mgmt/engine"
)
// Sendable contains a general implementation with some of the properties and
// methods needed to implement sending from resources. You'll need to implement
// the Sends method, and call the Send method in CheckApply via the Init API.
type Sendable struct {
send interface{}
// Bug5819 works around issue https://github.com/golang/go/issues/5819
Bug5819 interface{} // XXX: workaround
}
// Sends returns a struct containing the defaults of the type we send. This
// needs to be implemented (overridden) by the struct with the Sendable trait to
// be able to send any values. The public struct field names are the keys used.
func (obj *Sendable) Sends() interface{} {
return nil
}
// Send is used to send a struct in CheckApply. This is typically wrapped in the
// resource API and consumed that way.
func (obj *Sendable) Send(st interface{}) error {
// TODO: can we (or should we) run the type checking here instead?
obj.send = st
return nil
}
// Sent returns the struct of values that have been sent by this resource.
func (obj *Sendable) Sent() interface{} {
return obj.send
}
// Recvable contains a general implementation with some of the properties and
// methods needed to implement receiving from resources.
type Recvable struct {
recv map[string]*engine.Send
// Bug5819 works around issue https://github.com/golang/go/issues/5819
Bug5819 interface{} // XXX: workaround
}
// SetRecv is used to inject incoming values into the resource.
func (obj *Recvable) SetRecv(recv map[string]*engine.Send) {
//if obj.recv == nil {
// obj.recv = make(map[string]*engine.Send)
//}
obj.recv = recv
}
// Recv is used to get information that was passed in. This data can then be
// used to run the Send/Recv data transfer.
func (obj *Recvable) Recv() map[string]*engine.Send {
return obj.recv
}

41
engine/util.go Normal file
View File

@@ -0,0 +1,41 @@
// Mgmt
// Copyright (C) 2013-2018+ James Shubin and the project contributors
// Written by James Shubin <james@shubin.ca> and the project contributors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package engine
import (
"sort"
)
// ResourceSlice is a linear list of resources. It can be sorted.
type ResourceSlice []Res
func (rs ResourceSlice) Len() int { return len(rs) }
func (rs ResourceSlice) Swap(i, j int) { rs[i], rs[j] = rs[j], rs[i] }
func (rs ResourceSlice) Less(i, j int) bool { return rs[i].String() < rs[j].String() }
// Sort the list of resources and return a copy without modifying the input.
func Sort(rs []Res) []Res {
resources := []Res{}
for _, r := range rs { // copy
resources = append(resources, r)
}
sort.Sort(ResourceSlice(resources))
return resources
// sort.Sort(ResourceSlice(rs)) // this is wrong, it would modify input!
//return rs
}

39
resources/util.go → engine/util/util.go
View File

@@ -15,7 +15,7 @@
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package resources
package util
import (
"bytes"
@@ -24,10 +24,10 @@ import (
"fmt"
"os/user"
"reflect"
"sort"
"strconv"
"strings"
"github.com/purpleidea/mgmt/engine"
"github.com/purpleidea/mgmt/lang/types"
errwrap "github.com/pkg/errors"
@@ -38,27 +38,8 @@ const (
StructTag = "lang"
)
// ResourceSlice is a linear list of resources. It can be sorted.
type ResourceSlice []Res
func (rs ResourceSlice) Len() int { return len(rs) }
func (rs ResourceSlice) Swap(i, j int) { rs[i], rs[j] = rs[j], rs[i] }
func (rs ResourceSlice) Less(i, j int) bool { return rs[i].String() < rs[j].String() }
// Sort the list of resources and return a copy without modifying the input.
func Sort(rs []Res) []Res {
resources := []Res{}
for _, r := range rs { // copy
resources = append(resources, r)
}
sort.Sort(ResourceSlice(resources))
return resources
// sort.Sort(ResourceSlice(rs)) // this is wrong, it would modify input!
//return rs
}
// ResToB64 encodes a resource to a base64 encoded string (after serialization).
func ResToB64(res Res) (string, error) {
func ResToB64(res engine.Res) (string, error) {
b := bytes.Buffer{}
e := gob.NewEncoder(&b)
err := e.Encode(&res) // pass with &
@@ -69,7 +50,7 @@ func ResToB64(res Res) (string, error) {
}
// B64ToRes decodes a resource from a base64 encoded string (after deserialization).
func B64ToRes(str string) (Res, error) {
func B64ToRes(str string) (engine.Res, error) {
var output interface{}
bb, err := base64.StdEncoding.DecodeString(str)
if err != nil {
@@ -80,7 +61,7 @@ func B64ToRes(str string) (Res, error) {
if err := d.Decode(&output); err != nil { // pass with &
return nil, errwrap.Wrapf(err, "gob failed to decode")
}
res, ok := output.(Res)
res, ok := output.(engine.Res)
if !ok {
return nil, fmt.Errorf("output `%v` is not a Res", output)
}
@@ -89,7 +70,7 @@ func B64ToRes(str string) (Res, error) {
// StructTagToFieldName returns a mapping from recommended alias to actual field
// name. It returns an error if it finds a collision. It uses the `lang` tags.
func StructTagToFieldName(res Res) (map[string]string, error) {
func StructTagToFieldName(res engine.Res) (map[string]string, error) {
// TODO: fallback to looking up yaml tags, although harder to parse
result := make(map[string]string) // `lang` field tag -> field name
st := reflect.TypeOf(res).Elem() // elem for ptr to res
@@ -113,7 +94,7 @@ func StructTagToFieldName(res Res) (map[string]string, error) {
// LowerStructFieldNameToFieldName returns a mapping from the lower case version
// of each field name to the actual field name. It only returns public fields.
// It returns an error if it finds a collision.
func LowerStructFieldNameToFieldName(res Res) (map[string]string, error) {
func LowerStructFieldNameToFieldName(res engine.Res) (map[string]string, error) {
result := make(map[string]string) // lower field name -> field name
st := reflect.TypeOf(res).Elem() // elem for ptr to res
for i := 0; i < st.NumField(); i++ {
@@ -142,7 +123,7 @@ func LowerStructFieldNameToFieldName(res Res) (map[string]string, error) {
// but this is currently not implemented.
// TODO: should this behaviour be changed?
func LangFieldNameToStructFieldName(kind string) (map[string]string, error) {
res, err := NewResource(kind)
res, err := engine.NewResource(kind)
if err != nil {
return nil, err
}
@@ -163,7 +144,7 @@ func LangFieldNameToStructFieldName(kind string) (map[string]string, error) {
// StructKindToFieldNameTypeMap returns a map from field name to expected type
// in the lang type system.
func StructKindToFieldNameTypeMap(kind string) (map[string]*types.Type, error) {
res, err := NewResource(kind)
res, err := engine.NewResource(kind)
if err != nil {
return nil, err
}
@@ -180,7 +161,7 @@ func StructKindToFieldNameTypeMap(kind string) (map[string]*types.Type, error) {
field := st.Field(i)
name := field.Name
// TODO: in future, skip over fields that don't have a `lang` tag
//if name == "BaseRes" { // TODO: hack!!!
//if name == "Base" { // TODO: hack!!!
// continue
//}

105
engine/util/util_test.go Normal file
View File

@@ -0,0 +1,105 @@
// Mgmt
// Copyright (C) 2013-2018+ James Shubin and the project contributors
// Written by James Shubin <james@shubin.ca> and the project contributors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package util
import (
"os/user"
"strconv"
"testing"
)
func TestUnknownGroup(t *testing.T) {
gid, err := GetGID("unknowngroup")
if err == nil {
t.Errorf("expected failure, but passed with: %d", gid)
}
}
func TestUnknownUser(t *testing.T) {
uid, err := GetUID("unknownuser")
if err == nil {
t.Errorf("expected failure, but passed with: %d", uid)
}
}
func TestCurrentUserGroupByName(t *testing.T) {
// get current user
userObj, err := user.Current()
if err != nil {
t.Errorf("error trying to lookup current user: %s", err.Error())
}
currentUID := userObj.Uid
currentGID := userObj.Gid
var uid int
var gid int
// now try to get the uid/gid via our API (via username and group name)
if uid, err = GetUID(userObj.Username); err != nil {
t.Errorf("error trying to lookup current user UID: %s", err.Error())
}
if strconv.Itoa(uid) != currentUID {
t.Errorf("uid didn't match current user's: %s vs %s", strconv.Itoa(uid), currentUID)
}
// macOS users do not have a group with their name on it, so not assuming this here
group, err := user.LookupGroupId(currentGID)
if err != nil {
t.Errorf("failed to lookup group by id: %s", currentGID)
}
if gid, err = GetGID(group.Name); err != nil {
t.Errorf("error trying to lookup current user UID: %s", err.Error())
}
if strconv.Itoa(gid) != currentGID {
t.Errorf("gid didn't match current user's: %s vs %s", strconv.Itoa(gid), currentGID)
}
}
func TestCurrentUserGroupById(t *testing.T) {
// get current user
userObj, err := user.Current()
if err != nil {
t.Errorf("error trying to lookup current user: %s", err.Error())
}
currentUID := userObj.Uid
currentGID := userObj.Gid
var uid int
var gid int
// now try to get the uid/gid via our API (via uid and gid)
if uid, err = GetUID(currentUID); err != nil {
t.Errorf("error trying to lookup current user UID: %s", err.Error())
}
if strconv.Itoa(uid) != currentUID {
t.Errorf("uid didn't match current user's: %s vs %s", strconv.Itoa(uid), currentUID)
}
if gid, err = GetGID(currentGID); err != nil {
t.Errorf("error trying to lookup current user UID: %s", err.Error())
}
if strconv.Itoa(gid) != currentGID {
t.Errorf("gid didn't match current user's: %s vs %s", strconv.Itoa(gid), currentGID)
}
}

48
engine/world.go Normal file
View File

@@ -0,0 +1,48 @@
// Mgmt
// Copyright (C) 2013-2018+ James Shubin and the project contributors
// Written by James Shubin <james@shubin.ca> and the project contributors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package engine
import (
"github.com/purpleidea/mgmt/etcd/scheduler"
)
// World is an interface to the rest of the different graph state. It allows
// the GAPI to store state and exchange information throughout the cluster. It
// is the interface each machine uses to communicate with the rest of the world.
type World interface { // TODO: is there a better name for this interface?
ResWatch() chan error
ResExport([]Res) error
// FIXME: should this method take a "filter" data struct instead of many args?
ResCollect(hostnameFilter, kindFilter []string) ([]Res, error)
StrWatch(namespace string) chan error
StrIsNotExist(error) bool
StrGet(namespace string) (string, error)
StrSet(namespace, value string) error
StrDel(namespace string) error
// XXX: add the exchange primitives in here directly?
StrMapWatch(namespace string) chan error
StrMapGet(namespace string) (map[string]string, error)
StrMapSet(namespace, value string) error
StrMapDel(namespace string) error
Scheduler(namespace string, opts ...scheduler.Option) (*scheduler.Result, error)
Fs(uri string) (Fs, error)
}

11
etcd/etcd.go
View File

@@ -32,7 +32,7 @@
// * If a seed is given, connect as a client, and optionally volunteer to be a server.
// * All volunteering clients should listen for a message from the master for nomination.
// * If a client has been nominated, it should startup a server.
// * All servers should list for their nomination to be removed and shutdown if so.
// * All servers should listen for their nomination to be removed and shutdown if so.
// * The elected leader should decide who to nominate/unnominate to keep the right number of servers.
//
// Smoke testing:
@@ -64,7 +64,7 @@ import (
"time"
"github.com/purpleidea/mgmt/converger"
"github.com/purpleidea/mgmt/event"
"github.com/purpleidea/mgmt/etcd/event"
"github.com/purpleidea/mgmt/util"
etcd "github.com/coreos/etcd/clientv3" // "clientv3"
@@ -283,6 +283,13 @@ func NewEmbdEtcd(hostname string, seeds, clientURLs, serverURLs, advertiseClient
obj.serverURLs = []url.URL{*u}
}
if converger != nil {
converger.AddStateFn("etcd-hostname", func(converged bool) error {
// send our individual state into etcd for others to see
return SetHostnameConverged(obj, hostname, converged) // TODO: what should happen on error?
})
}
return obj
}

49
event/event.go → etcd/event/event.go
View File

@@ -22,54 +22,10 @@ import (
"fmt"
)
//go:generate stringer -type=Kind -output=kind_stringer.go
// Kind represents the type of event being passed.
type Kind int
// The different event kinds are used in different contexts.
const (
EventNil Kind = iota
EventExit
EventStart
EventPause
EventPoke
EventBackPoke
)
// Resp is a channel to be used for boolean responses. A nil represents an ACK,
// and a non-nil represents a NACK (false). This also lets us use custom errors.
type Resp chan error
// Event is the main struct that stores event information and responses.
type Event struct {
Kind Kind
Resp Resp // channel to send an ack response on, nil to skip
//Wg *sync.WaitGroup // receiver barrier to Wait() for everyone else on
Err error // store an error in our event
}
// ACK sends a single acknowledgement on the channel if one was requested.
func (event *Event) ACK() {
if event.Resp != nil { // if they've requested an ACK
event.Resp.ACK()
}
}
// NACK sends a negative acknowledgement message on the channel if one was requested.
func (event *Event) NACK() {
if event.Resp != nil { // if they've requested a NACK
event.Resp.NACK()
}
}
// ACKNACK sends a custom ACK or NACK message on the channel if one was requested.
func (event *Event) ACKNACK(err error) {
if event.Resp != nil { // if they've requested a NACK
event.Resp.ACKNACK(err)
}
}
// NewResp is just a helper to return the right type of response channel.
func NewResp() Resp {
resp := make(chan error)
@@ -112,8 +68,3 @@ func (resp Resp) ACKWait() {
}
}
}
// Error returns the stored error value.
func (event *Event) Error() error {
return event.Err
}

29
etcd/resources.go
View File

@@ -22,7 +22,8 @@ import (
"log"
"strings"
"github.com/purpleidea/mgmt/resources"
"github.com/purpleidea/mgmt/engine"
engineUtil "github.com/purpleidea/mgmt/engine/util"
"github.com/purpleidea/mgmt/util"
etcd "github.com/coreos/etcd/clientv3"
@@ -60,7 +61,7 @@ func WatchResources(obj *EmbdEtcd) chan error {
}
// SetResources exports all of the resources which we pass in to etcd.
func SetResources(obj *EmbdEtcd, hostname string, resourceList []resources.Res) error {
func SetResources(obj *EmbdEtcd, hostname string, resourceList []engine.Res) error {
// key structure is $NS/exported/$hostname/resources/$uid = $data
var kindFilter []string // empty to get from everyone
@@ -79,12 +80,12 @@ func SetResources(obj *EmbdEtcd, hostname string, resourceList []resources.Res)
ifs := []etcd.Cmp{} // list matching the desired state
ops := []etcd.Op{} // list of ops in this transaction
for _, res := range resourceList {
if res.GetKind() == "" {
log.Fatalf("Etcd: SetResources: Error: Empty kind: %v", res.GetName())
if res.Kind() == "" {
log.Fatalf("Etcd: SetResources: Error: Empty kind: %v", res.Name())
}
uid := fmt.Sprintf("%s/%s", res.GetKind(), res.GetName())
uid := fmt.Sprintf("%s/%s", res.Kind(), res.Name())
path := fmt.Sprintf("%s/exported/%s/resources/%s", NS, hostname, uid)
if data, err := resources.ResToB64(res); err == nil {
if data, err := engineUtil.ResToB64(res); err == nil {
ifs = append(ifs, etcd.Compare(etcd.Value(path), "=", data)) // desired state
ops = append(ops, etcd.OpPut(path, data))
} else {
@@ -92,9 +93,9 @@ func SetResources(obj *EmbdEtcd, hostname string, resourceList []resources.Res)
}
}
match := func(res resources.Res, resourceList []resources.Res) bool { // helper lambda
match := func(res engine.Res, resourceList []engine.Res) bool { // helper lambda
for _, x := range resourceList {
if res.GetKind() == x.GetKind() && res.GetName() == x.GetName() {
if res.Kind() == x.Kind() && res.Name() == x.Name() {
return true
}
}
@@ -104,10 +105,10 @@ func SetResources(obj *EmbdEtcd, hostname string, resourceList []resources.Res)
hasDeletes := false
// delete old, now unused resources here...
for _, res := range originals {
if res.GetKind() == "" {
log.Fatalf("Etcd: SetResources: Error: Empty kind: %v", res.GetName())
if res.Kind() == "" {
log.Fatalf("Etcd: SetResources: Error: Empty kind: %v", res.Name())
}
uid := fmt.Sprintf("%s/%s", res.GetKind(), res.GetName())
uid := fmt.Sprintf("%s/%s", res.Kind(), res.Name())
path := fmt.Sprintf("%s/exported/%s/resources/%s", NS, hostname, uid)
if match(res, resourceList) { // if we match, no need to delete!
@@ -135,10 +136,10 @@ func SetResources(obj *EmbdEtcd, hostname string, resourceList []resources.Res)
// TODO: Expand this with a more powerful filter based on what we eventually
// support in our collect DSL. Ideally a server side filter like WithFilter()
// We could do this if the pattern was $NS/exported/$kind/$hostname/$uid = $data.
func GetResources(obj *EmbdEtcd, hostnameFilter, kindFilter []string) ([]resources.Res, error) {
func GetResources(obj *EmbdEtcd, hostnameFilter, kindFilter []string) ([]engine.Res, error) {
// key structure is $NS/exported/$hostname/resources/$uid = $data
path := fmt.Sprintf("%s/exported/", NS)
resourceList := []resources.Res{}
resourceList := []engine.Res{}
keyMap, err := obj.Get(path, etcd.WithPrefix(), etcd.WithSort(etcd.SortByKey, etcd.SortAscend))
if err != nil {
return nil, fmt.Errorf("could not get resources: %v", err)
@@ -170,7 +171,7 @@ func GetResources(obj *EmbdEtcd, hostnameFilter, kindFilter []string) ([]resourc
continue
}
if obj, err := resources.B64ToRes(val); err == nil {
if obj, err := engineUtil.B64ToRes(val); err == nil {
log.Printf("Etcd: Get: (Hostname, Kind, Name): (%s, %s, %s)", hostname, kind, name)
resourceList = append(resourceList, obj)
} else {

14
etcd/world.go
View File

@@ -22,18 +22,18 @@ import (
"net/url"
"strings"
"github.com/purpleidea/mgmt/engine"
etcdfs "github.com/purpleidea/mgmt/etcd/fs"
"github.com/purpleidea/mgmt/etcd/scheduler"
"github.com/purpleidea/mgmt/resources"
)
// World is an etcd backed implementation of the World interface.
type World struct {
Hostname string // uuid for the consumer of these
EmbdEtcd *EmbdEtcd
MetadataPrefix string // expected metadata prefix
StoragePrefix string // storage prefix for etcdfs storage
StandaloneFs resources.Fs // store an fs here for local usage
MetadataPrefix string // expected metadata prefix
StoragePrefix string // storage prefix for etcdfs storage
StandaloneFs engine.Fs // store an fs here for local usage
Debug bool
Logf func(format string, v ...interface{})
}
@@ -46,13 +46,13 @@ func (obj *World) ResWatch() chan error {
// ResExport exports a list of resources under our hostname namespace.
// Subsequent calls replace the previously set collection atomically.
func (obj *World) ResExport(resourceList []resources.Res) error {
func (obj *World) ResExport(resourceList []engine.Res) error {
return SetResources(obj.EmbdEtcd, obj.Hostname, resourceList)
}
// ResCollect gets the collection of exported resources which match the filter.
// It does this atomically so that a call always returns a complete collection.
func (obj *World) ResCollect(hostnameFilter, kindFilter []string) ([]resources.Res, error) {
func (obj *World) ResCollect(hostnameFilter, kindFilter []string) ([]engine.Res, error) {
// XXX: should we be restricted to retrieving resources that were
// exported with a tag that allows or restricts our hostname? We could
// enforce that here if the underlying API supported it... Add this?
@@ -122,7 +122,7 @@ func (obj *World) Scheduler(namespace string, opts ...scheduler.Option) (*schedu
// execution that doesn't span more than a single host, this file system might
// actually be a local or memory backed file system, so actually only
// distributed within the boredom that is a single host cluster.
func (obj *World) Fs(uri string) (resources.Fs, error) {
func (obj *World) Fs(uri string) (engine.Fs, error) {
u, err := url.Parse(uri)
if err != nil {
return nil, err

59
examples/lib/exec-send-recv.go
View File

@@ -10,10 +10,11 @@ import (
"syscall"
"time"
"github.com/purpleidea/mgmt/engine"
"github.com/purpleidea/mgmt/engine/resources"
"github.com/purpleidea/mgmt/gapi"
mgmt "github.com/purpleidea/mgmt/lib"
"github.com/purpleidea/mgmt/pgraph"
"github.com/purpleidea/mgmt/resources"
"github.com/urfave/cli"
)
@@ -49,7 +50,7 @@ func NewMyGAPI(data gapi.Data, name string, interval uint) (*MyGAPI, error) {
// should take the prefix of the registered name. On activation, if there are
// any validation problems, you should return an error. If this was not
// activated, then you should return a nil GAPI and a nil error.
func (obj *MyGAPI) Cli(c *cli.Context, fs resources.Fs) (*gapi.Deploy, error) {
func (obj *MyGAPI) Cli(c *cli.Context, fs engine.Fs) (*gapi.Deploy, error) {
if s := c.String(obj.Name); c.IsSet(obj.Name) {
if s != "" {
return nil, fmt.Errorf("input is not empty")
@@ -103,66 +104,46 @@ func (obj *MyGAPI) Graph() (*pgraph.Graph, error) {
return nil, err
}
metaparams := resources.DefaultMetaParams
exec1 := &resources.ExecRes{
BaseRes: resources.BaseRes{
Name: "exec1",
Kind: "exec",
MetaParams: metaparams,
},
Cmd: "echo hello world && echo goodbye world 1>&2", // to stdout && stderr
Shell: "/bin/bash",
}
g.AddVertex(exec1)
output := &resources.FileRes{
BaseRes: resources.BaseRes{
Name: "output",
Kind: "file",
MetaParams: metaparams,
// send->recv!
Recv: map[string]*resources.Send{
"Content": {Res: exec1, Key: "Output"},
},
},
Path: "/tmp/mgmt/output",
State: "present",
}
// XXX: add send->recv!
//Recv: map[string]*engine.Send{
// "Content": {Res: exec1, Key: "Output"},
//},
g.AddVertex(output)
g.AddEdge(exec1, output, &resources.Edge{Name: "e0"})
g.AddEdge(exec1, output, &engine.Edge{Name: "e0"})
stdout := &resources.FileRes{
BaseRes: resources.BaseRes{
Name: "stdout",
Kind: "file",
MetaParams: metaparams,
// send->recv!
Recv: map[string]*resources.Send{
"Content": {Res: exec1, Key: "Stdout"},
},
},
Path: "/tmp/mgmt/stdout",
State: "present",
}
// XXX: add send->recv!
//Recv: map[string]*engine.Send{
// "Content": {Res: exec1, Key: "Stdout"},
//},
g.AddVertex(stdout)
g.AddEdge(exec1, stdout, &resources.Edge{Name: "e1"})
g.AddEdge(exec1, stdout, &engine.Edge{Name: "e1"})
stderr := &resources.FileRes{
BaseRes: resources.BaseRes{
Name: "stderr",
Kind: "file",
MetaParams: metaparams,
// send->recv!
Recv: map[string]*resources.Send{
"Content": {Res: exec1, Key: "Stderr"},
},
},
Path: "/tmp/mgmt/stderr",
State: "present",
}
// XXX: add send->recv!
//Recv: map[string]*engine.Send{
// "Content": {Res: exec1, Key: "Stderr"},
//},
g.AddVertex(stderr)
g.AddEdge(exec1, stderr, &resources.Edge{Name: "e2"})
g.AddEdge(exec1, stderr, &engine.Edge{Name: "e2"})
//g, err := config.NewGraphFromConfig(obj.data.Hostname, obj.data.World, obj.data.Noop)
return g, nil

31
examples/lib/libmgmt-subgraph0.go
View File

@@ -10,10 +10,11 @@ import (
"syscall"
"time"
"github.com/purpleidea/mgmt/engine"
"github.com/purpleidea/mgmt/engine/resources"
"github.com/purpleidea/mgmt/gapi"
mgmt "github.com/purpleidea/mgmt/lib"
"github.com/purpleidea/mgmt/pgraph"
"github.com/purpleidea/mgmt/resources"
errwrap "github.com/pkg/errors"
"github.com/urfave/cli"
@@ -54,7 +55,7 @@ func NewMyGAPI(data gapi.Data, name string, interval uint) (*MyGAPI, error) {
// should take the prefix of the registered name. On activation, if there are
// any validation problems, you should return an error. If this was not
// activated, then you should return a nil GAPI and a nil error.
func (obj *MyGAPI) Cli(c *cli.Context, fs resources.Fs) (*gapi.Deploy, error) {
func (obj *MyGAPI) Cli(c *cli.Context, fs engine.Fs) (*gapi.Deploy, error) {
if s := c.String(obj.Name); c.IsSet(obj.Name) {
if s != "" {
return nil, fmt.Errorf("input is not empty")
@@ -103,27 +104,13 @@ func (obj *MyGAPI) subGraph() (*pgraph.Graph, error) {
return nil, err
}
metaparams := resources.DefaultMetaParams
f1 := &resources.FileRes{
BaseRes: resources.BaseRes{
Name: "file1",
Kind: "file",
MetaParams: metaparams,
},
Path: "/tmp/mgmt/sub1",
Path: "/tmp/mgmt/sub1",
State: "present",
}
g.AddVertex(f1)
n1 := &resources.NoopRes{
BaseRes: resources.BaseRes{
Name: "noop1",
Kind: "noop",
MetaParams: metaparams,
},
}
n1 := &resources.NoopRes{}
g.AddVertex(n1)
return g, nil
@@ -140,14 +127,8 @@ func (obj *MyGAPI) Graph() (*pgraph.Graph, error) {
return nil, err
}
metaparams := resources.DefaultMetaParams
content := "I created a subgraph!\n"
f0 := &resources.FileRes{
BaseRes: resources.BaseRes{
Name: "README",
MetaParams: metaparams,
},
Path: "/tmp/mgmt/README",
Content: &content,
State: "present",
@@ -160,7 +141,7 @@ func (obj *MyGAPI) Graph() (*pgraph.Graph, error) {
}
edgeGenFn := func(v1, v2 pgraph.Vertex) pgraph.Edge {
edge := &resources.Edge{
edge := &engine.Edge{
Name: fmt.Sprintf("edge: %s->%s", v1, v2),
}

40
examples/lib/libmgmt-subgraph1.go
View File

@@ -10,10 +10,11 @@ import (
"syscall"
"time"
"github.com/purpleidea/mgmt/engine"
"github.com/purpleidea/mgmt/engine/resources"
"github.com/purpleidea/mgmt/gapi"
mgmt "github.com/purpleidea/mgmt/lib"
"github.com/purpleidea/mgmt/pgraph"
"github.com/purpleidea/mgmt/resources"
"github.com/urfave/cli"
)
@@ -49,7 +50,7 @@ func NewMyGAPI(data gapi.Data, name string, interval uint) (*MyGAPI, error) {
// should take the prefix of the registered name. On activation, if there are
// any validation problems, you should return an error. If this was not
// activated, then you should return a nil GAPI and a nil error.
func (obj *MyGAPI) Cli(c *cli.Context, fs resources.Fs) (*gapi.Deploy, error) {
func (obj *MyGAPI) Cli(c *cli.Context, fs engine.Fs) (*gapi.Deploy, error) {
if s := c.String(obj.Name); c.IsSet(obj.Name) {
if s != "" {
return nil, fmt.Errorf("input is not empty")
@@ -103,15 +104,8 @@ func (obj *MyGAPI) Graph() (*pgraph.Graph, error) {
return nil, err
}
metaparams := resources.DefaultMetaParams
content := "I created a subgraph!\n"
f0 := &resources.FileRes{
BaseRes: resources.BaseRes{
Name: "README",
Kind: "file",
MetaParams: metaparams,
},
Path: "/tmp/mgmt/README",
Content: &content,
State: "present",
@@ -126,40 +120,24 @@ func (obj *MyGAPI) Graph() (*pgraph.Graph, error) {
// add elements into the sub graph
f1 := &resources.FileRes{
BaseRes: resources.BaseRes{
Name: "file1",
Kind: "file",
MetaParams: metaparams,
},
Path: "/tmp/mgmt/sub1",
State: "present",
}
subGraph.AddVertex(f1)
n1 := &resources.NoopRes{
BaseRes: resources.BaseRes{
Name: "noop1",
Kind: "noop",
MetaParams: metaparams,
},
}
n1 := &resources.NoopRes{}
subGraph.AddVertex(n1)
e0 := &resources.Edge{Name: "e0"}
e0 := &engine.Edge{Name: "e0"}
e0.Notify = true // send a notification from v0 to v1
subGraph.AddEdge(f1, n1, e0)
// create the actual resource to hold the sub graph
subGraphRes0 := &resources.GraphRes{ // TODO: should we name this SubGraphRes ?
BaseRes: resources.BaseRes{
Name: "subgraph1",
Kind: "graph",
MetaParams: metaparams,
},
Graph: subGraph,
}
g.AddVertex(subGraphRes0) // add it to the main graph
//subGraphRes0 := &resources.GraphRes{ // TODO: should we name this SubGraphRes ?
// Graph: subGraph,
//}
//g.AddVertex(subGraphRes0) // add it to the main graph
//g, err := config.NewGraphFromConfig(obj.data.Hostname, obj.data.World, obj.data.Noop)
return g, nil

246
examples/lib/libmgmt1.go
View File

@@ -1,246 +0,0 @@
// libmgmt example
package main
import (
"fmt"
"log"
"os"
"os/signal"
"sync"
"syscall"
"time"
"github.com/purpleidea/mgmt/gapi"
mgmt "github.com/purpleidea/mgmt/lib"
"github.com/purpleidea/mgmt/pgraph"
"github.com/purpleidea/mgmt/resources"
"github.com/purpleidea/mgmt/yamlgraph"
"github.com/urfave/cli"
)
// XXX: this has not been updated to latest GAPI/Deploy API. Patches welcome!
const (
// Name is the name of this frontend.
Name = "libmgmt"
)
// MyGAPI implements the main GAPI interface.
type MyGAPI struct {
Name string // graph name
Interval uint // refresh interval, 0 to never refresh
data gapi.Data
initialized bool
closeChan chan struct{}
wg sync.WaitGroup // sync group for tunnel go routines
}
// NewMyGAPI creates a new MyGAPI struct and calls Init().
func NewMyGAPI(data gapi.Data, name string, interval uint) (*MyGAPI, error) {
obj := &MyGAPI{
Name: name,
Interval: interval,
}
return obj, obj.Init(data)
}
// Cli takes a cli.Context, and returns our GAPI if activated. All arguments
// should take the prefix of the registered name. On activation, if there are
// any validation problems, you should return an error. If this was not
// activated, then you should return a nil GAPI and a nil error.
func (obj *MyGAPI) Cli(c *cli.Context, fs resources.Fs) (*gapi.Deploy, error) {
if s := c.String(obj.Name); c.IsSet(obj.Name) {
if s != "" {
return nil, fmt.Errorf("input is not empty")
}
return &gapi.Deploy{
Name: obj.Name,
Noop: c.GlobalBool("noop"),
Sema: c.GlobalInt("sema"),
GAPI: &MyGAPI{
// TODO: add properties here...
},
}, nil
}
return nil, nil // we weren't activated!
}
// CliFlags returns a list of flags used by this deploy subcommand.
func (obj *MyGAPI) CliFlags() []cli.Flag {
return []cli.Flag{
cli.StringFlag{
Name: obj.Name,
Value: "",
Usage: "run",
},
}
}
// Init initializes the MyGAPI struct.
func (obj *MyGAPI) Init(data gapi.Data) error {
if obj.initialized {
return fmt.Errorf("already initialized")
}
if obj.Name == "" {
return fmt.Errorf("the graph name must be specified")
}
obj.data = data // store for later
obj.closeChan = make(chan struct{})
obj.initialized = true
return nil
}
// Graph returns a current Graph.
func (obj *MyGAPI) Graph() (*pgraph.Graph, error) {
if !obj.initialized {
return nil, fmt.Errorf("%s: MyGAPI is not initialized", Name)
}
n1, err := resources.NewNamedResource("noop", "noop1")
if err != nil {
return nil, err
}
// NOTE: This is considered the legacy method to build graphs. Avoid
// importing the legacy `yamlgraph` lib if possible for custom graphs.
// we can still build a graph via the yaml method
gc := &yamlgraph.GraphConfig{
Graph: obj.Name,
Resources: yamlgraph.Resources{ // must redefine anonymous struct :(
// in alphabetical order
Exec: []*resources.ExecRes{},
File: []*resources.FileRes{},
Msg: []*resources.MsgRes{},
Noop: []*resources.NoopRes{n1.(*resources.NoopRes)},
Pkg: []*resources.PkgRes{},
Svc: []*resources.SvcRes{},
Timer: []*resources.TimerRes{},
Virt: []*resources.VirtRes{},
},
//Collector: []collectorResConfig{},
//Edges: []Edge{},
Comment: "comment!",
}
g, err := gc.NewGraphFromConfig(obj.data.Hostname, obj.data.World, obj.data.Noop)
return g, err
}
// Next returns nil errors every time there could be a new graph.
func (obj *MyGAPI) Next() chan gapi.Next {
ch := make(chan gapi.Next)
obj.wg.Add(1)
go func() {
defer obj.wg.Done()
defer close(ch) // this will run before the obj.wg.Done()
if !obj.initialized {
next := gapi.Next{
Err: fmt.Errorf("%s: MyGAPI is not initialized", Name),
Exit: true, // exit, b/c programming error?
}
ch <- next
}
startChan := make(chan struct{}) // start signal
close(startChan) // kick it off!
ticker := make(<-chan time.Time)
if obj.data.NoStreamWatch || obj.Interval <= 0 {
ticker = nil
} else {
// arbitrarily change graph every interval seconds
t := time.NewTicker(time.Duration(obj.Interval) * time.Second)
defer t.Stop()
ticker = t.C
}
for {
select {
case <-startChan: // kick the loop once at start
startChan = nil // disable
// pass
case <-ticker:
// pass
case <-obj.closeChan:
return
}
log.Printf("%s: Generating new graph...", Name)
select {
case ch <- gapi.Next{}: // trigger a run
case <-obj.closeChan:
return
}
}
}()
return ch
}
// Close shuts down the MyGAPI.
func (obj *MyGAPI) Close() error {
if !obj.initialized {
return fmt.Errorf("%s: MyGAPI is not initialized", Name)
}
close(obj.closeChan)
obj.wg.Wait()
obj.initialized = false // closed = true
return nil
}
// Run runs an embedded mgmt server.
func Run() error {
obj := &mgmt.Main{}
obj.Program = "libmgmt" // TODO: set on compilation
obj.Version = "0.0.1" // TODO: set on compilation
obj.TmpPrefix = true
obj.IdealClusterSize = -1
obj.ConvergedTimeout = -1
obj.Noop = true
//obj.GAPI = &MyGAPI{ // graph API
// Name: "libmgmt", // TODO: set on compilation
// Interval: 15, // arbitrarily change graph every 15 seconds
//}
if err := obj.Init(); err != nil {
return err
}
// install the exit signal handler
exit := make(chan struct{})
defer close(exit)
go func() {
signals := make(chan os.Signal, 1)
signal.Notify(signals, os.Interrupt) // catch ^C
//signal.Notify(signals, os.Kill) // catch signals
signal.Notify(signals, syscall.SIGTERM)
select {
case sig := <-signals: // any signal will do
if sig == os.Interrupt {
log.Println("Interrupted by ^C")
obj.Exit(nil)
return
}
log.Println("Interrupted by signal")
obj.Exit(fmt.Errorf("killed by %v", sig))
return
case <-exit:
return
}
}()
return obj.Run()
}
func main() {
log.Printf("Hello!")
if err := Run(); err != nil {
fmt.Println(err)
os.Exit(1)
return
}
log.Printf("Goodbye!")
}

8
examples/lib/libmgmt2.go
View File

@@ -11,10 +11,10 @@ import (
"syscall"
"time"
"github.com/purpleidea/mgmt/engine"
"github.com/purpleidea/mgmt/gapi"
mgmt "github.com/purpleidea/mgmt/lib"
"github.com/purpleidea/mgmt/pgraph"
"github.com/purpleidea/mgmt/resources"
"github.com/urfave/cli"
)
@@ -52,7 +52,7 @@ func NewMyGAPI(data gapi.Data, name string, interval uint, count uint) (*MyGAPI,
// should take the prefix of the registered name. On activation, if there are
// any validation problems, you should return an error. If this was not
// activated, then you should return a nil GAPI and a nil error.
func (obj *MyGAPI) Cli(c *cli.Context, fs resources.Fs) (*gapi.Deploy, error) {
func (obj *MyGAPI) Cli(c *cli.Context, fs engine.Fs) (*gapi.Deploy, error) {
if s := c.String(obj.Name); c.IsSet(obj.Name) {
if s != "" {
return nil, fmt.Errorf("input is not empty")
@@ -107,13 +107,13 @@ func (obj *MyGAPI) Graph() (*pgraph.Graph, error) {
}
var vertex pgraph.Vertex
for i := uint(0); i < obj.Count; i++ {
n, err := resources.NewNamedResource("noop", fmt.Sprintf("noop%d", i))
n, err := engine.NewNamedResource("noop", fmt.Sprintf("noop%d", i))
if err != nil {
return nil, err
}
g.AddVertex(n)
if i > 0 {
g.AddEdge(vertex, n, &resources.Edge{Name: fmt.Sprintf("e%d", i)})
g.AddEdge(vertex, n, &engine.Edge{Name: fmt.Sprintf("e%d", i)})
}
vertex = n // save
}

44
examples/lib/libmgmt3.go
View File

@@ -10,10 +10,11 @@ import (
"syscall"
"time"
"github.com/purpleidea/mgmt/engine"
"github.com/purpleidea/mgmt/engine/resources"
"github.com/purpleidea/mgmt/gapi"
mgmt "github.com/purpleidea/mgmt/lib"
"github.com/purpleidea/mgmt/pgraph"
"github.com/purpleidea/mgmt/resources"
"github.com/urfave/cli"
)
@@ -49,7 +50,7 @@ func NewMyGAPI(data gapi.Data, name string, interval uint) (*MyGAPI, error) {
// should take the prefix of the registered name. On activation, if there are
// any validation problems, you should return an error. If this was not
// activated, then you should return a nil GAPI and a nil error.
func (obj *MyGAPI) Cli(c *cli.Context, fs resources.Fs) (*gapi.Deploy, error) {
func (obj *MyGAPI) Cli(c *cli.Context, fs engine.Fs) (*gapi.Deploy, error) {
if s := c.String(obj.Name); c.IsSet(obj.Name) {
if s != "" {
return nil, fmt.Errorf("input is not empty")
@@ -103,15 +104,8 @@ func (obj *MyGAPI) Graph() (*pgraph.Graph, error) {
return nil, err
}
metaparams := resources.DefaultMetaParams
content := "Delete me to trigger a notification!\n"
f0 := &resources.FileRes{
BaseRes: resources.BaseRes{
Name: "README",
Kind: "file",
MetaParams: metaparams,
},
Path: "/tmp/mgmt/README",
Content: &content,
State: "present",
@@ -120,50 +114,34 @@ func (obj *MyGAPI) Graph() (*pgraph.Graph, error) {
g.AddVertex(f0)
p1 := &resources.PasswordRes{
BaseRes: resources.BaseRes{
Name: "password1",
Kind: "password",
MetaParams: metaparams,
},
Length: 8, // generated string will have this many characters
Saved: true, // this causes passwords to be stored in plain text!
}
g.AddVertex(p1)
f1 := &resources.FileRes{
BaseRes: resources.BaseRes{
Name: "file1",
Kind: "file",
MetaParams: metaparams,
// send->recv!
Recv: map[string]*resources.Send{
"Content": {Res: p1, Key: "Password"},
},
},
Path: "/tmp/mgmt/secret",
//Content: p1.Password, // won't work
State: "present",
}
// XXX: add send->recv!
//Recv: map[string]*engine.Send{
// "Content": {Res: p1, Key: "Password"},
//},
g.AddVertex(f1)
n1 := &resources.NoopRes{
BaseRes: resources.BaseRes{
Name: "noop1",
Kind: "noop",
MetaParams: metaparams,
},
}
n1 := &resources.NoopRes{}
g.AddVertex(n1)
e0 := &resources.Edge{Name: "e0"}
e0 := &engine.Edge{Name: "e0"}
e0.Notify = true // send a notification from f0 to p1
g.AddEdge(f0, p1, e0)
g.AddEdge(p1, f1, &resources.Edge{Name: "e1"})
g.AddEdge(p1, f1, &engine.Edge{Name: "e1"})
e2 := &resources.Edge{Name: "e2"}
e2 := &engine.Edge{Name: "e2"}
e2.Notify = true // send a notification from f1 to n1
g.AddEdge(f1, n1, e2)

2
gapi/deploy.go
View File

@@ -34,7 +34,7 @@ func init() {
// TODO: add staged rollout functionality to this struct
// TODO: add proper authentication with gpg key signing
type Deploy struct {
Name string // lang, hcl, puppet, yaml, yaml2, etc...
Name string // lang, puppet, yaml, etc...
//Sync bool // wait for everyone to close previous GAPI before switching
Noop bool
Sema int // sema override

150
gapi/empty/empty.go Normal file
View File

@@ -0,0 +1,150 @@
// Mgmt
// Copyright (C) 2013-2018+ James Shubin and the project contributors
// Written by James Shubin <james@shubin.ca> and the project contributors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package empty
import (
"fmt"
"sync"
"github.com/purpleidea/mgmt/engine"
"github.com/purpleidea/mgmt/gapi"
"github.com/purpleidea/mgmt/pgraph"
"github.com/urfave/cli"
)
const (
// Name is the name of this frontend.
Name = "empty"
)
func init() {
gapi.Register(Name, func() gapi.GAPI { return &GAPI{} }) // register
}
// GAPI implements the main lang GAPI interface.
type GAPI struct {
data gapi.Data
initialized bool
closeChan chan struct{}
wg *sync.WaitGroup // sync group for tunnel go routines
}
// Cli takes a cli.Context, and returns our GAPI if activated. All arguments
// should take the prefix of the registered name. On activation, if there are
// any validation problems, you should return an error. If this was not
// activated, then you should return a nil GAPI and a nil error.
func (obj *GAPI) Cli(c *cli.Context, fs engine.Fs) (*gapi.Deploy, error) {
if s := c.String(Name); c.IsSet(Name) {
if s == "" {
return nil, fmt.Errorf("input code is empty")
}
return &gapi.Deploy{
Name: Name,
//Noop: false,
GAPI: &GAPI{},
}, nil
}
return nil, nil // we weren't activated!
}
// CliFlags returns a list of flags used by this deploy subcommand.
func (obj *GAPI) CliFlags() []cli.Flag {
return []cli.Flag{
cli.StringFlag{
Name: Name,
Value: "",
Usage: "empty graph to deploy",
},
}
}
// Init initializes the lang GAPI struct.
func (obj *GAPI) Init(data gapi.Data) error {
if obj.initialized {
return fmt.Errorf("already initialized")
}
obj.data = data // store for later
obj.closeChan = make(chan struct{})
obj.wg = &sync.WaitGroup{}
obj.initialized = true
return nil
}
// Graph returns a current Graph.
func (obj *GAPI) Graph() (*pgraph.Graph, error) {
if !obj.initialized {
return nil, fmt.Errorf("%s: GAPI is not initialized", Name)
}
obj.data.Logf("generating empty graph...")
g, err := pgraph.NewGraph("empty")
if err != nil {
return nil, err
}
return g, nil
}
// Next returns nil errors every time there could be a new graph.
func (obj *GAPI) Next() chan gapi.Next {
ch := make(chan gapi.Next)
obj.wg.Add(1)
go func() {
defer obj.wg.Done()
defer close(ch) // this will run before the obj.wg.Done()
if !obj.initialized {
next := gapi.Next{
Err: fmt.Errorf("%s: GAPI is not initialized", Name),
Exit: true, // exit, b/c programming error?
}
select {
case ch <- next:
case <-obj.closeChan:
}
return
}
// send only one event
next := gapi.Next{
Exit: false,
Err: nil,
}
select {
case ch <- next: // trigger a run (send a msg)
// pass
// unblock if we exit while waiting to send!
case <-obj.closeChan:
return
}
}()
return ch
}
// Close shuts down the lang GAPI.
func (obj *GAPI) Close() error {
if !obj.initialized {
return fmt.Errorf("%s: GAPI is not initialized", Name)
}
close(obj.closeChan)
obj.wg.Wait()
obj.initialized = false // closed = true
return nil
}

6
gapi/gapi.go
View File

@@ -22,8 +22,8 @@ import (
"encoding/gob"
"fmt"
"github.com/purpleidea/mgmt/engine"
"github.com/purpleidea/mgmt/pgraph"
"github.com/purpleidea/mgmt/resources"
"github.com/urfave/cli"
)
@@ -46,7 +46,7 @@ func Register(name string, fn func() GAPI) {
type Data struct {
Program string // name of the originating program
Hostname string // uuid for the host, required for GAPI
World resources.World
World engine.World
Noop bool
NoConfigWatch bool
NoStreamWatch bool
@@ -70,7 +70,7 @@ type Next struct {
// GAPI is a Graph API that represents incoming graphs and change streams.
type GAPI interface {
Cli(c *cli.Context, fs resources.Fs) (*Deploy, error)
Cli(c *cli.Context, fs engine.Fs) (*Deploy, error)
CliFlags() []cli.Flag
Init(Data) error // initializes the GAPI and passes in useful data

8
gapi/helpers.go
View File

@@ -20,7 +20,7 @@ package gapi
import (
"io/ioutil"
"github.com/purpleidea/mgmt/resources"
"github.com/purpleidea/mgmt/engine"
"github.com/purpleidea/mgmt/util"
errwrap "github.com/pkg/errors"
@@ -31,7 +31,7 @@ import (
const Umask = 0666
// CopyFileToFs copies a file from src path on the local fs to a dst path on fs.
func CopyFileToFs(fs resources.Fs, src, dst string) error {
func CopyFileToFs(fs engine.Fs, src, dst string) error {
data, err := ioutil.ReadFile(src)
if err != nil {
return errwrap.Wrapf(err, "can't read from file `%s`", src)
@@ -44,7 +44,7 @@ func CopyFileToFs(fs resources.Fs, src, dst string) error {
// CopyStringToFs copies a file from src path on the local fs to a dst path on
// fs.
func CopyStringToFs(fs resources.Fs, str, dst string) error {
func CopyStringToFs(fs engine.Fs, str, dst string) error {
if err := fs.WriteFile(dst, []byte(str), Umask); err != nil {
return errwrap.Wrapf(err, "can't write to file `%s`", dst)
}
@@ -52,6 +52,6 @@ func CopyStringToFs(fs resources.Fs, str, dst string) error {
}
// CopyDirToFs copies a dir from src path on the local fs to a dst path on fs.
func CopyDirToFs(fs resources.Fs, src, dst string) error {
func CopyDirToFs(fs engine.Fs, src, dst string) error {
return util.CopyDiskToFs(fs, src, dst, false)
}

8
integration/basic_test.go
View File

@@ -24,6 +24,8 @@ import (
"path"
"sort"
"testing"
"github.com/purpleidea/mgmt/util"
)
func TestInstance0(t *testing.T) {
@@ -67,7 +69,7 @@ func TestInstance1(t *testing.T) {
values := []test{}
{
code := Code(`
code := util.Code(`
$root = getenv("MGMT_TEST_ROOT")
file "${root}/mgmt-hello-world" {
@@ -150,7 +152,7 @@ func TestCluster1(t *testing.T) {
values := []test{}
{
code := Code(`
code := util.Code(`
$root = getenv("MGMT_TEST_ROOT")
file "${root}/mgmt-hostname" {
@@ -174,7 +176,7 @@ func TestCluster1(t *testing.T) {
})
}
{
code := Code(`
code := util.Code(`
$root = getenv("MGMT_TEST_ROOT")
file "${root}/mgmt-hostname" {

6
integration/instance.go
View File

@@ -52,11 +52,11 @@ const (
// longTimeout is a high bound of time we're willing to wait for events.
// If we exceed this timeout, then it's likely we are blocked somewhere.
longTimeout = 30 // seconds
longTimeout = 60 // seconds
// convergedTimeout is the number of seconds we wait for our instance to
// remain unchanged to be considered as converged.
convergedTimeout = 5 // seconds
convergedTimeout = 15 // seconds
// dirMode is the the mode used when making directories.
dirMode = 0755
@@ -297,7 +297,7 @@ func (obj *Instance) Wait(ctx context.Context) error {
for {
select {
// FIXME: instead of sending one event here, the recwatch
// library should sent one initial event at startup...
// library should send one initial event at startup...
case <-startup:
startup = nil
// send an initial event

32
integration/util.go
View File

@@ -25,7 +25,6 @@ import (
"path/filepath"
"runtime"
"strconv"
"strings"
errwrap "github.com/pkg/errors"
)
@@ -48,37 +47,6 @@ func BinaryPath() (string, error) {
return path.Join(root, binaryName), nil
}
// Code takes a code block as a backtick enclosed `heredoc` and removes any
// common indentation from each line. This helps inline code as strings to be
// formatted nicely without unnecessary indentation. It also drops the very
// first line of code if it has zero length.
func Code(code string) string {
output := []string{}
lines := strings.Split(code, "\n")
var found bool
var strip string // prefix to remove
for i, x := range lines {
if !found && len(x) > 0 {
for j := 0; j < len(x); j++ {
if x[j] != '\t' {
break
}
strip += "\t"
}
// otherwise, there's no indentation
found = true
}
if i == 0 && len(x) == 0 { // drop first line if it's empty
continue
}
s := strings.TrimPrefix(x, strip)
output = append(output, s)
}
return strings.Join(output, "\n")
}
// ParsePort parses a URL and returns the port that was found.
func ParsePort(input string) (int, error) {
u, err := url.Parse(input)

23
integration/util_test.go
View File

@@ -41,29 +41,6 @@ func TestBinaryPath(t *testing.T) {
_ = fi
}
func TestCodeIndent(t *testing.T) {
c1 := Code(
`
$root = getenv("MGMT_TEST_ROOT")
file "${root}/mgmt-hello-world" {
content => "hello world from @purpleidea\n",
state => "exists",
}
`)
c2 :=
`$root = getenv("MGMT_TEST_ROOT")
file "${root}/mgmt-hello-world" {
content => "hello world from @purpleidea\n",
state => "exists",
}
`
if c1 != c2 {
t.Errorf("code samples differ")
}
}
func TestParsePort(t *testing.T) {
if port, err := ParsePort("http://127.0.0.1:2379"); err != nil {
t.Errorf("could not determine port: %+v", err)

12
lang/funcs/engine.go
View File

@@ -22,10 +22,10 @@ import (
"strings"
"sync"
"github.com/purpleidea/mgmt/engine"
"github.com/purpleidea/mgmt/lang/interfaces"
"github.com/purpleidea/mgmt/lang/types"
"github.com/purpleidea/mgmt/pgraph"
"github.com/purpleidea/mgmt/resources"
multierr "github.com/hashicorp/go-multierror"
errwrap "github.com/pkg/errors"
@@ -103,7 +103,7 @@ func (obj *Edge) String() string {
type Engine struct {
Graph *pgraph.Graph
Hostname string
World resources.World
World engine.World
Debug bool
Logf func(format string, v ...interface{})
@@ -161,15 +161,15 @@ func (obj *Engine) Init() error {
return fmt.Errorf("vertex (%+v) was not an expr", vertex)
}
if obj.Debug {
obj.Logf("Loading func `%s`", vertex)
}
obj.state[vertex] = &State{Expr: expr} // store some state!
if e := obj.state[vertex].Init(); e != nil {
err = multierr.Append(err, e) // list of errors
}
if obj.Debug {
obj.Logf("Loading func `%s`", vertex)
}
}
if err != nil { // usually due to `not found` errors
return errwrap.Wrapf(err, "could not load requested funcs")

4
lang/funcs/facts/facts.go
View File

@@ -21,10 +21,10 @@ package facts
import (
"fmt"
"github.com/purpleidea/mgmt/engine"
"github.com/purpleidea/mgmt/lang/funcs"
"github.com/purpleidea/mgmt/lang/interfaces"
"github.com/purpleidea/mgmt/lang/types"
"github.com/purpleidea/mgmt/resources"
)
// RegisteredFacts is a global map of all possible facts which can be used. You
@@ -61,7 +61,7 @@ type Init struct {
Hostname string // uuid for the host
//Noop bool
Output chan types.Value // Stream must close `output` chan
World resources.World
World engine.World
}
// Fact is the interface that any valid fact must fulfill. It is very simple,

37
lang/gapi.go
View File

@@ -19,13 +19,12 @@ package lang
import (
"fmt"
"log"
"strings"
"sync"
"github.com/purpleidea/mgmt/engine"
"github.com/purpleidea/mgmt/gapi"
"github.com/purpleidea/mgmt/pgraph"
"github.com/purpleidea/mgmt/resources"
multierr "github.com/hashicorp/go-multierror"
errwrap "github.com/pkg/errors"
@@ -59,7 +58,7 @@ type GAPI struct {
// should take the prefix of the registered name. On activation, if there are
// any validation problems, you should return an error. If this was not
// activated, then you should return a nil GAPI and a nil error.
func (obj *GAPI) Cli(c *cli.Context, fs resources.Fs) (*gapi.Deploy, error) {
func (obj *GAPI) Cli(c *cli.Context, fs engine.Fs) (*gapi.Deploy, error) {
if s := c.String(Name); c.IsSet(Name) {
if s == "" {
return nil, fmt.Errorf("input code is empty")
@@ -135,6 +134,10 @@ func (obj *GAPI) LangInit() error {
Hostname: obj.data.Hostname,
World: obj.data.World,
Debug: obj.data.Debug,
Logf: func(format string, v ...interface{}) {
// TODO: add the Name prefix in parent logger
obj.data.Logf(Name+": "+format, v...)
},
}
if err := obj.lang.Init(); err != nil {
return errwrap.Wrapf(err, "can't init the lang")
@@ -209,11 +212,11 @@ func (obj *GAPI) Next() chan gapi.Next {
case <-obj.closeChan:
return
}
log.Printf("%s: Generating new graph...", Name)
obj.data.Logf("generating new graph...")
// skip this to pass through the err if present
if langSwap && err == nil {
log.Printf("%s: Swap!", Name)
obj.data.Logf("swap!")
// run up to these three but fail on err
if e := obj.LangClose(); e != nil { // close any old lang
err = e // pass through the err
@@ -230,7 +233,29 @@ func (obj *GAPI) Next() chan gapi.Next {
} else {
if obj.data.NoStreamWatch { // TODO: do we want to allow this for the lang?
streamChan = nil
obj.data.Logf("warning: language will not stream")
// send only one event
limitChan := make(chan error)
obj.wg.Add(1)
go func() {
defer obj.wg.Done()
defer close(limitChan)
select {
// only one
case err, ok := <-obj.lang.Stream():
if !ok {
return
}
select {
case limitChan <- err:
case <-obj.closeChan:
return
}
case <-obj.closeChan:
return
}
}()
streamChan = limitChan
} else {
// stream for lang events
streamChan = obj.lang.Stream() // update stream

6
lang/interfaces/ast.go
View File

@@ -18,9 +18,9 @@
package interfaces
import (
"github.com/purpleidea/mgmt/engine"
"github.com/purpleidea/mgmt/lang/types"
"github.com/purpleidea/mgmt/pgraph"
"github.com/purpleidea/mgmt/resources"
)
const (
@@ -111,7 +111,7 @@ type Edge struct {
// Output is a collection of data returned by a Stmt.
type Output struct { // returned by Stmt
Resources []resources.Res
Resources []engine.Res
Edges []*Edge
//Exported []*Exports // TODO: add exported resources
}
@@ -120,7 +120,7 @@ type Output struct { // returned by Stmt
// lists initialized appropriately.
func (obj *Output) Empty() *Output {
return &Output{
Resources: []resources.Res{},
Resources: []engine.Res{},
Edges: []*Edge{},
}
}

4
lang/interfaces/func.go
View File

@@ -18,8 +18,8 @@
package interfaces
import (
"github.com/purpleidea/mgmt/engine"
"github.com/purpleidea/mgmt/lang/types"
"github.com/purpleidea/mgmt/resources"
)
// Info is a static representation of some information about the function. It is
@@ -39,7 +39,7 @@ type Init struct {
//Noop bool
Input chan types.Value // Engine will close `input` chan
Output chan types.Value // Stream must close `output` chan
World resources.World
World engine.World
Debug bool
Logf func(format string, v ...interface{})
}

46
lang/interpret.go
View File

@@ -20,9 +20,9 @@ package lang // TODO: move this into a sub package of lang/$name?
import (
"fmt"
"github.com/purpleidea/mgmt/engine"
"github.com/purpleidea/mgmt/lang/interfaces"
"github.com/purpleidea/mgmt/pgraph"
"github.com/purpleidea/mgmt/resources"
errwrap "github.com/pkg/errors"
)
@@ -42,26 +42,26 @@ func interpret(ast interfaces.Stmt) (*pgraph.Graph, error) {
return nil, errwrap.Wrapf(err, "could not create new graph")
}
var lookup = make(map[string]map[string]resources.Res) // map[kind]map[name]Res
var lookup = make(map[string]map[string]engine.Res) // map[kind]map[name]Res
// build the send/recv mapping; format: map[kind]map[name]map[field]*Send
var receive = make(map[string]map[string]map[string]*resources.Send)
var receive = make(map[string]map[string]map[string]*engine.Send)
for _, res := range output.Resources {
graph.AddVertex(res)
kind := res.GetKind()
name := res.GetName()
kind := res.Kind()
name := res.Name()
if _, exists := lookup[kind]; !exists {
lookup[kind] = make(map[string]resources.Res)
receive[kind] = make(map[string]map[string]*resources.Send)
lookup[kind] = make(map[string]engine.Res)
receive[kind] = make(map[string]map[string]*engine.Send)
}
if _, exists := receive[kind][name]; !exists {
receive[kind][name] = make(map[string]*resources.Send)
receive[kind][name] = make(map[string]*engine.Send)
}
if r, exists := lookup[kind][name]; exists { // found same name
if !r.Compare(res) {
if err := engine.ResCmp(r, res); err != nil {
// TODO: print a diff of the two resources
return nil, fmt.Errorf("incompatible duplicate resource `%s` found", res)
return nil, errwrap.Wrapf(err, "incompatible duplicate resource `%s` found", res)
}
// more than one compatible resource exists... we allow
// duplicates, if they're going to not conflict...
@@ -71,9 +71,9 @@ func interpret(ast interfaces.Stmt) (*pgraph.Graph, error) {
}
for _, e := range output.Edges {
var v1, v2 resources.Res
var v1, v2 engine.Res
var exists bool // = true
var m map[string]resources.Res
var m map[string]engine.Res
var notify = e.Notify
if m, exists = lookup[e.Kind1]; exists {
@@ -91,10 +91,10 @@ func interpret(ast interfaces.Stmt) (*pgraph.Graph, error) {
if existingEdge := graph.FindEdge(v1, v2); existingEdge != nil {
// collate previous Notify signals to this edge with OR
notify = notify || (existingEdge.(*resources.Edge)).Notify
notify = notify || (existingEdge.(*engine.Edge)).Notify
}
edge := &resources.Edge{
edge := &engine.Edge{
Name: fmt.Sprintf("%s -> %s", v1, v2),
Notify: notify,
}
@@ -117,8 +117,14 @@ func interpret(ast interfaces.Stmt) (*pgraph.Graph, error) {
}
}
res, ok := v1.(engine.SendableRes)
if !ok {
return nil, fmt.Errorf("cannot send from resource: %s", engine.Stringer(v1))
}
// XXX: type check the send/recv relationship somewhere
// store mapping for later
receive[e.Kind2][e.Name2][e.Recv] = &resources.Send{Res: v1, Key: e.Send}
receive[e.Kind2][e.Name2][e.Recv] = &engine.Send{Res: res, Key: e.Send}
}
// we need to first build up a map of all the resources handles, because
@@ -130,7 +136,15 @@ func interpret(ast interfaces.Stmt) (*pgraph.Graph, error) {
// TODO: do this in a deterministic order
for kind, x := range receive {
for name, recv := range x {
lookup[kind][name].SetRecv(recv) // set it!
if len(recv) == 0 { // skip empty maps from allocation!
continue
}
r := lookup[kind][name]
res, ok := r.(engine.RecvableRes)
if !ok {
return nil, fmt.Errorf("cannot recv to resource: %s", engine.Repr(kind, name))
}
res.SetRecv(recv) // set it!
}
}

13
lang/interpret_test.go
View File

@@ -22,10 +22,11 @@ import (
"strings"
"testing"
"github.com/purpleidea/mgmt/engine"
"github.com/purpleidea/mgmt/engine/resources"
"github.com/purpleidea/mgmt/lang/interfaces"
"github.com/purpleidea/mgmt/lang/unification"
"github.com/purpleidea/mgmt/pgraph"
"github.com/purpleidea/mgmt/resources"
)
func vertexAstCmpFn(v1, v2 pgraph.Vertex) (bool, error) {
@@ -496,7 +497,7 @@ func TestAstInterpret0(t *testing.T) {
}
{
graph, _ := pgraph.NewGraph("g")
t1, _ := resources.NewNamedResource("test", "t1")
t1, _ := engine.NewNamedResource("test", "t1")
x := t1.(*resources.TestRes)
int64ptr := int64(42)
x.Int64Ptr = &int64ptr
@@ -522,7 +523,7 @@ func TestAstInterpret0(t *testing.T) {
}
{
graph, _ := pgraph.NewGraph("g")
t1, _ := resources.NewNamedResource("test", "t1")
t1, _ := engine.NewNamedResource("test", "t1")
x := t1.(*resources.TestRes)
stringptr := "wow"
x.StringPtr = &stringptr
@@ -540,21 +541,21 @@ func TestAstInterpret0(t *testing.T) {
{
// FIXME: add a better vertexCmpFn so we can compare send/recv!
graph, _ := pgraph.NewGraph("g")
t1, _ := resources.NewNamedResource("test", "t1")
t1, _ := engine.NewNamedResource("test", "t1")
{
x := t1.(*resources.TestRes)
int64Ptr := int64(42)
x.Int64Ptr = &int64Ptr
graph.AddVertex(t1)
}
t2, _ := resources.NewNamedResource("test", "t2")
t2, _ := engine.NewNamedResource("test", "t2")
{
x := t2.(*resources.TestRes)
int64Ptr := int64(13)
x.Int64Ptr = &int64Ptr
graph.AddVertex(t2)
}
edge := &resources.Edge{
edge := &engine.Edge{
Name: fmt.Sprintf("%s -> %s", t1, t2),
Notify: false,
}

46
lang/lang.go
View File

@@ -20,9 +20,9 @@ package lang // TODO: move this into a sub package of lang/$name?
import (
"fmt"
"io"
"log"
"sync"
"github.com/purpleidea/mgmt/engine"
"github.com/purpleidea/mgmt/lang/funcs"
_ "github.com/purpleidea/mgmt/lang/funcs/core" // import so the funcs register
_ "github.com/purpleidea/mgmt/lang/funcs/facts/core" // import so the facts register
@@ -31,7 +31,6 @@ import (
"github.com/purpleidea/mgmt/lang/interfaces"
"github.com/purpleidea/mgmt/lang/unification"
"github.com/purpleidea/mgmt/pgraph"
"github.com/purpleidea/mgmt/resources"
errwrap "github.com/pkg/errors"
)
@@ -50,8 +49,9 @@ const (
type Lang struct {
Input io.Reader // os.Stdin or anything that satisfies this interface
Hostname string
World resources.World
World engine.World
Debug bool
Logf func(format string, v ...interface{})
ast interfaces.Stmt // store main prog AST here
funcs *funcs.Engine // function event engine
@@ -79,13 +79,13 @@ func (obj *Lang) Init() error {
loadedSignal := func() { close(obj.loadedChan) } // only run once!
// run the lexer/parser and build an AST
log.Printf("%s: Lexing/Parsing...", Name)
obj.Logf("lexing/parsing...")
ast, err := LexParse(obj.Input)
if err != nil {
return errwrap.Wrapf(err, "could not generate AST")
}
if obj.Debug {
log.Printf("%s: behold, the AST: %+v", Name, ast)
obj.Logf("behold, the AST: %+v", ast)
}
// TODO: should we validate the structure of the AST?
@@ -94,7 +94,7 @@ func (obj *Lang) Init() error {
// return errwrap.Wrapf(err, "could not validate AST")
//}
log.Printf("%s: Interpolating...", Name)
obj.Logf("interpolating...")
// interpolate strings and other expansionable nodes in AST
interpolated, err := ast.Interpolate()
if err != nil {
@@ -111,7 +111,7 @@ func (obj *Lang) Init() error {
},
}
log.Printf("%s: Building Scope...", Name)
obj.Logf("building scope...")
// propagate the scope down through the AST...
if err := obj.ast.SetScope(scope); err != nil {
return errwrap.Wrapf(err, "could not set scope")
@@ -120,15 +120,15 @@ func (obj *Lang) Init() error {
// apply type unification
logf := func(format string, v ...interface{}) {
if obj.Debug { // unification only has debug messages...
log.Printf(Name+": unification: "+format, v...)
obj.Logf("unification: "+format, v...)
}
}
log.Printf("%s: Running Type Unification...", Name)
obj.Logf("running type unification...")
if err := unification.Unify(obj.ast, unification.SimpleInvariantSolverLogger(logf)); err != nil {
return errwrap.Wrapf(err, "could not unify types")
}
log.Printf("%s: Building Function Graph...", Name)
obj.Logf("building function graph...")
// we assume that for some given code, the list of funcs doesn't change
// iow, we don't support variable, variables or absurd things like that
graph, err := obj.ast.Graph() // build the graph of functions
@@ -137,24 +137,24 @@ func (obj *Lang) Init() error {
}
if obj.Debug {
log.Printf("%s: function graph: %+v", Name, graph)
graph.Logf("%s: ", Name) // log graph with this printf prefix...
obj.Logf("function graph: %+v", graph)
graph.Logf(obj.Logf) // log graph output with this logger...
}
if graph.NumVertices() == 0 { // no funcs to load!
// send only one signal since we won't ever send after this!
log.Printf("%s: Static graph found", Name)
obj.Logf("static graph found")
obj.wg.Add(1)
go func() {
defer obj.wg.Done()
defer close(obj.streamChan) // no more events are coming!
close(obj.loadedChan) // signal
select {
case obj.streamChan <- nil: // send one signal
// pass
case <-obj.closeChan:
return
}
close(obj.loadedChan) // signal
}()
return nil // exit early, no funcs to load!
}
@@ -165,22 +165,22 @@ func (obj *Lang) Init() error {
World: obj.World,
Debug: obj.Debug,
Logf: func(format string, v ...interface{}) {
log.Printf(Name+": funcs: "+format, v...)
obj.Logf("funcs: "+format, v...)
},
Glitch: false, // FIXME: verify this functionality is perfect!
}
log.Printf("%s: Function Engine Initializing...", Name)
obj.Logf("function engine initializing...")
if err := obj.funcs.Init(); err != nil {
return errwrap.Wrapf(err, "init error with func engine")
}
log.Printf("%s: Function Engine Validating...", Name)
obj.Logf("function engine validating...")
if err := obj.funcs.Validate(); err != nil {
return errwrap.Wrapf(err, "validate error with func engine")
}
log.Printf("%s: Function Engine Starting...", Name)
obj.Logf("function engine starting...")
// On failure, we expect the caller to run Close() to shutdown all of
// the currently initialized (and running) funcs... This is needed if
// we successfully ran `Run` but isn't needed only for Init/Validate.
@@ -189,11 +189,11 @@ func (obj *Lang) Init() error {
}
// wait for some activity
log.Printf("%s: Stream...", Name)
obj.Logf("stream...")
stream := obj.funcs.Stream()
obj.wg.Add(1)
go func() {
log.Printf("%s: Loop...", Name)
obj.Logf("loop...")
defer obj.wg.Done()
defer close(obj.streamChan) // no more events are coming!
for {
@@ -202,7 +202,7 @@ func (obj *Lang) Init() error {
select {
case err, ok = <-stream:
if !ok {
log.Printf("%s: Stream closed", Name)
obj.Logf("stream closed")
return
}
if err == nil {
@@ -217,7 +217,7 @@ func (obj *Lang) Init() error {
select {
case obj.streamChan <- err: // send
if err != nil {
log.Printf("%s: Stream error: %+v", Name, err)
obj.Logf("Stream error: %+v", err)
return
}
@@ -246,7 +246,7 @@ func (obj *Lang) Interpret() (*pgraph.Graph, error) {
return nil, fmt.Errorf("funcs aren't loaded yet")
}
log.Printf("%s: Running interpret...", Name)
obj.Logf("running interpret...")
if obj.funcs != nil { // no need to rlock if we have a static graph
obj.funcs.RLock()
}

49
lang/lang_test.go
View File

@@ -22,9 +22,10 @@ import (
"strings"
"testing"
"github.com/purpleidea/mgmt/engine"
"github.com/purpleidea/mgmt/engine/resources"
_ "github.com/purpleidea/mgmt/lang/funcs/facts/core" // load facts
"github.com/purpleidea/mgmt/pgraph"
"github.com/purpleidea/mgmt/resources"
multierr "github.com/hashicorp/go-multierror"
errwrap "github.com/pkg/errors"
@@ -63,8 +64,8 @@ func vertexCmpFn(v1, v2 pgraph.Vertex) (bool, error) {
return false, fmt.Errorf("oops, empty vertex")
}
r1, r2 := v1.(resources.Res), v2.(resources.Res)
if !r1.Compare(r2) {
r1, r2 := v1.(engine.Res), v2.(engine.Res)
if err := r1.Cmp(r2); err != nil {
//fmt.Printf("r1: %+v\n", *(r1.(*resources.TestRes).Int64Ptr))
//fmt.Printf("r2: %+v\n", *(r2.(*resources.TestRes).Int64Ptr))
return false, nil
@@ -80,11 +81,15 @@ func edgeCmpFn(e1, e2 pgraph.Edge) (bool, error) {
return e1.String() == e2.String(), nil
}
func runInterpret(code string) (*pgraph.Graph, error) {
func runInterpret(t *testing.T, code string) (*pgraph.Graph, error) {
str := strings.NewReader(code)
logf := func(format string, v ...interface{}) {
t.Logf("test: lang: "+format, v...)
}
lang := &Lang{
Input: str, // string as an interface that satisfies io.Reader
Debug: true,
Logf: logf,
}
if err := lang.Init(); err != nil {
return nil, errwrap.Wrapf(err, "init failed")
@@ -119,7 +124,7 @@ func runInterpret(code string) (*pgraph.Graph, error) {
func TestInterpret0(t *testing.T) {
code := ``
graph, err := runInterpret(code)
graph, err := runInterpret(t, code)
if err != nil {
t.Errorf("runInterpret failed: %+v", err)
return
@@ -132,13 +137,13 @@ func TestInterpret0(t *testing.T) {
func TestInterpret1(t *testing.T) {
code := `noop "n1" {}`
graph, err := runInterpret(code)
graph, err := runInterpret(t, code)
if err != nil {
t.Errorf("runInterpret failed: %+v", err)
return
}
n1, _ := resources.NewNamedResource("noop", "n1")
n1, _ := engine.NewNamedResource("noop", "n1")
expected := &pgraph.Graph{}
expected.AddVertex(n1)
@@ -151,14 +156,14 @@ func TestInterpret2(t *testing.T) {
noop "n1" {}
noop "n2" {}
`
graph, err := runInterpret(code)
graph, err := runInterpret(t, code)
if err != nil {
t.Errorf("runInterpret failed: %+v", err)
return
}
n1, _ := resources.NewNamedResource("noop", "n1")
n2, _ := resources.NewNamedResource("noop", "n2")
n1, _ := engine.NewNamedResource("noop", "n1")
n2, _ := engine.NewNamedResource("noop", "n2")
expected := &pgraph.Graph{}
expected.AddVertex(n1)
@@ -174,7 +179,7 @@ func TestInterpret3(t *testing.T) {
int8 => 88888888,
}
`
_, err := runInterpret(code)
_, err := runInterpret(t, code)
if err == nil {
t.Errorf("expected overflow failure, but it passed")
}
@@ -194,13 +199,13 @@ func TestInterpret4(t *testing.T) {
comment => "☺\thello\u263a\nwo\"rld\\2\u263a", # must escape these
}
`
graph, err := runInterpret(code)
graph, err := runInterpret(t, code)
if err != nil {
t.Errorf("runInterpret failed: %+v", err)
return
}
t1, _ := resources.NewNamedResource("test", "t1")
t1, _ := engine.NewNamedResource("test", "t1")
x := t1.(*resources.TestRes)
str := " !\"#$%&'()*+,-./0123456790:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_abcdefghijklmnopqrstuvwxyz{|}~"
x.StringPtr = &str
@@ -228,13 +233,13 @@ func TestInterpret5(t *testing.T) {
}
}
`
graph, err := runInterpret(code)
graph, err := runInterpret(t, code)
if err != nil {
t.Errorf("runInterpret failed: %+v", err)
return
}
t1, _ := resources.NewNamedResource("test", "t1")
t1, _ := engine.NewNamedResource("test", "t1")
x := t1.(*resources.TestRes)
x.Int64 = 42
str := "hello!"
@@ -262,7 +267,7 @@ func TestInterpret6(t *testing.T) {
}
}
`
graph, err := runInterpret(code)
graph, err := runInterpret(t, code)
if err != nil {
t.Errorf("runInterpret failed: %+v", err)
return
@@ -271,7 +276,7 @@ func TestInterpret6(t *testing.T) {
expected := &pgraph.Graph{}
{
r, _ := resources.NewNamedResource("test", "t1")
r, _ := engine.NewNamedResource("test", "t1")
x := r.(*resources.TestRes)
x.Int64 = 42
str := "hello"
@@ -279,7 +284,7 @@ func TestInterpret6(t *testing.T) {
expected.AddVertex(x)
}
{
r, _ := resources.NewNamedResource("test", "t2")
r, _ := engine.NewNamedResource("test", "t2")
x := r.(*resources.TestRes)
x.Int64 = 13
str := "world"
@@ -319,7 +324,7 @@ func TestInterpretMany(t *testing.T) {
}
{
graph, _ := pgraph.NewGraph("g")
r, _ := resources.NewNamedResource("test", "t")
r, _ := engine.NewNamedResource("test", "t")
x := r.(*resources.TestRes)
i := int64(42 + 13)
x.Int64Ptr = &i
@@ -337,7 +342,7 @@ func TestInterpretMany(t *testing.T) {
}
{
graph, _ := pgraph.NewGraph("g")
r, _ := resources.NewNamedResource("test", "t")
r, _ := engine.NewNamedResource("test", "t")
x := r.(*resources.TestRes)
i := int64(42 + 13 + 99)
x.Int64Ptr = &i
@@ -355,7 +360,7 @@ func TestInterpretMany(t *testing.T) {
}
{
graph, _ := pgraph.NewGraph("g")
r, _ := resources.NewNamedResource("test", "t")
r, _ := engine.NewNamedResource("test", "t")
x := r.(*resources.TestRes)
i := int64(42 + 13 - 99)
x.Int64Ptr = &i
@@ -386,7 +391,7 @@ func TestInterpretMany(t *testing.T) {
t.Logf("\n\ntest #%d (%s) ----------------\n\n", index, name)
graph, err := runInterpret(code)
graph, err := runInterpret(t, code)
if !fail && err != nil {
t.Errorf("test #%d: FAIL", index)
t.Errorf("test #%d: runInterpret failed with: %+v", index, err)

15
lang/structs.go
View File

@@ -23,13 +23,14 @@ import (
"reflect"
"strings"
"github.com/purpleidea/mgmt/engine"
engineUtil "github.com/purpleidea/mgmt/engine/util"
"github.com/purpleidea/mgmt/lang/funcs"
"github.com/purpleidea/mgmt/lang/funcs/structs"
"github.com/purpleidea/mgmt/lang/interfaces"
"github.com/purpleidea/mgmt/lang/types"
"github.com/purpleidea/mgmt/lang/unification"
"github.com/purpleidea/mgmt/pgraph"
"github.com/purpleidea/mgmt/resources"
"github.com/purpleidea/mgmt/util"
errwrap "github.com/pkg/errors"
@@ -240,7 +241,7 @@ func (obj *StmtRes) Output() (*interfaces.Output, error) {
// TODO: test for []str instead, and loop
name := nameValue.Str() // must not panic
res, err := resources.NewNamedResource(obj.Kind, name)
res, err := engine.NewNamedResource(obj.Kind, name)
if err != nil {
return nil, errwrap.Wrapf(err, "cannot create resource kind `%s` with named `%s`", obj.Kind, name)
}
@@ -250,7 +251,7 @@ func (obj *StmtRes) Output() (*interfaces.Output, error) {
panic(fmt.Sprintf("expected struct, got: %s", k))
}
mapping, err := resources.LangFieldNameToStructFieldName(obj.Kind)
mapping, err := engineUtil.LangFieldNameToStructFieldName(obj.Kind)
if err != nil {
return nil, err
}
@@ -377,7 +378,7 @@ func (obj *StmtRes) Output() (*interfaces.Output, error) {
}
return &interfaces.Output{
Resources: []resources.Res{res},
Resources: []engine.Res{res},
Edges: edges,
}, nil
}
@@ -580,7 +581,7 @@ func (obj *StmtResField) Unify(kind string) ([]interfaces.Invariant, error) {
// TODO: unfortunately this gets called separately for each field... if
// we could cache this, it might be worth looking into for performance!
typMap, err := resources.LangFieldNameToStructType(kind)
typMap, err := engineUtil.LangFieldNameToStructType(kind)
if err != nil {
return nil, err
}
@@ -1142,7 +1143,7 @@ func (obj *StmtIf) Output() (*interfaces.Output, error) {
return nil, err
}
resources := []resources.Res{}
resources := []engine.Res{}
if output != nil {
resources = append(resources, output.Resources...)
//edges = output.Edges
@@ -1267,7 +1268,7 @@ func (obj *StmtProg) Graph() (*pgraph.Graph, error) {
// analogous function for expressions is Value. Those Value functions might get
// called by this Output function if they are needed to produce the output.
func (obj *StmtProg) Output() (*interfaces.Output, error) {
resources := []resources.Res{}
resources := []engine.Res{}
edges := []*interfaces.Edge{}
for _, stmt := range obj.Prog {

63
lib/cli.go
View File

@@ -23,6 +23,7 @@ import (
"os"
"os/signal"
"sort"
"sync"
"syscall"
"github.com/purpleidea/mgmt/bindata"
@@ -126,44 +127,72 @@ func run(c *cli.Context) error {
obj.PgpIdentity = &us
}
obj.Prometheus = c.Bool("prometheus")
obj.PrometheusListen = c.String("prometheus-listen")
if err := obj.Validate(); err != nil {
return err
}
if err := obj.Init(); err != nil {
return err
}
obj.Prometheus = c.Bool("prometheus")
obj.PrometheusListen = c.String("prometheus-listen")
// install the exit signal handler
wg := &sync.WaitGroup{}
defer wg.Wait()
exit := make(chan struct{})
defer close(exit)
wg.Add(1)
go func() {
signals := make(chan os.Signal, 1)
defer wg.Done()
// must have buffer for max number of signals
signals := make(chan os.Signal, 3+1) // 3 * ^C + 1 * SIGTERM
signal.Notify(signals, os.Interrupt) // catch ^C
//signal.Notify(signals, os.Kill) // catch signals
signal.Notify(signals, syscall.SIGTERM)
var count uint8
for {
select {
case sig := <-signals: // any signal will do
if sig != os.Interrupt {
log.Printf("Interrupted by signal")
obj.Interrupt(fmt.Errorf("killed by %v", sig))
return
}
select {
case sig := <-signals: // any signal will do
if sig == os.Interrupt {
log.Println("Interrupted by ^C")
obj.Exit(nil)
switch count {
case 0:
log.Printf("Interrupted by ^C")
obj.Exit(nil)
case 1:
log.Printf("Interrupted by ^C (fast pause)")
obj.FastExit(nil)
case 2:
log.Printf("Interrupted by ^C (hard interrupt)")
obj.Interrupt(nil)
}
count++
case <-exit:
return
}
log.Println("Interrupted by signal")
obj.Exit(fmt.Errorf("killed by %v", sig))
return
case <-exit:
return
}
}()
if err := obj.Run(); err != nil {
reterr := obj.Run()
if reterr != nil {
// log the error message returned
log.Printf("Main: Error: %v", err)
log.Printf("Main: Error: %v", reterr)
}
if err := obj.Close(); err != nil {
log.Printf("Main: Close: %v", err)
//return cli.NewExitError(err.Error(), 1) // TODO: ?
return cli.NewExitError("", 1)
}
return nil
return reterr
}
// CLI is the entry point for using mgmt normally from the CLI.

1
lib/deploy.go
View File

@@ -25,7 +25,6 @@ import (
"github.com/purpleidea/mgmt/etcd"
etcdfs "github.com/purpleidea/mgmt/etcd/fs"
"github.com/purpleidea/mgmt/gapi"
// these imports are so that GAPIs register themselves in init()
_ "github.com/purpleidea/mgmt/lang"
_ "github.com/purpleidea/mgmt/puppet"

2
lib/hello.go
View File

@@ -44,5 +44,5 @@ func hello(program, version string, flags Flags) {
}
log.Printf("This is: %s, version: %s", program, version)
log.Printf("Main: Start: %v", start)
log.Printf("main: Start: %v", start)
}

Some files were not shown because too many files have changed in this diff Show More