lourenco/mgmt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

lang: Add meta parameter parsing to resources

Browse Source 
Now we can actually specify metaparameters in the resources!
This commit is contained in:
James Shubin
2019-01-10 22:40:39 -05:00
parent 8e0bde3071
commit ad30737119
9 changed files with 646 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
docs/language-guide.md
View File

@@ -214,6 +214,50 @@ it evaluates to `true`, then the parameter will be used. If no `elvis` operator
is specified, then the parameter value will also be used. If the parameter is
not specified, then it will obviously not be used.
Resources may specify meta parameters. To do so, you must add them as you would
a regular parameter, except that they start with `Meta` and are capitalized. Eg:
```mcl
file "/tmp/f1" {
content => "hello!\n",
Meta:noop => true,
Meta:delay => $b ?: 42,
}
```
As you can see, they also support the elvis operator, and you can add as many as
you like. While it is not recommended to add the same meta parameter more than
once, it does not currently cause an error, and even though the result of doing
so is officially undefined, it will currently take the last specified value.
You may also specify a single meta parameter struct. This is useful if you'd
like to reuse a value, or build a combined value programmatically. For example:
```mcl
file "/tmp/f1" {
content => "hello!\n",
Meta => $b ?: struct{
noop => false,
retry => -1,
delay => 0,
poll => 5,
limit => 4.2,
burst => 3,
sema => ["foo:1", "bar:3",],
},
}
```
Remember that the top-level `Meta` field supports the elvis operator, while the
individual struct fields in the struct type do not. This is to be expected, but
since they are syntactically similar, it is worth mentioning to avoid confusion.
Please note that at the moment, you must specify a full metaparams struct, since
partial struct types are currently not supported in the language. Patches are
welcome if you'd like to add this tricky feature!
Resources may also declare edges internally. The edges may point to or from
another resource, and may optionally include a notification. The four properties
are: `Before`, `Depend`, `Notify` and `Listen`. The first two represent normal

38
lang/interpret_test.go
View File

@@ -963,6 +963,44 @@ func TestAstInterpret0(t *testing.T) {
graph: graph,
})
}
{
graph, _ := pgraph.NewGraph("g")
t1, _ := engine.NewNamedResource("test", "t1")
x := t1.(*resources.TestRes)
stringptr := "this is meta"
x.StringPtr = &stringptr
m := &engine.MetaParams{
Noop: true, // overwritten
Retry: -1,
Delay: 0,
Poll: 5,
Limit: 4.2,
Burst: 3,
Sema: []string{"foo:1", "bar:3"},
}
x.SetMetaParams(m)
graph.AddVertex(t1)
testCases = append(testCases, test{
name: "resource with meta params",
code: `
test "t1" {
stringptr => "this is meta",
Meta => struct{
noop => false,
retry => -1,
delay => 0,
poll => 5,
limit => 4.2,
burst => 3,
sema => ["foo:1", "bar:3",],
},
Meta:noop => true,
}
`,
graph: graph,
})
}
names := []string{}
for index, tc := range testCases { // run all the tests

31
lang/interpret_test/TestAstFunc1/metaparams0.graph Normal file
View File

@@ -0,0 +1,31 @@
Edge: bool(false) -> struct(noop: bool(false); retry: int(-1); delay: int(0); poll: int(5); limit: float(4.2); burst: int(3); sema: list(str(foo:1), str(bar:3))) # noop
Edge: bool(true) -> var(b) # b
Edge: bool(true) -> var(b) # b
Edge: float(4.2) -> struct(noop: bool(false); retry: int(-1); delay: int(0); poll: int(5); limit: float(4.2); burst: int(3); sema: list(str(foo:1), str(bar:3))) # limit
Edge: int(-1) -> struct(noop: bool(false); retry: int(-1); delay: int(0); poll: int(5); limit: float(4.2); burst: int(3); sema: list(str(foo:1), str(bar:3))) # retry
Edge: int(0) -> struct(noop: bool(false); retry: int(-1); delay: int(0); poll: int(5); limit: float(4.2); burst: int(3); sema: list(str(foo:1), str(bar:3))) # delay
Edge: int(3) -> struct(noop: bool(false); retry: int(-1); delay: int(0); poll: int(5); limit: float(4.2); burst: int(3); sema: list(str(foo:1), str(bar:3))) # burst
Edge: int(5) -> struct(noop: bool(false); retry: int(-1); delay: int(0); poll: int(5); limit: float(4.2); burst: int(3); sema: list(str(foo:1), str(bar:3))) # poll
Edge: list(str(foo:1), str(bar:3)) -> struct(noop: bool(false); retry: int(-1); delay: int(0); poll: int(5); limit: float(4.2); burst: int(3); sema: list(str(foo:1), str(bar:3))) # sema
Edge: str(bar:3) -> list(str(foo:1), str(bar:3)) # 1
Edge: str(foo:1) -> list(str(foo:1), str(bar:3)) # 0
Edge: str(hello world) -> call:fmt.printf(str(hello world)) # a
Vertex: bool(false)
Vertex: bool(false)
Vertex: bool(true)
Vertex: bool(true)
Vertex: call:fmt.printf(str(hello world))
Vertex: float(4.2)
Vertex: int(-1)
Vertex: int(0)
Vertex: int(3)
Vertex: int(42)
Vertex: int(5)
Vertex: list(str(foo:1), str(bar:3))
Vertex: str(bar:3)
Vertex: str(foo:1)
Vertex: str(greeting)
Vertex: str(hello world)
Vertex: struct(noop: bool(false); retry: int(-1); delay: int(0); poll: int(5); limit: float(4.2); burst: int(3); sema: list(str(foo:1), str(bar:3)))
Vertex: var(b)
Vertex: var(b)

20
lang/interpret_test/TestAstFunc1/metaparams0/main.mcl Normal file
View File

@@ -0,0 +1,20 @@
import "fmt"
$b = true
test "greeting" {
anotherstr => fmt.printf("hello world"),
Meta => $b ?: struct{
noop => false,
retry => -1,
delay => 0,
poll => 5,
limit => 4.2,
burst => 3,
sema => ["foo:1", "bar:3",],
},
Meta:noop => false,
Meta:noop => true, # duplicates allowed atm, but not recommended!
Meta:poll => $b ?: 42,
}

5
lang/lang_test.go
View File

@@ -75,6 +75,11 @@ func vertexCmpFn(v1, v2 pgraph.Vertex) (bool, error) {
return false, nil
}
m1, m2 := r1.MetaParams(), r2.MetaParams()
if err := m1.Cmp(m2); err != nil {
return false, nil
}
return v1.String() == v2.String(), nil
}

1
lang/lexparse.go
View File

@@ -52,6 +52,7 @@ const (
ErrLexerFloatOverflow = interfaces.Error("float: overflow")
ErrParseError = interfaces.Error("parser")
ErrParseSetType = interfaces.Error("can't set return type in parser")
ErrParseResFieldInvalid = interfaces.Error("can't use unknown resource field")
ErrParseAdditionalEquals = interfaces.Error(errstrParseAdditionalEquals)
ErrParseExpectingComma = interfaces.Error(errstrParseExpectingComma)
)

103
lang/lexparse_test.go
View File

@@ -1078,6 +1078,109 @@ func TestLexParse0(t *testing.T) {
exp: exp,
})
}
{
exp := &StmtProg{
Prog: []interfaces.Stmt{
&StmtRes{
Kind: "test",
Name: &ExprStr{
V: "t1",
},
Contents: []StmtResContents{
&StmtResMeta{
Property: "noop",
MetaExpr: &ExprBool{
V: true,
},
},
&StmtResMeta{
Property: "delay",
MetaExpr: &ExprInt{
V: 42,
},
Condition: &ExprBool{
V: true,
},
},
},
},
&StmtRes{
Kind: "test",
Name: &ExprStr{
V: "t2",
},
Contents: []StmtResContents{
&StmtResMeta{
Property: "limit",
MetaExpr: &ExprFloat{
V: 0.45,
},
},
&StmtResMeta{
Property: "burst",
MetaExpr: &ExprInt{
V: 4,
},
},
},
},
&StmtRes{
Kind: "test",
Name: &ExprStr{
V: "t3",
},
Contents: []StmtResContents{
&StmtResMeta{
Property: "noop",
MetaExpr: &ExprBool{
V: true,
},
},
&StmtResMeta{
Property: "meta",
MetaExpr: &ExprStruct{
Fields: []*ExprStructField{
{Name: "poll", Value: &ExprInt{V: 5}},
{Name: "retry", Value: &ExprInt{V: 3}},
{
Name: "sema",
Value: &ExprList{
Elements: []interfaces.Expr{
&ExprStr{V: "foo:1"},
&ExprStr{V: "bar:3"},
},
},
},
},
},
},
},
}},
}
testCases = append(testCases, test{
name: "res meta stmt",
code: `
test "t1" {
Meta:noop => true,
Meta:delay => true ?: 42,
}
test "t2" {
Meta:limit => 0.45,
Meta:burst => 4,
}
test "t3" {
Meta:noop => true, # meta params can be combined
Meta => struct{
poll => 5,
retry => 3,
sema => ["foo:1", "bar:3",],
},
}
`,
fail: false,
exp: exp,
})
}
{
testCases = append(testCases, test{
name: "parser set type incompatibility str",

83
lang/parser.y
View File

@@ -69,6 +69,7 @@ func init() {
resContents []StmtResContents // interface
resField *StmtResField
resEdge *StmtResEdge
resMeta *StmtResMeta
edgeHalfList []*StmtEdgeHalf
edgeHalf *StmtEdgeHalf
@@ -945,6 +946,26 @@ resource_body:
posLast(yylex, yyDollar) // our pos
$$.resContents = append($1.resContents, $2.resEdge)
}
| resource_body resource_meta
{
posLast(yylex, yyDollar) // our pos
$$.resContents = append($1.resContents, $2.resMeta)
}
| resource_body conditional_resource_meta
{
posLast(yylex, yyDollar) // our pos
$$.resContents = append($1.resContents, $2.resMeta)
}
| resource_body resource_meta_struct
{
posLast(yylex, yyDollar) // our pos
$$.resContents = append($1.resContents, $2.resMeta)
}
| resource_body conditional_resource_meta_struct
{
posLast(yylex, yyDollar) // our pos
$$.resContents = append($1.resContents, $2.resMeta)
}
;
resource_field:
IDENTIFIER ROCKET expr COMMA
@@ -991,6 +1012,68 @@ conditional_resource_edge:
}
}
;
resource_meta:
// Meta:noop => true,
CAPITALIZED_IDENTIFIER COLON IDENTIFIER ROCKET expr COMMA
{
posLast(yylex, yyDollar) // our pos
if strings.ToLower($1.str) != strings.ToLower(MetaField) {
// this will ultimately cause a parser error to occur...
yylex.Error(fmt.Sprintf("%s: %s", ErrParseResFieldInvalid, $1.str))
}
$$.resMeta = &StmtResMeta{
Property: $3.str,
MetaExpr: $5.expr,
}
}
;
conditional_resource_meta:
// Meta:limit => $present ?: 4,
CAPITALIZED_IDENTIFIER COLON IDENTIFIER ROCKET expr ELVIS expr COMMA
{
posLast(yylex, yyDollar) // our pos
if strings.ToLower($1.str) != strings.ToLower(MetaField) {
// this will ultimately cause a parser error to occur...
yylex.Error(fmt.Sprintf("%s: %s", ErrParseResFieldInvalid, $1.str))
}
$$.resMeta = &StmtResMeta{
Property: $3.str,
MetaExpr: $7.expr,
Condition: $5.expr,
}
}
;
resource_meta_struct:
// Meta => struct{meta => true, retry => 3,},
CAPITALIZED_IDENTIFIER ROCKET expr COMMA
{
posLast(yylex, yyDollar) // our pos
if strings.ToLower($1.str) != strings.ToLower(MetaField) {
// this will ultimately cause a parser error to occur...
yylex.Error(fmt.Sprintf("%s: %s", ErrParseResFieldInvalid, $1.str))
}
$$.resMeta = &StmtResMeta{
Property: $1.str,
MetaExpr: $3.expr,
}
}
;
conditional_resource_meta_struct:
// Meta => $present ?: struct{poll => 60, sema => ["foo:1", "bar:3",],},
CAPITALIZED_IDENTIFIER ROCKET expr ELVIS expr COMMA
{
posLast(yylex, yyDollar) // our pos
if strings.ToLower($1.str) != strings.ToLower(MetaField) {
// this will ultimately cause a parser error to occur...
yylex.Error(fmt.Sprintf("%s: %s", ErrParseResFieldInvalid, $1.str))
}
$$.resMeta = &StmtResMeta{
Property: $1.str,
MetaExpr: $5.expr,
Condition: $3.expr,
}
}
;
edge:
// TODO: we could technically prevent single edge_half pieces from being
// parsed, but it's probably more work than is necessary...

322
lang/structs.go
View File

@@ -36,6 +36,7 @@ import (
"github.com/purpleidea/mgmt/util"
errwrap "github.com/pkg/errors"
"golang.org/x/time/rate"
)
const (
@@ -55,6 +56,9 @@ const (
// This is most similar to "require" in Puppet.
EdgeDepend = "depend"
// MetaField is the prefix used to specify a meta parameter for the res.
MetaField = "meta"
// AllowUserDefinedPolyFunc specifies if we allow user-defined
// polymorphic functions or not. At the moment this is not implemented.
// XXX: not implemented
@@ -302,7 +306,6 @@ func (obj *StmtRes) Graph() (*pgraph.Graph, error) {
// analogous function for expressions is Value. Those Value functions might get
// called by this Output function if they are needed to produce the output. In
// the case of this resource statement, this is definitely the case.
// XXX: Add MetaParams as a simple meta field with a struct of the right type...
func (obj *StmtRes) Output() (*interfaces.Output, error) {
nameValue, err := obj.Name.Value()
if err != nil {
@@ -447,6 +450,12 @@ func (obj *StmtRes) Output() (*interfaces.Output, error) {
return nil, errwrap.Wrapf(err, "error building edges")
}
metaparams, err := obj.metaparams()
if err != nil {
return nil, errwrap.Wrapf(err, "error building meta params")
}
res.SetMetaParams(metaparams)
return &interfaces.Output{
Resources: []engine.Res{res},
Edges: edges,
@@ -565,6 +574,115 @@ func (obj *StmtRes) edges() ([]*interfaces.Edge, error) {
return edges, nil
}
// metaparams is a helper function to generate the metaparams that come from the
// resource.
func (obj *StmtRes) metaparams() (*engine.MetaParams, error) {
meta := engine.DefaultMetaParams.Copy() // defaults
for _, line := range obj.Contents {
x, ok := line.(*StmtResMeta)
if !ok {
continue
}
if x.Condition != nil {
b, err := x.Condition.Value()
if err != nil {
return nil, err
}
if !b.Bool() { // if value exists, and is false, skip it
continue
}
}
v, err := x.MetaExpr.Value()
if err != nil {
return nil, err
}
switch p := strings.ToLower(x.Property); p {
// TODO: we could add these fields dynamically if we were fancy!
case "noop":
meta.Noop = v.Bool() // must not panic
case "retry":
x := v.Int() // must not panic
// TODO: check that it doesn't overflow
meta.Retry = int16(x)
case "delay":
x := v.Int() // must not panic
// TODO: check that it isn't signed
meta.Delay = uint64(x)
case "poll":
x := v.Int() // must not panic
// TODO: check that it doesn't overflow and isn't signed
meta.Poll = uint32(x)
case "limit": // rate.Limit
x := v.Float() // must not panic
meta.Limit = rate.Limit(x)
case "burst":
x := v.Int() // must not panic
// TODO: check that it doesn't overflow
meta.Burst = int(x)
case "sema": // []string
values := []string{}
for _, x := range v.List() { // must not panic
s := x.Str() // must not panic
values = append(values, s)
}
meta.Sema = values
case MetaField:
if val, exists := v.Struct()["noop"]; exists {
meta.Noop = val.Bool() // must not panic
}
if val, exists := v.Struct()["retry"]; exists {
x := val.Int() // must not panic
// TODO: check that it doesn't overflow
meta.Retry = int16(x)
}
if val, exists := v.Struct()["delay"]; exists {
x := val.Int() // must not panic
// TODO: check that it isn't signed
meta.Delay = uint64(x)
}
if val, exists := v.Struct()["poll"]; exists {
x := val.Int() // must not panic
// TODO: check that it doesn't overflow and isn't signed
meta.Poll = uint32(x)
}
if val, exists := v.Struct()["limit"]; exists {
x := val.Float() // must not panic
meta.Limit = rate.Limit(x)
}
if val, exists := v.Struct()["burst"]; exists {
x := val.Int() // must not panic
// TODO: check that it doesn't overflow
meta.Burst = int(x)
}
if val, exists := v.Struct()["sema"]; exists {
values := []string{}
for _, x := range val.List() { // must not panic
s := x.Str() // must not panic
values = append(values, s)
}
meta.Sema = values
}
default:
return nil, fmt.Errorf("unknown property: %s", p)
}
}
return meta, nil
}
// StmtResContents is the interface that is met by the resource contents. Look
// closely for while it is similar to the Stmt interface, it is quite different.
type StmtResContents interface {
@@ -884,6 +1002,208 @@ func (obj *StmtResEdge) Graph() (*pgraph.Graph, error) {
return graph, nil
}
// StmtResMeta represents a single meta value in the parsed resource
// representation. It can also contain a struct that contains one or more meta
// parameters. If it contains such a struct, then the `Property` field contains
// the string found in the MetaField constant, otherwise this field will
// correspond to the particular meta parameter specified. This does not satisfy
// the Stmt interface.
type StmtResMeta struct {
Property string // TODO: iota constant instead?
MetaExpr interfaces.Expr
Condition interfaces.Expr // the value will be used if nil or true
}
// Apply is a general purpose iterator method that operates on any AST node. It
// is not used as the primary AST traversal function because it is less readable
// and easy to reason about than manually implementing traversal for each node.
// Nevertheless, it is a useful facility for operations that might only apply to
// a select number of node types, since they won't need extra noop iterators...
func (obj *StmtResMeta) Apply(fn func(interfaces.Node) error) error {
if obj.Condition != nil {
if err := obj.Condition.Apply(fn); err != nil {
return err
}
}
if err := obj.MetaExpr.Apply(fn); err != nil {
return err
}
return fn(obj)
}
// Init initializes this branch of the AST, and returns an error if it fails to
// validate.
func (obj *StmtResMeta) Init(data *interfaces.Data) error {
if obj.Property == "" {
return fmt.Errorf("empty property")
}
switch p := strings.ToLower(obj.Property); p {
// TODO: we could add these fields dynamically if we were fancy!
case "noop":
case "retry":
case "delay":
case "poll":
case "limit":
case "burst":
case "sema":
case MetaField:
default:
return fmt.Errorf("invalid property: `%s`", obj.Property)
}
if obj.Condition != nil {
if err := obj.Condition.Init(data); err != nil {
return err
}
}
return obj.MetaExpr.Init(data)
}
// Interpolate returns a new node (aka a copy) once it has been expanded. This
// generally increases the size of the AST when it is used. It calls Interpolate
// on any child elements and builds the new node with those new node contents.
// This interpolate is different It is different from the interpolate found in
// the Expr and Stmt interfaces because it returns a different type as output.
func (obj *StmtResMeta) Interpolate() (StmtResContents, error) {
interpolated, err := obj.MetaExpr.Interpolate()
if err != nil {
return nil, err
}
var condition interfaces.Expr
if obj.Condition != nil {
condition, err = obj.Condition.Interpolate()
if err != nil {
return nil, err
}
}
return &StmtResMeta{
Property: obj.Property,
MetaExpr: interpolated,
Condition: condition,
}, nil
}
// SetScope stores the scope for later use in this resource and it's children,
// which it propagates this downwards to.
func (obj *StmtResMeta) SetScope(scope *interfaces.Scope) error {
if err := obj.MetaExpr.SetScope(scope); err != nil {
return err
}
if obj.Condition != nil {
if err := obj.Condition.SetScope(scope); err != nil {
return err
}
}
return nil
}
// Unify returns the list of invariants that this node produces. It recursively
// calls Unify on any children elements that exist in the AST, and returns the
// collection to the caller. It is different from the Unify found in the Expr
// and Stmt interfaces because it adds an input parameter.
// XXX: Allow specifying partial meta param structs and unify the subset type.
// XXX: The resource fields have the same limitation with field structs.
func (obj *StmtResMeta) Unify(kind string) ([]interfaces.Invariant, error) {
var invariants []interfaces.Invariant
invars, err := obj.MetaExpr.Unify()
if err != nil {
return nil, err
}
invariants = append(invariants, invars...)
// conditional expression might have some children invariants to share
if obj.Condition != nil {
condition, err := obj.Condition.Unify()
if err != nil {
return nil, err
}
invariants = append(invariants, condition...)
// the condition must ultimately be a boolean
conditionInvar := &unification.EqualsInvariant{
Expr: obj.Condition,
Type: types.TypeBool,
}
invariants = append(invariants, conditionInvar)
}
// add additional invariants based on what's in obj.Property !!!
var typ *types.Type
switch p := strings.ToLower(obj.Property); p {
// TODO: we could add these fields dynamically if we were fancy!
case "noop":
typ = types.TypeBool
case "retry":
typ = types.TypeInt
case "delay":
typ = types.TypeInt
case "poll":
typ = types.TypeInt
case "limit": // rate.Limit
typ = types.TypeFloat
case "burst":
typ = types.TypeInt
case "sema":
typ = types.NewType("[]str")
case MetaField:
// FIXME: allow partial subsets of this struct, and in any order
// FIXME: we might need an updated unification engine to do this
typ = types.NewType("struct{noop bool; retry int; delay int; poll int; limit float; burst int; sema []str}")
default:
return nil, fmt.Errorf("unknown property: %s", p)
}
invar := &unification.EqualsInvariant{
Expr: obj.MetaExpr,
Type: typ,
}
invariants = append(invariants, invar)
return invariants, nil
}
// Graph returns the reactive function graph which is expressed by this node. It
// includes any vertices produced by this node, and the appropriate edges to any
// vertices that are produced by its children. Nodes which fulfill the Expr
// interface directly produce vertices (and possible children) where as nodes
// that fulfill the Stmt interface do not produces vertices, where as their
// children might. It is interesting to note that nothing directly adds an edge
// to the resources created, but rather, once all the values (expressions) with
// no outgoing edges have produced at least a single value, then the resources
// know they're able to be built.
func (obj *StmtResMeta) Graph() (*pgraph.Graph, error) {
graph, err := pgraph.NewGraph("resmeta")
if err != nil {
return nil, errwrap.Wrapf(err, "could not create graph")
}
g, err := obj.MetaExpr.Graph()
if err != nil {
return nil, err
}
graph.AddGraph(g)
if obj.Condition != nil {
g, err := obj.Condition.Graph()
if err != nil {
return nil, err
}
graph.AddGraph(g)
}
return graph, nil
}
// StmtEdge is a representation of a dependency. It also supports send/recv.
// Edges represents that the first resource (Kind/Name) listed in the
// EdgeHalfList should happen in the resource graph *before* the next resource