// Mgmt // Copyright (C) James Shubin and the project contributors // Written by James Shubin and the project contributors // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // This program is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU General Public License // along with this program. If not, see . // // Additional permission under GNU GPL version 3 section 7 // // If you modify this program, or any covered work, by linking or combining it // with embedded mcl code and modules (and that the embedded mcl code and // modules which link with this program, contain a copy of their source code in // the authoritative form) containing parts covered by the terms of any other // license, the licensors of this program grant you additional permission to // convey the resulting work. Furthermore, the licensors of this program grant // the original author, James Shubin, additional permission to update this // additional permission if he deems it necessary to achieve the goals of this // additional permission. // Package interpret contains the implementation of the actual interpret // function that takes an AST and returns a resource graph. package interpret import ( "fmt" "github.com/purpleidea/mgmt/engine" engineUtil "github.com/purpleidea/mgmt/engine/util" "github.com/purpleidea/mgmt/lang/interfaces" "github.com/purpleidea/mgmt/pgraph" "github.com/purpleidea/mgmt/util/errwrap" ) // Interpreter is a base struct for handling the Interpret operation. There is // nothing stateful here, you don't need to preserve this between runs. type Interpreter struct { // Debug represents if we're running in debug mode or not. Debug bool // Logf is a logger which should be used. Logf func(format string, v ...interface{}) // lookup stores the resources found by kind and name. It doesn't store // any resources which are hidden since those could have duplicates. // format: map[kind]map[name]Res lookup map[engine.ResPtrUID]engine.Res // lookupHidden stores the hidden resources found by kind and name. It // doesn't store any normal resources which are not hidden. // format formerly: map[kind]map[name]Res lookupHidden map[engine.ResPtrUID][]engine.Res // receive doesn't need a special extension for hidden resources since // they can't send, only recv, and senders can't have incompatible dupes // format formerly: map[kind]map[name]map[field]*Send receive map[engine.ResPtrUID]map[string]*engine.Send // export tracks the unique combinations we export. (kind, name, host) export map[engine.ResDelete]struct{} } // Interpret runs the program and outputs a generated resource graph. It // requires an AST, and the table of values required to populate that AST. Type // unification, and earlier steps should obviously be run first so that you can // actually get a useful resource graph out of this instead of an error! func (obj *Interpreter) Interpret(ast interfaces.Stmt, table interfaces.Table) (*pgraph.Graph, error) { // build the kind,name -> res mapping obj.lookup = make(map[engine.ResPtrUID]engine.Res) obj.lookupHidden = make(map[engine.ResPtrUID][]engine.Res) // build the send/recv mapping obj.receive = make(map[engine.ResPtrUID]map[string]*engine.Send) // build the exports obj.export = make(map[engine.ResDelete]struct{}) // Remember that if a resource is "Hidden", then make sure it is NOT // sending to anyone, since it would never produce a value. It can // receive values, since those might be used during export. // // Remember that if a resource is "Hidden", then it may exist alongside // another resource with the same kind+name without triggering the // "inequivalent duplicate resource" style of errors. Of course multiple // hidden resources with the same kind+name may also exist // simultaneously, just keep in mind that it means that an edge pointing // to a particular kind+name now actually may point to more than one! // // This is needed because of two reasons: (1) because a regular resource // will likely never be compatible with a "Hidden" and "Exported" // resource because one resource might have the Meta:hidden and // Meta:export params and one might not; (2) because you may wish to // have two different hidden resources of different params which export // to different hosts, which means they would likely not be compatible. // // Since we can have more than one "Hidden" and "Exported" resource with // the same name and kind, it's important that we don't export that data // to the same (kind, name, host) location since we'd have multiple // writers to the same key in our World store. We could consider // checking for compatibility, but that's more difficult to achieve. The // "any" host is treated as a special key, which punts this duplicate // problem to being a collection problem. (Which could happen with two // different hosts each exporting a different value to a single host.) // // Remember that the resource graph that this function returns, may now // contain two or more identically named kind+name resources, if at // least one of them is "Hidden". If they are entirely identical, then // it's acceptable to merge them. They may _not_ be merged with the // CompatibleRes API, since on resource "collection" a param may be // changed which could conceivably be incompatible with how we ran the // AdaptCmp API when we merged them. output, err := ast.Output(table) // contains resList, edgeList, etc... if err != nil { return nil, err } graph, err := pgraph.NewGraph("interpret") // give graph a default name if err != nil { return nil, errwrap.Wrapf(err, "could not create new graph") } for _, res := range output.Resources { kind := res.Kind() name := res.Name() meta := res.MetaParams() ruid := engine.ResPtrUID{ Kind: kind, Name: name, } for _, host := range meta.Export { uid := engine.ResDelete{ Kind: kind, Name: name, Host: host, } if _, exists := obj.export[uid]; exists { return nil, fmt.Errorf("duplicate export: %s to %s", res, host) } obj.export[uid] = struct{}{} } if meta.Hidden { rs := obj.lookupHidden[ruid] if len(rs) > 0 { // We only need to check against the last added // resource since this should be commutative, // and as we add more they check themselves in. r := rs[len(rs)-1] // XXX: If we want to check against the regular // resources in obj.lookup, then do it here. // If they're different, then we deduplicate. if err := engine.ResCmp(r, res); err == nil { continue } } // add to temporary lookup table obj.lookupHidden[ruid] = append(obj.lookupHidden[ruid], res) continue } if r, exists := obj.lookup[ruid]; exists { // found same name // XXX: If we want to check against the special hidden // resources in obj.lookupHidden, then do it here. // if the resources support the compatibility API, then // we can attempt to merge them intelligently... r1, ok1 := r.(engine.CompatibleRes) r2, ok2 := res.(engine.CompatibleRes) if ok1 && ok2 { if err := engine.AdaptCmp(r1, r2); err != nil { // TODO: print a diff of the two resources return nil, errwrap.Wrapf(err, "incompatible duplicate resource `%s` found", res) } merged, err := engine.ResMerge(r1, r2) if err != nil { return nil, errwrap.Wrapf(err, "could not merge duplicate resources") } obj.lookup[ruid] = merged // they match here, we don't need to test below! continue } if err := engine.ResCmp(r, res); err != nil { // TODO: print a diff of the two resources return nil, errwrap.Wrapf(err, "inequivalent duplicate resource `%s` found", res) } // more than one identical resource exists. we can allow // duplicates, if they're not going to conflict... since // it was identical, we leave the earlier version in the // graph since they're exactly equivalent anyways. // TODO: does it matter which one we add to the graph? // currently we add the first one that was found... continue } obj.lookup[ruid] = res // add to temporary lookup table //graph.AddVertex(res) // do this below once this table is final } // ensure all the vertices exist... for _, res := range obj.lookup { graph.AddVertex(res) } for _, rs := range obj.lookupHidden { for _, res := range rs { graph.AddVertex(res) } } for _, edge := range output.Edges { v1s := obj.lookupAll(edge.Kind1, edge.Name1) if len(v1s) == 0 { return nil, fmt.Errorf("edge cannot find resource kind: %s named: `%s`", edge.Kind1, edge.Name1) } v2s := obj.lookupAll(edge.Kind2, edge.Name2) if len(v2s) == 0 { return nil, fmt.Errorf("edge cannot find resource kind: %s named: `%s`", edge.Kind2, edge.Name2) } // Make edges pair wise between each two. Normally these loops // only have one iteration each unless we have Hidden resources. for _, v1 := range v1s { for _, v2 := range v2s { e := obj.makeEdge(graph, v1, v2, edge) graph.AddEdge(v1, v2, e) // identical duplicates are ignored } } // send recv if (edge.Send == "") != (edge.Recv == "") { // xor return nil, fmt.Errorf("you must specify both send/recv fields or neither") } if edge.Send == "" || edge.Recv == "" { // is there send/recv to do or not? continue } for _, v1 := range v1s { for _, v2 := range v2s { if err := obj.makeSendRecv(v1, v2, edge); err != nil { return nil, err } } } } // we need to first build up a map of all the resources handles, because // we don't know which order send/recv pairs will arrive in, and we need // to ensure the right pointer exists before we reference it... finally, // we build up a list of send/recv mappings to ensure we don't overwrite // pre-existing mappings, so we can now set them all at once at the end! // TODO: do this in a deterministic order for st, recv := range obj.receive { kind := st.Kind name := st.Name if len(recv) == 0 { // skip empty maps from allocation! continue } if r := obj.lookupRes(kind, name); r != nil { res, ok := r.(engine.RecvableRes) if !ok { return nil, fmt.Errorf("cannot recv to resource: %s", engine.Repr(kind, name)) } res.SetRecv(recv) // set it! } // hidden for _, r := range obj.lookupHiddenRes(kind, name) { res, ok := r.(engine.RecvableRes) if !ok { return nil, fmt.Errorf("cannot recv to resource: %s", engine.Repr(kind, name)) } res.SetRecv(recv) // set it! } } // ensure that we have a DAG! if _, err := graph.TopologicalSort(); err != nil { errNotAcyclic, ok := err.(*pgraph.ErrNotAcyclic) if !ok { return nil, err // programming error } obj.Logf("%s", err) for _, vertex := range errNotAcyclic.Cycle { obj.Logf("* %s", vertex) } return nil, errwrap.Wrapf(err, "resource graph has cycles") } return graph, nil } // lookupRes is a simple helper function. Returns nil if not found. func (obj *Interpreter) lookupRes(kind, name string) engine.Res { ruid := engine.ResPtrUID{ Kind: kind, Name: name, } res, exists := obj.lookup[ruid] if !exists { return nil } return res } // lookupHiddenRes is a simple helper function. Returns any found. func (obj *Interpreter) lookupHiddenRes(kind, name string) []engine.Res { ruid := engine.ResPtrUID{ Kind: kind, Name: name, } res, exists := obj.lookupHidden[ruid] if !exists { return nil } return res } // lookupAll is a simple helper function. Returns any found. func (obj *Interpreter) lookupAll(kind, name string) []pgraph.Vertex { vs := []pgraph.Vertex{} if r := obj.lookupRes(kind, name); r != nil { vs = append(vs, r) } for _, r := range obj.lookupHiddenRes(kind, name) { vs = append(vs, r) } return vs } // makeEdge is a simple helper function. func (obj *Interpreter) makeEdge(graph *pgraph.Graph, v1, v2 pgraph.Vertex, edge *interfaces.Edge) *engine.Edge { var notify = edge.Notify if existingEdge := graph.FindEdge(v1, v2); existingEdge != nil { // collate previous Notify signals to this edge with OR notify = notify || (existingEdge.(*engine.Edge)).Notify } return &engine.Edge{ Name: fmt.Sprintf("%s -> %s", v1, v2), Notify: notify, } } // makeSendRecv is a simple helper function. func (obj *Interpreter) makeSendRecv(v1, v2 pgraph.Vertex, edge *interfaces.Edge) error { ruid := engine.ResPtrUID{ Kind: edge.Kind2, Name: edge.Name2, } if _, exists := obj.receive[ruid]; !exists { obj.receive[ruid] = make(map[string]*engine.Send) } // check for pre-existing send/recv at this key if existingSend, exists := obj.receive[ruid][edge.Recv]; exists { // ignore identical duplicates // TODO: does this safe ignore work with duplicate compatible resources? if existingSend.Res != v1 || existingSend.Key != edge.Send { return fmt.Errorf("resource: `%s` has duplicate receive on: `%s` param", engine.Repr(edge.Kind2, edge.Name2), edge.Recv) } } if res, ok := v1.(engine.Res); ok && res.MetaParams().Hidden && edge.Send != "" { return fmt.Errorf("cannot send from hidden resource: %s", engine.Stringer(res)) } res1, ok := v1.(engine.SendableRes) if !ok { return fmt.Errorf("cannot send from resource: %s", engine.Stringer(res1)) } res2, ok := v2.(engine.RecvableRes) if !ok { return fmt.Errorf("cannot recv to resource: %s", engine.Stringer(res2)) } if err := engineUtil.StructFieldCompat(res1.Sends(), edge.Send, res2, edge.Recv); err != nil { return errwrap.Wrapf(err, "cannot send/recv from %s.%s to %s.%s", engine.Stringer(res1), edge.Send, engine.Stringer(res2), edge.Recv) } // XXX: Not doing this for now, see the interface for more information. // TODO: We could instead pass in edge.Send so it would know precisely! //res1.SendSetActive(true) // tell it that it will be sending (optimization) // store mapping for later obj.receive[ruid][edge.Recv] = &engine.Send{Res: res1, Key: edge.Send} return nil }