From b1f93b40aec46ad533f40df5b77e3c315a8d82f5 Mon Sep 17 00:00:00 2001 From: James Shubin Date: Fri, 1 Mar 2019 11:08:19 -0500 Subject: [PATCH] lang: funcs: Add runner pure func execution This adds a function runner that runs pure functions. It will hopefully be useful for speculative execution of functions for compile time determination of types. --- lang/funcs/core/core_test.go | 165 +++++++++++++++++++++++++++++++++++ lang/funcs/funcs.go | 165 +++++++++++++++++++++++++++++++++++ lang/interfaces/func.go | 1 + 3 files changed, 331 insertions(+) create mode 100644 lang/funcs/core/core_test.go diff --git a/lang/funcs/core/core_test.go b/lang/funcs/core/core_test.go new file mode 100644 index 00000000..ebf1fbc5 --- /dev/null +++ b/lang/funcs/core/core_test.go @@ -0,0 +1,165 @@ +// Mgmt +// Copyright (C) 2013-2018+ James Shubin and the project contributors +// Written by James Shubin and the project contributors +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +// +build !root + +package core + +import ( + "fmt" + "reflect" + "testing" + + "github.com/purpleidea/mgmt/lang/funcs" + "github.com/purpleidea/mgmt/lang/types" + "github.com/purpleidea/mgmt/util" + + "github.com/davecgh/go-spew/spew" + "github.com/kylelemons/godebug/pretty" +) + +func TestPureFuncExec0(t *testing.T) { + type test struct { // an individual test + name string + funcname string + args []types.Value + fail bool + expect types.Value + } + testCases := []test{} + + //{ + // testCases = append(testCases, test{ + // name: "", + // funcname: "", + // args: []types.Value{ + // }, + // fail: false, + // expect: nil, + // }) + //} + { + testCases = append(testCases, test{ + name: "strings.to_lower 0", + funcname: "strings.to_lower", + args: []types.Value{ + &types.StrValue{ + V: "HELLO", + }, + }, + fail: false, + expect: &types.StrValue{ + V: "hello", + }, + }) + } + { + testCases = append(testCases, test{ + name: "datetime.now fail", + funcname: "datetime.now", + args: nil, + fail: true, + expect: nil, + }) + } + // TODO: run unification in PureFuncExec if it makes sense to do so... + //{ + // testCases = append(testCases, test{ + // name: "len 0", + // funcname: "len", + // args: []types.Value{ + // &types.StrValue{ + // V: "Hello, world!", + // }, + // }, + // fail: false, + // expect: &types.IntValue{ + // V: 13, + // }, + // }) + //} + + names := []string{} + for index, tc := range testCases { // run all the tests + if tc.name == "" { + t.Errorf("test #%d: not named", index) + continue + } + if util.StrInList(tc.name, names) { + t.Errorf("test #%d: duplicate sub test name of: %s", index, tc.name) + continue + } + names = append(names, tc.name) + + //if index != 3 { // hack to run a subset (useful for debugging) + //if (index != 20 && index != 21) { + //if tc.name != "nil" { + // continue + //} + + t.Run(fmt.Sprintf("test #%d (%s)", index, tc.name), func(t *testing.T) { + name, funcname, args, fail, expect := tc.name, tc.funcname, tc.args, tc.fail, tc.expect + + t.Logf("\n\ntest #%d (%s) ----------------\n\n", index, name) + + f, err := funcs.Lookup(funcname) + if err != nil { + t.Errorf("test #%d: func lookup failed with: %+v", index, err) + return + } + + result, err := funcs.PureFuncExec(f, args) + + if !fail && err != nil { + t.Errorf("test #%d: func failed with: %+v", index, err) + return + } + if fail && err == nil { + t.Errorf("test #%d: func passed, expected fail", index) + return + } + if !fail && result == nil { + t.Errorf("test #%d: func output was nil", index) + return + } + + if !reflect.DeepEqual(result, expect) { + // double check because DeepEqual is different since the func exists + diff := pretty.Compare(result, expect) + if diff != "" { // bonus + t.Errorf("test #%d: result did not match expected", index) + // TODO: consider making our own recursive print function + t.Logf("test #%d: actual: \n\n%s\n", index, spew.Sdump(result)) + t.Logf("test #%d: expected: \n\n%s", index, spew.Sdump(expect)) + + // more details, for tricky cases: + diffable := &pretty.Config{ + Diffable: true, + IncludeUnexported: true, + //PrintStringers: false, + //PrintTextMarshalers: false, + //SkipZeroFields: false, + } + t.Logf("test #%d: actual: \n\n%s\n", index, diffable.Sprint(result)) + t.Logf("test #%d: expected: \n\n%s", index, diffable.Sprint(expect)) + t.Logf("test #%d: diff:\n%s", index, diff) + return + } + } + }) + } +} diff --git a/lang/funcs/funcs.go b/lang/funcs/funcs.go index 7f41a9ae..604c247a 100644 --- a/lang/funcs/funcs.go +++ b/lang/funcs/funcs.go @@ -21,8 +21,14 @@ package funcs import ( "fmt" "strings" + "sync" "github.com/purpleidea/mgmt/lang/interfaces" + "github.com/purpleidea/mgmt/lang/types" + "github.com/purpleidea/mgmt/util" + + multierr "github.com/hashicorp/go-multierror" + errwrap "github.com/pkg/errors" ) const ( @@ -128,3 +134,162 @@ func Map() map[string]func() interfaces.Func { } return m } + +// PureFuncExec is usually used to provisionally speculate about the result of a +// pure function, by running it once, and returning the result. Pure functions +// are expected to only produce one value that depends only on the input values. +// This won't run any slow functions either. +func PureFuncExec(handle interfaces.Func, args []types.Value) (types.Value, error) { + hostname := "" // XXX: add to interface + debug := false // XXX: add to interface + logf := func(format string, v ...interface{}) {} // XXX: add to interface + + info := handle.Info() + if !info.Pure { + return nil, fmt.Errorf("func is not pure") + } + // if function is expensive to run, we won't run it provisionally + if info.Slow { + return nil, fmt.Errorf("func is slow") + } + + if err := handle.Validate(); err != nil { + return nil, errwrap.Wrapf(err, "could not validate func") + } + + sig := handle.Info().Sig + if sig.Kind != types.KindFunc { + return nil, fmt.Errorf("must be kind func") + } + + wg := &sync.WaitGroup{} + defer wg.Wait() + + errch := make(chan error) + input := make(chan types.Value) // we close this when we're done + output := make(chan types.Value) // we create it, func closes it + + init := &interfaces.Init{ + Hostname: hostname, + Input: input, + Output: output, + World: nil, // should not be used for pure functions + Debug: debug, + Logf: func(format string, v ...interface{}) { + logf("func: "+format, v...) + }, + } + + if err := handle.Init(init); err != nil { + return nil, errwrap.Wrapf(err, "could not init func") + } + + close1 := make(chan struct{}) + close2 := make(chan struct{}) + wg.Add(1) + go func() { + defer wg.Done() + defer close(errch) // last one turns out the lights + select { + case <-close1: + } + select { + case <-close2: + } + }() + + wg.Add(1) + go func() { + defer wg.Done() + defer close(close1) + if debug { + logf("Running func") + } + err := handle.Stream() // sends to output chan + if debug { + logf("Exiting func") + } + if err == nil { + return + } + // we closed with an error... + select { + case errch <- errwrap.Wrapf(err, "problem streaming func"): + } + }() + + wg.Add(1) + go func() { + defer wg.Done() + defer close(close2) + defer close(input) // we only send one value + if len(args) == 0 { + return + } + si := &types.Type{ + // input to functions are structs + Kind: types.KindStruct, + Map: handle.Info().Sig.Map, + Ord: handle.Info().Sig.Ord, + } + st := types.NewStruct(si) + + for i, arg := range args { + name := util.NumToAlpha(i) // assume (incorrectly) for now... + if err := st.Set(name, arg); err != nil { // populate struct + select { + case errch <- errwrap.Wrapf(err, "struct set failure"): + } + return + } + } + + select { + case input <- st: // send to function (must not block) + case <-close1: // unblock the input send in case stream closed + select { + case errch <- fmt.Errorf("stream closed early"): + } + } + }() + + once := false + var result types.Value + var reterr error +Loop: + for { + select { + case value, ok := <-output: // read from channel + if !ok { + output = nil + continue Loop // only exit via errch closing! + } + if once { + reterr = fmt.Errorf("got more than one value") + continue // only exit via errch closing! + } + once = true + result = value // save value + + case err, ok := <-errch: // handle possible errors + if !ok { + break Loop + } + e := errwrap.Wrapf(err, "problem streaming func") + if reterr != nil { + reterr = multierr.Append(reterr, e) + } else { + reterr = e + } + } + } + + if err := handle.Close(); err != nil { + if reterr != nil { + err = multierr.Append(err, reterr) + } + return nil, errwrap.Wrapf(err, "problem closing func") + } + + return result, reterr +} diff --git a/lang/interfaces/func.go b/lang/interfaces/func.go index 2d06ecd5..2c8410b9 100644 --- a/lang/interfaces/func.go +++ b/lang/interfaces/func.go @@ -28,6 +28,7 @@ import ( type Info struct { Pure bool // is the function pure? (can it be memoized?) Memo bool // should the function be memoized? (false if too much output) + Slow bool // is the function slow? (avoid speculative execution) Sig *types.Type // the signature of the function, must be KindFunc Err error // is this a valid function, or was it created improperly? }