lang: Split lang package out into many subpackages

This is a giant refactor to split the giant lang package into many
subpackages. The most difficult piece was figuring out how to extract
the extra ast structs into their own package, because they needed to
call two functions which also needed to import the ast.

The solution was to separate out those functions into their own
packages, and to pass them into the ast at the root when they're needed,
and to let the relevant ast portions call a handle.

This isn't terribly ugly because we already had a giant data struct
woven through the ast.

The bad part is rebasing any WIP work on top of this.

lang/Makefile

@@ -20,23 +20,23 @@ SHELL = /usr/bin/env bash
 
 all: build
 
-build: lexer.nn.go y.go interpolate/parse.generated.go
+build: parser/lexer.nn.go parser/y.go interpolate/parse.generated.go
 	@# recursively run make in child dir named types
 	@$(MAKE) --quiet -C types
 
 clean:
 	$(MAKE) --quiet -C types clean
-	@rm -f lexer.nn.go y.go y.output interpolate/parse.generated.go || true
+	@rm -f parser/lexer.nn.go parser/y.go parser/y.output interpolate/parse.generated.go || true
 
-lexer.nn.go: lexer.nex
+parser/lexer.nn.go: parser/lexer.nex
 	@echo "Generating: lexer..."
-	nex -e lexer.nex
-	@ROOT="$$( cd "$$( dirname "$${BASH_SOURCE[0]}" )" && cd .. && pwd )" && $$ROOT/misc/header.sh 'lexer.nn.go'
+	nex -e -o $@ $<
+	@ROOT="$$( cd "$$( dirname "$${BASH_SOURCE[0]}" )" && cd .. && pwd )" && $$ROOT/misc/header.sh 'parser/lexer.nn.go'
 
-y.go: parser.y
+parser/y.go: parser/parser.y
 	@echo "Generating: parser..."
-	goyacc parser.y
-	@ROOT="$$( cd "$$( dirname "$${BASH_SOURCE[0]}" )" && cd .. && pwd )" && $$ROOT/misc/header.sh 'y.go'
+	goyacc -v parser/y.output -o $@ $<
+	@ROOT="$$( cd "$$( dirname "$${BASH_SOURCE[0]}" )" && cd .. && pwd )" && $$ROOT/misc/header.sh 'parser/y.go'
 
 interpolate/parse.generated.go: interpolate/parse.rl
 	@echo "Generating: interpolation..."

lang/ast/scope_test.go

@@ -17,7 +17,7 @@
 
 // +build !root
 
-package lang
+package ast
 
 import (
 	"fmt"

lang/ast/structs.go

@@ -15,7 +15,7 @@
 // You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package lang // TODO: move this into a sub package of lang/$name?
+package ast
 
 import (
 	"bytes"
@@ -3065,13 +3065,13 @@ func (obj *StmtProg) importSystemScope(name string) (*interfaces.Scope, error) {
 
 	isEmpty := true // assume empty (which should cause an error)
 
-	funcs := FuncPrefixToFunctionsScope(name) // runs funcs.LookupPrefix
-	if len(funcs) > 0 {
+	functions := FuncPrefixToFunctionsScope(name) // runs funcs.LookupPrefix
+	if len(functions) > 0 {
 		isEmpty = false
 	}
 
 	// perform any normal "startup" for these functions...
-	for _, fn := range funcs {
+	for _, fn := range functions {
 		// XXX: is this the right place for this, or should it be elsewhere?
 		// XXX: do we need a modified obj.data for this b/c it's in a scope?
 		if err := fn.Init(obj.data); err != nil {
@@ -3084,7 +3084,7 @@ func (obj *StmtProg) importSystemScope(name string) (*interfaces.Scope, error) {
 	scope := &interfaces.Scope{
 		// TODO: we could use the core API for variables somehow...
 		//Variables: make(map[string]interfaces.Expr),
-		Functions: funcs, // map[string]interfaces.Expr
+		Functions: functions, // map[string]interfaces.Expr
 		// TODO: we could add a core API for classes too!
 		//Classes: make(map[string]interfaces.Stmt),
 	}
@@ -3101,7 +3101,7 @@ func (obj *StmtProg) importSystemScope(name string) (*interfaces.Scope, error) {
 	// XXX: consider using a virtual `append *` statement to combine these instead.
 	for _, p := range paths {
 		// we only want code from this prefix
-		prefix := CoreDir + name + "/"
+		prefix := funcs.CoreDir + name + "/"
 		if !strings.HasPrefix(p, prefix) {
 			continue
 		}
@@ -3126,7 +3126,7 @@ func (obj *StmtProg) importSystemScope(name string) (*interfaces.Scope, error) {
 		reader := bytes.NewReader(b) // wrap the byte stream
 
 		// now run the lexer/parser to do the import
-		ast, err := LexParse(reader)
+		ast, err := obj.data.LexParser(reader)
 		if err != nil {
 			return nil, errwrap.Wrapf(err, "could not generate AST from import `%s`", name)
 		}
@@ -3240,7 +3240,7 @@ func (obj *StmtProg) importScopeWithInputs(s string, scope *interfaces.Scope, pa
 	metadata.Metadata = obj.data.Metadata
 
 	// now run the lexer/parser to do the import
-	ast, err := LexParse(reader)
+	ast, err := obj.data.LexParser(reader)
 	if err != nil {
 		return nil, errwrap.Wrapf(err, "could not generate AST from import")
 	}
@@ -3252,15 +3252,18 @@ func (obj *StmtProg) importScopeWithInputs(s string, scope *interfaces.Scope, pa
 	// init and validate the structure of the AST
 	data := &interfaces.Data{
 		// TODO: add missing fields here if/when needed
-		Fs:         obj.data.Fs,
-		FsURI:      obj.data.FsURI,
-		Base:       output.Base, // new base dir (absolute path)
-		Files:      files,
-		Imports:    parentVertex, // the parent vertex that imported me
-		Metadata:   metadata,
-		Modules:    obj.data.Modules,
-		Downloader: obj.data.Downloader,
-		//World: obj.data.World,
+		Fs:       obj.data.Fs,
+		FsURI:    obj.data.FsURI,
+		Base:     output.Base, // new base dir (absolute path)
+		Files:    files,
+		Imports:  parentVertex, // the parent vertex that imported me
+		Metadata: metadata,
+		Modules:  obj.data.Modules,
+
+		LexParser:       obj.data.LexParser,
+		Downloader:      obj.data.Downloader,
+		StrInterpolater: obj.data.StrInterpolater,
+		//World: obj.data.World, // TODO: do we need this?
 
 		//Prefix: obj.Prefix, // TODO: add a path on?
 		Debug: obj.data.Debug,
@@ -3359,7 +3362,7 @@ func (obj *StmtProg) SetScope(scope *interfaces.Scope) error {
 			return fmt.Errorf("import `%s` already exists in this scope", imp.Name)
 		}
 
-		result, err := ParseImportName(imp.Name)
+		result, err := langutil.ParseImportName(imp.Name)
 		if err != nil {
 			return errwrap.Wrapf(err, "import `%s` is not valid", imp.Name)
 		}
@@ -3447,16 +3450,16 @@ func (obj *StmtProg) SetScope(scope *interfaces.Scope) error {
 	}
 
 	// now collect all the functions, and group by name (if polyfunc is ok)
-	funcs := make(map[string][]*StmtFunc)
+	functions := make(map[string][]*StmtFunc)
 	for _, x := range obj.Body {
 		fn, ok := x.(*StmtFunc)
 		if !ok {
 			continue
 		}
 
-		_, exists := funcs[fn.Name]
+		_, exists := functions[fn.Name]
 		if !exists {
-			funcs[fn.Name] = []*StmtFunc{} // initialize
+			functions[fn.Name] = []*StmtFunc{} // initialize
 		}
 
 		// check for duplicates *in this scope*
@@ -3464,11 +3467,11 @@ func (obj *StmtProg) SetScope(scope *interfaces.Scope) error {
 			return fmt.Errorf("func `%s` already exists in this scope", fn.Name)
 		}
 
-		// collect funcs (if multiple, this is a polyfunc)
-		funcs[fn.Name] = append(funcs[fn.Name], fn)
+		// collect functions (if multiple, this is a polyfunc)
+		functions[fn.Name] = append(functions[fn.Name], fn)
 	}
 
-	for name, fnList := range funcs {
+	for name, fnList := range functions {
 		if obj.data.Debug { // TODO: is this message ever useful?
 			obj.data.Logf("prog: set scope: collect: (%+v -> %d): %+v (%T)", name, len(fnList), fnList[0].Func, fnList[0].Func)
 		}
@@ -4816,7 +4819,7 @@ func (obj *ExprStr) Init(data *interfaces.Data) error {
 // which need interpolation. If any are found, it returns a larger AST which has
 // a function which returns a string as its root. Otherwise it returns itself.
 func (obj *ExprStr) Interpolate() (interfaces.Expr, error) {
-	pos := &Pos{
+	pos := &interfaces.Pos{
 		// column/line number, starting at 1
 		//Column: -1, // TODO
 		//Line: -1, // TODO
@@ -4825,22 +4828,27 @@ func (obj *ExprStr) Interpolate() (interfaces.Expr, error) {
 
 	data := &interfaces.Data{
 		// TODO: add missing fields here if/when needed
-		Fs:         obj.data.Fs,
-		FsURI:      obj.data.FsURI,
-		Base:       obj.data.Base,
-		Files:      obj.data.Files,
-		Imports:    obj.data.Imports,
-		Metadata:   obj.data.Metadata,
-		Modules:    obj.data.Modules,
-		Downloader: obj.data.Downloader,
-		//World:      obj.data.World,
+		Fs:       obj.data.Fs,
+		FsURI:    obj.data.FsURI,
+		Base:     obj.data.Base,
+		Files:    obj.data.Files,
+		Imports:  obj.data.Imports,
+		Metadata: obj.data.Metadata,
+		Modules:  obj.data.Modules,
+
+		LexParser:       obj.data.LexParser,
+		Downloader:      obj.data.Downloader,
+		StrInterpolater: obj.data.StrInterpolater,
+		//World: obj.data.World, // TODO: do we need this?
+
 		Prefix: obj.data.Prefix,
 		Debug:  obj.data.Debug,
 		Logf: func(format string, v ...interface{}) {
 			obj.data.Logf("interpolate: "+format, v...)
 		},
 	}
-	result, err := InterpolateStr(obj.V, pos, data)
+
+	result, err := obj.data.StrInterpolater(obj.V, pos, data)
 	if err != nil {
 		return nil, err
 	}

lang/ast/util.go

@@ -15,7 +15,7 @@
 // You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package lang
+package ast
 
 import (
 	"fmt"
@@ -27,6 +27,7 @@ import (
 	"github.com/purpleidea/mgmt/lang/funcs/vars"
 	"github.com/purpleidea/mgmt/lang/interfaces"
 	"github.com/purpleidea/mgmt/lang/types"
+	"github.com/purpleidea/mgmt/util/errwrap"
 )
 
 // FuncPrefixToFunctionsScope is a helper function to return the functions
@@ -215,3 +216,29 @@ func ValueToExpr(val types.Value) (interfaces.Expr, error) {
 
 	return expr, expr.SetType(val.Type())
 }
+
+// CollectFiles collects all the files used in the AST. You will see more files
+// based on how many compiling steps have run. In general, this is useful for
+// collecting all the files needed to store in our file system for a deploy.
+func CollectFiles(ast interfaces.Stmt) ([]string, error) {
+	// collect the list of files
+	fileList := []string{}
+	fn := func(node interfaces.Node) error {
+		// redundant check for example purposes
+		stmt, ok := node.(interfaces.Stmt)
+		if !ok {
+			return nil
+		}
+		body, ok := stmt.(*StmtProg)
+		if !ok {
+			return nil
+		}
+		// collect into global
+		fileList = append(fileList, body.importFiles...)
+		return nil
+	}
+	if err := ast.Apply(fn); err != nil {
+		return nil, errwrap.Wrapf(err, "can't retrieve paths")
+	}
+	return fileList, nil
+}

lang/funcs/funcs.go

@@ -40,6 +40,9 @@ const (
 	// we don't want to allow this as the first or last character in a name.
 	// NOTE: the template library will panic if it is one of: .-#
 	ReplaceChar = "_"
+
+	// CoreDir is the directory prefix where core bindata mcl code is added.
+	CoreDir = "core/"
 )
 
 // registeredFuncs is a global map of all possible funcs which can be used. You

lang/fuzz/fuzz.go

@@ -20,7 +20,7 @@ package fuzz
 import (
 	"bytes"
 
-	"github.com/purpleidea/mgmt/lang"
+	"github.com/purpleidea/mgmt/lang/parser"
 )
 
 // Fuzz is repeatedly called by go-fuzz with semi-random inputs in an attempt to
@@ -32,7 +32,7 @@ import (
 // gives new coverage; and 0 otherwise; other values are reserved for future
 // use.
 func Fuzz(data []byte) int {
-	ast, err := lang.LexParse(bytes.NewReader(data))
+	ast, err := parser.LexParse(bytes.NewReader(data))
 	if err != nil {
 		if ast != nil {
 			panic("ast != nil on error")

lang/gapi/gapi.go

@@ -25,10 +25,13 @@ import (
 
 	"github.com/purpleidea/mgmt/gapi"
 	"github.com/purpleidea/mgmt/lang"
+	"github.com/purpleidea/mgmt/lang/ast"
 	"github.com/purpleidea/mgmt/lang/download"
 	"github.com/purpleidea/mgmt/lang/funcs/vars"
 	"github.com/purpleidea/mgmt/lang/inputs"
 	"github.com/purpleidea/mgmt/lang/interfaces"
+	"github.com/purpleidea/mgmt/lang/interpolate"
+	"github.com/purpleidea/mgmt/lang/parser"
 	"github.com/purpleidea/mgmt/lang/unification"
 	"github.com/purpleidea/mgmt/pgraph"
 	"github.com/purpleidea/mgmt/util"
@@ -191,12 +194,12 @@ func (obj *GAPI) Cli(cliInfo *gapi.CliInfo) (*gapi.Deploy, error) {
 	// TODO: do the paths need to be cleaned for "../" before comparison?
 
 	logf("lexing/parsing...")
-	ast, err := lang.LexParse(bytes.NewReader(output.Main))
+	xast, err := parser.LexParse(bytes.NewReader(output.Main))
 	if err != nil {
 		return nil, errwrap.Wrapf(err, "could not generate AST")
 	}
 	if debug {
-		logf("behold, the AST: %+v", ast)
+		logf("behold, the AST: %+v", xast)
 	}
 
 	var downloader interfaces.Downloader
@@ -239,16 +242,19 @@ func (obj *GAPI) Cli(cliInfo *gapi.CliInfo) (*gapi.Deploy, error) {
 	// init and validate the structure of the AST
 	data := &interfaces.Data{
 		// TODO: add missing fields here if/when needed
-		Fs:         localFs,       // the local fs!
-		FsURI:      localFs.URI(), // TODO: is this right?
-		Base:       output.Base,   // base dir (absolute path) that this is rooted in
-		Files:      output.Files,
-		Imports:    importVertex,
-		Metadata:   output.Metadata,
-		Modules:    modules,
-		Downloader: downloader,
+		Fs:       localFs,       // the local fs!
+		FsURI:    localFs.URI(), // TODO: is this right?
+		Base:     output.Base,   // base dir (absolute path) that this is rooted in
+		Files:    output.Files,
+		Imports:  importVertex,
+		Metadata: output.Metadata,
+		Modules:  modules,
 
+		LexParser:       parser.LexParse,
+		Downloader:      downloader,
+		StrInterpolater: interpolate.InterpolateStr,
 		//World: obj.World, // TODO: do we need this?
+
 		Prefix: prefix,
 		Debug:  debug,
 		Logf: func(format string, v ...interface{}) {
@@ -257,25 +263,25 @@ func (obj *GAPI) Cli(cliInfo *gapi.CliInfo) (*gapi.Deploy, error) {
 		},
 	}
 	// some of this might happen *after* interpolate in SetScope or Unify...
-	if err := ast.Init(data); err != nil {
+	if err := xast.Init(data); err != nil {
 		return nil, errwrap.Wrapf(err, "could not init and validate AST")
 	}
 
 	logf("interpolating...")
 	// interpolate strings and other expansionable nodes in AST
-	interpolated, err := ast.Interpolate()
+	interpolated, err := xast.Interpolate()
 	if err != nil {
 		return nil, errwrap.Wrapf(err, "could not interpolate AST")
 	}
 
 	variables := map[string]interfaces.Expr{
-		"purpleidea": &lang.ExprStr{V: "hello world!"}, // james says hi
+		"purpleidea": &ast.ExprStr{V: "hello world!"}, // james says hi
 		// TODO: change to a func when we can change hostname dynamically!
-		"hostname": &lang.ExprStr{V: ""}, // NOTE: empty b/c not used
+		"hostname": &ast.ExprStr{V: ""}, // NOTE: empty b/c not used
 	}
-	consts := lang.VarPrefixToVariablesScope(vars.ConstNamespace) // strips prefix!
-	addback := vars.ConstNamespace + interfaces.ModuleSep         // add it back...
-	variables, err = lang.MergeExprMaps(variables, consts, addback)
+	consts := ast.VarPrefixToVariablesScope(vars.ConstNamespace) // strips prefix!
+	addback := vars.ConstNamespace + interfaces.ModuleSep        // add it back...
+	variables, err = ast.MergeExprMaps(variables, consts, addback)
 	if err != nil {
 		return nil, errwrap.Wrapf(err, "couldn't merge in consts")
 	}
@@ -284,7 +290,7 @@ func (obj *GAPI) Cli(cliInfo *gapi.CliInfo) (*gapi.Deploy, error) {
 	scope := &interfaces.Scope{
 		Variables: variables,
 		// all the built-in top-level, core functions enter here...
-		Functions: lang.FuncPrefixToFunctionsScope(""), // runs funcs.LookupPrefix
+		Functions: ast.FuncPrefixToFunctionsScope(""), // runs funcs.LookupPrefix
 	}
 
 	logf("building scope...")
@@ -316,7 +322,7 @@ func (obj *GAPI) Cli(cliInfo *gapi.CliInfo) (*gapi.Deploy, error) {
 	}
 
 	// get the list of needed files (this is available after SetScope)
-	fileList, err := lang.CollectFiles(interpolated)
+	fileList, err := ast.CollectFiles(interpolated)
 	if err != nil {
 		return nil, errwrap.Wrapf(err, "could not collect files")
 	}
@@ -664,7 +670,7 @@ func (obj *GAPI) Get(getInfo *gapi.GetInfo) error {
 	// TODO: do the paths need to be cleaned for "../" before comparison?
 
 	logf("lexing/parsing...")
-	ast, err := lang.LexParse(bytes.NewReader(output.Main))
+	ast, err := parser.LexParse(bytes.NewReader(output.Main))
 	if err != nil {
 		return errwrap.Wrapf(err, "could not generate AST")
 	}
@@ -709,16 +715,19 @@ func (obj *GAPI) Get(getInfo *gapi.GetInfo) error {
 	// init and validate the structure of the AST
 	data := &interfaces.Data{
 		// TODO: add missing fields here if/when needed
-		Fs:         localFs,       // the local fs!
-		FsURI:      localFs.URI(), // TODO: is this right?
-		Base:       output.Base,   // base dir (absolute path) that this is rooted in
-		Files:      output.Files,
-		Imports:    importVertex,
-		Metadata:   output.Metadata,
-		Modules:    modules,
-		Downloader: downloader,
+		Fs:       localFs,       // the local fs!
+		FsURI:    localFs.URI(), // TODO: is this right?
+		Base:     output.Base,   // base dir (absolute path) that this is rooted in
+		Files:    output.Files,
+		Imports:  importVertex,
+		Metadata: output.Metadata,
+		Modules:  modules,
 
+		LexParser:       parser.LexParse,
+		Downloader:      downloader,
+		StrInterpolater: interpolate.InterpolateStr,
 		//World: obj.World, // TODO: do we need this?
+
 		Prefix: prefix,
 		Debug:  debug,
 		Logf: func(format string, v ...interface{}) {

lang/interfaces/ast.go

@@ -19,6 +19,7 @@ package interfaces
 
 import (
 	"fmt"
+	"io"
 	"sort"
 
 	"github.com/purpleidea/mgmt/engine"
@@ -186,6 +187,16 @@ type Data struct {
 	// deploys, however that is not blocked at the level of this interface.
 	Downloader Downloader
 
+	// LexParser is a function that needs to get passed in to run the lexer
+	// and parser to build the initial AST. This is passed in this way to
+	// avoid dependency cycles.
+	LexParser func(io.Reader) (Stmt, error)
+
+	// StrInterpolater is a function that needs to get passed in to run the
+	// string interpolation. This is passed in this way to avoid dependency
+	// cycles.
+	StrInterpolater func(string, *Pos, *Data) (Expr, error)
+
 	//World engine.World // TODO: do we need this?
 
 	// Prefix provides a unique path prefix that we can namespace in. It is

lang/interfaces/parser.go

@@ -0,0 +1,27 @@
+// Mgmt
+// Copyright (C) 2013-2021+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package interfaces
+
+// Pos represents a position in the code. This is used by the parser and string
+// interpolation.
+// TODO: consider expanding with range characteristics.
+type Pos struct {
+	Line     int    // line number starting at 1
+	Column   int    // column number starting at 1
+	Filename string // optional source filename, if known
+}

lang/interpolate/interpolate.go

@@ -15,13 +15,14 @@
 // You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package lang // TODO: move this into a sub package of lang/$name?
+package interpolate
 
 import (
 	"fmt"
 
+	"github.com/purpleidea/mgmt/lang/ast"
+	"github.com/purpleidea/mgmt/lang/funcs"
 	"github.com/purpleidea/mgmt/lang/interfaces"
-	"github.com/purpleidea/mgmt/lang/interpolate"
 	"github.com/purpleidea/mgmt/util/errwrap"
 
 	"github.com/hashicorp/hil"
@@ -35,16 +36,8 @@ const (
 	UseHilInterpolation = false
 )
 
-// Pos represents a position in the code.
-// TODO: consider expanding with range characteristics.
-type Pos struct {
-	Line     int    // line number starting at 1
-	Column   int    // column number starting at 1
-	Filename string // optional source filename, if known
-}
-
 // InterpolateStr interpolates a string and returns the representative AST.
-func InterpolateStr(str string, pos *Pos, data *interfaces.Data) (interfaces.Expr, error) {
+func InterpolateStr(str string, pos *interfaces.Pos, data *interfaces.Data) (interfaces.Expr, error) {
 	if data.Debug {
 		data.Logf("interpolating: %s", str)
 	}
@@ -57,8 +50,8 @@ func InterpolateStr(str string, pos *Pos, data *interfaces.Data) (interfaces.Exp
 
 // InterpolateRagel interpolates a string and returns the representative AST. It
 // uses the ragel parser to perform the string interpolation.
-func InterpolateRagel(str string, pos *Pos, data *interfaces.Data) (interfaces.Expr, error) {
-	sequence, err := interpolate.Parse(str)
+func InterpolateRagel(str string, pos *interfaces.Pos, data *interfaces.Data) (interfaces.Expr, error) {
+	sequence, err := Parse(str)
 	if err != nil {
 		return nil, errwrap.Wrapf(err, "parser failed")
 	}
@@ -67,14 +60,14 @@ func InterpolateRagel(str string, pos *Pos, data *interfaces.Data) (interfaces.E
 	for _, term := range sequence {
 
 		switch t := term.(type) {
-		case interpolate.Literal:
-			expr := &ExprStr{
+		case Literal:
+			expr := &ast.ExprStr{
 				V: t.Value,
 			}
 			exprs = append(exprs, expr)
 
-		case interpolate.Variable:
-			expr := &ExprVar{
+		case Variable:
+			expr := &ast.ExprVar{
 				Name: t.Name,
 			}
 			exprs = append(exprs, expr)
@@ -85,7 +78,7 @@ func InterpolateRagel(str string, pos *Pos, data *interfaces.Data) (interfaces.E
 
 	// If we didn't find anything of value, we got an empty string...
 	if len(sequence) == 0 && str == "" { // be doubly sure...
-		expr := &ExprStr{
+		expr := &ast.ExprStr{
 			V: "",
 		}
 		exprs = append(exprs, expr)
@@ -108,7 +101,7 @@ func InterpolateRagel(str string, pos *Pos, data *interfaces.Data) (interfaces.E
 
 // InterpolateHil interpolates a string and returns the representative AST. This
 // particular implementation uses the hashicorp hil library and syntax to do so.
-func InterpolateHil(str string, pos *Pos, data *interfaces.Data) (interfaces.Expr, error) {
+func InterpolateHil(str string, pos *interfaces.Pos, data *interfaces.Data) (interfaces.Expr, error) {
 	var line, column int = -1, -1
 	var filename string
 	if pos != nil {
@@ -132,15 +125,19 @@ func InterpolateHil(str string, pos *Pos, data *interfaces.Data) (interfaces.Exp
 
 	transformData := &interfaces.Data{
 		// TODO: add missing fields here if/when needed
-		Fs:         data.Fs,
-		FsURI:      data.FsURI,
-		Base:       data.Base,
-		Files:      data.Files,
-		Imports:    data.Imports,
-		Metadata:   data.Metadata,
-		Modules:    data.Modules,
-		Downloader: data.Downloader,
-		//World:      data.World,
+		Fs:       data.Fs,
+		FsURI:    data.FsURI,
+		Base:     data.Base,
+		Files:    data.Files,
+		Imports:  data.Imports,
+		Metadata: data.Metadata,
+		Modules:  data.Modules,
+
+		LexParser:       data.LexParser,
+		Downloader:      data.Downloader,
+		StrInterpolater: data.StrInterpolater,
+		//World: data.World, // TODO: do we need this?
+
 		Prefix: data.Prefix,
 		Debug:  data.Debug,
 		Logf: func(format string, v ...interface{}) {
@@ -205,7 +202,7 @@ func hilTransform(root hilast.Node, data *interfaces.Data) (interfaces.Expr, err
 			args = append(args, arg)
 		}
 
-		return &ExprCall{
+		return &ast.ExprCall{
 			Name: node.Func, // name
 			Args: args,
 		}, nil
@@ -217,23 +214,23 @@ func hilTransform(root hilast.Node, data *interfaces.Data) (interfaces.Expr, err
 
 		switch node.Typex {
 		case hilast.TypeBool:
-			return &ExprBool{
+			return &ast.ExprBool{
 				V: node.Value.(bool),
 			}, nil
 
 		case hilast.TypeString:
-			return &ExprStr{
+			return &ast.ExprStr{
 				V: node.Value.(string),
 			}, nil
 
 		case hilast.TypeInt:
-			return &ExprInt{
+			return &ast.ExprInt{
 				// node.Value is an int stored as an interface
 				V: int64(node.Value.(int)),
 			}, nil
 
 		case hilast.TypeFloat:
-			return &ExprFloat{
+			return &ast.ExprFloat{
 				V: node.Value.(float64),
 			}, nil
 
@@ -249,7 +246,7 @@ func hilTransform(root hilast.Node, data *interfaces.Data) (interfaces.Expr, err
 		if data.Debug {
 			data.Logf("got variable access type: %+v", node)
 		}
-		return &ExprVar{
+		return &ast.ExprVar{
 			Name: node.Name,
 		}, nil
 
@@ -284,7 +281,7 @@ func concatExprListIntoCall(exprs []interfaces.Expr) (interfaces.Expr, error) {
 		return nil, fmt.Errorf("empty list")
 	}
 
-	operator := &ExprStr{
+	operator := &ast.ExprStr{
 		V: "+", // for PLUS this is a `+` character
 	}
 
@@ -293,11 +290,11 @@ func concatExprListIntoCall(exprs []interfaces.Expr) (interfaces.Expr, error) {
 	}
 	//if len(exprs) == 1 {
 	//	arg := exprs[0]
-	//	emptyStr := &ExprStr{
+	//	emptyStr := &ast.ExprStr{
 	//		V: "", // empty str
 	//	}
-	//	return &ExprCall{
-	//		Name: operatorFuncName, // concatenate the two strings with + operator
+	//	return &ast.ExprCall{
+	//		Name: funcs.OperatorFuncName, // concatenate the two strings with + operator
 	//		Args: []interfaces.Expr{
 	//			operator, // operator first
 	//			arg,      // string arg
@@ -313,9 +310,9 @@ func concatExprListIntoCall(exprs []interfaces.Expr) (interfaces.Expr, error) {
 		return nil, err
 	}
 
-	return &ExprCall{
+	return &ast.ExprCall{
 		// NOTE: if we don't set the data field we need Init() called on it!
-		Name: operatorFuncName, // concatenate the two strings with + operator
+		Name: funcs.OperatorFuncName, // concatenate the two strings with + operator
 		Args: []interfaces.Expr{
 			operator, // operator first
 			head,     // string arg
@@ -333,7 +330,7 @@ func simplifyExprList(exprs []interfaces.Expr) ([]interfaces.Expr, error) {
 
 	for _, x := range exprs {
 		switch v := x.(type) {
-		case *ExprStr:
+		case *ast.ExprStr:
 			if !last {
 				last = true
 				result = append(result, x)
@@ -342,7 +339,7 @@ func simplifyExprList(exprs []interfaces.Expr) ([]interfaces.Expr, error) {
 
 			// combine!
 			expr := result[len(result)-1] // there has to be at least one
-			str, ok := expr.(*ExprStr)
+			str, ok := expr.(*ast.ExprStr)
 			if !ok {
 				// programming error
 				return nil, fmt.Errorf("unexpected type (%T)", expr)
@@ -351,7 +348,7 @@ func simplifyExprList(exprs []interfaces.Expr) ([]interfaces.Expr, error) {
 			//last = true // redundant, it's already true
 			// ... and don't append, we've combined!
 
-		case *ExprVar:
+		case *ast.ExprVar:
 			last = false // the next one can't combine with me
 			result = append(result, x)
 

lang/interpolate/interpolate_test.go

@@ -17,7 +17,7 @@
 
 // +build !root
 
-package lang
+package interpolate
 
 import (
 	"fmt"
@@ -25,7 +25,10 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/purpleidea/mgmt/lang/ast"
+	"github.com/purpleidea/mgmt/lang/funcs"
 	"github.com/purpleidea/mgmt/lang/interfaces"
+	"github.com/purpleidea/mgmt/lang/parser"
 	"github.com/purpleidea/mgmt/util"
 
 	"github.com/davecgh/go-spew/spew"
@@ -45,28 +48,28 @@ func TestInterpolate0(t *testing.T) {
 	// names, and then run `go test -run <pattern>` with the name(s) to run.
 
 	{
-		ast := &StmtProg{
+		xast := &ast.StmtProg{
 			Body: []interfaces.Stmt{},
 		}
 		testCases = append(testCases, test{ // 0
 			"nil",
 			``,
 			false,
-			ast,
+			xast,
 		})
 	}
 	{
-		ast := &StmtProg{
+		xast := &ast.StmtProg{
 			Body: []interfaces.Stmt{
-				&StmtRes{
+				&ast.StmtRes{
 					Kind: "test",
-					Name: &ExprStr{
+					Name: &ast.ExprStr{
 						V: "t1",
 					},
-					Contents: []StmtResContents{
-						&StmtResField{
+					Contents: []ast.StmtResContents{
+						&ast.StmtResField{
 							Field: "stringptr",
-							Value: &ExprStr{
+							Value: &ast.ExprStr{
 								V: "foo",
 							},
 						},
@@ -82,33 +85,33 @@ func TestInterpolate0(t *testing.T) {
 			}
 			`,
 			fail: false,
-			ast:  ast,
+			ast:  xast,
 		})
 	}
 	{
-		fieldName := &ExprCall{
-			Name: operatorFuncName,
+		fieldName := &ast.ExprCall{
+			Name: funcs.OperatorFuncName,
 			Args: []interfaces.Expr{
-				&ExprStr{
+				&ast.ExprStr{
 					V: "+",
 				},
-				&ExprStr{
+				&ast.ExprStr{
 					V: "foo-",
 				},
-				&ExprVar{
+				&ast.ExprVar{
 					Name: "x",
 				},
 			},
 		}
-		ast := &StmtProg{
+		xast := &ast.StmtProg{
 			Body: []interfaces.Stmt{
-				&StmtRes{
+				&ast.StmtRes{
 					Kind: "test",
-					Name: &ExprStr{
+					Name: &ast.ExprStr{
 						V: "t1",
 					},
-					Contents: []StmtResContents{
-						&StmtResField{
+					Contents: []ast.StmtResContents{
+						&ast.StmtResField{
 							Field: "stringptr",
 							Value: fieldName,
 						},
@@ -125,21 +128,21 @@ func TestInterpolate0(t *testing.T) {
 			}
 			`,
 			fail: false,
-			ast:  ast,
+			ast:  xast,
 		})
 	}
 	{
-		ast := &StmtProg{
+		xast := &ast.StmtProg{
 			Body: []interfaces.Stmt{
-				&StmtRes{
+				&ast.StmtRes{
 					Kind: "test",
-					Name: &ExprStr{
+					Name: &ast.ExprStr{
 						V: "t1",
 					},
-					Contents: []StmtResContents{
-						&StmtResField{
+					Contents: []ast.StmtResContents{
+						&ast.StmtResField{
 							Field: "stringptr",
-							Value: &ExprStr{
+							Value: &ast.ExprStr{
 								V: "${hello}",
 							},
 						},
@@ -155,21 +158,21 @@ func TestInterpolate0(t *testing.T) {
 			}
 			`,
 			fail: false,
-			ast:  ast,
+			ast:  xast,
 		})
 	}
 	{
-		ast := &StmtProg{
+		xast := &ast.StmtProg{
 			Body: []interfaces.Stmt{
-				&StmtRes{
+				&ast.StmtRes{
 					Kind: "test",
-					Name: &ExprStr{
+					Name: &ast.ExprStr{
 						V: "t1",
 					},
-					Contents: []StmtResContents{
-						&StmtResField{
+					Contents: []ast.StmtResContents{
+						&ast.StmtResField{
 							Field: "stringptr",
-							Value: &ExprStr{
+							Value: &ast.ExprStr{
 								V: `\` + `$` + `{hello}`,
 							},
 						},
@@ -185,7 +188,7 @@ func TestInterpolate0(t *testing.T) {
 			}
 			`,
 			fail: false,
-			ast:  ast,
+			ast:  xast,
 		})
 	}
 
@@ -204,7 +207,7 @@ func TestInterpolate0(t *testing.T) {
 			code, fail, exp := tc.code, tc.fail, tc.ast
 
 			str := strings.NewReader(code)
-			ast, err := LexParse(str)
+			ast, err := parser.LexParse(str)
 			if err != nil {
 				t.Errorf("test #%d: FAIL", index)
 				t.Errorf("test #%d: lex/parse failed with: %+v", index, err)
@@ -213,6 +216,9 @@ func TestInterpolate0(t *testing.T) {
 			t.Logf("test #%d: AST: %+v", index, ast)
 
 			data := &interfaces.Data{
+				// TODO: add missing fields here if/when needed
+				StrInterpolater: InterpolateStr,
+
 				Debug: testing.Verbose(), // set via the -test.v flag to `go test`
 				Logf: func(format string, v ...interface{}) {
 					t.Logf("ast: "+format, v...)
@@ -296,17 +302,17 @@ func TestInterpolateBasicStmt(t *testing.T) {
 	//	})
 	//}
 	{
-		ast := &StmtProg{
+		xast := &ast.StmtProg{
 			Body: []interfaces.Stmt{
-				&StmtRes{
+				&ast.StmtRes{
 					Kind: "test",
-					Name: &ExprStr{
+					Name: &ast.ExprStr{
 						V: "t1",
 					},
-					Contents: []StmtResContents{
-						&StmtResField{
+					Contents: []ast.StmtResContents{
+						&ast.StmtResField{
 							Field: "stringptr",
-							Value: &ExprStr{
+							Value: &ast.ExprStr{
 								V: "foo",
 							},
 						},
@@ -314,17 +320,17 @@ func TestInterpolateBasicStmt(t *testing.T) {
 				},
 			},
 		}
-		exp := &StmtProg{
+		exp := &ast.StmtProg{
 			Body: []interfaces.Stmt{
-				&StmtRes{
+				&ast.StmtRes{
 					Kind: "test",
-					Name: &ExprStr{
+					Name: &ast.ExprStr{
 						V: "t1",
 					},
-					Contents: []StmtResContents{
-						&StmtResField{
+					Contents: []ast.StmtResContents{
+						&ast.StmtResField{
 							Field: "stringptr",
-							Value: &ExprStr{
+							Value: &ast.ExprStr{
 								V: "foo",
 							},
 						},
@@ -334,23 +340,23 @@ func TestInterpolateBasicStmt(t *testing.T) {
 		}
 		testCases = append(testCases, test{
 			name: "basic resource",
-			ast:  ast,
+			ast:  xast,
 			fail: false,
 			exp:  exp,
 		})
 	}
 	{
-		ast := &StmtProg{
+		xast := &ast.StmtProg{
 			Body: []interfaces.Stmt{
-				&StmtRes{
+				&ast.StmtRes{
 					Kind: "test",
-					Name: &ExprStr{
+					Name: &ast.ExprStr{
 						V: "t${blah}",
 					},
-					Contents: []StmtResContents{
-						&StmtResField{
+					Contents: []ast.StmtResContents{
+						&ast.StmtResField{
 							Field: "stringptr",
-							Value: &ExprStr{
+							Value: &ast.ExprStr{
 								V: "foo",
 							},
 						},
@@ -358,29 +364,29 @@ func TestInterpolateBasicStmt(t *testing.T) {
 				},
 			},
 		}
-		resName := &ExprCall{
-			Name: operatorFuncName,
+		resName := &ast.ExprCall{
+			Name: funcs.OperatorFuncName,
 			Args: []interfaces.Expr{
-				&ExprStr{
+				&ast.ExprStr{
 					V: "+",
 				},
-				&ExprStr{
+				&ast.ExprStr{
 					V: "t",
 				},
-				&ExprVar{
+				&ast.ExprVar{
 					Name: "blah",
 				},
 			},
 		}
-		exp := &StmtProg{
+		exp := &ast.StmtProg{
 			Body: []interfaces.Stmt{
-				&StmtRes{
+				&ast.StmtRes{
 					Kind: "test",
 					Name: resName,
-					Contents: []StmtResContents{
-						&StmtResField{
+					Contents: []ast.StmtResContents{
+						&ast.StmtResField{
 							Field: "stringptr",
-							Value: &ExprStr{
+							Value: &ast.ExprStr{
 								V: "foo",
 							},
 						},
@@ -390,23 +396,23 @@ func TestInterpolateBasicStmt(t *testing.T) {
 		}
 		testCases = append(testCases, test{
 			name: "expanded resource",
-			ast:  ast,
+			ast:  xast,
 			fail: false,
 			exp:  exp,
 		})
 	}
 	{
-		ast := &StmtProg{
+		xast := &ast.StmtProg{
 			Body: []interfaces.Stmt{
-				&StmtRes{
+				&ast.StmtRes{
 					Kind: "test",
-					Name: &ExprStr{
+					Name: &ast.ExprStr{
 						V: "t${42}", // incorrect type
 					},
-					Contents: []StmtResContents{
-						&StmtResField{
+					Contents: []ast.StmtResContents{
+						&ast.StmtResField{
 							Field: "stringptr",
-							Value: &ExprStr{
+							Value: &ast.ExprStr{
 								V: "foo",
 							},
 						},
@@ -414,30 +420,30 @@ func TestInterpolateBasicStmt(t *testing.T) {
 				},
 			},
 		}
-		resName := &ExprCall{
-			Name: operatorFuncName,
+		resName := &ast.ExprCall{
+			Name: funcs.OperatorFuncName,
 			// incorrect sig for this function, and now invalid interpolation
 			Args: []interfaces.Expr{
-				&ExprStr{
+				&ast.ExprStr{
 					V: "+",
 				},
-				&ExprStr{
+				&ast.ExprStr{
 					V: "t",
 				},
-				&ExprInt{
+				&ast.ExprInt{
 					V: 42,
 				},
 			},
 		}
-		exp := &StmtProg{
+		exp := &ast.StmtProg{
 			Body: []interfaces.Stmt{
-				&StmtRes{
+				&ast.StmtRes{
 					Kind: "test",
 					Name: resName,
-					Contents: []StmtResContents{
-						&StmtResField{
+					Contents: []ast.StmtResContents{
+						&ast.StmtResField{
 							Field: "stringptr",
-							Value: &ExprStr{
+							Value: &ast.ExprStr{
 								V: "foo",
 							},
 						},
@@ -448,7 +454,7 @@ func TestInterpolateBasicStmt(t *testing.T) {
 		_ = exp // historical
 		testCases = append(testCases, test{
 			name: "expanded invalid resource name",
-			ast:  ast,
+			ast:  xast,
 			fail: true,
 			//exp:  exp,
 		})
@@ -466,6 +472,8 @@ func TestInterpolateBasicStmt(t *testing.T) {
 
 			data := &interfaces.Data{
 				// TODO: add missing fields here if/when needed
+				StrInterpolater: InterpolateStr,
+
 				Debug: testing.Verbose(), // set via the -test.v flag to `go test`
 				Logf: func(format string, v ...interface{}) {
 					t.Logf("ast: "+format, v...)
@@ -535,51 +543,51 @@ func TestInterpolateBasicExpr(t *testing.T) {
 	//	})
 	//}
 	{
-		ast := &ExprStr{
+		xast := &ast.ExprStr{
 			V: "hello",
 		}
-		exp := &ExprStr{
+		exp := &ast.ExprStr{
 			V: "hello",
 		}
 		testCases = append(testCases, test{
 			name: "basic string",
-			ast:  ast,
+			ast:  xast,
 			fail: false,
 			exp:  exp,
 		})
 	}
 	{
-		ast := &ExprStr{
+		xast := &ast.ExprStr{
 			V: "hello ${person_name}",
 		}
-		exp := &ExprCall{
-			Name: operatorFuncName,
+		exp := &ast.ExprCall{
+			Name: funcs.OperatorFuncName,
 			Args: []interfaces.Expr{
-				&ExprStr{
+				&ast.ExprStr{
 					V: "+",
 				},
-				&ExprStr{
+				&ast.ExprStr{
 					V: "hello ",
 				},
-				&ExprVar{
+				&ast.ExprVar{
 					Name: "person_name",
 				},
 			},
 		}
 		testCases = append(testCases, test{
 			name: "basic expansion",
-			ast:  ast,
+			ast:  xast,
 			fail: false,
 			exp:  exp,
 		})
 	}
 	{
-		ast := &ExprStr{
+		xast := &ast.ExprStr{
 			V: "hello ${x ${y} z}",
 		}
 		testCases = append(testCases, test{
 			name: "invalid expansion",
-			ast:  ast,
+			ast:  xast,
 			fail: true,
 		})
 	}
@@ -587,31 +595,31 @@ func TestInterpolateBasicExpr(t *testing.T) {
 	// library, but are not yet supported by our translation layer, nor do
 	// they necessarily work or make much sense at this point in time...
 	//{
-	//	ast := &ExprStr{
+	//	xast := &ast.ExprStr{
 	//		V: `hello ${func("hello ${var.foo}")}`,
 	//	}
 	//	exp := nil // TODO: add this
 	//	testCases = append(testCases, test{
 	//		name: "double expansion",
-	//		ast:  ast,
+	//		ast:  xast,
 	//		fail: false,
 	//		exp:  exp,
 	//	})
 	//}
 	{
-		ast := &ExprStr{
+		xast := &ast.ExprStr{
 			V: "sweetie${3.14159}", // invalid
 		}
-		exp := &ExprCall{
-			Name: operatorFuncName,
+		exp := &ast.ExprCall{
+			Name: funcs.OperatorFuncName,
 			Args: []interfaces.Expr{
-				&ExprStr{
+				&ast.ExprStr{
 					V: "+",
 				},
-				&ExprStr{
+				&ast.ExprStr{
 					V: "sweetie",
 				},
-				&ExprFloat{
+				&ast.ExprFloat{
 					V: 3.14159,
 				},
 			},
@@ -619,24 +627,24 @@ func TestInterpolateBasicExpr(t *testing.T) {
 		_ = exp // historical
 		testCases = append(testCases, test{
 			name: "float expansion",
-			ast:  ast,
+			ast:  xast,
 			fail: true,
 		})
 	}
 	{
-		ast := &ExprStr{
+		xast := &ast.ExprStr{
 			V: "i am: ${sys.hostname()}",
 		}
-		exp := &ExprCall{
-			Name: operatorFuncName,
+		exp := &ast.ExprCall{
+			Name: funcs.OperatorFuncName,
 			Args: []interfaces.Expr{
-				&ExprStr{
+				&ast.ExprStr{
 					V: "+",
 				},
-				&ExprStr{
+				&ast.ExprStr{
 					V: "i am: ",
 				},
-				&ExprCall{
+				&ast.ExprCall{
 					Name: "sys.hostname",
 					Args: []interfaces.Expr{},
 				},
@@ -645,30 +653,30 @@ func TestInterpolateBasicExpr(t *testing.T) {
 		_ = exp // historical
 		testCases = append(testCases, test{
 			name: "function expansion",
-			ast:  ast,
+			ast:  xast,
 			fail: true,
 		})
 	}
 	{
-		ast := &ExprStr{
+		xast := &ast.ExprStr{
 			V: "i am: ${blah(21, 12.3)}",
 		}
-		exp := &ExprCall{
-			Name: operatorFuncName,
+		exp := &ast.ExprCall{
+			Name: funcs.OperatorFuncName,
 			Args: []interfaces.Expr{
-				&ExprStr{
+				&ast.ExprStr{
 					V: "+",
 				},
-				&ExprStr{
+				&ast.ExprStr{
 					V: "i am: ",
 				},
-				&ExprCall{
+				&ast.ExprCall{
 					Name: "blah",
 					Args: []interfaces.Expr{
-						&ExprInt{
+						&ast.ExprInt{
 							V: 21,
 						},
-						&ExprFloat{
+						&ast.ExprFloat{
 							V: 12.3,
 						},
 					},
@@ -678,31 +686,31 @@ func TestInterpolateBasicExpr(t *testing.T) {
 		_ = exp // historical
 		testCases = append(testCases, test{
 			name: "function expansion arg",
-			ast:  ast,
+			ast:  xast,
 			fail: true,
 		})
 	}
 	// FIXME: i am broken, i don't deal well with negatives for some reason
 	//{
-	//	ast := &ExprStr{
+	//	xast := &ast.ExprStr{
 	//		V: "i am: ${blah(21, -12.3)}",
 	//	}
-	//	exp := &ExprCall{
-	//		Name: operatorFuncName,
+	//	exp := &ast.ExprCall{
+	//		Name: funcs.OperatorFuncName,
 	//		Args: []interfaces.Expr{
-	//			&ExprStr{
+	//			&ast.ExprStr{
 	//				V: "+",
 	//			},
-	//			&ExprStr{
+	//			&ast.ExprStr{
 	//				V: "i am: ",
 	//			},
-	//			&ExprCall{
+	//			&ast.ExprCall{
 	//				Name: "blah",
 	//				Args: []interfaces.Expr{
-	//					&ExprInt{
+	//					&ast.ExprInt{
 	//						V: 21,
 	//					},
-	//					&ExprFloat{
+	//					&ast.ExprFloat{
 	//						V: -12.3,
 	//					},
 	//				},
@@ -711,58 +719,58 @@ func TestInterpolateBasicExpr(t *testing.T) {
 	//	}
 	//	testCases = append(testCases, test{
 	//		name: "function expansion arg negative",
-	//		ast:  ast,
+	//		ast:  xast,
 	//		fail: false,
 	//		exp:  exp,
 	//	})
 	//}
 	// FIXME: i am broken :(
 	//{
-	//	ast := &ExprStr{
+	//	xast := &ast.ExprStr{
 	//		V: "sweetie${-3.14159}", // FIXME: only the negative breaks this
 	//	}
-	//	exp := &ExprCall{
-	//		Name: operatorFuncName,
+	//	exp := &ast.ExprCall{
+	//		Name: funcs.OperatorFuncName,
 	//		Args: []interfaces.Expr{
-	//			&ExprStr{
+	//			&ast.ExprStr{
 	//				V: "+",
 	//			},
-	//			&ExprStr{
+	//			&ast.ExprStr{
 	//				V: "sweetie",
 	//			},
-	//			&ExprFloat{
+	//			&ast.ExprFloat{
 	//				V: -3.14159,
 	//			},
 	//		},
 	//	}
 	//	testCases = append(testCases, test{
 	//		name: "negative float expansion",
-	//		ast:  ast,
+	//		ast:  xast,
 	//		fail: false,
 	//		exp:  exp,
 	//	})
 	//}
 	// FIXME: i am also broken, but less important
 	//{
-	//	ast := &ExprStr{
+	//	xast := &ast.ExprStr{
 	//		V: `i am: ${blah(42, "${foo}")}`,
 	//	}
-	//	exp := &ExprCall{
-	//		Name: operatorFuncName,
+	//	exp := &ast.ExprCall{
+	//		Name: funcs.OperatorFuncName,
 	//		Args: []interfaces.Expr{
-	//			&ExprStr{
+	//			&ast.ExprStr{
 	//				V: "+",
 	//			},
-	//			&ExprStr{
+	//			&ast.ExprStr{
 	//				V: "i am: ",
 	//			},
-	//			&ExprCall{
+	//			&ast.ExprCall{
 	//				Name: "blah",
 	//				Args: []interfaces.Expr{
-	//					&ExprInt{
+	//					&ast.ExprInt{
 	//						V: 42,
 	//					},
-	//					&ExprVar{
+	//					&ast.ExprVar{
 	//						Name: "foo",
 	//					},
 	//				},
@@ -771,7 +779,7 @@ func TestInterpolateBasicExpr(t *testing.T) {
 	//	}
 	//	testCases = append(testCases, test{
 	//		name: "function expansion arg with var",
-	//		ast:  ast,
+	//		ast:  xast,
 	//		fail: false,
 	//		exp:  exp,
 	//	})
@@ -789,6 +797,8 @@ func TestInterpolateBasicExpr(t *testing.T) {
 
 			data := &interfaces.Data{
 				// TODO: add missing fields here if/when needed
+				StrInterpolater: InterpolateStr,
+
 				Debug: testing.Verbose(), // set via the -test.v flag to `go test`
 				Logf: func(format string, v ...interface{}) {
 					t.Logf("ast: "+format, v...)

lang/interpret_test.go

@@ -33,11 +33,14 @@ import (
 	"github.com/purpleidea/mgmt/engine/graph/autoedge"
 	"github.com/purpleidea/mgmt/engine/resources"
 	"github.com/purpleidea/mgmt/etcd"
+	"github.com/purpleidea/mgmt/lang/ast"
 	"github.com/purpleidea/mgmt/lang/funcs"
 	"github.com/purpleidea/mgmt/lang/funcs/vars"
 	"github.com/purpleidea/mgmt/lang/inputs"
 	"github.com/purpleidea/mgmt/lang/interfaces"
+	"github.com/purpleidea/mgmt/lang/interpolate"
 	"github.com/purpleidea/mgmt/lang/interpret"
+	"github.com/purpleidea/mgmt/lang/parser"
 	"github.com/purpleidea/mgmt/lang/unification"
 	"github.com/purpleidea/mgmt/pgraph"
 	"github.com/purpleidea/mgmt/util"
@@ -83,11 +86,11 @@ func (obj *edge) String() string {
 func TestAstFunc0(t *testing.T) {
 	scope := &interfaces.Scope{ // global scope
 		Variables: map[string]interfaces.Expr{
-			"hello":  &ExprStr{V: "world"},
-			"answer": &ExprInt{V: 42},
+			"hello":  &ast.ExprStr{V: "world"},
+			"answer": &ast.ExprInt{V: 42},
 		},
 		// all the built-in top-level, core functions enter here...
-		Functions: FuncPrefixToFunctionsScope(""), // runs funcs.LookupPrefix
+		Functions: ast.FuncPrefixToFunctionsScope(""), // runs funcs.LookupPrefix
 	}
 
 	type test struct { // an individual test
@@ -190,7 +193,7 @@ func TestAstFunc0(t *testing.T) {
 	}
 	{
 		graph, _ := pgraph.NewGraph("g")
-		v1, v2, v3, v4, v5 := vtex(`str("t")`), vtex(`str("+")`), vtex("int(42)"), vtex("int(13)"), vtex(fmt.Sprintf(`call:%s(str("+"), int(42), int(13))`, operatorFuncName))
+		v1, v2, v3, v4, v5 := vtex(`str("t")`), vtex(`str("+")`), vtex("int(42)"), vtex("int(13)"), vtex(fmt.Sprintf(`call:%s(str("+"), int(42), int(13))`, funcs.OperatorFuncName))
 		graph.AddVertex(&v1, &v2, &v3, &v4, &v5)
 		e1, e2, e3 := edge("op"), edge("a"), edge("b")
 		graph.AddEdge(&v2, &v5, &e1)
@@ -212,8 +215,8 @@ func TestAstFunc0(t *testing.T) {
 		graph, _ := pgraph.NewGraph("g")
 		v1, v2, v3 := vtex(`str("t")`), vtex(`str("-")`), vtex(`str("+")`)
 		v4, v5, v6 := vtex("int(42)"), vtex("int(13)"), vtex("int(99)")
-		v7 := vtex(fmt.Sprintf(`call:%s(str("+"), int(42), int(13))`, operatorFuncName))
-		v8 := vtex(fmt.Sprintf(`call:%s(str("-"), call:%s(str("+"), int(42), int(13)), int(99))`, operatorFuncName, operatorFuncName))
+		v7 := vtex(fmt.Sprintf(`call:%s(str("+"), int(42), int(13))`, funcs.OperatorFuncName))
+		v8 := vtex(fmt.Sprintf(`call:%s(str("-"), call:%s(str("+"), int(42), int(13)), int(99))`, funcs.OperatorFuncName, funcs.OperatorFuncName))
 
 		graph.AddVertex(&v1, &v2, &v3, &v4, &v5, &v6, &v7, &v8)
 		e1, e2, e3 := edge("op"), edge("a"), edge("b")
@@ -242,7 +245,7 @@ func TestAstFunc0(t *testing.T) {
 		v1, v2 := vtex("bool(true)"), vtex(`str("t")`)
 		v3, v4 := vtex("int(13)"), vtex("int(42)")
 		v5, v6 := vtex("var(i)"), vtex("var(x)")
-		v7, v8 := vtex(`str("+")`), vtex(fmt.Sprintf(`call:%s(str("+"), int(42), var(i))`, operatorFuncName))
+		v7, v8 := vtex(`str("+")`), vtex(fmt.Sprintf(`call:%s(str("+"), int(42), var(i))`, funcs.OperatorFuncName))
 
 		e1, e2, e3, e4, e5 := edge("op"), edge("a"), edge("b"), edge("var:i"), edge("var:x")
 		graph.AddVertex(&v1, &v2, &v3, &v4, &v5, &v6, &v7, &v8)
@@ -437,29 +440,31 @@ func TestAstFunc0(t *testing.T) {
 
 			t.Logf("\n\ntest #%d (%s) ----------------\n\n", index, name)
 			str := strings.NewReader(code)
-			ast, err := LexParse(str)
+			xast, err := parser.LexParse(str)
 			if err != nil {
 				t.Errorf("test #%d: FAIL", index)
 				t.Errorf("test #%d: lex/parse failed with: %+v", index, err)
 				return
 			}
-			t.Logf("test #%d: AST: %+v", index, ast)
+			t.Logf("test #%d: AST: %+v", index, xast)
 
 			data := &interfaces.Data{
 				// TODO: add missing fields here if/when needed
+				StrInterpolater: interpolate.InterpolateStr,
+
 				Debug: testing.Verbose(), // set via the -test.v flag to `go test`
 				Logf: func(format string, v ...interface{}) {
 					t.Logf("ast: "+format, v...)
 				},
 			}
 			// some of this might happen *after* interpolate in SetScope or Unify...
-			if err := ast.Init(data); err != nil {
+			if err := xast.Init(data); err != nil {
 				t.Errorf("test #%d: FAIL", index)
 				t.Errorf("test #%d: could not init and validate AST: %+v", index, err)
 				return
 			}
 
-			iast, err := ast.Interpolate()
+			iast, err := xast.Interpolate()
 			if err != nil {
 				t.Errorf("test #%d: FAIL", index)
 				t.Errorf("test #%d: interpolate failed with: %+v", index, err)
@@ -562,13 +567,13 @@ func TestAstFunc1(t *testing.T) {
 	t.Logf("tests directory is: %s", dir)
 
 	variables := map[string]interfaces.Expr{
-		"purpleidea": &ExprStr{V: "hello world!"}, // james says hi
+		"purpleidea": &ast.ExprStr{V: "hello world!"}, // james says hi
 		// TODO: change to a func when we can change hostname dynamically!
-		"hostname": &ExprStr{V: ""}, // NOTE: empty b/c not used
+		"hostname": &ast.ExprStr{V: ""}, // NOTE: empty b/c not used
 	}
-	consts := VarPrefixToVariablesScope(vars.ConstNamespace) // strips prefix!
-	addback := vars.ConstNamespace + interfaces.ModuleSep    // add it back...
-	variables, err = MergeExprMaps(variables, consts, addback)
+	consts := ast.VarPrefixToVariablesScope(vars.ConstNamespace) // strips prefix!
+	addback := vars.ConstNamespace + interfaces.ModuleSep        // add it back...
+	variables, err = ast.MergeExprMaps(variables, consts, addback)
 	if err != nil {
 		t.Errorf("couldn't merge in consts: %+v", err)
 		return
@@ -577,7 +582,7 @@ func TestAstFunc1(t *testing.T) {
 	scope := &interfaces.Scope{ // global scope
 		Variables: variables,
 		// all the built-in top-level, core functions enter here...
-		Functions: FuncPrefixToFunctionsScope(""), // runs funcs.LookupPrefix
+		Functions: ast.FuncPrefixToFunctionsScope(""), // runs funcs.LookupPrefix
 	}
 
 	type errs struct {
@@ -778,7 +783,7 @@ func TestAstFunc1(t *testing.T) {
 			logf("main:\n%s", output.Main) // debug
 
 			reader := bytes.NewReader(output.Main)
-			ast, err := LexParse(reader)
+			xast, err := parser.LexParse(reader)
 			if (!fail || !failLexParse) && err != nil {
 				t.Errorf("test #%d: FAIL", index)
 				t.Errorf("test #%d: lex/parse failed with: %+v", index, err)
@@ -800,7 +805,7 @@ func TestAstFunc1(t *testing.T) {
 				return
 			}
 
-			t.Logf("test #%d: AST: %+v", index, ast)
+			t.Logf("test #%d: AST: %+v", index, xast)
 
 			importGraph, err := pgraph.NewGraph("importGraph")
 			if err != nil {
@@ -824,13 +829,16 @@ func TestAstFunc1(t *testing.T) {
 				Metadata: output.Metadata,
 				Modules:  "/" + interfaces.ModuleDirectory, // not really needed here afaict
 
+				LexParser:       parser.LexParse,
+				StrInterpolater: interpolate.InterpolateStr,
+
 				Debug: testing.Verbose(), // set via the -test.v flag to `go test`
 				Logf: func(format string, v ...interface{}) {
 					logf("ast: "+format, v...)
 				},
 			}
 			// some of this might happen *after* interpolate in SetScope or Unify...
-			err = ast.Init(data)
+			err = xast.Init(data)
 			if (!fail || !failInit) && err != nil {
 				t.Errorf("test #%d: FAIL", index)
 				t.Errorf("test #%d: could not init and validate AST: %+v", index, err)
@@ -852,7 +860,7 @@ func TestAstFunc1(t *testing.T) {
 				return
 			}
 
-			iast, err := ast.Interpolate()
+			iast, err := xast.Interpolate()
 			if err != nil {
 				t.Errorf("test #%d: FAIL", index)
 				t.Errorf("test #%d: interpolate failed with: %+v", index, err)
@@ -1017,13 +1025,13 @@ func TestAstFunc2(t *testing.T) {
 	t.Logf("tests directory is: %s", dir)
 
 	variables := map[string]interfaces.Expr{
-		"purpleidea": &ExprStr{V: "hello world!"}, // james says hi
+		"purpleidea": &ast.ExprStr{V: "hello world!"}, // james says hi
 		// TODO: change to a func when we can change hostname dynamically!
-		"hostname": &ExprStr{V: ""}, // NOTE: empty b/c not used
+		"hostname": &ast.ExprStr{V: ""}, // NOTE: empty b/c not used
 	}
-	consts := VarPrefixToVariablesScope(vars.ConstNamespace) // strips prefix!
-	addback := vars.ConstNamespace + interfaces.ModuleSep    // add it back...
-	variables, err = MergeExprMaps(variables, consts, addback)
+	consts := ast.VarPrefixToVariablesScope(vars.ConstNamespace) // strips prefix!
+	addback := vars.ConstNamespace + interfaces.ModuleSep        // add it back...
+	variables, err = ast.MergeExprMaps(variables, consts, addback)
 	if err != nil {
 		t.Errorf("couldn't merge in consts: %+v", err)
 		return
@@ -1032,7 +1040,7 @@ func TestAstFunc2(t *testing.T) {
 	scope := &interfaces.Scope{ // global scope
 		Variables: variables,
 		// all the built-in top-level, core functions enter here...
-		Functions: FuncPrefixToFunctionsScope(""), // runs funcs.LookupPrefix
+		Functions: ast.FuncPrefixToFunctionsScope(""), // runs funcs.LookupPrefix
 	}
 
 	type errs struct {
@@ -1274,7 +1282,7 @@ func TestAstFunc2(t *testing.T) {
 			logf("main:\n%s", output.Main) // debug
 
 			reader := bytes.NewReader(output.Main)
-			ast, err := LexParse(reader)
+			xast, err := parser.LexParse(reader)
 			if (!fail || !failLexParse) && err != nil {
 				t.Errorf("test #%d: FAIL", index)
 				t.Errorf("test #%d: lex/parse failed with: %+v", index, err)
@@ -1296,7 +1304,7 @@ func TestAstFunc2(t *testing.T) {
 				return
 			}
 
-			t.Logf("test #%d: AST: %+v", index, ast)
+			t.Logf("test #%d: AST: %+v", index, xast)
 
 			importGraph, err := pgraph.NewGraph("importGraph")
 			if err != nil {
@@ -1320,13 +1328,16 @@ func TestAstFunc2(t *testing.T) {
 				Metadata: output.Metadata,
 				Modules:  "/" + interfaces.ModuleDirectory, // not really needed here afaict
 
+				LexParser:       parser.LexParse,
+				StrInterpolater: interpolate.InterpolateStr,
+
 				Debug: testing.Verbose(), // set via the -test.v flag to `go test`
 				Logf: func(format string, v ...interface{}) {
 					logf("ast: "+format, v...)
 				},
 			}
 			// some of this might happen *after* interpolate in SetScope or Unify...
-			err = ast.Init(data)
+			err = xast.Init(data)
 			if (!fail || !failInit) && err != nil {
 				t.Errorf("test #%d: FAIL", index)
 				t.Errorf("test #%d: could not init and validate AST: %+v", index, err)
@@ -1348,7 +1359,7 @@ func TestAstFunc2(t *testing.T) {
 				return
 			}
 
-			iast, err := ast.Interpolate()
+			iast, err := xast.Interpolate()
 			if (!fail || !failInterpolate) && err != nil {
 				t.Errorf("test #%d: FAIL", index)
 				t.Errorf("test #%d: Interpolate failed with: %+v", index, err)
@@ -1806,12 +1817,12 @@ func TestAstInterpret0(t *testing.T) {
 		t.Logf("\n\ntest #%d (%s) ----------------\n\n", index, name)
 
 		str := strings.NewReader(code)
-		ast, err := LexParse(str)
+		xast, err := parser.LexParse(str)
 		if err != nil {
 			t.Errorf("test #%d: lex/parse failed with: %+v", index, err)
 			continue
 		}
-		t.Logf("test #%d: AST: %+v", index, ast)
+		t.Logf("test #%d: AST: %+v", index, xast)
 
 		data := &interfaces.Data{
 			// TODO: add missing fields here if/when needed
@@ -1821,7 +1832,7 @@ func TestAstInterpret0(t *testing.T) {
 			},
 		}
 		// some of this might happen *after* interpolate in SetScope or Unify...
-		if err := ast.Init(data); err != nil {
+		if err := xast.Init(data); err != nil {
 			t.Errorf("test #%d: FAIL", index)
 			t.Errorf("test #%d: could not init and validate AST: %+v", index, err)
 			return
@@ -1831,7 +1842,7 @@ func TestAstInterpret0(t *testing.T) {
 		// perform type unification, run the function graph engine, and
 		// only gives you limited results... don't expect normal code to
 		// run and produce meaningful things in this test...
-		graph, err := interpret.Interpret(ast)
+		graph, err := interpret.Interpret(xast)
 
 		if !fail && err != nil {
 			t.Errorf("test #%d: interpret failed with: %+v", index, err)

lang/lang.go

@@ -23,25 +23,21 @@ import (
 	"sync"
 
 	"github.com/purpleidea/mgmt/engine"
+	"github.com/purpleidea/mgmt/lang/ast"
 	"github.com/purpleidea/mgmt/lang/funcs"
 	_ "github.com/purpleidea/mgmt/lang/funcs/core" // import so the funcs register
 	"github.com/purpleidea/mgmt/lang/funcs/vars"
 	"github.com/purpleidea/mgmt/lang/inputs"
 	"github.com/purpleidea/mgmt/lang/interfaces"
+	"github.com/purpleidea/mgmt/lang/interpolate"
 	"github.com/purpleidea/mgmt/lang/interpret"
+	"github.com/purpleidea/mgmt/lang/parser"
 	"github.com/purpleidea/mgmt/lang/unification"
 	"github.com/purpleidea/mgmt/pgraph"
 	"github.com/purpleidea/mgmt/util"
 	"github.com/purpleidea/mgmt/util/errwrap"
 )
 
-const (
-	// make these available internally without requiring the import
-	operatorFuncName = funcs.OperatorFuncName
-	historyFuncName  = funcs.HistoryFuncName
-	containsFuncName = funcs.ContainsFuncName
-)
-
 // Lang is the main language lexer/parser object.
 type Lang struct {
 	Fs    engine.Fs // connected fs where input dir or metadata exists
@@ -117,12 +113,12 @@ func (obj *Lang) Init() error {
 	// run the lexer/parser and build an AST
 	obj.Logf("lexing/parsing...")
 	// this reads an io.Reader, which might be a stream of multiple files...
-	ast, err := LexParse(reader)
+	xast, err := parser.LexParse(reader)
 	if err != nil {
 		return errwrap.Wrapf(err, "could not generate AST")
 	}
 	if obj.Debug {
-		obj.Logf("behold, the AST: %+v", ast)
+		obj.Logf("behold, the AST: %+v", xast)
 	}
 
 	importGraph, err := pgraph.NewGraph("importGraph")
@@ -147,7 +143,11 @@ func (obj *Lang) Init() error {
 		Metadata: output.Metadata,
 		Modules:  "/" + interfaces.ModuleDirectory, // do not set from env for a deploy!
 
+		LexParser:       parser.LexParse,
+		Downloader:      nil, // XXX: is this used here?
+		StrInterpolater: interpolate.InterpolateStr,
 		//World: obj.World, // TODO: do we need this?
+
 		Prefix: obj.Prefix,
 		Debug:  obj.Debug,
 		Logf: func(format string, v ...interface{}) {
@@ -156,26 +156,26 @@ func (obj *Lang) Init() error {
 		},
 	}
 	// some of this might happen *after* interpolate in SetScope or Unify...
-	if err := ast.Init(data); err != nil {
+	if err := xast.Init(data); err != nil {
 		return errwrap.Wrapf(err, "could not init and validate AST")
 	}
 
 	obj.Logf("interpolating...")
 	// interpolate strings and other expansionable nodes in AST
-	interpolated, err := ast.Interpolate()
+	interpolated, err := xast.Interpolate()
 	if err != nil {
 		return errwrap.Wrapf(err, "could not interpolate AST")
 	}
 	obj.ast = interpolated
 
 	variables := map[string]interfaces.Expr{
-		"purpleidea": &ExprStr{V: "hello world!"}, // james says hi
+		"purpleidea": &ast.ExprStr{V: "hello world!"}, // james says hi
 		// TODO: change to a func when we can change hostname dynamically!
-		"hostname": &ExprStr{V: obj.Hostname},
+		"hostname": &ast.ExprStr{V: obj.Hostname},
 	}
-	consts := VarPrefixToVariablesScope(vars.ConstNamespace) // strips prefix!
-	addback := vars.ConstNamespace + interfaces.ModuleSep    // add it back...
-	variables, err = MergeExprMaps(variables, consts, addback)
+	consts := ast.VarPrefixToVariablesScope(vars.ConstNamespace) // strips prefix!
+	addback := vars.ConstNamespace + interfaces.ModuleSep        // add it back...
+	variables, err = ast.MergeExprMaps(variables, consts, addback)
 	if err != nil {
 		return errwrap.Wrapf(err, "couldn't merge in consts")
 	}
@@ -184,7 +184,7 @@ func (obj *Lang) Init() error {
 	scope := &interfaces.Scope{
 		Variables: variables,
 		// all the built-in top-level, core functions enter here...
-		Functions: FuncPrefixToFunctionsScope(""), // runs funcs.LookupPrefix
+		Functions: ast.FuncPrefixToFunctionsScope(""), // runs funcs.LookupPrefix
 	}
 
 	obj.Logf("building scope...")

lang/parser/lexer.nex

@@ -402,7 +402,7 @@
 // You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package lang
+package parser
 
 import (
 	"fmt"

lang/parser/lexparse.go

@@ -15,13 +15,12 @@
 // You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package lang // TODO: move this into a sub package of lang/$name?
+package parser
 
 import (
 	"bufio"
 	"fmt"
 	"io"
-	"net/url"
 	"path"
 	"sort"
 	"strings"
@@ -32,17 +31,6 @@ import (
 	"github.com/purpleidea/mgmt/util/errwrap"
 )
 
-const (
-	// ModuleMagicPrefix is the prefix which, if found as a prefix to the
-	// last token in an import path, will be removed silently if there are
-	// remaining characters following the name. If this is the empty string
-	// then it will be ignored.
-	ModuleMagicPrefix = "mgmt-"
-
-	// CoreDir is the directory prefix where core bindata mcl code is added.
-	CoreDir = "core/"
-)
-
 // These constants represent the different possible lexer/parser errors.
 const (
 	ErrLexerUnrecognized      = interfaces.Error("unrecognized")
@@ -236,144 +224,3 @@ func DirectoryReader(fs engine.Fs, dir string) (io.Reader, map[uint64]string, er
 
 	return io.MultiReader(readers...), offsets, nil
 }
-
-// ParseImportName parses an import name and returns the default namespace name
-// that should be used with it. For example, if the import name was:
-// "git://example.com/purpleidea/Module-Name", this might return an alias of
-// "module_name". It also returns a bunch of other data about the parsed import.
-// TODO: check for invalid or unwanted special characters
-func ParseImportName(name string) (*interfaces.ImportData, error) {
-	magicPrefix := ModuleMagicPrefix
-	if name == "" {
-		return nil, fmt.Errorf("empty name")
-	}
-	if strings.HasPrefix(name, "/") {
-		return nil, fmt.Errorf("absolute paths are not allowed")
-	}
-
-	u, err := url.Parse(name)
-	if err != nil {
-		return nil, errwrap.Wrapf(err, "name is not a valid url")
-	}
-	if u.Path == "" {
-		return nil, fmt.Errorf("empty path")
-	}
-	p := u.Path
-	// catch bad paths like: git:////home/james/ (note the quad slash!)
-	// don't penalize if we have a dir with a trailing slash at the end
-	if s := path.Clean(u.Path); u.Path != s && u.Path != s+"/" {
-		// TODO: are there any cases where this is not what we want?
-		return nil, fmt.Errorf("dirty path, cleaned it's: `%s`", s)
-	}
-
-	for strings.HasSuffix(p, "/") { // remove trailing slashes
-		p = p[:len(p)-len("/")]
-	}
-
-	split := strings.Split(p, "/") // take last chunk if slash separated
-	s := split[0]
-	if len(split) > 1 {
-		s = split[len(split)-1] // pick last chunk
-	}
-
-	// TODO: should we treat a special name: "purpleidea/mgmt-foo" as "foo"?
-	if magicPrefix != "" && strings.HasPrefix(s, magicPrefix) && len(s) > len(magicPrefix) {
-		s = s[len(magicPrefix):]
-	}
-
-	s = strings.Replace(s, "-", "_", -1) // XXX: allow underscores in IDENTIFIER
-	if strings.HasPrefix(s, "_") || strings.HasSuffix(s, "_") {
-		return nil, fmt.Errorf("name can't begin or end with dash or underscore")
-	}
-	alias := strings.ToLower(s)
-
-	// if this is a local import, it's a straight directory path
-	// if it's an fqdn import, it should contain a metadata file
-
-	// if there's no protocol prefix, then this must be a local path
-	isLocal := u.Scheme == ""
-	// if it has a trailing slash or .mcl extension it's not a system import
-	isSystem := isLocal && !strings.HasSuffix(u.Path, "/") && !strings.HasSuffix(u.Path, interfaces.DotFileNameExtension)
-	// is it a local file?
-	isFile := !isSystem && isLocal && strings.HasSuffix(u.Path, interfaces.DotFileNameExtension)
-	xpath := u.Path // magic path
-	if isSystem {
-		xpath = ""
-	}
-	if !isLocal {
-		host := u.Host // host or host:port
-		split := strings.Split(host, ":")
-		if l := len(split); l == 1 || l == 2 {
-			host = split[0]
-		} else {
-			return nil, fmt.Errorf("incorrect number of colons (%d) in hostname", l)
-		}
-		xpath = path.Join(host, xpath)
-	}
-	if !isLocal && !strings.HasSuffix(xpath, "/") {
-		xpath = xpath + "/"
-	}
-	// we're a git repo with a local path instead of an fqdn over http!
-	// this still counts as isLocal == false, since it's still a remote
-	if u.Host == "" && strings.HasPrefix(u.Path, "/") {
-		xpath = strings.TrimPrefix(xpath, "/") // make it a relative dir
-	}
-	if strings.HasPrefix(xpath, "/") { // safety check (programming error?)
-		return nil, fmt.Errorf("can't parse strange import")
-	}
-
-	// build a url to clone from if we're not local...
-	// TODO: consider adding some logic that is similar to the logic in:
-	// https://github.com/golang/go/blob/054640b54df68789d9df0e50575d21d9dbffe99f/src/cmd/go/internal/get/vcs.go#L972
-	// so that we can more correctly figure out the correct url to clone...
-	xurl := ""
-	if !isLocal {
-		u.Fragment = ""
-		// TODO: maybe look for ?sha1=... or ?tag=... to pick a real ref
-		u.RawQuery = ""
-		u.ForceQuery = false
-		xurl = u.String()
-	}
-
-	// if u.Path is local file like: foo/server.mcl alias should be "server"
-	// we should trim the alias to remove the .mcl (the dir is already gone)
-	if isFile && strings.HasSuffix(alias, interfaces.DotFileNameExtension) {
-		alias = strings.TrimSuffix(alias, interfaces.DotFileNameExtension)
-	}
-
-	return &interfaces.ImportData{
-		Name:     name, // save the original value here
-		Alias:    alias,
-		IsSystem: isSystem,
-		IsLocal:  isLocal,
-		IsFile:   isFile,
-		Path:     xpath,
-		URL:      xurl,
-	}, nil
-}
-
-// CollectFiles collects all the files used in the AST. You will see more files
-// based on how many compiling steps have run. In general, this is useful for
-// collecting all the files needed to store in our file system for a deploy.
-func CollectFiles(ast interfaces.Stmt) ([]string, error) {
-	// collect the list of files
-	fileList := []string{}
-	fn := func(node interfaces.Node) error {
-		// redundant check for example purposes
-		stmt, ok := node.(interfaces.Stmt)
-		if !ok {
-			return nil
-		}
-		prog, ok := stmt.(*StmtProg)
-		if !ok {
-			return nil
-		}
-		// collect into global
-		fileList = append(fileList, prog.importFiles...)
-		return nil
-	}
-	if err := ast.Apply(fn); err != nil {
-		return nil, errwrap.Wrapf(err, "can't retrieve paths")
-	}
-	return fileList, nil
-}

lang/parser/parser.y

@@ -16,12 +16,14 @@
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 %{
-package lang
+package parser
 
 import (
 	"fmt"
 	"strings"
 
+	"github.com/purpleidea/mgmt/lang/ast"
+	"github.com/purpleidea/mgmt/lang/funcs"
 	"github.com/purpleidea/mgmt/lang/interfaces"
 	"github.com/purpleidea/mgmt/lang/types"
 	"github.com/purpleidea/mgmt/util"
@@ -56,22 +58,22 @@ func init() {
 	exprs []interfaces.Expr
 	expr  interfaces.Expr
 
-	mapKVs []*ExprMapKV
-	mapKV  *ExprMapKV
+	mapKVs []*ast.ExprMapKV
+	mapKV  *ast.ExprMapKV
 
-	structFields []*ExprStructField
-	structField  *ExprStructField
+	structFields []*ast.ExprStructField
+	structField  *ast.ExprStructField
 
 	args []*interfaces.Arg
 	arg  *interfaces.Arg
 
-	resContents []StmtResContents // interface
-	resField    *StmtResField
-	resEdge     *StmtResEdge
-	resMeta     *StmtResMeta
+	resContents []ast.StmtResContents // interface
+	resField    *ast.StmtResField
+	resEdge     *ast.StmtResEdge
+	resMeta     *ast.StmtResMeta
 
-	edgeHalfList []*StmtEdgeHalf
-	edgeHalf     *StmtEdgeHalf
+	edgeHalfList []*ast.StmtEdgeHalf
+	edgeHalf     *ast.StmtEdgeHalf
 }
 
 %token OPEN_CURLY CLOSE_CURLY
@@ -133,7 +135,7 @@ prog:
 	/* end of list */
 	{
 		posLast(yylex, yyDollar) // our pos
-		$$.stmt = &StmtProg{
+		$$.stmt = &ast.StmtProg{
 			Body: []interfaces.Stmt{},
 		}
 	}
@@ -141,12 +143,12 @@ prog:
 	{
 		posLast(yylex, yyDollar) // our pos
 		// TODO: should we just skip comments for now?
-		//if _, ok := $2.stmt.(*StmtComment); !ok {
+		//if _, ok := $2.stmt.(*ast.StmtComment); !ok {
 		//}
-		if stmt, ok := $1.stmt.(*StmtProg); ok {
+		if stmt, ok := $1.stmt.(*ast.StmtProg); ok {
 			stmts := stmt.Body
 			stmts = append(stmts, $2.stmt)
-			$$.stmt = &StmtProg{
+			$$.stmt = &ast.StmtProg{
 				Body: stmts,
 			}
 		}
@@ -156,7 +158,7 @@ stmt:
 	COMMENT
 	{
 		posLast(yylex, yyDollar) // our pos
-		$$.stmt = &StmtComment{
+		$$.stmt = &ast.StmtComment{
 			Value: $1.str,
 		}
 	}
@@ -178,7 +180,7 @@ stmt:
 |	IF expr OPEN_CURLY prog CLOSE_CURLY
 	{
 		posLast(yylex, yyDollar) // our pos
-		$$.stmt = &StmtIf{
+		$$.stmt = &ast.StmtIf{
 			Condition:  $2.expr,
 			ThenBranch: $4.stmt,
 			//ElseBranch: nil,
@@ -187,7 +189,7 @@ stmt:
 |	IF expr OPEN_CURLY prog CLOSE_CURLY ELSE OPEN_CURLY prog CLOSE_CURLY
 	{
 		posLast(yylex, yyDollar) // our pos
-		$$.stmt = &StmtIf{
+		$$.stmt = &ast.StmtIf{
 			Condition:  $2.expr,
 			ThenBranch: $4.stmt,
 			ElseBranch: $8.stmt,
@@ -200,9 +202,9 @@ stmt:
 |	FUNC_IDENTIFIER IDENTIFIER OPEN_PAREN args CLOSE_PAREN OPEN_CURLY expr CLOSE_CURLY
 	{
 		posLast(yylex, yyDollar) // our pos
-		$$.stmt = &StmtFunc{
+		$$.stmt = &ast.StmtFunc{
 			Name: $2.str,
-			Func: &ExprFunc{
+			Func: &ast.ExprFunc{
 				Args: $4.args,
 				//Return: nil,
 				Body: $7.expr,
@@ -213,7 +215,7 @@ stmt:
 |	FUNC_IDENTIFIER IDENTIFIER OPEN_PAREN args CLOSE_PAREN type OPEN_CURLY expr CLOSE_CURLY
 	{
 		posLast(yylex, yyDollar) // our pos
-		fn := &ExprFunc{
+		fn := &ast.ExprFunc{
 			Args:   $4.args,
 			Return: $6.typ, // return type is known
 			Body:   $8.expr,
@@ -242,7 +244,7 @@ stmt:
 				yylex.Error(fmt.Sprintf("%s: %+v", ErrParseSetType, err))
 			}
 		}
-		$$.stmt = &StmtFunc{
+		$$.stmt = &ast.StmtFunc{
 			Name: $2.str,
 			Func: fn,
 		}
@@ -251,7 +253,7 @@ stmt:
 |	CLASS_IDENTIFIER IDENTIFIER OPEN_CURLY prog CLOSE_CURLY
 	{
 		posLast(yylex, yyDollar) // our pos
-		$$.stmt = &StmtClass{
+		$$.stmt = &ast.StmtClass{
 			Name: $2.str,
 			Args: nil,
 			Body: $4.stmt,
@@ -262,7 +264,7 @@ stmt:
 |	CLASS_IDENTIFIER IDENTIFIER OPEN_PAREN args CLOSE_PAREN OPEN_CURLY prog CLOSE_CURLY
 	{
 		posLast(yylex, yyDollar) // our pos
-		$$.stmt = &StmtClass{
+		$$.stmt = &ast.StmtClass{
 			Name: $2.str,
 			Args: $4.args,
 			Body: $7.stmt,
@@ -272,7 +274,7 @@ stmt:
 |	INCLUDE_IDENTIFIER dotted_identifier
 	{
 		posLast(yylex, yyDollar) // our pos
-		$$.stmt = &StmtInclude{
+		$$.stmt = &ast.StmtInclude{
 			Name: $2.str,
 		}
 	}
@@ -280,7 +282,7 @@ stmt:
 |	INCLUDE_IDENTIFIER dotted_identifier OPEN_PAREN call_args CLOSE_PAREN
 	{
 		posLast(yylex, yyDollar) // our pos
-		$$.stmt = &StmtInclude{
+		$$.stmt = &ast.StmtInclude{
 			Name: $2.str,
 			Args: $4.exprs,
 		}
@@ -289,7 +291,7 @@ stmt:
 |	IMPORT_IDENTIFIER STRING
 	{
 		posLast(yylex, yyDollar) // our pos
-		$$.stmt = &StmtImport{
+		$$.stmt = &ast.StmtImport{
 			Name: $2.str,
 			//Alias: "",
 		}
@@ -298,7 +300,7 @@ stmt:
 |	IMPORT_IDENTIFIER STRING AS_IDENTIFIER IDENTIFIER
 	{
 		posLast(yylex, yyDollar) // our pos
-		$$.stmt = &StmtImport{
+		$$.stmt = &ast.StmtImport{
 			Name:  $2.str,
 			Alias: $4.str,
 		}
@@ -307,7 +309,7 @@ stmt:
 |	IMPORT_IDENTIFIER STRING AS_IDENTIFIER MULTIPLY
 	{
 		posLast(yylex, yyDollar) // our pos
-		$$.stmt = &StmtImport{
+		$$.stmt = &ast.StmtImport{
 			Name:  $2.str,
 			Alias: $4.str,
 		}
@@ -325,28 +327,28 @@ expr:
 	BOOL
 	{
 		posLast(yylex, yyDollar) // our pos
-		$$.expr = &ExprBool{
+		$$.expr = &ast.ExprBool{
 			V: $1.bool,
 		}
 	}
 |	STRING
 	{
 		posLast(yylex, yyDollar) // our pos
-		$$.expr = &ExprStr{
+		$$.expr = &ast.ExprStr{
 			V: $1.str,
 		}
 	}
 |	INTEGER
 	{
 		posLast(yylex, yyDollar) // our pos
-		$$.expr = &ExprInt{
+		$$.expr = &ast.ExprInt{
 			V: $1.int,
 		}
 	}
 |	FLOAT
 	{
 		posLast(yylex, yyDollar) // our pos
-		$$.expr = &ExprFloat{
+		$$.expr = &ast.ExprFloat{
 			V: $1.float,
 		}
 	}
@@ -389,7 +391,7 @@ expr:
 |	IF expr OPEN_CURLY expr CLOSE_CURLY ELSE OPEN_CURLY expr CLOSE_CURLY
 	{
 		posLast(yylex, yyDollar) // our pos
-		$$.expr = &ExprIf{
+		$$.expr = &ast.ExprIf{
 			Condition:  $2.expr,
 			ThenBranch: $4.expr,
 			ElseBranch: $8.expr,
@@ -407,7 +409,7 @@ list:
 	OPEN_BRACK list_elements CLOSE_BRACK
 	{
 		posLast(yylex, yyDollar) // our pos
-		$$.expr = &ExprList{
+		$$.expr = &ast.ExprList{
 			Elements: $2.exprs,
 		}
 	}
@@ -436,7 +438,7 @@ map:
 	OPEN_CURLY map_kvs CLOSE_CURLY
 	{
 		posLast(yylex, yyDollar) // our pos
-		$$.expr = &ExprMap{
+		$$.expr = &ast.ExprMap{
 			KVs: $2.mapKVs,
 		}
 	}
@@ -445,7 +447,7 @@ map_kvs:
 	/* end of list */
 	{
 		posLast(yylex, yyDollar) // our pos
-		$$.mapKVs = []*ExprMapKV{}
+		$$.mapKVs = []*ast.ExprMapKV{}
 	}
 |	map_kvs map_kv
 	{
@@ -457,7 +459,7 @@ map_kv:
 	expr ROCKET expr COMMA
 	{
 		posLast(yylex, yyDollar) // our pos
-		$$.mapKV = &ExprMapKV{
+		$$.mapKV = &ast.ExprMapKV{
 			Key: $1.expr,
 			Val: $3.expr,
 		}
@@ -468,7 +470,7 @@ struct:
 	STRUCT_IDENTIFIER OPEN_CURLY struct_fields CLOSE_CURLY
 	{
 		posLast(yylex, yyDollar) // our pos
-		$$.expr = &ExprStruct{
+		$$.expr = &ast.ExprStruct{
 			Fields: $3.structFields,
 		}
 	}
@@ -477,7 +479,7 @@ struct_fields:
 	/* end of list */
 	{
 		posLast(yylex, yyDollar) // our pos
-		$$.structFields = []*ExprStructField{}
+		$$.structFields = []*ast.ExprStructField{}
 	}
 |	struct_fields struct_field
 	{
@@ -489,7 +491,7 @@ struct_field:
 	IDENTIFIER ROCKET expr COMMA
 	{
 		posLast(yylex, yyDollar) // our pos
-		$$.structField = &ExprStructField{
+		$$.structField = &ast.ExprStructField{
 			Name:  $1.str,
 			Value: $3.expr,
 		}
@@ -500,7 +502,7 @@ call:
 	dotted_identifier OPEN_PAREN call_args CLOSE_PAREN
 	{
 		posLast(yylex, yyDollar) // our pos
-		$$.expr = &ExprCall{
+		$$.expr = &ast.ExprCall{
 			Name: $1.str,
 			Args: $3.exprs,
 			//Var: false, // default
@@ -511,7 +513,7 @@ call:
 |	VAR_IDENTIFIER OPEN_PAREN call_args CLOSE_PAREN
 	{
 		posLast(yylex, yyDollar) // our pos
-		$$.expr = &ExprCall{
+		$$.expr = &ast.ExprCall{
 			Name: $1.str,
 			Args: $3.exprs,
 			// Instead of `Var: true`, we could have added a `$`
@@ -522,11 +524,11 @@ call:
 |	expr PLUS expr
 	{
 		posLast(yylex, yyDollar) // our pos
-		$$.expr = &ExprCall{
-			Name: operatorFuncName,
+		$$.expr = &ast.ExprCall{
+			Name: funcs.OperatorFuncName,
 			Args: []interfaces.Expr{
-				&ExprStr{          // operator first
-					V: $2.str, // for PLUS this is a `+` character
+				&ast.ExprStr{          // operator first
+					V: $2.str,     // for PLUS this is a `+` character
 				},
 				$1.expr,
 				$3.expr,
@@ -536,10 +538,10 @@ call:
 |	expr MINUS expr
 	{
 		posLast(yylex, yyDollar) // our pos
-		$$.expr = &ExprCall{
-			Name: operatorFuncName,
+		$$.expr = &ast.ExprCall{
+			Name: funcs.OperatorFuncName,
 			Args: []interfaces.Expr{
-				&ExprStr{ // operator first
+				&ast.ExprStr{ // operator first
 					V: $2.str,
 				},
 				$1.expr,
@@ -550,10 +552,10 @@ call:
 |	expr MULTIPLY expr
 	{
 		posLast(yylex, yyDollar) // our pos
-		$$.expr = &ExprCall{
-			Name: operatorFuncName,
+		$$.expr = &ast.ExprCall{
+			Name: funcs.OperatorFuncName,
 			Args: []interfaces.Expr{
-				&ExprStr{ // operator first
+				&ast.ExprStr{ // operator first
 					V: $2.str,
 				},
 				$1.expr,
@@ -564,10 +566,10 @@ call:
 |	expr DIVIDE expr
 	{
 		posLast(yylex, yyDollar) // our pos
-		$$.expr = &ExprCall{
-			Name: operatorFuncName,
+		$$.expr = &ast.ExprCall{
+			Name: funcs.OperatorFuncName,
 			Args: []interfaces.Expr{
-				&ExprStr{ // operator first
+				&ast.ExprStr{ // operator first
 					V: $2.str,
 				},
 				$1.expr,
@@ -578,10 +580,10 @@ call:
 |	expr EQ expr
 	{
 		posLast(yylex, yyDollar) // our pos
-		$$.expr = &ExprCall{
-			Name: operatorFuncName,
+		$$.expr = &ast.ExprCall{
+			Name: funcs.OperatorFuncName,
 			Args: []interfaces.Expr{
-				&ExprStr{ // operator first
+				&ast.ExprStr{ // operator first
 					V: $2.str,
 				},
 				$1.expr,
@@ -592,10 +594,10 @@ call:
 |	expr NEQ expr
 	{
 		posLast(yylex, yyDollar) // our pos
-		$$.expr = &ExprCall{
-			Name: operatorFuncName,
+		$$.expr = &ast.ExprCall{
+			Name: funcs.OperatorFuncName,
 			Args: []interfaces.Expr{
-				&ExprStr{ // operator first
+				&ast.ExprStr{ // operator first
 					V: $2.str,
 				},
 				$1.expr,
@@ -606,10 +608,10 @@ call:
 |	expr LT expr
 	{
 		posLast(yylex, yyDollar) // our pos
-		$$.expr = &ExprCall{
-			Name: operatorFuncName,
+		$$.expr = &ast.ExprCall{
+			Name: funcs.OperatorFuncName,
 			Args: []interfaces.Expr{
-				&ExprStr{ // operator first
+				&ast.ExprStr{ // operator first
 					V: $2.str,
 				},
 				$1.expr,
@@ -620,10 +622,10 @@ call:
 |	expr GT expr
 	{
 		posLast(yylex, yyDollar) // our pos
-		$$.expr = &ExprCall{
-			Name: operatorFuncName,
+		$$.expr = &ast.ExprCall{
+			Name: funcs.OperatorFuncName,
 			Args: []interfaces.Expr{
-				&ExprStr{ // operator first
+				&ast.ExprStr{ // operator first
 					V: $2.str,
 				},
 				$1.expr,
@@ -634,10 +636,10 @@ call:
 |	expr LTE expr
 	{
 		posLast(yylex, yyDollar) // our pos
-		$$.expr = &ExprCall{
-			Name: operatorFuncName,
+		$$.expr = &ast.ExprCall{
+			Name: funcs.OperatorFuncName,
 			Args: []interfaces.Expr{
-				&ExprStr{ // operator first
+				&ast.ExprStr{ // operator first
 					V: $2.str,
 				},
 				$1.expr,
@@ -648,10 +650,10 @@ call:
 |	expr GTE expr
 	{
 		posLast(yylex, yyDollar) // our pos
-		$$.expr = &ExprCall{
-			Name: operatorFuncName,
+		$$.expr = &ast.ExprCall{
+			Name: funcs.OperatorFuncName,
 			Args: []interfaces.Expr{
-				&ExprStr{ // operator first
+				&ast.ExprStr{ // operator first
 					V: $2.str,
 				},
 				$1.expr,
@@ -662,10 +664,10 @@ call:
 |	expr AND expr
 	{
 		posLast(yylex, yyDollar) // our pos
-		$$.expr = &ExprCall{
-			Name: operatorFuncName,
+		$$.expr = &ast.ExprCall{
+			Name: funcs.OperatorFuncName,
 			Args: []interfaces.Expr{
-				&ExprStr{ // operator first
+				&ast.ExprStr{ // operator first
 					V: $2.str,
 				},
 				$1.expr,
@@ -676,10 +678,10 @@ call:
 |	expr OR expr
 	{
 		posLast(yylex, yyDollar) // our pos
-		$$.expr = &ExprCall{
-			Name: operatorFuncName,
+		$$.expr = &ast.ExprCall{
+			Name: funcs.OperatorFuncName,
 			Args: []interfaces.Expr{
-				&ExprStr{ // operator first
+				&ast.ExprStr{ // operator first
 					V: $2.str,
 				},
 				$1.expr,
@@ -690,10 +692,10 @@ call:
 |	NOT expr
 	{
 		posLast(yylex, yyDollar) // our pos
-		$$.expr = &ExprCall{
-			Name: operatorFuncName,
+		$$.expr = &ast.ExprCall{
+			Name: funcs.OperatorFuncName,
 			Args: []interfaces.Expr{
-				&ExprStr{ // operator first
+				&ast.ExprStr{ // operator first
 					V: $1.str,
 				},
 				$2.expr,
@@ -704,13 +706,13 @@ call:
 	// get the N-th historical value, eg: $foo{3} is equivalent to: history($foo, 3)
 	{
 		posLast(yylex, yyDollar) // our pos
-		$$.expr = &ExprCall{
-			Name: historyFuncName,
+		$$.expr = &ast.ExprCall{
+			Name: funcs.HistoryFuncName,
 			Args: []interfaces.Expr{
-				&ExprVar{
+				&ast.ExprVar{
 					Name: $1.str,
 				},
-				&ExprInt{
+				&ast.ExprInt{
 					V: $1.int,
 				},
 			},
@@ -722,13 +724,13 @@ call:
 //|	dotted_var_identifier OPEN_CURLY INTEGER CLOSE_CURLY
 //	{
 //		posLast(yylex, yyDollar) // our pos
-//		$$.expr = &ExprCall{
-//			Name: historyFuncName,
+//		$$.expr = &ast.ExprCall{
+//			Name: funcs.HistoryFuncName,
 //			Args: []interfaces.Expr{
-//				&ExprVar{
+//				&ast.ExprVar{
 //					Name: $1.str,
 //				},
-//				&ExprInt{
+//				&ast.ExprInt{
 //					V: $3.int,
 //				},
 //			},
@@ -737,8 +739,8 @@ call:
 |	expr IN expr
 	{
 		posLast(yylex, yyDollar) // our pos
-		$$.expr = &ExprCall{
-			Name: containsFuncName,
+		$$.expr = &ast.ExprCall{
+			Name: funcs.ContainsFuncName,
 			Args: []interfaces.Expr{
 				$1.expr,
 				$3.expr,
@@ -770,7 +772,7 @@ var:
 	dotted_var_identifier
 	{
 		posLast(yylex, yyDollar) // our pos
-		$$.expr = &ExprVar{
+		$$.expr = &ast.ExprVar{
 			Name: $1.str,
 		}
 	}
@@ -783,7 +785,7 @@ func:
 	FUNC_IDENTIFIER OPEN_PAREN args CLOSE_PAREN OPEN_CURLY expr CLOSE_CURLY
 	{
 		posLast(yylex, yyDollar) // our pos
-		$$.expr = &ExprFunc{
+		$$.expr = &ast.ExprFunc{
 			Args: $3.args,
 			//Return: nil,
 			Body: $6.expr,
@@ -793,7 +795,7 @@ func:
 |	FUNC_IDENTIFIER OPEN_PAREN args CLOSE_PAREN type OPEN_CURLY expr CLOSE_CURLY
 	{
 		posLast(yylex, yyDollar) // our pos
-		$$.expr = &ExprFunc{
+		$$.expr = &ast.ExprFunc{
 			Args:   $3.args,
 			Return: $5.typ, // return type is known
 			Body:   $7.expr,
@@ -863,7 +865,7 @@ bind:
 	VAR_IDENTIFIER EQUALS expr
 	{
 		posLast(yylex, yyDollar) // our pos
-		$$.stmt = &StmtBind{
+		$$.stmt = &ast.StmtBind{
 			Ident: $1.str,
 			Value: $3.expr,
 		}
@@ -878,7 +880,7 @@ bind:
 			// this will ultimately cause a parser error to occur...
 			yylex.Error(fmt.Sprintf("%s: %+v", ErrParseSetType, err))
 		}
-		$$.stmt = &StmtBind{
+		$$.stmt = &ast.StmtBind{
 			Ident: $1.str,
 			Value: expr,
 		}
@@ -893,7 +895,7 @@ rbind:
 		// XXX: this kind of bind is different than the others, because
 		// it can only really be used for send->recv stuff, eg:
 		// foo.SomeString -> bar.SomeOtherString
-		$$.expr = &StmtBind{
+		$$.expr = &ast.StmtBind{
 			Ident: $1.str,
 			Value: $3.stmt,
 		}
@@ -905,7 +907,7 @@ resource:
 	RES_IDENTIFIER expr OPEN_CURLY resource_body CLOSE_CURLY
 	{
 		posLast(yylex, yyDollar) // our pos
-		$$.stmt = &StmtRes{
+		$$.stmt = &ast.StmtRes{
 			Kind:     $1.str,
 			Name:     $2.expr,
 			Contents: $4.resContents,
@@ -916,7 +918,7 @@ resource:
 |	IDENTIFIER expr OPEN_CURLY resource_body CLOSE_CURLY
 	{
 		posLast(yylex, yyDollar) // our pos
-		$$.stmt = &StmtRes{
+		$$.stmt = &ast.StmtRes{
 			Kind:     $1.str,
 			Name:     $2.expr,
 			Contents: $4.resContents,
@@ -927,7 +929,7 @@ resource_body:
 	/* end of list */
 	{
 		posLast(yylex, yyDollar) // our pos
-		$$.resContents = []StmtResContents{}
+		$$.resContents = []ast.StmtResContents{}
 	}
 |	resource_body resource_field
 	{
@@ -974,7 +976,7 @@ resource_field:
 	IDENTIFIER ROCKET expr COMMA
 	{
 		posLast(yylex, yyDollar) // our pos
-		$$.resField = &StmtResField{
+		$$.resField = &ast.StmtResField{
 			Field: $1.str,
 			Value: $3.expr,
 		}
@@ -985,7 +987,7 @@ conditional_resource_field:
 	IDENTIFIER ROCKET expr ELVIS expr COMMA
 	{
 		posLast(yylex, yyDollar) // our pos
-		$$.resField = &StmtResField{
+		$$.resField = &ast.StmtResField{
 			Field:     $1.str,
 			Value:     $5.expr,
 			Condition: $3.expr,
@@ -997,7 +999,7 @@ resource_edge:
 	CAPITALIZED_IDENTIFIER ROCKET edge_half COMMA
 	{
 		posLast(yylex, yyDollar) // our pos
-		$$.resEdge = &StmtResEdge{
+		$$.resEdge = &ast.StmtResEdge{
 			Property: $1.str,
 			EdgeHalf: $3.edgeHalf,
 		}
@@ -1008,7 +1010,7 @@ conditional_resource_edge:
 	CAPITALIZED_IDENTIFIER ROCKET expr ELVIS edge_half COMMA
 	{
 		posLast(yylex, yyDollar) // our pos
-		$$.resEdge = &StmtResEdge{
+		$$.resEdge = &ast.StmtResEdge{
 			Property:  $1.str,
 			EdgeHalf:  $5.edgeHalf,
 			Condition: $3.expr,
@@ -1020,11 +1022,11 @@ resource_meta:
 	CAPITALIZED_IDENTIFIER COLON IDENTIFIER ROCKET expr COMMA
 	{
 		posLast(yylex, yyDollar) // our pos
-		if strings.ToLower($1.str) != strings.ToLower(MetaField) {
+		if strings.ToLower($1.str) != strings.ToLower(ast.MetaField) {
 			// this will ultimately cause a parser error to occur...
 			yylex.Error(fmt.Sprintf("%s: %s", ErrParseResFieldInvalid, $1.str))
 		}
-		$$.resMeta = &StmtResMeta{
+		$$.resMeta = &ast.StmtResMeta{
 			Property: $3.str,
 			MetaExpr: $5.expr,
 		}
@@ -1035,11 +1037,11 @@ conditional_resource_meta:
 	CAPITALIZED_IDENTIFIER COLON IDENTIFIER ROCKET expr ELVIS expr COMMA
 	{
 		posLast(yylex, yyDollar) // our pos
-		if strings.ToLower($1.str) != strings.ToLower(MetaField) {
+		if strings.ToLower($1.str) != strings.ToLower(ast.MetaField) {
 			// this will ultimately cause a parser error to occur...
 			yylex.Error(fmt.Sprintf("%s: %s", ErrParseResFieldInvalid, $1.str))
 		}
-		$$.resMeta = &StmtResMeta{
+		$$.resMeta = &ast.StmtResMeta{
 			Property:  $3.str,
 			MetaExpr:  $7.expr,
 			Condition: $5.expr,
@@ -1051,11 +1053,11 @@ resource_meta_struct:
 	CAPITALIZED_IDENTIFIER ROCKET expr COMMA
 	{
 		posLast(yylex, yyDollar) // our pos
-		if strings.ToLower($1.str) != strings.ToLower(MetaField) {
+		if strings.ToLower($1.str) != strings.ToLower(ast.MetaField) {
 			// this will ultimately cause a parser error to occur...
 			yylex.Error(fmt.Sprintf("%s: %s", ErrParseResFieldInvalid, $1.str))
 		}
-		$$.resMeta = &StmtResMeta{
+		$$.resMeta = &ast.StmtResMeta{
 			Property: $1.str,
 			MetaExpr: $3.expr,
 		}
@@ -1066,11 +1068,11 @@ conditional_resource_meta_struct:
 	CAPITALIZED_IDENTIFIER ROCKET expr ELVIS expr COMMA
 	{
 		posLast(yylex, yyDollar) // our pos
-		if strings.ToLower($1.str) != strings.ToLower(MetaField) {
+		if strings.ToLower($1.str) != strings.ToLower(ast.MetaField) {
 			// this will ultimately cause a parser error to occur...
 			yylex.Error(fmt.Sprintf("%s: %s", ErrParseResFieldInvalid, $1.str))
 		}
-		$$.resMeta = &StmtResMeta{
+		$$.resMeta = &ast.StmtResMeta{
 			Property:  $1.str,
 			MetaExpr:  $5.expr,
 			Condition: $3.expr,
@@ -1084,7 +1086,7 @@ edge:
 	edge_half_list
 	{
 		posLast(yylex, yyDollar) // our pos
-		$$.stmt = &StmtEdge{
+		$$.stmt = &ast.StmtEdge{
 			EdgeHalfList: $1.edgeHalfList,
 			//Notify: false, // unused here
 		}
@@ -1093,8 +1095,8 @@ edge:
 |	edge_half_sendrecv ARROW edge_half_sendrecv
 	{
 		posLast(yylex, yyDollar) // our pos
-		$$.stmt = &StmtEdge{
-			EdgeHalfList: []*StmtEdgeHalf{
+		$$.stmt = &ast.StmtEdge{
+			EdgeHalfList: []*ast.StmtEdgeHalf{
 				$1.edgeHalf,
 				$3.edgeHalf,
 			},
@@ -1106,7 +1108,7 @@ edge_half_list:
 	edge_half
 	{
 		posLast(yylex, yyDollar) // our pos
-		$$.edgeHalfList = []*StmtEdgeHalf{$1.edgeHalf}
+		$$.edgeHalfList = []*ast.StmtEdgeHalf{$1.edgeHalf}
 	}
 |	edge_half_list ARROW edge_half
 	{
@@ -1119,7 +1121,7 @@ edge_half:
 	capitalized_res_identifier OPEN_BRACK expr CLOSE_BRACK
 	{
 		posLast(yylex, yyDollar) // our pos
-		$$.edgeHalf = &StmtEdgeHalf{
+		$$.edgeHalf = &ast.StmtEdgeHalf{
 			Kind: $1.str,
 			Name: $3.expr,
 			//SendRecv: "", // unused
@@ -1131,7 +1133,7 @@ edge_half_sendrecv:
 	capitalized_res_identifier OPEN_BRACK expr CLOSE_BRACK DOT IDENTIFIER
 	{
 		posLast(yylex, yyDollar) // our pos
-		$$.edgeHalf = &StmtEdgeHalf{
+		$$.edgeHalf = &ast.StmtEdgeHalf{
 			Kind: $1.str,
 			Name: $3.expr,
 			SendRecv: $6.str,

lang/unification_test.go

@@ -17,13 +17,15 @@
 
 // +build !root
 
-package lang
+package lang // XXX: move this to the unification package
 
 import (
 	"fmt"
 	"strings"
 	"testing"
 
+	"github.com/purpleidea/mgmt/lang/ast"
+	"github.com/purpleidea/mgmt/lang/funcs"
 	"github.com/purpleidea/mgmt/lang/funcs/vars"
 	"github.com/purpleidea/mgmt/lang/interfaces"
 	"github.com/purpleidea/mgmt/lang/types"
@@ -52,14 +54,14 @@ func TestUnification1(t *testing.T) {
 	//	})
 	//}
 	{
-		expr := &ExprStr{V: "hello"}
-		stmt := &StmtProg{
+		expr := &ast.ExprStr{V: "hello"}
+		stmt := &ast.StmtProg{
 			Body: []interfaces.Stmt{
-				&StmtRes{
+				&ast.StmtRes{
 					Kind: "test",
-					Name: &ExprStr{V: "t1"},
-					Contents: []StmtResContents{
-						&StmtResField{
+					Name: &ast.ExprStr{V: "t1"},
+					Contents: []ast.StmtResContents{
+						&ast.StmtResField{
 							Field: "str",
 							Value: expr,
 						},
@@ -77,23 +79,23 @@ func TestUnification1(t *testing.T) {
 		})
 	}
 	{
-		v1 := &ExprStr{}
-		v2 := &ExprStr{}
-		v3 := &ExprStr{}
-		expr := &ExprList{
+		v1 := &ast.ExprStr{}
+		v2 := &ast.ExprStr{}
+		v3 := &ast.ExprStr{}
+		expr := &ast.ExprList{
 			Elements: []interfaces.Expr{
 				v1,
 				v2,
 				v3,
 			},
 		}
-		stmt := &StmtProg{
+		stmt := &ast.StmtProg{
 			Body: []interfaces.Stmt{
-				&StmtRes{
+				&ast.StmtRes{
 					Kind: "test",
-					Name: &ExprStr{V: "test"},
-					Contents: []StmtResContents{
-						&StmtResField{
+					Name: &ast.ExprStr{V: "test"},
+					Contents: []ast.StmtResContents{
+						&ast.StmtResField{
 							Field: "slicestring",
 							Value: expr,
 						},
@@ -114,26 +116,26 @@ func TestUnification1(t *testing.T) {
 		})
 	}
 	{
-		k1 := &ExprInt{}
-		k2 := &ExprInt{}
-		k3 := &ExprInt{}
-		v1 := &ExprFloat{}
-		v2 := &ExprFloat{}
-		v3 := &ExprFloat{}
-		expr := &ExprMap{
-			KVs: []*ExprMapKV{
+		k1 := &ast.ExprInt{}
+		k2 := &ast.ExprInt{}
+		k3 := &ast.ExprInt{}
+		v1 := &ast.ExprFloat{}
+		v2 := &ast.ExprFloat{}
+		v3 := &ast.ExprFloat{}
+		expr := &ast.ExprMap{
+			KVs: []*ast.ExprMapKV{
 				{Key: k1, Val: v1},
 				{Key: k2, Val: v2},
 				{Key: k3, Val: v3},
 			},
 		}
-		stmt := &StmtProg{
+		stmt := &ast.StmtProg{
 			Body: []interfaces.Stmt{
-				&StmtRes{
+				&ast.StmtRes{
 					Kind: "test",
-					Name: &ExprStr{V: "test"},
-					Contents: []StmtResContents{
-						&StmtResField{
+					Name: &ast.ExprStr{V: "test"},
+					Contents: []ast.StmtResContents{
+						&ast.StmtResField{
 							Field: "mapintfloat",
 							Value: expr,
 						},
@@ -157,25 +159,25 @@ func TestUnification1(t *testing.T) {
 		})
 	}
 	{
-		b := &ExprBool{}
-		s := &ExprStr{}
-		i := &ExprInt{}
-		f := &ExprFloat{}
-		expr := &ExprStruct{
-			Fields: []*ExprStructField{
+		b := &ast.ExprBool{}
+		s := &ast.ExprStr{}
+		i := &ast.ExprInt{}
+		f := &ast.ExprFloat{}
+		expr := &ast.ExprStruct{
+			Fields: []*ast.ExprStructField{
 				{Name: "somebool", Value: b},
 				{Name: "somestr", Value: s},
 				{Name: "someint", Value: i},
 				{Name: "somefloat", Value: f},
 			},
 		}
-		stmt := &StmtProg{
+		stmt := &ast.StmtProg{
 			Body: []interfaces.Stmt{
-				&StmtRes{
+				&ast.StmtRes{
 					Kind: "test",
-					Name: &ExprStr{V: "test"},
-					Contents: []StmtResContents{
-						&StmtResField{
+					Name: &ast.ExprStr{V: "test"},
+					Contents: []ast.StmtResContents{
+						&ast.StmtResField{
 							Field: "mixedstruct",
 							Value: expr,
 						},
@@ -200,30 +202,30 @@ func TestUnification1(t *testing.T) {
 		// test "n1" {
 		//	int64ptr => 13 + 42,
 		//}
-		expr := &ExprCall{
-			Name: operatorFuncName,
+		expr := &ast.ExprCall{
+			Name: funcs.OperatorFuncName,
 			Args: []interfaces.Expr{
-				&ExprStr{
+				&ast.ExprStr{
 					V: "+",
 				},
-				&ExprInt{
+				&ast.ExprInt{
 					V: 13,
 				},
 
-				&ExprInt{
+				&ast.ExprInt{
 					V: 42,
 				},
 			},
 		}
-		stmt := &StmtProg{
+		stmt := &ast.StmtProg{
 			Body: []interfaces.Stmt{
-				&StmtRes{
+				&ast.StmtRes{
 					Kind: "test",
-					Name: &ExprStr{
+					Name: &ast.ExprStr{
 						V: "n1",
 					},
-					Contents: []StmtResContents{
-						&StmtResField{
+					Contents: []ast.StmtResContents{
+						&ast.StmtResField{
 							Field: "int64ptr",
 							Value: expr, // func
 						},
@@ -244,41 +246,41 @@ func TestUnification1(t *testing.T) {
 		//test "n1" {
 		//	int64ptr => 13 + 42 - 4,
 		//}
-		innerFunc := &ExprCall{
-			Name: operatorFuncName,
+		innerFunc := &ast.ExprCall{
+			Name: funcs.OperatorFuncName,
 			Args: []interfaces.Expr{
-				&ExprStr{
+				&ast.ExprStr{
 					V: "-",
 				},
-				&ExprInt{
+				&ast.ExprInt{
 					V: 42,
 				},
-				&ExprInt{
+				&ast.ExprInt{
 					V: 4,
 				},
 			},
 		}
-		expr := &ExprCall{
-			Name: operatorFuncName,
+		expr := &ast.ExprCall{
+			Name: funcs.OperatorFuncName,
 			Args: []interfaces.Expr{
-				&ExprStr{
+				&ast.ExprStr{
 					V: "+",
 				},
-				&ExprInt{
+				&ast.ExprInt{
 					V: 13,
 				},
 				innerFunc, // nested func, can we unify?
 			},
 		}
-		stmt := &StmtProg{
+		stmt := &ast.StmtProg{
 			Body: []interfaces.Stmt{
-				&StmtRes{
+				&ast.StmtRes{
 					Kind: "test",
-					Name: &ExprStr{
+					Name: &ast.ExprStr{
 						V: "n1",
 					},
-					Contents: []StmtResContents{
-						&StmtResField{
+					Contents: []ast.StmtResContents{
+						&ast.StmtResField{
 							Field: "int64ptr",
 							Value: expr,
 						},
@@ -300,41 +302,41 @@ func TestUnification1(t *testing.T) {
 		//test "n1" {
 		//	float32 => -25.38789 + 32.6 + 13.7,
 		//}
-		innerFunc := &ExprCall{
-			Name: operatorFuncName,
+		innerFunc := &ast.ExprCall{
+			Name: funcs.OperatorFuncName,
 			Args: []interfaces.Expr{
-				&ExprStr{
+				&ast.ExprStr{
 					V: "+",
 				},
-				&ExprFloat{
+				&ast.ExprFloat{
 					V: 32.6,
 				},
-				&ExprFloat{
+				&ast.ExprFloat{
 					V: 13.7,
 				},
 			},
 		}
-		expr := &ExprCall{
-			Name: operatorFuncName,
+		expr := &ast.ExprCall{
+			Name: funcs.OperatorFuncName,
 			Args: []interfaces.Expr{
-				&ExprStr{
+				&ast.ExprStr{
 					V: "+",
 				},
-				&ExprFloat{
+				&ast.ExprFloat{
 					V: -25.38789,
 				},
 				innerFunc, // nested func, can we unify?
 			},
 		}
-		stmt := &StmtProg{
+		stmt := &ast.StmtProg{
 			Body: []interfaces.Stmt{
-				&StmtRes{
+				&ast.StmtRes{
 					Kind: "test",
-					Name: &ExprStr{
+					Name: &ast.ExprStr{
 						V: "n1",
 					},
-					Contents: []StmtResContents{
-						&StmtResField{
+					Contents: []ast.StmtResContents{
+						&ast.StmtResField{
 							Field: "float32",
 							Value: expr,
 						},
@@ -357,35 +359,35 @@ func TestUnification1(t *testing.T) {
 		//test "t1" {
 		//	int64 => $x,
 		//}
-		innerFunc := &ExprCall{
-			Name: operatorFuncName,
+		innerFunc := &ast.ExprCall{
+			Name: funcs.OperatorFuncName,
 			Args: []interfaces.Expr{
-				&ExprStr{
+				&ast.ExprStr{
 					V: "-",
 				},
-				&ExprInt{
+				&ast.ExprInt{
 					V: 42,
 				},
-				&ExprInt{
+				&ast.ExprInt{
 					V: 13,
 				},
 			},
 		}
-		stmt := &StmtProg{
+		stmt := &ast.StmtProg{
 			Body: []interfaces.Stmt{
-				&StmtBind{
+				&ast.StmtBind{
 					Ident: "x",
 					Value: innerFunc,
 				},
-				&StmtRes{
+				&ast.StmtRes{
 					Kind: "test",
-					Name: &ExprStr{
+					Name: &ast.ExprStr{
 						V: "t1",
 					},
-					Contents: []StmtResContents{
-						&StmtResField{
+					Contents: []ast.StmtResContents{
+						&ast.StmtResField{
 							Field: "int64",
-							Value: &ExprVar{
+							Value: &ast.ExprVar{
 								Name: "x",
 							},
 						},
@@ -407,32 +409,32 @@ func TestUnification1(t *testing.T) {
 		//test "t1" {
 		//	anotherstr => $x,
 		//}
-		innerFunc := &ExprCall{
+		innerFunc := &ast.ExprCall{
 			Name: "template",
 			Args: []interfaces.Expr{
-				&ExprStr{
+				&ast.ExprStr{
 					V: "hello",
 				},
-				&ExprInt{
+				&ast.ExprInt{
 					V: 42,
 				},
 			},
 		}
-		stmt := &StmtProg{
+		stmt := &ast.StmtProg{
 			Body: []interfaces.Stmt{
-				&StmtBind{
+				&ast.StmtBind{
 					Ident: "x",
 					Value: innerFunc,
 				},
-				&StmtRes{
+				&ast.StmtRes{
 					Kind: "test",
-					Name: &ExprStr{
+					Name: &ast.ExprStr{
 						V: "t1",
 					},
-					Contents: []StmtResContents{
-						&StmtResField{
+					Contents: []ast.StmtResContents{
+						&ast.StmtResField{
 							Field: "anotherstr",
-							Value: &ExprVar{
+							Value: &ast.ExprVar{
 								Name: "x",
 							},
 						},
@@ -455,38 +457,38 @@ func TestUnification1(t *testing.T) {
 		//test "t1" {
 		//	anotherstr => $x,
 		//}
-		innerFunc := &ExprCall{
+		innerFunc := &ast.ExprCall{
 			Name: "template",
 			Args: []interfaces.Expr{
-				&ExprStr{
+				&ast.ExprStr{
 					V: "hello", // whatever...
 				},
-				&ExprVar{
+				&ast.ExprVar{
 					Name: "v",
 				},
 			},
 		}
-		stmt := &StmtProg{
+		stmt := &ast.StmtProg{
 			Body: []interfaces.Stmt{
-				&StmtBind{
+				&ast.StmtBind{
 					Ident: "v",
-					Value: &ExprInt{
+					Value: &ast.ExprInt{
 						V: 42,
 					},
 				},
-				&StmtBind{
+				&ast.StmtBind{
 					Ident: "x",
 					Value: innerFunc,
 				},
-				&StmtRes{
+				&ast.StmtRes{
 					Kind: "test",
-					Name: &ExprStr{
+					Name: &ast.ExprStr{
 						V: "t1",
 					},
-					Contents: []StmtResContents{
-						&StmtResField{
+					Contents: []ast.StmtResContents{
+						&ast.StmtResField{
 							Field: "anotherstr",
-							Value: &ExprVar{
+							Value: &ast.ExprVar{
 								Name: "x",
 							},
 						},
@@ -508,20 +510,20 @@ func TestUnification1(t *testing.T) {
 		//test "t1" {
 		//	stringptr => datetime.now(),	# bad (str vs. int)
 		//}
-		expr := &ExprCall{
+		expr := &ast.ExprCall{
 			Name: "datetime.now",
 			Args: []interfaces.Expr{},
 		}
-		stmt := &StmtProg{
+		stmt := &ast.StmtProg{
 			Body: []interfaces.Stmt{
-				&StmtImport{
+				&ast.StmtImport{
 					Name: "datetime",
 				},
-				&StmtRes{
+				&ast.StmtRes{
 					Kind: "test",
-					Name: &ExprStr{V: "t1"},
-					Contents: []StmtResContents{
-						&StmtResField{
+					Name: &ast.ExprStr{V: "t1"},
+					Contents: []ast.StmtResContents{
+						&ast.StmtResField{
 							Field: "stringptr",
 							Value: expr,
 						},
@@ -540,27 +542,27 @@ func TestUnification1(t *testing.T) {
 		//test "t1" {
 		//	stringptr => sys.getenv("GOPATH", "bug"),	# bad (two args vs. one)
 		//}
-		expr := &ExprCall{
+		expr := &ast.ExprCall{
 			Name: "sys.getenv",
 			Args: []interfaces.Expr{
-				&ExprStr{
+				&ast.ExprStr{
 					V: "GOPATH",
 				},
-				&ExprStr{
+				&ast.ExprStr{
 					V: "bug",
 				},
 			},
 		}
-		stmt := &StmtProg{
+		stmt := &ast.StmtProg{
 			Body: []interfaces.Stmt{
-				&StmtImport{
+				&ast.StmtImport{
 					Name: "sys",
 				},
-				&StmtRes{
+				&ast.StmtRes{
 					Kind: "test",
-					Name: &ExprStr{V: "t1"},
-					Contents: []StmtResContents{
-						&StmtResField{
+					Name: &ast.ExprStr{V: "t1"},
+					Contents: []ast.StmtResContents{
+						&ast.StmtResField{
 							Field: "stringptr",
 							Value: expr,
 						},
@@ -580,27 +582,27 @@ func TestUnification1(t *testing.T) {
 	//	//test "t1" {
 	//	//	stringptr => fmt.printf("hello %s and %s", "one"),	# bad
 	//	//}
-	//	expr := &ExprCall{
+	//	expr := &ast.ExprCall{
 	//		Name: "fmt.printf",
 	//		Args: []interfaces.Expr{
-	//			&ExprStr{
+	//			&ast.ExprStr{
 	//				V: "hello %s and %s",
 	//			},
-	//			&ExprStr{
+	//			&ast.ExprStr{
 	//				V: "one",
 	//			},
 	//		},
 	//	}
-	//	stmt := &StmtProg{
+	//	stmt := &ast.StmtProg{
 	//		Body: []interfaces.Stmt{
-	//			&StmtImport{
+	//			&ast.StmtImport{
 	//				Name: "fmt",
 	//			},
-	//			&StmtRes{
+	//			&ast.StmtRes{
 	//				Kind: "test",
-	//				Name: &ExprStr{V: "t1"},
-	//				Contents: []StmtResContents{
-	//					&StmtResField{
+	//				Name: &ast.ExprStr{V: "t1"},
+	//				Contents: []ast.StmtResContents{
+	//					&ast.StmtResField{
 	//						Field: "stringptr",
 	//						Value: expr,
 	//					},
@@ -619,33 +621,33 @@ func TestUnification1(t *testing.T) {
 	//	//test "t1" {
 	//	//	stringptr => fmt.printf("hello %s and %s", "one", "two", "three"),	# bad
 	//	//}
-	//	expr := &ExprCall{
+	//	expr := &ast.ExprCall{
 	//		Name: "fmt.printf",
 	//		Args: []interfaces.Expr{
-	//			&ExprStr{
+	//			&ast.ExprStr{
 	//				V: "hello %s and %s",
 	//			},
-	//			&ExprStr{
+	//			&ast.ExprStr{
 	//				V: "one",
 	//			},
-	//			&ExprStr{
+	//			&ast.ExprStr{
 	//				V: "two",
 	//			},
-	//			&ExprStr{
+	//			&ast.ExprStr{
 	//				V: "three",
 	//			},
 	//		},
 	//	}
-	//	stmt := &StmtProg{
+	//	stmt := &ast.StmtProg{
 	//		Body: []interfaces.Stmt{
-	//			&StmtImport{
+	//			&ast.StmtImport{
 	//				Name: "fmt",
 	//			},
-	//			&StmtRes{
+	//			&ast.StmtRes{
 	//				Kind: "test",
-	//				Name: &ExprStr{V: "t1"},
-	//				Contents: []StmtResContents{
-	//					&StmtResField{
+	//				Name: &ast.ExprStr{V: "t1"},
+	//				Contents: []ast.StmtResContents{
+	//					&ast.StmtResField{
 	//						Field: "stringptr",
 	//						Value: expr,
 	//					},
@@ -664,30 +666,30 @@ func TestUnification1(t *testing.T) {
 		//test "t1" {
 		//	stringptr => fmt.printf("hello %s and %s", "one", "two"),
 		//}
-		expr := &ExprCall{
+		expr := &ast.ExprCall{
 			Name: "fmt.printf",
 			Args: []interfaces.Expr{
-				&ExprStr{
+				&ast.ExprStr{
 					V: "hello %s and %s",
 				},
-				&ExprStr{
+				&ast.ExprStr{
 					V: "one",
 				},
-				&ExprStr{
+				&ast.ExprStr{
 					V: "two",
 				},
 			},
 		}
-		stmt := &StmtProg{
+		stmt := &ast.StmtProg{
 			Body: []interfaces.Stmt{
-				&StmtImport{
+				&ast.StmtImport{
 					Name: "fmt",
 				},
-				&StmtRes{
+				&ast.StmtRes{
 					Kind: "test",
-					Name: &ExprStr{V: "t1"},
-					Contents: []StmtResContents{
-						&StmtResField{
+					Name: &ast.ExprStr{V: "t1"},
+					Contents: []ast.StmtResContents{
+						&ast.StmtResField{
 							Field: "stringptr",
 							Value: expr,
 						},
@@ -714,37 +716,37 @@ func TestUnification1(t *testing.T) {
 		//test "t1" {
 		//	stringptr => fmt.printf("hello %s", $x),
 		//}
-		cond := &ExprIf{
-			Condition:  &ExprBool{V: true},
-			ThenBranch: &ExprInt{V: 42},
-			ElseBranch: &ExprInt{V: 13},
+		cond := &ast.ExprIf{
+			Condition:  &ast.ExprBool{V: true},
+			ThenBranch: &ast.ExprInt{V: 42},
+			ElseBranch: &ast.ExprInt{V: 13},
 		}
 		cond.SetType(types.TypeStr) // should fail unification
-		expr := &ExprCall{
+		expr := &ast.ExprCall{
 			Name: "fmt.printf",
 			Args: []interfaces.Expr{
-				&ExprStr{
+				&ast.ExprStr{
 					V: "hello %s",
 				},
-				&ExprVar{
+				&ast.ExprVar{
 					Name: "x", // the var
 				},
 			},
 		}
-		stmt := &StmtProg{
+		stmt := &ast.StmtProg{
 			Body: []interfaces.Stmt{
-				&StmtImport{
+				&ast.StmtImport{
 					Name: "fmt",
 				},
-				&StmtBind{
+				&ast.StmtBind{
 					Ident: "x", // the var
 					Value: cond,
 				},
-				&StmtRes{
+				&ast.StmtRes{
 					Kind: "test",
-					Name: &ExprStr{V: "t1"},
-					Contents: []StmtResContents{
-						&StmtResField{
+					Name: &ast.ExprStr{V: "t1"},
+					Contents: []ast.StmtResContents{
+						&ast.StmtResField{
 							Field: "anotherstr",
 							Value: expr,
 						},
@@ -766,38 +768,38 @@ func TestUnification1(t *testing.T) {
 		//test "t1" {
 		//	stringptr => fmt.printf("hello %s", $x),
 		//}
-		wvar := &ExprBool{V: true}
-		xvar := &ExprVar{Name: "w"}
+		wvar := &ast.ExprBool{V: true}
+		xvar := &ast.ExprVar{Name: "w"}
 		xvar.SetType(types.TypeStr) // should fail unification
-		expr := &ExprCall{
+		expr := &ast.ExprCall{
 			Name: "fmt.printf",
 			Args: []interfaces.Expr{
-				&ExprStr{
+				&ast.ExprStr{
 					V: "hello %s",
 				},
-				&ExprVar{
+				&ast.ExprVar{
 					Name: "x", // the var
 				},
 			},
 		}
-		stmt := &StmtProg{
+		stmt := &ast.StmtProg{
 			Body: []interfaces.Stmt{
-				&StmtImport{
+				&ast.StmtImport{
 					Name: "fmt",
 				},
-				&StmtBind{
+				&ast.StmtBind{
 					Ident: "w",
 					Value: wvar,
 				},
-				&StmtBind{
+				&ast.StmtBind{
 					Ident: "x", // the var
 					Value: xvar,
 				},
-				&StmtRes{
+				&ast.StmtRes{
 					Kind: "test",
-					Name: &ExprStr{V: "t1"},
-					Contents: []StmtResContents{
-						&StmtResField{
+					Name: &ast.ExprStr{V: "t1"},
+					Contents: []ast.StmtResContents{
+						&ast.StmtResField{
 							Field: "anotherstr",
 							Value: expr,
 						},
@@ -825,16 +827,16 @@ func TestUnification1(t *testing.T) {
 		}
 		names = append(names, tc.name)
 		t.Run(fmt.Sprintf("test #%d (%s)", index, tc.name), func(t *testing.T) {
-			ast, fail, expect, experr, experrstr := tc.ast, tc.fail, tc.expect, tc.experr, tc.experrstr
+			xast, fail, expect, experr, experrstr := tc.ast, tc.fail, tc.expect, tc.experr, tc.experrstr
 
 			//str := strings.NewReader(code)
-			//ast, err := LexParse(str)
+			//xast, err := parser.LexParse(str)
 			//if err != nil {
 			//	t.Errorf("test #%d: lex/parse failed with: %+v", index, err)
 			//	return
 			//}
 			// TODO: print out the AST's so that we can see the types
-			t.Logf("\n\ntest #%d: AST (before): %+v\n", index, ast)
+			t.Logf("\n\ntest #%d: AST (before): %+v\n", index, xast)
 
 			data := &interfaces.Data{
 				// TODO: add missing fields here if/when needed
@@ -844,7 +846,7 @@ func TestUnification1(t *testing.T) {
 				},
 			}
 			// some of this might happen *after* interpolate in SetScope or Unify...
-			if err := ast.Init(data); err != nil {
+			if err := xast.Init(data); err != nil {
 				t.Errorf("test #%d: FAIL", index)
 				t.Errorf("test #%d: could not init and validate AST: %+v", index, err)
 				return
@@ -860,13 +862,13 @@ func TestUnification1(t *testing.T) {
 			//t.Logf("test #%d: astInterpolated: %+v", index, astInterpolated)
 
 			variables := map[string]interfaces.Expr{
-				"purpleidea": &ExprStr{V: "hello world!"}, // james says hi
-				//"hostname": &ExprStr{V: obj.Hostname},
+				"purpleidea": &ast.ExprStr{V: "hello world!"}, // james says hi
+				//"hostname": &ast.ExprStr{V: obj.Hostname},
 			}
-			consts := VarPrefixToVariablesScope(vars.ConstNamespace) // strips prefix!
-			addback := vars.ConstNamespace + interfaces.ModuleSep    // add it back...
+			consts := ast.VarPrefixToVariablesScope(vars.ConstNamespace) // strips prefix!
+			addback := vars.ConstNamespace + interfaces.ModuleSep        // add it back...
 			var err error
-			variables, err = MergeExprMaps(variables, consts, addback)
+			variables, err = ast.MergeExprMaps(variables, consts, addback)
 			if err != nil {
 				t.Errorf("test #%d: FAIL", index)
 				t.Errorf("test #%d: couldn't merge in consts: %+v", index, err)
@@ -877,10 +879,10 @@ func TestUnification1(t *testing.T) {
 			scope := &interfaces.Scope{
 				Variables: variables,
 				// all the built-in top-level, core functions enter here...
-				Functions: FuncPrefixToFunctionsScope(""), // runs funcs.LookupPrefix
+				Functions: ast.FuncPrefixToFunctionsScope(""), // runs funcs.LookupPrefix
 			}
 			// propagate the scope down through the AST...
-			if err := ast.SetScope(scope); err != nil {
+			if err := xast.SetScope(scope); err != nil {
 				t.Errorf("test #%d: FAIL", index)
 				t.Errorf("test #%d: set scope failed with: %+v", index, err)
 				return
@@ -891,7 +893,7 @@ func TestUnification1(t *testing.T) {
 				t.Logf(fmt.Sprintf("test #%d", index)+": unification: "+format, v...)
 			}
 			unifier := &unification.Unifier{
-				AST:    ast,
+				AST:    xast,
 				Solver: unification.SimpleInvariantSolverLogger(logf),
 				Debug:  testing.Verbose(),
 				Logf:   logf,
@@ -899,7 +901,7 @@ func TestUnification1(t *testing.T) {
 			err = unifier.Unify()
 
 			// TODO: print out the AST's so that we can see the types
-			t.Logf("\n\ntest #%d: AST (after): %+v\n", index, ast)
+			t.Logf("\n\ntest #%d: AST (after): %+v\n", index, xast)
 
 			if !fail && err != nil {
 				t.Errorf("test #%d: FAIL", index)

lang/util/util.go

@@ -19,13 +19,24 @@ package util
 
 import (
 	"fmt"
+	"net/url"
+	"path"
 	"regexp"
 	"strings"
 
+	"github.com/purpleidea/mgmt/lang/interfaces"
 	"github.com/purpleidea/mgmt/lang/types"
 	"github.com/purpleidea/mgmt/util/errwrap"
 )
 
+const (
+	// ModuleMagicPrefix is the prefix which, if found as a prefix to the
+	// last token in an import path, will be removed silently if there are
+	// remaining characters following the name. If this is the empty string
+	// then it will be ignored.
+	ModuleMagicPrefix = "mgmt-"
+)
+
 // HasDuplicateTypes returns an error if the list of types is not unique.
 func HasDuplicateTypes(typs []*types.Type) error {
 	// FIXME: do this comparison in < O(n^2) ?
@@ -127,3 +138,118 @@ func ValidateVarName(name string) error {
 
 	return nil
 }
+
+// ParseImportName parses an import name and returns the default namespace name
+// that should be used with it. For example, if the import name was:
+// "git://example.com/purpleidea/Module-Name", this might return an alias of
+// "module_name". It also returns a bunch of other data about the parsed import.
+// TODO: check for invalid or unwanted special characters
+func ParseImportName(name string) (*interfaces.ImportData, error) {
+	magicPrefix := ModuleMagicPrefix
+	if name == "" {
+		return nil, fmt.Errorf("empty name")
+	}
+	if strings.HasPrefix(name, "/") {
+		return nil, fmt.Errorf("absolute paths are not allowed")
+	}
+
+	u, err := url.Parse(name)
+	if err != nil {
+		return nil, errwrap.Wrapf(err, "name is not a valid url")
+	}
+	if u.Path == "" {
+		return nil, fmt.Errorf("empty path")
+	}
+	p := u.Path
+	// catch bad paths like: git:////home/james/ (note the quad slash!)
+	// don't penalize if we have a dir with a trailing slash at the end
+	if s := path.Clean(u.Path); u.Path != s && u.Path != s+"/" {
+		// TODO: are there any cases where this is not what we want?
+		return nil, fmt.Errorf("dirty path, cleaned it's: `%s`", s)
+	}
+
+	for strings.HasSuffix(p, "/") { // remove trailing slashes
+		p = p[:len(p)-len("/")]
+	}
+
+	split := strings.Split(p, "/") // take last chunk if slash separated
+	s := split[0]
+	if len(split) > 1 {
+		s = split[len(split)-1] // pick last chunk
+	}
+
+	// TODO: should we treat a special name: "purpleidea/mgmt-foo" as "foo"?
+	if magicPrefix != "" && strings.HasPrefix(s, magicPrefix) && len(s) > len(magicPrefix) {
+		s = s[len(magicPrefix):]
+	}
+
+	s = strings.Replace(s, "-", "_", -1) // XXX: allow underscores in IDENTIFIER
+	if strings.HasPrefix(s, "_") || strings.HasSuffix(s, "_") {
+		return nil, fmt.Errorf("name can't begin or end with dash or underscore")
+	}
+	alias := strings.ToLower(s)
+
+	// if this is a local import, it's a straight directory path
+	// if it's an fqdn import, it should contain a metadata file
+
+	// if there's no protocol prefix, then this must be a local path
+	isLocal := u.Scheme == ""
+	// if it has a trailing slash or .mcl extension it's not a system import
+	isSystem := isLocal && !strings.HasSuffix(u.Path, "/") && !strings.HasSuffix(u.Path, interfaces.DotFileNameExtension)
+	// is it a local file?
+	isFile := !isSystem && isLocal && strings.HasSuffix(u.Path, interfaces.DotFileNameExtension)
+	xpath := u.Path // magic path
+	if isSystem {
+		xpath = ""
+	}
+	if !isLocal {
+		host := u.Host // host or host:port
+		split := strings.Split(host, ":")
+		if l := len(split); l == 1 || l == 2 {
+			host = split[0]
+		} else {
+			return nil, fmt.Errorf("incorrect number of colons (%d) in hostname", l)
+		}
+		xpath = path.Join(host, xpath)
+	}
+	if !isLocal && !strings.HasSuffix(xpath, "/") {
+		xpath = xpath + "/"
+	}
+	// we're a git repo with a local path instead of an fqdn over http!
+	// this still counts as isLocal == false, since it's still a remote
+	if u.Host == "" && strings.HasPrefix(u.Path, "/") {
+		xpath = strings.TrimPrefix(xpath, "/") // make it a relative dir
+	}
+	if strings.HasPrefix(xpath, "/") { // safety check (programming error?)
+		return nil, fmt.Errorf("can't parse strange import")
+	}
+
+	// build a url to clone from if we're not local...
+	// TODO: consider adding some logic that is similar to the logic in:
+	// https://github.com/golang/go/blob/054640b54df68789d9df0e50575d21d9dbffe99f/src/cmd/go/internal/get/vcs.go#L972
+	// so that we can more correctly figure out the correct url to clone...
+	xurl := ""
+	if !isLocal {
+		u.Fragment = ""
+		// TODO: maybe look for ?sha1=... or ?tag=... to pick a real ref
+		u.RawQuery = ""
+		u.ForceQuery = false
+		xurl = u.String()
+	}
+
+	// if u.Path is local file like: foo/server.mcl alias should be "server"
+	// we should trim the alias to remove the .mcl (the dir is already gone)
+	if isFile && strings.HasSuffix(alias, interfaces.DotFileNameExtension) {
+		alias = strings.TrimSuffix(alias, interfaces.DotFileNameExtension)
+	}
+
+	return &interfaces.ImportData{
+		Name:     name, // save the original value here
+		Alias:    alias,
+		IsSystem: isSystem,
+		IsLocal:  isLocal,
+		IsFile:   isFile,
+		Path:     xpath,
+		URL:      xurl,
+	}, nil
+}

test/test-golint.sh

@@ -27,7 +27,7 @@ if [ "$COMMITS" != "" ] && [ "$COMMITS" -gt "1" ]; then
 fi
 
 # find all go files, exluding temporary directories and generated files
-LINT=$(find * -maxdepth 9 -iname '*.go' -not -path 'old/*' -not -path 'tmp/*' -not -path 'bindata/*' -not -path 'lang/y.go' -not -path 'lang/lexer.nn.go' -not -path 'lang/interpolate/parse.generated.go' -not -path 'lang/types/kind_stringer.go' -not -path 'vendor/*' -exec golint {} \;)	# current golint output
+LINT=$(find * -maxdepth 9 -iname '*.go' -not -path 'old/*' -not -path 'tmp/*' -not -path 'bindata/*' -not -path 'lang/parser/y.go' -not -path 'lang/parser/lexer.nn.go' -not -path 'lang/interpolate/parse.generated.go' -not -path 'lang/types/kind_stringer.go' -not -path 'vendor/*' -exec golint {} \;)	# current golint output
 
 COUNT=`echo -e "$LINT" | wc -l`	# number of golint problems in current branch
 [ "$LINT" = "" ] && echo PASS && exit	# everything is "perfect"
@@ -55,7 +55,7 @@ while read -r line; do
 done <<< "$NUMSTAT1"	# three < is the secret to putting a variable into read
 
 git checkout "$PREVIOUS" &>/dev/null	# previous commit
-LINT1=$(find * -maxdepth 9 -iname '*.go' -not -path 'old/*' -not -path 'tmp/*' -not -path 'bindata/*' -not -path 'lang/y.go' -not -path 'lang/lexer.nn.go' -not -path 'vendor/*' -exec golint {} \;)
+LINT1=$(find * -maxdepth 9 -iname '*.go' -not -path 'old/*' -not -path 'tmp/*' -not -path 'bindata/*' -not -path 'lang/parser/y.go' -not -path 'lang/parser/lexer.nn.go' -not -path 'vendor/*' -exec golint {} \;)
 COUNT1=`echo -e "$LINT1" | wc -l`	# number of golint problems in older branch
 
 # clean up

test/test-gometalinter.sh

@@ -45,8 +45,8 @@ gml="$gml --enable=misspell"
 
 # exclude generated files:
 # TODO: at least until https://github.com/alecthomas/gometalinter/issues/270
-gml="$gml --exclude=lang/lexer.nn.go"
-gml="$gml --exclude=lang/y.go"
+gml="$gml --exclude=lang/parser/lexer.nn.go"
+gml="$gml --exclude=lang/parser/y.go"
 gml="$gml --exclude=bindata/bindata.go"
 gml="$gml --exclude=lang/types/kind_stringer.go"
 gml="$gml --exclude=lang/interpolate/parse.generated.go"

test/test-govet.sh

@@ -109,7 +109,7 @@ function reflowed-comments() {
 		return 0
 	fi
 
-	if [ "$1" = './lang/lexer.nn.go' ]; then
+	if [ "$1" = './lang/parser/lexer.nn.go' ]; then
 		return 0
 	fi