lang: Add unification optimizations

This adds a unification optimizations API, and uses it to optimize the
embedded provisioner. With these turned on, type unification drops from
around 1m45s to 2.5s which is a 40x speedup.

Makefile

@@ -63,7 +63,7 @@ SRPM_BASE = $(PROGRAM)-$(VERSION)-$(RELEASE).src.rpm
 RPM = rpmbuild/RPMS/$(PROGRAM)-$(VERSION)-$(RELEASE).$(ARCH).rpm
 USERNAME := $(shell cat ~/.config/copr 2>/dev/null | grep username | awk -F '=' '{print $$2}' | tr -d ' ')
 SERVER = 'dl.fedoraproject.org'
-REMOTE_PATH = 'pub/alt/$(USERNAME)/$(PROGRAM)'
+REMOTE_PATH = '/srv/pub/alt/$(USERNAME)/$(PROGRAM)'
 ifneq ($(GOTAGS),)
 	BUILD_FLAGS = -tags '$(GOTAGS)'
 endif

cli/util/args.go

@@ -84,8 +84,10 @@ type LangArgs struct {
 	OnlyDownload bool `arg:"--only-download" help:"stop after downloading any missing imports"`
 	Update       bool `arg:"--update" help:"update all dependencies to the latest versions"`
 
-	OnlyUnify bool `arg:"--only-unify" help:"stop after type unification"`
+	OnlyUnify          bool     `arg:"--only-unify" help:"stop after type unification"`
-	SkipUnify bool `arg:"--skip-unify" help:"skip type unification"`
+	SkipUnify          bool     `arg:"--skip-unify" help:"skip type unification"`
+	UnifySolver        *string  `arg:"--unify-name" help:"pick a specific unification solver"`
+	UnifyOptimizations []string `arg:"--unify-optimizations" help:"list of unification optimizations to request (experts only)"`
 
 	Depth int `arg:"--depth" default:"-1" help:"max recursion depth limit (-1 is unlimited)"`
 

docs/release-notes/0.0.25

@@ -0,0 +1,343 @@
+I've just released version 0.0.25 of mgmt!
+
+> 686 files changed, 28391 insertions(+), 6935 deletions(-)
+
+This is the first release that I consider to be generally useful at
+solving real-world problems, without needing to be an mgmt expert. It's
+also the first release that includes a very real `mcl` codebase. An
+accompanying blog post is also available:
+https://purpleidea.com/blog/2024/03/27/a-new-provisioning-tool/
+
+With that, here are a few highlights from the release:
+
+* We have a new mgmt partner program. Please sign-up for early access
+to these release notes, along with other special privileges. Details
+at: https://bit.ly/mgmt-partner-program
+
+* You can build self-contained mgmt binaries that contain a custom
+application. An initial "provisioning tool" has been built in this way.
+Please see the blog post for more details.
+
+* Default lookup functions exist in the language, with syntactic sugar
+(the || operator) , so you can get a default value if one doesn't
+exist, eg: $some_struct->some_struct_field || "some_default_value".
+
+* Resource fields can now accept interface{} (any) types.
+
+* A panic feature now exists in the language.
+
+* The exec resource has new `donecmd` and `creates` fields. Of note,
+`creates` supports watches too!
+
+* Send/recv now works for autogrouped resources!
+
+* Added `include as` (for classes) to the language. Nested
+(sugar/hierarchical) classes are now supported to make this more
+powerful!
+
+* Stats are printed if the function engine is waiting for too long.
+
+* There's a new http:flag resource, and also an http:proxy resource so
+that we can have caching http proxies!
+
+* Added a firewalld resource for opening ports!
+
+* Added a dhcp:range resource which is very powerful and has a fun API!
+
+* Added the "embedded" and "entry" packages, for building standalone
+tools. This goes perfectly with the new CLI library that we ported
+everything to.
+
+And much more...
+
+
+DOWNLOAD
+
+Prebuilt binaries are available here for this release:
+https://github.com/purpleidea/mgmt/releases/tag/0.0.25
+
+They can also be found on the Fedora mirror:
+https://dl.fedoraproject.org/pub/alt/purpleidea/mgmt/releases/0.0.25/
+
+
+NEWS
+
+* We changed the logical operators in mcl to use well-known English
+tokens: OR, AND, NOT. (but in lowercase of course)
+
+* The history function has been temporarily removed from the syntactic
+core. We'll add it back if we find it's useful to have sugar!
+
+* A bunch of lexer/parser cleanups and improvements were made.
+
+* Default lookup functions for lists, maps, and structs have been
+added. These come with syntactic sugar as mentioned above. (We plan to
+keep this syntax, but we're open to feedback and changes if they're
+good.)
+
+* Resources can accept the interface{} (any) type, although this should
+be used sparingly.
+
+* We added a new mcl test suite that checks resource output too!
+
+* Added a new `value` resource. This is a special resource kind that
+can be used for building some powerful state machines. Recommended for
+experienced users only.
+
+* Improved the golang function generation to allow functions that take
+[]str, so now we have a bunch more functions (like join) in our stdlib
+for free.
+
+* Add some mac address formatting functions. (core/net)
+
+* Added a panic resource and panic function into the core language.
+This is useful for safely shutting down a running mcl program to
+prevent something disastrous or unhandled.
+
+* Added a `donecmd` field to the exec resource. This runs a command
+after a successful CheckApply. This replaces the `&& echo foo > done`
+pattern that you'd see in some code.
+
+* Added a new internal `local` API that can be used for local machine
+operations. So far, read/writing/watching values that are stored
+locally.
+
+* Added `value` functions which bridge the `value` resource via the
+`local` API. To be used sparingly!
+
+* Bumped to golang 1.20, and we'll probably move again before the next
+release.
+
+* Allow send/recv with autogrouped resources. This adds many
+possibilities, in particular with the server style resources.
+
+* Added a bunch of tests for sneaky corner cases. Some of these were
+hard to write, but I think they're worth it.
+
+* ExprBind is now monomorphic! This was a design mistake that we
+introduced, but have since repaired. We now have far fewer copies
+running in the function graph, and things are much more efficient. This
+means lambdas can only have one type when used at two different call
+sites, which is much more logical, safer, faster and memory efficient.
+
+* Added an --only-unify option if you want to test your code but not
+run it.
+
+* Added a concat function for the common case of interpolation. This
+makes type unification significantly faster.
+
+* Eliminated some "benign" races. You might find this commit
+interesting to read: bc63b7608e84f60bf9d568188814d411a0688738
+
+* A pgraph bug was found and fixed. A test was added too! It's amazing
+this was here for so long, it just shows how subtle graph
+datastructures can be.
+
+* Added `include as` (for classes) to the language which lets our
+classes produce values which can then be used elsewhere. I decided this
+feature would be necessary after writing a bunch of mcl. It does have
+an extraneous scoping bug, but not anything that causes problems.
+
+* Nested classes are now supported. This lets you write the equivalent
+of nested classes, without actually having to nest them! This is not
+inheritance, but rather a way of handling scope and passing it
+downwards.
+
+* Improved the Ordering compiler step to catch a bunch of unhandled
+bugs. Sam is a genius and was able to figure out some of these using
+wizardry.
+
+* Added some convert functions to the mcl package.
+
+* Allow edges with colons...
+
+* ...Because we now support a new hierarchical autogrouping algorithm!
+This let's us have some very powerful resources.
+
+* ...Like http:*, dhcp:*, and so on, but we could even go deeper!
+
+* Fixed a super sneaky bug with resource swapping. Due to how we Cmp,
+this now preserves state more often, and in particular when we need it.
+I'm fairly certain that some code in a WIP branch of mine was actually
+blocked because of this issue. Pleased to run into it again, but now
+with a fix in place!
+
+* Added an http:flag resource. This let's a `wget` or similar call back
+to the http:server to kick off an action.
+
+* The http:flag resource supports directories now.
+
+* Stats are printed if the function engine is waiting for too long.
+This is mostly useful for developers who are building new functions and
+have a bug in their midst!
+
+* We added a --skip-unify option to prevent the double unification when
+running locally. When using `mgmt run` to test locally, we type check,
+and then deploy to ourselves, which then naturally type checks again.
+This skips the first one, which would be unsafe generally, but is
+perfectly safe when we're running a single instance.
+
+* Added a new http:proxy resource, and then tweaked it's API, and then
+added http streaming. This is an incredibly powerful resource that lets
+us build a caching http proxy with a single resource. I can't wait to
+see what else it gets used for. I'm using it for provisioning. It's not
+performance optimized at the moment as it uses a single mutex for
+everything, but this could be extended if we wanted to scale this out.
+
+* Added a ton of measuring/timing of common operations. This confirmed
+my belief that autoedges were slower than necessary. There are two ways
+to improve this. We might end up doing either one or both. Autogrouping
+is currently much faster than needed, so no improvements planned for
+now!
+
+* Started to clean up the internal FS API's. It would be really great
+if the core golang team would add something so we could get rid of the
+afero external interfaces.
+
+* Added an "embedded" package to offer API's related to embedded mcl
+programs! This lets us build standalone binaries which are powered by
+mcl.
+
+* Moved to a new CLI (go-arg) library. This has a few downsides, but
+they are fixable upstream, and this vastly improved our code quality
+and API's. This needed to happen, what with the mess that was
+urfave/cli. Look at our diff's, they're really elegant! This let us
+clean up our lib structs as well!
+
+* Added an "entry" package to kick-off the embedded API stuff. This
+uses the new CLI API's that we just built. The end-user now has a
+really easy time building new tools.
+
+* Added a bunch of util functions to aid in building certain standalone
+tools. I'm willing to accept more contributions in this space if
+they're sane, and related to our general mission. Please ask and then
+send patches if you're unsure.
+
+* Added a firewalld resource which makes opening up ports automatic
+when we need them. Perfect for the standalone laptop use-case.
+
+* Made type unification cancellable in case you get into a long-running
+scenario and want to end early.
+
+* Added a `creates` field to the exec resource. Very useful, and also
+supports watches! This is very useful for the common uses of exec.
+
+* Added a dhcp:range resource to offer any number of IP addresses to
+devices that we don't know the mac addresses of in advance. This makes
+building a provisioning tool even more ergonomic.
+
+* Optimized the name invariants since we can usually avoid an exclusive
+invariant in the common case. This roughly halved the type unification
+time. More improvements coming too!
+
+* Caught a sneaky list type that could get through type unification
+when it was interpolated alone. This now enforces the string invariant
+when we specify it, which is an important language design distinction.
+We added tests for this of course too!
+
+* The "log" package has been entirely refactored and is only visible in
+one place at the top of the program. Nice! I have a design for a
+"better logger / user interface" if we ever want to improve on this.
+
+* Added release targets for standalone binary builds. I also improved
+the Makefile release magic significantly.
+
+* Made a lot of small "polish" improvements to various resources.
+
+* Most interestingly, an embedded provisioner application has been
+built and made available in full. Please test and share with others.
+Hopefully this will encourage more interest in the project.
+
+* We're looking for help writing Amazon, Google, DigitalOcean, Hetzner,
+etc, resources if anyone is interested, reach out to us. Particularly
+if there is support from those organizations as well.
+
+* Many other bug fixes, changes, etc...
+
+* See the git log for more NEWS, and for anything notable I left out!
+
+
+BUGS/TODO
+
+* Function values getting _passed_ to resources doesn't work yet, but
+it's not a blocker, but it would definitely be useful. We're looking
+into it.
+
+* Function graphs are unnecessarily dynamic. We might make them more
+static so that we don't need as many transactions. This is really a
+compiler optimization and not a bug, but it's something important we'd
+like to have.
+
+* Running two Txn's during the same pause would be really helpful. I'm
+not sure how much of a performance improvement we'd get from this, but
+it would sure be interesting to build. If you want to build a fancy
+synchronization primitive, then let us know! Again this is not a bug.
+
+* Type unification performance can be improved drastically. I will have
+to implement the fast algorithm so that we can scale to very large mcl
+programs. Help is wanted if you are familiar with "unionfind" and/or
+type unification.
+
+
+TALKS
+
+I don't have anything planned until CfgMgmtCamp 2025. If you'd like to
+book me for a private event, or sponsor my travel for your conference,
+please let me know.
+
+I recently gave two talks: one at CfgMgmtCamp 2024, and one at FOSDEM
+in the golang room. Both are available online and demonstrated an
+earlier version of the provisioning tool which is fully available
+today. The talks can be found here: https://purpleidea.com/talks/
+
+
+PARTNER PROGRAM
+
+We have a new mgmt partner program which gets you early access to
+releases, bug fixes, support, and many other goodies. Please sign-up
+today: https://bit.ly/mgmt-partner-program
+
+
+MISC
+
+Our mailing list host (Red Hat) is no longer letting non-Red Hat
+employees use their infrastructure. We're looking for a new home. I've
+opened a ticket with Freedesktop. If you have any sway with them or
+other recommendations, please let me know:
+https://gitlab.freedesktop.org/freedesktop/freedesktop/-/issues/1082
+
+We're still looking for new contributors, and there are easy, medium
+and hard issues available! You're also welcome to suggest your own!
+Please join us in #mgmtconfig on Libera IRC or Matrix (preferred) and
+ping us if you'd like help getting started! For details please see:
+
+https://github.com/purpleidea/mgmt/blob/master/docs/faq.md#how-do-i-con
+tribute-to-the-project-if-i-dont-know-golang
+
+Many tagged #mgmtlove issues exist:
+https://github.com/purpleidea/mgmt/issues?q=is%3Aissue+is%3Aopen+label%
+3Amgmtlove
+
+Although asking in IRC/matrix is the best way to find something to work
+on.
+
+
+MENTORING
+
+We offer mentoring for new golang/mgmt hackers who want to get
+involved. This is fun and friendly! You get to improve your skills,
+and we get some patches in return. Ping me off-list for details.
+
+
+THANKS
+
+Thanks (alphabetically) to everyone who contributed to the latest
+release:
+Eng Zer Jun, James Shubin, Oliver Lowe, Samuel Gélineau
+We had 4 unique committers since 0.0.24, and have had 90 overall.
+run 'git log 0.0.24..0.0.25' to see what has changed since 0.0.24
+
+
+Happy hacking,
+James
+@purpleidea

lang/core/embedded/provisioner/provisioner.go

@@ -46,6 +46,7 @@ import (
 	"github.com/purpleidea/mgmt/lang/embedded"
 	"github.com/purpleidea/mgmt/lang/funcs/simple"
 	"github.com/purpleidea/mgmt/lang/types"
+	"github.com/purpleidea/mgmt/lang/unification/simplesolver" // TODO: remove me!
 	"github.com/purpleidea/mgmt/util"
 	"github.com/purpleidea/mgmt/util/errwrap"
 	"github.com/purpleidea/mgmt/util/password"
@@ -383,6 +384,12 @@ func (obj *provisioner) Customize(a interface{}) (*cli.RunArgs, error) {
 
 	// Make any changes here that we want to...
 	runArgs.RunLang.SkipUnify = true // speed things up for known good code
+	name := simplesolver.Name
+	// TODO: Remove these optimizations when the solver is faster overall.
+	runArgs.RunLang.UnifySolver = &name
+	runArgs.RunLang.UnifyOptimizations = []string{
+		simplesolver.OptimizationSkipFuncCmp,
+	}
 	libConfig.TmpPrefix = true
 	libConfig.NoPgp = true
 

lang/gapi/gapi.go

@@ -70,6 +70,9 @@ func init() {
 type GAPI struct {
 	InputURI string // input URI of code file system to run
 
+	// Data is some additional data for the lang struct.
+	Data *lang.Data
+
 	lang   *lang.Lang // lang struct
 	wgRun  *sync.WaitGroup
 	ctx    context.Context
@@ -261,6 +264,15 @@ func (obj *GAPI) Cli(info *gapi.Info) (*gapi.Deploy, error) {
 		return nil, nil // success!
 	}
 
+	unificationStrategy := make(map[string]string)
+	if name := args.UnifySolver; name != nil && *name != "" {
+		unificationStrategy[unification.StrategyNameKey] = *name
+	}
+	if len(args.UnifyOptimizations) > 0 {
+		// TODO: use a query string parser instead?
+		unificationStrategy[unification.StrategyOptimizationsKey] = strings.Join(args.UnifyOptimizations, ",")
+	}
+
 	if !args.SkipUnify {
 		// apply type unification
 		unificationLogf := func(format string, v ...interface{}) {
@@ -269,13 +281,19 @@ func (obj *GAPI) Cli(info *gapi.Info) (*gapi.Deploy, error) {
 			}
 		}
 		logf("running type unification...")
-		startTime := time.Now()
+
-		unifier := &unification.Unifier{
+		solver, err := unification.LookupDefault()
-			AST:    iast,
+		if err != nil {
-			Solver: unification.SimpleInvariantSolverLogger(unificationLogf),
+			return nil, errwrap.Wrapf(err, "could not get default solver")
-			Debug:  debug,
-			Logf:   unificationLogf,
 		}
+		unifier := &unification.Unifier{
+			AST:      iast,
+			Solver:   solver,
+			Strategy: unificationStrategy,
+			Debug:    debug,
+			Logf:     unificationLogf,
+		}
+		startTime := time.Now()
 		unifyErr := unifier.Unify(context.TODO())
 		delta := time.Since(startTime)
 		formatted := delta.String()
@@ -406,7 +424,10 @@ func (obj *GAPI) Cli(info *gapi.Info) (*gapi.Deploy, error) {
 		Sema: info.Flags.Sema,
 		GAPI: &GAPI{
 			InputURI: fs.URI(),
-			// TODO: add properties here...
+			Data: &lang.Data{
+				UnificationStrategy: unificationStrategy,
+				// TODO: add properties here...
+			},
 		},
 	}, nil
 }
@@ -446,6 +467,7 @@ func (obj *GAPI) LangInit(ctx context.Context) error {
 		Fs:    fs,
 		FsURI: obj.InputURI,
 		Input: input,
+		Data:  obj.Data,
 
 		Hostname: obj.data.Hostname,
 		Local:    obj.data.Local,

lang/interpret_test.go

@@ -458,9 +458,15 @@ func TestAstFunc1(t *testing.T) {
 			xlogf := func(format string, v ...interface{}) {
 				logf("unification: "+format, v...)
 			}
+			solver, err := unification.LookupDefault()
+			if err != nil {
+				t.Errorf("test #%d: FAIL", index)
+				t.Errorf("test #%d: solver lookup failed with: %+v", index, err)
+				return
+			}
 			unifier := &unification.Unifier{
 				AST:    iast,
-				Solver: unification.SimpleInvariantSolverLogger(xlogf),
+				Solver: solver,
 				Debug:  testing.Verbose(),
 				Logf:   xlogf,
 			}
@@ -1028,9 +1034,15 @@ func TestAstFunc2(t *testing.T) {
 			xlogf := func(format string, v ...interface{}) {
 				logf("unification: "+format, v...)
 			}
+			solver, err := unification.LookupDefault()
+			if err != nil {
+				t.Errorf("test #%d: FAIL", index)
+				t.Errorf("test #%d: solver lookup failed with: %+v", index, err)
+				return
+			}
 			unifier := &unification.Unifier{
 				AST:    iast,
-				Solver: unification.SimpleInvariantSolverLogger(xlogf),
+				Solver: solver,
 				Debug:  testing.Verbose(),
 				Logf:   xlogf,
 			}
@@ -1830,9 +1842,15 @@ func TestAstFunc3(t *testing.T) {
 			xlogf := func(format string, v ...interface{}) {
 				logf("unification: "+format, v...)
 			}
+			solver, err := unification.LookupDefault()
+			if err != nil {
+				t.Errorf("test #%d: FAIL", index)
+				t.Errorf("test #%d: solver lookup failed with: %+v", index, err)
+				return
+			}
 			unifier := &unification.Unifier{
 				AST:    iast,
-				Solver: unification.SimpleInvariantSolverLogger(xlogf),
+				Solver: solver,
 				Debug:  testing.Verbose(),
 				Logf:   xlogf,
 			}

lang/lang.go

@@ -50,6 +50,7 @@ import (
 	"github.com/purpleidea/mgmt/lang/interpret"
 	"github.com/purpleidea/mgmt/lang/parser"
 	"github.com/purpleidea/mgmt/lang/unification"
+	_ "github.com/purpleidea/mgmt/lang/unification/solvers" // import so the solvers register
 	"github.com/purpleidea/mgmt/pgraph"
 	"github.com/purpleidea/mgmt/util"
 	"github.com/purpleidea/mgmt/util/errwrap"
@@ -63,6 +64,18 @@ const (
 	EngineStartupStatsTimeout = 10
 )
 
+// Data is some data that is passed into the Lang struct. It is presented here
+// as a single struct with room for multiple fields so that it can be changed or
+// extended in the future without having to re-plumb through all the fields it
+// contains
+type Data struct {
+	// UnificationStrategy is a hack to tune unification performance until
+	// we have an overall cleaner unification algorithm in place.
+	UnificationStrategy map[string]string
+
+	// TODO: Add other fields here if necessary.
+}
+
 // Lang is the main language lexer/parser object.
 type Lang struct {
 	Fs    engine.Fs // connected fs where input dir or metadata exists
@@ -78,6 +91,9 @@ type Lang struct {
 	// run the raw string as mcl code.
 	Input string
 
+	// Data is some additional data for the lang struct.
+	Data *Data
+
 	Hostname string
 	Local    *local.API
 	World    engine.World
@@ -100,6 +116,12 @@ type Lang struct {
 // watching them, *before* we pull their values, that way we'll know if they
 // changed from the values we wanted.
 func (obj *Lang) Init(ctx context.Context) error {
+	if obj.Data == nil {
+		return fmt.Errorf("lang struct was not built properly")
+	}
+	if obj.Data.UnificationStrategy == nil {
+		return fmt.Errorf("lang struct was not built properly")
+	}
 	if obj.Debug {
 		obj.Logf("input: %s", obj.Input)
 		tree, err := util.FsTree(obj.Fs, "/") // should look like gapi
@@ -220,18 +242,29 @@ func (obj *Lang) Init(ctx context.Context) error {
 
 	// apply type unification
 	logf := func(format string, v ...interface{}) {
+		// TODO: Remove the masked logger here when unification is clean!
 		if obj.Debug { // unification only has debug messages...
 			obj.Logf("unification: "+format, v...)
 		}
 	}
 	obj.Logf("running type unification...")
-	timing = time.Now()
+
-	unifier := &unification.Unifier{
+	var solver unification.Solver
-		AST:    obj.ast,
+	if name, exists := obj.Data.UnificationStrategy["solver"]; exists && name != "" {
-		Solver: unification.SimpleInvariantSolverLogger(logf),
+		if solver, err = unification.Lookup(name); err != nil {
-		Debug:  obj.Debug,
+			return errwrap.Wrapf(err, "could not get solver: %s", name)
-		Logf:   logf,
+		}
+	} else if solver, err = unification.LookupDefault(); err != nil {
+		return errwrap.Wrapf(err, "could not get default solver")
 	}
+	unifier := &unification.Unifier{
+		AST:      obj.ast,
+		Solver:   solver,
+		Strategy: obj.Data.UnificationStrategy,
+		Debug:    obj.Debug,
+		Logf:     logf,
+	}
+	timing = time.Now()
 	// NOTE: This is the "real" Unify that runs. (This is not for deploy.)
 	unifyErr := unifier.Unify(ctx)
 	obj.Logf("type unification took: %s", time.Since(timing))

lang/lang_test.go

@@ -137,7 +137,10 @@ func runInterpret(t *testing.T, code string) (_ *pgraph.Graph, reterr error) {
 	lang := &Lang{
 		Fs:    fs,
 		Input: "/" + interfaces.MetadataFilename, // start path in fs
-		Debug: testing.Verbose(),                 // set via the -test.v flag to `go test`
+		Data: &Data{
+			UnificationStrategy: make(map[string]string), // empty
+		},
+		Debug: testing.Verbose(), // set via the -test.v flag to `go test`
 		Logf:  logf,
 	}
 	if err := lang.Init(ctx); err != nil {

lang/unification/interfaces.go

@@ -0,0 +1,243 @@
+// Mgmt
+// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+//
+// Additional permission under GNU GPL version 3 section 7
+//
+// If you modify this program, or any covered work, by linking or combining it
+// with embedded mcl code and modules (and that the embedded mcl code and
+// modules which link with this program, contain a copy of their source code in
+// the authoritative form) containing parts covered by the terms of any other
+// license, the licensors of this program grant you additional permission to
+// convey the resulting work. Furthermore, the licensors of this program grant
+// the original author, James Shubin, additional permission to update this
+// additional permission if he deems it necessary to achieve the goals of this
+// additional permission.
+
+package unification
+
+import (
+	"context"
+	"fmt"
+	"sort"
+
+	"github.com/purpleidea/mgmt/lang/interfaces"
+	"github.com/purpleidea/mgmt/lang/types"
+)
+
+const (
+	// ErrAmbiguous means we couldn't find a solution, but we weren't
+	// inconsistent.
+	ErrAmbiguous = interfaces.Error("can't unify, no equalities were consumed, we're ambiguous")
+
+	// StrategyNameKey is the string key used when choosing a solver name.
+	StrategyNameKey = "name"
+
+	// StrategyOptimizationsKey is the string key used to tell the solver
+	// about the specific optimizations you'd like to request. The format
+	// can be specific to each solver.
+	StrategyOptimizationsKey = "optimizations"
+)
+
+// Init contains some handles that are used to initialize every solver. Each
+// individual solver can choose to omit using some of the fields.
+type Init struct {
+	// Strategy is a hack to tune unification performance until we have an
+	// overall cleaner unification algorithm in place.
+	Strategy map[string]string
+
+	Debug bool
+	Logf  func(format string, v ...interface{})
+}
+
+// Solver is the general interface that any solver needs to implement.
+type Solver interface {
+	// Init initializes the solver struct before first use.
+	Init(*Init) error
+
+	// Solve performs the actual solving. It must return as soon as possible
+	// if the context is closed.
+	Solve(ctx context.Context, invariants []interfaces.Invariant, expected []interfaces.Expr) (*InvariantSolution, error)
+}
+
+// registeredSolvers is a global map of all possible unification solvers which
+// can be used. You should never touch this map directly. Use methods like
+// Register instead.
+var registeredSolvers = make(map[string]func() Solver) // must initialize
+
+// Register takes a solver and its name and makes it available for use. It is
+// commonly called in the init() method of the solver at program startup. There
+// is no matching Unregister function.
+func Register(name string, solver func() Solver) {
+	if _, exists := registeredSolvers[name]; exists {
+		panic(fmt.Sprintf("a solver named %s is already registered", name))
+	}
+
+	//gob.Register(solver())
+	registeredSolvers[name] = solver
+}
+
+// Lookup returns a pointer to the solver's struct.
+func Lookup(name string) (Solver, error) {
+	solver, exists := registeredSolvers[name]
+	if !exists {
+		return nil, fmt.Errorf("not found")
+	}
+	return solver(), nil
+}
+
+// LookupDefault attempts to return a "default" solver.
+func LookupDefault() (Solver, error) {
+	if len(registeredSolvers) == 0 {
+		return nil, fmt.Errorf("no registered solvers")
+	}
+	if len(registeredSolvers) == 1 {
+		for _, solver := range registeredSolvers {
+			return solver(), nil // return the first and only one
+		}
+	}
+
+	// TODO: Should we remove this empty string feature?
+	// If one was registered with no name, then use that as the default.
+	if solver, exists := registeredSolvers[""]; exists { // empty name
+		return solver(), nil
+	}
+
+	return nil, fmt.Errorf("no registered default solver")
+}
+
+// DebugSolverState helps us in understanding the state of the type unification
+// solver in a more mainstream format.
+// Example:
+//
+// solver state:
+//
+// *	str("foo") :: str
+// *	call:f(str("foo")) [0xc000ac9f10] :: ?1
+// *	var(x) [0xc00088d840] :: ?2
+// *	param(x) [0xc00000f950] :: ?3
+// *	func(x) { var(x) } [0xc0000e9680] :: ?4
+// *	?2 = ?3
+// *	?4 = func(arg0 str) ?1
+// *	?4 = func(x str) ?2
+// *	?1 = ?2
+func DebugSolverState(solved map[interfaces.Expr]*types.Type, equalities []interfaces.Invariant) string {
+	s := ""
+
+	// all the relevant Exprs
+	count := 0
+	exprs := make(map[interfaces.Expr]int)
+	for _, equality := range equalities {
+		for _, expr := range equality.ExprList() {
+			count++
+			exprs[expr] = count // for sorting
+		}
+	}
+
+	// print the solved Exprs first
+	for expr, typ := range solved {
+		s += fmt.Sprintf("%v :: %v\n", expr, typ)
+		delete(exprs, expr)
+	}
+
+	sortedExprs := []interfaces.Expr{}
+	for k := range exprs {
+		sortedExprs = append(sortedExprs, k)
+	}
+	sort.Slice(sortedExprs, func(i, j int) bool { return exprs[sortedExprs[i]] < exprs[sortedExprs[j]] })
+
+	// for each remaining expr, generate a shorter name than the full pointer
+	nextVar := 1
+	shortNames := map[interfaces.Expr]string{}
+	for _, expr := range sortedExprs {
+		shortNames[expr] = fmt.Sprintf("?%d", nextVar)
+		nextVar++
+		s += fmt.Sprintf("%p %v :: %s\n", expr, expr, shortNames[expr])
+	}
+
+	// print all the equalities using the short names
+	for _, equality := range equalities {
+		switch e := equality.(type) {
+		case *interfaces.EqualsInvariant:
+			_, ok := solved[e.Expr]
+			if !ok {
+				s += fmt.Sprintf("%s = %v\n", shortNames[e.Expr], e.Type)
+			} else {
+				// if solved, then this is redundant, don't print anything
+			}
+
+		case *interfaces.EqualityInvariant:
+			type1, ok1 := solved[e.Expr1]
+			type2, ok2 := solved[e.Expr2]
+			if !ok1 && !ok2 {
+				s += fmt.Sprintf("%s = %s\n", shortNames[e.Expr1], shortNames[e.Expr2])
+			} else if ok1 && !ok2 {
+				s += fmt.Sprintf("%s = %s\n", type1, shortNames[e.Expr2])
+			} else if !ok1 && ok2 {
+				s += fmt.Sprintf("%s = %s\n", shortNames[e.Expr1], type2)
+			} else {
+				// if completely solved, then this is redundant, don't print anything
+			}
+
+		case *interfaces.EqualityWrapFuncInvariant:
+			funcType, funcOk := solved[e.Expr1]
+
+			args := ""
+			argsOk := true
+			for i, argName := range e.Expr2Ord {
+				if i > 0 {
+					args += ", "
+				}
+				argExpr := e.Expr2Map[argName]
+				argType, ok := solved[argExpr]
+				if !ok {
+					args += fmt.Sprintf("%s %s", argName, shortNames[argExpr])
+					argsOk = false
+				} else {
+					args += fmt.Sprintf("%s %s", argName, argType)
+				}
+			}
+
+			outType, outOk := solved[e.Expr2Out]
+
+			if !funcOk || !argsOk || !outOk {
+				if !funcOk && !outOk {
+					s += fmt.Sprintf("%s = func(%s) %s\n", shortNames[e.Expr1], args, shortNames[e.Expr2Out])
+				} else if !funcOk && outOk {
+					s += fmt.Sprintf("%s = func(%s) %s\n", shortNames[e.Expr1], args, outType)
+				} else if funcOk && !outOk {
+					s += fmt.Sprintf("%s = func(%s) %s\n", funcType, args, shortNames[e.Expr2Out])
+				} else {
+					s += fmt.Sprintf("%s = func(%s) %s\n", funcType, args, outType)
+				}
+			}
+
+		case *interfaces.CallFuncArgsValueInvariant:
+			// skip, not used in the examples I care about
+
+		case *interfaces.AnyInvariant:
+			// skip, not used in the examples I care about
+
+		case *interfaces.SkipInvariant:
+			// we don't care about this one
+
+		default:
+			s += fmt.Sprintf("%v\n", equality)
+		}
+	}
+
+	return s
+}

lang/unification/simplesolver/simplesolver.go

@@ -27,25 +27,34 @@
 // additional permission if he deems it necessary to achieve the goals of this
 // additional permission.
 
-package unification // TODO: can we put this solver in a sub-package?
+package simplesolver
 
 import (
 	"context"
 	"fmt"
 	"sort"
+	"strings"
 
 	"github.com/purpleidea/mgmt/lang/interfaces"
 	"github.com/purpleidea/mgmt/lang/types"
+	"github.com/purpleidea/mgmt/lang/unification"
 	"github.com/purpleidea/mgmt/util/errwrap"
 )
 
 const (
 	// Name is the prefix for our solver log messages.
-	Name = "solver: simple"
+	Name = "simple"
 
-	// ErrAmbiguous means we couldn't find a solution, but we weren't
+	// OptimizationSkipFuncCmp is the magic flag name to include the skip
-	// inconsistent.
+	// func cmp optimization which can speed up some simple programs. If
-	ErrAmbiguous = interfaces.Error("can't unify, no equalities were consumed, we're ambiguous")
+	// this is specified, then OptimizationHeuristicalDrop is redundant.
+	OptimizationSkipFuncCmp = "skip-func-cmp"
+
+	// OptimizationHeuristicalDrop is the magic flag name to include to tell
+	// the solver to drop some func compares. This is a less aggressive form
+	// of OptimizationSkipFuncCmp. This is redundant if
+	// OptimizationSkipFuncCmp is true.
+	OptimizationHeuristicalDrop = "heuristical-drop"
 
 	// AllowRecursion specifies whether we're allowed to use the recursive
 	// solver or not. It uses an absurd amount of memory, and might hang
@@ -61,154 +70,63 @@ const (
 	RecursionInvariantLimit = 5 // TODO: pick a better value ?
 )
 
-// SimpleInvariantSolverLogger is a wrapper which returns a
+func init() {
-// SimpleInvariantSolver with the log parameter of your choice specified. The
+	unification.Register(Name, func() unification.Solver { return &SimpleInvariantSolver{} })
-// result satisfies the correct signature for the solver parameter of the
-// Unification function.
-// TODO: Get rid of this function and consider just using the struct directly.
-func SimpleInvariantSolverLogger(logf func(format string, v ...interface{})) func(context.Context, []interfaces.Invariant, []interfaces.Expr) (*InvariantSolution, error) {
-	return func(ctx context.Context, invariants []interfaces.Invariant, expected []interfaces.Expr) (*InvariantSolution, error) {
-		sis := &SimpleInvariantSolver{
-			Debug: false, // TODO: consider plumbing this through
-			Logf:  logf,
-		}
-		return sis.Solve(ctx, invariants, expected)
-	}
-}
-
-// DebugSolverState helps us in understanding the state of the type unification
-// solver in a more mainstream format.
-// Example:
-//
-// solver state:
-//
-// *	str("foo") :: str
-// *	call:f(str("foo")) [0xc000ac9f10] :: ?1
-// *	var(x) [0xc00088d840] :: ?2
-// *	param(x) [0xc00000f950] :: ?3
-// *	func(x) { var(x) } [0xc0000e9680] :: ?4
-// *	?2 = ?3
-// *	?4 = func(arg0 str) ?1
-// *	?4 = func(x str) ?2
-// *	?1 = ?2
-func DebugSolverState(solved map[interfaces.Expr]*types.Type, equalities []interfaces.Invariant) string {
-	s := ""
-
-	// all the relevant Exprs
-	count := 0
-	exprs := make(map[interfaces.Expr]int)
-	for _, equality := range equalities {
-		for _, expr := range equality.ExprList() {
-			count++
-			exprs[expr] = count // for sorting
-		}
-	}
-
-	// print the solved Exprs first
-	for expr, typ := range solved {
-		s += fmt.Sprintf("%v :: %v\n", expr, typ)
-		delete(exprs, expr)
-	}
-
-	sortedExprs := []interfaces.Expr{}
-	for k := range exprs {
-		sortedExprs = append(sortedExprs, k)
-	}
-	sort.Slice(sortedExprs, func(i, j int) bool { return exprs[sortedExprs[i]] < exprs[sortedExprs[j]] })
-
-	// for each remaining expr, generate a shorter name than the full pointer
-	nextVar := 1
-	shortNames := map[interfaces.Expr]string{}
-	for _, expr := range sortedExprs {
-		shortNames[expr] = fmt.Sprintf("?%d", nextVar)
-		nextVar++
-		s += fmt.Sprintf("%p %v :: %s\n", expr, expr, shortNames[expr])
-	}
-
-	// print all the equalities using the short names
-	for _, equality := range equalities {
-		switch e := equality.(type) {
-		case *interfaces.EqualsInvariant:
-			_, ok := solved[e.Expr]
-			if !ok {
-				s += fmt.Sprintf("%s = %v\n", shortNames[e.Expr], e.Type)
-			} else {
-				// if solved, then this is redundant, don't print anything
-			}
-
-		case *interfaces.EqualityInvariant:
-			type1, ok1 := solved[e.Expr1]
-			type2, ok2 := solved[e.Expr2]
-			if !ok1 && !ok2 {
-				s += fmt.Sprintf("%s = %s\n", shortNames[e.Expr1], shortNames[e.Expr2])
-			} else if ok1 && !ok2 {
-				s += fmt.Sprintf("%s = %s\n", type1, shortNames[e.Expr2])
-			} else if !ok1 && ok2 {
-				s += fmt.Sprintf("%s = %s\n", shortNames[e.Expr1], type2)
-			} else {
-				// if completely solved, then this is redundant, don't print anything
-			}
-
-		case *interfaces.EqualityWrapFuncInvariant:
-			funcType, funcOk := solved[e.Expr1]
-
-			args := ""
-			argsOk := true
-			for i, argName := range e.Expr2Ord {
-				if i > 0 {
-					args += ", "
-				}
-				argExpr := e.Expr2Map[argName]
-				argType, ok := solved[argExpr]
-				if !ok {
-					args += fmt.Sprintf("%s %s", argName, shortNames[argExpr])
-					argsOk = false
-				} else {
-					args += fmt.Sprintf("%s %s", argName, argType)
-				}
-			}
-
-			outType, outOk := solved[e.Expr2Out]
-
-			if !funcOk || !argsOk || !outOk {
-				if !funcOk && !outOk {
-					s += fmt.Sprintf("%s = func(%s) %s\n", shortNames[e.Expr1], args, shortNames[e.Expr2Out])
-				} else if !funcOk && outOk {
-					s += fmt.Sprintf("%s = func(%s) %s\n", shortNames[e.Expr1], args, outType)
-				} else if funcOk && !outOk {
-					s += fmt.Sprintf("%s = func(%s) %s\n", funcType, args, shortNames[e.Expr2Out])
-				} else {
-					s += fmt.Sprintf("%s = func(%s) %s\n", funcType, args, outType)
-				}
-			}
-
-		case *interfaces.CallFuncArgsValueInvariant:
-			// skip, not used in the examples I care about
-
-		case *interfaces.AnyInvariant:
-			// skip, not used in the examples I care about
-
-		case *interfaces.SkipInvariant:
-			// we don't care about this one
-
-		default:
-			s += fmt.Sprintf("%v\n", equality)
-		}
-	}
-
-	return s
 }
 
 // SimpleInvariantSolver is an iterative invariant solver for AST expressions.
 // It is intended to be very simple, even if it's computationally inefficient.
 // TODO: Move some of the global solver constants into this struct as params.
 type SimpleInvariantSolver struct {
+	// Strategy is a series of methodologies to heuristically improve the
+	// solver.
+	Strategy map[string]string
+
 	Debug bool
 	Logf  func(format string, v ...interface{})
+
+	// skipFuncCmp tells the solver to skip the slow loop entirely. This may
+	// prevent some correct programs from completing type unification.
+	skipFuncCmp bool
+
+	// heuristicalDrop tells the solver to drop some func compares. This was
+	// determined heuristically and needs checking to see if it's even a
+	// sensible algorithmic approach. This is a less aggressive form of
+	// skipFuncCmp. This is redundant if skipFuncCmp is true.
+	heuristicalDrop bool
+
+	// zTotal is a heuristic counter to measure the size of the slow loop.
+	zTotal int
+}
+
+// Init contains some handles that are used to initialize the solver.
+func (obj *SimpleInvariantSolver) Init(init *unification.Init) error {
+	obj.Strategy = init.Strategy
+
+	obj.Debug = init.Debug
+	obj.Logf = init.Logf
+
+	optimizations, exists := init.Strategy[unification.StrategyOptimizationsKey]
+	if !exists {
+		return nil
+	}
+	// TODO: use a query string parser instead?
+	for _, x := range strings.Split(optimizations, ",") {
+		if x == OptimizationSkipFuncCmp {
+			obj.skipFuncCmp = true
+			continue
+		}
+		if x == OptimizationHeuristicalDrop {
+			obj.heuristicalDrop = true
+			continue
+		}
+	}
+
+	return nil
 }
 
 // Solve is the actual solve implementation of the solver.
-func (obj *SimpleInvariantSolver) Solve(ctx context.Context, invariants []interfaces.Invariant, expected []interfaces.Expr) (*InvariantSolution, error) {
+func (obj *SimpleInvariantSolver) Solve(ctx context.Context, invariants []interfaces.Invariant, expected []interfaces.Expr) (*unification.InvariantSolution, error) {
 	process := func(invariants []interfaces.Invariant) ([]interfaces.Invariant, []*interfaces.ExclusiveInvariant, error) {
 		equalities := []interfaces.Invariant{}
 		exclusives := []*interfaces.ExclusiveInvariant{}
@@ -351,7 +269,7 @@ func (obj *SimpleInvariantSolver) Solve(ctx context.Context, invariants []interf
 
 	// list all the expr's connected to expr, use pairs as chains
 	listConnectedFn := func(expr interfaces.Expr, exprs []*interfaces.EqualityInvariant) []interfaces.Expr {
-		pairsType := pairs(exprs)
+		pairsType := unification.Pairs(exprs)
 		return pairsType.DFS(expr)
 	}
 
@@ -770,7 +688,12 @@ Loop:
 					return false
 				}
 				// is there another EqualityWrapFuncInvariant with the same Expr1 pointer?
-				for _, fn := range fnInvariants {
+				fnDone := make(map[int]struct{})
+				for z, fn := range fnInvariants {
+					if obj.skipFuncCmp {
+						break
+					}
+					obj.zTotal++
 					// XXX: I think we're busy in this loop a lot.
 					select {
 					case <-ctx.Done():
@@ -817,6 +740,7 @@ Loop:
 							eqInvariants = append(eqInvariants, newEq)
 							// TODO: add it as a generator instead?
 							equalities = append(equalities, newEq)
+							fnDone[z] = struct{}{} // XXX: heuristical drop
 						}
 
 						// both solved or both unsolved we skip
@@ -830,6 +754,7 @@ Loop:
 									solved[rhsExpr] = lhsTyp // yay, we learned something!
 									//used = append(used, i) // mark equality as used up when complete!
 									obj.Logf("%s: solved partial rhs func arg equality", Name)
+									fnDone[z] = struct{}{} // XXX: heuristical drop
 								} else if err := newTyp.Cmp(lhsTyp); err != nil {
 									return nil, errwrap.Wrapf(err, "can't unify, invariant illogicality with partial rhs func arg equality")
 								}
@@ -850,6 +775,7 @@ Loop:
 									solved[lhsExpr] = rhsTyp // yay, we learned something!
 									//used = append(used, i) // mark equality as used up when complete!
 									obj.Logf("%s: solved partial lhs func arg equality", Name)
+									fnDone[z] = struct{}{} // XXX: heuristical drop
 								} else if err := newTyp.Cmp(rhsTyp); err != nil {
 									return nil, errwrap.Wrapf(err, "can't unify, invariant illogicality with partial lhs func arg equality")
 								}
@@ -880,6 +806,7 @@ Loop:
 						eqInvariants = append(eqInvariants, newEq)
 						// TODO: add it as a generator instead?
 						equalities = append(equalities, newEq)
+						fnDone[z] = struct{}{} // XXX: heuristical drop
 					}
 
 					// both solved or both unsolved we skip
@@ -893,6 +820,7 @@ Loop:
 								solved[rhsExpr] = lhsTyp // yay, we learned something!
 								//used = append(used, i) // mark equality as used up when complete!
 								obj.Logf("%s: solved partial rhs func return equality", Name)
+								fnDone[z] = struct{}{} // XXX: heuristical drop
 							} else if err := newTyp.Cmp(lhsTyp); err != nil {
 								return nil, errwrap.Wrapf(err, "can't unify, invariant illogicality with partial rhs func return equality")
 							}
@@ -913,6 +841,7 @@ Loop:
 								solved[lhsExpr] = rhsTyp // yay, we learned something!
 								//used = append(used, i) // mark equality as used up when complete!
 								obj.Logf("%s: solved partial lhs func return equality", Name)
+								fnDone[z] = struct{}{} // XXX: heuristical drop
 							} else if err := newTyp.Cmp(rhsTyp); err != nil {
 								return nil, errwrap.Wrapf(err, "can't unify, invariant illogicality with partial lhs func return equality")
 							}
@@ -924,6 +853,21 @@ Loop:
 						}
 					}
 
+				} // end big slow loop
+				if obj.heuristicalDrop {
+					fnDoneList := []int{}
+					for k := range fnDone {
+						fnDoneList = append(fnDoneList, k)
+					}
+					sort.Ints(fnDoneList)
+
+					for z := len(fnDoneList) - 1; z >= 0; z-- {
+						zx := fnDoneList[z] // delete index that was marked as done!
+						fnInvariants = append(fnInvariants[:zx], fnInvariants[zx+1:]...)
+						if obj.Debug {
+							obj.Logf("zTotal: %d, had: %d, removing: %d", obj.zTotal, len(fnInvariants), len(fnDoneList))
+						}
+					}
 				}
 
 				// can we solve anything?
@@ -1272,7 +1216,7 @@ Loop:
 					obj.Logf("%s: unsolved: %+v", Name, x)
 				}
 			}
-			obj.Logf("%s: solver state:\n%s", Name, DebugSolverState(solved, equalities))
+			obj.Logf("%s: solver state:\n%s", Name, unification.DebugSolverState(solved, equalities))
 
 			// Lastly, we could loop through each exclusive and see
 			// if it only has a single, easy solution. For example,
@@ -1338,7 +1282,7 @@ Loop:
 			}
 
 			// let's try each combination, one at a time...
-			for i, ex := range exclusivesProduct(exclusives) { // [][]interfaces.Invariant
+			for i, ex := range unification.ExclusivesProduct(exclusives) { // [][]interfaces.Invariant
 				select {
 				case <-ctx.Done():
 					return nil, ctx.Err()
@@ -1378,7 +1322,7 @@ Loop:
 			for expr, typ := range solved {
 				obj.Logf("%s: solved: (%p) => %+v", Name, expr, typ)
 			}
-			return nil, ErrAmbiguous
+			return nil, unification.ErrAmbiguous
 		}
 		// delete used equalities, in reverse order to preserve indexing!
 		for i := len(used) - 1; i >= 0; i-- {
@@ -1403,7 +1347,8 @@ Loop:
 		}
 		solutions = append(solutions, invar)
 	}
-	return &InvariantSolution{
+	obj.Logf("zTotal: %d", obj.zTotal)
+	return &unification.InvariantSolution{
 		Solutions: solutions,
 	}, nil
 }

lang/unification/solvers/simplesolver_test.go

@@ -29,7 +29,7 @@
 
 //go:build !root
 
-package unification
+package solvers
 
 import (
 	"context"
@@ -40,6 +40,7 @@ import (
 	"github.com/purpleidea/mgmt/lang/ast"
 	"github.com/purpleidea/mgmt/lang/interfaces"
 	"github.com/purpleidea/mgmt/lang/types"
+	"github.com/purpleidea/mgmt/lang/unification"
 	"github.com/purpleidea/mgmt/util"
 )
 
@@ -259,14 +260,27 @@ func TestSimpleSolver1(t *testing.T) {
 		t.Run(fmt.Sprintf("test #%d (%s)", index, tc.name), func(t *testing.T) {
 			invariants, expected, fail, expect, experr, experrstr := tc.invariants, tc.expected, tc.fail, tc.expect, tc.experr, tc.experrstr
 
+			debug := testing.Verbose()
 			logf := func(format string, v ...interface{}) {
 				t.Logf(fmt.Sprintf("test #%d", index)+": "+format, v...)
 			}
-			debug := testing.Verbose()
 
-			solver := SimpleInvariantSolverLogger(logf) // generates a solver with built-in logging
+			solver, err := unification.LookupDefault()
-
+			if err != nil {
-			solution, err := solver(context.TODO(), invariants, expected)
+				t.Errorf("test #%d: FAIL", index)
+				t.Errorf("test #%d: solver lookup failed with: %+v", index, err)
+				return
+			}
+			init := &unification.Init{
+				Debug: debug,
+				Logf:  logf,
+			}
+			if err := solver.Init(init); err != nil {
+				t.Errorf("test #%d: FAIL", index)
+				t.Errorf("test #%d: solver init failed with: %+v", index, err)
+				return
+			}
+			solution, err := solver.Solve(context.TODO(), invariants, expected)
 			t.Logf("test #%d: solver completed with: %+v", index, err)
 
 			if !fail && err != nil {

lang/unification/solvers/solvers.go

@@ -0,0 +1,37 @@
+// Mgmt
+// Copyright (C) 2013-2024+ James Shubin and the project contributors
+// Written by James Shubin <james@shubin.ca> and the project contributors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+//
+// Additional permission under GNU GPL version 3 section 7
+//
+// If you modify this program, or any covered work, by linking or combining it
+// with embedded mcl code and modules (and that the embedded mcl code and
+// modules which link with this program, contain a copy of their source code in
+// the authoritative form) containing parts covered by the terms of any other
+// license, the licensors of this program grant you additional permission to
+// convey the resulting work. Furthermore, the licensors of this program grant
+// the original author, James Shubin, additional permission to update this
+// additional permission if he deems it necessary to achieve the goals of this
+// additional permission.
+
+// Package solvers is used to have a central place to import all solvers from.
+// It is also a good locus to run all of the unification tests from.
+package solvers
+
+import (
+	// import so the solver registers
+	_ "github.com/purpleidea/mgmt/lang/unification/simplesolver"
+)

lang/unification/solvers/unification_test.go

@@ -29,7 +29,7 @@
 
 //go:build !root
 
-package lang // XXX: move this to the unification package
+package solvers
 
 import (
 	"context"
@@ -37,6 +37,7 @@ import (
 	"strings"
 	"testing"
 
+	_ "github.com/purpleidea/mgmt/engine/resources" // import so the resources register
 	"github.com/purpleidea/mgmt/lang/ast"
 	"github.com/purpleidea/mgmt/lang/funcs"
 	"github.com/purpleidea/mgmt/lang/funcs/vars"
@@ -848,13 +849,21 @@ func TestUnification1(t *testing.T) {
 			}
 
 			// apply type unification
+			debug := testing.Verbose()
 			logf := func(format string, v ...interface{}) {
 				t.Logf(fmt.Sprintf("test #%d", index)+": unification: "+format, v...)
 			}
+
+			solver, err := unification.LookupDefault()
+			if err != nil {
+				t.Errorf("test #%d: FAIL", index)
+				t.Errorf("test #%d: solver lookup failed with: %+v", index, err)
+				return
+			}
 			unifier := &unification.Unifier{
 				AST:    xast,
-				Solver: unification.SimpleInvariantSolverLogger(logf),
+				Solver: solver,
-				Debug:  testing.Verbose(),
+				Debug:  debug,
 				Logf:   logf,
 			}
 			err = unifier.Unify(context.TODO())

lang/unification/unification.go

@@ -46,8 +46,11 @@ type Unifier struct {
 	AST interfaces.Stmt
 
 	// Solver is the solver algorithm implementation to use.
-	// XXX: Solver should be a solver interface, not a function signature.
+	Solver Solver
-	Solver func(context.Context, []interfaces.Invariant, []interfaces.Expr) (*InvariantSolution, error)
+
+	// Strategy is a hack to tune unification performance until we have an
+	// overall cleaner unification algorithm in place.
+	Strategy map[string]string
 
 	Debug bool
 	Logf  func(format string, v ...interface{})
@@ -76,6 +79,15 @@ func (obj *Unifier) Unify(ctx context.Context) error {
 		return fmt.Errorf("the Logf function is missing")
 	}
 
+	init := &Init{
+		Strategy: obj.Strategy,
+		Logf:     obj.Logf,
+		Debug:    obj.Debug,
+	}
+	if err := obj.Solver.Init(init); err != nil {
+		return err
+	}
+
 	if obj.Debug {
 		obj.Logf("tree: %+v", obj.AST)
 	}
@@ -98,7 +110,7 @@ func (obj *Unifier) Unify(ctx context.Context) error {
 	exprMap := ExprListToExprMap(exprs)    // makes searching faster
 	exprList := ExprMapToExprList(exprMap) // makes it unique (no duplicates)
 
-	solved, err := obj.Solver(ctx, invariants, exprList)
+	solved, err := obj.Solver.Solve(ctx, invariants, exprList)
 	if err != nil {
 		return err
 	}
@@ -194,14 +206,14 @@ func (obj *InvariantSolution) ExprList() []interfaces.Expr {
 	return exprs
 }
 
-// exclusivesProduct returns a list of different products produced from the
+// ExclusivesProduct returns a list of different products produced from the
 // combinatorial product of the list of exclusives. Each ExclusiveInvariant must
 // contain between one and more Invariants. This takes every combination of
 // Invariants (choosing one from each ExclusiveInvariant) and returns that list.
 // In other words, if you have three exclusives, with invariants named (A1, B1),
 // (A2), and (A3, B3, C3) you'll get: (A1, A2, A3), (A1, A2, B3), (A1, A2, C3),
 // (B1, A2, A3), (B1, A2, B3), (B1, A2, C3) as results for this function call.
-func exclusivesProduct(exclusives []*interfaces.ExclusiveInvariant) [][]interfaces.Invariant {
+func ExclusivesProduct(exclusives []*interfaces.ExclusiveInvariant) [][]interfaces.Invariant {
 	if len(exclusives) == 0 {
 		return nil
 	}

lang/unification/util.go

@@ -76,13 +76,13 @@ func ExprContains(needle interfaces.Expr, haystack []interfaces.Expr) bool {
 	return false
 }
 
-// pairs is a simple list of pairs of expressions which can be used as a simple
+// Pairs is a simple list of pairs of expressions which can be used as a simple
 // undirected graph structure, or as a simple list of equalities.
-type pairs []*interfaces.EqualityInvariant
+type Pairs []*interfaces.EqualityInvariant
 
 // Vertices returns the list of vertices that the input expr is directly
 // connected to.
-func (obj pairs) Vertices(expr interfaces.Expr) []interfaces.Expr {
+func (obj Pairs) Vertices(expr interfaces.Expr) []interfaces.Expr {
 	m := make(map[interfaces.Expr]struct{})
 	for _, x := range obj {
 		if x.Expr1 == x.Expr2 { // skip circular
@@ -106,7 +106,7 @@ func (obj pairs) Vertices(expr interfaces.Expr) []interfaces.Expr {
 }
 
 // DFS returns a depth first search for the graph, starting at the input vertex.
-func (obj pairs) DFS(start interfaces.Expr) []interfaces.Expr {
+func (obj Pairs) DFS(start interfaces.Expr) []interfaces.Expr {
 	var d []interfaces.Expr // discovered
 	var s []interfaces.Expr // stack
 	found := false

util/distro.go

@@ -126,8 +126,8 @@ func LatestFedoraVersion(ctx context.Context, arch string) (string, error) {
 	m := 0
 	for _, r := range data {
 		version, err := strconv.Atoi(r.Version)
-		if err != nil {
+		if err != nil { // skip strings like "40 Beta"
-			return "", err
+			continue
 		}
 		if arch != "" && arch != r.Arch { // skip others
 			continue