engine, lang: Modern exported resources

I've been waiting to write this patch for a long time. I firmly believe
that the idea of "exported resources" was truly a brilliant one, but
which was never even properly understood by its original inventors! This
patch set aims to show how it should have been done.

The main differences are:

* Real-time modelling, since "once per run" makes no sense.
* Filter with code/functions not language syntax.
* Directed exporting to limit the intended recipients.

The next step is to add more "World" reading and filtering functions to
make it easy and expressive to make your selection of resources to
collect!

engine/graph/actions.go

@@ -181,6 +181,18 @@ func (obj *Engine) Process(ctx context.Context, vertex pgraph.Vertex) error {
 		refreshableRes.SetRefresh(refresh) // tell the resource
 	}
 
+	// Run the exported resource exporter!
+	var exportOK bool
+	var exportErr error
+	wg := &sync.WaitGroup{}
+	wg.Add(1)
+	// (Run this concurrently with the CheckApply related stuff below...)
+	go func() {
+		defer wg.Done()
+		// doesn't really need to be in parallel, but we can...
+		exportOK, exportErr = obj.Exporter.Export(ctx, res)
+	}()
+
 	// Check cached state, to skip CheckApply, but can't skip if refreshing!
 	// If the resource doesn't implement refresh, skip the refresh test.
 	// FIXME: if desired, check that we pass through refresh notifications!
@@ -190,6 +202,13 @@ func (obj *Engine) Process(ctx context.Context, vertex pgraph.Vertex) error {
 	} else if noop && (refresh && isRefreshableRes) { // had a refresh to do w/ noop!
 		checkOK, err = false, nil // therefore the state is wrong
 
+	} else if res.MetaParams().Hidden {
+		// We're not running CheckApply
+		if obj.Debug {
+			obj.Logf("%s: Hidden", res)
+		}
+		checkOK, err = true, nil // default
+
 	} else {
 		// run the CheckApply!
 		if obj.Debug {
@@ -201,6 +220,13 @@ func (obj *Engine) Process(ctx context.Context, vertex pgraph.Vertex) error {
 			obj.Logf("%s: CheckApply(%t): Return(%t, %s)", res, !noop, checkOK, engineUtil.CleanError(err))
 		}
 	}
+	wg.Wait()
+	checkOK = checkOK && exportOK // always combine
+	if err == nil {               // If CheckApply didn't error, look at exportOK.
+		// This is because if CheckApply errors we don't need to care or
+		// tell anyone about an exporting error.
+		err = exportErr
+	}
 
 	if checkOK && err != nil { // should never return this way
 		return fmt.Errorf("%s: resource programming error: CheckApply(%t): %t, %+v", res, !noop, checkOK, err)
@@ -304,14 +330,24 @@ func (obj *Engine) Worker(vertex pgraph.Vertex) error {
 	}
 
 	// initialize or reinitialize the meta state for this resource uid
-	obj.mlock.Lock()
-	if _, exists := obj.metas[engine.PtrUID(res)]; !exists || res.MetaParams().Reset {
-		obj.metas[engine.PtrUID(res)] = &engine.MetaState{
-			CheckApplyRetry: res.MetaParams().Retry, // lookup the retry value
-		}
+	// if we're using a Hidden resource, we don't support this feature
+	// TODO: should we consider supporting it? is it really necessary?
+	// XXX: to support this for Hidden, we'd need to handle dupe names
+	metas := &engine.MetaState{
+		CheckApplyRetry: res.MetaParams().Retry, // lookup the retry value
+	}
+	if !res.MetaParams().Hidden {
+		// Skip this if Hidden since we can have a hidden res that has
+		// the same kind+name as a regular res, and this would conflict.
+		obj.mlock.Lock()
+		if _, exists := obj.metas[engine.PtrUID(res)]; !exists || res.MetaParams().Reset {
+			obj.metas[engine.PtrUID(res)] = &engine.MetaState{
+				CheckApplyRetry: res.MetaParams().Retry, // lookup the retry value
+			}
+		}
+		metas = obj.metas[engine.PtrUID(res)] // handle
+		obj.mlock.Unlock()
 	}
-	metas := obj.metas[engine.PtrUID(res)] // handle
-	obj.mlock.Unlock()
 
 	//defer close(obj.state[vertex].stopped) // done signal
 
@@ -376,10 +412,21 @@ func (obj *Engine) Worker(vertex pgraph.Vertex) error {
 					delay = 0 // reset
 					continue
 				}
+
+			} else if res.MetaParams().Hidden {
+				// We're not running Watch
+				if obj.Debug {
+					obj.Logf("%s: Hidden", res)
+				}
+				obj.state[vertex].cuid.StartTimer() // TODO: Should we do this?
+				err = obj.state[vertex].hidden(obj.state[vertex].doneCtx)
+				obj.state[vertex].cuid.StopTimer() // TODO: Should we do this?
+
 			} else if interval := res.MetaParams().Poll; interval > 0 { // poll instead of watching :(
 				obj.state[vertex].cuid.StartTimer()
 				err = obj.state[vertex].poll(obj.state[vertex].doneCtx, interval)
 				obj.state[vertex].cuid.StopTimer() // clean up nicely
+
 			} else {
 				obj.state[vertex].cuid.StartTimer()
 				if obj.Debug {