From d01c16845067764bc063d47f066a2bb56d85948a Mon Sep 17 00:00:00 2001 From: James Shubin Date: Mon, 22 Jan 2024 16:53:33 -0500 Subject: [PATCH] lang: Add per-test config with count maximums Some of our special tests can only be run once per `go test` invocation. That is, using the test -count flag will cause a guaranteed failure since we depend on a global being initialized only once as part of that test. This adds a per-test config option so that a user can specify to never run a particular test more than once. This lets us continue to use the -count flag with the test suite, without it causing some tests to fail. --- lang/interpret_test.go | 103 ++++++++++++++++++ .../TestAstFunc2/test-one-instance-bind.txtar | 4 + .../test-one-instance-class-arg.txtar | 4 + .../test-one-instance-func-arg.txtar | 4 + .../test-one-instance-lambda-arg.txtar | 4 + 5 files changed, 119 insertions(+) diff --git a/lang/interpret_test.go b/lang/interpret_test.go index 9bae4487..790b6b1a 100644 --- a/lang/interpret_test.go +++ b/lang/interpret_test.go @@ -22,6 +22,7 @@ package lang import ( "bytes" "context" + "encoding/json" "fmt" "io/ioutil" "os" @@ -62,6 +63,24 @@ const ( runGraphviz = false // run graphviz in tests? ) +var ( + testMutex *sync.Mutex // guards testCounter + testCounter map[string]uint // counts how many times each test ran +) + +func init() { + testMutex = &sync.Mutex{} + testCounter = make(map[string]uint) +} + +// ConfigProperties are some values that are used to specify how each test runs. +type ConfigProperties struct { + + // MaximumCount specifies how many times this test can run safely in a + // single iteration. If zero then this means infinite. + MaximumCount uint `json:"maximum-count"` +} + // TestAstFunc1 is a more advanced version which pulls code from physical dirs. func TestAstFunc1(t *testing.T) { const magicError = "# err: " @@ -169,6 +188,7 @@ func TestAstFunc1(t *testing.T) { // copy files out into the test temp directory var testOutput []byte + var testConfig []byte found := false for _, file := range archive.Files { if file.Name == "OUTPUT" { @@ -176,6 +196,10 @@ func TestAstFunc1(t *testing.T) { found = true continue } + if file.Name == "CONFIG" { + testConfig = file.Data + continue + } name := filepath.Join(tmpdir, file.Name) dir := filepath.Dir(name) @@ -189,6 +213,29 @@ func TestAstFunc1(t *testing.T) { } } + var c ConfigProperties // add pointer to get nil if empty + if len(testConfig) > 0 { + if err := json.Unmarshal(testConfig, &c); err != nil { + t.Errorf("err parsing txtar(%s) config: %+v", txtarFile, err) + return + } + } + if testing.Verbose() { + t.Logf("config: %+v", c) + } + + testMutex.Lock() // global + count := testCounter[t.Name()] // global + testCounter[t.Name()]++ + testMutex.Unlock() + + if c.MaximumCount != 0 && count >= c.MaximumCount { + if count == c.MaximumCount { // logf once + t.Logf("Skipping test after count: %d", count) + } + return + } + if !found { // skip missing tests return } @@ -624,6 +671,7 @@ func TestAstFunc2(t *testing.T) { // copy files out into the test temp directory var testOutput []byte + var testConfig []byte found := false for _, file := range archive.Files { if file.Name == "OUTPUT" { @@ -631,6 +679,10 @@ func TestAstFunc2(t *testing.T) { found = true continue } + if file.Name == "CONFIG" { + testConfig = file.Data + continue + } name := filepath.Join(tmpdir, file.Name) dir := filepath.Dir(name) @@ -644,6 +696,29 @@ func TestAstFunc2(t *testing.T) { } } + var c ConfigProperties // add pointer to get nil if empty + if len(testConfig) > 0 { + if err := json.Unmarshal(testConfig, &c); err != nil { + t.Errorf("err parsing txtar(%s) config: %+v", txtarFile, err) + return + } + } + if testing.Verbose() { + t.Logf("config: %+v", c) + } + + testMutex.Lock() // global + count := testCounter[t.Name()] // global + testCounter[t.Name()]++ + testMutex.Unlock() + + if c.MaximumCount != 0 && count >= c.MaximumCount { + if count == c.MaximumCount { // logf once + t.Logf("Skipping test after count: %d", count) + } + return + } + if !found { // skip missing tests return } @@ -1404,6 +1479,7 @@ func TestAstFunc3(t *testing.T) { // copy files out into the test temp directory var testOutput []byte + var testConfig []byte found := false for _, file := range archive.Files { if file.Name == "OUTPUT" { @@ -1411,6 +1487,10 @@ func TestAstFunc3(t *testing.T) { found = true continue } + if file.Name == "CONFIG" { + testConfig = file.Data + continue + } name := filepath.Join(tmpdir, file.Name) dir := filepath.Dir(name) @@ -1424,6 +1504,29 @@ func TestAstFunc3(t *testing.T) { } } + var c ConfigProperties // add pointer to get nil if empty + if len(testConfig) > 0 { + if err := json.Unmarshal(testConfig, &c); err != nil { + t.Errorf("err parsing txtar(%s) config: %+v", txtarFile, err) + return + } + } + if testing.Verbose() { + t.Logf("config: %+v", c) + } + + testMutex.Lock() // global + count := testCounter[t.Name()] // global + testCounter[t.Name()]++ + testMutex.Unlock() + + if c.MaximumCount != 0 && count >= c.MaximumCount { + if count == c.MaximumCount { // logf once + t.Logf("Skipping test after count: %d", count) + } + return + } + if !found { // skip missing tests return } diff --git a/lang/interpret_test/TestAstFunc2/test-one-instance-bind.txtar b/lang/interpret_test/TestAstFunc2/test-one-instance-bind.txtar index c322ad70..e9dc8c53 100644 --- a/lang/interpret_test/TestAstFunc2/test-one-instance-bind.txtar +++ b/lang/interpret_test/TestAstFunc2/test-one-instance-bind.txtar @@ -1,3 +1,7 @@ +-- CONFIG -- +{ + "maximum-count": 1 +} -- main.mcl -- import "test" # one_instance_a should only produce one value, and will error if initialized twice diff --git a/lang/interpret_test/TestAstFunc2/test-one-instance-class-arg.txtar b/lang/interpret_test/TestAstFunc2/test-one-instance-class-arg.txtar index 2cfc2159..eac0517c 100644 --- a/lang/interpret_test/TestAstFunc2/test-one-instance-class-arg.txtar +++ b/lang/interpret_test/TestAstFunc2/test-one-instance-class-arg.txtar @@ -1,3 +1,7 @@ +-- CONFIG -- +{ + "maximum-count": 1 +} -- main.mcl -- import "test" diff --git a/lang/interpret_test/TestAstFunc2/test-one-instance-func-arg.txtar b/lang/interpret_test/TestAstFunc2/test-one-instance-func-arg.txtar index 94a7950e..2b83785e 100644 --- a/lang/interpret_test/TestAstFunc2/test-one-instance-func-arg.txtar +++ b/lang/interpret_test/TestAstFunc2/test-one-instance-func-arg.txtar @@ -1,3 +1,7 @@ +-- CONFIG -- +{ + "maximum-count": 1 +} -- main.mcl -- import "test" diff --git a/lang/interpret_test/TestAstFunc2/test-one-instance-lambda-arg.txtar b/lang/interpret_test/TestAstFunc2/test-one-instance-lambda-arg.txtar index 91fdba81..0aefe596 100644 --- a/lang/interpret_test/TestAstFunc2/test-one-instance-lambda-arg.txtar +++ b/lang/interpret_test/TestAstFunc2/test-one-instance-lambda-arg.txtar @@ -1,3 +1,7 @@ +-- CONFIG -- +{ + "maximum-count": 1 +} -- main.mcl -- import "test"