Support unimplemented tests in test runner.

This change allows the shim to return a magic error code (89) to
indicate that it doesn't implement some of the given flags for a test.
Unimplemented tests are, by default, an error. The --allow-unimplemented
flag to the test runner causes them to be ignored.

This is done in preparation for non-BoringSSL shims.

Change-Id: Iecfd545b9cf44df5e25b719bfd06275c8149311a
Reviewed-on: https://boringssl-review.googlesource.com/8970
Reviewed-by: David Benjamin <davidben@google.com>
Commit-Queue: David Benjamin <davidben@google.com>
CQ-Verified: CQ bot account: commit-bot@chromium.org <commit-bot@chromium.org>

ssl/test/runner/runner.go

@@ -21,6 +21,7 @@ import (
 	"crypto/x509"
 	"encoding/base64"
 	"encoding/pem"
+	"errors"
 	"flag"
 	"fmt"
 	"io"
@@ -39,22 +40,23 @@ import (
 )
 
 var (
-	useValgrind     = flag.Bool("valgrind", false, "If true, run code under valgrind")
-	useGDB          = flag.Bool("gdb", false, "If true, run BoringSSL code under gdb")
-	useLLDB         = flag.Bool("lldb", false, "If true, run BoringSSL code under lldb")
-	flagDebug       = flag.Bool("debug", false, "Hexdump the contents of the connection")
-	mallocTest      = flag.Int64("malloc-test", -1, "If non-negative, run each test with each malloc in turn failing from the given number onwards.")
-	mallocTestDebug = flag.Bool("malloc-test-debug", false, "If true, ask bssl_shim to abort rather than fail a malloc. This can be used with a specific value for --malloc-test to identity the malloc failing that is causing problems.")
-	jsonOutput      = flag.String("json-output", "", "The file to output JSON results to.")
-	pipe            = flag.Bool("pipe", false, "If true, print status output suitable for piping into another program.")
-	testToRun       = flag.String("test", "", "The name of a test to run, or empty to run all tests")
-	numWorkers      = flag.Int("num-workers", runtime.NumCPU(), "The number of workers to run in parallel.")
-	shimPath        = flag.String("shim-path", "../../../build/ssl/test/bssl_shim", "The location of the shim binary.")
-	resourceDir     = flag.String("resource-dir", ".", "The directory in which to find certificate and key files.")
-	fuzzer          = flag.Bool("fuzzer", false, "If true, tests against a BoringSSL built in fuzzer mode.")
-	transcriptDir   = flag.String("transcript-dir", "", "The directory in which to write transcripts.")
-	idleTimeout     = flag.Duration("idle-timeout", 15*time.Second, "The number of seconds to wait for a read or write to bssl_shim.")
-	deterministic   = flag.Bool("deterministic", false, "If true, uses a deterministic PRNG in the runner.")
+	useValgrind        = flag.Bool("valgrind", false, "If true, run code under valgrind")
+	useGDB             = flag.Bool("gdb", false, "If true, run BoringSSL code under gdb")
+	useLLDB            = flag.Bool("lldb", false, "If true, run BoringSSL code under lldb")
+	flagDebug          = flag.Bool("debug", false, "Hexdump the contents of the connection")
+	mallocTest         = flag.Int64("malloc-test", -1, "If non-negative, run each test with each malloc in turn failing from the given number onwards.")
+	mallocTestDebug    = flag.Bool("malloc-test-debug", false, "If true, ask bssl_shim to abort rather than fail a malloc. This can be used with a specific value for --malloc-test to identity the malloc failing that is causing problems.")
+	jsonOutput         = flag.String("json-output", "", "The file to output JSON results to.")
+	pipe               = flag.Bool("pipe", false, "If true, print status output suitable for piping into another program.")
+	testToRun          = flag.String("test", "", "The name of a test to run, or empty to run all tests")
+	numWorkers         = flag.Int("num-workers", runtime.NumCPU(), "The number of workers to run in parallel.")
+	shimPath           = flag.String("shim-path", "../../../build/ssl/test/bssl_shim", "The location of the shim binary.")
+	resourceDir        = flag.String("resource-dir", ".", "The directory in which to find certificate and key files.")
+	fuzzer             = flag.Bool("fuzzer", false, "If true, tests against a BoringSSL built in fuzzer mode.")
+	transcriptDir      = flag.String("transcript-dir", "", "The directory in which to write transcripts.")
+	idleTimeout        = flag.Duration("idle-timeout", 15*time.Second, "The number of seconds to wait for a read or write to bssl_shim.")
+	deterministic      = flag.Bool("deterministic", false, "If true, uses a deterministic PRNG in the runner.")
+	allowUnimplemented = flag.Bool("allow-unimplemented", false, "If true, report pass even if some tests are unimplemented.")
 )
 
 type testCert int
@@ -681,13 +683,10 @@ func lldbOf(path string, args ...string) *exec.Cmd {
 	return exec.Command("xterm", xtermArgs...)
 }
 
-type moreMallocsError struct{}
-
-func (moreMallocsError) Error() string {
-	return "child process did not exhaust all allocation calls"
-}
-
-var errMoreMallocs = moreMallocsError{}
+var (
+	errMoreMallocs   = errors.New("child process did not exhaust all allocation calls")
+	errUnimplemented = errors.New("child process does not implement needed flags")
+)
 
 // accept accepts a connection from listener, unless waitChan signals a process
 // exit first.
@@ -883,8 +882,11 @@ func runTest(test *testCase, shimPath string, mallocNumToFail int64) error {
 
 	childErr := <-waitChan
 	if exitError, ok := childErr.(*exec.ExitError); ok {
-		if exitError.Sys().(syscall.WaitStatus).ExitStatus() == 88 {
+		switch exitError.Sys().(syscall.WaitStatus).ExitStatus() {
+		case 88:
 			return errMoreMallocs
+		case 89:
+			return errUnimplemented
 		}
 	}
 
@@ -7701,7 +7703,7 @@ type statusMsg struct {
 }
 
 func statusPrinter(doneChan chan *testOutput, statusChan chan statusMsg, total int) {
-	var started, done, failed, lineLen int
+	var started, done, failed, unimplemented, lineLen int
 
 	testOutput := newTestOutput()
 	for msg := range statusChan {
@@ -7720,9 +7722,18 @@ func statusPrinter(doneChan chan *testOutput, statusChan chan statusMsg, total i
 			done++
 
 			if msg.err != nil {
-				fmt.Printf("FAILED (%s)\n%s\n", msg.test.name, msg.err)
-				failed++
-				testOutput.addResult(msg.test.name, "FAIL")
+				if msg.err == errUnimplemented {
+					if *pipe {
+						// Print each test instead of a status line.
+						fmt.Printf("UNIMPLEMENTED (%s)\n", msg.test.name)
+					}
+					unimplemented++
+					testOutput.addResult(msg.test.name, "UNIMPLEMENTED")
+				} else {
+					fmt.Printf("FAILED (%s)\n%s\n", msg.test.name, msg.err)
+					failed++
+					testOutput.addResult(msg.test.name, "FAIL")
+				}
 			} else {
 				if *pipe {
 					// Print each test instead of a status line.
@@ -7734,7 +7745,7 @@ func statusPrinter(doneChan chan *testOutput, statusChan chan statusMsg, total i
 
 		if !*pipe {
 			// Print a new status line.
-			line := fmt.Sprintf("%d/%d/%d/%d", failed, done, started, total)
+			line := fmt.Sprintf("%d/%d/%d/%d/%d", failed, unimplemented, done, started, total)
 			lineLen = len(line)
 			os.Stdout.WriteString(line)
 		}
@@ -7816,7 +7827,11 @@ func main() {
 		}
 	}
 
-	if !testOutput.allPassed {
+	if !*allowUnimplemented && testOutput.NumFailuresByType["UNIMPLEMENTED"] > 0 {
+		os.Exit(1)
+	}
+
+	if !testOutput.noneFailed {
 		os.Exit(1)
 	}
 }

ssl/test/runner/test_output.go

@@ -29,7 +29,7 @@ type testOutput struct {
 	SecondsSinceEpoch float64               `json:"seconds_since_epoch"`
 	NumFailuresByType map[string]int        `json:"num_failures_by_type"`
 	Tests             map[string]testResult `json:"tests"`
-	allPassed         bool
+	noneFailed        bool
 }
 
 type testResult struct {
@@ -45,7 +45,7 @@ func newTestOutput() *testOutput {
 		SecondsSinceEpoch: float64(time.Now().UnixNano()) / float64(time.Second/time.Nanosecond),
 		NumFailuresByType: make(map[string]int),
 		Tests:             make(map[string]testResult),
-		allPassed:         true,
+		noneFailed:        true,
 	}
 }
 
@@ -59,8 +59,8 @@ func (t *testOutput) addResult(name, result string) {
 		IsUnexpected: result != "PASS",
 	}
 	t.NumFailuresByType[result]++
-	if result != "PASS" {
-		t.allPassed = false
+	if result != "PASS" && result != "UNIMPLEMENTED" {
+		t.noneFailed = false
 	}
 }