From 842ae6cad06c0c80886021b8fc86b0ffc16ab59c Mon Sep 17 00:00:00 2001 From: EKR Date: Wed, 27 Jul 2016 09:22:05 +0200 Subject: [PATCH] Support unimplemented tests in test runner. This change allows the shim to return a magic error code (89) to indicate that it doesn't implement some of the given flags for a test. Unimplemented tests are, by default, an error. The --allow-unimplemented flag to the test runner causes them to be ignored. This is done in preparation for non-BoringSSL shims. Change-Id: Iecfd545b9cf44df5e25b719bfd06275c8149311a Reviewed-on: https://boringssl-review.googlesource.com/8970 Reviewed-by: David Benjamin Commit-Queue: David Benjamin CQ-Verified: CQ bot account: commit-bot@chromium.org --- ssl/test/runner/runner.go | 75 ++++++++++++++++++++-------------- ssl/test/runner/test_output.go | 8 ++-- 2 files changed, 49 insertions(+), 34 deletions(-) diff --git a/ssl/test/runner/runner.go b/ssl/test/runner/runner.go index 8ad99626..d3cd77ff 100644 --- a/ssl/test/runner/runner.go +++ b/ssl/test/runner/runner.go @@ -21,6 +21,7 @@ import ( "crypto/x509" "encoding/base64" "encoding/pem" + "errors" "flag" "fmt" "io" @@ -39,22 +40,23 @@ import ( ) var ( - useValgrind = flag.Bool("valgrind", false, "If true, run code under valgrind") - useGDB = flag.Bool("gdb", false, "If true, run BoringSSL code under gdb") - useLLDB = flag.Bool("lldb", false, "If true, run BoringSSL code under lldb") - flagDebug = flag.Bool("debug", false, "Hexdump the contents of the connection") - mallocTest = flag.Int64("malloc-test", -1, "If non-negative, run each test with each malloc in turn failing from the given number onwards.") - mallocTestDebug = flag.Bool("malloc-test-debug", false, "If true, ask bssl_shim to abort rather than fail a malloc. This can be used with a specific value for --malloc-test to identity the malloc failing that is causing problems.") - jsonOutput = flag.String("json-output", "", "The file to output JSON results to.") - pipe = flag.Bool("pipe", false, "If true, print status output suitable for piping into another program.") - testToRun = flag.String("test", "", "The name of a test to run, or empty to run all tests") - numWorkers = flag.Int("num-workers", runtime.NumCPU(), "The number of workers to run in parallel.") - shimPath = flag.String("shim-path", "../../../build/ssl/test/bssl_shim", "The location of the shim binary.") - resourceDir = flag.String("resource-dir", ".", "The directory in which to find certificate and key files.") - fuzzer = flag.Bool("fuzzer", false, "If true, tests against a BoringSSL built in fuzzer mode.") - transcriptDir = flag.String("transcript-dir", "", "The directory in which to write transcripts.") - idleTimeout = flag.Duration("idle-timeout", 15*time.Second, "The number of seconds to wait for a read or write to bssl_shim.") - deterministic = flag.Bool("deterministic", false, "If true, uses a deterministic PRNG in the runner.") + useValgrind = flag.Bool("valgrind", false, "If true, run code under valgrind") + useGDB = flag.Bool("gdb", false, "If true, run BoringSSL code under gdb") + useLLDB = flag.Bool("lldb", false, "If true, run BoringSSL code under lldb") + flagDebug = flag.Bool("debug", false, "Hexdump the contents of the connection") + mallocTest = flag.Int64("malloc-test", -1, "If non-negative, run each test with each malloc in turn failing from the given number onwards.") + mallocTestDebug = flag.Bool("malloc-test-debug", false, "If true, ask bssl_shim to abort rather than fail a malloc. This can be used with a specific value for --malloc-test to identity the malloc failing that is causing problems.") + jsonOutput = flag.String("json-output", "", "The file to output JSON results to.") + pipe = flag.Bool("pipe", false, "If true, print status output suitable for piping into another program.") + testToRun = flag.String("test", "", "The name of a test to run, or empty to run all tests") + numWorkers = flag.Int("num-workers", runtime.NumCPU(), "The number of workers to run in parallel.") + shimPath = flag.String("shim-path", "../../../build/ssl/test/bssl_shim", "The location of the shim binary.") + resourceDir = flag.String("resource-dir", ".", "The directory in which to find certificate and key files.") + fuzzer = flag.Bool("fuzzer", false, "If true, tests against a BoringSSL built in fuzzer mode.") + transcriptDir = flag.String("transcript-dir", "", "The directory in which to write transcripts.") + idleTimeout = flag.Duration("idle-timeout", 15*time.Second, "The number of seconds to wait for a read or write to bssl_shim.") + deterministic = flag.Bool("deterministic", false, "If true, uses a deterministic PRNG in the runner.") + allowUnimplemented = flag.Bool("allow-unimplemented", false, "If true, report pass even if some tests are unimplemented.") ) type testCert int @@ -681,13 +683,10 @@ func lldbOf(path string, args ...string) *exec.Cmd { return exec.Command("xterm", xtermArgs...) } -type moreMallocsError struct{} - -func (moreMallocsError) Error() string { - return "child process did not exhaust all allocation calls" -} - -var errMoreMallocs = moreMallocsError{} +var ( + errMoreMallocs = errors.New("child process did not exhaust all allocation calls") + errUnimplemented = errors.New("child process does not implement needed flags") +) // accept accepts a connection from listener, unless waitChan signals a process // exit first. @@ -883,8 +882,11 @@ func runTest(test *testCase, shimPath string, mallocNumToFail int64) error { childErr := <-waitChan if exitError, ok := childErr.(*exec.ExitError); ok { - if exitError.Sys().(syscall.WaitStatus).ExitStatus() == 88 { + switch exitError.Sys().(syscall.WaitStatus).ExitStatus() { + case 88: return errMoreMallocs + case 89: + return errUnimplemented } } @@ -7701,7 +7703,7 @@ type statusMsg struct { } func statusPrinter(doneChan chan *testOutput, statusChan chan statusMsg, total int) { - var started, done, failed, lineLen int + var started, done, failed, unimplemented, lineLen int testOutput := newTestOutput() for msg := range statusChan { @@ -7720,9 +7722,18 @@ func statusPrinter(doneChan chan *testOutput, statusChan chan statusMsg, total i done++ if msg.err != nil { - fmt.Printf("FAILED (%s)\n%s\n", msg.test.name, msg.err) - failed++ - testOutput.addResult(msg.test.name, "FAIL") + if msg.err == errUnimplemented { + if *pipe { + // Print each test instead of a status line. + fmt.Printf("UNIMPLEMENTED (%s)\n", msg.test.name) + } + unimplemented++ + testOutput.addResult(msg.test.name, "UNIMPLEMENTED") + } else { + fmt.Printf("FAILED (%s)\n%s\n", msg.test.name, msg.err) + failed++ + testOutput.addResult(msg.test.name, "FAIL") + } } else { if *pipe { // Print each test instead of a status line. @@ -7734,7 +7745,7 @@ func statusPrinter(doneChan chan *testOutput, statusChan chan statusMsg, total i if !*pipe { // Print a new status line. - line := fmt.Sprintf("%d/%d/%d/%d", failed, done, started, total) + line := fmt.Sprintf("%d/%d/%d/%d/%d", failed, unimplemented, done, started, total) lineLen = len(line) os.Stdout.WriteString(line) } @@ -7816,7 +7827,11 @@ func main() { } } - if !testOutput.allPassed { + if !*allowUnimplemented && testOutput.NumFailuresByType["UNIMPLEMENTED"] > 0 { + os.Exit(1) + } + + if !testOutput.noneFailed { os.Exit(1) } } diff --git a/ssl/test/runner/test_output.go b/ssl/test/runner/test_output.go index 2112092e..eb546382 100644 --- a/ssl/test/runner/test_output.go +++ b/ssl/test/runner/test_output.go @@ -29,7 +29,7 @@ type testOutput struct { SecondsSinceEpoch float64 `json:"seconds_since_epoch"` NumFailuresByType map[string]int `json:"num_failures_by_type"` Tests map[string]testResult `json:"tests"` - allPassed bool + noneFailed bool } type testResult struct { @@ -45,7 +45,7 @@ func newTestOutput() *testOutput { SecondsSinceEpoch: float64(time.Now().UnixNano()) / float64(time.Second/time.Nanosecond), NumFailuresByType: make(map[string]int), Tests: make(map[string]testResult), - allPassed: true, + noneFailed: true, } } @@ -59,8 +59,8 @@ func (t *testOutput) addResult(name, result string) { IsUnexpected: result != "PASS", } t.NumFailuresByType[result]++ - if result != "PASS" { - t.allPassed = false + if result != "PASS" && result != "UNIMPLEMENTED" { + t.noneFailed = false } }