Support unimplemented tests in test runner.

This change allows the shim to return a magic error code (89) to
indicate that it doesn't implement some of the given flags for a test.
Unimplemented tests are, by default, an error. The --allow-unimplemented
flag to the test runner causes them to be ignored.

This is done in preparation for non-BoringSSL shims.

Change-Id: Iecfd545b9cf44df5e25b719bfd06275c8149311a
Reviewed-on: https://boringssl-review.googlesource.com/8970
Reviewed-by: David Benjamin <davidben@google.com>
Commit-Queue: David Benjamin <davidben@google.com>
CQ-Verified: CQ bot account: commit-bot@chromium.org <commit-bot@chromium.org>
This commit is contained in:
EKR 2016-07-27 09:22:05 +02:00 committed by CQ bot account: commit-bot@chromium.org
parent 7241ca5ce4
commit 842ae6cad0
2 changed files with 49 additions and 34 deletions

View File

@ -21,6 +21,7 @@ import (
"crypto/x509"
"encoding/base64"
"encoding/pem"
"errors"
"flag"
"fmt"
"io"
@ -39,22 +40,23 @@ import (
)
var (
useValgrind = flag.Bool("valgrind", false, "If true, run code under valgrind")
useGDB = flag.Bool("gdb", false, "If true, run BoringSSL code under gdb")
useLLDB = flag.Bool("lldb", false, "If true, run BoringSSL code under lldb")
flagDebug = flag.Bool("debug", false, "Hexdump the contents of the connection")
mallocTest = flag.Int64("malloc-test", -1, "If non-negative, run each test with each malloc in turn failing from the given number onwards.")
mallocTestDebug = flag.Bool("malloc-test-debug", false, "If true, ask bssl_shim to abort rather than fail a malloc. This can be used with a specific value for --malloc-test to identity the malloc failing that is causing problems.")
jsonOutput = flag.String("json-output", "", "The file to output JSON results to.")
pipe = flag.Bool("pipe", false, "If true, print status output suitable for piping into another program.")
testToRun = flag.String("test", "", "The name of a test to run, or empty to run all tests")
numWorkers = flag.Int("num-workers", runtime.NumCPU(), "The number of workers to run in parallel.")
shimPath = flag.String("shim-path", "../../../build/ssl/test/bssl_shim", "The location of the shim binary.")
resourceDir = flag.String("resource-dir", ".", "The directory in which to find certificate and key files.")
fuzzer = flag.Bool("fuzzer", false, "If true, tests against a BoringSSL built in fuzzer mode.")
transcriptDir = flag.String("transcript-dir", "", "The directory in which to write transcripts.")
idleTimeout = flag.Duration("idle-timeout", 15*time.Second, "The number of seconds to wait for a read or write to bssl_shim.")
deterministic = flag.Bool("deterministic", false, "If true, uses a deterministic PRNG in the runner.")
useValgrind = flag.Bool("valgrind", false, "If true, run code under valgrind")
useGDB = flag.Bool("gdb", false, "If true, run BoringSSL code under gdb")
useLLDB = flag.Bool("lldb", false, "If true, run BoringSSL code under lldb")
flagDebug = flag.Bool("debug", false, "Hexdump the contents of the connection")
mallocTest = flag.Int64("malloc-test", -1, "If non-negative, run each test with each malloc in turn failing from the given number onwards.")
mallocTestDebug = flag.Bool("malloc-test-debug", false, "If true, ask bssl_shim to abort rather than fail a malloc. This can be used with a specific value for --malloc-test to identity the malloc failing that is causing problems.")
jsonOutput = flag.String("json-output", "", "The file to output JSON results to.")
pipe = flag.Bool("pipe", false, "If true, print status output suitable for piping into another program.")
testToRun = flag.String("test", "", "The name of a test to run, or empty to run all tests")
numWorkers = flag.Int("num-workers", runtime.NumCPU(), "The number of workers to run in parallel.")
shimPath = flag.String("shim-path", "../../../build/ssl/test/bssl_shim", "The location of the shim binary.")
resourceDir = flag.String("resource-dir", ".", "The directory in which to find certificate and key files.")
fuzzer = flag.Bool("fuzzer", false, "If true, tests against a BoringSSL built in fuzzer mode.")
transcriptDir = flag.String("transcript-dir", "", "The directory in which to write transcripts.")
idleTimeout = flag.Duration("idle-timeout", 15*time.Second, "The number of seconds to wait for a read or write to bssl_shim.")
deterministic = flag.Bool("deterministic", false, "If true, uses a deterministic PRNG in the runner.")
allowUnimplemented = flag.Bool("allow-unimplemented", false, "If true, report pass even if some tests are unimplemented.")
)
type testCert int
@ -681,13 +683,10 @@ func lldbOf(path string, args ...string) *exec.Cmd {
return exec.Command("xterm", xtermArgs...)
}
type moreMallocsError struct{}
func (moreMallocsError) Error() string {
return "child process did not exhaust all allocation calls"
}
var errMoreMallocs = moreMallocsError{}
var (
errMoreMallocs = errors.New("child process did not exhaust all allocation calls")
errUnimplemented = errors.New("child process does not implement needed flags")
)
// accept accepts a connection from listener, unless waitChan signals a process
// exit first.
@ -883,8 +882,11 @@ func runTest(test *testCase, shimPath string, mallocNumToFail int64) error {
childErr := <-waitChan
if exitError, ok := childErr.(*exec.ExitError); ok {
if exitError.Sys().(syscall.WaitStatus).ExitStatus() == 88 {
switch exitError.Sys().(syscall.WaitStatus).ExitStatus() {
case 88:
return errMoreMallocs
case 89:
return errUnimplemented
}
}
@ -7701,7 +7703,7 @@ type statusMsg struct {
}
func statusPrinter(doneChan chan *testOutput, statusChan chan statusMsg, total int) {
var started, done, failed, lineLen int
var started, done, failed, unimplemented, lineLen int
testOutput := newTestOutput()
for msg := range statusChan {
@ -7720,9 +7722,18 @@ func statusPrinter(doneChan chan *testOutput, statusChan chan statusMsg, total i
done++
if msg.err != nil {
fmt.Printf("FAILED (%s)\n%s\n", msg.test.name, msg.err)
failed++
testOutput.addResult(msg.test.name, "FAIL")
if msg.err == errUnimplemented {
if *pipe {
// Print each test instead of a status line.
fmt.Printf("UNIMPLEMENTED (%s)\n", msg.test.name)
}
unimplemented++
testOutput.addResult(msg.test.name, "UNIMPLEMENTED")
} else {
fmt.Printf("FAILED (%s)\n%s\n", msg.test.name, msg.err)
failed++
testOutput.addResult(msg.test.name, "FAIL")
}
} else {
if *pipe {
// Print each test instead of a status line.
@ -7734,7 +7745,7 @@ func statusPrinter(doneChan chan *testOutput, statusChan chan statusMsg, total i
if !*pipe {
// Print a new status line.
line := fmt.Sprintf("%d/%d/%d/%d", failed, done, started, total)
line := fmt.Sprintf("%d/%d/%d/%d/%d", failed, unimplemented, done, started, total)
lineLen = len(line)
os.Stdout.WriteString(line)
}
@ -7816,7 +7827,11 @@ func main() {
}
}
if !testOutput.allPassed {
if !*allowUnimplemented && testOutput.NumFailuresByType["UNIMPLEMENTED"] > 0 {
os.Exit(1)
}
if !testOutput.noneFailed {
os.Exit(1)
}
}

View File

@ -29,7 +29,7 @@ type testOutput struct {
SecondsSinceEpoch float64 `json:"seconds_since_epoch"`
NumFailuresByType map[string]int `json:"num_failures_by_type"`
Tests map[string]testResult `json:"tests"`
allPassed bool
noneFailed bool
}
type testResult struct {
@ -45,7 +45,7 @@ func newTestOutput() *testOutput {
SecondsSinceEpoch: float64(time.Now().UnixNano()) / float64(time.Second/time.Nanosecond),
NumFailuresByType: make(map[string]int),
Tests: make(map[string]testResult),
allPassed: true,
noneFailed: true,
}
}
@ -59,8 +59,8 @@ func (t *testOutput) addResult(name, result string) {
IsUnexpected: result != "PASS",
}
t.NumFailuresByType[result]++
if result != "PASS" {
t.allPassed = false
if result != "PASS" && result != "UNIMPLEMENTED" {
t.noneFailed = false
}
}