- /* Copyright (c) 2015, Google Inc.
- *
- * Permission to use, copy, modify, and/or distribute this software for any
- * purpose with or without fee is hereby granted, provided that the above
- * copyright notice and this permission notice appear in all copies.
- *
- * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
- * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
- * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
- * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
- * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
- * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
- * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */
-
- package main
-
- import (
- "bufio"
- "bytes"
- "encoding/json"
- "errors"
- "flag"
- "fmt"
- "math/rand"
- "os"
- "os/exec"
- "path"
- "runtime"
- "strconv"
- "strings"
- "sync"
- "syscall"
-
- "boringssl.googlesource.com/boringssl/util/testresult"
- )
-
- // TODO(davidben): Link tests with the malloc shim and port -malloc-test to this runner.
-
- var (
- useValgrind = flag.Bool("valgrind", false, "If true, run code under valgrind")
- useCallgrind = flag.Bool("callgrind", false, "If true, run code under valgrind to generate callgrind traces.")
- useGDB = flag.Bool("gdb", false, "If true, run BoringSSL code under gdb")
- useSDE = flag.Bool("sde", false, "If true, run BoringSSL code under Intel's SDE for each supported chip")
- sdePath = flag.String("sde-path", "sde", "The path to find the sde binary.")
- buildDir = flag.String("build-dir", "build", "The build directory to run the tests from.")
- numWorkers = flag.Int("num-workers", runtime.NumCPU(), "Runs the given number of workers when testing.")
- jsonOutput = flag.String("json-output", "", "The file to output JSON results to.")
- mallocTest = flag.Int64("malloc-test", -1, "If non-negative, run each test with each malloc in turn failing from the given number onwards.")
- mallocTestDebug = flag.Bool("malloc-test-debug", false, "If true, ask each test to abort rather than fail a malloc. This can be used with a specific value for --malloc-test to identity the malloc failing that is causing problems.")
- simulateARMCPUs = flag.Bool("simulate-arm-cpus", simulateARMCPUsDefault(), "If true, runs tests simulating different ARM CPUs.")
- )
-
- func simulateARMCPUsDefault() bool {
- return runtime.GOOS == "linux" && (runtime.GOARCH == "arm" || runtime.GOARCH == "arm64")
- }
-
- type test struct {
- args []string
- shard, numShards int
- // cpu, if not empty, contains a code to simulate. For SDE, run `sde64
- // -help` to get a list of these codes. For ARM, see gtest_main.cc for
- // the supported values.
- cpu string
- }
-
- type result struct {
- Test test
- Passed bool
- Error error
- }
-
- // sdeCPUs contains a list of CPU code that we run all tests under when *useSDE
- // is true.
- var sdeCPUs = []string{
- "p4p", // Pentium4 Prescott
- "mrm", // Merom
- "pnr", // Penryn
- "nhm", // Nehalem
- "wsm", // Westmere
- "snb", // Sandy Bridge
- "ivb", // Ivy Bridge
- "hsw", // Haswell
- "bdw", // Broadwell
- "skx", // Skylake Server
- "skl", // Skylake Client
- "cnl", // Cannonlake
- "knl", // Knights Landing
- "slt", // Saltwell
- "slm", // Silvermont
- "glm", // Goldmont
- "knm", // Knights Mill
- }
-
- var armCPUs = []string{
- "none", // No support for any ARM extensions.
- "neon", // Support for NEON.
- "crypto", // Support for NEON and crypto extensions.
- }
-
- func valgrindOf(dbAttach bool, path string, args ...string) *exec.Cmd {
- valgrindArgs := []string{"--error-exitcode=99", "--track-origins=yes", "--leak-check=full", "--quiet"}
- if dbAttach {
- valgrindArgs = append(valgrindArgs, "--db-attach=yes", "--db-command=xterm -e gdb -nw %f %p")
- }
- valgrindArgs = append(valgrindArgs, path)
- valgrindArgs = append(valgrindArgs, args...)
-
- return exec.Command("valgrind", valgrindArgs...)
- }
-
- func callgrindOf(path string, args ...string) *exec.Cmd {
- valgrindArgs := []string{"-q", "--tool=callgrind", "--dump-instr=yes", "--collect-jumps=yes", "--callgrind-out-file=" + *buildDir + "/callgrind/callgrind.out.%p"}
- valgrindArgs = append(valgrindArgs, path)
- valgrindArgs = append(valgrindArgs, args...)
-
- return exec.Command("valgrind", valgrindArgs...)
- }
-
- func gdbOf(path string, args ...string) *exec.Cmd {
- xtermArgs := []string{"-e", "gdb", "--args"}
- xtermArgs = append(xtermArgs, path)
- xtermArgs = append(xtermArgs, args...)
-
- return exec.Command("xterm", xtermArgs...)
- }
-
- func sdeOf(cpu, path string, args ...string) *exec.Cmd {
- sdeArgs := []string{"-" + cpu}
- // The kernel's vdso code for gettimeofday sometimes uses the RDTSCP
- // instruction. Although SDE has a -chip_check_vsyscall flag that
- // excludes such code by default, it does not seem to work. Instead,
- // pass the -chip_check_exe_only flag which retains test coverage when
- // statically linked and excludes the vdso.
- if cpu == "p4p" || cpu == "pnr" || cpu == "mrm" || cpu == "slt" {
- sdeArgs = append(sdeArgs, "-chip_check_exe_only")
- }
- sdeArgs = append(sdeArgs, "--", path)
- sdeArgs = append(sdeArgs, args...)
- return exec.Command(*sdePath, sdeArgs...)
- }
-
- var (
- errMoreMallocs = errors.New("child process did not exhaust all allocation calls")
- errTestSkipped = errors.New("test was skipped")
- )
-
- func runTestOnce(test test, mallocNumToFail int64) (passed bool, err error) {
- prog := path.Join(*buildDir, test.args[0])
- args := append([]string{}, test.args[1:]...)
- if *simulateARMCPUs && test.cpu != "" {
- args = append(args, "--cpu=" + test.cpu)
- }
- if *useSDE {
- // SDE is neither compatible with the unwind tester nor automatically
- // detected.
- args = append(args, "--no_unwind_tests")
- }
- var cmd *exec.Cmd
- if *useValgrind {
- cmd = valgrindOf(false, prog, args...)
- } else if *useCallgrind {
- cmd = callgrindOf(prog, args...)
- } else if *useGDB {
- cmd = gdbOf(prog, args...)
- } else if *useSDE {
- cmd = sdeOf(test.cpu, prog, args...)
- } else {
- cmd = exec.Command(prog, args...)
- }
- var outBuf bytes.Buffer
- cmd.Stdout = &outBuf
- cmd.Stderr = &outBuf
- if mallocNumToFail >= 0 {
- cmd.Env = os.Environ()
- cmd.Env = append(cmd.Env, "MALLOC_NUMBER_TO_FAIL="+strconv.FormatInt(mallocNumToFail, 10))
- if *mallocTestDebug {
- cmd.Env = append(cmd.Env, "MALLOC_ABORT_ON_FAIL=1")
- }
- cmd.Env = append(cmd.Env, "_MALLOC_CHECK=1")
- }
-
- if err := cmd.Start(); err != nil {
- return false, err
- }
- if err := cmd.Wait(); err != nil {
- if exitError, ok := err.(*exec.ExitError); ok {
- switch exitError.Sys().(syscall.WaitStatus).ExitStatus() {
- case 88:
- return false, errMoreMallocs
- case 89:
- fmt.Print(string(outBuf.Bytes()))
- return false, errTestSkipped
- }
- }
- fmt.Print(string(outBuf.Bytes()))
- return false, err
- }
-
- // Account for Windows line-endings.
- stdout := bytes.Replace(outBuf.Bytes(), []byte("\r\n"), []byte("\n"), -1)
-
- if bytes.HasSuffix(stdout, []byte("PASS\n")) &&
- (len(stdout) == 5 || stdout[len(stdout)-6] == '\n') {
- return true, nil
- }
-
- // Also accept a googletest-style pass line. This is left here in
- // transition until the tests are all converted and this script made
- // unnecessary.
- if bytes.Contains(stdout, []byte("\n[ PASSED ]")) {
- return true, nil
- }
-
- fmt.Print(string(outBuf.Bytes()))
- return false, nil
- }
-
- func runTest(test test) (bool, error) {
- if *mallocTest < 0 {
- return runTestOnce(test, -1)
- }
-
- for mallocNumToFail := int64(*mallocTest); ; mallocNumToFail++ {
- if passed, err := runTestOnce(test, mallocNumToFail); err != errMoreMallocs {
- if err != nil {
- err = fmt.Errorf("at malloc %d: %s", mallocNumToFail, err)
- }
- return passed, err
- }
- }
- }
-
- // setWorkingDirectory walks up directories as needed until the current working
- // directory is the top of a BoringSSL checkout.
- func setWorkingDirectory() {
- for i := 0; i < 64; i++ {
- if _, err := os.Stat("BUILDING.md"); err == nil {
- return
- }
- os.Chdir("..")
- }
-
- panic("Couldn't find BUILDING.md in a parent directory!")
- }
-
- func parseTestConfig(filename string) ([]test, error) {
- in, err := os.Open(filename)
- if err != nil {
- return nil, err
- }
- defer in.Close()
-
- decoder := json.NewDecoder(in)
- var testArgs [][]string
- if err := decoder.Decode(&testArgs); err != nil {
- return nil, err
- }
-
- var result []test
- for _, args := range testArgs {
- result = append(result, test{args: args})
- }
- return result, nil
- }
-
- func worker(tests <-chan test, results chan<- result, done *sync.WaitGroup) {
- defer done.Done()
- for test := range tests {
- passed, err := runTest(test)
- results <- result{test, passed, err}
- }
- }
-
- func (t test) shortName() string {
- return t.args[0] + t.shardMsg() + t.cpuMsg()
- }
-
- func (t test) longName() string {
- return strings.Join(t.args, " ") + t.cpuMsg()
- }
-
- func (t test) shardMsg() string {
- if t.numShards == 0 {
- return ""
- }
-
- return fmt.Sprintf(" [shard %d/%d]", t.shard+1, t.numShards)
- }
-
- func (t test) cpuMsg() string {
- if len(t.cpu) == 0 {
- return ""
- }
-
- return fmt.Sprintf(" (for CPU %q)", t.cpu)
- }
-
- func (t test) getGTestShards() ([]test, error) {
- if *numWorkers == 1 || len(t.args) != 1 {
- return []test{t}, nil
- }
-
- // Only shard the three GTest-based tests.
- if t.args[0] != "crypto/crypto_test" && t.args[0] != "ssl/ssl_test" && t.args[0] != "decrepit/decrepit_test" {
- return []test{t}, nil
- }
-
- prog := path.Join(*buildDir, t.args[0])
- cmd := exec.Command(prog, "--gtest_list_tests")
- var stdout bytes.Buffer
- cmd.Stdout = &stdout
- if err := cmd.Start(); err != nil {
- return nil, err
- }
- if err := cmd.Wait(); err != nil {
- return nil, err
- }
-
- var group string
- var tests []string
- scanner := bufio.NewScanner(&stdout)
- for scanner.Scan() {
- line := scanner.Text()
-
- // Remove the parameter comment and trailing space.
- if idx := strings.Index(line, "#"); idx >= 0 {
- line = line[:idx]
- }
- line = strings.TrimSpace(line)
- if len(line) == 0 {
- continue
- }
-
- if line[len(line)-1] == '.' {
- group = line
- continue
- }
-
- if len(group) == 0 {
- return nil, fmt.Errorf("found test case %q without group", line)
- }
- tests = append(tests, group+line)
- }
-
- const testsPerShard = 20
- if len(tests) <= testsPerShard {
- return []test{t}, nil
- }
-
- // Slow tests which process large test vector files tend to be grouped
- // together, so shuffle the order.
- shuffled := make([]string, len(tests))
- perm := rand.Perm(len(tests))
- for i, j := range perm {
- shuffled[i] = tests[j]
- }
-
- var shards []test
- for i := 0; i < len(shuffled); i += testsPerShard {
- n := len(shuffled) - i
- if n > testsPerShard {
- n = testsPerShard
- }
- shard := t
- shard.args = []string{shard.args[0], "--gtest_filter=" + strings.Join(shuffled[i:i+n], ":")}
- shard.shard = len(shards)
- shards = append(shards, shard)
- }
-
- for i := range shards {
- shards[i].numShards = len(shards)
- }
-
- return shards, nil
- }
-
- func main() {
- flag.Parse()
- setWorkingDirectory()
-
- testCases, err := parseTestConfig("util/all_tests.json")
- if err != nil {
- fmt.Printf("Failed to parse input: %s\n", err)
- os.Exit(1)
- }
-
- var wg sync.WaitGroup
- tests := make(chan test, *numWorkers)
- results := make(chan result, *numWorkers)
-
- for i := 0; i < *numWorkers; i++ {
- wg.Add(1)
- go worker(tests, results, &wg)
- }
-
- go func() {
- for _, test := range testCases {
- if *useSDE {
- // SDE generates plenty of tasks and gets slower
- // with additional sharding.
- for _, cpu := range sdeCPUs {
- testForCPU := test
- testForCPU.cpu = cpu
- tests <- testForCPU
- }
- } else if *simulateARMCPUs {
- // This mode is run instead of the default path,
- // so also include the native flow.
- tests <- test
- for _, cpu := range armCPUs {
- testForCPU := test
- testForCPU.cpu = cpu
- tests <- testForCPU
- }
- } else {
- shards, err := test.getGTestShards()
- if err != nil {
- fmt.Printf("Error listing tests: %s\n", err)
- os.Exit(1)
- }
- for _, shard := range shards {
- tests <- shard
- }
- }
- }
- close(tests)
-
- wg.Wait()
- close(results)
- }()
-
- testOutput := testresult.NewResults()
- var failed, skipped []test
- for testResult := range results {
- test := testResult.Test
- args := test.args
-
- if testResult.Error == errTestSkipped {
- fmt.Printf("%s\n", test.longName())
- fmt.Printf("%s was skipped\n", args[0])
- skipped = append(skipped, test)
- testOutput.AddSkip(test.longName())
- } else if testResult.Error != nil {
- fmt.Printf("%s\n", test.longName())
- fmt.Printf("%s failed to complete: %s\n", args[0], testResult.Error)
- failed = append(failed, test)
- testOutput.AddResult(test.longName(), "CRASH")
- } else if !testResult.Passed {
- fmt.Printf("%s\n", test.longName())
- fmt.Printf("%s failed to print PASS on the last line.\n", args[0])
- failed = append(failed, test)
- testOutput.AddResult(test.longName(), "FAIL")
- } else {
- fmt.Printf("%s\n", test.shortName())
- testOutput.AddResult(test.longName(), "PASS")
- }
- }
-
- if *jsonOutput != "" {
- if err := testOutput.WriteToFile(*jsonOutput); err != nil {
- fmt.Fprintf(os.Stderr, "Error: %s\n", err)
- }
- }
-
- if len(skipped) > 0 {
- fmt.Printf("\n%d of %d tests were skipped:\n", len(skipped), len(testCases))
- for _, test := range skipped {
- fmt.Printf("\t%s%s\n", strings.Join(test.args, " "), test.cpuMsg())
- }
- }
-
- if len(failed) > 0 {
- fmt.Printf("\n%d of %d tests failed:\n", len(failed), len(testCases))
- for _, test := range failed {
- fmt.Printf("\t%s%s\n", strings.Join(test.args, " "), test.cpuMsg())
- }
- os.Exit(1)
- }
-
- fmt.Printf("\nAll tests passed!\n")
- }
|