Home | History | Annotate | Download | only in util 
      1 /* Copyright (c) 2015, Google Inc.
      2  *
      3  * Permission to use, copy, modify, and/or distribute this software for any
      4  * purpose with or without fee is hereby granted, provided that the above
      5  * copyright notice and this permission notice appear in all copies.
      6  *
      7  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
      8  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
      9  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
     10  * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
     11  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
     12  * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
     13  * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */
     14 
     15 package main
     16 
     17 import (
     18 	"bytes"
     19 	"encoding/json"
     20 	"flag"
     21 	"fmt"
     22 	"os"
     23 	"os/exec"
     24 	"path"
     25 	"strconv"
     26 	"strings"
     27 	"syscall"
     28 	"time"
     29 )
     30 
     31 // TODO(davidben): Link tests with the malloc shim and port -malloc-test to this runner.
     32 
     33 var (
     34 	useValgrind     = flag.Bool("valgrind", false, "If true, run code under valgrind")
     35 	useGDB          = flag.Bool("gdb", false, "If true, run BoringSSL code under gdb")
     36 	buildDir        = flag.String("build-dir", "build", "The build directory to run the tests from.")
     37 	jsonOutput      = flag.String("json-output", "", "The file to output JSON results to.")
     38 	mallocTest      = flag.Int64("malloc-test", -1, "If non-negative, run each test with each malloc in turn failing from the given number onwards.")
     39 	mallocTestDebug = flag.Bool("malloc-test-debug", false, "If true, ask each test to abort rather than fail a malloc. This can be used with a specific value for --malloc-test to identity the malloc failing that is causing problems.")
     40 )
     41 
     42 type test []string
     43 
     44 // testOutput is a representation of Chromium's JSON test result format. See
     45 // https://www.chromium.org/developers/the-json-test-results-format
     46 type testOutput struct {
     47 	Version           int                   `json:"version"`
     48 	Interrupted       bool                  `json:"interrupted"`
     49 	PathDelimiter     string                `json:"path_delimiter"`
     50 	SecondsSinceEpoch float64               `json:"seconds_since_epoch"`
     51 	NumFailuresByType map[string]int        `json:"num_failures_by_type"`
     52 	Tests             map[string]testResult `json:"tests"`
     53 }
     54 
     55 type testResult struct {
     56 	Actual       string `json:"actual"`
     57 	Expected     string `json:"expected"`
     58 	IsUnexpected bool   `json:"is_unexpected"`
     59 }
     60 
     61 func newTestOutput() *testOutput {
     62 	return &testOutput{
     63 		Version:           3,
     64 		PathDelimiter:     ".",
     65 		SecondsSinceEpoch: float64(time.Now().UnixNano()) / float64(time.Second/time.Nanosecond),
     66 		NumFailuresByType: make(map[string]int),
     67 		Tests:             make(map[string]testResult),
     68 	}
     69 }
     70 
     71 func (t *testOutput) addResult(name, result string) {
     72 	if _, found := t.Tests[name]; found {
     73 		panic(name)
     74 	}
     75 	t.Tests[name] = testResult{
     76 		Actual:       result,
     77 		Expected:     "PASS",
     78 		IsUnexpected: result != "PASS",
     79 	}
     80 	t.NumFailuresByType[result]++
     81 }
     82 
     83 func (t *testOutput) writeTo(name string) error {
     84 	file, err := os.Create(name)
     85 	if err != nil {
     86 		return err
     87 	}
     88 	defer file.Close()
     89 	out, err := json.MarshalIndent(t, "", "  ")
     90 	if err != nil {
     91 		return err
     92 	}
     93 	_, err = file.Write(out)
     94 	return err
     95 }
     96 
     97 func valgrindOf(dbAttach bool, path string, args ...string) *exec.Cmd {
     98 	valgrindArgs := []string{"--error-exitcode=99", "--track-origins=yes", "--leak-check=full"}
     99 	if dbAttach {
    100 		valgrindArgs = append(valgrindArgs, "--db-attach=yes", "--db-command=xterm -e gdb -nw %f %p")
    101 	}
    102 	valgrindArgs = append(valgrindArgs, path)
    103 	valgrindArgs = append(valgrindArgs, args...)
    104 
    105 	return exec.Command("valgrind", valgrindArgs...)
    106 }
    107 
    108 func gdbOf(path string, args ...string) *exec.Cmd {
    109 	xtermArgs := []string{"-e", "gdb", "--args"}
    110 	xtermArgs = append(xtermArgs, path)
    111 	xtermArgs = append(xtermArgs, args...)
    112 
    113 	return exec.Command("xterm", xtermArgs...)
    114 }
    115 
    116 type moreMallocsError struct{}
    117 
    118 func (moreMallocsError) Error() string {
    119 	return "child process did not exhaust all allocation calls"
    120 }
    121 
    122 var errMoreMallocs = moreMallocsError{}
    123 
    124 func runTestOnce(test test, mallocNumToFail int64) (passed bool, err error) {
    125 	prog := path.Join(*buildDir, test[0])
    126 	args := test[1:]
    127 	var cmd *exec.Cmd
    128 	if *useValgrind {
    129 		cmd = valgrindOf(false, prog, args...)
    130 	} else if *useGDB {
    131 		cmd = gdbOf(prog, args...)
    132 	} else {
    133 		cmd = exec.Command(prog, args...)
    134 	}
    135 	var stdoutBuf bytes.Buffer
    136 	var stderrBuf bytes.Buffer
    137 	cmd.Stdout = &stdoutBuf
    138 	cmd.Stderr = &stderrBuf
    139 	if mallocNumToFail >= 0 {
    140 		cmd.Env = os.Environ()
    141 		cmd.Env = append(cmd.Env, "MALLOC_NUMBER_TO_FAIL="+strconv.FormatInt(mallocNumToFail, 10))
    142 		if *mallocTestDebug {
    143 			cmd.Env = append(cmd.Env, "MALLOC_ABORT_ON_FAIL=1")
    144 		}
    145 		cmd.Env = append(cmd.Env, "_MALLOC_CHECK=1")
    146 	}
    147 
    148 	if err := cmd.Start(); err != nil {
    149 		return false, err
    150 	}
    151 	if err := cmd.Wait(); err != nil {
    152 		if exitError, ok := err.(*exec.ExitError); ok {
    153 			if exitError.Sys().(syscall.WaitStatus).ExitStatus() == 88 {
    154 				return false, errMoreMallocs
    155 			}
    156 		}
    157 		fmt.Print(string(stderrBuf.Bytes()))
    158 		return false, err
    159 	}
    160 	fmt.Print(string(stderrBuf.Bytes()))
    161 
    162 	// Account for Windows line-endings.
    163 	stdout := bytes.Replace(stdoutBuf.Bytes(), []byte("\r\n"), []byte("\n"), -1)
    164 
    165 	if bytes.HasSuffix(stdout, []byte("PASS\n")) &&
    166 		(len(stdout) == 5 || stdout[len(stdout)-6] == '\n') {
    167 		return true, nil
    168 	}
    169 	return false, nil
    170 }
    171 
    172 func runTest(test test) (bool, error) {
    173 	if *mallocTest < 0 {
    174 		return runTestOnce(test, -1)
    175 	}
    176 
    177 	for mallocNumToFail := int64(*mallocTest); ; mallocNumToFail++ {
    178 		if passed, err := runTestOnce(test, mallocNumToFail); err != errMoreMallocs {
    179 			if err != nil {
    180 				err = fmt.Errorf("at malloc %d: %s", mallocNumToFail, err)
    181 			}
    182 			return passed, err
    183 		}
    184 	}
    185 }
    186 
    187 // shortTestName returns the short name of a test. Except for evp_test, it
    188 // assumes that any argument which ends in .txt is a path to a data file and not
    189 // relevant to the test's uniqueness.
    190 func shortTestName(test test) string {
    191 	var args []string
    192 	for _, arg := range test {
    193 		if test[0] == "crypto/evp/evp_test" || !strings.HasSuffix(arg, ".txt") {
    194 			args = append(args, arg)
    195 		}
    196 	}
    197 	return strings.Join(args, " ")
    198 }
    199 
    200 // setWorkingDirectory walks up directories as needed until the current working
    201 // directory is the top of a BoringSSL checkout.
    202 func setWorkingDirectory() {
    203 	for i := 0; i < 64; i++ {
    204 		if _, err := os.Stat("BUILDING.md"); err == nil {
    205 			return
    206 		}
    207 		os.Chdir("..")
    208 	}
    209 
    210 	panic("Couldn't find BUILDING.md in a parent directory!")
    211 }
    212 
    213 func parseTestConfig(filename string) ([]test, error) {
    214 	in, err := os.Open(filename)
    215 	if err != nil {
    216 		return nil, err
    217 	}
    218 	defer in.Close()
    219 
    220 	decoder := json.NewDecoder(in)
    221 	var result []test
    222 	if err := decoder.Decode(&result); err != nil {
    223 		return nil, err
    224 	}
    225 	return result, nil
    226 }
    227 
    228 func main() {
    229 	flag.Parse()
    230 	setWorkingDirectory()
    231 
    232 	tests, err := parseTestConfig("util/all_tests.json")
    233 	if err != nil {
    234 		fmt.Printf("Failed to parse input: %s\n", err)
    235 		os.Exit(1)
    236 	}
    237 
    238 	testOutput := newTestOutput()
    239 	var failed []test
    240 	for _, test := range tests {
    241 		fmt.Printf("%s\n", strings.Join([]string(test), " "))
    242 
    243 		name := shortTestName(test)
    244 		passed, err := runTest(test)
    245 		if err != nil {
    246 			fmt.Printf("%s failed to complete: %s\n", test[0], err)
    247 			failed = append(failed, test)
    248 			testOutput.addResult(name, "CRASHED")
    249 		} else if !passed {
    250 			fmt.Printf("%s failed to print PASS on the last line.\n", test[0])
    251 			failed = append(failed, test)
    252 			testOutput.addResult(name, "FAIL")
    253 		} else {
    254 			testOutput.addResult(name, "PASS")
    255 		}
    256 	}
    257 
    258 	if *jsonOutput != "" {
    259 		if err := testOutput.writeTo(*jsonOutput); err != nil {
    260 			fmt.Fprintf(os.Stderr, "Error: %s\n", err)
    261 		}
    262 	}
    263 
    264 	if len(failed) > 0 {
    265 		fmt.Printf("\n%d of %d tests failed:\n", len(failed), len(tests))
    266 		for _, test := range failed {
    267 			fmt.Printf("\t%s\n", strings.Join([]string(test), " "))
    268 		}
    269 		os.Exit(1)
    270 	}
    271 
    272 	fmt.Printf("\nAll tests passed!\n")
    273 }
    274