ceremonyclient/nekryptology/cmd/benchcomp/main.go
2023-07-05 00:32:28 -05:00

138 lines
3.5 KiB
Go

//
// Copyright Coinbase, Inc. All Rights Reserved.
//
// SPDX-License-Identifier: Apache-2.0
//
// benchcomp implements a command that receives two benchmarks files as input and flags the benchmarks that have
// degraded by more than a threshold amount. The main goal of this tool is to be used in CI
// to check each PR.
package main
import (
"bytes"
"flag"
"fmt"
"io"
"io/ioutil"
"github.com/pkg/errors"
"golang.org/x/tools/benchmark/parse"
)
const THRESHOLD = 1.1
func main() {
cReader, nReader, err := parseCmdArgs()
if err != nil {
panic(err)
}
if err := Compare(cReader, nReader); err != nil {
panic(err)
}
}
func parseCmdArgs() (io.Reader, io.Reader, error) {
cFlag := flag.String("current", "current-bench.log", "The patch to the log file containing the output of the current benchmark result.")
nFlag := flag.String("new", "new-bench.log", "The patch to the log file containing the output of the new benchmark result.")
flag.Parse()
cBytes, err := ioutil.ReadFile(*cFlag)
if err != nil {
return nil, nil, fmt.Errorf("reading current log file %v", err)
}
nBytes, err := ioutil.ReadFile(*nFlag)
if err != nil {
return nil, nil, fmt.Errorf("reading new log file %v", err)
}
return bytes.NewBuffer(cBytes), bytes.NewBuffer(nBytes), nil
}
// Compare expects two readers which contain the output of two runs of `go test -bench` command and throws an error if
// the performance has degraded by more than `THRESHOLD` amount.
func Compare(currBench, newBench io.Reader) error {
c, n, err := parseBenchmarks(currBench, newBench)
if err != nil {
return errors.Wrap(err, "parsing benchmark outputs")
}
perfDeviations := make([]string, 0)
for bench := range c {
if _, ok := n[bench]; !ok {
// New benchmark, skipping
continue
} else {
currB := c[bench]
newB := n[bench]
err = compareBenches(currB, newB)
if err != nil {
perfDeviations = append(perfDeviations, fmt.Sprintf("%v", err))
}
}
}
if len(perfDeviations) != 0 {
return fmt.Errorf("%#v", perfDeviations)
}
return nil
}
func parseBenchmarks(currBench, newBench io.Reader) (parse.Set, parse.Set, error) {
c, err := parse.ParseSet(currBench)
if err != nil {
return nil, nil, errors.Wrap(err, "parsing current benchmark output")
}
n, err := parse.ParseSet(newBench)
if err != nil {
return nil, nil, errors.Wrap(err, "parsing new benchmark output")
}
return c, n, nil
}
func compareBenches(currB, newB []*parse.Benchmark) error {
currMap := make(map[string]*parse.Benchmark)
newMap := make(map[string]*parse.Benchmark)
for _, b := range currB {
// TODO: double check what is in Name
currMap[b.Name] = b
}
for _, b := range newB {
// TODO: double check what is in Name
newMap[b.Name] = b
}
for name := range currMap {
if _, ok := newMap[name]; ok {
compare := []struct {
current float64
new float64
}{
{
current: float64(currMap[name].AllocedBytesPerOp),
new: float64(newMap[name].AllocedBytesPerOp),
},
{
current: float64(currMap[name].AllocsPerOp),
new: float64(newMap[name].AllocsPerOp),
},
{
current: currMap[name].NsPerOp,
new: newMap[name].NsPerOp,
},
{
current: currMap[name].MBPerS,
new: newMap[name].MBPerS,
},
}
for _, t := range compare {
if t.new > t.current*THRESHOLD {
percent := (t.new - t.current) * 100 / t.current
return fmt.Errorf("benchmark %s exceeded previous benchmark by %0.2f percent. Current: %0.2f, New: %0.2f", name, percent, t.current, t.new)
}
}
}
}
return nil
}