mirror of
				https://source.quilibrium.com/quilibrium/ceremonyclient.git
				synced 2025-11-04 00:27:41 +00:00 
			
		
		
		
	
		
			
				
	
	
		
			138 lines
		
	
	
		
			3.5 KiB
		
	
	
	
		
			Go
		
	
	
	
	
	
			
		
		
	
	
			138 lines
		
	
	
		
			3.5 KiB
		
	
	
	
		
			Go
		
	
	
	
	
	
//
 | 
						|
// Copyright Coinbase, Inc. All Rights Reserved.
 | 
						|
//
 | 
						|
// SPDX-License-Identifier: Apache-2.0
 | 
						|
//
 | 
						|
 | 
						|
// benchcomp implements a command that receives two benchmarks files as input and flags the benchmarks that have
 | 
						|
// degraded by more than a threshold amount. The main goal of this tool is to be used in CI
 | 
						|
// to check each PR.
 | 
						|
package main
 | 
						|
 | 
						|
import (
 | 
						|
	"bytes"
 | 
						|
	"flag"
 | 
						|
	"fmt"
 | 
						|
	"io"
 | 
						|
	"io/ioutil"
 | 
						|
 | 
						|
	"github.com/pkg/errors"
 | 
						|
	"golang.org/x/tools/benchmark/parse"
 | 
						|
)
 | 
						|
 | 
						|
const THRESHOLD = 1.1
 | 
						|
 | 
						|
func main() {
 | 
						|
	cReader, nReader, err := parseCmdArgs()
 | 
						|
	if err != nil {
 | 
						|
		panic(err)
 | 
						|
	}
 | 
						|
 | 
						|
	if err := Compare(cReader, nReader); err != nil {
 | 
						|
		panic(err)
 | 
						|
	}
 | 
						|
}
 | 
						|
 | 
						|
func parseCmdArgs() (io.Reader, io.Reader, error) {
 | 
						|
	cFlag := flag.String("current", "current-bench.log", "The patch to the log file containing the output of the current benchmark result.")
 | 
						|
	nFlag := flag.String("new", "new-bench.log", "The patch to the log file containing the output of the new benchmark result.")
 | 
						|
	flag.Parse()
 | 
						|
	cBytes, err := ioutil.ReadFile(*cFlag)
 | 
						|
	if err != nil {
 | 
						|
		return nil, nil, fmt.Errorf("reading current log file %v", err)
 | 
						|
	}
 | 
						|
	nBytes, err := ioutil.ReadFile(*nFlag)
 | 
						|
	if err != nil {
 | 
						|
		return nil, nil, fmt.Errorf("reading new log file %v", err)
 | 
						|
	}
 | 
						|
 | 
						|
	return bytes.NewBuffer(cBytes), bytes.NewBuffer(nBytes), nil
 | 
						|
}
 | 
						|
 | 
						|
// Compare expects two readers which contain the output of two runs of `go test -bench` command and throws an error if
 | 
						|
// the performance has degraded by more than `THRESHOLD` amount.
 | 
						|
func Compare(currBench, newBench io.Reader) error {
 | 
						|
	c, n, err := parseBenchmarks(currBench, newBench)
 | 
						|
	if err != nil {
 | 
						|
		return errors.Wrap(err, "parsing benchmark outputs")
 | 
						|
	}
 | 
						|
 | 
						|
	perfDeviations := make([]string, 0)
 | 
						|
	for bench := range c {
 | 
						|
		if _, ok := n[bench]; !ok {
 | 
						|
			// New benchmark, skipping
 | 
						|
			continue
 | 
						|
		} else {
 | 
						|
			currB := c[bench]
 | 
						|
			newB := n[bench]
 | 
						|
			err = compareBenches(currB, newB)
 | 
						|
			if err != nil {
 | 
						|
				perfDeviations = append(perfDeviations, fmt.Sprintf("%v", err))
 | 
						|
			}
 | 
						|
		}
 | 
						|
	}
 | 
						|
 | 
						|
	if len(perfDeviations) != 0 {
 | 
						|
		return fmt.Errorf("%#v", perfDeviations)
 | 
						|
	}
 | 
						|
	return nil
 | 
						|
}
 | 
						|
 | 
						|
func parseBenchmarks(currBench, newBench io.Reader) (parse.Set, parse.Set, error) {
 | 
						|
	c, err := parse.ParseSet(currBench)
 | 
						|
	if err != nil {
 | 
						|
		return nil, nil, errors.Wrap(err, "parsing current benchmark output")
 | 
						|
	}
 | 
						|
	n, err := parse.ParseSet(newBench)
 | 
						|
	if err != nil {
 | 
						|
		return nil, nil, errors.Wrap(err, "parsing new benchmark output")
 | 
						|
	}
 | 
						|
 | 
						|
	return c, n, nil
 | 
						|
}
 | 
						|
 | 
						|
func compareBenches(currB, newB []*parse.Benchmark) error {
 | 
						|
	currMap := make(map[string]*parse.Benchmark)
 | 
						|
	newMap := make(map[string]*parse.Benchmark)
 | 
						|
	for _, b := range currB {
 | 
						|
		// TODO: double check what is in Name
 | 
						|
		currMap[b.Name] = b
 | 
						|
	}
 | 
						|
	for _, b := range newB {
 | 
						|
		// TODO: double check what is in Name
 | 
						|
		newMap[b.Name] = b
 | 
						|
	}
 | 
						|
	for name := range currMap {
 | 
						|
		if _, ok := newMap[name]; ok {
 | 
						|
			compare := []struct {
 | 
						|
				current float64
 | 
						|
				new     float64
 | 
						|
			}{
 | 
						|
				{
 | 
						|
					current: float64(currMap[name].AllocedBytesPerOp),
 | 
						|
					new:     float64(newMap[name].AllocedBytesPerOp),
 | 
						|
				},
 | 
						|
				{
 | 
						|
					current: float64(currMap[name].AllocsPerOp),
 | 
						|
					new:     float64(newMap[name].AllocsPerOp),
 | 
						|
				},
 | 
						|
				{
 | 
						|
					current: currMap[name].NsPerOp,
 | 
						|
					new:     newMap[name].NsPerOp,
 | 
						|
				},
 | 
						|
				{
 | 
						|
					current: currMap[name].MBPerS,
 | 
						|
					new:     newMap[name].MBPerS,
 | 
						|
				},
 | 
						|
			}
 | 
						|
			for _, t := range compare {
 | 
						|
				if t.new > t.current*THRESHOLD {
 | 
						|
					percent := (t.new - t.current) * 100 / t.current
 | 
						|
					return fmt.Errorf("benchmark %s exceeded previous benchmark by %0.2f percent. Current: %0.2f, New: %0.2f", name, percent, t.current, t.new)
 | 
						|
				}
 | 
						|
			}
 | 
						|
		}
 | 
						|
	}
 | 
						|
	return nil
 | 
						|
}
 |