Skip to content

Commit 3ea558e

Browse files
authored
Merge pull request #332 from atlassian/new-load-tester
This adds a new load tester
2 parents 53a5e85 + 4f38ea5 commit 3ea558e

File tree

6 files changed

+339
-4
lines changed

6 files changed

+339
-4
lines changed

README.md

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -308,6 +308,16 @@ Incorrect meta tag values will be handled in best effort manner, i.e.
308308

309309
This is an experimental feature and it may be removed or changed in future versions.
310310

311+
312+
Load testing
313+
------------
314+
There is a tool under `cmd/loader` with support for a number of options which can be used to generate synthetic statsd
315+
load. There is also another load generation tool under `cmd/tester` which is deprecated and will be removed in a
316+
future release.
317+
318+
Help for the loader tool can be found through `--help`.
319+
320+
311321
Sending metrics
312322
---------------
313323
The server listens for UDP packets on the address given by the `--metrics-addr` flag,
@@ -316,12 +326,13 @@ flag (space separated list of backend names).
316326

317327
Currently supported backends are:
318328

319-
* graphite
329+
* cloudwatch
320330
* datadog
331+
* graphite
332+
* influxdb
333+
* newrelic
321334
* statsdaemon
322335
* stdout
323-
* cloudwatch
324-
* newrelic
325336

326337
The format of each metric is:
327338

cmd/loader/args.go

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
package main
2+
3+
import (
4+
"fmt"
5+
"os"
6+
7+
"github.com/jessevdk/go-flags"
8+
)
9+
10+
type commandOptions struct {
11+
Target string `short:"a" long:"address" default:"127.0.0.1:8125" description:"Address to send metrics" `
12+
MetricPrefix string `short:"p" long:"metric-prefix" default:"loadtest." description:"Metric name prefix" `
13+
MetricSuffix string ` long:"metric-suffix" default:".%d" description:"Metric suffix with cardinality marker" `
14+
Rate uint `short:"r" long:"rate" default:"1000" description:"Target packets per second" `
15+
DatagramSize uint ` long:"buffer-size" default:"1500" description:"Maximum size of datagram to send" `
16+
Workers uint `short:"w" long:"workers" default:"1" description:"Number of parallel workers to use" `
17+
Counts struct {
18+
Counter uint64 ` short:"c" long:"counter-count" description:"Number of counters to send" `
19+
Gauge uint64 ` short:"g" long:"gauge-count" description:"Number of gauges to send" `
20+
Set uint64 ` short:"s" long:"set-count" description:"Number of sets to send" `
21+
Timer uint64 ` short:"t" long:"timer-count" description:"Number of timers to send" `
22+
} `group:"Metric count"`
23+
NameCard struct {
24+
Counter uint ` long:"counter-cardinality" default:"1" description:"Cardinality of counter names" `
25+
Gauge uint ` long:"gauge-cardinality" default:"1" description:"Cardinality of gauges names" `
26+
Set uint ` long:"set-cardinality" default:"1" description:"Cardinality of set names" `
27+
Timer uint ` long:"timer-cardinality" default:"1" description:"Cardinality of timer names" `
28+
} `group:"Name cardinality"`
29+
TagCard struct {
30+
Counter []uint ` long:"counter-tag-cardinality" description:"Cardinality of count tags" `
31+
Gauge []uint ` long:"gauge-tag-cardinality" description:"Cardinality of gauge tags" `
32+
Set []uint ` long:"set-tag-cardinality" description:"Cardinality of set tags" `
33+
Timer []uint ` long:"timer-tag-cardinality" description:"Cardinality of timer tags" `
34+
} `group:"Tag cardinality"`
35+
ValueRange struct {
36+
Counter uint ` long:"counter-value-limit" default:"0" description:"Maximum value of counters minus one" `
37+
Gauge uint ` long:"gauge-value-limit" default:"1" description:"Maximum value of gauges" `
38+
Set uint ` long:"set-value-cardinality" default:"1" description:"Maximum number of values to send per set"`
39+
Timer uint ` long:"timer-value-limit" default:"1" description:"Maximum value of timers" `
40+
} `group:"Value range"`
41+
}
42+
43+
func parseArgs(args []string) commandOptions {
44+
var opts commandOptions
45+
parser := flags.NewParser(&opts, flags.HelpFlag | flags.PassDoubleDash)
46+
parser.LongDescription = "" + // because gofmt
47+
"When specifying cardinality, the tag cardinality can be specified multiple times,\n" +
48+
"and each tag will be named tagN:M. The maximum total cardinality will be:\n\n" +
49+
"|name| * |tag1| * |tag2| * ... * |tagN|\n\n" +
50+
"Care should be taken to not cause a combinatorial explosion."
51+
52+
positional, err := parser.ParseArgs(args)
53+
if err != nil {
54+
if !isHelp(err) {
55+
parser.WriteHelp(os.Stderr)
56+
_, _ = fmt.Fprintf(os.Stderr, "\n\nerror parsing command line: %v\n", err)
57+
os.Exit(1)
58+
}
59+
parser.WriteHelp(os.Stdout)
60+
os.Exit(0)
61+
}
62+
63+
if len(positional) != 0 {
64+
// Near as I can tell there's no way to say no positional arguments allowed.
65+
parser.WriteHelp(os.Stderr)
66+
_, _ = fmt.Fprintf(os.Stderr, "\n\nno positional arguments allowed\n")
67+
os.Exit(1)
68+
}
69+
70+
if opts.Counts.Counter+opts.Counts.Gauge+opts.Counts.Set+opts.Counts.Timer == 0 {
71+
parser.WriteHelp(os.Stderr)
72+
_, _ = fmt.Fprintf(os.Stderr, "\n\nAt least one of counter-count, gauge-count, set-count, or timer-count must be non-zero\n")
73+
os.Exit(1)
74+
}
75+
return opts
76+
}
77+
78+
// isHelp is a helper to test the error from ParseArgs() to
79+
// determine if the help message was written. It is safe to
80+
// call without first checking that error is nil.
81+
func isHelp(err error) bool {
82+
// This was copied from https://github.com/jessevdk/go-flags/blame/master/help.go#L499, as there has not been an
83+
// official release yet with this code. Renamed from WriteHelp to isHelp, as flags.ErrHelp is still returned when
84+
// flags.HelpFlag is set, flags.PrintError is clear, and -h/--help is passed on the command line, even though the
85+
// help is not displayed in such a situation.
86+
if err == nil { // No error
87+
return false
88+
}
89+
90+
flagError, ok := err.(*flags.Error)
91+
if !ok { // Not a go-flag error
92+
return false
93+
}
94+
95+
if flagError.Type != flags.ErrHelp { // Did not print the help message
96+
return false
97+
}
98+
99+
return true
100+
}

cmd/loader/generation.go

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
package main
2+
3+
import (
4+
"fmt"
5+
"math/rand"
6+
"strconv"
7+
"strings"
8+
"sync/atomic"
9+
)
10+
11+
type metricData struct {
12+
count uint64 // atomic
13+
nameFormat string
14+
nameCardinality uint
15+
tagCardinality []uint
16+
valueLimit uint
17+
}
18+
19+
type metricGenerator struct {
20+
rnd *rand.Rand
21+
22+
counters metricData
23+
gauges metricData
24+
sets metricData
25+
timers metricData
26+
}
27+
28+
func (md *metricData) genName(sb *strings.Builder, r *rand.Rand) {
29+
sb.WriteString(fmt.Sprintf(md.nameFormat, r.Intn(int(md.nameCardinality))))
30+
sb.WriteByte(':')
31+
}
32+
33+
func (md *metricData) genTags(sb *strings.Builder, r *rand.Rand) {
34+
if len(md.tagCardinality) > 0 {
35+
sb.WriteString("|#")
36+
for idx, c := range md.tagCardinality {
37+
if idx > 0 {
38+
sb.WriteByte(',')
39+
}
40+
sb.WriteString(fmt.Sprintf("tag%d:%d", idx, r.Intn(int(c))))
41+
}
42+
}
43+
sb.WriteByte('\n')
44+
}
45+
46+
func (mg *metricGenerator) nextCounter(sb *strings.Builder) {
47+
atomic.AddUint64(&mg.counters.count, ^uint64(0))
48+
mg.counters.genName(sb, mg.rnd)
49+
sb.WriteString(strconv.Itoa(1 + mg.rnd.Intn(int(mg.counters.valueLimit+1))))
50+
sb.WriteString("|c")
51+
mg.counters.genTags(sb, mg.rnd)
52+
}
53+
54+
func (mg *metricGenerator) nextGauge(sb *strings.Builder) {
55+
atomic.AddUint64(&mg.gauges.count, ^uint64(0))
56+
mg.gauges.genName(sb, mg.rnd)
57+
sb.WriteString(strconv.Itoa(mg.rnd.Intn(int(mg.gauges.valueLimit))))
58+
sb.WriteString("|g")
59+
mg.gauges.genTags(sb, mg.rnd)
60+
}
61+
62+
func (mg *metricGenerator) nextSet(sb *strings.Builder) {
63+
atomic.AddUint64(&mg.sets.count, ^uint64(0))
64+
mg.sets.genName(sb, mg.rnd)
65+
sb.WriteString(strconv.Itoa(mg.rnd.Intn(int(mg.sets.valueLimit))))
66+
sb.WriteString("|s")
67+
mg.sets.genTags(sb, mg.rnd)
68+
}
69+
70+
func (mg *metricGenerator) nextTimer(sb *strings.Builder) {
71+
atomic.AddUint64(&mg.timers.count, ^uint64(0))
72+
mg.timers.genName(sb, mg.rnd)
73+
sb.WriteString(strconv.FormatFloat(mg.rnd.Float64()*float64(mg.timers.valueLimit), 'g', -1, 64))
74+
sb.WriteString("|ms")
75+
mg.timers.genTags(sb, mg.rnd)
76+
}
77+
78+
func (mg *metricGenerator) next(sb *strings.Builder) bool {
79+
// We can safely read these non-atomically, because this goroutine is the only one that writes to them.
80+
total := mg.counters.count + mg.gauges.count + mg.sets.count + mg.timers.count
81+
if total == 0 {
82+
return false
83+
}
84+
85+
n := uint64(mg.rnd.Int63n(int64(total)))
86+
if n < mg.counters.count {
87+
mg.nextCounter(sb)
88+
} else if n < mg.counters.count+mg.gauges.count {
89+
mg.nextGauge(sb)
90+
} else if n < mg.counters.count+mg.gauges.count+mg.sets.count {
91+
mg.nextSet(sb)
92+
} else {
93+
mg.nextTimer(sb)
94+
}
95+
return true
96+
}

cmd/loader/main.go

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
package main
2+
3+
import (
4+
"bytes"
5+
"fmt"
6+
"math/rand"
7+
"net"
8+
"os"
9+
"strings"
10+
"sync/atomic"
11+
"time"
12+
)
13+
14+
func main() {
15+
opts := parseArgs(os.Args[1:])
16+
17+
pendingWorkers := make(chan struct{}, opts.Workers)
18+
metricGenerators := make([]*metricGenerator, 0, opts.Workers)
19+
for i := uint(0); i < opts.Workers; i++ {
20+
generator := &metricGenerator{
21+
rnd: rand.New(rand.NewSource(rand.Int63())),
22+
counters: metricData{
23+
nameFormat: fmt.Sprintf("%scounter%s", opts.MetricPrefix, opts.MetricSuffix),
24+
count: opts.Counts.Counter / uint64(opts.Workers),
25+
nameCardinality: opts.NameCard.Counter,
26+
tagCardinality: opts.TagCard.Counter,
27+
valueLimit: opts.ValueRange.Counter,
28+
},
29+
gauges: metricData{
30+
nameFormat: fmt.Sprintf("%sgauge%s", opts.MetricPrefix, opts.MetricSuffix),
31+
count: opts.Counts.Gauge / uint64(opts.Workers),
32+
nameCardinality: opts.NameCard.Gauge,
33+
tagCardinality: opts.TagCard.Gauge,
34+
valueLimit: opts.ValueRange.Gauge,
35+
},
36+
sets: metricData{
37+
nameFormat: fmt.Sprintf("%sset%s", opts.MetricPrefix, opts.MetricSuffix),
38+
count: opts.Counts.Set / uint64(opts.Workers),
39+
nameCardinality: opts.NameCard.Set,
40+
tagCardinality: opts.TagCard.Set,
41+
valueLimit: opts.ValueRange.Set,
42+
},
43+
timers: metricData{
44+
nameFormat: fmt.Sprintf("%stimer%s", opts.MetricPrefix, opts.MetricSuffix),
45+
count: opts.Counts.Timer / uint64(opts.Workers),
46+
nameCardinality: opts.NameCard.Timer,
47+
tagCardinality: opts.TagCard.Timer,
48+
valueLimit: opts.ValueRange.Timer,
49+
},
50+
}
51+
metricGenerators = append(metricGenerators, generator)
52+
go sendMetricsWorker(
53+
opts.Target,
54+
opts.DatagramSize,
55+
opts.Rate/opts.Workers,
56+
generator,
57+
pendingWorkers,
58+
)
59+
}
60+
61+
runningWorkers := opts.Workers
62+
statusTicker := time.NewTicker(1 * time.Second)
63+
for runningWorkers > 0 {
64+
select {
65+
case <-pendingWorkers:
66+
runningWorkers--
67+
case <-statusTicker.C:
68+
counters := uint64(0)
69+
gauges := uint64(0)
70+
sets := uint64(0)
71+
timers := uint64(0)
72+
for _, mg := range metricGenerators {
73+
counters += atomic.LoadUint64(&mg.counters.count)
74+
gauges += atomic.LoadUint64(&mg.gauges.count)
75+
sets += atomic.LoadUint64(&mg.sets.count)
76+
timers += atomic.LoadUint64(&mg.timers.count)
77+
}
78+
fmt.Printf("%d counters, %d gauges, %d sets, %d timers\n", counters, gauges, sets, timers)
79+
}
80+
}
81+
}
82+
83+
func sendMetricsWorker(
84+
address string,
85+
bufSize uint,
86+
rate uint,
87+
generator *metricGenerator,
88+
chDone chan<- struct{},
89+
) {
90+
s, err := net.DialTimeout("udp", address, 1*time.Second)
91+
if err != nil {
92+
panic(err)
93+
}
94+
95+
b := &bytes.Buffer{}
96+
97+
interval := time.Second / time.Duration(rate)
98+
99+
next := time.Now().Add(interval)
100+
101+
sb := &strings.Builder{}
102+
for generator.next(sb) {
103+
if uint(b.Len()+sb.Len()) > bufSize {
104+
timeToFlush := time.Until(next)
105+
if timeToFlush > 0 {
106+
time.Sleep(timeToFlush)
107+
}
108+
_, err := s.Write(b.Bytes())
109+
if err != nil {
110+
fmt.Printf("Pausing for 1 second, error sending packet: %v\n", err)
111+
time.Sleep(1*time.Second)
112+
}
113+
b.Reset()
114+
next = next.Add(interval)
115+
}
116+
b.WriteString(sb.String())
117+
sb.Reset()
118+
}
119+
120+
if b.Len() > 0 {
121+
_, err := s.Write(b.Bytes())
122+
if err != nil {
123+
panic(err)
124+
}
125+
}
126+
chDone <- struct{}{}
127+
}

go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,11 @@ require (
1515
github.com/gorilla/mux v1.7.3
1616
github.com/howeyc/fsnotify v0.9.0 // indirect
1717
github.com/imdario/mergo v0.3.8 // indirect
18+
github.com/jessevdk/go-flags v1.4.0
1819
github.com/json-iterator/go v1.1.9
1920
github.com/jstemmer/go-junit-report v0.9.1
2021
github.com/libp2p/go-reuseport v0.0.1
2122
github.com/magiconair/properties v1.8.1
22-
github.com/mozilla/tls-observatory v0.0.0-20190404164649-a3c1b6cfecfd
2323
github.com/sirupsen/logrus v1.4.2
2424
github.com/spf13/pflag v1.0.5
2525
github.com/spf13/viper v1.6.2

go.sum

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -198,6 +198,7 @@ github.com/imdario/mergo v0.3.8 h1:CGgOkSJeqMRmt0D9XLWExdT4m4F1vd3FV3VPt+0VxkQ=
198198
github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
199199
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
200200
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
201+
github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
201202
github.com/jingyugao/rowserrcheck v0.0.0-20191204022205-72ab7603b68a h1:GmsqmapfzSJkm28dhRoHz2tLRbJmqhU86IPgBtN3mmk=
202203
github.com/jingyugao/rowserrcheck v0.0.0-20191204022205-72ab7603b68a/go.mod h1:xRskid8CManxVta/ALEhJha/pweKBaVG6fWgc0yH25s=
203204
github.com/jirfag/go-printf-func-name v0.0.0-20191110105641-45db9963cdd3 h1:jNYPNLe3d8smommaoQlK7LOA5ESyUJJ+Wf79ZtA7Vp4=

0 commit comments

Comments
 (0)