1 # Shuffle order of tests and benchmarks
2
3 # Run tests
4 go test -v foo_test.go
5 ! stdout '-test.shuffle '
6 stdout '(?s)TestOne(.*)TestTwo(.*)TestThree'
7
8 go test -v -shuffle=off foo_test.go
9 ! stdout '-test.shuffle '
10 stdout '(?s)TestOne(.*)TestTwo(.*)TestThree'
11
12 go test -v -shuffle=42 foo_test.go
13 stdout '^-test.shuffle 42'
14 stdout '(?s)TestThree(.*)TestOne(.*)TestTwo'
15
16 go test -v -shuffle=43 foo_test.go
17 stdout '^-test.shuffle 43'
18 stdout '(?s)TestThree(.*)TestTwo(.*)TestOne'
19
20 go test -v -shuffle=44 foo_test.go
21 stdout '^-test.shuffle 44'
22 stdout '(?s)TestOne(.*)TestThree(.*)TestTwo'
23
24 go test -v -shuffle=0 foo_test.go
25 stdout '^-test.shuffle 0'
26 stdout '(?s)TestTwo(.*)TestOne(.*)TestThree'
27
28 go test -v -shuffle -1 foo_test.go
29 stdout '^-test.shuffle -1'
30 stdout '(?s)TestThree(.*)TestOne(.*)TestTwo'
31
32 go test -v -shuffle=on foo_test.go
33 stdout '^-test.shuffle '
34 stdout '(?s)=== RUN TestOne(.*)--- PASS: TestOne'
35 stdout '(?s)=== RUN TestTwo(.*)--- PASS: TestTwo'
36 stdout '(?s)=== RUN TestThree(.*)--- PASS: TestThree'
37
38
39 # Run tests and benchmarks
40 go test -v -bench=. foo_test.go
41 ! stdout '-test.shuffle '
42 stdout '(?s)TestOne(.*)TestTwo(.*)TestThree(.*)BenchmarkOne(.*)BenchmarkTwo(.*)BenchmarkThree'
43
44 go test -v -bench=. -shuffle=off foo_test.go
45 ! stdout '-test.shuffle '
46 stdout '(?s)TestOne(.*)TestTwo(.*)TestThree(.*)BenchmarkOne(.*)BenchmarkTwo(.*)BenchmarkThree'
47
48 go test -v -bench=. -shuffle=42 foo_test.go
49 stdout '^-test.shuffle 42'
50 stdout '(?s)TestThree(.*)TestOne(.*)TestTwo(.*)BenchmarkThree(.*)BenchmarkOne(.*)BenchmarkTwo'
51
52 go test -v -bench=. -shuffle=43 foo_test.go
53 stdout '^-test.shuffle 43'
54 stdout '(?s)TestThree(.*)TestTwo(.*)TestOne(.*)BenchmarkThree(.*)BenchmarkOne(.*)BenchmarkTwo'
55
56 go test -v -bench=. -shuffle=44 foo_test.go
57 stdout '^-test.shuffle 44'
58 stdout '(?s)TestOne(.*)TestThree(.*)TestTwo(.*)BenchmarkTwo(.*)BenchmarkOne(.*)BenchmarkThree'
59
60 go test -v -bench=. -shuffle=0 foo_test.go
61 stdout '^-test.shuffle 0'
62 stdout '(?s)TestTwo(.*)TestOne(.*)TestThree(.*)BenchmarkThree(.*)BenchmarkOne(.*)BenchmarkTwo'
63
64 go test -v -bench=. -shuffle -1 foo_test.go
65 stdout '^-test.shuffle -1'
66 stdout '(?s)TestThree(.*)TestOne(.*)TestTwo(.*)BenchmarkOne(.*)BenchmarkThree(.*)BenchmarkTwo'
67
68 go test -v -bench=. -shuffle=on foo_test.go
69 stdout '^-test.shuffle '
70 stdout '(?s)=== RUN TestOne(.*)--- PASS: TestOne'
71 stdout '(?s)=== RUN TestTwo(.*)--- PASS: TestTwo'
72 stdout '(?s)=== RUN TestThree(.*)--- PASS: TestThree'
73 stdout -count=2 'BenchmarkOne'
74 stdout -count=2 'BenchmarkTwo'
75 stdout -count=2 'BenchmarkThree'
76
77
78 # When running go test -count=N, each of the N runs distinct runs should maintain the same
79 # shuffled order of these tests.
80 go test -v -shuffle=43 -count=4 foo_test.go
81 stdout '^-test.shuffle 43'
82 stdout '(?s)TestThree(.*)TestTwo(.*)TestOne(.*)TestThree(.*)TestTwo(.*)TestOne(.*)TestThree(.*)TestTwo(.*)TestOne(.*)TestThree(.*)TestTwo(.*)TestOne'
83
84 go test -v -bench=. -shuffle=44 -count=2 foo_test.go
85 stdout '^-test.shuffle 44'
86 stdout '(?s)TestOne(.*)TestThree(.*)TestTwo(.*)TestOne(.*)TestThree(.*)TestTwo(.*)BenchmarkTwo(.*)BenchmarkOne(.*)BenchmarkThree(.*)'
87
88
89 # The feature should work with test binaries as well
90 go test -c
91 exec ./m.test -test.shuffle=off
92 ! stdout '^-test.shuffle '
93
94 exec ./m.test -test.shuffle=on
95 stdout '^-test.shuffle '
96
97 exec ./m.test -test.v -test.bench=. -test.shuffle=0 foo_test.go
98 stdout '^-test.shuffle 0'
99 stdout '(?s)TestTwo(.*)TestOne(.*)TestThree(.*)BenchmarkThree(.*)BenchmarkOne(.*)BenchmarkTwo'
100
101 exec ./m.test -test.v -test.bench=. -test.shuffle=123 foo_test.go
102 stdout '^-test.shuffle 123'
103 stdout '(?s)TestThree(.*)TestOne(.*)TestTwo(.*)BenchmarkThree(.*)BenchmarkTwo(.*)BenchmarkOne'
104
105 exec ./m.test -test.v -test.bench=. -test.shuffle=-1 foo_test.go
106 stdout '^-test.shuffle -1'
107 stdout '(?s)TestThree(.*)TestOne(.*)TestTwo(.*)BenchmarkOne(.*)BenchmarkThree(.*)BenchmarkTwo'
108
109 exec ./m.test -test.v -test.bench=. -test.shuffle=44 -test.count=2 foo_test.go
110 stdout '^-test.shuffle 44'
111 stdout '(?s)TestOne(.*)TestThree(.*)TestTwo(.*)TestOne(.*)TestThree(.*)TestTwo(.*)BenchmarkTwo(.*)BenchmarkOne(.*)BenchmarkThree(.*)'
112
113
114 # Negative testcases for invalid input
115 ! go test -shuffle -count=2
116 stderr 'invalid value "-count=2" for flag -shuffle: -shuffle argument must be "on", "off", or an int64: strconv.ParseInt: parsing "-count=2": invalid syntax'
117
118 ! go test -shuffle=
119 stderr '(?s)invalid value "" for flag -shuffle: -shuffle argument must be "on", "off", or an int64: strconv.ParseInt: parsing "": invalid syntax'
120
121 ! go test -shuffle=' '
122 stderr '(?s)invalid value " " for flag -shuffle: -shuffle argument must be "on", "off", or an int64: strconv.ParseInt: parsing " ": invalid syntax'
123
124 ! go test -shuffle=true
125 stderr 'invalid value "true" for flag -shuffle: -shuffle argument must be "on", "off", or an int64: strconv.ParseInt: parsing "true": invalid syntax'
126
127 ! go test -shuffle='abc'
128 stderr 'invalid value "abc" for flag -shuffle: -shuffle argument must be "on", "off", or an int64: strconv.ParseInt: parsing "abc": invalid syntax'
129
130 -- go.mod --
131 module m
132
133 go 1.16
134 -- foo_test.go --
135 package foo
136
137 import "testing"
138
139 func TestOne(t *testing.T) {}
140 func TestTwo(t *testing.T) {}
141 func TestThree(t *testing.T) {}
142
143 func BenchmarkOne(b *testing.B) {}
144 func BenchmarkTwo(b *testing.B) {}
145 func BenchmarkThree(b *testing.B) {}
146
147 -- foo.go --
148 package foo
149
View as plain text