forked from dapphub/dapptools
-
Notifications
You must be signed in to change notification settings - Fork 3
/
Copy pathrun-blockchain-tests
executable file
·136 lines (124 loc) · 3.25 KB
/
run-blockchain-tests
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
#!/usr/bin/env bash
set -e
# Invoke with hevm e.g.
# hevm compliance --tests ~/ethereum-tests --skip modexp --timeout 20 --html
HEVM=${HEVM:-hevm}
if [[ "$#" -lt 1 ]]; then
echo >&2 "usage: $(basename "$0") <tests-dir>"
exit 1
fi
tests=$1
html=$2
match=$3
skip=$4
timeout=${5:-10}
_html () {
cat <<.
<!doctype html>
<title>hevm test results</title>
<style>
* { font-family:
"latin modern mono", "fantasque sans mono",
inconsolata, menlo, monospace;
font-size: 22px;
line-height: 26px; }
body { margin: 2rem; }
header { text-align: center; margin: 4rem 0; }
table { border-collapse: collapse; width: 100%; }
tr:nth-child(even) { background: rgba(0, 0, 0, 0.05); }
td:not(:first-child):not(:last-child) { padding: 0 1rem; }
.category { opacity: 0.6; text-align: right }
a { color: darkblue; text-decoration: none; }
h1, h2 { text-align: center; margin-top: 2rem }
.testcase { font-weight: bold }
#failed .testcase { color: rgb(200, 0, 0) }
#balancefailed .testcase { color: rgb(200, 200, 0) }
#passed .testcase { color: rgb(0, 150, 0) }
#skipped .testcase { color: rgb(0, 100, 250) }
#timeout .testcase { color: rgb(0, 0, 250) }
</style>
<header>
<h1>hevm consensus test report</h1>
<p>$(date +%Y-%m-%d)</p>
<p>$(echo "$npass passed, $nbal bad-balance, $nnon bad-nonce, $nstr bad-storage, $nfail failed, $nskip skipped, $ntime timeout")</p>
(Test suite: <span class=GeneralStateTests</span>GeneralStateTests</span> for Istanbul)
</header>
<h2>Failed tests</h2>
<table id=failed>
<tbody>
$(echo $noncefailed)
$(echo $storagefailed)
$(echo $failed)
</table>
<h2>Failed tests (due to balance only)</h2>
<table id=balancefailed>
<tbody>
$(echo $balancefailed)
</table>
<h2>Timeout tests</h2>
<table id=timeout>
<tbody>
$(echo $timeouts)
</table>
<h2>Skipped tests</h2>
<table id=skipped>
<tbody>
$(echo $skipped)
</table>
<h2>Passed tests</h2>
<table id=passed>
<tbody>
$(echo $passed)
</table>
.
}
shopt -s nocasematch
{
cd "$tests"
for x in BlockchainTests/GeneralStateTests/*/*; do
if [[ $x =~ .*$match.* ]] && [[ -n $skip && $x =~ .*$skip.* ]]; then
for job in $(<$x jq '.|keys[]' -r); do
echo -n "$job " ; echo "skip"
done
elif [[ $x =~ .*$match.* ]]; then
set +e
"$HEVM" bc-test --file $x --timeout $timeout 2>&1
set -e
fi
done
} | {
while read test outcome; do
echo >&2 "$test $outcome"
row="<tr><td class=testcase>$test<td>$outcome"
row+=$'\n'
case $outcome in
ok) passed+=$row ;;
bad-balance) balancefailed+=$row ;;
bad-nonce) noncefailed+=$row ;;
bad-storage) storagefailed+=$row ;;
timeout) timeouts+=$row ;;
skip) skipped+=$row ;;
*) failed+=$row ;;
esac
done
sum () { echo -ne "$1" | wc -l | awk '{print $1}'; }
npass=$(sum "$passed")
nbal=$(sum "$balancefailed")
nnon=$(sum "$noncefailed")
nstr=$(sum "$storagefailed")
nfail=$(sum "$failed")
ntime=$(sum "$timeouts")
nskip=$(sum "$skipped")
echo >&2 "passed: $npass"
echo >&2 "bad-balance: $nbal"
echo >&2 "bad-nonce: $nnon"
echo >&2 "bad-storage: $nstr"
echo >&2 "failed: $nfail"
echo >&2 "timeout: $ntime"
echo >&2 "skipped: $nskip"
if [[ $html == "True" ]]; then
_html
fi
nbad=$(($nbal + $nnon + $nstr + $nfail))
[[ $nbad -gt 0 ]] && exit 1 || exit 0
}