1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
|
#!/usr/bin/env python
#
# Given a previous good compile narrow down miscompiles.
# Expects two directories named "before" and "after" each containing a set of
# assembly or object files where the "after" version is assumed to be broken.
# You also have to provide a script called "link_test". It is called with a list
# of files which should be linked together and result tested. "link_test" should
# returns with exitcode 0 if the linking and testing succeeded.
#
# abtest.py operates by taking all files from the "before" directory and
# in each step replacing one of them with a file from the "bad" directory.
#
# Additionally you can perform the same steps with a single .s file. In this
# mode functions are identified by "# -- Begin FunctionName" and
# "# -- End FunctionName" markers. The abtest.py then takes all functions from
# the file in the "before" directory and replaces one function with the
# corresponding function from the "bad" file in each step.
#
# Example usage to identify miscompiled files:
# 1. Create a link_test script, make it executable. Simple Example:
# clang "$@" -o /tmp/test && /tmp/test || echo "PROBLEM"
# 2. Run the script to figure out which files are miscompiled:
# > ./abtest.py
# somefile.s: ok
# someotherfile.s: skipped: same content
# anotherfile.s: failed: './link_test' exitcode != 0
# ...
# Example usage to identify miscompiled functions inside a file:
# 3. First you have to mark begin and end of the functions.
# The script comes with some examples called mark_xxx.py.
# Unfortunately this is very specific to your environment and it is likely
# that you have to write a custom version for your environment.
# > for i in before/*.s after/*.s; do mark_xxx.py $i; done
# 4. Run the tests on a single file (assuming before/file.s and
# after/file.s exist)
# > ./abtest.py file.s
# funcname1 [0/XX]: ok
# funcname2 [1/XX]: ok
# funcname3 [2/XX]: skipped: same content
# funcname4 [3/XX]: failed: './link_test' exitcode != 0
# ...
from fnmatch import filter
from sys import stderr
import argparse
import filecmp
import os
import subprocess
import sys
LINKTEST="./link_test"
ESCAPE="\033[%sm"
BOLD=ESCAPE % "1"
RED=ESCAPE % "31"
NORMAL=ESCAPE % "0"
FAILED=RED+"failed"+NORMAL
def find(dir, file_filter=None):
files = [walkdir[0]+"/"+file for walkdir in os.walk(dir) for file in walkdir[2]]
if file_filter != None:
files = filter(files, file_filter)
return files
def error(message):
stderr.write("Error: %s\n" % (message,))
def warn(message):
stderr.write("Warning: %s\n" % (message,))
def extract_functions(file):
functions = []
in_function = None
for line in open(file):
if line.startswith("# -- Begin "):
if in_function != None:
warn("Missing end of function %s" % (in_function,))
funcname = line[12:-1]
in_function = funcname
text = line
elif line.startswith("# -- End "):
function_name = line[10:-1]
if in_function != function_name:
warn("End %s does not match begin %s" % (function_name, in_function))
else:
text += line
functions.append( (in_function, text) )
in_function = None
elif in_function != None:
text += line
return functions
def replace_function(file, function, replacement, dest):
out = open(dest, "w")
skip = False
found = False
in_function = None
for line in open(file):
if line.startswith("# -- Begin "):
if in_function != None:
warn("Missing end of function %s" % (in_function,))
funcname = line[12:-1]
in_function = funcname
if in_function == function:
out.write(replacement)
skip = True
elif line.startswith("# -- End "):
function_name = line[10:-1]
if in_function != function_name:
warn("End %s does not match begin %s" % (function_name, in_function))
in_function = None
if skip:
skip = False
continue
if not skip:
out.write(line)
def announce_test(name):
stderr.write("%s%s%s: " % (BOLD, name, NORMAL))
stderr.flush()
def announce_result(result, info):
stderr.write(result)
if info != "":
stderr.write(": %s" % info)
stderr.write("\n")
stderr.flush()
def testrun(files):
linkline="%s %s" % (LINKTEST, " ".join(files),)
res = subprocess.call(linkline, shell=True)
if res != 0:
announce_result(FAILED, "'%s' exitcode != 0" % LINKTEST)
return False
else:
announce_result("ok", "")
return True
def check_files():
"""Check files mode"""
for i in range(0, len(NO_PREFIX)):
f = NO_PREFIX[i]
b=baddir+"/"+f
if b not in BAD_FILES:
warn("There is no corresponding file to '%s' in %s" \
% (gooddir+"/"+f, baddir))
continue
announce_test(f + " [%s/%s]" % (i+1, len(NO_PREFIX)))
# combine files (everything from good except f)
testfiles=[]
skip=False
for c in NO_PREFIX:
badfile = baddir+"/"+c
goodfile = gooddir+"/"+c
if c == f:
testfiles.append(badfile)
if filecmp.cmp(goodfile, badfile):
announce_result("skipped", "same content")
skip = True
break
else:
testfiles.append(goodfile)
if skip:
continue
testrun(testfiles)
def check_functions_in_file(base, goodfile, badfile):
functions = extract_functions(goodfile)
if len(functions) == 0:
warn("Couldn't find any function in %s, missing annotations?" % (goodfile,))
return
badfunctions = dict(extract_functions(badfile))
if len(functions) == 0:
warn("Couldn't find any function in %s, missing annotations?" % (badfile,))
return
COMBINED="/tmp/combined.s"
i = 0
for (func,func_text) in functions:
announce_test(func + " [%s/%s]" % (i+1, len(functions)))
i+=1
if func not in badfunctions:
warn("Function '%s' missing from bad file" % func)
continue
if badfunctions[func] == func_text:
announce_result("skipped", "same content")
continue
replace_function(goodfile, func, badfunctions[func], COMBINED)
testfiles=[]
for c in NO_PREFIX:
if c == base:
testfiles.append(COMBINED)
continue
testfiles.append(gooddir + "/" + c)
testrun(testfiles)
parser = argparse.ArgumentParser()
parser.add_argument('--a', dest='dir_a', default='before')
parser.add_argument('--b', dest='dir_b', default='after')
parser.add_argument('--insane', help='Skip sanity check', action='store_true')
parser.add_argument('file', metavar='file', nargs='?')
config = parser.parse_args()
gooddir=config.dir_a
baddir=config.dir_b
BAD_FILES=find(baddir, "*")
GOOD_FILES=find(gooddir, "*")
NO_PREFIX=sorted([x[len(gooddir)+1:] for x in GOOD_FILES])
# "Checking whether build environment is sane ..."
if not config.insane:
announce_test("sanity check")
if not os.access(LINKTEST, os.X_OK):
error("Expect '%s' to be present and executable" % (LINKTEST,))
exit(1)
res = testrun(GOOD_FILES)
if not res:
# "build environment is grinning and holding a spatula. Guess not."
linkline="%s %s" % (LINKTEST, " ".join(GOOD_FILES),)
stderr.write("\n%s\n\n" % linkline)
stderr.write("Returned with exitcode != 0\n")
sys.exit(1)
if config.file is not None:
# File exchange mode
goodfile = gooddir+"/"+config.file
badfile = baddir+"/"+config.file
check_functions_in_file(config.file, goodfile, badfile)
else:
# Function exchange mode
check_files()
|
