451ab91e |
1 | #!/usr/bin/env python |
2 | |
3 | #/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * |
4 | # * Mupen64plus - regression-video.py * |
5 | # * Mupen64Plus homepage: http://code.google.com/p/mupen64plus/ * |
6 | # * Copyright (C) 2008-2012 Richard Goedeken * |
7 | # * * |
8 | # * This program is free software; you can redistribute it and/or modify * |
9 | # * it under the terms of the GNU General Public License as published by * |
10 | # * the Free Software Foundation; either version 2 of the License, or * |
11 | # * (at your option) any later version. * |
12 | # * * |
13 | # * This program is distributed in the hope that it will be useful, * |
14 | # * but WITHOUT ANY WARRANTY; without even the implied warranty of * |
15 | # * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * |
16 | # * GNU General Public License for more details. * |
17 | # * * |
18 | # * You should have received a copy of the GNU General Public License * |
19 | # * along with this program; if not, write to the * |
20 | # * Free Software Foundation, Inc., * |
21 | # * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * |
22 | # * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
23 | |
24 | from optparse import OptionParser |
25 | from threading import Thread |
26 | from datetime import date |
27 | import subprocess |
28 | import commands |
29 | import shutil |
30 | import stat |
31 | import sys |
32 | import os |
33 | |
34 | # set global report string |
35 | report = "Mupen64Plus Regression Test report\n----------------------------------\n" |
36 | |
37 | #****************************************************************************** |
38 | # main functions |
39 | # |
40 | |
41 | def main(rootdir, cfgfile, nobuild, noemail): |
42 | global report |
43 | # set up child directory paths |
44 | srcdir = os.path.join(rootdir, "source") |
45 | shotdir = os.path.join(rootdir, "current") |
46 | refdir = os.path.join(rootdir, "reference") |
47 | archivedir = os.path.join(rootdir, "archive") |
48 | # run the test procedure |
49 | tester = RegTester(rootdir, srcdir, shotdir) |
50 | rval = 0 |
51 | while True: |
52 | # Step 1: load the test config file |
53 | if not tester.LoadConfig(cfgfile): |
54 | rval = 1 |
55 | break |
56 | # Step 2: check out from Mercurial |
57 | if not nobuild: |
58 | if not tester.CheckoutSource(srcdir): |
59 | rval = 2 |
60 | break |
61 | # Step 3: run test builds |
62 | if not nobuild: |
63 | for modname in tester.modulesAndParams: |
64 | module = tester.modulesAndParams[modname] |
65 | if "testbuilds" not in module: |
66 | continue |
67 | modurl = module["url"] |
68 | modfilename = modurl.split('/')[-1] |
69 | testlist = [ name.strip() for name in module["testbuilds"].split(',') ] |
70 | makeparams = [ params.strip() for params in module["testbuildparams"].split(',') ] |
71 | if len(testlist) != len(makeparams): |
72 | report += "Config file error for test builds in %s. Build name list and makefile parameter list have different lengths.\n" % modname |
73 | testbuilds = min(len(testlist), len(makeparams)) |
74 | for i in range(testbuilds): |
75 | buildname = testlist[i] |
76 | buildmake = makeparams[i] |
77 | BuildSource(srcdir, modfilename, modname, buildname, buildmake, module["outputfiles"], True) |
78 | # Step 4: build the binary for the video regression test |
79 | if not nobuild: |
80 | for modname in tester.modulesAndParams: |
81 | module = tester.modulesAndParams[modname] |
82 | modurl = module["url"] |
83 | modfilename = modurl.split('/')[-1] |
84 | videobuild = module["videobuild"] |
85 | videomake = module["videobuildparams"] |
86 | if not BuildSource(srcdir, modfilename, modname, videobuild, videomake, module["outputfiles"], False): |
87 | rval = 3 |
88 | break |
89 | if rval != 0: |
90 | break |
91 | # Step 5: run the tests, check the results |
92 | if not tester.RunTests(): |
93 | rval = 4 |
94 | break |
95 | if not tester.CheckResults(refdir): |
96 | rval = 5 |
97 | break |
98 | # test procedure is finished |
99 | break |
100 | # Step 6: send email report and archive the results |
101 | if not noemail: |
102 | if not tester.SendReport(): |
103 | rval = 6 |
104 | if not tester.ArchiveResults(archivedir): |
105 | rval = 7 |
106 | # all done with test process |
107 | return rval |
108 | |
109 | #****************************************************************************** |
110 | # Checkout & build functions |
111 | # |
112 | |
113 | def BuildSource(srcdir, moddir, modname, buildname, buildmake, outputfiles, istest): |
114 | global report |
115 | makepath = os.path.join(srcdir, moddir, "projects", "unix") |
116 | # print build report message and clear counters |
117 | testbuildcommand = "make -C %s %s" % (makepath, buildmake) |
118 | if istest: |
119 | report += "Running %s test build \"%s\"\n" % (modname, buildname) |
120 | else: |
121 | report += "Building %s \"%s\" for video test\n" % (modname, buildname) |
122 | warnings = 0 |
123 | errors = 0 |
124 | # run make and capture the output |
125 | output = commands.getoutput(testbuildcommand) |
126 | makelines = output.split("\n") |
127 | # print warnings and errors |
128 | for line in makelines: |
129 | if "error:" in line: |
130 | report += " " + line + "\n" |
131 | errors += 1 |
132 | if "warning:" in line: |
133 | report += " " + line + "\n" |
134 | warnings += 1 |
135 | report += "%i errors. %i warnings.\n" % (errors, warnings) |
136 | if errors > 0 and not istest: |
137 | return False |
138 | # check for output files |
139 | for filename in outputfiles.split(','): |
140 | if not os.path.exists(os.path.join(makepath, filename)): |
141 | report += "Build failed: '%s' not found\n" % filename |
142 | errors += 1 |
143 | if errors > 0 and not istest: |
144 | return False |
145 | # clean up if this was a test |
146 | if istest: |
147 | os.system("make -C %s clean" % makepath) |
148 | # if this wasn't a test, then copy our output files and data files |
149 | if not istest: |
150 | for filename in outputfiles.split(','): |
151 | shutil.move(os.path.join(makepath, filename), srcdir) |
152 | datapath = os.path.join(srcdir, moddir, "data") |
153 | if os.path.isdir(datapath): |
154 | copytree(datapath, os.path.join(srcdir, "data")) |
155 | # build was successful! |
156 | return True |
157 | |
158 | #****************************************************************************** |
159 | # Test execution classes |
160 | # |
161 | class RegTester: |
162 | def __init__(self, rootdir, bindir, screenshotdir): |
163 | self.rootdir = rootdir |
164 | self.bindir = bindir |
165 | self.screenshotdir = screenshotdir |
166 | self.generalParams = { } |
167 | self.gamesAndParams = { } |
168 | self.modulesAndParams = { } |
169 | self.videoplugins = [ "mupen64plus-video-rice.so" ] |
170 | self.thisdate = str(date.today()) |
171 | |
172 | def LoadConfig(self, filename): |
173 | global report |
174 | # read the config file |
175 | report += "\nLoading regression test configuration.\n" |
176 | try: |
177 | cfgfile = open(os.path.join(self.rootdir, filename), "r") |
178 | cfglines = cfgfile.read().split("\n") |
179 | cfgfile.close() |
180 | except Exception, e: |
181 | report += "Error in RegTestConfigParser::LoadConfig(): %s" % e |
182 | return False |
183 | # parse the file |
184 | GameFilename = None |
185 | ModuleName = None |
186 | for line in cfglines: |
187 | # strip leading and trailing whitespace |
188 | line = line.strip() |
189 | # test for comment |
190 | if len(line) == 0 or line[0] == '#': |
191 | continue |
192 | # test for new game filename |
193 | if line[0] == '[' and line [-1] == ']': |
194 | GameFilename = line[1:-1] |
195 | if GameFilename in self.gamesAndParams: |
196 | report += " Warning: Config file '%s' contains duplicate game entry '%s'\n" % (filename, GameFilename) |
197 | else: |
198 | self.gamesAndParams[GameFilename] = { } |
199 | continue |
200 | # test for new source module build |
201 | if line[0] == '{' and line [-1] == '}': |
202 | ModuleName = line[1:-1] |
203 | if ModuleName in self.modulesAndParams: |
204 | report += " Warning: Config file '%s' contains duplicate source module '%s'\n" % (filename, ModuleName) |
205 | else: |
206 | self.modulesAndParams[ModuleName] = { } |
207 | continue |
208 | # print warning and continue if it's not a (key = value) pair |
209 | pivot = line.find('=') |
210 | if pivot == -1: |
211 | report += " Warning: Config file '%s' contains unrecognized line: '%s'\n" % (filename, line) |
212 | continue |
213 | # parse key, value |
214 | key = line[:pivot].strip().lower() |
215 | value = line[pivot+1:].strip() |
216 | if ModuleName is None: |
217 | paramDict = self.generalParams |
218 | elif GameFilename is None: |
219 | paramDict = self.modulesAndParams[ModuleName] |
220 | else: |
221 | paramDict = self.gamesAndParams[GameFilename] |
222 | if key in paramDict: |
223 | report += " Warning: duplicate key '%s'\n" % key |
224 | continue |
225 | paramDict[key] = value |
226 | # check for required parameters |
227 | if "rompath" not in self.generalParams: |
228 | report += " Error: rompath is not given in config file\n" |
229 | return False |
230 | # config is loaded |
231 | return True |
232 | |
233 | def CheckoutSource(self, srcdir): |
234 | global report |
235 | # remove any current source directory |
236 | if not deltree(srcdir): |
237 | return False |
238 | os.mkdir(srcdir) |
239 | os.mkdir(os.path.join(srcdir, "data")) |
240 | # loop through all of the source modules |
241 | for modname in self.modulesAndParams: |
242 | module = self.modulesAndParams[modname] |
243 | if "url" not in module: |
244 | report += "Error: no Hg repository URL for module %s\n\n" % modname |
245 | return False |
246 | modurl = module["url"] |
247 | modfilename = modurl.split("/")[-1] |
248 | # call Hg to checkout Mupen64Plus source module |
249 | output = commands.getoutput("hg clone --cwd %s %s" % (srcdir, modurl)) |
250 | # parse the output |
251 | lastline = output.split("\n")[-1] |
252 | if "0 files unresolved" not in lastline: |
253 | report += "Hg Error: %s\n\n" % lastline |
254 | return False |
255 | # get the revision info |
256 | RevFound = False |
257 | output = commands.getoutput("hg tip -R %s" % os.path.join(srcdir, modfilename)) |
258 | for line in output.split('\n'): |
259 | words = line.split() |
260 | if len(words) == 2 and words[0] == 'changeset:': |
261 | report += "Hg Checkout %s: changeset %s\n" % (modfilename, words[1]) |
262 | RevFound = True |
263 | if not RevFound: |
264 | report += "Hg Error: couldn't find revision information\n\n" |
265 | return False |
266 | return True |
267 | |
268 | def RunTests(self): |
269 | global report |
270 | rompath = self.generalParams["rompath"] |
271 | if not os.path.exists(rompath): |
272 | report += " Error: ROM directory '%s' does not exist!\n" % rompath |
273 | return False |
274 | # Remove any current screenshot directory |
275 | if not deltree(self.screenshotdir): |
276 | return False |
277 | # Data initialization and start message |
278 | os.mkdir(self.screenshotdir) |
279 | for plugin in self.videoplugins: |
280 | videoname = plugin[:plugin.find('.')] |
281 | os.mkdir(os.path.join(self.screenshotdir, videoname)) |
282 | report += "\nRunning regression tests on %i games.\n" % len(self.gamesAndParams) |
283 | # loop over each game filename given in regtest config file |
284 | for GameFilename in self.gamesAndParams: |
285 | GameParams = self.gamesAndParams[GameFilename] |
286 | # if no screenshots parameter given for this game then skip it |
287 | if "screenshots" not in GameParams: |
288 | report += " Warning: no screenshots taken for game '%s'\n" % GameFilename |
289 | continue |
290 | # make a list of screenshots and check it |
291 | shotlist = [ str(int(framenum.strip())) for framenum in GameParams["screenshots"].split(',') ] |
292 | if len(shotlist) < 1 or (len(shotlist) == 1 and shotlist[0] == '0'): |
293 | report += " Warning: invalid screenshot list for game '%s'\n" % GameFilename |
294 | continue |
295 | # run a test for each video plugin |
296 | for plugin in self.videoplugins: |
297 | videoname = plugin[:plugin.find('.')] |
298 | # check if this plugin should be skipped |
299 | if "skipvideo" in GameParams: |
300 | skipit = False |
301 | skiplist = [ name.strip() for name in GameParams["skipvideo"].split(',') ] |
302 | for skiptag in skiplist: |
303 | if skiptag.lower() in plugin.lower(): |
304 | skipit = True |
305 | if skipit: |
306 | continue |
307 | # construct the command line |
308 | exepath = os.path.join(self.bindir, "mupen64plus") |
309 | exeparms = [ "--corelib", os.path.join(self.bindir, "libmupen64plus.so.2") ] |
310 | exeparms += [ "--testshots", ",".join(shotlist) ] |
311 | exeparms += [ "--sshotdir", os.path.join(self.screenshotdir, videoname) ] |
312 | exeparms += [ "--plugindir", self.bindir ] |
313 | exeparms += [ "--datadir", os.path.join(self.bindir, "data") ] |
314 | myconfig = os.path.join(self.rootdir, "config") |
315 | exeparms += [ "--configdir", myconfig ] |
316 | exeparms += [ "--gfx", plugin ] |
317 | exeparms += [ "--emumode", "2" ] |
318 | exeparms += [ os.path.join(rompath, GameFilename) ] |
319 | # run it, but if it takes too long print an error and kill it |
320 | testrun = RegTestRunner(exepath, exeparms) |
321 | testrun.start() |
322 | testrun.join(60.0) |
323 | if testrun.isAlive(): |
324 | report += " Error: Test run timed out after 60 seconds: '%s'\n" % " ".join(exeparms) |
325 | os.kill(testrun.pid, 9) |
326 | testrun.join(10.0) |
327 | |
328 | # all tests have been run |
329 | return True |
330 | |
331 | def CheckResults(self, refdir): |
332 | global report |
333 | # print message |
334 | warnings = 0 |
335 | errors = 0 |
336 | report += "\nChecking regression test results\n" |
337 | # get lists of files in the reference folders |
338 | refshots = { } |
339 | if not os.path.exists(refdir): |
340 | os.mkdir(refdir) |
341 | for plugin in self.videoplugins: |
342 | videoname = plugin[:plugin.find('.')] |
343 | videodir = os.path.join(refdir, videoname) |
344 | if not os.path.exists(videodir): |
345 | os.mkdir(videodir) |
346 | refshots[videoname] = [ ] |
347 | else: |
348 | refshots[videoname] = [ filename for filename in os.listdir(videodir) ] |
349 | # get lists of files produced by current test runs |
350 | newshots = { } |
351 | for plugin in self.videoplugins: |
352 | videoname = plugin[:plugin.find('.')] |
353 | videodir = os.path.join(self.screenshotdir, videoname) |
354 | if not os.path.exists(videodir): |
355 | newshots[videoname] = [ ] |
356 | else: |
357 | newshots[videoname] = [ filename for filename in os.listdir(videodir) ] |
358 | # make list of matching ref/test screenshots, and look for missing reference screenshots |
359 | checklist = { } |
360 | for plugin in self.videoplugins: |
361 | videoname = plugin[:plugin.find('.')] |
362 | checklist[videoname] = [ ] |
363 | for filename in newshots[videoname]: |
364 | if filename in refshots[videoname]: |
365 | checklist[videoname] += [ filename ] |
366 | else: |
367 | report += " Warning: reference screenshot '%s/%s' missing. Copying from current test run\n" % (videoname, filename) |
368 | shutil.copy(os.path.join(self.screenshotdir, videoname, filename), os.path.join(refdir, videoname)) |
369 | warnings += 1 |
370 | # look for missing test screenshots |
371 | for plugin in self.videoplugins: |
372 | videoname = plugin[:plugin.find('.')] |
373 | for filename in refshots[videoname]: |
374 | if filename not in newshots[videoname]: |
375 | report += " Error: Test screenshot '%s/%s' missing.\n" % (videoname, filename) |
376 | errors += 1 |
377 | # do image comparisons |
378 | for plugin in self.videoplugins: |
379 | videoname = plugin[:plugin.find('.')] |
380 | for filename in checklist[videoname]: |
381 | refimage = os.path.join(refdir, videoname, filename) |
382 | testimage = os.path.join(self.screenshotdir, videoname, filename) |
383 | diffimage = os.path.join(self.screenshotdir, videoname, os.path.splitext(filename)[0] + "_DIFF.png") |
384 | cmd = ("/usr/bin/compare", "-metric", "PSNR", refimage, testimage, diffimage) |
385 | pipe = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout |
386 | similarity = pipe.read().strip() |
387 | pipe.close() |
388 | try: |
389 | db = float(similarity) |
390 | except: |
391 | db = 0 |
392 | if db > 60.0: |
393 | os.unlink(diffimage) |
394 | else: |
395 | report += " Warning: test image '%s/%s' does not match reference. PSNR = %s\n" % (videoname, filename, similarity) |
396 | warnings += 1 |
397 | # give report and return |
398 | report += "%i errors. %i warnings.\n" % (errors, warnings) |
399 | return True |
400 | |
401 | def SendReport(self): |
402 | global report |
403 | # if there are no email addresses in the config file, then just we're done |
404 | if "sendemail" not in self.generalParams: |
405 | return True |
406 | if len(self.generalParams["sendemail"]) < 5: |
407 | return True |
408 | # construct the email message header |
409 | emailheader = "To: %s\n" % self.generalParams["sendemail"] |
410 | emailheader += "From: Mupen64Plus-Tester@fascination.homelinux.net\n" |
411 | emailheader += "Subject: %s Regression Test Results for Mupen64Plus\n" % self.thisdate |
412 | emailheader += "Reply-to: do-not-reply@fascination.homelinux.net\n" |
413 | emailheader += "Content-Type: text/plain; charset=UTF-8\n" |
414 | emailheader += "Content-Transfer-Encoding: 8bit\n\n" |
415 | # open a pipe to sendmail and dump our report |
416 | try: |
417 | pipe = subprocess.Popen(("/usr/sbin/sendmail", "-t"), stdin=subprocess.PIPE).stdin |
418 | pipe.write(emailheader) |
419 | pipe.write(report) |
420 | pipe.close() |
421 | except Exception, e: |
422 | report += "Exception encountered when calling sendmail: '%s'\n" % e |
423 | report += "Email header:\n%s\n" % emailheader |
424 | return False |
425 | return True |
426 | |
427 | def ArchiveResults(self, archivedir): |
428 | global report |
429 | # create archive dir if it doesn't exist |
430 | if not os.path.exists(archivedir): |
431 | os.mkdir(archivedir) |
432 | # move the images into a subdirectory of 'archive' given by date |
433 | subdir = os.path.join(archivedir, self.thisdate) |
434 | if os.path.exists(subdir): |
435 | if not deltree(subdir): |
436 | return False |
437 | if os.path.exists(self.screenshotdir): |
438 | shutil.move(self.screenshotdir, subdir) |
439 | # copy the report into the archive directory |
440 | f = open(os.path.join(archivedir, "report_%s.txt" % self.thisdate), "w") |
441 | f.write(report) |
442 | f.close() |
443 | # archival is complete |
444 | return True |
445 | |
446 | |
447 | class RegTestRunner(Thread): |
448 | def __init__(self, exepath, exeparms): |
449 | self.exepath = exepath |
450 | self.exeparms = exeparms |
451 | self.pid = 0 |
452 | self.returnval = None |
453 | Thread.__init__(self) |
454 | |
455 | def run(self): |
456 | # start the process |
457 | testprocess = subprocess.Popen([self.exepath] + self.exeparms) |
458 | # get the PID of the new test process |
459 | self.pid = testprocess.pid |
460 | # wait for the test to complete |
461 | self.returnval = testprocess.wait() |
462 | |
463 | |
464 | #****************************************************************************** |
465 | # Generic helper functions |
466 | # |
467 | |
468 | def deltree(dirname): |
469 | global report |
470 | if not os.path.exists(dirname): |
471 | return True |
472 | try: |
473 | for path in (os.path.join(dirname, filename) for filename in os.listdir(dirname)): |
474 | if os.path.isdir(path): |
475 | if not deltree(path): |
476 | return False |
477 | else: |
478 | os.unlink(path) |
479 | os.rmdir(dirname) |
480 | except Exception, e: |
481 | report += "Error in deltree(): %s\n" % e |
482 | return False |
483 | |
484 | return True |
485 | |
486 | def copytree(srcpath, dstpath): |
487 | if not os.path.isdir(srcpath) or not os.path.isdir(dstpath): |
488 | return False |
489 | for filename in os.listdir(srcpath): |
490 | filepath = os.path.join(srcpath, filename) |
491 | if os.path.isdir(filepath): |
492 | subdstpath = os.path.join(dstpath, filename) |
493 | os.mkdir(subdstpath) |
494 | copytree(filepath, subdstpath) |
495 | else: |
496 | shutil.copy(filepath, dstpath) |
497 | return True |
498 | |
499 | #****************************************************************************** |
500 | # main function call for standard script execution |
501 | # |
502 | |
503 | if __name__ == "__main__": |
504 | # parse the command-line arguments |
505 | parser = OptionParser() |
506 | parser.add_option("-n", "--nobuild", dest="nobuild", default=False, action="store_true", |
507 | help="Assume source code is present; don't check out and build") |
508 | parser.add_option("-e", "--noemail", dest="noemail", default=False, action="store_true", |
509 | help="don't send email or archive results") |
510 | parser.add_option("-t", "--testpath", dest="testpath", |
511 | help="Set root of testing directory to PATH", metavar="PATH") |
512 | parser.add_option("-c", "--cfgfile", dest="cfgfile", default="daily-tests.cfg", |
513 | help="Use regression test config file FILE", metavar="FILE") |
514 | (opts, args) = parser.parse_args() |
515 | # check test path |
516 | if opts.testpath is None: |
517 | # change directory to the directory containing this script and set root test path to "." |
518 | scriptdir = os.path.dirname(sys.argv[0]) |
519 | os.chdir(scriptdir) |
520 | rootdir = "." |
521 | else: |
522 | rootdir = opts.testpath |
523 | # call the main function |
524 | rval = main(rootdir, opts.cfgfile, opts.nobuild, opts.noemail) |
525 | sys.exit(rval) |
526 | |
527 | |