WebKit Bugzilla
New
Browse
Search+
Log In
×
Sign in with GitHub
or
Remember my login
Create Account
·
Forgot Password
Forgotten password account recovery
[patch]
Patch
bug-100030-20130306233454.patch (text/plain), 17.38 KB, created by
Glenn Adams
on 2013-03-06 22:38:50 PST
(
hide
)
Description:
Patch
Filename:
MIME Type:
Creator:
Glenn Adams
Created:
2013-03-06 22:38:50 PST
Size:
17.38 KB
patch
obsolete
>Subversion Revision: 144866 >diff --git a/Tools/ChangeLog b/Tools/ChangeLog >index a44fb40bf2e80f9dccf02fbb86f1e9a76938ee91..0d61e7a4603d80a41a80f4a9d418b3f21df959fb 100644 >--- a/Tools/ChangeLog >+++ b/Tools/ChangeLog >@@ -1,3 +1,34 @@ >+2013-03-06 Glenn Adams <glenn@skynav.com> >+ >+ run-perf-tests should have a --repeat option >+ https://bugs.webkit.org/show_bug.cgi?id=100030 >+ >+ Reviewed by NOBODY (OOPS!). >+ >+ Add --repeat option to run-perf-tests, with default value of 1, which runs >+ test set repeat count times then uploads and shows results as appropriate. >+ >+ Incidentally fixed typo: s/suceeds/succeeds/. >+ >+ * Scripts/webkitpy/performance_tests/perftestsrunner.py: >+ (PerfTestsRunner._parse_args): Add repeat option. >+ (PerfTestsRunner.run): Add outer repeat loop. >+ (PerfTestsRunner._generate_results): Split from original _generate_and_show_results. >+ (PerfTestsRunner._upload_and_show_results): Split from original _generate_and_show_results. >+ * Scripts/webkitpy/performance_tests/perftestsrunner_integrationtest.py: >+ (MainTest._test_run_with_json_output): Add repeat optional argument for generating expected logs. Fix typo. >+ (MainTest._test_run_with_json_output.mock_upload_json): Fix typo. >+ (MainTest.test_run_with_json_output): Fix typo. >+ (MainTest.test_run_with_description): Fix typo. >+ (MainTest.test_run_respects_no_results): Fix typo. >+ (MainTest.test_run_with_slave_config_json): Fix typo. >+ (MainTest.test_run_with_multiple_repositories): Fix typo. >+ (MainTest.test_run_with_upload_json): Fix typo. >+ (MainTest.test_run_with_upload_json_should_generate_perf_webkit_json): Fix typo. >+ (MainTest.test_run_with_repeat): Added new test for repeat count semantics. >+ * Scripts/webkitpy/performance_tests/perftestsrunner_unittest.py: >+ (MainTest.test_parse_args): Add test for repeat option parsing. >+ > 2013-03-05 Dirk Pranke <dpranke@chromium.org> > > [chromium] making DumpRenderTree_resources a separate bundle breaks xcode >diff --git a/Tools/Scripts/webkitpy/performance_tests/perftestsrunner.py b/Tools/Scripts/webkitpy/performance_tests/perftestsrunner.py >index c5a0409c403291ccd263621e61bd784525e3f0eb..0b970688191eeea15b45181a1379948b7a4069a4 100644 >--- a/Tools/Scripts/webkitpy/performance_tests/perftestsrunner.py >+++ b/Tools/Scripts/webkitpy/performance_tests/perftestsrunner.py >@@ -126,6 +126,8 @@ class PerfTestsRunner(object): > optparse.make_option("--additional-drt-flag", action="append", > default=[], help="Additional command line flag to pass to DumpRenderTree " > "Specify multiple times to add multiple flags."), >+ optparse.make_option("--repeat", default=1, type="int", >+ help="Specify number of times to run test set (default: 1)."), > ] > return optparse.OptionParser(option_list=(perf_option_list)).parse_args(args) > >@@ -177,24 +179,35 @@ class PerfTestsRunner(object): > _log.error("Build not up to date for %s" % self._port._path_to_driver()) > return self.EXIT_CODE_BAD_BUILD > >- tests = self._collect_tests() >- _log.info("Running %d tests" % len(tests)) >+ runCount = 0 >+ repeat = self._options.repeat >+ while (runCount < repeat): >+ runCount += 1 > >- for test in tests: >- if not test.prepare(self._options.time_out_ms): >- return self.EXIT_CODE_BAD_PREPARATION >+ tests = self._collect_tests() >+ runs = ' [Run %d of %d]' % (runCount, repeat) if repeat > 1 else '' >+ _log.info("Running %d tests%s" % (len(tests), runs)) > >- try: >- if needs_http: >- self._start_http_servers() >- unexpected = self._run_tests_set(sorted(list(tests), key=lambda test: test.test_name())) >+ for test in tests: >+ if not test.prepare(self._options.time_out_ms): >+ return self.EXIT_CODE_BAD_PREPARATION >+ >+ try: >+ if needs_http: >+ self._start_http_servers() >+ unexpected = self._run_tests_set(sorted(list(tests), key=lambda test: test.test_name())) >+ >+ finally: >+ if needs_http: >+ self._stop_http_servers() > >- finally: >- if needs_http: >- self._stop_http_servers() >+ if self._options.generate_results and not self._options.profile: >+ exit_code = self._generate_results() >+ if exit_code: >+ return exit_code > > if self._options.generate_results and not self._options.profile: >- exit_code = self._generate_and_show_results() >+ exit_code = self._upload_and_show_results() > if exit_code: > return exit_code > >@@ -206,7 +219,7 @@ class PerfTestsRunner(object): > return output_json_path > return self._host.filesystem.join(self._port.perf_results_directory(), self._DEFAULT_JSON_FILENAME) > >- def _generate_and_show_results(self): >+ def _generate_results(self): > options = self._options > output_json_path = self._output_json_path() > output = self._generate_results_dict(self._timestamp, options.description, options.platform, options.builder_name, options.build_number) >@@ -223,6 +236,10 @@ class PerfTestsRunner(object): > results_page_path = self._host.filesystem.splitext(output_json_path)[0] + '.html' > self._generate_output_files(output_json_path, results_page_path, output) > >+ def _upload_and_show_results(self): >+ options = self._options >+ output_json_path = self._output_json_path() >+ > if options.test_results_server: > if options.test_results_server == 'webkit-perf.appspot.com': > options.test_results_server = 'perf.webkit.org' >@@ -231,6 +248,7 @@ class PerfTestsRunner(object): > return self.EXIT_CODE_FAILED_UPLOADING > > if options.show_results: >+ results_page_path = self._host.filesystem.splitext(output_json_path)[0] + '.html' > self._port.show_results_html_file(results_page_path) > > def _generate_results_dict(self, timestamp, description, platform, builder_name, build_number): >diff --git a/Tools/Scripts/webkitpy/performance_tests/perftestsrunner_integrationtest.py b/Tools/Scripts/webkitpy/performance_tests/perftestsrunner_integrationtest.py >index da844790a707e2f5444bd5b613a35c5a6412332d..ff2ab493c49832dae215f1b35cb2e1e19103b113 100644 >--- a/Tools/Scripts/webkitpy/performance_tests/perftestsrunner_integrationtest.py >+++ b/Tools/Scripts/webkitpy/performance_tests/perftestsrunner_integrationtest.py >@@ -305,7 +305,7 @@ class MainTest(unittest.TestCase): > self.assertEqual(parser_tests['memory-test']['metrics']['JSHeap'], MemoryTestData.js_heap_results) > self.assertEqual(parser_tests['memory-test']['metrics']['Malloc'], MemoryTestData.malloc_results) > >- def _test_run_with_json_output(self, runner, filesystem, upload_suceeds=False, results_shown=True, expected_exit_code=0): >+ def _test_run_with_json_output(self, runner, filesystem, upload_succeeds=False, results_shown=True, expected_exit_code=0, repeat=1): > filesystem.write_text_file(runner._base_path + '/inspector/pass.html', 'some content') > filesystem.write_text_file(runner._base_path + '/Bindings/event-target-wrapper.html', 'some content') > >@@ -316,8 +316,8 @@ class MainTest(unittest.TestCase): > self.assertIn(hostname, ['some.host']) > self.assertIn(json_path, ['/mock-checkout/output.json']) > self.assertIn(host_path, [None, '/api/report']) >- uploaded[0] = upload_suceeds >- return upload_suceeds >+ uploaded[0] = upload_succeeds >+ return upload_succeeds > > runner._upload_json = mock_upload_json > runner._timestamp = 123456789 >@@ -330,12 +330,15 @@ class MainTest(unittest.TestCase): > stdout, stderr, logs = output_capture.restore_output() > > if not expected_exit_code: >- expected_logs = 'Running 2 tests\n' + EventTargetWrapperTestData.output + InspectorPassTestData.output >+ expected_logs = '' >+ for i in xrange(repeat): >+ runs = ' [Run %d of %d]' % (i + 1, repeat) if repeat > 1 else '' >+ expected_logs += 'Running 2 tests%s\n' % runs + EventTargetWrapperTestData.output + InspectorPassTestData.output > if results_shown: > expected_logs += 'MOCK: user.open_url: file://...\n' > self.assertEqual(self._normalize_output(logs), expected_logs) > >- self.assertEqual(uploaded[0], upload_suceeds) >+ self.assertEqual(uploaded[0], upload_succeeds) > > return logs > >@@ -347,7 +350,7 @@ class MainTest(unittest.TestCase): > def test_run_with_json_output(self): > runner, port = self.create_runner_and_setup_results_template(args=['--output-json-path=/mock-checkout/output.json', > '--test-results-server=some.host']) >- self._test_run_with_json_output(runner, port.host.filesystem, upload_suceeds=True) >+ self._test_run_with_json_output(runner, port.host.filesystem, upload_succeeds=True) > self.assertEqual(self._load_output_json(runner), [{ > "buildTime": "2013-02-08T15:19:37.460000", "tests": self._event_target_wrapper_and_inspector_results, > "revisions": {"WebKit": {"timestamp": "2013-02-01 08:48:05 +0000", "revision": "5678"}}}]) >@@ -359,7 +362,7 @@ class MainTest(unittest.TestCase): > def test_run_with_description(self): > runner, port = self.create_runner_and_setup_results_template(args=['--output-json-path=/mock-checkout/output.json', > '--test-results-server=some.host', '--description', 'some description']) >- self._test_run_with_json_output(runner, port.host.filesystem, upload_suceeds=True) >+ self._test_run_with_json_output(runner, port.host.filesystem, upload_succeeds=True) > self.assertEqual(self._load_output_json(runner), [{ > "buildTime": "2013-02-08T15:19:37.460000", "description": "some description", > "tests": self._event_target_wrapper_and_inspector_results, >@@ -377,7 +380,7 @@ class MainTest(unittest.TestCase): > def test_run_respects_no_results(self): > runner, port = self.create_runner(args=['--output-json-path=/mock-checkout/output.json', > '--test-results-server=some.host', '--no-results']) >- self._test_run_with_json_output(runner, port.host.filesystem, upload_suceeds=False, results_shown=False) >+ self._test_run_with_json_output(runner, port.host.filesystem, upload_succeeds=False, results_shown=False) > self.assertFalse(port.host.filesystem.isfile('/mock-checkout/output.json')) > > def test_run_generates_json_by_default(self): >@@ -479,7 +482,7 @@ class MainTest(unittest.TestCase): > runner, port = self.create_runner_and_setup_results_template(args=['--output-json-path=/mock-checkout/output.json', > '--slave-config-json-path=/mock-checkout/slave-config.json', '--test-results-server=some.host']) > port.host.filesystem.write_text_file('/mock-checkout/slave-config.json', '{"key": "value"}') >- self._test_run_with_json_output(runner, port.host.filesystem, upload_suceeds=True) >+ self._test_run_with_json_output(runner, port.host.filesystem, upload_succeeds=True) > self.assertEqual(self._load_output_json(runner), [{ > "buildTime": "2013-02-08T15:19:37.460000", "tests": self._event_target_wrapper_and_inspector_results, > "revisions": {"WebKit": {"timestamp": "2013-02-01 08:48:05 +0000", "revision": "5678"}}, "builderKey": "value"}]) >@@ -498,7 +501,7 @@ class MainTest(unittest.TestCase): > runner, port = self.create_runner_and_setup_results_template(args=['--output-json-path=/mock-checkout/output.json', > '--test-results-server=some.host']) > port.repository_paths = lambda: [('webkit', '/mock-checkout'), ('some', '/mock-checkout/some')] >- self._test_run_with_json_output(runner, port.host.filesystem, upload_suceeds=True) >+ self._test_run_with_json_output(runner, port.host.filesystem, upload_succeeds=True) > self.assertEqual(self._load_output_json(runner), [{ > "buildTime": "2013-02-08T15:19:37.460000", "tests": self._event_target_wrapper_and_inspector_results, > "revisions": {"webkit": {"timestamp": "2013-02-01 08:48:05 +0000", "revision": "5678"}, >@@ -508,13 +511,13 @@ class MainTest(unittest.TestCase): > runner, port = self.create_runner_and_setup_results_template(args=['--output-json-path=/mock-checkout/output.json', > '--test-results-server', 'some.host', '--platform', 'platform1', '--builder-name', 'builder1', '--build-number', '123']) > >- self._test_run_with_json_output(runner, port.host.filesystem, upload_suceeds=True) >+ self._test_run_with_json_output(runner, port.host.filesystem, upload_succeeds=True) > generated_json = json.loads(port.host.filesystem.files['/mock-checkout/output.json']) > self.assertEqual(generated_json[0]['platform'], 'platform1') > self.assertEqual(generated_json[0]['builderName'], 'builder1') > self.assertEqual(generated_json[0]['buildNumber'], 123) > >- self._test_run_with_json_output(runner, port.host.filesystem, upload_suceeds=False, expected_exit_code=PerfTestsRunner.EXIT_CODE_FAILED_UPLOADING) >+ self._test_run_with_json_output(runner, port.host.filesystem, upload_succeeds=False, expected_exit_code=PerfTestsRunner.EXIT_CODE_FAILED_UPLOADING) > > def test_run_with_upload_json_should_generate_perf_webkit_json(self): > runner, port = self.create_runner_and_setup_results_template(args=['--output-json-path=/mock-checkout/output.json', >@@ -522,7 +525,7 @@ class MainTest(unittest.TestCase): > '--slave-config-json-path=/mock-checkout/slave-config.json']) > port.host.filesystem.write_text_file('/mock-checkout/slave-config.json', '{"key": "value1"}') > >- self._test_run_with_json_output(runner, port.host.filesystem, upload_suceeds=True) >+ self._test_run_with_json_output(runner, port.host.filesystem, upload_succeeds=True) > generated_json = json.loads(port.host.filesystem.files['/mock-checkout/output.json']) > self.assertTrue(isinstance(generated_json, list)) > self.assertEqual(len(generated_json), 1) >@@ -542,3 +545,24 @@ class MainTest(unittest.TestCase): > self.assertEqual(output['tests']['Bindings']['tests']['event-target-wrapper'], { > 'url': 'http://trac.webkit.org/browser/trunk/PerformanceTests/Bindings/event-target-wrapper.html', > 'metrics': {'Time': {'current': [[1486.0, 1471.0, 1510.0, 1505.0, 1478.0, 1490.0]] * 4}}}) >+ >+ def test_run_with_repeat(self): >+ runner, port = self.create_runner_and_setup_results_template(args=['--output-json-path=/mock-checkout/output.json', >+ '--test-results-server=some.host', '--repeat', '5']) >+ self._test_run_with_json_output(runner, port.host.filesystem, upload_succeeds=True, repeat=5) >+ self.assertEqual(self._load_output_json(runner), [ >+ {"buildTime": "2013-02-08T15:19:37.460000", >+ "tests": self._event_target_wrapper_and_inspector_results, >+ "revisions": {"WebKit": {"timestamp": "2013-02-01 08:48:05 +0000", "revision": "5678"}}}, >+ {"buildTime": "2013-02-08T15:19:37.460000", >+ "tests": self._event_target_wrapper_and_inspector_results, >+ "revisions": {"WebKit": {"timestamp": "2013-02-01 08:48:05 +0000", "revision": "5678"}}}, >+ {"buildTime": "2013-02-08T15:19:37.460000", >+ "tests": self._event_target_wrapper_and_inspector_results, >+ "revisions": {"WebKit": {"timestamp": "2013-02-01 08:48:05 +0000", "revision": "5678"}}}, >+ {"buildTime": "2013-02-08T15:19:37.460000", >+ "tests": self._event_target_wrapper_and_inspector_results, >+ "revisions": {"WebKit": {"timestamp": "2013-02-01 08:48:05 +0000", "revision": "5678"}}}, >+ {"buildTime": "2013-02-08T15:19:37.460000", >+ "tests": self._event_target_wrapper_and_inspector_results, >+ "revisions": {"WebKit": {"timestamp": "2013-02-01 08:48:05 +0000", "revision": "5678"}}}]) >diff --git a/Tools/Scripts/webkitpy/performance_tests/perftestsrunner_unittest.py b/Tools/Scripts/webkitpy/performance_tests/perftestsrunner_unittest.py >index a88eef5130662bf658d7ac5c5401028c62f7070b..f9d55919d36f31fcc9f9f57b83d60eb47ae4d939 100644 >--- a/Tools/Scripts/webkitpy/performance_tests/perftestsrunner_unittest.py >+++ b/Tools/Scripts/webkitpy/performance_tests/perftestsrunner_unittest.py >@@ -137,6 +137,7 @@ class MainTest(unittest.TestCase): > '--test-results-server=somehost', > '--additional-drt-flag=--enable-threaded-parser', > '--additional-drt-flag=--awesomesauce', >+ '--repeat=5', > '--debug']) > self.assertTrue(options.build) > self.assertEqual(options.build_directory, 'folder42') >@@ -151,6 +152,7 @@ class MainTest(unittest.TestCase): > self.assertEqual(options.slave_config_json_path, 'a/source.json') > self.assertEqual(options.test_results_server, 'somehost') > self.assertEqual(options.additional_drt_flag, ['--enable-threaded-parser', '--awesomesauce']) >+ self.assertEqual(options.repeat, 5) > > def test_upload_json(self): > runner, port = self.create_runner()
You cannot view the attachment while viewing its details because your browser does not support IFRAMEs.
View the attachment on a separate page
.
View Attachment As Diff
View Attachment As Raw
Actions:
View
|
Formatted Diff
|
Diff
Attachments on
bug 100030
:
170614
|
170691
|
170694
|
191921
|
192059