1 |
2 |
jamieiles |
#!/usr/bin/env python
|
2 |
|
|
#
|
3 |
|
|
# Copyright 2005 Google Inc. All Rights Reserved.
|
4 |
|
|
#
|
5 |
|
|
# Redistribution and use in source and binary forms, with or without
|
6 |
|
|
# modification, are permitted provided that the following conditions are
|
7 |
|
|
# met:
|
8 |
|
|
#
|
9 |
|
|
# * Redistributions of source code must retain the above copyright
|
10 |
|
|
# notice, this list of conditions and the following disclaimer.
|
11 |
|
|
# * Redistributions in binary form must reproduce the above
|
12 |
|
|
# copyright notice, this list of conditions and the following disclaimer
|
13 |
|
|
# in the documentation and/or other materials provided with the
|
14 |
|
|
# distribution.
|
15 |
|
|
# * Neither the name of Google Inc. nor the names of its
|
16 |
|
|
# contributors may be used to endorse or promote products derived from
|
17 |
|
|
# this software without specific prior written permission.
|
18 |
|
|
#
|
19 |
|
|
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
20 |
|
|
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
21 |
|
|
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
22 |
|
|
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
23 |
|
|
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
24 |
|
|
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
25 |
|
|
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
26 |
|
|
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
27 |
|
|
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
28 |
|
|
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
29 |
|
|
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
30 |
|
|
|
31 |
|
|
"""Unit test for Google Test test filters.
|
32 |
|
|
|
33 |
|
|
A user can specify which test(s) in a Google Test program to run via either
|
34 |
|
|
the GTEST_FILTER environment variable or the --gtest_filter flag.
|
35 |
|
|
This script tests such functionality by invoking
|
36 |
|
|
gtest_filter_unittest_ (a program written with Google Test) with different
|
37 |
|
|
environments and command line flags.
|
38 |
|
|
|
39 |
|
|
Note that test sharding may also influence which tests are filtered. Therefore,
|
40 |
|
|
we test that here also.
|
41 |
|
|
"""
|
42 |
|
|
|
43 |
|
|
__author__ = 'wan@google.com (Zhanyong Wan)'
|
44 |
|
|
|
45 |
|
|
import os
|
46 |
|
|
import re
|
47 |
|
|
try:
|
48 |
|
|
from sets import Set as set # For Python 2.3 compatibility
|
49 |
|
|
except ImportError:
|
50 |
|
|
pass
|
51 |
|
|
import sys
|
52 |
|
|
|
53 |
|
|
import gtest_test_utils
|
54 |
|
|
|
55 |
|
|
# Constants.
|
56 |
|
|
|
57 |
|
|
# Checks if this platform can pass empty environment variables to child
|
58 |
|
|
# processes. We set an env variable to an empty string and invoke a python
|
59 |
|
|
# script in a subprocess to print whether the variable is STILL in
|
60 |
|
|
# os.environ. We then use 'eval' to parse the child's output so that an
|
61 |
|
|
# exception is thrown if the input is anything other than 'True' nor 'False'.
|
62 |
|
|
os.environ['EMPTY_VAR'] = ''
|
63 |
|
|
child = gtest_test_utils.Subprocess(
|
64 |
|
|
[sys.executable, '-c', 'import os; print(\'EMPTY_VAR\' in os.environ)'])
|
65 |
|
|
CAN_PASS_EMPTY_ENV = eval(child.output)
|
66 |
|
|
|
67 |
|
|
|
68 |
|
|
# Check if this platform can unset environment variables in child processes.
|
69 |
|
|
# We set an env variable to a non-empty string, unset it, and invoke
|
70 |
|
|
# a python script in a subprocess to print whether the variable
|
71 |
|
|
# is NO LONGER in os.environ.
|
72 |
|
|
# We use 'eval' to parse the child's output so that an exception
|
73 |
|
|
# is thrown if the input is neither 'True' nor 'False'.
|
74 |
|
|
os.environ['UNSET_VAR'] = 'X'
|
75 |
|
|
del os.environ['UNSET_VAR']
|
76 |
|
|
child = gtest_test_utils.Subprocess(
|
77 |
|
|
[sys.executable, '-c', 'import os; print(\'UNSET_VAR\' not in os.environ)'])
|
78 |
|
|
CAN_UNSET_ENV = eval(child.output)
|
79 |
|
|
|
80 |
|
|
|
81 |
|
|
# Checks if we should test with an empty filter. This doesn't
|
82 |
|
|
# make sense on platforms that cannot pass empty env variables (Win32)
|
83 |
|
|
# and on platforms that cannot unset variables (since we cannot tell
|
84 |
|
|
# the difference between "" and NULL -- Borland and Solaris < 5.10)
|
85 |
|
|
CAN_TEST_EMPTY_FILTER = (CAN_PASS_EMPTY_ENV and CAN_UNSET_ENV)
|
86 |
|
|
|
87 |
|
|
|
88 |
|
|
# The environment variable for specifying the test filters.
|
89 |
|
|
FILTER_ENV_VAR = 'GTEST_FILTER'
|
90 |
|
|
|
91 |
|
|
# The environment variables for test sharding.
|
92 |
|
|
TOTAL_SHARDS_ENV_VAR = 'GTEST_TOTAL_SHARDS'
|
93 |
|
|
SHARD_INDEX_ENV_VAR = 'GTEST_SHARD_INDEX'
|
94 |
|
|
SHARD_STATUS_FILE_ENV_VAR = 'GTEST_SHARD_STATUS_FILE'
|
95 |
|
|
|
96 |
|
|
# The command line flag for specifying the test filters.
|
97 |
|
|
FILTER_FLAG = 'gtest_filter'
|
98 |
|
|
|
99 |
|
|
# The command line flag for including disabled tests.
|
100 |
|
|
ALSO_RUN_DISABED_TESTS_FLAG = 'gtest_also_run_disabled_tests'
|
101 |
|
|
|
102 |
|
|
# Command to run the gtest_filter_unittest_ program.
|
103 |
|
|
COMMAND = gtest_test_utils.GetTestExecutablePath('gtest_filter_unittest_')
|
104 |
|
|
|
105 |
|
|
# Regex for determining whether parameterized tests are enabled in the binary.
|
106 |
|
|
PARAM_TEST_REGEX = re.compile(r'/ParamTest')
|
107 |
|
|
|
108 |
|
|
# Regex for parsing test case names from Google Test's output.
|
109 |
|
|
TEST_CASE_REGEX = re.compile(r'^\[\-+\] \d+ tests? from (\w+(/\w+)?)')
|
110 |
|
|
|
111 |
|
|
# Regex for parsing test names from Google Test's output.
|
112 |
|
|
TEST_REGEX = re.compile(r'^\[\s*RUN\s*\].*\.(\w+(/\w+)?)')
|
113 |
|
|
|
114 |
|
|
# The command line flag to tell Google Test to output the list of tests it
|
115 |
|
|
# will run.
|
116 |
|
|
LIST_TESTS_FLAG = '--gtest_list_tests'
|
117 |
|
|
|
118 |
|
|
# Indicates whether Google Test supports death tests.
|
119 |
|
|
SUPPORTS_DEATH_TESTS = 'HasDeathTest' in gtest_test_utils.Subprocess(
|
120 |
|
|
[COMMAND, LIST_TESTS_FLAG]).output
|
121 |
|
|
|
122 |
|
|
# Full names of all tests in gtest_filter_unittests_.
|
123 |
|
|
PARAM_TESTS = [
|
124 |
|
|
'SeqP/ParamTest.TestX/0',
|
125 |
|
|
'SeqP/ParamTest.TestX/1',
|
126 |
|
|
'SeqP/ParamTest.TestY/0',
|
127 |
|
|
'SeqP/ParamTest.TestY/1',
|
128 |
|
|
'SeqQ/ParamTest.TestX/0',
|
129 |
|
|
'SeqQ/ParamTest.TestX/1',
|
130 |
|
|
'SeqQ/ParamTest.TestY/0',
|
131 |
|
|
'SeqQ/ParamTest.TestY/1',
|
132 |
|
|
]
|
133 |
|
|
|
134 |
|
|
DISABLED_TESTS = [
|
135 |
|
|
'BarTest.DISABLED_TestFour',
|
136 |
|
|
'BarTest.DISABLED_TestFive',
|
137 |
|
|
'BazTest.DISABLED_TestC',
|
138 |
|
|
'DISABLED_FoobarTest.Test1',
|
139 |
|
|
'DISABLED_FoobarTest.DISABLED_Test2',
|
140 |
|
|
'DISABLED_FoobarbazTest.TestA',
|
141 |
|
|
]
|
142 |
|
|
|
143 |
|
|
if SUPPORTS_DEATH_TESTS:
|
144 |
|
|
DEATH_TESTS = [
|
145 |
|
|
'HasDeathTest.Test1',
|
146 |
|
|
'HasDeathTest.Test2',
|
147 |
|
|
]
|
148 |
|
|
else:
|
149 |
|
|
DEATH_TESTS = []
|
150 |
|
|
|
151 |
|
|
# All the non-disabled tests.
|
152 |
|
|
ACTIVE_TESTS = [
|
153 |
|
|
'FooTest.Abc',
|
154 |
|
|
'FooTest.Xyz',
|
155 |
|
|
|
156 |
|
|
'BarTest.TestOne',
|
157 |
|
|
'BarTest.TestTwo',
|
158 |
|
|
'BarTest.TestThree',
|
159 |
|
|
|
160 |
|
|
'BazTest.TestOne',
|
161 |
|
|
'BazTest.TestA',
|
162 |
|
|
'BazTest.TestB',
|
163 |
|
|
] + DEATH_TESTS + PARAM_TESTS
|
164 |
|
|
|
165 |
|
|
param_tests_present = None
|
166 |
|
|
|
167 |
|
|
# Utilities.
|
168 |
|
|
|
169 |
|
|
environ = os.environ.copy()
|
170 |
|
|
|
171 |
|
|
|
172 |
|
|
def SetEnvVar(env_var, value):
|
173 |
|
|
"""Sets the env variable to 'value'; unsets it when 'value' is None."""
|
174 |
|
|
|
175 |
|
|
if value is not None:
|
176 |
|
|
environ[env_var] = value
|
177 |
|
|
elif env_var in environ:
|
178 |
|
|
del environ[env_var]
|
179 |
|
|
|
180 |
|
|
|
181 |
|
|
def RunAndReturnOutput(args = None):
|
182 |
|
|
"""Runs the test program and returns its output."""
|
183 |
|
|
|
184 |
|
|
return gtest_test_utils.Subprocess([COMMAND] + (args or []),
|
185 |
|
|
env=environ).output
|
186 |
|
|
|
187 |
|
|
|
188 |
|
|
def RunAndExtractTestList(args = None):
|
189 |
|
|
"""Runs the test program and returns its exit code and a list of tests run."""
|
190 |
|
|
|
191 |
|
|
p = gtest_test_utils.Subprocess([COMMAND] + (args or []), env=environ)
|
192 |
|
|
tests_run = []
|
193 |
|
|
test_case = ''
|
194 |
|
|
test = ''
|
195 |
|
|
for line in p.output.split('\n'):
|
196 |
|
|
match = TEST_CASE_REGEX.match(line)
|
197 |
|
|
if match is not None:
|
198 |
|
|
test_case = match.group(1)
|
199 |
|
|
else:
|
200 |
|
|
match = TEST_REGEX.match(line)
|
201 |
|
|
if match is not None:
|
202 |
|
|
test = match.group(1)
|
203 |
|
|
tests_run.append(test_case + '.' + test)
|
204 |
|
|
return (tests_run, p.exit_code)
|
205 |
|
|
|
206 |
|
|
|
207 |
|
|
def InvokeWithModifiedEnv(extra_env, function, *args, **kwargs):
|
208 |
|
|
"""Runs the given function and arguments in a modified environment."""
|
209 |
|
|
try:
|
210 |
|
|
original_env = environ.copy()
|
211 |
|
|
environ.update(extra_env)
|
212 |
|
|
return function(*args, **kwargs)
|
213 |
|
|
finally:
|
214 |
|
|
environ.clear()
|
215 |
|
|
environ.update(original_env)
|
216 |
|
|
|
217 |
|
|
|
218 |
|
|
def RunWithSharding(total_shards, shard_index, command):
|
219 |
|
|
"""Runs a test program shard and returns exit code and a list of tests run."""
|
220 |
|
|
|
221 |
|
|
extra_env = {SHARD_INDEX_ENV_VAR: str(shard_index),
|
222 |
|
|
TOTAL_SHARDS_ENV_VAR: str(total_shards)}
|
223 |
|
|
return InvokeWithModifiedEnv(extra_env, RunAndExtractTestList, command)
|
224 |
|
|
|
225 |
|
|
# The unit test.
|
226 |
|
|
|
227 |
|
|
|
228 |
|
|
class GTestFilterUnitTest(gtest_test_utils.TestCase):
|
229 |
|
|
"""Tests the env variable or the command line flag to filter tests."""
|
230 |
|
|
|
231 |
|
|
# Utilities.
|
232 |
|
|
|
233 |
|
|
def AssertSetEqual(self, lhs, rhs):
|
234 |
|
|
"""Asserts that two sets are equal."""
|
235 |
|
|
|
236 |
|
|
for elem in lhs:
|
237 |
|
|
self.assert_(elem in rhs, '%s in %s' % (elem, rhs))
|
238 |
|
|
|
239 |
|
|
for elem in rhs:
|
240 |
|
|
self.assert_(elem in lhs, '%s in %s' % (elem, lhs))
|
241 |
|
|
|
242 |
|
|
def AssertPartitionIsValid(self, set_var, list_of_sets):
|
243 |
|
|
"""Asserts that list_of_sets is a valid partition of set_var."""
|
244 |
|
|
|
245 |
|
|
full_partition = []
|
246 |
|
|
for slice_var in list_of_sets:
|
247 |
|
|
full_partition.extend(slice_var)
|
248 |
|
|
self.assertEqual(len(set_var), len(full_partition))
|
249 |
|
|
self.assertEqual(set(set_var), set(full_partition))
|
250 |
|
|
|
251 |
|
|
def AdjustForParameterizedTests(self, tests_to_run):
|
252 |
|
|
"""Adjust tests_to_run in case value parameterized tests are disabled."""
|
253 |
|
|
|
254 |
|
|
global param_tests_present
|
255 |
|
|
if not param_tests_present:
|
256 |
|
|
return list(set(tests_to_run) - set(PARAM_TESTS))
|
257 |
|
|
else:
|
258 |
|
|
return tests_to_run
|
259 |
|
|
|
260 |
|
|
def RunAndVerify(self, gtest_filter, tests_to_run):
|
261 |
|
|
"""Checks that the binary runs correct set of tests for a given filter."""
|
262 |
|
|
|
263 |
|
|
tests_to_run = self.AdjustForParameterizedTests(tests_to_run)
|
264 |
|
|
|
265 |
|
|
# First, tests using the environment variable.
|
266 |
|
|
|
267 |
|
|
# Windows removes empty variables from the environment when passing it
|
268 |
|
|
# to a new process. This means it is impossible to pass an empty filter
|
269 |
|
|
# into a process using the environment variable. However, we can still
|
270 |
|
|
# test the case when the variable is not supplied (i.e., gtest_filter is
|
271 |
|
|
# None).
|
272 |
|
|
# pylint: disable-msg=C6403
|
273 |
|
|
if CAN_TEST_EMPTY_FILTER or gtest_filter != '':
|
274 |
|
|
SetEnvVar(FILTER_ENV_VAR, gtest_filter)
|
275 |
|
|
tests_run = RunAndExtractTestList()[0]
|
276 |
|
|
SetEnvVar(FILTER_ENV_VAR, None)
|
277 |
|
|
self.AssertSetEqual(tests_run, tests_to_run)
|
278 |
|
|
# pylint: enable-msg=C6403
|
279 |
|
|
|
280 |
|
|
# Next, tests using the command line flag.
|
281 |
|
|
|
282 |
|
|
if gtest_filter is None:
|
283 |
|
|
args = []
|
284 |
|
|
else:
|
285 |
|
|
args = ['--%s=%s' % (FILTER_FLAG, gtest_filter)]
|
286 |
|
|
|
287 |
|
|
tests_run = RunAndExtractTestList(args)[0]
|
288 |
|
|
self.AssertSetEqual(tests_run, tests_to_run)
|
289 |
|
|
|
290 |
|
|
def RunAndVerifyWithSharding(self, gtest_filter, total_shards, tests_to_run,
|
291 |
|
|
args=None, check_exit_0=False):
|
292 |
|
|
"""Checks that binary runs correct tests for the given filter and shard.
|
293 |
|
|
|
294 |
|
|
Runs all shards of gtest_filter_unittest_ with the given filter, and
|
295 |
|
|
verifies that the right set of tests were run. The union of tests run
|
296 |
|
|
on each shard should be identical to tests_to_run, without duplicates.
|
297 |
|
|
|
298 |
|
|
Args:
|
299 |
|
|
gtest_filter: A filter to apply to the tests.
|
300 |
|
|
total_shards: A total number of shards to split test run into.
|
301 |
|
|
tests_to_run: A set of tests expected to run.
|
302 |
|
|
args : Arguments to pass to the to the test binary.
|
303 |
|
|
check_exit_0: When set to a true value, make sure that all shards
|
304 |
|
|
return 0.
|
305 |
|
|
"""
|
306 |
|
|
|
307 |
|
|
tests_to_run = self.AdjustForParameterizedTests(tests_to_run)
|
308 |
|
|
|
309 |
|
|
# Windows removes empty variables from the environment when passing it
|
310 |
|
|
# to a new process. This means it is impossible to pass an empty filter
|
311 |
|
|
# into a process using the environment variable. However, we can still
|
312 |
|
|
# test the case when the variable is not supplied (i.e., gtest_filter is
|
313 |
|
|
# None).
|
314 |
|
|
# pylint: disable-msg=C6403
|
315 |
|
|
if CAN_TEST_EMPTY_FILTER or gtest_filter != '':
|
316 |
|
|
SetEnvVar(FILTER_ENV_VAR, gtest_filter)
|
317 |
|
|
partition = []
|
318 |
|
|
for i in range(0, total_shards):
|
319 |
|
|
(tests_run, exit_code) = RunWithSharding(total_shards, i, args)
|
320 |
|
|
if check_exit_0:
|
321 |
|
|
self.assertEqual(0, exit_code)
|
322 |
|
|
partition.append(tests_run)
|
323 |
|
|
|
324 |
|
|
self.AssertPartitionIsValid(tests_to_run, partition)
|
325 |
|
|
SetEnvVar(FILTER_ENV_VAR, None)
|
326 |
|
|
# pylint: enable-msg=C6403
|
327 |
|
|
|
328 |
|
|
def RunAndVerifyAllowingDisabled(self, gtest_filter, tests_to_run):
|
329 |
|
|
"""Checks that the binary runs correct set of tests for the given filter.
|
330 |
|
|
|
331 |
|
|
Runs gtest_filter_unittest_ with the given filter, and enables
|
332 |
|
|
disabled tests. Verifies that the right set of tests were run.
|
333 |
|
|
|
334 |
|
|
Args:
|
335 |
|
|
gtest_filter: A filter to apply to the tests.
|
336 |
|
|
tests_to_run: A set of tests expected to run.
|
337 |
|
|
"""
|
338 |
|
|
|
339 |
|
|
tests_to_run = self.AdjustForParameterizedTests(tests_to_run)
|
340 |
|
|
|
341 |
|
|
# Construct the command line.
|
342 |
|
|
args = ['--%s' % ALSO_RUN_DISABED_TESTS_FLAG]
|
343 |
|
|
if gtest_filter is not None:
|
344 |
|
|
args.append('--%s=%s' % (FILTER_FLAG, gtest_filter))
|
345 |
|
|
|
346 |
|
|
tests_run = RunAndExtractTestList(args)[0]
|
347 |
|
|
self.AssertSetEqual(tests_run, tests_to_run)
|
348 |
|
|
|
349 |
|
|
def setUp(self):
|
350 |
|
|
"""Sets up test case.
|
351 |
|
|
|
352 |
|
|
Determines whether value-parameterized tests are enabled in the binary and
|
353 |
|
|
sets the flags accordingly.
|
354 |
|
|
"""
|
355 |
|
|
|
356 |
|
|
global param_tests_present
|
357 |
|
|
if param_tests_present is None:
|
358 |
|
|
param_tests_present = PARAM_TEST_REGEX.search(
|
359 |
|
|
RunAndReturnOutput()) is not None
|
360 |
|
|
|
361 |
|
|
def testDefaultBehavior(self):
|
362 |
|
|
"""Tests the behavior of not specifying the filter."""
|
363 |
|
|
|
364 |
|
|
self.RunAndVerify(None, ACTIVE_TESTS)
|
365 |
|
|
|
366 |
|
|
def testDefaultBehaviorWithShards(self):
|
367 |
|
|
"""Tests the behavior without the filter, with sharding enabled."""
|
368 |
|
|
|
369 |
|
|
self.RunAndVerifyWithSharding(None, 1, ACTIVE_TESTS)
|
370 |
|
|
self.RunAndVerifyWithSharding(None, 2, ACTIVE_TESTS)
|
371 |
|
|
self.RunAndVerifyWithSharding(None, len(ACTIVE_TESTS) - 1, ACTIVE_TESTS)
|
372 |
|
|
self.RunAndVerifyWithSharding(None, len(ACTIVE_TESTS), ACTIVE_TESTS)
|
373 |
|
|
self.RunAndVerifyWithSharding(None, len(ACTIVE_TESTS) + 1, ACTIVE_TESTS)
|
374 |
|
|
|
375 |
|
|
def testEmptyFilter(self):
|
376 |
|
|
"""Tests an empty filter."""
|
377 |
|
|
|
378 |
|
|
self.RunAndVerify('', [])
|
379 |
|
|
self.RunAndVerifyWithSharding('', 1, [])
|
380 |
|
|
self.RunAndVerifyWithSharding('', 2, [])
|
381 |
|
|
|
382 |
|
|
def testBadFilter(self):
|
383 |
|
|
"""Tests a filter that matches nothing."""
|
384 |
|
|
|
385 |
|
|
self.RunAndVerify('BadFilter', [])
|
386 |
|
|
self.RunAndVerifyAllowingDisabled('BadFilter', [])
|
387 |
|
|
|
388 |
|
|
def testFullName(self):
|
389 |
|
|
"""Tests filtering by full name."""
|
390 |
|
|
|
391 |
|
|
self.RunAndVerify('FooTest.Xyz', ['FooTest.Xyz'])
|
392 |
|
|
self.RunAndVerifyAllowingDisabled('FooTest.Xyz', ['FooTest.Xyz'])
|
393 |
|
|
self.RunAndVerifyWithSharding('FooTest.Xyz', 5, ['FooTest.Xyz'])
|
394 |
|
|
|
395 |
|
|
def testUniversalFilters(self):
|
396 |
|
|
"""Tests filters that match everything."""
|
397 |
|
|
|
398 |
|
|
self.RunAndVerify('*', ACTIVE_TESTS)
|
399 |
|
|
self.RunAndVerify('*.*', ACTIVE_TESTS)
|
400 |
|
|
self.RunAndVerifyWithSharding('*.*', len(ACTIVE_TESTS) - 3, ACTIVE_TESTS)
|
401 |
|
|
self.RunAndVerifyAllowingDisabled('*', ACTIVE_TESTS + DISABLED_TESTS)
|
402 |
|
|
self.RunAndVerifyAllowingDisabled('*.*', ACTIVE_TESTS + DISABLED_TESTS)
|
403 |
|
|
|
404 |
|
|
def testFilterByTestCase(self):
|
405 |
|
|
"""Tests filtering by test case name."""
|
406 |
|
|
|
407 |
|
|
self.RunAndVerify('FooTest.*', ['FooTest.Abc', 'FooTest.Xyz'])
|
408 |
|
|
|
409 |
|
|
BAZ_TESTS = ['BazTest.TestOne', 'BazTest.TestA', 'BazTest.TestB']
|
410 |
|
|
self.RunAndVerify('BazTest.*', BAZ_TESTS)
|
411 |
|
|
self.RunAndVerifyAllowingDisabled('BazTest.*',
|
412 |
|
|
BAZ_TESTS + ['BazTest.DISABLED_TestC'])
|
413 |
|
|
|
414 |
|
|
def testFilterByTest(self):
|
415 |
|
|
"""Tests filtering by test name."""
|
416 |
|
|
|
417 |
|
|
self.RunAndVerify('*.TestOne', ['BarTest.TestOne', 'BazTest.TestOne'])
|
418 |
|
|
|
419 |
|
|
def testFilterDisabledTests(self):
|
420 |
|
|
"""Select only the disabled tests to run."""
|
421 |
|
|
|
422 |
|
|
self.RunAndVerify('DISABLED_FoobarTest.Test1', [])
|
423 |
|
|
self.RunAndVerifyAllowingDisabled('DISABLED_FoobarTest.Test1',
|
424 |
|
|
['DISABLED_FoobarTest.Test1'])
|
425 |
|
|
|
426 |
|
|
self.RunAndVerify('*DISABLED_*', [])
|
427 |
|
|
self.RunAndVerifyAllowingDisabled('*DISABLED_*', DISABLED_TESTS)
|
428 |
|
|
|
429 |
|
|
self.RunAndVerify('*.DISABLED_*', [])
|
430 |
|
|
self.RunAndVerifyAllowingDisabled('*.DISABLED_*', [
|
431 |
|
|
'BarTest.DISABLED_TestFour',
|
432 |
|
|
'BarTest.DISABLED_TestFive',
|
433 |
|
|
'BazTest.DISABLED_TestC',
|
434 |
|
|
'DISABLED_FoobarTest.DISABLED_Test2',
|
435 |
|
|
])
|
436 |
|
|
|
437 |
|
|
self.RunAndVerify('DISABLED_*', [])
|
438 |
|
|
self.RunAndVerifyAllowingDisabled('DISABLED_*', [
|
439 |
|
|
'DISABLED_FoobarTest.Test1',
|
440 |
|
|
'DISABLED_FoobarTest.DISABLED_Test2',
|
441 |
|
|
'DISABLED_FoobarbazTest.TestA',
|
442 |
|
|
])
|
443 |
|
|
|
444 |
|
|
def testWildcardInTestCaseName(self):
|
445 |
|
|
"""Tests using wildcard in the test case name."""
|
446 |
|
|
|
447 |
|
|
self.RunAndVerify('*a*.*', [
|
448 |
|
|
'BarTest.TestOne',
|
449 |
|
|
'BarTest.TestTwo',
|
450 |
|
|
'BarTest.TestThree',
|
451 |
|
|
|
452 |
|
|
'BazTest.TestOne',
|
453 |
|
|
'BazTest.TestA',
|
454 |
|
|
'BazTest.TestB', ] + DEATH_TESTS + PARAM_TESTS)
|
455 |
|
|
|
456 |
|
|
def testWildcardInTestName(self):
|
457 |
|
|
"""Tests using wildcard in the test name."""
|
458 |
|
|
|
459 |
|
|
self.RunAndVerify('*.*A*', ['FooTest.Abc', 'BazTest.TestA'])
|
460 |
|
|
|
461 |
|
|
def testFilterWithoutDot(self):
|
462 |
|
|
"""Tests a filter that has no '.' in it."""
|
463 |
|
|
|
464 |
|
|
self.RunAndVerify('*z*', [
|
465 |
|
|
'FooTest.Xyz',
|
466 |
|
|
|
467 |
|
|
'BazTest.TestOne',
|
468 |
|
|
'BazTest.TestA',
|
469 |
|
|
'BazTest.TestB',
|
470 |
|
|
])
|
471 |
|
|
|
472 |
|
|
def testTwoPatterns(self):
|
473 |
|
|
"""Tests filters that consist of two patterns."""
|
474 |
|
|
|
475 |
|
|
self.RunAndVerify('Foo*.*:*A*', [
|
476 |
|
|
'FooTest.Abc',
|
477 |
|
|
'FooTest.Xyz',
|
478 |
|
|
|
479 |
|
|
'BazTest.TestA',
|
480 |
|
|
])
|
481 |
|
|
|
482 |
|
|
# An empty pattern + a non-empty one
|
483 |
|
|
self.RunAndVerify(':*A*', ['FooTest.Abc', 'BazTest.TestA'])
|
484 |
|
|
|
485 |
|
|
def testThreePatterns(self):
|
486 |
|
|
"""Tests filters that consist of three patterns."""
|
487 |
|
|
|
488 |
|
|
self.RunAndVerify('*oo*:*A*:*One', [
|
489 |
|
|
'FooTest.Abc',
|
490 |
|
|
'FooTest.Xyz',
|
491 |
|
|
|
492 |
|
|
'BarTest.TestOne',
|
493 |
|
|
|
494 |
|
|
'BazTest.TestOne',
|
495 |
|
|
'BazTest.TestA',
|
496 |
|
|
])
|
497 |
|
|
|
498 |
|
|
# The 2nd pattern is empty.
|
499 |
|
|
self.RunAndVerify('*oo*::*One', [
|
500 |
|
|
'FooTest.Abc',
|
501 |
|
|
'FooTest.Xyz',
|
502 |
|
|
|
503 |
|
|
'BarTest.TestOne',
|
504 |
|
|
|
505 |
|
|
'BazTest.TestOne',
|
506 |
|
|
])
|
507 |
|
|
|
508 |
|
|
# The last 2 patterns are empty.
|
509 |
|
|
self.RunAndVerify('*oo*::', [
|
510 |
|
|
'FooTest.Abc',
|
511 |
|
|
'FooTest.Xyz',
|
512 |
|
|
])
|
513 |
|
|
|
514 |
|
|
def testNegativeFilters(self):
|
515 |
|
|
self.RunAndVerify('*-BazTest.TestOne', [
|
516 |
|
|
'FooTest.Abc',
|
517 |
|
|
'FooTest.Xyz',
|
518 |
|
|
|
519 |
|
|
'BarTest.TestOne',
|
520 |
|
|
'BarTest.TestTwo',
|
521 |
|
|
'BarTest.TestThree',
|
522 |
|
|
|
523 |
|
|
'BazTest.TestA',
|
524 |
|
|
'BazTest.TestB',
|
525 |
|
|
] + DEATH_TESTS + PARAM_TESTS)
|
526 |
|
|
|
527 |
|
|
self.RunAndVerify('*-FooTest.Abc:BazTest.*', [
|
528 |
|
|
'FooTest.Xyz',
|
529 |
|
|
|
530 |
|
|
'BarTest.TestOne',
|
531 |
|
|
'BarTest.TestTwo',
|
532 |
|
|
'BarTest.TestThree',
|
533 |
|
|
] + DEATH_TESTS + PARAM_TESTS)
|
534 |
|
|
|
535 |
|
|
self.RunAndVerify('BarTest.*-BarTest.TestOne', [
|
536 |
|
|
'BarTest.TestTwo',
|
537 |
|
|
'BarTest.TestThree',
|
538 |
|
|
])
|
539 |
|
|
|
540 |
|
|
# Tests without leading '*'.
|
541 |
|
|
self.RunAndVerify('-FooTest.Abc:FooTest.Xyz:BazTest.*', [
|
542 |
|
|
'BarTest.TestOne',
|
543 |
|
|
'BarTest.TestTwo',
|
544 |
|
|
'BarTest.TestThree',
|
545 |
|
|
] + DEATH_TESTS + PARAM_TESTS)
|
546 |
|
|
|
547 |
|
|
# Value parameterized tests.
|
548 |
|
|
self.RunAndVerify('*/*', PARAM_TESTS)
|
549 |
|
|
|
550 |
|
|
# Value parameterized tests filtering by the sequence name.
|
551 |
|
|
self.RunAndVerify('SeqP/*', [
|
552 |
|
|
'SeqP/ParamTest.TestX/0',
|
553 |
|
|
'SeqP/ParamTest.TestX/1',
|
554 |
|
|
'SeqP/ParamTest.TestY/0',
|
555 |
|
|
'SeqP/ParamTest.TestY/1',
|
556 |
|
|
])
|
557 |
|
|
|
558 |
|
|
# Value parameterized tests filtering by the test name.
|
559 |
|
|
self.RunAndVerify('*/0', [
|
560 |
|
|
'SeqP/ParamTest.TestX/0',
|
561 |
|
|
'SeqP/ParamTest.TestY/0',
|
562 |
|
|
'SeqQ/ParamTest.TestX/0',
|
563 |
|
|
'SeqQ/ParamTest.TestY/0',
|
564 |
|
|
])
|
565 |
|
|
|
566 |
|
|
def testFlagOverridesEnvVar(self):
|
567 |
|
|
"""Tests that the filter flag overrides the filtering env. variable."""
|
568 |
|
|
|
569 |
|
|
SetEnvVar(FILTER_ENV_VAR, 'Foo*')
|
570 |
|
|
args = ['--%s=%s' % (FILTER_FLAG, '*One')]
|
571 |
|
|
tests_run = RunAndExtractTestList(args)[0]
|
572 |
|
|
SetEnvVar(FILTER_ENV_VAR, None)
|
573 |
|
|
|
574 |
|
|
self.AssertSetEqual(tests_run, ['BarTest.TestOne', 'BazTest.TestOne'])
|
575 |
|
|
|
576 |
|
|
def testShardStatusFileIsCreated(self):
|
577 |
|
|
"""Tests that the shard file is created if specified in the environment."""
|
578 |
|
|
|
579 |
|
|
shard_status_file = os.path.join(gtest_test_utils.GetTempDir(),
|
580 |
|
|
'shard_status_file')
|
581 |
|
|
self.assert_(not os.path.exists(shard_status_file))
|
582 |
|
|
|
583 |
|
|
extra_env = {SHARD_STATUS_FILE_ENV_VAR: shard_status_file}
|
584 |
|
|
try:
|
585 |
|
|
InvokeWithModifiedEnv(extra_env, RunAndReturnOutput)
|
586 |
|
|
finally:
|
587 |
|
|
self.assert_(os.path.exists(shard_status_file))
|
588 |
|
|
os.remove(shard_status_file)
|
589 |
|
|
|
590 |
|
|
def testShardStatusFileIsCreatedWithListTests(self):
|
591 |
|
|
"""Tests that the shard file is created with the "list_tests" flag."""
|
592 |
|
|
|
593 |
|
|
shard_status_file = os.path.join(gtest_test_utils.GetTempDir(),
|
594 |
|
|
'shard_status_file2')
|
595 |
|
|
self.assert_(not os.path.exists(shard_status_file))
|
596 |
|
|
|
597 |
|
|
extra_env = {SHARD_STATUS_FILE_ENV_VAR: shard_status_file}
|
598 |
|
|
try:
|
599 |
|
|
output = InvokeWithModifiedEnv(extra_env,
|
600 |
|
|
RunAndReturnOutput,
|
601 |
|
|
[LIST_TESTS_FLAG])
|
602 |
|
|
finally:
|
603 |
|
|
# This assertion ensures that Google Test enumerated the tests as
|
604 |
|
|
# opposed to running them.
|
605 |
|
|
self.assert_('[==========]' not in output,
|
606 |
|
|
'Unexpected output during test enumeration.\n'
|
607 |
|
|
'Please ensure that LIST_TESTS_FLAG is assigned the\n'
|
608 |
|
|
'correct flag value for listing Google Test tests.')
|
609 |
|
|
|
610 |
|
|
self.assert_(os.path.exists(shard_status_file))
|
611 |
|
|
os.remove(shard_status_file)
|
612 |
|
|
|
613 |
|
|
if SUPPORTS_DEATH_TESTS:
|
614 |
|
|
def testShardingWorksWithDeathTests(self):
|
615 |
|
|
"""Tests integration with death tests and sharding."""
|
616 |
|
|
|
617 |
|
|
gtest_filter = 'HasDeathTest.*:SeqP/*'
|
618 |
|
|
expected_tests = [
|
619 |
|
|
'HasDeathTest.Test1',
|
620 |
|
|
'HasDeathTest.Test2',
|
621 |
|
|
|
622 |
|
|
'SeqP/ParamTest.TestX/0',
|
623 |
|
|
'SeqP/ParamTest.TestX/1',
|
624 |
|
|
'SeqP/ParamTest.TestY/0',
|
625 |
|
|
'SeqP/ParamTest.TestY/1',
|
626 |
|
|
]
|
627 |
|
|
|
628 |
|
|
for flag in ['--gtest_death_test_style=threadsafe',
|
629 |
|
|
'--gtest_death_test_style=fast']:
|
630 |
|
|
self.RunAndVerifyWithSharding(gtest_filter, 3, expected_tests,
|
631 |
|
|
check_exit_0=True, args=[flag])
|
632 |
|
|
self.RunAndVerifyWithSharding(gtest_filter, 5, expected_tests,
|
633 |
|
|
check_exit_0=True, args=[flag])
|
634 |
|
|
|
635 |
|
|
if __name__ == '__main__':
|
636 |
|
|
gtest_test_utils.Main()
|