OpenCores
URL https://opencores.org/ocsvn/s80186/s80186/trunk

Subversion Repositories s80186

[/] [s80186/] [trunk/] [vendor/] [googletest/] [googletest/] [test/] [gtest_shuffle_test.py] - Blame information for rev 2

Details | Compare with Previous | View Log

Line No. Rev Author Line
1 2 jamieiles
#!/usr/bin/env python
2
#
3
# Copyright 2009 Google Inc. All Rights Reserved.
4
#
5
# Redistribution and use in source and binary forms, with or without
6
# modification, are permitted provided that the following conditions are
7
# met:
8
#
9
#     * Redistributions of source code must retain the above copyright
10
# notice, this list of conditions and the following disclaimer.
11
#     * Redistributions in binary form must reproduce the above
12
# copyright notice, this list of conditions and the following disclaimer
13
# in the documentation and/or other materials provided with the
14
# distribution.
15
#     * Neither the name of Google Inc. nor the names of its
16
# contributors may be used to endorse or promote products derived from
17
# this software without specific prior written permission.
18
#
19
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
 
31
"""Verifies that test shuffling works."""
32
 
33
__author__ = 'wan@google.com (Zhanyong Wan)'
34
 
35
import os
36
import gtest_test_utils
37
 
38
# Command to run the gtest_shuffle_test_ program.
39
COMMAND = gtest_test_utils.GetTestExecutablePath('gtest_shuffle_test_')
40
 
41
# The environment variables for test sharding.
42
TOTAL_SHARDS_ENV_VAR = 'GTEST_TOTAL_SHARDS'
43
SHARD_INDEX_ENV_VAR = 'GTEST_SHARD_INDEX'
44
 
45
TEST_FILTER = 'A*.A:A*.B:C*'
46
 
47
ALL_TESTS = []
48
ACTIVE_TESTS = []
49
FILTERED_TESTS = []
50
SHARDED_TESTS = []
51
 
52
SHUFFLED_ALL_TESTS = []
53
SHUFFLED_ACTIVE_TESTS = []
54
SHUFFLED_FILTERED_TESTS = []
55
SHUFFLED_SHARDED_TESTS = []
56
 
57
 
58
def AlsoRunDisabledTestsFlag():
59
  return '--gtest_also_run_disabled_tests'
60
 
61
 
62
def FilterFlag(test_filter):
63
  return '--gtest_filter=%s' % (test_filter,)
64
 
65
 
66
def RepeatFlag(n):
67
  return '--gtest_repeat=%s' % (n,)
68
 
69
 
70
def ShuffleFlag():
71
  return '--gtest_shuffle'
72
 
73
 
74
def RandomSeedFlag(n):
75
  return '--gtest_random_seed=%s' % (n,)
76
 
77
 
78
def RunAndReturnOutput(extra_env, args):
79
  """Runs the test program and returns its output."""
80
 
81
  environ_copy = os.environ.copy()
82
  environ_copy.update(extra_env)
83
 
84
  return gtest_test_utils.Subprocess([COMMAND] + args, env=environ_copy).output
85
 
86
 
87
def GetTestsForAllIterations(extra_env, args):
88
  """Runs the test program and returns a list of test lists.
89
 
90
  Args:
91
    extra_env: a map from environment variables to their values
92
    args: command line flags to pass to gtest_shuffle_test_
93
 
94
  Returns:
95
    A list where the i-th element is the list of tests run in the i-th
96
    test iteration.
97
  """
98
 
99
  test_iterations = []
100
  for line in RunAndReturnOutput(extra_env, args).split('\n'):
101
    if line.startswith('----'):
102
      tests = []
103
      test_iterations.append(tests)
104
    elif line.strip():
105
      tests.append(line.strip())  # 'TestCaseName.TestName'
106
 
107
  return test_iterations
108
 
109
 
110
def GetTestCases(tests):
111
  """Returns a list of test cases in the given full test names.
112
 
113
  Args:
114
    tests: a list of full test names
115
 
116
  Returns:
117
    A list of test cases from 'tests', in their original order.
118
    Consecutive duplicates are removed.
119
  """
120
 
121
  test_cases = []
122
  for test in tests:
123
    test_case = test.split('.')[0]
124
    if not test_case in test_cases:
125
      test_cases.append(test_case)
126
 
127
  return test_cases
128
 
129
 
130
def CalculateTestLists():
131
  """Calculates the list of tests run under different flags."""
132
 
133
  if not ALL_TESTS:
134
    ALL_TESTS.extend(
135
        GetTestsForAllIterations({}, [AlsoRunDisabledTestsFlag()])[0])
136
 
137
  if not ACTIVE_TESTS:
138
    ACTIVE_TESTS.extend(GetTestsForAllIterations({}, [])[0])
139
 
140
  if not FILTERED_TESTS:
141
    FILTERED_TESTS.extend(
142
        GetTestsForAllIterations({}, [FilterFlag(TEST_FILTER)])[0])
143
 
144
  if not SHARDED_TESTS:
145
    SHARDED_TESTS.extend(
146
        GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3',
147
                                  SHARD_INDEX_ENV_VAR: '1'},
148
                                 [])[0])
149
 
150
  if not SHUFFLED_ALL_TESTS:
151
    SHUFFLED_ALL_TESTS.extend(GetTestsForAllIterations(
152
        {}, [AlsoRunDisabledTestsFlag(), ShuffleFlag(), RandomSeedFlag(1)])[0])
153
 
154
  if not SHUFFLED_ACTIVE_TESTS:
155
    SHUFFLED_ACTIVE_TESTS.extend(GetTestsForAllIterations(
156
        {}, [ShuffleFlag(), RandomSeedFlag(1)])[0])
157
 
158
  if not SHUFFLED_FILTERED_TESTS:
159
    SHUFFLED_FILTERED_TESTS.extend(GetTestsForAllIterations(
160
        {}, [ShuffleFlag(), RandomSeedFlag(1), FilterFlag(TEST_FILTER)])[0])
161
 
162
  if not SHUFFLED_SHARDED_TESTS:
163
    SHUFFLED_SHARDED_TESTS.extend(
164
        GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3',
165
                                  SHARD_INDEX_ENV_VAR: '1'},
166
                                 [ShuffleFlag(), RandomSeedFlag(1)])[0])
167
 
168
 
169
class GTestShuffleUnitTest(gtest_test_utils.TestCase):
170
  """Tests test shuffling."""
171
 
172
  def setUp(self):
173
    CalculateTestLists()
174
 
175
  def testShufflePreservesNumberOfTests(self):
176
    self.assertEqual(len(ALL_TESTS), len(SHUFFLED_ALL_TESTS))
177
    self.assertEqual(len(ACTIVE_TESTS), len(SHUFFLED_ACTIVE_TESTS))
178
    self.assertEqual(len(FILTERED_TESTS), len(SHUFFLED_FILTERED_TESTS))
179
    self.assertEqual(len(SHARDED_TESTS), len(SHUFFLED_SHARDED_TESTS))
180
 
181
  def testShuffleChangesTestOrder(self):
182
    self.assert_(SHUFFLED_ALL_TESTS != ALL_TESTS, SHUFFLED_ALL_TESTS)
183
    self.assert_(SHUFFLED_ACTIVE_TESTS != ACTIVE_TESTS, SHUFFLED_ACTIVE_TESTS)
184
    self.assert_(SHUFFLED_FILTERED_TESTS != FILTERED_TESTS,
185
                 SHUFFLED_FILTERED_TESTS)
186
    self.assert_(SHUFFLED_SHARDED_TESTS != SHARDED_TESTS,
187
                 SHUFFLED_SHARDED_TESTS)
188
 
189
  def testShuffleChangesTestCaseOrder(self):
190
    self.assert_(GetTestCases(SHUFFLED_ALL_TESTS) != GetTestCases(ALL_TESTS),
191
                 GetTestCases(SHUFFLED_ALL_TESTS))
192
    self.assert_(
193
        GetTestCases(SHUFFLED_ACTIVE_TESTS) != GetTestCases(ACTIVE_TESTS),
194
        GetTestCases(SHUFFLED_ACTIVE_TESTS))
195
    self.assert_(
196
        GetTestCases(SHUFFLED_FILTERED_TESTS) != GetTestCases(FILTERED_TESTS),
197
        GetTestCases(SHUFFLED_FILTERED_TESTS))
198
    self.assert_(
199
        GetTestCases(SHUFFLED_SHARDED_TESTS) != GetTestCases(SHARDED_TESTS),
200
        GetTestCases(SHUFFLED_SHARDED_TESTS))
201
 
202
  def testShuffleDoesNotRepeatTest(self):
203
    for test in SHUFFLED_ALL_TESTS:
204
      self.assertEqual(1, SHUFFLED_ALL_TESTS.count(test),
205
                       '%s appears more than once' % (test,))
206
    for test in SHUFFLED_ACTIVE_TESTS:
207
      self.assertEqual(1, SHUFFLED_ACTIVE_TESTS.count(test),
208
                       '%s appears more than once' % (test,))
209
    for test in SHUFFLED_FILTERED_TESTS:
210
      self.assertEqual(1, SHUFFLED_FILTERED_TESTS.count(test),
211
                       '%s appears more than once' % (test,))
212
    for test in SHUFFLED_SHARDED_TESTS:
213
      self.assertEqual(1, SHUFFLED_SHARDED_TESTS.count(test),
214
                       '%s appears more than once' % (test,))
215
 
216
  def testShuffleDoesNotCreateNewTest(self):
217
    for test in SHUFFLED_ALL_TESTS:
218
      self.assert_(test in ALL_TESTS, '%s is an invalid test' % (test,))
219
    for test in SHUFFLED_ACTIVE_TESTS:
220
      self.assert_(test in ACTIVE_TESTS, '%s is an invalid test' % (test,))
221
    for test in SHUFFLED_FILTERED_TESTS:
222
      self.assert_(test in FILTERED_TESTS, '%s is an invalid test' % (test,))
223
    for test in SHUFFLED_SHARDED_TESTS:
224
      self.assert_(test in SHARDED_TESTS, '%s is an invalid test' % (test,))
225
 
226
  def testShuffleIncludesAllTests(self):
227
    for test in ALL_TESTS:
228
      self.assert_(test in SHUFFLED_ALL_TESTS, '%s is missing' % (test,))
229
    for test in ACTIVE_TESTS:
230
      self.assert_(test in SHUFFLED_ACTIVE_TESTS, '%s is missing' % (test,))
231
    for test in FILTERED_TESTS:
232
      self.assert_(test in SHUFFLED_FILTERED_TESTS, '%s is missing' % (test,))
233
    for test in SHARDED_TESTS:
234
      self.assert_(test in SHUFFLED_SHARDED_TESTS, '%s is missing' % (test,))
235
 
236
  def testShuffleLeavesDeathTestsAtFront(self):
237
    non_death_test_found = False
238
    for test in SHUFFLED_ACTIVE_TESTS:
239
      if 'DeathTest.' in test:
240
        self.assert_(not non_death_test_found,
241
                     '%s appears after a non-death test' % (test,))
242
      else:
243
        non_death_test_found = True
244
 
245
  def _VerifyTestCasesDoNotInterleave(self, tests):
246
    test_cases = []
247
    for test in tests:
248
      [test_case, _] = test.split('.')
249
      if test_cases and test_cases[-1] != test_case:
250
        test_cases.append(test_case)
251
        self.assertEqual(1, test_cases.count(test_case),
252
                         'Test case %s is not grouped together in %s' %
253
                         (test_case, tests))
254
 
255
  def testShuffleDoesNotInterleaveTestCases(self):
256
    self._VerifyTestCasesDoNotInterleave(SHUFFLED_ALL_TESTS)
257
    self._VerifyTestCasesDoNotInterleave(SHUFFLED_ACTIVE_TESTS)
258
    self._VerifyTestCasesDoNotInterleave(SHUFFLED_FILTERED_TESTS)
259
    self._VerifyTestCasesDoNotInterleave(SHUFFLED_SHARDED_TESTS)
260
 
261
  def testShuffleRestoresOrderAfterEachIteration(self):
262
    # Get the test lists in all 3 iterations, using random seed 1, 2,
263
    # and 3 respectively.  Google Test picks a different seed in each
264
    # iteration, and this test depends on the current implementation
265
    # picking successive numbers.  This dependency is not ideal, but
266
    # makes the test much easier to write.
267
    [tests_in_iteration1, tests_in_iteration2, tests_in_iteration3] = (
268
        GetTestsForAllIterations(
269
            {}, [ShuffleFlag(), RandomSeedFlag(1), RepeatFlag(3)]))
270
 
271
    # Make sure running the tests with random seed 1 gets the same
272
    # order as in iteration 1 above.
273
    [tests_with_seed1] = GetTestsForAllIterations(
274
        {}, [ShuffleFlag(), RandomSeedFlag(1)])
275
    self.assertEqual(tests_in_iteration1, tests_with_seed1)
276
 
277
    # Make sure running the tests with random seed 2 gets the same
278
    # order as in iteration 2 above.  Success means that Google Test
279
    # correctly restores the test order before re-shuffling at the
280
    # beginning of iteration 2.
281
    [tests_with_seed2] = GetTestsForAllIterations(
282
        {}, [ShuffleFlag(), RandomSeedFlag(2)])
283
    self.assertEqual(tests_in_iteration2, tests_with_seed2)
284
 
285
    # Make sure running the tests with random seed 3 gets the same
286
    # order as in iteration 3 above.  Success means that Google Test
287
    # correctly restores the test order before re-shuffling at the
288
    # beginning of iteration 3.
289
    [tests_with_seed3] = GetTestsForAllIterations(
290
        {}, [ShuffleFlag(), RandomSeedFlag(3)])
291
    self.assertEqual(tests_in_iteration3, tests_with_seed3)
292
 
293
  def testShuffleGeneratesNewOrderInEachIteration(self):
294
    [tests_in_iteration1, tests_in_iteration2, tests_in_iteration3] = (
295
        GetTestsForAllIterations(
296
            {}, [ShuffleFlag(), RandomSeedFlag(1), RepeatFlag(3)]))
297
 
298
    self.assert_(tests_in_iteration1 != tests_in_iteration2,
299
                 tests_in_iteration1)
300
    self.assert_(tests_in_iteration1 != tests_in_iteration3,
301
                 tests_in_iteration1)
302
    self.assert_(tests_in_iteration2 != tests_in_iteration3,
303
                 tests_in_iteration2)
304
 
305
  def testShuffleShardedTestsPreservesPartition(self):
306
    # If we run M tests on N shards, the same M tests should be run in
307
    # total, regardless of the random seeds used by the shards.
308
    [tests1] = GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3',
309
                                         SHARD_INDEX_ENV_VAR: '0'},
310
                                        [ShuffleFlag(), RandomSeedFlag(1)])
311
    [tests2] = GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3',
312
                                         SHARD_INDEX_ENV_VAR: '1'},
313
                                        [ShuffleFlag(), RandomSeedFlag(20)])
314
    [tests3] = GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3',
315
                                         SHARD_INDEX_ENV_VAR: '2'},
316
                                        [ShuffleFlag(), RandomSeedFlag(25)])
317
    sorted_sharded_tests = tests1 + tests2 + tests3
318
    sorted_sharded_tests.sort()
319
    sorted_active_tests = []
320
    sorted_active_tests.extend(ACTIVE_TESTS)
321
    sorted_active_tests.sort()
322
    self.assertEqual(sorted_active_tests, sorted_sharded_tests)
323
 
324
if __name__ == '__main__':
325
  gtest_test_utils.Main()

powered by: WebSVN 2.1.0

© copyright 1999-2024 OpenCores.org, equivalent to Oliscience, all rights reserved. OpenCores®, registered trademark.