init: gtest and quiz 1.
This commit is contained in:
461
third-party/googletest-1.17.0/googletest/test/googletest-failfast-unittest.py
vendored
Executable file
461
third-party/googletest-1.17.0/googletest/test/googletest-failfast-unittest.py
vendored
Executable file
@@ -0,0 +1,461 @@
|
||||
#!/usr/bin/env python
|
||||
#
|
||||
# Copyright 2020 Google Inc. All Rights Reserved.
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are
|
||||
# met:
|
||||
#
|
||||
# * Redistributions of source code must retain the above copyright
|
||||
# notice, this list of conditions and the following disclaimer.
|
||||
# * Redistributions in binary form must reproduce the above
|
||||
# copyright notice, this list of conditions and the following disclaimer
|
||||
# in the documentation and/or other materials provided with the
|
||||
# distribution.
|
||||
# * Neither the name of Google Inc. nor the names of its
|
||||
# contributors may be used to endorse or promote products derived from
|
||||
# this software without specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
"""Unit test for Google Test fail_fast.
|
||||
|
||||
A user can specify if a Google Test program should continue test execution
|
||||
after a test failure via the GTEST_FAIL_FAST environment variable or the
|
||||
--gtest_fail_fast flag. The default value of the flag can also be changed
|
||||
by Bazel fail fast environment variable TESTBRIDGE_TEST_RUNNER_FAIL_FAST.
|
||||
|
||||
This script tests such functionality by invoking googletest-failfast-unittest_
|
||||
(a program written with Google Test) with different environments and command
|
||||
line flags.
|
||||
"""
|
||||
|
||||
import os
|
||||
from googletest.test import gtest_test_utils
|
||||
|
||||
# Constants.
|
||||
|
||||
# Bazel testbridge environment variable for fail fast
|
||||
BAZEL_FAIL_FAST_ENV_VAR = 'TESTBRIDGE_TEST_RUNNER_FAIL_FAST'
|
||||
|
||||
# The environment variable for specifying fail fast.
|
||||
FAIL_FAST_ENV_VAR = 'GTEST_FAIL_FAST'
|
||||
|
||||
# The command line flag for specifying fail fast.
|
||||
FAIL_FAST_FLAG = 'gtest_fail_fast'
|
||||
|
||||
# The command line flag to run disabled tests.
|
||||
RUN_DISABLED_FLAG = 'gtest_also_run_disabled_tests'
|
||||
|
||||
# The command line flag for specifying a filter.
|
||||
FILTER_FLAG = 'gtest_filter'
|
||||
|
||||
# Command to run the googletest-failfast-unittest_ program.
|
||||
COMMAND = gtest_test_utils.GetTestExecutablePath(
|
||||
'googletest-failfast-unittest_'
|
||||
)
|
||||
|
||||
# The command line flag to tell Google Test to output the list of tests it
|
||||
# will run.
|
||||
LIST_TESTS_FLAG = '--gtest_list_tests'
|
||||
|
||||
# Indicates whether Google Test supports death tests.
|
||||
SUPPORTS_DEATH_TESTS = (
|
||||
'HasDeathTest'
|
||||
in gtest_test_utils.Subprocess([COMMAND, LIST_TESTS_FLAG]).output
|
||||
)
|
||||
|
||||
# Utilities.
|
||||
|
||||
environ = os.environ.copy()
|
||||
|
||||
|
||||
def SetEnvVar(env_var, value):
|
||||
"""Sets the env variable to 'value'; unsets it when 'value' is None."""
|
||||
|
||||
if value is not None:
|
||||
environ[env_var] = value
|
||||
elif env_var in environ:
|
||||
del environ[env_var]
|
||||
|
||||
|
||||
def RunAndReturnOutput(test_suite=None, fail_fast=None, run_disabled=False):
|
||||
"""Runs the test program and returns its output."""
|
||||
|
||||
args = []
|
||||
xml_path = os.path.join(
|
||||
gtest_test_utils.GetTempDir(), '.GTestFailFastUnitTest.xml'
|
||||
)
|
||||
args += ['--gtest_output=xml:' + xml_path]
|
||||
if fail_fast is not None:
|
||||
if isinstance(fail_fast, str):
|
||||
args += ['--%s=%s' % (FAIL_FAST_FLAG, fail_fast)]
|
||||
elif fail_fast:
|
||||
args += ['--%s' % FAIL_FAST_FLAG]
|
||||
else:
|
||||
args += ['--no%s' % FAIL_FAST_FLAG]
|
||||
if test_suite:
|
||||
args += ['--%s=%s.*' % (FILTER_FLAG, test_suite)]
|
||||
if run_disabled:
|
||||
args += ['--%s' % RUN_DISABLED_FLAG]
|
||||
txt_out = gtest_test_utils.Subprocess([COMMAND] + args, env=environ).output
|
||||
with open(xml_path) as xml_file:
|
||||
return txt_out, xml_file.read()
|
||||
|
||||
|
||||
# The unit test.
|
||||
class GTestFailFastUnitTest(gtest_test_utils.TestCase):
|
||||
"""Tests the env variable or the command line flag for fail_fast."""
|
||||
|
||||
def testDefaultBehavior(self):
|
||||
"""Tests the behavior of not specifying the fail_fast."""
|
||||
|
||||
txt, _ = RunAndReturnOutput()
|
||||
self.assertIn('22 FAILED TEST', txt)
|
||||
|
||||
def testGoogletestFlag(self):
|
||||
txt, _ = RunAndReturnOutput(test_suite='HasSimpleTest', fail_fast=True)
|
||||
self.assertIn('1 FAILED TEST', txt)
|
||||
self.assertIn('[ SKIPPED ] 3 tests', txt)
|
||||
|
||||
txt, _ = RunAndReturnOutput(test_suite='HasSimpleTest', fail_fast=False)
|
||||
self.assertIn('4 FAILED TEST', txt)
|
||||
self.assertNotIn('[ SKIPPED ]', txt)
|
||||
|
||||
def testGoogletestEnvVar(self):
|
||||
"""Tests the behavior of specifying fail_fast via Googletest env var."""
|
||||
|
||||
try:
|
||||
SetEnvVar(FAIL_FAST_ENV_VAR, '1')
|
||||
txt, _ = RunAndReturnOutput('HasSimpleTest')
|
||||
self.assertIn('1 FAILED TEST', txt)
|
||||
self.assertIn('[ SKIPPED ] 3 tests', txt)
|
||||
|
||||
SetEnvVar(FAIL_FAST_ENV_VAR, '0')
|
||||
txt, _ = RunAndReturnOutput('HasSimpleTest')
|
||||
self.assertIn('4 FAILED TEST', txt)
|
||||
self.assertNotIn('[ SKIPPED ]', txt)
|
||||
finally:
|
||||
SetEnvVar(FAIL_FAST_ENV_VAR, None)
|
||||
|
||||
def testBazelEnvVar(self):
|
||||
"""Tests the behavior of specifying fail_fast via Bazel testbridge."""
|
||||
|
||||
try:
|
||||
SetEnvVar(BAZEL_FAIL_FAST_ENV_VAR, '1')
|
||||
txt, _ = RunAndReturnOutput('HasSimpleTest')
|
||||
self.assertIn('1 FAILED TEST', txt)
|
||||
self.assertIn('[ SKIPPED ] 3 tests', txt)
|
||||
|
||||
SetEnvVar(BAZEL_FAIL_FAST_ENV_VAR, '0')
|
||||
txt, _ = RunAndReturnOutput('HasSimpleTest')
|
||||
self.assertIn('4 FAILED TEST', txt)
|
||||
self.assertNotIn('[ SKIPPED ]', txt)
|
||||
finally:
|
||||
SetEnvVar(BAZEL_FAIL_FAST_ENV_VAR, None)
|
||||
|
||||
def testFlagOverridesEnvVar(self):
|
||||
"""Tests precedence of flag over env var."""
|
||||
|
||||
try:
|
||||
SetEnvVar(FAIL_FAST_ENV_VAR, '0')
|
||||
txt, _ = RunAndReturnOutput('HasSimpleTest', True)
|
||||
self.assertIn('1 FAILED TEST', txt)
|
||||
self.assertIn('[ SKIPPED ] 3 tests', txt)
|
||||
finally:
|
||||
SetEnvVar(FAIL_FAST_ENV_VAR, None)
|
||||
|
||||
def testGoogletestEnvVarOverridesBazelEnvVar(self):
|
||||
"""Tests that the Googletest native env var over Bazel testbridge."""
|
||||
|
||||
try:
|
||||
SetEnvVar(BAZEL_FAIL_FAST_ENV_VAR, '0')
|
||||
SetEnvVar(FAIL_FAST_ENV_VAR, '1')
|
||||
txt, _ = RunAndReturnOutput('HasSimpleTest')
|
||||
self.assertIn('1 FAILED TEST', txt)
|
||||
self.assertIn('[ SKIPPED ] 3 tests', txt)
|
||||
finally:
|
||||
SetEnvVar(FAIL_FAST_ENV_VAR, None)
|
||||
SetEnvVar(BAZEL_FAIL_FAST_ENV_VAR, None)
|
||||
|
||||
def testEventListener(self):
|
||||
txt, _ = RunAndReturnOutput(test_suite='HasSkipTest', fail_fast=True)
|
||||
self.assertIn('1 FAILED TEST', txt)
|
||||
self.assertIn('[ SKIPPED ] 3 tests', txt)
|
||||
for expected_count, callback in [
|
||||
(1, 'OnTestSuiteStart'),
|
||||
(5, 'OnTestStart'),
|
||||
(5, 'OnTestEnd'),
|
||||
(5, 'OnTestPartResult'),
|
||||
(1, 'OnTestSuiteEnd'),
|
||||
]:
|
||||
self.assertEqual(
|
||||
expected_count,
|
||||
txt.count(callback),
|
||||
'Expected %d calls to callback %s match count on output: %s '
|
||||
% (expected_count, callback, txt),
|
||||
)
|
||||
|
||||
txt, _ = RunAndReturnOutput(test_suite='HasSkipTest', fail_fast=False)
|
||||
self.assertIn('3 FAILED TEST', txt)
|
||||
self.assertIn('[ SKIPPED ] 1 test', txt)
|
||||
for expected_count, callback in [
|
||||
(1, 'OnTestSuiteStart'),
|
||||
(5, 'OnTestStart'),
|
||||
(5, 'OnTestEnd'),
|
||||
(5, 'OnTestPartResult'),
|
||||
(1, 'OnTestSuiteEnd'),
|
||||
]:
|
||||
self.assertEqual(
|
||||
expected_count,
|
||||
txt.count(callback),
|
||||
'Expected %d calls to callback %s match count on output: %s '
|
||||
% (expected_count, callback, txt),
|
||||
)
|
||||
|
||||
def assertXmlResultCount(self, result, count, xml):
|
||||
self.assertEqual(
|
||||
count,
|
||||
xml.count('result="%s"' % result),
|
||||
'Expected \'result="%s"\' match count of %s: %s '
|
||||
% (result, count, xml),
|
||||
)
|
||||
|
||||
def assertXmlStatusCount(self, status, count, xml):
|
||||
self.assertEqual(
|
||||
count,
|
||||
xml.count('status="%s"' % status),
|
||||
'Expected \'status="%s"\' match count of %s: %s '
|
||||
% (status, count, xml),
|
||||
)
|
||||
|
||||
def assertFailFastXmlAndTxtOutput(
|
||||
self,
|
||||
fail_fast,
|
||||
test_suite,
|
||||
passed_count,
|
||||
failure_count,
|
||||
skipped_count,
|
||||
suppressed_count,
|
||||
run_disabled=False,
|
||||
):
|
||||
"""Assert XML and text output of a test execution."""
|
||||
|
||||
txt, xml = RunAndReturnOutput(test_suite, fail_fast, run_disabled)
|
||||
if failure_count > 0:
|
||||
self.assertIn('%s FAILED TEST' % failure_count, txt)
|
||||
if suppressed_count > 0:
|
||||
self.assertIn('%s DISABLED TEST' % suppressed_count, txt)
|
||||
if skipped_count > 0:
|
||||
self.assertIn('[ SKIPPED ] %s tests' % skipped_count, txt)
|
||||
self.assertXmlStatusCount(
|
||||
'run', passed_count + failure_count + skipped_count, xml
|
||||
)
|
||||
self.assertXmlStatusCount('notrun', suppressed_count, xml)
|
||||
self.assertXmlResultCount('completed', passed_count + failure_count, xml)
|
||||
self.assertXmlResultCount('skipped', skipped_count, xml)
|
||||
self.assertXmlResultCount('suppressed', suppressed_count, xml)
|
||||
|
||||
def assertFailFastBehavior(
|
||||
self,
|
||||
test_suite,
|
||||
passed_count,
|
||||
failure_count,
|
||||
skipped_count,
|
||||
suppressed_count,
|
||||
run_disabled=False,
|
||||
):
|
||||
"""Assert --fail_fast via flag."""
|
||||
|
||||
for fail_fast in ('true', '1', 't', True):
|
||||
self.assertFailFastXmlAndTxtOutput(
|
||||
fail_fast,
|
||||
test_suite,
|
||||
passed_count,
|
||||
failure_count,
|
||||
skipped_count,
|
||||
suppressed_count,
|
||||
run_disabled,
|
||||
)
|
||||
|
||||
def assertNotFailFastBehavior(
|
||||
self,
|
||||
test_suite,
|
||||
passed_count,
|
||||
failure_count,
|
||||
skipped_count,
|
||||
suppressed_count,
|
||||
run_disabled=False,
|
||||
):
|
||||
"""Assert --nofail_fast via flag."""
|
||||
|
||||
for fail_fast in ('false', '0', 'f', False):
|
||||
self.assertFailFastXmlAndTxtOutput(
|
||||
fail_fast,
|
||||
test_suite,
|
||||
passed_count,
|
||||
failure_count,
|
||||
skipped_count,
|
||||
suppressed_count,
|
||||
run_disabled,
|
||||
)
|
||||
|
||||
def testFlag_HasFixtureTest(self):
|
||||
"""Tests the behavior of fail_fast and TEST_F."""
|
||||
self.assertFailFastBehavior(
|
||||
test_suite='HasFixtureTest',
|
||||
passed_count=1,
|
||||
failure_count=1,
|
||||
skipped_count=3,
|
||||
suppressed_count=0,
|
||||
)
|
||||
self.assertNotFailFastBehavior(
|
||||
test_suite='HasFixtureTest',
|
||||
passed_count=1,
|
||||
failure_count=4,
|
||||
skipped_count=0,
|
||||
suppressed_count=0,
|
||||
)
|
||||
|
||||
def testFlag_HasSimpleTest(self):
|
||||
"""Tests the behavior of fail_fast and TEST."""
|
||||
self.assertFailFastBehavior(
|
||||
test_suite='HasSimpleTest',
|
||||
passed_count=1,
|
||||
failure_count=1,
|
||||
skipped_count=3,
|
||||
suppressed_count=0,
|
||||
)
|
||||
self.assertNotFailFastBehavior(
|
||||
test_suite='HasSimpleTest',
|
||||
passed_count=1,
|
||||
failure_count=4,
|
||||
skipped_count=0,
|
||||
suppressed_count=0,
|
||||
)
|
||||
|
||||
def testFlag_HasParametersTest(self):
|
||||
"""Tests the behavior of fail_fast and TEST_P."""
|
||||
self.assertFailFastBehavior(
|
||||
test_suite='HasParametersSuite/HasParametersTest',
|
||||
passed_count=0,
|
||||
failure_count=1,
|
||||
skipped_count=3,
|
||||
suppressed_count=0,
|
||||
)
|
||||
self.assertNotFailFastBehavior(
|
||||
test_suite='HasParametersSuite/HasParametersTest',
|
||||
passed_count=0,
|
||||
failure_count=4,
|
||||
skipped_count=0,
|
||||
suppressed_count=0,
|
||||
)
|
||||
|
||||
def testFlag_HasDisabledTest(self):
|
||||
"""Tests the behavior of fail_fast and Disabled test cases."""
|
||||
self.assertFailFastBehavior(
|
||||
test_suite='HasDisabledTest',
|
||||
passed_count=1,
|
||||
failure_count=1,
|
||||
skipped_count=2,
|
||||
suppressed_count=1,
|
||||
run_disabled=False,
|
||||
)
|
||||
self.assertNotFailFastBehavior(
|
||||
test_suite='HasDisabledTest',
|
||||
passed_count=1,
|
||||
failure_count=3,
|
||||
skipped_count=0,
|
||||
suppressed_count=1,
|
||||
run_disabled=False,
|
||||
)
|
||||
|
||||
def testFlag_HasDisabledRunDisabledTest(self):
|
||||
"""Tests the behavior of fail_fast and Disabled test cases enabled."""
|
||||
self.assertFailFastBehavior(
|
||||
test_suite='HasDisabledTest',
|
||||
passed_count=1,
|
||||
failure_count=1,
|
||||
skipped_count=3,
|
||||
suppressed_count=0,
|
||||
run_disabled=True,
|
||||
)
|
||||
self.assertNotFailFastBehavior(
|
||||
test_suite='HasDisabledTest',
|
||||
passed_count=1,
|
||||
failure_count=4,
|
||||
skipped_count=0,
|
||||
suppressed_count=0,
|
||||
run_disabled=True,
|
||||
)
|
||||
|
||||
def testFlag_HasDisabledSuiteTest(self):
|
||||
"""Tests the behavior of fail_fast and Disabled test suites."""
|
||||
self.assertFailFastBehavior(
|
||||
test_suite='DISABLED_HasDisabledSuite',
|
||||
passed_count=0,
|
||||
failure_count=0,
|
||||
skipped_count=0,
|
||||
suppressed_count=5,
|
||||
run_disabled=False,
|
||||
)
|
||||
self.assertNotFailFastBehavior(
|
||||
test_suite='DISABLED_HasDisabledSuite',
|
||||
passed_count=0,
|
||||
failure_count=0,
|
||||
skipped_count=0,
|
||||
suppressed_count=5,
|
||||
run_disabled=False,
|
||||
)
|
||||
|
||||
def testFlag_HasDisabledSuiteRunDisabledTest(self):
|
||||
"""Tests the behavior of fail_fast and Disabled test suites enabled."""
|
||||
self.assertFailFastBehavior(
|
||||
test_suite='DISABLED_HasDisabledSuite',
|
||||
passed_count=1,
|
||||
failure_count=1,
|
||||
skipped_count=3,
|
||||
suppressed_count=0,
|
||||
run_disabled=True,
|
||||
)
|
||||
self.assertNotFailFastBehavior(
|
||||
test_suite='DISABLED_HasDisabledSuite',
|
||||
passed_count=1,
|
||||
failure_count=4,
|
||||
skipped_count=0,
|
||||
suppressed_count=0,
|
||||
run_disabled=True,
|
||||
)
|
||||
|
||||
if SUPPORTS_DEATH_TESTS:
|
||||
|
||||
def testFlag_HasDeathTest(self):
|
||||
"""Tests the behavior of fail_fast and death tests."""
|
||||
self.assertFailFastBehavior(
|
||||
test_suite='HasDeathTest',
|
||||
passed_count=1,
|
||||
failure_count=1,
|
||||
skipped_count=3,
|
||||
suppressed_count=0,
|
||||
)
|
||||
self.assertNotFailFastBehavior(
|
||||
test_suite='HasDeathTest',
|
||||
passed_count=1,
|
||||
failure_count=4,
|
||||
skipped_count=0,
|
||||
suppressed_count=0,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
gtest_test_utils.Main()
|
||||
Reference in New Issue
Block a user