First Commit
This commit is contained in:
355
3rdparty/shaderc/glslc/test/glslc_test_framework.py
vendored
Executable file
355
3rdparty/shaderc/glslc/test/glslc_test_framework.py
vendored
Executable file
@@ -0,0 +1,355 @@
|
||||
#!/usr/bin/env python
|
||||
# Copyright 2015 The Shaderc Authors. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Manages and runs tests from the current working directory.
|
||||
|
||||
This will traverse the current working directory and look for python files that
|
||||
contain subclasses of GlslCTest.
|
||||
|
||||
If a class has an @inside_glslc_testsuite decorator, an instance of that
|
||||
class will be created and serve as a test case in that testsuite. The test
|
||||
case is then run by the following steps:
|
||||
|
||||
1. A temporary directory will be created.
|
||||
2. The glslc_args member variable will be inspected and all placeholders in it
|
||||
will be expanded by calling instantiate_for_glslc_args() on placeholders.
|
||||
The transformed list elements are then supplied as glslc arguments.
|
||||
3. If the environment member variable exists, its write() method will be
|
||||
invoked.
|
||||
4. All expected_* member variables will be inspected and all placeholders in
|
||||
them will be expanded by calling instantiate_for_expectation() on those
|
||||
placeholders. After placeholder expansion, if the expected_* variable is
|
||||
a list, its element will be joined together with '' to form a single
|
||||
string. These expected_* variables are to be used by the check_*() methods.
|
||||
5. glslc will be run with the arguments supplied in glslc_args.
|
||||
6. All check_*() member methods will be called by supplying a TestStatus as
|
||||
argument. Each check_*() method is expected to return a (Success, Message)
|
||||
pair where Success is a boolean indicating success and Message is an error
|
||||
message.
|
||||
7. If any check_*() method fails, the error message is outputted and the
|
||||
current test case fails.
|
||||
|
||||
If --leave-output was not specified, all temporary files and directories will
|
||||
be deleted.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import fnmatch
|
||||
import inspect
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from collections import defaultdict
|
||||
from placeholder import PlaceHolder
|
||||
|
||||
|
||||
EXPECTED_BEHAVIOR_PREFIX = 'expected_'
|
||||
VALIDATE_METHOD_PREFIX = 'check_'
|
||||
|
||||
|
||||
def get_all_variables(instance):
|
||||
"""Returns the names of all the variables in instance."""
|
||||
return [v for v in dir(instance) if not callable(getattr(instance, v))]
|
||||
|
||||
|
||||
def get_all_methods(instance):
|
||||
"""Returns the names of all methods in instance."""
|
||||
return [m for m in dir(instance) if callable(getattr(instance, m))]
|
||||
|
||||
|
||||
def get_all_superclasses(cls):
|
||||
"""Returns all superclasses of a given class. Omits root 'object' superclass.
|
||||
|
||||
Returns:
|
||||
A list of superclasses of the given class. The order guarantees that
|
||||
* A Base class precedes its derived classes, e.g., for "class B(A)", it
|
||||
will be [..., A, B, ...].
|
||||
* When there are multiple base classes, base classes declared first
|
||||
precede those declared later, e.g., for "class C(A, B), it will be
|
||||
[..., A, B, C, ...]
|
||||
"""
|
||||
classes = []
|
||||
for superclass in cls.__bases__:
|
||||
for c in get_all_superclasses(superclass):
|
||||
if c is not object and c not in classes:
|
||||
classes.append(c)
|
||||
for superclass in cls.__bases__:
|
||||
if superclass is not object and superclass not in classes:
|
||||
classes.append(superclass)
|
||||
return classes
|
||||
|
||||
|
||||
def get_all_test_methods(test_class):
|
||||
"""Gets all validation methods.
|
||||
|
||||
Returns:
|
||||
A list of validation methods. The order guarantees that
|
||||
* A method defined in superclass precedes one defined in subclass,
|
||||
e.g., for "class A(B)", methods defined in B precedes those defined
|
||||
in A.
|
||||
* If a subclass has more than one superclass, e.g., "class C(A, B)",
|
||||
then methods defined in A precedes those defined in B.
|
||||
"""
|
||||
classes = get_all_superclasses(test_class)
|
||||
classes.append(test_class)
|
||||
all_tests = [m for c in classes
|
||||
for m in get_all_methods(c)
|
||||
if m.startswith(VALIDATE_METHOD_PREFIX)]
|
||||
unique_tests = []
|
||||
for t in all_tests:
|
||||
if t not in unique_tests:
|
||||
unique_tests.append(t)
|
||||
return unique_tests
|
||||
|
||||
|
||||
class GlslCTest:
|
||||
"""Base class for glslc test cases.
|
||||
|
||||
Subclasses define test cases' facts (shader source code, glslc command,
|
||||
result validation), which will be used by the TestCase class for running
|
||||
tests. Subclasses should define glslc_args (specifying glslc command
|
||||
arguments), and at least one check_*() method (for result validation) for
|
||||
a full-fledged test case. All check_*() methods should take a TestStatus
|
||||
parameter and return a (Success, Message) pair, in which Success is a
|
||||
boolean indicating success and Message is an error message. The test passes
|
||||
iff all check_*() methods returns true.
|
||||
|
||||
Often, a test case class will delegate the check_* behaviors by inheriting
|
||||
from other classes.
|
||||
"""
|
||||
|
||||
def name(self):
|
||||
return self.__class__.__name__
|
||||
|
||||
|
||||
class TestStatus:
|
||||
"""A struct for holding run status of a test case."""
|
||||
|
||||
def __init__(self, test_manager, returncode, stdout, stderr, directory, inputs, input_filenames):
|
||||
self.test_manager = test_manager
|
||||
self.returncode = returncode
|
||||
self.stdout = stdout
|
||||
self.stderr = stderr
|
||||
# temporary directory where the test runs
|
||||
self.directory = directory
|
||||
# List of inputs, as PlaceHolder objects.
|
||||
self.inputs = inputs
|
||||
# the names of input shader files (potentially including paths)
|
||||
self.input_filenames = input_filenames
|
||||
|
||||
|
||||
class GlslCTestException(Exception):
|
||||
"""GlslCTest exception class."""
|
||||
pass
|
||||
|
||||
|
||||
def inside_glslc_testsuite(testsuite_name):
|
||||
"""Decorator for subclasses of GlslCTest.
|
||||
|
||||
This decorator checks that a class meets the requirements (see below)
|
||||
for a test case class, and then puts the class in a certain testsuite.
|
||||
* The class needs to be a subclass of GlslCTest.
|
||||
* The class needs to have glslc_args defined as a list.
|
||||
* The class needs to define at least one check_*() methods.
|
||||
* All expected_* variables required by check_*() methods can only be
|
||||
of bool, str, or list type.
|
||||
* Python runtime will throw an exception if the expected_* member
|
||||
attributes required by check_*() methods are missing.
|
||||
"""
|
||||
def actual_decorator(cls):
|
||||
if not inspect.isclass(cls):
|
||||
raise GlslCTestException('Test case should be a class')
|
||||
if not issubclass(cls, GlslCTest):
|
||||
raise GlslCTestException(
|
||||
'All test cases should be subclasses of GlslCTest')
|
||||
if 'glslc_args' not in get_all_variables(cls):
|
||||
raise GlslCTestException('No glslc_args found in the test case')
|
||||
if not isinstance(cls.glslc_args, list):
|
||||
raise GlslCTestException('glslc_args needs to be a list')
|
||||
if not any([
|
||||
m.startswith(VALIDATE_METHOD_PREFIX)
|
||||
for m in get_all_methods(cls)]):
|
||||
raise GlslCTestException(
|
||||
'No check_*() methods found in the test case')
|
||||
if not all([
|
||||
isinstance(v, (bool, str, list))
|
||||
for v in get_all_variables(cls)]):
|
||||
raise GlslCTestException(
|
||||
'expected_* variables are only allowed to be bool, str, or '
|
||||
'list type.')
|
||||
cls.parent_testsuite = testsuite_name
|
||||
return cls
|
||||
return actual_decorator
|
||||
|
||||
|
||||
class TestManager:
|
||||
"""Manages and runs a set of tests."""
|
||||
|
||||
def __init__(self, executable_path, disassembler_path):
|
||||
self.executable_path = executable_path
|
||||
self.disassembler_path = disassembler_path
|
||||
self.num_successes = 0
|
||||
self.num_failures = 0
|
||||
self.num_tests = 0
|
||||
self.leave_output = False
|
||||
self.tests = defaultdict(list)
|
||||
|
||||
def notify_result(self, test_case, success, message):
|
||||
"""Call this to notify the manager of the results of a test run."""
|
||||
self.num_successes += 1 if success else 0
|
||||
self.num_failures += 0 if success else 1
|
||||
counter_string = str(
|
||||
self.num_successes + self.num_failures) + '/' + str(self.num_tests)
|
||||
print('%-10s %-40s ' % (counter_string, test_case.test.name()) +
|
||||
('Passed' if success else '-Failed-'))
|
||||
if not success:
|
||||
print(' '.join(test_case.command))
|
||||
print(message)
|
||||
|
||||
def add_test(self, testsuite, test):
|
||||
"""Add this to the current list of test cases."""
|
||||
self.tests[testsuite].append(TestCase(test, self))
|
||||
self.num_tests += 1
|
||||
|
||||
def run_tests(self):
|
||||
for suite in self.tests:
|
||||
print('Glslc test suite: "{suite}"'.format(suite=suite))
|
||||
for x in self.tests[suite]:
|
||||
x.runTest()
|
||||
|
||||
|
||||
class TestCase:
|
||||
"""A single test case that runs in its own directory."""
|
||||
|
||||
def __init__(self, test, test_manager):
|
||||
self.test = test
|
||||
self.test_manager = test_manager
|
||||
self.inputs = [] # inputs, as PlaceHolder objects.
|
||||
self.file_shaders = [] # filenames of shader files.
|
||||
self.stdin_shader = None # text to be passed to glslc as stdin
|
||||
|
||||
def setUp(self):
|
||||
"""Creates environment and instantiates placeholders for the test case."""
|
||||
|
||||
self.directory = tempfile.mkdtemp(dir=os.getcwd())
|
||||
glslc_args = self.test.glslc_args
|
||||
# Instantiate placeholders in glslc_args
|
||||
self.test.glslc_args = [
|
||||
arg.instantiate_for_glslc_args(self)
|
||||
if isinstance(arg, PlaceHolder) else arg
|
||||
for arg in self.test.glslc_args]
|
||||
# Get all shader files' names
|
||||
self.inputs = [arg for arg in glslc_args if isinstance(arg, PlaceHolder)]
|
||||
self.file_shaders = [arg.filename for arg in self.inputs]
|
||||
|
||||
if 'environment' in get_all_variables(self.test):
|
||||
self.test.environment.write(self.directory)
|
||||
|
||||
expectations = [v for v in get_all_variables(self.test)
|
||||
if v.startswith(EXPECTED_BEHAVIOR_PREFIX)]
|
||||
# Instantiate placeholders in expectations
|
||||
for expectation_name in expectations:
|
||||
expectation = getattr(self.test, expectation_name)
|
||||
if isinstance(expectation, list):
|
||||
expanded_expections = [
|
||||
element.instantiate_for_expectation(self)
|
||||
if isinstance(element, PlaceHolder) else element
|
||||
for element in expectation]
|
||||
setattr(
|
||||
self.test, expectation_name,
|
||||
''.join(expanded_expections))
|
||||
elif isinstance(expectation, PlaceHolder):
|
||||
setattr(self.test, expectation_name,
|
||||
expectation.instantiate_for_expectation(self))
|
||||
|
||||
|
||||
def tearDown(self):
|
||||
"""Removes the directory if we were not instructed to do otherwise."""
|
||||
if not self.test_manager.leave_output:
|
||||
shutil.rmtree(self.directory)
|
||||
|
||||
def runTest(self):
|
||||
"""Sets up and runs a test, reports any failures and then cleans up."""
|
||||
self.setUp()
|
||||
success = False
|
||||
message = ''
|
||||
try:
|
||||
self.command = [self.test_manager.executable_path]
|
||||
self.command.extend(self.test.glslc_args)
|
||||
|
||||
process = subprocess.Popen(
|
||||
args=self.command, stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
||||
cwd=self.directory)
|
||||
output = process.communicate(self.stdin_shader)
|
||||
test_status = TestStatus(
|
||||
self.test_manager,
|
||||
process.returncode, output[0], output[1],
|
||||
self.directory, self.inputs, self.file_shaders)
|
||||
run_results = [getattr(self.test, test_method)(test_status)
|
||||
for test_method in get_all_test_methods(
|
||||
self.test.__class__)]
|
||||
success, message = list(zip(*run_results))
|
||||
success = all(success)
|
||||
message = '\n'.join(message)
|
||||
except Exception as e:
|
||||
success = False
|
||||
message = str(e)
|
||||
self.test_manager.notify_result(self, success, message)
|
||||
self.tearDown()
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('glslc', metavar='path/to/glslc', type=str, nargs=1,
|
||||
help='Path to glslc')
|
||||
parser.add_argument('spirvdis', metavar='path/to/glslc', type=str, nargs=1,
|
||||
help='Path to spirv-dis')
|
||||
parser.add_argument('--leave-output', action='store_const', const=1,
|
||||
help='Do not clean up temporary directories')
|
||||
parser.add_argument('--test-dir', nargs=1,
|
||||
help='Directory to gather the tests from')
|
||||
args = parser.parse_args()
|
||||
default_path = sys.path
|
||||
root_dir = os.getcwd()
|
||||
if args.test_dir:
|
||||
root_dir = args.test_dir[0]
|
||||
manager = TestManager(args.glslc[0], args.spirvdis[0])
|
||||
if args.leave_output:
|
||||
manager.leave_output = True
|
||||
for root, _, filenames in os.walk(root_dir):
|
||||
for filename in fnmatch.filter(filenames, '*.py'):
|
||||
if filename.endswith('unittest.py'):
|
||||
# Skip unit tests, which are for testing functions of
|
||||
# the test framework.
|
||||
continue
|
||||
sys.path = default_path
|
||||
sys.path.append(root)
|
||||
try:
|
||||
mod = __import__(os.path.splitext(filename)[0])
|
||||
for _, obj, in inspect.getmembers(mod):
|
||||
if inspect.isclass(obj) and hasattr(obj, 'parent_testsuite'):
|
||||
manager.add_test(obj.parent_testsuite, obj())
|
||||
except:
|
||||
print("Failed to load " + filename)
|
||||
raise
|
||||
manager.run_tests()
|
||||
if manager.num_failures > 0:
|
||||
sys.exit(-1)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user