TestFinder.java revision 6:5a1b0714df0e
1155093Smarius/* 2155093Smarius * Copyright (c) 2010, 2013, Oracle and/or its affiliates. All rights reserved. 3155093Smarius * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4155093Smarius * 5155093Smarius * This code is free software; you can redistribute it and/or modify it 6155093Smarius * under the terms of the GNU General Public License version 2 only, as 7155093Smarius * published by the Free Software Foundation. Oracle designates this 8155093Smarius * particular file as subject to the "Classpath" exception as provided 9155093Smarius * by Oracle in the LICENSE file that accompanied this code. 10155093Smarius * 11155093Smarius * This code is distributed in the hope that it will be useful, but WITHOUT 12155093Smarius * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13155093Smarius * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 14155093Smarius * version 2 for more details (a copy is included in the LICENSE file that 15155093Smarius * accompanied this code). 16155093Smarius * 17155093Smarius * You should have received a copy of the GNU General Public License version 18155093Smarius * 2 along with this work; if not, write to the Free Software Foundation, 19155093Smarius * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 20155093Smarius * 21155093Smarius * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 22155093Smarius * or visit www.oracle.com if you need additional information or have any 23155093Smarius * questions. 24155093Smarius */ 25155093Smarius 26155093Smariuspackage jdk.nashorn.internal.test.framework; 27155093Smarius 28155093Smariusimport static jdk.nashorn.internal.test.framework.TestConfig.OPTIONS_CHECK_COMPILE_MSG; 29155093Smariusimport static jdk.nashorn.internal.test.framework.TestConfig.OPTIONS_COMPARE; 30155093Smariusimport static jdk.nashorn.internal.test.framework.TestConfig.OPTIONS_EXPECT_COMPILE_FAIL; 31155093Smariusimport static jdk.nashorn.internal.test.framework.TestConfig.OPTIONS_EXPECT_RUN_FAIL; 32155093Smariusimport static jdk.nashorn.internal.test.framework.TestConfig.OPTIONS_IGNORE_STD_ERROR; 33155093Smariusimport static jdk.nashorn.internal.test.framework.TestConfig.OPTIONS_RUN; 34155093Smariusimport static jdk.nashorn.internal.test.framework.TestConfig.TEST_JS_ENABLE_STRICT_MODE; 35155093Smariusimport static jdk.nashorn.internal.test.framework.TestConfig.TEST_JS_EXCLUDES_FILE; 36155093Smariusimport static jdk.nashorn.internal.test.framework.TestConfig.TEST_JS_EXCLUDE_DIR; 37155093Smariusimport static jdk.nashorn.internal.test.framework.TestConfig.TEST_JS_EXCLUDE_LIST; 38155093Smariusimport static jdk.nashorn.internal.test.framework.TestConfig.TEST_JS_FRAMEWORK; 39155093Smariusimport static jdk.nashorn.internal.test.framework.TestConfig.TEST_JS_INCLUDES; 40155093Smariusimport static jdk.nashorn.internal.test.framework.TestConfig.TEST_JS_LIST; 41155093Smariusimport static jdk.nashorn.internal.test.framework.TestConfig.TEST_JS_ROOTS; 42155093Smariusimport static jdk.nashorn.internal.test.framework.TestConfig.TEST_JS_UNCHECKED_DIR; 43155093Smarius 44155093Smariusimport java.io.File; 45155093Smariusimport java.io.IOException; 46155093Smariusimport java.nio.file.FileSystem; 47155093Smariusimport java.nio.file.FileSystems; 48155093Smariusimport java.nio.file.FileVisitOption; 49155093Smariusimport java.nio.file.FileVisitResult; 50155093Smariusimport java.nio.file.Files; 51155093Smariusimport java.nio.file.Path; 52155093Smariusimport java.nio.file.SimpleFileVisitor; 53155093Smariusimport java.nio.file.attribute.BasicFileAttributes; 54155093Smariusimport java.util.ArrayList; 55155093Smariusimport java.util.Collections; 56155093Smariusimport java.util.EnumSet; 57155093Smariusimport java.util.HashMap; 58155093Smariusimport java.util.HashSet; 59155093Smariusimport java.util.List; 60155093Smariusimport java.util.Map; 61155093Smariusimport java.util.Scanner; 62155093Smariusimport java.util.Set; 63155093Smariusimport javax.xml.xpath.XPath; 64155093Smariusimport javax.xml.xpath.XPathConstants; 65155093Smariusimport javax.xml.xpath.XPathExpressionException; 66155093Smariusimport javax.xml.xpath.XPathFactory; 67155093Smariusimport org.w3c.dom.NodeList; 68155093Smariusimport org.xml.sax.InputSource; 69155093Smarius 70155093Smarius/** 71155093Smarius * Utility class to find/parse script test files and to create 'test' instances. 72155093Smarius * Actual 'test' object type is decided by clients of this class. 73155093Smarius */ 74155093Smariusfinal class TestFinder { 75155093Smarius private TestFinder() {} 76155093Smarius 77155093Smarius interface TestFactory<T> { 78155093Smarius // 'test' instance type is decided by the client. 79155093Smarius T createTest(final String framework, final File testFile, final List<String> engineOptions, final Map<String, String> testOptions, final List<String> arguments); 80155093Smarius // place to log messages from TestFinder 81155093Smarius void log(String mg); 82155093Smarius } 83155093Smarius 84155093Smarius 85155093Smarius // finds all tests from configuration and calls TestFactory to create 'test' instance for each script test found 86155093Smarius static <T> void findAllTests(final List<T> tests, final Set<String> orphans, final TestFactory<T> testFactory) throws Exception { 87155093Smarius final String framework = System.getProperty(TEST_JS_FRAMEWORK); 88155093Smarius final String testList = System.getProperty(TEST_JS_LIST); 89155093Smarius if (testList == null || testList.length() == 0) { 90155093Smarius // Run the tests under the test roots dir, selected by the 91155093Smarius // TEST_JS_INCLUDES patterns 92155093Smarius final String testRootsString = System.getProperty(TEST_JS_ROOTS, "test/script"); 93155093Smarius if (testRootsString == null || testRootsString.length() == 0) { 94155093Smarius throw new Exception("Error: " + TEST_JS_ROOTS + " must be set"); 95155093Smarius } 96155093Smarius final String testRoots[] = testRootsString.split(" "); 97155093Smarius final FileSystem fileSystem = FileSystems.getDefault(); 98155093Smarius final Set<String> testExcludeSet = getExcludeSet(); 99155093Smarius final Path[] excludePaths = getExcludeDirs(); 100155093Smarius for (final String root : testRoots) { 101155093Smarius final Path dir = fileSystem.getPath(root); 102155093Smarius findTests(framework, dir, tests, orphans, excludePaths, testExcludeSet, testFactory); 103155093Smarius } 104155093Smarius } else { 105155093Smarius // TEST_JS_LIST contains a blank speparated list of test file names. 106155093Smarius final String strArray[] = testList.split(" "); 107155093Smarius for (final String ss : strArray) { 108155093Smarius handleOneTest(framework, new File(ss).toPath(), tests, orphans, testFactory); 109155093Smarius } 110155093Smarius } 111155093Smarius } 112155093Smarius 113155093Smarius private static boolean inExcludePath(final Path file, final Path[] excludePaths) { 114155093Smarius if (excludePaths == null) { 115155093Smarius return false; 116155093Smarius } 117155093Smarius 118155093Smarius for (final Path excludePath : excludePaths) { 119155093Smarius if (file.startsWith(excludePath)) { 120155093Smarius return true; 121155093Smarius } 122155093Smarius } 123155093Smarius return false; 124155093Smarius } 125155093Smarius 126155093Smarius private static <T> void findTests(final String framework, final Path dir, final List<T> tests, final Set<String> orphanFiles, final Path[] excludePaths, final Set<String> excludedTests, final TestFactory<T> factory) throws Exception { 127155093Smarius final String pattern = System.getProperty(TEST_JS_INCLUDES); 128155093Smarius final String extension = pattern == null ? "js" : pattern; 129155093Smarius final Exception[] exceptions = new Exception[1]; 130155093Smarius final List<String> excludedActualTests = new ArrayList<>(); 131155093Smarius 132155093Smarius if (! dir.toFile().isDirectory()) { 133155093Smarius factory.log("WARNING: " + dir + " not found or not a directory"); 134155093Smarius } 135155093Smarius 136155093Smarius Files.walkFileTree(dir, EnumSet.of(FileVisitOption.FOLLOW_LINKS), Integer.MAX_VALUE, new SimpleFileVisitor<Path>() { 137155093Smarius @Override 138155093Smarius public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException { 139155093Smarius final String fileName = file.getName(file.getNameCount() - 1).toString(); 140155093Smarius if (fileName.endsWith(extension)) { 141155093Smarius final String namex = file.toString().replace('\\', '/'); 142155093Smarius if (!inExcludePath(file, excludePaths) && !excludedTests.contains(file.getFileName().toString())) { 143155093Smarius try { 144155093Smarius handleOneTest(framework, file, tests, orphanFiles, factory); 145155093Smarius } catch (final Exception ex) { 146155093Smarius exceptions[0] = ex; 147155093Smarius return FileVisitResult.TERMINATE; 148155093Smarius } 149155093Smarius } else { 150155093Smarius excludedActualTests.add(namex); 151155093Smarius } 152155093Smarius } 153155093Smarius return FileVisitResult.CONTINUE; 154155093Smarius } 155155093Smarius }); 156155093Smarius Collections.sort(excludedActualTests); 157155093Smarius 158155093Smarius for (final String excluded : excludedActualTests) { 159155093Smarius factory.log("Excluding " + excluded); 160155093Smarius } 161155093Smarius 162155093Smarius if (exceptions[0] != null) { 163155093Smarius throw exceptions[0]; 164155093Smarius } 165155093Smarius } 166155093Smarius 167155093Smarius private static final String uncheckedDirs[] = System.getProperty(TEST_JS_UNCHECKED_DIR, "test/script/external/test262/").split(" "); 168155093Smarius 169155093Smarius private static boolean isUnchecked(final Path testFile) { 170155093Smarius for (final String uncheckedDir : uncheckedDirs) { 171155093Smarius if (testFile.startsWith(uncheckedDir)) { 172155093Smarius return true; 173155093Smarius } 174155093Smarius } 175155093Smarius return false; 176155093Smarius } 177155093Smarius 178155093Smarius private static <T> void handleOneTest(final String framework, final Path testFile, final List<T> tests, final Set<String> orphans, TestFactory<T> factory) throws Exception { 179155093Smarius final String name = testFile.getFileName().toString(); 180155093Smarius 181155093Smarius assert name.lastIndexOf(".js") > 0 : "not a JavaScript: " + name; 182155093Smarius 183155093Smarius // defaults: testFile is a test and should be run 184155093Smarius boolean isTest = isUnchecked(testFile); 185155093Smarius boolean isNotTest = false; 186155093Smarius boolean shouldRun = true; 187155093Smarius boolean compileFailure = false; 188155093Smarius boolean runFailure = false; 189155093Smarius boolean checkCompilerMsg = false; 190155093Smarius boolean noCompare = false; 191155093Smarius boolean ignoreStdError = false; 192155093Smarius 193155093Smarius final List<String> engineOptions = new ArrayList<>(); 194155093Smarius final List<String> scriptArguments = new ArrayList<>(); 195155093Smarius boolean inComment = false; 196155093Smarius 197155093Smarius try (Scanner scanner = new Scanner(testFile)) { 198155093Smarius while (scanner.hasNext()) { 199155093Smarius // TODO: Scan for /ref=file qualifiers, etc, to determine run 200155093Smarius // behavior 201155093Smarius String token = scanner.next(); 202155093Smarius if (token.startsWith("/*")) { 203155093Smarius inComment = true; 204155093Smarius } else if (token.endsWith(("*/"))) { 205155093Smarius inComment = false; 206155093Smarius } else if (!inComment) { 207155093Smarius continue; 208155093Smarius } 209155093Smarius 210155093Smarius // remove whitespace and trailing semicolons, if any 211155093Smarius // (trailing semicolons are found in some sputnik tests) 212155093Smarius token = token.trim(); 213155093Smarius final int semicolon = token.indexOf(';'); 214155093Smarius if (semicolon > 0) { 215155093Smarius token = token.substring(0, semicolon); 216155093Smarius } 217155093Smarius switch (token) { 218155093Smarius case "@test": 219155093Smarius isTest = true; 220155093Smarius break; 221155093Smarius case "@test/fail": 222155093Smarius isTest = true; 223155093Smarius compileFailure = true; 224155093Smarius break; 225155093Smarius case "@test/compile-error": 226155093Smarius isTest = true; 227155093Smarius compileFailure = true; 228155093Smarius checkCompilerMsg = true; 229155093Smarius shouldRun = false; 230155093Smarius break; 231155093Smarius case "@test/warning": 232155093Smarius isTest = true; 233155093Smarius checkCompilerMsg = true; 234155093Smarius break; 235155093Smarius case "@test/nocompare": 236155093Smarius isTest = true; 237155093Smarius noCompare = true; 238155093Smarius break; 239155093Smarius case "@subtest": 240155093Smarius isTest = false; 241155093Smarius isNotTest = true; 242155093Smarius break; 243155093Smarius case "@runif": 244155093Smarius if (System.getProperty(scanner.next()) != null) { 245155093Smarius shouldRun = true; 246155093Smarius } else { 247155093Smarius isTest = false; 248155093Smarius isNotTest = true; 249155093Smarius } 250155093Smarius break; 251155093Smarius case "@run": 252155093Smarius shouldRun = true; 253155093Smarius break; 254155093Smarius case "@run/fail": 255155093Smarius shouldRun = true; 256155093Smarius runFailure = true; 257155093Smarius break; 258155093Smarius case "@run/ignore-std-error": 259155093Smarius shouldRun = true; 260155093Smarius ignoreStdError = true; 261155093Smarius break; 262155093Smarius case "@argument": 263155093Smarius scriptArguments.add(scanner.next()); 264155093Smarius break; 265155093Smarius case "@option": 266155093Smarius engineOptions.add(scanner.next()); 267155093Smarius break; 268155093Smarius } 269155093Smarius 270155093Smarius // negative tests are expected to fail at runtime only 271155093Smarius // for those tests that are expected to fail at compile time, 272155093Smarius // add @test/compile-error 273155093Smarius if (token.equals("@negative") || token.equals("@strict_mode_negative")) { 274155093Smarius shouldRun = true; 275155093Smarius runFailure = true; 276155093Smarius } 277155093Smarius 278155093Smarius if (token.equals("@strict_mode") || token.equals("@strict_mode_negative") || token.equals("@onlyStrict") || token.equals("@noStrict")) { 279155093Smarius if (!strictModeEnabled()) { 280155093Smarius return; 281155093Smarius } 282155093Smarius } 283155093Smarius } 284155093Smarius } catch (final Exception ignored) { 285155093Smarius return; 286155093Smarius } 287155093Smarius 288155093Smarius if (isTest) { 289155093Smarius final Map<String, String> testOptions = new HashMap<>(); 290155093Smarius if (compileFailure) { 291155093Smarius testOptions.put(OPTIONS_EXPECT_COMPILE_FAIL, "true"); 292155093Smarius } 293155093Smarius if (shouldRun) { 294155093Smarius testOptions.put(OPTIONS_RUN, "true"); 295155093Smarius } 296155093Smarius if (runFailure) { 297155093Smarius testOptions.put(OPTIONS_EXPECT_RUN_FAIL, "true"); 298155093Smarius } 299155093Smarius if (checkCompilerMsg) { 300155093Smarius testOptions.put(OPTIONS_CHECK_COMPILE_MSG, "true"); 301155093Smarius } 302155093Smarius if (!noCompare) { 303155093Smarius testOptions.put(OPTIONS_COMPARE, "true"); 304155093Smarius } 305155093Smarius if (ignoreStdError) { 306155093Smarius testOptions.put(OPTIONS_IGNORE_STD_ERROR, "true"); 307155093Smarius } 308155093Smarius 309155093Smarius tests.add(factory.createTest(framework, testFile.toFile(), engineOptions, testOptions, scriptArguments)); 310155093Smarius } else if (!isNotTest) { 311155093Smarius orphans.add(name); 312155093Smarius } 313155093Smarius } 314155093Smarius 315155093Smarius private static boolean strictModeEnabled() { 316155093Smarius return Boolean.getBoolean(TEST_JS_ENABLE_STRICT_MODE); 317155093Smarius } 318155093Smarius 319155093Smarius private static Set<String> getExcludeSet() throws XPathExpressionException { 320155093Smarius final String testExcludeList = System.getProperty(TEST_JS_EXCLUDE_LIST); 321155093Smarius 322155093Smarius String[] testExcludeArray = {}; 323155093Smarius if (testExcludeList != null) { 324155093Smarius testExcludeArray = testExcludeList.split(" "); 325155093Smarius } 326155093Smarius final Set<String> testExcludeSet = new HashSet<>(testExcludeArray.length); 327155093Smarius for (final String test : testExcludeArray) { 328155093Smarius testExcludeSet.add(test); 329155093Smarius } 330155093Smarius 331155093Smarius final String testExcludesFile = System.getProperty(TEST_JS_EXCLUDES_FILE); 332155093Smarius if (testExcludesFile != null && !testExcludesFile.isEmpty()) { 333155093Smarius try { 334155093Smarius loadExcludesFile(testExcludesFile, testExcludeSet); 335155093Smarius } catch (final XPathExpressionException e) { 336155093Smarius System.err.println("Error: unable to load test excludes from " + testExcludesFile); 337155093Smarius e.printStackTrace(); 338155093Smarius throw e; 339155093Smarius } 340155093Smarius } 341155093Smarius return testExcludeSet; 342155093Smarius } 343155093Smarius 344155093Smarius private static void loadExcludesFile(final String testExcludesFile, final Set<String> testExcludeSet) throws XPathExpressionException { 345155093Smarius final XPath xpath = XPathFactory.newInstance().newXPath(); 346155093Smarius final NodeList testIds = (NodeList)xpath.evaluate("/excludeList/test/@id", new InputSource(testExcludesFile), XPathConstants.NODESET); 347155093Smarius for (int i = testIds.getLength() - 1; i >= 0; i--) { 348155093Smarius testExcludeSet.add(testIds.item(i).getNodeValue()); 349155093Smarius } 350155093Smarius } 351155093Smarius 352155093Smarius private static Path[] getExcludeDirs() { 353155093Smarius final String excludeDirs[] = System.getProperty(TEST_JS_EXCLUDE_DIR, "test/script/currently-failing").split(" "); 354155093Smarius final Path[] excludePaths = new Path[excludeDirs.length]; 355155093Smarius final FileSystem fileSystem = FileSystems.getDefault(); 356155093Smarius int i = 0; 357155093Smarius for (final String excludeDir : excludeDirs) { 358155093Smarius excludePaths[i++] = fileSystem.getPath(excludeDir); 359155093Smarius } 360155093Smarius return excludePaths; 361155093Smarius } 362155093Smarius} 363155093Smarius