OctaneTest.java revision 1088:7e62d98d4625 
1/*
2 * Copyright (c) 2010, 2013, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.  Oracle designates this
8 * particular file as subject to the "Classpath" exception as provided
9 * by Oracle in the LICENSE file that accompanied this code.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 */
25
26package jdk.nashorn.internal.performance;
27
28import java.io.BufferedInputStream;
29import java.io.BufferedReader;
30import java.io.ByteArrayInputStream;
31import java.io.ByteArrayOutputStream;
32import java.io.File;
33import java.io.IOException;
34import java.io.InputStream;
35import java.io.InputStreamReader;
36import java.io.PrintStream;
37import java.lang.reflect.Method;
38import java.net.URL;
39import java.text.NumberFormat;
40import java.util.List;
41import org.testng.annotations.Test;
42
43@SuppressWarnings("javadoc")
44public class OctaneTest {
45
46    @Test
47    public void box2DTest() {
48        genericTest("Box2D");
49    }
50
51    @Test
52    public void codeLoadTest() {
53        genericTest("Code-Load");
54    }
55
56    @Test
57    public void cryptoTest() {
58        genericTest("Crypto");
59    }
60
61    @Test
62    public void deltaBlueTest() {
63        genericTest("DeltaBlue");
64    }
65
66    @Test
67    public void earleyBoyerTest() {
68    genericTest("Earley-Boyer");
69    }
70
71    @Test
72    public void gbEMUTest() {
73        genericTest("GBEMU");
74    }
75
76/*    @Test
77    public void mandreelTest() {
78        genericTest("Mandreel");
79    }*/
80
81    @Test
82    public void navierStokesTest() {
83        genericTest("Navier-Stokes");
84    }
85
86    @Test
87    public void pdfJSTest() {
88        genericTest("PDFJS");
89    }
90
91    @Test
92    public void raytraceTest() {
93        genericTest("RayTrace");
94    }
95
96    @Test
97    public void regexpTest() {
98        genericTest("RegExp");
99    }
100
101    @Test
102    public void richardsTest() {
103        genericTest("Richards");
104    }
105
106    @Test
107    public void splayTest() {
108        genericTest("Splay");
109    }
110
111    @Test
112/*    public void typeScriptTest() {
113        genericTest("TypeScript");
114    }
115
116    @Test
117    public void zlibTest() {
118        genericTest("zlib");
119    }/*/
120
121    public void genericTest(final String benchmark) {
122        try {
123            final String mainScript      = "test/script/basic/run-octane.js";
124            final String benchmarkScript = "test/script/external/octane/benchmarks/" + benchmark.toLowerCase() + ".js";
125            final String[] args = {
126                "--",
127                benchmarkScript,
128                "--verbose"
129            };
130            final Double score = genericNashornTest(benchmark, mainScript, args);
131
132            Double rhinoScore = null;
133            if (checkRhinoPresence()) {
134                args[0]=mainScript; //rhino does not need this "--"
135                rhinoScore = genericRhinoTest(benchmark,  args, System.getProperty("rhino.jar"));
136            }
137
138            Double v8Score = null;
139            if (checkV8Presence()) {
140                v8Score = genericV8Test(benchmark, "test/script/basic/run-octane.js -- test/script/external/octane/benchmarks/" + benchmark.toLowerCase() + ".js --verbose");
141            }
142
143            generateOutput(benchmark.toLowerCase(), score, rhinoScore, v8Score);
144
145        } catch (final Throwable e) {
146            e.printStackTrace();
147        }
148    }
149
150    public Double genericNashornTest(final String benchmark, final String testPath, final String[] args) throws Throwable {
151        try {
152            final PerformanceWrapper wrapper = new PerformanceWrapper();
153
154            final ByteArrayOutputStream baos = new ByteArrayOutputStream();
155            final PrintStream ps = new PrintStream(baos);
156
157            final java.io.File test=new java.io.File(testPath);
158            final File absoluteFile=test.getAbsoluteFile();
159            @SuppressWarnings("deprecation")
160            final
161            URL testURL=absoluteFile.toURL();
162
163            wrapper.runExecuteOnlyTest(testPath, 0, 0, testURL.toString(), ps, System.err, args);
164
165            final byte[] output = baos.toByteArray();
166            final List<String> result = outputToStrings(output);
167
168            final Double _result = filterBenchmark(result, benchmark);
169
170            return _result;
171        } catch (final Throwable e) {
172            e.printStackTrace();
173            throw e;
174        }
175    }
176
177    public Double genericRhinoTest(final String benchmark, final String[] testPath, final String jarPath) throws Throwable {
178
179        final PrintStream systemOut = System.out;
180
181        try {
182            final ClassLoader loader = java.net.URLClassLoader.newInstance(new URL[] { new URL(jarPath) }, getClass().getClassLoader());
183            final Class<?> clazz = Class.forName("org.mozilla.javascript.tools.shell.Main", true, loader);
184
185            final Class<?>[] pars = { String[].class };
186            final Method mthd = clazz.getMethod("main", pars);
187
188            final ByteArrayOutputStream baos = new ByteArrayOutputStream();
189            final PrintStream ps = new PrintStream(baos);
190
191            System.setOut(ps);
192            //final String[] realArgs = testPath.split(" ");//{ testPath };
193            mthd.invoke(null, ((Object)testPath));
194            System.setOut(systemOut);
195
196            final byte[] output = baos.toByteArray();
197            final List<String> result = outputToStrings(output);
198            return filterBenchmark(result, benchmark);
199
200        } catch (final Throwable e) {
201            System.setOut(systemOut);
202            e.printStackTrace();
203            throw e;
204        }
205    }
206
207    public Double genericV8Test(final String benchmark, final String testPath) throws Throwable {
208
209        System.out.println("genericV8Test");
210        if (!checkV8Presence()) {
211            return null;
212        }
213        final String v8shell = System.getProperty("v8.shell.full.path");
214        final PrintStream systemOut = System.out;
215
216        try {
217
218            final Process process = Runtime.getRuntime().exec(v8shell + " " + testPath);
219            process.waitFor();
220            final InputStream processOut = process.getInputStream();
221            final BufferedInputStream bis = new BufferedInputStream(processOut);
222
223            final byte[] output = new byte[bis.available()];
224            bis.read(output, 0, bis.available());
225            final List<String> result = outputToStrings(output);
226            return filterBenchmark(result, benchmark);
227
228        } catch (final Throwable e) {
229            System.setOut(systemOut);
230            e.printStackTrace();
231            throw e;
232        }
233    }
234
235    protected List<String> outputToStrings(final byte[] output) throws IOException {
236        final ByteArrayInputStream bais = new ByteArrayInputStream(output);
237        final InputStreamReader reader = new InputStreamReader(bais);
238        final BufferedReader bufReader = new BufferedReader(reader);
239        final List<String> list = new java.util.LinkedList<>();
240        while (bufReader.ready()) {
241            list.add(bufReader.readLine());
242        }
243        return list;
244    }
245
246    protected Double filterBenchmark(final List<String> output, final String benchmarkName) throws Exception {
247        Double currentScore = 0.;
248        for (final String s : output) {
249            //if (s.trim().startsWith(benchmarkName)) {
250            if (s.trim().startsWith("Score")) {
251                final String[] split = s.split(":");
252                if (split.length != 2) {
253                    for (final String outString : output) {
254                        System.out.println("outString (score format)"+outString);
255                    }
256                    throw new IllegalArgumentException("Invalid benchmark output format");
257                }
258
259                final NumberFormat nf = NumberFormat.getInstance();
260                final Number _newCurrentScore = nf.parse(split[1].trim());
261                final Double newCurrentScore = _newCurrentScore.doubleValue();
262                if (currentScore < newCurrentScore) {
263                    currentScore = newCurrentScore;
264                }
265            }
266        }
267//        System.out.println("filterBenchmark current score:"+currentScore);
268        return currentScore;
269    }
270
271    void generateOutput(final String benchmark, final Double nashorn, final Double rhino, final Double v8) throws Exception {
272        Double nashornToRhino = null;
273        Double nashornToV8 = null;
274        if (rhino != null && rhino != 0) {
275            nashornToRhino = nashorn.doubleValue() / rhino.doubleValue();
276        }
277        if (v8 != null && rhino != 0) {
278            nashornToV8 = nashorn.doubleValue() / v8.doubleValue();
279        }
280        final String normalizedBenchmark=benchmark.replace("-", "");
281        System.out.println("benchmark-" + normalizedBenchmark + "-nashorn=" + nashorn);
282        AuroraWrapper.addResults(AuroraWrapper.createOrOpenDocument(), "benchmark-" + normalizedBenchmark + "-nashorn", nashorn.toString());
283
284        if (rhino != null) {
285            System.out.println("benchmark-" + normalizedBenchmark + "-rhino=" + rhino);
286            AuroraWrapper.addResults(AuroraWrapper.createOrOpenDocument(), "benchmark-" + normalizedBenchmark + "-rhino", rhino.toString());
287        }
288        if (v8 != null) {
289            AuroraWrapper.addResults(AuroraWrapper.createOrOpenDocument(), "benchmark-" + normalizedBenchmark + "-v8", v8.toString());
290        }
291        if (nashornToRhino != null) {
292            System.out.println("benchmark-" + normalizedBenchmark + "-nashorn-to-rhino=" + ((float)((int)(nashornToRhino * 100))) / 100);
293            AuroraWrapper.addResults(AuroraWrapper.createOrOpenDocument(), "benchmark-" + normalizedBenchmark + "-nashorn-to-rhino", "" + ((float)((int)(nashornToRhino * 100))) / 100);
294        }
295        if (nashornToV8 != null) {
296            System.out.println("benchmark-" + normalizedBenchmark + "-nashorn-to-v8=" + ((float)((int)(nashornToV8 * 100))) / 100);
297            AuroraWrapper.addResults(AuroraWrapper.createOrOpenDocument(), "benchmark-" + normalizedBenchmark + "-nashorn-to-v8", "" + ((float)((int)(nashornToV8 * 100))) / 100);
298        }
299    }
300
301    boolean checkRhinoPresence() {
302        final String rhinojar = System.getProperty("rhino.jar");
303        if (rhinojar != null) {
304            // System.out.println("Rhino jar found; performing comparison testing");
305            return true;
306        }
307        return false;
308    }
309
310    boolean checkV8Presence() {
311        final String v8shell = System.getProperty("v8.shell.full.path");
312        if (v8shell != null) {
313            // System.out.println("d8 found; performing comparison testing");
314            return true;
315        }
316        return false;
317    }
318
319}
320