Closed
Bug 450945
Opened 18 years ago
Closed 18 years ago
TM: sunspider results display incorrect with JIT
Categories
(Core :: JavaScript Engine, defect)
Tracking
()
RESOLVED
FIXED
People
(Reporter: shaver, Assigned: gal)
Details
(Keywords: testcase)
The category sums look right, but for bitops and after the individuals look pretty suspect! sunspider-compare looks right with the JIT on.
Heading to the airport now, can't really debug further atm.
============================================
RESULTS (means and 95% confidence intervals)
--------------------------------------------
Total: 1647.7ms +/- 0.2%
--------------------------------------------
[...]
access: 181.3ms +/- 0.3%
binary-trees: 45.2ms +/- 0.7%
fannkuch: 61.3ms +/- 0.6%
nbody: 59.2ms +/- 0.5%
nsieve: 15.6ms +/- 2.4%
bitops: 42.0ms +/- 1.1%
3bit-bits-in-byte: 0.1ms +/- 226.0%
bits-in-byte: 0.2ms +/- 226.0%
bitwise-and: 0.2ms +/- 226.0%
nsieve-bits: 0.1ms +/- 226.0%
controlflow: 33.5ms +/- 1.1%
recursive: 0.2ms +/- 226.0%
crypto: 96.8ms +/- 0.5%
aes: 0.2ms +/- 226.0%
md5: 0.1ms +/- 226.0%
sha1: 0.0ms +/- NaN%
date: 243.4ms +/- 0.2%
format-tofte: 0.1ms +/- 226.0%
format-xparb: 0.2ms +/- 226.0%
math: 82.8ms +/- 0.5%
cordic: 0.2ms +/- 226.0%
partial-sums: 0.1ms +/- 226.0%
spectral-norm: 0.1ms +/- 226.0%
regexp: 255.7ms +/- 0.2%
dna: 0.3ms +/- 226.0%
string: 516.4ms +/- 0.2%
base64: 0.1ms +/- 226.0%
fasta: 0.2ms +/- 226.0%
tagcloud: 0.2ms +/- 226.0%
unpack-code: 0.1ms +/- 226.0%
validate-input: 0.2ms +/- 226.0%
|
||
Comment 1•18 years ago
|
||
WFM. Maybe make clean?
|
Assignee | |
Comment 2•18 years ago
|
||
The sunspider analysis code uses some funny global along and off-path and only recently started working in the JIT. I wouldn't be entirely surprised if something is wrong.
|
|
Assignee | |
Comment 3•18 years ago
|
||
This just went into the tree and might be related:
http://hg.mozilla.org/index.cgi/tracemonkey/rev/6c1e358ce7a9
If an inner tree didn't know about a global at the side exit we are coming out of, but latter added it along another one (and modifies it there), and the outer tree uses the global after the tree call, it didn't read back the modified value (since it looked at the inner tree's exit typemap only and didn't merge in any types the inner tree picked up along the way).
|
|
Assignee | |
Comment 4•18 years ago
|
||
Some of the most recent changes require a clobber (see changelog). Shaver, can you try this again?
|
Reporter | |
Comment 5•18 years ago
|
||
Updated, clobbered, still see it.
1e456b6bceae
|
|
Assignee | |
Updated•18 years ago
|
Assignee: general → gal
|
|
Assignee | |
Comment 6•18 years ago
|
||
Isolated test case for the shell without actual benchmarking:
var tests = [ "3d-cube", "3d-morph", "3d-raytrace", "access-binary-trees", "access-fannkuch", "access-nbody", "access-nsieve", "bitops-3bit-bits-in-byte", "bitops-bits-in-byte", "bitops-bitwise-and", "bitops-nsieve-bits", "controlflow-recursive", "crypto-aes", "crypto-md5", "crypto-sha1", "date-format-tofte", "date-format-xparb", "math-cordic", "math-partial-sums", "math-spectral-norm", "regexp-dna", "string-base64", "string-fasta", "string-tagcloud", "string-unpack-code", "string-validate-input" ];
var categories = [ "3d", "access", "bitops", "controlflow", "crypto", "date", "math", "regexp", "string" ];
var output =
[{
"3d-cube": 46,
"3d-morph": 29,
"3d-raytrace": 56,
"access-binary-trees": 46,
"access-fannkuch": 59,
"access-nbody": 32,
"access-nsieve": 14,
"bitops-3bit-bits-in-byte": 2,
"bitops-bits-in-byte": 9,
"bitops-bitwise-and": 5,
"bitops-nsieve-bits": 26,
"controlflow-recursive": 32,
"crypto-aes": 30,
"crypto-md5": 25,
"crypto-sha1": 10,
"date-format-tofte": 113,
"date-format-xparb": 103,
"math-cordic": 21,
"math-partial-sums": 16,
"math-spectral-norm": 7,
"regexp-dna": 227,
"string-base64": 17,
"string-fasta": 122,
"string-tagcloud": 110,
"string-unpack-code": 174,
"string-validate-input": 43
}];
var count = output.length;
var itemTotals = {};
itemTotals.length = count;
var total = 0;
var categoryTotals = {};
var testTotalsByCategory = {};
var mean = 0;
var categoryMeans = {};
var testMeansByCategory = {};
var stdDev = 0;
var categoryStdDevs = {};
var testStdDevsByCategory = {};
var stdErr = 0;
var categoryStdErrs = {};
var testStdErrsByCategory = {};
function initialize()
{
itemTotals = {total: []};
for (var i = 0; i < categories.length; i++) {
var category = categories[i];
itemTotals[category] = [];
categoryTotals[category] = 0;
testTotalsByCategory[category] = {};
categoryMeans[category] = 0;
testMeansByCategory[category] = {};
categoryStdDevs[category] = 0;
testStdDevsByCategory[category] = {};
categoryStdErrs[category] = 0;
testStdErrsByCategory[category] = {};
}
for (var i = 0; i < tests.length; i++) {
var test = tests[i];
itemTotals[test] = [];
var category = test.replace(/-.*/, "");
testTotalsByCategory[category][test] = 0;
testMeansByCategory[category][test] = 0;
testStdDevsByCategory[category][test] = 0;
testStdErrsByCategory[category][test] = 0;
}
for (var i = 0; i < count; i++) {
itemTotals["total"][i] = 0;
for (var category in categoryTotals) {
itemTotals[category][i] = 0;
for (var test in testTotalsByCategory[category]) {
itemTotals[test][i] = 0;
}
}
}
}
function computeItemTotals()
{
for (var i = 0; i < output.length; i++) {
var result = output[i];
for (var test in result) {
var time = result[test];
var category = test.replace(/-.*/, "");
itemTotals["total"][i] += time;
itemTotals[category][i] += time;
itemTotals[test][i] += time;
}
}
}
function computeTotals()
{
for (var i = 0; i < output.length; i++) {
var result = output[i];
for (var test in result) {
var time = result[test];
var category = test.replace(/-.*/, "");
total += time;
categoryTotals[category] += time;
testTotalsByCategory[category][test] += time;
}
}
}
function computeMeans()
{
mean = total / count;
for (var category in categoryTotals) {
categoryMeans[category] = categoryTotals[category] / count;
for (var test in testTotalsByCategory[category]) {
testMeansByCategory[category][test] = testTotalsByCategory[category][test] / count;
}
}
}
function standardDeviation(mean, items)
{
var deltaSquaredSum = 0;
for (var i = 0; i < items.length; i++) {
var delta = items[i] - mean;
deltaSquaredSum += delta * delta;
}
variance = deltaSquaredSum / (items.length - 1);
return Math.sqrt(variance);
}
function computeStdDevs()
{
stdDev = standardDeviation(mean, itemTotals["total"]);
for (var category in categoryStdDevs) {
categoryStdDevs[category] = standardDeviation(categoryMeans[category], itemTotals[category]);
}
for (var category in categoryStdDevs) {
for (var test in testStdDevsByCategory[category]) {
testStdDevsByCategory[category][test] = standardDeviation(testMeansByCategory[category][test], itemTotals[test]);
}
}
}
function computeStdErrors()
{
var sqrtCount = Math.sqrt(count);
stdErr = stdDev / sqrtCount;
for (var category in categoryStdErrs) {
categoryStdErrs[category] = categoryStdDevs[category] / sqrtCount;
}
for (var category in categoryStdDevs) {
for (var test in testStdErrsByCategory[category]) {
testStdErrsByCategory[category][test] = testStdDevsByCategory[category][test] / sqrtCount;
}
}
}
var tDistribution = [NaN, NaN, 12.71, 4.30, 3.18, 2.78, 2.57, 2.45, 2.36, 2.31, 2.26, 2.23, 2.20, 2.18, 2.16, 2.14, 2.13, 2.12, 2.11, 2.10, 2.09, 2.09, 2.08, 2.07, 2.07, 2.06, 2.06, 2.06, 2.05, 2.05, 2.05, 2.04, 2.04, 2.04, 2.03, 2.03, 2.03, 2.03, 2.03, 2.02, 2.02, 2.02, 2.02, 2.02, 2.02, 2.02, 2.01, 2.01, 2.01, 2.01, 2.01, 2.01, 2.01, 2.01, 2.01, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.96];
var tMax = tDistribution.length;
var tLimit = 1.96;
function tDist(n)
{
if (n > tMax)
return tLimit;
return tDistribution[n];
}
function formatResult(meanWidth, mean, stdErr, n)
{
var meanString = mean.toFixed(1).toString();
while (meanString.length < meanWidth) {
meanString = " " + meanString;
}
if (n == 1)
return meanString + "ms";
return meanString + "ms +/- " + ((tDist(n) * stdErr / mean) * 100).toFixed(1) + "%";
}
function computeLabelWidth()
{
var width = "Total".length;
for (var category in categoryMeans) {
if (category.length + 2 > width)
width = category.length + 2;
}
for (var i = 0; i < tests.length; i++) {
var shortName = tests[i].replace(/^[^-]*-/, "");
if (shortName.length + 4 > width)
width = shortName.length + 4;
}
return width;
}
function computeMeanWidth()
{
var width = mean.toFixed(1).toString().length;
for (var category in categoryMeans) {
var candidate = categoryMeans[category].toFixed(2).toString().length;
if (candidate > width)
width = candidate;
for (var test in testMeansByCategory[category]) {
var candidate = testMeansByCategory[category][test].toFixed(2).toString().length;
if (candidate > width)
width = candidate;
}
}
return width;
}
function resultLine(labelWidth, indent, label, meanWidth, mean, stdErr)
{
var result = "";
for (i = 0; i < indent; i++) {
result += " ";
}
result += label + ": ";
for (i = 0; i < (labelWidth - (label.length + indent)); i++) {
result += " ";
}
return result + formatResult(meanWidth, mean, stdErr, count);
}
function printOutput()
{
var labelWidth = computeLabelWidth();
var meanWidth = computeMeanWidth();
print("\n");
print("============================================");
if (count == 1)
print("RESULTS");
else
print("RESULTS (means and 95% confidence intervals)");
print("--------------------------------------------");
print(resultLine(labelWidth, 0, "Total", meanWidth, mean, stdErr));
print("--------------------------------------------");
for (var category in categoryMeans) {
print("");
print(resultLine(labelWidth, 2, category, meanWidth, categoryMeans[category], categoryStdErrs[category]));
for (var test in testMeansByCategory[category]) {
var shortName = test.replace(/^[^-]*-/, "");
print(resultLine(labelWidth, 4, shortName, meanWidth, testMeansByCategory[category][test], testStdErrsByCategory[category][test]));
}
}
}
initialize();
computeItemTotals();
computeTotals();
computeMeans();
computeStdDevs();
computeStdErrors();
printOutput();
|
|
Assignee | |
Comment 7•18 years ago
|
||
Fixed. Turns out new and math.random are not pure functions we should CSE or constant-fold.
http://hg.mozilla.org/index.cgi/tracemonkey/rev/773396cbf8b1
Status: NEW → RESOLVED
Closed: 18 years ago
Resolution: --- → FIXED
You need to log in
before you can comment on or make changes to this bug.
Description
•