Skip to content
Snippets Groups Projects
Commit 29ed1900 authored by Michael Niedermayer's avatar Michael Niedermayer
Browse files

swr-test: fix rounding error leading to NAN

parent 28d37f71
No related branches found
No related tags found
No related merge requests found
......@@ -357,6 +357,7 @@ int main(int argc, char **argv){
maxdiff= FFMAX(maxdiff, FFABS(a-b));
}
sse= sum_aa + sum_bb - 2*sum_ab;
if(sse < 0 && sse > -0.00001) sse=0; //fix rounding error
fprintf(stderr, "[e:%f c:%f max:%f] len:%5d\n", sqrt(sse/out_count), sum_ab/(sqrt(sum_aa*sum_bb)), maxdiff, out_count);
}
......@@ -386,6 +387,7 @@ int main(int argc, char **argv){
maxdiff= FFMAX(maxdiff, FFABS(a-b));
}
sse= sum_aa + sum_bb - 2*sum_ab;
if(sse < 0 && sse > -0.00001) sse=0; //fix rounding error
fprintf(stderr, "[e:%f c:%f max:%f] len:%5d F:%3d\n", sqrt(sse/flush_count), sum_ab/(sqrt(sum_aa*sum_bb)), maxdiff, flush_count, flush_i);
}
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment