Lines Matching defs:precision
45 int precision; // number of digits to the right of decimal
51 private FormattedFloatingDecimal( boolean negSign, int decExponent, char []digits, int n, boolean e, int precision, Form form )
58 this.precision = precision;
423 // rounds at a particular precision.
434 // only one digit (0 or 1) is returned because the precision
472 public FormattedFloatingDecimal( double d, int precision, Form form )
479 this.precision = precision;
539 public FormattedFloatingDecimal( float f, int precision, Form form )
546 this.precision = precision;
1004 exp = checkExponent(decExponent + precision);
1005 digits = applyPrecision(decExponent + precision);
1008 exp = checkExponent(precision + 1);
1009 digits = applyPrecision(precision + 1);
1012 exp = checkExponent(precision);
1013 digits = applyPrecision(precision);
1014 // adjust precision to be the number of digits to right of decimal
1016 if (exp - 1 < -4 || exp - 1 >= precision) {
1018 precision--;
1021 precision = precision - exp;
1055 int t = Math.min(nDigits - charLength, precision);
1062 int t = Math.min(nDigits - charLength, precision);
1078 int t = Math.min(-exp, precision);
1085 int t = Math.min(digits.length, precision + exp);
1089 // copy only when significant digits are within the precision
1106 int t = Math.min(nDigits -1, precision);
1156 * for a single-precision float.
1597 * In double-precision, this is an exact floating integer.
1645 * The alternative is to reproduce the whole multiple-precision
1646 * algorythm for float precision, or to try to parameterize it