395N/A/*
3261N/A * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
395N/A * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
395N/A *
395N/A * This code is free software; you can redistribute it and/or modify it
395N/A * under the terms of the GNU General Public License version 2 only, as
395N/A * published by the Free Software Foundation.
395N/A *
395N/A * This code is distributed in the hope that it will be useful, but WITHOUT
395N/A * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
395N/A * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
395N/A * version 2 for more details (a copy is included in the LICENSE file that
395N/A * accompanied this code).
395N/A *
395N/A * You should have received a copy of the GNU General Public License version
395N/A * 2 along with this work; if not, write to the Free Software Foundation,
395N/A * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
395N/A *
2362N/A * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
2362N/A * or visit www.oracle.com if you need additional information or have any
2362N/A * questions.
395N/A */
395N/A
395N/A/*
395N/A * @test
395N/A * @bug 6380723
395N/A * @summary Decode many byte sequences in many ways
395N/A * @run main/timeout=1800 FindDecoderBugs
395N/A * @author Martin Buchholz
395N/A */
395N/A
395N/Aimport java.util.*;
395N/Aimport java.util.regex.*;
395N/Aimport java.nio.*;
395N/Aimport java.nio.charset.*;
395N/A
395N/Apublic class FindDecoderBugs {
395N/A
395N/A static boolean isBroken(String csn) {
395N/A if (csn.equals("x-COMPOUND_TEXT")) return true;
395N/A return false;
395N/A }
395N/A
395N/A static <T extends Comparable<? super T>> List<T> sort(Collection<T> c) {
395N/A List<T> list = new ArrayList<T>(c);
395N/A Collections.sort(list);
395N/A return list;
395N/A }
395N/A
395N/A static class TooManyFailures extends RuntimeException {
395N/A private static final long serialVersionUID = 0L;
395N/A }
395N/A
395N/A static String string(byte[] a) {
395N/A final StringBuilder sb = new StringBuilder();
395N/A for (byte b : a) {
395N/A if (sb.length() != 0) sb.append(' ');
395N/A sb.append(String.format("%02x", b & 0xff));
395N/A }
395N/A return sb.toString();
395N/A }
395N/A
395N/A static String string(char[] a) {
395N/A final StringBuilder sb = new StringBuilder();
395N/A for (char c : a) {
395N/A if (sb.length() != 0) sb.append(' ');
395N/A sb.append(String.format("\\u%04x", (int) c));
395N/A }
395N/A return sb.toString();
395N/A }
395N/A
395N/A static class Reporter {
395N/A // Some machinery to make sure only a small number of errors
395N/A // that are "too similar" are reported.
395N/A static class Counts extends HashMap<String, Long> {
395N/A private static final long serialVersionUID = -1;
395N/A long inc(String signature) {
395N/A Long count = get(signature);
395N/A if (count == null) count = 0L;
395N/A put(signature, count+1);
395N/A return count+1;
395N/A }
395N/A }
395N/A
395N/A final Counts failureCounts = new Counts();
395N/A final static long maxFailures = 2;
395N/A
395N/A final static Pattern hideBytes = Pattern.compile("\"[0-9a-f ]+\"");
395N/A final static Pattern hideChars = Pattern.compile("\\\\u[0-9a-f]{4}");
395N/A
395N/A boolean bug(String format, Object... args) {
395N/A String signature = String.format(format, args);
395N/A signature = hideBytes.matcher(signature).replaceAll("\"??\"");
395N/A signature = hideChars.matcher(signature).replaceAll("\\u????");
395N/A failed++;
395N/A if (failureCounts.inc(signature) <= maxFailures) {
395N/A System.out.printf(format, args);
395N/A System.out.println();
395N/A return true;
395N/A }
395N/A return false;
395N/A }
395N/A
395N/A void summarize() {
395N/A for (String key : sort(failureCounts.keySet()))
395N/A System.out.printf("-----%n%s%nfailures=%d%n",
395N/A key, failureCounts.get(key));
395N/A }
395N/A }
395N/A
395N/A static final Reporter reporter = new Reporter();
395N/A
395N/A static class Result {
395N/A final int limit;
395N/A final int ipos;
395N/A final boolean direct;
395N/A final byte[] ia;
395N/A final char[] oa;
395N/A final CoderResult cr;
395N/A
395N/A Result(ByteBuffer ib, CharBuffer ob, CoderResult cr) {
395N/A ipos = ib.position();
395N/A ia = toArray(ib);
395N/A oa = toArray(ob);
395N/A direct = ib.isDirect();
395N/A limit = ob.limit();
395N/A this.cr = cr;
395N/A }
395N/A
395N/A static byte[] toArray(ByteBuffer b) {
395N/A int pos = b.position();
395N/A byte[] a = new byte[b.limit()];
395N/A b.position(0);
395N/A b.get(a);
395N/A b.position(pos);
395N/A return a;
395N/A }
395N/A
395N/A static char[] toArray(CharBuffer b) {
395N/A char[] a = new char[b.position()];
395N/A b.position(0);
395N/A b.get(a);
395N/A return a;
395N/A }
395N/A
395N/A static boolean eq(Result x, Result y) {
395N/A return x == y ||
395N/A (x != null && y != null &&
395N/A (Arrays.equals(x.oa, y.oa) &&
395N/A x.ipos == y.ipos &&
395N/A x.cr == y.cr));
395N/A }
395N/A
395N/A public String toString() {
395N/A return String.format("\"%s\"[%d/%d] => %s \"%s\"[%d/%d]%s",
395N/A string(ia), ipos, ia.length,
395N/A cr, string(oa), oa.length, limit,
395N/A (direct ? " (direct)" : ""));
395N/A }
395N/A }
395N/A
395N/A // legend: r=regular d=direct In=Input Ou=Output
395N/A static final int maxBufSize = 20;
395N/A static final ByteBuffer[] ribs = new ByteBuffer[maxBufSize];
395N/A static final ByteBuffer[] dibs = new ByteBuffer[maxBufSize];
395N/A
395N/A static final CharBuffer[] robs = new CharBuffer[maxBufSize];
395N/A static final CharBuffer[] dobs = new CharBuffer[maxBufSize];
395N/A static {
395N/A for (int i = 0; i < maxBufSize; i++) {
395N/A ribs[i] = ByteBuffer.allocate(i);
395N/A dibs[i] = ByteBuffer.allocateDirect(i);
395N/A robs[i] = CharBuffer.allocate(i);
395N/A dobs[i] = ByteBuffer.allocateDirect(i*2).asCharBuffer();
395N/A }
395N/A }
395N/A
395N/A static class CharsetTester {
395N/A private final Charset cs;
395N/A private static final long maxFailures = 5;
395N/A private long failures = 0;
395N/A // private static final long maxCharsetFailures = Long.MAX_VALUE;
395N/A private static final long maxCharsetFailures = 10000L;
395N/A private final long failed0 = failed;
395N/A
395N/A CharsetTester(Charset cs) {
395N/A this.cs = cs;
395N/A }
395N/A
395N/A static boolean bug(String format, Object... args) {
395N/A return reporter.bug(format, args);
395N/A }
395N/A
395N/A Result recode(ByteBuffer ib, CharBuffer ob) {
395N/A try {
395N/A char canary = '\u4242';
395N/A ib.clear(); // Prepare to read
395N/A ob.clear(); // Prepare to write
395N/A for (int i = 0; i < ob.limit(); i++)
395N/A ob.put(i, canary);
395N/A CharsetDecoder coder = cs.newDecoder();
395N/A CoderResult cr = coder.decode(ib, ob, false);
395N/A equal(ib.limit(), ib.capacity());
395N/A equal(ob.limit(), ob.capacity());
395N/A Result r = new Result(ib, ob, cr);
395N/A if (cr.isError())
395N/A check(cr.length() > 0);
395N/A if (cr.isOverflow() && ob.remaining() > 10)
395N/A bug("OVERFLOW, but there's lots of room: %s %s",
395N/A cs, r);
395N/A// if (cr.isOverflow() && ib.remaining() == 0)
395N/A// bug("OVERFLOW, yet remaining() == 0: %s %s",
395N/A// cs, r);
395N/A if (cr.isError() && ib.remaining() < cr.length())
395N/A bug("remaining() < CoderResult.length(): %s %s",
395N/A cs, r);
395N/A// if (ib.position() == 0 && ob.position() > 0)
395N/A// reporter. bug("output only if input consumed: %s %s",
395N/A// cs, r);
395N/A // Should we warn if cr.isUnmappable() ??
395N/A CoderResult cr2 = coder.decode(ib, ob, false);
395N/A if (ib.position() != r.ipos ||
395N/A ob.position() != r.oa.length ||
395N/A cr != cr2)
395N/A bug("Coding operation not idempotent: %s%n %s%n %s",
395N/A cs, r, new Result(ib, ob, cr2));
395N/A if (ob.position() < ob.limit() &&
395N/A ob.get(ob.position()) != canary)
395N/A bug("Buffer overrun: %s %s %s",
395N/A cs, r, ob.get(ob.position()));
395N/A return r;
395N/A } catch (Throwable t) {
395N/A if (bug("Unexpected exception: %s %s %s",
395N/A cs, t.getClass().getSimpleName(),
395N/A new Result(ib, ob, null)))
395N/A t.printStackTrace();
395N/A return null;
395N/A }
395N/A }
395N/A
395N/A Result recode2(byte[] ia, int n) {
395N/A int len = ia.length;
395N/A ByteBuffer rib = ByteBuffer.wrap(ia);
395N/A ByteBuffer dib = dibs[len];
395N/A dib.clear(); dib.put(ia); dib.clear();
395N/A CharBuffer rob = robs[n];
395N/A CharBuffer dob = dobs[n];
395N/A equal(rob.limit(), n);
395N/A equal(dob.limit(), n);
395N/A check(dib.isDirect());
395N/A check(dob.isDirect());
395N/A Result r1 = recode(rib, rob);
395N/A Result r2 = recode(dib, dob);
395N/A if (r1 != null && r2 != null && ! Result.eq(r1, r2))
395N/A bug("Results differ for direct buffers: %s%n %s%n %s",
395N/A cs, r1, r2);
395N/A return r1;
395N/A }
395N/A
395N/A Result test(byte[] ia) {
395N/A if (failed - failed0 >= maxCharsetFailures)
395N/A throw new TooManyFailures();
395N/A
395N/A Result roomy = recode2(ia, maxBufSize - 1);
395N/A if (roomy == null) return roomy;
395N/A int olen = roomy.oa.length;
395N/A if (olen > 0) {
395N/A if (roomy.ipos == roomy.ia.length) {
395N/A Result perfectFit = recode2(ia, olen);
395N/A if (! Result.eq(roomy, perfectFit))
395N/A bug("Results differ: %s%n %s%n %s",
395N/A cs, roomy, perfectFit);
395N/A }
395N/A for (int i = 0; i < olen; i++) {
395N/A Result claustrophobic = recode2(ia, i);
395N/A if (claustrophobic == null) return roomy;
395N/A if (roomy.cr.isUnderflow() &&
395N/A ! claustrophobic.cr.isOverflow())
395N/A bug("Expected OVERFLOW: %s%n %s%n %s",
395N/A cs, roomy, claustrophobic);
395N/A }
395N/A }
395N/A return roomy;
395N/A }
395N/A
395N/A void testExhaustively(byte[] prefix, int n) {
395N/A int len = prefix.length;
395N/A byte[] ia = Arrays.copyOf(prefix, len + 1);
395N/A for (int i = 0; i < 0x100; i++) {
395N/A ia[len] = (byte) i;
395N/A if (n == 1)
395N/A test(ia);
395N/A else
395N/A testExhaustively(ia, n - 1);
395N/A }
395N/A }
395N/A
395N/A void testRandomly(byte[] prefix, int n) {
395N/A int len = prefix.length;
395N/A byte[] ia = Arrays.copyOf(prefix, len + n);
2538N/A for (int i = 0; i < 5000; i++) {
395N/A for (int j = 0; j < n; j++)
395N/A ia[len + j] = randomByte();
395N/A test(ia);
395N/A }
395N/A }
395N/A
395N/A void testPrefix(byte[] prefix) {
395N/A if (prefix.length > 0)
395N/A System.out.printf("Testing prefix %s%n", string(prefix));
395N/A
395N/A test(prefix);
395N/A
395N/A testExhaustively(prefix, 1);
395N/A testExhaustively(prefix, 2);
395N/A // Can you spare a week of CPU time?
395N/A // testExhaustively(cs, tester, prefix, 3);
395N/A
395N/A testRandomly(prefix, 3);
395N/A testRandomly(prefix, 4);
395N/A }
395N/A }
395N/A
395N/A private final static Random rnd = new Random();
395N/A private static byte randomByte() {
395N/A return (byte) rnd.nextInt(0x100);
395N/A }
395N/A private static byte[] randomBytes(int len) {
395N/A byte[] a = new byte[len];
395N/A for (int i = 0; i < len; i++)
395N/A a[i] = randomByte();
395N/A return a;
395N/A }
395N/A
395N/A private static final byte SS2 = (byte) 0x8e;
395N/A private static final byte SS3 = (byte) 0x8f;
395N/A private static final byte ESC = (byte) 0x1b;
395N/A private static final byte SO = (byte) 0x0e;
395N/A private static final byte SI = (byte) 0x0f;
395N/A
395N/A private final static byte[][] stateChangers = {
395N/A {SS2}, {SS3}, {SO}, {SI}
395N/A };
395N/A
395N/A private final static byte[][]escapeSequences = {
395N/A {ESC, '(', 'B'},
395N/A {ESC, '(', 'I'},
395N/A {ESC, '(', 'J'},
395N/A {ESC, '$', '@'},
395N/A {ESC, '$', 'A'},
395N/A {ESC, '$', ')', 'A'},
395N/A {ESC, '$', ')', 'C'},
395N/A {ESC, '$', ')', 'G'},
395N/A {ESC, '$', '*', 'H'},
395N/A {ESC, '$', '+', 'I'},
395N/A {ESC, '$', 'B'},
395N/A {ESC, 'N'},
395N/A {ESC, 'O'},
395N/A {ESC, '$', '(', 'D'},
395N/A };
395N/A
395N/A private static boolean isStateChanger(Charset cs, byte[] ia) {
395N/A Result r = new CharsetTester(cs).recode2(ia, 9);
395N/A return r == null ? false :
395N/A (r.cr.isUnderflow() &&
395N/A r.ipos == ia.length &&
395N/A r.oa.length == 0);
395N/A }
395N/A
395N/A private final static byte[][] incompletePrefixes = {
395N/A {ESC},
395N/A {ESC, '('},
395N/A {ESC, '$'},
395N/A {ESC, '$', '(',},
395N/A };
395N/A
395N/A private static boolean isIncompletePrefix(Charset cs, byte[] ia) {
395N/A Result r = new CharsetTester(cs).recode2(ia, 9);
395N/A return r == null ? false :
395N/A (r.cr.isUnderflow() &&
395N/A r.ipos == 0 &&
395N/A r.oa.length == 0);
395N/A }
395N/A
395N/A private static void testCharset(Charset cs) throws Throwable {
395N/A final String csn = cs.name();
395N/A
395N/A if (isBroken(csn)) {
395N/A System.out.printf("Skipping possibly broken charset %s%n", csn);
395N/A return;
395N/A }
395N/A System.out.println(csn);
395N/A CharsetTester tester = new CharsetTester(cs);
395N/A
395N/A tester.testPrefix(new byte[0]);
395N/A
395N/A if (! csn.matches("(?:x-)?(?:UTF|JIS(?:_X)?0).*")) {
395N/A for (byte[] prefix : stateChangers)
395N/A if (isStateChanger(cs, prefix))
395N/A tester.testPrefix(prefix);
395N/A
395N/A for (byte[] prefix : incompletePrefixes)
395N/A if (isIncompletePrefix(cs, prefix))
395N/A tester.testPrefix(prefix);
395N/A
395N/A if (isIncompletePrefix(cs, new byte[] {ESC}))
395N/A for (byte[] prefix : escapeSequences)
395N/A if (isStateChanger(cs, prefix))
395N/A tester.testPrefix(prefix);
395N/A }
395N/A }
395N/A
395N/A private static void realMain(String[] args) {
395N/A for (Charset cs : sort(Charset.availableCharsets().values())) {
395N/A try {
395N/A testCharset(cs);
395N/A } catch (TooManyFailures e) {
395N/A System.out.printf("Too many failures for %s%n", cs);
395N/A } catch (Throwable t) {
395N/A unexpected(t);
395N/A }
395N/A }
395N/A reporter.summarize();
395N/A }
395N/A
395N/A //--------------------- Infrastructure ---------------------------
395N/A static volatile long passed = 0, failed = 0;
395N/A static void pass() {passed++;}
395N/A static void fail() {failed++; Thread.dumpStack();}
395N/A static void fail(String format, Object... args) {
395N/A System.out.println(String.format(format, args)); failed++;}
395N/A static void fail(String msg) {System.out.println(msg); fail();}
395N/A static void unexpected(Throwable t) {failed++; t.printStackTrace();}
395N/A static void check(boolean cond) {if (cond) pass(); else fail();}
395N/A static void equal(Object x, Object y) {
395N/A if (x == null ? y == null : x.equals(y)) pass();
395N/A else fail(x + " not equal to " + y);}
2538N/A static void equal(int x, int y) {
2538N/A if (x == y) pass();
2538N/A else fail(x + " not equal to " + y);}
395N/A public static void main(String[] args) throws Throwable {
395N/A try {realMain(args);} catch (Throwable t) {unexpected(t);}
395N/A System.out.printf("%nPassed = %d, failed = %d%n%n", passed, failed);
395N/A if (failed > 0) throw new AssertionError("Some tests failed");}
395N/A}