import java.util.*;

import java.util.regex.*;

import java.nio.*;

import java.nio.charset.*;

public class FindEncoderBugs {

static boolean isBroken(String csn) {

if (csn.equals("x-COMPOUND_TEXT")) return true;

return false;

}

static <T extends Comparable<? super T>> List<T> sort(Collection<T> c) {

List<T> list = new ArrayList<T>(c);

Collections.sort(list);

return list;

}

static class TooManyFailures extends RuntimeException {

private static final long serialVersionUID = 0L;

}

static String string(byte[] a) {

final StringBuilder sb = new StringBuilder();

for (byte b : a) {

if (sb.length() != 0) sb.append(' ');

sb.append(String.format("%02x", b & 0xff));

}

return sb.toString();

}

static String string(char[] a) {

final StringBuilder sb = new StringBuilder();

for (char c : a) {

if (sb.length() != 0) sb.append(' ');

sb.append(String.format("\\u%04x", (int) c));

}

return sb.toString();

}

static class Reporter {

// Some machinery to make sure only a small number of errors

// that are "too similar" are reported.

static class Counts extends HashMap<String, Long> {

private static final long serialVersionUID = -1;

long inc(String signature) {

Long count = get(signature);

if (count == null) count = 0L;

put(signature, count+1);

return count+1;

}

}

final Counts failureCounts = new Counts();

final static long maxFailures = 2;

final static Pattern hideBytes = Pattern.compile("\"[0-9a-f ]+\"");

final static Pattern hideChars = Pattern.compile("\\\\u[0-9a-f]{4}");

boolean bug(String format, Object... args) {

String signature = String.format(format, args);

// signature = hideBytes.matcher(signature).replaceAll("\"??\"");

// signature = hideChars.matcher(signature).replaceAll("\\u????");

failed++;

if (failureCounts.inc(signature) <= maxFailures) {

System.out.printf(format, args);

System.out.println();

return true;

}

return false;

}

void summarize() {

for (String key : sort(failureCounts.keySet()))

System.out.printf("-----%n%s%nfailures=%d%n",

key, failureCounts.get(key));

}

}

static final Reporter reporter = new Reporter();

static class Result {

final int limit;

final int ipos;

final boolean direct;

final char[] ia;

final byte[] oa;

final CoderResult cr;

private static byte[] toByteArray(ByteBuffer bb) {

byte[] bytes = new byte[bb.position()];

for (int i = 0; i < bytes.length; i++)

bytes[i] = bb.get(i);

return bytes;

}

Result(CharBuffer ib, ByteBuffer ob, CoderResult cr) {

ipos = ib.position();

ia = toArray(ib);

oa = toArray(ob);

direct = ib.isDirect();

limit = ob.limit();

this.cr = cr;

}

static char[] toArray(CharBuffer b) {

int pos = b.position();

char[] a = new char[b.limit()];

b.position(0);

b.get(a);

b.position(pos);

return a;

}

static byte[] toArray(ByteBuffer b) {

byte[] a = new byte[b.position()];

b.position(0);

b.get(a);

return a;

}

static boolean eq(Result x, Result y) {

return x == y ||

(x != null && y != null &&

(Arrays.equals(x.oa, y.oa) &&

x.ipos == y.ipos &&

x.cr == y.cr));

}

public String toString() {

return String.format("\"%s\"[%d/%d] => %s \"%s\"[%d/%d]%s",

string(ia), ipos, ia.length,

cr, string(oa), oa.length, limit,

(direct ? " (direct)" : ""));

}

}

static class CharsetTester {

private final Charset cs;

private final boolean hasBom;

private static final int maxFailures = 5;

private int failures = 0;

// private static final long maxCharsetFailures = Long.MAX_VALUE;

private static final long maxCharsetFailures = 10000L;

private final long failed0 = failed;

// legend: r=regular d=direct In=Input Ou=Output

static final int maxBufSize = 20;

static final CharBuffer[] rInBuffers = new CharBuffer[maxBufSize];

static final CharBuffer[] dInBuffers = new CharBuffer[maxBufSize];

static final ByteBuffer[] rOuBuffers = new ByteBuffer[maxBufSize];

static final ByteBuffer[] dOuBuffers = new ByteBuffer[maxBufSize];

static {

for (int i = 0; i < maxBufSize; i++) {

rInBuffers[i] = CharBuffer.allocate(i);

dInBuffers[i] = ByteBuffer.allocateDirect(i*2).asCharBuffer();

rOuBuffers[i] = ByteBuffer.allocate(i);

dOuBuffers[i] = ByteBuffer.allocateDirect(i);

}

}

CharsetTester(Charset cs) {

this.cs = cs;

this.hasBom =

cs.name().matches(".*BOM.*") ||

cs.name().equals("UTF-16");

}

static boolean bug(String format, Object... args) {

return reporter.bug(format, args);

}

static boolean hasBom(byte[] a) {

switch (a.length) {

case 2: case 4:

int sum = 0;

for (byte x : a)

sum += x;

return sum == (byte) 0xfe + (byte) 0xff;

default: return false;

}

}

void testSurrogates() {

int failures = 0;

for (int i = 0; i < 10; i++) {

Result r = test(new char[] { randomHighSurrogate() });

if (r == null) break;

if (! (r.cr.isUnderflow() &&

r.ipos == 0))

bug("Lone high surrogate not UNDERFLOW: %s %s",

cs, r);

}

for (int i = 0; i < 10; i++) {

Result r = test(new char[] { randomLowSurrogate() });

if (r == null) break;

if (! (r.cr.isMalformed() && r.cr.length() == 1))

bug("Lone low surrogate not MALFORMED[1]: %s %s",

cs, r);

}

char[] chars = new char[2];

for (int i = 0; i < 10; i++) {

chars[0] = randomLowSurrogate(); // Always illegal

chars[1] = randomChar();

Result r = test(chars);

if (r == null) break;

if (! (r.cr.isMalformed() &&

r.cr.length() == 1 &&

(r.ipos == 0 || (hasBom && hasBom(r.oa))))) {

if (failures++ > 5) return;

bug("Unpaired low surrogate not MALFORMED[1]: %s %s",

cs, r);

}

}

for (int i = 0; i < 10; i++) {

chars[0] = randomHighSurrogate();

do {

chars[1] = randomChar();

} while (Character.isLowSurrogate(chars[1]));

Result r = test(chars);

if (r == null) break;

if (! (r.cr.isMalformed() &&

r.cr.length() == 1 &&

(r.ipos == 0 || (hasBom && hasBom(r.oa))))) {

if (failures++ > 5) return;

bug("Unpaired high surrogate not MALFORMED[1]: %s %s",

cs, r);

}

}

for (int i = 0; i < 1000; i++) {

chars[0] = randomHighSurrogate();

chars[1] = randomLowSurrogate();

Result r = test(chars);

if (r == null) break;

if (! ((r.cr.isUnmappable() &&

r.cr.length() == 2 &&

r.oa.length == 0)

||

(r.cr.isUnderflow() &&

r.oa.length > 0 &&

r.ipos == 2))) {

if (failures++ > 5) return;

bug("Legal supplementary character bug: %s %s",

cs, r);

}

}

}

Result recode(CharBuffer ib, ByteBuffer ob) {

try {

byte canary = 22;

ib.clear(); // Prepare to read

ob.clear(); // Prepare to write

for (int i = 0; i < ob.limit(); i++)

ob.put(i, canary);

CharsetEncoder coder = cs.newEncoder();

CoderResult cr = coder.encode(ib, ob, false);

equal(ib.limit(), ib.capacity());

equal(ob.limit(), ob.capacity());

Result r = new Result(ib, ob, cr);

if (cr.isError())

check(cr.length() > 0);

if (cr.isOverflow() && ob.remaining() > 10)

bug("OVERFLOW, but there's lots of room: %s %s",

cs, r);

if (cr.isError() && ib.remaining() < cr.length())

bug("remaining() < CoderResult.length(): %s %s",

cs, r);

CoderResult cr2 = coder.encode(ib, ob, false);

if (ib.position() != r.ipos ||

ob.position() != r.oa.length ||

cr != cr2)

bug("Coding operation not idempotent: %s%n %s%n %s",

cs, r, new Result(ib, ob, cr2));

if (ob.position() < ob.limit() &&

ob.get(ob.position()) != canary)

bug("Buffer overrun: %s %s %s",

cs, r, ob.get(ob.position()));

return r;

} catch (Throwable t) {

if (bug("Unexpected exception: %s %s %s",

cs, t.getClass().getSimpleName(),

new Result(ib, ob, null)))

t.printStackTrace();

return null;

}

}

Result recode2(char[] ia, int n) {

int len = ia.length;

CharBuffer rib = CharBuffer.wrap(ia);

CharBuffer dib = dInBuffers[len];

dib.clear(); dib.put(ia); dib.clear();

ByteBuffer rob = rOuBuffers[n];

ByteBuffer dob = dOuBuffers[n];

equal(rob.limit(), n);

equal(dob.limit(), n);

check(dib.isDirect());

check(dob.isDirect());

Result r1 = recode(rib, rob);

Result r2 = recode(dib, dob);

if (r1 != null && r2 != null && ! Result.eq(r1, r2))

bug("Results differ for direct buffers: %s%n %s%n %s",

cs, r1, r2);

return r1;

}

Result test(char[] ia) {

if (failed - failed0 >= maxCharsetFailures)

throw new TooManyFailures();

Result roomy = recode2(ia, maxBufSize - 1);

if (roomy == null) return roomy;

int olen = roomy.oa.length;

if (olen > 0) {

if (roomy.ipos == roomy.ia.length) {

Result perfectFit = recode2(ia, olen);

if (! Result.eq(roomy, perfectFit))

bug("Results differ: %s%n %s%n %s",

cs, roomy, perfectFit);

}

for (int i = 0; i < olen; i++) {

Result claustrophobic = recode2(ia, i);

if (claustrophobic == null) return roomy;

if (roomy.cr.isUnderflow() &&

! claustrophobic.cr.isOverflow())

bug("Expected OVERFLOW: %s%n %s%n %s",

cs, roomy, claustrophobic);

}

}

return roomy;

}

void testExhaustively(char[] prefix, int n) {

int len = prefix.length;

char[] ia = Arrays.copyOf(prefix, len + 1);

for (int i = 0; i < 0x10000; i++) {

ia[len] = (char) i;

if (n == 1)

test(ia);

else

testExhaustively(ia, n - 1);

}

}

void testRandomly(char[] prefix, int n) {

int len = prefix.length;

char[] ia = Arrays.copyOf(prefix, len + n);

for (int i = 0; i < 10000; i++) {

for (int j = 0; j < n; j++)

ia[len + j] = randomChar();

test(ia);

}

}

void testPrefix(char[] prefix) {

if (prefix.length > 0)

System.out.printf("Testing prefix %s%n", string(prefix));

test(prefix);

testExhaustively(prefix, 1);

// Can you spare a year of CPU time?

//testExhaustively(prefix, 2);

testRandomly(prefix, 2);

testRandomly(prefix, 3);

}

}

private final static Random rnd = new Random();

private static char randomChar() {

return (char) rnd.nextInt(Character.MAX_VALUE);

}

private static char randomHighSurrogate() {

return (char) (Character.MIN_HIGH_SURROGATE + rnd.nextInt(1024));

}

private static char randomLowSurrogate() {

return (char) (Character.MIN_LOW_SURROGATE + rnd.nextInt(1024));

}

private static void testCharset(Charset cs) throws Throwable {

if (! cs.canEncode())

return;

final String csn = cs.name();

if (isBroken(csn)) {

System.out.printf("Skipping possibly broken charset %s%n", csn);

return;

}

System.out.println(csn);

CharsetTester tester = new CharsetTester(cs);

tester.testSurrogates();

tester.testPrefix(new char[] {});

if (csn.equals("x-ISCII91")) {

System.out.println("More ISCII testing...");

new CharsetTester(cs).testPrefix(new char[]{'\u094d'}); // Halant

new CharsetTester(cs).testPrefix(new char[]{'\u093c'}); // Nukta

}

}

private static void realMain(String[] args) {

for (Charset cs : sort(Charset.availableCharsets().values())) {

try {

testCharset(cs);

} catch (TooManyFailures e) {

System.out.printf("Too many failures for %s%n", cs);

} catch (Throwable t) {

unexpected(t);

}

}

reporter.summarize();

}

//--------------------- Infrastructure ---------------------------

static volatile long passed = 0, failed = 0;

static void pass() {passed++;}

static void fail() {failed++; Thread.dumpStack();}

static void fail(String format, Object... args) {

System.out.println(String.format(format, args)); failed++;}

static void fail(String msg) {System.out.println(msg); fail();}

static void unexpected(Throwable t) {failed++; t.printStackTrace();}

static void check(boolean cond) {if (cond) pass(); else fail();}

static void equal(Object x, Object y) {

if (x == null ? y == null : x.equals(y)) pass();

else fail(x + " not equal to " + y);}

public static void main(String[] args) throws Throwable {

try {realMain(args);} catch (Throwable t) {unexpected(t);}

System.out.printf("%nPassed = %d, failed = %d%n%n", passed, failed);

if (failed > 0) throw new AssertionError("Some tests failed");}

}