import java.nio.charset.*;

import java.nio.*;

import java.util.*;

public class TestUTF8 {

static char[] decode(byte[] bb, String csn, boolean testDirect)

throws Exception {

CharsetDecoder dec = Charset.forName(csn).newDecoder();

ByteBuffer bbf;

CharBuffer cbf;

if (testDirect) {

bbf = ByteBuffer.allocateDirect(bb.length);

cbf = ByteBuffer.allocateDirect(bb.length*2).asCharBuffer();

bbf.put(bb).flip();

} else {

bbf = ByteBuffer.wrap(bb);

cbf = CharBuffer.allocate(bb.length);

}

CoderResult cr = dec.decode(bbf, cbf, true);

if (cr != CoderResult.UNDERFLOW)

throw new RuntimeException("Decoding err: " + csn);

char[] cc = new char[cbf.position()];

cbf.flip(); cbf.get(cc);

return cc;

}

static CoderResult decodeCR(byte[] bb, String csn, boolean testDirect)

throws Exception {

CharsetDecoder dec = Charset.forName(csn).newDecoder();

ByteBuffer bbf;

CharBuffer cbf;

if (testDirect) {

bbf = ByteBuffer.allocateDirect(bb.length);

cbf = ByteBuffer.allocateDirect(bb.length*2).asCharBuffer();

bbf.put(bb).flip();

} else {

bbf = ByteBuffer.wrap(bb);

cbf = CharBuffer.allocate(bb.length);

}

return dec.decode(bbf, cbf, true);

}

// copy/paste of the StringCoding.decode()

static char[] decode(Charset cs, byte[] ba, int off, int len) {

CharsetDecoder cd = cs.newDecoder();

int en = (int)(len * cd.maxCharsPerByte());

char[] ca = new char[en];

if (len == 0)

return ca;

cd.onMalformedInput(CodingErrorAction.REPLACE)

.onUnmappableCharacter(CodingErrorAction.REPLACE)

.reset();

ByteBuffer bb = ByteBuffer.wrap(ba, off, len);

CharBuffer cb = CharBuffer.wrap(ca);

try {

CoderResult cr = cd.decode(bb, cb, true);

if (!cr.isUnderflow())

cr.throwException();

cr = cd.flush(cb);

if (!cr.isUnderflow())

cr.throwException();

} catch (CharacterCodingException x) {

throw new Error(x);

}

return Arrays.copyOf(ca, cb.position());

}

static byte[] encode(char[] cc, String csn, boolean testDirect)

throws Exception {

ByteBuffer bbf;

CharBuffer cbf;

CharsetEncoder enc = Charset.forName(csn).newEncoder();

if (testDirect) {

bbf = ByteBuffer.allocateDirect(cc.length * 4);

cbf = ByteBuffer.allocateDirect(cc.length * 2).asCharBuffer();

cbf.put(cc).flip();

} else {

bbf = ByteBuffer.allocate(cc.length * 4);

cbf = CharBuffer.wrap(cc);

}

CoderResult cr = enc.encode(cbf, bbf, true);

if (cr != CoderResult.UNDERFLOW)

throw new RuntimeException("Encoding err: " + csn);

byte[] bb = new byte[bbf.position()];

bbf.flip(); bbf.get(bb);

return bb;

}

static CoderResult encodeCR(char[] cc, String csn, boolean testDirect)

throws Exception {

ByteBuffer bbf;

CharBuffer cbf;

CharsetEncoder enc = Charset.forName(csn).newEncoder();

if (testDirect) {

bbf = ByteBuffer.allocateDirect(cc.length * 4);

cbf = ByteBuffer.allocateDirect(cc.length * 2).asCharBuffer();

cbf.put(cc).flip();

} else {

bbf = ByteBuffer.allocate(cc.length * 4);

cbf = CharBuffer.wrap(cc);

}

return enc.encode(cbf, bbf, true);

}

static char[] getUTFChars() {

char[] cc = new char[0x10000 - 0xe000 + 0xd800 + //bmp

(0x110000 - 0x10000) * 2]; //supp

int pos = 0;

int i = 0;

for (i = 0; i < 0xd800; i++)

cc[pos++] = (char)i;

for (i = 0xe000; i < 0x10000; i++)

cc[pos++] = (char)i;

for (i = 0x10000; i < 0x110000; i++) {

pos += Character.toChars(i, cc, pos);

}

return cc;

}

static int to3ByteUTF8(char c, byte[] bb, int pos) {

bb[pos++] = (byte)(0xe0 | ((c >> 12)));

bb[pos++] = (byte)(0x80 | ((c >> 06) & 0x3f));

bb[pos++] = (byte)(0x80 | ((c >> 00) & 0x3f));

return 3;

}

static void checkRoundtrip(String csn) throws Exception {

System.out.printf(" Check roundtrip <%s>...", csn);

char[] cc = getUTFChars();

byte[] bb = encode(cc, csn, false);

char[] ccO = decode(bb, csn, false);

if (!Arrays.equals(cc, ccO)) {

System.out.printf(" non-direct failed");

}

bb = encode(cc, csn, true);

ccO = decode(bb, csn, true);

if (!Arrays.equals(cc, ccO)) {

System.out.print(" (direct) failed");

}

// String.getBytes()/toCharArray() goes to ArrayDe/Encoder path

if (!Arrays.equals(bb, new String(cc).getBytes(csn))) {

System.out.printf(" String.getBytes() failed");

}

if (!Arrays.equals(cc, new String(bb, csn).toCharArray())) {

System.out.printf(" String.toCharArray() failed");

}

System.out.println();

}

static void check6ByteSurrs(String csn) throws Exception {

System.out.printf(" Check 6-byte Surrogates <%s>...%n", csn);

byte[] bb = new byte[(0x110000 - 0x10000) * 6];

char[] cc = new char[(0x110000 - 0x10000) * 2];

int bpos = 0;

int cpos = 0;

for (int i = 0x10000; i < 0x110000; i++) {

Character.toChars(i, cc, cpos);

bpos += to3ByteUTF8(cc[cpos], bb, bpos);

bpos += to3ByteUTF8(cc[cpos + 1], bb, bpos);

cpos += 2;

}

char[] ccO = decode(bb, csn, false);

if (!Arrays.equals(cc, ccO)) {

System.out.printf(" decoding failed%n");

}

ccO = decode(bb, csn, true);

if (!Arrays.equals(cc, ccO)) {

System.out.printf(" decoding(direct) failed%n");

}

// new String(bb, csn).getBytes(csn) will not return

// the 6 bytes surrogates as in bb, so only test

// toCharArray() here.

if (!Arrays.equals(cc, new String(bb, csn).toCharArray())) {

System.out.printf(" String.toCharArray() failed");

}

}

static void compare(String csn1, String csn2) throws Exception {

System.out.printf(" Diff <%s> <%s>...%n", csn1, csn2);

char[] cc = getUTFChars();

byte[] bb1 = encode(cc, csn1, false);

byte[] bb2 = encode(cc, csn2, false);

if (!Arrays.equals(bb1, bb2))

System.out.printf(" encoding failed%n");

char[] cc1 = decode(bb1, csn1, false);

char[] cc2 = decode(bb1, csn2, false);

if (!Arrays.equals(cc1, cc2)) {

System.out.printf(" decoding failed%n");

}

bb1 = encode(cc, csn1, true);

bb2 = encode(cc, csn2, true);

if (!Arrays.equals(bb1, bb2))

System.out.printf(" encoding (direct) failed%n");

cc1 = decode(bb1, csn1, true);

cc2 = decode(bb1, csn2, true);

if (!Arrays.equals(cc1, cc2)) {

System.out.printf(" decoding (direct) failed%n");

}

}

// The first byte is the length of malformed bytes

static byte[][] malformed = {

// One-byte sequences:

{1, (byte)0xFF },

{1, (byte)0xC0 },

{1, (byte)0x80 },

{1, (byte)0xFF, (byte)0xFF}, // all ones

{1, (byte)0xA0, (byte)0x80}, // 101x first byte first nibble

// Two-byte sequences:

{1, (byte)0xC0, (byte)0x80}, // invalid first byte

{1, (byte)0xC1, (byte)0xBF}, // invalid first byte

{1, (byte)0xC2, (byte)0x00}, // invalid second byte

{1, (byte)0xC2, (byte)0xC0}, // invalid second byte

{1, (byte)0xD0, (byte)0x00}, // invalid second byte

{1, (byte)0xD0, (byte)0xC0}, // invalid second byte

{1, (byte)0xDF, (byte)0x00}, // invalid second byte

{1, (byte)0xDF, (byte)0xC0}, // invalid second byte

// Three-byte sequences

{1, (byte)0xE0, (byte)0x80, (byte)0x80}, // 111x first byte first nibble

{1, (byte)0xE0, (byte)0x80, (byte)0x80 }, // U+0000 zero-padded

{1, (byte)0xE0, (byte)0x81, (byte)0xBF }, // U+007F zero-padded

{1, (byte)0xE0, (byte)0x9F, (byte)0xBF }, // U+07FF zero-padded

{1, (byte)0xE0, (byte)0xC0, (byte)0xBF }, // invalid second byte

{2, (byte)0xE0, (byte)0xA0, (byte)0x7F }, // invalid third byte

{2, (byte)0xE0, (byte)0xA0, (byte)0xC0 }, // invalid third byte

{1, (byte)0xFF, (byte)0xFF, (byte)0xFF }, // all ones

{1, (byte)0xE0, (byte)0xC0, (byte)0x80 }, // invalid second byte

{1, (byte)0xE0, (byte)0x80, (byte)0xC0 }, // invalid first byte

// Four-byte sequences

{1, (byte)0xF0, (byte)0x80, (byte)0x80, (byte)0x80 }, // U+0000 zero-padded

{1, (byte)0xF0, (byte)0x80, (byte)0x81, (byte)0xBF }, // U+007F zero-padded

{1, (byte)0xF0, (byte)0x80, (byte)0x9F, (byte)0xBF }, // U+007F zero-padded

{1, (byte)0xF0, (byte)0x8F, (byte)0xBF, (byte)0xBF }, // U+07FF zero-padded

{1, (byte)0xFF, (byte)0xFF, (byte)0xFF, (byte)0xFF }, // all ones

{1, (byte)0xF0, (byte)0x80, (byte)0x80, (byte)0x80}, // invalid second byte

{1, (byte)0xF0, (byte)0xC0, (byte)0x80, (byte)0x80 }, // invalid second byte

{2, (byte)0xF0, (byte)0x90, (byte)0xC0, (byte)0x80 }, // invalid third byte

{3, (byte)0xF0, (byte)0x90, (byte)0x80, (byte)0xC0 }, // invalid third byte

{1, (byte)0xF1, (byte)0xC0, (byte)0x80, (byte)0x80 }, // invalid second byte

{2, (byte)0xF1, (byte)0x80, (byte)0xC0, (byte)0x80 }, // invalid third byte

{3, (byte)0xF1, (byte)0x80, (byte)0x80, (byte)0xC0 }, // invalid forth byte

{1, (byte)0xF4, (byte)0x90, (byte)0x80, (byte)0xC0 }, // out-range 4-byte

{1, (byte)0xF4, (byte)0xC0, (byte)0x80, (byte)0xC0 }, // out-range 4-byte

{1, (byte)0xF5, (byte)0x80, (byte)0x80, (byte)0xC0 }, // out-range 4-byte

// Five-byte sequences

{5, (byte)0xF8, (byte)0x80, (byte)0x80, (byte)0x80, (byte)0x80}, // invalid first byte

{5, (byte)0xF8, (byte)0x80, (byte)0x80, (byte)0x80, (byte)0x80 }, // U+0000 zero-padded

{5, (byte)0xF8, (byte)0x80, (byte)0x80, (byte)0x81, (byte)0xBF }, // U+007F zero-padded

{5, (byte)0xF8, (byte)0x80, (byte)0x80, (byte)0x9F, (byte)0xBF }, // U+07FF zero-padded

{5, (byte)0xF8, (byte)0x80, (byte)0x8F, (byte)0xBF, (byte)0xBF }, // U+FFFF zero-padded

{1, (byte)0xF8, (byte)0xC0, (byte)0x80, (byte)0x80, (byte)0x80},

{2, (byte)0xF8, (byte)0x80, (byte)0xC0, (byte)0x80, (byte)0x80 },

{3, (byte)0xF8, (byte)0x80, (byte)0x80, (byte)0xC1, (byte)0xBF },

{4, (byte)0xF8, (byte)0x80, (byte)0x80, (byte)0x9F, (byte)0xC0 },

// Six-byte sequences

{6, (byte)0xFC, (byte)0x80, (byte)0x80, (byte)0x80, (byte)0x80, (byte)0x80 }, // U+0000 zero-padded

{6, (byte)0xFC, (byte)0x80, (byte)0x80, (byte)0x80, (byte)0x81, (byte)0xBF }, // U+007F zero-padded

{6, (byte)0xFC, (byte)0x80, (byte)0x80, (byte)0x80, (byte)0x9F, (byte)0xBF }, // U+07FF zero-padded

{6, (byte)0xFC, (byte)0x80, (byte)0x80, (byte)0x8F, (byte)0xBF, (byte)0xBF }, // U+FFFF zero-padded

{1, (byte)0xF8, (byte)0xC0, (byte)0x80, (byte)0x80, (byte)0x80, (byte)0x80 },

{2, (byte)0xF8, (byte)0x80, (byte)0xC0, (byte)0x80, (byte)0x80, (byte)0x80 },

{3, (byte)0xF8, (byte)0x80, (byte)0x80, (byte)0xC1, (byte)0xBF, (byte)0x80 },

{4, (byte)0xF8, (byte)0x80, (byte)0x80, (byte)0x9F, (byte)0xC0, (byte)0x80 },

{5, (byte)0xF8, (byte)0x80, (byte)0x80, (byte)0x9F, (byte)0x80, (byte)0xC0 },

};

static void checkMalformed(String csn) throws Exception {

boolean failed = false;

System.out.printf(" Check malformed <%s>...%n", csn);

Charset cs = Charset.forName(csn);

for (boolean direct: new boolean[] {false, true}) {

for (byte[] bins : malformed) {

int mlen = bins[0];

byte[] bin = Arrays.copyOfRange(bins, 1, bins.length);

CoderResult cr = decodeCR(bin, csn, direct);

String ashex = "";

for (int i = 0; i < bin.length; i++) {

if (i > 0) ashex += " ";

ashex += Integer.toBinaryString((int)bin[i] & 0xff);

}

if (!cr.isMalformed()) {

System.out.printf(" FAIL(direct=%b): [%s] not malformed.%n", direct, ashex);

failed = true;

} else if (cr.length() != mlen) {

System.out.printf(" FAIL(direct=%b): [%s] malformed[len=%d].%n", direct, ashex, cr.length());

failed = true;

}

if (!Arrays.equals(decode(cs, bin, 0, bin.length),

new String(bin, csn).toCharArray())) {

System.out.printf(" FAIL(new String(bb, %s)) failed%n", csn);

failed = true;

}

}

}

if (failed)

throw new RuntimeException("Check malformed failed " + csn);

}

static boolean check(CharsetDecoder dec, byte[] utf8s, boolean direct, int[] flow) {

int inPos = flow[0];

int inLen = flow[1];

int outPos = flow[2];

int outLen = flow[3];

int expedInPos = flow[4];

int expedOutPos = flow[5];

CoderResult expedCR = (flow[6]==0)?CoderResult.UNDERFLOW

:CoderResult.OVERFLOW;

ByteBuffer bbf;

CharBuffer cbf;

if (direct) {

bbf = ByteBuffer.allocateDirect(inPos + utf8s.length);

cbf = ByteBuffer.allocateDirect((outPos + outLen)*2).asCharBuffer();

} else {

bbf = ByteBuffer.allocate(inPos + utf8s.length);

cbf = CharBuffer.allocate(outPos + outLen);

}

bbf.position(inPos);

bbf.put(utf8s).flip().position(inPos).limit(inPos + inLen);

cbf.position(outPos);

dec.reset();

CoderResult cr = dec.decode(bbf, cbf, false);

if (cr != expedCR ||

bbf.position() != expedInPos ||

cbf.position() != expedOutPos) {

System.out.printf("Expected(direct=%5b): [", direct);

for (int i:flow) System.out.print(" " + i);

System.out.println("] CR=" + cr +

", inPos=" + bbf.position() +

", outPos=" + cbf.position());

return false;

}

return true;

}

static void checkUnderOverflow(String csn) throws Exception {

System.out.printf(" Check under/overflow <%s>...%n", csn);

CharsetDecoder dec = Charset.forName(csn).newDecoder();

boolean failed = false;

byte[] utf8s = new String("\u007f\u07ff\ue000\ud800\udc00").getBytes("UTF-8");

int inlen = utf8s.length;

for (int inoff = 0; inoff < 20; inoff++) {

for (int outoff = 0; outoff < 20; outoff++) {

int[][] Flows = {

//inpos, inLen, outPos, outLen, inPosEP, outposEP, under(0)/over(1)

{inoff, inlen, outoff, 1, inoff + 1, outoff + 1, 1},

{inoff, inlen, outoff, 2, inoff + 3, outoff + 2, 1},

{inoff, inlen, outoff, 3, inoff + 6, outoff + 3, 1},

{inoff, inlen, outoff, 4, inoff + 6, outoff + 3, 1},

{inoff, inlen, outoff, 5, inoff + 10,outoff + 5, 0},

// underflow

{inoff, 1, outoff, 5, inoff + 1, outoff + 1, 0},

{inoff, 2, outoff, 5, inoff + 1, outoff + 1, 0},

{inoff, 3, outoff, 5, inoff + 3, outoff + 2, 0},

{inoff, 4, outoff, 5, inoff + 3, outoff + 2, 0},

{inoff, 5, outoff, 5, inoff + 3, outoff + 2, 0},

{inoff, 6, outoff, 5, inoff + 6, outoff + 3, 0},

{inoff, 7, outoff, 5, inoff + 6, outoff + 3, 0},

{inoff, 8, outoff, 5, inoff + 6, outoff + 3, 0},

{inoff, 9, outoff, 5, inoff + 6, outoff + 3, 0},

{inoff, 10, outoff, 5, inoff + 10,outoff + 5, 0},

// 2-byte underflow/overflow

{inoff, 2, outoff, 1, inoff + 1, outoff + 1, 0},

{inoff, 3, outoff, 1, inoff + 1, outoff + 1, 1},

// 3-byte underflow/overflow

{inoff, 4, outoff, 2, inoff + 3, outoff + 2, 0},

{inoff, 5, outoff, 2, inoff + 3, outoff + 2, 0},

{inoff, 6, outoff, 2, inoff + 3, outoff + 2, 1},

// 4-byte underflow/overflow

{inoff, 7, outoff, 4, inoff + 6, outoff + 3, 0},

{inoff, 8, outoff, 4, inoff + 6, outoff + 3, 0},

{inoff, 9, outoff, 4, inoff + 6, outoff + 3, 0},

{inoff, 10, outoff, 4, inoff + 6, outoff + 3, 1},

};

for (boolean direct: new boolean[] {false, true}) {

for (int[] flow: Flows) {

if (!check(dec, utf8s, direct, flow))

failed = true;

}

}}}

if (failed)

throw new RuntimeException("Check under/overflow failed " + csn);

}

public static void main(String[] args) throws Exception {

checkRoundtrip("UTF-8");

check6ByteSurrs("UTF-8");

//compare("UTF-8", "UTF-8-OLD");

checkMalformed("UTF-8");

checkUnderOverflow("UTF-8");

}

}