1245N/A

1245N/A

1245N/Aimport java.nio.charset.*;

1245N/Aimport java.nio.*;

1245N/Aimport java.util.*;

1245N/A

1245N/Apublic class TestIBMDB {

1245N/A static class Time {

1245N/A long t;

1245N/A }

1306N/A static int iteration = 200;

1245N/A

1245N/A static char[] decode(byte[] bb, Charset cs, boolean testDirect, Time t)

1245N/A throws Exception {

1245N/A String csn = cs.name();

1245N/A CharsetDecoder dec = cs.newDecoder();

1245N/A ByteBuffer bbf;

1245N/A CharBuffer cbf;

1245N/A if (testDirect) {

1245N/A bbf = ByteBuffer.allocateDirect(bb.length);

1245N/A cbf = ByteBuffer.allocateDirect(bb.length*2).asCharBuffer();

1245N/A bbf.put(bb);

1245N/A } else {

1245N/A bbf = ByteBuffer.wrap(bb);

1245N/A cbf = CharBuffer.allocate(bb.length);

1245N/A }

1245N/A CoderResult cr = null;

1245N/A long t1 = System.nanoTime()/1000;

1245N/A for (int i = 0; i < iteration; i++) {

1245N/A bbf.rewind();

1245N/A cbf.clear();

1245N/A dec.reset();

1245N/A cr = dec.decode(bbf, cbf, true);

1245N/A }

1245N/A long t2 = System.nanoTime()/1000;

1245N/A t.t = (t2 - t1)/iteration;

1245N/A if (cr != CoderResult.UNDERFLOW) {

1245N/A System.out.println("DEC-----------------");

1245N/A int pos = bbf.position();

1245N/A System.out.printf(" cr=%s, bbf.pos=%d, bb[pos]=%x,%x,%x,%x%n",

1245N/A cr.toString(), pos,

1245N/A bb[pos++]&0xff, bb[pos++]&0xff,bb[pos++]&0xff, bb[pos++]&0xff);

1245N/A throw new RuntimeException("Decoding err: " + csn);

1245N/A }

1245N/A char[] cc = new char[cbf.position()];

1245N/A cbf.flip(); cbf.get(cc);

1245N/A return cc;

1245N/A

1245N/A }

1245N/A

1245N/A static CoderResult decodeCR(byte[] bb, Charset cs, boolean testDirect)

1245N/A throws Exception {

1245N/A CharsetDecoder dec = cs.newDecoder();

1245N/A ByteBuffer bbf;

1245N/A CharBuffer cbf;

1245N/A if (testDirect) {

1245N/A bbf = ByteBuffer.allocateDirect(bb.length);

1245N/A cbf = ByteBuffer.allocateDirect(bb.length*2).asCharBuffer();

1245N/A bbf.put(bb).flip();

1245N/A } else {

1245N/A bbf = ByteBuffer.wrap(bb);

1245N/A cbf = CharBuffer.allocate(bb.length);

1245N/A }

1245N/A CoderResult cr = null;

1245N/A for (int i = 0; i < iteration; i++) {

1245N/A bbf.rewind();

1245N/A cbf.clear();

1245N/A dec.reset();

1245N/A cr = dec.decode(bbf, cbf, true);

1245N/A }

1245N/A return cr;

1245N/A }

1245N/A

1245N/A static byte[] encode(char[] cc, Charset cs, boolean testDirect, Time t)

1245N/A throws Exception {

1245N/A ByteBuffer bbf;

1245N/A CharBuffer cbf;

1245N/A CharsetEncoder enc = cs.newEncoder();

1245N/A String csn = cs.name();

1245N/A if (testDirect) {

1245N/A bbf = ByteBuffer.allocateDirect(cc.length * 4);

1245N/A cbf = ByteBuffer.allocateDirect(cc.length * 2).asCharBuffer();

1245N/A cbf.put(cc).flip();

1245N/A } else {

1245N/A bbf = ByteBuffer.allocate(cc.length * 4);

1245N/A cbf = CharBuffer.wrap(cc);

1245N/A }

1245N/A CoderResult cr = null;

1245N/A long t1 = System.nanoTime()/1000;

1245N/A for (int i = 0; i < iteration; i++) {

1245N/A cbf.rewind();

1245N/A bbf.clear();

1245N/A enc.reset();

1245N/A cr = enc.encode(cbf, bbf, true);

1245N/A }

1245N/A long t2 = System.nanoTime()/1000;

1245N/A t.t = (t2 - t1)/iteration;

1245N/A if (cr != CoderResult.UNDERFLOW) {

1245N/A System.out.println("ENC-----------------");

1245N/A int pos = cbf.position();

1245N/A System.out.printf(" cr=%s, cbf.pos=%d, cc[pos]=%x%n",

1245N/A cr.toString(), pos, cc[pos]&0xffff);

1245N/A throw new RuntimeException("Encoding err: " + csn);

1245N/A }

1245N/A byte[] bb = new byte[bbf.position()];

1245N/A bbf.flip(); bbf.get(bb);

1245N/A return bb;

1245N/A }

1245N/A

1245N/A static CoderResult encodeCR(char[] cc, Charset cs, boolean testDirect)

1245N/A throws Exception {

1245N/A ByteBuffer bbf;

1245N/A CharBuffer cbf;

1245N/A CharsetEncoder enc = cs.newEncoder();

1245N/A if (testDirect) {

1245N/A bbf = ByteBuffer.allocateDirect(cc.length * 4);

1245N/A cbf = ByteBuffer.allocateDirect(cc.length * 2).asCharBuffer();

1245N/A cbf.put(cc).flip();

1245N/A } else {

1245N/A bbf = ByteBuffer.allocate(cc.length * 4);

1245N/A cbf = CharBuffer.wrap(cc);

1245N/A }

1245N/A CoderResult cr = null;

1245N/A for (int i = 0; i < iteration; i++) {

1245N/A cbf.rewind();

1245N/A bbf.clear();

1245N/A enc.reset();

1245N/A cr = enc.encode(cbf, bbf, true);

1245N/A }

1245N/A return cr;

1245N/A }

1245N/A

1245N/A static void printEntry(char c, Charset cs) {

1245N/A byte[] bb = new String(new char[] {c}).getBytes(cs);

1245N/A for (byte b:bb)

1245N/A System.out.printf("%x", b&0xff);

1245N/A System.out.printf(" %x", c & 0xffff);

1245N/A String s2 = new String(bb, cs);

1245N/A System.out.printf(" %x%n", s2.charAt(0) & 0xffff);

1245N/A }

1245N/A

1245N/A // check and compare canEncoding/Encoding

1245N/A static char[] checkEncoding(Charset oldCS, Charset newCS)

1245N/A throws Exception {

1245N/A System.out.printf("Encoding <%s> <%s>...%n", oldCS.name(), newCS.name());

1245N/A CharsetEncoder encOLD = oldCS.newEncoder();

1245N/A CharsetEncoder encNew = newCS.newEncoder();

1245N/A char[] cc = new char[0x10000];

1245N/A int pos = 0;

1245N/A boolean is970 = "x-IBM970-Old".equals(oldCS.name());

1245N/A

1245N/A for (char c = 0; c < 0xffff; c++) {

1245N/A boolean canOld = encOLD.canEncode(c);

1245N/A boolean canNew = encNew.canEncode(c);

1245N/A

1306N/A if (is970 && c == 0x2299)

1306N/A continue;

1306N/A

1245N/A if (canOld != canNew) {

1245N/A if (canNew) {

1245N/A System.out.printf(" NEW(only): ");

1245N/A printEntry(c, newCS);

1245N/A } else {

1306N/A if (is970) {

1306N/A byte[] bb = new String(new char[] {c}).getBytes(oldCS);

1306N/A if (bb.length == 2 && bb[0] == (byte)0xa2 && bb[1] == (byte)0xc1) {

1306N/A // we know 970 has bogus nnnn -> a2c1 -> 2299

1306N/A continue;

1306N/A }

1306N/A }

1245N/A System.out.printf(" OLD(only): ");

1245N/A printEntry(c, oldCS);

1245N/A }

1245N/A } else if (canNew) {

1245N/A byte[] bbNew = new String(new char[] {c}).getBytes(newCS);

1245N/A byte[] bbOld = new String(new char[] {c}).getBytes(oldCS);

1245N/A if (!Arrays.equals(bbNew, bbOld)) {

1245N/A System.out.printf(" c->b NEW: ");

1245N/A printEntry(c, newCS);

1245N/A System.out.printf(" c->b OLD: ");

1245N/A printEntry(c, oldCS);

1245N/A } else {

1245N/A String sNew = new String(bbNew, newCS);

1245N/A String sOld = new String(bbOld, oldCS);

1245N/A if (!sNew.equals(sOld)) {

1245N/A System.out.printf(" b2c NEW (c=%x):", c&0xffff);

1245N/A printEntry(sNew.charAt(0), newCS);

1245N/A System.out.printf(" b2c OLD:");

1245N/A printEntry(sOld.charAt(0), oldCS);

1245N/A }

1245N/A }

1245N/A }

1245N/A if (canNew & canOld) { // added only both for now

1245N/A cc[pos++] = c;

1245N/A }

1245N/A }

1245N/A return Arrays.copyOf(cc, pos);

1245N/A }

1245N/A

1245N/A

1245N/A // check and compare canEncoding/Encoding

1245N/A static void checkDecoding(Charset oldCS, Charset newCS)

1245N/A throws Exception

1245N/A {

1245N/A System.out.printf("Decoding <%s> <%s>...%n", oldCS.name(), newCS.name());

1245N/A boolean isEBCDIC = oldCS.name().startsWith("x-IBM93");

1245N/A

1245N/A //Try singlebyte first

1245N/A byte[] bb = new byte[1];

1245N/A System.out.printf(" trying SB...%n");

1245N/A for (int b = 0; b < 0x100; b++) {

1245N/A bb[0] = (byte)b;

1245N/A String sOld = new String(bb, oldCS);

1245N/A String sNew = new String(bb, newCS);

1245N/A if (!sOld.equals(sNew)) {

1306N/A System.out.printf(" b=%x: %x/%d(old) %x/%d(new)%n",

1306N/A b& 0xff,

1306N/A sOld.charAt(0) & 0xffff, sOld.length(),

1306N/A sNew.charAt(0) & 0xffff, sNew.length());

1245N/A }

1245N/A }

1245N/A

1245N/A System.out.printf(" trying DB...%n");

1245N/A bb = new byte[isEBCDIC?4:2];

1245N/A int b1Min = 0x40;

1245N/A int b1Max = 0xfe;

1245N/A for (int b1 = 0x40; b1 < 0xff; b1++) {

1306N/A if (!isEBCDIC) {

1306N/A // decodable singlebyte b1

1306N/A bb[0] = (byte)b1;

1306N/A String sOld = new String(bb, oldCS);

1306N/A String sNew = new String(bb, newCS);

1306N/A if (!sOld.equals(sNew)) {

1306N/A if (sOld.length() != 2 && sOld.charAt(0) != 0) {

1306N/A // only prints we are NOT expected. above two are known issue

1306N/A System.out.printf(" b1=%x: %x/%d(old) %x/%d(new)%n",

1306N/A b1 & 0xff,

1306N/A sOld.charAt(0) & 0xffff, sOld.length(),

1306N/A sNew.charAt(0) & 0xffff, sNew.length());

1306N/A continue;

1306N/A }

1306N/A }

1306N/A }

1245N/A for (int b2 = 0x40; b2 < 0xff; b2++) {

1245N/A if (isEBCDIC) {

1245N/A bb[0] = 0x0e;

1245N/A bb[1] = (byte)b1;

1245N/A bb[2] = (byte)b2;

1245N/A bb[3] = 0x0f;

1245N/A } else {

1245N/A bb[0] = (byte)b1;

1245N/A bb[1] = (byte)b2;

1245N/A }

1245N/A String sOld = new String(bb, oldCS);

1245N/A String sNew = new String(bb, newCS);

1245N/A //if (!sOld.equals(sNew)) {

1245N/A if (sOld.charAt(0) != sNew.charAt(0)) {

1245N/A

1245N/Aif (sOld.charAt(0) == 0 && sNew.charAt(0) == 0xfffd)

1245N/A continue; // known issude in old implementation

1245N/A

1306N/A System.out.printf(" bb=<%x,%x> c(old)=%x, c(new)=%x%n",

1245N/A b1, b2, sOld.charAt(0) & 0xffff, sNew.charAt(0) & 0xffff);

1245N/A }

1245N/A }

1245N/A }

1245N/A }

1245N/A

1245N/A static void checkInit(String csn) throws Exception {

1245N/A System.out.printf("Check init <%s>...%n", csn);

1245N/A Charset.forName("Big5"); // load in the ExtendedCharsets

1245N/A long t1 = System.nanoTime()/1000;

1245N/A Charset cs = Charset.forName(csn);

1245N/A long t2 = System.nanoTime()/1000;

1245N/A System.out.printf(" charset :%d%n", t2 - t1);

1245N/A t1 = System.nanoTime()/1000;

1245N/A cs.newDecoder();

1245N/A t2 = System.nanoTime()/1000;

1245N/A System.out.printf(" new Decoder :%d%n", t2 - t1);

1245N/A

1245N/A t1 = System.nanoTime()/1000;

1245N/A cs.newEncoder();

1245N/A t2 = System.nanoTime()/1000;

1245N/A System.out.printf(" new Encoder :%d%n", t2 - t1);

1245N/A }

1245N/A

1245N/A static void compare(Charset cs1, Charset cs2, char[] cc) throws Exception {

1245N/A System.gc(); // enqueue finalizable objects

1245N/A Thread.sleep(1000);

1245N/A System.gc(); // enqueue finalizable objects

1245N/A

1245N/A String csn1 = cs1.name();

1245N/A String csn2 = cs2.name();

1245N/A System.out.printf("Diff <%s> <%s>...%n", csn1, csn2);

1245N/A

1245N/A Time t1 = new Time();

1245N/A Time t2 = new Time();

1245N/A

1245N/A byte[] bb1 = encode(cc, cs1, false, t1);

1245N/A byte[] bb2 = encode(cc, cs2, false, t2);

1245N/A

1245N/A System.out.printf(" Encoding TimeRatio %s/%s: %d,%d :%f%n",

1245N/A csn2, csn1,

1245N/A t2.t, t1.t,

1245N/A (double)(t2.t)/(t1.t));

1245N/A if (!Arrays.equals(bb1, bb2)) {

1245N/A System.out.printf(" encoding failed%n");

1245N/A }

1245N/A

1245N/A char[] cc2 = decode(bb1, cs2, false, t2);

1245N/A char[] cc1 = decode(bb1, cs1, false, t1);

1245N/A System.out.printf(" Decoding TimeRatio %s/%s: %d,%d :%f%n",

1245N/A csn2, csn1,

1245N/A t2.t, t1.t,

1245N/A (double)(t2.t)/(t1.t));

1245N/A if (!Arrays.equals(cc1, cc2)) {

1245N/A System.out.printf(" decoding failed%n");

1245N/A }

1245N/A

1245N/A bb1 = encode(cc, cs1, true, t1);

1245N/A bb2 = encode(cc, cs2, true, t2);

1245N/A

1245N/A System.out.printf(" Encoding(dir) TimeRatio %s/%s: %d,%d :%f%n",

1245N/A csn2, csn1,

1245N/A t2.t, t1.t,

1245N/A (double)(t2.t)/(t1.t));

1245N/A

1245N/A if (!Arrays.equals(bb1, bb2))

1245N/A System.out.printf(" encoding (direct) failed%n");

1245N/A

1245N/A cc1 = decode(bb1, cs1, true, t1);

1245N/A cc2 = decode(bb1, cs2, true, t2);

1245N/A System.out.printf(" Decoding(dir) TimeRatio %s/%s: %d,%d :%f%n",

1245N/A csn2, csn1,

1245N/A t2.t, t1.t,

1245N/A (double)(t2.t)/(t1.t));

1245N/A if (!Arrays.equals(cc1, cc2)) {

1245N/A System.out.printf(" decoding (direct) failed%n");

1245N/A }

1245N/A }

1245N/A

1245N/A /* The first byte is the length of malformed bytes

1245N/A

1245N/A static void checkMalformed(Charset cs, byte[][] malformed)

1245N/A throws Exception

1245N/A {

1245N/A boolean failed = false;

1245N/A String csn = cs.name();

1245N/A System.out.printf("Check malformed <%s>...%n", csn);

1245N/A for (boolean direct: new boolean[] {false, true}) {

1245N/A for (byte[] bins : malformed) {

1245N/A int mlen = bins[0];

1245N/A byte[] bin = Arrays.copyOfRange(bins, 1, bins.length);

1245N/A CoderResult cr = decodeCR(bin, cs, direct);

1245N/A String ashex = "";

1245N/A for (int i = 0; i < bin.length; i++) {

1245N/A if (i > 0) ashex += " ";

1245N/A ashex += Integer.toString((int)bin[i] & 0xff, 16);

1245N/A }

1245N/A if (!cr.isMalformed()) {

1245N/A System.out.printf(" FAIL(direct=%b): [%s] not malformed. -->cr=%s\n", direct, ashex, cr.toString());

1245N/A failed = true;

1245N/A } else if (cr.length() != mlen) {

1245N/A System.out.printf(" FAIL(direct=%b): [%s] malformed[len=%d].\n", direct, ashex, cr.length());

1245N/A failed = true;

1245N/A }

1245N/A }

1245N/A }

1245N/A if (failed)

1245N/A throw new RuntimeException("Check malformed failed " + csn);

1245N/A }

1245N/A

1245N/A static boolean check(CharsetDecoder dec, byte[] bytes, boolean direct, int[] flow) {

1245N/A int inPos = flow[0];

1245N/A int inLen = flow[1];

1245N/A int outPos = flow[2];

1245N/A int outLen = flow[3];

1245N/A int expedInPos = flow[4];

1245N/A int expedOutPos = flow[5];

1245N/A CoderResult expedCR = (flow[6]==0)?CoderResult.UNDERFLOW

1245N/A :CoderResult.OVERFLOW;

1245N/A ByteBuffer bbf;

1245N/A CharBuffer cbf;

1245N/A if (direct) {

1245N/A bbf = ByteBuffer.allocateDirect(inPos + bytes.length);

1245N/A cbf = ByteBuffer.allocateDirect((outPos + outLen)*2).asCharBuffer();

1245N/A } else {

1245N/A bbf = ByteBuffer.allocate(inPos + bytes.length);

1245N/A cbf = CharBuffer.allocate(outPos + outLen);

1245N/A }

1245N/A bbf.position(inPos);

1245N/A bbf.put(bytes).flip().position(inPos).limit(inPos + inLen);

1245N/A cbf.position(outPos);

1245N/A dec.reset();

1245N/A CoderResult cr = dec.decode(bbf, cbf, false);

1245N/A if (cr != expedCR ||

1245N/A bbf.position() != expedInPos ||

1245N/A cbf.position() != expedOutPos) {

1245N/A System.out.printf("Expected(direct=%5b): [", direct);

1245N/A for (int i:flow) System.out.print(" " + i);

1245N/A System.out.println("] CR=" + cr +

1245N/A ", inPos=" + bbf.position() +

1245N/A ", outPos=" + cbf.position());

1245N/A return false;

1245N/A }

1245N/A return true;

1245N/A }

1245N/A

1245N/A static void checkUnderOverflow(Charset cs) throws Exception {

1245N/A String csn = cs.name();

1245N/A System.out.printf("Check under/overflow <%s>...%n", csn);

1245N/A CharsetDecoder dec = cs.newDecoder();

1245N/A boolean failed = false;

1245N/A

1245N/A //7f, a1a1, 8ea2a1a1, 8ea3a1a1, 8ea7a1a1

1245N/A //0 1 2 3 7 11

1245N/A byte[] bytes = new String("\u007f\u3000\u4e42\u4e28\ud840\udc55").getBytes("EUC_TW");

1245N/A int inlen = bytes.length;

1245N/A

1245N/A int MAXOFF = 20;

1245N/A for (int inoff = 0; inoff < MAXOFF; inoff++) {

1245N/A for (int outoff = 0; outoff < MAXOFF; outoff++) {

1245N/A int[][] Flows = {

1245N/A //inpos, inLen, outPos, outLen, inPosEP, outposEP, under(0)/over(1)

1245N/A //overflow

1245N/A {inoff, inlen, outoff, 1, inoff + 1, outoff + 1, 1},

1245N/A {inoff, inlen, outoff, 2, inoff + 3, outoff + 2, 1},

1245N/A {inoff, inlen, outoff, 3, inoff + 7, outoff + 3, 1},

1245N/A {inoff, inlen, outoff, 4, inoff + 11, outoff + 4, 1},

1245N/A {inoff, inlen, outoff, 5, inoff + 11, outoff + 4, 1},

1245N/A {inoff, inlen, outoff, 6, inoff + 15, outoff + 6, 0},

1245N/A //underflow

1245N/A {inoff, 1, outoff, 6, inoff + 1, outoff + 1, 0},

1245N/A {inoff, 2, outoff, 6, inoff + 1, outoff + 1, 0},

1245N/A {inoff, 3, outoff, 6, inoff + 3, outoff + 2, 0},

1245N/A {inoff, 4, outoff, 6, inoff + 3, outoff + 2, 0},

1245N/A {inoff, 5, outoff, 6, inoff + 3, outoff + 2, 0},

1245N/A {inoff, 8, outoff, 6, inoff + 7, outoff + 3, 0},

1245N/A {inoff, 9, outoff, 6, inoff + 7, outoff + 3, 0},

1245N/A {inoff, 10, outoff, 6, inoff + 7, outoff + 3, 0},

1245N/A {inoff, 11, outoff, 6, inoff +11, outoff + 4, 0},

1245N/A {inoff, 12, outoff, 6, inoff +11, outoff + 4, 0},

1245N/A {inoff, 15, outoff, 6, inoff +15, outoff + 6, 0},

1245N/A // 2-byte under/overflow

1245N/A {inoff, 2, outoff, 1, inoff + 1, outoff + 1, 0},

1245N/A {inoff, 3, outoff, 1, inoff + 1, outoff + 1, 1},

1245N/A {inoff, 3, outoff, 2, inoff + 3, outoff + 2, 0},

1245N/A };

1245N/A for (boolean direct: new boolean[] {false, true}) {

1245N/A for (int[] flow: Flows) {

1245N/A if (!check(dec, bytes, direct, flow))

1245N/A failed = true;

1245N/A }

1245N/A }}}

1245N/A if (failed)

1245N/A throw new RuntimeException("Check under/overflow failed " + csn);

1245N/A }

1245N/A

1245N/A static String[] csnames = new String[] {

1245N/A "IBM930",

1245N/A "IBM933",

1245N/A "IBM935",

1245N/A "IBM937",

1245N/A "IBM939",

1245N/A "IBM942",

1245N/A "IBM943",

1245N/A "IBM948",

1245N/A "IBM949",

1245N/A "IBM950",

1245N/A "IBM970",

1245N/A "IBM942C",

1245N/A "IBM943C",

1245N/A "IBM949C",

1245N/A "IBM1381",

1245N/A "IBM1383",

1306N/A

1306N/A "EUC_CN",

1306N/A "EUC_KR",

1306N/A "GBK",

1306N/A "Johab",

1306N/A "MS932",

1306N/A "MS936",

1306N/A "MS949",

1306N/A "MS950",

1245N/A };

1245N/A

1245N/A public static void main(String[] args) throws Exception {

1245N/A for (String csname: csnames) {

1245N/A System.out.printf("-----------------------------------%n");

1245N/A String oldname = csname + "_OLD";

1245N/A checkInit(csname);

1245N/A Charset csOld = (Charset)Class.forName(oldname).newInstance();

1245N/A Charset csNew = Charset.forName(csname);

1245N/A char[] cc = checkEncoding(csOld, csNew);

1245N/A checkDecoding(csOld, csNew);

1245N/A compare(csNew, csOld, cc);

1245N/A

1245N/A if (csname.startsWith("x-IBM93")) {

1245N/A //ecdbic

1245N/A checkMalformed(csNew, new byte[][] {

1245N/A {1, 0x26, 0x0f, 0x27}, // in SBSC, no SI

1245N/A {1, 0x0e, 0x41, 0x41, 0xe}, // in DBSC, no SO

1245N/A {2, 0x0e, 0x40, 0x41, 0xe}, // illegal DB

1245N/A });

1245N/A } else if (csname.equals("x-IBM970") ||

1245N/A csname.equals("x-IBM1383")) {

1245N/A //euc_simple

1245N/A checkMalformed(csNew, new byte[][] {

1245N/A {1, 0x26, (byte)0x8f, 0x27}, // SS2

1245N/A {1, (byte)0xa1, (byte)0xa1, (byte)0x8e, 0x51}, // SS3

1245N/A });

1245N/A }

1245N/A }

1245N/A }

1245N/A}