TextAnalyzerTest.java revision 957
953N/Apackage org.opensolaris.opengrok.analysis;
953N/A
953N/Aimport java.io.ByteArrayInputStream;
953N/Aimport java.io.IOException;
953N/Aimport java.io.InputStreamReader;
957N/Aimport java.io.Reader;
953N/Aimport java.nio.ByteBuffer;
953N/Aimport java.nio.charset.Charset;
953N/A
953N/Aimport junit.framework.Assert;
953N/A
953N/Aimport org.apache.lucene.document.Document;
953N/Aimport org.junit.Test;
953N/A
953N/Apublic class TextAnalyzerTest {
953N/A private String defaultEncoding = new InputStreamReader(new ByteArrayInputStream(new byte[0])).getEncoding();
953N/A private String encoding;
953N/A private String contents;
953N/A
953N/A @Test
953N/A public void defaultEncoding() throws IOException {
953N/A new TestableTextAnalyzer().analyze(new Document(),
953N/A new ByteArrayInputStream("hello".getBytes()));
953N/A
953N/A Assert.assertEquals(defaultEncoding, encoding);
953N/A
953N/A Assert.assertEquals("hello", contents);
953N/A }
953N/A
953N/A @Test
953N/A public void resetsStreamOnShortInput() throws IOException {
953N/A new TestableTextAnalyzer().analyze(new Document(),
953N/A new ByteArrayInputStream("hi".getBytes()));
953N/A
953N/A Assert.assertEquals(defaultEncoding, encoding);
953N/A
953N/A Assert.assertEquals("hi", contents);
953N/A }
953N/A
953N/A @Test
953N/A public void utf8WithBOM() throws IOException {
953N/A byte[] buffer = new byte[] {(byte) 239, (byte) 187, (byte) 191, 'h', 'e', 'l', 'l', 'o'};
953N/A new TestableTextAnalyzer().analyze(new Document(),
953N/A new ByteArrayInputStream(buffer));
953N/A
953N/A Assert.assertEquals("hello", contents);
953N/A Assert.assertEquals("UTF8", encoding);
953N/A }
953N/A
953N/A @Test
953N/A public void utf16WithBOM() throws IOException {
953N/A final ByteBuffer utf16str = Charset.forName("UTF-16").encode("hello");
953N/A byte[] bytes = new byte[utf16str.remaining()];
953N/A utf16str.get(bytes, 0, bytes.length);
953N/A
953N/A new TestableTextAnalyzer().analyze(new Document(),
953N/A new ByteArrayInputStream(bytes));
953N/A
953N/A Assert.assertEquals("UTF-16", encoding);
953N/A
953N/A Assert.assertEquals("hello", contents);
953N/A }
953N/A
953N/A @Test
953N/A public void utf16WithBOMAlternate() throws IOException {
953N/A final ByteBuffer utf16str = Charset.forName("UTF-16").encode("hello");
953N/A byte[] bytes = new byte[utf16str.remaining()];
953N/A utf16str.get(bytes, 0, bytes.length);
953N/A
953N/A for (int i=0; i<bytes.length; i+=2) {
953N/A byte b = bytes[i];
953N/A bytes[i] = bytes[i+1];
953N/A bytes[i+1] = b;
953N/A }
953N/A
953N/A new TestableTextAnalyzer().analyze(new Document(),
953N/A new ByteArrayInputStream(bytes));
953N/A
953N/A Assert.assertEquals("UTF-16", encoding);
953N/A
953N/A Assert.assertEquals("hello", contents);
953N/A }
953N/A
953N/A public class TestableTextAnalyzer extends TextAnalyzer {
953N/A public TestableTextAnalyzer() {
953N/A super(null);
953N/A }
953N/A
953N/A @Override
957N/A protected void analyze(Document doc, Reader r) throws IOException {
957N/A encoding = ((InputStreamReader) r).getEncoding();
953N/A
953N/A char[] buf = new char[1024];
953N/A int br = r.read(buf);
953N/A
953N/A contents = new String(buf, 0, br);
953N/A }
953N/A }
953N/A}