package org.apache.lucene.index;

import org.apache.lucene.util.*;

import org.apache.lucene.store.*;

import org.apache.lucene.search.*;

import org.apache.lucene.analysis.*;

import org.apache.lucene.analysis.tokenattributes.*;

import org.apache.lucene.document.*;

import org.apache.lucene.index.FieldInfo.IndexOptions;

import java.io.IOException;

import java.io.UnsupportedEncodingException;

import java.util.ArrayList;

import java.util.Collections;

import java.util.List;

import org.junit.Ignore;

public class Test2BTerms extends LuceneTestCase {

private final class MyTokenStream extends TokenStream {

private final int tokensPerDoc;

private int tokenCount;

private final CharTermAttribute charTerm;

private final static int TOKEN_LEN = 5;

private final char[] chars;

public final List<String> savedTerms = new ArrayList<String>();

private int nextSave;

public MyTokenStream(int tokensPerDoc) {

super();

this.tokensPerDoc = tokensPerDoc;

charTerm = addAttribute(CharTermAttribute.class);

chars = charTerm.resizeBuffer(TOKEN_LEN);

charTerm.setLength(TOKEN_LEN);

nextSave = _TestUtil.nextInt(random, 500000, 1000000);

}

@Override

public boolean incrementToken() {

if (tokenCount >= tokensPerDoc) {

return false;

}

_TestUtil.randomFixedLengthUnicodeString(random, chars, 0, TOKEN_LEN);

tokenCount++;

if (--nextSave == 0) {

final String s = new String(chars, 0, TOKEN_LEN);

System.out.println("TEST: save term=" + s + " [" + toHexString(s) + "]");

savedTerms.add(s);

nextSave = _TestUtil.nextInt(random, 500000, 1000000);

}

return true;

}

@Override

public void reset() {

tokenCount = 0;

}

}

@Ignore("Takes ~4 hours to run on a fast machine!!")

public void test2BTerms() throws IOException {

final long TERM_COUNT = ((long) Integer.MAX_VALUE) + 100000000;

final int TERMS_PER_DOC = _TestUtil.nextInt(random, 100000, 1000000);

List<String> savedTerms = null;

MockDirectoryWrapper dir = newFSDirectory(_TestUtil.getTempDir("2BTerms"));

dir.setThrottling(MockDirectoryWrapper.Throttling.NEVER);

dir.setCheckIndexOnClose(false); // don't double-checkindex

//Directory dir = newFSDirectory(new File("/p/lucene/indices/2bindex"));

if (true) {

IndexWriter w = new IndexWriter(dir,

new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))

.setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH)

.setRAMBufferSizeMB(256.0)

.setMergeScheduler(new ConcurrentMergeScheduler())

.setMergePolicy(newLogMergePolicy(false, 10))

.setOpenMode(IndexWriterConfig.OpenMode.CREATE));

MergePolicy mp = w.getConfig().getMergePolicy();

if (mp instanceof LogByteSizeMergePolicy) {

// 1 petabyte:

((LogByteSizeMergePolicy) mp).setMaxMergeMB(1024*1024*1024);

}

Document doc = new Document();

final MyTokenStream ts = new MyTokenStream(TERMS_PER_DOC);

Field field = new Field("field", ts);

field.setIndexOptions(IndexOptions.DOCS_ONLY);

field.setOmitNorms(true);

doc.add(field);

//w.setInfoStream(System.out);

final int numDocs = (int) (TERM_COUNT/TERMS_PER_DOC);

System.out.println("TERMS_PER_DOC=" + TERMS_PER_DOC);

System.out.println("numDocs=" + numDocs);

for(int i=0;i<numDocs;i++) {

final long t0 = System.currentTimeMillis();

w.addDocument(doc);

System.out.println(i + " of " + numDocs + " " + (System.currentTimeMillis()-t0) + " msec");

}

savedTerms = ts.savedTerms;

System.out.println("TEST: full merge");

w.forceMerge(1);

System.out.println("TEST: close writer");

w.close();

}

System.out.println("TEST: open reader");

final IndexReader r = IndexReader.open(dir);

if (savedTerms == null) {

savedTerms = findTerms(r);

}

final int numSavedTerms = savedTerms.size();

final List<String> bigOrdTerms = new ArrayList<String>(savedTerms.subList(numSavedTerms-10, numSavedTerms));

System.out.println("TEST: test big ord terms...");

testSavedTerms(r, bigOrdTerms);

System.out.println("TEST: test all saved terms...");

testSavedTerms(r, savedTerms);

r.close();

System.out.println("TEST: now CheckIndex...");

CheckIndex.Status status = _TestUtil.checkIndex(dir);

final long tc = status.segmentInfos.get(0).termIndexStatus.termCount;

assertTrue("count " + tc + " is not > " + Integer.MAX_VALUE, tc > Integer.MAX_VALUE);

dir.close();

}

private List<String> findTerms(IndexReader r) throws IOException {

System.out.println("TEST: findTerms");

final TermEnum termEnum = r.terms();

final List<String> savedTerms = new ArrayList<String>();

int nextSave = _TestUtil.nextInt(random, 500000, 1000000);

while(termEnum.next()) {

if (--nextSave == 0) {

savedTerms.add(termEnum.term().text());

System.out.println("TEST: add " + termEnum.term());

nextSave = _TestUtil.nextInt(random, 500000, 1000000);

}

}

return savedTerms;

}

private String toHexString(String s) {

byte[] bytes;

try {

bytes = s.getBytes("UTF-8");

} catch (UnsupportedEncodingException uee) {

throw new RuntimeException(uee);

}

StringBuilder sb = new StringBuilder();

for(byte b : bytes) {

if (sb.length() > 0) {

sb.append(' ');

}

sb.append(Integer.toHexString(b&0xFF));

}

return sb.toString();

}

private void testSavedTerms(IndexReader r, List<String> terms) throws IOException {

System.out.println("TEST: run " + terms.size() + " terms on reader=" + r);

IndexSearcher s = new IndexSearcher(r);

Collections.shuffle(terms);

boolean failed = false;

for(int iter=0;iter<10*terms.size();iter++) {

final String term = terms.get(random.nextInt(terms.size()));

System.out.println("TEST: search " + term + " [" + toHexString(term) + "]");

final long t0 = System.currentTimeMillis();

final int count = s.search(new TermQuery(new Term("field", term)), 1).totalHits;

if (count <= 0) {

System.out.println(" FAILED: count=" + count);

failed = true;

}

final long t1 = System.currentTimeMillis();

System.out.println(" took " + (t1-t0) + " millis");

final TermEnum termEnum = r.terms(new Term("field", term));

final String text = termEnum.term().text();

if (!term.equals(text)) {

System.out.println(" FAILED: wrong term: got " + text + " [" + toHexString(text) + "]");

failed = true;

}

}

assertFalse(failed);

}

}