JFlexTokenizer.java revision 816
304N/A/*
304N/A * CDDL HEADER START
304N/A *
304N/A * The contents of this file are subject to the terms of the
1068N/A * Common Development and Distribution License (the "License").
304N/A * You may not use this file except in compliance with the License.
304N/A *
919N/A * See LICENSE.txt included in this distribution for the specific
919N/A * language governing permissions and limitations under the License.
919N/A *
919N/A * When distributing Covered Code, include this CDDL HEADER in each
919N/A * file and include the License file at LICENSE.txt.
919N/A * If applicable, add the following below this CDDL HEADER, with the
919N/A * fields enclosed by brackets "[]" replaced with your own identifying
919N/A * information: Portions Copyright [yyyy] [name of copyright owner]
919N/A *
919N/A * CDDL HEADER END
919N/A */
919N/A
919N/A/*
919N/A * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
919N/A * Use is subject to license terms.
919N/A */
919N/Apackage org.opensolaris.opengrok.analysis;
304N/A
304N/Aimport org.apache.lucene.analysis.Token;
304N/Aimport org.apache.lucene.analysis.Tokenizer;
304N/A
493N/A/**
304N/A * this class was created because of lucene 2.4.1 update which introduced char[] in Tokens instead of String
970N/A *
970N/A * Created on August 24, 2009
970N/A * @author Lubos Kosco
970N/A */
304N/A
1068N/Apublic abstract class JFlexTokenizer extends Tokenizer {
304N/A
911N/A // just for passing reference
1068N/A private Token internal=null;
1068N/A
911N/A // default jflex scanner method
304N/A abstract public Token yylex() throws java.io.IOException ;
304N/A
304N/A /**
304N/A * This is a convenience method for having correctly generated classes who reuse Tokens and save gc for lucene summarizer
304N/A * you MUST consume the returned token to properly get the null value !
493N/A * @param preusableToken
304N/A * @return null if no more tokens, otherwise a pointer to the modified token
970N/A * @throws java.io.IOException
970N/A */
970N/A @Override
304N/A public final Token next(Token preusableToken) throws java.io.IOException {
internal=this.yylex();
if (internal!=null) {
preusableToken.reinit(internal);
return preusableToken; }
preusableToken=null;
return null;
}
}