JFlexTokenizer.java revision eb32a77fdb57f20c042b7b79b28a4fb4060cb949
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco/*
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco * CDDL HEADER START
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco *
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco * The contents of this file are subject to the terms of the
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco * Common Development and Distribution License (the "License").
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco * You may not use this file except in compliance with the License.
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco *
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco * See LICENSE.txt included in this distribution for the specific
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco * language governing permissions and limitations under the License.
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco *
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco * When distributing Covered Code, include this CDDL HEADER in each
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco * file and include the License file at LICENSE.txt.
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco * If applicable, add the following below this CDDL HEADER, with the
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco * fields enclosed by brackets "[]" replaced with your own identifying
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco * information: Portions Copyright [yyyy] [name of copyright owner]
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco *
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco * CDDL HEADER END
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco */
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco/*
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco * Use is subject to license terms.
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco */
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Koscopackage org.opensolaris.opengrok.analysis;
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Koscoimport org.apache.lucene.analysis.Token;
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Koscoimport org.apache.lucene.analysis.Tokenizer;
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco/**
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco * this class was created because of lucene 2.4.1 update which introduced char[] in Tokens instead of String
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco *
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco * Created on August 24, 2009
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco * @author Lubos Kosco
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco */
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Koscopublic abstract class JFlexTokenizer extends Tokenizer {
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco // just for passing reference
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco private Token internal=null;
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco // default jflex scanner method
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco abstract public Token yylex() throws java.io.IOException ;
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco /**
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco * This is a convenience method for having correctly generated classes who reuse Tokens and save gc for lucene summarizer
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco * you MUST consume the returned token to properly get the null value !
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco * @param preusableToken
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco * @return null if no more tokens, otherwise a pointer to the modified token
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco * @throws java.io.IOException
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco */
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco @Override
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco public final Token next(Token preusableToken) throws java.io.IOException {
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco internal=this.yylex();
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco if (internal!=null) {
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco preusableToken.reinit(internal);
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco return preusableToken; }
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco preusableToken=null;
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco return null;
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco }
eb32a77fdb57f20c042b7b79b28a4fb4060cb949Lubos Kosco}