Lines Matching refs:tokens
56 /** Converts text to tokens. */
128 Token[] tokens = getTokens(text); // parse text to token array
130 if (tokens.length == 0) {
166 for (int i = 0; i < tokens.length; i++) {
170 if (highlight.contains(tokens[i].toString())) {
176 int endToken = Math.min(i+SUM_CONTEXT, tokens.length);
177 int offset = tokens[startToken].startOffset();
199 Token t = tokens[j];
205 endToken = Math.min(j+SUM_CONTEXT, tokens.length);
218 // So if (j < tokens.length) then there is still trailing
222 if (j < tokens.length) {
223 excerpt.add(new Summary.Fragment(text.substring(offset,tokens[j].endOffset())));
250 int excerptLen = Math.min(SUM_LENGTH, tokens.length);
253 excerpt.add(new Summary.Fragment(text.substring(tokens[0].startOffset(), tokens[excerptLen-1].startOffset())));
278 if (tokenCount > 0 && lastExcerptPos < tokens.length) {
301 * a stream of tokens