Searched refs:tokenStream (Results 1 - 25 of 136) sorted by relevance

123456

/lucene-3.6.0/lucene/core/src/test/org/apache/lucene/analysis/
H A DTestPerFieldAnalzyerWrapper.java36 TokenStream tokenStream = analyzer.tokenStream("field",
38 CharTermAttribute termAtt = tokenStream.getAttribute(CharTermAttribute.class);
40 assertTrue(tokenStream.incrementToken());
45 tokenStream = analyzer.tokenStream("special",
47 termAtt = tokenStream.getAttribute(CharTermAttribute.class);
48 assertTrue(tokenStream.incrementToken());
H A DTestDuelingAnalyzers.java54 assertEquals(s, left.tokenStream("foo", newStringReader(s)),
55 right.tokenStream("foo", newStringReader(s)));
74 assertEquals(s, left.tokenStream("foo", newStringReader(s)),
75 right.tokenStream("foo", newStringReader(s)));
90 assertEquals(s, left.tokenStream("foo", newStringReader(s)),
91 right.tokenStream("foo", newStringReader(s)));
109 assertEquals(s, left.tokenStream("foo", newStringReader(s)),
110 right.tokenStream("foo", newStringReader(s)));
125 assertEquals(s, left.tokenStream("foo", newStringReader(s)),
126 right.tokenStream("fo
[all...]
H A DTestStopAnalyzer.java48 TokenStream stream = stop.tokenStream("test", reader);
64 TokenStream stream = newStop.tokenStream("test", reader);
84 TokenStream stream = newStop.tokenStream("test", reader);
/lucene-3.6.0/lucene/backwards/src/test/org/apache/lucene/analysis/
H A DTestPerFieldAnalzyerWrapper.java36 TokenStream tokenStream = analyzer.tokenStream("field",
38 CharTermAttribute termAtt = tokenStream.getAttribute(CharTermAttribute.class);
40 assertTrue(tokenStream.incrementToken());
45 tokenStream = analyzer.tokenStream("special",
47 termAtt = tokenStream.getAttribute(CharTermAttribute.class);
48 assertTrue(tokenStream.incrementToken());
/lucene-3.6.0/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/
H A DFragmenter.java31 * interested in from tokenStream and then access the values in {@link #isNewFragment()}.
34 * @param tokenStream the {@link TokenStream} to be fragmented
36 public void start(String originalText, TokenStream tokenStream); argument
H A DNullFragmenter.java26 public void start(String s, TokenStream tokenStream) { argument
H A DHighlighter.java81 TokenStream tokenStream = analyzer.reusableTokenStream(fieldName, new StringReader(text));
82 return getBestFragment(tokenStream, text);
91 * @param tokenStream a stream of tokens identified in the text parameter, including offset information.
101 public final String getBestFragment(TokenStream tokenStream, String text) argument
104 String[] results = getBestFragments(tokenStream,text, 1);
133 TokenStream tokenStream = analyzer.reusableTokenStream(fieldName, new StringReader(text));
134 return getBestFragments(tokenStream, text, maxNumFragments);
151 TokenStream tokenStream,
158 TextFragment[] frag =getBestTextFragments(tokenStream,text, true,maxNumFragments);
177 * @param tokenStream
150 getBestFragments( TokenStream tokenStream, String text, int maxNumFragments) argument
184 getBestTextFragments( TokenStream tokenStream, String text, boolean mergeContiguousFragments, int maxNumFragments) argument
461 getBestFragments( TokenStream tokenStream, String text, int maxNumFragments, String separator) argument
[all...]
H A DScorer.java34 * @param tokenStream the {@link TokenStream} that will be scored.
40 public TokenStream init(TokenStream tokenStream) throws IOException; argument
H A DSimpleSpanFragmenter.java100 public void start(String originalText, TokenStream tokenStream) { argument
104 termAtt = tokenStream.addAttribute(CharTermAttribute.class);
105 posIncAtt = tokenStream.addAttribute(PositionIncrementAttribute.class);
106 offsetAtt = tokenStream.addAttribute(OffsetAttribute.class);
H A DTokenGroup.java43 public TokenGroup(TokenStream tokenStream) { argument
44 offsetAtt = tokenStream.addAttribute(OffsetAttribute.class);
45 termAtt = tokenStream.addAttribute(CharTermAttribute.class);
H A DQueryScorer.java177 public TokenStream init(TokenStream tokenStream) throws IOException { argument
179 termAtt = tokenStream.addAttribute(CharTermAttribute.class);
180 posIncAtt = tokenStream.addAttribute(PositionIncrementAttribute.class);
185 return initExtractor(tokenStream);
210 private TokenStream initExtractor(TokenStream tokenStream) throws IOException { argument
217 tokenStream, field);
220 tokenStream, field, reader);
/lucene-3.6.0/solr/core/src/java/org/apache/solr/analysis/
H A DSolrAnalyzer.java46 public TokenStream tokenStream(String fieldName, Reader reader) { method in class:SolrAnalyzer
52 private final TokenStream tokenStream; field in class:SolrAnalyzer.TokenStreamInfo
53 public TokenStreamInfo(Tokenizer tokenizer, TokenStream tokenStream) { argument
55 this.tokenStream = tokenStream;
58 public TokenStream getTokenStream() { return tokenStream; }
66 // if (true) return tokenStream(fieldName, reader);
/lucene-3.6.0/solr/core/src/java/org/apache/solr/handler/
H A DAnalysisRequestHandlerBase.java89 TokenStream tokenStream = null;
91 tokenStream = analyzer.reusableTokenStream(context.getFieldName(), new StringReader(value));
96 namedList.add(tokenStream.getClass().getName(), convertTokensToNamedLists(analyzeTokenStream(tokenStream), context));
116 TokenStream tokenStream = tfac.create(tokenizerChain.charStream(new StringReader(value)));
117 List<AttributeSource> tokens = analyzeTokenStream(tokenStream);
119 namedList.add(tokenStream.getClass().getName(), convertTokensToNamedLists(tokens, context));
127 tokenStream = tokenFilterFactory.create(listBasedTokenStream);
128 tokens = analyzeTokenStream(tokenStream);
129 namedList.add(tokenStream
180 analyzeTokenStream(TokenStream tokenStream) argument
[all...]
/lucene-3.6.0/lucene/core/src/java/org/apache/lucene/analysis/
H A DLimitTokenCountAnalyzer.java42 public TokenStream tokenStream(String fieldName, Reader reader) { method in class:LimitTokenCountAnalyzer
44 delegate.tokenStream(fieldName, reader), maxTokenCount
H A DAnalyzer.java37 * Therefore all non-abstract subclasses must be final or their {@link #tokenStream}
56 Modifier.isFinal(clazz.getMethod("tokenStream", String.class, Reader.class).getModifiers()) &&
59 "Analyzer implementation classes or at least their tokenStream() and reusableTokenStream() implementations must be final";
70 public abstract TokenStream tokenStream(String fieldName, Reader reader); method in class:Analyzer
80 return tokenStream(fieldName, reader);
126 * @return position increment gap, added to the next token emitted from {@link #tokenStream(String,Reader)}
141 * @return offset gap, added to the next token emitted from {@link #tokenStream(String,Reader)}
/lucene-3.6.0/lucene/core/src/test/org/apache/lucene/
H A DTestAssertions.java30 public final TokenStream tokenStream(String s, Reader r) { return null; } method in class:TestAssertions.TestAnalyzer1
37 public TokenStream tokenStream(String s, Reader r) { return null; } method in class:TestAssertions.TestAnalyzer2
44 public TokenStream tokenStream(String s, Reader r) { return null; } method in class:TestAssertions.TestAnalyzer3
51 public final TokenStream tokenStream(String s, Reader r) { return null; } method in class:TestAssertions.TestAnalyzer4
/lucene-3.6.0/lucene/backwards/src/test/org/apache/lucene/
H A DTestAssertions.java30 public final TokenStream tokenStream(String s, Reader r) { return null; } method in class:TestAssertions.TestAnalyzer1
37 public TokenStream tokenStream(String s, Reader r) { return null; } method in class:TestAssertions.TestAnalyzer2
44 public TokenStream tokenStream(String s, Reader r) { return null; } method in class:TestAssertions.TestAnalyzer3
51 public final TokenStream tokenStream(String s, Reader r) { return null; } method in class:TestAssertions.TestAnalyzer4
/lucene-3.6.0/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/
H A DHighlighterTest.java176 TokenStream tokenStream = new StandardAnalyzer(TEST_VERSION_CURRENT).tokenStream(fieldName, new StringReader(text));
183 String rv = highlighter.getBestFragments(tokenStream, text, 1, "(FIELD TEXT TRUNCATED)");
197 TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME,
201 String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,
246 TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
250 String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,
268 TokenStream tokenStream
1806 public TokenStream tokenStream(String arg0, Reader arg1) { method in class:SynonymAnalyzer
[all...]
/lucene-3.6.0/lucene/contrib/analyzers/kuromoji/src/test/org/apache/lucene/analysis/ja/
H A DTestJapaneseTokenizer.java142 TokenStream ts = analyzer.tokenStream("bogus", new StringReader("くよくよくよくよくよくよくよくよくよくよくよくよくよくよくよくよくよくよくよくよ"));
166 //TokenStream ts = a.tokenStream("foo", new StringReader("妹の咲子です。俺と年子で、今受験生です。"));
167 TokenStream ts = analyzer.tokenStream("foo", new StringReader("&#x250cdf66<!--\"<!--#<!--;?><!--#<!--#><!---->?>-->;"));
198 TokenStream ts = analyzer.tokenStream("foo", new StringReader(s));
219 TokenStream ts = analyzer.tokenStream("foo", new StringReader(s));
229 TokenStream ts = analyzerNoPunct.tokenStream("foo", new StringReader("。、。。"));
236 TokenStream ts = extendedModeAnalyzerNoPunct.tokenStream("foo", new StringReader("......"));
245 assertTokenStreamContents(analyzerNoPunct.tokenStream("foo", new StringReader("これは本ではない")),
252 assertTokenStreamContents(analyzerNoPunct.tokenStream("foo", new StringReader("これは本ではない ")),
263 assertTokenStreamContents(analyzer.tokenStream("fo
[all...]
/lucene-3.6.0/lucene/core/src/java/org/apache/lucene/document/
H A DField.java279 public TokenStream tokenStreamValue() { return tokenStream; }
333 public void setTokenStream(TokenStream tokenStream) { argument
336 this.tokenStream = tokenStream;
480 * @param tokenStream The TokenStream with the content
481 * @throws NullPointerException if name or tokenStream is <code>null</code>
483 public Field(String name, TokenStream tokenStream) { argument
484 this(name, tokenStream, TermVector.NO);
495 * @param tokenStream The TokenStream with the content
497 * @throws NullPointerException if name or tokenStream i
499 Field(String name, TokenStream tokenStream, TermVector termVector) argument
[all...]
/lucene-3.6.0/solr/core/src/java/org/apache/solr/highlight/
H A DGapFragmenter.java93 public void start(String originalText, TokenStream tokenStream) { argument
94 offsetAtt = tokenStream.getAttribute(OffsetAttribute.class);
95 posIncAtt = tokenStream.getAttribute(PositionIncrementAttribute.class);
/lucene-3.6.0/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ar/
H A DTestArabicStemFilter.java125 ArabicLetterTokenizer tokenStream = new ArabicLetterTokenizer(TEST_VERSION_CURRENT, new StringReader("ساهدهات"));
127 ArabicStemFilter filter = new ArabicStemFilter(new KeywordMarkerFilter(tokenStream, set));
132 ArabicLetterTokenizer tokenStream = new ArabicLetterTokenizer(TEST_VERSION_CURRENT, new StringReader(input));
133 ArabicStemFilter filter = new ArabicStemFilter(tokenStream);
H A DTestArabicNormalizationFilter.java92 ArabicLetterTokenizer tokenStream = new ArabicLetterTokenizer(TEST_VERSION_CURRENT, new StringReader(input));
93 ArabicNormalizationFilter filter = new ArabicNormalizationFilter(tokenStream);
/lucene-3.6.0/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/fa/
H A DTestPersianNormalizationFilter.java62 ArabicLetterTokenizer tokenStream = new ArabicLetterTokenizer(TEST_VERSION_CURRENT,
65 tokenStream);
/lucene-3.6.0/lucene/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/
H A DSmartChineseAnalyzer.java140 public TokenStream tokenStream(String fieldName, Reader reader) { method in class:SmartChineseAnalyzer
154 Tokenizer tokenStream; field in class:SmartChineseAnalyzer.SavedStreams
165 streams.tokenStream = new SentenceTokenizer(reader);
166 streams.filteredTokenStream = new WordTokenFilter(streams.tokenStream);
172 streams.tokenStream.reset(reader);

Completed in 108 milliseconds

123456