Searched defs:tokens (Results 1 - 25 of 34) sorted by relevance

12

/lucene-3.6.0/lucene/core/src/test/org/apache/lucene/analysis/
H A DTestCachingTokenFilter.java36 private String[] tokens = new String[] {"term1", "term2", "term3", "term2"}; field in class:TestCachingTokenFilter
50 if (index == tokens.length) {
54 termAtt.append(tokens[index++]);
66 // 1) we consume all tokens twice before we add the doc to the index
71 // 2) now add the document to the index and verify if all tokens are indexed
93 // 3) reset stream and consume tokens again
104 assertTrue(count < tokens.length);
105 assertEquals(tokens[count], termAtt.toString());
109 assertEquals(tokens.length, count);
/lucene-3.6.0/lucene/backwards/src/test/org/apache/lucene/analysis/
H A DTestCachingTokenFilter.java35 private String[] tokens = new String[] {"term1", "term2", "term3", "term2"}; field in class:TestCachingTokenFilter
49 if (index == tokens.length) {
53 termAtt.append(tokens[index++]);
65 // 1) we consume all tokens twice before we add the doc to the index
70 // 2) now add the document to the index and verify if all tokens are indexed
92 // 3) reset stream and consume tokens again
103 assertTrue(count < tokens.length);
104 assertEquals(tokens[count], termAtt.toString());
108 assertEquals(tokens.length, count);
/lucene-3.6.0/lucene/core/src/test/org/apache/lucene/index/
H A DTestPositionBasedTermVectorMapper.java25 protected String[] tokens; field in class:TestPositionBasedTermVectorMapper
33 tokens = new String[]{"here", "is", "some", "text", "to", "test", "extra"};
34 thePositions = new int[tokens.length][];
35 offsets = new TermVectorOffsetInfo[tokens.length][];
38 for (int i = 0; i < tokens.length - 1; i++)
50 thePositions[tokens.length - 1] = new int[1];
51 thePositions[tokens.length - 1][0] = 0;//put this at the same position as "here"
52 offsets[tokens.length - 1] = new TermVectorOffsetInfo[1];
53 offsets[tokens.length - 1][0] = new TermVectorOffsetInfo(0, 1);
59 mapper.setExpectations("test", tokens
[all...]
H A DTestTermVectorsReader.java59 TestToken[] tokens = new TestToken[testTerms.length * TERM_FREQ]; field in class:TestTermVectorsReader
81 TestToken token = tokens[tokenUpto++] = new TestToken();
88 Arrays.sort(tokens);
140 if (tokenUpto >= tokens.length)
143 final TestToken testToken = tokens[tokenUpto++];
148 posIncrAtt.setPositionIncrement(testToken.pos - tokens[tokenUpto-2].pos);
/lucene-3.6.0/lucene/test-framework/src/java/org/apache/lucene/analysis/
H A DCannedTokenStream.java31 private final Token[] tokens; field in class:CannedTokenStream
38 public CannedTokenStream(Token[] tokens) { argument
39 this.tokens = tokens;
44 if (upto < tokens.length) {
45 final Token token = tokens[upto++];
/lucene-3.6.0/lucene/backwards/src/test/org/apache/lucene/index/
H A DTestPositionBasedTermVectorMapper.java25 protected String[] tokens; field in class:TestPositionBasedTermVectorMapper
33 tokens = new String[]{"here", "is", "some", "text", "to", "test", "extra"};
34 thePositions = new int[tokens.length][];
35 offsets = new TermVectorOffsetInfo[tokens.length][];
38 for (int i = 0; i < tokens.length - 1; i++)
50 thePositions[tokens.length - 1] = new int[1];
51 thePositions[tokens.length - 1][0] = 0;//put this at the same position as "here"
52 offsets[tokens.length - 1] = new TermVectorOffsetInfo[1];
53 offsets[tokens.length - 1][0] = new TermVectorOffsetInfo(0, 1);
59 mapper.setExpectations("test", tokens
[all...]
/lucene-3.6.0/solr/core/src/java/org/apache/solr/spelling/
H A DSpellingOptions.java33 * The tokens to spell check
35 public Collection<Token> tokens; field in class:SpellingOptions
68 public SpellingOptions(Collection<Token> tokens, int count) { argument
69 this.tokens = tokens;
73 public SpellingOptions(Collection<Token> tokens, IndexReader reader) { argument
74 this.tokens = tokens;
78 public SpellingOptions(Collection<Token> tokens, IndexReader reader, int count) { argument
79 this.tokens
85 SpellingOptions(Collection<Token> tokens, IndexReader reader, int count, boolean onlyMorePopular, boolean extendedResults, float accuracy, SolrParams customParams) argument
[all...]
H A DSpellingResult.java34 private Collection<Token> tokens; field in class:SpellingResult
48 public SpellingResult(Collection<Token> tokens) { argument
49 this.tokens = tokens;
144 * @return The original tokens
147 return tokens;
150 public void setTokens(Collection<Token> tokens) { argument
151 this.tokens = tokens;
H A DSolrSpellChecker.java99 public SpellingResult getSuggestions(Collection<Token> tokens, IndexReader reader) throws IOException { argument
100 return getSuggestions(tokens, reader, 1, false, false);
111 public SpellingResult getSuggestions(Collection<Token> tokens, IndexReader reader, int count) throws IOException { argument
112 return getSuggestions(tokens, reader, count, false, false);
124 public SpellingResult getSuggestions(Collection<Token> tokens, IndexReader reader, boolean onlyMorePopular, boolean extendedResults) throws IOException { argument
125 return getSuggestions(tokens, reader, 1, onlyMorePopular, extendedResults);
132 * @param tokens The Tokens to be spell checked.
142 public abstract SpellingResult getSuggestions(Collection<Token> tokens, IndexReader reader, int count, argument
159 return getSuggestions(options.tokens, options.reader, options.count, options.onlyMorePopular, options.extendedResults);
/lucene-3.6.0/solr/core/src/test/org/apache/solr/spelling/
H A DSpellingQueryConverterTest.java42 Collection<Token> tokens = converter.convert("field:foo");
43 assertTrue("tokens is null and it shouldn't be", tokens != null);
44 assertTrue("tokens Size: " + tokens.size() + " is not: " + 1, tokens.size() == 1);
53 Collection<Token> tokens = converter.convert(original);
54 assertTrue("tokens is null and it shouldn't be", tokens != null);
55 assertEquals("tokens Siz
84 isOffsetCorrect(String s, Collection<Token> tokens) argument
[all...]
/lucene-3.6.0/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/
H A DTokenSourcesTest.java51 private Token[] tokens; field in class:TokenSourcesTest.OverlappingTokenStream
62 if (this.i >= this.tokens.length) {
66 termAttribute.setEmpty().append(this.tokens[i]);
67 offsetAttribute.setOffset(this.tokens[i].startOffset(),
68 this.tokens[i].endOffset());
69 positionIncrementAttribute.setPositionIncrement(this.tokens[i]
77 this.tokens = new Token[] {
84 this.tokens[1].setPositionIncrement(0);
H A DHighlighterPhraseTest.java298 private Token[] tokens; field in class:HighlighterPhraseTest.TokenStreamSparse
313 if (this.i >= this.tokens.length) {
317 termAttribute.setEmpty().append(this.tokens[i]);
318 offsetAttribute.setOffset(this.tokens[i].startOffset(), this.tokens[i]
320 positionIncrementAttribute.setPositionIncrement(this.tokens[i]
328 this.tokens = new Token[] {
333 this.tokens[3].setPositionIncrement(2);
338 private Token[] tokens; field in class:HighlighterPhraseTest.TokenStreamConcurrent
353 if (this.i >= this.tokens
[all...]
/lucene-3.6.0/lucene/contrib/icu/src/test/org/apache/lucene/analysis/icu/segmentation/
H A DTestLaoBreakIterator.java43 private void assertBreaksTo(BreakIterator iterator, String sourceText, String tokens[]) { argument
49 for (int i = 0; i < tokens.length; i++) {
57 assertEquals(tokens[i], new String(text, start, end - start));
/lucene-3.6.0/lucene/contrib/spellchecker/src/java/org/apache/lucene/search/suggest/tst/
H A DTSTAutocomplete.java29 * @param tokens
38 public void balancedTree(Object[] tokens, Object[] vals, int lo, int hi, argument
42 root = insert(root, (String) tokens[mid], vals[mid], 0);
43 balancedTree(tokens, vals, lo, mid - 1, root);
44 balancedTree(tokens, vals, mid + 1, hi, root);
/lucene-3.6.0/solr/core/src/test/org/apache/solr/analysis/
H A DTestTrimFilter.java91 final Token tokens[]; field in class:TestTrimFilter.IterTokenStream
100 public IterTokenStream(Token... tokens) { argument
102 this.tokens = tokens;
105 public IterTokenStream(Collection<Token> tokens) { argument
106 this(tokens.toArray(new Token[tokens.size()]));
111 if (index >= tokens.length)
115 Token token = tokens[index++];
H A DTestRemoveDuplicatesTokenFilter.java52 public void testDups(final String expected, final Token... tokens) argument
55 final Iterator<Token> toks = Arrays.asList(tokens).iterator();
H A DTestSlowSynonymFilter.java81 map.add(strings("a b"), tokens("ab"), orig, merge);
82 map.add(strings("a c"), tokens("ac"), orig, merge);
83 map.add(strings("a"), tokens("aa"), orig, merge);
84 map.add(strings("b"), tokens("bb"), orig, merge);
85 map.add(strings("z x c v"), tokens("zxcv"), orig, merge);
86 map.add(strings("x c"), tokens("xc"), orig, merge);
98 map.add(strings("a b"), tokens("ab"), orig, merge);
99 map.add(strings("a b"), tokens("ab"), orig, merge);
105 map.add(strings("zoo"), tokens("zoo"), orig, merge);
107 map.add(strings("zoo"), tokens("zo
334 private List<Token> tokens(String str) { method in class:TestSlowSynonymFilter
381 final Token tokens[]; field in class:TestSlowSynonymFilter.IterTokenStream
390 IterTokenStream(Token... tokens) argument
395 IterTokenStream(Collection<Token> tokens) argument
[all...]
/lucene-3.6.0/solr/core/src/test/org/apache/solr/handler/component/
H A DDummyCustomParamSpellChecker.java50 public SpellingResult getSuggestions(Collection<Token> tokens, IndexReader reader, int count, boolean onlyMorePopular, boolean extendedResults) throws IOException { argument
51 return getSuggestions(new SpellingOptions(tokens, reader, count, onlyMorePopular, extendedResults, 0, null));
/lucene-3.6.0/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/
H A DTokenGroup.java26 * One, or several overlapping tokens, along with the score(s) and the scope of
32 Token [] tokens=new Token[MAX_NUM_TOKENS_PER_GROUP]; field in class:TokenGroup
72 tokens[numTokens] = token;
93 return tokens[index];
113 * @return the number of tokens in this group
127 * @return all tokens' scores summed up
/lucene-3.6.0/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/
H A DIndexTimeSynonymTest.java281 void makeSynonymIndex( String value, Token... tokens ) throws Exception {
282 Analyzer analyzer = new TokenArrayAnalyzer( tokens );
297 Token[] tokens; field in class:IndexTimeSynonymTest.TokenArrayAnalyzer
298 public TokenArrayAnalyzer( Token... tokens ){
299 this.tokens = tokens;
310 if( p >= tokens.length ) return false;
312 tokens[p++].copyTo(reusableToken);
/lucene-3.6.0/lucene/contrib/misc/src/java/org/apache/lucene/index/
H A DTermVectorAccessor.java69 private List<String> tokens; field in class:TermVectorAccessor
90 if (tokens == null) {
91 tokens = new ArrayList<String>(500);
95 tokens.clear();
107 tokens.add(termEnum.term().text());
126 mapper.setExpectations(field, tokens.size(), false, !mapper.isIgnoringPositions());
127 for (int i = 0; i < tokens.size(); i++) {
128 mapper.map(tokens.get(i), frequencies.get(i).intValue(), (TermVectorOffsetInfo[]) null, positions.get(i));
/lucene-3.6.0/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/
H A DCompoundWordTokenFilterBase.java71 protected final LinkedList<CompoundToken> tokens; field in class:CompoundWordTokenFilterBase
154 this.tokens=new LinkedList<CompoundToken>();
178 if (!tokens.isEmpty()) {
180 CompoundToken token = tokens.removeFirst();
193 // only capture the state if we really need it for producing new tokens
194 if (!tokens.isEmpty()) {
205 /** Decomposes the current {@link #termAtt} and places {@link CompoundToken} instances in the {@link #tokens} list.
213 tokens.clear();
/lucene-3.6.0/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/shingle/
H A DTestShingleMatrixFilter.java69 LinkedList<Token> tokens;
73 tokens = new LinkedList<Token>();
74 tokens.add(createToken("please", 0, 6));
75 tokens.add(createToken("divide", 7, 13));
76 tokens.add(createToken("this", 14, 18));
77 tokens.add(createToken("sentence", 19, 27));
78 tokens.add(createToken("into", 28, 32));
79 tokens.add(createToken("shingles", 33, 39));
81 tls = new TokenListStream(tokens);
105 LinkedList<Token> tokens;
489 private Collection<Token> tokens; field in class:TestShingleMatrixFilter.TokenListStream
497 TokenListStream(Collection<Token> tokens) argument
[all...]
/lucene-3.6.0/solr/core/src/java/org/apache/solr/spelling/suggest/
H A DSuggester.java176 public SpellingResult getSuggestions(Collection<Token> tokens, IndexReader reader, int count, boolean onlyMorePopular, boolean extendedResults) throws IOException { argument
177 return getSuggestions(new SpellingOptions(tokens, reader, count, onlyMorePopular, extendedResults, Float.MIN_VALUE, null));
182 LOG.debug("getSuggestions: " + options.tokens);
189 for (Token t : options.tokens) {
/lucene-3.6.0/solr/solrj/src/java/org/apache/solr/client/solrj/response/
H A DAnalysisResponseBase.java69 List<NamedList> tokens = (List<NamedList>) phaseEntry.getValue();
70 for (NamedList token : tokens) {
113 * A phase in the analysis process. The phase holds the tokens produced in this phase and the name of the class that
119 private List<TokenInfo> tokens = new ArrayList<TokenInfo>(); field in class:AnalysisResponseBase.AnalysisPhase
135 tokens.add(tokenInfo);
139 * Returns a list of tokens which represent the token stream produced in this phase.
141 * @return A list of tokens which represent the token stream produced in this phase.
144 return tokens;
174 * @param match Indicates whether this token matches one of the the query tokens.
243 * Returns whether this token matches one of the query tokens (i
[all...]

Completed in 995 milliseconds

12