Searched refs:token (Results 1 - 25 of 74) sorted by relevance

123

/lucene-3.6.0/lucene/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/
H A DSegTokenFilter.java39 * @param token input {@link SegToken}
42 public SegToken filter(SegToken token) { argument
43 switch (token.wordType) {
46 for (int i = 0; i < token.charArray.length; i++) {
47 if (token.charArray[i] >= 0xFF10)
48 token.charArray[i] -= 0xFEE0;
50 if (token.charArray[i] >= 0x0041 && token.charArray[i] <= 0x005A) /* lowercase latin */
51 token.charArray[i] += 0x0020;
55 for (int i = 0; i < token
[all...]
H A DHHMMSegmenter.java48 SegToken token;
70 token = new SegToken(charArray, i, j, WordType.CHINESE_WORD,
72 segGraph.addToken(token);
80 token = new SegToken(charArray, i, j, WordType.CHINESE_WORD,
82 segGraph.addToken(token);
117 token = new SegToken(charArray, i, j, wordType, frequency);
118 segGraph.addToken(token);
135 token = new SegToken(charArray, i, j, wordType, frequency);
136 segGraph.addToken(token);
144 token
[all...]
H A DSegGraph.java71 * Set the {@link SegToken#index} for each token, based upon its order by startOffset.
95 * Add a {@link SegToken} to the mapping, creating a new mapping at the token's startOffset if one does not exist.
96 * @param token {@link SegToken}
98 public void addToken(SegToken token) { argument
99 int s = token.startOffset;
102 newlist.add(token);
106 tokenList.add(token);
/lucene-3.6.0/lucene/contrib/analyzers/kuromoji/src/java/org/apache/lucene/analysis/ja/tokenattributes/
H A DBaseFormAttributeImpl.java28 private Token token; field in class:BaseFormAttributeImpl
31 return token == null ? null : token.getBaseForm();
34 public void setToken(Token token) { argument
35 this.token = token;
40 token = null;
46 t.setToken(token);
H A DInflectionAttributeImpl.java29 private Token token; field in class:InflectionAttributeImpl
32 return token == null ? null : token.getInflectionType();
36 return token == null ? null : token.getInflectionForm();
39 public void setToken(Token token) { argument
40 this.token = token;
45 token = null;
51 t.setToken(token);
[all...]
H A DReadingAttributeImpl.java29 private Token token; field in class:ReadingAttributeImpl
32 return token == null ? null : token.getReading();
36 return token == null ? null : token.getPronunciation();
39 public void setToken(Token token) { argument
40 this.token = token;
45 token = null;
51 t.setToken(token);
[all...]
H A DPartOfSpeechAttributeImpl.java29 private Token token; field in class:PartOfSpeechAttributeImpl
32 return token == null ? null : token.getPartOfSpeech();
35 public void setToken(Token token) { argument
36 this.token = token;
41 token = null;
47 t.setToken(token);
H A DBaseFormAttribute.java31 public void setToken(Token token); argument
H A DInflectionAttribute.java32 public void setToken(Token token); argument
H A DPartOfSpeechAttribute.java28 public void setToken(Token token); argument
H A DReadingAttribute.java32 public void setToken(Token token); argument
/lucene-3.6.0/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/
H A DTestSingleTokenTokenFilter.java30 Token token = new Token();
31 SingleTokenTokenStream ts = new SingleTokenTokenStream(token);
37 assertEquals(token, tokenAtt);
40 token = new Token("hallo", 10, 20, "someType");
41 ts.setToken(token);
45 assertEquals(token, tokenAtt);
/lucene-3.6.0/solr/core/src/java/org/apache/solr/spelling/
H A DSpellingResult.java37 * Key == token
38 * Value = Map -> key is the suggestion, value is the frequency of the token in the collection
55 * @param token The token to associate the suggestions with
58 public void add(Token token, List<String> suggestions) { argument
59 LinkedHashMap<String, Integer> map = this.suggestions.get(token);
62 this.suggestions.put(token, map);
71 public void add(Token token, int docFreq) { argument
72 addFrequency(token, docFreq);
76 * Adds an original token wit
81 addFrequency(Token token, int docFreq) argument
94 add(Token token, String suggestion, int docFreq) argument
112 get(Token token) argument
122 getTokenFrequency(Token token) argument
[all...]
H A DSpellingQueryConverter.java102 //TODO: Extract the words using a simple regex, but not query stuff, and then analyze them to produce the token stream
128 Token token = new Token();
129 token.copyBuffer(termAtt.buffer(), 0, termAtt.length());
130 token.setStartOffset(offset + offsetAtt.startOffset());
131 token.setEndOffset(offset + offsetAtt.endOffset());
132 token.setFlags(flagsAtt.getFlags());
133 token.setType(typeAtt.type());
134 token.setPayload(payloadAtt.getPayload());
135 token.setPositionIncrement(posIncAtt.getPositionIncrement());
136 result.add(token);
[all...]
/lucene-3.6.0/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/
H A DSingleTokenTokenStream.java28 * A {@link TokenStream} containing a single token.
34 // The token needs to be immutable, so work with clones!
38 public SingleTokenTokenStream(Token token) { argument
41 assert token != null;
42 this.singleToken = (Token) token.clone();
69 public void setToken(Token token) { argument
70 this.singleToken = (Token) token.clone();
H A DPrefixAwareTokenFilter.java34 * Joins two token streams and leaves the last token of the first stream available
35 * to be used when updating the token values in the second stream based on that token.
37 * The default implementation adds last prefix token end offset to the suffix token start and end offsets.
115 private void setCurrentToken(Token token) { argument
116 if (token == null) return;
118 termAtt.copyBuffer(token.buffer(), 0, token
126 getNextPrefixInputToken(Token token) argument
137 getNextSuffixInputToken(Token token) argument
[all...]
/lucene-3.6.0/solr/core/src/test/org/apache/solr/handler/
H A DAnalysisRequestHandlerTestBase.java32 protected void assertToken(NamedList token, TokenInfo info) { argument
33 assertEquals(info.getText(), token.get("text"));
35 assertEquals(info.getRawText(), token.get("raw_text"));
37 assertEquals(info.getType(), token.get("type"));
38 assertEquals(new Integer(info.getStart()), token.get("start"));
39 assertEquals(new Integer(info.getEnd()), token.get("end"));
40 assertEquals(new Integer(info.getPosition()), token.get("position"));
41 assertArrayEquals(info.getPositionHistory(), ArrayUtils.toPrimitive((Integer[]) token.get("positionHistory")));
43 assertEquals(Boolean.TRUE, token.get("match"));
46 assertEquals(info.getPayload(), token
[all...]
H A DAnalysisRequestHandlerTest.java64 NamedList<Object> token;
66 token = tokens.get("token", 0);
67 value = (String) token.get("value");
69 token = tokens.get("token", 1);
70 value = (String) token.get("value");
73 token = tokens.get("token", 2);
74 value = (String) token
[all...]
/lucene-3.6.0/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/
H A DShingleMatrixFilter.java44 * <p>A ShingleMatrixFilter constructs shingles (token n-grams) from a token stream.
45 * In other words, it creates combinations of tokens as a single token.
65 * and leaves the third, the z-axis, for for multi token synonyms.
75 * <p>Consider this token matrix:
135 * @param token
139 public abstract TokenPositioner getTokenPositioner(Token token) throws IOException; argument
144 * @param token
147 public abstract void setTokenPositioner(Token token, ShingleMatrixFilter.TokenPositioner tokenPositioner); argument
151 * @param token
154 getWeight(Token token) argument
161 setWeight(Token token, float weight) argument
392 getNextInputToken(Token token) argument
403 getNextToken(Token token) argument
585 updateToken(Token token, List<Token> shingle, int currentPermutationStartOffset, List<Row> currentPermutationRows, List<Token> currentPermuationTokens) argument
726 Column(Token token) argument
934 getTokenPositioner(Token token) argument
939 setTokenPositioner(Token token, TokenPositioner tokenPositioner) argument
943 getWeight(Token token) argument
948 setWeight(Token token, float weight) argument
962 getTokenPositioner(Token token) argument
971 setTokenPositioner(Token token, TokenPositioner tokenPositioner) argument
976 getWeight(Token token) argument
981 setWeight(Token token, float weight) argument
1003 getTokenPositioner(Token token) argument
1022 setTokenPositioner(Token token, TokenPositioner tokenPositioner) argument
1033 getWeight(Token token) argument
1047 setWeight(Token token, float weight) argument
[all...]
/lucene-3.6.0/solr/core/src/java/org/apache/solr/analysis/
H A DReversedWildcardFilterFactory.java44 * fraction of the query token length. Defaults to 0.0f (disabled).</li>
46 * token after the last wildcard character. For good performance this should be
90 * a query token should be reversed in order to use the
92 * @param token input token.
93 * @return true if input token should be reversed, false otherwise.
95 public boolean shouldReverse(String token) { argument
96 int posQ = token.indexOf('?');
97 int posA = token.indexOf('*');
103 int len = token
[all...]
H A DBufferedTokenStream.java88 * Process a token. Subclasses may read more tokens from the input stream,
89 * write more tokens to the output stream, or simply return the next token
90 * to be output. Subclasses may return null if the token is to be dropped.
93 * the token output stream.
110 * Read a token from the buffered input stream.
122 * Push a token back into the buffered input stream, such that it will
153 Token token = new Token();
154 token.copyBuffer(termAtt.buffer(), 0, termAtt.length());
155 token.setOffset(offsetAtt.startOffset(), offsetAtt.endOffset());
156 token
165 writeToken(Token token) argument
[all...]
/lucene-3.6.0/lucene/contrib/spellchecker/src/java/org/apache/lucene/search/suggest/tst/
H A DTernaryTreeNode.java31 * occurring in the inserted token.
40 String token; field in class:TernaryTreeNode
/lucene-3.6.0/lucene/test-framework/src/java/org/apache/lucene/analysis/
H A DCannedTokenStream.java45 final Token token = tokens[upto++];
50 termAtt.append(token.toString());
51 posIncrAtt.setPositionIncrement(token.getPositionIncrement());
52 offsetAtt.setOffset(token.startOffset(), token.endOffset());
53 payloadAtt.setPayload(token.getPayload());
/lucene-3.6.0/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/
H A DTokenStreamFromTermPositionVector.java62 Token token;
64 token = new Token(terms[i].toCharArray(), 0, terms[i].length(),
67 token = new Token();
68 token.setEmpty().append(terms[i]);
73 token.setPositionIncrement(termPositions[j]);
74 this.positionedTokens.add(token);
79 for (final Token token : this.positionedTokens) {
80 int thisPosition = token.getPositionIncrement();
81 token.setPositionIncrement(thisPosition - lastPosition);
/lucene-3.6.0/lucene/contrib/icu/src/java/org/apache/lucene/analysis/icu/
H A DICUTransformFilter.java119 private CharTermAttribute token; field in class:ICUTransformFilter.ReplaceableTermAttribute
121 void setText(final CharTermAttribute token) { argument
122 this.token = token;
123 this.buffer = token.buffer();
124 this.length = token.length();
158 token.setLength(length = newLength);
167 token.setLength(length = newLength);
176 buffer = token.resizeBuffer(newLength);

Completed in 104 milliseconds

123