/lucene-3.6.0/lucene/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/ |
H A D | SegTokenFilter.java | 39 * @param token input {@link SegToken} 42 public SegToken filter(SegToken token) { argument 43 switch (token.wordType) { 46 for (int i = 0; i < token.charArray.length; i++) { 47 if (token.charArray[i] >= 0xFF10) 48 token.charArray[i] -= 0xFEE0; 50 if (token.charArray[i] >= 0x0041 && token.charArray[i] <= 0x005A) /* lowercase latin */ 51 token.charArray[i] += 0x0020; 55 for (int i = 0; i < token [all...] |
H A D | HHMMSegmenter.java | 48 SegToken token; 70 token = new SegToken(charArray, i, j, WordType.CHINESE_WORD, 72 segGraph.addToken(token); 80 token = new SegToken(charArray, i, j, WordType.CHINESE_WORD, 82 segGraph.addToken(token); 117 token = new SegToken(charArray, i, j, wordType, frequency); 118 segGraph.addToken(token); 135 token = new SegToken(charArray, i, j, wordType, frequency); 136 segGraph.addToken(token); 144 token [all...] |
H A D | SegGraph.java | 71 * Set the {@link SegToken#index} for each token, based upon its order by startOffset. 95 * Add a {@link SegToken} to the mapping, creating a new mapping at the token's startOffset if one does not exist. 96 * @param token {@link SegToken} 98 public void addToken(SegToken token) { argument 99 int s = token.startOffset; 102 newlist.add(token); 106 tokenList.add(token);
|
/lucene-3.6.0/lucene/contrib/analyzers/kuromoji/src/java/org/apache/lucene/analysis/ja/tokenattributes/ |
H A D | BaseFormAttributeImpl.java | 28 private Token token; field in class:BaseFormAttributeImpl 31 return token == null ? null : token.getBaseForm(); 34 public void setToken(Token token) { argument 35 this.token = token; 40 token = null; 46 t.setToken(token);
|
H A D | InflectionAttributeImpl.java | 29 private Token token; field in class:InflectionAttributeImpl 32 return token == null ? null : token.getInflectionType(); 36 return token == null ? null : token.getInflectionForm(); 39 public void setToken(Token token) { argument 40 this.token = token; 45 token = null; 51 t.setToken(token); [all...] |
H A D | ReadingAttributeImpl.java | 29 private Token token; field in class:ReadingAttributeImpl 32 return token == null ? null : token.getReading(); 36 return token == null ? null : token.getPronunciation(); 39 public void setToken(Token token) { argument 40 this.token = token; 45 token = null; 51 t.setToken(token); [all...] |
H A D | PartOfSpeechAttributeImpl.java | 29 private Token token; field in class:PartOfSpeechAttributeImpl 32 return token == null ? null : token.getPartOfSpeech(); 35 public void setToken(Token token) { argument 36 this.token = token; 41 token = null; 47 t.setToken(token);
|
H A D | BaseFormAttribute.java | 31 public void setToken(Token token); argument
|
H A D | InflectionAttribute.java | 32 public void setToken(Token token); argument
|
H A D | PartOfSpeechAttribute.java | 28 public void setToken(Token token); argument
|
H A D | ReadingAttribute.java | 32 public void setToken(Token token); argument
|
/lucene-3.6.0/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/ |
H A D | TestSingleTokenTokenFilter.java | 30 Token token = new Token(); 31 SingleTokenTokenStream ts = new SingleTokenTokenStream(token); 37 assertEquals(token, tokenAtt); 40 token = new Token("hallo", 10, 20, "someType"); 41 ts.setToken(token); 45 assertEquals(token, tokenAtt);
|
/lucene-3.6.0/solr/core/src/java/org/apache/solr/spelling/ |
H A D | SpellingResult.java | 37 * Key == token 38 * Value = Map -> key is the suggestion, value is the frequency of the token in the collection 55 * @param token The token to associate the suggestions with 58 public void add(Token token, List<String> suggestions) { argument 59 LinkedHashMap<String, Integer> map = this.suggestions.get(token); 62 this.suggestions.put(token, map); 71 public void add(Token token, int docFreq) { argument 72 addFrequency(token, docFreq); 76 * Adds an original token wit 81 addFrequency(Token token, int docFreq) argument 94 add(Token token, String suggestion, int docFreq) argument 112 get(Token token) argument 122 getTokenFrequency(Token token) argument [all...] |
H A D | SpellingQueryConverter.java | 102 //TODO: Extract the words using a simple regex, but not query stuff, and then analyze them to produce the token stream 128 Token token = new Token(); 129 token.copyBuffer(termAtt.buffer(), 0, termAtt.length()); 130 token.setStartOffset(offset + offsetAtt.startOffset()); 131 token.setEndOffset(offset + offsetAtt.endOffset()); 132 token.setFlags(flagsAtt.getFlags()); 133 token.setType(typeAtt.type()); 134 token.setPayload(payloadAtt.getPayload()); 135 token.setPositionIncrement(posIncAtt.getPositionIncrement()); 136 result.add(token); [all...] |
/lucene-3.6.0/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/ |
H A D | SingleTokenTokenStream.java | 28 * A {@link TokenStream} containing a single token. 34 // The token needs to be immutable, so work with clones! 38 public SingleTokenTokenStream(Token token) { argument 41 assert token != null; 42 this.singleToken = (Token) token.clone(); 69 public void setToken(Token token) { argument 70 this.singleToken = (Token) token.clone();
|
H A D | PrefixAwareTokenFilter.java | 34 * Joins two token streams and leaves the last token of the first stream available 35 * to be used when updating the token values in the second stream based on that token. 37 * The default implementation adds last prefix token end offset to the suffix token start and end offsets. 115 private void setCurrentToken(Token token) { argument 116 if (token == null) return; 118 termAtt.copyBuffer(token.buffer(), 0, token 126 getNextPrefixInputToken(Token token) argument 137 getNextSuffixInputToken(Token token) argument [all...] |
/lucene-3.6.0/solr/core/src/test/org/apache/solr/handler/ |
H A D | AnalysisRequestHandlerTestBase.java | 32 protected void assertToken(NamedList token, TokenInfo info) { argument 33 assertEquals(info.getText(), token.get("text")); 35 assertEquals(info.getRawText(), token.get("raw_text")); 37 assertEquals(info.getType(), token.get("type")); 38 assertEquals(new Integer(info.getStart()), token.get("start")); 39 assertEquals(new Integer(info.getEnd()), token.get("end")); 40 assertEquals(new Integer(info.getPosition()), token.get("position")); 41 assertArrayEquals(info.getPositionHistory(), ArrayUtils.toPrimitive((Integer[]) token.get("positionHistory"))); 43 assertEquals(Boolean.TRUE, token.get("match")); 46 assertEquals(info.getPayload(), token [all...] |
H A D | AnalysisRequestHandlerTest.java | 64 NamedList<Object> token; 66 token = tokens.get("token", 0); 67 value = (String) token.get("value"); 69 token = tokens.get("token", 1); 70 value = (String) token.get("value"); 73 token = tokens.get("token", 2); 74 value = (String) token [all...] |
/lucene-3.6.0/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ |
H A D | ShingleMatrixFilter.java | 44 * <p>A ShingleMatrixFilter constructs shingles (token n-grams) from a token stream. 45 * In other words, it creates combinations of tokens as a single token. 65 * and leaves the third, the z-axis, for for multi token synonyms. 75 * <p>Consider this token matrix: 135 * @param token 139 public abstract TokenPositioner getTokenPositioner(Token token) throws IOException; argument 144 * @param token 147 public abstract void setTokenPositioner(Token token, ShingleMatrixFilter.TokenPositioner tokenPositioner); argument 151 * @param token 154 getWeight(Token token) argument 161 setWeight(Token token, float weight) argument 392 getNextInputToken(Token token) argument 403 getNextToken(Token token) argument 585 updateToken(Token token, List<Token> shingle, int currentPermutationStartOffset, List<Row> currentPermutationRows, List<Token> currentPermuationTokens) argument 726 Column(Token token) argument 934 getTokenPositioner(Token token) argument 939 setTokenPositioner(Token token, TokenPositioner tokenPositioner) argument 943 getWeight(Token token) argument 948 setWeight(Token token, float weight) argument 962 getTokenPositioner(Token token) argument 971 setTokenPositioner(Token token, TokenPositioner tokenPositioner) argument 976 getWeight(Token token) argument 981 setWeight(Token token, float weight) argument 1003 getTokenPositioner(Token token) argument 1022 setTokenPositioner(Token token, TokenPositioner tokenPositioner) argument 1033 getWeight(Token token) argument 1047 setWeight(Token token, float weight) argument [all...] |
/lucene-3.6.0/solr/core/src/java/org/apache/solr/analysis/ |
H A D | ReversedWildcardFilterFactory.java | 44 * fraction of the query token length. Defaults to 0.0f (disabled).</li> 46 * token after the last wildcard character. For good performance this should be 90 * a query token should be reversed in order to use the 92 * @param token input token. 93 * @return true if input token should be reversed, false otherwise. 95 public boolean shouldReverse(String token) { argument 96 int posQ = token.indexOf('?'); 97 int posA = token.indexOf('*'); 103 int len = token [all...] |
H A D | BufferedTokenStream.java | 88 * Process a token. Subclasses may read more tokens from the input stream, 89 * write more tokens to the output stream, or simply return the next token 90 * to be output. Subclasses may return null if the token is to be dropped. 93 * the token output stream. 110 * Read a token from the buffered input stream. 122 * Push a token back into the buffered input stream, such that it will 153 Token token = new Token(); 154 token.copyBuffer(termAtt.buffer(), 0, termAtt.length()); 155 token.setOffset(offsetAtt.startOffset(), offsetAtt.endOffset()); 156 token 165 writeToken(Token token) argument [all...] |
/lucene-3.6.0/lucene/contrib/spellchecker/src/java/org/apache/lucene/search/suggest/tst/ |
H A D | TernaryTreeNode.java | 31 * occurring in the inserted token. 40 String token; field in class:TernaryTreeNode
|
/lucene-3.6.0/lucene/test-framework/src/java/org/apache/lucene/analysis/ |
H A D | CannedTokenStream.java | 45 final Token token = tokens[upto++]; 50 termAtt.append(token.toString()); 51 posIncrAtt.setPositionIncrement(token.getPositionIncrement()); 52 offsetAtt.setOffset(token.startOffset(), token.endOffset()); 53 payloadAtt.setPayload(token.getPayload());
|
/lucene-3.6.0/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/ |
H A D | TokenStreamFromTermPositionVector.java | 62 Token token; 64 token = new Token(terms[i].toCharArray(), 0, terms[i].length(), 67 token = new Token(); 68 token.setEmpty().append(terms[i]); 73 token.setPositionIncrement(termPositions[j]); 74 this.positionedTokens.add(token); 79 for (final Token token : this.positionedTokens) { 80 int thisPosition = token.getPositionIncrement(); 81 token.setPositionIncrement(thisPosition - lastPosition);
|
/lucene-3.6.0/lucene/contrib/icu/src/java/org/apache/lucene/analysis/icu/ |
H A D | ICUTransformFilter.java | 119 private CharTermAttribute token; field in class:ICUTransformFilter.ReplaceableTermAttribute 121 void setText(final CharTermAttribute token) { argument 122 this.token = token; 123 this.buffer = token.buffer(); 124 this.length = token.length(); 158 token.setLength(length = newLength); 167 token.setLength(length = newLength); 176 buffer = token.resizeBuffer(newLength);
|