org.apache.lucene.analysis.Token.setPositionIncrement()方法的使用及代码示例

x33g5p2x  于2022-01-30 转载在 其他  
字(10.1k)|赞(0)|评价(0)|浏览(142)

本文整理了Java中org.apache.lucene.analysis.Token.setPositionIncrement()方法的一些代码示例，展示了Token.setPositionIncrement()的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台，是从一些精选项目中提取出来的代码，具有较强的参考意义，能在一定程度帮忙到你。Token.setPositionIncrement()方法的具体详情如下：
包路径：org.apache.lucene.analysis.Token
类名称：Token
方法名：setPositionIncrement

Token.setPositionIncrement介绍

暂无

代码示例

代码示例来源：origin: treygrainger/solr-in-action

/**
 * Convert the mapping{position, terms} to a list of tokens with appropriate
 * position increments.
 */
private static LinkedList<Token> mergeToSingleTokenStream(
  SortedMap<Integer, LinkedList<Token>> tokenHash) {
 LinkedList<Token> result = new LinkedList<Token>();
 int currentPosition = 0;
 for (int newPosition : tokenHash.keySet()) {
  int incrementTokenIndex = result.size();
  LinkedList<Token> brothers = tokenHash.get(newPosition);
  /*
   * The first item in the list gets the position increment; the rest have a
   * position increment of 0.
   */
  int positionIncrement = newPosition - currentPosition;
  // set all token to 0 increment
  for (Token token : brothers) {
   token.setPositionIncrement(0);
   result.add(token);
  }
  // increment position of the first token
  if (result.size() > incrementTokenIndex
    && result.get(incrementTokenIndex) != null) {
   result.get(incrementTokenIndex).setPositionIncrement(positionIncrement);
  }
  currentPosition = newPosition;
 }
 return result;
}

代码示例来源：origin: org.dspace.dependencies.solr/dspace-solr-core

/** Construct a compound token. */
private Token gramToken(Token first, Token second) {
 buffer.setLength(0);
 buffer.append(first.termText());
 buffer.append(SEPARATOR);
 buffer.append(second.termText());
 Token result = new Token(buffer.toString(), first.startOffset(), second
   .endOffset(), "gram");
 result.setPositionIncrement(0);
 return result;
}

代码示例来源：origin: ajermakovics/eclipse-instasearch

private void splitIntoTokens()
{
  String term = termAtt.term();
  String[] termParts = splitTerm(term);
  if(termParts.length > 1)
  {
    int termPos = offsetAtt.startOffset();
    
    for (int i = 0; i < termParts.length; i++) 
    {
      String termPart = termParts[i];
      int termPartPos = termPos + term.indexOf(termPart);
      int termPartEndPos = termPartPos + termPart.length();
      Token newToken = new Token(termPart, termPartPos, termPartEndPos);
      newToken.setPositionIncrement(0); // in the same position
      tokens.add( newToken );
    }
  }
}

代码示例来源：origin: org.dspace.dependencies.solr/dspace-solr-core

concatTok.setPositionIncrement(generateSubwords==true ? 0 : posOffset);
queue.add(concatTok);

代码示例来源：origin: org.compass-project/compass

private void addAliasesToStack(Token token) {
    String[] synonyms = synonymLookupProvider.lookupSynonyms(token.termText());

    if (synonyms == null) {
      return;
    }

    for (int i = 0; i < synonyms.length; i++) {
      Token synToken = new Token(synonyms[i], token.startOffset(), token.endOffset(), TOKEN_TYPE_SYNONYM);
      synToken.setPositionIncrement(0);
      synonymStack.addFirst(synToken);
    }
  }
}

代码示例来源：origin: org.apache.lucene/lucene-analyzers

/**
 * Final touch of a shingle token before it is passed on to the consumer from method {@link #incrementToken()}.
 *
 * Calculates and sets type, flags, position increment, start/end offsets and weight.
 *
 * @param token Shingle token
 * @param shingle Tokens used to produce the shingle token.
 * @param currentPermutationStartOffset Start offset in parameter currentPermutationTokens
 * @param currentPermutationRows index to Matrix.Column.Row from the position of tokens in parameter currentPermutationTokens
 * @param currentPermuationTokens tokens of the current permutation of rows in the matrix.
 */
public void updateToken(Token token, List<Token> shingle, int currentPermutationStartOffset, List<Row> currentPermutationRows, List<Token> currentPermuationTokens) {
 token.setType(ShingleMatrixFilter.class.getName());
 token.setFlags(0);
 token.setPositionIncrement(1);
 token.setStartOffset(shingle.get(0).startOffset());
 token.setEndOffset(shingle.get(shingle.size() - 1).endOffset());
 settingsCodec.setWeight(token, calculateShingleWeight(token, shingle, currentPermutationStartOffset, currentPermutationRows, currentPermuationTokens));
}

代码示例来源：origin: org.dspace.dependencies.solr/dspace-solr-core

return token;
Token mergedToken = new Token(termText.toString(), startOffset, token.endOffset(), token.type());
mergedToken.setPositionIncrement(firstPositionIncrement);
return mergedToken;

代码示例来源：origin: org.dspace.dependencies.solr/dspace-solr-core

Token tok = new Token(tok1.startOffset(), tok1.endOffset(), tok1.type());
tok.setTermBuffer(tok1.termBuffer(), 0, tok1.termLength());
tok.setPositionIncrement(pos1-pos);
result.add(tok);
pos=pos1;
Token tok = new Token(tok2.startOffset(), tok2.endOffset(), tok2.type());
tok.setTermBuffer(tok2.termBuffer(), 0, tok2.termLength());
tok.setPositionIncrement(pos2-pos);
result.add(tok);
pos=pos2;

代码示例来源：origin: org.apache.lucene/lucene-analyzers

private Token getNextInputToken(Token token) throws IOException {
 if (!input.incrementToken()) return null;
 token.copyBuffer(in_termAtt.buffer(), 0, in_termAtt.length());
 token.setPositionIncrement(in_posIncrAtt.getPositionIncrement());
 token.setFlags(in_flagsAtt.getFlags());
 token.setOffset(in_offsetAtt.startOffset(), in_offsetAtt.endOffset());
 token.setType(in_typeAtt.type());
 token.setPayload(in_payloadAtt.getPayload());
 return token;
}

代码示例来源：origin: org.compass-project/compass

nextToken.startOffset() + curOffset,
    nextToken.endOffset() + curOffset);
offsetToken.setPositionIncrement(nextToken.getPositionIncrement() + extra * 10);
return offsetToken;

代码示例来源：origin: org.infinispan/infinispan-embedded-query

private Token getNextSuffixInputToken(Token token) throws IOException {
 if (!suffix.incrementToken()) return null;
 token.copyBuffer(termAtt.buffer(), 0, termAtt.length());
 token.setPositionIncrement(posIncrAtt.getPositionIncrement());
 token.setFlags(flagsAtt.getFlags());
 token.setOffset(offsetAtt.startOffset(), offsetAtt.endOffset());
 token.setType(typeAtt.type());
 token.setPayload(payloadAtt.getPayload());
 return token;
}

代码示例来源：origin: org.apache.lucene/com.springsource.org.apache.lucene

/**
 * Returns the next input Token whose term() is not a stop word.
 */
public final Token next(final Token reusableToken) throws IOException {
 assert reusableToken != null;
 // return the first non-stop word found
 int skippedPositions = 0;
 for (Token nextToken = input.next(reusableToken); nextToken != null; nextToken = input.next(reusableToken)) {
  if (!stopWords.contains(nextToken.termBuffer(), 0, nextToken.termLength())) {
   if (enablePositionIncrements) {
    nextToken.setPositionIncrement(nextToken.getPositionIncrement() + skippedPositions);
   }
   return nextToken;
  }
  skippedPositions += nextToken.getPositionIncrement();
 }
 // reached EOS -- return null
 return null;
}

代码示例来源：origin: org.apache.lucene/lucene-analyzers

private Token getNextSuffixInputToken(Token token) throws IOException {
 if (!suffix.incrementToken()) return null;
 token.copyBuffer(termAtt.buffer(), 0, termAtt.length());
 token.setPositionIncrement(posIncrAtt.getPositionIncrement());
 token.setFlags(flagsAtt.getFlags());
 token.setOffset(offsetAtt.startOffset(), offsetAtt.endOffset());
 token.setType(typeAtt.type());
 token.setPayload(payloadAtt.getPayload());
 return token;
}

代码示例来源：origin: org.apache.lucene/lucene-core-jfrog

/**
 * Returns the next input Token whose term() is not a stop word.
 */
public final Token next(final Token reusableToken) throws IOException {
 assert reusableToken != null;
 // return the first non-stop word found
 int skippedPositions = 0;
 for (Token nextToken = input.next(reusableToken); nextToken != null; nextToken = input.next(reusableToken)) {
  if (!stopWords.contains(nextToken.termBuffer(), 0, nextToken.termLength())) {
   if (enablePositionIncrements) {
    nextToken.setPositionIncrement(nextToken.getPositionIncrement() + skippedPositions);
   }
   return nextToken;
  }
  skippedPositions += nextToken.getPositionIncrement();
 }
 // reached EOS -- return null
 return null;
}

代码示例来源：origin: org.apache.lucene/lucene-analyzers

private Token getNextPrefixInputToken(Token token) throws IOException {
 if (!prefix.incrementToken()) return null;
 token.copyBuffer(p_termAtt.buffer(), 0, p_termAtt.length());
 token.setPositionIncrement(p_posIncrAtt.getPositionIncrement());
 token.setFlags(p_flagsAtt.getFlags());
 token.setOffset(p_offsetAtt.startOffset(), p_offsetAtt.endOffset());
 token.setType(p_typeAtt.type());
 token.setPayload(p_payloadAtt.getPayload());
 return token;
}

代码示例来源：origin: org.infinispan/infinispan-embedded-query

private Token getNextPrefixInputToken(Token token) throws IOException {
 if (!prefix.incrementToken()) return null;
 token.copyBuffer(p_termAtt.buffer(), 0, p_termAtt.length());
 token.setPositionIncrement(p_posIncrAtt.getPositionIncrement());
 token.setFlags(p_flagsAtt.getFlags());
 token.setOffset(p_offsetAtt.startOffset(), p_offsetAtt.endOffset());
 token.setType(p_typeAtt.type());
 token.setPayload(p_payloadAtt.getPayload());
 return token;
}

代码示例来源：origin: org.apache.lucene/lucene-analyzers

private Token getNextToken(Token token) throws IOException {
 if (!this.incrementToken()) return null;
 token.copyBuffer(termAtt.buffer(), 0, termAtt.length());
 token.setPositionIncrement(posIncrAtt.getPositionIncrement());
 token.setFlags(flagsAtt.getFlags());
 token.setOffset(offsetAtt.startOffset(), offsetAtt.endOffset());
 token.setType(typeAtt.type());
 token.setPayload(payloadAtt.getPayload());
 return token;
}

代码示例来源：origin: DiceTechJobs/SolrPlugins

private Collection<Token> getTokens(String q, Analyzer analyzer) throws IOException {
  Collection<Token> result = new ArrayList<Token>();
  assert analyzer != null;
  TokenStream ts = analyzer.tokenStream("", q);
  try {
    ts.reset();
    // TODO: support custom attributes
    CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
    OffsetAttribute offsetAtt = ts.addAttribute(OffsetAttribute.class);
    TypeAttribute typeAtt = ts.addAttribute(TypeAttribute.class);
    FlagsAttribute flagsAtt = ts.addAttribute(FlagsAttribute.class);
    PayloadAttribute payloadAtt = ts.addAttribute(PayloadAttribute.class);
    PositionIncrementAttribute posIncAtt = ts.addAttribute(PositionIncrementAttribute.class);
    while (ts.incrementToken()){
      Token token = new Token();
      token.copyBuffer(termAtt.buffer(), 0, termAtt.length());
      token.setOffset(offsetAtt.startOffset(), offsetAtt.endOffset());
      token.setType(typeAtt.type());
      token.setFlags(flagsAtt.getFlags());
      token.setPayload(payloadAtt.getPayload());
      token.setPositionIncrement(posIncAtt.getPositionIncrement());
      result.add(token);
    }
    ts.end();
    return result;
  } finally {
    IOUtils.closeWhileHandlingException(ts);
  }
}

代码示例来源：origin: org.apache.lucene/com.springsource.org.apache.lucene

reusableToken.setPositionIncrement(posIncr);
scanner.getText(reusableToken);
final int start = scanner.yychar();

代码示例来源：origin: org.apache.lucene/lucene-core-jfrog

reusableToken.setPositionIncrement(posIncr);
scanner.getText(reusableToken);
final int start = scanner.yychar();

内容来源于网络，如有侵权，请联系作者删除！

token

相关文章

    查看更多

    热门标签

    更多
    JavaquerypythonNode开发语言requestUtil数据库Table后端算法LoggerMessageElementParser

    最新文章

    更多
    • 蜀ICP备13028337号-1 大数据知识库 https://www.saoniuhuo.com © All rights reserved
    • 本站内容来源互联网,如果侵犯您的权益请联系我们删除, 联系方式：448109455@qq.com