高亮exception
2014-04-01 23:23:43,568 ERROR [com.taobao.tsearcher.service.component.SearchHandler] - -->SearchException
com.taobao.tsearcher.common.exeception.TSearcherException: org.apache.lucene.search.highlight.InvalidTokenOffsetsException: Token 测试 exceeds length of provided text sized 2
问题所在
PaodingTokenizer 里面的reset、end 方法的细节处理。
PaodingTokenizer 相关代码和说明如下
@OverridepublicbooleanincrementToken()throwsIOException{clearAttributes();//这里需要注意while(tokenIteractor==null||!tokenIteractor.hasNext()){//System.out.println(dissected);intread=0;intremainning=-1;if(dissected>=beef.length()){remainning=0;}elseif(dissected<0){remainning=bufferLength+dissected;}if(remainning>=0){if(remainning>0){System.arraycopy(buffer, -dissected, buffer, 0, remainning);}read=input.read(buffer, remainning, bufferLength-remainning);inputLength+=read;intcharCount=remainning+read;if(charCount<0){returnfalse;}if(charCount<bufferLength){buffer[charCount++]=0;}beef.set(0, charCount);offset+=Math.abs(dissected);//offset-=remainning;dissected=0;}dissected=knife.dissect((Collector)this, beef, dissected);//offset+=read;//!!!tokenIteractor=tokenCollector.iterator();}Tokentoken=tokenIteractor.next();//log.warn("token.startOffset="+token.startOffset()+",token.endoffset="+token.endOffset());//termAtt.setTermBuffer(token.term());//termAtt.setEmpty();//termAtt.append(token);//v3.*，在4.*中append不行的//termAtt.copyBuffer(token.buffer(), 0, token.length());//wrongforpaodingV4.*termAtt.copyBuffer(token.buffer(), 0, token.buffer().length);//V4.*offsetAtt.setOffset(correctOffset(token.startOffset()), correctOffset(token.endOffset()));//log.warn("token="+token+",startOffset="+offsetAtt.startOffset()+",endOffset="+offsetAtt.endOffset());typeAtt.setType("paoding");returntrue;}@Overridepublicvoidreset()throwsIOException{super.reset();offset=0;inputLength=0;offsetAtt.setOffset(0, 0);termAtt.setEmpty();}@Overridepublicfinalvoidend()throwsIOException{//setfinaloffsetsuper.end();dissected=0;beef.set(0, 0);finalintfinalOffset=correctOffset(offset);this.offsetAtt.setOffset(finalOffset, finalOffset);}
