Suggestion needed for a case of Lucene Migration with TokenStream
"Saha, Rajib" <[email protected]> Thu, 24 Apr 2025 05:35:53 +0000
| Newsgroups | gmane.comp.jakarta.lucene.user |
|---|---|
| Message-ID | <DB9PR02MB83710558C7E75A8A8CF2F5B690852@DB9PR02MB8371.eurprd02.prod.outlook.com> |
Hi Experts,
We are migrating Lucene from 2.4.1 to 8.11.2.
During Migration for a part of code, we are getting below exception in 8.11.2 based changes from Red line colored.
=============
java.lang.IllegalArgumentException: TokenStream fields must be indexed and tokenized
at org.apache.lucene.document.Field.<init>(Field.java:152)
I tied few options. But, could not able to resolve the error. Beiiw
Can somebody of you please help me to identify, where it is going as wrong?
We had code based on 2.4.1 as like below:
===================================
Int currentVal<http://10.238.236.101:8080/source/s?defs=currentVal&project=2025_RTM> = //some value
PayloadTokenStream<http://10.238.236.101:8080/source/s?defs=PayloadTokenStream&project=2025_RTM> tokenStream<http://10.238.236.101:8080/source/s?refs=tokenStream&project=2025_RTM> = new PayloadTokenStream<http://10.238.236.101:8080/source/s?defs=PayloadTokenStream&project=2025_RTM>();
tokenStream<http://10.238.236.101:8080/source/xref/2025_RTM/platform.services.search.java/framework/java/serviceplugins/src/com/sap/businessobjects/platform/search/lucene/index/engine/LuceneIndexEngine.java#tokenStream>.setPayload<http://10.238.236.101:8080/source/s?defs=setPayload&project=2025_RTM>(currentVal<http://10.238.236.101:8080/source/s?defs=currentVal&project=2025_RTM>);
lucField<http://10.238.236.101:8080/source/xref/2025_RTM/platform.services.search.java/framework/java/serviceplugins/src/com/sap/businessobjects/platform/search/lucene/index/engine/LuceneIndexEngine.java#lucField> = new Field<http://10.238.236.101:8080/source/s?defs=Field&project=2025_RTM>(config<http://10.238.236.101:8080/source/s?defs=config&project=2025_RTM>.payloadUid<http://10.238.236.101:8080/source/s?defs=payloadUid&project=2025_RTM>().name<http://10.238.236.101:8080/source/s?defs=name&project=2025_RTM>, tokenStream<http://10.238.236.101:8080/source/xref/2025_RTM/platform.services.search.java/framework/java/serviceplugins/src/com/sap/businessobjects/platform/search/lucene/index/engine/LuceneIndexEngine.java#tokenStream>);
doc<http://10.238.236.101:8080/source/s?defs=doc&project=2025_RTM>.add<http://10.238.236.101:8080/source/s?defs=add&project=2025_RTM>(lucField<http://10.238.236.101:8080/source/xref/2025_RTM/platform.services.search.java/framework/java/serviceplugins/src/com/sap/businessobjects/platform/search/lucene/index/engine/LuceneIndexEngine.java#lucField>);
......
public class PayloadTokenStream<http://10.238.236.101:8080/source/s?refs=PayloadTokenStream&project=2025_RTM> extends TokenStream<http://10.238.236.101:8080/source/s?defs=TokenStream&project=2025_RTM>{
public static String<http://10.238.236.101:8080/source/s?defs=String&project=2025_RTM> UID_PAYLOAD_START_VAL<http://10.238.236.101:8080/source/s?refs=UID_PAYLOAD_START_VAL&project=2025_RTM> = "_UID_";
private Token<http://10.238.236.101:8080/source/s?defs=Token&project=2025_RTM> token<http://10.238.236.101:8080/source/s?refs=token&project=2025_RTM> = new Token<http://10.238.236.101:8080/source/s?defs=Token&project=2025_RTM>(UID_PAYLOAD_START_VAL<http://10.238.236.101:8080/source/xref/2025_RTM/platform.services.search.java/framework/java/sdk/src/com/sap/businessobjects/platform/search/sdk/index/PayloadTokenStream.java#UID_PAYLOAD_START_VAL>,0,0);
private byte[] buffer<http://10.238.236.101:8080/source/s?refs=buffer&project=2025_RTM> = new byte[4];
private boolean returnToken<http://10.238.236.101:8080/source/s?refs=returnToken&project=2025_RTM> = false;
public void setPayload<http://10.238.236.101:8080/source/s?refs=setPayload&project=2025_RTM>(int uid<http://10.238.236.101:8080/source/s?refs=uid&project=2025_RTM>){
buffer<http://10.238.236.101:8080/source/xref/2025_RTM/platform.services.search.java/framework/java/sdk/src/com/sap/businessobjects/platform/search/sdk/index/PayloadTokenStream.java#buffer>[0] = (byte)uid<http://10.238.236.101:8080/source/xref/2025_RTM/platform.services.search.java/framework/java/sdk/src/com/sap/businessobjects/platform/search/sdk/index/PayloadTokenStream.java#uid>;
buffer<http://10.238.236.101:8080/source/xref/2025_RTM/platform.services.search.java/framework/java/sdk/src/com/sap/businessobjects/platform/search/sdk/index/PayloadTokenStream.java#buffer>[1] = (byte)(uid<http://10.238.236.101:8080/source/xref/2025_RTM/platform.services.search.java/framework/java/sdk/src/com/sap/businessobjects/platform/search/sdk/index/PayloadTokenStream.java#uid>>>8);
buffer<http://10.238.236.101:8080/source/xref/2025_RTM/platform.services.search.java/framework/java/sdk/src/com/sap/businessobjects/platform/search/sdk/index/PayloadTokenStream.java#buffer>[2] = (byte)(uid<http://10.238.236.101:8080/source/xref/2025_RTM/platform.services.search.java/framework/java/sdk/src/com/sap/businessobjects/platform/search/sdk/index/PayloadTokenStream.java#uid>>>16);
buffer<http://10.238.236.101:8080/source/xref/2025_RTM/platform.services.search.java/framework/java/sdk/src/com/sap/businessobjects/platform/search/sdk/index/PayloadTokenStream.java#buffer>[3] = (byte)(uid<http://10.238.236.101:8080/source/xref/2025_RTM/platform.services.search.java/framework/java/sdk/src/com/sap/businessobjects/platform/search/sdk/index/PayloadTokenStream.java#uid>>>24);
token<http://10.238.236.101:8080/source/xref/2025_RTM/platform.services.search.java/framework/java/sdk/src/com/sap/businessobjects/platform/search/sdk/index/PayloadTokenStream.java#token>.setPayload<http://10.238.236.101:8080/source/xref/2025_RTM/platform.services.search.java/framework/java/sdk/src/com/sap/businessobjects/platform/search/sdk/index/PayloadTokenStream.java#setPayload>(new Payload<http://10.238.236.101:8080/source/s?defs=Payload&project=2025_RTM>(buffer<http://10.238.236.101:8080/source/xref/2025_RTM/platform.services.search.java/framework/java/sdk/src/com/sap/businessobjects/platform/search/sdk/index/PayloadTokenStream.java#buffer>));
returnToken = true;
}
public Token<http://10.238.236.101:8080/source/s?defs=Token&project=2025_RTM> next<http://10.238.236.101:8080/source/s?refs=next&project=2025_RTM>() throws IOException<http://10.238.236.101:8080/source/s?defs=IOException&project=2025_RTM>{
if (returnToken<http://10.238.236.101:8080/source/xref/2025_RTM/platform.services.search.java/framework/java/sdk/src/com/sap/businessobjects/platform/search/sdk/index/PayloadTokenStream.java#returnToken>){ returnToken<http://10.238.236.101:8080/source/xref/2025_RTM/platform.services.search.java/framework/java/sdk/src/com/sap/businessobjects/platform/search/sdk/index/PayloadTokenStream.java#returnToken> = false; return token<http://10.238.236.101:8080/source/xref/2025_RTM/platform.services.search.java/framework/java/sdk/src/com/sap/businessobjects/platform/search/sdk/index/PayloadTokenStream.java#token>; }
else { return null<http://10.238.236.101:8080/source/s?defs=null&project=2025_RTM>; }
}
}
We have made code based on 8.11.2 as like below:
==========================================
PayloadTokenStream tokenStream = new PayloadTokenStream();
tokenStream.setPayload(currentVal);
FieldType fieldType = new FieldType();
lucField = new Field(config.payloadUid().name, tokenStream, fieldType);
doc.add(lucField);
----
public class PayloadTokenStream<http://10.238.236.101:8080/source/s?refs=PayloadTokenStream&project=2025_RTM> extends TokenStream<http://10.238.236.101:8080/source/s?defs=TokenStream&project=2025_RTM>{
public static String<http://10.238.236.101:8080/source/s?defs=String&project=2025_RTM> UID_PAYLOAD_START_VAL<http://10.238.236.101:8080/source/s?refs=UID_PAYLOAD_START_VAL&project=2025_RTM> = "_UID_";
private byte[] buffer<http://10.238.236.101:8080/source/s?refs=buffer&project=2025_RTM> = new byte[4];
private boolean returnToken<http://10.238.236.101:8080/source/s?refs=returnToken&project=2025_RTM> = false;
public void setPayload<http://10.238.236.101:8080/source/s?refs=setPayload&project=2025_RTM>(int uid<http://10.238.236.101:8080/source/s?refs=uid&project=2025_RTM>){
buffer<http://10.238.236.101:8080/source/xref/2025_RTM/platform.services.search.java/framework/java/sdk/src/com/sap/businessobjects/platform/search/sdk/index/PayloadTokenStream.java#buffer>[0] = (byte)uid<http://10.238.236.101:8080/source/xref/2025_RTM/platform.services.search.java/framework/java/sdk/src/com/sap/businessobjects/platform/search/sdk/index/PayloadTokenStream.java#uid>;
buffer<http://10.238.236.101:8080/source/xref/2025_RTM/platform.services.search.java/framework/java/sdk/src/com/sap/businessobjects/platform/search/sdk/index/PayloadTokenStream.java#buffer>[1] = (byte)(uid<http://10.238.236.101:8080/source/xref/2025_RTM/platform.services.search.java/framework/java/sdk/src/com/sap/businessobjects/platform/search/sdk/index/PayloadTokenStream.java#uid>>>8);
buffer<http://10.238.236.101:8080/source/xref/2025_RTM/platform.services.search.java/framework/java/sdk/src/com/sap/businessobjects/platform/search/sdk/index/PayloadTokenStream.java#buffer>[2] = (byte)(uid<http://10.238.236.101:8080/source/xref/2025_RTM/platform.services.search.java/framework/java/sdk/src/com/sap/businessobjects/platform/search/sdk/index/PayloadTokenStream.java#uid>>>16);
buffer<http://10.238.236.101:8080/source/xref/2025_RTM/platform.services.search.java/framework/java/sdk/src/com/sap/businessobjects/platform/search/sdk/index/PayloadTokenStream.java#buffer>[3] = (byte)(uid<http://10.238.236.101:8080/source/xref/2025_RTM/platform.services.search.java/framework/java/sdk/src/com/sap/businessobjects/platform/search/sdk/index/PayloadTokenStream.java#uid>>>24);
PayloadAttributeImpl attributeImpl = new PayloadAttributeImpl(new BytesRef(buffer));
addAttributeImpl(attributeImpl);
returnToken = true;
}
public boolean incrementToken() throws IOException {
if (returnToken){
returnToken = false;
return true;
}
else {
return false;
}
}
}
Regards
Rajib