Author: natalia
Date: Fri Oct 19 17:32:43 2007
New Revision: 586647
URL: http://svn.apache.org/viewvc?rev=586647&view=rev
Log:
Added extention XPath function 'ftcontains' for full text search
Added:
xml/xindice/trunk/java/src/org/apache/xindice/core/query/FuncFTContains.java (with props)
xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/SpecialQueryParser.java (with props)
Modified:
xml/xindice/trunk/java/src/org/apache/xindice/core/indexer/LuceneIndexer.java
xml/xindice/trunk/java/src/org/apache/xindice/core/query/XPathQueryResolver.java
Modified: xml/xindice/trunk/java/src/org/apache/xindice/core/indexer/LuceneIndexer.java
URL: http://svn.apache.org/viewvc/xml/xindice/trunk/java/src/org/apache/xindice/core/indexer/LuceneIndexer.java?rev=586647&r1=586646&r2=586647&view=diff
==============================================================================
--- xml/xindice/trunk/java/src/org/apache/xindice/core/indexer/LuceneIndexer.java (original)
+++ xml/xindice/trunk/java/src/org/apache/xindice/core/indexer/LuceneIndexer.java Fri Oct 19 17:32:43 2007
@@ -91,10 +91,9 @@
private static final String PATTERN_ALIAS = "alias";
public static final String KEYNAME = "key";
- public static final String TEXTNAME = "text";
// Default analyzer to use
- private static final String DEFANALYZER = "org.apache.lucene.analysis.SimpleAnalyzer";
+ public static final String DEFANALYZER = "org.apache.lucene.analysis.SimpleAnalyzer";
private static final IndexMatch[] EMPTY_MATCHES = new IndexMatch[0];
private File idxFile;
@@ -147,6 +146,31 @@
*/
public IndexPattern[] getPatterns() {
return (IndexPattern[]) patterns.keySet().toArray(new IndexPattern[0]);
+ }
+
+ /**
+ * Return alias for the given pattern. If this exact pattern is not indexed,
+ * method will look for matching indexed pattern.
+ * @param pattern IndexPattern
+ * @return Alias for the closest matching pattern or null, if there is none
+ */
+ public String getPatternAlias(IndexPattern pattern) {
+ if (patterns.containsKey(pattern)) {
+ return (String) patterns.get(pattern);
+ }
+
+ int match = 0;
+ IndexPattern matchPattern = null;
+ for (Iterator i = patterns.keySet().iterator(); i.hasNext(); ) {
+ IndexPattern p = (IndexPattern) i.next();
+ int cMatch = pattern.getMatchLevel(p);
+ if (cMatch > match) {
+ match = cMatch;
+ matchPattern = p;
+ }
+ }
+
+ return (String) patterns.get(matchPattern);
}
/**
Added: xml/xindice/trunk/java/src/org/apache/xindice/core/query/FuncFTContains.java
URL: http://svn.apache.org/viewvc/xml/xindice/trunk/java/src/org/apache/xindice/core/query/FuncFTContains.java?rev=586647&view=auto
==============================================================================
--- xml/xindice/trunk/java/src/org/apache/xindice/core/query/FuncFTContains.java (added)
+++ xml/xindice/trunk/java/src/org/apache/xindice/core/query/FuncFTContains.java Fri Oct 19 17:32:43 2007
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * $Id$
+ */
+
+package org.apache.xindice.core.query;
+
+import org.apache.xpath.XPathContext;
+import org.apache.xpath.functions.FunctionOneArg;
+import org.apache.xpath.objects.XObject;
+import org.apache.xpath.objects.XBoolean;
+import org.apache.lucene.queryParser.ParseException;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.xindice.core.query.ftsearch.Searcher;
+import org.apache.xindice.core.data.NodeSet;
+import org.apache.xindice.core.Collection;
+import org.apache.xindice.core.indexer.Indexer;
+import org.apache.xindice.core.indexer.LuceneIndexer;
+import org.apache.xindice.xml.dom.NodeListImpl;
+import org.w3c.dom.Node;
+
+import javax.xml.transform.TransformerException;
+
+/**
+ * Full text search extention function for XPath.
+ *
+ * @version $Revision$, $Date$
+ */
+public class FuncFTContains extends FunctionOneArg {
+
+ /**
+ * Execute text search function. The function returns XBoolean.S_TRUE
+ * if the current node text matches the query, XBoolean.S_FALSE otherwise.
+ * @param xctxt The current execution context.
+ * @return XBoolean.S_TRUE or XBoolean.S_FALSE.
+ *
+ * @throws javax.xml.transform.TransformerException
+ */
+ public XObject execute(XPathContext xctxt) throws TransformerException {
+
+ XPathQueryResolver.XPathResolverContext ctxt = (XPathQueryResolver.XPathResolverContext) xctxt;
+
+ // it is not guaranteed that analyzer will be set at this time, even if there is
+ // a suitable index in the collection, for various reasons XPathQueryResolver may
+ // decide not to run optimization for the function
+ Analyzer analyzer = (Analyzer) ctxt.getParameter(XPathQueryResolver.PARAM_ANALYZER);
+ if (analyzer == null) {
+ try {
+ Collection collection = (Collection) ctxt.getParameter(XPathQueryResolver.PARAM_COLLECTION);
+ Indexer idx = collection.getIndexManager().getBestIndexer(Indexer.STYLE_FULLTEXT, null);
+ if (idx instanceof LuceneIndexer) {
+ analyzer = ((LuceneIndexer) idx).getAnalyzer();
+ } else {
+ analyzer = (Analyzer) Class.forName(LuceneIndexer.DEFANALYZER).newInstance();
+ }
+
+ // parameters are intentionally altered so next node or next document won't need
+ // to do an analyzer lookup
+ ctxt.setParameter(XPathQueryResolver.PARAM_ANALYZER, analyzer);
+ } catch (Exception e) {
+ throw new TransformerException("Could not get text analyzer");
+ }
+ }
+
+ String query = getArg0().execute(xctxt).str();
+ try {
+ int ctxtNode = xctxt.getCurrentNode();
+ Node node = xctxt.getDTM(ctxtNode).getNode(ctxtNode);
+ NodeListImpl list = new NodeListImpl(null);
+ list.add(node);
+
+ Searcher searcher = new Searcher(list, analyzer);
+ NodeSet nodes = searcher.search(query);
+
+ return nodes.hasMoreNodes() ? XBoolean.S_TRUE : XBoolean.S_FALSE;
+ } catch (ParseException e) {
+ throw new TransformerException("Error in text query", e);
+ }
+ }
+}
Propchange: xml/xindice/trunk/java/src/org/apache/xindice/core/query/FuncFTContains.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: xml/xindice/trunk/java/src/org/apache/xindice/core/query/FuncFTContains.java
------------------------------------------------------------------------------
svn:keywords = Id Revision Author Date
Modified: xml/xindice/trunk/java/src/org/apache/xindice/core/query/XPathQueryResolver.java
URL: http://svn.apache.org/viewvc/xml/xindice/trunk/java/src/org/apache/xindice/core/query/XPathQueryResolver.java?rev=586647&r1=586646&r2=586647&view=diff
==============================================================================
--- xml/xindice/trunk/java/src/org/apache/xindice/core/query/XPathQueryResolver.java (original)
+++ xml/xindice/trunk/java/src/org/apache/xindice/core/query/XPathQueryResolver.java Fri Oct 19 17:32:43 2007
@@ -23,6 +23,7 @@
import org.apache.commons.logging.LogFactory;
import org.apache.xindice.core.Collection;
import org.apache.xindice.core.DBException;
+import org.apache.xindice.core.query.ftsearch.SpecialQueryParser;
import org.apache.xindice.core.data.Entry;
import org.apache.xindice.core.data.Key;
import org.apache.xindice.core.data.NodeSet;
@@ -33,6 +34,7 @@
import org.apache.xindice.core.indexer.IndexPattern;
import org.apache.xindice.core.indexer.IndexQuery;
import org.apache.xindice.core.indexer.Indexer;
+import org.apache.xindice.core.indexer.LuceneIndexer;
import org.apache.xindice.core.indexer.helpers.IndexQueryANY;
import org.apache.xindice.core.indexer.helpers.IndexQueryEQ;
import org.apache.xindice.core.indexer.helpers.IndexQueryGEQ;
@@ -65,6 +67,7 @@
import org.apache.xpath.objects.XNumber;
import org.apache.xpath.objects.XObject;
import org.apache.xpath.objects.XString;
+import org.apache.lucene.analysis.Analyzer;
import org.w3c.dom.DOMException;
import org.w3c.dom.Element;
@@ -79,11 +82,14 @@
import javax.xml.transform.TransformerException;
import java.lang.reflect.Constructor;
+import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.SortedSet;
import java.util.TreeSet;
+import java.util.HashMap;
+import java.util.Map;
/**
* XPathQueryResolver
@@ -101,6 +107,9 @@
public static final String STYLE_XPATH = "XPath";
+ static final String PARAM_COLLECTION = "collection";
+ static final String PARAM_ANALYZER = "analyzer";
+
// Maps Xalan Comparisons To IndexQuery
private static final int[] OPMAP = {
IndexQuery.NEQ, IndexQuery.EQ, IndexQuery.LEQ, IndexQuery.LT, IndexQuery.GEQ, IndexQuery.GT
@@ -111,18 +120,31 @@
private static final boolean XCOMPILER3;
// Xalan Compiler constructor
private static final Constructor XCOMPILER;
+ // XPath constructor
+ private static final Constructor XPATH;
static {
boolean c3;
Constructor c;
+ Constructor x;
try {
c = Compiler.class.getConstructor(
new Class[] { ErrorListener.class, SourceLocator.class, FunctionTable.class });
+
+ x = XPath.class.getConstructor(
+ new Class[] { String.class, SourceLocator.class, PrefixResolver.class,
+ int.class, ErrorListener.class, FunctionTable.class });
+
c3 = true;
} catch (NoSuchMethodException nsme) {
try {
c = Compiler.class.getConstructor(
new Class[] { ErrorListener.class, SourceLocator.class });
+
+ x = XPath.class.getConstructor(
+ new Class[] { String.class, SourceLocator.class, PrefixResolver.class,
+ int.class, ErrorListener.class });
+
c3 = false;
} catch (NoSuchMethodException e) {
// Should not happen
@@ -133,20 +155,47 @@
XCOMPILER3 = c3;
XCOMPILER = c;
+ XPATH = x;
}
private DefaultErrorHandler errorListener;
private FunctionTable functionTable;
private boolean autoIndex;
+ private int funcFTContainsId;
public XPathQueryResolver() {
super();
errorListener = new DefaultErrorHandler();
- if (XCOMPILER3) {
- functionTable = new FunctionTable();
+
+ Object num = null;
+ try {
+ Method install;
+ Object function;
+
+ if (XCOMPILER3) {
+ functionTable = new FunctionTable();
+ function = FuncFTContains.class;
+ install = functionTable.getClass().getMethod("installFunction",
+ new Class[] {String.class, Class.class});
+ num = install.invoke(functionTable, new Object[] {"ftcontains", function});
+ } else {
+ function = new FuncFTContains();
+ install = FunctionTable.class.getMethod("installFunction",
+ new Class[] {String.class, Expression.class});
+ num = install.invoke(FunctionTable.class, new Object[] {"ftcontains", function});
+ }
+ } catch (Exception e) {
+ // no extentions will be available
+ log.error("Could not invoke installFunction method. Incompatible Xalan version?");
+ }
+
+ if (num != null) {
+ funcFTContainsId = ((Integer) num).intValue();
+ } else {
+ log.error("Could not install ftcontains function.");
}
}
@@ -201,6 +250,8 @@
public Compiler cmp;
public XPath xp;
public Key[] keys;
+ public Analyzer analyzer;
+ private HashMap parameters;
public XPathQuery(Collection context, String query, NamespaceMap nsMap, Key[] keys)
throws QueryException {
@@ -209,6 +260,9 @@
this.nsMap = nsMap;
this.keys = keys;
+ parameters = new HashMap();
+ parameters.put(PARAM_COLLECTION, context);
+
Expression ex;
try {
if (nsMap != null) {
@@ -283,7 +337,7 @@
keySet = (Key[]) set.toArray(EMPTY_KEYS);
}
- return new ResultSet(context, pr, keySet, query);
+ return new ResultSet(context, pr, keySet, query, parameters);
} catch (Exception e) {
if (e instanceof QueryException) {
throw (QueryException) e.fillInStackTrace();
@@ -728,6 +782,10 @@
case FunctionTable.FUNC_TRUE:
return XBoolean.S_TRUE;
default:
+ // custom extention
+ if (id == funcFTContainsId) {
+ return funcFTContains(owner, args);
+ }
return null;
}
}
@@ -860,6 +918,26 @@
return null;
}
+ private Object funcFTContains(String owner, List args) throws Exception {
+ if (args.size() != 1) {
+ return null;
+ }
+
+ if (parameters == null) {
+ parameters = new HashMap();
+ }
+
+ Object o = args.get(0);
+
+ if (o instanceof XString) {
+ // extract text query
+ String query = ((XString) o).str();
+ return queryTextIndex(owner, query);
+ }
+
+ return null;
+ }
+
private Object funcFloor(List args) throws Exception {
if (args.size() == 1) {
Object o = args.get(0);
@@ -1115,6 +1193,46 @@
return null;
}
+ private Object queryTextIndex(String ps, String query) throws Exception {
+ IndexPattern pattern = new IndexPattern(symbols, ps, nsMap);
+
+ // check if there is full text indexer for this collection
+ Indexer idx = context.getIndexManager().getBestIndexer(Indexer.STYLE_FULLTEXT, pattern);
+ if (idx instanceof LuceneIndexer) {
+ LuceneIndexer textInd = ((LuceneIndexer) idx);
+ analyzer = textInd.getAnalyzer();
+ parameters.put(PARAM_ANALYZER, analyzer);
+
+ // see if index has matching pattern
+ String alias = textInd.getPatternAlias(pattern);
+
+ if (alias != null) {
+ // Queries that contain 'NOT', '!', '-' operators cannot be used here
+ // because LuceneIndexer searches for documents, in that context
+ // "NOT term" query means to find documents where 'term' does not
+ // appear in certain field at all. For XPath, however, it means that
+ // 'term' must not appear in text of the element that currently under
+ // evaluation, but may appear in the other elements that match the
+ // same IndexPattern.
+ //
+ // To make sure that all potentially matching documents are returned
+ // by the search, all subqueries with these operators are ignored
+ // on this step.
+ org.apache.lucene.search.Query parsedQuery = new SpecialQueryParser(alias, analyzer).parse(query);
+ IndexMatch[] matches = textInd.queryMatches(parsedQuery);
+ Key[] keys = QueryEngine.getUniqueKeys(matches);
+
+ return new NamedKeys(ps, ps.indexOf('@') != -1, keys);
+ }
+ } else {
+ // there is no Lucene indexer, fall back to default analyzer
+ analyzer = (Analyzer) Class.forName(LuceneIndexer.DEFANALYZER).newInstance();
+ parameters.put(PARAM_ANALYZER, analyzer);
+ }
+
+ return null;
+ }
+
/**
* queryComparison performs a comparison query use the operands that are passed to it, and returns the resulting
* Keys.
@@ -1220,12 +1338,14 @@
public int keyPos = 0;
public NodeIterator ni;
public Object node;
+ private Map parameters;
- public ResultSet(Collection context, PrefixResolver pr, Key[] keySet, String query) {
+ public ResultSet(Collection context, PrefixResolver pr, Key[] keySet, String query, Map parameters) {
this.context = context;
this.pr = pr;
this.keySet = keySet;
this.query = query;
+ this.parameters = parameters;
errors = new ErrorListener() {
public void fatalError(TransformerException te) {
@@ -1254,7 +1374,21 @@
}
}
- private void prepareNextNode() throws XMLDBException, TransformerException, DBException {
+ private XPath createXPath(PrefixResolver pfx) {
+ try {
+ if (XCOMPILER3) {
+ return (XPath) XPATH.newInstance(
+ new Object[] {query, null, pfx, new Integer(XPath.SELECT), errors, functionTable});
+ } else {
+ return (XPath) XPATH.newInstance(
+ new Object[] {query, null, pfx, new Integer(XPath.SELECT), errors});
+ }
+ } catch (Exception e) {
+ throw new RuntimeException("Could not instantiate Compiler: " + e);
+ }
+ }
+
+ private void prepareNextNode() throws XMLDBException, TransformerException, DBException {
node = null;
while (keyPos < keySet.length) {
@@ -1275,15 +1409,15 @@
continue;
}
- XPathContext xpc = new XPathContext();
+ XPathResolverContext xpc = new XPathResolverContext(parameters);
PrefixResolver pfx;
if (pr == null) {
pfx = new PrefixResolverDefault(d.getDocumentElement());
- xp = new XPath(query, null, pfx, XPath.SELECT, errors);
+ xp = createXPath(pfx);
} else {
pfx = pr;
if (xp == null) {
- xp = new XPath(query, null, pfx, XPath.SELECT, errors);
+ xp = createXPath(pfx);
}
}
@@ -1407,4 +1541,23 @@
}
private final static NodeIterator EMPTY_NODE_ITERATOR = new EmptyNodeIterator();
+
+ /**
+ * XPathContext with optional parameters
+ */
+ static class XPathResolverContext extends XPathContext {
+ private Map parameters;
+
+ public XPathResolverContext(Map parameters) {
+ this.parameters = parameters;
+ }
+
+ public Object getParameter(String name) {
+ return parameters.get(name);
+ }
+
+ public void setParameter(String name, Object object) {
+ parameters.put(name, object);
+ }
+ }
}
Added: xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/SpecialQueryParser.java
URL: http://svn.apache.org/viewvc/xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/SpecialQueryParser.java?rev=586647&view=auto
==============================================================================
--- xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/SpecialQueryParser.java (added)
+++ xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/SpecialQueryParser.java Fri Oct 19 17:32:43 2007
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * $Id$
+ */
+
+package org.apache.xindice.core.query.ftsearch;
+
+import org.apache.lucene.queryParser.QueryParser;
+import org.apache.lucene.queryParser.CharStream;
+import org.apache.lucene.queryParser.QueryParserTokenManager;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.BooleanClause;
+
+import java.util.Vector;
+
+/**
+ * Specialized parser for text queries that ignores query clauses that have
+ * 'prohibited' modifier. This is only used for XPath full text search
+ * extention function to search existing text index, if any.
+ *
+ * @version $Revision$, $Date$
+ */
+public class SpecialQueryParser extends QueryParser {
+ private static final int CONJ_AND = 1;
+ private static final int CONJ_OR = 2;
+
+ private static final int MOD_NOT = 10;
+ private static final int MOD_REQ = 11;
+
+ public SpecialQueryParser(String f, Analyzer a) {
+ super(f, a);
+ }
+
+ public SpecialQueryParser(CharStream stream) {
+ super(stream);
+ }
+
+ public SpecialQueryParser(QueryParserTokenManager tm) {
+ super(tm);
+ }
+
+ /**
+ * This method is slightly modified copy of superclass method, where it ignores
+ * boolean clauses that have 'prohibited' modifier.
+ * @see QueryParser#addClause(java.util.Vector, int, int, org.apache.lucene.search.Query)
+ */
+ protected void addClause(Vector clauses, int conj, int mods, Query q) {
+
+ // If this term is introduced by AND, make the preceding term required,
+ // unless it's already prohibited
+ if (clauses.size() > 0 && conj == CONJ_AND) {
+ BooleanClause c = (BooleanClause) clauses.elementAt(clauses.size()-1);
+ if (!c.isProhibited())
+ c.setOccur(BooleanClause.Occur.MUST);
+ }
+
+ if (clauses.size() > 0 && getDefaultOperator() == AND_OPERATOR && conj == CONJ_OR) {
+ // If this term is introduced by OR, make the preceding term optional,
+ // unless it's prohibited (that means we leave -a OR b but +a OR b-->a OR b)
+ // notice if the input is a OR b, first term is parsed as required; without
+ // this modification a OR b would parsed as +a OR b
+ BooleanClause c = (BooleanClause) clauses.elementAt(clauses.size()-1);
+ if (!c.isProhibited())
+ c.setOccur(BooleanClause.Occur.SHOULD);
+ }
+
+ // We might have been passed a null query; the term might have been
+ // filtered away by the analyzer.
+ if (q == null)
+ return;
+
+ boolean required, prohibited;
+ if (getDefaultOperator() == OR_OPERATOR) {
+ // We set REQUIRED if we're introduced by AND or +; PROHIBITED if
+ // introduced by NOT or -; make sure not to set both.
+ prohibited = (mods == MOD_NOT);
+ required = (mods == MOD_REQ);
+ if (conj == CONJ_AND && !prohibited) {
+ required = true;
+ }
+ } else {
+ // We set PROHIBITED if we're introduced by NOT or -; We set REQUIRED
+ // if not PROHIBITED and not introduced by OR
+ prohibited = (mods == MOD_NOT);
+ required = (!prohibited && conj != CONJ_OR);
+ }
+ if (required && !prohibited) {
+ clauses.addElement(new BooleanClause(q, BooleanClause.Occur.MUST));
+ } else if (!required && !prohibited) {
+ clauses.addElement(new BooleanClause(q, BooleanClause.Occur.SHOULD));
+ } else if (required && prohibited) {
+ throw new RuntimeException("Clause cannot be both required and prohibited");
+ }
+ }
+}
Propchange: xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/SpecialQueryParser.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/SpecialQueryParser.java
------------------------------------------------------------------------------
svn:keywords = Id Revision Author Date
lmpx.com only provides a reader for public news (NNTP) servers. It is not
affiliated with the servers or forums shown here and is not responsible for
the content of articles, which is written by their respective authors.