Author: natalia
Date: Sat Oct 13 19:41:50 2007
New Revision: 584476
URL: http://svn.apache.org/viewvc?rev=584476&view=rev
Log:
Lucene-based full text search in the content of DOM nodes
Added:
xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/
xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/NodeReader.java (with props)
xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/NodeTermDocs.java (with props)
xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/NodeTermEnum.java (with props)
xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/NodeTermPositions.java (with props)
xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/Searcher.java (with props)
Added: xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/NodeReader.java
URL: http://svn.apache.org/viewvc/xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/NodeReader.java?rev=584476&view=auto
==============================================================================
--- xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/NodeReader.java (added)
+++ xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/NodeReader.java Sat Oct 13 19:41:50 2007
@@ -0,0 +1,286 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * $Id$
+ */
+
+package org.apache.xindice.core.query.ftsearch;
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.TermFreqVector;
+import org.apache.lucene.index.TermEnum;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TermDocs;
+import org.apache.lucene.index.TermPositions;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.FieldSelector;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Token;
+import org.apache.lucene.search.DefaultSimilarity;
+import org.apache.xindice.xml.dom.NodeImpl;
+import org.w3c.dom.NodeList;
+import org.w3c.dom.Node;
+
+import java.io.IOException;
+import java.io.StringReader;
+import java.util.Collection;
+import java.util.Arrays;
+import java.util.ArrayList;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.HashSet;
+import java.util.List;
+
+/**
+ * Implementation of IndexReader that works with set of DOM nodes in memory.
+ * Set of nodes is constant - no modifications of it are allowed.
+ *
+ * @version $Revision$, $Date$
+ */
+public class NodeReader extends IndexReader {
+ // list of DOM nodes
+ private final ArrayList[] nodes;
+
+ private byte[] norms;
+
+ // maps term to a list of documents where it appears
+ private final SortedMap termMap;
+
+ protected NodeReader(NodeList list, Analyzer analyzer) {
+ super(null);
+
+ nodes = new ArrayList[list.getLength()];
+
+ for (int i = 0; i < nodes.length; i++) {
+ nodes[i] = new ArrayList();
+ Node node = list.item(i);
+
+ String text;
+ if (node instanceof NodeImpl) {
+ // DOM Level 3 method
+ text = ((NodeImpl) node).getTextContent();
+ } else {
+ text = getTextContent(node);
+ }
+
+ TokenStream stream = analyzer.tokenStream("", new StringReader(text));
+ try {
+ Token token;
+ while ((token = stream.next()) != null) {
+ nodes[i].add(token.termText());
+ }
+ } catch (IOException e) {
+ // won't happen
+ }
+ }
+
+ // init norms
+ norms = new byte[nodes.length];
+ Arrays.fill(norms, DefaultSimilarity.encodeNorm(1.0f));
+
+ // build term enumeration
+ termMap = buildTermMap();
+ }
+
+ public TermFreqVector[] getTermFreqVectors(int docNumber) {
+ throw new UnsupportedOperationException();
+ }
+
+ public TermFreqVector getTermFreqVector(int docNumber, String field) {
+ throw new UnsupportedOperationException();
+ }
+
+ public int numDocs() {
+ return nodes.length;
+ }
+
+ public int maxDoc() {
+ return nodes.length;
+ }
+
+ /**
+ * Method is not supported.
+ */
+ public Document document(int n, FieldSelector fieldSelector) {
+ return null;
+ }
+
+ /**
+ * Deletion is not supported.
+ */
+ public boolean isDeleted(int n) {
+ return false;
+ }
+
+ /**
+ * Deletion is not supported.
+ */
+ public boolean hasDeletions() {
+ return false;
+ }
+
+ public byte[] norms(String field) throws IOException {
+ return field.length() == 0 ? norms : null;
+ }
+
+ public void norms(String field, byte[] bytes, int offset) {
+ System.arraycopy(norms, 0, bytes, offset, maxDoc());
+ }
+
+ protected void doSetNorm(int doc, String field, byte value) {
+ if (field.length() > 0) {
+ return;
+ }
+
+ norms[doc] = value;
+ }
+
+ public TermEnum terms() {
+ return new NodeTermEnum(termMap);
+ }
+
+ public TermEnum terms(Term t) {
+ return new NodeTermEnum(termMap, t);
+ }
+
+ /**
+ * Builds the map of all the terms in all the nodes to the list of
+ * node numbers where those terms appear.
+ * @return Map with keys sorted in ascending order
+ */
+ private SortedMap buildTermMap() {
+ SortedMap map = new TreeMap();
+
+ for (int i = 0; i < nodes.length; i++) {
+ for (int j = 0; j < nodes[i].size(); j++) {
+ String term = (String) nodes[i].get(j);
+
+ List docs;
+ if (map.containsKey(term)) {
+ docs = (List) map.get(term);
+ } else {
+ docs = new ArrayList();
+ }
+
+ docs.add(new Integer(i));
+ map.put(term, docs);
+ }
+ }
+
+ return map;
+ }
+
+ public int docFreq(Term t) {
+ List docs = (List) termMap.get(t.text());
+
+ // no such term
+ if (docs == null) {
+ return 0;
+ }
+
+ HashSet set = new HashSet();
+ set.addAll(docs);
+ return set.size();
+ }
+
+ public TermDocs termDocs() {
+ return new NodeTermDocs(this);
+ }
+
+ public TermPositions termPositions() {
+ return new NodeTermPositions(this);
+ }
+
+ /**
+ * Deletion is not supported.
+ */
+ protected void doDelete(int docNum) {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * Deletion is not supported.
+ */
+ protected void doUndeleteAll() {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * Not applicable
+ */
+ protected void doCommit() {
+ }
+
+ /**
+ * Not applicable
+ */
+ protected void doClose() {
+ }
+
+ /**
+ * Field names are not supported.
+ */
+ public Collection getFieldNames(FieldOption fldOption) {
+ throw new UnsupportedOperationException();
+ }
+
+ ArrayList[] getNodes() {
+ return nodes;
+ }
+
+ SortedMap getTermMap() {
+ return termMap;
+ }
+
+ /**
+ * Get text content of a DOM node. This is the same as DOM Level 3 method
+ * getTextContent().
+ * @param node
+ * @return The text content of this node and its descendants.
+ */
+ private String getTextContent(Node node) {
+ String text = null;
+ switch (node.getNodeType()) {
+ case Node.ATTRIBUTE_NODE:
+ case Node.CDATA_SECTION_NODE:
+ case Node.COMMENT_NODE:
+ case Node.PROCESSING_INSTRUCTION_NODE:
+ case Node.TEXT_NODE:
+ text = node.getNodeValue();
+ break;
+ case Node.ELEMENT_NODE:
+ case Node.DOCUMENT_FRAGMENT_NODE:
+ case Node.ENTITY_NODE:
+ case Node.ENTITY_REFERENCE_NODE:
+ StringBuffer val = new StringBuffer();
+
+ NodeList children = node.getChildNodes();
+ if (children == null || children.getLength() == 0) {
+ text = "";
+ break;
+ }
+
+ for (int i = 0; i < children.getLength(); i++) {
+ val.append(getTextContent(children.item(i)));
+ }
+ text = val.toString();
+ break;
+ }
+
+ return text;
+ }
+}
Propchange: xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/NodeReader.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/NodeReader.java
------------------------------------------------------------------------------
svn:keywords = Id Revision Author Date
Added: xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/NodeTermDocs.java
URL: http://svn.apache.org/viewvc/xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/NodeTermDocs.java?rev=584476&view=auto
==============================================================================
--- xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/NodeTermDocs.java (added)
+++ xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/NodeTermDocs.java Sat Oct 13 19:41:50 2007
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * $Id$
+ */
+
+package org.apache.xindice.core.query.ftsearch;
+
+import org.apache.lucene.index.TermDocs;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TermEnum;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.List;
+
+/**
+ * TermDocs for searching over DOM nodes in memory
+ *
+ * @version $Revision$, $Date$
+ */
+public class NodeTermDocs implements TermDocs {
+ protected int num;
+ protected ArrayList[] nodes;
+ protected Map termMap;
+ protected int[] freqs;
+ private boolean init;
+
+ public NodeTermDocs(NodeReader reader) {
+ nodes = reader.getNodes();
+ termMap = reader.getTermMap();
+
+ freqs = new int[nodes.length];
+ }
+
+ public void seek(Term term) {
+ List docs = (List) termMap.get(term.text());
+ if (docs != null) {
+ for (Iterator i = docs.iterator(); i.hasNext(); ) {
+ int idx = ((Integer) i.next()).intValue();
+ freqs[idx]++;
+ }
+ }
+ }
+
+ public void seek(TermEnum termEnum) {
+ seek(termEnum.term());
+ }
+
+ public int doc() {
+ return num;
+ }
+
+ public int freq() {
+ return freqs[num];
+ }
+
+ public boolean next() {
+ if (!init) {
+ init = true;
+ num = 0;
+ } else {
+ num++;
+ }
+
+ while(num < nodes.length && freqs[num] == 0) {
+ num++;
+ }
+
+ return num < nodes.length;
+ }
+
+ public int read(int[] docs, int[] freqs) {
+ int count = 0;
+ for (int i = 0; i < docs.length && num < nodes.length; i++, num++) {
+ if (this.freqs[num] > 0) {
+ freqs[count] = this.freqs[num];
+ docs[count] = num;
+ count++;
+ }
+ }
+
+ return count;
+ }
+
+ public boolean skipTo(int target) {
+ num = target;
+ return num < nodes.length;
+ }
+
+ /**
+ * Not applicable
+ */
+ public void close() {
+ }
+}
Propchange: xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/NodeTermDocs.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/NodeTermDocs.java
------------------------------------------------------------------------------
svn:keywords = Id Revision Author Date
Added: xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/NodeTermEnum.java
URL: http://svn.apache.org/viewvc/xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/NodeTermEnum.java?rev=584476&view=auto
==============================================================================
--- xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/NodeTermEnum.java (added)
+++ xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/NodeTermEnum.java Sat Oct 13 19:41:50 2007
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * $Id$
+ */
+
+package org.apache.xindice.core.query.ftsearch;
+
+import org.apache.lucene.index.TermEnum;
+import org.apache.lucene.index.Term;
+
+import java.util.Iterator;
+import java.util.SortedMap;
+import java.util.HashSet;
+import java.util.List;
+
+/**
+ * Ordered set of terms used by NodeReader.
+ *
+ * @version $Revision$, $Date$
+ */
+public class NodeTermEnum extends TermEnum {
+ // map that backs this enumeration
+ private SortedMap termMap;
+
+ private Iterator iterator;
+
+ // value of the current term
+ private String curTerm;
+
+ public NodeTermEnum(SortedMap termMap) {
+ this.termMap = termMap;
+ iterator = this.termMap.keySet().iterator();
+ if (iterator.hasNext()) {
+ curTerm = (String) iterator.next();
+ }
+ }
+
+ public NodeTermEnum(SortedMap termMap, Term t) {
+ this(termMap.tailMap(t.text()));
+ }
+
+ public boolean next() {
+ if (iterator.hasNext()) {
+ curTerm = (String) iterator.next();
+ return true;
+ }
+
+ return false;
+ }
+
+ public Term term() {
+ if (curTerm != null) {
+ return new Term("", curTerm);
+ }
+
+ return null;
+ }
+
+ public int docFreq() {
+ List docs = (List) termMap.get(curTerm);
+
+ HashSet set = new HashSet();
+ set.addAll(docs);
+ return set.size();
+ }
+
+ /**
+ * Not applicable
+ */
+ public void close() {
+ }
+}
Propchange: xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/NodeTermEnum.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/NodeTermEnum.java
------------------------------------------------------------------------------
svn:keywords = Id Revision Author Date
Added: xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/NodeTermPositions.java
URL: http://svn.apache.org/viewvc/xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/NodeTermPositions.java?rev=584476&view=auto
==============================================================================
--- xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/NodeTermPositions.java (added)
+++ xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/NodeTermPositions.java Sat Oct 13 19:41:50 2007
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * $Id$
+ */
+
+package org.apache.xindice.core.query.ftsearch;
+
+import org.apache.lucene.index.TermPositions;
+import org.apache.lucene.index.Term;
+
+import java.io.IOException;
+
+/**
+ * TermPositions for searching over DOM nodes in memory
+ *
+ * @version $Revision$, $Date$
+ */
+public class NodeTermPositions extends NodeTermDocs implements TermPositions {
+ private int pos;
+ private String term;
+
+ public NodeTermPositions(NodeReader reader) {
+ super(reader);
+ }
+
+ public void seek(Term term) {
+ this.term = term.text();
+ super.seek(term);
+ }
+
+ public int nextPosition() throws IOException {
+ while (pos < nodes[num].size()) {
+ if ((nodes[num].get(pos)).equals(term)) {
+ return pos;
+ }
+ pos++;
+ }
+
+ return 0;
+ }
+
+ public boolean next() {
+ if (super.next()) {
+ pos = 0;
+ return true;
+ }
+
+ return false;
+ }
+
+ public int getPayloadLength() {
+ throw new UnsupportedOperationException();
+ }
+
+ public byte[] getPayload(byte[] data, int offset) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ public boolean isPayloadAvailable() {
+ return false;
+ }
+}
Propchange: xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/NodeTermPositions.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/NodeTermPositions.java
------------------------------------------------------------------------------
svn:keywords = Id Revision Author Date
Added: xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/Searcher.java
URL: http://svn.apache.org/viewvc/xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/Searcher.java?rev=584476&view=auto
==============================================================================
--- xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/Searcher.java (added)
+++ xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/Searcher.java Sat Oct 13 19:41:50 2007
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * $Id$
+ */
+
+package org.apache.xindice.core.query.ftsearch;
+
+import org.w3c.dom.NodeList;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.queryParser.QueryParser;
+import org.apache.lucene.queryParser.ParseException;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Hits;
+import org.apache.lucene.search.Query;
+import org.apache.xindice.core.data.NodeSet;
+
+import java.io.IOException;
+
+/**
+ * Searcher executes full text queries against list of nodes and returns
+ * nodes that match the query in the order of relevance (score) - most relevant
+ * results will be return first.<br>
+ * <br>
+ * Query syntax is the same as syntax of Lucene query, except it does not use
+ * field names.
+ *
+ * @version $Revision$, $Date$
+ */
+public class Searcher {
+ private NodeList nodes;
+ private Analyzer analyzer;
+
+ /**
+ * Builds new Searcher based on list of nodes and analyzer.
+ *
+ * @param nodes List of nodes to search
+ * @param analyzer Analyzer that will be used to tokenize text of the nodes.
+ * Choice of analyzer affects query results.
+ * @see org.apache.lucene.analysis.Analyzer
+ */
+ public Searcher(NodeList nodes, Analyzer analyzer) {
+ this.nodes = nodes;
+ this.analyzer = analyzer;
+ }
+
+ /**
+ * Executes query against list of nodes and returns matches in the order
+ * of relevance (score).
+ *
+ * @param query Full text query
+ * @return NodeSet that contains matching nodes
+ * @throws ParseException Query failed to be parsed
+ */
+ public NodeSet search(String query) throws ParseException {
+ Query compQuery = new QueryParser("", analyzer).parse(query);
+ NodeReader reader = new NodeReader(nodes, analyzer);
+ IndexSearcher searcher = new IndexSearcher(reader);
+ Hits hits = null;
+ try {
+ hits = searcher.search(compQuery);
+ } catch (IOException e) {
+ // this searcher does not use file IO, exception won't happen
+ }
+
+ return new ResultSet(hits);
+ }
+
+ private class ResultSet implements NodeSet {
+ private Hits hits;
+ private int count;
+
+ private ResultSet(Hits hits) {
+ this.hits = hits;
+ }
+
+ public boolean hasMoreNodes() {
+ return count < hits.length();
+ }
+
+ public Object getNextNode() {
+ try {
+ return nodes.item(hits.id(count++));
+ } catch (IOException e) {
+ // ignore, does not use IO
+ }
+
+ return null;
+ }
+ }
+}
Propchange: xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/Searcher.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: xml/xindice/trunk/java/src/org/apache/xindice/core/query/ftsearch/Searcher.java
------------------------------------------------------------------------------
svn:keywords = Id Revision Author Date
lmpx.com only provides a reader for public news (NNTP) servers. It is not
affiliated with the servers or forums shown here and is not responsible for
the content of articles, which is written by their respective authors.