Modified: lenya/branches/revolution/1.3.x/src/java/org/apache/lenya/lucene/html/ParseException.java
URL: http://svn.apache.org/viewvc/lenya/branches/revolution/1.3.x/src/java/org/apache/lenya/lucene/html/ParseException.java?rev=617035&r1=617034&r2=617035&view=diff
==============================================================================
--- lenya/branches/revolution/1.3.x/src/java/org/apache/lenya/lucene/html/ParseException.java (original)
+++ lenya/branches/revolution/1.3.x/src/java/org/apache/lenya/lucene/html/ParseException.java Wed Jan 30 23:44:03 2008
@@ -14,227 +14,160 @@
* limitations under the License.
*
*/
-
/* $Id$ */
-
package org.apache.lenya.lucene.html;
-
/**
- * This exception is thrown when parse errors are encountered. You can explicitly create objects of
- * this exception type by calling the method generateParseException in the generated parser. You
- * can modify this class to customize your error reporting mechanisms so long as you retain the
- * public fields.
+ * This exception is thrown when parse errors are encountered. You can explicitly create objects of this exception type by calling the method generateParseException in the generated parser. You can modify this class to customize your error reporting mechanisms so long as you retain the public fields.
*/
public class ParseException extends Exception {
- /**
- * This variable determines which constructor was used to create this object and thereby
- * affects the semantics of the "getMessage" method (see below).
- */
- protected boolean specialConstructor;
-
- /**
- * This is the last token that has been consumed successfully. If this object has been created
- * due to a parse error, the token followng this token will (therefore) be the first error
- * token.
- */
- public Token currentToken;
-
- /**
- * Each entry in this array is an array of integers. Each array of integers represents a
- * sequence of tokens (by their ordinal values) that is expected at this point of the parse.
- */
- public int[][] expectedTokenSequences;
-
- /**
- * This is a reference to the "tokenImage" array of the generated parser within which the parse
- * error occurred. This array is defined in the generated ...Constants interface.
- */
- public String[] tokenImage;
-
- /** The end of line string for this machine. */
- protected String eol = System.getProperty("line.separator", "\n");
-
- /**
- * This constructor is used by the method "generateParseException" in the generated parser.
- * Calling this constructor generates a new object of this type with the fields
- * "currentToken", "expectedTokenSequences", and "tokenImage" set. The boolean flag
- * "specialConstructor" is also set to true to indicate that this constructor was used to
- * create this object. This constructor calls its super class with the empty string to force
- * the "toString" method of parent class "Throwable" to print the error message in the form:
- * ParseException: <result of getMessage>
- *
- * @param currentTokenVal DOCUMENT ME!
- * @param expectedTokenSequencesVal DOCUMENT ME!
- * @param tokenImageVal DOCUMENT ME!
- */
- public ParseException(Token currentTokenVal, int[][] expectedTokenSequencesVal,
- String[] tokenImageVal) {
- super("");
- specialConstructor = true;
- currentToken = currentTokenVal;
- expectedTokenSequences = expectedTokenSequencesVal;
- tokenImage = tokenImageVal;
- }
-
- /**
- * The following constructors are for use by you for whatever purpose you can think of.
- * Constructing the exception in this manner makes the exception behave in the normal way -
- * i.e., as documented in the class "Throwable". The fields "errorToken",
- * "expectedTokenSequences", and "tokenImage" do not contain relevant information. The JavaCC
- * generated code does not use these constructors.
- */
- public ParseException() {
- super();
- specialConstructor = false;
- }
-
- /**
- * Creates a new ParseException object.
- *
- * @param message DOCUMENT ME!
- */
- public ParseException(String message) {
- super(message);
- specialConstructor = false;
- }
-
- /**
- * This method has the standard behavior when this object has been created using the standard
- * constructors. Otherwise, it uses "currentToken" and "expectedTokenSequences" to generate a
- * parse error message and returns it. If this object has been created due to a parse error,
- * and you do not catch it (it gets thrown from the parser), then this method is called during
- * the printing of the final stack trace, and hence the correct error message gets displayed.
- *
- * @return DOCUMENT ME!
- */
- public String getMessage() {
- if (!specialConstructor) {
- return super.getMessage();
- }
-
- String expected = "";
- int maxSize = 0;
-
- for (int i = 0; i < expectedTokenSequences.length; i++) {
- if (maxSize < expectedTokenSequences[i].length) {
- maxSize = expectedTokenSequences[i].length;
- }
-
- for (int j = 0; j < expectedTokenSequences[i].length; j++) {
- expected += (tokenImage[expectedTokenSequences[i][j]] + " ");
- }
-
- if (expectedTokenSequences[i][expectedTokenSequences[i].length - 1] != 0) {
- expected += "...";
- }
-
- expected += (eol + " ");
- }
-
- String retval = "Encountered \"";
- Token tok = currentToken.next;
-
- for (int i = 0; i < maxSize; i++) {
- if (i != 0) {
- retval += " ";
- }
-
- if (tok.kind == 0) {
- retval += tokenImage[0];
-
- break;
- }
-
- retval += add_escapes(tok.image);
- tok = tok.next;
- }
-
- retval += ("\" at line " + currentToken.next.beginLine + ", column " +
- currentToken.next.beginColumn);
- retval += ("." + eol);
-
- if (expectedTokenSequences.length == 1) {
- retval += ("Was expecting:" + eol + " ");
- } else {
- retval += ("Was expecting one of:" + eol + " ");
- }
-
- retval += expected;
-
- return retval;
- }
-
- /**
- * Used to convert raw characters to their escaped version when these raw version cannot be
- * used as part of an ASCII string literal.
- *
- * @param str DOCUMENT ME!
- *
- * @return DOCUMENT ME!
- */
- protected String add_escapes(String str) {
- StringBuffer retval = new StringBuffer();
- char ch;
-
- for (int i = 0; i < str.length(); i++) {
- switch (str.charAt(i)) {
+ private static final long serialVersionUID = 1L;
+ /**
+ * This variable determines which constructor was used to create this object and thereby affects the semantics of the "getMessage" method (see below).
+ */
+ protected boolean specialConstructor;
+ /**
+ * This is the last token that has been consumed successfully. If this object has been created due to a parse error, the token followng this token will (therefore) be the first error token.
+ */
+ public Token currentToken;
+ /**
+ * Each entry in this array is an array of integers. Each array of integers represents a sequence of tokens (by their ordinal values) that is expected at this point of the parse.
+ */
+ public int[][] expectedTokenSequences;
+ /**
+ * This is a reference to the "tokenImage" array of the generated parser within which the parse error occurred. This array is defined in the generated ...Constants interface.
+ */
+ public String[] tokenImage;
+ /** The end of line string for this machine. */
+ protected String eol = System.getProperty("line.separator", "\n");
+ /**
+ * This constructor is used by the method "generateParseException" in the generated parser. Calling this constructor generates a new object of this type with the fields "currentToken", "expectedTokenSequences", and "tokenImage" set. The boolean flag "specialConstructor" is also set to true to indicate that this constructor was used to create this object. This constructor calls its super class with the empty string to force the "toString" method of parent class "Throwable" to print the error message in the form: ParseException: <result of getMessage>
+ *
+ * @param currentTokenVal
+ * DOCUMENT ME!
+ * @param expectedTokenSequencesVal
+ * DOCUMENT ME!
+ * @param tokenImageVal
+ * DOCUMENT ME!
+ */
+ public ParseException(Token currentTokenVal, int[][] expectedTokenSequencesVal, String[] tokenImageVal) {
+ super("");
+ specialConstructor = true;
+ currentToken = currentTokenVal;
+ expectedTokenSequences = expectedTokenSequencesVal;
+ tokenImage = tokenImageVal;
+ }
+ /**
+ * The following constructors are for use by you for whatever purpose you can think of. Constructing the exception in this manner makes the exception behave in the normal way - i.e., as documented in the class "Throwable". The fields "errorToken", "expectedTokenSequences", and "tokenImage" do not contain relevant information. The JavaCC generated code does not use these constructors.
+ */
+ public ParseException() {
+ super();
+ specialConstructor = false;
+ }
+ /**
+ * Creates a new ParseException object.
+ *
+ * @param message
+ * DOCUMENT ME!
+ */
+ public ParseException(String message) {
+ super(message);
+ specialConstructor = false;
+ }
+ /**
+ * This method has the standard behavior when this object has been created using the standard constructors. Otherwise, it uses "currentToken" and "expectedTokenSequences" to generate a parse error message and returns it. If this object has been created due to a parse error, and you do not catch it (it gets thrown from the parser), then this method is called during the printing of the final stack trace, and hence the correct error message gets displayed.
+ *
+ * @return DOCUMENT ME!
+ */
+ public String getMessage() {
+ if(!specialConstructor){
+ return super.getMessage();
+ }
+ String expected = "";
+ int maxSize = 0;
+ for(int i = 0; i < expectedTokenSequences.length; i++){
+ if(maxSize < expectedTokenSequences[i].length){
+ maxSize = expectedTokenSequences[i].length;
+ }
+ for(int j = 0; j < expectedTokenSequences[i].length; j++){
+ expected += (tokenImage[expectedTokenSequences[i][j]] + " ");
+ }
+ if(expectedTokenSequences[i][expectedTokenSequences[i].length - 1] != 0){
+ expected += "...";
+ }
+ expected += (eol + " ");
+ }
+ String retval = "Encountered \"";
+ Token tok = currentToken.next;
+ for(int i = 0; i < maxSize; i++){
+ if(i != 0){
+ retval += " ";
+ }
+ if(tok.kind == 0){
+ retval += tokenImage[0];
+ break;
+ }
+ retval += add_escapes(tok.image);
+ tok = tok.next;
+ }
+ retval += ("\" at line " + currentToken.next.beginLine + ", column " + currentToken.next.beginColumn);
+ retval += ("." + eol);
+ if(expectedTokenSequences.length == 1){
+ retval += ("Was expecting:" + eol + " ");
+ }else{
+ retval += ("Was expecting one of:" + eol + " ");
+ }
+ retval += expected;
+ return retval;
+ }
+ /**
+ * Used to convert raw characters to their escaped version when these raw version cannot be used as part of an ASCII string literal.
+ *
+ * @param str
+ * DOCUMENT ME!
+ *
+ * @return DOCUMENT ME!
+ */
+ protected String add_escapes(String str) {
+ StringBuffer retval = new StringBuffer();
+ char ch;
+ for(int i = 0; i < str.length(); i++){
+ switch(str.charAt(i)){
case 0:
-
- continue;
-
+ continue;
case '\b':
- retval.append("\\b");
-
- continue;
-
+ retval.append("\\b");
+ continue;
case '\t':
- retval.append("\\t");
-
- continue;
-
+ retval.append("\\t");
+ continue;
case '\n':
- retval.append("\\n");
-
- continue;
-
+ retval.append("\\n");
+ continue;
case '\f':
- retval.append("\\f");
-
- continue;
-
+ retval.append("\\f");
+ continue;
case '\r':
- retval.append("\\r");
-
- continue;
-
+ retval.append("\\r");
+ continue;
case '\"':
- retval.append("\\\"");
-
- continue;
-
+ retval.append("\\\"");
+ continue;
case '\'':
- retval.append("\\\'");
-
- continue;
-
+ retval.append("\\\'");
+ continue;
case '\\':
- retval.append("\\\\");
-
- continue;
-
+ retval.append("\\\\");
+ continue;
default:
-
- if (((ch = str.charAt(i)) < 0x20) || (ch > 0x7e)) {
- String s = "0000" + Integer.toString(ch, 16);
- retval.append("\\u" + s.substring(s.length() - 4, s.length()));
- } else {
- retval.append(ch);
- }
-
- continue;
- }
- }
-
- return retval.toString();
- }
+ if(((ch = str.charAt(i)) < 0x20) || (ch > 0x7e)){
+ String s = "0000" + Integer.toString(ch, 16);
+ retval.append("\\u" + s.substring(s.length() - 4, s.length()));
+ }else{
+ retval.append(ch);
+ }
+ continue;
+ }
+ }
+ return retval.toString();
+ }
}
Modified: lenya/branches/revolution/1.3.x/src/java/org/apache/lenya/lucene/html/TokenMgrError.java
URL: http://svn.apache.org/viewvc/lenya/branches/revolution/1.3.x/src/java/org/apache/lenya/lucene/html/TokenMgrError.java?rev=617035&r1=617034&r2=617035&view=diff
==============================================================================
--- lenya/branches/revolution/1.3.x/src/java/org/apache/lenya/lucene/html/TokenMgrError.java (original)
+++ lenya/branches/revolution/1.3.x/src/java/org/apache/lenya/lucene/html/TokenMgrError.java Wed Jan 30 23:44:03 2008
@@ -14,173 +14,139 @@
* limitations under the License.
*
*/
-
/* $Id$ */
-
package org.apache.lenya.lucene.html;
-
public class TokenMgrError extends Error {
- /*
- * Ordinals for various reasons why an Error of this type can be thrown.
- */
-
- /** Lexical error occured. */
- static final int LEXICAL_ERROR = 0;
-
- /** An attempt wass made to create a second instance of a static token manager. */
- static final int STATIC_LEXER_ERROR = 1;
-
- /** Tried to change to an invalid lexical state. */
- static final int INVALID_LEXICAL_STATE = 2;
-
- /** Detected (and bailed out of) an infinite loop in the token manager. */
- static final int LOOP_DETECTED = 3;
-
- /** Indicates the reason why the exception is thrown. It will have one of the above 4 values. */
- int errorCode;
-
- /*
- * Constructors of various flavors follow.
- */
- public TokenMgrError() {
- }
-
- /**
- * Creates a new TokenMgrError object.
- *
- * @param message DOCUMENT ME!
- * @param reason DOCUMENT ME!
- */
- public TokenMgrError(String message, int reason) {
- super(message);
- errorCode = reason;
- }
-
- /**
- * Creates a new TokenMgrError object.
- *
- * @param EOFSeen DOCUMENT ME!
- * @param lexState DOCUMENT ME!
- * @param errorLine DOCUMENT ME!
- * @param errorColumn DOCUMENT ME!
- * @param errorAfter DOCUMENT ME!
- * @param curChar DOCUMENT ME!
- * @param reason DOCUMENT ME!
- */
- public TokenMgrError(boolean EOFSeen, int lexState, int errorLine, int errorColumn,
- String errorAfter, char curChar, int reason) {
- this(LexicalError(EOFSeen, lexState, errorLine, errorColumn, errorAfter, curChar), reason);
- }
-
- /**
- * Replaces unprintable characters by their espaced (or unicode escaped) equivalents in the
- * given string
- *
- * @param str DOCUMENT ME!
- *
- * @return DOCUMENT ME!
- */
- protected static final String addEscapes(String str) {
- StringBuffer retval = new StringBuffer();
- char ch;
-
- for (int i = 0; i < str.length(); i++) {
- switch (str.charAt(i)) {
+ /*
+ * Ordinals for various reasons why an Error of this type can be thrown.
+ */
+ private static final long serialVersionUID = 1L;
+ /** Lexical error occured. */
+ static final int LEXICAL_ERROR = 0;
+ /** An attempt wass made to create a second instance of a static token manager. */
+ static final int STATIC_LEXER_ERROR = 1;
+ /** Tried to change to an invalid lexical state. */
+ static final int INVALID_LEXICAL_STATE = 2;
+ /** Detected (and bailed out of) an infinite loop in the token manager. */
+ static final int LOOP_DETECTED = 3;
+ /** Indicates the reason why the exception is thrown. It will have one of the above 4 values. */
+ int errorCode;
+ /*
+ * Constructors of various flavors follow.
+ */
+ public TokenMgrError() {
+ }
+ /**
+ * Creates a new TokenMgrError object.
+ *
+ * @param message
+ * DOCUMENT ME!
+ * @param reason
+ * DOCUMENT ME!
+ */
+ public TokenMgrError(String message, int reason) {
+ super(message);
+ errorCode = reason;
+ }
+ /**
+ * Creates a new TokenMgrError object.
+ *
+ * @param EOFSeen
+ * DOCUMENT ME!
+ * @param lexState
+ * DOCUMENT ME!
+ * @param errorLine
+ * DOCUMENT ME!
+ * @param errorColumn
+ * DOCUMENT ME!
+ * @param errorAfter
+ * DOCUMENT ME!
+ * @param curChar
+ * DOCUMENT ME!
+ * @param reason
+ * DOCUMENT ME!
+ */
+ public TokenMgrError(boolean EOFSeen, int lexState, int errorLine, int errorColumn, String errorAfter, char curChar, int reason) {
+ this(LexicalError(EOFSeen, lexState, errorLine, errorColumn, errorAfter, curChar), reason);
+ }
+ /**
+ * Replaces unprintable characters by their espaced (or unicode escaped) equivalents in the given string
+ *
+ * @param str
+ * DOCUMENT ME!
+ *
+ * @return DOCUMENT ME!
+ */
+ protected static final String addEscapes(String str) {
+ StringBuffer retval = new StringBuffer();
+ char ch;
+ for(int i = 0; i < str.length(); i++){
+ switch(str.charAt(i)){
case 0:
-
- continue;
-
+ continue;
case '\b':
- retval.append("\\b");
-
- continue;
-
+ retval.append("\\b");
+ continue;
case '\t':
- retval.append("\\t");
-
- continue;
-
+ retval.append("\\t");
+ continue;
case '\n':
- retval.append("\\n");
-
- continue;
-
+ retval.append("\\n");
+ continue;
case '\f':
- retval.append("\\f");
-
- continue;
-
+ retval.append("\\f");
+ continue;
case '\r':
- retval.append("\\r");
-
- continue;
-
+ retval.append("\\r");
+ continue;
case '\"':
- retval.append("\\\"");
-
- continue;
-
+ retval.append("\\\"");
+ continue;
case '\'':
- retval.append("\\\'");
-
- continue;
-
+ retval.append("\\\'");
+ continue;
case '\\':
- retval.append("\\\\");
-
- continue;
-
+ retval.append("\\\\");
+ continue;
default:
-
- if (((ch = str.charAt(i)) < 0x20) || (ch > 0x7e)) {
- String s = "0000" + Integer.toString(ch, 16);
- retval.append("\\u" + s.substring(s.length() - 4, s.length()));
- } else {
- retval.append(ch);
- }
-
- continue;
- }
- }
-
- return retval.toString();
- }
-
- /**
- * Returns a detailed message for the Error when it is thrown by the token manager to indicate
- * a lexical error. Parameters : EOFSeen : indicates if EOF caused the lexicl error
- * curLexState : lexical state in which this error occured errorLine : line number when the
- * error occured errorColumn : column number when the error occured errorAfter : prefix that
- * was seen before this error occured curchar : the offending character Note: You can
- * customize the lexical error message by modifying this method.
- *
- * @param EOFSeen DOCUMENT ME!
- * @param lexState DOCUMENT ME!
- * @param errorLine DOCUMENT ME!
- * @param errorColumn DOCUMENT ME!
- * @param errorAfter DOCUMENT ME!
- * @param curChar DOCUMENT ME!
- *
- * @return DOCUMENT ME!
- */
- private static final String LexicalError(boolean EOFSeen, int lexState, int errorLine,
- int errorColumn, String errorAfter, char curChar) {
- return ("Lexical error at line " + errorLine + ", column " + errorColumn +
- ". Encountered: " +
- (EOFSeen ? "<EOF> "
- : (("\"" + addEscapes(String.valueOf(curChar)) + "\"") + " (" + (int) curChar +
- "), ")) + "after : \"" + addEscapes(errorAfter) + "\"");
- }
-
- /**
- * You can also modify the body of this method to customize your error messages. For example,
- * cases like LOOP_DETECTED and INVALID_LEXICAL_STATE are not of end-users concern, so you can
- * return something like : "Internal Error : Please file a bug report .... " from this method
- * for such cases in the release version of your parser.
- *
- * @return DOCUMENT ME!
- */
- public String getMessage() {
- return super.getMessage();
- }
+ if(((ch = str.charAt(i)) < 0x20) || (ch > 0x7e)){
+ String s = "0000" + Integer.toString(ch, 16);
+ retval.append("\\u" + s.substring(s.length() - 4, s.length()));
+ }else{
+ retval.append(ch);
+ }
+ continue;
+ }
+ }
+ return retval.toString();
+ }
+ /**
+ * Returns a detailed message for the Error when it is thrown by the token manager to indicate a lexical error. Parameters : EOFSeen : indicates if EOF caused the lexicl error curLexState : lexical state in which this error occured errorLine : line number when the error occured errorColumn : column number when the error occured errorAfter : prefix that was seen before this error occured curchar : the offending character Note: You can customize the lexical error message by modifying this method.
+ *
+ * @param EOFSeen
+ * DOCUMENT ME!
+ * @param lexState
+ * DOCUMENT ME!
+ * @param errorLine
+ * DOCUMENT ME!
+ * @param errorColumn
+ * DOCUMENT ME!
+ * @param errorAfter
+ * DOCUMENT ME!
+ * @param curChar
+ * DOCUMENT ME!
+ *
+ * @return DOCUMENT ME!
+ */
+ private static final String LexicalError(boolean EOFSeen, int lexState, int errorLine, int errorColumn, String errorAfter, char curChar) {
+ return("Lexical error at line " + errorLine + ", column " + errorColumn + ". Encountered: " + (EOFSeen ? "<EOF> " : (("\"" + addEscapes(String.valueOf(curChar)) + "\"") + " (" + (int) curChar + "), ")) + "after : \"" + addEscapes(errorAfter) + "\"");
+ }
+ /**
+ * You can also modify the body of this method to customize your error messages. For example, cases like LOOP_DETECTED and INVALID_LEXICAL_STATE are not of end-users concern, so you can return something like : "Internal Error : Please file a bug report .... " from this method for such cases in the release version of your parser.
+ *
+ * @return DOCUMENT ME!
+ */
+ public String getMessage() {
+ return super.getMessage();
+ }
}
Modified: lenya/branches/revolution/1.3.x/src/java/org/apache/lenya/lucene/index/AbstractDocumentCreator.java
URL: http://svn.apache.org/viewvc/lenya/branches/revolution/1.3.x/src/java/org/apache/lenya/lucene/index/AbstractDocumentCreator.java?rev=617035&r1=617034&r2=617035&view=diff
==============================================================================
--- lenya/branches/revolution/1.3.x/src/java/org/apache/lenya/lucene/index/AbstractDocumentCreator.java (original)
+++ lenya/branches/revolution/1.3.x/src/java/org/apache/lenya/lucene/index/AbstractDocumentCreator.java Wed Jan 30 23:44:03 2008
@@ -16,84 +16,82 @@
*/
/* $Id$ */
package org.apache.lenya.lucene.index;
-
import java.io.File;
-import org.apache.log4j.Category;
+import org.apache.log4j.Logger;
import org.apache.lucene.document.DateTools;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
-
public class AbstractDocumentCreator implements DocumentCreator {
- Category log = Category.getInstance(AbstractDocumentCreator.class);
- /** Creates a new instance of AbstractDocumentCreator */
- public AbstractDocumentCreator() {
- }
- /**
- * DOCUMENT ME!
- *
- * @param file
- * DOCUMENT ME!
- * @param htdocsDumpDir
- * DOCUMENT ME!
- *
- * @return DOCUMENT ME!
- *
- * @throws Exception
- * DOCUMENT ME!
- */
- public Document getDocument(File file, File htdocsDumpDir) throws Exception {
- // make a new, empty document
- Document doc = new Document();
- // Add the url as a field named "url". Use an UnIndexed field, so
- // that the url is just stored with the document, but is not searchable.
- String requestURI = file.getPath().replace(File.separatorChar, '/').substring(htdocsDumpDir.getPath().length());
- if (requestURI.substring(requestURI.length() - 8).equals(".pdf.txt")) {
- requestURI = requestURI.substring(0, requestURI.length() - 4); // Remove
- // .txt
- // extension
- // from
- // PDF
- // text
- // file
- }
- // doc.add(Field.UnIndexed("url", requestURI));
- doc.add(new Field("url", requestURI, Field.Store.YES, Field.Index.NO));
- // Add the mime-type as a field named "mime-type"
- if (requestURI.substring(requestURI.length() - 5).equals(".html")) {
- // doc.add(Field.UnIndexed("mime-type", "text/html"));
- doc.add(new Field("mime-type", "text/html", Field.Store.YES, Field.Index.NO));
- } else if (requestURI.substring(requestURI.length() - 4).equals(".txt")) {
- // doc.add(Field.UnIndexed("mime-type", "text/plain"));
- doc.add(new Field("mime-type", "text/plain", Field.Store.YES, Field.Index.NO));
- } else if (requestURI.substring(requestURI.length() - 4).equals(".pdf")) {
- // doc.add(Field.UnIndexed("mime-type", "application/pdf"));
- doc.add(new Field("mime-type", "application/pdf", Field.Store.YES, Field.Index.NO));
- } else {
- // Don't add any mime-type field
- // doc.add(Field.UnIndexed("mime-type", "null"));
- }
- // Add the last modified date of the file a field named "modified". Use
- // a
- // Keyword field, so that it's searchable, but so that no attempt is
- // made
- // to tokenize the field into words.
- // doc.add(Field.Keyword("modified",
- // DateField.timeToString(file.lastModified())));
- doc.add(new Field("modified", DateTools.timeToString(file.lastModified(), DateTools.Resolution.MILLISECOND), Field.Store.YES, Field.Index.UN_TOKENIZED, Field.TermVector.YES));
- // Add the id as a field, so that index can be incrementally maintained.
- String id = IndexIterator.createID(file, htdocsDumpDir);
- log.debug(id);
- // doc.add(Field.Keyword("id", id));
- doc.add(new Field("id", id, Field.Store.YES, Field.Index.UN_TOKENIZED, Field.TermVector.YES));
- // Add the uid as a field, so that index can be incrementally
- // maintained.
- // This field is not stored with document, it is indexed, but it is not
- // tokenized prior to indexing.
- String uid = IndexIterator.createUID(file, htdocsDumpDir);
- log.debug(uid);
- doc.add(new Field("uid", uid,
- // false, true, false));
- Field.Store.NO, Field.Index.TOKENIZED, Field.TermVector.NO));
- return doc;
- }
+ private static Logger log = Logger.getLogger(AbstractDocumentCreator.class);
+ /** Creates a new instance of AbstractDocumentCreator */
+ public AbstractDocumentCreator() {
+ }
+ /**
+ * DOCUMENT ME!
+ *
+ * @param file
+ * DOCUMENT ME!
+ * @param htdocsDumpDir
+ * DOCUMENT ME!
+ *
+ * @return DOCUMENT ME!
+ *
+ * @throws Exception
+ * DOCUMENT ME!
+ */
+ public Document getDocument(File file, File htdocsDumpDir) throws Exception {
+ // make a new, empty document
+ Document doc = new Document();
+ // Add the url as a field named "url". Use an UnIndexed field, so
+ // that the url is just stored with the document, but is not searchable.
+ String requestURI = file.getPath().replace(File.separatorChar, '/').substring(htdocsDumpDir.getPath().length());
+ if(requestURI.substring(requestURI.length() - 8).equals(".pdf.txt")){
+ requestURI = requestURI.substring(0, requestURI.length() - 4); // Remove
+ // .txt
+ // extension
+ // from
+ // PDF
+ // text
+ // file
+ }
+ // doc.add(Field.UnIndexed("url", requestURI));
+ doc.add(new Field("url", requestURI, Field.Store.YES, Field.Index.NO));
+ // Add the mime-type as a field named "mime-type"
+ if(requestURI.substring(requestURI.length() - 5).equals(".html")){
+ // doc.add(Field.UnIndexed("mime-type", "text/html"));
+ doc.add(new Field("mime-type", "text/html", Field.Store.YES, Field.Index.NO));
+ }else if(requestURI.substring(requestURI.length() - 4).equals(".txt")){
+ // doc.add(Field.UnIndexed("mime-type", "text/plain"));
+ doc.add(new Field("mime-type", "text/plain", Field.Store.YES, Field.Index.NO));
+ }else if(requestURI.substring(requestURI.length() - 4).equals(".pdf")){
+ // doc.add(Field.UnIndexed("mime-type", "application/pdf"));
+ doc.add(new Field("mime-type", "application/pdf", Field.Store.YES, Field.Index.NO));
+ }else{
+ // Don't add any mime-type field
+ // doc.add(Field.UnIndexed("mime-type", "null"));
+ }
+ // Add the last modified date of the file a field named "modified". Use
+ // a
+ // Keyword field, so that it's searchable, but so that no attempt is
+ // made
+ // to tokenize the field into words.
+ // doc.add(Field.Keyword("modified",
+ // DateField.timeToString(file.lastModified())));
+ doc.add(new Field("modified", DateTools.timeToString(file.lastModified(), DateTools.Resolution.MILLISECOND), Field.Store.YES, Field.Index.UN_TOKENIZED, Field.TermVector.YES));
+ // Add the id as a field, so that index can be incrementally maintained.
+ String id = IndexIterator.createID(file, htdocsDumpDir);
+ log.debug(id);
+ // doc.add(Field.Keyword("id", id));
+ doc.add(new Field("id", id, Field.Store.YES, Field.Index.UN_TOKENIZED, Field.TermVector.YES));
+ // Add the uid as a field, so that index can be incrementally
+ // maintained.
+ // This field is not stored with document, it is indexed, but it is not
+ // tokenized prior to indexing.
+ String uid = IndexIterator.createUID(file, htdocsDumpDir);
+ log.debug(uid);
+ doc.add(new Field("uid", uid,
+ // false, true, false));
+ Field.Store.NO, Field.Index.TOKENIZED, Field.TermVector.NO));
+ return doc;
+ }
}
Modified: lenya/branches/revolution/1.3.x/src/java/org/apache/lenya/lucene/index/AbstractIndexer.java
URL: http://svn.apache.org/viewvc/lenya/branches/revolution/1.3.x/src/java/org/apache/lenya/lucene/index/AbstractIndexer.java?rev=617035&r1=617034&r2=617035&view=diff
==============================================================================
--- lenya/branches/revolution/1.3.x/src/java/org/apache/lenya/lucene/index/AbstractIndexer.java (original)
+++ lenya/branches/revolution/1.3.x/src/java/org/apache/lenya/lucene/index/AbstractIndexer.java Wed Jan 30 23:44:03 2008
@@ -16,338 +16,326 @@
*/
/* $Id$ */
package org.apache.lenya.lucene.index;
-
import java.io.File;
import java.io.FileFilter;
import java.io.IOException;
import java.util.Arrays;
-import org.apache.log4j.Category;
import org.apache.lenya.lucene.IndexConfiguration;
+import org.apache.log4j.Logger;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.Term;
import org.w3c.dom.Element;
-
/**
- * Abstract base class for indexers. The factory method
- * {@link #getDocumentCreator(String[])} is used to create a DocumentCreator
- * from the command-line arguments.
+ * Abstract base class for indexers. The factory method {@link #getDocumentCreator(String[])} is used to create a DocumentCreator from the command-line arguments.
*/
public abstract class AbstractIndexer implements Indexer {
- private static Category log = Category.getInstance(AbstractIndexer.class);
- private DocumentCreator documentCreator;
- private Element indexer;
- private String configFileName;
- /**
- * Creates a new instance of AbstractIndexer
- */
- public AbstractIndexer() {
- }
- /**
- * Returns the DocumentCreator of this indexer.
- */
- protected DocumentCreator getDocumentCreator() {
- return documentCreator;
- }
- /**
- * Initializes this indexer with command-line parameters.
- */
- public void configure(Element indexer, String configFileName) throws Exception {
- documentCreator = createDocumentCreator(indexer, configFileName);
- this.indexer = indexer;
- this.configFileName = configFileName;
- }
- /**
- * DOCUMENT ME!
- *
- * @param element
- * DOCUMENT ME!
- *
- * @return DOCUMENT ME!
- *
- * @throws Exception
- * DOCUMENT ME!
- */
- public abstract DocumentCreator createDocumentCreator(Element indexer, String configFileName) throws Exception;
- /**
- * Updates the index incrementally. Walk directory hierarchy in uid order,
- * while keeping uid iterator from existing index in sync. Mismatches
- * indicate one of:
- * <ol>
- * <li>old documents to be deleted</li>
- * <li>unchanged documents, to be left alone, or</li>
- * <li>new documents, to be indexed.</li>
- * </ol>
- */
- public void updateIndex(File dumpDirectory, File index) throws Exception {
- deleteStaleDocuments(dumpDirectory, index);
- doIndex(dumpDirectory, index, false);
- }
- /**
- * Updates the index re one document
- *
- * <ol>
- * <li>old documents to be deleted</li>
- * <li>unchanged documents, to be left alone, or</li>
- * <li>new documents, to be indexed.</li>
- * </ol>
- */
- public void indexDocument(File file) throws Exception {
- IndexConfiguration config = new IndexConfiguration(configFileName);
- log.debug("File: " + file);
- File dumpDir = new File(config.resolvePath(config.getHTDocsDumpDir()));
- log.debug("Dump dir: " + dumpDir);
- File indexDir = new File(config.resolvePath(config.getIndexDir()));
- log.debug("Index dir: " + indexDir);
- String id = IndexIterator.createID(file, dumpDir);
- boolean createNewIndex = false;
- if (!IndexReader.indexExists(indexDir)) {
- log.warn("Index does not exist yet: " + indexDir);
- createNewIndex = true;
- } else {
- // Delete from index
- IndexReader reader = IndexReader.open(indexDir.getAbsolutePath());
- Term term = new Term("id", id);
- log.debug(term.toString());
- int numberOfDeletedDocuments = reader.deleteDocuments(term);
- if (numberOfDeletedDocuments == 1) {
- log.info("Document has been deleted: " + term);
- } else {
- log.warn("No such document found in this index: " + term);
- }
- // log.debug("Number of deleted documents: " +
- // numberOfDeletedDocuments);
- // log.debug("Current number of documents in this index: " +
- // reader.numDocs());
- reader.close();
- }
- // Append to index
- Document doc = getDocumentCreator().getDocument(new File(dumpDir, id), dumpDir);
- IndexWriter writer = new IndexWriter(indexDir, new StandardAnalyzer(), createNewIndex);
- writer.setMaxFieldLength(1000000);
- writer.addDocument(doc);
- // log.debug("Document has been added: " + doc);
- log.info("Document has been added: " + id);
- writer.optimize();
- writer.close();
- }
- /**
- * Creates a new index.
- */
- public void createIndex(File dumpDirectory, File index) throws Exception {
- doIndex(dumpDirectory, index, true);
- }
- /**
- * Index files
- *
- * @param dumpDirectory
- * Directory where the files to be indexed are located
- * @param index
- * Directory where the index shall be located
- * @param create
- * <strong>true</strong> means the index will be created from
- * scratch, <strong>false</strong> means it will be indexed
- * incrementally
- */
- public void doIndex(File dumpDirectory, File index, boolean create) {
- if (!index.isDirectory()) {
- index.mkdirs();
- log.warn("Directory has been created: " + index.getAbsolutePath());
- }
- try {
- IndexWriter writer = new IndexWriter(index.getAbsolutePath(), new StandardAnalyzer(), create);
- writer.setMaxFieldLength(1000000);
- IndexInformation info = new IndexInformation(index.getAbsolutePath(), dumpDirectory, getFilter(indexer, configFileName), create);
- IndexHandler handler;
- if (create) {
- handler = new CreateIndexHandler(dumpDirectory, info, writer);
- } else {
- handler = new UpdateIndexHandler(dumpDirectory, info, writer);
- }
- IndexIterator iterator = new IndexIterator(index.getAbsolutePath(), getFilter(indexer, configFileName));
- iterator.addHandler(handler);
- iterator.iterate(dumpDirectory);
- writer.optimize();
- writer.close();
- } catch (IOException e) {
+ private static Logger log = Logger.getLogger(AbstractIndexer.class);
+ private DocumentCreator documentCreator;
+ private Element indexer;
+ private String configFileName;
+ /**
+ * Creates a new instance of AbstractIndexer
+ */
+ public AbstractIndexer() {
+ }
+ /**
+ * Returns the DocumentCreator of this indexer.
+ */
+ protected DocumentCreator getDocumentCreator() {
+ return documentCreator;
+ }
+ /**
+ * Initializes this indexer with command-line parameters.
+ */
+ public void configure(Element indexer, String configFileName) throws Exception {
+ documentCreator = createDocumentCreator(indexer, configFileName);
+ this.indexer = indexer;
+ this.configFileName = configFileName;
+ }
+ /**
+ * DOCUMENT ME!
+ *
+ * @param element
+ * DOCUMENT ME!
+ *
+ * @return DOCUMENT ME!
+ *
+ * @throws Exception
+ * DOCUMENT ME!
+ */
+ public abstract DocumentCreator createDocumentCreator(Element indexer, String configFileName) throws Exception;
+ /**
+ * Updates the index incrementally. Walk directory hierarchy in uid order, while keeping uid iterator from existing index in sync. Mismatches indicate one of:
+ * <ol>
+ * <li>old documents to be deleted</li>
+ * <li>unchanged documents, to be left alone, or</li>
+ * <li>new documents, to be indexed.</li>
+ * </ol>
+ */
+ public void updateIndex(File dumpDirectory, File index) throws Exception {
+ deleteStaleDocuments(dumpDirectory, index);
+ doIndex(dumpDirectory, index, false);
+ }
+ /**
+ * Updates the index re one document
+ *
+ * <ol>
+ * <li>old documents to be deleted</li>
+ * <li>unchanged documents, to be left alone, or</li>
+ * <li>new documents, to be indexed.</li>
+ * </ol>
+ */
+ public void indexDocument(File file) throws Exception {
+ IndexConfiguration config = new IndexConfiguration(configFileName);
+ log.debug("File: " + file);
+ File dumpDir = new File(config.resolvePath(config.getHTDocsDumpDir()));
+ log.debug("Dump dir: " + dumpDir);
+ File indexDir = new File(config.resolvePath(config.getIndexDir()));
+ log.debug("Index dir: " + indexDir);
+ String id = IndexIterator.createID(file, dumpDir);
+ boolean createNewIndex = false;
+ if(!IndexReader.indexExists(indexDir)){
+ log.warn("Index does not exist yet: " + indexDir);
+ createNewIndex = true;
+ }else{
+ // Delete from index
+ IndexReader reader = IndexReader.open(indexDir.getAbsolutePath());
+ Term term = new Term("id", id);
+ log.debug(term.toString());
+ int numberOfDeletedDocuments = reader.deleteDocuments(term);
+ if(numberOfDeletedDocuments == 1){
+ log.info("Document has been deleted: " + term);
+ }else{
+ log.warn("No such document found in this index: " + term);
+ }
+ // log.debug("Number of deleted documents: " +
+ // numberOfDeletedDocuments);
+ // log.debug("Current number of documents in this index: " +
+ // reader.numDocs());
+ reader.close();
+ }
+ // Append to index
+ Document doc = getDocumentCreator().getDocument(new File(dumpDir, id), dumpDir);
+ IndexWriter writer = new IndexWriter(indexDir, new StandardAnalyzer(), createNewIndex);
+ writer.setMaxFieldLength(1000000);
+ writer.addDocument(doc);
+ // log.debug("Document has been added: " + doc);
+ log.info("Document has been added: " + id);
+ writer.optimize();
+ writer.close();
+ }
+ /**
+ * Creates a new index.
+ */
+ public void createIndex(File dumpDirectory, File index) throws Exception {
+ doIndex(dumpDirectory, index, true);
+ }
+ /**
+ * Index files
+ *
+ * @param dumpDirectory
+ * Directory where the files to be indexed are located
+ * @param index
+ * Directory where the index shall be located
+ * @param create
+ * <strong>true</strong> means the index will be created from scratch, <strong>false</strong> means it will be indexed incrementally
+ */
+ public void doIndex(File dumpDirectory, File index, boolean create) {
+ if(!index.isDirectory()){
+ index.mkdirs();
+ log.warn("Directory has been created: " + index.getAbsolutePath());
+ }
+ try{
+ IndexWriter writer = new IndexWriter(index.getAbsolutePath(), new StandardAnalyzer(), create);
+ writer.setMaxFieldLength(1000000);
+ IndexInformation info = new IndexInformation(index.getAbsolutePath(), dumpDirectory, getFilter(indexer, configFileName), create);
+ IndexHandler handler;
+ if(create){
+ handler = new CreateIndexHandler(dumpDirectory, info, writer);
+ }else{
+ handler = new UpdateIndexHandler(dumpDirectory, info, writer);
+ }
+ IndexIterator iterator = new IndexIterator(index.getAbsolutePath(), getFilter(indexer, configFileName));
+ iterator.addHandler(handler);
+ iterator.iterate(dumpDirectory);
+ writer.optimize();
+ writer.close();
+ }catch(IOException e){
+ log.error(e);
+ }
+ }
+ /**
+ * Delete the stale documents.
+ */
+ protected void deleteStaleDocuments(File dumpDirectory, File index) throws Exception {
+ log.debug("Deleting stale documents");
+ IndexIterator iterator = new IndexIterator(index.getAbsolutePath(), getFilter(indexer, configFileName));
+ iterator.addHandler(new DeleteHandler());
+ iterator.iterate(dumpDirectory);
+ log.debug("Deleting stale documents finished");
+ }
+ /**
+ * Returns the filter used to receive the indexable files. Might be overwritten by inherited class.
+ */
+ public FileFilter getFilter(Element indexer, String configFileName) {
+ String[] indexableExtensions = {"html", "htm", "txt"};
+ return new AbstractIndexer.DefaultIndexFilter(indexableExtensions);
+ }
+ /**
+ * FileFilter used to obtain the files to index.
+ */
+ public class DefaultIndexFilter implements FileFilter {
+ protected String[] indexableExtensions;
+ /**
+ * Default indexable extensions: html, htm, txt
+ */
+ public DefaultIndexFilter() {
+ String[] iE = {"html", "htm", "txt"};
+ indexableExtensions = iE;
+ }
+ /**
+ *
+ */
+ public DefaultIndexFilter(String[] indexableExtensions) {
+ this.indexableExtensions = indexableExtensions;
+ }
+ /**
+ * Tests whether or not the specified abstract pathname should be included in a pathname list.
+ *
+ * @param pathname
+ * The abstract pathname to be tested
+ * @return <code>true</code> if and only if <code>pathname</code> should be included
+ *
+ */
+ public boolean accept(File file) {
+ boolean accept;
+ if(file.isDirectory()){
+ accept = true;
+ }else{
+ String fileName = file.getName();
+ String extension = fileName.substring(fileName.lastIndexOf(".") + 1);
+ accept = Arrays.asList(indexableExtensions).contains(extension);
+ }
+ return accept;
+ }
+ }
+ /**
+ * Deletes all stale documents up to the document representing the next file. The following documents are deleted:
+ * <ul>
+ * <li>representing files that where removed</li>
+ * <li>representing the same file but are older than the current file</li>
+ * </ul>
+ */
+ public class DeleteHandler extends AbstractIndexIteratorHandler {
+ /**
+ * Handles a stale document.
+ *
+ */
+ public void handleStaleDocument(IndexReader reader, Term term) {
+ log.debug("deleting " + IndexIterator.uid2url(term.text()));
+ try{
+ int deletedDocuments = reader.deleteDocuments(term);
+ log.debug("deleted " + deletedDocuments + " documents.");
+ }catch(IOException e){
log.error(e);
- }
- }
- /**
- * Delete the stale documents.
- */
- protected void deleteStaleDocuments(File dumpDirectory, File index) throws Exception {
- log.debug("Deleting stale documents");
- IndexIterator iterator = new IndexIterator(index.getAbsolutePath(), getFilter(indexer, configFileName));
- iterator.addHandler(new DeleteHandler());
- iterator.iterate(dumpDirectory);
- log.debug("Deleting stale documents finished");
- }
- /**
- * Returns the filter used to receive the indexable files. Might be
- * overwritten by inherited class.
- */
- public FileFilter getFilter(Element indexer, String configFileName) {
- String[] indexableExtensions = { "html", "htm", "txt" };
- return new AbstractIndexer.DefaultIndexFilter(indexableExtensions);
- }
- /**
- * FileFilter used to obtain the files to index.
- */
- public class DefaultIndexFilter implements FileFilter {
- protected String[] indexableExtensions;
- /**
- * Default indexable extensions: html, htm, txt
- */
- public DefaultIndexFilter() {
- String[] iE = { "html", "htm", "txt" };
- indexableExtensions = iE;
- }
- /**
- *
- */
- public DefaultIndexFilter(String[] indexableExtensions) {
- this.indexableExtensions = indexableExtensions;
- }
- /**
- * Tests whether or not the specified abstract pathname should be
- * included in a pathname list.
- *
- * @param pathname
- * The abstract pathname to be tested
- * @return <code>true</code> if and only if <code>pathname</code>
- * should be included
- *
- */
- public boolean accept(File file) {
- boolean accept;
- if (file.isDirectory()) {
- accept = true;
- } else {
- String fileName = file.getName();
- String extension = fileName.substring(fileName.lastIndexOf(".") + 1);
- accept = Arrays.asList(indexableExtensions).contains(extension);
- }
- return accept;
- }
- }
- /**
- * Deletes all stale documents up to the document representing the next
- * file. The following documents are deleted:
- * <ul>
- * <li>representing files that where removed</li>
- * <li>representing the same file but are older than the current file</li>
- * </ul>
- */
- public class DeleteHandler extends AbstractIndexIteratorHandler {
- /**
- * Handles a stale document.
- *
- */
- public void handleStaleDocument(IndexReader reader, Term term) {
- log.debug("deleting " + IndexIterator.uid2url(term.text()));
- try {
- int deletedDocuments = reader.deleteDocuments(term);
- log.debug("deleted " + deletedDocuments + " documents.");
- } catch (IOException e) {
- log.error(e);
- }
- }
- }
- /**
- * DOCUMENT ME!
- */
- public class IndexHandler extends AbstractIndexIteratorHandler {
- /**
- * Creates a new IndexHandler object.
- *
- * @param dumpDirectory
- * DOCUMENT ME!
- * @param info
- * DOCUMENT ME!
- * @param writer
- * DOCUMENT ME!
- */
- public IndexHandler(File dumpDirectory, IndexInformation info, IndexWriter writer) {
- this.info = info;
- this.dumpDirectory = dumpDirectory;
- this.writer = writer;
- }
- private IndexInformation info;
- protected IndexInformation getInformation() {
- return info;
- }
- private File dumpDirectory;
- protected File getDumpDirectory() {
- return dumpDirectory;
- }
- private IndexWriter writer;
- protected IndexWriter getWriter() {
- return writer;
- }
- /**
- * Add document to index
- */
- protected void addFile(File file) {
- log.debug("adding document: " + file.getAbsolutePath());
- try {
- Document doc = getDocumentCreator().getDocument(file, dumpDirectory);
- writer.addDocument(doc);
- } catch (Exception e) {
- log.error(e);
- }
- info.increase();
- log.info(info.printProgress());
- }
- }
- /**
- * DOCUMENT ME!
- */
- public class CreateIndexHandler extends IndexHandler {
- /**
- * Creates a new CreateIndexHandler object.
- *
- * @param dumpDirectory
- * DOCUMENT ME!
- * @param info
- * DOCUMENT ME!
- * @param writer
- * DOCUMENT ME!
- */
- public CreateIndexHandler(File dumpDirectory, IndexInformation info, IndexWriter writer) {
- super(dumpDirectory, info, writer);
- }
- /**
- * Handles a file. Used when creating a new index.
- */
- public void handleFile(IndexReader reader, File file) {
- addFile(file);
- }
- }
- /**
- * DOCUMENT ME!
- */
- public class UpdateIndexHandler extends IndexHandler {
- /**
- * Creates a new UpdateIndexHandler object.
- *
- * @param dumpDirectory
- * DOCUMENT ME!
- * @param info
- * DOCUMENT ME!
- * @param writer
- * DOCUMENT ME!
- */
- public UpdateIndexHandler(File dumpDirectory, IndexInformation info, IndexWriter writer) {
- super(dumpDirectory, info, writer);
- }
- /**
- * Handles a new document. Used when updating the index.
- */
- public void handleNewDocument(IndexReader reader, Term term, File file) {
- addFile(file);
- }
- }
+ }
+ }
+ }
+ /**
+ * DOCUMENT ME!
+ */
+ public class IndexHandler extends AbstractIndexIteratorHandler {
+ /**
+ * Creates a new IndexHandler object.
+ *
+ * @param dumpDirectory
+ * DOCUMENT ME!
+ * @param info
+ * DOCUMENT ME!
+ * @param writer
+ * DOCUMENT ME!
+ */
+ public IndexHandler(File dumpDirectory, IndexInformation info, IndexWriter writer) {
+ this.info = info;
+ this.dumpDirectory = dumpDirectory;
+ this.writer = writer;
+ }
+ private IndexInformation info;
+ protected IndexInformation getInformation() {
+ return info;
+ }
+ private File dumpDirectory;
+ protected File getDumpDirectory() {
+ return dumpDirectory;
+ }
+ private IndexWriter writer;
+ protected IndexWriter getWriter() {
+ return writer;
+ }
+ /**
+ * Add document to index
+ */
+ protected void addFile(File file) {
+ log.debug("adding document: " + file.getAbsolutePath());
+ try{
+ Document doc = getDocumentCreator().getDocument(file, dumpDirectory);
+ writer.addDocument(doc);
+ }catch(Exception e){
+ log.error(e);
+ }
+ info.increase();
+ log.info(info.printProgress());
+ }
+ }
+ /**
+ * DOCUMENT ME!
+ */
+ public class CreateIndexHandler extends IndexHandler {
+ /**
+ * Creates a new CreateIndexHandler object.
+ *
+ * @param dumpDirectory
+ * DOCUMENT ME!
+ * @param info
+ * DOCUMENT ME!
+ * @param writer
+ * DOCUMENT ME!
+ */
+ public CreateIndexHandler(File dumpDirectory, IndexInformation info, IndexWriter writer) {
+ super(dumpDirectory, info, writer);
+ }
+ /**
+ * Handles a file. Used when creating a new index.
+ */
+ public void handleFile(IndexReader reader, File file) {
+ addFile(file);
+ }
+ }
+ /**
+ * DOCUMENT ME!
+ */
+ public class UpdateIndexHandler extends IndexHandler {
+ /**
+ * Creates a new UpdateIndexHandler object.
+ *
+ * @param dumpDirectory
+ * DOCUMENT ME!
+ * @param info
+ * DOCUMENT ME!
+ * @param writer
+ * DOCUMENT ME!
+ */
+ public UpdateIndexHandler(File dumpDirectory, IndexInformation info, IndexWriter writer) {
+ super(dumpDirectory, info, writer);
+ }
+ /**
+ * Handles a new document. Used when updating the index.
+ */
+ public void handleNewDocument(IndexReader reader, Term term, File file) {
+ addFile(file);
+ }
+ }
}
Modified: lenya/branches/revolution/1.3.x/src/java/org/apache/lenya/lucene/index/ConfigurableDocumentCreator.java
URL: http://svn.apache.org/viewvc/lenya/branches/revolution/1.3.x/src/java/org/apache/lenya/lucene/index/ConfigurableDocumentCreator.java?rev=617035&r1=617034&r2=617035&view=diff
==============================================================================
--- lenya/branches/revolution/1.3.x/src/java/org/apache/lenya/lucene/index/ConfigurableDocumentCreator.java (original)
+++ lenya/branches/revolution/1.3.x/src/java/org/apache/lenya/lucene/index/ConfigurableDocumentCreator.java Wed Jan 30 23:44:03 2008
@@ -14,11 +14,8 @@
* limitations under the License.
*
*/
-
/* $Id$ */
-
package org.apache.lenya.lucene.index;
-
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
@@ -27,7 +24,6 @@
import java.io.StringWriter;
import java.io.Writer;
import java.lang.reflect.Method;
-
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.transform.OutputKeys;
@@ -36,212 +32,171 @@
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;
import javax.xml.transform.stream.StreamSource;
-
import org.apache.lenya.lucene.parser.HTMLParser;
import org.apache.lenya.lucene.parser.HTMLParserFactory;
import org.apache.lenya.lucene.parser.StringCleaner;
import org.apache.lenya.xml.DocumentHelper;
import org.apache.lenya.xml.NamespaceHelper;
-import org.apache.log4j.Category;
+import org.apache.log4j.Logger;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.InputSource;
-
/**
* Uses XSLT to transform a XML into a Lucene document
*/
public class ConfigurableDocumentCreator extends AbstractDocumentCreator {
- Category log = Category.getInstance(ConfigurableDocumentCreator.class);
-
- public static final String LUCENE_NAMESPACE = "http://apache.org/cocoon/lenya/lucene/1.0";
- public static final String XHTML_NAMESPACE = "http://www.w3.org/1999/xhtml";
-
- /**
- * Creates a new ConfigurableDocumentCreator object.
- *
- * @param stylesheet DOCUMENT ME!
- */
- public ConfigurableDocumentCreator(String stylesheet) {
- this.stylesheet = stylesheet;
- }
-
- private String stylesheet;
-
- /**
- * DOCUMENT ME!
- *
- * @return DOCUMENT ME!
- */
- public String getStylesheet() {
- return stylesheet;
- }
-
- /**
- * Transform source document into lucene document and generate a Lucene Document instance
- *
- * @param file DOCUMENT ME!
- * @param htdocsDumpDir DOCUMENT ME!
- *
- * @return DOCUMENT ME!
- *
- * @throws Exception DOCUMENT ME!
- */
- public Document getDocument(File file, File htdocsDumpDir) throws Exception {
- log.debug(".getDocument() : indexing " + file.getAbsolutePath());
- try {
-
- org.w3c.dom.Document sourceDocument = null;
- DocumentBuilderFactory parserFactory = DocumentBuilderFactory.newInstance();
- parserFactory.setValidating(false);
- parserFactory.setNamespaceAware(true);
- parserFactory.setIgnoringElementContentWhitespace(true);
- DocumentBuilder mybuilder = parserFactory.newDocumentBuilder();
- sourceDocument = mybuilder.parse(file.getAbsolutePath());
-
-
-// FIXME: What is this good for: <?xml version="1.0"?><body>...</body>
-/*
- NamespaceHelper documentHelper = new NamespaceHelper(XHTML_NAMESPACE, "xhtml", "html");
- org.w3c.dom.Document sourceDocument = documentHelper.getDocument();
-
- Element rootNode = sourceDocument.getDocumentElement();
-
- String bodyText = getBodyText(file);
- Element bodyElement = documentHelper.createElement("body", bodyText);
- rootNode.appendChild(bodyElement);
-*/
-
-
-
-
- DOMSource documentSource = new DOMSource(sourceDocument);
- Writer documentWriter = new StringWriter();
-
- TransformerFactory tFactory = TransformerFactory.newInstance();
- Transformer documentTransformer = tFactory.newTransformer(new StreamSource(new StringReader(getStylesheet())));
- documentTransformer.setOutputProperty(OutputKeys.INDENT, "yes");
- documentTransformer.setOutputProperty(OutputKeys.ENCODING, "ISO-8859-1");
-
- String fileName = file.getName();
-
- if (fileName.endsWith(".pdf.txt")) {
- fileName = fileName.substring(0, fileName.lastIndexOf(".txt"));
- }
-
- documentTransformer.setParameter("filename", fileName);
- documentTransformer.transform(documentSource, new StreamResult(documentWriter));
-
- // DEBUG: debug lucene documents
- //dumpLuceneDocument(file, documentWriter);
-
- DocumentBuilder builder = DocumentHelper.createBuilder();
- org.w3c.dom.Document luceneDocument = builder.parse(new InputSource(new StringReader(documentWriter.toString())));
-
- NamespaceHelper helper = new NamespaceHelper(LUCENE_NAMESPACE, "luc", luceneDocument);
- Element root = luceneDocument.getDocumentElement();
- Element[] fieldElements = helper.getChildren(root, "field");
-
- Document document = super.getDocument(file, htdocsDumpDir);
-
- Class[] parameterTypes = { String.class, String.class };
-
- for (int i = 0; i < fieldElements.length; i++) {
- String name = fieldElements[i].getAttribute("name");
- String type = fieldElements[i].getAttribute("type");
- String text = getText(fieldElements[i]);
-
- Method method = Field.class.getMethod(type, parameterTypes);
-
- String[] args = { name, text };
-
- Field field = (Field) method.invoke(null, args);
- document.add(field);
-
- }
-
- return document;
- } catch (Exception e) {
- throw e;
- }
- }
-
- /**
- * Writes the lucene XML document to a file.
- */
- protected void dumpLuceneDocument(File file, Writer writer) throws IOException {
- log.debug(".dumpLuceneDocument(): Dump document: " + file.getAbsolutePath());
-
- File luceneDocumentFile = new File(file.getAbsolutePath() + ".xluc");
- luceneDocumentFile.createNewFile();
-
- FileWriter fileWriter = new FileWriter(luceneDocumentFile);
- fileWriter.write(writer.toString());
- fileWriter.close();
- }
-
- /**
- * DOCUMENT ME!
- *
- * @param node DOCUMENT ME!
- *
- * @return DOCUMENT ME!
- */
- public static String getText(Node node) {
- StringBuffer result = new StringBuffer();
-
- if (!node.hasChildNodes()) {
- return "";
- }
-
- NodeList list = node.getChildNodes();
-
- for (int i = 0; i < list.getLength(); i++) {
- Node subnode = list.item(i);
-
- if (subnode.getNodeType() == Node.TEXT_NODE) {
- result.append(subnode.getNodeValue());
- } else if (subnode.getNodeType() == Node.CDATA_SECTION_NODE) {
- result.append(subnode.getNodeValue());
- } else if (subnode.getNodeType() == Node.ENTITY_REFERENCE_NODE) {
- // Recurse into the subtree for text
- // (and ignore comments)
- result.append(getText(subnode));
- }
- }
-
- return result.toString();
- }
-
- /**
- * DOCUMENT ME!
- *
- * @param file DOCUMENT ME!
- *
- * @return DOCUMENT ME!
- *
- * @throws Exception DOCUMENT ME!
- */
- public static String getBodyText(File file) throws Exception {
- HTMLParser parser = HTMLParserFactory.newInstance(file);
- parser.parse(file);
-
- Reader reader = parser.getReader();
- Writer writer = new StringWriter();
-
- int c;
-
- while ((c = reader.read()) != -1)
- writer.write(c);
-
- String content = writer.toString();
- reader.close();
- writer.close();
-
- content = StringCleaner.clean(content);
-
- return content;
- }
+ private static Logger log = Logger.getLogger(ConfigurableDocumentCreator.class);
+ public static final String LUCENE_NAMESPACE = "http://apache.org/cocoon/lenya/lucene/1.0";
+ public static final String XHTML_NAMESPACE = "http://www.w3.org/1999/xhtml";
+ /**
+ * Creates a new ConfigurableDocumentCreator object.
+ *
+ * @param stylesheet
+ * DOCUMENT ME!
+ */
+ public ConfigurableDocumentCreator(String stylesheet) {
+ this.stylesheet = stylesheet;
+ }
+ private String stylesheet;
+ /**
+ * DOCUMENT ME!
+ *
+ * @return DOCUMENT ME!
+ */
+ public String getStylesheet() {
+ return stylesheet;
+ }
+ /**
+ * Transform source document into lucene document and generate a Lucene Document instance
+ *
+ * @param file
+ * DOCUMENT ME!
+ * @param htdocsDumpDir
+ * DOCUMENT ME!
+ *
+ * @return DOCUMENT ME!
+ *
+ * @throws Exception
+ * DOCUMENT ME!
+ */
+ public Document getDocument(File file, File htdocsDumpDir) throws Exception {
+ log.debug(".getDocument() : indexing " + file.getAbsolutePath());
+ try{
+ org.w3c.dom.Document sourceDocument = null;
+ DocumentBuilderFactory parserFactory = DocumentBuilderFactory.newInstance();
+ parserFactory.setValidating(false);
+ parserFactory.setNamespaceAware(true);
+ parserFactory.setIgnoringElementContentWhitespace(true);
+ DocumentBuilder mybuilder = parserFactory.newDocumentBuilder();
+ sourceDocument = mybuilder.parse(file.getAbsolutePath());
+ // FIXME: What is this good for: <?xml version="1.0"?><body>...</body>
+ /*
+ * NamespaceHelper documentHelper = new NamespaceHelper(XHTML_NAMESPACE, "xhtml", "html"); org.w3c.dom.Document sourceDocument = documentHelper.getDocument();
+ *
+ * Element rootNode = sourceDocument.getDocumentElement();
+ *
+ * String bodyText = getBodyText(file); Element bodyElement = documentHelper.createElement("body", bodyText); rootNode.appendChild(bodyElement);
+ */
+ DOMSource documentSource = new DOMSource(sourceDocument);
+ Writer documentWriter = new StringWriter();
+ TransformerFactory tFactory = TransformerFactory.newInstance();
+ Transformer documentTransformer = tFactory.newTransformer(new StreamSource(new StringReader(getStylesheet())));
+ documentTransformer.setOutputProperty(OutputKeys.INDENT, "yes");
+ documentTransformer.setOutputProperty(OutputKeys.ENCODING, "ISO-8859-1");
+ String fileName = file.getName();
+ if(fileName.endsWith(".pdf.txt")){
+ fileName = fileName.substring(0, fileName.lastIndexOf(".txt"));
+ }
+ documentTransformer.setParameter("filename", fileName);
+ documentTransformer.transform(documentSource, new StreamResult(documentWriter));
+ // DEBUG: debug lucene documents
+ // dumpLuceneDocument(file, documentWriter);
+ DocumentBuilder builder = DocumentHelper.createBuilder();
+ org.w3c.dom.Document luceneDocument = builder.parse(new InputSource(new StringReader(documentWriter.toString())));
+ NamespaceHelper helper = new NamespaceHelper(LUCENE_NAMESPACE, "luc", luceneDocument);
+ Element root = luceneDocument.getDocumentElement();
+ Element[] fieldElements = helper.getChildren(root, "field");
+ Document document = super.getDocument(file, htdocsDumpDir);
+ Class[] parameterTypes = {String.class, String.class};
+ for(int i = 0; i < fieldElements.length; i++){
+ String name = fieldElements[i].getAttribute("name");
+ String type = fieldElements[i].getAttribute("type");
+ String text = getText(fieldElements[i]);
+ Method method = Field.class.getMethod(type, parameterTypes);
+ String[] args = {name, text};
+ Field field = (Field) method.invoke(null, args);
+ document.add(field);
+ }
+ return document;
+ }catch(Exception e){
+ throw e;
+ }
+ }
+ /**
+ * Writes the lucene XML document to a file.
+ */
+ protected void dumpLuceneDocument(File file, Writer writer) throws IOException {
+ log.debug(".dumpLuceneDocument(): Dump document: " + file.getAbsolutePath());
+ File luceneDocumentFile = new File(file.getAbsolutePath() + ".xluc");
+ luceneDocumentFile.createNewFile();
+ FileWriter fileWriter = new FileWriter(luceneDocumentFile);
+ fileWriter.write(writer.toString());
+ fileWriter.close();
+ }
+ /**
+ * DOCUMENT ME!
+ *
+ * @param node
+ * DOCUMENT ME!
+ *
+ * @return DOCUMENT ME!
+ */
+ public static String getText(Node node) {
+ StringBuffer result = new StringBuffer();
+ if(!node.hasChildNodes()){
+ return "";
+ }
+ NodeList list = node.getChildNodes();
+ for(int i = 0; i < list.getLength(); i++){
+ Node subnode = list.item(i);
+ if(subnode.getNodeType() == Node.TEXT_NODE){
+ result.append(subnode.getNodeValue());
+ }else if(subnode.getNodeType() == Node.CDATA_SECTION_NODE){
+ result.append(subnode.getNodeValue());
+ }else if(subnode.getNodeType() == Node.ENTITY_REFERENCE_NODE){
+ // Recurse into the subtree for text
+ // (and ignore comments)
+ result.append(getText(subnode));
+ }
+ }
+ return result.toString();
+ }
+ /**
+ * DOCUMENT ME!
+ *
+ * @param file
+ * DOCUMENT ME!
+ *
+ * @return DOCUMENT ME!
+ *
+ * @throws Exception
+ * DOCUMENT ME!
+ */
+ public static String getBodyText(File file) throws Exception {
+ HTMLParser parser = HTMLParserFactory.newInstance(file);
+ parser.parse(file);
+ Reader reader = parser.getReader();
+ Writer writer = new StringWriter();
+ int c;
+ while((c = reader.read()) != -1)
+ writer.write(c);
+ String content = writer.toString();
+ reader.close();
+ writer.close();
+ content = StringCleaner.clean(content);
+ return content;
+ }
}
Modified: lenya/branches/revolution/1.3.x/src/java/org/apache/lenya/lucene/index/ConfigurableIndexer.java
URL: http://svn.apache.org/viewvc/lenya/branches/revolution/1.3.x/src/java/org/apache/lenya/lucene/index/ConfigurableIndexer.java?rev=617035&r1=617034&r2=617035&view=diff
==============================================================================
--- lenya/branches/revolution/1.3.x/src/java/org/apache/lenya/lucene/index/ConfigurableIndexer.java (original)
+++ lenya/branches/revolution/1.3.x/src/java/org/apache/lenya/lucene/index/ConfigurableIndexer.java Wed Jan 30 23:44:03 2008
@@ -14,190 +14,167 @@
* limitations under the License.
*
*/
-
/* $Id$ */
-
package org.apache.lenya.lucene.index;
-
import java.io.File;
import java.io.FileFilter;
import java.io.StringWriter;
import java.io.Writer;
import java.net.URI;
import java.net.URL;
-
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;
import javax.xml.transform.stream.StreamSource;
-
import org.apache.cocoon.util.NetUtils;
import org.apache.lenya.xml.DocumentHelper;
-import org.apache.log4j.Category;
+import org.apache.log4j.Logger;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
-
public class ConfigurableIndexer extends AbstractIndexer {
- Category log = Category.getInstance(ConfigurableIndexer.class);
-
- /**
- * Instantiate a Document Creator for creating Lucene Documents
- *
- * @param element <code>indexer</code> node
- *
- * @return DocumentCreator
- *
- * @throws Exception DOCUMENT ME!
- */
- public DocumentCreator createDocumentCreator(Element indexer, String configFileName) throws Exception {
- log.debug(".createDocumentCreatort(): Element name: " + indexer.getNodeName());
-
- // FIXME: concat these files ...
- String configurationFileName = new File(configFileName).getParent() + File.separator + getLuceneDocConfigFileName(indexer);
- File configurationFile = new File(configurationFileName);
- String stylesheet = getStylesheet(configurationFile);
- return new ConfigurableDocumentCreator(stylesheet);
- }
-
- public static final String CONFIGURATION_CREATOR_STYLESHEET = "org/apache/lenya/lucene/index/configuration2xslt.xsl";
-
- /**
- * Converts the configuration file to an XSLT stylesheet and returns a reader that reads this stylesheet.
- */
- protected String getStylesheet(File configurationFile) throws Exception {
- log.debug(".getStylesheet(): Configuration file: " + configurationFile.getAbsolutePath());
-
- URL configurationCreatorURL = ConfigurableIndexer.class.getClassLoader().getResource(CONFIGURATION_CREATOR_STYLESHEET);
- File configurationStylesheetFile = new File(new URI(NetUtils.encodePath(configurationCreatorURL.toString())));
- Document configurationDocument = DocumentHelper.readDocument(configurationFile);
-
- TransformerFactory tFactory = TransformerFactory.newInstance();
- Transformer configurationTransformer = tFactory.newTransformer(new StreamSource(configurationStylesheetFile));
-
- DOMSource source = new DOMSource(configurationDocument);
- Writer stylesheetWriter = new StringWriter();
- configurationTransformer.transform(source, new StreamResult(stylesheetWriter));
-
- // Show meta stylesheet which has been created by configuration2xslt.xsl
- log.debug(".getStylesheet(): Meta Stylesheet: " + stylesheetWriter.toString());
-
- return stylesheetWriter.toString();
- }
-
- /**
- * Returns the filter used to receive the indexable files.
- */
- public FileFilter getFilter(Element indexer, String configFileName) {
- if (extensionsExists(indexer)) {
- String[] indexableExtensions = new String[1];
- indexableExtensions[0] = getExtensions(indexer);
- return new AbstractIndexer.DefaultIndexFilter(indexableExtensions);
- } else if (filterExists(indexer)) {
- return getFilterFromConfiguration(indexer);
- }
-
- return new AbstractIndexer.DefaultIndexFilter();
- }
-
- /**
- *
- */
- private String getLuceneDocConfigFileName(Element indexer) {
- String luceneDocConfigFileName = null;
-
- NodeList nl = indexer.getChildNodes();
- for (int i = 0; i < nl.getLength(); i++) {
- Node node = nl.item(i);
- if (node.getNodeType() == Node.ELEMENT_NODE && node.getNodeName().equals("configuration")) {
- log.debug(".getLuceneDocConfigFileName(): Node configuration exists!");
- luceneDocConfigFileName = ((Element)node).getAttribute("src");
- }
- }
- if (luceneDocConfigFileName == null) {
- log.error(".getLuceneDocConfigFileName(): ERROR: Lucene Document Configuration is not specified (indexer/configuration/@src)");
- }
- log.debug(".getLuceneDocConfigFileName(): Lucene Document Configuration: " + luceneDocConfigFileName);
- return luceneDocConfigFileName;
- }
-
- /**
- *
- */
- private String getExtensions(Element indexer) {
- String extensions = null;
-
- NodeList nl = indexer.getChildNodes();
- for (int i = 0; i < nl.getLength(); i++) {
- Node node = nl.item(i);
- if (node.getNodeType() == Node.ELEMENT_NODE && node.getNodeName().equals("extensions")) {
- log.debug("Node extensions exists!");
- extensions = ((Element)node).getAttribute("src");
- }
- }
- if (extensions == null) {
- log.error("Extensions have not been specified (indexer/extensions/@src)");
- }
- log.debug("Extensions: " + extensions);
- return extensions;
- }
-
- /**
- *
- */
- private FileFilter getFilterFromConfiguration(Element indexer) {
- String className = null;
-
- NodeList nl = indexer.getChildNodes();
- for (int i = 0; i < nl.getLength(); i++) {
- Node node = nl.item(i);
- if (node.getNodeType() == Node.ELEMENT_NODE && node.getNodeName().equals("filter")) {
- log.debug("Node filter exists!");
- className = ((Element)node).getAttribute("class");
- }
- }
- if (className == null) {
- log.error("Class name has not been specified (indexer/filter/@class)");
- return null;
- }
- log.debug("Class name: " + className);
- try {
- return (FileFilter)Class.forName(className).newInstance();
- } catch(Exception e) {
- log.error("" + e);
- }
- return null;
- }
-
- /**
- * Check if node <extensions src="..."/> exists
- */
- private boolean extensionsExists(Element indexer) {
- NodeList nl = indexer.getChildNodes();
- for (int i = 0; i < nl.getLength(); i++) {
- Node node = nl.item(i);
- if (node.getNodeType() == Node.ELEMENT_NODE && node.getNodeName().equals("extensions")) {
- log.debug("Node <extensions src=\"...\"/> exist");
- return true;
- }
- }
- return false;
- }
-
- /**
- *
- */
- private boolean filterExists(Element indexer) {
- NodeList nl = indexer.getChildNodes();
- for (int i = 0; i < nl.getLength(); i++) {
- Node node = nl.item(i);
- if (node.getNodeType() == Node.ELEMENT_NODE && node.getNodeName().equals("filter")) {
- log.debug("Node filter exists!");
- return true;
- }
- }
- return false;
- }
+ private static Logger log = Logger.getLogger(ConfigurableIndexer.class);
+ /**
+ * Instantiate a Document Creator for creating Lucene Documents
+ *
+ * @param element
+ * <code>indexer</code> node
+ *
+ * @return DocumentCreator
+ *
+ * @throws Exception
+ * DOCUMENT ME!
+ */
+ public DocumentCreator createDocumentCreator(Element indexer, String configFileName) throws Exception {
+ log.debug(".createDocumentCreatort(): Element name: " + indexer.getNodeName());
+ // FIXME: concat these files ...
+ String configurationFileName = new File(configFileName).getParent() + File.separator + getLuceneDocConfigFileName(indexer);
+ File configurationFile = new File(configurationFileName);
+ String stylesheet = getStylesheet(configurationFile);
+ return new ConfigurableDocumentCreator(stylesheet);
+ }
+ public static final String CONFIGURATION_CREATOR_STYLESHEET = "org/apache/lenya/lucene/index/configuration2xslt.xsl";
+ /**
+ * Converts the configuration file to an XSLT stylesheet and returns a reader that reads this stylesheet.
+ */
+ protected String getStylesheet(File configurationFile) throws Exception {
+ log.debug(".getStylesheet(): Configuration file: " + configurationFile.getAbsolutePath());
+ URL configurationCreatorURL = ConfigurableIndexer.class.getClassLoader().getResource(CONFIGURATION_CREATOR_STYLESHEET);
+ File configurationStylesheetFile = new File(new URI(NetUtils.encodePath(configurationCreatorURL.toString())));
+ Document configurationDocument = DocumentHelper.readDocument(configurationFile);
+ TransformerFactory tFactory = TransformerFactory.newInstance();
+ Transformer configurationTransformer = tFactory.newTransformer(new StreamSource(configurationStylesheetFile));
+ DOMSource source = new DOMSource(configurationDocument);
+ Writer stylesheetWriter = new StringWriter();
+ configurationTransformer.transform(source, new StreamResult(stylesheetWriter));
+ // Show meta stylesheet which has been created by configuration2xslt.xsl
+ log.debug(".getStylesheet(): Meta Stylesheet: " + stylesheetWriter.toString());
+ return stylesheetWriter.toString();
+ }
+ /**
+ * Returns the filter used to receive the indexable files.
+ */
+ public FileFilter getFilter(Element indexer, String configFileName) {
+ if(extensionsExists(indexer)){
+ String[] indexableExtensions = new String[1];
+ indexableExtensions[0] = getExtensions(indexer);
+ return new AbstractIndexer.DefaultIndexFilter(indexableExtensions);
+ }else if(filterExists(indexer)){
+ return getFilterFromConfiguration(indexer);
+ }
+ return new AbstractIndexer.DefaultIndexFilter();
+ }
+ /**
+ *
+ */
+ private String getLuceneDocConfigFileName(Element indexer) {
+ String luceneDocConfigFileName = null;
+ NodeList nl = indexer.getChildNodes();
+ for(int i = 0; i < nl.getLength(); i++){
+ Node node = nl.item(i);
+ if(node.getNodeType() == Node.ELEMENT_NODE && node.getNodeName().equals("configuration")){
+ log.debug(".getLuceneDocConfigFileName(): Node configuration exists!");
+ luceneDocConfigFileName = ((Element) node).getAttribute("src");
+ }
+ }
+ if(luceneDocConfigFileName == null){
+ log.error(".getLuceneDocConfigFileName(): ERROR: Lucene Document Configuration is not specified (indexer/configuration/@src)");
+ }
+ log.debug(".getLuceneDocConfigFileName(): Lucene Document Configuration: " + luceneDocConfigFileName);
+ return luceneDocConfigFileName;
+ }
+ /**
+ *
+ */
+ private String getExtensions(Element indexer) {
+ String extensions = null;
+ NodeList nl = indexer.getChildNodes();
+ for(int i = 0; i < nl.getLength(); i++){
+ Node node = nl.item(i);
+ if(node.getNodeType() == Node.ELEMENT_NODE && node.getNodeName().equals("extensions")){
+ log.debug("Node extensions exists!");
+ extensions = ((Element) node).getAttribute("src");
+ }
+ }
+ if(extensions == null){
+ log.error("Extensions have not been specified (indexer/extensions/@src)");
+ }
+ log.debug("Extensions: " + extensions);
+ return extensions;
+ }
+ /**
+ *
+ */
+ private FileFilter getFilterFromConfiguration(Element indexer) {
+ String className = null;
+ NodeList nl = indexer.getChildNodes();
+ for(int i = 0; i < nl.getLength(); i++){
+ Node node = nl.item(i);
+ if(node.getNodeType() == Node.ELEMENT_NODE && node.getNodeName().equals("filter")){
+ log.debug("Node filter exists!");
+ className = ((Element) node).getAttribute("class");
+ }
+ }
+ if(className == null){
+ log.error("Class name has not been specified (indexer/filter/@class)");
+ return null;
+ }
+ log.debug("Class name: " + className);
+ try{
+ return (FileFilter) Class.forName(className).newInstance();
+ }catch(Exception e){
+ log.error("" + e);
+ }
+ return null;
+ }
+ /**
+ * Check if node <extensions src="..."/> exists
+ */
+ private boolean extensionsExists(Element indexer) {
+ NodeList nl = indexer.getChildNodes();
+ for(int i = 0; i < nl.getLength(); i++){
+ Node node = nl.item(i);
+ if(node.getNodeType() == Node.ELEMENT_NODE && node.getNodeName().equals("extensions")){
+ log.debug("Node <extensions src=\"...\"/> exist");
+ return true;
+ }
+ }
+ return false;
+ }
+ /**
+ *
+ */
+ private boolean filterExists(Element indexer) {
+ NodeList nl = indexer.getChildNodes();
+ for(int i = 0; i < nl.getLength(); i++){
+ Node node = nl.item(i);
+ if(node.getNodeType() == Node.ELEMENT_NODE && node.getNodeName().equals("filter")){
+ log.debug("Node filter exists!");
+ return true;
+ }
+ }
+ return false;
+ }
}
lmpx.com only provides a reader for public news (NNTP) servers. It is not
affiliated with the servers or forums shown here and is not responsible for
the content of articles, which is written by their respective authors.