[mb-commits] r13694 - in search_server/trunk: index/src/main/java/org/musicbrainz/search/index index/src/test/java/org/musicbrainz/search/index servlet/src/main/java/org/musicbrainz/search/servlet/mmd2 servlet/src/main/resources servlet/src/test/java/org/musicbrainz/search/servlet
[email protected] Tue, 04 Jun 2013 20:32:24 +0000
| Newsgroups | gmane.comp.audio.musicbrainz.cvs |
|---|---|
| Message-ID | <E1Ujxu4-0007rM-AK@wiley> |
Author: ijabz
Date: 2013-06-04 20:32:24 +0000 (Tue, 04 Jun 2013)
New Revision: 13694
Modified:
search_server/trunk/index/src/main/java/org/musicbrainz/search/index/AreaIndex.java
search_server/trunk/index/src/main/java/org/musicbrainz/search/index/AreaIndexField.java
search_server/trunk/index/src/main/java/org/musicbrainz/search/index/LabelIndex.java
search_server/trunk/index/src/test/java/org/musicbrainz/search/index/AbstractIndexTest.java
search_server/trunk/index/src/test/java/org/musicbrainz/search/index/AreaIndexTest.java
search_server/trunk/servlet/src/main/java/org/musicbrainz/search/servlet/mmd2/AreaWriter.java
search_server/trunk/servlet/src/main/java/org/musicbrainz/search/servlet/mmd2/ArtistWriter.java
search_server/trunk/servlet/src/main/resources/oxml.xml
search_server/trunk/servlet/src/test/java/org/musicbrainz/search/servlet/FindAreaTest.java
Log:
SEARCH-286:Add support for searching/displaying area aliases
Modified: search_server/trunk/index/src/main/java/org/musicbrainz/search/index/AreaIndex.java
===================================================================
--- search_server/trunk/index/src/main/java/org/musicbrainz/search/index/AreaIndex.java 2013-06-04 19:37:16 UTC (rev 13693)
+++ search_server/trunk/index/src/main/java/org/musicbrainz/search/index/AreaIndex.java 2013-06-04 20:32:24 UTC (rev 13694)
@@ -32,14 +32,15 @@
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexWriter;
-import org.musicbrainz.mmd2.AreaList;
-import org.musicbrainz.mmd2.DefAreaElementInner;
-import org.musicbrainz.mmd2.Label;
-import org.musicbrainz.mmd2.ObjectFactory;
+import org.musicbrainz.mmd2.*;
import org.musicbrainz.search.MbDocument;
import java.io.IOException;
import java.sql.*;
+import java.util.HashMap;
+import java.util.LinkedHashSet;
+import java.util.Map;
+import java.util.Set;
public class AreaIndex extends DatabaseIndex {
@@ -89,23 +90,79 @@
" LEFT JOIN area_type at ON a.type = at.id " +
" WHERE a.id BETWEEN ? AND ? " +
" ORDER BY a.id");
+
+ addPreparedStatement("ALIASES",
+ "SELECT a.area as area, a.name as alias, a.sort_name as alias_sortname, a.primary_for_locale, a.locale, att.name as type," +
+ "a.begin_date_year, a.begin_date_month, a.begin_date_day, a.end_date_year, a.end_date_month, a.end_date_day" +
+ " FROM area_alias a" +
+ " LEFT JOIN area_alias_type att on (a.type=att.id)" +
+ " WHERE area BETWEEN ? AND ?" +
+ " ORDER BY area, alias, alias_sortname");
+
}
public void indexData(IndexWriter indexWriter, int min, int max) throws SQLException, IOException {
- PreparedStatement st = getPreparedStatement("AREA");
+ ObjectFactory of = new ObjectFactory();
+
+ // Get area aliases
+ Map<Integer, Set<Alias>> aliases = new HashMap<Integer, Set<Alias>>();
+ PreparedStatement st = getPreparedStatement("ALIASES");
st.setInt(1, min);
st.setInt(2, max);
ResultSet rs = st.executeQuery();
while (rs.next()) {
- indexWriter.addDocument(documentFromResultSet(rs));
+ int areaId = rs.getInt("area");
+ Set<Alias> list;
+ if (!aliases.containsKey(areaId)) {
+ list = new LinkedHashSet<Alias>();
+ aliases.put(areaId, list);
+ } else {
+ list = aliases.get(areaId);
+ }
+ Alias alias = of.createAlias();
+ alias.setContent(rs.getString("alias"));
+ alias.setSortName(rs.getString("alias_sortname"));
+ boolean isPrimary = rs.getBoolean("primary_for_locale");
+ if(isPrimary) {
+ alias.setPrimary("primary");
+ }
+ String locale = rs.getString("locale");
+ if(locale!=null) {
+ alias.setLocale(locale);
+ }
+ String type = rs.getString("type");
+ if(type!=null) {
+ alias.setType(type);
+ }
+
+ String begin = Utils.formatDate(rs.getInt("begin_date_year"), rs.getInt("begin_date_month"), rs.getInt("begin_date_day"));
+ if(!Strings.isNullOrEmpty(begin)) {
+ alias.setBeginDate(begin);
+ }
+
+ String end = Utils.formatDate(rs.getInt("end_date_year"), rs.getInt("end_date_month"), rs.getInt("end_date_day"));
+ if(!Strings.isNullOrEmpty(end)) {
+ alias.setEndDate(end);
+ }
+ list.add(alias);
}
rs.close();
+ st = getPreparedStatement("AREA");
+ st.setInt(1, min);
+ st.setInt(2, max);
+ rs = st.executeQuery();
+ while (rs.next()) {
+ indexWriter.addDocument(documentFromResultSet(rs, aliases));
+ }
+ rs.close();
+
}
- public Document documentFromResultSet(ResultSet rs) throws SQLException {
+ public Document documentFromResultSet(ResultSet rs,
+ Map<Integer, Set<Alias>> aliases) throws SQLException {
MbDocument doc = new MbDocument();
ObjectFactory of = new ObjectFactory();
@@ -114,7 +171,8 @@
DefAreaElementInner area = of.createDefAreaElementInner();
areaList.getArea().add(area);
- doc.addField(AreaIndexField.ID, rs.getString("id"));
+ int areaId = rs.getInt("id");
+ doc.addField(AreaIndexField.ID, areaId);
String guid = rs.getString("gid");
doc.addField(AreaIndexField.AREA_ID, guid);
@@ -134,6 +192,18 @@
area.setType(type);
}
+ if (aliases.containsKey(areaId)) {
+ AliasList aliasList = of.createAliasList();
+ for (Alias nextAlias : aliases.get(areaId)) {
+ doc.addField(AreaIndexField.ALIAS, nextAlias.getContent());
+ if(!nextAlias.getSortName().equals(nextAlias.getContent())) {
+ doc.addField(AreaIndexField.ALIAS, nextAlias.getSortName());
+ }
+ aliasList.getAlias().add(nextAlias);
+ }
+ area.setAliasList(aliasList);
+ }
+
String store = MMDSerializer.serialize(areaList);
doc.addField(AreaIndexField.AREA_STORE, store);
return doc.getLuceneDocument();
Modified: search_server/trunk/index/src/main/java/org/musicbrainz/search/index/AreaIndexField.java
===================================================================
--- search_server/trunk/index/src/main/java/org/musicbrainz/search/index/AreaIndexField.java 2013-06-04 19:37:16 UTC (rev 13693)
+++ search_server/trunk/index/src/main/java/org/musicbrainz/search/index/AreaIndexField.java 2013-06-04 20:32:24 UTC (rev 13694)
@@ -33,6 +33,7 @@
import org.apache.lucene.document.FieldType;
import org.musicbrainz.search.analysis.CaseInsensitiveKeywordAnalyzer;
import org.musicbrainz.search.analysis.MusicbrainzAnalyzer;
+import org.musicbrainz.search.analysis.MusicbrainzWithPosGapAnalyzer;
/**
* Fields created in Lucene Search Index
@@ -40,6 +41,7 @@
public enum AreaIndexField implements IndexField {
ID ("_id", MusicBrainzFieldTypes.TEXT_STORED_ANALYZED_NO_NORMS, new KeywordAnalyzer()),
+ ALIAS ("alias", MusicBrainzFieldTypes.TEXT_NOT_STORED_ANALYZED, new MusicbrainzWithPosGapAnalyzer()),
AREA ("area", MusicBrainzFieldTypes.TEXT_STORED_ANALYZED, new MusicbrainzAnalyzer()),
AREA_ID ("aid", MusicBrainzFieldTypes.TEXT_STORED_NOT_ANALYZED_NO_NORMS, new KeywordAnalyzer()),
AREA_STORE ("areastore", MusicBrainzFieldTypes.TEXT_STORED_NOT_INDEXED),
Modified: search_server/trunk/index/src/main/java/org/musicbrainz/search/index/LabelIndex.java
===================================================================
--- search_server/trunk/index/src/main/java/org/musicbrainz/search/index/LabelIndex.java 2013-06-04 19:37:16 UTC (rev 13693)
+++ search_server/trunk/index/src/main/java/org/musicbrainz/search/index/LabelIndex.java 2013-06-04 20:32:24 UTC (rev 13694)
@@ -234,6 +234,8 @@
Label label = of.createLabel();
int labelId = rs.getInt("id");
+ doc.addField(LabelIndexField.ID, labelId);
+
String labelGuid = rs.getString("gid");
doc.addField(LabelIndexField.LABEL_ID, labelGuid);
label.setId(labelGuid);
Modified: search_server/trunk/index/src/test/java/org/musicbrainz/search/index/AbstractIndexTest.java
===================================================================
--- search_server/trunk/index/src/test/java/org/musicbrainz/search/index/AbstractIndexTest.java 2013-06-04 19:37:16 UTC (rev 13693)
+++ search_server/trunk/index/src/test/java/org/musicbrainz/search/index/AbstractIndexTest.java 2013-06-04 20:32:24 UTC (rev 13694)
@@ -996,6 +996,8 @@
stmt.addBatch("INSERT INTO work_alias_type (id, name) VALUES (1, 'Search hint')");
+ stmt.addBatch("INSERT INTO area_alias_type (id, name) VALUES (1, 'AliasType')");
+
}
protected void insertReplicationInfo(Statement stmt) throws Exception {
Modified: search_server/trunk/index/src/test/java/org/musicbrainz/search/index/AreaIndexTest.java
===================================================================
--- search_server/trunk/index/src/test/java/org/musicbrainz/search/index/AreaIndexTest.java 2013-06-04 19:37:16 UTC (rev 13693)
+++ search_server/trunk/index/src/test/java/org/musicbrainz/search/index/AreaIndexTest.java 2013-06-04 20:32:24 UTC (rev 13694)
@@ -6,12 +6,14 @@
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.store.RAMDirectory;
import org.junit.Test;
+import org.musicbrainz.mmd2.Alias;
import org.musicbrainz.mmd2.AreaList;
import org.musicbrainz.mmd2.DefAreaElementInner;
import java.sql.Statement;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
public class AreaIndexTest extends AbstractIndexTest {
@@ -36,6 +38,8 @@
Statement stmt = conn.createStatement();
stmt.addBatch("INSERT INTO area (id, gid,name,sort_name, type) VALUES (1, 'aa95182f-df0a-3ad6-8bfb-4b63482cd276', 'Afghanistan','Afghanistan', 1)");
stmt.addBatch("INSERT INTO area_type(id, name) VALUES (1, 'Country')");
+ stmt.addBatch("INSERT INTO area_alias (id, area, sort_name, name, primary_for_locale, locale, type ) VALUES (3, 1, 'Afghan', 'Afghany', true, 'en',1)");
+
stmt.executeBatch();
stmt.close();
}
@@ -88,9 +92,24 @@
ir.close();
}
+ @Test
+ public void testIndexAreaAlias() throws Exception {
+ addAreaOne();
+ RAMDirectory ramDir = new RAMDirectory();
+ createIndex(ramDir);
+
+ IndexReader ir = DirectoryReader.open(ramDir);
+ assertEquals(2, ir.numDocs());
+ {
+ checkTerm(ir, AreaIndexField.ALIAS, "afghan");
+
+ }
+ ir.close();
+ }
+
@Test
- public void testIndexArea() throws Exception {
+ public void testStoredIndexArea() throws Exception {
addAreaOne();
RAMDirectory ramDir = new RAMDirectory();
@@ -107,6 +126,14 @@
assertEquals("Afghanistan", area.getName());
assertEquals("Afghanistan", area.getSortName());
assertEquals("Country", area.getType());
+ assertNotNull(area.getAliasList());
+ assertEquals(1,area.getAliasList().getAlias().size());
+
+ Alias alias = area.getAliasList().getAlias().get(0);
+ assertEquals("Afghany",alias.getContent());
+ assertEquals("Afghan",alias.getSortName());
+ assertEquals("en",alias.getLocale());
+ assertEquals("AliasType",alias.getType());
}
ir.close();
}
Modified: search_server/trunk/servlet/src/main/java/org/musicbrainz/search/servlet/mmd2/AreaWriter.java
===================================================================
--- search_server/trunk/servlet/src/main/java/org/musicbrainz/search/servlet/mmd2/AreaWriter.java 2013-06-04 19:37:16 UTC (rev 13693)
+++ search_server/trunk/servlet/src/main/java/org/musicbrainz/search/servlet/mmd2/AreaWriter.java 2013-06-04 20:32:24 UTC (rev 13694)
@@ -29,10 +29,7 @@
package org.musicbrainz.search.servlet.mmd2;
-import org.musicbrainz.mmd2.DefAreaElementInner;
-import org.musicbrainz.mmd2.Metadata;
-import org.musicbrainz.mmd2.ObjectFactory;
-import org.musicbrainz.mmd2.AreaList;
+import org.musicbrainz.mmd2.*;
import org.musicbrainz.search.MbDocument;
import org.musicbrainz.search.index.MMDSerializer;
import org.musicbrainz.search.index.AreaIndexField;
@@ -90,4 +87,43 @@
area.setScore(String.valueOf(result.getNormalizedScore()));
list.add(area);
}
+
+ /**
+ * Overridden to ensure all attributes are set for each alias
+ *
+ * @param metadata
+ */
+ @Override
+ public void adjustForJson(Metadata metadata) {
+
+ if (metadata.getAreaList().getArea().size()>0) {
+ for(DefAreaElementInner area:metadata.getAreaList().getArea()) {
+ if(area.getAliasList()!=null) {
+ for (Alias alias : area.getAliasList().getAlias()) {
+
+ if (alias.getBeginDate() == null) {
+ alias.setBeginDate("");
+ }
+ if (alias.getEndDate() == null) {
+ alias.setEndDate("");
+ }
+ if (alias.getType() == null) {
+ alias.setType("");
+ }
+ if (alias.getLocale() == null) {
+ alias.setLocale("");
+ }
+ //On Xml output as primary, but in json they have changed to true/false
+ if (alias.getPrimary() == null) {
+ alias.setPrimary("false");
+ }
+ else {
+ alias.setPrimary("true");
+ }
+ }
+ }
+ }
+ }
+
+ }
}
\ No newline at end of file
Modified: search_server/trunk/servlet/src/main/java/org/musicbrainz/search/servlet/mmd2/ArtistWriter.java
===================================================================
--- search_server/trunk/servlet/src/main/java/org/musicbrainz/search/servlet/mmd2/ArtistWriter.java 2013-06-04 19:37:16 UTC (rev 13693)
+++ search_server/trunk/servlet/src/main/java/org/musicbrainz/search/servlet/mmd2/ArtistWriter.java 2013-06-04 20:32:24 UTC (rev 13694)
@@ -89,7 +89,7 @@
}
/**
- * Overriden to ensure all attributes are set for each alias
+ * Overridden to ensure all attributes are set for each alias
*
* @param metadata
*/
Modified: search_server/trunk/servlet/src/main/resources/oxml.xml
===================================================================
--- search_server/trunk/servlet/src/main/resources/oxml.xml 2013-06-04 19:37:16 UTC (rev 13693)
+++ search_server/trunk/servlet/src/main/resources/oxml.xml 2013-06-04 20:32:24 UTC (rev 13694)
@@ -3,9 +3,13 @@
xmlns="http://www.eclipse.org/eclipselink/xsds/persistence/oxm"
package-name="org.musicbrainz.mmd2">
<java-types>
-
<java-type name="Metadata">
<java-attributes>
+ <xml-element java-attribute="areaList" xml-path="."/>
+ </java-attributes>
+ </java-type>
+ <java-type name="Metadata">
+ <java-attributes>
<xml-element java-attribute="artistList" xml-path="."/>
</java-attributes>
</java-type>
@@ -171,6 +175,11 @@
<xml-element java-attribute="puid" name="puids"/>
</java-attributes>
</java-type>
+ <java-type name="AreaList">
+ <java-attributes>
+ <xml-element java-attribute="area" name="areas"/>
+ </java-attributes>
+ </java-type>
<java-type name="ReleaseList">
<java-attributes>
<xml-element java-attribute="release" name="releases"/>
@@ -256,6 +265,11 @@
<xml-element java-attribute="labelInfo" name="label-info"/>
</java-attributes>
</java-type>
+ <java-type name="DefAreaElementInner">
+ <java-attributes>
+ <xml-element java-attribute="aliasList" xml-path="."/>
+ </java-attributes>
+ </java-type>
<java-type name="Artist">
<java-attributes>
<xml-element java-attribute="aliasList" xml-path="."/>
Modified: search_server/trunk/servlet/src/test/java/org/musicbrainz/search/servlet/FindAreaTest.java
===================================================================
--- search_server/trunk/servlet/src/test/java/org/musicbrainz/search/servlet/FindAreaTest.java 2013-06-04 19:37:16 UTC (rev 13693)
+++ search_server/trunk/servlet/src/test/java/org/musicbrainz/search/servlet/FindAreaTest.java 2013-06-04 20:32:24 UTC (rev 13694)
@@ -53,14 +53,16 @@
doc.addField(AreaIndexField.SORTNAME, "Afghanistan");
area.setSortName("Afghanistan");
- /*
- doc.addField(AreaIndexField.ALIAS, "Jockeys");
+
+ doc.addField(AreaIndexField.ALIAS, "Afghany");
AliasList aliasList = of.createAliasList();
Alias alias = of.createAlias();
aliasList.getAlias().add(alias);
- alias.setContent("Jockeys");
+ alias.setContent("Afghany");
+ alias.setSortName("Afghan");
area.setAliasList(aliasList);
+ /*
doc.addField(AreaIndexField.CODE, 1234);
area.setAreaCode(BigInteger.valueOf(1234));
@@ -120,6 +122,27 @@
}
@Test
+ public void testFindAreaBySortName() throws Exception {
+ Results res = ss.search("sortname:\"Afghanistan\"", 0, 10);
+ assertEquals(1, res.getTotalHits());
+ Result result = res.results.get(0);
+ MbDocument doc = result.getDoc();
+ assertEquals("ff571ff4-04cb-4b9c-8a1c-354c330f863c", doc.get(AreaIndexField.AREA_ID));
+ assertEquals("Afghanistan", doc.get(AreaIndexField.AREA));
+ }
+
+ @Test
+ public void testFindAreaByAlias() throws Exception {
+ Results res = ss.search("alias:\"Afghany\"", 0, 10);
+ assertEquals(1, res.getTotalHits());
+ Result result = res.results.get(0);
+ MbDocument doc = result.getDoc();
+ assertEquals("ff571ff4-04cb-4b9c-8a1c-354c330f863c", doc.get(AreaIndexField.AREA_ID));
+ assertEquals("Afghanistan", doc.get(AreaIndexField.AREA));
+ }
+
+
+ @Test
public void testFindAreaByDismax1() throws Exception {
Results res = sd.search("Afghanistan", 0, 10);
assertEquals(1, res.getTotalHits());
@@ -154,7 +177,8 @@
assertTrue(output.contains("type=\"Country\""));
assertTrue(output.contains("<name>Afghanistan</name>"));
assertTrue(output.contains("<sort-name>Afghanistan</sort-name>"));
-/* assertTrue(output.contains("<alias>Jockeys</alias>"));
+ assertTrue(output.contains("<alias sort-name=\"Afghan\">Afghany</alias></alias-list>"));
+ /*
assertTrue(output.contains("<begin>1993</begin"));
assertTrue(output.contains("<end>2004</end>"));
assertTrue(output.contains("<area-code>1234</area-code>"));
@@ -217,13 +241,14 @@
assertTrue(output.contains("\"type\":\"Country\""));
assertTrue(output.contains("name\":\"Afghanistan\""));
assertTrue(output.contains("\"sort-name\":\"Afghanistan\""));
+ assertTrue(output.contains("\"aliases\":[{\"locale\":\"\",\"sort-name\":\"Afghan\",\"type\":\"\",\"primary\":\"false\",\"begin-date\":\"\",\"end-date\":\"\",\"value\":\"Afghany\""));
+
/*
assertTrue(output.contains("life-span\":{\"begin\":\"1993\""));
assertTrue(output.contains("\"country\":\"GB\""));
assertTrue(output.contains("\"tags\":[{\"count\":22,\"name\":\"dance\"}]"));
assertTrue(output.contains("\"ended\":true"));
assertTrue(output.contains("\"ipis\":[\"1001\""));
- assertTrue(output.contains("\"aliases\":[{\"locale\":\"\",\"type\":\"\",\"primary\":\"false\",\"begin-date\":\"\",\"end-date\":\"\",\"value\":\"Jockeys\"}]"));
assertTrue(output.contains("\"end\":\"2004\""));
assertTrue(output.contains("\"area-code\":1234"));
assertTrue(output.contains("\"count\":1"));