Re: Search working? [1 Attachment]
"Sebastian Tänzer [email protected] [zms-developers]" <[email protected]> Mon, 1 Jun 2015 23:05:05 +0200
| Newsgroups | gmane.comp.cms.zms.devel |
|---|---|
| Message-ID | <[email protected]> |
--Apple-Mail=_3F5BB0E8-53DB-4D01-89EB-0D05A3DC2FF6 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset=utf-8 Hello Rainer, basically it=E2=80=99s working and the approach behind it is quite flexible= . There=E2=80=99s a bug in the current SVN release I fixed in the Connector a= ttached to this mail or at http://pastebin.com/eu2Y1vJF <http://pastebin.co= m/eu2Y1vJF> I don=E2=80=99t know it that is the most elegant way but at least it works = ;-) Best Sebastian - t=C3=A4nzermedien GmbH - Agentur f=C3=BCr Online-Medien Kekul=C3=A9str. 39 | 53115 Bonn, Germany E-Mail: [email protected] | Web: http://www.taenzer.me=20=20 Tel.: 0228-304134-0 | Fax.: 0228-304134-99 --Apple-Mail=_3F5BB0E8-53DB-4D01-89EB-0D05A3DC2FF6 Content-Type: multipart/mixed; boundary="Apple-Mail=_53D0A02B-2298-40B6-AAE8-6E3EA30F6D3E" --Apple-Mail=_53D0A02B-2298-40B6-AAE8-6E3EA30F6D3E Content-Transfer-Encoding: quoted-printable Content-Type: text/html; charset=utf-8 <html><head><meta http-equiv=3D"Content-Type" content=3D"text/html charset= =3Dutf-8"></head><body style=3D"word-wrap: break-word; -webkit-nbsp-mode: s= pace; -webkit-line-break: after-white-space;" class=3D""><div class=3D"">He= llo Rainer,</div><div class=3D""><br class=3D""></div><div class=3D"">basic= ally it=E2=80=99s working and the approach behind it is quite flexible.</di= v><div class=3D""><br class=3D""></div><div class=3D"">There=E2=80=99s a bu= g in the current SVN release I fixed in the Connector attached to this mail= or at <a href=3D"http://pastebin.com/eu2Y1vJF" class=3D"">http://past= ebin.com/eu2Y1vJF</a></div><div class=3D""><br class=3D""></div><div class= =3D"">I don=E2=80=99t know it that is the most elegant way but at least it = works ;-)</div><div class=3D""><br class=3D""></div><div class=3D"">Best</d= iv><div class=3D"">Sebastian</div><div class=3D""><div class=3D""><br class= =3D""></div><div class=3D"">-</div><div class=3D"">t=C3=A4nzermedien GmbH -= Agentur f=C3=BCr Online-Medien</div><div class=3D"">Kekul=C3=A9str. 39 | 5= 3115 Bonn, Germany</div><div class=3D"">E-Mail: <a href=3D"mailto:st@taenze= r.me" class=3D"">[email protected]</a> | Web: <a href=3D"http://www.taenzer.me"= class=3D"">http://www.taenzer.me</a> </div><div class=3D"">Tel.: 022= 8-304134-0 | Fax.: 0228-304134-99</div></div><div class=3D""><br class=3D""= ></div></body></html> --Apple-Mail=_53D0A02B-2298-40B6-AAE8-6E3EA30F6D3E Content-Disposition: attachment; filename=ZMSZCatalogConnector.py Content-Type: text/x-python-script; name="ZMSZCatalogConnector.py" Content-Transfer-Encoding: quoted-printable #= ##########################################################################= #####=0D # ZMSZCatalogConnector.py=0D #=0D # This program is free software; you can redistribute it and/or=0D # modify it under the terms of the GNU General Public License=0D # as published by the Free Software Foundation; either version 2=0D # of the License, or (at your option) any later version.=0D #=0D # This program is distributed in the hope that it will be useful,=0D # but WITHOUT ANY WARRANTY; without even the implied warranty of=0D # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the=0D # GNU General Public License for more details.=0D #=0D # You should have received a copy of the GNU General Public License=0D # along with this program; if not, write to the Free Software=0D # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, = USA.=0D = ##########################################################################= ######=0D =0D =0D # Imports.=0D from Products.PageTemplates.PageTemplateFile import PageTemplateFile=0D from Products.ZCatalog import ZCatalog=0D import urllib=0D import zope.interface=0D # Product Imports.=0D import _globals=0D import IZMSCatalogConnector=0D import ZMSZCatalogAdapter=0D import ZMSItem=0D =0D =0D # = --------------------------------------------------------------------------= ----=0D # ZMSZCatalogConnector.intValue:=0D # = --------------------------------------------------------------------------= ----=0D def intValue(v):=0D try:=0D i =3D int(v)=0D except:=0D i =3D 0=0D return i=0D =0D =0D # = --------------------------------------------------------------------------= ----=0D # ZMSZCatalogConnector.Empty:=0D # = --------------------------------------------------------------------------= ----=0D class Empty: =0D pass=0D =0D =0D # = --------------------------------------------------------------------------= ----=0D # ZMSZCatalogConnector.addLexicon:=0D # = --------------------------------------------------------------------------= ----=0D def addLexicon( container, cat):=0D =0D # Remove Lexicon=0D ids =3D cat.objectIds( ['ZCTextIndex Lexicon','ZCTextIndex Unicode = Lexicon'])=0D if len( ids) > 0:=0D cat.manage_delObjects( ids)=0D =0D # Add Lexicon=0D index_type =3D = container.getConfProperty('ZCatalog.TextIndexType','ZCTextIndex')=0D if index_type =3D=3D 'ZCTextIndex':=0D elements =3D []=0D wordSplitter =3D Empty()=0D wordSplitter.group =3D 'Word Splitter'=0D wordSplitter.name =3D 'HTML aware splitter'=0D elements.append(wordSplitter)=0D caseNormalizer =3D Empty()=0D caseNormalizer.group =3D 'Case Normalizer'=0D caseNormalizer.name =3D 'Case Normalizer'=0D elements.append(caseNormalizer)=0D stopWords =3D Empty()=0D stopWords.group =3D 'Stop Words'=0D stopWords.name =3D 'Remove listed and single char words'=0D elements.append(stopWords)=0D try:=0D cat.manage_addProduct['ZCTextIndex'].manage_addLexicon('Lexicon', = 'Default lexicon', elements)=0D except:=0D pass=0D =0D =0D # = --------------------------------------------------------------------------= ----=0D # ZMSZCatalogConnector.recreateCatalog:=0D # = --------------------------------------------------------------------------= ----=0D def recreateCatalog(self, zcm, lang):=0D =0D #-- Create catalog=0D cat_id =3D 'catalog_%s'%lang=0D if cat_id in self.objectIds():=0D self.manage_delObjects([cat_id])=0D cat_title =3D 'Default catalog'=0D zcatalog =3D ZCatalog.ZCatalog(id=3Dcat_id, title=3Dcat_title, = container=3Dself)=0D self._setObject(zcatalog.id, zcatalog)=0D zcatalog =3D getattr(self,cat_id)=0D =0D #-- Add lexicon=0D addLexicon( self, zcatalog)=0D =0D #-- Add columns=0D for index_name in = ['id','meta_id','absolute_url','zcat_column_custom']:=0D zcatalog.manage_addColumn(index_name)=0D =0D #-- Add Indexes (incl. Columns)=0D index_type =3D = zcm.getConfProperty('ZCatalog.TextIndexType','ZCTextIndex')=0D for attr_id in zcm.getAttrIds():=0D index_name =3D 'zcat_index_%s'%attr_id=0D extra =3D None=0D if index_type =3D=3D 'ZCTextIndex':=0D extra =3D Empty()=0D extra.doc_attr =3D index_name=0D extra.index_type =3D 'Okapi BM25 Rank'=0D extra.lexicon_id =3D 'Lexicon'=0D else:=0D extra =3D {}=0D extra['default_encoding'] =3D 'utf-8'=0D extra['indexed_fields'] =3D index_name=0D extra['fields'] =3D [index_name]=0D extra['near_distance'] =3D 5=0D extra['splitter_casefolding'] =3D 1=0D extra['splitter_max_len'] =3D 64=0D extra['splitter_separators'] =3D '.+-_@'=0D extra['splitter_single_chars'] =3D 0=0D if index_type =3D=3D 'TextIndexNG2':=0D extra['use_converters'] =3D 1=0D extra['use_normalizer'] =3D ''=0D # setattr(index_extra,'use_stemmer','')=0D extra['use_stopwords'] =3D ''=0D elif index_type =3D=3D 'TextIndexNG3':=0D extra['languages'] =3D (lang,)=0D extra['query_parser'] =3D 'txng.parsers.en'=0D extra['index_unknown_languages'] =3D True=0D extra['dedicated_storage'] =3D True=0D extra['use_stopwords'] =3D False=0D extra['use_normalizer'] =3D False=0D extra['use_converters'] =3D True=0D extra['use_stemmer'] =3D False=0D zcatalog.manage_addColumn(index_name)=0D zcatalog.manage_addIndex(index_name,index_type,extra)=0D =0D =0D = ##########################################################################= ######=0D = ##########################################################################= ######=0D ###=0D ### Class=0D ###=0D = ##########################################################################= ######=0D = ##########################################################################= ######=0D class ZMSZCatalogConnector(=0D ZMSItem.ZMSItem):=0D zope.interface.implements(=0D IZMSCatalogConnector.IZMSCatalogConnector)=0D =0D # Properties.=0D # -----------=0D meta_type =3D 'ZMSZCatalogConnector'=0D icon =3D "/misc_/ZCatalog/ZCatalog.gif"=0D =0D # Management Interface.=0D # ---------------------=0D manage_input_form =3D = PageTemplateFile('zpt/ZMSZCatalogAdapter/manage_zcatalog_connector',global= s())=0D =0D # Management Permissions.=0D # -----------------------=0D __administratorPermissions__ =3D (=0D 'manage_changeProperties', 'manage_main',=0D )=0D __ac_permissions__=3D(=0D ('ZMS Administrator', __administratorPermissions__),=0D )=0D =0D = ##########################################################################= ##=0D # ZMSZCatalogConnector.__init__: =0D #=0D # Constructor.=0D = ##########################################################################= ##=0D def __init__(self):=0D self.id =3D 'zcatalog_connector'=0D =0D =0D # = --------------------------------------------------------------------------= =0D # ZMSZCatalogConnector.search_xml:=0D # = --------------------------------------------------------------------------= =0D def search_xml(self, q, page_index=3D0, page_size=3D10, = REQUEST=3DNone, RESPONSE=3DNone):=0D """ ZMSZCatalogConnector.search_xml """=0D # Check constraints.=0D page_index =3D int(page_index)=0D page_size =3D int(page_size)=0D REQUEST.set('lang',REQUEST.get('lang',self.getPrimaryLanguage()))=0D= RESPONSE =3D REQUEST.RESPONSE=0D content_type =3D 'text/xml;charset=3Dutf-8'=0D RESPONSE.setHeader('Content-Type',content_type)=0D RESPONSE.setHeader('Cache-Control', 'no-cache')=0D RESPONSE.setHeader('Pragma', 'no-cache')=0D # Execute query.=0D status =3D 0=0D msg =3D ''=0D results =3D []=0D try: =0D results =3D self.search(q)=0D except:=0D import sys=0D _globals.writeError(self,'[search_xml]')=0D t,v,tb =3D sys.exc_info()=0D status =3D 400=0D msg =3D v=0D # Assemble xml.=0D xml =3D self.getXmlHeader()=0D xml +=3D '<response>'=0D xml +=3D '<lst name=3D"responseHeader">'=0D xml +=3D '<int name=3D"status">%i</int>'%status=0D xml +=3D '<lst name=3D"params">'=0D for key in REQUEST.form.keys():=0D xml +=3D '<int name=3D"%s">%s</int>'%(key,str(REQUEST.form[key]))=0D= xml +=3D '</lst>'=0D xml +=3D '</lst>'=0D if status > 0:=0D xml +=3D '<lst name=3D"error">'=0D xml +=3D '<int name=3D"msg">%s</int>'%msg=0D xml +=3D '<int name=3D"code">%i</int>'%status=0D xml +=3D '</lst>'=0D else:=0D xml +=3D '<result name=3D"response" numFound=3D"%i" = start=3D"%i">'%(len(results),page_index*page_size)=0D if len(results) > page_size:=0D results =3D = results[page_index*page_size:(page_index+1)*page_size]=0D for result in results:=0D xml +=3D '<doc>'=0D for k in result.keys():=0D v =3D result[k]=0D if k =3D=3D 'absolute_url':=0D k =3D 'loc'=0D elif k =3D=3D 'zcat_column_custom':=0D k =3D 'custom'=0D elif k =3D=3D 'standard_html':=0D v =3D ZMSZCatalogAdapter.remove_tags(self,v)=0D xml +=3D '<arr name=3D"%s">'%k=0D if k =3D=3D 'custom':=0D xml +=3D '<str>%s</str>'%v =0D else: =0D xml +=3D '<str><![CDATA[%s]]></str>'%v =0D xml +=3D '</arr>'=0D xml +=3D '</doc>'=0D xml +=3D '</result>'=0D xml +=3D '</response>'=0D return xml=0D =0D =0D # = --------------------------------------------------------------------------= =0D # ZMSZCatalogConnector.suggest_xml:=0D # = --------------------------------------------------------------------------= =0D def suggest_xml(self, q, limit=3D5, REQUEST=3DNone, RESPONSE=3DNone):=0D= """ ZMSZCatalogConnector.suggest_xml """=0D # Check constraints.=0D REQUEST.set('lang',REQUEST.get('lang',self.getPrimaryLanguage()))=0D= RESPONSE =3D REQUEST.RESPONSE=0D content_type =3D 'text/xml;charset=3Dutf-8'=0D RESPONSE.setHeader('Content-Type',content_type)=0D RESPONSE.setHeader('Cache-Control', 'no-cache')=0D RESPONSE.setHeader('Pragma', 'no-cache')=0D # Execute query.=0D status =3D 0=0D msg =3D ''=0D results =3D []=0D try: =0D results =3D self.suggest(q,limit)=0D except:=0D import sys=0D _globals.writeError(self,'[suggest_xml]')=0D t,v,tb =3D sys.exc_info()=0D status =3D 400=0D msg =3D v=0D # Assemble xml.=0D xml =3D self.getXmlHeader()=0D xml +=3D '<response>'=0D xml +=3D '<lst name=3D"responseHeader">'=0D xml +=3D '<int name=3D"status">%i</int>'%status=0D xml +=3D '</lst>'=0D if status > 0:=0D xml +=3D '<lst name=3D"error">'=0D xml +=3D '<int name=3D"msg">%s</int>'%msg=0D xml +=3D '<int name=3D"code">%i</int>'%status=0D xml +=3D '</lst>'=0D else:=0D xml +=3D '<lst>'=0D xml +=3D '<lst name=3D"suggestions">'=0D xml +=3D '<int name=3D"numFound">%i</int>'%len(results)=0D xml +=3D '<arr name=3D"suggestion">'=0D for result in results:=0D xml +=3D '<str>%s</str>'%result=0D xml +=3D '</arr>'=0D xml +=3D '</lst>'=0D xml +=3D '</lst>'=0D xml +=3D '</response>'=0D return xml=0D =0D =0D # = --------------------------------------------------------------------------= =0D # ZMSZCatalogConnector.search:=0D # = --------------------------------------------------------------------------= =0D def search(self, qs, order=3DNone):=0D rtn =3D []=0D =0D # ZCatalog.=0D request =3D self.REQUEST=0D lang =3D request.get('lang',self.getPrimaryLanguage())=0D zcatalog =3D getattr(self,'catalog_%s'%lang)=0D =0D # Find search-results.=0D items =3D []=0D for index in zcatalog.indexes():=0D if index.find('zcat_index_')=3D=3D0:=0D query =3D {index:self.search_encode( qs)}=0D qr =3D zcatalog(query)=0D _globals.writeLog( self, "[search]: = %s=3D%i"%(str(query),len(qr)))=0D for item in qr:=0D if item not in items:=0D items.append( item.aq_base )=0D =0D # Process search-results.=0D results =3D []=0D for item in items:=0D data_record_id =3D item.data_record_id_=0D path =3D zcatalog.getpath(data_record_id)=0D # Append to valid results.=0D if len(filter(lambda x: x[1]['path']=3D=3Dpath, results)) =3D=3D = 0:=0D result =3D {}=0D result['path'] =3D path=0D result['score'] =3D intValue(item.data_record_score_)=0D result['normscore'] =3D = intValue(item.data_record_normalized_score_)=0D for column in zcatalog.schema():=0D k =3D column=0D if k.find('zcat_index_')=3D=3D0:=0D k =3D k[len('zcat_index_'):]=0D result[k] =3D getattr(item,column,None)=0D results.append((item.data_record_score_,result))=0D =0D # Sort search-results.=0D results.sort()=0D results.reverse()=0D =0D # Append search-results.=0D rtn.extend(map(lambda ob: ob[1],results))=0D =0D # Return list of search-results in correct sort-order.=0D return rtn=0D =0D =0D # = --------------------------------------------------------------------------= =0D # ZMSZCatalogConnector.suggest:=0D # = --------------------------------------------------------------------------= =0D def suggest(self, q, limit=3D5):=0D rtn =3D []=0D =0D # ZCatalog.=0D request =3D self.REQUEST=0D lang =3D request.get('lang',self.getPrimaryLanguage())=0D zcatalog =3D getattr(self,'catalog_%s'%lang)=0D =0D # Lexicon.=0D lexicon =3D zcatalog.Lexicon=0D for w in lexicon.words():=0D if w[0] not in ['_','0','1','2','3','4','5','6','7','8','9'] and = w.lower().find(q) >=3D 0 and w not in rtn:=0D rtn.append(w)=0D if len(rtn) >=3D limit:=0D break=0D =0D # Return list of suggestions.=0D return rtn=0D =0D =0D # = --------------------------------------------------------------------------= =0D # ZMSZCatalogConnector._update:=0D # = --------------------------------------------------------------------------= =0D def _update(self, node, d):=0D zcm =3D self.getCatalogAdapter()=0D lang =3D self.REQUEST.get('lang')=0D # Prepare object.=0D extra_column_ids =3D ['custom']=0D for attr_id in extra_column_ids:=0D attr_name =3D 'zcat_column_%s'%attr_id=0D value =3D d.get(attr_id)=0D setattr(node,attr_name,value)=0D for attr_id in zcm.getAttrIds():=0D attr_name =3D 'zcat_index_%s'%attr_id=0D value =3D node.attr(attr_id)=0D setattr(node,attr_name,value)=0D # Reindex object.=0D node.default_catalog =3D 'catalog_%s'%lang=0D node.reindex_object()=0D # Unprepare object.=0D for attr_id in extra_column_ids:=0D attr_name =3D 'zcat_column_%s'%attr_id=0D delattr(node,attr_name)=0D for attr_id in zcm.getAttrIds():=0D attr_name =3D 'zcat_index_%s'%attr_id=0D delattr(node,attr_name)=0D =0D =0D # = --------------------------------------------------------------------------= =0D # ZMSZCatalogConnector.reindex_all:=0D # = --------------------------------------------------------------------------= =0D def reindex_all(self):=0D zcm =3D self.getCatalogAdapter()=0D request =3D self.REQUEST=0D container =3D self.getDocumentElement()=0D for lang in container.getLangIds():=0D request.set('lang',lang)=0D # Recreate catalog.=0D recreateCatalog(container,self.aq_parent,lang)=0D # Reindex items to catalog.=0D def cb(node,d):=0D self._update(node,d)=0D zcm.get_sitemap(cb,container,recursive=3DTrue)=0D =0D =0D # = --------------------------------------------------------------------------= =0D # ZMSZCatalogConnector.reindex_node:=0D # = --------------------------------------------------------------------------= =0D def reindex_node(self, node):=0D node.writeLog('[ZMSZCatalogConnector.reindex_node]')=0D zcm =3D self.getCatalogAdapter()=0D # Reindex item to catalog.=0D def cb(node,d):=0D self._update(node,d)=0D zcm.get_sitemap(cb,node,recursive=3DFalse)=0D =0D =0D def get_sitemap(self):=0D """=0D Returns sitemap.=0D @rtype: C{str}=0D """=0D zcm =3D self.getCatalogAdapter()=0D request =3D self.REQUEST=0D RESPONSE =3D request.RESPONSE=0D RESPONSE.setHeader('Content-Type','text/plain; charset=3Dutf-8')=0D= l =3D []=0D def cb(node,d):=0D l.append(d)=0D zcm.get_sitemap(cb,self.getDocumentElement(),recursive=3DTrue)=0D return self.str_json(l)=0D =0D =0D = ##########################################################################= ##=0D # ZMSZCatalogConnector.manage_changeProperties:=0D #=0D # Change properties.=0D = ##########################################################################= ##=0D def manage_changeProperties(self, selected, btn, lang, REQUEST):=0D message =3D ''=0D =0D # Reindex.=0D # --------=0D if btn =3D=3D 'Reindex' and selected:=0D reindex =3D self.reindex_all()=0D message +=3D '%s reindexed (%s)\n'%(self.id,str(reindex))=0D =0D # Remove.=0D # -------=0D elif btn =3D=3D 'Remove':=0D ids =3D REQUEST.get('zcatalog_objectIds',[])=0D if len(ids) > 0:=0D self.getDocumentElement().manage_delObjects(ids)=0D message +=3D self.getZMILangStr('MSG_DELETED')%len(ids)=0D =0D return message=0D =0D = ##########################################################################= ######=0D --Apple-Mail=_53D0A02B-2298-40B6-AAE8-6E3EA30F6D3E Content-Transfer-Encoding: 7bit Content-Type: text/html; charset=us-ascii <html><head><meta http-equiv="Content-Type" content="text/html charset=us-ascii"></head><body style="word-wrap: break-word; -webkit-nbsp-mode: space; -webkit-line-break: after-white-space;" class=""></body></html> --Apple-Mail=_53D0A02B-2298-40B6-AAE8-6E3EA30F6D3E-- --Apple-Mail=_3F5BB0E8-53DB-4D01-89EB-0D05A3DC2FF6--