topic classifier

the first version of my newspaper article topic classifier is done.

here is the code. maybe it could become some sort of idiom after some
improvement. any comments and feedback are much appreciated!

#Written by Thomas M. Hagen
#for Aksis (www.aksis.uib.no)
#Topic classifier for newspaper articles
#Last update: Feb 2009

window <bychunk>
{
    input (:file:) [dag\/20090116.dag] #nl - nordlys
    call /:chunks:/
    output /\nDone!\n/
    exit
}

:chunks: #splits the file into '#http://'-initiated chunks:
{ #first chunk:
    match (:: :url: :chunk: :next_url:) [:file:]
/(#http:.*?)(<.*?)(#http:|EOF)/
    isolate (:piece:) /:*:url::*:chunk:/
    call /:main:/ [:*:piece:]
    { # the next chunks:
    match <fromnext> (:: :url: :chunk: :next_url:) [:file:]
/(#http:.*?)(<.*?)(#http:)/
    isolate (:piece:) /:*:url::*:chunk:/
    call /:main:/ [:*:piece:]
    liaf
    }
    return
}

:main: (:args:)
{
    {
    call /:classify:/ [:*:args:]
    output /\nThe piece from which the following is an exerpt has been
classified as ':*:conclusion:':\n/
    call /:html_strip:/ [:*:args:]
    isolate (:interaction:) /:*:conclusion::*:args::*:stats:/
    call /:interact:/ [:*:interaction:]
    }
    return
}

:classify: (:args:)
{
    match (:: :url: :chunk:) [:args:] /(#http:.*?)(<.*)/
    isolate (:piece:) /:*:url::*:chunk:/
    {
    isolate (:stats:)
    classify <osbf microgroom> [:piece:] \
        (bus.cfc cli.cfc con.cfc cul.cfc dep.cfc \
        dis.cfc edu.cfc ene.cfc ent.cfc env.cfc \
        fac.cfc fam.cfc hea.cfc int.cfc lei.cfc \
        loc.cfc nat.cfc new.cfc out.cfc pol.cfc \
        sci.cfc spo.cfc tec.cfc tra.cfc unc.cfc \
        wor.cfc you.cfc) (:stats:)
    match (:long-conc:) [:stats:] /\(.*?\.cfc/
    match (:: :paren: :conclusion: :ext:) [:long-conc:] /(\()(.*)(\.cfc)/
    return
    }
}

:interact: (:args:)
{
    {
    match (:: :conclusion: :chunk: :stats:) [:args:]
/(.*?)(#http:.*)(CLASSIFY.*)/
    isolate (:user-feedback:)
    output [stdout] /\n\nCategories:\n/
    output [stdout] /bus (business),       ene (energy),        lei
(leisure),  tec (technology),\n/
    output [stdout] /cli (climate),        ent (entertainment), loc
(local),    tra (travel),\n/
    output [stdout] /con (consumer),       env (environment),   nat
(nature),   wor (worklife\/professional),\n/
    output [stdout] /cul (culture),        fac (fact),          new
(news),     you (youth).\n/
    output [stdout] /dep (depth analysis), fam (family),        pol
(politics),\n/
    output [stdout] /dis (discussion),     hea (health),        sci
(science),\n/
    output [stdout] /edu (education),      int (interview),     spo
(sports),\n/
    output [stdout] /\nSpecial categories:\n/
    output [stdout] /unc (uncertain),      out (stuff that doesn't belong in
the corpus).\n/
    output [stdout] /\nWas the text classified correctly? If yes, type
'yes'.\n/
    output [stdout] /If not type the correct three-letter category: /
    input <byline> [stdin] (:user-feedback:)
    match <absent> [:user-feedback:] /yes/  #when you DON'T say yes, it
re-trains the piece.
    output /\nThank you! The piece will be retrained as
':*:user-feedback:'.\n/
    isolate (:conc-file:) /:*:conclusion:.cfc/
    isolate (:cfc-file:) /:*:user-feedback:.cfc/
    learn <osbf microgroom refute> [:chunk:] (:*:conc-file:)
    output /\nThe piece has been refuted from ':*:conclusion:'.\n/
    {
        match <absent> [:user-feedback:] /out/ #do NOT learn it if it
doesn't belong in the corpus
        learn <osbf microgroom> [:chunk:] (:*:cfc-file:)
        output /\nThe piece has been retrained as ':*:user-feedback:'.\n/
    }
    return
    }
    output /\nThe piece was classified correctly.\n/
    {
    #relearn if confidence is not high enough:
    match (:: :pR:) [:stats:] /Best match.*: (.*)  \nTotal/
    isolate (:boolean:)
    eval (:boolean:) /:@:(:*:pR: < 10) + (2 > 1):/ #i get errors when the
bool = 0, so i rig it to be 1 or 2.
    match [:boolean:] /2/
    output /\nBecause confidence was low, statistics are collected from this
piece anyway.\n/
    learn <osbf microgroom> (:*:conc-file:)  [:piece:]
    }
    output /Stats will NOT be collected.\n/
    return
}

:html_strip: (:args:)
{
    {
    match (:: :url: :chunk:) [:args:] /(#http:.*?)(<.*)/
    isolate (:piece:) /:*:url::*:chunk:/
    output /:*:url:/
    #first occurrence:
    match (:: :start_tag: :html_tag: :non_html: :another_html_tag:)
[:piece:] /(#http:.*?)(<.*?>)(.*?)(<.*?>)/
    call /:peel:/ [:*:non_html:]
    { #next occurrences:
        match <fromnext> (:: :html_tag: :non_html: :another_html_tag:)
[:piece:] /(<.*?>)(.*?)(<.*?>)/
        call /:peel:/ [:*:non_html:]
        liaf
    }
    match
    }
}

:peel: #removes some boilerplate to improve terminal readability
{
    isolate (:bool:)
    eval (:bool:) /:@:(:#:non_html: > 150) + (2 > 1):/ #don't bother
printing it if it ain't at least 150 chars long.
    {
    match [:bool:] /2/
    match <absent> [:non_html:] /OAS_/
    match <absent> [:non_html:] /function/
    match <absent> [:non_html:] /tmsec/
    match <absent> [:non_html:] /http:/
    match <absent> [:non_html:] /@import/
    match <absent> [:non_html:] /30 &&/
    match <absent> [:non_html:] /0&&/
    match <absent> [:non_html:] /\n\n/
    match <absent> [:non_html:] /\(CD-&/
     match <absent> [:non_html:] /Abonnementet l/
     match <absent> [:non_html:] /\//
     match <absent> [:non_html:] /document/
     match <absent> [:non_html:] /topHeight/
     match <absent> [:non_html:] /Karate\./
     match <absent> [:non_html:] /var /
     match <absent> [:non_html:] /All utgivelse av stoff i Adr/
     match <absent> [:non_html:] /tw_widget/
     match <absent> [:non_html:] /yahooppc/
     match <absent> [:non_html:] /sesamannonse/
     match <absent> [:non_html:] /footer-menu/
    match <absent> [:non_html:] /Auda! Her skulle du/
     match <absent> [:non_html:] /wineBanner/
     match <absent> [:non_html:] /font-size/
    output /:*:non_html:/
    }
    return
}

------------------------------------------------------------------------------
Create and Deploy Rich Internet Apps outside the browser with Adobe(R)AIR(TM)
software. With Adobe AIR, Ajax developers can use existing skills and code to
build responsive, highly engaging applications that combine the power of local
resources and data with the reach of the web. Download the Adobe AIR SDK and
Ajax docs to start building applications today-http://p.sf.net/sfu/adobe-com

_______________________________________________
Crm114-general mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/crm114-general