topic classifier without example

#Written by Thomas M. Hagen
#for Aksis (www.aksis.uib.no)
#Topic classifier for newspaper articles
#Last update: Feb 2009

window <bychunk>
{
    input (:file:) [dag\/20090104.dag] #vg - verdens gnag
    call /:chunks:/
    output /\nDone!\n/
    exit
}

:chunks: #splits the file into '#http://'-initiated chunks:
{ #first chunk:
		match (:: :url: :chunk: :next_url:) [:file:] /(#http:.*?)(<.*?)(#http:|EOF)/ 
		isolate (:piece:) /:*:url::*:chunk:/
		call /:main:/ [:*:piece:]
		{ # the next chunks:
				match <fromnext> (:: :url: :chunk: :next_url:) [:file:] /(#http:.*?)(<.*?)(#http:)/
				isolate (:piece:) /:*:url::*:chunk:/
				call /:main:/ [:*:piece:]
				liaf
		}
		return
}

:main: (:args:) 
{
		{
				call /:classify:/ [:*:args:]
				output /\nThe piece from which the following is an exerpt has been classified as ':*:conclusion:':\n/
				call /:html_strip:/ [:*:args:]
				isolate (:interaction:) /:*:conclusion::*:args::*:stats:/
				call /:interact:/ [:*:interaction:]
		}
		return
}

:classify: (:args:)
{
		match (:: :url: :chunk:) [:args:] /(#http:.*?)(<.*)/
		isolate (:piece:) /:*:url::*:chunk:/ 
		{ 
				isolate (:stats:)
				classify <osbf microgroom> [:piece:] \
								(bus.cfc cli.cfc con.cfc cul.cfc dep.cfc \
								dis.cfc edu.cfc ene.cfc ent.cfc env.cfc \
								fac.cfc fam.cfc hea.cfc int.cfc lei.cfc \
								loc.cfc nat.cfc new.cfc out.cfc pol.cfc \
								sci.cfc spo.cfc tec.cfc tra.cfc unc.cfc \
								wor.cfc you.cfc) (:stats:)
				match (:long-conc:) [:stats:] /\(.*?\.cfc/
				match (:: :paren: :conclusion: :ext:) [:long-conc:] /(\()(.*)(\.cfc)/
				return
		}
}

:interact: (:args:)
{
		{
				match (:: :conclusion: :chunk: :stats:) [:args:] /(.*?)(#http:.*)(CLASSIFY.*)/
				isolate (:user-feedback:)
				output [stdout] /\n\nCategories:\n/
				output [stdout] /bus (business),       ene (energy),        lei (leisure),  tec (technology),\n/
				output [stdout] /cli (climate),        ent (entertainment), loc (local),    tra (travel),\n/
				output [stdout] /con (consumer),       env (environment),   nat (nature),   wor (worklife\/professional),\n/
				output [stdout] /cul (culture),        fac (fact),          new (news),     you (youth).\n/
				output [stdout] /dep (depth analysis), fam (family),        pol (politics),\n/
				output [stdout] /dis (discussion),     hea (health),        sci (science),\n/
				output [stdout] /edu (education),      int (interview),     spo (sports),\n/
				output [stdout] /\nSpecial categories:\n/
				output [stdout] /unc (uncertain),      out (stuff that doesn't belong in the corpus).\n/
				output [stdout] /\nWas the text classified correctly? If yes, type 'yes'.\n/
				output [stdout] /If not type the correct three-letter category: /
				input <byline> [stdin] (:user-feedback:)
				match <absent> [:user-feedback:] /yes/  #when you DON'T say yes, it re-trains the piece.
				output /\nThank you! The piece will be retrained as ':*:user-feedback:'.\n/
				isolate (:conc-file:) /:*:conclusion:.cfc/
				isolate (:cfc-file:) /:*:user-feedback:.cfc/
				learn <osbf microgroom refute> [:chunk:] (:*:conc-file:)
				output /\nThe piece has been refuted from ':*:conclusion:'.\n/
				{
						match <absent> [:user-feedback:] /out/ #do NOT learn it if it doesn't belong in the corpus
						learn <osbf microgroom> [:chunk:] (:*:cfc-file:)
						output /\nThe piece has been retrained as ':*:user-feedback:'.\n/
				}
				return
		}
		output /\nThe piece was classified correctly.\n/
		{
				#relearn if confidence is not high enough:
				match (:: :pR:) [:stats:] /Best match.*: (.*)  \nTotal/
				isolate (:boolean:)
				eval (:boolean:) /:@:(:*:pR: < 10) + (2 > 1):/ #i get errors when the bool = 0, so i rig it to be 1 or 2.
				match [:boolean:] /2/
				output /\nBecause confidence was low, statistics are collected from this piece anyway.\n/
				learn <osbf microgroom> (:*:conc-file:)  [:piece:] 
		}
		output /Stats will NOT be collected.\n/
		return
}

:html_strip: (:args:) 
{
		{
				match (:: :url: :chunk:) [:args:] /(#http:.*?)(<.*)/ 
				isolate (:piece:) /:*:url::*:chunk:/
				output /:*:url:/
				#first occurrence:
				match (:: :start_tag: :html_tag: :non_html: :another_html_tag:) [:piece:] /(#http:.*?)(<.*?>)(.*?)(<.*?>)/ 
				call /:peel:/ [:*:non_html:]
				{ #next occurrences:
						match <fromnext> (:: :html_tag: :non_html: :another_html_tag:) [:piece:] /(<.*?>)(.*?)(<.*?>)/ 
						call /:peel:/ [:*:non_html:]
						liaf
				}
				match 
		}
}

:peel: #removes some boilerplate to improve terminal readability
{
		isolate (:bool:)
		eval (:bool:) /:@:(:#:non_html: > 150) + (2 > 1):/ #don't bother printing it if it ain't at least 150 chars long.
		{
				match [:bool:] /2/
				match <absent> [:non_html:] /OAS_/
				match <absent> [:non_html:] /function/
				match <absent> [:non_html:] /tmsec/
				match <absent> [:non_html:] /http:/
				match <absent> [:non_html:] /@import/
				match <absent> [:non_html:] /30 &&/
				match <absent> [:non_html:] /0&&/
				match <absent> [:non_html:] /\n\n/
				match <absent> [:non_html:] /\(CD-&/
				match <absent> [:non_html:] /Abonnementet l/
				match <absent> [:non_html:] /\//
				match <absent> [:non_html:] /document/
				match <absent> [:non_html:] /topHeight/
				match <absent> [:non_html:] /Karate\./
				match <absent> [:non_html:] /var /
				match <absent> [:non_html:] /All utgivelse av stoff i Adr/
				match <absent> [:non_html:] /tw_widget/
				match <absent> [:non_html:] /yahooppc/
				match <absent> [:non_html:] /sesamannonse/
				match <absent> [:non_html:] /footer-menu/
				match <absent> [:non_html:] /Auda! Her skulle du/
				match <absent> [:non_html:] /wineBanner/
				match <absent> [:non_html:] /font-size/
				output /:*:non_html:/
		}
		return
}