Re: Bugs in FIFO handling
"Trever L. Adams" <[email protected]>
| Newsgroups | gmane.mail.spam.crm114 |
|---|---|
| Message-ID | <[email protected]> |
Bill Yerazunis wrote: > Nope... no ideas. > > But if you can post source to a test case that shows the misbehavior, > that would be very useful! > > - Bill Yerazunis > > Sorry for the delay in responding. The requested code is attached as four attachments. The hyperspace files are fairly large, so I will not post them. Thanks, Trever ------------------------------------------------------------------------- This SF.Net email is sponsored by the Moblin Your Move Developer's challenge Build the coolest Linux based applications with Moblin SDK & win great prizes Grand prize is a trip for two to an Open Source event anywhere in the world http://moblin-contest.org/redirect.php?banner_id=100&url=/ _______________________________________________ Crm114-general mailing list [email protected] https://lists.sourceforge.net/lists/listinfo/crm114-general
dehtml.crm
(text/plain, 4.7 KB)
# Copyright (C) 2008 Trever L. Adams; licensed under the
# GNU Public License (GPL) version 2. A copy of this license is included
# in the distribution media, or obtain one from www.fsf.org .
{
}
return
# Normalize currency
:normalize_currency:
{
isolate (:mycount: :mycount2: :tocount: :tocount2:)
{
alter (:tocount2:) //
alter (:tocount:) //
alter (:mycount:) //
# crm114 doesn't seem to understand utf-8
# currency symbols: £€¥元₨₪₱؋៛₡¢圓₩₭₮₨ƒ₦﷼฿₴₫£
match <fromnext> /[$][[:space:]]*([[:digit:]]+)(\.[[:digit:]]*)?/ (:keep: :tocount: :tocount2:)
# output /ORIGKEEP: :*:keep: TO COUNT: :*:tocount: :*:tocount2: COUNTS :*:mycount: :*:mycount2:\n/
eval (:mycount:) /:@: :#:tocount: :/
eval (:mycount2:) /:@: :#:tocount2: -1 :/
alter (:keep:) /$/
{
eval (:mycount:) /:@: :*:mycount: - 1 :/
eval / :@: :*:mycount: >= 0 :/
alter (:keep:) /:*:keep:X/
liaf
}
{
eval / :@: :*:mycount2: >= 0 :/
alter (:keep:) /:*:keep:./
}
{
eval (:mycount2:) /:@: :*:mycount2: - 1 :/
eval / :@: :*:mycount2: >= 0 :/
alter (:keep:) /:*:keep:X/
liaf
}
# output /KEEP: :*:keep: TO COUNT: :*:mycount: :*:mycount2:\n/
liaf
}
}
return
# Remove spaces
:kill_spaces:
{
# Reset to top
match /[[:space:]][[:space:]]/
# Remove spaces
{
match <fromcurrent> /[[:space:]][[:space:]]+/ (:deadspace:)
alter (:deadspace:) /\n/
liaf
}
}
return
# Turn all of the content of META tags into inline text
:find_meta:
{
# Reset to top
match <nocase> /<meta/
{
# Find metas
{
isolate (:meta_txt: :good:)
alter (:meta_txt:) //
alter (:good:) //
match <nocase fromcurrent> /<meta ([^>]*)>/ (:whole_meta_tag: :meta_txt:)
{
{
match <nocase> [:whole_meta_tag:] /.*description"?([^>]*)>/ (:meta_txt: :good:)
call /:keep_good_meta:/ [:*:meta_txt:] (:meta_txt:)
alter (:whole_meta_tag:) /:*:meta_txt:/
}
alius
{
match <nocase> [:whole_meta_tag:] /.*keywords"?([^>]*)>/ (:meta_txt: :good:)
call /:keep_good_meta:/ [:*:meta_txt:] (:meta_txt:)
alter (:whole_meta_tag:) /:*:meta_txt:/
}
alius
{
alter (:whole_meta_tag:) //
}
}
liaf
}
}
}
return
# Keep the good meta tags
:keep_good_meta: (:kgm_input:)
{
isolate (:kgm_meta_txt:)
{
match <nocase> [:kgm_input:] /content\="?([^"]*)"?/ (:kgm_meta_txt: :kgm_good:)
alter (:kgm_meta_txt:) /:*:kgm_good:/
}
return /:*:kgm_meta_txt: /
}
:de_html: (:show:)
{
isolate (:image_texts:)
alter (:image_texts:) //
# Remove all comments
{
# Reset to top
match /<!--/
{
match <fromcurrent> /<!--.*?-->/ (:whole_comment:)
alter (:whole_comment:) //
liaf
}
}
# Remove scripts
{
# Reset to top
match <nocase> /<[ \n\r]*script/
{
{
match <nocase fromcurrent> /<[ \n\r]*script[^>]*>.*?<[ \n\r]*\/script[^>]*>/ (:whole_script:)
alter (:whole_script:) //
liaf
}
}
}
# Remove style tags
{
# Reset to top
match <nocase> /<[ \n\r]*style/
{
match <nocase fromcurrent> /<[ \n\r]*style[^>]*>.*?<[ \n\r]*\/style[^>]*>/ (:whole_style:)
alter (:whole_style:) //
liaf
}
}
# Keep keywords and descriptions, kill all other meta tags
call /:find_meta:/
# Keep only the alt and title sections of an IMG tag
{
# Reset to top
match <nocase> /<img/
{
isolate (:title_txt: :alt_txt:)
match <nocase fromcurrent> /<img ([^>]*)>/ (:whole_img_tag:)
{
{
{
match <nocase> [:whole_img_tag:] /title="([^"]*)"/ (:trash: :title_txt:)
}
alius
{
match <nocase> [:whole_img_tag:] /title='([^']*)'/ (:trash: :title_txt:)
}
alius
{
match <nocase> [:whole_img_tag:] /title=[[:graph:]]*)/ (:trash: :title_txt:)
}
}
{
{
match <nocase> [:whole_img_tag:] /alt="([^"]*)"/ (:trash: :alt_txt:)
}
alius
{
match <nocase> [:whole_img_tag:] /alt='([^']*)'/ (:trash: :alt_txt:)
}
alius
{
match <nocase> [:whole_img_tag:] /alt=[[:graph:]]*)/ (:trash: :alt_txt:)
}
}
}
{
match [:alt_txt:] /.+[[:graph:]].*/
alter (:image_texts:) /:*:image_texts: :*:alt_txt:/
}
{
match [:title_txt:] /.+[[:graph:]].*/
alter (:image_texts:) /:*:image_texts: :*:title_txt:/
}
alter (:whole_img_tag:) / /
liaf
}
}
# Get rid of every other tag, without mercy!
{
# Reset to top
match /./
{
match <fromcurrent> /<[^>]*>/ (:some_tag:)
alter (:some_tag:) / /
liaf
}
}
# Remove all
{
# Reset to top
match / /
{
match <fromcurrent> / / (:nbsp:)
alter (:nbsp:) / /
liaf
}
}
# Remove spaces (makes it slower, doesn't seem to help anything)
# call /:kill_spaces:/
alter (:_dw:) /:*:_dw: :*:image_texts:/
# output [/tmp/duh2.txt] /:*:_dw:/
# accept
}
return
web_categorize.crm
(text/plain, 1.9 KB)
# Copyright (C) 2008 Trever L. Adams; licensed under the
# GNU Public License (GPL) version 2. A copy of this license is included
# in the distribution media, or obtain one from www.fsf.org .
{
window
isolate (:pipe: :out: :mydata: :file: :fifo: :trash:)
alter (:pipe:) /\/tmp\/webcatfifo/
call /:first:/
:loop:
{
window <eofretry>
input <byline> [ :*:pipe: ] (:mydata:)
match [:mydata:] /xyx[[:space:]]*\|(.*?)\|[[:space::]*\|(.*)\|[[:space:]]*zxz/ (:trash: :file: :fifo:)
syscall <async> /:call_judge: <:*:file: >>:*:fifo: /
liaf
}
output /OOPS, we barfed\n/
goto /:loop:/
exit
}
:first:
{
isolate (:trash: :fifo: :stupid:)
input <byline> [ :*:pipe: ]
match /xyx[[:space:]]*\|(.*?)\|[[:space::]*\|(.*)\|[[:space:]]*zxz/ (:trash: :file: :stupid:)
alter (:fifo:) /:*:stupid:/
input [:*:file:]
call /:judge:/
output [:*:fifo:] /:*:out:/
}
return
:call_judge:
{
isolate (:out:)
input
call /:judge:/
output /:*:out:/
# output [stderr] /OUT - :*:out:\n/
exit
}
:judge: (:needinput:)
{
call /:normalize_currency:/
call /:de_html:/
isolate (:stats: :mystats: :mymatch: :mypR: :duh:)
insert categories.crm
match [:stats:] /Best match to file #.*? \(:*:chs_dir:(.+?).chs\) prob: .*? pR: (.*?[[:space:]])/ (:mystats: :mymatch: :mypR:)
{
{
eval (:duh:) /:@:(:*:mypR: < 5):/
eval (:duh:) /:@:(:*:mypR: > -2):/
alter (:out:) /X-CATEGORY-TEXT-AMBIGUOUS: :*:mymatch:\n/
alter (:out:) /:*:out:X-CATEGORY-TEXT-AMBIGUOUS-LEVEL: :*:mypR:\n/
}
alius
{
eval (:duh:) /:@:(:*:mypR: > 5):/
alter (:out:) /X-CATEGORY-TEXT: :*:mymatch:\n/
alter (:out:) /:*:out:X-CATEGORY-TEXT-LEVEL: :*:mypR:\n/
}
alius
{
alter (:out:) /X-CATEGORY-TEXT-NEAREST: :*:mymatch:\n/
alter (:out:) /:*:out:X-CATEGORY-TEXT-NEAREST-LEVEL: :*:mypR:\n/
}
}
# output /:*:stats:\n/
# output /:*:_dw:/
# eval (:mycount:) /:@: :#:_dw: :/
# output /SIZE: :*:mycount:\n/
}
return
insert dehtml.crm
web_test.c
(text/plain, 736 B)
#include <sys/types.h>
#include <sys/stat.h>
#include <stdlib.h>
#include <unistd.h>
#include <fcntl.h>
#include <stdio.h>
#include <string.h>
void do_request(int feedto, int i, char *name);
void main(void)
{
int feedto=0;
// mkfifo(mkstemp(CRMFILTXXXXXX), 777);
do_request(feedto, 1, "/tmp/weather.html");
do_request(feedto, 2, "/tmp/duh.html");
do_request(feedto, 3, "/tmp/enya.html");
do_request(feedto, 4, "/tmp/testimony.html");
}
void do_request(int feedto, int i, char *name)
{
char duh[300];
feedto=open("/tmp/webcatfifo", O_WRONLY | O_APPEND);
snprintf(duh, 300, "xyx |%s| |/tmp/resultfifo%d| zxz\n", name, i);
lockf(feedto, F_LOCK, 0);
write(feedto, duh, strlen(duh));
flock(feedto, F_ULOCK, 0);
close(feedto);
}
categories.crm
(text/plain, 1.6 KB)
isolate (:chs_dir:) alter (:chs_dir:) /chs_files\// classify <hyperspace unique > 0.25 (\ chs_files/adult.artnudes.chs chs_files/adult.chs chs_files/adult.naturism.chs chs_files/adult.porn.chs chs_files/adult.sexuality.chs chs_files/antisocial.warez.chs chs_files/clothing.chs chs_files/clothing.intimates.female.chs chs_files/clothing.intimates.male.chs chs_files/clothing.swimsuit.female.chs chs_files/clothing.swimsuit.male.chs chs_files/commerce.auctions.chs chs_files/commerce.banking.chs chs_files/commerce.jewelry.chs chs_files/commerce.payment.chs chs_files/commerce.shopping.chs chs_files/downloads.cellphones.chs chs_files/downloads.desktopsillies.chs chs_files/education.chs chs_files/e-mail.chs chs_files/entertainment.chs chs_files/entertainment.gambling.chs chs_files/entertainment.gossip.chs chs_files/entertainment.onlinegames.chs chs_files/entertainment.sports.chs chs_files/entertainment.vacation.chs chs_files/government.chs chs_files/health.addictions.chs chs_files/health.chs chs_files/health.sexual.chs chs_files/information.childcare.chs chs_files/information.culinary.chs chs_files/information.gardening.chs chs_files/information.homerepair.chs chs_files/information.news.chs chs_files/information.pets.chs chs_files/information.weather.chs chs_files/instantmessaging.chs chs_files/intoxicants.BeerLiquor.chs chs_files/jobsearch.chs chs_files/objectionable.HomoBiSexuality.chs chs_files/personalfinance.chs chs_files/personalfinance.investing.chs chs_files/radio.chs chs_files/realestate.chs chs_files/religion.chs chs_files/social.blogs.chs chs_files/social.dating.chs chs_files/social.reconnect.chs chs_files/weapons.chs) (:stats:)
signature.asc
(application/pgp-signature, 259 B)
-----BEGIN PGP SIGNATURE----- Version: GnuPG v1.4.9 (GNU/Linux) Comment: Using GnuPG with Fedora - http://enigmail.mozdev.org iEYEARECAAYFAkk1fbwACgkQTmpiivh5HXMesACgjhYA4o12CQl+gswTs4V8yyOZ cYQAn0P836FznFAlpxkz9pgAhA+8XURk =MsVW -----END PGP SIGNATURE-----