Re: Bugs in FIFO handling

Bill Yerazunis wrote:
> Trevor:
>
> The files came through as a base64 attachment, not as ASCII 
> plaintext.
>
> Can you resend them, but just insert the file via your text
> editor?
>
> Also- did putting everything in ONE file directory fix the 
> problem?  If it did, my hunch would be to carefully check 
> the file protections because they (and not the code) are the
> most likely culprits.
>
>    - Bill Yerazunis
>
>   
Alright, files coming. These are two separate project. I haven't tried 
putting everything in one directory yet.
I will recheck permissions when I mess with crm again for anti-spam.

categories.crm:
isolate (:chs_dir:)
alter (:chs_dir:) /chs_files\//
classify <hyperspace unique > 0.25 (\
 chs_files/adult.artnudes.chs chs_files/adult.chs 
chs_files/adult.naturism.chs chs_files/adult.porn.chs 
chs_files/adult.sexuality.chs chs_files/antisocial.warez.chs 
chs_files/clothing.chs chs_files/clothing.intimates.female.chs 
chs_files/clothing.intimates.male.chs 
chs_files/clothing.swimsuit.female.chs 
chs_files/clothing.swimsuit.male.chs chs_files/commerce.auctions.chs 
chs_files/commerce.banking.chs chs_files/commerce.jewelry.chs 
chs_files/commerce.payment.chs chs_files/commerce.shopping.chs 
chs_files/downloads.cellphones.chs 
chs_files/downloads.desktopsillies.chs chs_files/education.chs 
chs_files/e-mail.chs chs_files/entertainment.chs 
chs_files/entertainment.gambling.chs chs_files/entertainment.gossip.chs 
chs_files/entertainment.onlinegames.chs 
chs_files/entertainment.sports.chs chs_files/entertainment.vacation.chs 
chs_files/government.chs chs_files/health.addictions.chs 
chs_files/health.chs chs_files/health.sexual.chs 
chs_files/information.childcare.chs chs_files/information.culinary.chs 
chs_files/information.gardening.chs chs_files/information.homerepair.chs 
chs_files/information.news.chs chs_files/information.pets.chs 
chs_files/information.weather.chs chs_files/instantmessaging.chs 
chs_files/intoxicants.BeerLiquor.chs chs_files/jobsearch.chs 
chs_files/objectionable.HomoBiSexuality.chs 
chs_files/personalfinance.chs chs_files/personalfinance.investing.chs 
chs_files/radio.chs chs_files/realestate.chs chs_files/religion.chs 
chs_files/social.blogs.chs chs_files/social.dating.chs 
chs_files/social.reconnect.chs chs_files/weapons.chs) (:stats:)

dehtml.crm:
#    Copyright (C) 2008 Trever L. Adams; licensed under the
#    GNU Public License (GPL) version 2.  A copy of this license is included
#    in the distribution media, or obtain one from www.fsf.org .

{
}
return

# Normalize currency
:normalize_currency:
{
isolate (:mycount: :mycount2: :tocount: :tocount2:)
    {
        alter (:tocount2:) //
        alter (:tocount:) //  
        alter (:mycount:) //
        # crm114 doesn't seem to understand utf-8
        # currency symbols: £€¥元₨₪₱؋៛₡¢圓₩₭₮₨ƒ₦﷼฿₴₫£
        match <fromnext> 
/[$][[:space:]]*([[:digit:]]+)(\.[[:digit:]]*)?/ (:keep: :tocount: 
:tocount2:)
#        output /ORIGKEEP: :*:keep: TO COUNT: :*:tocount: :*:tocount2: 
COUNTS :*:mycount: :*:mycount2:\n/
        eval (:mycount:) /:@: :#:tocount: :/
        eval (:mycount2:) /:@: :#:tocount2: -1 :/
        alter (:keep:) /$/
        {
            eval (:mycount:) /:@: :*:mycount: - 1 :/
            eval / :@: :*:mycount: >= 0 :/
            alter (:keep:) /:*:keep:X/
            liaf
   
        }
        {
            eval / :@: :*:mycount2: >= 0 :/
            alter (:keep:) /:*:keep:./
        }
        {
            eval (:mycount2:) /:@: :*:mycount2: - 1 :/
            eval / :@: :*:mycount2: >= 0 :/
            alter (:keep:) /:*:keep:X/
            liaf   
        }
#        output /KEEP: :*:keep: TO COUNT: :*:mycount: :*:mycount2:\n/
        liaf
    }
}
return

# Remove spaces
:kill_spaces:
{
    # Reset to top
    match /[[:space:]][[:space:]]/
    # Remove spaces
    {
    match <fromcurrent> /[[:space:]][[:space:]]+/ (:deadspace:)
    alter (:deadspace:) /\n/
    liaf
    }
}
return

# Turn all of the content of META tags into inline text
:find_meta:
{
    # Reset to top
    match <nocase> /<meta/
    {
        # Find metas
        {
            isolate (:meta_txt: :good:)
            alter (:meta_txt:) //
            alter (:good:) //
            match <nocase fromcurrent> /<meta ([^>]*)>/ 
(:whole_meta_tag: :meta_txt:)
            {
                {
                    match <nocase> [:whole_meta_tag:] 
/.*description"?([^>]*)>/ (:meta_txt: :good:)
                    call /:keep_good_meta:/ [:*:meta_txt:] (:meta_txt:)
                    alter (:whole_meta_tag:) /:*:meta_txt:/
                }
                alius
                {
                    match <nocase> [:whole_meta_tag:] 
/.*keywords"?([^>]*)>/ (:meta_txt: :good:)
                    call /:keep_good_meta:/ [:*:meta_txt:] (:meta_txt:)
                    alter (:whole_meta_tag:) /:*:meta_txt:/
                }
                alius
                {
                    alter (:whole_meta_tag:) //
                }
            }
            liaf
        }
    }
}
return

# Keep the good meta tags
:keep_good_meta: (:kgm_input:)
{
isolate (:kgm_meta_txt:)
    {
        match <nocase> [:kgm_input:] /content\="?([^"]*)"?/ 
(:kgm_meta_txt: :kgm_good:)
        alter (:kgm_meta_txt:) /:*:kgm_good:/
    }
    return /:*:kgm_meta_txt: /
}

:de_html: (:show:)
{
isolate (:image_texts:)
alter (:image_texts:) //

    #  Remove all comments
    {
       # Reset to top
    match /<!--/
    {
        match <fromcurrent> /<!--.*?-->/ (:whole_comment:)
        alter (:whole_comment:) //
        liaf
    }
    }

    # Remove scripts
    {
    # Reset to top
    match <nocase> /<[ \n\r]*script/
    {
        {
            match <nocase fromcurrent> /<[ \n\r]*script[^>]*>.*?<[ 
\n\r]*\/script[^>]*>/ (:whole_script:)
            alter (:whole_script:) //
            liaf
        }
    }
    }

    # Remove style tags
    {
    # Reset to top
    match <nocase> /<[ \n\r]*style/
    {
        match <nocase fromcurrent> /<[ \n\r]*style[^>]*>.*?<[ 
\n\r]*\/style[^>]*>/ (:whole_style:)
        alter (:whole_style:) //
        liaf
    }
    }
    # Keep keywords and descriptions, kill all other meta tags
    call /:find_meta:/

    #  Keep only the alt and title sections of an IMG tag
    {
    # Reset to top
    match <nocase> /<img/
    {
        isolate (:title_txt: :alt_txt:)
        match <nocase fromcurrent>  /<img ([^>]*)>/ (:whole_img_tag:)
        {
            {
                {
                    match <nocase> [:whole_img_tag:] /title="([^"]*)"/ 
(:trash: :title_txt:)
                }
                alius
                {
                    match <nocase> [:whole_img_tag:] /title='([^']*)'/ 
(:trash: :title_txt:)
                }
                alius
                {
                    match <nocase> [:whole_img_tag:] 
/title=[[:graph:]]*)/ (:trash: :title_txt:)
                }
            }
            {
                {
                    match <nocase> [:whole_img_tag:] /alt="([^"]*)"/ 
(:trash: :alt_txt:)
                }
                alius
                {
                    match <nocase> [:whole_img_tag:] /alt='([^']*)'/ 
(:trash: :alt_txt:)
                }
                alius
                {
                    match <nocase> [:whole_img_tag:] /alt=[[:graph:]]*)/ 
(:trash: :alt_txt:)
                }
            }
        }
        {
            match [:alt_txt:] /.+[[:graph:]].*/
            alter (:image_texts:)  /:*:image_texts: :*:alt_txt:/
        }
        {
            match [:title_txt:] /.+[[:graph:]].*/
            alter (:image_texts:)  /:*:image_texts: :*:title_txt:/
        }
        alter (:whole_img_tag:) / /
        liaf
    }
    }
    #   Get rid of every other tag, without mercy!
    {
    # Reset to top
    match /./
    {
        match <fromcurrent> /<[^>]*>/ (:some_tag:)
        alter (:some_tag:) / /
        liaf
    }
    }

    #  Remove all &nbsp;
    {
       # Reset to top
    match /&nbsp;/
    {
        match <fromcurrent> /&nbsp;/ (:nbsp:)
        alter (:nbsp:) / /
        liaf
    }
    }

    # Remove spaces (makes it slower, doesn't seem to help anything)
#    call /:kill_spaces:/
    alter (:_dw:) /:*:_dw: :*:image_texts:/
#    output [/tmp/duh2.txt] /:*:_dw:/
#    accept
}
return

web_categorize.crm:
#    Copyright (C) 2008 Trever L. Adams; licensed under the
#    GNU Public License (GPL) version 2.  A copy of this license is included
#    in the distribution media, or obtain one from www.fsf.org .

{
    window
    isolate (:pipe: :out: :mydata: :file: :fifo: :trash:)
    alter (:pipe:) /\/tmp\/webcatfifo/

    call /:first:/
    :loop:
    {
            window <eofretry>
            input <byline> [ :*:pipe: ] (:mydata:)
            match [:mydata:] 
/xyx[[:space:]]*\|(.*?)\|[[:space::]*\|(.*)\|[[:space:]]*zxz/ (:trash: 
:file: :fifo:)
            syscall <async> /:call_judge: <:*:file: >>:*:fifo: /
             liaf
    }
    output /OOPS, we barfed\n/
    goto /:loop:/
    exit
}

:first:
{
    isolate (:trash: :fifo: :stupid:)
    input <byline> [ :*:pipe: ]
    match /xyx[[:space:]]*\|(.*?)\|[[:space::]*\|(.*)\|[[:space:]]*zxz/ 
(:trash: :file: :stupid:)
    alter (:fifo:) /:*:stupid:/
    input [:*:file:]
    call /:judge:/
    output [:*:fifo:] /:*:out:/
}
return

:call_judge:
{
    isolate (:out:)
    input
    call /:judge:/
    output /:*:out:/
#    output [stderr] /OUT - :*:out:\n/
    exit
}

:judge: (:needinput:)
{
    call /:normalize_currency:/
    call /:de_html:/
    isolate (:stats: :mystats: :mymatch: :mypR: :duh:)
    insert categories.crm
    match [:stats:] /Best match to file #.*? \(:*:chs_dir:(.+?).chs\) 
prob: .*?  pR: (.*?[[:space:]])/ (:mystats: :mymatch: :mypR:)
    {
        {
            eval (:duh:) /:@:(:*:mypR: < 5):/
            eval (:duh:) /:@:(:*:mypR: > -2):/
            alter (:out:) /X-CATEGORY-TEXT-AMBIGUOUS: :*:mymatch:\n/
            alter (:out:) /:*:out:X-CATEGORY-TEXT-AMBIGUOUS-LEVEL: 
:*:mypR:\n/
        }
        alius
        {
            eval (:duh:) /:@:(:*:mypR: > 5):/
            alter (:out:) /X-CATEGORY-TEXT: :*:mymatch:\n/
            alter (:out:) /:*:out:X-CATEGORY-TEXT-LEVEL: :*:mypR:\n/
        }
        alius
        {
            alter (:out:) /X-CATEGORY-TEXT-NEAREST: :*:mymatch:\n/
            alter (:out:) /:*:out:X-CATEGORY-TEXT-NEAREST-LEVEL: :*:mypR:\n/
        }
    }

#    output /:*:stats:\n/
#    output /:*:_dw:/
#    eval (:mycount:) /:@: :#:_dw: :/
#    output /SIZE: :*:mycount:\n/
}
return

insert dehtml.crm


web_test.c:
#include <sys/types.h>
#include <sys/stat.h>
#include <stdlib.h>
#include <unistd.h>
#include <fcntl.h>
#include <stdio.h>
#include <string.h>

void do_request(int feedto, int i, char *name);

void main(void)
{
int feedto=0;
//    mkfifo(mkstemp(CRMFILTXXXXXX), 777);
    do_request(feedto, 1, "/tmp/weather.html");
    do_request(feedto, 2, "/tmp/duh.html");
    do_request(feedto, 3, "/tmp/enya.html");
    do_request(feedto, 4, "/tmp/testimony.html");
}

void do_request(int feedto, int i, char *name)
{
char duh[300];
    feedto=open("/tmp/webcatfifo", O_WRONLY | O_APPEND);
    snprintf(duh, 300, "xyx |%s| |/tmp/resultfifo%d| zxz\n", name, i);
    lockf(feedto, F_LOCK, 0);
    write(feedto, duh, strlen(duh));
    flock(feedto, F_ULOCK, 0);
    close(feedto);
}

-------------------------------------------------------------------------
This SF.Net email is sponsored by the Moblin Your Move Developer's challenge
Build the coolest Linux based applications with Moblin SDK & win great prizes
Grand prize is a trip for two to an Open Source event anywhere in the world
http://moblin-contest.org/redirect.php?banner_id=100&url=/

_______________________________________________
Crm114-general mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/crm114-general

-----BEGIN PGP SIGNATURE-----
Version: GnuPG v1.4.9 (GNU/Linux)
Comment: Using GnuPG with Fedora - http://enigmail.mozdev.org

iEYEARECAAYFAkk1iPAACgkQTmpiivh5HXOY5wCcCTBghq7pSi1aPc9euZyOJ9pt
8UAAoIOGpeiLYYrVrw48mZcAxE6fXnRF
=nRAD
-----END PGP SIGNATURE-----