Re: ocamllex eating up line breaks even when not asked to

I cannot reproduce the output you say you observe. Instead I have

# parse_all "abc\ndef";;
- : token list = [UNRESERVED "abc"; CHAR '\n'; UNRESERVED "def"]

which makes sense: the only part of your code that handles LINEBREAK (and
PUNCTUATION) is the keyword list, but the only lexing rule that queries the
keyword list is "id", which is only invoked for alphanumeric characters.
"\n" is thus not recognized by this rule and handled by the only rule which
it makes, namely the (_ as c { CHAR c }) rule.

On Wed, May 6, 2015 at 11:22 AM, [email protected] [ocaml_beginners]
<[email protected]> wrote:

>
>
>  Hello all, when I use the lexer from the .mll file below, I do not get
> what I want
>   For example, when I type parse_all « abc\ndef » in the interpreter, I am
> expecting
> to be answered [UNRESERVED « abc »; LINEBREAK ‘\n’ ; UNRESERVED « def » ]
>  but instead I get [UNRESERVED « abc »; UNRESERVED « def » ].
>    What did I do wrong ?
>
>
> Contents of mysql_lexer.mll file :
>
> {
>
>   let create_hashtable size init =
>     let tbl = Hashtbl.create size in
>     List.iter (fun (key, data) -> Hashtbl.add tbl key data) init;
>     tbl
>
>   type token =
>     | OP of char
>     | INT of int
>     | FLOAT of float
>     | CHAR of char
>     | PUNCTUATION_MARK of char
>     | LINEBREAK of char
>     | RESERVED of string
>     | UNRESERVED of string
>
>   let reserved_words=
>      [
>        "AUTO_INCREMENT";
>        "CHARSET";
>        "COLLATE";
>        "CREATE";
>        "DEFAULT";
>        "ENGINE";
>        "INSERT";
>        "INTO";
>        "KEY";
>        "NOT";
>        "NULL";
>        "PRIMARY";
>        "SET";
>        "SQL_MODE";
>        "time_zone";
>        "VALUES"
>      ]
>
>   let punctuation_marks=
>      [
>        ';';
>        '\"';
>        '\'';
>        '=';
>        '`';
>        ',';
>        '?';
>        ':';
>        '(';
>        ')';
>        '{';
>        '}';
>      ]
>
>     let linebreaks=
>       [
>        '\n';
>        '\r'
>       ]
>
>   let keyword_list =
>    (List.map (fun c->(String.make 1 c,PUNCTUATION_MARK c))
> punctuation_marks )
>   @(List.map (fun x->(x,RESERVED x)) reserved_words )
>   @(List.map (fun c->(String.make 1 c,LINEBREAK c)) linebreaks )
>
>   let keyword_table =
>     create_hashtable (List.length keyword_list) keyword_list
>
>
> }
>
> let digit = ['0'-'9']
> let id = ['a'-'z' 'A'-'Z' '0'-'9']*
>
> rule parse_just_one_item = parse
>   | digit+ as inum
>       { let num = int_of_string inum in
>         let tok=INT num in
>         tok
>
>     }
>   | digit+ '.' digit* as fnum
>       { let num = float_of_string fnum in
>       let tok=FLOAT num in
>         tok
>     }
>   | id as word
>       { try
>         let token = Hashtbl.find keyword_table word in
>         token
>       with Not_found ->
>         let token=UNRESERVED word in
>         token
>     }
>   | '+'
>   | '-'
>   | '*'
>   | '/' as op
>       { OP op }
>   | [' ' '\t' ]    (* eat up whitespace *)
>        { parse_just_one_item lexbuf }
>   | _ as c
>       {
>       CHAR c
>     }
>   | eof
>       { raise End_of_file }
>
> {
>
> let accu=ref([])
>
> let memorize x=(accu:=x::(!accu))
>
> let parse_all_silently s =
>           let _=(accu:=[]) in
>           try
>             let lexbuf = Lexing.from_string s in
>             while true do
>               let result = parse_just_one_item lexbuf in
>                 memorize result;
>             done
>           with End_of_file ->
>             ()
>
> let parse_all s=let _=parse_all_silently s in List.rev(!accu)
>
> }
>
>
>
>
>
>
> 
>