ocamllex eating up line breaks even when not asked to

"[email protected] [ocaml_beginners]" <[email protected]>
Newsgroups gmane.comp.lang.ocaml.beginners
Message-ID <[email protected]>
 Hello all, when I use the lexer from the .mll file below, I do not get what I want
  For example, when I type parse_all « abc\ndef » in the interpreter, I am expecting
to be answered [UNRESERVED « abc »; LINEBREAK ‘\n’ ; UNRESERVED « def » ]
 but instead I get [UNRESERVED « abc »; UNRESERVED « def » ].
   What did I do wrong ?
  

Contents of mysql_lexer.mll file : 

{

  let create_hashtable size init =
    let tbl = Hashtbl.create size in
    List.iter (fun (key, data) -> Hashtbl.add tbl key data) init;
    tbl

  type token =
    | OP of char
    | INT of int
    | FLOAT of float
    | CHAR of char
    | PUNCTUATION_MARK of char
    | LINEBREAK of char
    | RESERVED of string
    | UNRESERVED of string
    
  let reserved_words=
     [
       "AUTO_INCREMENT";
       "CHARSET";
       "COLLATE";
       "CREATE";
       "DEFAULT";
       "ENGINE";
       "INSERT";
       "INTO";
       "KEY";
       "NOT";
       "NULL";
       "PRIMARY";
       "SET";
       "SQL_MODE";
       "time_zone";
       "VALUES"
     ]  
     
  let punctuation_marks=
     [
       ';';
       '\"';
       '\'';
       '=';
       '`';
       ',';
       '?';
       ':';
       '(';
       ')';
       '{';
       '}';   
     ]   
    
    let linebreaks=
      [
       '\n';
       '\r'
      ]
    
  let keyword_list =  
   (List.map (fun c->(String.make 1 c,PUNCTUATION_MARK c)) punctuation_marks )
  @(List.map (fun x->(x,RESERVED x)) reserved_words )
  @(List.map (fun c->(String.make 1 c,LINEBREAK c)) linebreaks )

  let keyword_table = 
    create_hashtable (List.length keyword_list) keyword_list


}

let digit = ['0'-'9']
let id = ['a'-'z' 'A'-'Z' '0'-'9']*

rule parse_just_one_item = parse
  | digit+ as inum
      { let num = int_of_string inum in
        let tok=INT num in
        tok
      
    }
  | digit+ '.' digit* as fnum
      { let num = float_of_string fnum in
      let tok=FLOAT num in
        tok
    }
  | id as word
      { try
        let token = Hashtbl.find keyword_table word in
        token
      with Not_found ->
        let token=UNRESERVED word in
        token
    }
  | '+'
  | '-'
  | '*'
  | '/' as op
      { OP op }
  | [' ' '\t' ]    (* eat up whitespace *)
       { parse_just_one_item lexbuf }
  | _ as c
      { 
      CHAR c
    }
  | eof
      { raise End_of_file }

{

let accu=ref([])
  
let memorize x=(accu:=x::(!accu))

let parse_all_silently s =
          let _=(accu:=[]) in
          try
            let lexbuf = Lexing.from_string s in
            while true do
              let result = parse_just_one_item lexbuf in
                memorize result; 
            done
          with End_of_file ->
            ()

let parse_all s=let _=parse_all_silently s in List.rev(!accu)
  
}
lmpx.com only provides a reader for public news (NNTP) servers. It is not affiliated with the servers or forums shown here and is not responsible for the content of articles, which is written by their respective authors.