ocamllex eating up line breaks even when not asked to
| Newsgroups | gmane.comp.lang.ocaml.beginners |
|---|---|
| Message-ID | <[email protected]> |
Hello all, when I use the lexer from the .mll file below, I do not get what I want
For example, when I type parse_all « abc\ndef » in the interpreter, I am expecting
to be answered [UNRESERVED « abc »; LINEBREAK ‘\n’ ; UNRESERVED « def » ]
but instead I get [UNRESERVED « abc »; UNRESERVED « def » ].
What did I do wrong ?
Contents of mysql_lexer.mll file :
{
let create_hashtable size init =
let tbl = Hashtbl.create size in
List.iter (fun (key, data) -> Hashtbl.add tbl key data) init;
tbl
type token =
| OP of char
| INT of int
| FLOAT of float
| CHAR of char
| PUNCTUATION_MARK of char
| LINEBREAK of char
| RESERVED of string
| UNRESERVED of string
let reserved_words=
[
"AUTO_INCREMENT";
"CHARSET";
"COLLATE";
"CREATE";
"DEFAULT";
"ENGINE";
"INSERT";
"INTO";
"KEY";
"NOT";
"NULL";
"PRIMARY";
"SET";
"SQL_MODE";
"time_zone";
"VALUES"
]
let punctuation_marks=
[
';';
'\"';
'\'';
'=';
'`';
',';
'?';
':';
'(';
')';
'{';
'}';
]
let linebreaks=
[
'\n';
'\r'
]
let keyword_list =
(List.map (fun c->(String.make 1 c,PUNCTUATION_MARK c)) punctuation_marks )
@(List.map (fun x->(x,RESERVED x)) reserved_words )
@(List.map (fun c->(String.make 1 c,LINEBREAK c)) linebreaks )
let keyword_table =
create_hashtable (List.length keyword_list) keyword_list
}
let digit = ['0'-'9']
let id = ['a'-'z' 'A'-'Z' '0'-'9']*
rule parse_just_one_item = parse
| digit+ as inum
{ let num = int_of_string inum in
let tok=INT num in
tok
}
| digit+ '.' digit* as fnum
{ let num = float_of_string fnum in
let tok=FLOAT num in
tok
}
| id as word
{ try
let token = Hashtbl.find keyword_table word in
token
with Not_found ->
let token=UNRESERVED word in
token
}
| '+'
| '-'
| '*'
| '/' as op
{ OP op }
| [' ' '\t' ] (* eat up whitespace *)
{ parse_just_one_item lexbuf }
| _ as c
{
CHAR c
}
| eof
{ raise End_of_file }
{
let accu=ref([])
let memorize x=(accu:=x::(!accu))
let parse_all_silently s =
let _=(accu:=[]) in
try
let lexbuf = Lexing.from_string s in
while true do
let result = parse_just_one_item lexbuf in
memorize result;
done
with End_of_file ->
()
let parse_all s=let _=parse_all_silently s in List.rev(!accu)
}