how to implement lexer with more than one state(s) ?
Jens Kallup <[email protected]>
| Newsgroups | gmane.comp.parsers.spirit.general |
|---|---|
| Message-ID | <[email protected]> |
Hello,
like the subject says:
how to implement lexer with more than one state?
here the code with the issue:
Jens
#include "includes/mainwindow.h"
#include <boost/config/warning_disable.hpp>
#include <boost/spirit/include/lex_lexertl.hpp>
#include <boost/spirit/include/phoenix.hpp>
#include <boost/spirit/include/qi.hpp>
#include <boost/spirit/include/qi_skip.hpp>
#include <boost/spirit/include/qi_eol.hpp>
#include <boost/spirit/include/phoenix_operator.hpp>
#include <boost/bind.hpp>
#include <boost/ref.hpp>
#include <iostream>
#include <fstream>
int lineno = 1;
using namespace std;
namespace lex = boost::spirit::lex;
namespace qi = boost::spirit::qi;
namespace ascii = boost::spirit::ascii;
namespace phx = boost::phoenix;
using boost::spirit::ascii::space; // use the ASCII space parser
using boost::spirit::ascii::char_;
using boost::phoenix::val;
enum TokenEnum {
TokPrintLn,
TokPrintStr
};
struct set_lexer_state
{
set_lexer_state(char const* state_)
: state(state_)
{
QString str = state_;
if (str == "PRINTLN") {
cout << "printer"
<< endl;
}
}
// This is called by the semantic action handling code during the
lexing
template <typename Iterator, typename Context>
void operator()(Iterator const&, Iterator const&
, BOOST_SCOPED_ENUM(boost::spirit::lex::pass_flags)&
, std::size_t&, Context& ctx) const
{
ctx.set_state_name(state.c_str());
}
std::string state;
};
struct add_token
{
// This is called by the semantic action handling code during the
lexing
template <typename Iterator, typename Context>
void operator()(Iterator const&, Iterator const&
, BOOST_SCOPED_ENUM(boost::spirit::lex::pass_flags)&
, std::size_t&, Context& ctx) const
{
ctx.set_state_name("PRINTLN");
set_lexer_state("INITIAL");
}
add_token(TokenEnum entype)
{
switch (entype)
{
case TokPrintStr:
{
std::cout << "11: " << std::endl;
} break;
}
set_lexer_state("INITIAL");
}
};
template <typename Lexer>
struct dbase_tokens : lex::lexer<Lexer>
{
dbase_tokens()
: dbase_tokens::base_type(lex::match_flags::match_default)
{
printLn = "\\?";
//cc_comment = "\\/\\/.*";
defEol = "[ \\r\\n]*";
//definition = "^\\n*";
//stringStr1 = "\\\".*\\\"";
//stringStr2 = "\\'.*\\'[ \\t\\n]*";
//stringStr3 = "\\[.*\\][ \\t\\n]*";
whitespace = "[ \\t\\r\\n]*";
this->self =
cc_comment [ set_lexer_state("COMMENT") ] |
printLn [ set_lexer_state("PRINTLN") ]
;
this->self("PRINTLN")
= defEol [ set_lexer_state("INITIAL") ]
//| stringStr1 [ add_token(TokenEnum::TokPrintStr) ]
//| stringStr2 [ add_token(TokenEnum::TokPrintStr) ]
//| stringStr3 [ add_token(TokenEnum::TokPrintStr) ]
// | cc_comment [ set_lexer_state("COMMENT") ]
// | definition [ lex::_pass = lex::pass_flags::pass_ignore ]
//| whitespace [ set_lexer_state("INITIAL") ]
;
this->self("COMMENT")
= defEol [ set_lexer_state("INITIAL") ]
//| definition [ lex::_pass = lex::pass_flags::pass_ignore ]
//| whitespace [ set_lexer_state("INITIAL") ]
;
}
lex::token_def<lex::omit> cc_comment, defEol,
printLn,
whitespace;
lex::token_def<string> definition,
stringStr1,
stringStr2,
stringStr3;
};
template <typename Iterator>
struct dbase_grammar : public qi::grammar<Iterator>
{
template <typename TokenDef>
dbase_grammar(TokenDef const& tok) :
dbase_grammar::base_type(start)
{
start = +symsbols;
symsbols
=
//tok.cc_comment |
tok.printLn
;
}
qi::rule<Iterator> start,
symsbols;
};
bool parseText(QString text)
{
std::string data(text.toStdString().c_str());
if (data.size() < 1) {
QMessageBox::information(0,"Error","No Data for parser.\nABORT.");
return false;
}
typedef std::string::iterator base_iterator_type;
typedef lex::lexertl::token<
base_iterator_type, boost::mpl::vector<std::string>
> token_type;
typedef lex::lexertl::actor_lexer<token_type> lexer_type;
typedef dbase_tokens<lexer_type> dbase_tokens;
typedef dbase_grammar<dbase_tokens::iterator_type> dbase_grammar;
dbase_tokens dbase_lexer;
dbase_grammar dbase_parser(dbase_lexer);
auto first = data.begin();
auto last = data.end();
bool r = false;
lex::tokenize(first, last, dbase_lexer);
first = data.begin();
r = lex::tokenize_and_parse(first, last, dbase_lexer, dbase_parser);
//r = qi::phrase_parse(first, last, ini_parser, ascii::space);
if (r && first == last) {
cout << "SUCCESS" << endl;
return true;
} else {
cout << "ERROR at " << endl;
cout << std::string(first, last) << endl;
cout << "abort." << endl;
} return false;
}
------------------------------------------------------------------------------
Attend Shape: An AT&T Tech Expo July 15-16. Meet us at AT&T Park in San
Francisco, CA to explore cutting-edge tech and listen to tech luminaries
present their vision of the future. This family event has something for
everyone, including kids. Get more information and register today.
http://sdm.link/attshape