how to implement lexer with more than one state(s) ?

Jens Kallup <[email protected]>
Newsgroups gmane.comp.parsers.spirit.general
Message-ID <[email protected]>
Hello,

like the subject says:
how to implement lexer with more than one state?

here the code with the issue:
Jens

#include "includes/mainwindow.h"

#include <boost/config/warning_disable.hpp>
#include <boost/spirit/include/lex_lexertl.hpp>
#include <boost/spirit/include/phoenix.hpp>
#include <boost/spirit/include/qi.hpp>
#include <boost/spirit/include/qi_skip.hpp>
#include <boost/spirit/include/qi_eol.hpp>
#include <boost/spirit/include/phoenix_operator.hpp>
#include <boost/bind.hpp>
#include <boost/ref.hpp>
#include <iostream>
#include <fstream>

int lineno = 1;


using namespace std;

namespace lex   = boost::spirit::lex;
namespace qi    = boost::spirit::qi;
namespace ascii = boost::spirit::ascii;

namespace phx   = boost::phoenix;

using boost::spirit::ascii::space; // use the ASCII space parser
using boost::spirit::ascii::char_;

using boost::phoenix::val;

enum TokenEnum {
     TokPrintLn,
     TokPrintStr
};

struct set_lexer_state
{
     set_lexer_state(char const* state_)
         : state(state_)
     {
         QString str = state_;
         if (str == "PRINTLN") {
             cout << "printer"
                  << endl;
         }
     }

     // This is called by the semantic action handling code during the 
lexing
     template <typename Iterator, typename Context>
     void operator()(Iterator const&, Iterator const&
                     , BOOST_SCOPED_ENUM(boost::spirit::lex::pass_flags)&
                     , std::size_t&, Context& ctx) const
     {
         ctx.set_state_name(state.c_str());
     }

     std::string state;
};

struct add_token
{
     // This is called by the semantic action handling code during the 
lexing
     template <typename Iterator, typename Context>
     void operator()(Iterator const&, Iterator const&
                     , BOOST_SCOPED_ENUM(boost::spirit::lex::pass_flags)&
                     , std::size_t&, Context& ctx) const
     {
         ctx.set_state_name("PRINTLN");
         set_lexer_state("INITIAL");
     }

     add_token(TokenEnum entype)
     {
         switch (entype)
         {
             case TokPrintStr:
             {
                 std::cout << "11: " << std::endl;
             }   break;
         }
         set_lexer_state("INITIAL");
     }
};

template <typename Lexer>
struct dbase_tokens : lex::lexer<Lexer>
{
     dbase_tokens()
         : dbase_tokens::base_type(lex::match_flags::match_default)
     {
         printLn    = "\\?";

         //cc_comment = "\\/\\/.*";

         defEol     = "[ \\r\\n]*";

         //definition = "^\\n*";
         //stringStr1 = "\\\".*\\\"";
         //stringStr2 = "\\'.*\\'[ \\t\\n]*";
         //stringStr3 = "\\[.*\\][ \\t\\n]*";

         whitespace = "[ \\t\\r\\n]*";

         this->self =
               cc_comment  [ set_lexer_state("COMMENT") ] |
               printLn     [ set_lexer_state("PRINTLN") ]
             ;

         this->self("PRINTLN")
             = defEol      [ set_lexer_state("INITIAL") ]

             //| stringStr1 [ add_token(TokenEnum::TokPrintStr) ]
             //| stringStr2 [ add_token(TokenEnum::TokPrintStr) ]
             //| stringStr3 [ add_token(TokenEnum::TokPrintStr) ]

           //  | cc_comment [ set_lexer_state("COMMENT") ]
           //  | definition [ lex::_pass = lex::pass_flags::pass_ignore ]
             //| whitespace  [ set_lexer_state("INITIAL") ]
             ;

         this->self("COMMENT")
             = defEol     [ set_lexer_state("INITIAL") ]
             //| definition [ lex::_pass = lex::pass_flags::pass_ignore ]
             //| whitespace [ set_lexer_state("INITIAL") ]
             ;
     }

     lex::token_def<lex::omit> cc_comment, defEol,
     printLn,
     whitespace;

     lex::token_def<string> definition,

     stringStr1,
     stringStr2,
     stringStr3;
};

template <typename Iterator>
struct dbase_grammar : public qi::grammar<Iterator>
{
     template <typename TokenDef>

     dbase_grammar(TokenDef const& tok) :
     dbase_grammar::base_type(start)
     {
         start = +symsbols;

         symsbols
               =
               //tok.cc_comment |
               tok.printLn
         ;

     }
     qi::rule<Iterator> start,
     symsbols;
};

bool parseText(QString text)
{
     std::string data(text.toStdString().c_str());
     if (data.size() < 1) {
         QMessageBox::information(0,"Error","No Data for parser.\nABORT.");
         return false;
     }

     typedef std::string::iterator base_iterator_type;
     typedef lex::lexertl::token<
             base_iterator_type, boost::mpl::vector<std::string>
         > token_type;
     typedef lex::lexertl::actor_lexer<token_type> lexer_type;
     typedef dbase_tokens<lexer_type> dbase_tokens;
     typedef dbase_grammar<dbase_tokens::iterator_type> dbase_grammar;

     dbase_tokens  dbase_lexer;
     dbase_grammar dbase_parser(dbase_lexer);

     auto first = data.begin();
     auto last  = data.end();

     bool r = false;

     lex::tokenize(first, last, dbase_lexer);
     first = data.begin();

     r = lex::tokenize_and_parse(first, last, dbase_lexer, dbase_parser);
     //r = qi::phrase_parse(first, last, ini_parser, ascii::space);
     if (r && first == last) {
         cout << "SUCCESS" << endl;
         return true;
     } else {
         cout << "ERROR at " << endl;
         cout << std::string(first, last) << endl;
         cout << "abort." << endl;
     }   return false;
}

------------------------------------------------------------------------------
Attend Shape: An AT&T Tech Expo July 15-16. Meet us at AT&T Park in San
Francisco, CA to explore cutting-edge tech and listen to tech luminaries
present their vision of the future. This family event has something for
everyone, including kids. Get more information and register today.
http://sdm.link/attshape
lmpx.com only provides a reader for public news (NNTP) servers. It is not affiliated with the servers or forums shown here and is not responsible for the content of articles, which is written by their respective authors.