tokenizer.hpp

Go to the documentation of this file.
00001 /* $Id: tokenizer.hpp 24080 2008-02-26 14:08:17Z suokko $ */
00002 /*
00003    Copyright (C) 2004 - 2008 by Philippe Plantier <ayin@anathas.org>
00004    Part of the Battle for Wesnoth Project http://www.wesnoth.org
00005 
00006    This program is free software; you can redistribute it and/or modify
00007    it under the terms of the GNU General Public License version 2
00008    or at your option any later version.
00009    This program is distributed in the hope that it will be useful,
00010    but WITHOUT ANY WARRANTY.
00011 
00012    See the COPYING file for more details.
00013 */
00014 
00015 //! @file serialization/tokenizer.hpp 
00016 //!
00017 
00018 #ifndef TOKENIZER_H_INCLUDED
00019 #define TOKENIZER_H_INCLUDED
00020 
00021 #include "util.hpp"
00022 
00023 #include <istream>
00024 #include <string>
00025 
00026 class config;
00027 
00028 struct token
00029 {
00030     token() :
00031         type(END),
00032         leading_spaces(),
00033         value()
00034         {}
00035 
00036     enum token_type {
00037         STRING,
00038         QSTRING,
00039         UNTERMINATED_QSTRING,
00040         MISC,
00041 
00042         LF = '\n',
00043         EQUALS = '=',
00044         COMMA = ',',
00045         PLUS = '+',
00046         SLASH = '/',
00047         OPEN_BRACKET = '[',
00048         CLOSE_BRACKED = ']',
00049         UNDERSCORE = '_',
00050         END
00051     } type;
00052 
00053     std::string leading_spaces;
00054     std::string value;
00055 };
00056 
00057 //! Abstract baseclass for the tokenizer
00058 class tokenizer
00059 {
00060 public:
00061     tokenizer(std::istream& in);
00062     ~tokenizer() {}
00063 
00064     const token& next_token();
00065     const token& current_token() const;
00066     std::string get_line() const;
00067     std::string& textdomain();
00068 
00069 protected:
00070     tokenizer();
00071     int current_;
00072     size_t lineno_;
00073 
00074     inline void next_char()
00075     {
00076         if (UNLIKELY(current_ == '\n'))
00077             lineno_++;
00078         this->next_char_fast();
00079     }
00080 
00081     inline void next_char_fast()
00082     {
00083         do {
00084             if (LIKELY(in_.good()))
00085             {
00086                 current_ = in_.get();
00087             }
00088             else
00089             {
00090                 current_ = EOF;
00091                 return;
00092             }
00093         }while (UNLIKELY(current_ == '\r'));
00094 #if 0
00095         // @todo: disabled untill campaign server is fixed
00096         if(LIKELY(in_.good())) {
00097             current_ = in_.get();
00098             if (UNLIKELY(current_ == '\r'))
00099             {
00100                 // we assume that there is only one '\r'
00101                 if(LIKELY(in_.good())) {
00102                     current_ = in_.get();
00103                 } else {
00104                     current_ = EOF;
00105                 }
00106             }
00107         } else {
00108             current_ = EOF;
00109         }
00110 #endif
00111     }
00112 
00113     inline int peek_char() const
00114     {
00115         return in_.peek();
00116     }
00117 
00118 private:
00119     bool is_space(const int c) const;
00120     bool is_alnum(const int c) const;
00121     void skip_comment();
00122 
00123     std::string textdomain_;
00124     std::string file_;
00125     size_t tokenstart_lineno_;
00126     token token_;
00127     std::istream& in_;
00128 };
00129 
00130 #endif
00131 

Generated by doxygen 1.5.5 on 23 May 2008 for The Battle for Wesnoth
Gna! | Forum | Wiki | CIA | devdocs