Split tokenizer and preprocessor out of the GLSL parser

[libs/gl.git] / source / glsl / tokenizer.h

diff --git a/source/glsl/tokenizer.h b/source/glsl/tokenizer.h
new file mode 100644 (file)
index 0000000..a732918
--- /dev/null
+++ b/source/glsl/tokenizer.h
@@ -0,0 +1,54 @@
+#ifndef MSP_GL_SL_TOKENIZER_H_
+#define MSP_GL_SL_TOKENIZER_H_
+
+#include <deque>
+#include <string>
+#include <sigc++/signal.h>
+
+namespace Msp {
+namespace GL {
+namespace SL {
+
+class Preprocessor;
+
+struct Location
+{
+       std::string name;
+       unsigned line;
+};
+
+class Tokenizer
+{
+public:
+       sigc::signal<void> signal_preprocess;
+
+private:
+       std::string::const_iterator iter;
+       std::string::const_iterator source_end;
+       Location location;
+       bool allow_preprocess;
+       std::string last_token;
+       std::deque<std::string> next_tokens;
+
+public:
+       Tokenizer();
+
+       void begin(const std::string &, const std::string &);
+       const std::string &peek_token(unsigned = 0);
+       const std::string &parse_token();
+       void expect(const std::string &);
+       const Location &get_location() const { return location; }
+private:
+       std::string parse_token_();
+       void preprocess();
+       std::string parse_identifier();
+       std::string parse_number();
+       std::string parse_other();
+       void skip_comment_and_whitespace();
+};
+
+} // namespace SL
+} // namespace GL
+} // namespace Msp
+
+#endif