Lines Matching refs:tokenizer
117 // The "tokenizer" class.
121 //! \brief A stream tokenizer.
124 //! tokenizer. It is only able to recognize one character-long delimiters,
131 class tokenizer {
154 tokenizer(IS&, bool, const token_type&, const token_type&,
168 tokenizer< IS >::tokenizer(IS& p_is,
186 tokenizer< IS >::lineno(void)
194 tokenizer< IS >::add_delim(char delim, const token_type& type)
202 tokenizer< IS >::add_keyword(const std::string& keyword,
210 tokenizer< IS >::add_quote(char ch, const token_type& type)
218 tokenizer< IS >::next(void)
306 tokenizer< IS >::rest_of_line(void)