28 m_tokenizer=tokenizer;
30 if (m_tokenizer!=NULL)
41 if (m_tokenizer!=NULL)
61 for (
index_t i=start; i<end; i++)
63 result[i-start]=m_text[i];
77 for (
index_t i=start; i<end; i++)
79 result[i-start]=m_text[i];
81 result[end-start]=
'\0';
91 return (
bool) strtod(token.
vector, NULL);
96 #define READ_INT_METHOD(fname, convf, sg_type) \
97 sg_type CParser::fname() \
99 SGVector<char> token=read_cstring(); \
102 return (sg_type) convf(token.vector, NULL, 10); \
104 return (sg_type) 0L; \
109 #undef READ_INT_METHOD
111 #define READ_REAL_METHOD(fname, convf, sg_type) \
112 sg_type CParser::fname() \
114 SGVector<char> token=read_cstring(); \
117 return (sg_type) convf(token.vector, NULL); \
119 return (sg_type) 0L; \
136 #undef READ_REAL_METHOD
142 if (m_tokenizer!=NULL)
150 m_tokenizer=tokenizer;
152 if (m_tokenizer!=NULL)
void set_text(SGVector< char > text)
#define READ_REAL_METHOD(fname, convf, sg_type)
virtual void set_text(SGVector< char > txt)
virtual void skip_token()
virtual SGVector< char > read_string()
The class CTokenizer acts as a base class in order to implement tokenizers. Sub-classes must implemen...
virtual bool has_next()=0
void set_tokenizer(CTokenizer *tokenizer)
all of classes and functions are contained in the shogun namespace
virtual index_t next_token_idx(index_t &start)=0
virtual SGVector< char > read_cstring()
#define READ_INT_METHOD(fname, convf, sg_type)