13 using namespace shogun;
27 m_max_token_length=10*1024*1024;
30 m_tokenizer=tokenizer;
41 m_max_token_length=max_token_length;
44 m_tokenizer=tokenizer;
58 if (m_stream==NULL || m_max_token_length==0 || m_tokenizer==NULL)
60 SG_ERROR(
"CLineReader::has_next():: Class is not initialized\n");
66 SG_ERROR(
"CLineReader::has_next():: Error reading file\n");
78 int32_t bytes_to_skip=0;
79 m_next_token_length=read(bytes_to_skip);
80 if (m_next_token_length==-1)
90 int32_t bytes_to_skip=0;
91 m_next_token_length=read(bytes_to_skip);
92 if (m_next_token_length==-1)
97 line=read_token(m_next_token_length-bytes_to_skip);
113 m_tokenizer=tokenizer;
118 void CLineReader::init()
124 m_max_token_length=0;
125 m_next_token_length=-1;
128 int32_t CLineReader::read(int32_t& bytes_to_skip)
131 int32_t bytes_to_read=0;
132 int32_t temp_bytes_to_skip=0;
136 if (bytes_to_skip==line_end)
149 if (m_buffer->
available() < m_max_token_length)
152 bytes_to_read=m_max_token_length;
157 m_buffer->
push(m_stream, bytes_to_read);
159 if (ferror(m_stream))
161 SG_ERROR(
"CLineReader::read(int32_t&):: Error reading file\n");
174 line=m_buffer->
pop(line_len);