Tokenizer.h

Go to the documentation of this file.

/*
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 3 of the License, or
 * (at your option) any later version.
 *
 * Written (W) 2013 Evangelos Anagnostopoulos
 * Copyright (C) 2013 Evangelos Anagnostopoulos
 */

#ifndef _TOKENIZER__H__
#define _TOKENIZER__H__

#include <shogun/base/SGObject.h>
#include <shogun/lib/SGString.h>
#include <shogun/lib/SGVector.h>

namespace shogun
{
class CSGObject;
template<class T> class SGVector;

class CTokenizer: public CSGObject
{
public:
    CTokenizer();

    CTokenizer(const CTokenizer& orig);

    virtual ~CTokenizer() { };

    virtual void set_text(SGVector<char> txt);

    virtual bool has_next()=0;

    virtual index_t next_token_idx(index_t& start)=0;

    virtual CTokenizer* get_copy()=0;

private:
    void init();

protected:
    SGVector<char> text;
};
}

#endif  /* _TOKENIZER__H__ */