DelimiterTokenizer.h

Go to the documentation of this file.

/*
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 3 of the License, or
 * (at your option) any later version.
 *
 * Written (W) 2013 Evangelos Anagnostopoulos
 * Copyright (C) 2013 Evangelos Anagnostopoulos
 */

#ifndef _DELIMITERTOKENIZER__H__
#define _DELIMITERTOKENIZER__H__

#include <shogun/lib/config.h>

#include <shogun/lib/Tokenizer.h>
#include <shogun/lib/SGVector.h>
#include <shogun/lib/common.h>

namespace shogun
{

class CDelimiterTokenizer: public CTokenizer
{
public:
    CDelimiterTokenizer(bool skip_delimiters = false);

    CDelimiterTokenizer(const CDelimiterTokenizer& orig);

    virtual ~CDelimiterTokenizer() {}

    virtual void set_text(SGVector<char> txt);

    virtual bool has_next();

    virtual index_t next_token_idx(index_t& start);

    virtual const char* get_name() const;

    void init_for_whitespace();

    CDelimiterTokenizer* get_copy();

    void clear_delimiters();

    bool get_skip_delimiters() const;

    void set_skip_delimiters(bool skip_delimiters);

private:
    void init();

public:
    SGVector<bool> delimiters;

protected:
    index_t last_idx;

    bool skip_consecutive_delimiters;
};
}
#endif  /* _WHITESPACETOKENIZER__H__ */