StreamingVwFeatures.h

Go to the documentation of this file.

/*
 * Copyright (c) 2009 Yahoo! Inc.  All rights reserved.  The copyrights
 * embodied in the content of this file are licensed under the BSD
 * (revised) open source license.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 3 of the License, or
 * (at your option) any later version.
 *
 * Written (W) 2011 Shashwat Lal Das
 * Adaptation of Vowpal Wabbit v5.1.
 * Copyright (C) 2011 Berlin Institute of Technology and Max-Planck-Society.
 */

#ifndef _STREAMING_VWFEATURES__H__
#define _STREAMING_VWFEATURES__H__

#include <shogun/lib/common.h>
#include <shogun/lib/DataType.h>
#include <shogun/mathematics/Math.h>

#include <shogun/io/InputParser.h>
#include <shogun/io/StreamingVwFile.h>
#include <shogun/io/StreamingVwCacheFile.h>
#include <shogun/features/StreamingDotFeatures.h>
#include <shogun/classifier/vw/vw_common.h>
#include <shogun/classifier/vw/vw_math.h>

namespace shogun
{
class CStreamingVwFeatures : public CStreamingDotFeatures
{
public:

    CStreamingVwFeatures()
        : CStreamingDotFeatures()
    {
        init();
        set_read_functions();
    }

    CStreamingVwFeatures(CStreamingVwFile* file,
                 bool is_labelled,
                 int32_t size)
        : CStreamingDotFeatures()
    {
        init(file, is_labelled, size);
        set_read_functions();
    }

    CStreamingVwFeatures(CStreamingVwCacheFile* file,
                 bool is_labelled,
                 int32_t size)
        : CStreamingDotFeatures()
    {
        init(file, is_labelled, size);
        set_read_functions();
    }

    ~CStreamingVwFeatures()
    {
        parser.end_parser();
        SG_UNREF(env);
    }

    CFeatures* duplicate() const
    {
        return new CStreamingVwFeatures(*this);
    }

    virtual void set_vector_reader();

    virtual void set_vector_and_label_reader();

    virtual void start_parser();

    virtual void end_parser();

    virtual void reset_stream()
    {
        if (working_file->is_seekable())
        {
            working_file->reset_stream();
            parser.exit_parser();
            parser.init(working_file, has_labels, parser.get_ring_size());
            parser.set_free_vector_after_release(false);
            parser.start_parser();
        }
        else
            SG_ERROR("The input cannot be reset! Please use 1 pass.\n");
    }

    virtual CVwEnvironment* get_env()
    {
        SG_REF(env);
        return env;
    }

    virtual void set_env(CVwEnvironment* vw_env)
    {
        env = vw_env;
        SG_REF(env);
    }

    virtual bool get_next_example();

    virtual VwExample* get_example();

    virtual float64_t get_label();

    virtual void release_example();

    inline virtual void expand_if_required(float32_t*& vec, int32_t& len)
    {
        int32_t dim = 1 << env->num_bits;
        if (dim > len)
        {
            vec = SG_REALLOC(float32_t, vec, dim);
            memset(&vec[len], 0, (dim-len) * sizeof(float32_t));
            len = dim;
        }
    }

    inline virtual void expand_if_required(float64_t*& vec, int32_t& len)
    {
        int32_t dim = 1 << env->num_bits;
        if (dim > len)
        {
            vec = SG_REALLOC(float64_t, vec, dim);
            memset(&vec[len], 0, (dim-len) * sizeof(float64_t));
            len = dim;
        }
    }

    virtual int32_t get_dim_feature_space() const;

    inline virtual float32_t real_weight(float32_t w, float32_t gravity)
    {
        float32_t wprime = 0;
        if (gravity < fabsf(w))
            wprime = CMath::sign(w)*(fabsf(w) - gravity);
        return wprime;
    }

    virtual float32_t dot(CStreamingDotFeatures *df);

    virtual float32_t dense_dot(VwExample* &ex, const float32_t* vec2);

    virtual float32_t dense_dot(const float32_t* vec2, int32_t vec2_len);

    virtual float32_t dense_dot(SGSparseVector<float32_t>* vec1, const float32_t* vec2);

    virtual float32_t dense_dot_truncated(const float32_t* vec2, VwExample* &ex, float32_t gravity);

    virtual void add_to_dense_vec(float32_t alpha, VwExample* &ex, float32_t* vec2, int32_t vec2_len, bool abs_val = false);

    virtual void add_to_dense_vec(float32_t alpha, float32_t* vec2, int32_t vec2_len, bool abs_val = false);

    virtual inline int32_t get_nnz_features_for_vector()
    {
        return current_length;
    }

    virtual int32_t get_num_features();

    virtual inline EFeatureType get_feature_type();

    virtual EFeatureClass get_feature_class();

    inline virtual const char* get_name() const { return "StreamingVwFeatures"; }

    inline virtual int32_t get_num_vectors() const
    {
        if (current_example)
            return 1;
        else
            return 0;
    }

    virtual int32_t get_size() { return sizeof(VwExample); }

private:
    virtual void init();

    virtual void init(CStreamingVwFile *file, bool is_labelled, int32_t size);

    virtual void init(CStreamingVwCacheFile *file, bool is_labelled, int32_t size);

    virtual void setup_example(VwExample* ae);

protected:

    CInputParser<VwExample> parser;

    vw_size_t example_count;

    float64_t current_label;

    int32_t current_length;

    CVwEnvironment* env;

    VwExample* current_example;
};
}
#endif // _STREAMING_VWFEATURES__H__