10 #ifndef __PROTOBUFFILE_H__
11 #define __PROTOBUFFILE_H__
22 #define ___APPLE_TYPE_BOOL TYPE_BOOL
28 #ifdef ___APPLE_TYPE_BOOL
29 #define TYPE_BOOL ___APPLE_TYPE_BOOL
30 #undef ___APPLE_TYPE_BOOL
34 #include <shogun/io/protobuf/ShogunVersion.pb.h>
35 #include <shogun/io/protobuf/Headers.pb.h>
36 #include <shogun/io/protobuf/Chunks.pb.h>
72 CProtobufFile(
const char* fname,
char rw=
'r',
const char* name=NULL);
77 #ifndef SWIG // SWIG should skip this
85 virtual void get_vector(int8_t*& vector, int32_t& len);
86 virtual void get_vector(uint8_t*& vector, int32_t& len);
87 virtual void get_vector(
char*& vector, int32_t& len);
88 virtual void get_vector(int32_t*& vector, int32_t& len);
89 virtual void get_vector(uint32_t*& vector, int32_t& len);
93 virtual void get_vector(int16_t*& vector, int32_t& len);
94 virtual void get_vector(uint16_t*& vector, int32_t& len);
95 virtual void get_vector(int64_t*& vector, int32_t& len);
96 virtual void get_vector(uint64_t*& vector, int32_t& len);
108 uint8_t*& matrix, int32_t& num_feat, int32_t& num_vec);
110 int8_t*& matrix, int32_t& num_feat, int32_t& num_vec);
112 char*& matrix, int32_t& num_feat, int32_t& num_vec);
114 int32_t*& matrix, int32_t& num_feat, int32_t& num_vec);
116 uint32_t*& matrix, int32_t& num_feat, int32_t& num_vec);
118 int64_t*& matrix, int32_t& num_feat, int32_t& num_vec);
120 uint64_t*& matrix, int32_t& num_feat, int32_t& num_vec);
122 float32_t*& matrix, int32_t& num_feat, int32_t& num_vec);
124 float64_t*& matrix, int32_t& num_feat, int32_t& num_vec);
126 floatmax_t*& matrix, int32_t& num_feat, int32_t& num_vec);
128 int16_t*& matrix, int32_t& num_feat, int32_t& num_vec);
130 uint16_t*& matrix, int32_t& num_feat, int32_t& num_vec);
142 uint8_t*& array, int32_t*& dims, int32_t& num_dims);
144 char*& array, int32_t*& dims, int32_t& num_dims);
146 int32_t*& array, int32_t*& dims, int32_t& num_dims);
148 float32_t*& array, int32_t*& dims, int32_t& num_dims);
150 float64_t*& array, int32_t*& dims, int32_t& num_dims);
152 int16_t*& array, int32_t*& dims, int32_t& num_dims);
154 uint16_t*& array, int32_t*& dims, int32_t& num_dims);
203 int32_t& max_string_len);
206 int32_t& max_string_len);
209 int32_t& max_string_len);
212 int32_t& max_string_len);
215 int32_t& max_string_len);
218 int32_t& max_string_len);
221 int32_t& max_string_len);
224 int32_t& max_string_len);
227 int32_t& max_string_len);
230 int32_t& max_string_len);
233 int32_t& max_string_len);
236 int32_t& max_string_len);
249 virtual void set_vector(
const int8_t* vector, int32_t len);
250 virtual void set_vector(
const uint8_t* vector, int32_t len);
251 virtual void set_vector(
const char* vector, int32_t len);
252 virtual void set_vector(
const int32_t* vector, int32_t len);
253 virtual void set_vector(
const uint32_t* vector, int32_t len);
257 virtual void set_vector(
const int16_t* vector, int32_t len);
258 virtual void set_vector(
const uint16_t* vector, int32_t len);
259 virtual void set_vector(
const int64_t* vector, int32_t len);
260 virtual void set_vector(
const uint64_t* vector, int32_t len);
271 const uint8_t* matrix, int32_t num_feat, int32_t num_vec);
273 const int8_t* matrix, int32_t num_feat, int32_t num_vec);
275 const char* matrix, int32_t num_feat, int32_t num_vec);
277 const int32_t* matrix, int32_t num_feat, int32_t num_vec);
279 const uint32_t* matrix, int32_t num_feat, int32_t num_vec);
281 const int64_t* matrix, int32_t num_feat, int32_t num_vec);
283 const uint64_t* matrix, int32_t num_feat, int32_t num_vec);
285 const float32_t* matrix, int32_t num_feat, int32_t num_vec);
287 const float64_t* matrix, int32_t num_feat, int32_t num_vec);
289 const floatmax_t* matrix, int32_t num_feat, int32_t num_vec);
291 const int16_t* matrix, int32_t num_feat, int32_t num_vec);
293 const uint16_t* matrix, int32_t num_feat, int32_t num_vec);
364 #endif // #ifndef SWIG // SWIG should skip this
366 virtual const char*
get_name()
const {
return "ProtobufFile"; }
373 void write_big_endian_uint(uint32_t number, uint8_t* array, uint32_t size);
376 uint32_t read_big_endian_uint(uint8_t* array, uint32_t size);
379 int32_t compute_num_messages(uint64_t len, int32_t sizeof_type)
const;
382 void read_and_validate_global_header(ShogunVersion_SGDataType type);
385 void write_global_header(ShogunVersion_SGDataType type);
389 VectorHeader read_vector_header();
390 MatrixHeader read_matrix_header();
391 SparseMatrixHeader read_sparse_matrix_header();
392 StringListHeader read_string_list_header();
397 void write_vector_header(int32_t len, int32_t num_messages);
398 void write_matrix_header(int32_t num_feat, int32_t num_vec, int32_t num_messages);
405 void write_sparse_matrix_header(
407 void write_sparse_matrix_header(
409 void write_sparse_matrix_header(
411 void write_sparse_matrix_header(
413 void write_sparse_matrix_header(
415 void write_sparse_matrix_header(
417 void write_sparse_matrix_header(
419 void write_sparse_matrix_header(
421 void write_sparse_matrix_header(
423 void write_sparse_matrix_header(
425 void write_sparse_matrix_header(
427 void write_sparse_matrix_header(
429 void write_sparse_matrix_header(
437 void write_string_list_header(
439 void write_string_list_header(
441 void write_string_list_header(
443 void write_string_list_header(
445 void write_string_list_header(
447 void write_string_list_header(
449 void write_string_list_header(
451 void write_string_list_header(
453 void write_string_list_header(
455 void write_string_list_header(
457 void write_string_list_header(
459 void write_string_list_header(
464 void read_message(google::protobuf::Message& message);
467 void write_message(
const google::protobuf::Message& message);
474 void read_memory_block(uint8_t*& vector, uint64_t len, int32_t num_messages);
475 void read_memory_block(int8_t*& vector, uint64_t len, int32_t num_messages);
476 void read_memory_block(
char*& vector, uint64_t len, int32_t num_messages);
477 void read_memory_block(int32_t*& vector, uint64_t len, int32_t num_messages);
478 void read_memory_block(uint32_t*& vector, uint64_t len, int32_t num_messages);
479 void read_memory_block(
float64_t*& vector, uint64_t len, int32_t num_messages);
480 void read_memory_block(
float32_t*& vector, uint64_t len, int32_t num_messages);
481 void read_memory_block(
floatmax_t*& vector, uint64_t len, int32_t num_messages);
482 void read_memory_block(int16_t*& vector, uint64_t len, int32_t num_messages);
483 void read_memory_block(uint16_t*& vector, uint64_t len, int32_t num_messages);
484 void read_memory_block(int64_t*& vector, uint64_t len, int32_t num_messages);
485 void read_memory_block(uint64_t*& vector, uint64_t len, int32_t num_messages);
493 void write_memory_block(
const int8_t* vector, uint64_t len, int32_t num_messages);
494 void write_memory_block(
const uint8_t* vector, uint64_t len, int32_t num_messages);
495 void write_memory_block(
const char* vector, uint64_t len, int32_t num_messages);
496 void write_memory_block(
const int32_t* vector, uint64_t len, int32_t num_messages);
497 void write_memory_block(
const uint32_t* vector, uint64_t len, int32_t num_messages);
498 void write_memory_block(
const float32_t* vector, uint64_t len, int32_t num_messages);
499 void write_memory_block(
const float64_t* vector, uint64_t len, int32_t num_messages);
500 void write_memory_block(
const floatmax_t* vector, uint64_t len, int32_t num_messages);
501 void write_memory_block(
const int16_t* vector, uint64_t len, int32_t num_messages);
502 void write_memory_block(
const uint16_t* vector, uint64_t len, int32_t num_messages);
503 void write_memory_block(
const int64_t* vector, uint64_t len, int32_t num_messages);
504 void write_memory_block(
const uint64_t* vector, uint64_t len, int32_t num_messages);
509 const SparseMatrixHeader& data_header);
511 const SparseMatrixHeader& data_header);
513 const SparseMatrixHeader& data_header);
515 const SparseMatrixHeader& data_header);
517 const SparseMatrixHeader& data_header);
519 const SparseMatrixHeader& data_header);
521 const SparseMatrixHeader& data_header);
523 const SparseMatrixHeader& data_header);
525 const SparseMatrixHeader& data_header);
527 const SparseMatrixHeader& data_header);
529 const SparseMatrixHeader& data_header);
531 const SparseMatrixHeader& data_header);
533 const SparseMatrixHeader& data_header);
539 void write_sparse_matrix(
541 void write_sparse_matrix(
543 void write_sparse_matrix(
545 void write_sparse_matrix(
547 void write_sparse_matrix(
549 void write_sparse_matrix(
551 void write_sparse_matrix(
553 void write_sparse_matrix(
555 void write_sparse_matrix(
557 void write_sparse_matrix(
559 void write_sparse_matrix(
561 void write_sparse_matrix(
563 void write_sparse_matrix(
569 const StringListHeader& data_header);
571 const StringListHeader& data_header);
573 const StringListHeader& data_header);
575 const StringListHeader& data_header);
577 const StringListHeader& data_header);
579 const StringListHeader& data_header);
581 const StringListHeader& data_header);
583 const StringListHeader& data_header);
585 const StringListHeader& data_header);
587 const StringListHeader& data_header);
589 const StringListHeader& data_header);
591 const StringListHeader& data_header);
597 void write_string_list(
599 void write_string_list(
601 void write_string_list(
603 void write_string_list(
605 void write_string_list(
607 void write_string_list(
609 void write_string_list(
611 void write_string_list(
613 void write_string_list(
615 void write_string_list(
617 void write_string_list(
619 void write_string_list(
628 int32_t message_size;
634 uint8_t uint_buffer[4];
virtual void set_string_list(const SGString< uint8_t > *strings, int32_t num_str)
virtual void get_string_list(SGString< uint8_t > *&strings, int32_t &num_str, int32_t &max_string_len)
virtual void get_ndarray(uint8_t *&array, int32_t *&dims, int32_t &num_dims)
virtual void get_vector(int8_t *&vector, int32_t &len)
virtual void get_matrix(uint8_t *&matrix, int32_t &num_feat, int32_t &num_vec)
virtual void set_sparse_matrix(const SGSparseVector< bool > *matrix, int32_t num_feat, int32_t num_vec)
A File access base class.
virtual void set_vector(const int8_t *vector, int32_t len)
virtual void get_sparse_matrix(SGSparseVector< bool > *&matrix, int32_t &num_feat, int32_t &num_vec)
all of classes and functions are contained in the shogun namespace
virtual void set_matrix(const uint8_t *matrix, int32_t num_feat, int32_t num_vec)
Class for work with binary file in protobuf format.
template class SGSparseVector The assumtion is that the stored SGSparseVectorEntry* vector is orde...
virtual const char * get_name() const