SHOGUN  4.0.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
ProtobufFile.cpp
Go to the documentation of this file.
1 /*
2  * This program is free software; you can redistribute it and/or modify
3  * it under the terms of the GNU General Public License as published by
4  * the Free Software Foundation; either version 3 of the License, or
5  * (at your option) any later version.
6  *
7  * Written (W) 2013 Evgeniy Andreev (gsomix)
8  */
9 #ifdef HAVE_PROTOBUF
10 
11 #include <shogun/io/ProtobufFile.h>
12 #include <shogun/io/SGIO.h>
13 
14 #include <shogun/lib/SGVector.h>
15 #include <shogun/lib/SGMatrix.h>
17 #include <shogun/lib/SGString.h>
18 #include <google/protobuf/message.h>
19 
20 using namespace shogun;
21 
23 {
24  init();
25 }
26 
27 CProtobufFile::CProtobufFile(FILE* f, const char* name) :
28  CFile(f, name)
29 {
30  init();
31 }
32 
33 CProtobufFile::CProtobufFile(const char* fname, char rw, const char* name) :
34  CFile(fname, rw, name)
35 {
36  init();
37 }
38 
40 {
41  SG_FREE(buffer);
42 }
43 
44 void CProtobufFile::init()
45 {
46  version=1;
47  message_size=1024*1024;
48 
49  buffer=SG_MALLOC(uint8_t, message_size*sizeof(uint32_t));
50 }
51 
52 #define GET_VECTOR(sg_type) \
53 void CProtobufFile::get_vector(sg_type*& vector, int32_t& len) \
54 { \
55  read_and_validate_global_header(ShogunVersion::VECTOR); \
56  VectorHeader data_header=read_vector_header(); \
57  len=data_header.len(); \
58  read_memory_block(vector, len, data_header.num_messages()); \
59 }
60 
61 GET_VECTOR(int8_t)
62 GET_VECTOR(uint8_t)
63 GET_VECTOR(char)
64 GET_VECTOR(int32_t)
65 GET_VECTOR(uint32_t)
69 GET_VECTOR(int16_t)
70 GET_VECTOR(uint16_t)
71 GET_VECTOR(int64_t)
72 GET_VECTOR(uint64_t)
73 #undef GET_VECTOR
74 
75 #define GET_MATRIX(read_func, sg_type) \
76 void CProtobufFile::get_matrix(sg_type*& matrix, int32_t& num_feat, int32_t& num_vec) \
77 { \
78  read_and_validate_global_header(ShogunVersion::MATRIX); \
79  MatrixHeader data_header=read_matrix_header(); \
80  num_feat=data_header.num_cols(); \
81  num_vec=data_header.num_rows(); \
82  read_memory_block(matrix, num_feat*num_vec, data_header.num_messages()); \
83 }
84 
85 GET_MATRIX(read_char, int8_t)
86 GET_MATRIX(read_byte, uint8_t)
87 GET_MATRIX(read_char, char)
88 GET_MATRIX(read_int, int32_t)
89 GET_MATRIX(read_uint, uint32_t)
90 GET_MATRIX(read_short_real, float32_t)
91 GET_MATRIX(read_real, float64_t)
92 GET_MATRIX(read_long_real, floatmax_t)
93 GET_MATRIX(read_short, int16_t)
94 GET_MATRIX(read_word, uint16_t)
95 GET_MATRIX(read_long, int64_t)
96 GET_MATRIX(read_ulong, uint64_t)
97 #undef GET_MATRIX
98 
99 #define GET_NDARRAY(read_func, sg_type) \
100 void CProtobufFile::get_ndarray(sg_type*& array, int32_t*& dims, int32_t& num_dims) \
101 { \
102  SG_NOTIMPLEMENTED \
103 }
104 
105 GET_NDARRAY(read_byte, uint8_t)
106 GET_NDARRAY(read_char, char)
107 GET_NDARRAY(read_int, int32_t)
108 GET_NDARRAY(read_short_real, float32_t)
109 GET_NDARRAY(read_real, float64_t)
110 GET_NDARRAY(read_short, int16_t)
111 GET_NDARRAY(read_word, uint16_t)
112 #undef GET_NDARRAY
113 
114 #define GET_SPARSE_MATRIX(sg_type) \
115 void CProtobufFile::get_sparse_matrix( \
116  SGSparseVector<sg_type>*& matrix, int32_t& num_feat, int32_t& num_vec) \
117 { \
118  read_and_validate_global_header(ShogunVersion::SPARSE_MATRIX); \
119  SparseMatrixHeader data_header=read_sparse_matrix_header(); \
120  num_feat=data_header.num_features(); \
121  num_vec=data_header.num_vectors(); \
122  read_sparse_matrix(matrix, data_header); \
123 }
124 
125 GET_SPARSE_MATRIX(bool)
126 GET_SPARSE_MATRIX(int8_t)
127 GET_SPARSE_MATRIX(uint8_t)
128 GET_SPARSE_MATRIX(char)
129 GET_SPARSE_MATRIX(int32_t)
130 GET_SPARSE_MATRIX(uint32_t)
134 GET_SPARSE_MATRIX(int16_t)
135 GET_SPARSE_MATRIX(uint16_t)
136 GET_SPARSE_MATRIX(int64_t)
137 GET_SPARSE_MATRIX(uint64_t)
138 #undef GET_SPARSE_MATRIX
139 
140 #define SET_VECTOR(sg_type) \
141 void CProtobufFile::set_vector(const sg_type* vector, int32_t len) \
142 { \
143  int32_t num_messages=compute_num_messages(len, sizeof(sg_type)); \
144  write_global_header(ShogunVersion::VECTOR); \
145  write_vector_header(len, num_messages); \
146  write_memory_block(vector, len, num_messages); \
147 }
148 
149 SET_VECTOR(int8_t)
150 SET_VECTOR(uint8_t)
151 SET_VECTOR(char)
152 SET_VECTOR(int32_t)
153 SET_VECTOR(uint32_t)
154 SET_VECTOR(int64_t)
155 SET_VECTOR(uint64_t)
159 SET_VECTOR(int16_t)
160 SET_VECTOR(uint16_t)
161 #undef SET_VECTOR
162 
163 #define SET_MATRIX(sg_type) \
164 void CProtobufFile::set_matrix(const sg_type* matrix, int32_t num_feat, int32_t num_vec) \
165 { \
166  int32_t num_messages=compute_num_messages(num_feat*num_vec, sizeof(sg_type)); \
167  write_global_header(ShogunVersion::MATRIX); \
168  write_matrix_header(num_feat, num_vec, num_messages); \
169  write_memory_block(matrix, num_feat*num_vec, num_messages); \
170 }
171 
172 SET_MATRIX(int8_t)
173 SET_MATRIX(uint8_t)
174 SET_MATRIX(char)
175 SET_MATRIX(int32_t)
176 SET_MATRIX(uint32_t)
177 SET_MATRIX(int64_t)
178 SET_MATRIX(uint64_t)
182 SET_MATRIX(int16_t)
183 SET_MATRIX(uint16_t)
184 #undef SET_MATRIX
185 
186 #define SET_SPARSE_MATRIX(sg_type) \
187 void CProtobufFile::set_sparse_matrix( \
188  const SGSparseVector<sg_type>* matrix, int32_t num_feat, int32_t num_vec) \
189 { \
190  write_global_header(ShogunVersion::SPARSE_MATRIX); \
191  write_sparse_matrix_header(matrix, num_feat, num_vec); \
192  write_sparse_matrix(matrix, num_vec); \
193 }
194 
195 SET_SPARSE_MATRIX(bool)
196 SET_SPARSE_MATRIX(int8_t)
197 SET_SPARSE_MATRIX(uint8_t)
198 SET_SPARSE_MATRIX(char)
199 SET_SPARSE_MATRIX(int32_t)
200 SET_SPARSE_MATRIX(uint32_t)
201 SET_SPARSE_MATRIX(int64_t)
202 SET_SPARSE_MATRIX(uint64_t)
206 SET_SPARSE_MATRIX(int16_t)
207 SET_SPARSE_MATRIX(uint16_t)
208 #undef SET_SPARSE_MATRIX
209 
210 #define GET_STRING_LIST(sg_type) \
211 void CProtobufFile::get_string_list( \
212  SGString<sg_type>*& strings, int32_t& num_str, \
213  int32_t& max_string_len) \
214 { \
215  read_and_validate_global_header(ShogunVersion::STRING_LIST); \
216  StringListHeader data_header=read_string_list_header(); \
217  num_str=data_header.num_str(); \
218  max_string_len=data_header.max_string_len(); \
219  read_string_list(strings, data_header); \
220 }
221 
222 GET_STRING_LIST(int8_t)
223 GET_STRING_LIST(uint8_t)
224 GET_STRING_LIST(char)
225 GET_STRING_LIST(int32_t)
226 GET_STRING_LIST(uint32_t)
227 GET_STRING_LIST(int64_t)
228 GET_STRING_LIST(uint64_t)
232 GET_STRING_LIST(int16_t)
233 GET_STRING_LIST(uint16_t)
234 #undef GET_STRING_LIST
235 
236 #define SET_STRING_LIST(sg_type) \
237 void CProtobufFile::set_string_list( \
238  const SGString<sg_type>* strings, int32_t num_str) \
239 { \
240  write_global_header(ShogunVersion::STRING_LIST); \
241  write_string_list_header(strings, num_str); \
242  write_string_list(strings, num_str); \
243 }
244 
245 SET_STRING_LIST(int8_t)
246 SET_STRING_LIST(uint8_t)
247 SET_STRING_LIST(char)
248 SET_STRING_LIST(int32_t)
249 SET_STRING_LIST(uint32_t)
250 SET_STRING_LIST(int64_t)
251 SET_STRING_LIST(uint64_t)
255 SET_STRING_LIST(int16_t)
256 SET_STRING_LIST(uint16_t)
257 #undef SET_STRING_LIST
258 
259 void CProtobufFile::write_big_endian_uint(uint32_t number, uint8_t* array, uint32_t size)
260 {
261  if (size<4)
262  SG_ERROR("array is too small to write\n");
263 
264  array[0]=(number>>24)&0xffu;
265  array[1]=(number>>16)&0xffu;
266  array[2]=(number>>8)&0xffu;
267  array[3]=number&0xffu;
268 }
269 
270 uint32_t CProtobufFile::read_big_endian_uint(uint8_t* array, uint32_t size)
271 {
272  if (size<4)
273  SG_ERROR("array is too small to read\n");
274 
275  return (array[0]<<24) | (array[1]<<16) | (array[2]<<8) | array[3];
276 }
277 
278 int32_t CProtobufFile::compute_num_messages(uint64_t len, int32_t sizeof_type) const
279 {
280  uint32_t elements_in_message=message_size/sizeof_type;
281  uint32_t num_messages=len/elements_in_message;
282  if (len % elements_in_message > 0)
283  num_messages++;
284 
285  return num_messages;
286 }
287 
288 void CProtobufFile::read_and_validate_global_header(ShogunVersion_SGDataType type)
289 {
290  ShogunVersion header;
291  read_message(header);
292  REQUIRE(header.version()==version, "wrong version\n")
293  REQUIRE(header.data_type()==type, "wrong type\n")
294 }
295 
296 void CProtobufFile::write_global_header(ShogunVersion_SGDataType type)
297 {
298  ShogunVersion header;
299  header.set_version(version);
300  header.set_data_type(type);
301  write_message(header);
302 }
303 
304 VectorHeader CProtobufFile::read_vector_header()
305 {
306  VectorHeader data_header;
307  read_message(data_header);
308 
309  return data_header;
310 }
311 
312 SparseMatrixHeader CProtobufFile::read_sparse_matrix_header()
313 {
314  SparseMatrixHeader data_header;
315  read_message(data_header);
316 
317  return data_header;
318 }
319 
320 MatrixHeader CProtobufFile::read_matrix_header()
321 {
322  MatrixHeader data_header;
323  read_message(data_header);
324 
325  return data_header;
326 }
327 
328 StringListHeader CProtobufFile::read_string_list_header()
329 {
330  StringListHeader data_header;
331  read_message(data_header);
332 
333  return data_header;
334 }
335 
336 void CProtobufFile::write_vector_header(int32_t len, int32_t num_messages)
337 {
338  VectorHeader data_header;
339  data_header.set_len(len);
340  data_header.set_num_messages(num_messages);
341  write_message(data_header);
342 }
343 
344 void CProtobufFile::write_matrix_header(int32_t num_feat, int32_t num_vec, int32_t num_messages)
345 {
346  MatrixHeader data_header;
347  data_header.set_num_cols(num_feat);
348  data_header.set_num_rows(num_vec);
349  data_header.set_num_messages(num_messages);
350  write_message(data_header);
351 }
352 
353 #define WRITE_SPARSE_MATRIX_HEADER(sg_type) \
354 void CProtobufFile::write_sparse_matrix_header( \
355  const SGSparseVector<sg_type>* matrix, int32_t num_feat, int32_t num_vec) \
356 { \
357  SparseMatrixHeader data_header; \
358  data_header.set_num_features(num_feat); \
359  data_header.set_num_vectors(num_vec); \
360  for (int32_t i=0; i<num_vec; i++) \
361  { \
362  data_header.add_num_feat_entries(matrix[i].num_feat_entries); \
363  } \
364  \
365  write_message(data_header); \
366 }
367 
381 #undef WRITE_SPARSE_MATRIX_HEADER
382 
383 #define WRITE_STRING_LIST_HEADER(sg_type) \
384 void CProtobufFile::write_string_list_header(const SGString<sg_type>* strings, int32_t num_str) \
385 { \
386  int32_t max_string_len=0; \
387  StringListHeader data_header; \
388  data_header.set_num_str(num_str); \
389  for (int32_t i=0; i<num_str; i++) \
390  { \
391  data_header.add_str_len(strings[i].slen); \
392  if (strings[i].slen>max_string_len) \
393  max_string_len=strings[i].slen; \
394  } \
395  data_header.set_max_string_len(max_string_len); \
396  write_message(data_header); \
397 }
398 
403 WRITE_STRING_LIST_HEADER(uint32_t)
405 WRITE_STRING_LIST_HEADER(uint64_t)
410 WRITE_STRING_LIST_HEADER(uint16_t)
411 #undef WRITE_STRING_LIST_HEADER
412 
413 void CProtobufFile::read_message(google::protobuf::Message& message)
414 {
415  uint32_t bytes_read=0;
416  uint32_t msg_size=0;
417 
418  // read size of message
419  bytes_read=fread(uint_buffer, sizeof(char), sizeof(uint32_t), file);
420  REQUIRE(bytes_read==sizeof(uint32_t), "IO error\n");
421  msg_size=read_big_endian_uint(uint_buffer, sizeof(uint32_t));
422  REQUIRE(msg_size>0, "message size should be more than zero\n");
423 
424  // read message
425  bytes_read=fread(buffer, sizeof(char), msg_size, file);
426  REQUIRE(bytes_read==msg_size, "IO error\n");
427 
428  // try to parse message from read data
429  REQUIRE(message.ParseFromArray(buffer, msg_size), "cannot parse header\n");
430 }
431 
432 void CProtobufFile::write_message(const google::protobuf::Message& message)
433 {
434  uint32_t bytes_write=0;
435  uint32_t msg_size=message.ByteSize();
436 
437  // write size of message
438  write_big_endian_uint(msg_size, uint_buffer, sizeof(uint32_t));
439  bytes_write=fwrite(uint_buffer, sizeof(char), sizeof(uint32_t), file);
440  REQUIRE(bytes_write==sizeof(uint32_t), "IO error\n");
441 
442  // write serialized message
443  message.SerializeToArray(buffer, msg_size);
444  bytes_write=fwrite(buffer, sizeof(char), msg_size, file);
445  REQUIRE(bytes_write==msg_size, "IO error\n");
446 }
447 
448 #define READ_MEMORY_BLOCK(chunk_type, sg_type) \
449 void CProtobufFile::read_memory_block(sg_type*& vector, uint64_t len, int32_t num_messages) \
450 { \
451  vector=SG_MALLOC(sg_type, len); \
452  \
453  chunk_type chunk; \
454  int32_t elements_in_message=message_size/sizeof(sg_type); \
455  for (int32_t i=0; i<num_messages; i++) \
456  { \
457  read_message(chunk); \
458  \
459  int32_t num_elements_to_read=0; \
460  if ((len-(i+1)*elements_in_message)<=0) \
461  num_elements_to_read=len-i*elements_in_message; \
462  else \
463  num_elements_to_read=elements_in_message; \
464  \
465  for (int32_t j=0; j<num_elements_to_read; j++) \
466  vector[j+i*elements_in_message]=chunk.data(j); \
467  } \
468 }
469 
470 READ_MEMORY_BLOCK(Int32Chunk, int8_t)
471 READ_MEMORY_BLOCK(UInt32Chunk, uint8_t)
472 READ_MEMORY_BLOCK(UInt32Chunk, char)
473 READ_MEMORY_BLOCK(Int32Chunk, int32_t)
474 READ_MEMORY_BLOCK(UInt32Chunk, uint32_t)
475 READ_MEMORY_BLOCK(Float32Chunk, float32_t)
476 READ_MEMORY_BLOCK(Float64Chunk, float64_t)
477 READ_MEMORY_BLOCK(Float64Chunk, floatmax_t)
478 READ_MEMORY_BLOCK(Int32Chunk, int16_t)
479 READ_MEMORY_BLOCK(UInt32Chunk, uint16_t)
480 READ_MEMORY_BLOCK(Int64Chunk, int64_t)
481 READ_MEMORY_BLOCK(UInt64Chunk, uint64_t)
482 #undef READ_MEMORY_BLOCK
483 
484 #define WRITE_MEMORY_BLOCK(chunk_type, sg_type) \
485 void CProtobufFile::write_memory_block(const sg_type* vector, uint64_t len, int32_t num_messages) \
486 { \
487  chunk_type chunk; \
488  int32_t elements_in_message=message_size/sizeof(sg_type); \
489  for (int32_t i=0; i<num_messages; i++) \
490  { \
491  \
492  int32_t num_elements_to_write=0; \
493  if ((len-(i+1)*elements_in_message)<=0) \
494  num_elements_to_write=len-i*elements_in_message; \
495  else \
496  num_elements_to_write=elements_in_message; \
497  \
498  for (int32_t j=0; j<num_elements_to_write; j++) \
499  chunk.add_data(vector[j+i*elements_in_message]); \
500  \
501  write_message(chunk); \
502  chunk.Clear(); \
503  } \
504 }
505 
506 WRITE_MEMORY_BLOCK(Int32Chunk, int8_t)
507 WRITE_MEMORY_BLOCK(UInt32Chunk, uint8_t)
508 WRITE_MEMORY_BLOCK(UInt32Chunk, char)
509 WRITE_MEMORY_BLOCK(Int32Chunk, int32_t)
510 WRITE_MEMORY_BLOCK(UInt64Chunk, uint32_t)
511 WRITE_MEMORY_BLOCK(Int64Chunk, int64_t)
512 WRITE_MEMORY_BLOCK(UInt64Chunk, uint64_t)
513 WRITE_MEMORY_BLOCK(Float32Chunk, float32_t)
514 WRITE_MEMORY_BLOCK(Float64Chunk, float64_t)
515 WRITE_MEMORY_BLOCK(Float64Chunk, floatmax_t)
516 WRITE_MEMORY_BLOCK(Int32Chunk, int16_t)
517 WRITE_MEMORY_BLOCK(UInt32Chunk, uint16_t)
518 #undef WRITE_MEMORY_BLOCK
519 
520 #define READ_SPARSE_MATRIX(chunk_type, sg_type) \
521 void CProtobufFile::read_sparse_matrix( \
522  SGSparseVector<sg_type>*& matrix, const SparseMatrixHeader& data_header) \
523 { \
524  matrix=SG_MALLOC(SGSparseVector<sg_type>, data_header.num_vectors()); \
525  \
526  UInt64Chunk feat_index_chunk; \
527  chunk_type entry_chunk; \
528  read_message(feat_index_chunk); \
529  read_message(entry_chunk); \
530  \
531  int32_t elements_in_message=message_size/sizeof(sg_type); \
532  int32_t buffer_counter=0; \
533  for (uint32_t i=0; i<data_header.num_vectors(); i++) \
534  { \
535  matrix[i]=SGSparseVector<sg_type>(data_header.num_feat_entries(i)); \
536  for (int32_t j=0; j<matrix[i].num_feat_entries; j++) \
537  { \
538  matrix[i].features[j].feat_index=feat_index_chunk.data(buffer_counter); \
539  matrix[i].features[j].entry=entry_chunk.data(buffer_counter); \
540  buffer_counter++; \
541  \
542  if (buffer_counter==elements_in_message) \
543  { \
544  read_message(feat_index_chunk); \
545  read_message(entry_chunk); \
546  buffer_counter=0; \
547  } \
548  } \
549  } \
550 }
551 
552 READ_SPARSE_MATRIX(BoolChunk, bool)
553 READ_SPARSE_MATRIX(Int32Chunk, int8_t)
554 READ_SPARSE_MATRIX(UInt32Chunk, uint8_t)
555 READ_SPARSE_MATRIX(UInt32Chunk, char)
556 READ_SPARSE_MATRIX(Int32Chunk, int32_t)
557 READ_SPARSE_MATRIX(UInt32Chunk, uint32_t)
558 READ_SPARSE_MATRIX(Float32Chunk, float32_t)
559 READ_SPARSE_MATRIX(Float64Chunk, float64_t)
560 READ_SPARSE_MATRIX(Float64Chunk, floatmax_t)
561 READ_SPARSE_MATRIX(Int32Chunk, int16_t)
562 READ_SPARSE_MATRIX(UInt32Chunk, uint16_t)
563 READ_SPARSE_MATRIX(Int64Chunk, int64_t)
564 READ_SPARSE_MATRIX(UInt64Chunk, uint64_t)
565 #undef READ_SPARSE_MATRIX
566 
567 #define WRITE_SPARSE_MATRIX(chunk_type, sg_type) \
568 void CProtobufFile::write_sparse_matrix( \
569  const SGSparseVector<sg_type>* matrix, int32_t num_vec) \
570 { \
571  UInt64Chunk feat_index_chunk; \
572  chunk_type entry_chunk; \
573  int32_t elements_in_message=message_size/sizeof(sg_type); \
574  int32_t buffer_counter=0; \
575  for (int32_t i=0; i<num_vec; i++) \
576  { \
577  for (int32_t j=0; j<matrix[i].num_feat_entries; j++) \
578  { \
579  feat_index_chunk.add_data(matrix[i].features[j].feat_index); \
580  entry_chunk.add_data(matrix[i].features[j].entry); \
581  buffer_counter++; \
582  \
583  if (buffer_counter==elements_in_message) \
584  { \
585  write_message(feat_index_chunk); \
586  write_message(entry_chunk); \
587  feat_index_chunk.Clear(); \
588  entry_chunk.Clear(); \
589  buffer_counter=0; \
590  } \
591  } \
592  } \
593  \
594  if (buffer_counter!=0) \
595  { \
596  write_message(feat_index_chunk); \
597  write_message(entry_chunk); \
598  } \
599 }
600 
601 WRITE_SPARSE_MATRIX(BoolChunk, bool)
602 WRITE_SPARSE_MATRIX(Int32Chunk, int8_t)
603 WRITE_SPARSE_MATRIX(UInt32Chunk, uint8_t)
604 WRITE_SPARSE_MATRIX(UInt32Chunk, char)
605 WRITE_SPARSE_MATRIX(Int32Chunk, int32_t)
606 WRITE_SPARSE_MATRIX(UInt64Chunk, uint32_t)
607 WRITE_SPARSE_MATRIX(Int64Chunk, int64_t)
608 WRITE_SPARSE_MATRIX(UInt64Chunk, uint64_t)
609 WRITE_SPARSE_MATRIX(Float32Chunk, float32_t)
610 WRITE_SPARSE_MATRIX(Float64Chunk, float64_t)
611 WRITE_SPARSE_MATRIX(Float64Chunk, floatmax_t)
612 WRITE_SPARSE_MATRIX(Int32Chunk, int16_t)
613 WRITE_SPARSE_MATRIX(UInt32Chunk, uint16_t)
614 #undef WRITE_SPARSE_MATRIX
615 
616 #define READ_STRING_LIST(chunk_type, sg_type) \
617 void CProtobufFile::read_string_list( \
618  SGString<sg_type>*& strings, const StringListHeader& data_header) \
619 { \
620  strings=SG_MALLOC(SGString<sg_type>, data_header.num_str()); \
621  \
622  chunk_type chunk; \
623  read_message(chunk); \
624  int32_t elements_in_message=message_size/sizeof(sg_type); \
625  int32_t buffer_counter=0; \
626  for (uint32_t i=0; i<data_header.num_str(); i++) \
627  { \
628  strings[i]=SGString<sg_type>(data_header.str_len(i)); \
629  for (int32_t j=0; j<strings[i].slen; j++) \
630  { \
631  strings[i].string[j]=chunk.data(buffer_counter); \
632  buffer_counter++; \
633  \
634  if (buffer_counter==elements_in_message) \
635  { \
636  read_message(chunk); \
637  buffer_counter=0; \
638  } \
639  } \
640  } \
641 }
642 
643 READ_STRING_LIST(Int32Chunk, int8_t)
644 READ_STRING_LIST(UInt32Chunk, uint8_t)
645 READ_STRING_LIST(UInt32Chunk, char)
646 READ_STRING_LIST(Int32Chunk, int32_t)
647 READ_STRING_LIST(UInt32Chunk, uint32_t)
648 READ_STRING_LIST(Float32Chunk, float32_t)
649 READ_STRING_LIST(Float64Chunk, float64_t)
650 READ_STRING_LIST(Float64Chunk, floatmax_t)
651 READ_STRING_LIST(Int32Chunk, int16_t)
652 READ_STRING_LIST(UInt32Chunk, uint16_t)
653 READ_STRING_LIST(Int64Chunk, int64_t)
654 READ_STRING_LIST(UInt64Chunk, uint64_t)
655 #undef READ_STRING_LIST
656 
657 #define WRITE_STRING_LIST(chunk_type, sg_type) \
658 void CProtobufFile::write_string_list( \
659  const SGString<sg_type>* strings, int32_t num_str) \
660 { \
661  chunk_type chunk; \
662  int32_t elements_in_message=message_size/sizeof(sg_type); \
663  int32_t buffer_counter=0; \
664  for (int32_t i=0; i<num_str; i++) \
665  { \
666  for (int32_t j=0; j<strings[i].slen; j++) \
667  { \
668  chunk.add_data(strings[i].string[j]); \
669  buffer_counter++; \
670  \
671  if (buffer_counter==elements_in_message) \
672  { \
673  write_message(chunk); \
674  chunk.Clear(); \
675  buffer_counter=0; \
676  } \
677  } \
678  } \
679  \
680  if (buffer_counter!=0) \
681  write_message(chunk); \
682 }
683 
684 WRITE_STRING_LIST(Int32Chunk, int8_t)
685 WRITE_STRING_LIST(UInt32Chunk, uint8_t)
686 WRITE_STRING_LIST(UInt32Chunk, char)
687 WRITE_STRING_LIST(Int32Chunk, int32_t)
688 WRITE_STRING_LIST(UInt64Chunk, uint32_t)
689 WRITE_STRING_LIST(Int64Chunk, int64_t)
690 WRITE_STRING_LIST(UInt64Chunk, uint64_t)
691 WRITE_STRING_LIST(Float32Chunk, float32_t)
692 WRITE_STRING_LIST(Float64Chunk, float64_t)
693 WRITE_STRING_LIST(Float64Chunk, floatmax_t)
694 WRITE_STRING_LIST(Int32Chunk, int16_t)
695 WRITE_STRING_LIST(UInt32Chunk, uint16_t)
696 #undef WRITE_STRING_LIST
697 
698 #endif /* HAVE_PROTOBUF */

SHOGUN Machine Learning Toolbox - Documentation