SHOGUN  3.2.1
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
MemoryMappedFile.h
Go to the documentation of this file.
1 /*
2  * This program is free software; you can redistribute it and/or modify
3  * it under the terms of the GNU General Public License as published by
4  * the Free Software Foundation; either version 3 of the License, or
5  * (at your option) any later version.
6  *
7  * Written (W) 2009 Soeren Sonnenburg
8  * Copyright (C) 2009 Fraunhofer Institute FIRST and Max-Planck-Society
9  */
10 
11 #ifndef __MEMORYMAPPEDFILE_H__
12 #define __MEMORYMAPPEDFILE_H__
13 
14 #include <shogun/lib/config.h>
15 
16 #include <shogun/io/SGIO.h>
17 #include <shogun/base/SGObject.h>
18 
19 #include <stdio.h>
20 #include <string.h>
21 #include <sys/mman.h>
22 #include <sys/stat.h>
23 #include <sys/types.h>
24 #include <fcntl.h>
25 #include <unistd.h>
26 
27 namespace shogun
28 {
33 template <class T> class CMemoryMappedFile : public CSGObject
34 {
35  public:
38  {
39  SG_UNSTABLE("CMemoryMappedFile::CMemoryMappedFile()",
40  "\n");
41 
42  fd = 0;
43  length = 0;
44  address = NULL;
45  rw = 'r';
47 
48  set_generic<T>();
49  }
50 
64  CMemoryMappedFile(const char* fname, char flag='r', int64_t fsize=0)
65  : CSGObject()
66  {
67  REQUIRE(flag=='w' || flag=='r', "Only 'r' and 'w' flags are allowed")
68 
70  rw=flag;
71 
72  int open_flags=O_RDONLY;
73  int mmap_prot=PROT_READ;
74  int mmap_flags=MAP_PRIVATE;
75 
76  if (rw=='w')
77  {
78  open_flags=O_RDWR | O_CREAT;
79  mmap_prot=PROT_READ|PROT_WRITE;
80  mmap_flags=MAP_SHARED;
81  }
82 
83  fd = open(fname, open_flags, S_IRWXU | S_IRWXG | S_IRWXO);
84  if (fd == -1)
85  SG_ERROR("Error opening file\n")
86 
87  if (rw=='w' && fsize)
88  {
89  uint8_t byte=0;
90  if (lseek(fd, fsize, SEEK_SET) != fsize || write(fd, &byte, 1) != 1)
91  SG_ERROR("Error creating file of size %ld bytes\n", fsize)
92  }
93 
94  struct stat sb;
95  if (fstat(fd, &sb) == -1)
96  SG_ERROR("Error determining file size\n")
97 
98  length = sb.st_size;
99  address = mmap(NULL, length, mmap_prot, mmap_flags, fd, 0);
100  if (address == MAP_FAILED)
101  SG_ERROR("Error mapping file")
102 
103  set_generic<T>();
104  }
105 
108  {
109  munmap(address, length);
110  if (rw=='w' && last_written_byte && ftruncate(fd, last_written_byte) == -1)
111 
112  {
113  close(fd);
114  SG_ERROR("Error Truncating file to %ld bytes\n", last_written_byte)
115  }
116  close(fd);
117  }
118 
128  inline T* get_map()
129  {
130  return (T*) address;
131  }
132 
137  uint64_t get_length()
138  {
139  return length/sizeof(T);
140  }
141 
146  uint64_t get_size()
147  {
148  return length;
149  }
150 
162  char* get_line(uint64_t& len, uint64_t& offs)
163  {
164  char* s = (char*) address;
165  for (uint64_t i=offs; i<length; i++)
166  {
167  if (s[i] == '\n')
168  {
169  char* line=&s[offs];
170  len=i-offs;
171  offs=i+1;
172  return line;
173  }
174  }
175 
176  len=0;
177  offs=length;
178  return NULL;
179  }
180 
191  void write_line(const char* line, uint64_t len, uint64_t& offs)
192  {
193  char* s = ((char*) address) + offs;
194  if (len+1+offs > length)
195  SG_ERROR("Writing beyond size of file\n")
196 
197  for (uint64_t i=0; i<len; i++)
198  s[i] = line[i];
199 
200  s[len]='\n';
201  offs+=length+1;
202  last_written_byte=offs-1;
203  }
204 
216  inline void set_truncate_size(uint64_t sz=0)
217  {
219  }
220 
225  int32_t get_num_lines()
226  {
227  char* s = (char*) address;
228  int32_t linecount=0;
229  for (uint64_t i=0; i<length; i++)
230  {
231  if (s[i] == '\n')
232  linecount++;
233  }
234 
235  return linecount;
236  }
237 
245  inline T operator[](uint64_t index) const
246  {
247  return ((T*)address)[index];
248  }
249 
257  inline T operator[](int32_t index) const
258  {
259  return ((T*)address)[index];
260  }
261 
263  virtual const char* get_name() const { return "MemoryMappedFile"; }
264 
265  protected:
267  int fd;
269  uint64_t length;
271  void* address;
273  char rw;
274 
277 };
278 }
279 #endif

SHOGUN Machine Learning Toolbox - Documentation