IOR
aiori-IME.c
Go to the documentation of this file.
1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
2  * vim:expandtab:shiftwidth=8:tabstop=8:
3  */
4 /******************************************************************************\
5 * *
6 * Copyright (c) 2003, The Regents of the University of California. *
7 * Copyright (c) 2018, DataDirect Networks. *
8 * See the file COPYRIGHT for a complete copyright notice and license. *
9 * *
10 ********************************************************************************
11 *
12 * Implement abstract I/O interface for DDN Infinite Memory Engine (IME).
13 *
14 \******************************************************************************/
15 
16 #ifdef HAVE_CONFIG_H
17 #include "config.h"
18 #endif
19 
20 #include <assert.h>
21 #include <stdio.h>
22 #include <stdlib.h>
23 #include <sys/stat.h>
24 #include <errno.h> /* sys_errlist */
25 #include <fcntl.h> /* IO operations */
26 
27 #include "ior.h"
28 #include "iordef.h"
29 #include "aiori.h"
30 #include "utilities.h"
31 #include "ime_native.h"
32 
33 #ifndef O_BINARY /* Required on Windows */
34 # define O_BINARY 0
35 #endif
36 
37 /**************************** P R O T O T Y P E S *****************************/
38 
39 static void *IME_Create(char *, IOR_param_t *);
40 static void *IME_Open(char *, IOR_param_t *);
41 static void IME_Close(void *, IOR_param_t *);
42 static void IME_Delete(char *, IOR_param_t *);
43 static char *IME_GetVersion();
44 static void IME_Fsync(void *, IOR_param_t *);
45 static int IME_Access(const char *, int, IOR_param_t *);
46 static IOR_offset_t IME_GetFileSize(IOR_param_t *, MPI_Comm, char *);
47 static IOR_offset_t IME_Xfer(int, void *, IOR_size_t *,
49 static int IME_StatFS(const char *, ior_aiori_statfs_t *,
50  IOR_param_t *);
51 static int IME_RmDir(const char *, IOR_param_t *);
52 static int IME_MkDir(const char *, mode_t, IOR_param_t *);
53 static int IME_Stat(const char *, struct stat *, IOR_param_t *);
54 static void IME_Initialize();
55 static void IME_Finalize();
56 
57 /************************** D E C L A R A T I O N S ***************************/
58 
59 extern int rank;
60 extern int rankOffset;
61 extern int verbose;
62 extern MPI_Comm testComm;
63 
65  .name = "IME",
66  .create = IME_Create,
67  .open = IME_Open,
68  .xfer = IME_Xfer,
69  .close = IME_Close,
70  .delete = IME_Delete,
71  .get_version = IME_GetVersion,
72  .fsync = IME_Fsync,
73  .get_file_size = IME_GetFileSize,
74  .access = IME_Access,
75  .statfs = IME_StatFS,
76  .rmdir = IME_RmDir,
77  .mkdir = IME_MkDir,
78  .stat = IME_Stat,
79  .initialize = IME_Initialize,
80  .finalize = IME_Finalize,
81 };
82 
83 /***************************** F U N C T I O N S ******************************/
84 
85 /*
86  * Initialize IME (before MPI is started).
87  */
88 static void IME_Initialize()
89 {
90  ime_native_init();
91 }
92 
93 /*
94  * Finlize IME (after MPI is shutdown).
95  */
96 static void IME_Finalize()
97 {
98  (void)ime_native_finalize();
99 }
100 
101 /*
102  * Try to access a file through the IME interface.
103  */
104 static int IME_Access(const char *path, int mode, IOR_param_t *param)
105 {
106  (void)param;
107 
108  return ime_native_access(path, mode);
109 }
110 
111 /*
112  * Creat and open a file through the IME interface.
113  */
114 static void *IME_Create(char *testFileName, IOR_param_t *param)
115 {
116  return IME_Open(testFileName, param);
117 }
118 
119 /*
120  * Open a file through the IME interface.
121  */
122 static void *IME_Open(char *testFileName, IOR_param_t *param)
123 {
124  int fd_oflag = O_BINARY;
125  int *fd;
126 
127  fd = (int *)malloc(sizeof(int));
128  if (fd == NULL)
129  ERR("Unable to malloc file descriptor");
130 
131  if (param->useO_DIRECT)
132  set_o_direct_flag(&fd_oflag);
133 
134  if (param->openFlags & IOR_RDONLY)
135  fd_oflag |= O_RDONLY;
136  if (param->openFlags & IOR_WRONLY)
137  fd_oflag |= O_WRONLY;
138  if (param->openFlags & IOR_RDWR)
139  fd_oflag |= O_RDWR;
140  if (param->openFlags & IOR_APPEND)
141  fd_oflag |= O_APPEND;
142  if (param->openFlags & IOR_CREAT)
143  fd_oflag |= O_CREAT;
144  if (param->openFlags & IOR_EXCL)
145  fd_oflag |= O_EXCL;
146  if (param->openFlags & IOR_TRUNC)
147  fd_oflag |= O_TRUNC;
148 
149  *fd = ime_native_open(testFileName, fd_oflag, 0664);
150  if (*fd < 0) {
151  free(fd);
152  ERR("cannot open file");
153  }
154 
155  return((void *)fd);
156 }
157 
158 /*
159  * Write or read access to file using the IM interface.
160  */
161 static IOR_offset_t IME_Xfer(int access, void *file, IOR_size_t *buffer,
162  IOR_offset_t length, IOR_param_t *param)
163 {
164  int xferRetries = 0;
165  long long remaining = (long long)length;
166  char *ptr = (char *)buffer;
167  int fd = *(int *)file;
168  long long rc;
169 
170  while (remaining > 0) {
171  /* write/read file */
172  if (access == WRITE) { /* WRITE */
173  if (verbose >= VERBOSE_4) {
174  fprintf(stdout, "task %d writing to offset %lld\n",
175  rank, param->offset + length - remaining);
176  }
177 
178  rc = ime_native_pwrite(fd, ptr, remaining, param->offset);
179 
180  if (param->fsyncPerWrite)
181  IME_Fsync(&fd, param);
182  } else { /* READ or CHECK */
183  if (verbose >= VERBOSE_4) {
184  fprintf(stdout, "task %d reading from offset %lld\n",
185  rank, param->offset + length - remaining);
186  }
187 
188  rc = ime_native_pread(fd, ptr, remaining, param->offset);
189  if (rc == 0)
190  ERR("hit EOF prematurely");
191  else if (rc < 0)
192  ERR("read failed");
193  }
194 
195  if (rc < remaining) {
196  fprintf(stdout, "WARNING: Task %d, partial %s, %lld of "
197  "%lld bytes at offset %lld\n",
198  rank, access == WRITE ? "write" : "read", rc,
199  remaining, param->offset + length - remaining );
200 
201  if (param->singleXferAttempt) {
202  MPI_CHECK(MPI_Abort(MPI_COMM_WORLD, -1),
203  "barrier error");
204  }
205 
206  if (xferRetries > MAX_RETRY) {
207  ERR( "too many retries -- aborting" );
208  }
209  } else if (rc > remaining) /* this should never happen */
210  ERR("too many bytes transferred!?!");
211 
212  assert(rc >= 0);
213  assert(rc <= remaining);
214  remaining -= rc;
215  ptr += rc;
216  xferRetries++;
217  }
218 
219  return(length);
220 }
221 
222 /*
223  * Perform fsync().
224  */
225 static void IME_Fsync(void *fd, IOR_param_t *param)
226 {
227  if (ime_native_fsync(*(int *)fd) != 0)
228  WARN("cannot perform fsync on file");
229 }
230 
231 /*
232  * Close a file through the IME interface.
233  */
234 static void IME_Close(void *fd, IOR_param_t *param)
235 {
236  if (ime_native_close(*(int *)fd) != 0)
237  {
238  free(fd);
239  ERR("cannot close file");
240  }
241  else
242  free(fd);
243 }
244 
245 /*
246  * Delete a file through the IME interface.
247  */
248 static void IME_Delete(char *testFileName, IOR_param_t *param)
249 {
250  char errmsg[256];
251  sprintf(errmsg, "[RANK %03d]:cannot delete file %s\n",
252  rank, testFileName);
253  if (ime_native_unlink(testFileName) != 0)
254  WARN(errmsg);
255 }
256 
257 /*
258  * Determine API version.
259  */
260 static char *IME_GetVersion()
261 {
262  static char ver[1024] = {};
263 #if (IME_NATIVE_API_VERSION >= 120)
264  strcpy(ver, ime_native_version());
265 #else
266  strcpy(ver, "not supported");
267 #endif
268  return ver;
269 }
270 
271 /*
272  * XXX: statfs call is currently not exposed by IME native interface.
273  */
274 static int IME_StatFS(const char *oid, ior_aiori_statfs_t *stat_buf,
276 {
277  (void)oid;
278  (void)stat_buf;
279  (void)param;
280 
281  WARN("statfs is currently not supported in IME backend!");
282  return -1;
283 }
284 
285 /*
286  * XXX: mkdir call is currently not exposed by IME native interface.
287  */
288 static int IME_MkDir(const char *oid, mode_t mode, IOR_param_t *param)
289 {
290  (void)oid;
291  (void)mode;
292  (void)param;
293 
294  WARN("mkdir is currently not supported in IME backend!");
295  return -1;
296 }
297 
298 /*
299  * XXX: rmdir call is curretly not exposed by IME native interface.
300  */
301 static int IME_RmDir(const char *oid, IOR_param_t *param)
302 {
303  (void)oid;
304  (void)param;
305 
306  WARN("rmdir is currently not supported in IME backend!");
307  return -1;
308 }
309 
310 /*
311  * Perform stat() through the IME interface.
312  */
313 static int IME_Stat(const char *path, struct stat *buf, IOR_param_t *param)
314 {
315  (void)param;
316 
317  return ime_native_stat(path, buf);
318 }
319 
320 /*
321  * Use IME stat() to return aggregate file size.
322  */
324  char *testFileName)
325 {
326  struct stat stat_buf;
327  IOR_offset_t aggFileSizeFromStat, tmpMin, tmpMax, tmpSum;
328 
329  if (ime_native_stat(testFileName, &stat_buf) != 0) {
330  ERR("cannot get status of written file");
331  }
332  aggFileSizeFromStat = stat_buf.st_size;
333 
334  if (test->filePerProc) {
335  MPI_CHECK(MPI_Allreduce(&aggFileSizeFromStat, &tmpSum, 1,
336  MPI_LONG_LONG_INT, MPI_SUM, testComm),
337  "cannot total data moved");
338  aggFileSizeFromStat = tmpSum;
339  } else {
340  MPI_CHECK(MPI_Allreduce(&aggFileSizeFromStat, &tmpMin, 1,
341  MPI_LONG_LONG_INT, MPI_MIN, testComm),
342  "cannot total data moved");
343  MPI_CHECK(MPI_Allreduce(&aggFileSizeFromStat, &tmpMax, 1,
344  MPI_LONG_LONG_INT, MPI_MAX, testComm),
345  "cannot total data moved");
346 
347  if (tmpMin != tmpMax) {
348  if (rank == 0) {
349  WARN("inconsistent file size by different tasks");
350  }
351  /* incorrect, but now consistent across tasks */
352  aggFileSizeFromStat = tmpMin;
353  }
354  }
355 
356  return(aggFileSizeFromStat);
357 }
static void * IME_Create(char *, IOR_param_t *)
Definition: aiori-IME.c:114
static void * IME_Open(char *, IOR_param_t *)
Definition: aiori-IME.c:122
#define O_BINARY
Definition: aiori-IME.c:34
void set_o_direct_flag(int *fd)
Definition: utilities.c:73
static void IME_Initialize()
Definition: aiori-IME.c:88
#define ERR(MSG)
Definition: iordef.h:169
int filePerProc
Definition: ior.h:104
static void IME_Fsync(void *, IOR_param_t *)
Definition: aiori-IME.c:225
CURLcode rc
Definition: aiori-S3.c:121
static int IME_RmDir(const char *, IOR_param_t *)
Definition: aiori-IME.c:301
static void IME_Finalize()
Definition: aiori-IME.c:96
static int IME_Stat(const char *, struct stat *, IOR_param_t *)
Definition: aiori-IME.c:313
#define IOR_APPEND
Definition: aiori.h:36
static IOR_offset_t IME_Xfer(int, void *, IOR_size_t *, IOR_offset_t, IOR_param_t *)
Definition: aiori-IME.c:161
#define IOR_RDONLY
Definition: aiori.h:33
unsigned int openFlags
Definition: ior.h:85
int fsyncPerWrite
Definition: ior.h:152
#define WRITE
Definition: iordef.h:95
#define IOR_CREAT
Definition: aiori.h:37
#define IOR_EXCL
Definition: aiori.h:39
static int IME_Access(const char *, int, IOR_param_t *)
Definition: aiori-IME.c:104
static void IME_Close(void *, IOR_param_t *)
Definition: aiori-IME.c:234
MPI_Comm testComm
Definition: utilities.c:61
#define IOR_TRUNC
Definition: aiori.h:38
#define MPI_CHECK(MPI_STATUS, MSG)
Definition: iordef.h:192
int singleXferAttempt
Definition: ior.h:151
#define MAX_RETRY
Definition: iordef.h:111
static IOR_param_t param
Definition: mdtest.c:153
static IOR_offset_t IME_GetFileSize(IOR_param_t *, MPI_Comm, char *)
Definition: aiori-IME.c:323
static void IME_Delete(char *, IOR_param_t *)
Definition: aiori-IME.c:248
#define IOR_WRONLY
Definition: aiori.h:34
int rankOffset
Definition: utilities.c:58
long long int IOR_size_t
Definition: iordef.h:124
#define WARN(MSG)
Definition: iordef.h:145
static int IME_MkDir(const char *, mode_t, IOR_param_t *)
Definition: aiori-IME.c:288
IOR_offset_t offset
Definition: ior.h:119
#define VERBOSE_4
Definition: iordef.h:106
#define IOR_RDWR
Definition: aiori.h:35
int verbose
Definition: utilities.c:60
char * name
Definition: aiori.h:67
long long int IOR_offset_t
Definition: iordef.h:123
ior_aiori_t ime_aiori
Definition: aiori-IME.c:64
static char * IME_GetVersion()
Definition: aiori-IME.c:260
int rank
Definition: utilities.c:57
int useO_DIRECT
Definition: ior.h:125
static int IME_StatFS(const char *, ior_aiori_statfs_t *, IOR_param_t *)
Definition: aiori-IME.c:274
#define NULL
Definition: iordef.h:79