IOR
aiori-NCMPI.c
Go to the documentation of this file.
1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
2  * vim:expandtab:shiftwidth=8:tabstop=8:
3  */
4 /******************************************************************************\
5 * *
6 * Copyright (c) 2003, The Regents of the University of California *
7 * See the file COPYRIGHT for a complete copyright notice and license. *
8 * *
9 ********************************************************************************
10 *
11 * Implement abstract I/O interface for Parallel NetCDF (NCMPI).
12 *
13 \******************************************************************************/
14 
15 #ifdef HAVE_CONFIG_H
16 #include "config.h"
17 #endif
18 
19 #include <stdio.h>
20 #include <stdlib.h>
21 #include <sys/stat.h>
22 #include <pnetcdf.h>
23 
24 #include "ior.h"
25 #include "iordef.h"
26 #include "aiori.h"
27 #include "utilities.h"
28 
29 #define NUM_DIMS 3 /* number of dimensions to data set */
30 
31 /*
32  * NCMPI_CHECK will display a custom error message and then exit the program
33  */
34 #define NCMPI_CHECK(NCMPI_RETURN, MSG) do { \
35  \
36  if (NCMPI_RETURN < 0) { \
37  fprintf(stdout, "** error **\n"); \
38  fprintf(stdout, "ERROR in %s (line %d): %s.\n", \
39  __FILE__, __LINE__, MSG); \
40  fprintf(stdout, "ERROR: %s.\n", ncmpi_strerror(NCMPI_RETURN)); \
41  fprintf(stdout, "** exiting **\n"); \
42  exit(-1); \
43  } \
44 } while(0)
45 
46 /**************************** P R O T O T Y P E S *****************************/
47 
48 static int GetFileMode(IOR_param_t *);
49 
50 static void *NCMPI_Create(char *, IOR_param_t *);
51 static void *NCMPI_Open(char *, IOR_param_t *);
52 static IOR_offset_t NCMPI_Xfer(int, void *, IOR_size_t *,
54 static void NCMPI_Close(void *, IOR_param_t *);
55 static void NCMPI_Delete(char *, IOR_param_t *);
56 static char *NCMPI_GetVersion();
57 static void NCMPI_Fsync(void *, IOR_param_t *);
58 static IOR_offset_t NCMPI_GetFileSize(IOR_param_t *, MPI_Comm, char *);
59 static int NCMPI_Access(const char *, int, IOR_param_t *);
60 
61 /************************** D E C L A R A T I O N S ***************************/
62 
64  .name = "NCMPI",
65  .create = NCMPI_Create,
66  .open = NCMPI_Open,
67  .xfer = NCMPI_Xfer,
68  .close = NCMPI_Close,
69  .delete = NCMPI_Delete,
70  .get_version = NCMPI_GetVersion,
71  .fsync = NCMPI_Fsync,
72  .get_file_size = NCMPI_GetFileSize,
73  .statfs = aiori_posix_statfs,
74  .mkdir = aiori_posix_mkdir,
75  .rmdir = aiori_posix_rmdir,
76  .access = NCMPI_Access,
77  .stat = aiori_posix_stat,
78 };
79 
80 /***************************** F U N C T I O N S ******************************/
81 
82 /*
83  * Create and open a file through the NCMPI interface.
84  */
85 static void *NCMPI_Create(char *testFileName, IOR_param_t * param)
86 {
87  int *fd;
88  int fd_mode;
89  MPI_Info mpiHints = MPI_INFO_NULL;
90 
91  /* Wei-keng Liao: read and set MPI file hints from hintsFile */
92  SetHints(&mpiHints, param->hintsFileName);
93  if (rank == 0 && param->showHints) {
94  fprintf(stdout, "\nhints passed to MPI_File_open() {\n");
95  ShowHints(&mpiHints);
96  fprintf(stdout, "}\n");
97  }
98 
99  fd = (int *)malloc(sizeof(int));
100  if (fd == NULL)
101  ERR("malloc() failed");
102 
103  fd_mode = GetFileMode(param);
104  NCMPI_CHECK(ncmpi_create(testComm, testFileName, fd_mode,
105  mpiHints, fd), "cannot create file");
106 
107  /* Wei-keng Liao: print the MPI file hints currently used */
108 /* WEL - add when ncmpi_get_file_info() is in current parallel-netcdf release
109  if (rank == 0 && param->showHints) {
110  MPI_CHECK(ncmpi_get_file_info(*fd, &mpiHints),
111  "cannot get file info");
112  fprintf(stdout, "\nhints returned from opened file {\n");
113  ShowHints(&mpiHints);
114  fprintf(stdout, "}\n");
115  }
116 */
117 
118  /* Wei-keng Liao: free up the mpiHints object */
119 /* WEL - this needs future fix from next release of PnetCDF
120  if (mpiHints != MPI_INFO_NULL)
121  MPI_CHECK(MPI_Info_free(&mpiHints), "cannot free file info");
122 */
123 
124  return (fd);
125 }
126 
127 /*
128  * Open a file through the NCMPI interface.
129  */
130 static void *NCMPI_Open(char *testFileName, IOR_param_t * param)
131 {
132  int *fd;
133  int fd_mode;
134  MPI_Info mpiHints = MPI_INFO_NULL;
135 
136  /* Wei-keng Liao: read and set MPI file hints from hintsFile */
137  SetHints(&mpiHints, param->hintsFileName);
138  if (rank == 0 && param->showHints) {
139  fprintf(stdout, "\nhints passed to MPI_File_open() {\n");
140  ShowHints(&mpiHints);
141  fprintf(stdout, "}\n");
142  }
143 
144  fd = (int *)malloc(sizeof(int));
145  if (fd == NULL)
146  ERR("malloc() failed");
147 
148  fd_mode = GetFileMode(param);
149  NCMPI_CHECK(ncmpi_open(testComm, testFileName, fd_mode,
150  mpiHints, fd), "cannot open file");
151 
152  /* Wei-keng Liao: print the MPI file hints currently used */
153 /* WEL - add when ncmpi_get_file_info() is in current parallel-netcdf release
154  if (rank == 0 && param->showHints) {
155  MPI_CHECK(ncmpi_get_file_info(*fd, &mpiHints),
156  "cannot get file info");
157  fprintf(stdout, "\nhints returned from opened file {\n");
158  ShowHints(&mpiHints);
159  fprintf(stdout, "}\n");
160  }
161 */
162 
163  /* Wei-keng Liao: free up the mpiHints object */
164 /* WEL - this needs future fix from next release of PnetCDF
165  if (mpiHints != MPI_INFO_NULL)
166  MPI_CHECK(MPI_Info_free(&mpiHints), "cannot free file info");
167 */
168 
169  return (fd);
170 }
171 
172 /*
173  * Write or read access to file using the NCMPI interface.
174  */
175 static IOR_offset_t NCMPI_Xfer(int access, void *fd, IOR_size_t * buffer,
176  IOR_offset_t length, IOR_param_t * param)
177 {
178  signed char *bufferPtr = (signed char *)buffer;
179  static int firstReadCheck = FALSE, startDataSet;
180  int var_id, dim_id[NUM_DIMS];
181  MPI_Offset bufSize[NUM_DIMS], offset[NUM_DIMS];
182  IOR_offset_t segmentPosition;
183  int segmentNum, transferNum;
184 
185  /* Wei-keng Liao: In ior.c line 1979 says "block size must be a multiple
186  of transfer size." Hence, length should always == param->transferSize
187  below. I leave it here to double check.
188  */
189  if (length != param->transferSize) {
190  char errMsg[256];
191  sprintf(errMsg, "length(%lld) != param->transferSize(%lld)\n",
192  length, param->transferSize);
193  NCMPI_CHECK(-1, errMsg);
194  }
195 
196  /* determine by offset if need to start data set */
197  if (param->filePerProc == TRUE) {
198  segmentPosition = (IOR_offset_t) 0;
199  } else {
200  segmentPosition =
201  (IOR_offset_t) ((rank + rankOffset) % param->numTasks)
202  * param->blockSize;
203  }
204  if ((int)(param->offset - segmentPosition) == 0) {
205  startDataSet = TRUE;
206  /*
207  * this toggle is for the read check operation, which passes through
208  * this function twice; note that this function will open a data set
209  * only on the first read check and close only on the second
210  */
211  if (access == READCHECK) {
212  if (firstReadCheck == TRUE) {
213  firstReadCheck = FALSE;
214  } else {
215  firstReadCheck = TRUE;
216  }
217  }
218  }
219 
220  if (startDataSet == TRUE &&
221  (access != READCHECK || firstReadCheck == TRUE)) {
222  if (access == WRITE) {
223  int numTransfers =
224  param->blockSize / param->transferSize;
225 
226  /* Wei-keng Liao: change 1D array to 3D array of dimensions:
227  [segmentCount*numTasksWorld][numTransfers][transferSize]
228  Requirement: none of these dimensions should be > 4G,
229  */
230  NCMPI_CHECK(ncmpi_def_dim
231  (*(int *)fd, "segments_times_np",
232  NC_UNLIMITED, &dim_id[0]),
233  "cannot define data set dimensions");
234  NCMPI_CHECK(ncmpi_def_dim
235  (*(int *)fd, "number_of_transfers",
236  numTransfers, &dim_id[1]),
237  "cannot define data set dimensions");
238  NCMPI_CHECK(ncmpi_def_dim
239  (*(int *)fd, "transfer_size",
240  param->transferSize, &dim_id[2]),
241  "cannot define data set dimensions");
242  NCMPI_CHECK(ncmpi_def_var
243  (*(int *)fd, "data_var", NC_BYTE, NUM_DIMS,
244  dim_id, &var_id),
245  "cannot define data set variables");
246  NCMPI_CHECK(ncmpi_enddef(*(int *)fd),
247  "cannot close data set define mode");
248 
249  } else {
250  NCMPI_CHECK(ncmpi_inq_varid
251  (*(int *)fd, "data_var", &var_id),
252  "cannot retrieve data set variable");
253  }
254 
255  if (param->collective == FALSE) {
256  NCMPI_CHECK(ncmpi_begin_indep_data(*(int *)fd),
257  "cannot enable independent data mode");
258  }
259 
260  param->var_id = var_id;
261  startDataSet = FALSE;
262  }
263 
264  var_id = param->var_id;
265 
266  /* Wei-keng Liao: calculate the segment number */
267  segmentNum = param->offset / (param->numTasks * param->blockSize);
268 
269  /* Wei-keng Liao: calculate the transfer number in each block */
270  transferNum = param->offset % param->blockSize / param->transferSize;
271 
272  /* Wei-keng Liao: read/write the 3rd dim of the dataset, each is of
273  amount param->transferSize */
274  bufSize[0] = 1;
275  bufSize[1] = 1;
276  bufSize[2] = param->transferSize;
277 
278  offset[0] = segmentNum * numTasksWorld + rank;
279  offset[1] = transferNum;
280  offset[2] = 0;
281 
282  /* access the file */
283  if (access == WRITE) { /* WRITE */
284  if (param->collective) {
285  NCMPI_CHECK(ncmpi_put_vara_schar_all
286  (*(int *)fd, var_id, offset, bufSize,
287  bufferPtr),
288  "cannot write to data set");
289  } else {
290  NCMPI_CHECK(ncmpi_put_vara_schar
291  (*(int *)fd, var_id, offset, bufSize,
292  bufferPtr),
293  "cannot write to data set");
294  }
295  } else { /* READ or CHECK */
296  if (param->collective == TRUE) {
297  NCMPI_CHECK(ncmpi_get_vara_schar_all
298  (*(int *)fd, var_id, offset, bufSize,
299  bufferPtr),
300  "cannot read from data set");
301  } else {
302  NCMPI_CHECK(ncmpi_get_vara_schar
303  (*(int *)fd, var_id, offset, bufSize,
304  bufferPtr),
305  "cannot read from data set");
306  }
307  }
308 
309  return (length);
310 }
311 
312 /*
313  * Perform fsync().
314  */
315 static void NCMPI_Fsync(void *fd, IOR_param_t * param)
316 {
317  ;
318 }
319 
320 /*
321  * Close a file through the NCMPI interface.
322  */
323 static void NCMPI_Close(void *fd, IOR_param_t * param)
324 {
325  if (param->collective == FALSE) {
326  NCMPI_CHECK(ncmpi_end_indep_data(*(int *)fd),
327  "cannot disable independent data mode");
328  }
329  NCMPI_CHECK(ncmpi_close(*(int *)fd), "cannot close file");
330  free(fd);
331 }
332 
333 /*
334  * Delete a file through the NCMPI interface.
335  */
336 static void NCMPI_Delete(char *testFileName, IOR_param_t * param)
337 {
338  return(MPIIO_Delete(testFileName, param));
339 }
340 
341 /*
342  * Determine api version.
343  */
344 static char* NCMPI_GetVersion()
345 {
346  return (char *)ncmpi_inq_libvers();
347 }
348 
349 /*
350  * Return the correct file mode for NCMPI.
351  */
353 {
354  int fd_mode = 0;
355 
356  /* set IOR file flags to NCMPI flags */
357  /* -- file open flags -- */
358  if (param->openFlags & IOR_RDONLY) {
359  fd_mode |= NC_NOWRITE;
360  }
361  if (param->openFlags & IOR_WRONLY) {
362  fprintf(stdout, "File write only not implemented in NCMPI\n");
363  }
364  if (param->openFlags & IOR_RDWR) {
365  fd_mode |= NC_WRITE;
366  }
367  if (param->openFlags & IOR_APPEND) {
368  fprintf(stdout, "File append not implemented in NCMPI\n");
369  }
370  if (param->openFlags & IOR_CREAT) {
371  fd_mode |= NC_CLOBBER;
372  }
373  if (param->openFlags & IOR_EXCL) {
374  fprintf(stdout, "Exclusive access not implemented in NCMPI\n");
375  }
376  if (param->openFlags & IOR_TRUNC) {
377  fprintf(stdout, "File truncation not implemented in NCMPI\n");
378  }
379  if (param->openFlags & IOR_DIRECT) {
380  fprintf(stdout, "O_DIRECT not implemented in NCMPI\n");
381  }
382 
383  /* Wei-keng Liao: to enable > 4GB file size */
384  fd_mode |= NC_64BIT_OFFSET;
385 
386  return (fd_mode);
387 }
388 
389 /*
390  * Use MPIIO call to get file size.
391  */
393  char *testFileName)
394 {
395  return(MPIIO_GetFileSize(test, testComm, testFileName));
396 }
397 
398 /*
399  * Use MPIIO call to check for access.
400  */
401 static int NCMPI_Access(const char *path, int mode, IOR_param_t *param)
402 {
403  return(MPIIO_Access(path, mode, param));
404 }
int showHints
Definition: ior.h:126
static char * NCMPI_GetVersion()
Definition: aiori-NCMPI.c:344
#define ERR(MSG)
Definition: iordef.h:169
void ShowHints(MPI_Info *mpiHints)
Definition: utilities.c:303
int filePerProc
Definition: ior.h:104
IOR_offset_t transferSize
Definition: ior.h:118
int aiori_posix_rmdir(const char *path, IOR_param_t *param)
Definition: aiori.c:134
static IOR_offset_t NCMPI_Xfer(int, void *, IOR_size_t *, IOR_offset_t, IOR_param_t *)
Definition: aiori-NCMPI.c:175
#define READCHECK
Definition: iordef.h:98
static int GetFileMode(IOR_param_t *)
Definition: aiori-NCMPI.c:352
#define IOR_APPEND
Definition: aiori.h:36
int aiori_posix_mkdir(const char *path, mode_t mode, IOR_param_t *param)
Definition: aiori.c:129
ior_aiori_t ncmpi_aiori
Definition: aiori-NCMPI.c:63
#define IOR_RDONLY
Definition: aiori.h:33
unsigned int openFlags
Definition: ior.h:85
#define WRITE
Definition: iordef.h:95
static IOR_offset_t NCMPI_GetFileSize(IOR_param_t *, MPI_Comm, char *)
Definition: aiori-NCMPI.c:392
int aiori_posix_statfs(const char *path, ior_aiori_statfs_t *stat_buf, IOR_param_t *param)
Definition: aiori.c:104
#define IOR_CREAT
Definition: aiori.h:37
#define IOR_EXCL
Definition: aiori.h:39
char * hintsFileName
Definition: ior.h:92
MPI_Comm testComm
Definition: utilities.c:61
#define IOR_TRUNC
Definition: aiori.h:38
Definition: ior.h:47
int numTasksWorld
Definition: utilities.c:56
void MPIIO_Delete(char *testFileName, IOR_param_t *param)
Definition: aiori-MPIIO.c:409
static IOR_param_t param
Definition: mdtest.c:153
#define NUM_DIMS
Definition: aiori-NCMPI.c:29
int MPIIO_Access(const char *path, int mode, IOR_param_t *param)
Definition: aiori-MPIIO.c:69
#define IOR_WRONLY
Definition: aiori.h:34
#define FALSE
Definition: iordef.h:71
int rankOffset
Definition: utilities.c:58
long long int IOR_size_t
Definition: iordef.h:124
int numTasks
Definition: ior.h:94
static void * NCMPI_Create(char *, IOR_param_t *)
Definition: aiori-NCMPI.c:85
int collective
Definition: ior.h:115
IOR_offset_t offset
Definition: ior.h:119
int aiori_posix_stat(const char *path, struct stat *buf, IOR_param_t *param)
Definition: aiori.c:144
static void NCMPI_Fsync(void *, IOR_param_t *)
Definition: aiori-NCMPI.c:315
void SetHints(MPI_Info *mpiHints, char *hintsFileName)
Definition: utilities.c:242
int var_id
Definition: ior.h:184
#define IOR_RDWR
Definition: aiori.h:35
static void * NCMPI_Open(char *, IOR_param_t *)
Definition: aiori-NCMPI.c:130
#define NCMPI_CHECK(NCMPI_RETURN, MSG)
Definition: aiori-NCMPI.c:34
char * name
Definition: aiori.h:67
long long int IOR_offset_t
Definition: iordef.h:123
static void NCMPI_Delete(char *, IOR_param_t *)
Definition: aiori-NCMPI.c:336
int rank
Definition: utilities.c:57
static void NCMPI_Close(void *, IOR_param_t *)
Definition: aiori-NCMPI.c:323
IOR_offset_t blockSize
Definition: ior.h:117
#define TRUE
Definition: iordef.h:75
#define IOR_DIRECT
Definition: aiori.h:40
IOR_offset_t MPIIO_GetFileSize(IOR_param_t *test, MPI_Comm testComm, char *testFileName)
Definition: aiori-MPIIO.c:472
#define NULL
Definition: iordef.h:79
static int NCMPI_Access(const char *, int, IOR_param_t *)
Definition: aiori-NCMPI.c:401