IOR
aiori-HDFS.c
Go to the documentation of this file.
1 /* -*- mode: c; indent-tabs-mode: t; -*-
2  * vim:noexpandtab:
3  *
4  * Editing with tabs allows different users to pick their own indentation
5  * appearance without changing the file.
6  */
7 
8 /*
9  * Copyright (c) 2009, Los Alamos National Security, LLC All rights reserved.
10  * Copyright 2009. Los Alamos National Security, LLC. This software was produced
11  * under U.S. Government contract DE-AC52-06NA25396 for Los Alamos National
12  * Laboratory (LANL), which is operated by Los Alamos National Security, LLC for
13  * the U.S. Department of Energy. The U.S. Government has rights to use,
14  * reproduce, and distribute this software. NEITHER THE GOVERNMENT NOR LOS
15  * ALAMOS NATIONAL SECURITY, LLC MAKES ANY WARRANTY, EXPRESS OR IMPLIED, OR
16  * ASSUMES ANY LIABILITY FOR THE USE OF THIS SOFTWARE. If software is
17  * modified to produce derivative works, such modified software should be
18  * clearly marked, so as not to confuse it with the version available from
19  * LANL.
20  *
21  * Additionally, redistribution and use in source and binary forms, with or
22  * without modification, are permitted provided that the following conditions are
23  * met:
24  *
25  * • Redistributions of source code must retain the above copyright notice,
26  * this list of conditions and the following disclaimer.
27  *
28  * • Redistributions in binary form must reproduce the above copyright notice,
29  * this list of conditions and the following disclaimer in the documentation
30  * and/or other materials provided with the distribution.
31  *
32  * • Neither the name of Los Alamos National Security, LLC, Los Alamos National
33  * Laboratory, LANL, the U.S. Government, nor the names of its contributors may be
34  * used to endorse or promote products derived from this software without specific
35  * prior written permission.
36  *
37  * THIS SOFTWARE IS PROVIDED BY LOS ALAMOS NATIONAL SECURITY, LLC AND CONTRIBUTORS
38  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
39  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
40  * ARE DISCLAIMED. IN NO EVENT SHALL LOS ALAMOS NATIONAL SECURITY, LLC OR
41  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
42  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
43  * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
44  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
45  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
46  * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
47  * OF SUCH DAMAGE.
48  */
49 
50 /******************************************************************************\
51 *
52 * Implement of abstract I/O interface for HDFS.
53 *
54 * HDFS has the added concept of a "File System Handle" which has to be
55 * connected before files are opened. We store this in the IOR_param_t
56 * object that is always passed to our functions. The thing that callers
57 * think of as the "fd" is an hdfsFile, (a pointer).
58 *
59 \******************************************************************************/
60 
61 #ifdef HAVE_CONFIG_H
62 # include "config.h"
63 #endif
64 
65 #include <stdio.h>
66 #include <stdlib.h>
67 
68 #ifdef __linux__
69 # include <sys/ioctl.h> /* necessary for: */
70 # define __USE_GNU /* O_DIRECT and */
71 # include <fcntl.h> /* IO operations */
72 # undef __USE_GNU
73 #endif /* __linux__ */
74 
75 #include <errno.h>
76 #include <fcntl.h> /* IO operations */
77 #include <sys/stat.h>
78 #include <assert.h>
79 /*
80 #ifdef HAVE_LUSTRE_LUSTRE_USER_H
81 #include <lustre/lustre_user.h>
82 #endif
83 */
84 
85 #include "ior.h"
86 #include "aiori.h"
87 #include "iordef.h"
88 
89 #ifndef open64 /* necessary for TRU64 -- */
90 # define open64 open /* unlikely, but may pose */
91 #endif /* not open64 */ /* conflicting prototypes */
92 
93 #ifndef lseek64 /* necessary for TRU64 -- */
94 # define lseek64 lseek /* unlikely, but may pose */
95 #endif /* not lseek64 */ /* conflicting prototypes */
96 
97 #ifndef O_BINARY /* Required on Windows */
98 # define O_BINARY 0
99 #endif
100 
101 #include "hdfs.h"
102 
103 /**************************** P R O T O T Y P E S *****************************/
104 static void *HDFS_Create(char *, IOR_param_t *);
105 static void *HDFS_Open(char *, IOR_param_t *);
106 static IOR_offset_t HDFS_Xfer(int, void *, IOR_size_t *,
108 static void HDFS_Close(void *, IOR_param_t *);
109 static void HDFS_Delete(char *, IOR_param_t *);
110 static void HDFS_SetVersion(IOR_param_t *);
111 static void HDFS_Fsync(void *, IOR_param_t *);
112 static IOR_offset_t HDFS_GetFileSize(IOR_param_t *, MPI_Comm, char *);
113 
114 /************************** D E C L A R A T I O N S ***************************/
115 
117  .name = "HDFS",
118  .create = HDFS_Create,
119  .open = HDFS_Open,
120  .xfer = HDFS_Xfer,
121  .close = HDFS_Close,
122  .delete = HDFS_Delete,
123  .set_version = HDFS_SetVersion,
124  .fsync = HDFS_Fsync,
125  .get_file_size = HDFS_GetFileSize,
126 };
127 
128 /***************************** F U N C T I O N S ******************************/
129 
130 /* This is identical to the one in aiori-POSIX.c Doesn't seem like
131  * it would be appropriate in utilities.c.
132  */
133 
135 {
136 /* note that TRU64 needs O_DIRECTIO, SunOS uses directio(),
137  and everyone else needs O_DIRECT */
138 #ifndef O_DIRECT
139 # ifndef O_DIRECTIO
140  WARN("cannot use O_DIRECT");
141 # define O_DIRECT 000000
142 # else /* O_DIRECTIO */
143 # define O_DIRECT O_DIRECTIO
144 # endif /* not O_DIRECTIO */
145 #endif /* not O_DIRECT */
146 
147  *fd |= O_DIRECT;
148 }
149 
150 
151 /*
152  * "Connect" to an HDFS file-system. HDFS requires this be done before and
153  * files are opened. It is easy for ior_aiori.open/create to assure that
154  * we connect, if we haven't already done so. However, there's not a
155  * simple way to assure that we disconnect after the last file-close. For
156  * now, we'll make a special call at the end of ior.c
157  *
158  * NOTE: It's okay to call this thing whenever you need to be sure the HDFS
159  * filesystem is connected.
160  */
161 static void hdfs_connect( IOR_param_t* param ) {
162  if (param->verbose >= VERBOSE_4) {
163  printf("-> hdfs_connect [nn:\"%s\", port:%d, user:%s]\n",
164  param->hdfs_name_node,
165  param->hdfs_name_node_port,
166  param->hdfs_user );
167  }
168 
169  if ( param->hdfs_fs ) {
170  if (param->verbose >= VERBOSE_4) {
171  printf("<- hdfs_connect [nothing to do]\n"); /* DEBUGGING */
172  }
173  return;
174  }
175 
176  /* initialize a builder, holding parameters for hdfsBuilderConnect() */
177  struct hdfsBuilder* builder = hdfsNewBuilder();
178  if ( ! builder )
179  ERR_SIMPLE("couldn't create an hdfsBuilder");
180 
181  hdfsBuilderSetForceNewInstance ( builder ); /* don't use cached instance */
182 
183  hdfsBuilderSetNameNode ( builder, param->hdfs_name_node );
184  hdfsBuilderSetNameNodePort( builder, param->hdfs_name_node_port );
185  hdfsBuilderSetUserName ( builder, param->hdfs_user );
186 
187  /* NOTE: hdfsBuilderConnect() frees the builder */
188  param->hdfs_fs = hdfsBuilderConnect( builder );
189  if ( ! param->hdfs_fs )
190  ERR_SIMPLE("hdsfsBuilderConnect failed");
191 
192  if (param->verbose >= VERBOSE_4) {
193  printf("<- hdfs_connect [success]\n");
194  }
195 }
196 
198  if (param->verbose >= VERBOSE_4) {
199  printf("-> hdfs_disconnect\n");
200  }
201  if ( param->hdfs_fs ) {
202  hdfsDisconnect( param->hdfs_fs );
203  param->hdfs_fs = NULL;
204  }
205  if (param->verbose >= VERBOSE_4) {
206  printf("<- hdfs_disconnect\n");
207  }
208 }
209 
210 
211 /*
212  * Create or open the file. Pass TRUE if creating and FALSE if opening an existing file.
213  * Return an hdfsFile.
214  */
215 
216 static void *HDFS_Create_Or_Open( char *testFileName, IOR_param_t *param, unsigned char createFile ) {
217  if (param->verbose >= VERBOSE_4) {
218  printf("-> HDFS_Create_Or_Open\n");
219  }
220 
221  hdfsFile hdfs_file = NULL;
222  int fd_oflags = 0, hdfs_return;
223 
224  /* initialize file-system handle, if needed */
225  hdfs_connect( param );
226 
227  /*
228  * Check for unsupported flags.
229  *
230  * If they want RDWR, we don't know if they're going to try to do both, so we
231  * can't default to either O_RDONLY or O_WRONLY. Thus, we error and exit.
232  *
233  * The other two, we just note that they are not supported and don't do them.
234  */
235 
236  if ( param->openFlags & IOR_RDWR ) {
237  ERR( "Opening or creating a file in RDWR is not implemented in HDFS" );
238  }
239 
240  if ( param->openFlags & IOR_EXCL ) {
241  fprintf( stdout, "Opening or creating a file in Exclusive mode is not implemented in HDFS\n" );
242  }
243 
244  if ( param->openFlags & IOR_APPEND ) {
245  fprintf( stdout, "Opening or creating a file for appending is not implemented in HDFS\n" );
246  }
247 
248  /*
249  * Setup the flags to be used.
250  */
251 
252  if ( createFile == TRUE ) {
253  fd_oflags = O_CREAT;
254  }
255 
256  if ( param->openFlags & IOR_WRONLY ) {
257  if ( !param->filePerProc ) {
258 
259  // in N-1 mode, only rank 0 truncates the file
260  if ( rank != 0 ) {
261  fd_oflags |= O_WRONLY;
262  } else {
263  fd_oflags |= O_TRUNC;
264  fd_oflags |= O_WRONLY;
265  }
266 
267  } else {
268  // in N-N mode, everyone does truncate
269  fd_oflags |= O_TRUNC;
270  fd_oflags |= O_WRONLY;
271  }
272 
273  } else {
274  fd_oflags |= O_RDONLY;
275  }
276 
277  /*
278  * Now see if O_DIRECT is needed.
279  */
280 
281  if ( param->useO_DIRECT == TRUE ) {
282  hdfs_set_o_direct_flag( &fd_oflags );
283  }
284 
285  /*
286  * For N-1 write, All other ranks wait for Rank 0 to open the file.
287  * this is bec 0 does truncate and rest don't
288  * it would be dangerous for all to do truncate as they might
289  * truncate each other's writes
290  */
291 
292  if (( param->openFlags & IOR_WRONLY ) &&
293  ( !param->filePerProc ) &&
294  ( rank != 0 )) {
295 
296  MPI_CHECK(MPI_Barrier(testComm), "barrier error");
297  }
298 
299  /*
300  * Now rank zero can open and truncate, if necessary.
301  */
302 
303  if (param->verbose >= VERBOSE_4) {
304  printf("\thdfsOpenFile(0x%llx, %s, 0%o, %d, %d, %d)\n",
305  param->hdfs_fs,
306  testFileName,
307  fd_oflags, /* shown in octal to compare w/ <bits/fcntl.h> */
308  param->transferSize,
309  param->hdfs_replicas,
310  param->hdfs_block_size);
311  }
312  hdfs_file = hdfsOpenFile( param->hdfs_fs,
313  testFileName,
314  fd_oflags,
315  param->transferSize,
316  param->hdfs_replicas,
317  param->hdfs_block_size);
318  if ( ! hdfs_file ) {
319  ERR( "Failed to open the file" );
320  }
321 
322  /*
323  * For N-1 write, Rank 0 waits for the other ranks to open the file after it has.
324  */
325 
326  if (( param->openFlags & IOR_WRONLY ) &&
327  ( !param->filePerProc ) &&
328  ( rank == 0 )) {
329 
330  MPI_CHECK(MPI_Barrier(testComm), "barrier error");
331  }
332 
333  if (param->verbose >= VERBOSE_4) {
334  printf("<- HDFS_Create_Or_Open\n");
335  }
336  return ((void *) hdfs_file );
337 }
338 
339 /*
340  * Create and open a file through the HDFS interface.
341  */
342 
343 static void *HDFS_Create( char *testFileName, IOR_param_t * param ) {
344  if (param->verbose >= VERBOSE_4) {
345  printf("-> HDFS_Create\n");
346  }
347 
348  if (param->verbose >= VERBOSE_4) {
349  printf("<- HDFS_Create\n");
350  }
351  return HDFS_Create_Or_Open( testFileName, param, TRUE );
352 }
353 
354 /*
355  * Open a file through the HDFS interface.
356  */
357 static void *HDFS_Open( char *testFileName, IOR_param_t * param ) {
358  if (param->verbose >= VERBOSE_4) {
359  printf("-> HDFS_Open\n");
360  }
361 
362  if ( param->openFlags & IOR_CREAT ) {
363  if (param->verbose >= VERBOSE_4) {
364  printf("<- HDFS_Open( ... TRUE)\n");
365  }
366  return HDFS_Create_Or_Open( testFileName, param, TRUE );
367  }
368  else {
369  if (param->verbose >= VERBOSE_4) {
370  printf("<- HDFS_Open( ... FALSE)\n");
371  }
372  return HDFS_Create_Or_Open( testFileName, param, FALSE );
373  }
374 }
375 
376 /*
377  * Write or read to file using the HDFS interface.
378  */
379 
380 static IOR_offset_t HDFS_Xfer(int access, void *file, IOR_size_t * buffer,
381  IOR_offset_t length, IOR_param_t * param) {
382  if (param->verbose >= VERBOSE_4) {
383  printf("-> HDFS_Xfer(acc:%d, file:0x%llx, buf:0x%llx, len:%llu, 0x%llx)\n",
384  access, file, buffer, length, param);
385  }
386 
387  int xferRetries = 0;
388  long long remaining = (long long)length;
389  char* ptr = (char *)buffer;
390  long long rc;
391  off_t offset = param->offset;
392  hdfsFS hdfs_fs = param->hdfs_fs; /* (void*) */
393  hdfsFile hdfs_file = (hdfsFile)file; /* (void*) */
394 
395 
396  while ( remaining > 0 ) {
397 
398  /* write/read file */
399  if (access == WRITE) { /* WRITE */
400  if (verbose >= VERBOSE_4) {
401  fprintf( stdout, "task %d writing to offset %lld\n",
402  rank,
403  param->offset + length - remaining);
404  }
405 
406  if (param->verbose >= VERBOSE_4) {
407  printf("\thdfsWrite( 0x%llx, 0x%llx, 0x%llx, %lld)\n",
408  hdfs_fs, hdfs_file, ptr, remaining ); /* DEBUGGING */
409  }
410  rc = hdfsWrite( hdfs_fs, hdfs_file, ptr, remaining );
411  if ( rc < 0 ) {
412  ERR( "hdfsWrite() failed" );
413  }
414 
415  offset += rc;
416 
417  if ( param->fsyncPerWrite == TRUE ) {
418  HDFS_Fsync( hdfs_file, param );
419  }
420  }
421  else { /* READ or CHECK */
422  if (verbose >= VERBOSE_4) {
423  fprintf( stdout, "task %d reading from offset %lld\n",
424  rank,
425  param->offset + length - remaining );
426  }
427 
428  if (param->verbose >= VERBOSE_4) {
429  printf("\thdfsRead( 0x%llx, 0x%llx, 0x%llx, %lld)\n",
430  hdfs_fs, hdfs_file, ptr, remaining ); /* DEBUGGING */
431  }
432  rc = hdfsRead( hdfs_fs, hdfs_file, ptr, remaining );
433 
434  if ( rc == 0 ) {
435  ERR( "hdfs_read() returned EOF prematurely" );
436  }
437 
438  if ( rc < 0 ) {
439  ERR( "hdfs_read() failed" );
440  }
441 
442  offset += rc;
443  }
444 
445 
446  if ( rc < remaining ) {
447  fprintf(stdout, "WARNING: Task %d, partial %s, %lld of %lld bytes at offset %lld\n",
448  rank,
449  access == WRITE ? "hdfsWrite()" : "hdfs_read()",
450  rc, remaining,
451  param->offset + length - remaining );
452 
453  if ( param->singleXferAttempt == TRUE ) {
454  MPI_CHECK( MPI_Abort( MPI_COMM_WORLD, -1 ), "barrier error" );
455  }
456 
457  if ( xferRetries > MAX_RETRY ) {
458  ERR( "too many retries -- aborting" );
459  }
460  }
461 
462  assert( rc >= 0 );
463  assert( rc <= remaining );
464  remaining -= rc;
465  ptr += rc;
466  xferRetries++;
467  }
468 
469  if (param->verbose >= VERBOSE_4) {
470  printf("<- HDFS_Xfer\n");
471  }
472  return ( length );
473 }
474 
475 /*
476  * Perform hdfs_sync().
477  */
478 
479 static void HDFS_Fsync( void *fd, IOR_param_t * param ) {
480  if (param->verbose >= VERBOSE_4) {
481  printf("-> HDFS_Fsync\n");
482  }
483  hdfsFS hdfs_fs = param->hdfs_fs; /* (void *) */
484  hdfsFile hdfs_file = (hdfsFile)fd; /* (void *) */
485 
486 #if 0
487  if (param->verbose >= VERBOSE_4) {
488  printf("\thdfsHSync(0x%llx, 0x%llx)\n", hdfs_fs, hdfs_file);
489  }
490  if ( hdfsHSync( hdfs_fs, hdfs_file ) != 0 ) {
491  EWARN( "hdfsHSync() failed" );
492  }
493 #elif 0
494  if (param->verbose >= VERBOSE_4) {
495  printf("\thdfsHFlush(0x%llx, 0x%llx)\n", hdfs_fs, hdfs_file);
496  }
497  if ( hdfsHFlush( hdfs_fs, hdfs_file ) != 0 ) {
498  EWARN( "hdfsHFlush() failed" );
499  }
500 #else
501  if (param->verbose >= VERBOSE_4) {
502  printf("\thdfsFlush(0x%llx, 0x%llx)\n", hdfs_fs, hdfs_file);
503  }
504  if ( hdfsFlush( hdfs_fs, hdfs_file ) != 0 ) {
505  EWARN( "hdfsFlush() failed" );
506  }
507 #endif
508 
509  if (param->verbose >= VERBOSE_4) {
510  printf("<- HDFS_Fsync\n");
511  }
512 }
513 
514 /*
515  * Close a file through the HDFS interface.
516  */
517 
518 static void HDFS_Close( void *fd, IOR_param_t * param ) {
519  if (param->verbose >= VERBOSE_4) {
520  printf("-> HDFS_Close\n");
521  }
522 
523  hdfsFS hdfs_fs = param->hdfs_fs; /* (void *) */
524  hdfsFile hdfs_file = (hdfsFile)fd; /* (void *) */
525 
526  int open_flags;
527 
528  if ( param->openFlags & IOR_WRONLY ) {
529  open_flags = O_CREAT | O_WRONLY;
530  } else {
531  open_flags = O_RDONLY;
532  }
533 
534  if ( hdfsCloseFile( hdfs_fs, hdfs_file ) != 0 ) {
535  ERR( "hdfsCloseFile() failed" );
536  }
537 
538  if (param->verbose >= VERBOSE_4) {
539  printf("<- HDFS_Close\n");
540  }
541 }
542 
543 /*
544  * Delete a file through the HDFS interface.
545  *
546  * NOTE: The signature for ior_aiori.delete doesn't include a parameter to
547  * select recursive deletes. We'll assume that that is never needed.
548  */
549 static void HDFS_Delete( char *testFileName, IOR_param_t * param ) {
550  if (param->verbose >= VERBOSE_4) {
551  printf("-> HDFS_Delete\n");
552  }
553 
554  char errmsg[256];
555 
556  /* initialize file-system handle, if needed */
557  hdfs_connect( param );
558 
559  if ( ! param->hdfs_fs )
560  ERR_SIMPLE( "Can't delete a file without an HDFS connection" );
561 
562  if ( hdfsDelete( param->hdfs_fs, testFileName, 0 ) != 0 ) {
563  sprintf(errmsg,
564  "[RANK %03d]: hdfsDelete() of file \"%s\" failed\n",
565  rank, testFileName);
566 
567  EWARN( errmsg );
568  }
569  if (param->verbose >= VERBOSE_4) {
570  printf("<- HDFS_Delete\n");
571  }
572 }
573 
574 /*
575  * Determine api version.
576  */
577 
579  if (param->verbose >= VERBOSE_4) {
580  printf("-> HDFS_SetVersion\n");
581  }
582 
583  strcpy( param->apiVersion, param->api );
584  if (param->verbose >= VERBOSE_4) {
585  printf("<- HDFS_SetVersion\n");
586  }
587 }
588 
589 /*
590  * Use hdfsGetPathInfo() to get info about file?
591  * Is there an fstat we can use on hdfs?
592  * Should we just use POSIX fstat?
593  */
594 
595 static IOR_offset_t
597  MPI_Comm testComm,
598  char * testFileName) {
599  if (param->verbose >= VERBOSE_4) {
600  printf("-> HDFS_GetFileSize(%s)\n", testFileName);
601  }
602 
603  IOR_offset_t aggFileSizeFromStat;
604  IOR_offset_t tmpMin, tmpMax, tmpSum;
605 
606  /* make sure file-system is connected */
607  hdfs_connect( param );
608 
609  /* file-info struct includes size in bytes */
610  if (param->verbose >= VERBOSE_4) {
611  printf("\thdfsGetPathInfo(%s) ...", testFileName);fflush(stdout);
612  }
613 
614  hdfsFileInfo* info = hdfsGetPathInfo( param->hdfs_fs, testFileName );
615  if ( ! info )
616  ERR_SIMPLE( "hdfsGetPathInfo() failed" );
617  if (param->verbose >= VERBOSE_4) {
618  printf("done.\n");fflush(stdout);
619  }
620 
621  aggFileSizeFromStat = info->mSize;
622 
623  if ( param->filePerProc == TRUE ) {
624  if (param->verbose >= VERBOSE_4) {
625  printf("\tall-reduce (1)\n");
626  }
627  MPI_CHECK(
628  MPI_Allreduce(
629  &aggFileSizeFromStat, &tmpSum, 1, MPI_LONG_LONG_INT, MPI_SUM, testComm ),
630  "cannot total data moved" );
631 
632  aggFileSizeFromStat = tmpSum;
633  }
634  else {
635  if (param->verbose >= VERBOSE_4) {
636  printf("\tall-reduce (2a)\n");
637  }
638  MPI_CHECK(
639  MPI_Allreduce(
640  &aggFileSizeFromStat, &tmpMin, 1, MPI_LONG_LONG_INT, MPI_MIN, testComm ),
641  "cannot total data moved" );
642 
643  if (param->verbose >= VERBOSE_4) {
644  printf("\tall-reduce (2b)\n");
645  }
646  MPI_CHECK(
647  MPI_Allreduce(
648  &aggFileSizeFromStat, &tmpMax, 1, MPI_LONG_LONG_INT, MPI_MAX, testComm ),
649  "cannot total data moved" );
650 
651  if ( tmpMin != tmpMax ) {
652  if ( rank == 0 ) {
653  WARN( "inconsistent file size by different tasks" );
654  }
655 
656  /* incorrect, but now consistent across tasks */
657  aggFileSizeFromStat = tmpMin;
658  }
659  }
660 
661  if (param->verbose >= VERBOSE_4) {
662  printf("<- HDFS_GetFileSize [%llu]\n", aggFileSizeFromStat);
663  }
664  return ( aggFileSizeFromStat );
665 }
char * hdfs_user
Definition: ior.h:168
#define ERR(MSG)
Definition: iordef.h:169
static void HDFS_Delete(char *, IOR_param_t *)
Definition: aiori-HDFS.c:549
int filePerProc
Definition: ior.h:104
CURLcode rc
Definition: aiori-S3.c:121
ior_aiori_t hdfs_aiori
Definition: aiori-HDFS.c:116
IOR_offset_t transferSize
Definition: ior.h:118
#define IOR_APPEND
Definition: aiori.h:36
char * apiVersion
Definition: ior.h:88
int hdfs_replicas
Definition: ior.h:172
unsigned int openFlags
Definition: ior.h:85
int fsyncPerWrite
Definition: ior.h:152
#define ERR_SIMPLE(MSG)
Definition: iordef.h:178
#define WRITE
Definition: iordef.h:95
#define EWARN(MSG)
Definition: iordef.h:156
static IOR_offset_t HDFS_Xfer(int, void *, IOR_size_t *, IOR_offset_t, IOR_param_t *)
Definition: aiori-HDFS.c:380
#define IOR_CREAT
Definition: aiori.h:37
#define IOR_EXCL
Definition: aiori.h:39
hdfsFS hdfs_fs
Definition: ior.h:171
tPort hdfs_name_node_port
Definition: ior.h:170
static void * HDFS_Create_Or_Open(char *testFileName, IOR_param_t *param, unsigned char createFile)
Definition: aiori-HDFS.c:216
MPI_Comm testComm
Definition: utilities.c:61
#define O_DIRECT
static void HDFS_SetVersion(IOR_param_t *)
Definition: aiori-HDFS.c:578
void * hdfsFS
Definition: ior.h:29
int verbose
Definition: ior.h:138
#define MPI_CHECK(MPI_STATUS, MSG)
Definition: iordef.h:192
int singleXferAttempt
Definition: ior.h:151
Definition: ior.h:47
#define MAX_RETRY
Definition: iordef.h:111
void hdfs_set_o_direct_flag(int *fd)
Definition: aiori-HDFS.c:134
static void hdfs_connect(IOR_param_t *param)
Definition: aiori-HDFS.c:161
static IOR_param_t param
Definition: mdtest.c:153
static void * HDFS_Create(char *, IOR_param_t *)
Definition: aiori-HDFS.c:343
#define IOR_WRONLY
Definition: aiori.h:34
#define FALSE
Definition: iordef.h:71
long long int IOR_size_t
Definition: iordef.h:124
#define WARN(MSG)
Definition: iordef.h:145
int hdfs_block_size
Definition: ior.h:173
static void hdfs_disconnect(IOR_param_t *param)
Definition: aiori-HDFS.c:197
IOR_offset_t offset
Definition: ior.h:119
#define VERBOSE_4
Definition: iordef.h:106
static IOR_offset_t HDFS_GetFileSize(IOR_param_t *, MPI_Comm, char *)
Definition: aiori-HDFS.c:596
#define IOR_RDWR
Definition: aiori.h:35
static void * HDFS_Open(char *, IOR_param_t *)
Definition: aiori-HDFS.c:357
int verbose
Definition: utilities.c:60
char * api
Definition: ior.h:87
char * name
Definition: aiori.h:67
long long int IOR_offset_t
Definition: iordef.h:123
int rank
Definition: utilities.c:57
int useO_DIRECT
Definition: ior.h:125
#define TRUE
Definition: iordef.h:75
static void HDFS_Fsync(void *, IOR_param_t *)
Definition: aiori-HDFS.c:479
const char * hdfs_name_node
Definition: ior.h:169
static void HDFS_Close(void *, IOR_param_t *)
Definition: aiori-HDFS.c:518
#define NULL
Definition: iordef.h:79