IOR
mdtest.c
Go to the documentation of this file.
1 /*
2  * Copyright (C) 2003, The Regents of the University of California.
3  * Produced at the Lawrence Livermore National Laboratory.
4  * Written by Christopher J. Morrone <morrone@llnl.gov>,
5  * Bill Loewe <loewe@loewe.net>, Tyce McLarty <mclarty@llnl.gov>,
6  * and Ryan Kroiss <rrkroiss@lanl.gov>.
7  * All rights reserved.
8  * UCRL-CODE-155800
9  *
10  * Please read the COPYRIGHT file.
11  *
12  * This program is free software; you can redistribute it and/or modify
13  * it under the terms of the GNU General Public License (as published by
14  * the Free Software Foundation) version 2, dated June 1991.
15  *
16  * This program is distributed in the hope that it will be useful,
17  * but WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
18  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19  * terms and conditions of the GNU General Public License for more details.
20  *
21  * You should have received a copy of the GNU General Public License
22  * along with this program; if not, write to the Free Software
23  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
24  *
25  * CVS info:
26  * $RCSfile: mdtest.c,v $
27  * $Revision: 1.4 $
28  * $Date: 2013/11/27 17:05:31 $
29  * $Author: brettkettering $
30  */
31 
32 #include <limits.h>
33 #include <math.h>
34 #include <stdio.h>
35 #include <stdlib.h>
36 #include <stdbool.h>
37 #include <inttypes.h>
38 #include <sys/types.h>
39 #include <sys/stat.h>
40 
41 #include "option.h"
42 #include "utilities.h"
43 
44 #if HAVE_SYS_PARAM_H
45 #include <sys/param.h>
46 #endif
47 
48 #if HAVE_SYS_MOUNT_H
49 #include <sys/mount.h>
50 #endif
51 
52 #if HAVE_SYS_STATFS_H
53 #include <sys/statfs.h>
54 #endif
55 
56 #if HAVE_SYS_STATVFS_H
57 #include <sys/statvfs.h>
58 #endif
59 
60 #include <fcntl.h>
61 #include <string.h>
62 #include <unistd.h>
63 #include <dirent.h>
64 #include <errno.h>
65 #include <time.h>
66 #include <sys/time.h>
67 
68 #include "aiori.h"
69 #include "ior.h"
70 #include "mdtest.h"
71 
72 #include <mpi.h>
73 
74 #define FILEMODE S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH
75 #define DIRMODE S_IRUSR|S_IWUSR|S_IXUSR|S_IRGRP|S_IWGRP|S_IXGRP|S_IROTH|S_IXOTH
76 #define RELEASE_VERS "1.9.3"
77 #define TEST_DIR "#test-dir"
78 #define ITEM_COUNT 25000
79 
80 #define LLU "%lu"
81 
82 static int size;
83 static uint64_t *rand_array;
84 static char testdir[MAX_PATHLEN];
85 static char testdirpath[MAX_PATHLEN];
87 static char **filenames;
88 static char hostname[MAX_PATHLEN];
89 static char mk_name[MAX_PATHLEN];
90 static char stat_name[MAX_PATHLEN];
91 static char read_name[MAX_PATHLEN];
92 static char rm_name[MAX_PATHLEN];
99 static char *write_buffer;
100 static char *read_buffer;
102 
103 
104 static int barriers;
105 static int create_only;
106 static int stat_only;
107 static int read_only;
108 static int remove_only;
109 static int leaf_only;
110 static unsigned branch_factor;
111 static int depth;
112 
113 /*
114  * This is likely a small value, but it's sometimes computed by
115  * branch_factor^(depth+1), so we'll make it a larger variable,
116  * just in case.
117  */
118 static uint64_t num_dirs_in_tree;
119 /*
120  * As we start moving towards Exascale, we could have billions
121  * of files in a directory. Make room for that possibility with
122  * a larger variable.
123  */
124 static uint64_t items;
125 static uint64_t items_per_dir;
126 static uint64_t num_dirs_in_tree_calc; /* this is a workaround until the overal code is refactored */
127 static int directory_loops;
128 static int print_time;
129 static int random_seed;
130 static int shared_file;
131 static int files_only;
132 static int dirs_only;
133 static int pre_delay;
136 static int throttle;
138 static size_t write_bytes;
140 static size_t read_bytes;
141 static int sync_file;
142 static int path_count;
143 static int nstride; /* neighbor stride */
144 
146 static pid_t pid;
147 static uid_t uid;
148 
149 /* just use the POSIX backend for now */
150 static const char *backend_name = "POSIX";
151 static const ior_aiori_t *backend;
152 
154 
155 /* This structure describes the processing status for stonewalling */
156 typedef struct{
157  double start_time;
158 
160 
161  uint64_t items_start;
162  uint64_t items_done;
163 
164  uint64_t items_per_dir;
166 
167 #define CHECK_STONE_WALL(p) (((p)->stone_wall_timer_seconds != 0) && ((GetTimeStamp() - (p)->start_time) > (p)->stone_wall_timer_seconds))
168 
169 /* for making/removing unique directory && stating/deleting subdirectory */
171 
172 
173 void offset_timers(double * t, int tcount) {
174  double toffset;
175  int i;
176 
177 
178  if (( rank == 0 ) && ( verbose >= 1 )) {
179  fprintf( out_logfile, "V-1: Entering offset_timers...\n" );
180  fflush( out_logfile );
181  }
182 
183  toffset = MPI_Wtime() - t[tcount];
184  for (i = 0; i < tcount+1; i++) {
185  t[i] += toffset;
186  }
187 }
188 
189 void parse_dirpath(char *dirpath_arg) {
190  char * tmp, * token;
191  char delimiter_string[3] = { '@', '\n', '\0' };
192  int i = 0;
193 
194 
195  if (( rank == 0 ) && ( verbose >= 1 )) {
196  fprintf( out_logfile, "V-1: Entering parse_dirpath...\n" );
197  fflush( out_logfile );
198  }
199 
200  tmp = dirpath_arg;
201 
202  if (* tmp != '\0') path_count++;
203  while (* tmp != '\0') {
204  if (* tmp == '@') {
205  path_count++;
206  }
207  tmp++;
208  }
209  // prevent changes to the original dirpath_arg
210  dirpath_arg = strdup(dirpath_arg);
211  filenames = (char **)malloc(path_count * sizeof(char **));
212  if (filenames == NULL || dirpath_arg == NULL) {
213  FAIL("out of memory");
214  }
215 
216  token = strtok(dirpath_arg, delimiter_string);
217  while (token != NULL) {
218  filenames[i] = token;
219  token = strtok(NULL, delimiter_string);
220  i++;
221  }
222 }
223 
224 static void prep_testdir(int j, int dir_iter){
225  int pos = sprintf(testdir, "%s", testdirpath);
226  if ( testdir[strlen( testdir ) - 1] != '/' ) {
227  pos += sprintf(& testdir[pos], "/");
228  }
229  pos += sprintf(& testdir[pos], "%s", TEST_DIR);
230  pos += sprintf(& testdir[pos], ".%d-%d", j, dir_iter);
231 }
232 
233 /*
234  * This function copies the unique directory name for a given option to
235  * the "to" parameter. Some memory must be allocated to the "to" parameter.
236  */
237 
238 void unique_dir_access(int opt, char *to) {
239  if (( rank == 0 ) && ( verbose >= 1 )) {
240  fprintf( out_logfile, "V-1: Entering unique_dir_access...\n" );
241  fflush( out_logfile );
242  }
243 
244  if (opt == MK_UNI_DIR) {
245  MPI_Barrier(testComm);
246  sprintf( to, "%s/%s", testdir, unique_chdir_dir );
247  } else if (opt == STAT_SUB_DIR) {
248  sprintf( to, "%s/%s", testdir, unique_stat_dir );
249  } else if (opt == READ_SUB_DIR) {
250  sprintf( to, "%s/%s", testdir, unique_read_dir );
251  } else if (opt == RM_SUB_DIR) {
252  sprintf( to, "%s/%s", testdir, unique_rm_dir );
253  } else if (opt == RM_UNI_DIR) {
254  sprintf( to, "%s/%s", testdir, unique_rm_uni_dir );
255  }
256 }
257 
258 static void create_remove_dirs (const char *path, bool create, uint64_t itemNum) {
259  char curr_item[MAX_PATHLEN];
260  const char *operation = create ? "create" : "remove";
261 
262  if (( rank == 0 ) &&
263  ( verbose >= 3 ) &&
264  (itemNum % ITEM_COUNT==0 && (itemNum != 0))) {
265 
266  fprintf(out_logfile, "V-3: %s dir: "LLU"\n", operation, itemNum);
267  fflush(out_logfile);
268  }
269 
270  //create dirs
271  sprintf(curr_item, "%s/dir.%s%" PRIu64, path, create ? mk_name : rm_name, itemNum);
272  if (rank == 0 && verbose >= 3) {
273  fprintf(out_logfile, "V-3: create_remove_items_helper (dirs %s): curr_item is \"%s\"\n", operation, curr_item);
274  fflush(out_logfile);
275  }
276 
277  if (create) {
278  if (backend->mkdir(curr_item, DIRMODE, &param) == -1) {
279  FAIL("unable to create directory");
280  }
281  } else {
282  if (backend->rmdir(curr_item, &param) == -1) {
283  FAIL("unable to remove directory");
284  }
285  }
286 }
287 
288 static void remove_file (const char *path, uint64_t itemNum) {
289  char curr_item[MAX_PATHLEN];
290 
291  if (( rank == 0 ) &&
292  ( verbose >= 3 ) &&
293  (itemNum % ITEM_COUNT==0 && (itemNum != 0))) {
294 
295  fprintf(out_logfile, "V-3: remove file: "LLU"\n", itemNum);
296  fflush(out_logfile);
297  }
298 
299  //remove files
300  sprintf(curr_item, "%s/file.%s"LLU"", path, rm_name, itemNum);
301  if (rank == 0 && verbose >= 3) {
302  fprintf(out_logfile, "V-3: create_remove_items_helper (non-dirs remove): curr_item is \"%s\"\n", curr_item);
303  fflush(out_logfile);
304  }
305  if (!(shared_file && rank != 0)) {
306  backend->delete (curr_item, &param);
307  }
308 }
309 
310 static void create_file (const char *path, uint64_t itemNum) {
311  char curr_item[MAX_PATHLEN];
312  void *aiori_fh;
313 
314  if (( rank == 0 ) &&
315  ( verbose >= 3 ) &&
316  (itemNum % ITEM_COUNT==0 && (itemNum != 0))) {
317 
318  fprintf(out_logfile, "V-3: create file: "LLU"\n", itemNum);
319  fflush(out_logfile);
320  }
321 
322  //create files
323  sprintf(curr_item, "%s/file.%s"LLU"", path, mk_name, itemNum);
324  if ((rank == 0 && verbose >= 3) || verbose >= 5) {
325  fprintf(out_logfile, "V-3: create_remove_items_helper (non-dirs create): curr_item is \"%s\"\n", curr_item);
326  fflush(out_logfile);
327  }
328 
329  if (collective_creates) {
330  param.openFlags = IOR_WRONLY;
331 
332  if (rank == 0 && verbose >= 3) {
333  fprintf(out_logfile, "V-3: create_remove_items_helper (collective): open...\n" );
334  fflush( out_logfile );
335  }
336 
337  aiori_fh = backend->open (curr_item, &param);
338  if (NULL == aiori_fh) {
339  FAIL("unable to open file");
340  }
341 
342  /*
343  * !collective_creates
344  */
345  } else {
346  param.openFlags = IOR_CREAT | IOR_WRONLY;
347  param.filePerProc = !shared_file;
348 
349  if (rank == 0 && verbose >= 3) {
350  fprintf(out_logfile, "V-3: create_remove_items_helper (non-collective, shared): open...\n" );
351  fflush( out_logfile );
352  }
353 
354  aiori_fh = backend->create (curr_item, &param);
355  if (NULL == aiori_fh) {
356  FAIL("unable to create file");
357  }
358  }
359 
360  if (write_bytes > 0) {
361  if (rank == 0 && verbose >= 3) {
362  fprintf(out_logfile, "V-3: create_remove_items_helper: write...\n" );
363  fflush( out_logfile );
364  }
365 
366  /*
367  * According to Bill Loewe, writes are only done one time, so they are always at
368  * offset 0 (zero).
369  */
370  param.offset = 0;
371  param.fsyncPerWrite = sync_file;
372  if ( write_bytes != (size_t) backend->xfer (WRITE, aiori_fh, (IOR_size_t *) write_buffer, write_bytes, &param)) {
373  FAIL("unable to write file");
374  }
375  }
376 
377  if (rank == 0 && verbose >= 3) {
378  fprintf(out_logfile, "V-3: create_remove_items_helper: close...\n" );
379  fflush( out_logfile );
380  }
381 
382  backend->close (aiori_fh, &param);
383 }
384 
385 /* helper for creating/removing items */
386 void create_remove_items_helper(const int dirs, const int create, const char *path,
387  uint64_t itemNum, rank_progress_t * progress) {
388 
389  if (( rank == 0 ) && ( verbose >= 1 )) {
390  fprintf( out_logfile, "V-1: Entering create_remove_items_helper...\n" );
391  fflush( out_logfile );
392  }
393 
394  for (uint64_t i = progress->items_start; i < progress->items_per_dir ; ++i) {
395  if (!dirs) {
396  if (create) {
397  create_file (path, itemNum + i);
398  } else {
399  remove_file (path, itemNum + i);
400  }
401  } else {
402  create_remove_dirs (path, create, itemNum + i);
403  }
404  if(CHECK_STONE_WALL(progress)){
405  if(progress->items_done == 0){
406  progress->items_done = i + 1;
407  }
408  return;
409  }
410  }
411  progress->items_done = progress->items_per_dir;
412 }
413 
414 /* helper function to do collective operations */
415 void collective_helper(const int dirs, const int create, const char* path, uint64_t itemNum, rank_progress_t * progress) {
416  char curr_item[MAX_PATHLEN];
417 
418  if (( rank == 0 ) && ( verbose >= 1 )) {
419  fprintf( out_logfile, "V-1: Entering collective_helper...\n" );
420  fflush( out_logfile );
421  }
422  for (uint64_t i = progress->items_start ; i < progress->items_per_dir ; ++i) {
423  if (dirs) {
424  create_remove_dirs (path, create, itemNum + i);
425  continue;
426  }
427 
428  sprintf(curr_item, "%s/file.%s"LLU"", path, create ? mk_name : rm_name, itemNum+i);
429  if (rank == 0 && verbose >= 3) {
430  fprintf(out_logfile, "V-3: create file: %s\n", curr_item);
431  fflush(out_logfile);
432  }
433 
434  if (create) {
435  void *aiori_fh;
436 
437  //create files
438  param.openFlags = IOR_WRONLY | IOR_CREAT;
439  aiori_fh = backend->create (curr_item, &param);
440  if (NULL == aiori_fh) {
441  FAIL("unable to create file");
442  }
443 
444  backend->close (aiori_fh, &param);
445  } else if (!(shared_file && rank != 0)) {
446  //remove files
447  backend->delete (curr_item, &param);
448  }
449  if(CHECK_STONE_WALL(progress)){
450  progress->items_done = i + 1;
451  return;
452  }
453  }
454  progress->items_done = progress->items_per_dir;
455 }
456 
457 /* recusive function to create and remove files/directories from the
458  directory tree */
459 void create_remove_items(int currDepth, const int dirs, const int create, const int collective, const char *path, uint64_t dirNum, rank_progress_t * progress) {
460  unsigned i;
461  char dir[MAX_PATHLEN];
462  char temp_path[MAX_PATHLEN];
463  unsigned long long currDir = dirNum;
464 
465 
466  if (( rank == 0 ) && ( verbose >= 1 )) {
467  fprintf( out_logfile, "V-1: Entering create_remove_items, currDepth = %d...\n", currDepth );
468  fflush( out_logfile );
469  }
470 
471 
472  memset(dir, 0, MAX_PATHLEN);
473  strcpy(temp_path, path);
474 
475  if (rank == 0 && verbose >= 3) {
476  fprintf(out_logfile, "V-3: create_remove_items (start): temp_path is \"%s\"\n", temp_path );
477  fflush(out_logfile);
478  }
479 
480  if (currDepth == 0) {
481  /* create items at this depth */
482  if (!leaf_only || (depth == 0 && leaf_only)) {
483  if (collective) {
484  collective_helper(dirs, create, temp_path, 0, progress);
485  } else {
486  create_remove_items_helper(dirs, create, temp_path, 0, progress);
487  }
488  }
489 
490  if (depth > 0) {
491  create_remove_items(++currDepth, dirs, create,
492  collective, temp_path, ++dirNum, progress);
493  }
494 
495  } else if (currDepth <= depth) {
496  /* iterate through the branches */
497  for (i=0; i<branch_factor; i++) {
498 
499  /* determine the current branch and append it to the path */
500  sprintf(dir, "%s.%llu/", base_tree_name, currDir);
501  strcat(temp_path, "/");
502  strcat(temp_path, dir);
503 
504  if (rank == 0 && verbose >= 3) {
505  fprintf(out_logfile, "V-3: create_remove_items (for loop): temp_path is \"%s\"\n", temp_path );
506  fflush(out_logfile);
507  }
508 
509  /* create the items in this branch */
510  if (!leaf_only || (leaf_only && currDepth == depth)) {
511  if (collective) {
512  collective_helper(dirs, create, temp_path, currDir*items_per_dir, progress);
513  } else {
514  create_remove_items_helper(dirs, create, temp_path, currDir*items_per_dir, progress);
515  }
516  }
517 
518  /* make the recursive call for the next level below this branch */
520  ++currDepth,
521  dirs,
522  create,
523  collective,
524  temp_path,
525  ( currDir * ( unsigned long long )branch_factor ) + 1,
526  progress
527  );
528  currDepth--;
529 
530  /* reset the path */
531  strcpy(temp_path, path);
532  currDir++;
533  }
534  }
535 }
536 
537 /* stats all of the items created as specified by the input parameters */
538 void mdtest_stat(const int random, const int dirs, const long dir_iter, const char *path, rank_progress_t * progress) {
539  struct stat buf;
540  uint64_t parent_dir, item_num = 0;
541  char item[MAX_PATHLEN], temp[MAX_PATHLEN];
542 
543  if (( rank == 0 ) && ( verbose >= 1 )) {
544  fprintf( out_logfile, "V-1: Entering mdtest_stat...\n" );
545  fflush( out_logfile );
546  }
547 
548  uint64_t stop_items = items;
549 
550  if( directory_loops != 1 ){
551  stop_items = items_per_dir;
552  }
553 
554  /* iterate over all of the item IDs */
555  for (uint64_t i = 0 ; i < stop_items ; ++i) {
556  /*
557  * It doesn't make sense to pass the address of the array because that would
558  * be like passing char **. Tested it on a Cray and it seems to work either
559  * way, but it seems that it is correct without the "&".
560  *
561  memset(&item, 0, MAX_PATHLEN);
562  */
563  memset(item, 0, MAX_PATHLEN);
564  memset(temp, 0, MAX_PATHLEN);
565 
566 
567  /* determine the item number to stat */
568  if (random) {
569  item_num = rand_array[i];
570  } else {
571  item_num = i;
572  }
573 
574  /* make adjustments if in leaf only mode*/
575  if (leaf_only) {
576  item_num += items_per_dir *
577  (num_dirs_in_tree - (uint64_t) pow( branch_factor, depth ));
578  }
579 
580  /* create name of file/dir to stat */
581  if (dirs) {
582  if (rank == 0 && verbose >= 3 && (i%ITEM_COUNT == 0) && (i != 0)) {
583  fprintf(out_logfile, "V-3: stat dir: "LLU"\n", i);
584  fflush(out_logfile);
585  }
586  sprintf(item, "dir.%s"LLU"", stat_name, item_num);
587  } else {
588  if (rank == 0 && verbose >= 3 && (i%ITEM_COUNT == 0) && (i != 0)) {
589  fprintf(out_logfile, "V-3: stat file: "LLU"\n", i);
590  fflush(out_logfile);
591  }
592  sprintf(item, "file.%s"LLU"", stat_name, item_num);
593  }
594 
595  /* determine the path to the file/dir to be stat'ed */
596  parent_dir = item_num / items_per_dir;
597 
598  if (parent_dir > 0) { //item is not in tree's root directory
599 
600  /* prepend parent directory to item's path */
601  sprintf(temp, "%s."LLU"/%s", base_tree_name, parent_dir, item);
602  strcpy(item, temp);
603 
604  //still not at the tree's root dir
605  while (parent_dir > branch_factor) {
606  parent_dir = (uint64_t) ((parent_dir-1) / branch_factor);
607  sprintf(temp, "%s."LLU"/%s", base_tree_name, parent_dir, item);
608  strcpy(item, temp);
609  }
610  }
611 
612  /* Now get item to have the full path */
613  sprintf( temp, "%s/%s", path, item );
614  strcpy( item, temp );
615 
616  /* below temp used to be hiername */
617  if (rank == 0 && verbose >= 3) {
618  if (dirs) {
619  fprintf(out_logfile, "V-3: mdtest_stat dir : %s\n", item);
620  } else {
621  fprintf(out_logfile, "V-3: mdtest_stat file: %s\n", item);
622  }
623  fflush(out_logfile);
624  }
625 
626  if (-1 == backend->stat (item, &buf, &param)) {
627  if (dirs) {
628  if ( verbose >= 3 ) {
629  fprintf( out_logfile, "V-3: Stat'ing directory \"%s\"\n", item );
630  fflush( out_logfile );
631  }
632  FAIL("unable to stat directory");
633  } else {
634  if ( verbose >= 3 ) {
635  fprintf( out_logfile, "V-3: Stat'ing file \"%s\"\n", item );
636  fflush( out_logfile );
637  }
638  FAIL("unable to stat file");
639  }
640  }
641  }
642 }
643 
644 
645 /* reads all of the items created as specified by the input parameters */
646 void mdtest_read(int random, int dirs, const long dir_iter, char *path) {
647  uint64_t parent_dir, item_num = 0;
648  char item[MAX_PATHLEN], temp[MAX_PATHLEN];
649  void *aiori_fh;
650 
651  if (( rank == 0 ) && ( verbose >= 1 )) {
652  fprintf( out_logfile, "V-1: Entering mdtest_read...\n" );
653  fflush( out_logfile );
654  }
655 
656  /* allocate read buffer */
657  if (read_bytes > 0) {
658  read_buffer = (char *)malloc(read_bytes);
659  if (read_buffer == NULL) {
660  FAIL("out of memory");
661  }
662  }
663 
664  uint64_t stop_items = items;
665 
666  if( directory_loops != 1 ){
667  stop_items = items_per_dir;
668  }
669 
670  /* iterate over all of the item IDs */
671  for (uint64_t i = 0 ; i < stop_items ; ++i) {
672  /*
673  * It doesn't make sense to pass the address of the array because that would
674  * be like passing char **. Tested it on a Cray and it seems to work either
675  * way, but it seems that it is correct without the "&".
676  *
677  * NTH: Both are technically correct in C.
678  *
679  * memset(&item, 0, MAX_PATHLEN);
680  */
681  memset(item, 0, MAX_PATHLEN);
682  memset(temp, 0, MAX_PATHLEN);
683 
684  /* determine the item number to read */
685  if (random) {
686  item_num = rand_array[i];
687  } else {
688  item_num = i;
689  }
690 
691  /* make adjustments if in leaf only mode*/
692  if (leaf_only) {
693  item_num += items_per_dir *
694  (num_dirs_in_tree - (uint64_t) pow (branch_factor, depth));
695  }
696 
697  /* create name of file to read */
698  if (!dirs) {
699  if (rank == 0 && verbose >= 3 && (i%ITEM_COUNT == 0) && (i != 0)) {
700  fprintf(out_logfile, "V-3: read file: "LLU"\n", i);
701  fflush(out_logfile);
702  }
703  sprintf(item, "file.%s"LLU"", read_name, item_num);
704  }
705 
706  /* determine the path to the file/dir to be read'ed */
707  parent_dir = item_num / items_per_dir;
708 
709  if (parent_dir > 0) { //item is not in tree's root directory
710 
711  /* prepend parent directory to item's path */
712  sprintf(temp, "%s."LLU"/%s", base_tree_name, parent_dir, item);
713  strcpy(item, temp);
714 
715  /* still not at the tree's root dir */
716  while (parent_dir > branch_factor) {
717  parent_dir = (unsigned long long) ((parent_dir-1) / branch_factor);
718  sprintf(temp, "%s."LLU"/%s", base_tree_name, parent_dir, item);
719  strcpy(item, temp);
720  }
721  }
722 
723  /* Now get item to have the full path */
724  sprintf( temp, "%s/%s", path, item );
725  strcpy( item, temp );
726 
727  /* below temp used to be hiername */
728  if (rank == 0 && verbose >= 3) {
729  if (!dirs) {
730  fprintf(out_logfile, "V-3: mdtest_read file: %s\n", item);
731  }
732  fflush(out_logfile);
733  }
734 
735  /* open file for reading */
736  param.openFlags = O_RDONLY;
737  aiori_fh = backend->open (item, &param);
738  if (NULL == aiori_fh) {
739  FAIL("unable to open file");
740  }
741 
742  /* read file */
743  if (read_bytes > 0) {
744  if (read_bytes != (size_t) backend->xfer (READ, aiori_fh, (IOR_size_t *) read_buffer, read_bytes, &param)) {
745  FAIL("unable to read file");
746  }
747  }
748 
749  /* close file */
750  backend->close (aiori_fh, &param);
751  }
752 }
753 
754 /* This method should be called by rank 0. It subsequently does all of
755  the creates and removes for the other ranks */
756 void collective_create_remove(const int create, const int dirs, const int ntasks, const char *path, rank_progress_t * progress) {
757  char temp[MAX_PATHLEN];
758 
759  if (( rank == 0 ) && ( verbose >= 1 )) {
760  fprintf( out_logfile, "V-1: Entering collective_create_remove...\n" );
761  fflush( out_logfile );
762  }
763 
764  /* rank 0 does all of the creates and removes for all of the ranks */
765  for (int i = 0 ; i < ntasks ; ++i) {
766  memset(temp, 0, MAX_PATHLEN);
767 
768  strcpy(temp, testdir);
769  strcat(temp, "/");
770 
771  /* set the base tree name appropriately */
772  if (unique_dir_per_task) {
773  sprintf(base_tree_name, "mdtest_tree.%d", i);
774  } else {
775  sprintf(base_tree_name, "mdtest_tree");
776  }
777 
778  /* Setup to do I/O to the appropriate test dir */
779  strcat(temp, base_tree_name);
780  strcat(temp, ".0");
781 
782  /* set all item names appropriately */
783  if (!shared_file) {
784  sprintf(mk_name, "mdtest.%d.", (i+(0*nstride))%ntasks);
785  sprintf(stat_name, "mdtest.%d.", (i+(1*nstride))%ntasks);
786  sprintf(read_name, "mdtest.%d.", (i+(2*nstride))%ntasks);
787  sprintf(rm_name, "mdtest.%d.", (i+(3*nstride))%ntasks);
788  }
789  if (unique_dir_per_task) {
790  sprintf(unique_mk_dir, "%s/mdtest_tree.%d.0", testdir,
791  (i+(0*nstride))%ntasks);
792  sprintf(unique_chdir_dir, "%s/mdtest_tree.%d.0", testdir,
793  (i+(1*nstride))%ntasks);
794  sprintf(unique_stat_dir, "%s/mdtest_tree.%d.0", testdir,
795  (i+(2*nstride))%ntasks);
796  sprintf(unique_read_dir, "%s/mdtest_tree.%d.0", testdir,
797  (i+(3*nstride))%ntasks);
798  sprintf(unique_rm_dir, "%s/mdtest_tree.%d.0", testdir,
799  (i+(4*nstride))%ntasks);
800  sprintf(unique_rm_uni_dir, "%s", testdir);
801  }
802 
803  /* Now that everything is set up as it should be, do the create or remove */
804  if (rank == 0 && verbose >= 3) {
805  fprintf(out_logfile, "V-3: collective_create_remove (create_remove_items): temp is \"%s\"\n", temp);
806  fflush( out_logfile );
807  }
808 
809  create_remove_items(0, dirs, create, 1, temp, 0, progress);
810  }
811 
812  /* reset all of the item names */
813  if (unique_dir_per_task) {
814  sprintf(base_tree_name, "mdtest_tree.0");
815  } else {
816  sprintf(base_tree_name, "mdtest_tree");
817  }
818  if (!shared_file) {
819  sprintf(mk_name, "mdtest.%d.", (0+(0*nstride))%ntasks);
820  sprintf(stat_name, "mdtest.%d.", (0+(1*nstride))%ntasks);
821  sprintf(read_name, "mdtest.%d.", (0+(2*nstride))%ntasks);
822  sprintf(rm_name, "mdtest.%d.", (0+(3*nstride))%ntasks);
823  }
824  if (unique_dir_per_task) {
825  sprintf(unique_mk_dir, "%s/mdtest_tree.%d.0", testdir,
826  (0+(0*nstride))%ntasks);
827  sprintf(unique_chdir_dir, "%s/mdtest_tree.%d.0", testdir,
828  (0+(1*nstride))%ntasks);
829  sprintf(unique_stat_dir, "%s/mdtest_tree.%d.0", testdir,
830  (0+(2*nstride))%ntasks);
831  sprintf(unique_read_dir, "%s/mdtest_tree.%d.0", testdir,
832  (0+(3*nstride))%ntasks);
833  sprintf(unique_rm_dir, "%s/mdtest_tree.%d.0", testdir,
834  (0+(4*nstride))%ntasks);
835  sprintf(unique_rm_uni_dir, "%s", testdir);
836  }
837 }
838 
839 void directory_test(const int iteration, const int ntasks, const char *path, rank_progress_t * progress) {
840  int size;
841  double t[5] = {0};
842  char temp_path[MAX_PATHLEN];
843 
844  MPI_Comm_size(testComm, &size);
845 
846  if (( rank == 0 ) && ( verbose >= 1 )) {
847  fprintf( out_logfile, "V-1: Entering directory_test...\n" );
848  fflush( out_logfile );
849  }
850 
851  MPI_Barrier(testComm);
852  t[0] = MPI_Wtime();
853 
854  /* create phase */
855  if(create_only) {
856  for (int dir_iter = 0; dir_iter < directory_loops; dir_iter ++){
857  prep_testdir(iteration, dir_iter);
858  if (unique_dir_per_task) {
859  unique_dir_access(MK_UNI_DIR, temp_path);
861  offset_timers(t, 0);
862  }
863  } else {
864  sprintf( temp_path, "%s/%s", testdir, path );
865  }
866 
867  if (verbose >= 3 && rank == 0) {
868  fprintf(out_logfile, "V-3: directory_test: create path is \"%s\"\n", temp_path );
869  fflush( out_logfile );
870  }
871 
872  /* "touch" the files */
873  if (collective_creates) {
874  if (rank == 0) {
875  collective_create_remove(1, 1, ntasks, temp_path, progress);
876  }
877  } else {
878  /* create directories */
879  create_remove_items(0, 1, 1, 0, temp_path, 0, progress);
880  }
881  }
882  }
883 
884  if (barriers) {
885  MPI_Barrier(testComm);
886  }
887  t[1] = MPI_Wtime();
888 
889  /* stat phase */
890  if (stat_only) {
891  for (int dir_iter = 0; dir_iter < directory_loops; dir_iter ++){
892  prep_testdir(iteration, dir_iter);
893  if (unique_dir_per_task) {
894  unique_dir_access(STAT_SUB_DIR, temp_path);
896  offset_timers(t, 1);
897  }
898  } else {
899  sprintf( temp_path, "%s/%s", testdir, path );
900  }
901 
902  if (verbose >= 3 && rank == 0) {
903  fprintf(out_logfile, "V-3: directory_test: stat path is \"%s\"\n", temp_path );
904  fflush( out_logfile );
905  }
906 
907  /* stat directories */
908  if (random_seed > 0) {
909  mdtest_stat(1, 1, dir_iter, temp_path, progress);
910  } else {
911  mdtest_stat(0, 1, dir_iter, temp_path, progress);
912  }
913  }
914  }
915 
916  if (barriers) {
917  MPI_Barrier(testComm);
918  }
919  t[2] = MPI_Wtime();
920 
921  /* read phase */
922  if (read_only) {
923  for (int dir_iter = 0; dir_iter < directory_loops; dir_iter ++){
924  prep_testdir(iteration, dir_iter);
925  if (unique_dir_per_task) {
926  unique_dir_access(READ_SUB_DIR, temp_path);
928  offset_timers(t, 2);
929  }
930  } else {
931  sprintf( temp_path, "%s/%s", testdir, path );
932  }
933 
934  if (verbose >= 3 && rank == 0) {
935  fprintf(out_logfile, "V-3: directory_test: read path is \"%s\"\n", temp_path );
936  fflush( out_logfile );
937  }
938 
939  /* read directories */
940  if (random_seed > 0) {
941  ; /* N/A */
942  } else {
943  ; /* N/A */
944  }
945  }
946  }
947 
948  if (barriers) {
949  MPI_Barrier(testComm);
950  }
951  t[3] = MPI_Wtime();
952 
953  if (remove_only) {
954  for (int dir_iter = 0; dir_iter < directory_loops; dir_iter ++){
955  prep_testdir(iteration, dir_iter);
956  if (unique_dir_per_task) {
957  unique_dir_access(RM_SUB_DIR, temp_path);
959  offset_timers(t, 3);
960  }
961  } else {
962  sprintf( temp_path, "%s/%s", testdir, path );
963  }
964 
965  if (verbose >= 3 && rank == 0) {
966  fprintf(out_logfile, "V-3: directory_test: remove directories path is \"%s\"\n", temp_path );
967  fflush( out_logfile );
968  }
969 
970  /* remove directories */
971  if (collective_creates) {
972  if (rank == 0) {
973  collective_create_remove(0, 1, ntasks, temp_path, progress);
974  }
975  } else {
976  create_remove_items(0, 1, 0, 0, temp_path, 0, progress);
977  }
978  }
979  }
980 
981  if (barriers) {
982  MPI_Barrier(testComm);
983  }
984  t[4] = MPI_Wtime();
985 
986  if (remove_only) {
987  if (unique_dir_per_task) {
988  unique_dir_access(RM_UNI_DIR, temp_path);
989  } else {
990  sprintf( temp_path, "%s/%s", testdir, path );
991  }
992 
993  if (verbose >= 3 && rank == 0) {
994  fprintf(out_logfile, "V-3: directory_test: remove unique directories path is \"%s\"\n", temp_path );
995  fflush( out_logfile );
996  }
997  }
998 
1000  offset_timers(t, 4);
1001  }
1002 
1003  /* calculate times */
1004  if (create_only) {
1005  summary_table[iteration].rate[0] = items*size/(t[1] - t[0]);
1006  summary_table[iteration].time[0] = t[1] - t[0];
1007  summary_table[iteration].items[0] = items*size;
1008  summary_table[iteration].stonewall_last_item[0] = items;
1009  }
1010  if (stat_only) {
1011  summary_table[iteration].rate[1] = items*size/(t[2] - t[1]);
1012  summary_table[iteration].time[1] = t[2] - t[1];
1013  summary_table[iteration].items[1] = items*size;
1014  summary_table[iteration].stonewall_last_item[1] = items;
1015  }
1016  if (read_only) {
1017  summary_table[iteration].rate[2] = items*size/(t[3] - t[2]);
1018  summary_table[iteration].time[2] = t[3] - t[2];
1019  summary_table[iteration].items[2] = items*size;
1020  summary_table[iteration].stonewall_last_item[2] = items;
1021  }
1022  if (remove_only) {
1023  summary_table[iteration].rate[3] = items*size/(t[4] - t[3]);
1024  summary_table[iteration].time[3] = t[4] - t[3];
1025  summary_table[iteration].items[3] = items*size;
1026  summary_table[iteration].stonewall_last_item[3] = items;
1027  }
1028 
1029  if (verbose >= 1 && rank == 0) {
1030  fprintf(out_logfile, "V-1: Directory creation: %14.3f sec, %14.3f ops/sec\n",
1031  t[1] - t[0], summary_table[iteration].rate[0]);
1032  fprintf(out_logfile, "V-1: Directory stat : %14.3f sec, %14.3f ops/sec\n",
1033  t[2] - t[1], summary_table[iteration].rate[1]);
1034 /* N/A
1035  fprintf(out_logfile, "V-1: Directory read : %14.3f sec, %14.3f ops/sec\n",
1036  t[3] - t[2], summary_table[iteration].rate[2]);
1037 */
1038  fprintf(out_logfile, "V-1: Directory removal : %14.3f sec, %14.3f ops/sec\n",
1039  t[4] - t[3], summary_table[iteration].rate[3]);
1040  fflush(out_logfile);
1041  }
1042 }
1043 
1044 /* Returns if the stonewall was hit */
1045 int updateStoneWallIterations(int iteration, rank_progress_t * progress, double tstart){
1046  int hit = 0;
1047  if (verbose >= 1 ) {
1048  fprintf( out_logfile, "V-1: rank %d stonewall hit with %lld items\n", rank, (long long) progress->items_done );
1049  fflush( out_logfile );
1050  }
1051  uint64_t done = progress->items_done;
1052  long long unsigned max_iter = 0;
1053  MPI_Allreduce(& progress->items_done, & max_iter, 1, MPI_LONG_LONG_INT, MPI_MAX, testComm);
1054  summary_table[iteration].stonewall_time[MDTEST_FILE_CREATE_NUM] = MPI_Wtime() - tstart;
1055 
1056  // continue to the maximum...
1057  long long min_accessed = 0;
1058  MPI_Reduce(& progress->items_done, & min_accessed, 1, MPI_LONG_LONG_INT, MPI_MIN, 0, testComm);
1059  long long sum_accessed = 0;
1060  MPI_Reduce(& progress->items_done, & sum_accessed, 1, MPI_LONG_LONG_INT, MPI_SUM, 0, testComm);
1061 
1062  if(items != (sum_accessed / size)){
1063  summary_table[iteration].stonewall_item_sum[MDTEST_FILE_CREATE_NUM] = sum_accessed;
1064  summary_table[iteration].stonewall_item_min[MDTEST_FILE_CREATE_NUM] = min_accessed * size;
1065  if (rank == 0){
1066  fprintf( out_logfile, "Continue stonewall hit min: %lld max: %lld avg: %.1f \n", min_accessed, max_iter, ((double) sum_accessed) / size);
1067  fflush( out_logfile );
1068  }
1069  hit = 1;
1070  }
1071  progress->items_start = done;
1072  progress->items_per_dir = max_iter;
1073 
1074  return hit;
1075 }
1076 
1077 void file_test(const int iteration, const int ntasks, const char *path, rank_progress_t * progress) {
1078  int size;
1079  double t[5] = {0};
1080  char temp_path[MAX_PATHLEN];
1081  MPI_Comm_size(testComm, &size);
1082 
1083  if (( rank == 0 ) && ( verbose >= 1 )) {
1084  fprintf( out_logfile, "V-1: Entering file_test...\n" );
1085  fflush( out_logfile );
1086  }
1087 
1088  MPI_Barrier(testComm);
1089  t[0] = MPI_Wtime();
1090 
1091  /* create phase */
1092  if (create_only ) {
1093  for (int dir_iter = 0; dir_iter < directory_loops; dir_iter ++){
1094  prep_testdir(iteration, dir_iter);
1095 
1096  if (unique_dir_per_task) {
1097  unique_dir_access(MK_UNI_DIR, temp_path);
1098  if (!time_unique_dir_overhead) {
1099  offset_timers(t, 0);
1100  }
1101  } else {
1102  sprintf( temp_path, "%s/%s", testdir, path );
1103  }
1104 
1105  if (verbose >= 3 && rank == 0) {
1106  fprintf(out_logfile, "V-3: file_test: create path is \"%s\"\n", temp_path );
1107  fflush( out_logfile );
1108  }
1109 
1110  /* "touch" the files */
1111  if (collective_creates) {
1112  if (rank == 0) {
1113  collective_create_remove(1, 0, ntasks, temp_path, progress);
1114  }
1115  MPI_Barrier(testComm);
1116  }
1117 
1118  /* create files */
1119  create_remove_items(0, 0, 1, 0, temp_path, 0, progress);
1121  int hit = updateStoneWallIterations(iteration, progress, t[0]);
1122 
1123  if (hit){
1124  progress->stone_wall_timer_seconds = 0;
1125  if (verbose > 1){
1126  printf("stonewall rank %d: %lld of %lld \n", rank, (long long) progress->items_start, (long long) progress->items_per_dir);
1127  }
1128  create_remove_items(0, 0, 1, 0, temp_path, 0, progress);
1129  // now reset the values
1131  items = progress->items_done;
1132  }
1135  }
1136  // reset stone wall timer to allow proper cleanup
1137  progress->stone_wall_timer_seconds = 0;
1138  }
1139  }
1140  }else{
1142  int64_t expected_items;
1143  /* The number of items depends on the stonewalling file */
1145  if(expected_items >= 0){
1146  items = expected_items;
1147  progress->items_per_dir = items;
1148  }
1149  if (rank == 0) {
1150  if(expected_items == -1){
1151  fprintf(out_logfile, "WARNING: could not read stonewall status file\n");
1152  }else if(verbose >= 1){
1153  fprintf(out_logfile, "Read stonewall status; items: "LLU"\n", items);
1154  }
1155  }
1156  }
1157  }
1158 
1159  if (barriers) {
1160  MPI_Barrier(testComm);
1161  }
1162  t[1] = MPI_Wtime();
1163 
1164  /* stat phase */
1165  if (stat_only ) {
1166  for (int dir_iter = 0; dir_iter < directory_loops; dir_iter ++){
1167  prep_testdir(iteration, dir_iter);
1168  if (unique_dir_per_task) {
1169  unique_dir_access(STAT_SUB_DIR, temp_path);
1170  if (!time_unique_dir_overhead) {
1171  offset_timers(t, 1);
1172  }
1173  } else {
1174  sprintf( temp_path, "%s/%s", testdir, path );
1175  }
1176 
1177  if (verbose >= 3 && rank == 0) {
1178  fprintf(out_logfile, "V-3: file_test: stat path is \"%s\"\n", temp_path );
1179  fflush( out_logfile );
1180  }
1181 
1182  /* stat files */
1183  mdtest_stat((random_seed > 0 ? 1 : 0), 0, dir_iter, temp_path, progress);
1184  }
1185  }
1186 
1187  if (barriers) {
1188  MPI_Barrier(testComm);
1189  }
1190  t[2] = MPI_Wtime();
1191 
1192  /* read phase */
1193  if (read_only ) {
1194  for (int dir_iter = 0; dir_iter < directory_loops; dir_iter ++){
1195  prep_testdir(iteration, dir_iter);
1196  if (unique_dir_per_task) {
1197  unique_dir_access(READ_SUB_DIR, temp_path);
1198  if (!time_unique_dir_overhead) {
1199  offset_timers(t, 2);
1200  }
1201  } else {
1202  sprintf( temp_path, "%s/%s", testdir, path );
1203  }
1204 
1205  if (verbose >= 3 && rank == 0) {
1206  fprintf(out_logfile, "V-3: file_test: read path is \"%s\"\n", temp_path );
1207  fflush( out_logfile );
1208  }
1209 
1210  /* read files */
1211  if (random_seed > 0) {
1212  mdtest_read(1,0, dir_iter, temp_path);
1213  } else {
1214  mdtest_read(0,0, dir_iter, temp_path);
1215  }
1216  }
1217  }
1218 
1219  if (barriers) {
1220  MPI_Barrier(testComm);
1221  }
1222  t[3] = MPI_Wtime();
1223 
1224  if (remove_only) {
1225  progress->items_start = 0;
1226 
1227  for (int dir_iter = 0; dir_iter < directory_loops; dir_iter ++){
1228  prep_testdir(iteration, dir_iter);
1229  if (unique_dir_per_task) {
1230  unique_dir_access(RM_SUB_DIR, temp_path);
1231  if (!time_unique_dir_overhead) {
1232  offset_timers(t, 3);
1233  }
1234  } else {
1235  sprintf( temp_path, "%s/%s", testdir, path );
1236  }
1237 
1238  if (verbose >= 3 && rank == 0) {
1239  fprintf(out_logfile, "V-3: file_test: rm directories path is \"%s\"\n", temp_path );
1240  fflush( out_logfile );
1241  }
1242 
1243  if (collective_creates) {
1244  if (rank == 0) {
1245  collective_create_remove(0, 0, ntasks, temp_path, progress);
1246  }
1247  } else {
1248  create_remove_items(0, 0, 0, 0, temp_path, 0, progress);
1249  }
1250  }
1251  }
1252 
1253  if (barriers) {
1254  MPI_Barrier(testComm);
1255  }
1256  t[4] = MPI_Wtime();
1257  if (remove_only) {
1258  if (unique_dir_per_task) {
1259  unique_dir_access(RM_UNI_DIR, temp_path);
1260  } else {
1261  strcpy( temp_path, path );
1262  }
1263 
1264  if (verbose >= 3 && rank == 0) {
1265  fprintf(out_logfile, "V-3: file_test: rm unique directories path is \"%s\"\n", temp_path );
1266  fflush( out_logfile );
1267  }
1268  }
1269 
1271  offset_timers(t, 4);
1272  }
1273 
1274  if(num_dirs_in_tree_calc){ /* this is temporary fix needed when using -n and -i together */
1276  }
1277 
1278  /* calculate times */
1279  if (create_only) {
1280  summary_table[iteration].rate[4] = items*size/(t[1] - t[0]);
1281  summary_table[iteration].time[4] = t[1] - t[0];
1282  summary_table[iteration].items[4] = items*size;
1283  summary_table[iteration].stonewall_last_item[4] = items;
1284  }
1285  if (stat_only) {
1286  summary_table[iteration].rate[5] = items*size/(t[2] - t[1]);
1287  summary_table[iteration].time[5] = t[2] - t[1];
1288  summary_table[iteration].items[5] = items*size;
1289  summary_table[iteration].stonewall_last_item[5] = items;
1290  }
1291  if (read_only) {
1292  summary_table[iteration].rate[6] = items*size/(t[3] - t[2]);
1293  summary_table[iteration].time[6] = t[3] - t[2];
1294  summary_table[iteration].items[6] = items*size;
1295  summary_table[iteration].stonewall_last_item[6] = items;
1296  }
1297  if (remove_only) {
1298  summary_table[iteration].rate[7] = items*size/(t[4] - t[3]);
1299  summary_table[iteration].time[7] = t[4] - t[3];
1300  summary_table[iteration].items[7] = items*size;
1301  summary_table[iteration].stonewall_last_item[7] = items;
1302  }
1303 
1304  if (verbose >= 1 && rank == 0) {
1305  fprintf(out_logfile, "V-1: File creation : %14.3f sec, %14.3f ops/sec\n",
1306  t[1] - t[0], summary_table[iteration].rate[4]);
1307  fprintf(out_logfile, "V-1: File stat : %14.3f sec, %14.3f ops/sec\n",
1308  t[2] - t[1], summary_table[iteration].rate[5]);
1309  fprintf(out_logfile, "V-1: File read : %14.3f sec, %14.3f ops/sec\n",
1310  t[3] - t[2], summary_table[iteration].rate[6]);
1311  fprintf(out_logfile, "V-1: File removal : %14.3f sec, %14.3f ops/sec\n",
1312  t[4] - t[3], summary_table[iteration].rate[7]);
1313  fflush(out_logfile);
1314  }
1315 }
1316 
1317 void print_help (void) {
1318  int j;
1319 
1320  char APIs[1024];
1321  aiori_supported_apis(APIs);
1322  char apiStr[1024];
1323  sprintf(apiStr, "API for I/O [%s]", APIs);
1324 
1325  fprintf(out_logfile,
1326  "Usage: mdtest [-b branching_factor] [-B] [-c] [-C] [-d testdir] [-D] [-e number_of_bytes_to_read]\n"
1327  " [-E] [-f first] [-F] [-h] [-i iterations] [-I items_per_dir] [-l last] [-L]\n"
1328  " [-n number_of_items] [-N stride_length] [-p seconds] [-r]\n"
1329  " [-R[seed]] [-s stride] [-S] [-t] [-T] [-u] [-v] [-a API]\n"
1330  " [-V verbosity_value] [-w number_of_bytes_to_write] [-W seconds] [-y] [-z depth] -Z\n"
1331  "\t-a: %s\n"
1332  "\t-b: branching factor of hierarchical directory structure\n"
1333  "\t-B: no barriers between phases\n"
1334  "\t-c: collective creates: task 0 does all creates\n"
1335  "\t-C: only create files/dirs\n"
1336  "\t-d: the directory in which the tests will run\n"
1337  "\t-D: perform test on directories only (no files)\n"
1338  "\t-e: bytes to read from each file\n"
1339  "\t-E: only read files/dir\n"
1340  "\t-f: first number of tasks on which the test will run\n"
1341  "\t-F: perform test on files only (no directories)\n"
1342  "\t-h: prints this help message\n"
1343  "\t-i: number of iterations the test will run\n"
1344  "\t-I: number of items per directory in tree\n"
1345  "\t-l: last number of tasks on which the test will run\n"
1346  "\t-L: files only at leaf level of tree\n"
1347  "\t-n: every process will creat/stat/read/remove # directories and files\n"
1348  "\t-N: stride # between neighbor tasks for file/dir operation (local=0)\n"
1349  "\t-p: pre-iteration delay (in seconds)\n"
1350  "\t-r: only remove files or directories left behind by previous runs\n"
1351  "\t-R: randomly stat files (optional argument for random seed)\n"
1352  "\t-s: stride between the number of tasks for each test\n"
1353  "\t-S: shared file access (file only, no directories)\n"
1354  "\t-t: time unique working directory overhead\n"
1355  "\t-T: only stat files/dirs\n"
1356  "\t-u: unique working directory for each task\n"
1357  "\t-v: verbosity (each instance of option increments by one)\n"
1358  "\t-V: verbosity value\n"
1359  "\t-w: bytes to write to each file after it is created\n"
1360  "\t-W: number in seconds; stonewall timer, write as many seconds and ensure all processes did the same number of operations (currently only stops during create phase)\n"
1361  "\t-x: StoneWallingStatusFile; contains the number of iterations of the creation phase, can be used to split phases across runs\n"
1362  "\t-y: sync file after writing\n"
1363  "\t-z: depth of hierarchical directory structure\n"
1364  "\t-Z: print time instead of rate\n",
1365  apiStr
1366  );
1367 
1368  MPI_Initialized(&j);
1369  if (j) {
1370  MPI_Finalize();
1371  }
1372  exit(0);
1373 }
1374 
1375 int calc_allreduce_index(int iter, int rank, int op){
1376  int tableSize = MDTEST_LAST_NUM;
1377  return iter * tableSize * size + rank * tableSize + op;
1378 }
1379 
1380 void summarize_results(int iterations) {
1381  char access[MAX_PATHLEN];
1382  int i, j, k;
1383  int start, stop, tableSize = MDTEST_LAST_NUM;
1384  double min, max, mean, sd, sum = 0, var = 0, curr = 0;
1385 
1386  double all[iterations * size * tableSize];
1387 
1388 
1389  if (( rank == 0 ) && ( verbose >= 1 )) {
1390  fprintf( out_logfile, "V-1: Entering summarize_results...\n" );
1391  fflush( out_logfile );
1392  }
1393 
1394  MPI_Barrier(testComm);
1395  for(int i=0; i < iterations; i++){
1396  if(print_time){
1397  MPI_Gather(& summary_table[i].time[0], tableSize, MPI_DOUBLE, & all[i*tableSize*size], tableSize, MPI_DOUBLE, 0, testComm);
1398  }else{
1399  MPI_Gather(& summary_table[i].rate[0], tableSize, MPI_DOUBLE, & all[i*tableSize*size], tableSize, MPI_DOUBLE, 0, testComm);
1400  }
1401  }
1402 
1403  if (rank == 0) {
1404 
1405  fprintf(out_logfile, "\nSUMMARY %s: (of %d iterations)\n", print_time ? "time": "rate", iterations);
1406  fprintf(out_logfile,
1407  " Operation Max Min Mean Std Dev\n");
1408  fprintf(out_logfile,
1409  " --------- --- --- ---- -------\n");
1410  fflush(out_logfile);
1411 
1412  /* if files only access, skip entries 0-3 (the dir tests) */
1413  if (files_only && !dirs_only) {
1414  start = 4;
1415  } else {
1416  start = 0;
1417  }
1418 
1419  /* if directories only access, skip entries 4-7 (the file tests) */
1420  if (dirs_only && !files_only) {
1421  stop = 4;
1422  } else {
1423  stop = 8;
1424  }
1425 
1426  /* special case: if no directory or file tests, skip all */
1427  if (!dirs_only && !files_only) {
1428  start = stop = 0;
1429  }
1430 
1431  /* calculate aggregates */
1432  if (barriers) {
1433  double maxes[iterations];
1434 
1435 
1436  /* Because each proc times itself, in the case of barriers we
1437  * have to backwards calculate the time to simulate the use
1438  * of barriers.
1439  */
1440  for (i = start; i < stop; i++) {
1441  for (j=0; j<iterations; j++) {
1442  maxes[j] = all[j*tableSize + i];
1443  for (k=0; k<size; k++) {
1444  curr = all[calc_allreduce_index(j, k, i)];
1445  //For verification in 3.2 printf("%d %d %d = %f\n", j, k, i, curr);
1446  if (maxes[j] < curr) {
1447  maxes[j] = curr;
1448  }
1449  }
1450  }
1451 
1452  min = max = maxes[0];
1453  for (j=0; j<iterations; j++) {
1454  if (min > maxes[j]) {
1455  min = maxes[j];
1456  }
1457  if (max < maxes[j]) {
1458  max = maxes[j];
1459  }
1460  sum += maxes[j];
1461  }
1462  mean = sum / iterations;
1463  for (j=0; j<iterations; j++) {
1464  var += pow((mean - maxes[j]), 2);
1465  }
1466  var = var / iterations;
1467  sd = sqrt(var);
1468  switch (i) {
1469  case 0: strcpy(access, "Directory creation:"); break;
1470  case 1: strcpy(access, "Directory stat :"); break;
1471  /* case 2: strcpy(access, "Directory read :"); break; */
1472  case 2: ; break; /* N/A */
1473  case 3: strcpy(access, "Directory removal :"); break;
1474  case 4: strcpy(access, "File creation :"); break;
1475  case 5: strcpy(access, "File stat :"); break;
1476  case 6: strcpy(access, "File read :"); break;
1477  case 7: strcpy(access, "File removal :"); break;
1478  default: strcpy(access, "ERR"); break;
1479  }
1480  if (i != 2) {
1481  fprintf(out_logfile, " %s ", access);
1482  fprintf(out_logfile, "%14.3f ", max);
1483  fprintf(out_logfile, "%14.3f ", min);
1484  fprintf(out_logfile, "%14.3f ", mean);
1485  fprintf(out_logfile, "%14.3f\n", sd);
1486  fflush(out_logfile);
1487  }
1488  sum = var = 0;
1489 
1490  }
1491 
1492  } else {
1493  for (i = start; i < stop; i++) {
1494  min = max = all[i];
1495  for (k=0; k < size; k++) {
1496  for (j = 0; j < iterations; j++) {
1497  curr = all[calc_allreduce_index(j, k, i)];
1498  if (min > curr) {
1499  min = curr;
1500  }
1501  if (max < curr) {
1502  max = curr;
1503  }
1504  sum += curr;
1505  }
1506  }
1507  mean = sum / (iterations * size);
1508  for (k=0; k<size; k++) {
1509  for (j = 0; j < iterations; j++) {
1510  var += pow((mean - all[(k*tableSize*iterations)
1511  + (j*tableSize) + i]), 2);
1512  }
1513  }
1514  var = var / (iterations * size);
1515  sd = sqrt(var);
1516  switch (i) {
1517  case 0: strcpy(access, "Directory creation:"); break;
1518  case 1: strcpy(access, "Directory stat :"); break;
1519  /* case 2: strcpy(access, "Directory read :"); break; */
1520  case 2: ; break; /* N/A */
1521  case 3: strcpy(access, "Directory removal :"); break;
1522  case 4: strcpy(access, "File creation :"); break;
1523  case 5: strcpy(access, "File stat :"); break;
1524  case 6: strcpy(access, "File read :"); break;
1525  case 7: strcpy(access, "File removal :"); break;
1526  default: strcpy(access, "ERR"); break;
1527  }
1528  if (i != 2) {
1529  fprintf(out_logfile, " %s ", access);
1530  fprintf(out_logfile, "%14.3f ", max);
1531  fprintf(out_logfile, "%14.3f ", min);
1532  fprintf(out_logfile, "%14.3f ", mean);
1533  fprintf(out_logfile, "%14.3f\n", sd);
1534  fflush(out_logfile);
1535  }
1536  sum = var = 0;
1537 
1538  }
1539  }
1540 
1541  /* calculate tree create/remove rates */
1542  for (i = 8; i < tableSize; i++) {
1543  min = max = all[i];
1544  for (j = 0; j < iterations; j++) {
1545  if(print_time){
1546  curr = summary_table[j].time[i];
1547  }else{
1548  curr = summary_table[j].rate[i];
1549  }
1550 
1551  if (min > curr) {
1552  min = curr;
1553  }
1554  if (max < curr) {
1555  max = curr;
1556  }
1557  sum += curr;
1558  }
1559  mean = sum / (iterations);
1560  for (j = 0; j < iterations; j++) {
1561  if(print_time){
1562  curr = summary_table[j].time[i];
1563  }else{
1564  curr = summary_table[j].rate[i];
1565  }
1566 
1567  var += pow((mean - curr), 2);
1568  }
1569  var = var / (iterations);
1570  sd = sqrt(var);
1571  switch (i) {
1572  case 8: strcpy(access, "Tree creation :"); break;
1573  case 9: strcpy(access, "Tree removal :"); break;
1574  default: strcpy(access, "ERR"); break;
1575  }
1576  fprintf(out_logfile, " %s ", access);
1577  fprintf(out_logfile, "%14.3f ", max);
1578  fprintf(out_logfile, "%14.3f ", min);
1579  fprintf(out_logfile, "%14.3f ", mean);
1580  fprintf(out_logfile, "%14.3f\n", sd);
1581  fflush(out_logfile);
1582  sum = var = 0;
1583  }
1584  }
1585 }
1586 
1587 /* Checks to see if the test setup is valid. If it isn't, fail. */
1588 void valid_tests() {
1589 
1590  if (((stone_wall_timer_seconds > 0) && (branch_factor > 1)) || ! barriers) {
1591  fprintf(out_logfile, "Error, stone wall timer does only work with a branch factor <= 1 and with barriers\n");
1592  MPI_Abort(testComm, 1);
1593  }
1594 
1595  if (!create_only && !stat_only && !read_only && !remove_only) {
1597  if (( rank == 0 ) && ( verbose >= 1 )) {
1598  fprintf( out_logfile, "V-1: main: Setting create/stat/read/remove_only to True\n" );
1599  fflush( out_logfile );
1600  }
1601  }
1602 
1603  if (( rank == 0 ) && ( verbose >= 1 )) {
1604  fprintf( out_logfile, "V-1: Entering valid_tests...\n" );
1605  fflush( out_logfile );
1606  }
1607 
1608  /* if dirs_only and files_only were both left unset, set both now */
1609  if (!dirs_only && !files_only) {
1610  dirs_only = files_only = 1;
1611  }
1612 
1613  /* if shared file 'S' access, no directory tests */
1614  if (shared_file) {
1615  dirs_only = 0;
1616  }
1617 
1618  /* check for no barriers with shifting processes for different phases.
1619  that is, one may not specify both -B and -N as it will introduce
1620  race conditions that may cause errors stat'ing or deleting after
1621  creates.
1622  */
1623  if (( barriers == 0 ) && ( nstride != 0 ) && ( rank == 0 )) {
1624  FAIL( "Possible race conditions will occur: -B not compatible with -N");
1625  }
1626 
1627  /* check for collective_creates incompatibilities */
1628  if (shared_file && collective_creates && rank == 0) {
1629  FAIL("-c not compatible with -S");
1630  }
1631  if (path_count > 1 && collective_creates && rank == 0) {
1632  FAIL("-c not compatible with multiple test directories");
1633  }
1634  if (collective_creates && !barriers) {
1635  FAIL("-c not compatible with -B");
1636  }
1637 
1638  if ( strcasecmp(backend_name, "POSIX") != 0 && strcasecmp(backend_name, "DUMMY") != 0) {
1639  FAIL("-a only supported interface is POSIX (and DUMMY) right now!");
1640  }
1641 
1642  /* check for shared file incompatibilities */
1643  if (unique_dir_per_task && shared_file && rank == 0) {
1644  FAIL("-u not compatible with -S");
1645  }
1646 
1647  /* check multiple directory paths and strided option */
1648  if (path_count > 1 && nstride > 0) {
1649  FAIL("cannot have multiple directory paths with -N strides between neighbor tasks");
1650  }
1651 
1652  /* check for shared directory and multiple directories incompatibility */
1653  if (path_count > 1 && unique_dir_per_task != 1) {
1654  FAIL("shared directory mode is not compatible with multiple directory paths");
1655  }
1656 
1657  /* check if more directory paths than ranks */
1658  if (path_count > size) {
1659  FAIL("cannot have more directory paths than MPI tasks");
1660  }
1661 
1662  /* check depth */
1663  if (depth < 0) {
1664  FAIL("depth must be greater than or equal to zero");
1665  }
1666  /* check branch_factor */
1667  if (branch_factor < 1 && depth > 0) {
1668  FAIL("branch factor must be greater than or equal to zero");
1669  }
1670  /* check for valid number of items */
1671  if ((items > 0) && (items_per_dir > 0)) {
1672  if(unique_dir_per_task){
1673  FAIL("only specify the number of items or the number of items per directory");
1674  }else if( items % items_per_dir != 0){
1675  FAIL("items must be a multiple of items per directory");
1676  }else if( stone_wall_timer_seconds != 0){
1677  FAIL("items + items_per_dir can only be set without stonewalling");
1678  }
1679  }
1680 
1681 }
1682 
1683 void show_file_system_size(char *file_system) {
1684  char real_path[MAX_PATHLEN];
1685  char file_system_unit_str[MAX_PATHLEN] = "GiB";
1686  char inode_unit_str[MAX_PATHLEN] = "Mi";
1687  int64_t file_system_unit_val = 1024 * 1024 * 1024;
1688  int64_t inode_unit_val = 1024 * 1024;
1689  int64_t total_file_system_size,
1690  free_file_system_size,
1691  total_inodes,
1692  free_inodes;
1693  double total_file_system_size_hr,
1694  used_file_system_percentage,
1695  used_inode_percentage;
1696  ior_aiori_statfs_t stat_buf;
1697  int ret;
1698 
1699  if (( rank == 0 ) && ( verbose >= 1 )) {
1700  fprintf( out_logfile, "V-1: Entering show_file_system_size...\n" );
1701  fflush( out_logfile );
1702  }
1703 
1704  ret = backend->statfs (file_system, &stat_buf, &param);
1705  if (0 != ret) {
1706  FAIL("unable to stat file system");
1707  }
1708 
1709  total_file_system_size = stat_buf.f_blocks * stat_buf.f_bsize;
1710  free_file_system_size = stat_buf.f_bfree * stat_buf.f_bsize;
1711 
1712  used_file_system_percentage = (1 - ((double)free_file_system_size
1713  / (double)total_file_system_size)) * 100;
1714  total_file_system_size_hr = (double)total_file_system_size
1715  / (double)file_system_unit_val;
1716  if (total_file_system_size_hr > 1024) {
1717  total_file_system_size_hr = total_file_system_size_hr / 1024;
1718  strcpy(file_system_unit_str, "TiB");
1719  }
1720 
1721  /* inodes */
1722  total_inodes = stat_buf.f_files;
1723  free_inodes = stat_buf.f_ffree;
1724 
1725  used_inode_percentage = (1 - ((double)free_inodes/(double)total_inodes))
1726  * 100;
1727 
1728  if (realpath(file_system, real_path) == NULL) {
1729  FAIL("unable to use realpath()");
1730  }
1731 
1732 
1733  /* show results */
1734  fprintf(out_logfile, "Path: %s\n", real_path);
1735  fprintf(out_logfile, "FS: %.1f %s Used FS: %2.1f%% ",
1736  total_file_system_size_hr, file_system_unit_str,
1737  used_file_system_percentage);
1738  fprintf(out_logfile, "Inodes: %.1f %s Used Inodes: %2.1f%%\n",
1739  (double)total_inodes / (double)inode_unit_val,
1740  inode_unit_str, used_inode_percentage);
1741  fflush(out_logfile);
1742 
1743  return;
1744 }
1745 
1747 {
1748  char dirpath[MAX_PATHLEN] = {0};
1749  int i;
1750  int directoryFound = 0;
1751 
1752 
1753  if (( rank == 0 ) && ( verbose >= 1 )) {
1754  fprintf( out_logfile, "V-1: Entering display_freespace...\n" );
1755  fflush( out_logfile );
1756  }
1757 
1758  if (verbose >= 3 && rank == 0) {
1759  fprintf(out_logfile, "V-3: testdirpath is \"%s\"\n", testdirpath );
1760  fflush( out_logfile );
1761  }
1762 
1763  strcpy(dirpath, testdirpath);
1764 
1765  /* get directory for outfile */
1766  i = strlen(dirpath);
1767  while (i-- > 0) {
1768  if (dirpath[i] == '/') {
1769  dirpath[i] = '\0';
1770  directoryFound = 1;
1771  break;
1772  }
1773  }
1774 
1775  /* if no directory/, use '.' */
1776  if (directoryFound == 0) {
1777  strcpy(dirpath, ".");
1778  }
1779 
1780  if (verbose >= 3 && rank == 0) {
1781  fprintf(out_logfile, "V-3: Before show_file_system_size, dirpath is \"%s\"\n", dirpath );
1782  fflush( out_logfile );
1783  }
1784 
1785  show_file_system_size(dirpath);
1786 
1787  if (verbose >= 3 && rank == 0) {
1788  fprintf(out_logfile, "V-3: After show_file_system_size, dirpath is \"%s\"\n", dirpath );
1789  fflush( out_logfile );
1790  }
1791 
1792  return;
1793 }
1794 
1796  int currDepth, char* path, int dirNum, rank_progress_t * progress) {
1797 
1798  unsigned i;
1799  char dir[MAX_PATHLEN];
1800 
1801 
1802  if (( rank == 0 ) && ( verbose >= 1 )) {
1803  fprintf( out_logfile, "V-1: Entering create_remove_directory_tree, currDepth = %d...\n", currDepth );
1804  fflush( out_logfile );
1805  }
1806 
1807  if (currDepth == 0) {
1808  sprintf(dir, "%s/%s.%d/", path, base_tree_name, dirNum);
1809 
1810  if (create) {
1811  if (rank == 0 && verbose >= 2) {
1812  fprintf(out_logfile, "V-2: Making directory \"%s\"\n", dir);
1813  fflush(out_logfile);
1814  }
1815 
1816  if (-1 == backend->mkdir (dir, DIRMODE, &param)) {
1817  fprintf(out_logfile, "error could not create directory \"%s\"\n", dir);
1818  }
1819  }
1820 
1821  create_remove_directory_tree(create, ++currDepth, dir, ++dirNum, progress);
1822 
1823  if (!create) {
1824  if (rank == 0 && verbose >= 2) {
1825  fprintf(out_logfile, "V-2: Remove directory \"%s\"\n", dir);
1826  fflush(out_logfile);
1827  }
1828 
1829  if (-1 == backend->rmdir(dir, &param)) {
1830  FAIL("Unable to remove directory");
1831  }
1832  }
1833  } else if (currDepth <= depth) {
1834 
1835  char temp_path[MAX_PATHLEN];
1836  strcpy(temp_path, path);
1837  int currDir = dirNum;
1838 
1839  for (i=0; i<branch_factor; i++) {
1840  sprintf(dir, "%s.%d/", base_tree_name, currDir);
1841  strcat(temp_path, dir);
1842 
1843  if (create) {
1844  if (rank == 0 && verbose >= 2) {
1845  fprintf(out_logfile, "V-2: Making directory \"%s\"\n", temp_path);
1846  fflush(out_logfile);
1847  }
1848 
1849  if (-1 == backend->mkdir(temp_path, DIRMODE, &param)) {
1850  FAIL("Unable to create directory");
1851  }
1852  }
1853 
1854  create_remove_directory_tree(create, ++currDepth,
1855  temp_path, (branch_factor*currDir)+1, progress);
1856  currDepth--;
1857 
1858  if (!create) {
1859  if (rank == 0 && verbose >= 2) {
1860  fprintf(out_logfile, "V-2: Remove directory \"%s\"\n", temp_path);
1861  fflush(out_logfile);
1862  }
1863 
1864  if (-1 == backend->rmdir(temp_path, &param)) {
1865  FAIL("Unable to remove directory");
1866  }
1867  }
1868 
1869  strcpy(temp_path, path);
1870  currDir++;
1871  }
1872  }
1873 }
1874 
1875 static void mdtest_iteration(int i, int j, MPI_Group testgroup, mdtest_results_t * summary_table){
1876  rank_progress_t progress_o;
1877  memset(& progress_o, 0 , sizeof(progress_o));
1878  progress_o.start_time = GetTimeStamp();
1880  progress_o.items_per_dir = items_per_dir;
1881  rank_progress_t * progress = & progress_o;
1882 
1883  /* start and end times of directory tree create/remove */
1884  double startCreate, endCreate;
1885  int k;
1886 
1887  if (rank == 0 && verbose >= 1) {
1888  fprintf(out_logfile, "V-1: main: * iteration %d *\n", j+1);
1889  fflush(out_logfile);
1890  }
1891 
1892  for (int dir_iter = 0; dir_iter < directory_loops; dir_iter ++){
1893  prep_testdir(j, dir_iter);
1894 
1895  if (verbose >= 2 && rank == 0) {
1896  fprintf(out_logfile, "V-2: main (for j loop): making testdir, \"%s\"\n", testdir );
1897  fflush( out_logfile );
1898  }
1899  if ((rank < path_count) && backend->access(testdir, F_OK, &param) != 0) {
1900  if (backend->mkdir(testdir, DIRMODE, &param) != 0) {
1901  FAIL("Unable to create test directory");
1902  }
1903  }
1904  }
1905 
1906  if (create_only) {
1907  /* create hierarchical directory structure */
1908  MPI_Barrier(testComm);
1909 
1910  startCreate = MPI_Wtime();
1911  for (int dir_iter = 0; dir_iter < directory_loops; dir_iter ++){
1912  prep_testdir(j, dir_iter);
1913 
1914  if (unique_dir_per_task) {
1915  if (collective_creates && (rank == 0)) {
1916  /*
1917  * This is inside two loops, one of which already uses "i" and the other uses "j".
1918  * I don't know how this ever worked. I'm changing this loop to use "k".
1919  */
1920  for (k=0; k<size; k++) {
1921  sprintf(base_tree_name, "mdtest_tree.%d", k);
1922 
1923  if (verbose >= 3 && rank == 0) {
1924  fprintf(out_logfile,
1925  "V-3: main (create hierarchical directory loop-collective): Calling create_remove_directory_tree with \"%s\"\n",
1926  testdir );
1927  fflush( out_logfile );
1928  }
1929 
1930  /*
1931  * Let's pass in the path to the directory we most recently made so that we can use
1932  * full paths in the other calls.
1933  */
1934  create_remove_directory_tree(1, 0, testdir, 0, progress);
1935  if(CHECK_STONE_WALL(progress)){
1936  size = k;
1937  break;
1938  }
1939  }
1940  } else if (!collective_creates) {
1941  if (verbose >= 3 && rank == 0) {
1942  fprintf(out_logfile,
1943  "V-3: main (create hierarchical directory loop-!collective_creates): Calling create_remove_directory_tree with \"%s\"\n",
1944  testdir );
1945  fflush( out_logfile );
1946  }
1947 
1948  /*
1949  * Let's pass in the path to the directory we most recently made so that we can use
1950  * full paths in the other calls.
1951  */
1952  create_remove_directory_tree(1, 0, testdir, 0, progress);
1953  }
1954  } else {
1955  if (rank == 0) {
1956  if (verbose >= 3 && rank == 0) {
1957  fprintf(out_logfile,
1958  "V-3: main (create hierarchical directory loop-!unque_dir_per_task): Calling create_remove_directory_tree with \"%s\"\n",
1959  testdir );
1960  fflush( out_logfile );
1961  }
1962 
1963  /*
1964  * Let's pass in the path to the directory we most recently made so that we can use
1965  * full paths in the other calls.
1966  */
1967  create_remove_directory_tree(1, 0 , testdir, 0, progress);
1968  }
1969  }
1970  }
1971  MPI_Barrier(testComm);
1972  endCreate = MPI_Wtime();
1973  summary_table->rate[8] =
1974  num_dirs_in_tree / (endCreate - startCreate);
1975  summary_table->time[8] = (endCreate - startCreate);
1976  summary_table->items[8] = num_dirs_in_tree;
1977  summary_table->stonewall_last_item[8] = num_dirs_in_tree;
1978  if (verbose >= 1 && rank == 0) {
1979  fprintf(out_logfile, "V-1: main: Tree creation : %14.3f sec, %14.3f ops/sec\n",
1980  (endCreate - startCreate), summary_table->rate[8]);
1981  fflush(out_logfile);
1982  }
1983  }
1984  sprintf(unique_mk_dir, "%s.0", base_tree_name);
1985  sprintf(unique_chdir_dir, "%s.0", base_tree_name);
1986  sprintf(unique_stat_dir, "%s.0", base_tree_name);
1987  sprintf(unique_read_dir, "%s.0", base_tree_name);
1988  sprintf(unique_rm_dir, "%s.0", base_tree_name);
1989  unique_rm_uni_dir[0] = 0;
1990 
1991  if (!unique_dir_per_task) {
1992  if (verbose >= 3 && rank == 0) {
1993  fprintf(out_logfile, "V-3: main: Using unique_mk_dir, \"%s\"\n", unique_mk_dir );
1994  fflush( out_logfile );
1995  }
1996  }
1997 
1998  if (rank < i) {
1999  if (!shared_file) {
2000  sprintf(mk_name, "mdtest.%d.", (rank+(0*nstride))%i);
2001  sprintf(stat_name, "mdtest.%d.", (rank+(1*nstride))%i);
2002  sprintf(read_name, "mdtest.%d.", (rank+(2*nstride))%i);
2003  sprintf(rm_name, "mdtest.%d.", (rank+(3*nstride))%i);
2004  }
2005  if (unique_dir_per_task) {
2006  sprintf(unique_mk_dir, "mdtest_tree.%d.0", (rank+(0*nstride))%i);
2007  sprintf(unique_chdir_dir, "mdtest_tree.%d.0", (rank+(1*nstride))%i);
2008  sprintf(unique_stat_dir, "mdtest_tree.%d.0", (rank+(2*nstride))%i);
2009  sprintf(unique_read_dir, "mdtest_tree.%d.0", (rank+(3*nstride))%i);
2010  sprintf(unique_rm_dir, "mdtest_tree.%d.0", (rank+(4*nstride))%i);
2011  unique_rm_uni_dir[0] = 0;
2012  }
2013 
2014  if (verbose >= 3 && rank == 0) {
2015  fprintf(out_logfile, "V-3: main: Copied unique_mk_dir, \"%s\", to topdir\n", unique_mk_dir );
2016  fflush( out_logfile );
2017  }
2018 
2019  if (dirs_only && !shared_file) {
2020  if (pre_delay) {
2022  }
2023  directory_test(j, i, unique_mk_dir, progress);
2024  }
2025  if (files_only) {
2026  if (pre_delay) {
2028  }
2029  file_test(j, i, unique_mk_dir, progress);
2030  }
2031  }
2032 
2033  /* remove directory structure */
2034  if (!unique_dir_per_task) {
2035  if (verbose >= 3 && rank == 0) {
2036  fprintf(out_logfile, "V-3: main: Using testdir, \"%s\"\n", testdir );
2037  fflush( out_logfile );
2038  }
2039  }
2040 
2041  MPI_Barrier(testComm);
2042  if (remove_only) {
2043  progress->items_start = 0;
2044  startCreate = MPI_Wtime();
2045  for (int dir_iter = 0; dir_iter < directory_loops; dir_iter ++){
2046  prep_testdir(j, dir_iter);
2047  if (unique_dir_per_task) {
2048  if (collective_creates && (rank == 0)) {
2049  /*
2050  * This is inside two loops, one of which already uses "i" and the other uses "j".
2051  * I don't know how this ever worked. I'm changing this loop to use "k".
2052  */
2053  for (k=0; k<size; k++) {
2054  sprintf(base_tree_name, "mdtest_tree.%d", k);
2055 
2056  if (verbose >= 3 && rank == 0) {
2057  fprintf(out_logfile,
2058  "V-3: main (remove hierarchical directory loop-collective): Calling create_remove_directory_tree with \"%s\"\n",
2059  testdir );
2060  fflush( out_logfile );
2061  }
2062 
2063  /*
2064  * Let's pass in the path to the directory we most recently made so that we can use
2065  * full paths in the other calls.
2066  */
2067  create_remove_directory_tree(0, 0, testdir, 0, progress);
2068  if(CHECK_STONE_WALL(progress)){
2069  size = k;
2070  break;
2071  }
2072  }
2073  } else if (!collective_creates) {
2074  if (verbose >= 3 && rank == 0) {
2075  fprintf(out_logfile,
2076  "V-3: main (remove hierarchical directory loop-!collective): Calling create_remove_directory_tree with \"%s\"\n",
2077  testdir );
2078  fflush( out_logfile );
2079  }
2080 
2081  /*
2082  * Let's pass in the path to the directory we most recently made so that we can use
2083  * full paths in the other calls.
2084  */
2085  create_remove_directory_tree(0, 0, testdir, 0, progress);
2086  }
2087  } else {
2088  if (rank == 0) {
2089  if (verbose >= 3 && rank == 0) {
2090  fprintf(out_logfile,
2091  "V-3: main (remove hierarchical directory loop-!unique_dir_per_task): Calling create_remove_directory_tree with \"%s\"\n",
2092  testdir );
2093  fflush( out_logfile );
2094  }
2095 
2096  /*
2097  * Let's pass in the path to the directory we most recently made so that we can use
2098  * full paths in the other calls.
2099  */
2100  create_remove_directory_tree(0, 0 , testdir, 0, progress);
2101  }
2102  }
2103  }
2104 
2105  MPI_Barrier(testComm);
2106  endCreate = MPI_Wtime();
2107  summary_table->rate[9] = num_dirs_in_tree / (endCreate - startCreate);
2108  summary_table->time[9] = endCreate - startCreate;
2109  summary_table->items[9] = num_dirs_in_tree;
2110  summary_table->stonewall_last_item[8] = num_dirs_in_tree;
2111  if (verbose >= 1 && rank == 0) {
2112  fprintf(out_logfile, "V-1: main Tree removal : %14.3f sec, %14.3f ops/sec\n",
2113  (endCreate - startCreate), summary_table->rate[9]);
2114  fflush(out_logfile);
2115  }
2116 
2117  if (( rank == 0 ) && ( verbose >=2 )) {
2118  fprintf( out_logfile, "V-2: main (at end of for j loop): Removing testdir of \"%s\"\n", testdir );
2119  fflush( out_logfile );
2120  }
2121 
2122  for (int dir_iter = 0; dir_iter < directory_loops; dir_iter ++){
2123  prep_testdir(j, dir_iter);
2124  if ((rank < path_count) && backend->access(testdir, F_OK, &param) == 0) {
2125  //if (( rank == 0 ) && access(testdir, F_OK) == 0) {
2126  if (backend->rmdir(testdir, &param) == -1) {
2127  FAIL("unable to remove directory");
2128  }
2129  }
2130  }
2131  } else {
2132  summary_table->rate[9] = 0;
2133  }
2134 }
2135 
2137  barriers = 1;
2138  branch_factor = 1;
2139  throttle = 1;
2141  create_only = 0;
2142  stat_only = 0;
2143  read_only = 0;
2144  remove_only = 0;
2145  leaf_only = 0;
2146  depth = 0;
2147  num_dirs_in_tree = 0;
2148  items_per_dir = 0;
2149  random_seed = 0;
2150  print_time = 0;
2151  shared_file = 0;
2152  files_only = 0;
2153  dirs_only = 0;
2154  pre_delay = 0;
2155  unique_dir_per_task = 0;
2157  items = 0;
2159  collective_creates = 0;
2160  write_bytes = 0;
2162  read_bytes = 0;
2163  sync_file = 0;
2164  path_count = 0;
2165  nstride = 0;
2166 }
2167 
2168 
2169 mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * world_out) {
2170  testComm = world_com;
2171  out_logfile = world_out;
2172  mpi_comm_world = world_com;
2173 
2174  init_clock();
2175 
2176  mdtest_init_args();
2177  int i, j;
2178  int nodeCount;
2179  MPI_Group worldgroup, testgroup;
2180  struct {
2181  int first;
2182  int last;
2183  int stride;
2184  } range = {0, 0, 1};
2185  int first = 1;
2186  int last = 0;
2187  int stride = 1;
2188  int iterations = 1;
2189 
2190  verbose = 0;
2191  int no_barriers = 0;
2192  char * path = "./out";
2193  int randomize = 0;
2194  char APIs[1024];
2195  aiori_supported_apis(APIs);
2196  char apiStr[1024];
2197  sprintf(apiStr, "API for I/O [%s]", APIs);
2198 
2199  option_help options [] = {
2200  {'a', NULL, apiStr, OPTION_OPTIONAL_ARGUMENT, 's', & backend_name},
2201  {'b', NULL, "branching factor of hierarchical directory structure", OPTION_OPTIONAL_ARGUMENT, 'd', & branch_factor},
2202  {'d', NULL, "the directory in which the tests will run", OPTION_OPTIONAL_ARGUMENT, 's', & path},
2203  {'B', NULL, "no barriers between phases", OPTION_OPTIONAL_ARGUMENT, 'd', & no_barriers},
2204  {'C', NULL, "only create files/dirs", OPTION_FLAG, 'd', & create_only},
2205  {'T', NULL, "only stat files/dirs", OPTION_FLAG, 'd', & stat_only},
2206  {'E', NULL, "only read files/dir", OPTION_FLAG, 'd', & read_only},
2207  {'r', NULL, "only remove files or directories left behind by previous runs", OPTION_FLAG, 'd', & remove_only},
2208  {'D', NULL, "perform test on directories only (no files)", OPTION_FLAG, 'd', & dirs_only},
2209  {'e', NULL, "bytes to read from each file", OPTION_OPTIONAL_ARGUMENT, 'l', & read_bytes},
2210  {'f', NULL, "first number of tasks on which the test will run", OPTION_OPTIONAL_ARGUMENT, 'd', & first},
2211  {'F', NULL, "perform test on files only (no directories)", OPTION_FLAG, 'd', & files_only},
2212  {'i', NULL, "number of iterations the test will run", OPTION_OPTIONAL_ARGUMENT, 'd', & iterations},
2213  {'I', NULL, "number of items per directory in tree", OPTION_OPTIONAL_ARGUMENT, 'l', & items_per_dir},
2214  {'l', NULL, "last number of tasks on which the test will run", OPTION_OPTIONAL_ARGUMENT, 'd', & last},
2215  {'L', NULL, "files only at leaf level of tree", OPTION_FLAG, 'd', & leaf_only},
2216  {'n', NULL, "every process will creat/stat/read/remove # directories and files", OPTION_OPTIONAL_ARGUMENT, 'l', & items},
2217  {'N', NULL, "stride # between neighbor tasks for file/dir operation (local=0)", OPTION_OPTIONAL_ARGUMENT, 'd', & nstride},
2218  {'p', NULL, "pre-iteration delay (in seconds)", OPTION_OPTIONAL_ARGUMENT, 'd', & pre_delay},
2219  {'R', NULL, "random access to files (only for stat)", OPTION_FLAG, 'd', & randomize},
2220  {0, "random-seed", "random seed for -R", OPTION_OPTIONAL_ARGUMENT, 'd', & random_seed},
2221  {'s', NULL, "stride between the number of tasks for each test", OPTION_OPTIONAL_ARGUMENT, 'd', & stride},
2222  {'S', NULL, "shared file access (file only, no directories)", OPTION_FLAG, 'd', & shared_file},
2223  {'c', NULL, "collective creates: task 0 does all creates", OPTION_FLAG, 'd', & collective_creates},
2224  {'t', NULL, "time unique working directory overhead", OPTION_FLAG, 'd', & time_unique_dir_overhead},
2225  {'u', NULL, "unique working directory for each task", OPTION_FLAG, 'd', & unique_dir_per_task},
2226  {'v', NULL, "verbosity (each instance of option increments by one)", OPTION_FLAG, 'd', & verbose},
2227  {'V', NULL, "verbosity value", OPTION_OPTIONAL_ARGUMENT, 'd', & verbose},
2228  {'w', NULL, "bytes to write to each file after it is created", OPTION_OPTIONAL_ARGUMENT, 'l', & write_bytes},
2229  {'W', NULL, "number in seconds; stonewall timer, write as many seconds and ensure all processes did the same number of operations (currently only stops during create phase)", OPTION_OPTIONAL_ARGUMENT, 'd', & stone_wall_timer_seconds},
2230  {'x', NULL, "StoneWallingStatusFile; contains the number of iterations of the creation phase, can be used to split phases across runs", OPTION_OPTIONAL_ARGUMENT, 's', & stoneWallingStatusFile},
2231  {'y', NULL, "sync file after writing", OPTION_FLAG, 'd', & sync_file},
2232  {'z', NULL, "depth of hierarchical directory structure", OPTION_OPTIONAL_ARGUMENT, 'd', & depth},
2233  {'Z', NULL, "print time instead of rate", OPTION_FLAG, 'd', & print_time},
2234  LAST_OPTION
2235  };
2236  airoi_parse_options(argc, argv, options);
2237 
2238  backend = aiori_select(backend_name);
2239  if (NULL == backend) {
2240  FAIL("Could not find suitable backend to use");
2241  }
2242 
2243  MPI_Comm_rank(testComm, &rank);
2244  MPI_Comm_size(testComm, &size);
2245 
2246  pid = getpid();
2247  uid = getuid();
2248 
2249  nodeCount = size / CountTasksPerNode(testComm);
2250 
2251  if (rank == 0) {
2252  fprintf(out_logfile, "-- started at %s --\n\n", PrintTimestamp());
2253  fprintf(out_logfile, "mdtest-%s was launched with %d total task(s) on %d node(s)\n",
2254  RELEASE_VERS, size, nodeCount);
2255  fflush(out_logfile);
2256  }
2257 
2258  if (rank == 0) {
2259  fprintf(out_logfile, "Command line used: %s", argv[0]);
2260  for (i = 1; i < argc; i++) {
2261  fprintf(out_logfile, " \"%s\"", argv[i]);
2262  }
2263  fprintf(out_logfile, "\n");
2264  fflush(out_logfile);
2265  }
2266 
2267  /* adjust special variables */
2268  barriers = ! no_barriers;
2269  if (path != NULL){
2270  parse_dirpath(path);
2271  }
2272  if( randomize > 0 ){
2273  if (random_seed == 0) {
2274  /* Ensure all procs have the same random number */
2275  random_seed = time(NULL);
2276  MPI_Barrier(testComm);
2277  MPI_Bcast(&random_seed, 1, MPI_INT, 0, testComm);
2278  }
2279  random_seed += rank;
2280  }
2281  if ((items > 0) && (items_per_dir > 0) && (! unique_dir_per_task)) {
2283  }else{
2284  directory_loops = 1;
2285  }
2286  valid_tests();
2287 
2288  if (( rank == 0 ) && ( verbose >= 1 )) {
2289  // option_print_current(options);
2290  fprintf (out_logfile, "api : %s\n", backend_name);
2291  fprintf( out_logfile, "barriers : %s\n", ( barriers ? "True" : "False" ));
2292  fprintf( out_logfile, "collective_creates : %s\n", ( collective_creates ? "True" : "False" ));
2293  fprintf( out_logfile, "create_only : %s\n", ( create_only ? "True" : "False" ));
2294  fprintf( out_logfile, "dirpath(s):\n" );
2295  for ( i = 0; i < path_count; i++ ) {
2296  fprintf( out_logfile, "\t%s\n", filenames[i] );
2297  }
2298  fprintf( out_logfile, "dirs_only : %s\n", ( dirs_only ? "True" : "False" ));
2299  fprintf( out_logfile, "read_bytes : "LLU"\n", read_bytes );
2300  fprintf( out_logfile, "read_only : %s\n", ( read_only ? "True" : "False" ));
2301  fprintf( out_logfile, "first : %d\n", first );
2302  fprintf( out_logfile, "files_only : %s\n", ( files_only ? "True" : "False" ));
2303  fprintf( out_logfile, "iterations : %d\n", iterations );
2304  fprintf( out_logfile, "items_per_dir : "LLU"\n", items_per_dir );
2305  fprintf( out_logfile, "last : %d\n", last );
2306  fprintf( out_logfile, "leaf_only : %s\n", ( leaf_only ? "True" : "False" ));
2307  fprintf( out_logfile, "items : "LLU"\n", items );
2308  fprintf( out_logfile, "nstride : %d\n", nstride );
2309  fprintf( out_logfile, "pre_delay : %d\n", pre_delay );
2310  fprintf( out_logfile, "remove_only : %s\n", ( leaf_only ? "True" : "False" ));
2311  fprintf( out_logfile, "random_seed : %d\n", random_seed );
2312  fprintf( out_logfile, "stride : %d\n", stride );
2313  fprintf( out_logfile, "shared_file : %s\n", ( shared_file ? "True" : "False" ));
2314  fprintf( out_logfile, "time_unique_dir_overhead: %s\n", ( time_unique_dir_overhead ? "True" : "False" ));
2315  fprintf( out_logfile, "stone_wall_timer_seconds: %d\n", stone_wall_timer_seconds);
2316  fprintf( out_logfile, "stat_only : %s\n", ( stat_only ? "True" : "False" ));
2317  fprintf( out_logfile, "unique_dir_per_task : %s\n", ( unique_dir_per_task ? "True" : "False" ));
2318  fprintf( out_logfile, "write_bytes : "LLU"\n", write_bytes );
2319  fprintf( out_logfile, "sync_file : %s\n", ( sync_file ? "True" : "False" ));
2320  fprintf( out_logfile, "depth : %d\n", depth );
2321  fflush( out_logfile );
2322  }
2323 
2324  /* setup total number of items and number of items per dir */
2325  if (depth <= 0) {
2326  num_dirs_in_tree = 1;
2327  } else {
2328  if (branch_factor < 1) {
2329  num_dirs_in_tree = 1;
2330  } else if (branch_factor == 1) {
2331  num_dirs_in_tree = depth + 1;
2332  } else {
2333  num_dirs_in_tree = (pow(branch_factor, depth+1) - 1) / (branch_factor - 1);
2334  }
2335  }
2336  if (items_per_dir > 0) {
2337  if(items == 0){
2338  if (leaf_only) {
2339  items = items_per_dir * (uint64_t) pow(branch_factor, depth);
2340  } else {
2342  }
2343  }else{
2345  }
2346  } else {
2347  if (leaf_only) {
2348  if (branch_factor <= 1) {
2349  items_per_dir = items;
2350  } else {
2351  items_per_dir = (uint64_t) (items / pow(branch_factor, depth));
2352  items = items_per_dir * (uint64_t) pow(branch_factor, depth);
2353  }
2354  } else {
2357  }
2358  }
2359 
2360  /* initialize rand_array */
2361  if (random_seed > 0) {
2362  srand(random_seed);
2363 
2364  uint64_t s;
2365 
2366  rand_array = (uint64_t *) malloc( items * sizeof(*rand_array));
2367 
2368  for (s=0; s < items; s++) {
2369  rand_array[s] = s;
2370  }
2371 
2372  /* shuffle list randomly */
2373  uint64_t n = items;
2374  while (n>1) {
2375  n--;
2376 
2377  /*
2378  * Generate a random number in the range 0 .. n
2379  *
2380  * rand() returns a number from 0 .. RAND_MAX. Divide that
2381  * by RAND_MAX and you get a floating point number in the
2382  * range 0 .. 1. Multiply that by n and you get a number in
2383  * the range 0 .. n.
2384  */
2385  uint64_t k = ( uint64_t ) ((( double )rand() / ( double )RAND_MAX ) * ( double )n );
2386 
2387  /*
2388  * Now move the nth element to the kth (randomly chosen)
2389  * element, and the kth element to the nth element.
2390  */
2391 
2392  uint64_t tmp = rand_array[k];
2393  rand_array[k] = rand_array[n];
2394  rand_array[n] = tmp;
2395  }
2396  }
2397 
2398  /* allocate and initialize write buffer with # */
2399  if (write_bytes > 0) {
2400  write_buffer = (char *)malloc(write_bytes);
2401  if (write_buffer == NULL) {
2402  FAIL("out of memory");
2403  }
2404  memset(write_buffer, 0x23, write_bytes);
2405  }
2406 
2407  /* setup directory path to work in */
2408  if (path_count == 0) { /* special case where no directory path provided with '-d' option */
2409  char *ret = getcwd(testdirpath, MAX_PATHLEN);
2410  if (ret == NULL) {
2411  FAIL("Unable to get current working directory");
2412  }
2413  path_count = 1;
2414  } else {
2416  }
2417 
2418  /* if directory does not exist, create it */
2419  if ((rank < path_count) && backend->access(testdirpath, F_OK, &param) != 0) {
2420  if (backend->mkdir(testdirpath, DIRMODE, &param) != 0) {
2421  FAIL("Unable to create test directory path");
2422  }
2423  }
2424 
2425  /* display disk usage */
2426  if (verbose >= 3 && rank == 0) {
2427  fprintf(out_logfile, "V-3: main (before display_freespace): testdirpath is \"%s\"\n", testdirpath );
2428  fflush( out_logfile );
2429  }
2430 
2431  if (rank == 0) display_freespace(testdirpath);
2432 
2433  if (verbose >= 3 && rank == 0) {
2434  fprintf(out_logfile, "V-3: main (after display_freespace): testdirpath is \"%s\"\n", testdirpath );
2435  fflush( out_logfile );
2436  }
2437 
2438  if (rank == 0) {
2439  if (random_seed > 0) {
2440  fprintf(out_logfile, "random seed: %d\n", random_seed);
2441  }
2442  }
2443 
2444  if (gethostname(hostname, MAX_PATHLEN) == -1) {
2445  perror("gethostname");
2446  MPI_Abort(testComm, 2);
2447  }
2448 
2449  if (last == 0) {
2450  first = size;
2451  last = size;
2452  }
2453 
2454  /* setup summary table for recording results */
2455  summary_table = (mdtest_results_t *) malloc(iterations * sizeof(mdtest_results_t));
2456  memset(summary_table, 0, sizeof(mdtest_results_t));
2457  for(int i=0; i < iterations; i++){
2458  for(int j=0; j < MDTEST_LAST_NUM; j++){
2459  summary_table[i].rate[j] = 0.0;
2460  summary_table[i].time[j] = 0.0;
2461  }
2462  }
2463 
2464  if (summary_table == NULL) {
2465  FAIL("out of memory");
2466  }
2467 
2468  if (unique_dir_per_task) {
2469  sprintf(base_tree_name, "mdtest_tree.%d", rank);
2470  } else {
2471  sprintf(base_tree_name, "mdtest_tree");
2472  }
2473 
2474  /* default use shared directory */
2475  strcpy(mk_name, "mdtest.shared.");
2476  strcpy(stat_name, "mdtest.shared.");
2477  strcpy(read_name, "mdtest.shared.");
2478  strcpy(rm_name, "mdtest.shared.");
2479 
2480  MPI_Comm_group(testComm, &worldgroup);
2481 
2482  /* Run the tests */
2483  for (i = first; i <= last && i <= size; i += stride) {
2484  range.last = i - 1;
2485  MPI_Group_range_incl(worldgroup, 1, (void *)&range, &testgroup);
2486  MPI_Comm_create(testComm, testgroup, &testComm);
2487  if (rank == 0) {
2488  uint64_t items_all = i * items;
2490  items_all *= num_dirs_in_tree_calc;
2491  }
2492  if (files_only && dirs_only) {
2493  fprintf(out_logfile, "\n%d tasks, "LLU" files/directories\n", i, items_all);
2494  } else if (files_only) {
2495  if (!shared_file) {
2496  fprintf(out_logfile, "\n%d tasks, "LLU" files\n", i, items_all);
2497  }
2498  else {
2499  fprintf(out_logfile, "\n%d tasks, 1 file\n", i);
2500  }
2501  } else if (dirs_only) {
2502  fprintf(out_logfile, "\n%d tasks, "LLU" directories\n", i, items_all);
2503  }
2504  }
2505  if (rank == 0 && verbose >= 1) {
2506  fprintf(out_logfile, "\n");
2507  fprintf(out_logfile, " Operation Duration Rate\n");
2508  fprintf(out_logfile, " --------- -------- ----\n");
2509  }
2510 
2511  for (j = 0; j < iterations; j++) {
2512  // keep track of the current status for stonewalling
2513  mdtest_iteration(i, j, testgroup, & summary_table[j]);
2514  }
2515  summarize_results(iterations);
2516  if (i == 1 && stride > 1) {
2517  i = 0;
2518  }
2519  }
2520 
2521  if (rank == 0) {
2522  fprintf(out_logfile, "\n-- finished at %s --\n", PrintTimestamp());
2523  fflush(out_logfile);
2524  }
2525 
2526  if (random_seed > 0) {
2527  free(rand_array);
2528  }
2529  return summary_table;
2530 }
static int shared_file
Definition: mdtest.c:130
uint64_t stonewall_item_sum[MDTEST_LAST_NUM]
Definition: mdtest.h:32
static char testdir[MAX_PATHLEN]
Definition: mdtest.c:84
double rate[MDTEST_LAST_NUM]
Definition: mdtest.h:24
static char unique_rm_dir[MAX_PATHLEN]
Definition: mdtest.c:97
uint64_t f_blocks
Definition: aiori.h:58
static char testdirpath[MAX_PATHLEN]
Definition: mdtest.c:85
void(* delete)(char *, IOR_param_t *)
Definition: aiori.h:73
static char unique_read_dir[MAX_PATHLEN]
Definition: mdtest.c:96
static char rm_name[MAX_PATHLEN]
Definition: mdtest.c:92
uint64_t f_bfree
Definition: aiori.h:59
int(* rmdir)(const char *path, IOR_param_t *param)
Definition: aiori.h:79
static int stone_wall_timer_seconds
Definition: mdtest.c:139
#define LAST_OPTION
Definition: option.h:36
static char stat_name[MAX_PATHLEN]
Definition: mdtest.c:90
static char unique_rm_uni_dir[MAX_PATHLEN]
Definition: mdtest.c:98
static size_t read_bytes
Definition: mdtest.c:140
int filePerProc
Definition: ior.h:104
static int size
Definition: mdtest.c:82
static int barriers
Definition: mdtest.c:104
static void create_file(const char *path, uint64_t itemNum)
Definition: mdtest.c:310
static const ior_aiori_t * backend
Definition: mdtest.c:151
void mdtest_init_args()
Definition: mdtest.c:2136
static int time_unique_dir_overhead
Definition: mdtest.c:135
int64_t ReadStoneWallingIterations(char *const filename)
Definition: utilities.c:580
mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE *world_out)
Definition: mdtest.c:2169
static int nstride
Definition: mdtest.c:143
static int unique_dir_per_task
Definition: mdtest.c:134
int CountTasksPerNode(MPI_Comm comm)
Definition: utilities.c:166
uint64_t stonewall_item_min[MDTEST_LAST_NUM]
Definition: mdtest.h:31
void parse_dirpath(char *dirpath_arg)
Definition: mdtest.c:189
static uint64_t items
Definition: mdtest.c:124
static int collective_creates
Definition: mdtest.c:137
#define DIRMODE
Definition: mdtest.c:75
uint64_t items_start
Definition: mdtest.c:161
uint64_t f_ffree
Definition: aiori.h:62
void print_help(void)
Definition: mdtest.c:1317
void offset_timers(double *t, int tcount)
Definition: mdtest.c:173
static void prep_testdir(int j, int dir_iter)
Definition: mdtest.c:224
void create_remove_items(int currDepth, const int dirs, const int create, const int collective, const char *path, uint64_t dirNum, rank_progress_t *progress)
Definition: mdtest.c:459
unsigned int openFlags
Definition: ior.h:85
int fsyncPerWrite
Definition: ior.h:152
static void mdtest_iteration(int i, int j, MPI_Group testgroup, mdtest_results_t *summary_table)
Definition: mdtest.c:1875
static unsigned branch_factor
Definition: mdtest.c:110
int(* access)(const char *path, int mode, IOR_param_t *param)
Definition: aiori.h:80
void mdtest_read(int random, int dirs, const long dir_iter, char *path)
Definition: mdtest.c:646
int stone_wall_timer_seconds
Definition: mdtest.c:159
void show_file_system_size(char *file_system)
Definition: mdtest.c:1683
#define WRITE
Definition: iordef.h:95
void(* close)(void *, IOR_param_t *)
Definition: aiori.h:72
void aiori_supported_apis(char *APIs)
Definition: aiori.c:84
double start_time
Definition: mdtest.c:157
static char * write_buffer
Definition: mdtest.c:99
static int directory_loops
Definition: mdtest.c:127
#define READ
Definition: iordef.h:97
static size_t write_bytes
Definition: mdtest.c:138
void mdtest_stat(const int random, const int dirs, const long dir_iter, const char *path, rank_progress_t *progress)
Definition: mdtest.c:538
char * PrintTimestamp()
Definition: utilities.c:565
void init_clock()
Definition: utilities.c:560
#define IOR_CREAT
Definition: aiori.h:37
const ior_aiori_t * aiori_select(const char *api)
Definition: aiori.c:184
void *(* open)(char *, IOR_param_t *)
Definition: aiori.h:69
static int read_only
Definition: mdtest.c:107
static char * stoneWallingStatusFile
Definition: mdtest.c:101
void collective_helper(const int dirs, const int create, const char *path, uint64_t itemNum, rank_progress_t *progress)
Definition: mdtest.c:415
void file_test(const int iteration, const int ntasks, const char *path, rank_progress_t *progress)
Definition: mdtest.c:1077
double time[MDTEST_LAST_NUM]
Definition: mdtest.h:25
static int files_only
Definition: mdtest.c:131
static char base_tree_name[MAX_PATHLEN]
Definition: mdtest.c:86
static int leaf_only
Definition: mdtest.c:109
uint64_t f_files
Definition: aiori.h:61
MPI_Comm testComm
Definition: utilities.c:61
static int print_time
Definition: mdtest.c:128
static mdtest_results_t * summary_table
Definition: mdtest.c:145
uint64_t f_bsize
Definition: aiori.h:57
uint64_t items[MDTEST_LAST_NUM]
Definition: mdtest.h:26
static int create_only
Definition: mdtest.c:105
void valid_tests()
Definition: mdtest.c:1588
static uint64_t num_dirs_in_tree
Definition: mdtest.c:118
static char unique_chdir_dir[MAX_PATHLEN]
Definition: mdtest.c:94
void collective_create_remove(const int create, const int dirs, const int ntasks, const char *path, rank_progress_t *progress)
Definition: mdtest.c:756
static uid_t uid
Definition: mdtest.c:147
void directory_test(const int iteration, const int ntasks, const char *path, rank_progress_t *progress)
Definition: mdtest.c:839
void create_remove_items_helper(const int dirs, const int create, const char *path, uint64_t itemNum, rank_progress_t *progress)
Definition: mdtest.c:386
double GetTimeStamp(void)
Definition: utilities.c:511
int updateStoneWallIterations(int iteration, rank_progress_t *progress, double tstart)
Definition: mdtest.c:1045
void display_freespace(char *testdirpath)
Definition: mdtest.c:1746
void create_remove_directory_tree(int create, int currDepth, char *path, int dirNum, rank_progress_t *progress)
Definition: mdtest.c:1795
static option_help options[]
Definition: aiori-DUMMY.c:31
static int sync_file
Definition: mdtest.c:141
uint64_t items_done
Definition: mdtest.c:162
static int dirs_only
Definition: mdtest.c:132
int(* mkdir)(const char *path, mode_t mode, IOR_param_t *param)
Definition: aiori.h:78
static IOR_param_t param
Definition: mdtest.c:153
static void remove_file(const char *path, uint64_t itemNum)
Definition: mdtest.c:288
void StoreStoneWallingIterations(char *const filename, int64_t count)
Definition: utilities.c:602
#define IOR_WRONLY
Definition: aiori.h:34
static char hostname[MAX_PATHLEN]
Definition: mdtest.c:88
static char unique_stat_dir[MAX_PATHLEN]
Definition: mdtest.c:95
static int path_count
Definition: mdtest.c:142
static char read_name[MAX_PATHLEN]
Definition: mdtest.c:91
#define RELEASE_VERS
Definition: mdtest.c:76
static uint64_t num_dirs_in_tree_calc
Definition: mdtest.c:126
static int random_seed
Definition: mdtest.c:129
static void create_remove_dirs(const char *path, bool create, uint64_t itemNum)
Definition: mdtest.c:258
void *(* create)(char *, IOR_param_t *)
Definition: aiori.h:68
long long int IOR_size_t
Definition: iordef.h:124
uint64_t stonewall_last_item[MDTEST_LAST_NUM]
Definition: mdtest.h:30
static int pre_delay
Definition: mdtest.c:133
static char * read_buffer
Definition: mdtest.c:100
static int throttle
Definition: mdtest.c:136
void summarize_results(int iterations)
Definition: mdtest.c:1380
static pid_t pid
Definition: mdtest.c:146
uint64_t items_per_dir
Definition: mdtest.c:164
IOR_offset_t offset
Definition: ior.h:119
#define MAX_PATHLEN
Definition: utilities.h:35
static char mk_name[MAX_PATHLEN]
Definition: mdtest.c:89
#define FAIL(msg)
Definition: utilities.h:47
static int remove_only
Definition: mdtest.c:108
static const char * backend_name
Definition: mdtest.c:150
double stonewall_time[MDTEST_LAST_NUM]
Definition: mdtest.h:29
int(* statfs)(const char *, ior_aiori_statfs_t *, IOR_param_t *param)
Definition: aiori.h:77
static char unique_mk_dir[MAX_PATHLEN]
Definition: mdtest.c:93
void DelaySecs(int delay)
Definition: utilities.c:617
#define TEST_DIR
Definition: mdtest.c:77
int verbose
Definition: utilities.c:60
MPI_Comm mpi_comm_world
Definition: utilities.c:62
int calc_allreduce_index(int iter, int rank, int op)
Definition: mdtest.c:1375
static char ** filenames
Definition: mdtest.c:87
static uint64_t items_per_dir
Definition: mdtest.c:125
FILE * out_logfile
Definition: utilities.c:63
void unique_dir_access(int opt, char *to)
Definition: mdtest.c:238
int rank
Definition: utilities.c:57
#define ITEM_COUNT
Definition: mdtest.c:78
int(* stat)(const char *path, struct stat *buf, IOR_param_t *param)
Definition: aiori.h:81
#define CHECK_STONE_WALL(p)
Definition: mdtest.c:167
static int stat_only
Definition: mdtest.c:106
IOR_offset_t(* xfer)(int, void *, IOR_size_t *, IOR_offset_t, IOR_param_t *)
Definition: aiori.h:70
#define LLU
Definition: mdtest.c:80
void airoi_parse_options(int argc, char **argv, option_help *global_options)
Definition: aiori.c:64
static int depth
Definition: mdtest.c:111
static uint64_t * rand_array
Definition: mdtest.c:83
#define NULL
Definition: iordef.h:79