IOR
aiori-DFS.c
Go to the documentation of this file.
1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
2  * vim:expandtab:shiftwidth=8:tabstop=8:
3  */
4 /*
5  * Copyright (C) 2018-2020 Intel Corporation
6  * See the file COPYRIGHT for a complete copyright notice and license.
7  */
8 
9 /*
10  * This file implements the abstract I/O interface for DAOS FS API.
11  */
12 
13 #define _BSD_SOURCE
14 
15 #ifdef HAVE_CONFIG_H
16 #include "config.h"
17 #endif
18 
19 #include <string.h>
20 #include <assert.h>
21 #include <errno.h>
22 #include <stdio.h>
23 #include <dirent.h>
24 #include <sys/types.h>
25 #include <sys/stat.h>
26 #include <unistd.h>
27 #include <fcntl.h>
28 #include <libgen.h>
29 
30 #include <gurt/common.h>
31 #include <gurt/hash.h>
32 #include <daos.h>
33 #include <daos_fs.h>
34 
35 #include "ior.h"
36 #include "iordef.h"
37 #include "aiori.h"
38 #include "utilities.h"
39 
40 dfs_t *dfs;
41 static daos_handle_t poh, coh;
42 static daos_oclass_id_t objectClass;
43 static daos_oclass_id_t dir_oclass;
44 static struct d_hash_table *dir_hash;
45 static bool dfs_init;
46 
47 struct aiori_dir_hdl {
48  d_list_t entry;
49  dfs_obj_t *oh;
50  char name[PATH_MAX];
51 };
52 
53 enum handleType {
57 };
58 
59 /************************** O P T I O N S *****************************/
60 struct dfs_options{
61  char *pool;
62 #if !defined(DAOS_API_VERSION_MAJOR) || DAOS_API_VERSION_MAJOR < 1
63  char *svcl;
64 #endif
65  char *group;
66  char *cont;
68  char *oclass;
69  char *dir_oclass;
70  char *prefix;
71  int destroy;
72 };
73 
74 static struct dfs_options o = {
75  .pool = NULL,
76 #if !defined(DAOS_API_VERSION_MAJOR) || DAOS_API_VERSION_MAJOR < 1
77  .svcl = NULL,
78 #endif
79  .group = NULL,
80  .cont = NULL,
81  .chunk_size = 1048576,
82  .oclass = NULL,
83  .dir_oclass = NULL,
84  .prefix = NULL,
85  .destroy = 0,
86 };
87 
88 static option_help options [] = {
89  {0, "dfs.pool", "pool uuid", OPTION_OPTIONAL_ARGUMENT, 's', & o.pool},
90 #if !defined(DAOS_API_VERSION_MAJOR) || DAOS_API_VERSION_MAJOR < 1
91  {0, "dfs.svcl", "pool SVCL", OPTION_OPTIONAL_ARGUMENT, 's', & o.svcl},
92 #endif
93  {0, "dfs.group", "server group", OPTION_OPTIONAL_ARGUMENT, 's', & o.group},
94  {0, "dfs.cont", "DFS container uuid", OPTION_OPTIONAL_ARGUMENT, 's', & o.cont},
95  {0, "dfs.chunk_size", "chunk size", OPTION_OPTIONAL_ARGUMENT, 'd', &o.chunk_size},
96  {0, "dfs.oclass", "object class", OPTION_OPTIONAL_ARGUMENT, 's', &o.oclass},
97  {0, "dfs.dir_oclass", "directory object class", OPTION_OPTIONAL_ARGUMENT, 's', &o.dir_oclass},
98  {0, "dfs.prefix", "mount prefix", OPTION_OPTIONAL_ARGUMENT, 's', & o.prefix},
99  {0, "dfs.destroy", "Destroy DFS Container", OPTION_FLAG, 'd', &o.destroy},
101 };
102 
103 /**************************** P R O T O T Y P E S *****************************/
104 static void *DFS_Create(char *, IOR_param_t *);
105 static void *DFS_Open(char *, IOR_param_t *);
106 static IOR_offset_t DFS_Xfer(int, void *, IOR_size_t *,
108 static void DFS_Close(void *, IOR_param_t *);
109 static void DFS_Delete(char *, IOR_param_t *);
110 static char* DFS_GetVersion();
111 static void DFS_Fsync(void *, IOR_param_t *);
112 static void DFS_Sync(IOR_param_t *);
113 static IOR_offset_t DFS_GetFileSize(IOR_param_t *, MPI_Comm, char *);
114 static int DFS_Statfs (const char *, ior_aiori_statfs_t *, IOR_param_t *);
115 static int DFS_Stat (const char *, struct stat *, IOR_param_t *);
116 static int DFS_Mkdir (const char *, mode_t, IOR_param_t *);
117 static int DFS_Rmdir (const char *, IOR_param_t *);
118 static int DFS_Access (const char *, int, IOR_param_t *);
119 static void DFS_Init();
120 static void DFS_Finalize();
121 static option_help * DFS_options();
122 
123 /************************** D E C L A R A T I O N S ***************************/
124 
126  .name = "DFS",
127  .create = DFS_Create,
128  .open = DFS_Open,
129  .xfer = DFS_Xfer,
130  .close = DFS_Close,
131  .delete = DFS_Delete,
132  .get_version = DFS_GetVersion,
133  .fsync = DFS_Fsync,
134  .sync = DFS_Sync,
135  .get_file_size = DFS_GetFileSize,
136  .statfs = DFS_Statfs,
137  .mkdir = DFS_Mkdir,
138  .rmdir = DFS_Rmdir,
139  .access = DFS_Access,
140  .stat = DFS_Stat,
141  .initialize = DFS_Init,
142  .finalize = DFS_Finalize,
143  .get_options = DFS_options,
144  .enable_mdtest = true,
145 };
146 
147 /***************************** F U N C T I O N S ******************************/
148 
149 /* For DAOS methods. */
150 #define DCHECK(rc, format, ...) \
151 do { \
152  int _rc = (rc); \
153  \
154  if (_rc != 0) { \
155  fprintf(stderr, "ERROR (%s:%d): %d: %d: " \
156  format"\n", __FILE__, __LINE__, rank, _rc, \
157  ##__VA_ARGS__); \
158  fflush(stderr); \
159  exit(-1); \
160  } \
161 } while (0)
162 
163 #define INFO(level, format, ...) \
164 do { \
165  if (verbose >= level) \
166  printf("[%d] "format"\n", rank, ##__VA_ARGS__); \
167 } while (0)
168 
169 #define GERR(format, ...) \
170 do { \
171  fprintf(stderr, format"\n", ##__VA_ARGS__); \
172  MPI_CHECK(MPI_Abort(MPI_COMM_WORLD, -1), "MPI_Abort() error"); \
173 } while (0)
174 
175 static inline struct aiori_dir_hdl *
176 hdl_obj(d_list_t *rlink)
177 {
178  return container_of(rlink, struct aiori_dir_hdl, entry);
179 }
180 
181 static bool
182 key_cmp(struct d_hash_table *htable, d_list_t *rlink,
183  const void *key, unsigned int ksize)
184 {
185  struct aiori_dir_hdl *hdl = hdl_obj(rlink);
186 
187  return (strcmp(hdl->name, (const char *)key) == 0);
188 }
189 
190 static void
191 rec_free(struct d_hash_table *htable, d_list_t *rlink)
192 {
193  struct aiori_dir_hdl *hdl = hdl_obj(rlink);
194 
195  assert(d_hash_rec_unlinked(&hdl->entry));
196  dfs_release(hdl->oh);
197  free(hdl);
198 }
199 
200 static d_hash_table_ops_t hdl_hash_ops = {
201  .hop_key_cmp = key_cmp,
202  .hop_rec_free = rec_free
203 };
204 
205 /* Distribute process 0's pool or container handle to others. */
206 static void
208 {
209  d_iov_t global;
210  int rc;
211 
212  global.iov_buf = NULL;
213  global.iov_buf_len = 0;
214  global.iov_len = 0;
215 
216  assert(type == POOL_HANDLE || type == CONT_HANDLE || type == DFS_HANDLE);
217  if (rank == 0) {
218  /* Get the global handle size. */
219  if (type == POOL_HANDLE)
220  rc = daos_pool_local2global(poh, &global);
221  else if (type == CONT_HANDLE)
222  rc = daos_cont_local2global(coh, &global);
223  else
224  rc = dfs_local2global(dfs, &global);
225  DCHECK(rc, "Failed to get global handle size");
226  }
227 
228  MPI_CHECK(MPI_Bcast(&global.iov_buf_len, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD),
229  "Failed to bcast global handle buffer size");
230 
231  global.iov_len = global.iov_buf_len;
232  global.iov_buf = malloc(global.iov_buf_len);
233  if (global.iov_buf == NULL)
234  ERR("Failed to allocate global handle buffer");
235 
236  if (rank == 0) {
237  if (type == POOL_HANDLE)
238  rc = daos_pool_local2global(poh, &global);
239  else if (type == CONT_HANDLE)
240  rc = daos_cont_local2global(coh, &global);
241  else
242  rc = dfs_local2global(dfs, &global);
243  DCHECK(rc, "Failed to create global handle");
244  }
245 
246  MPI_CHECK(MPI_Bcast(global.iov_buf, global.iov_buf_len, MPI_BYTE, 0, MPI_COMM_WORLD),
247  "Failed to bcast global pool handle");
248 
249  if (rank != 0) {
250  if (type == POOL_HANDLE)
251  rc = daos_pool_global2local(global, &poh);
252  else if (type == CONT_HANDLE)
253  rc = daos_cont_global2local(poh, global, &coh);
254  else
255  rc = dfs_global2local(poh, coh, 0, global, &dfs);
256  DCHECK(rc, "Failed to get local handle");
257  }
258 
259  free(global.iov_buf);
260 }
261 
262 static int
263 parse_filename(const char *path, char **_obj_name, char **_cont_name)
264 {
265  char *f1 = NULL;
266  char *f2 = NULL;
267  char *fname = NULL;
268  char *cont_name = NULL;
269  int rc = 0;
270 
271  if (path == NULL || _obj_name == NULL || _cont_name == NULL)
272  return -EINVAL;
273 
274  if (strcmp(path, "/") == 0) {
275  *_cont_name = strdup("/");
276  if (*_cont_name == NULL)
277  return -ENOMEM;
278  *_obj_name = NULL;
279  return 0;
280  }
281 
282  f1 = strdup(path);
283  if (f1 == NULL) {
284  rc = -ENOMEM;
285  goto out;
286  }
287 
288  f2 = strdup(path);
289  if (f2 == NULL) {
290  rc = -ENOMEM;
291  goto out;
292  }
293 
294  fname = basename(f1);
295  cont_name = dirname(f2);
296 
297  if (cont_name[0] == '.' || cont_name[0] != '/') {
298  char cwd[1024];
299 
300  if (getcwd(cwd, 1024) == NULL) {
301  rc = -ENOMEM;
302  goto out;
303  }
304 
305  if (strcmp(cont_name, ".") == 0) {
306  cont_name = strdup(cwd);
307  if (cont_name == NULL) {
308  rc = -ENOMEM;
309  goto out;
310  }
311  } else {
312  char *new_dir = calloc(strlen(cwd) + strlen(cont_name)
313  + 1, sizeof(char));
314  if (new_dir == NULL) {
315  rc = -ENOMEM;
316  goto out;
317  }
318 
319  strcpy(new_dir, cwd);
320  if (cont_name[0] == '.') {
321  strcat(new_dir, &cont_name[1]);
322  } else {
323  strcat(new_dir, "/");
324  strcat(new_dir, cont_name);
325  }
326  cont_name = new_dir;
327  }
328  *_cont_name = cont_name;
329  } else {
330  *_cont_name = strdup(cont_name);
331  if (*_cont_name == NULL) {
332  rc = -ENOMEM;
333  goto out;
334  }
335  }
336 
337  *_obj_name = strdup(fname);
338  if (*_obj_name == NULL) {
339  free(*_cont_name);
340  *_cont_name = NULL;
341  rc = -ENOMEM;
342  goto out;
343  }
344 
345 out:
346  if (f1)
347  free(f1);
348  if (f2)
349  free(f2);
350  return rc;
351 }
352 
353 static void
354 share_file_handle(dfs_obj_t **file, MPI_Comm comm)
355 {
356  d_iov_t global;
357  int rc;
358 
359  global.iov_buf = NULL;
360  global.iov_buf_len = 0;
361  global.iov_len = 0;
362 
363  if (rank == 0) {
364  rc = dfs_obj_local2global(dfs, *file, &global);
365  DCHECK(rc, "Failed to get global handle size");
366  }
367 
368  MPI_CHECK(MPI_Bcast(&global.iov_buf_len, 1, MPI_UINT64_T, 0, testComm),
369  "Failed to bcast global handle buffer size");
370 
371  global.iov_len = global.iov_buf_len;
372  global.iov_buf = malloc(global.iov_buf_len);
373  if (global.iov_buf == NULL)
374  ERR("Failed to allocate global handle buffer");
375 
376  if (rank == 0) {
377  rc = dfs_obj_local2global(dfs, *file, &global);
378  DCHECK(rc, "Failed to create global handle");
379  }
380 
381  MPI_CHECK(MPI_Bcast(global.iov_buf, global.iov_buf_len, MPI_BYTE, 0, testComm),
382  "Failed to bcast global pool handle");
383 
384  if (rank != 0) {
385  rc = dfs_obj_global2local(dfs, 0, global, file);
386  DCHECK(rc, "Failed to get local handle");
387  }
388 
389  free(global.iov_buf);
390 }
391 
392 static dfs_obj_t *
393 lookup_insert_dir(const char *name, mode_t *mode)
394 {
395  struct aiori_dir_hdl *hdl;
396  d_list_t *rlink;
397  int rc;
398 
399  rlink = d_hash_rec_find(dir_hash, name, strlen(name));
400  if (rlink != NULL) {
401  hdl = hdl_obj(rlink);
402  return hdl->oh;
403  }
404 
405  hdl = calloc(1, sizeof(struct aiori_dir_hdl));
406  if (hdl == NULL)
407  GERR("failed to alloc dir handle");
408 
409  strncpy(hdl->name, name, PATH_MAX-1);
410  hdl->name[PATH_MAX-1] = '\0';
411 
412  rc = dfs_lookup(dfs, name, O_RDWR, &hdl->oh, mode, NULL);
413  if (rc)
414  return NULL;
415  if (mode && S_ISREG(*mode))
416  return hdl->oh;
417 
418  rc = d_hash_rec_insert(dir_hash, hdl->name, strlen(hdl->name),
419  &hdl->entry, true);
420  DCHECK(rc, "Failed to insert dir handle in hashtable");
421 
422  return hdl->oh;
423 }
424 
426  return options;
427 }
428 
429 static void
431  int rc;
432 
434  if (dfs_init)
435  return;
436 
438  if (o.pool == NULL || o.cont == NULL)
439  return;
440 
441 #if !defined(DAOS_API_VERSION_MAJOR) || DAOS_API_VERSION_MAJOR < 1
442  if (o.svcl == NULL)
443  return;
444 #endif
445 
446  rc = daos_init();
447  DCHECK(rc, "Failed to initialize daos");
448 
449  if (o.oclass) {
450  objectClass = daos_oclass_name2id(o.oclass);
451  if (objectClass == OC_UNKNOWN)
452  GERR("Invalid DAOS object class %s\n", o.oclass);
453  }
454 
455  if (o.dir_oclass) {
456  dir_oclass = daos_oclass_name2id(o.dir_oclass);
457  if (dir_oclass == OC_UNKNOWN)
458  GERR("Invalid DAOS directory object class %s\n", o.dir_oclass);
459  }
460 
461  rc = d_hash_table_create(0, 16, NULL, &hdl_hash_ops, &dir_hash);
462  DCHECK(rc, "Failed to initialize dir hashtable");
463 
464  if (rank == 0) {
465  uuid_t pool_uuid, co_uuid;
466  daos_pool_info_t pool_info;
467  daos_cont_info_t co_info;
468 
469  rc = uuid_parse(o.pool, pool_uuid);
470  DCHECK(rc, "Failed to parse 'Pool uuid': %s", o.pool);
471 
472  rc = uuid_parse(o.cont, co_uuid);
473  DCHECK(rc, "Failed to parse 'Cont uuid': %s", o.cont);
474 
475  INFO(VERBOSE_1, "Pool uuid = %s", o.pool);
476  INFO(VERBOSE_1, "DFS Container namespace uuid = %s", o.cont);
477 
478 #if !defined(DAOS_API_VERSION_MAJOR) || DAOS_API_VERSION_MAJOR < 1
479  d_rank_list_t *svcl = NULL;
480 
481  svcl = daos_rank_list_parse(o.svcl, ":");
482  if (svcl == NULL)
483  ERR("Failed to allocate svcl");
484  INFO(VERBOSE_1, "Pool svcl = %s", o.svcl);
485 
487  rc = daos_pool_connect(pool_uuid, o.group, svcl, DAOS_PC_RW,
488  &poh, &pool_info, NULL);
489  d_rank_list_free(svcl);
490 #else
491  rc = daos_pool_connect(pool_uuid, o.group, DAOS_PC_RW,
492  &poh, &pool_info, NULL);
493 #endif
494  DCHECK(rc, "Failed to connect to pool");
495 
496  rc = daos_cont_open(poh, co_uuid, DAOS_COO_RW, &coh, &co_info,
497  NULL);
498  /* If NOEXIST we create it */
499  if (rc == -DER_NONEXIST) {
500  INFO(VERBOSE_1, "Creating DFS Container ...\n");
501 
502  rc = dfs_cont_create(poh, co_uuid, NULL, &coh, NULL);
503  if (rc)
504  DCHECK(rc, "Failed to create container");
505  } else if (rc) {
506  DCHECK(rc, "Failed to create container");
507  }
508 
509  rc = dfs_mount(poh, coh, O_RDWR, &dfs);
510  DCHECK(rc, "Failed to mount DFS namespace");
511  }
512 
516 
517  if (o.prefix) {
518  rc = dfs_set_prefix(dfs, o.prefix);
519  DCHECK(rc, "Failed to set DFS Prefix");
520  }
521  dfs_init = true;
522 }
523 
524 static void
526 {
527  int rc;
528 
529  MPI_Barrier(MPI_COMM_WORLD);
530  d_hash_table_destroy(dir_hash, true /* force */);
531 
532  rc = dfs_umount(dfs);
533  DCHECK(rc, "Failed to umount DFS namespace");
534  MPI_Barrier(MPI_COMM_WORLD);
535 
536  rc = daos_cont_close(coh, NULL);
537  DCHECK(rc, "Failed to close container %s (%d)", o.cont, rc);
538  MPI_Barrier(MPI_COMM_WORLD);
539 
540  if (o.destroy) {
541  if (rank == 0) {
542  uuid_t uuid;
543  double t1, t2;
544 
545  INFO(VERBOSE_1, "Destorying DFS Container: %s\n", o.cont);
546  uuid_parse(o.cont, uuid);
547  t1 = MPI_Wtime();
548  rc = daos_cont_destroy(poh, uuid, 1, NULL);
549  t2 = MPI_Wtime();
550  if (rc == 0)
551  INFO(VERBOSE_1, "Container Destroy time = %f secs", t2-t1);
552  }
553 
554  MPI_Bcast(&rc, 1, MPI_INT, 0, MPI_COMM_WORLD);
555  if (rc) {
556  if (rank == 0)
557  DCHECK(rc, "Failed to destroy container %s (%d)", o.cont, rc);
558  MPI_Abort(MPI_COMM_WORLD, -1);
559  }
560  }
561 
562  if (rank == 0)
563  INFO(VERBOSE_1, "Disconnecting from DAOS POOL\n");
564 
565  rc = daos_pool_disconnect(poh, NULL);
566  DCHECK(rc, "Failed to disconnect from pool");
567 
568  MPI_CHECK(MPI_Barrier(MPI_COMM_WORLD), "barrier error");
569 
570  if (rank == 0)
571  INFO(VERBOSE_1, "Finalizing DAOS..\n");
572 
573  rc = daos_fini();
574  DCHECK(rc, "Failed to finalize DAOS");
575 
577  o.pool = NULL;
578 #if !defined(DAOS_API_VERSION_MAJOR) || DAOS_API_VERSION_MAJOR < 1
579  o.svcl = NULL;
580 #endif
581  o.group = NULL;
582  o.cont = NULL;
583  o.chunk_size = 1048576;
584  o.oclass = NULL;
585  o.dir_oclass = NULL;
586  o.prefix = NULL;
587  o.destroy = 0;
588  objectClass = 0;
589  dir_oclass = 0;
590  dfs_init = false;
591 }
592 
593 /*
594  * Creat and open a file through the DFS interface.
595  */
596 static void *
597 DFS_Create(char *testFileName, IOR_param_t *param)
598 {
599  char *name = NULL, *dir_name = NULL;
600  dfs_obj_t *obj = NULL, *parent = NULL;
601  mode_t mode = 0664;
602  int fd_oflag = 0;
603  int rc;
604 
605  assert(param);
606 
607  rc = parse_filename(testFileName, &name, &dir_name);
608  DCHECK(rc, "Failed to parse path %s", testFileName);
609  assert(dir_name);
610  assert(name);
611 
612  mode = S_IFREG | mode;
613  if (param->filePerProc || rank == 0) {
614  fd_oflag |= O_CREAT | O_RDWR | O_EXCL;
615 
616  parent = lookup_insert_dir(dir_name, NULL);
617  if (parent == NULL)
618  GERR("Failed to lookup parent dir");
619 
620  rc = dfs_open(dfs, parent, name, mode, fd_oflag,
621  objectClass, o.chunk_size, NULL, &obj);
622  DCHECK(rc, "dfs_open() of %s Failed", name);
623  }
624  if (!param->filePerProc) {
626  }
627 
628  if (name)
629  free(name);
630  if (dir_name)
631  free(dir_name);
632 
633  return ((void *)obj);
634 }
635 
636 /*
637  * Open a file through the DFS interface.
638  */
639 static void *
640 DFS_Open(char *testFileName, IOR_param_t *param)
641 {
642  char *name = NULL, *dir_name = NULL;
643  dfs_obj_t *obj = NULL, *parent = NULL;
644  mode_t mode;
645  int rc;
646  int fd_oflag = 0;
647 
648  fd_oflag |= O_RDWR;
649  mode = S_IFREG | param->mode;
650 
651  rc = parse_filename(testFileName, &name, &dir_name);
652  DCHECK(rc, "Failed to parse path %s", testFileName);
653 
654  assert(dir_name);
655  assert(name);
656 
657  if (param->filePerProc || rank == 0) {
658  parent = lookup_insert_dir(dir_name, NULL);
659  if (parent == NULL)
660  GERR("Failed to lookup parent dir");
661 
662  rc = dfs_open(dfs, parent, name, mode, fd_oflag, objectClass,
663  o.chunk_size, NULL, &obj);
664  DCHECK(rc, "dfs_open() of %s Failed", name);
665  }
666 
667  if (!param->filePerProc) {
669  }
670 
671  if (name)
672  free(name);
673  if (dir_name)
674  free(dir_name);
675 
676  return ((void *)obj);
677 }
678 
679 /*
680  * Write or read access to file using the DFS interface.
681  */
682 static IOR_offset_t
683 DFS_Xfer(int access, void *file, IOR_size_t *buffer, IOR_offset_t length,
685 {
686  int xferRetries = 0;
687  long long remaining = (long long)length;
688  char *ptr = (char *)buffer;
689  daos_size_t ret;
690  int rc;
691  dfs_obj_t *obj;
692 
693  obj = (dfs_obj_t *)file;
694 
695  while (remaining > 0) {
696  d_iov_t iov;
697  d_sg_list_t sgl;
698 
700  sgl.sg_nr = 1;
701  sgl.sg_nr_out = 0;
702  d_iov_set(&iov, (void *)ptr, remaining);
703  sgl.sg_iovs = &iov;
704 
705  /* write/read file */
706  if (access == WRITE) {
707  rc = dfs_write(dfs, obj, &sgl, param->offset, NULL);
708  if (rc) {
709  fprintf(stderr, "dfs_write() failed (%d)\n", rc);
710  return -1;
711  }
712  ret = remaining;
713  } else {
714  rc = dfs_read(dfs, obj, &sgl, param->offset, &ret, NULL);
715  if (rc || ret == 0)
716  fprintf(stderr, "dfs_read() failed(%d)\n", rc);
717  }
718 
719  if (ret < remaining) {
720  if (param->singleXferAttempt == TRUE)
721  exit(-1);
722  if (xferRetries > MAX_RETRY)
723  ERR("too many retries -- aborting");
724  }
725 
726  assert(ret >= 0);
727  assert(ret <= remaining);
728  remaining -= ret;
729  ptr += ret;
730  xferRetries++;
731  }
732 
733  return (length);
734 }
735 
736 /*
737  * Perform fsync().
738  */
739 static void
741 {
742  /* no cache in DFS, so this is a no-op currently */
743  dfs_sync(dfs);
744  return;
745 }
746 
747 /*
748  * Perform sync() on the dfs mount.
749  */
750 static void
752 {
753  /* no cache in DFS, so this is a no-op currently */
754  dfs_sync(dfs);
755  return;
756 }
757 
758 /*
759  * Close a file through the DFS interface.
760  */
761 static void
763 {
764  dfs_release((dfs_obj_t *)fd);
765 }
766 
767 /*
768  * Delete a file through the DFS interface.
769  */
770 static void
771 DFS_Delete(char *testFileName, IOR_param_t * param)
772 {
773  char *name = NULL, *dir_name = NULL;
774  dfs_obj_t *parent = NULL;
775  int rc;
776 
777  rc = parse_filename(testFileName, &name, &dir_name);
778  DCHECK(rc, "Failed to parse path %s", testFileName);
779 
780  assert(dir_name);
781  assert(name);
782 
783  parent = lookup_insert_dir(dir_name, NULL);
784  if (parent == NULL)
785  GERR("Failed to lookup parent dir");
786 
787  rc = dfs_remove(dfs, parent, name, false, NULL);
788  DCHECK(rc, "dfs_remove() of %s Failed", name);
789 
790  if (name)
791  free(name);
792  if (dir_name)
793  free(dir_name);
794 }
795 
796 static char* DFS_GetVersion()
797 {
798  static char ver[1024] = {};
799 
800  sprintf(ver, "%s", "DAOS");
801  return ver;
802 }
803 
804 /*
805  * Use DFS stat() to return aggregate file size.
806  */
807 static IOR_offset_t
808 DFS_GetFileSize(IOR_param_t * test, MPI_Comm comm, char *testFileName)
809 {
810  dfs_obj_t *obj;
811  daos_size_t fsize, tmpMin, tmpMax, tmpSum;
812  int rc;
813 
814  if (test->filePerProc || rank == 0) {
815  rc = dfs_lookup(dfs, testFileName, O_RDONLY, &obj, NULL, NULL);
816  if (rc) {
817  fprintf(stderr, "dfs_lookup() of %s Failed (%d)", testFileName, rc);
818  return -1;
819  }
820 
821  rc = dfs_get_size(dfs, obj, &fsize);
822  dfs_release(obj);
823  if (rc)
824  return -1;
825  }
826 
827  if (test->filePerProc == TRUE) {
828  MPI_CHECK(MPI_Allreduce(&fsize, &tmpSum, 1,
829  MPI_LONG_LONG_INT, MPI_SUM, comm),
830  "cannot total data moved");
831  fsize = tmpSum;
832  } else {
833  MPI_CHECK(MPI_Allreduce(&fsize, &tmpMin, 1,
834  MPI_LONG_LONG_INT, MPI_MIN, comm),
835  "cannot total data moved");
836  MPI_CHECK(MPI_Allreduce(&fsize, &tmpMax, 1,
837  MPI_LONG_LONG_INT, MPI_MAX, comm),
838  "cannot total data moved");
839  if (tmpMin != tmpMax) {
840  if (rank == 0) {
841  WARN("inconsistent file size by different tasks");
842  }
843  /* incorrect, but now consistent across tasks */
844  fsize = tmpMin;
845  }
846  }
847 
848  return (fsize);
849 }
850 
851 static int
852 DFS_Statfs(const char *path, ior_aiori_statfs_t *sfs, IOR_param_t * param)
853 {
854  return 0;
855 }
856 
857 static int
858 DFS_Mkdir(const char *path, mode_t mode, IOR_param_t * param)
859 {
860  dfs_obj_t *parent = NULL;
861  char *name = NULL, *dir_name = NULL;
862  int rc;
863 
864  rc = parse_filename(path, &name, &dir_name);
865  DCHECK(rc, "Failed to parse path %s", path);
866 
867  assert(dir_name);
868  if (!name)
869  return 0;
870 
871  parent = lookup_insert_dir(dir_name, NULL);
872  if (parent == NULL)
873  GERR("Failed to lookup parent dir");
874 
875  rc = dfs_mkdir(dfs, parent, name, mode, dir_oclass);
876  DCHECK(rc, "dfs_mkdir() of %s Failed", name);
877 
878  if (name)
879  free(name);
880  if (dir_name)
881  free(dir_name);
882  return rc;
883 }
884 
885 static int
886 DFS_Rmdir(const char *path, IOR_param_t * param)
887 {
888  dfs_obj_t *parent = NULL;
889  char *name = NULL, *dir_name = NULL;
890  int rc;
891 
892  rc = parse_filename(path, &name, &dir_name);
893  DCHECK(rc, "Failed to parse path %s", path);
894 
895  assert(dir_name);
896  assert(name);
897 
898  parent = lookup_insert_dir(dir_name, NULL);
899  if (parent == NULL)
900  GERR("Failed to lookup parent dir");
901 
902  rc = dfs_remove(dfs, parent, name, false, NULL);
903  DCHECK(rc, "dfs_remove() of %s Failed", name);
904 
905  if (name)
906  free(name);
907  if (dir_name)
908  free(dir_name);
909  if (rc)
910  return -1;
911  return rc;
912 }
913 
914 static int
915 DFS_Access(const char *path, int mode, IOR_param_t * param)
916 {
917  dfs_obj_t *obj = NULL;
918  mode_t fmode;
919 
920  obj = lookup_insert_dir(path, &fmode);
921  if (obj == NULL)
922  return -1;
923 
925  if (S_ISREG(fmode))
926  dfs_release(obj);
927 
928  return 0;
929 }
930 
931 static int
932 DFS_Stat(const char *path, struct stat *buf, IOR_param_t * param)
933 {
934  dfs_obj_t *parent = NULL;
935  char *name = NULL, *dir_name = NULL;
936  int rc;
937 
938  rc = parse_filename(path, &name, &dir_name);
939  DCHECK(rc, "Failed to parse path %s", path);
940 
941  assert(dir_name);
942  assert(name);
943 
944  parent = lookup_insert_dir(dir_name, NULL);
945  if (parent == NULL)
946  GERR("Failed to lookup parent dir");
947 
948  rc = dfs_stat(dfs, parent, name, buf);
949 
950  if (name)
951  free(name);
952  if (dir_name)
953  free(dir_name);
954  if (rc)
955  return -1;
956  return rc;
957 }
static void HandleDistribute(enum handleType type)
Definition: aiori-DFS.c:207
static void rec_free(struct d_hash_table *htable, d_list_t *rlink)
Definition: aiori-DFS.c:191
static option_help * DFS_options()
Definition: aiori-DFS.c:425
handleType
Definition: aiori-DFS.c:53
static int DFS_Stat(const char *, struct stat *, IOR_param_t *)
Definition: aiori-DFS.c:932
static void DFS_Delete(char *, IOR_param_t *)
Definition: aiori-DFS.c:771
int chunk_size
Definition: aiori-DFS.c:67
#define ERR(MSG)
Definition: iordef.h:184
char * pool
Definition: aiori-DFS.c:61
static struct aiori_dir_hdl * hdl_obj(d_list_t *rlink)
Definition: aiori-DFS.c:176
#define LAST_OPTION
Definition: option.h:37
int filePerProc
Definition: ior.h:111
#define DCHECK(rc, format,...)
Definition: aiori-DFS.c:150
static daos_handle_t coh
Definition: aiori-DFS.c:41
dfs_obj_t * oh
Definition: aiori-DFS.c:49
CURLcode rc
Definition: aiori-S3.c:121
dfs_t * dfs
Definition: aiori-DFS.c:40
int destroy
Definition: aiori-DFS.c:71
static void DFS_Finalize()
Definition: aiori-DFS.c:525
static int parse_filename(const char *path, char **_obj_name, char **_cont_name)
Definition: aiori-DFS.c:263
static void * DFS_Create(char *, IOR_param_t *)
Definition: aiori-DFS.c:597
static d_hash_table_ops_t hdl_hash_ops
Definition: aiori-DFS.c:200
static void DFS_Sync(IOR_param_t *)
Definition: aiori-DFS.c:751
#define GERR(format,...)
Definition: aiori-DFS.c:169
#define WRITE
Definition: iordef.h:95
static void share_file_handle(dfs_obj_t **file, MPI_Comm comm)
Definition: aiori-DFS.c:354
unsigned int mode
Definition: ior.h:87
static struct dfs_options o
Definition: aiori-DFS.c:74
static int DFS_Statfs(const char *, ior_aiori_statfs_t *, IOR_param_t *)
Definition: aiori-DFS.c:852
#define INFO(level, format,...)
Definition: aiori-DFS.c:163
char * prefix
Definition: aiori-DFS.c:70
static struct d_hash_table * dir_hash
Definition: aiori-DFS.c:44
MPI_Comm testComm
Definition: utilities.c:60
char * cont
Definition: aiori-DFS.c:66
#define MPI_CHECK(MPI_STATUS, MSG)
Definition: iordef.h:224
#define PATH_MAX
Definition: iordef.h:112
static daos_oclass_id_t dir_oclass
Definition: aiori-DFS.c:43
static void * DFS_Open(char *, IOR_param_t *)
Definition: aiori-DFS.c:640
static IOR_offset_t DFS_GetFileSize(IOR_param_t *, MPI_Comm, char *)
Definition: aiori-DFS.c:808
char * svcl
Definition: aiori-DFS.c:63
int singleXferAttempt
Definition: ior.h:161
#define MAX_RETRY
Definition: iordef.h:110
static bool dfs_init
Definition: aiori-DFS.c:45
static daos_handle_t poh
Definition: aiori-DFS.c:41
static IOR_param_t param
Definition: mdtest.c:170
static daos_oclass_id_t objectClass
Definition: aiori-DFS.c:42
static void DFS_Close(void *, IOR_param_t *)
Definition: aiori-DFS.c:762
ior_aiori_t dfs_aiori
Definition: aiori-DFS.c:125
long long int IOR_size_t
Definition: iordef.h:123
#define WARN(MSG)
Definition: iordef.h:144
char * dir_oclass
Definition: aiori-DFS.c:69
static int DFS_Access(const char *, int, IOR_param_t *)
Definition: aiori-DFS.c:915
static void DFS_Fsync(void *, IOR_param_t *)
Definition: aiori-DFS.c:740
static char * DFS_GetVersion()
Definition: aiori-DFS.c:796
static dfs_obj_t * lookup_insert_dir(const char *name, mode_t *mode)
Definition: aiori-DFS.c:393
static option_help options[]
Definition: aiori-DFS.c:88
static void DFS_Init()
Definition: aiori-DFS.c:430
IOR_offset_t offset
Definition: ior.h:126
char * group
Definition: aiori-DFS.c:65
char * oclass
Definition: aiori-DFS.c:68
d_list_t entry
Definition: aiori-DFS.c:48
char name[PATH_MAX]
Definition: aiori-DFS.c:50
#define VERBOSE_1
Definition: iordef.h:102
static int DFS_Mkdir(const char *, mode_t, IOR_param_t *)
Definition: aiori-DFS.c:858
static IOR_offset_t DFS_Xfer(int, void *, IOR_size_t *, IOR_offset_t, IOR_param_t *)
Definition: aiori-DFS.c:683
static int DFS_Rmdir(const char *, IOR_param_t *)
Definition: aiori-DFS.c:886
char * name
Definition: aiori.h:68
long long int IOR_offset_t
Definition: iordef.h:122
int rank
Definition: utilities.c:57
#define TRUE
Definition: iordef.h:75
static bool key_cmp(struct d_hash_table *htable, d_list_t *rlink, const void *key, unsigned int ksize)
Definition: aiori-DFS.c:182
#define NULL
Definition: iordef.h:79