IOR
aiori-RADOS.c
Go to the documentation of this file.
1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
2  * vim:expandtab:shiftwidth=8:tabstop=8:
3  */
4 /******************************************************************************\
5 * *
6 * (C) 2015 The University of Chicago *
7 * *
8 * See COPYRIGHT in top-level directory. *
9 * *
10 ********************************************************************************
11 *
12 * Implement abstract I/O interface for RADOS.
13 *
14 \******************************************************************************/
15 
16 #ifdef HAVE_CONFIG_H
17 # include "config.h"
18 #endif
19 
20 #include <stdio.h>
21 #include <stdlib.h>
22 #include <sys/stat.h>
23 #include <rados/librados.h>
24 
25 #include "ior.h"
26 #include "iordef.h"
27 #include "aiori.h"
28 #include "utilities.h"
29 
30 /************************** O P T I O N S *****************************/
32  char * user;
33  char * conf;
34  char * pool;
35 };
36 
37 static struct rados_options o = {
38  .user = NULL,
39  .conf = NULL,
40  .pool = NULL,
41 };
42 
43 static option_help options [] = {
44  {0, "rados.user", "Username for the RADOS cluster", OPTION_REQUIRED_ARGUMENT, 's', & o.user},
45  {0, "rados.conf", "Config file for the RADOS cluster", OPTION_REQUIRED_ARGUMENT, 's', & o.conf},
46  {0, "rados.pool", "RADOS pool to use for I/O", OPTION_REQUIRED_ARGUMENT, 's', & o.pool},
48 };
49 
50 
51 /**************************** P R O T O T Y P E S *****************************/
52 static void *RADOS_Create(char *, IOR_param_t *);
53 static void *RADOS_Open(char *, IOR_param_t *);
54 static IOR_offset_t RADOS_Xfer(int, void *, IOR_size_t *,
56 static void RADOS_Close(void *, IOR_param_t *);
57 static void RADOS_Delete(char *, IOR_param_t *);
58 static void RADOS_Fsync(void *, IOR_param_t *);
59 static IOR_offset_t RADOS_GetFileSize(IOR_param_t *, MPI_Comm, char *);
60 static int RADOS_StatFS(const char *, ior_aiori_statfs_t *, IOR_param_t *);
61 static int RADOS_MkDir(const char *, mode_t, IOR_param_t *);
62 static int RADOS_RmDir(const char *, IOR_param_t *);
63 static int RADOS_Access(const char *, int, IOR_param_t *);
64 static int RADOS_Stat(const char *, struct stat *, IOR_param_t *);
65 static option_help * RADOS_options();
66 
67 /************************** D E C L A R A T I O N S ***************************/
69  .name = "RADOS",
70  .create = RADOS_Create,
71  .open = RADOS_Open,
72  .xfer = RADOS_Xfer,
73  .close = RADOS_Close,
74  .delete = RADOS_Delete,
75  .get_version = aiori_get_version,
76  .fsync = RADOS_Fsync,
77  .get_file_size = RADOS_GetFileSize,
78  .statfs = RADOS_StatFS,
79  .mkdir = RADOS_MkDir,
80  .rmdir = RADOS_RmDir,
81  .access = RADOS_Access,
82  .stat = RADOS_Stat,
83  .get_options = RADOS_options,
84 };
85 
86 #define RADOS_ERR(__err_str, __ret) do { \
87  errno = -__ret; \
88  ERR(__err_str); \
89 } while(0)
90 
91 /***************************** F U N C T I O N S ******************************/
93  return options;
94 }
95 
97 {
98  int ret;
99 
100  /* create RADOS cluster handle */
101  ret = rados_create(&param->rados_cluster, o.user);
102  if (ret)
103  RADOS_ERR("unable to create RADOS cluster handle", ret);
104 
105  /* set the handle using the Ceph config */
106  ret = rados_conf_read_file(param->rados_cluster, o.conf);
107  if (ret)
108  RADOS_ERR("unable to read RADOS config file", ret);
109 
110  /* connect to the RADOS cluster */
111  ret = rados_connect(param->rados_cluster);
112  if (ret)
113  RADOS_ERR("unable to connect to the RADOS cluster", ret);
114 
115  /* create an io context for the pool we are operating on */
116  ret = rados_ioctx_create(param->rados_cluster, o.pool, &param->rados_ioctx);
117  if (ret)
118  RADOS_ERR("unable to create an I/O context for the RADOS cluster", ret);
119 
120  return;
121 }
122 
124 {
125  /* ioctx destroy */
126  rados_ioctx_destroy(param->rados_ioctx);
127 
128  /* shutdown */
129  rados_shutdown(param->rados_cluster);
130 }
131 
132 static void *RADOS_Create_Or_Open(char *testFileName, IOR_param_t * param, int create_flag)
133 {
134  int ret;
135  char *oid;
136 
137  RADOS_Cluster_Init(param);
138 
139  if (param->useO_DIRECT == TRUE)
140  WARN("direct I/O mode is not implemented in RADOS\n");
141 
142  oid = strdup(testFileName);
143  if (!oid)
144  ERR("unable to allocate RADOS oid");
145 
146  if (create_flag)
147  {
148  rados_write_op_t create_op;
149  int rados_create_flag;
150 
151  if (param->openFlags & IOR_EXCL)
152  rados_create_flag = LIBRADOS_CREATE_EXCLUSIVE;
153  else
154  rados_create_flag = LIBRADOS_CREATE_IDEMPOTENT;
155 
156  /* create a RADOS "write op" for creating the object */
157  create_op = rados_create_write_op();
158  rados_write_op_create(create_op, rados_create_flag, NULL);
159  ret = rados_write_op_operate(create_op, param->rados_ioctx, oid,
160  NULL, 0);
161  rados_release_write_op(create_op);
162  if (ret)
163  RADOS_ERR("unable to create RADOS object", ret);
164  }
165  else
166  {
167  /* XXX actually, we should probably assert oid existence here? */
168  }
169 
170  return (void *)oid;
171 }
172 
173 static void *RADOS_Create(char *testFileName, IOR_param_t * param)
174 {
175  return RADOS_Create_Or_Open(testFileName, param, TRUE);
176 }
177 
178 static void *RADOS_Open(char *testFileName, IOR_param_t * param)
179 {
180  if (param->openFlags & IOR_CREAT)
181  return RADOS_Create_Or_Open(testFileName, param, TRUE);
182  else
183  return RADOS_Create_Or_Open(testFileName, param, FALSE);
184 }
185 
186 static IOR_offset_t RADOS_Xfer(int access, void *fd, IOR_size_t * buffer,
187  IOR_offset_t length, IOR_param_t * param)
188 {
189  int ret;
190  char *oid = (char *)fd;
191 
192  if (access == WRITE)
193  {
194  rados_write_op_t write_op;
195 
196  write_op = rados_create_write_op();
197  rados_write_op_write(write_op, (const char *)buffer,
198  length, param->offset);
199  ret = rados_write_op_operate(write_op, param->rados_ioctx,
200  oid, NULL, 0);
201  rados_release_write_op(write_op);
202  if (ret)
203  RADOS_ERR("unable to write RADOS object", ret);
204  }
205  else /* READ */
206  {
207  int read_ret;
208  size_t bytes_read;
209  rados_read_op_t read_op;
210 
211  read_op = rados_create_read_op();
212  rados_read_op_read(read_op, param->offset, length, (char *)buffer,
213  &bytes_read, &read_ret);
214  ret = rados_read_op_operate(read_op, param->rados_ioctx, oid, 0);
215  rados_release_read_op(read_op);
216  if (ret || read_ret || ((IOR_offset_t)bytes_read != length))
217  RADOS_ERR("unable to read RADOS object", ret);
218  }
219 
220  return length;
221 }
222 
223 static void RADOS_Fsync(void *fd, IOR_param_t * param)
224 {
225  return;
226 }
227 
228 static void RADOS_Close(void *fd, IOR_param_t * param)
229 {
230  char *oid = (char *)fd;
231 
232  /* object does not need to be "closed", but we should tear the cluster down */
233  RADOS_Cluster_Finalize(param);
234  free(oid);
235 
236  return;
237 }
238 
239 static void RADOS_Delete(char *testFileName, IOR_param_t * param)
240 {
241  int ret;
242  char *oid = testFileName;
243  rados_write_op_t remove_op;
244 
245  /* we have to reestablish cluster connection here... */
246  RADOS_Cluster_Init(param);
247 
248  /* remove the object */
249  remove_op = rados_create_write_op();
250  rados_write_op_remove(remove_op);
251  ret = rados_write_op_operate(remove_op, param->rados_ioctx,
252  oid, NULL, 0);
253  rados_release_write_op(remove_op);
254  if (ret)
255  RADOS_ERR("unable to remove RADOS object", ret);
256 
257  RADOS_Cluster_Finalize(param);
258 
259  return;
260 }
261 
263  char *testFileName)
264 {
265  int ret;
266  char *oid = testFileName;
267  rados_read_op_t stat_op;
268  uint64_t oid_size;
269  int stat_ret;
270  IOR_offset_t aggSizeFromStat, tmpMin, tmpMax, tmpSum;
271 
272  /* we have to reestablish cluster connection here... */
273  RADOS_Cluster_Init(test);
274 
275  /* stat the object */
276  stat_op = rados_create_read_op();
277  rados_read_op_stat(stat_op, &oid_size, NULL, &stat_ret);
278  ret = rados_read_op_operate(stat_op, test->rados_ioctx, oid, 0);
279  rados_release_read_op(stat_op);
280  if (ret || stat_ret)
281  RADOS_ERR("unable to stat RADOS object", stat_ret);
282  aggSizeFromStat = oid_size;
283 
284  if (test->filePerProc == TRUE)
285  {
286  MPI_CHECK(MPI_Allreduce(&aggSizeFromStat, &tmpSum, 1,
287  MPI_LONG_LONG_INT, MPI_SUM, testComm),
288  "cannot total data moved");
289  aggSizeFromStat = tmpSum;
290  }
291  else
292  {
293  MPI_CHECK(MPI_Allreduce(&aggSizeFromStat, &tmpMin, 1,
294  MPI_LONG_LONG_INT, MPI_MIN, testComm),
295  "cannot total data moved");
296  MPI_CHECK(MPI_Allreduce(&aggSizeFromStat, &tmpMax, 1,
297  MPI_LONG_LONG_INT, MPI_MAX, testComm),
298  "cannot total data moved");
299  if (tmpMin != tmpMax)
300  {
301  if (rank == 0)
302  WARN("inconsistent file size by different tasks");
303 
304  /* incorrect, but now consistent across tasks */
305  aggSizeFromStat = tmpMin;
306  }
307  }
308 
310 
311  return aggSizeFromStat;
312 }
313 
314 static int RADOS_StatFS(const char *oid, ior_aiori_statfs_t *stat_buf,
316 {
317  WARN("statfs not supported in RADOS backend!");
318  return -1;
319 }
320 
321 static int RADOS_MkDir(const char *oid, mode_t mode, IOR_param_t *param)
322 {
323  WARN("mkdir not supported in RADOS backend!");
324  return -1;
325 }
326 
327 static int RADOS_RmDir(const char *oid, IOR_param_t *param)
328 {
329  WARN("rmdir not supported in RADOS backend!");
330  return -1;
331 }
332 
333 static int RADOS_Access(const char *oid, int mode, IOR_param_t *param)
334 {
335  rados_read_op_t read_op;
336  int ret;
337  int prval;
338  uint64_t oid_size;
339 
340  /* we have to reestablish cluster connection here... */
341  RADOS_Cluster_Init(param);
342 
343  /* use read_op stat to check for oid existence */
344  read_op = rados_create_read_op();
345  rados_read_op_stat(read_op, &oid_size, NULL, &prval);
346  ret = rados_read_op_operate(read_op, param->rados_ioctx, oid, 0);
347  rados_release_read_op(read_op);
348 
349  RADOS_Cluster_Finalize(param);
350 
351  if (ret | prval)
352  return -1;
353  else
354  return 0;
355 }
356 
357 static int RADOS_Stat(const char *oid, struct stat *buf, IOR_param_t *param)
358 {
359  WARN("stat not supported in RADOS backend!");
360  return -1;
361 }
static IOR_offset_t RADOS_GetFileSize(IOR_param_t *, MPI_Comm, char *)
Definition: aiori-RADOS.c:262
#define ERR(MSG)
Definition: iordef.h:169
#define LAST_OPTION
Definition: option.h:36
static int RADOS_RmDir(const char *, IOR_param_t *)
Definition: aiori-RADOS.c:327
int filePerProc
Definition: ior.h:104
#define RADOS_ERR(__err_str, __ret)
Definition: aiori-RADOS.c:86
static void RADOS_Close(void *, IOR_param_t *)
Definition: aiori-RADOS.c:228
static void * RADOS_Create(char *, IOR_param_t *)
Definition: aiori-RADOS.c:173
static void RADOS_Fsync(void *, IOR_param_t *)
Definition: aiori-RADOS.c:223
static void * RADOS_Create_Or_Open(char *testFileName, IOR_param_t *param, int create_flag)
Definition: aiori-RADOS.c:132
ior_aiori_t rados_aiori
Definition: aiori-RADOS.c:68
unsigned int openFlags
Definition: ior.h:85
#define WRITE
Definition: iordef.h:95
static void * RADOS_Open(char *, IOR_param_t *)
Definition: aiori-RADOS.c:178
static void RADOS_Delete(char *, IOR_param_t *)
Definition: aiori-RADOS.c:239
#define IOR_CREAT
Definition: aiori.h:37
#define IOR_EXCL
Definition: aiori.h:39
char * aiori_get_version()
Definition: aiori.c:149
MPI_Comm testComm
Definition: utilities.c:61
#define MPI_CHECK(MPI_STATUS, MSG)
Definition: iordef.h:192
static int RADOS_Access(const char *, int, IOR_param_t *)
Definition: aiori-RADOS.c:333
static struct rados_options o
Definition: aiori-RADOS.c:37
static IOR_param_t param
Definition: mdtest.c:153
static int RADOS_Stat(const char *, struct stat *, IOR_param_t *)
Definition: aiori-RADOS.c:357
#define FALSE
Definition: iordef.h:71
long long int IOR_size_t
Definition: iordef.h:124
#define WARN(MSG)
Definition: iordef.h:145
rados_t rados_cluster
Definition: ior.h:180
static void RADOS_Cluster_Init(IOR_param_t *param)
Definition: aiori-RADOS.c:96
static int RADOS_StatFS(const char *, ior_aiori_statfs_t *, IOR_param_t *)
Definition: aiori-RADOS.c:314
static IOR_offset_t RADOS_Xfer(int, void *, IOR_size_t *, IOR_offset_t, IOR_param_t *)
Definition: aiori-RADOS.c:186
IOR_offset_t offset
Definition: ior.h:119
static option_help * RADOS_options()
Definition: aiori-RADOS.c:92
static int RADOS_MkDir(const char *, mode_t, IOR_param_t *)
Definition: aiori-RADOS.c:321
char * name
Definition: aiori.h:67
rados_ioctx_t rados_ioctx
Definition: ior.h:181
long long int IOR_offset_t
Definition: iordef.h:123
int rank
Definition: utilities.c:57
int useO_DIRECT
Definition: ior.h:125
#define TRUE
Definition: iordef.h:75
static void RADOS_Cluster_Finalize(IOR_param_t *param)
Definition: aiori-RADOS.c:123
static option_help options[]
Definition: aiori-RADOS.c:43
#define NULL
Definition: iordef.h:79