23 # include <sys/ioctl.h> 35 #ifdef HAVE_LINUX_LUSTRE_LUSTRE_USER_H 36 # include <linux/lustre/lustre_user.h> 37 #elif defined(HAVE_LUSTRE_LUSTRE_USER_H) 38 # include <lustre/lustre_user.h> 43 #ifdef HAVE_GPFS_FCNTL_H 44 # include <gpfs_fcntl.h> 47 #ifdef HAVE_BEEGFS_BEEGFS_H 48 #include <beegfs/beegfs.h> 63 # define lseek64 lseek 97 #ifdef HAVE_GPFS_FCNTL_H 98 void gpfs_free_all_locks(
int fd)
102 gpfsFcntlHeader_t header;
103 gpfsFreeRange_t release;
105 release_all.header.totalLength =
sizeof(release_all);
106 release_all.header.fcntlVersion = GPFS_FCNTL_CURRENT_VERSION;
107 release_all.header.fcntlReserved = 0;
109 release_all.release.structLen =
sizeof(release_all.release);
110 release_all.release.structType = GPFS_FREE_RANGE;
111 release_all.release.start = 0;
112 release_all.release.length = 0;
114 rc = gpfs_fcntl(fd, &release_all);
116 EWARN(
"gpfs_fcntl release all locks hint failed.");
123 gpfsFcntlHeader_t header;
124 gpfsAccessRange_t access;
127 take_locks.header.totalLength =
sizeof(take_locks);
128 take_locks.header.fcntlVersion = GPFS_FCNTL_CURRENT_VERSION;
129 take_locks.header.fcntlReserved = 0;
131 take_locks.access.structLen =
sizeof(take_locks.access);
132 take_locks.access.structType = GPFS_ACCESS_RANGE;
133 take_locks.access.start = param->
offset;
134 take_locks.access.length = length;
135 take_locks.access.isWrite = (access ==
WRITE);
137 rc = gpfs_fcntl(fd, &take_locks);
139 EWARN(
"gpfs_fcntl access range hint failed.");
147 gpfsFcntlHeader_t header;
148 gpfsFreeRange_t free;
152 free_locks.header.totalLength =
sizeof(free_locks);
153 free_locks.header.fcntlVersion = GPFS_FCNTL_CURRENT_VERSION;
154 free_locks.header.fcntlReserved = 0;
156 free_locks.free.structLen =
sizeof(free_locks.free);
157 free_locks.free.structType = GPFS_FREE_RANGE;
158 free_locks.free.start = param->
offset;
159 free_locks.free.length = length;
161 rc = gpfs_fcntl(fd, &free_locks);
163 EWARN(
"gpfs_fcntl free range hint failed.");
169 #ifdef HAVE_BEEGFS_BEEGFS_H 171 int mkTempInDir(
char* dirPath)
173 unsigned long len = strlen(dirPath) + 8;
174 char* tmpfilename = (
char*)malloc(
sizeof (
char)*len+1);
175 snprintf(tmpfilename, len,
"%s/XXXXXX", dirPath);
177 int fd = mkstemp(tmpfilename);
184 bool beegfs_getStriping(
char* dirPath, u_int16_t* numTargetsOut,
unsigned* chunkSizeOut)
188 int fd = mkTempInDir(dirPath);
190 unsigned stripePattern = 0;
191 retVal = beegfs_getStripeInfo(fd, &stripePattern, chunkSizeOut, numTargetsOut);
198 bool beegfs_isOptionSet(
int opt) {
205 bool beegfs_createFilePath(
char* filepath, mode_t mode,
int numTargets,
int chunkSize)
208 char* dirTmp = strdup(filepath);
209 char* dir = dirname(dirTmp);
210 DIR* parentDirS = opendir(dir);
212 ERR(
"Failed to get directory");
216 int parentDirFd = dirfd(parentDirS);
219 ERR(
"Failed to get directory descriptor");
223 bool isBeegfs = beegfs_testIsBeeGFS(parentDirFd);
226 WARN(
"Not a BeeGFS file system");
230 if ( !beegfs_isOptionSet(numTargets)
231 || !beegfs_isOptionSet(chunkSize)) {
232 u_int16_t defaultNumTargets = 0;
233 unsigned defaultChunkSize = 0;
234 bool haveDefaults = beegfs_getStriping(dir,
238 ERR(
"Failed to get default BeeGFS striping values");
240 numTargets = beegfs_isOptionSet(numTargets) ?
241 numTargets : defaultNumTargets;
242 chunkSize = beegfs_isOptionSet(chunkSize) ?
243 chunkSize : defaultChunkSize;
246 char* filenameTmp = strdup(filepath);
247 char* filename = basename(filepath);
248 bool isFileCreated = beegfs_createFile(parentDirFd, filename,
249 mode, numTargets, chunkSize);
251 ERR(
"Could not create file");
256 closedir(parentDirS);
272 fd = (
int *)malloc(
sizeof(
int));
274 ERR(
"Unable to malloc file descriptor");
279 #ifdef HAVE_LUSTRE_LUSTRE_USER_H 283 #define FASYNC 00020000 293 *fd =
open64(testFileName, fd_oflag, 0664);
295 ERR(
"open64() failed");
297 struct lov_user_md opts = { 0 };
300 opts.lmm_magic = LOV_USER_MAGIC;
309 O_CREAT | O_EXCL | O_RDWR | O_LOV_DELAY_CREATE;
310 *fd =
open64(testFileName, fd_oflag, 0664);
312 fprintf(stdout,
"\nUnable to open '%s': %s\n",
313 testFileName, strerror(
errno));
315 "MPI_Abort() error");
316 }
else if (ioctl(*fd, LL_IOC_LOV_SETSTRIPE, &opts)) {
317 char *errmsg =
"stripe already set";
319 errmsg = strerror(
errno);
321 "\nError on ioctl for '%s' (%d): %s\n",
322 testFileName, *fd, errmsg);
324 "MPI_Abort() error");
333 fd_oflag |= O_CREAT | O_RDWR;
335 #ifdef HAVE_BEEGFS_BEEGFS_H 338 bool result = beegfs_createFilePath(testFileName,
343 fd_oflag &= ~O_CREAT;
345 EWARN(
"BeeGFS tuning failed");
350 *fd =
open64(testFileName, fd_oflag, 0664);
352 ERR(
"open64() failed");
354 #ifdef HAVE_LUSTRE_LUSTRE_USER_H 358 int lustre_ioctl_flags = LL_FILE_IGNORE_LOCK;
359 if (ioctl(*fd, LL_IOC_SETFLAGS, &lustre_ioctl_flags) == -1)
360 ERR(
"ioctl(LL_IOC_SETFLAGS) failed");
364 #ifdef HAVE_GPFS_FCNTL_H 369 gpfs_free_all_locks(*fd);
383 fd = (
int *)malloc(
sizeof(
int));
385 ERR(
"Unable to malloc file descriptor");
391 *fd =
open64(testFileName, fd_oflag);
393 ERR(
"open64 failed");
395 #ifdef HAVE_LUSTRE_LUSTRE_USER_H 397 int lustre_ioctl_flags = LL_FILE_IGNORE_LOCK;
400 "** Disabling lustre range locking **\n");
402 if (ioctl(*fd, LL_IOC_SETFLAGS, &lustre_ioctl_flags) == -1)
403 ERR(
"ioctl(LL_IOC_SETFLAGS) failed");
407 #ifdef HAVE_GPFS_FCNTL_H 409 gpfs_free_all_locks(*fd);
422 long long remaining = (
long long)length;
423 char *ptr = (
char *)buffer;
429 #ifdef HAVE_GPFS_FCNTL_H 431 gpfs_access_start(fd, length, param, access);
438 ERR(
"lseek64() failed");
440 while (remaining > 0) {
442 if (access ==
WRITE) {
445 "task %d writing to offset %lld\n",
447 param->
offset + length - remaining);
449 rc = write(fd, ptr, remaining);
451 ERR(
"write() failed");
457 "task %d reading from offset %lld\n",
459 param->
offset + length - remaining);
461 rc = read(fd, ptr, remaining);
463 ERR(
"read() returned EOF prematurely");
465 ERR(
"read() failed");
467 if (rc < remaining) {
469 "WARNING: Task %d, partial %s, %lld of %lld bytes at offset %lld\n",
471 access ==
WRITE ?
"write()" :
"read()",
473 param->
offset + length - remaining);
478 ERR(
"too many retries -- aborting");
481 assert(rc <= remaining);
486 #ifdef HAVE_GPFS_FCNTL_H 488 gpfs_access_end(fd, length, param, access);
499 if (fsync(*(
int *)fd) != 0)
500 EWARN(
"fsync() failed");
508 if (close(*(
int *)fd) != 0)
509 ERR(
"close() failed");
519 sprintf(errmsg,
"[RANK %03d]: unlink() of file \"%s\" failed\n",
521 if (unlink(testFileName) != 0)
531 struct stat stat_buf;
532 IOR_offset_t aggFileSizeFromStat, tmpMin, tmpMax, tmpSum;
534 if (stat(testFileName, &stat_buf) != 0) {
535 ERR(
"stat() failed");
537 aggFileSizeFromStat = stat_buf.st_size;
540 MPI_CHECK(MPI_Allreduce(&aggFileSizeFromStat, &tmpSum, 1,
541 MPI_LONG_LONG_INT, MPI_SUM, testComm),
542 "cannot total data moved");
543 aggFileSizeFromStat = tmpSum;
545 MPI_CHECK(MPI_Allreduce(&aggFileSizeFromStat, &tmpMin, 1,
546 MPI_LONG_LONG_INT, MPI_MIN, testComm),
547 "cannot total data moved");
548 MPI_CHECK(MPI_Allreduce(&aggFileSizeFromStat, &tmpMax, 1,
549 MPI_LONG_LONG_INT, MPI_MAX, testComm),
550 "cannot total data moved");
551 if (tmpMin != tmpMax) {
553 WARN(
"inconsistent file size by different tasks");
556 aggFileSizeFromStat = tmpMin;
560 return (aggFileSizeFromStat);
static void POSIX_Fsync(void *, IOR_param_t *)
void set_o_direct_flag(int *fd)
void * POSIX_Open(char *testFileName, IOR_param_t *param)
int aiori_posix_rmdir(const char *path, IOR_param_t *param)
int aiori_posix_mkdir(const char *path, mode_t mode, IOR_param_t *param)
int aiori_posix_statfs(const char *path, ior_aiori_statfs_t *stat_buf, IOR_param_t *param)
void * POSIX_Create(char *testFileName, IOR_param_t *param)
char * aiori_get_version()
void POSIX_Delete(char *testFileName, IOR_param_t *param)
#define MPI_CHECK(MPI_STATUS, MSG)
IOR_offset_t POSIX_GetFileSize(IOR_param_t *test, MPI_Comm testComm, char *testFileName)
void POSIX_Close(void *fd, IOR_param_t *param)
int aiori_posix_stat(const char *path, struct stat *buf, IOR_param_t *param)
int aiori_posix_access(const char *path, int mode, IOR_param_t *param)
long long int IOR_offset_t
static IOR_offset_t POSIX_Xfer(int, void *, IOR_size_t *, IOR_offset_t, IOR_param_t *)