IOR
Macros | Functions | Variables
utilities.c File Reference
#include <stdarg.h>
#include <stdio.h>
#include <stdlib.h>
#include <errno.h>
#include <fcntl.h>
#include <math.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <time.h>
#include <regex.h>
#include <sys/statfs.h>
#include <sys/time.h>
#include "utilities.h"
#include "aiori.h"
#include "ior.h"
#include "ior-internal.h"
Include dependency graph for utilities.c:

Go to the source code of this file.

Macros

#define RANDALGO_GOLDEN_RATIO_PRIME   0x9e37fffffffc0001UL
 
#define O_DIRECT   000000
 

Functions

void update_write_memory_pattern (uint64_t item, char *buf, size_t bytes, int rand_seed, int pretendRank, ior_dataPacketType_e dataPacketType, ior_memory_flags type)
 
void generate_memory_pattern (char *buf, size_t bytes, int rand_seed, int pretendRank, ior_dataPacketType_e dataPacketType, ior_memory_flags type)
 
void invalidate_buffer_pattern (char *buffer, size_t bytes, ior_memory_flags type)
 
int verify_memory_pattern (uint64_t item, char *buffer, size_t bytes, int rand_seed, int pretendRank, ior_dataPacketType_e dataPacketType, ior_memory_flags type)
 
void * safeMalloc (uint64_t size)
 
void FailMessage (int rank, const char *location, char *format,...)
 
size_t NodeMemoryStringToBytes (char *size_str)
 
ior_dataPacketType_e parsePacketType (char t)
 
void updateParsedOptions (IOR_param_t *options, options_all_t *global_options)
 
void set_o_direct_flag (int *flag)
 
char * CurrentTimeString (void)
 
void DumpBuffer (void *buffer, size_t size)
 
int QueryNodeMapping (MPI_Comm comm, int print_nodemap)
 
void initCUDA (int blockMapping, int rank, int numNodes, int tasksPerNode, int useGPUID)
 
int GetNumNodes (MPI_Comm comm)
 
int GetNumTasks (MPI_Comm comm)
 
int GetNumTasksOnNode0 (MPI_Comm comm)
 
void ExtractHint (char *settingVal, char *valueVal, char *hintString)
 
void SetHints (MPI_Info *mpiHints, char *hintsFileName)
 
void ShowHints (MPI_Info *mpiHints)
 
IOR_offset_t StringToBytes (char *size_str)
 
void ShowFileSystemSize (char *filename, const struct ior_aiori *backend, void *backend_options)
 
int Regex (char *string, char *pattern)
 
double GetTimeStamp (void)
 
static double TimeDeviation (MPI_Comm com)
 
void init_clock (MPI_Comm com)
 
char * PrintTimestamp ()
 
int64_t ReadStoneWallingIterations (char *const filename, MPI_Comm com)
 
void StoreStoneWallingIterations (char *const filename, int64_t count)
 
void DelaySecs (int delay)
 
char * HumanReadable (IOR_offset_t value, int base)
 
unsigned long GetProcessorAndCore (int *chip, int *core)
 
void * aligned_buffer_alloc (size_t size, ior_memory_flags type)
 
void aligned_buffer_free (void *buf, ior_memory_flags gpu)
 

Variables

int errno
 
int numTasks
 
int rank = 0
 
int rankOffset = 0
 
int verbose = VERBOSE_0
 
MPI_Comm testComm = MPI_COMM_NULL
 
FILE * out_logfile = NULL
 
FILE * out_resultfile = NULL
 
enum OutputFormat_t outputFormat
 

Macro Definition Documentation

◆ O_DIRECT

#define O_DIRECT   000000

Referenced by set_o_direct_flag().

◆ RANDALGO_GOLDEN_RATIO_PRIME

#define RANDALGO_GOLDEN_RATIO_PRIME   0x9e37fffffffc0001UL

Definition at line 62 of file utilities.c.

Referenced by update_write_memory_pattern(), and verify_memory_pattern().

Function Documentation

◆ aligned_buffer_alloc()

void* aligned_buffer_alloc ( size_t  size,
ior_memory_flags  type 
)

Definition at line 1070 of file utilities.c.

References ERR, IOR_MEMORY_TYPE_GPU_DEVICE_ONLY, IOR_MEMORY_TYPE_GPU_MANAGED_CHECK_CPU, IOR_MEMORY_TYPE_GPU_MANAGED_CHECK_GPU, and safeMalloc().

Referenced by mdtest_read(), mdtest_run(), run_benchmark(), run_precreate(), WriteOrRead(), and XferBuffersSetup().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ aligned_buffer_free()

void aligned_buffer_free ( void *  buf,
ior_memory_flags  gpu 
)

Definition at line 1119 of file utilities.c.

References ERR, and WARN.

Referenced by mdtest_read(), mdtest_run(), run_benchmark(), run_precreate(), WriteOrRead(), and XferBuffersFree().

Here is the caller graph for this function:

◆ CurrentTimeString()

char* CurrentTimeString ( void  )

Definition at line 356 of file utilities.c.

References ERR, and NULL.

Referenced by PrintHeader(), PrintTestEnds(), ShowTestStart(), and TestIoSys().

Here is the caller graph for this function:

◆ DelaySecs()

void DelaySecs ( int  delay)

Definition at line 974 of file utilities.c.

References out_logfile, rank, verbose, and VERBOSE_1.

Referenced by mdtest_iteration(), and TestIoSys().

Here is the caller graph for this function:

◆ DumpBuffer()

void DumpBuffer ( void *  buffer,
size_t  size 
)

Definition at line 381 of file utilities.c.

References IOR_format, and out_logfile.

◆ ExtractHint()

void ExtractHint ( char *  settingVal,
char *  valueVal,
char *  hintString 
)

Definition at line 628 of file utilities.c.

References NULL, and out_logfile.

Referenced by SetHints().

Here is the caller graph for this function:

◆ FailMessage()

void FailMessage ( int  rank,
const char *  location,
char *  format,
  ... 
)

Definition at line 247 of file utilities.c.

References out_logfile, PrintTimestamp(), and testComm.

Here is the call graph for this function:

◆ generate_memory_pattern()

void generate_memory_pattern ( char *  buf,
size_t  bytes,
int  rand_seed,
int  pretendRank,
ior_dataPacketType_e  dataPacketType,
ior_memory_flags  type 
)

Fills a buffer with bytes of a given pattern. Not performance-sensitive because it is called once per test.

Parameters
bufpointer to byte buffer to fill
bytesnumber of bytes to produce to fill buffer
rand_seedseed to use for PRNG
pretendRankunique identifier for this process
dataPacketTypeidentifier to designate pattern to fill buffer

Definition at line 137 of file utilities.c.

References DATA_INCOMPRESSIBLE, DATA_OFFSET, DATA_RANDOM, DATA_TIMESTAMP, generate_memory_pattern_gpu(), IOR_MEMORY_TYPE_GPU_DEVICE_ONLY, and IOR_MEMORY_TYPE_GPU_MANAGED_CHECK_GPU.

Referenced by mdtest_run(), run_benchmark(), run_precreate(), and TestIoSys().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ GetNumNodes()

int GetNumNodes ( MPI_Comm  comm)

Definition at line 476 of file utilities.c.

References GetNumTasks(), GetNumTasksOnNode0(), MPI_CHECK, numTasks, and rank.

Referenced by InitTests(), md_workbench_run(), and mdtest_run().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ GetNumTasks()

int GetNumTasks ( MPI_Comm  comm)

Definition at line 513 of file utilities.c.

References MPI_CHECK, and numTasks.

Referenced by GetNumNodes(), and InitTests().

Here is the caller graph for this function:

◆ GetNumTasksOnNode0()

int GetNumTasksOnNode0 ( MPI_Comm  comm)

Definition at line 551 of file utilities.c.

References FAIL, MAX_PATHLEN, MPI_CHECK, out_logfile, rank, and verbose.

Referenced by GetNumNodes(), InitTests(), md_workbench_run(), and mdtest_run().

Here is the caller graph for this function:

◆ GetProcessorAndCore()

unsigned long GetProcessorAndCore ( int *  chip,
int *  core 
)

Definition at line 1057 of file utilities.c.

Referenced by GetTestFileName(), and HumanReadable().

Here is the caller graph for this function:

◆ GetTimeStamp()

double GetTimeStamp ( void  )

◆ HumanReadable()

char* HumanReadable ( IOR_offset_t  value,
int  base 
)

Definition at line 987 of file utilities.c.

References BASE_TEN, BASE_TWO, GetProcessorAndCore(), GIBIBYTE, GIGABYTE, MAX_STR, MEBIBYTE, MEGABYTE, and NULL.

Referenced by ShowSetup().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ init_clock()

void init_clock ( MPI_Comm  com)

Definition at line 917 of file utilities.c.

Referenced by InitTests(), md_workbench_run(), and mdtest_run().

Here is the caller graph for this function:

◆ initCUDA()

void initCUDA ( int  blockMapping,
int  rank,
int  numNodes,
int  tasksPerNode,
int  useGPUID 
)

Definition at line 438 of file utilities.c.

References ERRF, and WARNF.

Referenced by md_workbench_run(), mdtest_run(), and test_initialize().

Here is the caller graph for this function:

◆ invalidate_buffer_pattern()

void invalidate_buffer_pattern ( char *  buffer,
size_t  bytes,
ior_memory_flags  type 
)

Definition at line 172 of file utilities.c.

References IOR_MEMORY_TYPE_GPU_DEVICE_ONLY, and IOR_MEMORY_TYPE_GPU_MANAGED_CHECK_GPU.

Referenced by mdtest_read(), run_benchmark(), and WriteOrReadSingle().

Here is the caller graph for this function:

◆ NodeMemoryStringToBytes()

size_t NodeMemoryStringToBytes ( char *  size_str)

Definition at line 259 of file utilities.c.

References ERR, rc, and string_to_bytes().

Referenced by DecodeDirective(), and updateParsedOptions().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ parsePacketType()

ior_dataPacketType_e parsePacketType ( char  t)

Definition at line 291 of file utilities.c.

References DATA_INCOMPRESSIBLE, DATA_OFFSET, DATA_RANDOM, DATA_TIMESTAMP, and ERRF.

Referenced by DecodeDirective(), md_workbench_run(), mdtest_run(), and updateParsedOptions().

Here is the caller graph for this function:

◆ PrintTimestamp()

char* PrintTimestamp ( )

Definition at line 921 of file utilities.c.

References NULL, out_logfile, rank, and verbose.

Referenced by FailMessage(), and mdtest_run().

Here is the caller graph for this function:

◆ QueryNodeMapping()

int QueryNodeMapping ( MPI_Comm  comm,
int  print_nodemap 
)

Definition at line 402 of file utilities.c.

References FAIL, MAX_PATHLEN, out_logfile, and rank.

Referenced by InitTests(), md_workbench_run(), and mdtest_run().

Here is the caller graph for this function:

◆ ReadStoneWallingIterations()

int64_t ReadStoneWallingIterations ( char *const  filename,
MPI_Comm  com 
)

Definition at line 936 of file utilities.c.

References NULL, and rank.

Referenced by file_test(), and TestIoSys().

Here is the caller graph for this function:

◆ Regex()

int Regex ( char *  string,
char *  pattern 
)

Definition at line 833 of file utilities.c.

References ERR.

◆ safeMalloc()

void* safeMalloc ( uint64_t  size)

◆ set_o_direct_flag()

void set_o_direct_flag ( int *  flag)

Definition at line 332 of file utilities.c.

References O_DIRECT, and WARN.

Referenced by IME_Open(), POSIX_Create(), and POSIX_Open().

Here is the caller graph for this function:

◆ SetHints()

void SetHints ( MPI_Info *  mpiHints,
char *  hintsFileName 
)

Definition at line 656 of file utilities.c.

References environ, ERR, ExtractHint(), MAX_STR, MPI_CHECK, NULL, and WARN.

Referenced by HDF5_Open(), MPIIO_Access(), MPIIO_GetFileSize(), MPIIO_Open(), NCMPI_Create(), and NCMPI_Open().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ ShowFileSystemSize()

void ShowFileSystemSize ( char *  filename,
const struct ior_aiori backend,
void *  backend_options 
)

◆ ShowHints()

void ShowHints ( MPI_Info *  mpiHints)

Definition at line 717 of file utilities.c.

References MPI_CHECK, and out_logfile.

Referenced by HDF5_Open(), MPIIO_Open(), NCMPI_Create(), and NCMPI_Open().

Here is the caller graph for this function:

◆ StoreStoneWallingIterations()

void StoreStoneWallingIterations ( char *const  filename,
int64_t  count 
)

Definition at line 959 of file utilities.c.

References FAIL, NULL, and rank.

Referenced by file_test_create(), and ShowTestEnd().

Here is the caller graph for this function:

◆ StringToBytes()

IOR_offset_t StringToBytes ( char *  size_str)

Definition at line 739 of file utilities.c.

References rc.

◆ TimeDeviation()

static double TimeDeviation ( MPI_Comm  com)
static

Definition at line 892 of file utilities.c.

References GetTimeStamp(), min, and MPI_CHECK.

Here is the call graph for this function:

◆ update_write_memory_pattern()

void update_write_memory_pattern ( uint64_t  item,
char *  buf,
size_t  bytes,
int  rand_seed,
int  pretendRank,
ior_dataPacketType_e  dataPacketType,
ior_memory_flags  type 
)

Modifies a buffer for a write. Performance sensitive because it is called before each write.

Parameters
bufpointer to byte buffer to fill
bytesnumber of bytes to produce to fill buffer
rand_seedseed to use for PRNG
pretendRankunique identifier for this process
dataPacketTypeidentifier to designate pattern to fill buffer

Definition at line 94 of file utilities.c.

References DATA_RANDOM, DATA_TIMESTAMP, IOR_MEMORY_TYPE_GPU_DEVICE_ONLY, IOR_MEMORY_TYPE_GPU_MANAGED_CHECK_GPU, RANDALGO_GOLDEN_RATIO_PRIME, and update_write_memory_pattern_gpu().

Referenced by create_file(), run_benchmark(), run_precreate(), and WriteOrReadSingle().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ updateParsedOptions()

void updateParsedOptions ( IOR_param_t options,
options_all_t global_options 
)

◆ verify_memory_pattern()

int verify_memory_pattern ( uint64_t  item,
char *  buffer,
size_t  bytes,
int  rand_seed,
int  pretendRank,
ior_dataPacketType_e  dataPacketType,
ior_memory_flags  type 
)

Definition at line 182 of file utilities.c.

References DATA_INCOMPRESSIBLE, DATA_OFFSET, DATA_RANDOM, DATA_TIMESTAMP, IOR_MEMORY_TYPE_GPU_DEVICE_ONLY, IOR_MEMORY_TYPE_GPU_MANAGED_CHECK_GPU, RANDALGO_GOLDEN_RATIO_PRIME, and verify_memory_pattern_gpu().

Referenced by CompareData(), create_file(), mdtest_read(), and run_benchmark().

Here is the call graph for this function:
Here is the caller graph for this function:

Variable Documentation

◆ errno

int errno

◆ numTasks

int numTasks

◆ out_logfile

FILE* out_logfile = NULL

◆ out_resultfile

FILE* out_resultfile = NULL

◆ outputFormat

enum OutputFormat_t outputFormat

◆ rank

int rank = 0

Definition at line 70 of file utilities.c.

Referenced by CEPHFS_GetFileSize(), CheckFileSize(), collective_helper(), CountErrors(), create_file(), DecodeDirective(), DelaySecs(), DFS_Create(), DFS_Finalize(), DFS_GetFileSize(), DFS_Init(), DFS_Open(), directory_test(), DisplayOutliers(), DistributeHints(), DUMMY_Close(), DUMMY_Create(), DUMMY_Sync(), DUMMY_Xfer(), file_hits_histogram(), file_test(), file_test_create(), GetNumNodes(), GetNumTasksOnNode0(), GetOffsetArrayRandom(), GetTestFileName(), HandleDistribute(), HDF5_GetFileSize(), HDF5_MkDir(), HDF5_Open(), HDF5_RmDir(), HDF5_Stat(), HDF5_StatFS(), HDF5_Xfer(), HDFS_Create_Or_Open(), HDFS_Delete(), HDFS_Xfer(), IME_Delete(), IME_Options(), IME_Xfer(), InitTests(), ior_main(), ior_run(), md_validate_tests(), mdtest_iteration(), mdtest_read(), mdtest_run(), MPIIO_Open(), MPIIO_Xfer(), NCMPI_Create(), NCMPI_Open(), NCMPI_Xfer(), PMDK_Delete(), PMDK_xfer_hints(), POSIX_Create(), POSIX_Delete(), POSIX_Rename(), POSIX_Xfer(), PrependDir(), PrintArrayEnd(), PrintArrayNamedStart(), PrintArrayStart(), PrintEndSection(), PrintHeader(), PrintLongSummaryAllTests(), PrintLongSummaryHeader(), PrintLongSummaryOneOperation(), PrintRemoveTiming(), PrintRepeatEnd(), PrintRepeatStart(), PrintShortSummary(), PrintTestEnds(), PrintTimestamp(), QueryNodeMapping(), ReadStoneWallingIterations(), ReduceIterResults(), remove_file(), RemoveFile(), S3_Close_internal(), s3_connect(), S3_Create_Or_Open_internal(), S3_final(), S3_GetFileSize(), S3_init(), S3_Xfer_internal(), SeekOffset(), SetupDataSet(), share_file_handle(), ShowTestEnd(), StoreRankInformation(), StoreStoneWallingIterations(), summarize_results(), test_initialize(), TestIoSys(), VerboseMessage(), WriteOrRead(), and WriteTimes().

◆ rankOffset

int rankOffset = 0

◆ testComm

MPI_Comm testComm = MPI_COMM_NULL

◆ verbose

int verbose = VERBOSE_0