|
IOR
|
#include <stdarg.h>#include <stdio.h>#include <stdlib.h>#include <errno.h>#include <fcntl.h>#include <math.h>#include <string.h>#include <sys/stat.h>#include <sys/types.h>#include <time.h>#include <regex.h>#include <sys/statfs.h>#include <sys/time.h>#include "utilities.h"#include "aiori.h"#include "ior.h"#include "ior-internal.h"
Go to the source code of this file.
Macros | |
| #define | RANDALGO_GOLDEN_RATIO_PRIME 0x9e37fffffffc0001UL |
| #define | O_DIRECT 000000 |
Functions | |
| void | update_write_memory_pattern (uint64_t item, char *buf, size_t bytes, int rand_seed, int pretendRank, ior_dataPacketType_e dataPacketType, ior_memory_flags type) |
| void | generate_memory_pattern (char *buf, size_t bytes, int rand_seed, int pretendRank, ior_dataPacketType_e dataPacketType, ior_memory_flags type) |
| void | invalidate_buffer_pattern (char *buffer, size_t bytes, ior_memory_flags type) |
| int | verify_memory_pattern (uint64_t item, char *buffer, size_t bytes, int rand_seed, int pretendRank, ior_dataPacketType_e dataPacketType, ior_memory_flags type) |
| void * | safeMalloc (uint64_t size) |
| void | FailMessage (int rank, const char *location, char *format,...) |
| size_t | NodeMemoryStringToBytes (char *size_str) |
| ior_dataPacketType_e | parsePacketType (char t) |
| void | updateParsedOptions (IOR_param_t *options, options_all_t *global_options) |
| void | set_o_direct_flag (int *flag) |
| char * | CurrentTimeString (void) |
| void | DumpBuffer (void *buffer, size_t size) |
| int | QueryNodeMapping (MPI_Comm comm, int print_nodemap) |
| void | initCUDA (int blockMapping, int rank, int numNodes, int tasksPerNode, int useGPUID) |
| int | GetNumNodes (MPI_Comm comm) |
| int | GetNumTasks (MPI_Comm comm) |
| int | GetNumTasksOnNode0 (MPI_Comm comm) |
| void | ExtractHint (char *settingVal, char *valueVal, char *hintString) |
| void | SetHints (MPI_Info *mpiHints, char *hintsFileName) |
| void | ShowHints (MPI_Info *mpiHints) |
| IOR_offset_t | StringToBytes (char *size_str) |
| void | ShowFileSystemSize (char *filename, const struct ior_aiori *backend, void *backend_options) |
| int | Regex (char *string, char *pattern) |
| double | GetTimeStamp (void) |
| static double | TimeDeviation (MPI_Comm com) |
| void | init_clock (MPI_Comm com) |
| char * | PrintTimestamp () |
| int64_t | ReadStoneWallingIterations (char *const filename, MPI_Comm com) |
| void | StoreStoneWallingIterations (char *const filename, int64_t count) |
| void | DelaySecs (int delay) |
| char * | HumanReadable (IOR_offset_t value, int base) |
| unsigned long | GetProcessorAndCore (int *chip, int *core) |
| void * | aligned_buffer_alloc (size_t size, ior_memory_flags type) |
| void | aligned_buffer_free (void *buf, ior_memory_flags gpu) |
Variables | |
| int | errno |
| int | numTasks |
| int | rank = 0 |
| int | rankOffset = 0 |
| int | verbose = VERBOSE_0 |
| MPI_Comm | testComm = MPI_COMM_NULL |
| FILE * | out_logfile = NULL |
| FILE * | out_resultfile = NULL |
| enum OutputFormat_t | outputFormat |
| #define O_DIRECT 000000 |
Referenced by set_o_direct_flag().
| #define RANDALGO_GOLDEN_RATIO_PRIME 0x9e37fffffffc0001UL |
Definition at line 62 of file utilities.c.
Referenced by update_write_memory_pattern(), and verify_memory_pattern().
| void* aligned_buffer_alloc | ( | size_t | size, |
| ior_memory_flags | type | ||
| ) |
Definition at line 1070 of file utilities.c.
References ERR, IOR_MEMORY_TYPE_GPU_DEVICE_ONLY, IOR_MEMORY_TYPE_GPU_MANAGED_CHECK_CPU, IOR_MEMORY_TYPE_GPU_MANAGED_CHECK_GPU, and safeMalloc().
Referenced by mdtest_read(), mdtest_run(), run_benchmark(), run_precreate(), WriteOrRead(), and XferBuffersSetup().


| void aligned_buffer_free | ( | void * | buf, |
| ior_memory_flags | gpu | ||
| ) |
Definition at line 1119 of file utilities.c.
Referenced by mdtest_read(), mdtest_run(), run_benchmark(), run_precreate(), WriteOrRead(), and XferBuffersFree().

| char* CurrentTimeString | ( | void | ) |
Definition at line 356 of file utilities.c.
Referenced by PrintHeader(), PrintTestEnds(), ShowTestStart(), and TestIoSys().

| void DelaySecs | ( | int | delay | ) |
Definition at line 974 of file utilities.c.
References out_logfile, rank, verbose, and VERBOSE_1.
Referenced by mdtest_iteration(), and TestIoSys().

| void DumpBuffer | ( | void * | buffer, |
| size_t | size | ||
| ) |
Definition at line 381 of file utilities.c.
References IOR_format, and out_logfile.
| void ExtractHint | ( | char * | settingVal, |
| char * | valueVal, | ||
| char * | hintString | ||
| ) |
Definition at line 628 of file utilities.c.
References NULL, and out_logfile.
Referenced by SetHints().

| void FailMessage | ( | int | rank, |
| const char * | location, | ||
| char * | format, | ||
| ... | |||
| ) |
Definition at line 247 of file utilities.c.
References out_logfile, PrintTimestamp(), and testComm.

| void generate_memory_pattern | ( | char * | buf, |
| size_t | bytes, | ||
| int | rand_seed, | ||
| int | pretendRank, | ||
| ior_dataPacketType_e | dataPacketType, | ||
| ior_memory_flags | type | ||
| ) |
Fills a buffer with bytes of a given pattern. Not performance-sensitive because it is called once per test.
| buf | pointer to byte buffer to fill |
| bytes | number of bytes to produce to fill buffer |
| rand_seed | seed to use for PRNG |
| pretendRank | unique identifier for this process |
| dataPacketType | identifier to designate pattern to fill buffer |
Definition at line 137 of file utilities.c.
References DATA_INCOMPRESSIBLE, DATA_OFFSET, DATA_RANDOM, DATA_TIMESTAMP, generate_memory_pattern_gpu(), IOR_MEMORY_TYPE_GPU_DEVICE_ONLY, and IOR_MEMORY_TYPE_GPU_MANAGED_CHECK_GPU.
Referenced by mdtest_run(), run_benchmark(), run_precreate(), and TestIoSys().


| int GetNumNodes | ( | MPI_Comm | comm | ) |
Definition at line 476 of file utilities.c.
References GetNumTasks(), GetNumTasksOnNode0(), MPI_CHECK, numTasks, and rank.
Referenced by InitTests(), md_workbench_run(), and mdtest_run().


| int GetNumTasks | ( | MPI_Comm | comm | ) |
Definition at line 513 of file utilities.c.
References MPI_CHECK, and numTasks.
Referenced by GetNumNodes(), and InitTests().

| int GetNumTasksOnNode0 | ( | MPI_Comm | comm | ) |
Definition at line 551 of file utilities.c.
References FAIL, MAX_PATHLEN, MPI_CHECK, out_logfile, rank, and verbose.
Referenced by GetNumNodes(), InitTests(), md_workbench_run(), and mdtest_run().

| unsigned long GetProcessorAndCore | ( | int * | chip, |
| int * | core | ||
| ) |
Definition at line 1057 of file utilities.c.
Referenced by GetTestFileName(), and HumanReadable().

| double GetTimeStamp | ( | void | ) |
Definition at line 876 of file utilities.c.
Referenced by add_timed_result(), directory_test(), file_test(), file_test_create(), md_workbench_run(), mdtest_iteration(), mdw_wait(), run_benchmark(), run_cleanup(), run_precreate(), test_time_elapsed(), TestIoSys(), TimeDeviation(), updateStoneWallIterations(), and WriteOrRead().

| char* HumanReadable | ( | IOR_offset_t | value, |
| int | base | ||
| ) |
Definition at line 987 of file utilities.c.
References BASE_TEN, BASE_TWO, GetProcessorAndCore(), GIBIBYTE, GIGABYTE, MAX_STR, MEBIBYTE, MEGABYTE, and NULL.
Referenced by ShowSetup().


| void init_clock | ( | MPI_Comm | com | ) |
Definition at line 917 of file utilities.c.
Referenced by InitTests(), md_workbench_run(), and mdtest_run().

| void initCUDA | ( | int | blockMapping, |
| int | rank, | ||
| int | numNodes, | ||
| int | tasksPerNode, | ||
| int | useGPUID | ||
| ) |
Definition at line 438 of file utilities.c.
Referenced by md_workbench_run(), mdtest_run(), and test_initialize().

| void invalidate_buffer_pattern | ( | char * | buffer, |
| size_t | bytes, | ||
| ior_memory_flags | type | ||
| ) |
Definition at line 172 of file utilities.c.
References IOR_MEMORY_TYPE_GPU_DEVICE_ONLY, and IOR_MEMORY_TYPE_GPU_MANAGED_CHECK_GPU.
Referenced by mdtest_read(), run_benchmark(), and WriteOrReadSingle().

| size_t NodeMemoryStringToBytes | ( | char * | size_str | ) |
Definition at line 259 of file utilities.c.
References ERR, rc, and string_to_bytes().
Referenced by DecodeDirective(), and updateParsedOptions().


| ior_dataPacketType_e parsePacketType | ( | char | t | ) |
Definition at line 291 of file utilities.c.
References DATA_INCOMPRESSIBLE, DATA_OFFSET, DATA_RANDOM, DATA_TIMESTAMP, and ERRF.
Referenced by DecodeDirective(), md_workbench_run(), mdtest_run(), and updateParsedOptions().

| char* PrintTimestamp | ( | ) |
Definition at line 921 of file utilities.c.
References NULL, out_logfile, rank, and verbose.
Referenced by FailMessage(), and mdtest_run().

| int QueryNodeMapping | ( | MPI_Comm | comm, |
| int | print_nodemap | ||
| ) |
Definition at line 402 of file utilities.c.
References FAIL, MAX_PATHLEN, out_logfile, and rank.
Referenced by InitTests(), md_workbench_run(), and mdtest_run().

| int64_t ReadStoneWallingIterations | ( | char *const | filename, |
| MPI_Comm | com | ||
| ) |
Definition at line 936 of file utilities.c.
Referenced by file_test(), and TestIoSys().

| int Regex | ( | char * | string, |
| char * | pattern | ||
| ) |
Definition at line 833 of file utilities.c.
References ERR.
| void* safeMalloc | ( | uint64_t | size | ) |
Definition at line 238 of file utilities.c.
Referenced by aligned_buffer_alloc(), AllocResults(), createGlobalOptions(), GetOffsetArrayRandom(), HDF5_Open(), mdtest_run(), parse_dirpath(), POSIX_Create(), POSIX_Open(), StoreRankInformation(), and summarize_results().

| void set_o_direct_flag | ( | int * | flag | ) |
Definition at line 332 of file utilities.c.
References O_DIRECT, and WARN.
Referenced by IME_Open(), POSIX_Create(), and POSIX_Open().

| void SetHints | ( | MPI_Info * | mpiHints, |
| char * | hintsFileName | ||
| ) |
Definition at line 656 of file utilities.c.
References environ, ERR, ExtractHint(), MAX_STR, MPI_CHECK, NULL, and WARN.
Referenced by HDF5_Open(), MPIIO_Access(), MPIIO_GetFileSize(), MPIIO_Open(), NCMPI_Create(), and NCMPI_Open().


| void ShowFileSystemSize | ( | char * | filename, |
| const struct ior_aiori * | backend, | ||
| void * | backend_options | ||
| ) |
Definition at line 770 of file utilities.c.
References ior_aiori_statfs::f_bfree, ior_aiori_statfs::f_blocks, ior_aiori_statfs::f_bsize, ior_aiori_statfs::f_ffree, ior_aiori_statfs::f_files, out_logfile, out_resultfile, OUTPUT_CSV, OUTPUT_DEFAULT, OUTPUT_JSON, outputFormat, ior_aiori::statfs, and WARN.
Referenced by mdtest_run(), and ShowTestStart().

| void ShowHints | ( | MPI_Info * | mpiHints | ) |
Definition at line 717 of file utilities.c.
References MPI_CHECK, and out_logfile.
Referenced by HDF5_Open(), MPIIO_Open(), NCMPI_Create(), and NCMPI_Open().

| void StoreStoneWallingIterations | ( | char *const | filename, |
| int64_t | count | ||
| ) |
Definition at line 959 of file utilities.c.
References FAIL, NULL, and rank.
Referenced by file_test_create(), and ShowTestEnd().

| IOR_offset_t StringToBytes | ( | char * | size_str | ) |
Definition at line 739 of file utilities.c.
References rc.
|
static |
Definition at line 892 of file utilities.c.
References GetTimeStamp(), min, and MPI_CHECK.

| void update_write_memory_pattern | ( | uint64_t | item, |
| char * | buf, | ||
| size_t | bytes, | ||
| int | rand_seed, | ||
| int | pretendRank, | ||
| ior_dataPacketType_e | dataPacketType, | ||
| ior_memory_flags | type | ||
| ) |
Modifies a buffer for a write. Performance sensitive because it is called before each write.
| buf | pointer to byte buffer to fill |
| bytes | number of bytes to produce to fill buffer |
| rand_seed | seed to use for PRNG |
| pretendRank | unique identifier for this process |
| dataPacketType | identifier to designate pattern to fill buffer |
Definition at line 94 of file utilities.c.
References DATA_RANDOM, DATA_TIMESTAMP, IOR_MEMORY_TYPE_GPU_DEVICE_ONLY, IOR_MEMORY_TYPE_GPU_MANAGED_CHECK_GPU, RANDALGO_GOLDEN_RATIO_PRIME, and update_write_memory_pattern_gpu().
Referenced by create_file(), run_benchmark(), run_precreate(), and WriteOrReadSingle().


| void updateParsedOptions | ( | IOR_param_t * | options, |
| options_all_t * | global_options | ||
| ) |
Definition at line 308 of file utilities.c.
References aiori_select(), airoi_update_module_options(), IOR_param_t::api, IOR_param_t::apiVersion, backend, IOR_param_t::backend, IOR_param_t::backend_options, IOR_param_t::buffer_type, IOR_param_t::dataPacketType, ERR, ior_aiori::get_version, IOR_param_t::incompressibleSeed, IOR_param_t::memoryPerNode, IOR_param_t::memoryPerNodeStr, NodeMemoryStringToBytes(), NULL, parsePacketType(), and IOR_param_t::setTimeStampSignature.
Referenced by ParseCommandLine().


| int verify_memory_pattern | ( | uint64_t | item, |
| char * | buffer, | ||
| size_t | bytes, | ||
| int | rand_seed, | ||
| int | pretendRank, | ||
| ior_dataPacketType_e | dataPacketType, | ||
| ior_memory_flags | type | ||
| ) |
Definition at line 182 of file utilities.c.
References DATA_INCOMPRESSIBLE, DATA_OFFSET, DATA_RANDOM, DATA_TIMESTAMP, IOR_MEMORY_TYPE_GPU_DEVICE_ONLY, IOR_MEMORY_TYPE_GPU_MANAGED_CHECK_GPU, RANDALGO_GOLDEN_RATIO_PRIME, and verify_memory_pattern_gpu().
Referenced by CompareData(), create_file(), mdtest_read(), and run_benchmark().


| int errno |
| int numTasks |
Referenced by DisplayOutliers(), file_hits_histogram(), GetNumNodes(), and GetNumTasks().
| FILE* out_logfile = NULL |
Definition at line 74 of file utilities.c.
Referenced by airoi_create_all_module_options(), CountErrors(), DecodeDirective(), DelaySecs(), DUMMY_Close(), DUMMY_Create(), DUMMY_Delete(), DUMMY_Fsync(), DUMMY_GetFileSize(), DUMMY_Open(), DUMMY_Xfer(), DumpBuffer(), ExtractHint(), FailMessage(), file_hits_histogram(), GetNumTasksOnNode0(), HogMemory(), ior_main(), ior_run(), md_workbench_run(), mdtest_run(), ParseLine(), PrintHeader(), PrintTimestamp(), QueryNodeMapping(), RemoveFile(), ShowFileSystemSize(), ShowHints(), ShowSetup(), ShowTestEnd(), summarize_results_rank0(), TestIoSys(), VerboseMessage(), WriteOrRead(), and WriteTimes().
| FILE* out_resultfile = NULL |
Definition at line 75 of file utilities.c.
Referenced by DecodeDirective(), ior_main(), ior_run(), mdtest_run(), PPDouble(), PrintArrayEnd(), PrintArrayNamedStart(), PrintArrayStart(), PrintEndSection(), PrintHeader(), PrintIndent(), PrintKeyVal(), PrintKeyValDouble(), PrintKeyValEnd(), PrintKeyValInt(), PrintKeyValStart(), PrintLongSummaryAllTests(), PrintLongSummaryHeader(), PrintLongSummaryOneOperation(), PrintNamedArrayStart(), PrintNamedSectionStart(), PrintNextToken(), PrintReducedResult(), PrintRemoveTiming(), PrintShortSummary(), PrintStartSection(), PrintTableHeader(), ShowFileSystemSize(), ShowSetup(), and ShowTestStart().
| enum OutputFormat_t outputFormat |
Definition at line 76 of file utilities.c.
Referenced by DecodeDirective(), PrintArrayEnd(), PrintArrayNamedStart(), PrintArrayStart(), PrintEndSection(), PrintHeader(), PrintIndent(), PrintKeyVal(), PrintKeyValDouble(), PrintKeyValEnd(), PrintKeyValInt(), PrintKeyValStart(), PrintLongSummaryAllTests(), PrintLongSummaryHeader(), PrintLongSummaryOneOperation(), PrintNamedArrayStart(), PrintNamedSectionStart(), PrintNextToken(), PrintReducedResult(), PrintRemoveTiming(), PrintRepeatStart(), PrintShortSummary(), PrintStartSection(), PrintTableHeader(), PrintTestEnds(), ShowFileSystemSize(), ShowSetup(), and ShowTestStart().
| int rank = 0 |
Definition at line 70 of file utilities.c.
Referenced by CEPHFS_GetFileSize(), CheckFileSize(), collective_helper(), CountErrors(), create_file(), DecodeDirective(), DelaySecs(), DFS_Create(), DFS_Finalize(), DFS_GetFileSize(), DFS_Init(), DFS_Open(), directory_test(), DisplayOutliers(), DistributeHints(), DUMMY_Close(), DUMMY_Create(), DUMMY_Sync(), DUMMY_Xfer(), file_hits_histogram(), file_test(), file_test_create(), GetNumNodes(), GetNumTasksOnNode0(), GetOffsetArrayRandom(), GetTestFileName(), HandleDistribute(), HDF5_GetFileSize(), HDF5_MkDir(), HDF5_Open(), HDF5_RmDir(), HDF5_Stat(), HDF5_StatFS(), HDF5_Xfer(), HDFS_Create_Or_Open(), HDFS_Delete(), HDFS_Xfer(), IME_Delete(), IME_Options(), IME_Xfer(), InitTests(), ior_main(), ior_run(), md_validate_tests(), mdtest_iteration(), mdtest_read(), mdtest_run(), MPIIO_Open(), MPIIO_Xfer(), NCMPI_Create(), NCMPI_Open(), NCMPI_Xfer(), PMDK_Delete(), PMDK_xfer_hints(), POSIX_Create(), POSIX_Delete(), POSIX_Rename(), POSIX_Xfer(), PrependDir(), PrintArrayEnd(), PrintArrayNamedStart(), PrintArrayStart(), PrintEndSection(), PrintHeader(), PrintLongSummaryAllTests(), PrintLongSummaryHeader(), PrintLongSummaryOneOperation(), PrintRemoveTiming(), PrintRepeatEnd(), PrintRepeatStart(), PrintShortSummary(), PrintTestEnds(), PrintTimestamp(), QueryNodeMapping(), ReadStoneWallingIterations(), ReduceIterResults(), remove_file(), RemoveFile(), S3_Close_internal(), s3_connect(), S3_Create_Or_Open_internal(), S3_final(), S3_GetFileSize(), S3_init(), S3_Xfer_internal(), SeekOffset(), SetupDataSet(), share_file_handle(), ShowTestEnd(), StoreRankInformation(), StoreStoneWallingIterations(), summarize_results(), test_initialize(), TestIoSys(), VerboseMessage(), WriteOrRead(), and WriteTimes().
| int rankOffset = 0 |
Definition at line 71 of file utilities.c.
Referenced by file_hits_histogram(), GetTestFileName(), HDF5_Xfer(), IME_Options(), MPIIO_Open(), NCMPI_Xfer(), PMDK_xfer_hints(), PrependDir(), RemoveFile(), SeekOffset(), SetupDataSet(), TestIoSys(), and WriteOrRead().
| MPI_Comm testComm = MPI_COMM_NULL |
Definition at line 73 of file utilities.c.
Referenced by CEPHFS_GetFileSize(), CheckFileSize(), CountErrors(), DFS_check_params(), DFS_Create(), DFS_Finalize(), DFS_GetFileSize(), DFS_Open(), directory_test(), DisplayOutliers(), FailMessage(), file_test(), file_test_create(), HandleDistribute(), HDF5_Open(), HDFS_Create_Or_Open(), IME_Options(), mdtest_iteration(), mdtest_run(), MPIIO_GetFileSize(), MPIIO_Open(), NCMPI_Create(), NCMPI_Open(), phase_end(), phase_prepare(), PMDK_xfer_hints(), POSIX_Create(), ReduceIterResults(), S3_Close_internal(), s3_connect(), S3_Create_Or_Open_internal(), S3_GetFileSize(), share_file_handle(), StoreRankInformation(), summarize_results(), test_finalize(), test_initialize(), TestIoSys(), unique_dir_access(), updateStoneWallIterations(), and WriteOrRead().
| int verbose = VERBOSE_0 |
Definition at line 72 of file utilities.c.
Referenced by CheckFileSize(), DelaySecs(), DUMMY_Close(), DUMMY_Create(), DUMMY_Delete(), DUMMY_Fsync(), DUMMY_GetFileSize(), DUMMY_Open(), DUMMY_Xfer(), GetNumTasksOnNode0(), HDFS_Close(), hdfs_connect(), HDFS_Create(), HDFS_Create_Or_Open(), HDFS_Delete(), hdfs_disconnect(), HDFS_Fsync(), HDFS_GetFileSize(), HDFS_Open(), HDFS_Xfer(), HogMemory(), IME_Options(), IME_Xfer(), InitTests(), ior_main(), mdtest_run(), PMDK_xfer_hints(), POSIX_check_params(), POSIX_Xfer(), PrintHeader(), PrintLongSummaryAllTests(), PrintLongSummaryHeader(), PrintLongSummaryOneOperation(), PrintRemoveTiming(), PrintShortSummary(), PrintTestEnds(), PrintTimestamp(), ProcessIterResults(), ReduceIterResults(), RemoveFile(), S3_Close_internal(), S3_Create_Or_Open_internal(), SetupDataSet(), ShowTestStart(), test_initialize(), TestIoSys(), updateStoneWallIterations(), VerboseMessage(), and WriteOrRead().
1.8.13