md-workbench.c File Reference

#include <mpi.h>
#include <time.h>
#include <stdio.h>
#include <errno.h>
#include <string.h>
#include <stdlib.h>
#include <math.h>
#include <assert.h>
#include "md-workbench.h"
#include "config.h"
#include "aiori.h"
#include "utilities.h"
#include "parse_options.h"

Include dependency graph for md-workbench.c:

Go to the source code of this file.

Data Structures
struct	op_stat_t
struct	time_result_t
struct	phase_stat_t
struct	benchmark_options

Macros
#define	_POSIX_C_SOURCE   199309L
#define	DIRMODE   S_IRUSR|S_IWUSR|S_IXUSR|S_IRGRP|S_IWGRP|S_IXGRP|S_IROTH|S_IXOTH
#define	CHECK_MPI_RET(ret)   if (ret != MPI_SUCCESS){ printf("Unexpected error in MPI on Line %d\n", __LINE__);}
#define	LLU   (long long unsigned)
#define	min(a, b)   (a < b ? a : b)
#define	oprintf(...)   do { fprintf(o.logfile, __VA_ARGS__); fflush(o.logfile); } while(0);

Functions
static void	def_dset_name (char *out_name, int n, int d)
static void	def_obj_name (char *out_name, int n, int d, int i)
void	init_options ()
static void	mdw_wait (double runtime)
static void	init_stats (phase_stat_t *p, size_t repeats)
static float	add_timed_result (double start, double phase_start_timer, time_result_t *results, size_t pos, double *max_time, double *out_op_time)
static void	print_detailed_stat_header ()
static int	sum_err (phase_stat_t *p)
static double	statistics_mean (int count, double *arr)
static double	statistics_std_dev (int count, double *arr)
static void	statistics_minmax (int count, double *arr, double *out_min, double *out_max)
static void	print_p_stat (char *buff, const char *name, phase_stat_t *p, double t, int print_global)
static int	compare_floats (time_result_t *x, time_result_t *y)
static double	runtime_quantile (int repeats, time_result_t *times, float quantile)
static uint64_t	aggregate_timers (int repeats, int max_repeats, time_result_t *times, time_result_t *global_times)
static void	compute_histogram (const char *name, time_result_t *times, time_statistics_t *stats, size_t repeats, int writeLatencyFile)
static void	end_phase (const char *name, phase_stat_t *p)
void	run_precreate (phase_stat_t *s, int current_index)
void	run_benchmark (phase_stat_t *s, int *current_index_p)
void	run_cleanup (phase_stat_t *s, int start_index)
static void	printTime ()
static int	return_position ()
static void	store_position (int position)
mdworkbench_results_t *	md_workbench_run (int argc, char **argv, MPI_Comm world_com, FILE *out_logfile)

Variables
struct benchmark_options	o
static option_help	options []

Macro Definition Documentation

_POSIX_C_SOURCE

#define _POSIX_C_SOURCE   199309L

Definition at line 1 of file md-workbench.c.

CHECK_MPI_RET

#define CHECK_MPI_RET

(

ret

)

if (ret != MPI_SUCCESS){ printf("Unexpected error in MPI on Line %d\n", __LINE__);}

Definition at line 25 of file md-workbench.c.

Referenced by aggregate_timers(), end_phase(), and run_benchmark().

DIRMODE

#define DIRMODE   S_IRUSR|S_IWUSR|S_IXUSR|S_IRGRP|S_IWGRP|S_IXGRP|S_IROTH|S_IXOTH

Definition at line 23 of file md-workbench.c.

Referenced by md_workbench_run(), and run_precreate().

LLU

#define LLU   (long long unsigned)

Definition at line 26 of file md-workbench.c.

min

#define min	(		a,
			b
	)		(a < b ? a : b)

Definition at line 27 of file md-workbench.c.

Referenced by Gfarm_get_file_size(), statistics_minmax(), summarize_results_rank0(), and TimeDeviation().

oprintf

#define oprintf

(

...

)

do { fprintf(o.logfile, __VA_ARGS__); fflush(o.logfile); } while(0);

Definition at line 29 of file md-workbench.c.

Referenced by end_phase(), md_workbench_run(), printTime(), run_benchmark(), run_cleanup(), and run_precreate().

Function Documentation

add_timed_result()

static float add_timed_result ( double  start, double  phase_start_timer, time_result_t *  results, size_t  pos, double *  max_time, double *  out_op_time  )

static

Definition at line 189 of file md-workbench.c.

References GetTimeStamp(), time_result_t::runtime, and time_result_t::time_since_app_start.

Referenced by run_benchmark(), run_cleanup(), and run_precreate().

Here is the call graph for this function:

Here is the caller graph for this function:

aggregate_timers()

static uint64_t aggregate_timers ( int  repeats, int  max_repeats, time_result_t *  times, time_result_t *  global_times  )

static

Definition at line 364 of file md-workbench.c.

References CHECK_MPI_RET, benchmark_options::com, o, benchmark_options::rank, and benchmark_options::size.

Referenced by end_phase().

Here is the caller graph for this function:

compare_floats()

static int compare_floats ( time_result_t *  x, time_result_t *  y  )

static

Definition at line 354 of file md-workbench.c.

References time_result_t::runtime.

Referenced by compute_histogram().

Here is the caller graph for this function:

compute_histogram()

static void compute_histogram ( const char *  name, time_result_t *  times, time_statistics_t *  stats, size_t  repeats, int  writeLatencyFile  )

static

Definition at line 387 of file md-workbench.c.

References compare_floats(), ERRF, benchmark_options::global_iteration, benchmark_options::latency_file_prefix, time_statistics_t::max, MAX_PATHLEN, time_statistics_t::median, time_statistics_t::min, NULL, o, time_statistics_t::q1, time_statistics_t::q3, time_statistics_t::q90, time_statistics_t::q99, benchmark_options::rank, benchmark_options::relative_waiting_factor, time_result_t::runtime, and runtime_quantile().

Referenced by end_phase().

Here is the call graph for this function:

Here is the caller graph for this function:

def_dset_name()

static void def_dset_name ( char *  out_name, int  n, int  d  )

static

Definition at line 135 of file md-workbench.c.

References o, and benchmark_options::prefix.

Referenced by run_cleanup(), and run_precreate().

Here is the caller graph for this function:

def_obj_name()

static void def_obj_name ( char *  out_name, int  n, int  d, int  i  )

static

Definition at line 139 of file md-workbench.c.

References o, and benchmark_options::prefix.

Referenced by run_benchmark(), run_cleanup(), and run_precreate().

Here is the caller graph for this function:

end_phase()

static void end_phase ( const char *  name, phase_stat_t *  p  )

static

Definition at line 417 of file md-workbench.c.

References aggregate_timers(), CHECK_MPI_RET, benchmark_options::com, compute_histogram(), mdworkbench_results_t::count, benchmark_options::dset_count, phase_stat_t::dset_create, init_stats(), benchmark_options::latency_keep_all, phase_stat_t::max_op_time, MAX_PATHLEN, benchmark_options::num, o, oprintf, benchmark_options::precreate, print_p_stat(), benchmark_options::process_report, benchmark_options::rank, benchmark_options::read_only, phase_stat_t::repeats, mdworkbench_results_t::result, benchmark_options::results, benchmark_options::size, mdworkbench_result_t::stats_create, phase_stat_t::stats_create, mdworkbench_result_t::stats_delete, phase_stat_t::stats_delete, mdworkbench_result_t::stats_read, phase_stat_t::stats_read, mdworkbench_result_t::stats_stat, phase_stat_t::stats_stat, phase_stat_t::stonewall_iterations, phase_stat_t::t, phase_stat_t::t_all, phase_stat_t::time_create, phase_stat_t::time_delete, phase_stat_t::time_read, and phase_stat_t::time_stat.

Referenced by md_workbench_run().

Here is the call graph for this function:

Here is the caller graph for this function:

init_options()

void init_options

(

)

Definition at line 143 of file md-workbench.c.

References benchmark_options::interface, and o.

Referenced by md_workbench_run().

Here is the caller graph for this function:

init_stats()

static void init_stats ( phase_stat_t *  p, size_t  repeats  )

static

Definition at line 179 of file md-workbench.c.

References phase_stat_t::repeats, phase_stat_t::time_create, phase_stat_t::time_delete, phase_stat_t::time_read, and phase_stat_t::time_stat.

Referenced by end_phase(), and md_workbench_run().

Here is the caller graph for this function:

md_workbench_run()

mdworkbench_results_t* md_workbench_run	(	int	argc,
		char **	argv,
		MPI_Comm	world_com,
		FILE *	out_logfile
	)

Definition at line 891 of file md-workbench.c.

References benchmark_options::adaptive_waiting_mode, aiori_select(), airoi_create_all_module_options(), airoi_update_module_options(), benchmark_options::backend, benchmark_options::backend_options, ior_aiori::check_params, benchmark_options::com, mdworkbench_results_t::count, benchmark_options::dataPacketType, DIRMODE, benchmark_options::dset_count, ior_aiori::enable_mdtest, end_phase(), ERR, benchmark_options::file_size, aiori_xfer_hint_t::filePerProc, ior_aiori::finalize, GetNumNodes(), GetNumTasksOnNode0(), GetTimeStamp(), benchmark_options::global_iteration, global_options, benchmark_options::gpuID, benchmark_options::gpuMemoryFlags, benchmark_options::hints, init_clock(), init_options(), init_stats(), initCUDA(), ior_aiori::initialize, benchmark_options::interface, IOR_MEMORY_TYPE_CPU, benchmark_options::iterations, benchmark_options::logfile, ior_aiori::mkdir, NULL, benchmark_options::num, o, oprintf, option_parse(), out_logfile, benchmark_options::packetTypeStr, parsePacketType(), benchmark_options::phase_benchmark, benchmark_options::phase_cleanup, benchmark_options::phase_precreate, phase_stat_t::phase_start_timer, benchmark_options::precreate, benchmark_options::prefix, print_detailed_stat_header(), benchmark_options::print_detailed_stats, printTime(), QueryNodeMapping(), benchmark_options::quiet_output, benchmark_options::random_seed, benchmark_options::rank, benchmark_options::relative_waiting_factor, benchmark_options::results, return_position(), ior_aiori::rmdir, run_benchmark(), run_cleanup(), run_precreate(), benchmark_options::size, benchmark_options::start_item_number, benchmark_options::stonewall_timer, benchmark_options::stonewall_timer_wear_out, store_position(), phase_stat_t::t, WARN, WARNF, and ior_aiori::xfer_hints.

Referenced by main().

Here is the call graph for this function:

Here is the caller graph for this function:

mdw_wait()

static void mdw_wait ( double  runtime )

static

Definition at line 160 of file md-workbench.c.

References GetTimeStamp(), NULL, o, and benchmark_options::relative_waiting_factor.

Referenced by run_benchmark().

Here is the call graph for this function:

Here is the caller graph for this function:

print_detailed_stat_header()

static void print_detailed_stat_header ( )

static

Definition at line 201 of file md-workbench.c.

Referenced by md_workbench_run().

Here is the caller graph for this function:

print_p_stat()

static void print_p_stat ( char *  buff, const char *  name, phase_stat_t *  p, double  t, int  print_global  )

static

Definition at line 237 of file md-workbench.c.

References mdworkbench_results_t::count, phase_stat_t::dset_create, phase_stat_t::dset_delete, op_stat_t::err, mdworkbench_result_t::errors, mdworkbench_results_t::errors, benchmark_options::file_size, mdworkbench_result_t::iterations_done, time_statistics_t::max, mdworkbench_result_t::max_op_time, phase_stat_t::max_op_time, time_statistics_t::median, time_statistics_t::min, o, phase_stat_t::obj_create, phase_stat_t::obj_delete, phase_stat_t::obj_read, phase_stat_t::obj_stat, benchmark_options::print_detailed_stats, time_statistics_t::q1, time_statistics_t::q3, time_statistics_t::q90, time_statistics_t::q99, benchmark_options::quiet_output, mdworkbench_result_t::rate, benchmark_options::read_only, benchmark_options::relative_waiting_factor, phase_stat_t::repeats, mdworkbench_results_t::result, benchmark_options::results, mdworkbench_result_t::runtime, benchmark_options::size, statistics_mean(), statistics_minmax(), statistics_std_dev(), phase_stat_t::stats_create, phase_stat_t::stats_delete, phase_stat_t::stats_read, phase_stat_t::stats_stat, phase_stat_t::stonewall_iterations, op_stat_t::suc, sum_err(), phase_stat_t::t, and phase_stat_t::t_all.

Referenced by end_phase().

Here is the call graph for this function:

Here is the caller graph for this function:

printTime()

static void printTime ( )

static

Definition at line 852 of file md-workbench.c.

References oprintf.

Referenced by md_workbench_run().

Here is the caller graph for this function:

return_position()

static int return_position ( )

static

Definition at line 859 of file md-workbench.c.

References benchmark_options::com, ERRF, o, benchmark_options::rank, and benchmark_options::run_info_file.

Referenced by md_workbench_run().

Here is the caller graph for this function:

run_benchmark()

void run_benchmark	(	phase_stat_t *	s,
		int *	current_index_p
	)

Definition at line 602 of file md-workbench.c.

References add_timed_result(), aligned_buffer_alloc(), aligned_buffer_free(), benchmark_options::backend, benchmark_options::backend_options, CHECK_MPI_RET, ior_aiori::close, benchmark_options::com, ior_aiori::create, benchmark_options::dataPacketType, def_obj_name(), ior_aiori::delete, benchmark_options::dset_count, op_stat_t::err, ERRF, FAIL, benchmark_options::file_size, generate_memory_pattern(), GetTimeStamp(), benchmark_options::gpuMemoryFlags, benchmark_options::ignore_precreate_errors, invalidate_buffer_pattern(), IOR_CREAT, IOR_RDONLY, IOR_WRONLY, phase_stat_t::max_op_time, MAX_PATHLEN, mdw_wait(), NULL, benchmark_options::num, o, phase_stat_t::obj_create, phase_stat_t::obj_delete, phase_stat_t::obj_read, phase_stat_t::obj_stat, benchmark_options::offset, ior_aiori::open, oprintf, phase_stat_t::phase_start_timer, benchmark_options::precreate, benchmark_options::random_seed, benchmark_options::rank, READ, benchmark_options::read_only, benchmark_options::relative_waiting_factor, phase_stat_t::repeats, benchmark_options::size, ior_aiori::stat, phase_stat_t::stonewall_iterations, benchmark_options::stonewall_timer, benchmark_options::stonewall_timer_wear_out, op_stat_t::suc, phase_stat_t::t, phase_stat_t::time_create, phase_stat_t::time_delete, phase_stat_t::time_read, phase_stat_t::time_stat, update_write_memory_pattern(), benchmark_options::verbosity, verify_memory_pattern(), benchmark_options::verify_read, WARNF, WRITE, and ior_aiori::xfer.

Referenced by md_workbench_run().

Here is the call graph for this function:

Here is the caller graph for this function:

run_cleanup()

void run_cleanup	(	phase_stat_t *	s,
		int	start_index
	)

Definition at line 776 of file md-workbench.c.

References add_timed_result(), benchmark_options::backend, benchmark_options::backend_options, def_dset_name(), def_obj_name(), ior_aiori::delete, benchmark_options::dset_count, phase_stat_t::dset_delete, GetTimeStamp(), phase_stat_t::max_op_time, MAX_PATHLEN, o, phase_stat_t::obj_delete, oprintf, phase_stat_t::phase_start_timer, benchmark_options::precreate, benchmark_options::rank, ior_aiori::rmdir, op_stat_t::suc, phase_stat_t::time_delete, and benchmark_options::verbosity.

Referenced by md_workbench_run().

Here is the call graph for this function:

Here is the caller graph for this function:

run_precreate()

void run_precreate	(	phase_stat_t *	s,
		int	current_index
	)

Definition at line 544 of file md-workbench.c.

References add_timed_result(), aligned_buffer_alloc(), aligned_buffer_free(), benchmark_options::backend, benchmark_options::backend_options, ior_aiori::close, ior_aiori::create, benchmark_options::dataPacketType, def_dset_name(), def_obj_name(), DIRMODE, benchmark_options::dset_count, phase_stat_t::dset_create, op_stat_t::err, ERRF, FAIL, benchmark_options::file_size, generate_memory_pattern(), GetTimeStamp(), benchmark_options::gpuMemoryFlags, benchmark_options::ignore_precreate_errors, IOR_CREAT, IOR_WRONLY, phase_stat_t::max_op_time, MAX_PATHLEN, ior_aiori::mkdir, NULL, o, phase_stat_t::obj_create, oprintf, phase_stat_t::phase_start_timer, benchmark_options::precreate, benchmark_options::random_seed, benchmark_options::rank, op_stat_t::suc, phase_stat_t::time_create, update_write_memory_pattern(), benchmark_options::verbosity, WRITE, and ior_aiori::xfer.

Referenced by md_workbench_run().

Here is the call graph for this function:

Here is the caller graph for this function:

runtime_quantile()

static double runtime_quantile ( int  repeats, time_result_t *  times, float  quantile  )

static

Definition at line 358 of file md-workbench.c.

References time_result_t::runtime.

Referenced by compute_histogram().

Here is the caller graph for this function:

statistics_mean()

static double statistics_mean ( int  count, double *  arr  )

static

Definition at line 209 of file md-workbench.c.

References o, and benchmark_options::size.

Referenced by print_p_stat(), and statistics_std_dev().

Here is the caller graph for this function:

statistics_minmax()

static void statistics_minmax ( int  count, double *  arr, double *  out_min, double *  out_max  )

static

Definition at line 226 of file md-workbench.c.

References min, o, and benchmark_options::size.

Referenced by print_p_stat().

Here is the caller graph for this function:

statistics_std_dev()

static double statistics_std_dev ( int  count, double *  arr  )

static

Definition at line 217 of file md-workbench.c.

References o, benchmark_options::size, and statistics_mean().

Referenced by print_p_stat().

Here is the call graph for this function:

Here is the caller graph for this function:

store_position()

static void store_position ( int  position )

static

Definition at line 878 of file md-workbench.c.

References ERRF, o, benchmark_options::rank, and benchmark_options::run_info_file.

Referenced by md_workbench_run().

Here is the caller graph for this function:

sum_err()

static int sum_err ( phase_stat_t *  p )

static

Definition at line 205 of file md-workbench.c.

References phase_stat_t::dset_create, phase_stat_t::dset_delete, op_stat_t::err, phase_stat_t::obj_create, phase_stat_t::obj_delete, phase_stat_t::obj_read, and phase_stat_t::obj_stat.

Referenced by print_p_stat().

Here is the caller graph for this function:

Variable Documentation

o

struct benchmark_options o

Definition at line 133 of file md-workbench.c.

Referenced by aggregate_timers(), aio_check_params(), aio_Close(), aio_create(), aio_finalize(), aio_Fsync(), aio_initialize(), aio_Open(), aio_options(), aio_Sync(), aio_Xfer(), CHFS_options(), compute_histogram(), createGlobalOptions(), def_dset_name(), def_obj_name(), DFS_check_params(), DFS_Create(), DFS_Finalize(), DFS_Init(), DFS_Open(), DFS_options(), DUMMY_Close(), DUMMY_Create(), DUMMY_options(), DUMMY_Sync(), DUMMY_Xfer(), end_phase(), HDF5_check_params(), HDF5_Open(), HDF5_options(), HDFS_access(), HDFS_Close(), HDFS_Create_Or_Open(), HDFS_Delete(), HDFS_Fsync(), HDFS_GetFileSize(), HDFS_mkdir(), HDFS_options(), HDFS_rmdir(), HDFS_stat(), HDFS_statfs(), HDFS_Xfer(), IME_Open(), IME_Options(), init_options(), ior_mmap_file(), md_workbench_run(), mdw_wait(), MMAP_Close(), MMAP_Fsync(), MMAP_options(), MMAP_Xfer(), MPIIO_options(), NCMPI_Create(), NCMPI_Open(), NCMPI_options(), NCMPI_Xfer(), option_parse(), option_parse_token(), POSIX_check_params(), POSIX_Close(), POSIX_Create(), POSIX_Open(), POSIX_options(), POSIX_Xfer(), print_current_option_section(), print_help_section(), print_p_stat(), RADOS_check_params(), RADOS_Initialize(), RADOS_options(), return_position(), run_benchmark(), run_cleanup(), run_precreate(), S3_check_params(), S3_Create(), S3_Delete(), S3_final(), S3_init(), S3_mkdir(), S3_Open(), S3_options(), S3_rmdir(), S3_stat(), S3_statfs(), S3_Xfer(), SeekOffset(), SetupDataSet(), statistics_mean(), statistics_minmax(), statistics_std_dev(), and store_position().

options

option_help options[]

static

Definition at line 811 of file md-workbench.c.