IOR
Macros | Typedefs | Functions | Variables
aiori-S3.c File Reference
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <assert.h>
#include "ior.h"
#include "aiori.h"
#include "iordef.h"
#include <curl/curl.h>
#include <libxml/parser.h>
#include <libxml/tree.h>
#include "aws4c.h"
#include "aws4c_extra.h"
Include dependency graph for aiori-S3.c:

Go to the source code of this file.

Macros

#define BUFF_SIZE   1024
 
#define IOR_CURL_INIT   0x01 /* curl top-level inits were perfomed once? */
 
#define IOR_CURL_NOCONTINUE   0x02
 
#define IOR_CURL_S3_EMC_EXT   0x04 /* allow EMC extensions to S3? */
 
#define CURL_ERR(MSG, CURL_ERRNO, PARAM)
 
#define CURL_WARN(MSG, CURL_ERRNO)
 

Typedefs

typedef void CURL
 
typedef void IOBuf
 

Functions

static void * S3_Create (char *, IOR_param_t *)
 
static void * S3_Open (char *, IOR_param_t *)
 
static IOR_offset_t S3_Xfer (int, void *, IOR_size_t *, IOR_offset_t, IOR_param_t *)
 
static void S3_Close (void *, IOR_param_t *)
 
static void * EMC_Create (char *, IOR_param_t *)
 
static void * EMC_Open (char *, IOR_param_t *)
 
static IOR_offset_t EMC_Xfer (int, void *, IOR_size_t *, IOR_offset_t, IOR_param_t *)
 
static void EMC_Close (void *, IOR_param_t *)
 
static void S3_Delete (char *, IOR_param_t *)
 
static void S3_Fsync (void *, IOR_param_t *)
 
static IOR_offset_t S3_GetFileSize (IOR_param_t *, MPI_Comm, char *)
 
static void S3_init ()
 
static void S3_finalize ()
 
static void s3_connect (IOR_param_t *param)
 
static void s3_disconnect (IOR_param_t *param)
 
void s3_MPU_reset (IOR_param_t *param)
 
static void * S3_Create_Or_Open_internal (char *testFileName, IOR_param_t *param, unsigned char createFile, int multi_part_upload_p)
 
static IOR_offset_t S3_Xfer_internal (int access, void *file, IOR_size_t *buffer, IOR_offset_t length, IOR_param_t *param, int multi_part_upload_p)
 
static void S3_Close_internal (void *fd, IOR_param_t *param, int multi_part_upload_p)
 
static void EMC_Delete (char *testFileName, IOR_param_t *param)
 

Variables

static char buff [BUFF_SIZE]
 
const int ETAG_SIZE = 32
 
CURLcode rc
 
const char * bucket_name = "ior"
 
IOBufio_buf
 
IOBufetags
 
ior_aiori_t s3_aiori
 
ior_aiori_t s3_plus_aiori
 
ior_aiori_t s3_emc_aiori
 

Macro Definition Documentation

◆ BUFF_SIZE

#define BUFF_SIZE   1024

◆ CURL_ERR

#define CURL_ERR (   MSG,
  CURL_ERRNO,
  PARAM 
)
Value:
do { \
fprintf(stdout, "ior ERROR: %s: %s (curl-errno=%d) (%s:%d)\n", \
MSG, curl_easy_strerror(CURL_ERRNO), CURL_ERRNO, \
__FILE__, __LINE__); \
fflush(stdout); \
MPI_Abort((PARAM)->testComm, -1); \
} while (0)
MPI_Comm testComm
Definition: utilities.c:61

Definition at line 231 of file aiori-S3.c.

◆ CURL_WARN

#define CURL_WARN (   MSG,
  CURL_ERRNO 
)
Value:
do { \
fprintf(stdout, "ior WARNING: %s: %s (curl-errno=%d) (%s:%d)\n", \
MSG, curl_easy_strerror(CURL_ERRNO), CURL_ERRNO, \
__FILE__, __LINE__); \
fflush(stdout); \
} while (0)

Definition at line 241 of file aiori-S3.c.

◆ IOR_CURL_INIT

#define IOR_CURL_INIT   0x01 /* curl top-level inits were perfomed once? */

Definition at line 129 of file aiori-S3.c.

Referenced by s3_connect().

◆ IOR_CURL_NOCONTINUE

#define IOR_CURL_NOCONTINUE   0x02

Definition at line 130 of file aiori-S3.c.

◆ IOR_CURL_S3_EMC_EXT

#define IOR_CURL_S3_EMC_EXT   0x04 /* allow EMC extensions to S3? */

Definition at line 131 of file aiori-S3.c.

Referenced by s3_connect().

Typedef Documentation

◆ CURL

typedef void CURL

Definition at line 137 of file aiori-S3.c.

◆ IOBuf

typedef void IOBuf

Definition at line 138 of file aiori-S3.c.

Function Documentation

◆ EMC_Close()

static void EMC_Close ( void *  fd,
IOR_param_t param 
)
static

Definition at line 1316 of file aiori-S3.c.

References FALSE, and S3_Close_internal().

Here is the call graph for this function:

◆ EMC_Create()

static void * EMC_Create ( char *  testFileName,
IOR_param_t param 
)
static

Definition at line 582 of file aiori-S3.c.

References FALSE, S3_Create_Or_Open_internal(), TRUE, IOR_param_t::verbose, and VERBOSE_2.

Here is the call graph for this function:

◆ EMC_Delete()

static void EMC_Delete ( char *  testFileName,
IOR_param_t param 
)
static

Definition at line 1363 of file aiori-S3.c.

References s3_connect(), IOR_param_t::verbose, and VERBOSE_2.

Here is the call graph for this function:

◆ EMC_Open()

static void * EMC_Open ( char *  testFileName,
IOR_param_t param 
)
static

Definition at line 620 of file aiori-S3.c.

References FALSE, IOR_CREAT, IOR_param_t::openFlags, S3_Create_Or_Open_internal(), TRUE, IOR_param_t::verbose, and VERBOSE_2.

Here is the call graph for this function:

◆ EMC_Xfer()

static IOR_offset_t EMC_Xfer ( int  access,
void *  file,
IOR_size_t buffer,
IOR_offset_t  length,
IOR_param_t param 
)
static

Definition at line 931 of file aiori-S3.c.

References FALSE, and S3_Xfer_internal().

Here is the call graph for this function:

◆ S3_Close()

static void S3_Close ( void *  fd,
IOR_param_t param 
)
static

Definition at line 1310 of file aiori-S3.c.

References S3_Close_internal(), and TRUE.

Here is the call graph for this function:

◆ S3_Close_internal()

static void S3_Close_internal ( void *  fd,
IOR_param_t param,
int  multi_part_upload_p 
)
static

Definition at line 1015 of file aiori-S3.c.

References buff, BUFF_SIZE, ERR_SIMPLE, ETAG_SIZE, IOR_param_t::filePerProc, MPI_CHECK, NULL, IOR_param_t::numTasks, IOR_param_t::open, rank, s3_MPU_reset(), IOR_param_t::segmentCount, IOR_param_t::testComm, IOR_param_t::UploadId, IOR_param_t::verbose, VERBOSE_2, VERBOSE_3, VERBOSE_4, and WRITE.

Referenced by EMC_Close(), and S3_Close().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ s3_connect()

static void s3_connect ( IOR_param_t param)
static

Definition at line 275 of file aiori-S3.c.

References bucket_name, buff, BUFF_SIZE, IOR_CURL_INIT, IOR_CURL_S3_EMC_EXT, MPI_CHECK, rank, IOR_param_t::testComm, IOR_param_t::verbose, and VERBOSE_2.

Referenced by EMC_Delete(), S3_Create_Or_Open_internal(), S3_Delete(), and S3_GetFileSize().

Here is the caller graph for this function:

◆ S3_Create()

static void * S3_Create ( char *  testFileName,
IOR_param_t param 
)
static

Definition at line 570 of file aiori-S3.c.

References S3_Create_Or_Open_internal(), TRUE, IOR_param_t::verbose, and VERBOSE_2.

Here is the call graph for this function:

◆ S3_Create_Or_Open_internal()

static void* S3_Create_Or_Open_internal ( char *  testFileName,
IOR_param_t param,
unsigned char  createFile,
int  multi_part_upload_p 
)
static

◆ S3_Delete()

static void S3_Delete ( char *  testFileName,
IOR_param_t param 
)
static

Definition at line 1334 of file aiori-S3.c.

References s3_connect(), IOR_param_t::verbose, and VERBOSE_2.

Here is the call graph for this function:

◆ s3_disconnect()

static void s3_disconnect ( IOR_param_t param)
static

Definition at line 382 of file aiori-S3.c.

References IOR_param_t::verbose, and VERBOSE_2.

◆ S3_finalize()

static void S3_finalize ( )
static

Definition at line 224 of file aiori-S3.c.

◆ S3_Fsync()

static void S3_Fsync ( void *  fd,
IOR_param_t param 
)
static

Definition at line 977 of file aiori-S3.c.

References IOR_param_t::verbose, and VERBOSE_2.

◆ S3_GetFileSize()

static IOR_offset_t S3_GetFileSize ( IOR_param_t param,
MPI_Comm  testComm,
char *  testFileName 
)
static

Definition at line 1405 of file aiori-S3.c.

References IOR_param_t::filePerProc, MPI_CHECK, rank, s3_connect(), IOR_param_t::testComm, TRUE, IOR_param_t::verbose, VERBOSE_2, and WARN.

Here is the call graph for this function:

◆ S3_init()

static void S3_init ( )
static

Definition at line 217 of file aiori-S3.c.

◆ s3_MPU_reset()

void s3_MPU_reset ( IOR_param_t param)

Definition at line 400 of file aiori-S3.c.

References IOR_param_t::part_number.

Referenced by S3_Close_internal().

Here is the caller graph for this function:

◆ S3_Open()

static void * S3_Open ( char *  testFileName,
IOR_param_t param 
)
static

Definition at line 600 of file aiori-S3.c.

References FALSE, IOR_CREAT, IOR_param_t::openFlags, S3_Create_Or_Open_internal(), TRUE, IOR_param_t::verbose, and VERBOSE_2.

Here is the call graph for this function:

◆ S3_Xfer()

static IOR_offset_t S3_Xfer ( int  access,
void *  file,
IOR_size_t buffer,
IOR_offset_t  length,
IOR_param_t param 
)
static

Definition at line 922 of file aiori-S3.c.

References S3_Xfer_internal(), and TRUE.

Here is the call graph for this function:

◆ S3_Xfer_internal()

static IOR_offset_t S3_Xfer_internal ( int  access,
void *  file,
IOR_size_t buffer,
IOR_offset_t  length,
IOR_param_t param,
int  multi_part_upload_p 
)
static

Variable Documentation

◆ bucket_name

const char* bucket_name = "ior"

Definition at line 124 of file aiori-S3.c.

Referenced by s3_connect().

◆ buff

char buff[BUFF_SIZE]
static

◆ ETAG_SIZE

const int ETAG_SIZE = 32

Definition at line 119 of file aiori-S3.c.

Referenced by S3_Close_internal(), and S3_Xfer_internal().

◆ etags

IOBuf* etags

Definition at line 142 of file aiori-S3.c.

◆ io_buf

IOBuf* io_buf

Definition at line 141 of file aiori-S3.c.

◆ rc

CURLcode rc

◆ s3_aiori

ior_aiori_t s3_aiori
Initial value:
= {
.name = "S3",
.create = S3_Create,
.open = S3_Open,
.xfer = S3_Xfer,
.close = S3_Close,
.delete = S3_Delete,
.get_version = aiori_get_version,
.fsync = S3_Fsync,
.get_file_size = S3_GetFileSize,
.initialize = S3_init,
.finalize = S3_finalize
}
static void S3_Fsync(void *, IOR_param_t *)
Definition: aiori-S3.c:977
static void S3_finalize()
Definition: aiori-S3.c:224
static void S3_Close(void *, IOR_param_t *)
Definition: aiori-S3.c:1310
static IOR_offset_t S3_GetFileSize(IOR_param_t *, MPI_Comm, char *)
Definition: aiori-S3.c:1405
char * aiori_get_version()
Definition: aiori.c:149
static void S3_init()
Definition: aiori-S3.c:217
static void * S3_Open(char *, IOR_param_t *)
Definition: aiori-S3.c:600
static void * S3_Create(char *, IOR_param_t *)
Definition: aiori-S3.c:570
static IOR_offset_t S3_Xfer(int, void *, IOR_size_t *, IOR_offset_t, IOR_param_t *)
Definition: aiori-S3.c:922
static void S3_Delete(char *, IOR_param_t *)
Definition: aiori-S3.c:1334

Definition at line 168 of file aiori-S3.c.

◆ s3_emc_aiori

ior_aiori_t s3_emc_aiori
Initial value:
= {
.name = "S3_EMC",
.create = EMC_Create,
.open = EMC_Open,
.xfer = EMC_Xfer,
.close = EMC_Close,
.delete = S3_Delete,
.set_version = S3_SetVersion,
.fsync = S3_Fsync,
.get_file_size = S3_GetFileSize,
.initialize = S3_init,
.finalize = S3_finalize
}
static void S3_Fsync(void *, IOR_param_t *)
Definition: aiori-S3.c:977
static void EMC_Close(void *, IOR_param_t *)
Definition: aiori-S3.c:1316
static IOR_offset_t EMC_Xfer(int, void *, IOR_size_t *, IOR_offset_t, IOR_param_t *)
Definition: aiori-S3.c:931
static void * EMC_Create(char *, IOR_param_t *)
Definition: aiori-S3.c:582
static void S3_finalize()
Definition: aiori-S3.c:224
static IOR_offset_t S3_GetFileSize(IOR_param_t *, MPI_Comm, char *)
Definition: aiori-S3.c:1405
static void * EMC_Open(char *, IOR_param_t *)
Definition: aiori-S3.c:620
static void S3_init()
Definition: aiori-S3.c:217
static void S3_Delete(char *, IOR_param_t *)
Definition: aiori-S3.c:1334

Definition at line 202 of file aiori-S3.c.

◆ s3_plus_aiori

ior_aiori_t s3_plus_aiori
Initial value:
= {
.name = "S3_plus",
.create = S3_Create,
.open = S3_Open,
.xfer = S3_Xfer,
.close = S3_Close,
.delete = S3_Delete,
.set_version = S3_SetVersion,
.fsync = S3_Fsync,
.get_file_size = S3_GetFileSize,
.initialize = S3_init,
.finalize = S3_finalize
}
static void S3_Fsync(void *, IOR_param_t *)
Definition: aiori-S3.c:977
static void S3_finalize()
Definition: aiori-S3.c:224
static void S3_Close(void *, IOR_param_t *)
Definition: aiori-S3.c:1310
static IOR_offset_t S3_GetFileSize(IOR_param_t *, MPI_Comm, char *)
Definition: aiori-S3.c:1405
static void S3_init()
Definition: aiori-S3.c:217
static void * S3_Open(char *, IOR_param_t *)
Definition: aiori-S3.c:600
static void * S3_Create(char *, IOR_param_t *)
Definition: aiori-S3.c:570
static IOR_offset_t S3_Xfer(int, void *, IOR_size_t *, IOR_offset_t, IOR_param_t *)
Definition: aiori-S3.c:922
static void S3_Delete(char *, IOR_param_t *)
Definition: aiori-S3.c:1334

Definition at line 185 of file aiori-S3.c.