From 8d0cddd21e7461a4bc26cac0c9d391dcfcf9a891 Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Wed, 13 Jun 2018 18:37:37 +0000 Subject: [PATCH 001/206] Add DFS ior/mdtest driver Signed-off-by: Mohamad Chaarawi --- configure.ac | 23 +- src/Makefile.am | 4 + src/aiori-DFS.c | 562 ++++++++++++++++++++++++++++++++++++++++++++++++ src/aiori.c | 3 + src/aiori.h | 4 + src/ior.c | 11 +- src/mdtest.c | 10 + 7 files changed, 614 insertions(+), 3 deletions(-) create mode 100755 src/aiori-DFS.c diff --git a/configure.ac b/configure.ac index 4f9461ce..af6e49ab 100755 --- a/configure.ac +++ b/configure.ac @@ -142,8 +142,27 @@ AM_COND_IF([USE_POSIX_AIORI],[ AC_DEFINE([USE_POSIX_AIORI], [], [Build POSIX backend AIORI]) ]) - - +# DFS IO support +AC_ARG_WITH([daos], + [AS_HELP_STRING([--with-daos], + [support IO with DFS backend @<:@default=no@:>@])], + [], + [with_daos=no]) + +AS_IF([test "x$with_daos" != xno], + DAOS="yes" + LDFLAGS="$LDFLAGS -L$with_daos/lib" + CPPFLAGS="$CPPFLAGS -I$with_daos/include" + AC_CHECK_HEADERS(daos_types.h,, [unset DAOS]) + AC_CHECK_LIB([uuid], [uuid_generate],, [unset DAOS]) + AC_CHECK_LIB([daos_common], [daos_sgl_init],, [unset DAOS]) + AC_CHECK_LIB([daos], [daos_init],, [unset DAOS]) + AC_CHECK_LIB([dfs], [dfs_mkdir],, [unset DAOS])) + +AM_CONDITIONAL([USE_DFS_AIORI], [test x$DAOS = xyes]) +AM_COND_IF([USE_DFS_AIORI],[ + AC_DEFINE([USE_DFS_AIORI], [], [Build DFS backend AIORI]) +]) # aws4c is needed for the S3 backend (see --with-S3, below). # Version 0.5.2 of aws4c is available at https://github.com/jti-lanl/aws4c.git diff --git a/src/Makefile.am b/src/Makefile.am index 7d2575ba..aea18245 100755 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -53,6 +53,10 @@ if USE_POSIX_AIORI extraSOURCES += aiori-POSIX.c endif +if USE_DFS_AIORI +extraSOURCES += aiori-DFS.c +endif + if USE_S3_AIORI extraSOURCES += aiori-S3.c diff --git a/src/aiori-DFS.c b/src/aiori-DFS.c new file mode 100755 index 00000000..ad0c712e --- /dev/null +++ b/src/aiori-DFS.c @@ -0,0 +1,562 @@ +/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- + * vim:expandtab:shiftwidth=8:tabstop=8: + */ +/******************************************************************************\ +* * +* Copyright (c) 2003, The Regents of the University of California * +* See the file COPYRIGHT for a complete copyright notice and license. * +* * +******************************************************************************** +* +* Implement of abstract I/O interface for DFS. +* +\******************************************************************************/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "ior.h" +#include "aiori.h" +#include "iordef.h" +#include "utilities.h" + +dfs_t *dfs; + +static int +parse_filename(const char *path, char **_obj_name, char **_cont_name) +{ + char *f1 = NULL; + char *f2 = NULL; + char *fname = NULL; + char *cont_name = NULL; + int rc = 0; + + if (path == NULL || _obj_name == NULL || _cont_name == NULL) + return -EINVAL; + + if (strcmp(path, "/") == 0) { + *_cont_name = strdup("/"); + if (*_cont_name == NULL) + return -ENOMEM; + *_obj_name = NULL; + return 0; + } + + f1 = strdup(path); + if (f1 == NULL) + D_GOTO(out, rc = -ENOMEM); + + f2 = strdup(path); + if (f2 == NULL) + D_GOTO(out, rc = -ENOMEM); + + fname = basename(f1); + cont_name = dirname(f2); + + if (cont_name[0] == '.' || cont_name[0] != '/') { + char *cwd; + + //getcwd(cwd, 1024); + cwd = strdup("/"); + if (strcmp(cont_name, ".") == 0) { + cont_name = strdup(cwd); + if (cont_name == NULL) + D_GOTO(out, rc = -ENOMEM); + } else { + char *new_dir = calloc(strlen(cwd) + strlen(cont_name) + + 1, sizeof(char)); + if (new_dir == NULL) + D_GOTO(out, rc = -ENOMEM); + + strcpy(new_dir, cwd); + if (cont_name[0] == '.') { + strcat(new_dir, &cont_name[1]); + } else { + strcat(new_dir, "/"); + strcat(new_dir, cont_name); + } + cont_name = new_dir; + } + *_cont_name = cont_name; + } else { + *_cont_name = strdup(cont_name); + if (*_cont_name == NULL) + D_GOTO(out, rc = -ENOMEM); + } + + *_obj_name = strdup(fname); + if (*_obj_name == NULL) { + free(*_cont_name); + *_cont_name = NULL; + D_GOTO(out, rc = -ENOMEM); + } + +out: + if (f1) + free(f1); + if (f2) + free(f2); + return rc; +} + +/**************************** P R O T O T Y P E S *****************************/ +static void *DFS_Create(char *, IOR_param_t *); +static void *DFS_Open(char *, IOR_param_t *); +static IOR_offset_t DFS_Xfer(int, void *, IOR_size_t *, + IOR_offset_t, IOR_param_t *); +static void DFS_Close(void *, IOR_param_t *); +static void DFS_Delete(char *, IOR_param_t *); +static void DFS_SetVersion(IOR_param_t *); +static void DFS_Fsync(void *, IOR_param_t *); +static IOR_offset_t DFS_GetFileSize(IOR_param_t *, MPI_Comm, char *); +static int DFS_Statfs (const char *, ior_aiori_statfs_t *, IOR_param_t *); +static int DFS_Stat (const char *, struct stat *, IOR_param_t *); +static int DFS_Mkdir (const char *, mode_t, IOR_param_t *); +static int DFS_Rmdir (const char *, IOR_param_t *); +static int DFS_Access (const char *, int, IOR_param_t *); + +/************************** D E C L A R A T I O N S ***************************/ + +ior_aiori_t dfs_aiori = { + .name = "DFS", + .create = DFS_Create, + .open = DFS_Open, + .xfer = DFS_Xfer, + .close = DFS_Close, + .delete = DFS_Delete, + .set_version = DFS_SetVersion, + .fsync = DFS_Fsync, + .get_file_size = DFS_GetFileSize, + .statfs = DFS_Statfs, + .mkdir = DFS_Mkdir, + .rmdir = DFS_Rmdir, + .access = DFS_Access, + .stat = DFS_Stat, +}; + +/***************************** F U N C T I O N S ******************************/ + +int +dfs_init(void) { + int rc; + + rc = daos_init(); + if (rc) { + fprintf(stderr, "daos_init() failed with %d\n", rc); + return rc; + } + + rc = dfs_mount(&dfs); + if (rc) { + fprintf(stderr, "dfs_mount failed (%d)\n", rc); + return 1; + } + + return rc; +} + +int dfs_finalize(void) +{ + dfs_umount(dfs); + daos_fini(); + return 0; +} + +/* + * Creat and open a file through the DFS interface. + */ +static void * +DFS_Create(char *testFileName, IOR_param_t *param) +{ + char *name = NULL, *dir_name = NULL; + dfs_obj_t *obj = NULL, *parent = NULL; + mode_t pmode; + int fd_oflag = 0; + int rc; + + fd_oflag |= O_CREAT | O_RDWR; + + rc = parse_filename(testFileName, &name, &dir_name); + if (rc) + goto out; + + assert(dir_name); + assert(name); + + rc = dfs_lookup(dfs, dir_name, &parent, &pmode); + if (rc || !S_ISDIR(pmode)) + goto out; + + mode_t mode = S_IFREG | param->mode; + rc = dfs_open(dfs, parent, name, mode, fd_oflag, NULL, &obj); + if (rc) + goto out; + +out: + if (name) + free(name); + if (dir_name) + free(dir_name); + if (parent) + dfs_release(parent); + + return ((void *)obj); +} + +/* + * Open a file through the DFS interface. + */ +static void *DFS_Open(char *testFileName, IOR_param_t *param) +{ + char *name = NULL, *dir_name = NULL; + dfs_obj_t *obj = NULL, *parent = NULL; + mode_t pmode; + int rc; + int fd_oflag = 0; + + fd_oflag |= O_RDWR; + + rc = parse_filename(testFileName, &name, &dir_name); + if (rc) + goto out; + + assert(dir_name); + assert(name); + + rc = dfs_lookup(dfs, dir_name, &parent, &pmode); + if (rc || !S_ISDIR(pmode)) + goto out; + + rc = dfs_open(dfs, parent, name, S_IFREG, fd_oflag, NULL, &obj); + if (rc) + goto out; + +out: + if (name) + free(name); + if (dir_name) + free(dir_name); + if (parent) + dfs_release(parent); + + return ((void *)obj); +} + +/* + * Write or read access to file using the DFS interface. + */ +static IOR_offset_t +DFS_Xfer(int access, void *file, IOR_size_t *buffer, IOR_offset_t length, + IOR_param_t *param) +{ + int xferRetries = 0; + long long remaining = (long long)length; + char *ptr = (char *)buffer; + daos_size_t ret; + int rc; + dfs_obj_t *obj; + + obj = (dfs_obj_t *)file; + + while (remaining > 0) { + daos_iov_t iov; + daos_sg_list_t sgl; + + /** set memory location */ + sgl.sg_nr = 1; + sgl.sg_nr_out = 0; + daos_iov_set(&iov, (void *)ptr, remaining); + sgl.sg_iovs = &iov; + + /* write/read file */ + if (access == WRITE) { + rc = dfs_write(dfs, obj, sgl, param->offset); + if (rc) + ERR("write() failed"); + ret = remaining; + } else { + rc = dfs_read(dfs, obj, sgl, param->offset, &ret); + if (rc || ret == 0) + ERR("read() failed"); + } + + if (ret < remaining) { + if (param->singleXferAttempt == TRUE) + MPI_CHECK(MPI_Abort(MPI_COMM_WORLD, -1), + "barrier error"); + if (xferRetries > MAX_RETRY) + ERR("too many retries -- aborting"); + } + + assert(ret >= 0); + assert(ret <= remaining); + remaining -= ret; + ptr += ret; + xferRetries++; + } + + return (length); +} + +/* + * Perform fsync(). + */ +static void DFS_Fsync(void *fd, IOR_param_t * param) +{ + return; +} + +/* + * Close a file through the DFS interface. + */ +static void DFS_Close(void *fd, IOR_param_t * param) +{ + dfs_release((dfs_obj_t *)fd); +} + +/* + * Delete a file through the DFS interface. + */ +static void DFS_Delete(char *testFileName, IOR_param_t * param) +{ + char *name = NULL, *dir_name = NULL; + dfs_obj_t *parent = NULL; + mode_t pmode; + int rc; + + rc = parse_filename(testFileName, &name, &dir_name); + if (rc) + goto out; + + assert(dir_name); + assert(name); + + rc = dfs_lookup(dfs, dir_name, &parent, &pmode); + if (rc || !S_ISDIR(pmode)) + goto out; + + rc = dfs_remove(dfs, parent, name); + if (rc) + goto out; + +out: + if (name) + free(name); + if (dir_name) + free(dir_name); + if (parent) + dfs_release(parent); +} + +/* + * Determine api version. + */ +static void DFS_SetVersion(IOR_param_t * test) +{ + strcpy(test->apiVersion, test->api); +} + +/* + * Use DFS stat() to return aggregate file size. + */ +static IOR_offset_t DFS_GetFileSize(IOR_param_t * test, MPI_Comm testComm, + char *testFileName) +{ + dfs_obj_t *obj; + daos_size_t fsize, tmpMin, tmpMax, tmpSum; + int rc; + + rc = dfs_lookup(dfs, testFileName, &obj, NULL); + if (rc) + return -1; + + rc = dfs_get_size(dfs, obj, &fsize); + if (rc) + return -1; + + dfs_release(obj); + + if (test->filePerProc == TRUE) { + MPI_CHECK(MPI_Allreduce(&fsize, &tmpSum, 1, + MPI_LONG_LONG_INT, MPI_SUM, testComm), + "cannot total data moved"); + fsize = tmpSum; + } else { + MPI_CHECK(MPI_Allreduce(&fsize, &tmpMin, 1, + MPI_LONG_LONG_INT, MPI_MIN, testComm), + "cannot total data moved"); + MPI_CHECK(MPI_Allreduce(&fsize, &tmpMax, 1, + MPI_LONG_LONG_INT, MPI_MAX, testComm), + "cannot total data moved"); + if (tmpMin != tmpMax) { + if (rank == 0) { + WARN("inconsistent file size by different tasks"); + } + /* incorrect, but now consistent across tasks */ + fsize = tmpMin; + } + } + + return (fsize); +} + +static int +DFS_Statfs(const char *path, ior_aiori_statfs_t *sfs, IOR_param_t * param) +{ + return 0; +} + +static int +DFS_Mkdir (const char *path, mode_t mode, IOR_param_t * param) +{ + dfs_obj_t *parent = NULL; + mode_t pmode; + char *name = NULL, *dir_name = NULL; + int rc; + + rc = parse_filename(path, &name, &dir_name); + if (rc) + return rc; + + assert(dir_name); + assert(name); + + rc = dfs_lookup(dfs, dir_name, &parent, &pmode); + if (rc || !S_ISDIR(pmode)) + goto out; + + rc = dfs_mkdir(dfs, parent, name, mode); + if (rc) + goto out; + +out: + if (name) + free(name); + if (dir_name) + free(dir_name); + if (parent) + dfs_release(parent); + return rc; +} + +static int +DFS_Rmdir (const char *path, IOR_param_t * param) +{ + dfs_obj_t *parent = NULL; + mode_t pmode; + char *name = NULL, *dir_name = NULL; + int rc; + + rc = parse_filename(path, &name, &dir_name); + if (rc) + return rc; + + assert(dir_name); + assert(name); + + rc = dfs_lookup(dfs, dir_name, &parent, &pmode); + if (rc || !S_ISDIR(pmode)) + goto out; + + rc = dfs_remove(dfs, parent, name); + if (rc) + goto out; + +out: + if (name) + free(name); + if (dir_name) + free(dir_name); + if (parent) + dfs_release(parent); + return rc; +} + +static int +DFS_Access (const char *path, int mode, IOR_param_t * param) +{ + dfs_obj_t *parent = NULL; + mode_t pmode; + char *name = NULL, *dir_name = NULL; + struct stat stbuf; + int rc; + + rc = parse_filename(path, &name, &dir_name); + if (rc) + return rc; + + assert(dir_name); + assert(name); + + rc = dfs_lookup(dfs, dir_name, &parent, &pmode); + if (rc || !S_ISDIR(pmode)) + goto out; + + if (strcmp(name, ".") == 0) { + free(name); + name = NULL; + } + rc = dfs_stat(dfs, parent, name, &stbuf); + if (rc) { + rc = -1; + errno = -ENOENT; + goto out; + } + +out: + if (name) + free(name); + if (dir_name) + free(dir_name); + if (parent) + dfs_release(parent); + return rc; +} + +static int +DFS_Stat (const char *path, struct stat *buf, IOR_param_t * param) +{ + dfs_obj_t *parent = NULL; + mode_t pmode; + char *name = NULL, *dir_name = NULL; + int rc; + + rc = parse_filename(path, &name, &dir_name); + if (rc) + return rc; + + assert(dir_name); + assert(name); + + rc = dfs_lookup(dfs, dir_name, &parent, &pmode); + if (rc || !S_ISDIR(pmode)) + goto out; + + rc = dfs_stat(dfs, parent, name, buf); + if (rc) + goto out; + +out: + if (name) + free(name); + if (dir_name) + free(dir_name); + if (parent) + dfs_release(parent); + return rc; +} diff --git a/src/aiori.c b/src/aiori.c index 677c1eaf..a40cbd6c 100644 --- a/src/aiori.c +++ b/src/aiori.c @@ -51,6 +51,9 @@ ior_aiori_t *available_aiori[] = { &s3_aiori, &s3_plus_aiori, &s3_emc_aiori, +#endif +#ifdef USE_DFS_AIORI + &dfs_aiori, #endif NULL }; diff --git a/src/aiori.h b/src/aiori.h index 4ee400a1..9e5695ed 100755 --- a/src/aiori.h +++ b/src/aiori.h @@ -88,11 +88,15 @@ extern ior_aiori_t mmap_aiori; extern ior_aiori_t s3_aiori; extern ior_aiori_t s3_plus_aiori; extern ior_aiori_t s3_emc_aiori; +extern ior_aiori_t dfs_aiori; const ior_aiori_t *aiori_select (const char *api); int aiori_count (void); const char *aiori_default (void); +int dfs_init(void); +int dfs_finalize(void); + IOR_offset_t MPIIO_GetFileSize(IOR_param_t * test, MPI_Comm testComm, char *testFileName); diff --git a/src/ior.c b/src/ior.c index b92b40db..e4dc03b9 100755 --- a/src/ior.c +++ b/src/ior.c @@ -733,7 +733,7 @@ static void DisplayUsage(char **argv) { char *opts[] = { "OPTIONS:", - " -a S api -- API for I/O [POSIX|MMAP|MPIIO|HDF5|HDFS|S3|S3_EMC|NCMPI]", + " -a S api -- API for I/O [POSIX|DFS|MMAP|MPIIO|HDF5|HDFS|S3|S3_EMC|NCMPI]", " -A N refNum -- user supplied reference number to include in the summary", " -b N blockSize -- contiguous bytes to write per task (e.g.: 8, 4k, 2m, 1g)", " -B useO_DIRECT -- uses O_DIRECT for POSIX, bypassing I/O buffers", @@ -2037,6 +2037,11 @@ static void TestIoSys(IOR_test_t *test) /* bind I/O calls to specific API */ AioriBind(params->api, params); +#ifdef USE_DFS_AIORI + if (strcmp(params->api, "DFS") == 0) + dfs_init(); +#endif + /* show test setup */ if (rank == 0 && verbose >= VERBOSE_0) ShowSetup(params); @@ -2310,6 +2315,10 @@ static void TestIoSys(IOR_test_t *test) /* Sync with the tasks that did not participate in this test */ MPI_CHECK(MPI_Barrier(MPI_COMM_WORLD), "barrier error"); +#ifdef USE_DFS_AIORI + if (strcmp(params->api, "DFS") == 0) + dfs_finalize(); +#endif } /* diff --git a/src/mdtest.c b/src/mdtest.c index aadbf781..3245c479 100644 --- a/src/mdtest.c +++ b/src/mdtest.c @@ -1936,6 +1936,11 @@ int main(int argc, char **argv) { } } +#ifdef USE_DFS_AIORI + if (strcmp(backend_name, "DFS") == 0) + dfs_init(); +#endif + if (!create_only && !stat_only && !read_only && !remove_only) { create_only = stat_only = read_only = remove_only = 1; if (( rank == 0 ) && ( verbose >= 1 )) { @@ -2411,6 +2416,11 @@ int main(int argc, char **argv) { free(rand_array); } +#ifdef USE_DFS_AIORI + if (strcmp(backend_name, "DFS") == 0) + dfs_finalize(); +#endif + MPI_Finalize(); exit(0); } From 1768eff5529f4b99cb322b096d2dd52546af8ac3 Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Wed, 20 Jun 2018 21:25:22 +0000 Subject: [PATCH 002/206] update DFS plugin Signed-off-by: Mohamad Chaarawi --- .gitignore | 4 ++ configure.ac | 14 +++++ src/aiori-DFS.c | 132 +++++++++++++++++++++++++++++++++++++++++++----- 3 files changed, 136 insertions(+), 14 deletions(-) diff --git a/.gitignore b/.gitignore index 73dd929e..5ba0f005 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,8 @@ Makefile.in aclocal.m4 config.log config.status +COPYING +INSTALL config/compile config/config.guess config/config.sub @@ -11,11 +13,13 @@ config/install-sh config/missing configure contrib/.deps/ +contrib/cbif contrib/Makefile contrib/Makefile.in doc/Makefile doc/Makefile.in src/.deps/ +src/mdtest src/Makefile src/Makefile.in src/config.h diff --git a/configure.ac b/configure.ac index af6e49ab..9394f597 100755 --- a/configure.ac +++ b/configure.ac @@ -142,6 +142,18 @@ AM_COND_IF([USE_POSIX_AIORI],[ AC_DEFINE([USE_POSIX_AIORI], [], [Build POSIX backend AIORI]) ]) +AC_ARG_WITH([cart], + [AS_HELP_STRING([--with-cart], + [Build DAOS ROMIO driver[default=no]])],, + [with_cart=no]) + +AS_IF([test "x$with_cart" != xno], + CART="yes" + LDFLAGS="$LDFLAGS -L$with_cart/lib" + CPPFLAGS="$CPPFLAGS -I$with_cart/include/" + AC_CHECK_HEADERS(gurt/common.h,, [unset CART]) + AC_CHECK_LIB([gurt], [d_rank_list_alloc],, [unset CART])) + # DFS IO support AC_ARG_WITH([daos], [AS_HELP_STRING([--with-daos], @@ -159,6 +171,8 @@ AS_IF([test "x$with_daos" != xno], AC_CHECK_LIB([daos], [daos_init],, [unset DAOS]) AC_CHECK_LIB([dfs], [dfs_mkdir],, [unset DAOS])) +AS_IF([test "x$CART" != xyes], [unset DAOS]) + AM_CONDITIONAL([USE_DFS_AIORI], [test x$DAOS = xyes]) AM_COND_IF([USE_DFS_AIORI],[ AC_DEFINE([USE_DFS_AIORI], [], [Build DFS backend AIORI]) diff --git a/src/aiori-DFS.c b/src/aiori-DFS.c index ad0c712e..4e70d3be 100755 --- a/src/aiori-DFS.c +++ b/src/aiori-DFS.c @@ -36,6 +36,7 @@ #include "utilities.h" dfs_t *dfs; +daos_handle_t poh, coh; static int parse_filename(const char *path, char **_obj_name, char **_cont_name) @@ -151,20 +152,121 @@ ior_aiori_t dfs_aiori = { /***************************** F U N C T I O N S ******************************/ +/* MSC - Make a generic DAOS function instead */ +static d_rank_list_t * +daos_rank_list_parse(const char *str, const char *sep) +{ + d_rank_t *buf; + int cap = 8; + d_rank_list_t *ranks = NULL; + char *s, *p; + int n = 0; + + buf = malloc(sizeof(d_rank_t) * cap); + if (buf == NULL) + goto out; + s = strdup(str); + if (s == NULL) + goto out_buf; + + while ((s = strtok_r(s, sep, &p)) != NULL) { + if (n == cap) { + d_rank_t *buf_new; + int cap_new; + + /* Double the buffer. */ + cap_new = cap * 2; + buf_new = malloc(sizeof(d_rank_t) * cap_new); + if (buf_new == NULL) + goto out_s; + memcpy(buf_new, buf, sizeof(d_rank_t) * n); + free(buf); + buf = buf_new; + cap = cap_new; + } + buf[n] = atoi(s); + n++; + s = NULL; + } + + ranks = d_rank_list_alloc(n); + if (ranks == NULL) + goto out_s; + memcpy(ranks->rl_ranks, buf, sizeof(*buf) * n); + +out_s: + if (s) + free(s); +out_buf: + free(buf); +out: + return ranks; +} + int dfs_init(void) { - int rc; - + char *pool_str, *svcl_str, *group_str; + uuid_t pool_uuid, co_uuid; + daos_pool_info_t pool_info; + daos_cont_info_t co_info; + d_rank_list_t *svcl = NULL; + int rc; + rc = daos_init(); if (rc) { fprintf(stderr, "daos_init() failed with %d\n", rc); return rc; } - rc = dfs_mount(&dfs); + pool_str = getenv("DAOS_POOL"); + if (!pool_str) { + fprintf(stderr, "missing pool uuid\n"); + return -1; + } + if (uuid_parse(pool_str, pool_uuid) < 0) { + fprintf(stderr, "Invalid pool uuid\n"); + return -1; + } + + svcl_str = getenv("DAOS_SVCL"); + if (!svcl_str) { + fprintf(stderr, "missing pool service rank list\n"); + return -1; + } + svcl = daos_rank_list_parse(svcl_str, ":"); + if (svcl == NULL) { + fprintf(stderr, "Invalid pool service rank list\n"); + return -1; + } + + group_str = getenv("DAOS_GROUP"); + + /** Connect to DAOS pool */ + rc = daos_pool_connect(pool_uuid, group_str, svcl, DAOS_PC_RW, + &poh, &pool_info, NULL); + if (rc < 0) { + fprintf(stderr, "Failed to connect to pool %s %s (%d)\n", + pool_str, svcl_str, rc); + return -1; + } + + uuid_generate(co_uuid); + rc = daos_cont_create(poh, co_uuid, NULL); + if (rc) { + fprintf(stderr, "Failed to create container (%d)\n", rc); + return -1; + } + + rc = daos_cont_open(poh, co_uuid, DAOS_COO_RW, &coh, &co_info, NULL); + if (rc) { + fprintf(stderr, "Failed to open container (%d)\n", rc); + return -1; + } + + rc = dfs_mount(poh, coh, &dfs); if (rc) { fprintf(stderr, "dfs_mount failed (%d)\n", rc); - return 1; + return -1; } return rc; @@ -173,6 +275,8 @@ dfs_init(void) { int dfs_finalize(void) { dfs_umount(dfs); + daos_cont_close(coh, NULL); + daos_pool_disconnect(poh, NULL); daos_fini(); return 0; } @@ -198,7 +302,7 @@ DFS_Create(char *testFileName, IOR_param_t *param) assert(dir_name); assert(name); - rc = dfs_lookup(dfs, dir_name, &parent, &pmode); + rc = dfs_lookup(dfs, dir_name, O_RDWR, &parent, &pmode); if (rc || !S_ISDIR(pmode)) goto out; @@ -238,7 +342,7 @@ static void *DFS_Open(char *testFileName, IOR_param_t *param) assert(dir_name); assert(name); - rc = dfs_lookup(dfs, dir_name, &parent, &pmode); + rc = dfs_lookup(dfs, dir_name, O_RDWR, &parent, &pmode); if (rc || !S_ISDIR(pmode)) goto out; @@ -346,11 +450,11 @@ static void DFS_Delete(char *testFileName, IOR_param_t * param) assert(dir_name); assert(name); - rc = dfs_lookup(dfs, dir_name, &parent, &pmode); + rc = dfs_lookup(dfs, dir_name, O_RDWR, &parent, &pmode); if (rc || !S_ISDIR(pmode)) goto out; - rc = dfs_remove(dfs, parent, name); + rc = dfs_remove(dfs, parent, name, false); if (rc) goto out; @@ -381,7 +485,7 @@ static IOR_offset_t DFS_GetFileSize(IOR_param_t * test, MPI_Comm testComm, daos_size_t fsize, tmpMin, tmpMax, tmpSum; int rc; - rc = dfs_lookup(dfs, testFileName, &obj, NULL); + rc = dfs_lookup(dfs, testFileName, O_RDONLY, &obj, NULL); if (rc) return -1; @@ -436,7 +540,7 @@ DFS_Mkdir (const char *path, mode_t mode, IOR_param_t * param) assert(dir_name); assert(name); - rc = dfs_lookup(dfs, dir_name, &parent, &pmode); + rc = dfs_lookup(dfs, dir_name, O_RDWR, &parent, &pmode); if (rc || !S_ISDIR(pmode)) goto out; @@ -469,11 +573,11 @@ DFS_Rmdir (const char *path, IOR_param_t * param) assert(dir_name); assert(name); - rc = dfs_lookup(dfs, dir_name, &parent, &pmode); + rc = dfs_lookup(dfs, dir_name, O_RDWR, &parent, &pmode); if (rc || !S_ISDIR(pmode)) goto out; - rc = dfs_remove(dfs, parent, name); + rc = dfs_remove(dfs, parent, name, false); if (rc) goto out; @@ -503,7 +607,7 @@ DFS_Access (const char *path, int mode, IOR_param_t * param) assert(dir_name); assert(name); - rc = dfs_lookup(dfs, dir_name, &parent, &pmode); + rc = dfs_lookup(dfs, dir_name, O_RDWR, &parent, &pmode); if (rc || !S_ISDIR(pmode)) goto out; @@ -543,7 +647,7 @@ DFS_Stat (const char *path, struct stat *buf, IOR_param_t * param) assert(dir_name); assert(name); - rc = dfs_lookup(dfs, dir_name, &parent, &pmode); + rc = dfs_lookup(dfs, dir_name, O_RDONLY, &parent, &pmode); if (rc || !S_ISDIR(pmode)) goto out; From 07ec65c0d58adeacac4dd989190201350182b771 Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Thu, 19 Jul 2018 21:36:29 +0000 Subject: [PATCH 003/206] remoce rank_list_parse as it is exposed by DAOS API now. Signed-off-by: Mohamad Chaarawi --- src/aiori-DFS.c | 51 ------------------------------------------------- 1 file changed, 51 deletions(-) diff --git a/src/aiori-DFS.c b/src/aiori-DFS.c index 4e70d3be..6f0c07bb 100755 --- a/src/aiori-DFS.c +++ b/src/aiori-DFS.c @@ -152,57 +152,6 @@ ior_aiori_t dfs_aiori = { /***************************** F U N C T I O N S ******************************/ -/* MSC - Make a generic DAOS function instead */ -static d_rank_list_t * -daos_rank_list_parse(const char *str, const char *sep) -{ - d_rank_t *buf; - int cap = 8; - d_rank_list_t *ranks = NULL; - char *s, *p; - int n = 0; - - buf = malloc(sizeof(d_rank_t) * cap); - if (buf == NULL) - goto out; - s = strdup(str); - if (s == NULL) - goto out_buf; - - while ((s = strtok_r(s, sep, &p)) != NULL) { - if (n == cap) { - d_rank_t *buf_new; - int cap_new; - - /* Double the buffer. */ - cap_new = cap * 2; - buf_new = malloc(sizeof(d_rank_t) * cap_new); - if (buf_new == NULL) - goto out_s; - memcpy(buf_new, buf, sizeof(d_rank_t) * n); - free(buf); - buf = buf_new; - cap = cap_new; - } - buf[n] = atoi(s); - n++; - s = NULL; - } - - ranks = d_rank_list_alloc(n); - if (ranks == NULL) - goto out_s; - memcpy(ranks->rl_ranks, buf, sizeof(*buf) * n); - -out_s: - if (s) - free(s); -out_buf: - free(buf); -out: - return ranks; -} - int dfs_init(void) { char *pool_str, *svcl_str, *group_str; From 5fb850c8108eb3c06fa3d0201908b3bc074a20bb Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Thu, 23 Aug 2018 21:58:53 +0000 Subject: [PATCH 004/206] - update the DFS driver to latest DFS API. - update cmd line options to add DAOS Pool and Container uuid and SVCL - Add init/finalize backend functions. Signed-off-by: Mohamad Chaarawi --- src/aiori-DFS.c | 181 ++++++++++++++++++++++++++++++-------------- src/aiori.h | 5 +- src/ior.c | 17 ++--- src/ior.h | 6 ++ src/mdtest.c | 95 ++++++++++++++++++----- src/parse_options.c | 9 ++- 6 files changed, 224 insertions(+), 89 deletions(-) diff --git a/src/aiori-DFS.c b/src/aiori-DFS.c index 6f0c07bb..f22bbbd6 100755 --- a/src/aiori-DFS.c +++ b/src/aiori-DFS.c @@ -6,6 +6,14 @@ * Copyright (c) 2003, The Regents of the University of California * * See the file COPYRIGHT for a complete copyright notice and license. * * * +* Copyright (C) 2018 Intel Corporation +* +* GOVERNMENT LICENSE RIGHTS-OPEN SOURCE SOFTWARE +* The Government's rights to use, modify, reproduce, release, perform, display, +* or disclose this software are subject to the terms of the Apache License as +* provided in Contract No. 8F-30005. +* Any reproduction of computer software, computer software documentation, or +* portions thereof marked with this legend must also reproduce the markings. ******************************************************************************** * * Implement of abstract I/O interface for DFS. @@ -70,10 +78,11 @@ parse_filename(const char *path, char **_obj_name, char **_cont_name) cont_name = dirname(f2); if (cont_name[0] == '.' || cont_name[0] != '/') { - char *cwd; + char cwd[1024]; + + if (getcwd(cwd, 1024) == NULL) + D_GOTO(out, rc = -ENOMEM); - //getcwd(cwd, 1024); - cwd = strdup("/"); if (strcmp(cont_name, ".") == 0) { cont_name = strdup(cwd); if (cont_name == NULL) @@ -130,6 +139,8 @@ static int DFS_Stat (const char *, struct stat *, IOR_param_t *); static int DFS_Mkdir (const char *, mode_t, IOR_param_t *); static int DFS_Rmdir (const char *, IOR_param_t *); static int DFS_Access (const char *, int, IOR_param_t *); +static int DFS_Init(IOR_param_t *param); +static int DFS_Finalize(IOR_param_t *param); /************************** D E C L A R A T I O N S ***************************/ @@ -148,85 +159,122 @@ ior_aiori_t dfs_aiori = { .rmdir = DFS_Rmdir, .access = DFS_Access, .stat = DFS_Stat, + .init = DFS_Init, + .finalize = DFS_Finalize, }; /***************************** F U N C T I O N S ******************************/ -int -dfs_init(void) { - char *pool_str, *svcl_str, *group_str; +static int +DFS_Init(IOR_param_t *param) { uuid_t pool_uuid, co_uuid; daos_pool_info_t pool_info; daos_cont_info_t co_info; d_rank_list_t *svcl = NULL; + bool cont_created = false; int rc; - - rc = daos_init(); - if (rc) { - fprintf(stderr, "daos_init() failed with %d\n", rc); - return rc; - } - pool_str = getenv("DAOS_POOL"); - if (!pool_str) { - fprintf(stderr, "missing pool uuid\n"); - return -1; - } - if (uuid_parse(pool_str, pool_uuid) < 0) { + if (uuid_parse(param->daosPool, pool_uuid) < 0) { fprintf(stderr, "Invalid pool uuid\n"); return -1; } - svcl_str = getenv("DAOS_SVCL"); - if (!svcl_str) { - fprintf(stderr, "missing pool service rank list\n"); + if (uuid_parse(param->daosCont, co_uuid) < 0) { + fprintf(stderr, "Invalid pool uuid\n"); return -1; } - svcl = daos_rank_list_parse(svcl_str, ":"); + + svcl = daos_rank_list_parse(param->daosPoolSvc, ":"); if (svcl == NULL) { fprintf(stderr, "Invalid pool service rank list\n"); return -1; } - group_str = getenv("DAOS_GROUP"); + printf("Pool uuid = %s, SVCL = %s\n", param->daosPool, + param->daosPoolSvc); + + printf("DFS Container namespace uuid = %s\n", param->daosCont); + + rc = daos_init(); + if (rc) { + fprintf(stderr, "daos_init() failed with %d\n", rc); + return rc; + } /** Connect to DAOS pool */ - rc = daos_pool_connect(pool_uuid, group_str, svcl, DAOS_PC_RW, - &poh, &pool_info, NULL); + rc = daos_pool_connect(pool_uuid, + strlen(param->daosGroup) ? param->daosGroup : NULL, + svcl, DAOS_PC_RW, &poh, &pool_info, NULL); if (rc < 0) { - fprintf(stderr, "Failed to connect to pool %s %s (%d)\n", - pool_str, svcl_str, rc); - return -1; + fprintf(stderr, "Failed to connect to pool (%d)\n", rc); + goto err_daos; } - uuid_generate(co_uuid); - rc = daos_cont_create(poh, co_uuid, NULL); - if (rc) { - fprintf(stderr, "Failed to create container (%d)\n", rc); - return -1; + rc = daos_cont_open(poh, co_uuid, DAOS_COO_RW, &coh, &co_info, NULL); + /* If NOEXIST we create it */ + if (rc == -DER_NONEXIST) { + printf("Creating DFS Container ...\n"); + rc = daos_cont_create(poh, co_uuid, NULL); + if (rc == 0) { + cont_created = true; + rc = daos_cont_open(poh, co_uuid, DAOS_COO_RW, &coh, + &co_info, NULL); + } } - - rc = daos_cont_open(poh, co_uuid, DAOS_COO_RW, &coh, &co_info, NULL); if (rc) { - fprintf(stderr, "Failed to open container (%d)\n", rc); - return -1; + fprintf(stderr, "Failed to create container (%d)\n", rc); + goto err_pool; } - rc = dfs_mount(poh, coh, &dfs); + rc = dfs_mount(poh, coh, O_RDWR, &dfs); if (rc) { fprintf(stderr, "dfs_mount failed (%d)\n", rc); - return -1; + goto err_cont; } +out: + daos_rank_list_free(svcl); return rc; +err_cont: + daos_cont_close(coh, NULL); +err_pool: + if (cont_created) + daos_cont_destroy(poh, co_uuid, 1, NULL); + daos_pool_disconnect(poh, NULL); +err_daos: + daos_fini(); + goto out; } -int dfs_finalize(void) +int +DFS_Finalize(IOR_param_t *param) { - dfs_umount(dfs); - daos_cont_close(coh, NULL); + int rc; + + rc = dfs_umount(dfs, true); + if (rc) { + fprintf(stderr, "dfs_umount() failed (%d)\n", rc); + return -1; + } + + rc = daos_cont_close(coh, NULL); + if (rc) { + fprintf(stderr, "daos_cont_close() failed (%d)\n", rc); + return -1; + } + daos_pool_disconnect(poh, NULL); - daos_fini(); + if (rc) { + fprintf(stderr, "daos_pool_disconnect() failed (%d)\n", rc); + return -1; + } + + rc = daos_fini(); + if (rc) { + fprintf(stderr, "daos_fini() failed (%d)\n", rc); + return -1; + } + return 0; } @@ -238,11 +286,14 @@ DFS_Create(char *testFileName, IOR_param_t *param) { char *name = NULL, *dir_name = NULL; dfs_obj_t *obj = NULL, *parent = NULL; - mode_t pmode; + mode_t pmode, mode; int fd_oflag = 0; int rc; + assert(param); + fd_oflag |= O_CREAT | O_RDWR; + mode = S_IFREG | param->mode; rc = parse_filename(testFileName, &name, &dir_name); if (rc) @@ -255,8 +306,8 @@ DFS_Create(char *testFileName, IOR_param_t *param) if (rc || !S_ISDIR(pmode)) goto out; - mode_t mode = S_IFREG | param->mode; - rc = dfs_open(dfs, parent, name, mode, fd_oflag, NULL, &obj); + rc = dfs_open(dfs, parent, name, mode, fd_oflag, DAOS_OC_LARGE_RW, + NULL, &obj); if (rc) goto out; @@ -274,7 +325,8 @@ DFS_Create(char *testFileName, IOR_param_t *param) /* * Open a file through the DFS interface. */ -static void *DFS_Open(char *testFileName, IOR_param_t *param) +static void * +DFS_Open(char *testFileName, IOR_param_t *param) { char *name = NULL, *dir_name = NULL; dfs_obj_t *obj = NULL, *parent = NULL; @@ -295,7 +347,7 @@ static void *DFS_Open(char *testFileName, IOR_param_t *param) if (rc || !S_ISDIR(pmode)) goto out; - rc = dfs_open(dfs, parent, name, S_IFREG, fd_oflag, NULL, &obj); + rc = dfs_open(dfs, parent, name, S_IFREG, fd_oflag, 0, NULL, &obj); if (rc) goto out; @@ -369,15 +421,18 @@ DFS_Xfer(int access, void *file, IOR_size_t *buffer, IOR_offset_t length, /* * Perform fsync(). */ -static void DFS_Fsync(void *fd, IOR_param_t * param) +static void +DFS_Fsync(void *fd, IOR_param_t * param) { + dfs_sync(dfs); return; } /* * Close a file through the DFS interface. */ -static void DFS_Close(void *fd, IOR_param_t * param) +static void +DFS_Close(void *fd, IOR_param_t * param) { dfs_release((dfs_obj_t *)fd); } @@ -385,7 +440,8 @@ static void DFS_Close(void *fd, IOR_param_t * param) /* * Delete a file through the DFS interface. */ -static void DFS_Delete(char *testFileName, IOR_param_t * param) +static void +DFS_Delete(char *testFileName, IOR_param_t * param) { char *name = NULL, *dir_name = NULL; dfs_obj_t *parent = NULL; @@ -419,7 +475,8 @@ static void DFS_Delete(char *testFileName, IOR_param_t * param) /* * Determine api version. */ -static void DFS_SetVersion(IOR_param_t * test) +static void +DFS_SetVersion(IOR_param_t * test) { strcpy(test->apiVersion, test->api); } @@ -427,8 +484,8 @@ static void DFS_SetVersion(IOR_param_t * test) /* * Use DFS stat() to return aggregate file size. */ -static IOR_offset_t DFS_GetFileSize(IOR_param_t * test, MPI_Comm testComm, - char *testFileName) +static IOR_offset_t +DFS_GetFileSize(IOR_param_t * test, MPI_Comm testComm, char *testFileName) { dfs_obj_t *obj; daos_size_t fsize, tmpMin, tmpMax, tmpSum; @@ -475,7 +532,7 @@ DFS_Statfs(const char *path, ior_aiori_statfs_t *sfs, IOR_param_t * param) } static int -DFS_Mkdir (const char *path, mode_t mode, IOR_param_t * param) +DFS_Mkdir(const char *path, mode_t mode, IOR_param_t * param) { dfs_obj_t *parent = NULL; mode_t pmode; @@ -504,11 +561,13 @@ DFS_Mkdir (const char *path, mode_t mode, IOR_param_t * param) free(dir_name); if (parent) dfs_release(parent); + if (rc) + return -1; return rc; } static int -DFS_Rmdir (const char *path, IOR_param_t * param) +DFS_Rmdir(const char *path, IOR_param_t * param) { dfs_obj_t *parent = NULL; mode_t pmode; @@ -537,11 +596,13 @@ DFS_Rmdir (const char *path, IOR_param_t * param) free(dir_name); if (parent) dfs_release(parent); + if (rc) + return -1; return rc; } static int -DFS_Access (const char *path, int mode, IOR_param_t * param) +DFS_Access(const char *path, int mode, IOR_param_t * param) { dfs_obj_t *parent = NULL; mode_t pmode; @@ -578,11 +639,13 @@ DFS_Access (const char *path, int mode, IOR_param_t * param) free(dir_name); if (parent) dfs_release(parent); + if (rc) + return -1; return rc; } static int -DFS_Stat (const char *path, struct stat *buf, IOR_param_t * param) +DFS_Stat(const char *path, struct stat *buf, IOR_param_t * param) { dfs_obj_t *parent = NULL; mode_t pmode; @@ -611,5 +674,7 @@ DFS_Stat (const char *path, struct stat *buf, IOR_param_t * param) free(dir_name); if (parent) dfs_release(parent); + if (rc) + return -1; return rc; } diff --git a/src/aiori.h b/src/aiori.h index 9e5695ed..b2c4818f 100755 --- a/src/aiori.h +++ b/src/aiori.h @@ -77,6 +77,8 @@ typedef struct ior_aiori { int (*rmdir) (const char *path, IOR_param_t * param); int (*access) (const char *path, int mode, IOR_param_t * param); int (*stat) (const char *path, struct stat *buf, IOR_param_t * param); + int (*init)(IOR_param_t *); + int (*finalize)(IOR_param_t *); } ior_aiori_t; extern ior_aiori_t hdf5_aiori; @@ -94,9 +96,6 @@ const ior_aiori_t *aiori_select (const char *api); int aiori_count (void); const char *aiori_default (void); -int dfs_init(void); -int dfs_finalize(void); - IOR_offset_t MPIIO_GetFileSize(IOR_param_t * test, MPI_Comm testComm, char *testFileName); diff --git a/src/ior.c b/src/ior.c index e4dc03b9..aa64436a 100755 --- a/src/ior.c +++ b/src/ior.c @@ -2037,10 +2037,10 @@ static void TestIoSys(IOR_test_t *test) /* bind I/O calls to specific API */ AioriBind(params->api, params); -#ifdef USE_DFS_AIORI - if (strcmp(params->api, "DFS") == 0) - dfs_init(); -#endif + /* initialize API session */ + if (backend->init != NULL) + if (backend->init(params) != 0) + ERR("Could not init backend"); /* show test setup */ if (rank == 0 && verbose >= VERBOSE_0) @@ -2312,13 +2312,12 @@ static void TestIoSys(IOR_test_t *test) free(timer[i]); } + /* finalize API session */ + if (backend->finalize != NULL) + backend->finalize(params); + /* Sync with the tasks that did not participate in this test */ MPI_CHECK(MPI_Barrier(MPI_COMM_WORLD), "barrier error"); - -#ifdef USE_DFS_AIORI - if (strcmp(params->api, "DFS") == 0) - dfs_finalize(); -#endif } /* diff --git a/src/ior.h b/src/ior.h index ce1b4ecd..31b962a6 100755 --- a/src/ior.h +++ b/src/ior.h @@ -213,6 +213,12 @@ typedef struct int beegfs_numTargets; /* number storage targets to use */ int beegfs_chunkSize; /* srtipe pattern for new files */ + /* daos variables */ + char daosGroup[MAX_STR]; /* group name */ + char daosPool[37]; /* pool UUID */ + char daosPoolSvc[MAX_STR]; /* pool service ranks */ + char daosCont[37]; /* Container UUID */ + int id; /* test's unique ID */ int intraTestBarriers; /* barriers between open/op and op/close */ } IOR_param_t; diff --git a/src/mdtest.c b/src/mdtest.c index 3245c479..b95211d7 100644 --- a/src/mdtest.c +++ b/src/mdtest.c @@ -1801,6 +1801,59 @@ void create_remove_directory_tree(int create, } } +/* + * Set flags from commandline string/value pairs. + */ +static void +DecodeDirective(char *line, IOR_param_t *params) +{ + char option[MAX_STR]; + char value[MAX_STR]; + int rc; + + rc = sscanf(line, " %[^=# \t\r\n] = %[^# \t\r\n] ", option, value); + if (rc != 2 && rank == 0) { + fprintf(stdout, "Syntax error in configuration options: %s\n", + line); + MPI_CHECK(MPI_Abort(MPI_COMM_WORLD, -1), "MPI_Abort() error"); + } + + if (strcasecmp(option, "daospool") == 0) { + strcpy(params->daosPool, value); + } else if (strcasecmp(option, "daospoolsvc") == 0) { + strcpy(params->daosPoolSvc, value); + } else if (strcasecmp(option, "daosgroup") == 0) { + strcpy(params->daosGroup, value); + } else if (strcasecmp(option, "daoscont") == 0) { + strcpy(params->daosCont, value); + } + else { + if (rank == 0) + fprintf(stdout, "Unrecognized parameter \"%s\"\n", + option); + MPI_CHECK(MPI_Abort(MPI_COMM_WORLD, -1), "MPI_Abort() error"); + } +} + +/* + * Parse a single line, which may contain multiple comma-seperated directives + */ +static void +ParseLine(char *line, IOR_param_t * test) +{ + char *start, *end; + + start = line; + do { + end = strchr(start, ','); + if (end != NULL) + *end = '\0'; + DecodeDirective(start, test); + start = end + 1; + } while (end != NULL); + +} + int main(int argc, char **argv) { int i, j, k, c; int nodeCount; @@ -1853,7 +1906,7 @@ int main(int argc, char **argv) { /* Parse command line options */ while (1) { - c = getopt(argc, argv, "a:b:BcCd:De:Ef:Fhi:I:l:Ln:N:p:rR::s:StTuvV:w:yz:"); + c = getopt(argc, argv, "a:b:BcCd:De:Ef:Fhi:I:l:Ln:N:O:p:rR::s:StTuvV:w:yz:"); if (c == -1) { break; } @@ -1898,6 +1951,9 @@ int main(int argc, char **argv) { //items = atoi(optarg); break; case 'N': nstride = atoi(optarg); break; + case 'O': + ParseLine(optarg, ¶m); + break; case 'p': pre_delay = atoi(optarg); break; case 'r': @@ -1936,11 +1992,6 @@ int main(int argc, char **argv) { } } -#ifdef USE_DFS_AIORI - if (strcmp(backend_name, "DFS") == 0) - dfs_init(); -#endif - if (!create_only && !stat_only && !read_only && !remove_only) { create_only = stat_only = read_only = remove_only = 1; if (( rank == 0 ) && ( verbose >= 1 )) { @@ -2082,6 +2133,11 @@ int main(int argc, char **argv) { FAIL("Could not find suitable backend to use"); } + /* initialize API session */ + if (backend->init != NULL) + if (backend->init(¶m) != 0) + FAIL("Could not init backend"); + /* if directory does not exist, create it */ if ((rank < path_count) && backend->access(testdirpath, F_OK, ¶m) != 0) { if (backend->mkdir(testdirpath, DIRMODE, ¶m) != 0) { @@ -2090,16 +2146,20 @@ int main(int argc, char **argv) { } /* display disk usage */ - if (verbose >= 3 && rank == 0) { - printf( "V-3: main (before display_freespace): testdirpath is \"%s\"\n", testdirpath ); - fflush( stdout ); - } + if (strcmp(backend->name, "DFS")) { + if (verbose >= 3 && rank == 0) { + printf( "V-3: main (before display_freespace): testdirpath is \"%s\"\n", + testdirpath ); + fflush( stdout ); + } - if (rank == 0) display_freespace(testdirpath); + if (rank == 0) display_freespace(testdirpath); - if (verbose >= 3 && rank == 0) { - printf( "V-3: main (after display_freespace): testdirpath is \"%s\"\n", testdirpath ); - fflush( stdout ); + if (verbose >= 3 && rank == 0) { + printf( "V-3: main (after display_freespace): testdirpath is \"%s\"\n", + testdirpath ); + fflush( stdout ); + } } if (rank == 0) { @@ -2416,10 +2476,9 @@ int main(int argc, char **argv) { free(rand_array); } -#ifdef USE_DFS_AIORI - if (strcmp(backend_name, "DFS") == 0) - dfs_finalize(); -#endif + /* finalize API session */ + if (backend->finalize != NULL) + backend->finalize(¶m); MPI_Finalize(); exit(0); diff --git a/src/parse_options.c b/src/parse_options.c index cfa388a4..11265595 100755 --- a/src/parse_options.c +++ b/src/parse_options.c @@ -327,7 +327,14 @@ void DecodeDirective(char *line, IOR_param_t *params) RecalculateExpectedFileSize(params); } else if (strcasecmp(option, "summaryalways") == 0) { params->summary_every_test = atoi(value); - } else { + } else if (strcasecmp(option, "daospool") == 0) { + strcpy(params->daosPool, value); + } else if (strcasecmp(option, "daospoolsvc") == 0) { + strcpy(params->daosPoolSvc, value); + } else if (strcasecmp(option, "daosgroup") == 0) { + strcpy(params->daosGroup, value); + } + else { if (rank == 0) fprintf(stdout, "Unrecognized parameter \"%s\"\n", option); From f4b03efd72dee9ecbead5a2d39baf8d0bbecdb4d Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Mon, 27 Aug 2018 17:22:38 +0000 Subject: [PATCH 005/206] Add the DAOS ior driver. Signed-off-by: Mohamad Chaarawi --- .gitignore | 9 + README_DAOS | 48 +++ configure.ac | 10 + doc/USER_GUIDE | 37 ++ src/Makefile.am | 5 + src/aiori-DAOS.c | 889 ++++++++++++++++++++++++++++++++++++++++++++ src/aiori.c | 11 +- src/aiori.h | 9 +- src/ior.c | 22 +- src/ior.h | 15 + src/list.h | 556 +++++++++++++++++++++++++++ src/mdtest-main.c | 4 +- src/parse_options.c | 36 +- 13 files changed, 1631 insertions(+), 20 deletions(-) create mode 100644 README_DAOS create mode 100644 src/aiori-DAOS.c create mode 100644 src/list.h diff --git a/.gitignore b/.gitignore index 73dd929e..327640c5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +tags Makefile Makefile.in aclocal.m4 @@ -9,10 +10,12 @@ config/config.sub config/depcomp config/install-sh config/missing +config/test-driver configure contrib/.deps/ contrib/Makefile contrib/Makefile.in +contrib/cbif doc/Makefile doc/Makefile.in src/.deps/ @@ -28,7 +31,13 @@ contrib/cbif.o src/*.o src/*.i src/*.s +src/*.a src/ior +src/mdtest +src/testlib +src/test/.deps/ +src/test/.dirstamp +src/test/lib.o doc/doxygen/build doc/sphinx/_*/ diff --git a/README_DAOS b/README_DAOS new file mode 100644 index 00000000..ba4fc3e5 --- /dev/null +++ b/README_DAOS @@ -0,0 +1,48 @@ +Building with DAOS API +---------------------- + +At step 1 above, one must specify "--with-daos". If the DAOS +headers and libraries are not installed at respective system +default locations, then one may also needs to set CPPFLAGS and +LDFLAGS accordingly. + +Running with DAOS API +--------------------- + +One must specify an existing pool using "-O +daospool=". IOR must be launched in a way that +attaches the IOR process group to the DAOS server process group. + +One must also specify a container UUID using "-o +". If the "-E" option is given, then this UUID +shall denote an existing container created by a "matching" IOR +run. Otherwise, IOR will create a new container with this UUID. +In the latter case, one may use uuidgen(1) to generate the UUID +of the new container. + +When benchmarking write performance, one likely do not want +"-W", which causes the write phase to do one additional memory +copy for every I/O. This is due to IOR's assumption that when a +DAOS_Xfer() call returns the buffer may be released. Therefore, +random data is written when "-W" is absent, while data is copied +from IOR buffers when "-W" is present. + +See doc/USER_GUIDE for all options and directives. Note that not +all combinations of options are supported. + +Examples that should work include: + + - "ior -a DAOS -w -W -o -O + daospool=,daospoolsvc=" writes into a new container + and verifies the data, using default daosRecordSize, transferSize, + daosStripeSize, blockSize, daosAios, etc. + + - "ior -a DAOS -w -W -r -R -o -b 1g -t 4m -C -O + daospool=,daospoolsvc=,daosrecordsize=1m, + daosstripesize=4m, daosstripecount=256,daosaios=8" does all IOR tests and + shifts ranks during checkWrite and checkRead. + + - "ior -a DAOS -w -r -o -b 8g -t 1m -C -O + daospool=,daospoolsvc=,daosrecordsize=1m,daosstripesize=4m, + daosstripecount=256,daosaios=8" may be a base to be tuned for performance + benchmarking. diff --git a/configure.ac b/configure.ac index 70423551..bb643d9e 100755 --- a/configure.ac +++ b/configure.ac @@ -166,6 +166,16 @@ AM_COND_IF([USE_RADOS_AIORI],[ AC_DEFINE([USE_RADOS_AIORI], [], [Build RADOS backend AIORI]) ]) +# DAOS support +AC_ARG_WITH([daos], + [AS_HELP_STRING([--with-daos], + [support IO with DAOS backend @<:@default=no@:>@])], + [], + [with_daos=no]) +AM_CONDITIONAL([USE_DAOS_AIORI], [test x$with_daos = xyes]) +AM_COND_IF([USE_DAOS_AIORI],[ + AC_DEFINE([USE_DAOS_AIORI], [], [Build DAOS backend AIORI]) +]) # aws4c is needed for the S3 backend (see --with-S3, below). diff --git a/doc/USER_GUIDE b/doc/USER_GUIDE index 7ea6e49a..d76266d7 100755 --- a/doc/USER_GUIDE +++ b/doc/USER_GUIDE @@ -367,6 +367,43 @@ BeeGFS-SPECIFIC (POSIX only): * beegfsChunkSize - set the striping chunk size. Must be a power of two, and greater than 64kiB, (e.g.: 256k, 1M, ...) +DAOS-ONLY: +========== + * daosGroup - group name [NULL] + + * daosPool - UUID of the pool [] + + * daosPoolSvc - pool service replica ranks (e.g., 1:2:3:4:5) [] + + * daosRecordSize - size (in bytes) of an akey record [256k] + NOTE: must divide transferSize + + * daosStripeSize - size (in bytes) of a chunk in a stripe [512k] + NOTE: must be a multiple of transferSize + + * daosStripeCount - number of stripes [64 * number of targets] + NOTE: i.e., number of dkeys + + * daosStripeMax - max length of each stripe [0] + NOTE: must be a multiple of daosStripeSize + NOTE: for write testing with small storage + NOTE: offsets in a stripe larger than daosStripeMax + are mapped to offset % daosStripeMax + + * daosAios - max number of asychonous I/Os [1] + + * daosWriteOnly - skip flushing and committing [0=FALSE] + + * daosEpoch - epoch to read or write [0] + NOTE: 0 denotes reading GHCE or writing GHCE + 1 + + * daosWait - epoch to wait when opening the container [0] + + * daosKill - kill a target in the middle of the test [0] + NOTE: must also specify daosObjectClass=repl + + * daosObjectClass - object class (tiny, small, large, repl, repl_max) + [large] *********************** * 5. VERBOSITY LEVELS * diff --git a/src/Makefile.am b/src/Makefile.am index 0e0b9164..32db201f 100755 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -70,6 +70,11 @@ extraSOURCES += aiori-RADOS.c extraLDADD += -lrados endif +if USE_DAOS_AIORI +extraSOURCES += aiori-DAOS.c list.h +extraLDADD += -ldaos -ldaos_common -luuid +endif + if USE_S3_AIORI extraSOURCES += aiori-S3.c if AWS4C_DIR diff --git a/src/aiori-DAOS.c b/src/aiori-DAOS.c new file mode 100644 index 00000000..9175a5a0 --- /dev/null +++ b/src/aiori-DAOS.c @@ -0,0 +1,889 @@ +/* + * -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- + * vim:expandtab:shiftwidth=8:tabstop=8: + */ +/* + * SPECIAL LICENSE RIGHTS-OPEN SOURCE SOFTWARE + * The Government's rights to use, modify, reproduce, release, perform, display, + * or disclose this software are subject to the terms of Contract No. B599860, + * and the terms of the GNU General Public License version 2. + * Any reproduction of computer software, computer software documentation, or + * portions thereof marked with this legend must also reproduce the markings. + */ +/* + * Copyright (c) 2013, 2016 Intel Corporation. + */ +/* + * This file implements the abstract I/O interface for DAOS. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ior.h" +#include "aiori.h" +#include "iordef.h" +#include "list.h" + +/**************************** P R O T O T Y P E S *****************************/ + +static void DAOS_Init(IOR_param_t *); +static void DAOS_Fini(IOR_param_t *); +static void *DAOS_Create(char *, IOR_param_t *); +static void *DAOS_Open(char *, IOR_param_t *); +static IOR_offset_t DAOS_Xfer(int, void *, IOR_size_t *, + IOR_offset_t, IOR_param_t *); +static void DAOS_Close(void *, IOR_param_t *); +static void DAOS_Delete(char *, IOR_param_t *); +static char* DAOS_GetVersion(); +static void DAOS_Fsync(void *, IOR_param_t *); +static IOR_offset_t DAOS_GetFileSize(IOR_param_t *, MPI_Comm, char *); + +/************************** D E C L A R A T I O N S ***************************/ + +ior_aiori_t daos_aiori = { + .name = "DAOS", + .create = DAOS_Create, + .open = DAOS_Open, + .xfer = DAOS_Xfer, + .close = DAOS_Close, + .delete = DAOS_Delete, + .get_version = DAOS_GetVersion, + .fsync = DAOS_Fsync, + .get_file_size = DAOS_GetFileSize, + .initialize = DAOS_Init, + .finalize = DAOS_Fini, +}; + +enum handleType { + POOL_HANDLE, + CONTAINER_HANDLE +}; + +struct fileDescriptor { + daos_handle_t container; + daos_cont_info_t containerInfo; + daos_handle_t object; + daos_epoch_t epoch; +}; + +struct aio { + cfs_list_t a_list; + char a_dkeyBuf[32]; + daos_key_t a_dkey; + daos_recx_t a_recx; + unsigned char a_csumBuf[32]; + daos_csum_buf_t a_csum; + daos_epoch_range_t a_epochRange; + daos_iod_t a_iod; + daos_iov_t a_iov; + daos_sg_list_t a_sgl; + struct daos_event a_event; +}; + +static daos_handle_t eventQueue; +static struct daos_event **events; +static unsigned char *buffers; +static int nAios; +static daos_handle_t pool; +static daos_pool_info_t poolInfo; +static daos_oclass_id_t objectClass = DAOS_OC_LARGE_RW; +static CFS_LIST_HEAD(aios); +static IOR_offset_t total_size; + +/***************************** F U N C T I O N S ******************************/ + +/* For DAOS methods. */ +#define DCHECK(rc, format, ...) \ +do { \ + int _rc = (rc); \ + \ + if (_rc < 0) { \ + fprintf(stdout, "ior ERROR (%s:%d): %d: %d: " \ + format"\n", __FILE__, __LINE__, rank, _rc, \ + ##__VA_ARGS__); \ + fflush(stdout); \ + MPI_Abort(MPI_COMM_WORLD, -1); \ + } \ +} while (0) + +#define INFO(level, param, format, ...) \ +do { \ + if (param->verbose >= level) \ + printf("[%d] "format"\n", rank, ##__VA_ARGS__); \ +} while (0) + +/* For generic errors like invalid command line options. */ +#define GERR(format, ...) \ +do { \ + fprintf(stdout, format"\n", ##__VA_ARGS__); \ + MPI_CHECK(MPI_Abort(MPI_COMM_WORLD, -1), "MPI_Abort() error"); \ +} while (0) + +/* Distribute process 0's pool or container handle to others. */ +static void HandleDistribute(daos_handle_t *handle, enum handleType type, + IOR_param_t *param) +{ + daos_iov_t global; + int rc; + + assert(type == POOL_HANDLE || !daos_handle_is_inval(pool)); + + global.iov_buf = NULL; + global.iov_buf_len = 0; + global.iov_len = 0; + + if (rank == 0) { + /* Get the global handle size. */ + if (type == POOL_HANDLE) + rc = daos_pool_local2global(*handle, &global); + else + rc = daos_cont_local2global(*handle, &global); + DCHECK(rc, "Failed to get global handle size"); + } + + MPI_CHECK(MPI_Bcast(&global.iov_buf_len, 1, MPI_UINT64_T, 0, + param->testComm), + "Failed to bcast global handle buffer size"); + + global.iov_buf = malloc(global.iov_buf_len); + if (global.iov_buf == NULL) + ERR("Failed to allocate global handle buffer"); + + if (rank == 0) { + if (type == POOL_HANDLE) + rc = daos_pool_local2global(*handle, &global); + else + rc = daos_cont_local2global(*handle, &global); + DCHECK(rc, "Failed to create global handle"); + } + + MPI_CHECK(MPI_Bcast(global.iov_buf, global.iov_buf_len, MPI_BYTE, 0, + param->testComm), + "Failed to bcast global pool handle"); + + if (rank != 0) { + /* A larger-than-actual length works just fine. */ + global.iov_len = global.iov_buf_len; + + if (type == POOL_HANDLE) + rc = daos_pool_global2local(global, handle); + else + rc = daos_cont_global2local(pool, global, handle); + DCHECK(rc, "Failed to get local handle"); + } + + free(global.iov_buf); +} + +static void ContainerOpen(char *testFileName, IOR_param_t *param, + daos_handle_t *container, daos_cont_info_t *info) +{ + int rc; + + if (rank == 0) { + uuid_t uuid; + unsigned int dFlags; + + rc = uuid_parse(testFileName, uuid); + DCHECK(rc, "Failed to parse 'testFile': %s", testFileName); + + if (param->open == WRITE && + param->useExistingTestFile == FALSE) { + INFO(VERBOSE_2, param, "Creating container %s", + testFileName); + + rc = daos_cont_create(pool, uuid, NULL /* ev */); + DCHECK(rc, "Failed to create container %s", + testFileName); + } + + INFO(VERBOSE_2, param, "Openning container %s", testFileName); + + if (param->open == WRITE) + dFlags = DAOS_COO_RW; + else + dFlags = DAOS_COO_RO; + + rc = daos_cont_open(pool, uuid, dFlags, container, info, + NULL /* ev */); + DCHECK(rc, "Failed to open container %s", testFileName); + + INFO(VERBOSE_2, param, "Container epoch state:"); + INFO(VERBOSE_2, param, " HCE: %lu", + info->ci_epoch_state.es_hce); + INFO(VERBOSE_2, param, " LRE: %lu", + info->ci_epoch_state.es_lre); + INFO(VERBOSE_2, param, " LHE: %lu (%lx)", + info->ci_epoch_state.es_lhe, info->ci_epoch_state.es_lhe); + INFO(VERBOSE_2, param, " GHCE: %lu", + info->ci_epoch_state.es_ghce); + INFO(VERBOSE_2, param, " GLRE: %lu", + info->ci_epoch_state.es_glre); + INFO(VERBOSE_2, param, " GHPCE: %lu", + info->ci_epoch_state.es_ghpce); + +#if 0 + if (param->open != WRITE && param->daosWait != 0) { + daos_epoch_t e; + + e = param->daosWait; + + INFO(VERBOSE_2, param, "Waiting for epoch %lu", e); + + rc = daos_epoch_wait(*container, &e, + NULL /* ignore HLE */, + NULL /* synchronous */); + DCHECK(rc, "Failed to wait for epoch %lu", + param->daosWait); + } + + if (param->open == WRITE && + param->useExistingTestFile == FALSE) { + daos_oclass_attr_t attr = { + .ca_schema = DAOS_OS_STRIPED, + .ca_resil_degree = 0, + .ca_resil = DAOS_RES_REPL, + .ca_grp_nr = 4, + .u.repl = { + .r_method = 0, + .r_num = 2 + } + }; + + INFO(VERBOSE_2, param, "Registering object class"); + + rc = daos_oclass_register(container, objectClass, &attr, + NULL /* ev */); + DCHECK(rc, "Failed to register object class"); + } +#endif + } + + HandleDistribute(container, CONTAINER_HANDLE, param); + + MPI_CHECK(MPI_Bcast(info, sizeof *info, MPI_BYTE, 0, param->testComm), + "Failed to broadcast container info"); +} + +static void ContainerClose(daos_handle_t container, IOR_param_t *param) +{ + int rc; + + if (rank != 0) { + rc = daos_cont_close(container, NULL /* ev */); + DCHECK(rc, "Failed to close container"); + } + + /* An MPI_Gather() call would probably be more efficient. */ + MPI_CHECK(MPI_Barrier(param->testComm), + "Failed to synchronize processes"); + + if (rank == 0) { + rc = daos_cont_close(container, NULL /* ev */); + DCHECK(rc, "Failed to close container"); + } +} + +static void ObjectOpen(daos_handle_t container, daos_handle_t *object, + daos_epoch_t epoch, IOR_param_t *param) +{ + daos_obj_id_t oid; + unsigned int flags; + int rc; + + oid.hi = 0; + oid.lo = 1; + daos_obj_id_generate(&oid, 0, objectClass); + +#if 0 + /** declaring object not implemented commenting it */ + if (rank == 0 && param->open == WRITE && + param->useExistingTestFile == FALSE) { + INFO(VERBOSE_2, param, "Declaring object"); + + rc = daos_obj_declare(container, oid, epoch, NULL /* oa */, + NULL /* ev */); + DCHECK(rc, "Failed to declare object"); + } +#endif + /* An MPI_Bcast() call would probably be more efficient. */ + MPI_CHECK(MPI_Barrier(param->testComm), + "Failed to synchronize processes"); + + if (param->open == WRITE) + flags = DAOS_OO_RW; + else + flags = DAOS_OO_RO; + + rc = daos_obj_open(container, oid, epoch, flags, object, NULL /* ev */); + DCHECK(rc, "Failed to open object"); +} + +static void ObjectClose(daos_handle_t object) +{ + int rc; + + rc = daos_obj_close(object, NULL /* ev */); + DCHECK(rc, "Failed to close object"); +} + +static void AIOInit(IOR_param_t *param) +{ + struct aio *aio; + int i; + int rc; + + rc = posix_memalign((void **) &buffers, sysconf(_SC_PAGESIZE), + param->transferSize * param->daosAios); + DCHECK(rc, "Failed to allocate buffer array"); + + for (i = 0; i < param->daosAios; i++) { + aio = malloc(sizeof *aio); + if (aio == NULL) + ERR("Failed to allocate aio array"); + + memset(aio, 0, sizeof *aio); + + aio->a_dkey.iov_buf = aio->a_dkeyBuf; + aio->a_dkey.iov_buf_len = sizeof aio->a_dkeyBuf; + + aio->a_recx.rx_nr = 1; + + aio->a_csum.cs_csum = &aio->a_csumBuf; + aio->a_csum.cs_buf_len = sizeof aio->a_csumBuf; + aio->a_csum.cs_len = aio->a_csum.cs_buf_len; + + aio->a_epochRange.epr_hi = DAOS_EPOCH_MAX; + + aio->a_iod.iod_name.iov_buf = "data"; + aio->a_iod.iod_name.iov_buf_len = + strlen(aio->a_iod.iod_name.iov_buf) + 1; + aio->a_iod.iod_name.iov_len = aio->a_iod.iod_name.iov_buf_len; + aio->a_iod.iod_nr = 1; + aio->a_iod.iod_type = DAOS_IOD_ARRAY; + aio->a_iod.iod_recxs = &aio->a_recx; + aio->a_iod.iod_csums = &aio->a_csum; + aio->a_iod.iod_eprs = &aio->a_epochRange; + aio->a_iod.iod_size = param->transferSize; + + aio->a_iov.iov_buf = buffers + param->transferSize * i; + aio->a_iov.iov_buf_len = param->transferSize; + aio->a_iov.iov_len = aio->a_iov.iov_buf_len; + + aio->a_sgl.sg_nr = 1; + aio->a_sgl.sg_iovs = &aio->a_iov; + + rc = daos_event_init(&aio->a_event, eventQueue, + NULL /* parent */); + DCHECK(rc, "Failed to initialize event for aio[%d]", i); + + cfs_list_add(&aio->a_list, &aios); + + INFO(VERBOSE_3, param, "Allocated AIO %p: buffer %p", aio, + aio->a_iov.iov_buf); + } + + nAios = param->daosAios; + + events = malloc((sizeof *events) * param->daosAios); + if (events == NULL) + ERR("Failed to allocate events array"); +} + +static void AIOFini(IOR_param_t *param) +{ + struct aio *aio; + struct aio *tmp; + + free(events); + + cfs_list_for_each_entry_safe(aio, tmp, &aios, a_list) { + INFO(VERBOSE_3, param, "Freeing AIO %p: buffer %p", aio, + aio->a_iov.iov_buf); + cfs_list_del_init(&aio->a_list); + daos_event_fini(&aio->a_event); + free(aio); + } + + free(buffers); +} + +static void AIOWait(IOR_param_t *param) +{ + struct aio *aio; + int i; + int rc; + + rc = daos_eq_poll(eventQueue, 0, DAOS_EQ_WAIT, param->daosAios, + events); + DCHECK(rc, "Failed to poll event queue"); + assert(rc <= param->daosAios - nAios); + + for (i = 0; i < rc; i++) { + int ret; + + aio = (struct aio *) + ((char *) events[i] - + (char *) (&((struct aio *) 0)->a_event)); + + DCHECK(aio->a_event.ev_error, "Failed to transfer (%lu, %lu)", + aio->a_iod.iod_recxs->rx_idx, + aio->a_iod.iod_recxs->rx_nr); + + daos_event_fini(&aio->a_event); + ret = daos_event_init(&aio->a_event, eventQueue, + NULL /* parent */); + DCHECK(ret, "Failed to reinitialize event for AIO %p", aio); + + cfs_list_move(&aio->a_list, &aios); + nAios++; + + if (param->verbose >= VERBOSE_3) + INFO(VERBOSE_3, param, "Completed AIO %p: buffer %p", aio, + aio->a_iov.iov_buf); + } + + INFO(VERBOSE_3, param, "Found %d completed AIOs (%d free %d busy)", rc, + nAios, param->daosAios - nAios); +} + +static void ObjectClassParse(const char *string) +{ + if (strcasecmp(string, "tiny") == 0) + objectClass = DAOS_OC_TINY_RW; + else if (strcasecmp(string, "small") == 0) + objectClass = DAOS_OC_SMALL_RW; + else if (strcasecmp(string, "large") == 0) + objectClass = DAOS_OC_LARGE_RW; + else if (strcasecmp(string, "echo") == 0) + objectClass = DAOS_OC_ECHO_RW; + else if (strcasecmp(string, "R2") == 0) + objectClass = DAOS_OC_R2_RW; + else if (strcasecmp(string, "R2S") == 0) + objectClass = DAOS_OC_R2S_RW; + else if (strcasecmp(string, "R3S") == 0) + objectClass = DAOS_OC_R3S_RW; + else if (strcasecmp(string, "R3") == 0) + objectClass = DAOS_OC_R3_RW; + else if (strcasecmp(string, "R4") == 0) + objectClass = DAOS_OC_R4_RW; + else if (strcasecmp(string, "R4S") == 0) + objectClass = DAOS_OC_R4S_RW; + else if (strcasecmp(string, "repl_max") == 0) + objectClass = DAOS_OC_REPL_MAX_RW; + else + GERR("Invalid 'daosObjectClass' argument: '%s'", string); +} + +static const char *GetGroup(IOR_param_t *param) +{ + if (strlen(param->daosGroup) == 0) + return NULL; + return param->daosGroup; +} + +static void ParseService(IOR_param_t *param, int max, d_rank_list_t *ranks) +{ + char *s; + + s = strdup(param->daosPoolSvc); + if (s == NULL) + GERR("failed to duplicate argument"); + ranks->rl_nr = 0; + while ((s = strtok(s, ":")) != NULL) { + if (ranks->rl_nr >= max) { + free(s); + GERR("at most %d pool service replicas supported", max); + } + ranks->rl_ranks[ranks->rl_nr] = atoi(s); + ranks->rl_nr++; + s = NULL; + } + free(s); +} + +static void DAOS_Init(IOR_param_t *param) +{ + int rc; + + if (strlen(param->daosObjectClass) != 0) + ObjectClassParse(param->daosObjectClass); + + if (param->filePerProc) + GERR("'filePerProc' not yet supported"); + if (param->daosStripeMax % param->daosStripeSize != 0) + GERR("'daosStripeMax' must be a multiple of 'daosStripeSize'"); + if (param->daosStripeSize % param->transferSize != 0) + GERR("'daosStripeSize' must be a multiple of 'transferSize'"); + if (param->transferSize % param->daosRecordSize != 0) + GERR("'transferSize' must be a multiple of 'daosRecordSize'"); + if (param->daosKill && ((objectClass != DAOS_OC_R2_RW) || + (objectClass != DAOS_OC_R3_RW) || + (objectClass != DAOS_OC_R4_RW) || + (objectClass != DAOS_OC_R2S_RW) || + (objectClass != DAOS_OC_R3S_RW) || + (objectClass != DAOS_OC_R4S_RW) || + (objectClass != DAOS_OC_REPL_MAX_RW))) + GERR("'daosKill' only makes sense with 'daosObjectClass=repl'"); + + if (rank == 0) + INFO(VERBOSE_0, param, "WARNING: USING daosStripeMax CAUSES READS TO RETURN INVALID DATA"); + + rc = daos_init(); + DCHECK(rc, "Failed to initialize daos"); + + rc = daos_eq_create(&eventQueue); + DCHECK(rc, "Failed to create event queue"); + + if (rank == 0) { + uuid_t uuid; + d_rank_t rank[13]; + d_rank_list_t ranks; + + if (strlen(param->daosPool) == 0) + GERR("'daosPool' must be specified"); + if (strlen(param->daosPoolSvc) == 0) + GERR("'daosPoolSvc' must be specified"); + + INFO(VERBOSE_2, param, "Connecting to pool %s %s", + param->daosPool, param->daosPoolSvc); + + rc = uuid_parse(param->daosPool, uuid); + DCHECK(rc, "Failed to parse 'daosPool': %s", param->daosPool); + ranks.rl_ranks = rank; + ParseService(param, sizeof(rank) / sizeof(rank[0]), &ranks); + + rc = daos_pool_connect(uuid, GetGroup(param), &ranks, + DAOS_PC_RW, &pool, &poolInfo, + NULL /* ev */); + DCHECK(rc, "Failed to connect to pool %s", param->daosPool); + } + + HandleDistribute(&pool, POOL_HANDLE, param); + + MPI_CHECK(MPI_Bcast(&poolInfo, sizeof poolInfo, MPI_BYTE, 0, + param->testComm), + "Failed to bcast pool info"); + + if (param->daosStripeCount == -1) + param->daosStripeCount = poolInfo.pi_ntargets * 64UL; +} + +static void DAOS_Fini(IOR_param_t *param) +{ + int rc; + + rc = daos_pool_disconnect(pool, NULL /* ev */); + DCHECK(rc, "Failed to disconnect from pool %s", param->daosPool); + + rc = daos_eq_destroy(eventQueue, 0 /* flags */); + DCHECK(rc, "Failed to destroy event queue"); + + rc = daos_fini(); + DCHECK(rc, "Failed to finalize daos"); +} + +static void *DAOS_Create(char *testFileName, IOR_param_t *param) +{ + return DAOS_Open(testFileName, param); +} + +static void *DAOS_Open(char *testFileName, IOR_param_t *param) +{ + struct fileDescriptor *fd; + daos_epoch_t ghce; + + fd = malloc(sizeof *fd); + if (fd == NULL) + ERR("Failed to allocate fd"); + + ContainerOpen(testFileName, param, &fd->container, &fd->containerInfo); + + ghce = fd->containerInfo.ci_epoch_state.es_ghce; + if (param->open == WRITE) { + if (param->daosEpoch == 0) + fd->epoch = ghce + 1; + else if (param->daosEpoch <= ghce) + GERR("Can't modify committed epoch\n"); + else + fd->epoch = param->daosEpoch; + } else { + if (param->daosEpoch == 0) { + if (param->daosWait == 0) + fd->epoch = ghce; + else + fd->epoch = param->daosWait; + } else if (param->daosEpoch > ghce) { + GERR("Can't read uncommitted epoch\n"); + } else { + fd->epoch = param->daosEpoch; + } + } + + if (rank == 0) + INFO(VERBOSE_2, param, "Accessing epoch %lu", fd->epoch); + + if (rank == 0 && param->open == WRITE) { + daos_epoch_t e = fd->epoch; + int rc; + + INFO(VERBOSE_2, param, "Holding epoch %lu", fd->epoch); + + rc = daos_epoch_hold(fd->container, &fd->epoch, + NULL /* state */, NULL /* ev */); + DCHECK(rc, "Failed to hold epoch"); + assert(fd->epoch == e); + } + + ObjectOpen(fd->container, &fd->object, fd->epoch, param); + + AIOInit(param); + + return fd; +} + +static void +kill_daos_server(IOR_param_t *param) +{ + daos_pool_info_t info; + d_rank_t rank, svc_ranks[13]; + d_rank_list_t svc, targets; + uuid_t uuid; + char *s; + int rc; + + rc = daos_pool_query(pool, NULL, &info, NULL); + DCHECK(rc, "Error in querying pool\n"); + + if (info.pi_ntargets - info.pi_ndisabled <= 1) + return; + /* choose the last alive one */ + rank = info.pi_ntargets - 1 - info.pi_ndisabled; + + rc = uuid_parse(param->daosPool, uuid); + DCHECK(rc, "Failed to parse 'daosPool': %s", param->daosPool); + + if (rc != 0) + printf("Killing tgt rank: %d (total of %d of %d already disabled)\n", + rank, info.pi_ndisabled, info.pi_ntargets); + fflush(stdout); + + rc = daos_mgmt_svc_rip(GetGroup(param), rank, true, NULL); + DCHECK(rc, "Error in killing server\n"); + + targets.rl_nr = 1; + targets.rl_ranks = &rank; + + svc.rl_ranks = svc_ranks; + ParseService(param, sizeof(svc_ranks)/ sizeof(svc_ranks[0]), &svc); + + rc = daos_pool_exclude(uuid, NULL, &svc, &targets, NULL); + DCHECK(rc, "Error in excluding pool from poolmap\n"); + + rc = daos_pool_query(pool, NULL, &info, NULL); + DCHECK(rc, "Error in querying pool\n"); + + printf("%d targets succesfully disabled\n", + info.pi_ndisabled); + +} + +static void +kill_and_sync(IOR_param_t *param) +{ + double start, end; + + start = MPI_Wtime(); + if (rank == 0) + kill_daos_server(param); + + if (rank == 0) + printf("Done killing and excluding\n"); + + MPI_CHECK(MPI_Barrier(param->testComm), + "Failed to synchronize processes"); + + end = MPI_Wtime(); + if (rank == 0) + printf("Time spent inducing failure: %lf\n", (end - start)); +} + +static IOR_offset_t DAOS_Xfer(int access, void *file, IOR_size_t *buffer, + IOR_offset_t length, IOR_param_t *param) +{ + struct fileDescriptor *fd = file; + struct aio *aio; + uint64_t stripe; + IOR_offset_t stripeOffset; + uint64_t round; + int rc; + + assert(length == param->transferSize); + assert(param->offset % length == 0); + + /** + * Currently killing only during writes + * Kills once when 1/2 of blocksize is + * written + **/ + total_size += length; + if (param->daosKill && (access == WRITE) && + ((param->blockSize)/2) == total_size) { + /** More than half written lets kill */ + if (rank == 0) + printf("Killing and Syncing\n", rank); + kill_and_sync(param); + param->daosKill = 0; + } + + /* + * Find an available AIO descriptor. If none, wait for one. + */ + while (nAios == 0) + AIOWait(param); + aio = cfs_list_entry(aios.next, struct aio, a_list); + cfs_list_move_tail(&aio->a_list, &aios); + nAios--; + + stripe = (param->offset / param->daosStripeSize) % + param->daosStripeCount; + rc = snprintf(aio->a_dkeyBuf, sizeof aio->a_dkeyBuf, "%lu", stripe); + assert(rc < sizeof aio->a_dkeyBuf); + aio->a_dkey.iov_len = strlen(aio->a_dkeyBuf) + 1; + round = param->offset / (param->daosStripeSize * param->daosStripeCount); + stripeOffset = param->daosStripeSize * round + + param->offset % param->daosStripeSize; + if (param->daosStripeMax != 0) + stripeOffset %= param->daosStripeMax; + aio->a_recx.rx_idx = stripeOffset / param->daosRecordSize; + aio->a_epochRange.epr_lo = fd->epoch; + + /* + * If the data written will be checked later, we have to copy in valid + * data instead of writing random bytes. If the data being read is for + * checking purposes, poison the buffer first. + */ + if (access == WRITE && param->checkWrite) + memcpy(aio->a_iov.iov_buf, buffer, length); + else if (access == WRITECHECK || access == READCHECK) + memset(aio->a_iov.iov_buf, '#', length); + + INFO(VERBOSE_3, param, "Starting AIO %p (%d free %d busy): access %d " + "dkey '%s' iod <%llu, %llu> sgl <%p, %lu>", aio, nAios, + param->daosAios - nAios, access, (char *) aio->a_dkey.iov_buf, + (unsigned long long) aio->a_iod.iod_recxs->rx_idx, + (unsigned long long) aio->a_iod.iod_recxs->rx_nr, + aio->a_sgl.sg_iovs->iov_buf, + (unsigned long long) aio->a_sgl.sg_iovs->iov_buf_len); + + if (access == WRITE) { + rc = daos_obj_update(fd->object, fd->epoch, &aio->a_dkey, + 1 /* nr */, &aio->a_iod, &aio->a_sgl, + &aio->a_event); + DCHECK(rc, "Failed to start update operation"); + } else { + rc = daos_obj_fetch(fd->object, fd->epoch, &aio->a_dkey, + 1 /* nr */, &aio->a_iod, &aio->a_sgl, + NULL /* maps */, &aio->a_event); + DCHECK(rc, "Failed to start fetch operation"); + } + + /* + * If this is a WRITECHECK or READCHECK, we are expected to fill data + * into the buffer before returning. Note that if this is a READ, we + * don't have to return valid data as WriteOrRead() doesn't care. + */ + if (access == WRITECHECK || access == READCHECK) { + while (param->daosAios - nAios > 0) + AIOWait(param); + memcpy(buffer, aio->a_sgl.sg_iovs->iov_buf, length); + } + + return length; +} + +static void DAOS_Close(void *file, IOR_param_t *param) +{ + struct fileDescriptor *fd = file; + int rc; + + while (param->daosAios - nAios > 0) + AIOWait(param); + AIOFini(param); + + ObjectClose(fd->object); + + if (param->open == WRITE && !param->daosWriteOnly) { + /* Wait for everybody for to complete the writes. */ + MPI_CHECK(MPI_Barrier(param->testComm), + "Failed to synchronize processes"); + + if (rank == 0) { + INFO(VERBOSE_2, param, "Flushing epoch %lu", fd->epoch); + + rc = daos_epoch_flush(fd->container, fd->epoch, + NULL /* state */, NULL /* ev */); + DCHECK(rc, "Failed to flush epoch"); + + INFO(VERBOSE_2, param, "Committing epoch %lu", + fd->epoch); + + rc = daos_epoch_commit(fd->container, fd->epoch, + NULL /* state */, NULL /* ev */); + DCHECK(rc, "Failed to commit object write"); + } + } + + ContainerClose(fd->container, param); + + free(fd); +} + +static void DAOS_Delete(char *testFileName, IOR_param_t *param) +{ + uuid_t uuid; + int rc; + + INFO(VERBOSE_2, param, "Deleting container %s", testFileName); + + rc = uuid_parse(testFileName, uuid); + DCHECK(rc, "Failed to parse 'testFile': %s", testFileName); + + rc = daos_cont_destroy(pool, uuid, 1 /* force */, NULL /* ev */); + if (rc != -DER_NONEXIST) + DCHECK(rc, "Failed to destroy container %s", testFileName); +} + +static char* DAOS_GetVersion() +{ + static char ver[1024] = {}; + + sprintf(ver, "%s", "DAOS"); + return ver; +} + +static void DAOS_Fsync(void *file, IOR_param_t *param) +{ + while (param->daosAios - nAios > 0) + AIOWait(param); +} + +static IOR_offset_t DAOS_GetFileSize(IOR_param_t *test, MPI_Comm testComm, + char *testFileName) +{ + /* + * Sizes are inapplicable to containers at the moment. + */ + return 0; +} diff --git a/src/aiori.c b/src/aiori.c index f5d5719f..5978ed20 100644 --- a/src/aiori.c +++ b/src/aiori.c @@ -57,6 +57,9 @@ ior_aiori_t *available_aiori[] = { #endif #ifdef USE_RADOS_AIORI &rados_aiori, +#endif +#ifdef USE_DAOS_AIORI + &daos_aiori, #endif NULL }; @@ -133,7 +136,7 @@ char* aiori_get_version() static int is_initialized = FALSE; -void aiori_initialize(){ +void aiori_initialize(IOR_test_t *tests_head){ if (is_initialized) return; is_initialized = TRUE; @@ -145,18 +148,18 @@ void aiori_initialize(){ for (ior_aiori_t **tmp = available_aiori ; *tmp != NULL; ++tmp) { if((*tmp)->initialize){ - (*tmp)->initialize(); + (*tmp)->initialize(tests_head ? &tests_head->params : NULL); } } } -void aiori_finalize(){ +void aiori_finalize(IOR_test_t *tests_head){ if (! is_initialized) return; is_initialized = FALSE; for (ior_aiori_t **tmp = available_aiori ; *tmp != NULL; ++tmp) { if((*tmp)->finalize){ - (*tmp)->finalize(); + (*tmp)->finalize(tests_head ? &tests_head->params : NULL); } } } diff --git a/src/aiori.h b/src/aiori.h index 0b0ffda8..c1f63e57 100755 --- a/src/aiori.h +++ b/src/aiori.h @@ -79,8 +79,8 @@ typedef struct ior_aiori { int (*rmdir) (const char *path, IOR_param_t * param); int (*access) (const char *path, int mode, IOR_param_t * param); int (*stat) (const char *path, struct stat *buf, IOR_param_t * param); - void (*initialize)(); /* called once per program before MPI is started */ - void (*finalize)(); /* called once per program after MPI is shutdown */ + void (*initialize)(IOR_param_t *); /* called once per program before MPI is started */ + void (*finalize)(IOR_param_t *); /* called once per program after MPI is shutdown */ option_help * (*get_options)(); } ior_aiori_t; @@ -96,9 +96,10 @@ extern ior_aiori_t s3_aiori; extern ior_aiori_t s3_plus_aiori; extern ior_aiori_t s3_emc_aiori; extern ior_aiori_t rados_aiori; +extern ior_aiori_t daos_aiori; -void aiori_initialize(); -void aiori_finalize(); +void aiori_initialize(IOR_test_t *th); +void aiori_finalize(IOR_test_t *th); const ior_aiori_t *aiori_select (const char *api); int aiori_count (void); void aiori_supported_apis(char * APIs); diff --git a/src/ior.c b/src/ior.c index 1fed65ec..fbb69222 100755 --- a/src/ior.c +++ b/src/ior.c @@ -98,8 +98,6 @@ int ior_main(int argc, char **argv) out_logfile = stdout; out_resultfile = stdout; - aiori_initialize(); - /* * check -h option from commandline without starting MPI; */ @@ -125,6 +123,8 @@ int ior_main(int argc, char **argv) PrintHeader(argc, argv); + aiori_initialize(tests_head); + /* perform each test */ for (tptr = tests_head; tptr != NULL; tptr = tptr->next) { verbose = tptr->params.verbose; @@ -143,6 +143,8 @@ int ior_main(int argc, char **argv) ShowTestEnd(tptr); } + aiori_finalize(tests_head); + if (verbose < 0) /* always print final summary */ verbose = 0; @@ -155,8 +157,6 @@ int ior_main(int argc, char **argv) MPI_CHECK(MPI_Finalize(), "cannot finalize MPI"); - aiori_finalize(); - return totalErrorCount; } @@ -199,6 +199,11 @@ void init_IOR_Param_t(IOR_param_t * p) p->setAlignment = 1; p->lustre_start_ost = -1; + p->daosRecordSize = 262144; + p->daosStripeSize = 524288; + p->daosStripeCount = -1; + p->daosAios = 1; + hdfs_user = getenv("USER"); if (!hdfs_user) hdfs_user = ""; @@ -297,7 +302,8 @@ static void CheckFileSize(IOR_test_t *test, IOR_offset_t dataMoved, int rep) 1, MPI_LONG_LONG_INT, MPI_SUM, testComm), "cannot total data moved"); - if (strcasecmp(params->api, "HDF5") != 0 && strcasecmp(params->api, "NCMPI") != 0) { + if (strcasecmp(params->api, "HDF5") != 0 && strcasecmp(params->api, "NCMPI") != 0 && + strcasecmp(params->api, "DAOS") != 0) { if (verbose >= VERBOSE_0 && rank == 0) { if ((params->expectedAggFileSize != results[rep].aggFileSizeFromXfer) @@ -913,7 +919,8 @@ static void RemoveFile(char *testFileName, int filePerProc, IOR_param_t * test) rankOffset = 0; GetTestFileName(testFileName, test); } - if (backend->access(testFileName, F_OK, test) == 0) { + if (backend->access(testFileName, F_OK, test) == 0 || + strcasecmp(test->api, "DAOS") == 0) { backend->delete(testFileName, test); } if (test->reorderTasksRandom == TRUE) { @@ -921,7 +928,8 @@ static void RemoveFile(char *testFileName, int filePerProc, IOR_param_t * test) GetTestFileName(testFileName, test); } } else { - if ((rank == 0) && (backend->access(testFileName, F_OK, test) == 0)) { + if ((rank == 0) && (backend->access(testFileName, F_OK, test) == 0 || + strcasecmp(test->api, "DAOS"))) { backend->delete(testFileName, test); } } diff --git a/src/ior.h b/src/ior.h index 43177fdc..4fa9052e 100755 --- a/src/ior.h +++ b/src/ior.h @@ -190,6 +190,21 @@ typedef struct int lustre_set_striping; /* flag that we need to set lustre striping */ int lustre_ignore_locks; + /* DAOS variables */ + char daosGroup[MAX_STR]; /* group name */ + char daosPool[37]; /* pool UUID */ + char daosPoolSvc[MAX_STR]; /* pool service ranks */ + int daosRecordSize; /* size of akey record (i.e., rx_rsize) */ + int daosStripeSize; + unsigned long daosStripeCount; + unsigned long daosStripeMax; /* max length of a stripe */ + int daosAios; /* max number of concurrent async I/Os */ + int daosWriteOnly; /* write only, no flush and commit */ + unsigned long daosEpoch; /* epoch to access */ + unsigned long daosWait; /* epoch to wait for before reading */ + int daosKill; /* kill a target while running IOR */ + char daosObjectClass[MAX_STR]; /* object class */ + /* gpfs variables */ int gpfs_hint_access; /* use gpfs "access range" hint */ int gpfs_release_token; /* immediately release GPFS tokens after diff --git a/src/list.h b/src/list.h new file mode 100644 index 00000000..dbe052cd --- /dev/null +++ b/src/list.h @@ -0,0 +1,556 @@ +/** + * GPL HEADER START + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 only, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License version 2 for more details (a copy is included + * in the LICENSE file that accompanied this code). + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * GPL HEADER END + */ +#ifndef __DAOS_LIST_H__ +#define __DAOS_LIST_H__ + +/* + * Simple doubly linked list implementation. + * + * Some of the internal functions ("__xxx") are useful when + * manipulating whole lists rather than single entries, as + * sometimes we already know the next/prev entries and we can + * generate better code by using them directly rather than + * using the generic single-entry routines. + */ + +#define prefetch(a) ((void)a) + +struct cfs_list_head { + struct cfs_list_head *next, *prev; +}; + +typedef struct cfs_list_head cfs_list_t; + +#define CFS_LIST_HEAD_INIT(name) { &(name), &(name) } + +#define CFS_LIST_HEAD(name) \ + cfs_list_t name = CFS_LIST_HEAD_INIT(name) + +#define CFS_INIT_LIST_HEAD(ptr) do { \ + (ptr)->next = (ptr); (ptr)->prev = (ptr); \ +} while (0) + +/** + * Insert a new entry between two known consecutive entries. + * + * This is only for internal list manipulation where we know + * the prev/next entries already! + */ +static inline void __cfs_list_add(cfs_list_t * new, + cfs_list_t * prev, + cfs_list_t * next) +{ + next->prev = new; + new->next = next; + new->prev = prev; + prev->next = new; +} + +/** + * Insert an entry at the start of a list. + * \param new new entry to be inserted + * \param head list to add it to + * + * Insert a new entry after the specified head. + * This is good for implementing stacks. + */ +static inline void cfs_list_add(cfs_list_t *new, + cfs_list_t *head) +{ + __cfs_list_add(new, head, head->next); +} + +/** + * Insert an entry at the end of a list. + * \param new new entry to be inserted + * \param head list to add it to + * + * Insert a new entry before the specified head. + * This is useful for implementing queues. + */ +static inline void cfs_list_add_tail(cfs_list_t *new, + cfs_list_t *head) +{ + __cfs_list_add(new, head->prev, head); +} + +/* + * Delete a list entry by making the prev/next entries + * point to each other. + * + * This is only for internal list manipulation where we know + * the prev/next entries already! + */ +static inline void __cfs_list_del(cfs_list_t *prev, + cfs_list_t *next) +{ + next->prev = prev; + prev->next = next; +} + +/** + * Remove an entry from the list it is currently in. + * \param entry the entry to remove + * Note: list_empty(entry) does not return true after this, the entry is in an + * undefined state. + */ +static inline void cfs_list_del(cfs_list_t *entry) +{ + __cfs_list_del(entry->prev, entry->next); +} + +/** + * Remove an entry from the list it is currently in and reinitialize it. + * \param entry the entry to remove. + */ +static inline void cfs_list_del_init(cfs_list_t *entry) +{ + __cfs_list_del(entry->prev, entry->next); + CFS_INIT_LIST_HEAD(entry); +} + +/** + * Remove an entry from the list it is currently in and insert it at the start + * of another list. + * \param list the entry to move + * \param head the list to move it to + */ +static inline void cfs_list_move(cfs_list_t *list, + cfs_list_t *head) +{ + __cfs_list_del(list->prev, list->next); + cfs_list_add(list, head); +} + +/** + * Remove an entry from the list it is currently in and insert it at the end of + * another list. + * \param list the entry to move + * \param head the list to move it to + */ +static inline void cfs_list_move_tail(cfs_list_t *list, + cfs_list_t *head) +{ + __cfs_list_del(list->prev, list->next); + cfs_list_add_tail(list, head); +} + +/** + * Test whether a list is empty + * \param head the list to test. + */ +static inline int cfs_list_empty(cfs_list_t *head) +{ + return head->next == head; +} + +/** + * Test whether a list is empty and not being modified + * \param head the list to test + * + * Tests whether a list is empty _and_ checks that no other CPU might be + * in the process of modifying either member (next or prev) + * + * NOTE: using cfs_list_empty_careful() without synchronization + * can only be safe if the only activity that can happen + * to the list entry is cfs_list_del_init(). Eg. it cannot be used + * if another CPU could re-list_add() it. + */ +static inline int cfs_list_empty_careful(const cfs_list_t *head) +{ + cfs_list_t *next = head->next; + return (next == head) && (next == head->prev); +} + +static inline void __cfs_list_splice(cfs_list_t *list, + cfs_list_t *head) +{ + cfs_list_t *first = list->next; + cfs_list_t *last = list->prev; + cfs_list_t *at = head->next; + + first->prev = head; + head->next = first; + + last->next = at; + at->prev = last; +} + +/** + * Join two lists + * \param list the new list to add. + * \param head the place to add it in the first list. + * + * The contents of \a list are added at the start of \a head. \a list is in an + * undefined state on return. + */ +static inline void cfs_list_splice(cfs_list_t *list, + cfs_list_t *head) +{ + if (!cfs_list_empty(list)) + __cfs_list_splice(list, head); +} + +/** + * Join two lists and reinitialise the emptied list. + * \param list the new list to add. + * \param head the place to add it in the first list. + * + * The contents of \a list are added at the start of \a head. \a list is empty + * on return. + */ +static inline void cfs_list_splice_init(cfs_list_t *list, + cfs_list_t *head) +{ + if (!cfs_list_empty(list)) { + __cfs_list_splice(list, head); + CFS_INIT_LIST_HEAD(list); + } +} + +/** + * Get the container of a list + * \param ptr the embedded list. + * \param type the type of the struct this is embedded in. + * \param member the member name of the list within the struct. + */ +#define cfs_list_entry(ptr, type, member) \ + ((type *)((char *)(ptr)-(char *)(&((type *)0)->member))) + +/** + * Iterate over a list + * \param pos the iterator + * \param head the list to iterate over + * + * Behaviour is undefined if \a pos is removed from the list in the body of the + * loop. + */ +#define cfs_list_for_each(pos, head) \ + for (pos = (head)->next, prefetch(pos->next); pos != (head); \ + pos = pos->next, prefetch(pos->next)) + +/** + * Iterate over a list safely + * \param pos the iterator + * \param n temporary storage + * \param head the list to iterate over + * + * This is safe to use if \a pos could be removed from the list in the body of + * the loop. + */ +#define cfs_list_for_each_safe(pos, n, head) \ + for (pos = (head)->next, n = pos->next; pos != (head); \ + pos = n, n = pos->next) + +/** + * Iterate over a list continuing after existing point + * \param pos the type * to use as a loop counter + * \param head the list head + * \param member the name of the list_struct within the struct + */ +#define cfs_list_for_each_entry_continue(pos, head, member) \ + for (pos = cfs_list_entry(pos->member.next, typeof(*pos), member); \ + prefetch(pos->member.next), &pos->member != (head); \ + pos = cfs_list_entry(pos->member.next, typeof(*pos), member)) + +/** + * \defgroup hlist Hash List + * Double linked lists with a single pointer list head. + * Mostly useful for hash tables where the two pointer list head is too + * wasteful. You lose the ability to access the tail in O(1). + * @{ + */ + +typedef struct cfs_hlist_node { + struct cfs_hlist_node *next, **pprev; +} cfs_hlist_node_t; + +typedef struct cfs_hlist_head { + cfs_hlist_node_t *first; +} cfs_hlist_head_t; + +/* @} */ + +/* + * "NULL" might not be defined at this point + */ +#ifdef NULL +#define NULL_P NULL +#else +#define NULL_P ((void *)0) +#endif + +/** + * \addtogroup hlist + * @{ + */ + +#define CFS_HLIST_HEAD_INIT { NULL_P } +#define CFS_HLIST_HEAD(name) cfs_hlist_head_t name = { NULL_P } +#define CFS_INIT_HLIST_HEAD(ptr) ((ptr)->first = NULL_P) +#define CFS_INIT_HLIST_NODE(ptr) ((ptr)->next = NULL_P, (ptr)->pprev = NULL_P) + +static inline int cfs_hlist_unhashed(const cfs_hlist_node_t *h) +{ + return !h->pprev; +} + +static inline int cfs_hlist_empty(const cfs_hlist_head_t *h) +{ + return !h->first; +} + +static inline void __cfs_hlist_del(cfs_hlist_node_t *n) +{ + cfs_hlist_node_t *next = n->next; + cfs_hlist_node_t **pprev = n->pprev; + *pprev = next; + if (next) + next->pprev = pprev; +} + +static inline void cfs_hlist_del(cfs_hlist_node_t *n) +{ + __cfs_hlist_del(n); +} + +static inline void cfs_hlist_del_init(cfs_hlist_node_t *n) +{ + if (n->pprev) { + __cfs_hlist_del(n); + CFS_INIT_HLIST_NODE(n); + } +} + +static inline void cfs_hlist_add_head(cfs_hlist_node_t *n, + cfs_hlist_head_t *h) +{ + cfs_hlist_node_t *first = h->first; + n->next = first; + if (first) + first->pprev = &n->next; + h->first = n; + n->pprev = &h->first; +} + +/* next must be != NULL */ +static inline void cfs_hlist_add_before(cfs_hlist_node_t *n, + cfs_hlist_node_t *next) +{ + n->pprev = next->pprev; + n->next = next; + next->pprev = &n->next; + *(n->pprev) = n; +} + +static inline void cfs_hlist_add_after(cfs_hlist_node_t *n, + cfs_hlist_node_t *next) +{ + next->next = n->next; + n->next = next; + next->pprev = &n->next; + + if(next->next) + next->next->pprev = &next->next; +} + +#define cfs_hlist_entry(ptr, type, member) container_of(ptr,type,member) + +#define cfs_hlist_for_each(pos, head) \ + for (pos = (head)->first; pos && (prefetch(pos->next), 1); \ + pos = pos->next) + +#define cfs_hlist_for_each_safe(pos, n, head) \ + for (pos = (head)->first; pos && (n = pos->next, 1); \ + pos = n) + +/** + * Iterate over an hlist of given type + * \param tpos the type * to use as a loop counter. + * \param pos the &struct hlist_node to use as a loop counter. + * \param head the head for your list. + * \param member the name of the hlist_node within the struct. + */ +#define cfs_hlist_for_each_entry(tpos, pos, head, member) \ + for (pos = (head)->first; \ + pos && ({ prefetch(pos->next); 1;}) && \ + ({ tpos = cfs_hlist_entry(pos, typeof(*tpos), member); 1;}); \ + pos = pos->next) + +/** + * Iterate over an hlist continuing after existing point + * \param tpos the type * to use as a loop counter. + * \param pos the &struct hlist_node to use as a loop counter. + * \param member the name of the hlist_node within the struct. + */ +#define cfs_hlist_for_each_entry_continue(tpos, pos, member) \ + for (pos = (pos)->next; \ + pos && ({ prefetch(pos->next); 1;}) && \ + ({ tpos = cfs_hlist_entry(pos, typeof(*tpos), member); 1;}); \ + pos = pos->next) + +/** + * Iterate over an hlist continuing from an existing point + * \param tpos the type * to use as a loop counter. + * \param pos the &struct hlist_node to use as a loop counter. + * \param member the name of the hlist_node within the struct. + */ +#define cfs_hlist_for_each_entry_from(tpos, pos, member) \ + for (; pos && ({ prefetch(pos->next); 1;}) && \ + ({ tpos = cfs_hlist_entry(pos, typeof(*tpos), member); 1;}); \ + pos = pos->next) + +/** + * Iterate over an hlist of given type safe against removal of list entry + * \param tpos the type * to use as a loop counter. + * \param pos the &struct hlist_node to use as a loop counter. + * \param n another &struct hlist_node to use as temporary storage + * \param head the head for your list. + * \param member the name of the hlist_node within the struct. + */ +#define cfs_hlist_for_each_entry_safe(tpos, pos, n, head, member) \ + for (pos = (head)->first; \ + pos && ({ n = pos->next; 1; }) && \ + ({ tpos = cfs_hlist_entry(pos, typeof(*tpos), member); 1;}); \ + pos = n) + +/* @} */ + +#ifndef cfs_list_for_each_prev +/** + * Iterate over a list in reverse order + * \param pos the &struct list_head to use as a loop counter. + * \param head the head for your list. + */ +#define cfs_list_for_each_prev(pos, head) \ + for (pos = (head)->prev, prefetch(pos->prev); pos != (head); \ + pos = pos->prev, prefetch(pos->prev)) + +#endif /* cfs_list_for_each_prev */ + +#ifndef cfs_list_for_each_entry +/** + * Iterate over a list of given type + * \param pos the type * to use as a loop counter. + * \param head the head for your list. + * \param member the name of the list_struct within the struct. + */ +#define cfs_list_for_each_entry(pos, head, member) \ + for (pos = cfs_list_entry((head)->next, typeof(*pos), member), \ + prefetch(pos->member.next); \ + &pos->member != (head); \ + pos = cfs_list_entry(pos->member.next, typeof(*pos), member), \ + prefetch(pos->member.next)) +#endif /* cfs_list_for_each_entry */ + +#ifndef cfs_list_for_each_entry_rcu +#define cfs_list_for_each_entry_rcu(pos, head, member) \ + list_for_each_entry(pos, head, member) +#endif + +#ifndef cfs_list_for_each_entry_rcu +#define cfs_list_for_each_entry_rcu(pos, head, member) \ + list_for_each_entry(pos, head, member) +#endif + +#ifndef cfs_list_for_each_entry_reverse +/** + * Iterate backwards over a list of given type. + * \param pos the type * to use as a loop counter. + * \param head the head for your list. + * \param member the name of the list_struct within the struct. + */ +#define cfs_list_for_each_entry_reverse(pos, head, member) \ + for (pos = cfs_list_entry((head)->prev, typeof(*pos), member); \ + prefetch(pos->member.prev), &pos->member != (head); \ + pos = cfs_list_entry(pos->member.prev, typeof(*pos), member)) +#endif /* cfs_list_for_each_entry_reverse */ + +#ifndef cfs_list_for_each_entry_safe +/** + * Iterate over a list of given type safe against removal of list entry + * \param pos the type * to use as a loop counter. + * \param n another type * to use as temporary storage + * \param head the head for your list. + * \param member the name of the list_struct within the struct. + */ +#define cfs_list_for_each_entry_safe(pos, n, head, member) \ + for (pos = cfs_list_entry((head)->next, typeof(*pos), member), \ + n = cfs_list_entry(pos->member.next, typeof(*pos), member); \ + &pos->member != (head); \ + pos = n, n = cfs_list_entry(n->member.next, typeof(*n), member)) + +#endif /* cfs_list_for_each_entry_safe */ + +#ifndef cfs_list_for_each_entry_safe_from +/** + * Iterate over a list continuing from an existing point + * \param pos the type * to use as a loop cursor. + * \param n another type * to use as temporary storage + * \param head the head for your list. + * \param member the name of the list_struct within the struct. + * + * Iterate over list of given type from current point, safe against + * removal of list entry. + */ +#define cfs_list_for_each_entry_safe_from(pos, n, head, member) \ + for (n = cfs_list_entry(pos->member.next, typeof(*pos), member); \ + &pos->member != (head); \ + pos = n, n = cfs_list_entry(n->member.next, typeof(*n), member)) +#endif /* cfs_list_for_each_entry_safe_from */ + +#define cfs_list_for_each_entry_typed(pos, head, type, member) \ + for (pos = cfs_list_entry((head)->next, type, member), \ + prefetch(pos->member.next); \ + &pos->member != (head); \ + pos = cfs_list_entry(pos->member.next, type, member), \ + prefetch(pos->member.next)) + +#define cfs_list_for_each_entry_reverse_typed(pos, head, type, member) \ + for (pos = cfs_list_entry((head)->prev, type, member); \ + prefetch(pos->member.prev), &pos->member != (head); \ + pos = cfs_list_entry(pos->member.prev, type, member)) + +#define cfs_list_for_each_entry_safe_typed(pos, n, head, type, member) \ + for (pos = cfs_list_entry((head)->next, type, member), \ + n = cfs_list_entry(pos->member.next, type, member); \ + &pos->member != (head); \ + pos = n, n = cfs_list_entry(n->member.next, type, member)) + +#define cfs_list_for_each_entry_safe_from_typed(pos, n, head, type, member) \ + for (n = cfs_list_entry(pos->member.next, type, member); \ + &pos->member != (head); \ + pos = n, n = cfs_list_entry(n->member.next, type, member)) + +#define cfs_hlist_for_each_entry_typed(tpos, pos, head, type, member) \ + for (pos = (head)->first; \ + pos && (prefetch(pos->next), 1) && \ + (tpos = cfs_hlist_entry(pos, type, member), 1); \ + pos = pos->next) + +#define cfs_hlist_for_each_entry_safe_typed(tpos, pos, n, head, type, member) \ + for (pos = (head)->first; \ + pos && (n = pos->next, 1) && \ + (tpos = cfs_hlist_entry(pos, type, member), 1); \ + pos = n) + +#endif /* __DAOS_LIST_H__ */ diff --git a/src/mdtest-main.c b/src/mdtest-main.c index 854456f6..f54cf664 100644 --- a/src/mdtest-main.c +++ b/src/mdtest-main.c @@ -2,12 +2,12 @@ #include "aiori.h" int main(int argc, char **argv) { - aiori_initialize(); MPI_Init(&argc, &argv); + aiori_initialize(NULL); mdtest_run(argc, argv, MPI_COMM_WORLD, stdout); + aiori_finalize(NULL); MPI_Finalize(); - aiori_finalize(); return 0; } diff --git a/src/parse_options.c b/src/parse_options.c index 80b99a31..0f251ef3 100755 --- a/src/parse_options.c +++ b/src/parse_options.c @@ -300,6 +300,34 @@ void DecodeDirective(char *line, IOR_param_t *params) params->numTasks = atoi(value); } else if (strcasecmp(option, "summaryalways") == 0) { params->summary_every_test = atoi(value); + } else if (strcasecmp(option, "daosgroup") == 0) { + strcpy(params->daosGroup, value); + } else if (strcasecmp(option, "daospool") == 0) { + strcpy(params->daosPool, value); + } else if (strcasecmp(option, "daospoolsvc") == 0) { + strcpy(params->daosPoolSvc, value); + } else if (strcasecmp(option, "daosrecordsize") == 0) { + params->daosRecordSize = string_to_bytes(value); + } else if (strcasecmp(option, "daosstripesize") == 0) { + printf("HERE %s\n", value); + params->daosStripeSize = string_to_bytes(value); + printf("HERE %d\n", params->daosStripeSize); + } else if (strcasecmp(option, "daosstripecount") == 0) { + params->daosStripeCount = atoi(value); + } else if (strcasecmp(option, "daosstripemax") == 0) { + params->daosStripeMax = string_to_bytes(value); + } else if (strcasecmp(option, "daosaios") == 0) { + params->daosAios = atoi(value); + } else if (strcasecmp(option, "daosepoch") == 0) { + params->daosEpoch = atoi(value); + } else if (strcasecmp(option, "daoswait") == 0) { + params->daosWait = atoi(value); + } else if (strcasecmp(option, "daoswriteonly") == 0) { + params->daosWriteOnly = atoi(value); + } else if (strcasecmp(option, "daoskill") == 0) { + params->daosKill = atoi(value); + } else if (strcasecmp(option, "daosobjectclass") == 0) { + strcpy(params->daosObjectClass, value); } else { if (rank == 0) fprintf(out_logfile, "Unrecognized parameter \"%s\"\n", @@ -311,11 +339,13 @@ void DecodeDirective(char *line, IOR_param_t *params) /* * Parse a single line, which may contain multiple comma-seperated directives */ -void ParseLine(char *line, IOR_param_t * test) +void ParseLine(const char *line, IOR_param_t * test) { char *start, *end; - start = line; + start = strdup(line); + if (start == NULL) + ERR("failed to duplicate line"); do { end = strchr(start, ','); if (end != NULL) @@ -422,7 +452,7 @@ IOR_test_t *ReadConfigScript(char *scriptName) static IOR_param_t * parameters; static void decodeDirectiveWrapper(char *line){ - DecodeDirective(line, parameters); + ParseLine(line, parameters); } /* From 98ae5df16df69ddb7aee72cf634e0b662d073db0 Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Tue, 28 Aug 2018 19:45:18 +0000 Subject: [PATCH 006/206] remove printf added by accident. Signed-off-by: Mohamad Chaarawi --- src/parse_options.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/parse_options.c b/src/parse_options.c index 0f251ef3..5769d74f 100755 --- a/src/parse_options.c +++ b/src/parse_options.c @@ -309,9 +309,7 @@ void DecodeDirective(char *line, IOR_param_t *params) } else if (strcasecmp(option, "daosrecordsize") == 0) { params->daosRecordSize = string_to_bytes(value); } else if (strcasecmp(option, "daosstripesize") == 0) { - printf("HERE %s\n", value); params->daosStripeSize = string_to_bytes(value); - printf("HERE %d\n", params->daosStripeSize); } else if (strcasecmp(option, "daosstripecount") == 0) { params->daosStripeCount = atoi(value); } else if (strcasecmp(option, "daosstripemax") == 0) { From 14d67c19d9b4abb9c382f8b8f9a840a223ead1a8 Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Wed, 29 Aug 2018 22:25:48 +0000 Subject: [PATCH 007/206] fix DAOS plugin options passing. Signed-off-by: Mohamad Chaarawi --- src/aiori-DAOS.c | 45 ++++++++++++++++++++++----------------------- src/ior.c | 2 +- 2 files changed, 23 insertions(+), 24 deletions(-) diff --git a/src/aiori-DAOS.c b/src/aiori-DAOS.c index 73d78376..3aadba93 100644 --- a/src/aiori-DAOS.c +++ b/src/aiori-DAOS.c @@ -98,21 +98,23 @@ static void DAOS_Delete(char *, IOR_param_t *); static char* DAOS_GetVersion(); static void DAOS_Fsync(void *, IOR_param_t *); static IOR_offset_t DAOS_GetFileSize(IOR_param_t *, MPI_Comm, char *); +static option_help * DAOS_options(); /************************** D E C L A R A T I O N S ***************************/ ior_aiori_t daos_aiori = { - .name = "DAOS", - .create = DAOS_Create, - .open = DAOS_Open, - .xfer = DAOS_Xfer, - .close = DAOS_Close, - .delete = DAOS_Delete, - .get_version = DAOS_GetVersion, - .fsync = DAOS_Fsync, - .get_file_size = DAOS_GetFileSize, - .initialize = DAOS_Init, - .finalize = DAOS_Fini, + .name = "DAOS", + .create = DAOS_Create, + .open = DAOS_Open, + .xfer = DAOS_Xfer, + .close = DAOS_Close, + .delete = DAOS_Delete, + .get_version = DAOS_GetVersion, + .fsync = DAOS_Fsync, + .get_file_size = DAOS_GetFileSize, + .initialize = DAOS_Init, + .finalize = DAOS_Fini, + .get_options = DAOS_options, }; enum handleType { @@ -536,13 +538,6 @@ static void ObjectClassParse(const char *string) GERR("Invalid 'daosObjectClass' argument: '%s'", string); } -static const char *GetGroup(IOR_param_t *param) -{ - if (strlen(o.daosGroup) == 0) - return NULL; - return o.daosGroup; -} - static void ParseService(IOR_param_t *param, int max, d_rank_list_t *ranks) { char *s; @@ -563,11 +558,15 @@ static void ParseService(IOR_param_t *param, int max, d_rank_list_t *ranks) free(s); } +static option_help * DAOS_options(){ + return options; +} + static void DAOS_Init(IOR_param_t *param) { int rc; - if (strlen(o.daosObjectClass) != 0) + if (o.daosObjectClass) ObjectClassParse(o.daosObjectClass); if (param->filePerProc) @@ -601,9 +600,9 @@ static void DAOS_Init(IOR_param_t *param) d_rank_t rank[13]; d_rank_list_t ranks; - if (strlen(o.daosPool) == 0) + if (o.daosPool == NULL) GERR("'daosPool' must be specified"); - if (strlen(o.daosPoolSvc) == 0) + if (o.daosPoolSvc == NULL) GERR("'daosPoolSvc' must be specified"); INFO(VERBOSE_2, param, "Connecting to pool %s %s", @@ -614,7 +613,7 @@ static void DAOS_Init(IOR_param_t *param) ranks.rl_ranks = rank; ParseService(param, sizeof(rank) / sizeof(rank[0]), &ranks); - rc = daos_pool_connect(uuid, GetGroup(param), &ranks, + rc = daos_pool_connect(uuid, o.daosGroup, &ranks, DAOS_PC_RW, &pool, &poolInfo, NULL /* ev */); DCHECK(rc, "Failed to connect to pool %s", o.daosPool); @@ -729,7 +728,7 @@ kill_daos_server(IOR_param_t *param) rank, info.pi_ndisabled, info.pi_ntargets); fflush(stdout); - rc = daos_mgmt_svc_rip(GetGroup(param), rank, true, NULL); + rc = daos_mgmt_svc_rip(o.daosGroup, rank, true, NULL); DCHECK(rc, "Error in killing server\n"); targets.rl_nr = 1; diff --git a/src/ior.c b/src/ior.c index 7c7dffe0..781a1345 100755 --- a/src/ior.c +++ b/src/ior.c @@ -1187,7 +1187,7 @@ static void TestIoSys(IOR_test_t *test) backend = aiori_select(params->api); if (backend->initialize) - backend->initialize(NULL); + backend->initialize(params); /* show test setup */ if (rank == 0 && verbose >= VERBOSE_0) From 2668363fb14d844c5b36b5531ef106220f150928 Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Thu, 30 Aug 2018 14:47:17 +0000 Subject: [PATCH 008/206] - update README for DAOS drivers. - update flag options in DAOS API - make svcl option in DFS API the same as DAOS API Signed-off-by: Mohamad Chaarawi --- README_DAOS | 104 +++++++++++++++++++++++++++++++++++++---------- src/aiori-DAOS.c | 4 +- src/aiori-DFS.c | 2 +- 3 files changed, 85 insertions(+), 25 deletions(-) diff --git a/README_DAOS b/README_DAOS index ba4fc3e5..409923de 100644 --- a/README_DAOS +++ b/README_DAOS @@ -1,25 +1,55 @@ -Building with DAOS API +Building ---------------------- -At step 1 above, one must specify "--with-daos". If the DAOS -headers and libraries are not installed at respective system -default locations, then one may also needs to set CPPFLAGS and -LDFLAGS accordingly. +The DAOS library must be installed on the system. + +./bootsrap +./configure --prefix=iorInstallDir --with-daos=DIR + +One must specify "--with-daos=/path/to/daos/install". When that is specified +the DAOS and DFS driver will be built. + +The DAOS driver uses the DAOS API to create a container (file) and update/fetch +an object to the container using the daos object API. The DAOS driver works with +IOR only (no mdtest support yet). + +The DFS (DAOS File System) driver creates an encapsulated namespace and emulates +the POSIX driver using the DFS API directly on top of DAOS. The DFS driver works +with both IOR and mdtest. Running with DAOS API --------------------- -One must specify an existing pool using "-O -daospool=". IOR must be launched in a way that -attaches the IOR process group to the DAOS server process group. +Driver specific options are specified at the end after "--". For example: + +ior -a DAOS [ior_options] -- [daos_options] -One must also specify a container UUID using "-o -". If the "-E" option is given, then this UUID +In the IOR options, the file name should be specified as a container uuid using +"-o ". If the "-E" option is given, then this UUID shall denote an existing container created by a "matching" IOR run. Otherwise, IOR will create a new container with this UUID. In the latter case, one may use uuidgen(1) to generate the UUID of the new container. +The DAOS options include: + +Required Options: +-p : pool uuid to connect to (has to be created beforehand) +-v : pool svcl list (: separated) + +Optional Options: +-g : group name of servers with the pool +-r : object record size for IO +-s +-c +-m +-a : number of concurrent async IOs +-w : Flag to indicate no commit, just update +-e +-t : wait for specific epoch before read +-k : flag to kill a rank during IO +-o : specific object class + When benchmarking write performance, one likely do not want "-W", which causes the write phase to do one additional memory copy for every I/O. This is due to IOR's assumption that when a @@ -32,17 +62,47 @@ all combinations of options are supported. Examples that should work include: - - "ior -a DAOS -w -W -o -O - daospool=,daospoolsvc=" writes into a new container - and verifies the data, using default daosRecordSize, transferSize, - daosStripeSize, blockSize, daosAios, etc. + - "ior -a DAOS -w -W -o -- -p -v " + writes into a new container and verifies the data, using default + daosRecordSize, transferSize, daosStripeSize, blockSize, daosAios, etc. + + - "ior -a DAOS -w -W -r -R -o -b 1g -t 4m -C -- + -p -v -r 1m -s 4m -c 256 -a 8" + does all IOR tests and shifts ranks during checkWrite and checkRead. + + - "ior -a DAOS -w -r -o -b 8g -t 1m -C -- + -p -v -r 1m -s 4m -c 256 -a 8" + may be a base to be tuned for performance benchmarking. - - "ior -a DAOS -w -W -r -R -o -b 1g -t 4m -C -O - daospool=,daospoolsvc=,daosrecordsize=1m, - daosstripesize=4m, daosstripecount=256,daosaios=8" does all IOR tests and - shifts ranks during checkWrite and checkRead. - - "ior -a DAOS -w -r -o -b 8g -t 1m -C -O - daospool=,daospoolsvc=,daosrecordsize=1m,daosstripesize=4m, - daosstripecount=256,daosaios=8" may be a base to be tuned for performance - benchmarking. +Running with DFS API +--------------------- + +Driver specific options are specified at the end after "--". For example: + +ior -a DFS [ior_options] -- [dfs_options] +mdtest -a DFS [mdtest_options] -- [dfs_options] + +Required Options: +-p : pool uuid to connect to (has to be created beforehand) +-v : pool svcl list (: separated) +-c : container uuid that will hold the encapsulated namespace + +Optional Options: +-g : group name of servers with the pool + +In the IOR options, the file name should be specified on the root dir directly +since ior does not create directories and the DFS container representing the +encapsulated namespace is not the same as the system namespace the user is +executing from. + +Examples that should work include: + - "ior -a DFS -w -W -o /test1 -- -p -v -c " + - "ior -a DFS -w -W -r -R -o /test2 -b 1g -t 4m -C -- -p -v -c " + - "ior -a DFS -w -r -o /test3 -b 8g -t 1m -C -- -p -v -c " + +Running mdtest, the user needs to specify a directory with -d where the test +tree will be created. Some examples: + - "mdtest -a DFS -n 100 -F -D -d /bla -- -p -v -c " + - "mdtest -a DFS -n 1000 -F -C -d /bla -- -p -v -c " + - "mdtest -a DFS -I 10 -z 5 -b 2 -L -d /bla -- -p -v -c " diff --git a/src/aiori-DAOS.c b/src/aiori-DAOS.c index 3aadba93..e3830d95 100644 --- a/src/aiori-DAOS.c +++ b/src/aiori-DAOS.c @@ -77,10 +77,10 @@ static option_help options [] = { {'c', "daosStripeCount", "Stripe Count", OPTION_OPTIONAL_ARGUMENT, 'u', &o.daosStripeCount}, {'m', "daosStripeMax", "Max Stripe",OPTION_OPTIONAL_ARGUMENT, 'u', &o.daosStripeMax}, {'a', "daosAios", "Concurrent Async IOs",OPTION_OPTIONAL_ARGUMENT, 'd', &o.daosAios}, - {'w', "daosWriteOnly", "Write Only, no commit",OPTION_OPTIONAL_ARGUMENT, 'd', &o.daosWriteOnly}, + {'w', "daosWriteOnly", "Write Only, no commit",OPTION_FLAG, 'd', &o.daosWriteOnly}, {'e', "daosEpoch", "Epoch Number to Access",OPTION_OPTIONAL_ARGUMENT, 'u', &o.daosEpoch}, {'t', "daosWait", "Epoch to wait for before read",OPTION_OPTIONAL_ARGUMENT, 'u', &o.daosWait}, - {'k', "daosKill", "Kill target while running",OPTION_OPTIONAL_ARGUMENT, 'd', &o.daosKill}, + {'k', "daosKill", "Kill target while running",OPTION_FLAG, 'd', &o.daosKill}, {'o', "daosObjectClass", "object class", OPTION_OPTIONAL_ARGUMENT, 's', &o.daosObjectClass}, LAST_OPTION }; diff --git a/src/aiori-DFS.c b/src/aiori-DFS.c index f33ed94c..aead7581 100755 --- a/src/aiori-DFS.c +++ b/src/aiori-DFS.c @@ -63,7 +63,7 @@ static struct dfs_options o = { static option_help options [] = { {'p', "pool", "DAOS pool uuid", OPTION_REQUIRED_ARGUMENT, 's', & o.pool}, - {'s', "svcl", "DAOS pool SVCL", OPTION_REQUIRED_ARGUMENT, 's', & o.svcl}, + {'v', "svcl", "DAOS pool SVCL", OPTION_REQUIRED_ARGUMENT, 's', & o.svcl}, {'g', "group", "DAOS server group", OPTION_OPTIONAL_ARGUMENT, 's', & o.group}, {'c', "cont", "DFS container uuid", OPTION_REQUIRED_ARGUMENT, 's', & o.cont}, LAST_OPTION From 2f713e947d4627c2dc1716b2e262f1968c6b24fb Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Wed, 12 Sep 2018 22:53:42 +0000 Subject: [PATCH 009/206] fix bug in driver name check for DAOS. Signed-off-by: Mohamad Chaarawi --- src/ior.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ior.c b/src/ior.c index 781a1345..71582108 100755 --- a/src/ior.c +++ b/src/ior.c @@ -921,7 +921,7 @@ static void RemoveFile(char *testFileName, int filePerProc, IOR_param_t * test) } } else { if ((rank == 0) && (backend->access(testFileName, F_OK, test) == 0 || - strcasecmp(test->api, "DAOS"))) { + strcasecmp(test->api, "DAOS") == 0)) { backend->delete(testFileName, test); } } From 0a7b1e38ea64d70c094305df6d64d5f604010083 Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Thu, 13 Sep 2018 13:30:10 +0000 Subject: [PATCH 010/206] fix verbose output in DAOS & DFS drivers Signed-off-by: Mohamad Chaarawi --- src/aiori-DAOS.c | 4 ++-- src/aiori-DFS.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/aiori-DAOS.c b/src/aiori-DAOS.c index e3830d95..657bc1d2 100644 --- a/src/aiori-DAOS.c +++ b/src/aiori-DAOS.c @@ -171,7 +171,7 @@ do { \ #define INFO(level, param, format, ...) \ do { \ - if (param->verbose >= level) \ + if (verbose >= level) \ printf("[%d] "format"\n", rank, ##__VA_ARGS__); \ } while (0) @@ -260,7 +260,7 @@ static void ContainerOpen(char *testFileName, IOR_param_t *param, testFileName); } - INFO(VERBOSE_2, param, "Openning container %s", testFileName); + INFO(VERBOSE_2, param, "Opening container %s", testFileName); if (param->open == WRITE) dFlags = DAOS_COO_RW; diff --git a/src/aiori-DFS.c b/src/aiori-DFS.c index aead7581..46edeb63 100755 --- a/src/aiori-DFS.c +++ b/src/aiori-DFS.c @@ -243,7 +243,7 @@ DFS_Init(IOR_param_t *param) { if (svcl == NULL) ERR("Failed to allocate svcl"); - if (verbose >= 3) { + if (verbose >= VERBOSE_1) { printf("Pool uuid = %s, SVCL = %s\n", o.pool, o.svcl); printf("DFS Container namespace uuid = %s\n", o.cont); } @@ -259,7 +259,7 @@ DFS_Init(IOR_param_t *param) { rc = daos_cont_open(poh, co_uuid, DAOS_COO_RW, &coh, &co_info, NULL); /* If NOEXIST we create it */ if (rc == -DER_NONEXIST) { - if (verbose >= 3) + if (verbose >= VERBOSE_1) printf("Creating DFS Container ...\n"); rc = daos_cont_create(poh, co_uuid, NULL); if (rc == 0) { From ab71b4fc2abdcd6c4575987be744b848a5410695 Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Thu, 13 Sep 2018 13:36:36 +0000 Subject: [PATCH 011/206] update DAOS README Signed-off-by: Mohamad Chaarawi --- README_DAOS | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/README_DAOS b/README_DAOS index 409923de..fb3c68ea 100644 --- a/README_DAOS +++ b/README_DAOS @@ -25,11 +25,10 @@ Driver specific options are specified at the end after "--". For example: ior -a DAOS [ior_options] -- [daos_options] In the IOR options, the file name should be specified as a container uuid using -"-o ". If the "-E" option is given, then this UUID -shall denote an existing container created by a "matching" IOR -run. Otherwise, IOR will create a new container with this UUID. -In the latter case, one may use uuidgen(1) to generate the UUID -of the new container. +"-o ". If the "-E" option is given, then this UUID shall denote +an existing container created by a "matching" IOR run. Otherwise, IOR will +create a new container with this UUID. In the latter case, one may use +uuidgen(1) to generate the UUID of the new container. The DAOS options include: @@ -50,15 +49,18 @@ Optional Options: -k : flag to kill a rank during IO -o : specific object class -When benchmarking write performance, one likely do not want -"-W", which causes the write phase to do one additional memory -copy for every I/O. This is due to IOR's assumption that when a -DAOS_Xfer() call returns the buffer may be released. Therefore, -random data is written when "-W" is absent, while data is copied -from IOR buffers when "-W" is present. - -See doc/USER_GUIDE for all options and directives. Note that not -all combinations of options are supported. +When benchmarking write performance, one likely does not want "-W", which causes +the write phase to do one additional memory copy for every I/O. This is due to +IOR's assumption that when a DAOS_Xfer() call returns the buffer may be +released. Therefore, random data is written when "-W" is absent, while data is +copied from IOR buffers when "-W" is present. + +See doc/USER_GUIDE for all options and directives. Note that not all +combinations of options are supported. For example specifying an epoch to access +and running ior with multiple iterations would cause all iterations other than +first one to fail because the epoch will be committed in the first iteration. In +that case, the epoch should not be specified and the DAOS driver would choose +the epoch to access automatically on each iteration. Examples that should work include: From fd34b08e05ab448089415dfa30bdbfb26a3604bf Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Tue, 18 Sep 2018 16:48:59 +0100 Subject: [PATCH 012/206] Added patch fix O_DIRECT again. --- src/utilities.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/utilities.c b/src/utilities.c index 7c997c14..b71ce41d 100755 --- a/src/utilities.c +++ b/src/utilities.c @@ -16,6 +16,10 @@ # include "config.h" #endif +#ifdef __linux__ +# define _GNU_SOURCE /* Needed for O_DIRECT in fcntl */ +#endif /* __linux__ */ + #include #include #include From 44e8f03917b8096106802b311efe17d762e3a77c Mon Sep 17 00:00:00 2001 From: Jean-Yves VET Date: Mon, 17 Sep 2018 17:57:14 +0200 Subject: [PATCH 013/206] Fix bug in computing max write and read bandwidths This patch fixes a bug when computing the maximum bandwidths in short summary output. --- src/ior-output.c | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/src/ior-output.c b/src/ior-output.c index 24a2e6ac..385e63a7 100644 --- a/src/ior-output.c +++ b/src/ior-output.c @@ -612,8 +612,8 @@ void PrintShortSummary(IOR_test_t * test) { IOR_param_t *params = &test->params; IOR_results_t *results = test->results; - double max_write = 0.0; - double max_read = 0.0; + double max_write_bw = 0.0; + double max_read_bw = 0.0; double bw; int reps; int i; @@ -625,33 +625,31 @@ void PrintShortSummary(IOR_test_t * test) reps = params->repetitions; - max_write = results[0].writeTime; - max_read = results[0].readTime; for (i = 0; i < reps; i++) { bw = (double)results[i].aggFileSizeForBW / results[i].writeTime; - max_write = MAX(bw, max_write); + max_write_bw = MAX(bw, max_write_bw); bw = (double)results[i].aggFileSizeForBW / results[i].readTime; - max_read = MAX(bw, max_read); + max_read_bw = MAX(bw, max_read_bw); } if(outputFormat == OUTPUT_DEFAULT){ if (params->writeFile) { fprintf(out_resultfile, "Max Write: %.2f MiB/sec (%.2f MB/sec)\n", - max_write/MEBIBYTE, max_write/MEGABYTE); + max_write_bw/MEBIBYTE, max_write_bw/MEGABYTE); } if (params->readFile) { fprintf(out_resultfile, "Max Read: %.2f MiB/sec (%.2f MB/sec)\n", - max_read/MEBIBYTE, max_read/MEGABYTE); + max_read_bw/MEBIBYTE, max_read_bw/MEGABYTE); } }else if (outputFormat == OUTPUT_JSON){ PrintNamedSectionStart("max"); if (params->writeFile) { - PrintKeyValDouble("writeMiB", max_write/MEBIBYTE); - PrintKeyValDouble("writeMB", max_write/MEGABYTE); + PrintKeyValDouble("writeMiB", max_write_bw/MEBIBYTE); + PrintKeyValDouble("writeMB", max_write_bw/MEGABYTE); } if (params->readFile) { - PrintKeyValDouble("readMiB", max_read/MEBIBYTE); - PrintKeyValDouble("readMB", max_read/MEGABYTE); + PrintKeyValDouble("readMiB", max_read_bw/MEBIBYTE); + PrintKeyValDouble("readMB", max_read_bw/MEGABYTE); } PrintEndSection(); } From 2bcaea564e20ea5a47d1bcae61fc76e2e990f0f3 Mon Sep 17 00:00:00 2001 From: Jean-Yves VET Date: Mon, 17 Sep 2018 11:38:57 +0200 Subject: [PATCH 014/206] Initialize only the required backends Context: IOR initializes all available backends. If one backend fails to initialize IOR cannot be used. This patch makes IOR initialize only the backends which will be used. The initialization is done after that the parameters are checked so that the help message can still be dispayed is something goes wrong. --- src/aiori.c | 102 ++++++++++++++++++++++++++++++++++++---------- src/aiori.h | 4 +- src/ior.c | 10 ++--- src/ior.h | 4 +- src/mdtest-main.c | 4 +- 5 files changed, 92 insertions(+), 32 deletions(-) diff --git a/src/aiori.c b/src/aiori.c index f5d5719f..292ef190 100644 --- a/src/aiori.c +++ b/src/aiori.c @@ -12,6 +12,8 @@ * \******************************************************************************/ +#include +#include #include "aiori.h" #if defined(HAVE_SYS_STATVFS_H) @@ -131,34 +133,90 @@ char* aiori_get_version() return ""; } -static int is_initialized = FALSE; +static bool is_initialized = false; -void aiori_initialize(){ - if (is_initialized) return; - is_initialized = TRUE; +static void init_or_fini_internal(const ior_aiori_t *test_backend, + const bool init) +{ + if (init) + { + if (test_backend->initialize) + test_backend->initialize(); + } + else + { + if (test_backend->finalize) + test_backend->finalize(); + } +} - /* Sanity check, we were compiled with SOME backend, right? */ - if (0 == aiori_count ()) { - ERR("No IO backends compiled into aiori. " - "Run 'configure --with-', and recompile."); - } +static void init_or_fini(IOR_test_t *tests, const bool init) +{ + /* Sanity check, we were compiled with SOME backend, right? */ + if (0 == aiori_count ()) { + ERR("No IO backends compiled into aiori. " + "Run 'configure --with-', and recompile."); + } - for (ior_aiori_t **tmp = available_aiori ; *tmp != NULL; ++tmp) { - if((*tmp)->initialize){ - (*tmp)->initialize(); - } - } + /* Pointer to the initialize of finalize function */ + + + /* if tests is NULL, initialize or finalize all available backends */ + if (tests == NULL) + { + for (ior_aiori_t **tmp = available_aiori ; *tmp != NULL; ++tmp) + init_or_fini_internal(*tmp, init); + + return; + } + + for (IOR_test_t *t = tests; t != NULL; t = t->next) + { + IOR_param_t *params = &t->params; + assert(params != NULL); + + const ior_aiori_t *test_backend = params->backend; + assert(test_backend != NULL); + + init_or_fini_internal(test_backend, init); + } } -void aiori_finalize(){ - if (! is_initialized) return; - is_initialized = FALSE; - for (ior_aiori_t **tmp = available_aiori ; *tmp != NULL; ++tmp) { - if((*tmp)->finalize){ - (*tmp)->finalize(); - } - } +/** + * Initialize IO backends. + * + * @param[in] tests Pointers to the first test + * + * This function initializes all backends which will be used. If tests is NULL + * all available backends are initialized. + */ +void aiori_initialize(IOR_test_t *tests) +{ + if (is_initialized) + return; + + init_or_fini(tests, true); + + is_initialized = true; +} + +/** + * Finalize IO backends. + * + * @param[in] tests Pointers to the first test + * + * This function finalizes all backends which were used. If tests is NULL + * all available backends are finialized. + */ +void aiori_finalize(IOR_test_t *tests) +{ + if (!is_initialized) + return; + + is_initialized = false; + + init_or_fini(tests, false); } const ior_aiori_t *aiori_select (const char *api) diff --git a/src/aiori.h b/src/aiori.h index 0b0ffda8..feb19b1a 100755 --- a/src/aiori.h +++ b/src/aiori.h @@ -97,8 +97,8 @@ extern ior_aiori_t s3_plus_aiori; extern ior_aiori_t s3_emc_aiori; extern ior_aiori_t rados_aiori; -void aiori_initialize(); -void aiori_finalize(); +void aiori_initialize(IOR_test_t * tests); +void aiori_finalize(IOR_test_t * tests); const ior_aiori_t *aiori_select (const char *api); int aiori_count (void); void aiori_supported_apis(char * APIs); diff --git a/src/ior.c b/src/ior.c index 57c186cb..cac5a002 100755 --- a/src/ior.c +++ b/src/ior.c @@ -98,8 +98,6 @@ int ior_main(int argc, char **argv) out_logfile = stdout; out_resultfile = stdout; - aiori_initialize(); - /* * check -h option from commandline without starting MPI; */ @@ -123,6 +121,8 @@ int ior_main(int argc, char **argv) InitTests(tests_head, mpi_comm_world); verbose = tests_head->params.verbose; + aiori_initialize(tests_head); + PrintHeader(argc, argv); /* perform each test */ @@ -151,11 +151,11 @@ int ior_main(int argc, char **argv) /* display finish time */ PrintTestEnds(); - DestroyTests(tests_head); - MPI_CHECK(MPI_Finalize(), "cannot finalize MPI"); - aiori_finalize(); + aiori_finalize(tests_head); + + DestroyTests(tests_head); return totalErrorCount; } diff --git a/src/ior.h b/src/ior.h index c283be79..67198c38 100755 --- a/src/ior.h +++ b/src/ior.h @@ -77,9 +77,11 @@ typedef struct IO_BUFFERS * USER_GUIDE */ +struct ior_aiori; + typedef struct { - const void * backend; + const struct ior_aiori * backend; char * debug; /* debug info string */ unsigned int mode; /* file permissions */ unsigned int openFlags; /* open flags (see also ) */ diff --git a/src/mdtest-main.c b/src/mdtest-main.c index 854456f6..8f4e6085 100644 --- a/src/mdtest-main.c +++ b/src/mdtest-main.c @@ -2,12 +2,12 @@ #include "aiori.h" int main(int argc, char **argv) { - aiori_initialize(); + aiori_initialize(NULL); MPI_Init(&argc, &argv); mdtest_run(argc, argv, MPI_COMM_WORLD, stdout); MPI_Finalize(); - aiori_finalize(); + aiori_finalize(NULL); return 0; } From 7d2464f7331ad41923c46e8e9f2b3da17a1349e1 Mon Sep 17 00:00:00 2001 From: Rob Latham Date: Thu, 20 Sep 2018 15:43:41 +0000 Subject: [PATCH 015/206] HDF5: option to enable collective metadata In HDF5-1.10 the library has an option to read and write the metadata of a file collectively. One has to opt-in to this optimization with a property list but particularly at large scale it can have tremendous performance implications. Further information: - https://support.hdfgroup.org/HDF5/docNewFeatures/CollMDataIO/RFC-CollectiveMetadataReads.pdf - https://support.hdfgroup.org/HDF5/doc/RM/RM_H5P.html#Property-SetAllCollMetadataOps --- configure.ac | 4 ++++ doc/sphinx/userDoc/options.rst | 3 +++ src/aiori-HDF5.c | 11 +++++++++++ src/ior.h | 1 + src/parse_options.c | 2 ++ 5 files changed, 21 insertions(+) diff --git a/configure.ac b/configure.ac index a59cba4d..a3c41b03 100755 --- a/configure.ac +++ b/configure.ac @@ -98,8 +98,12 @@ AC_ARG_WITH([hdf5], AM_CONDITIONAL([USE_HDF5_AIORI], [test x$with_hdf5 = xyes]) AM_COND_IF([USE_HDF5_AIORI],[ AC_DEFINE([USE_HDF5_AIORI], [], [Build HDF5 backend AIORI]) + AC_SEARCH_LIBS([H5Pset_all_coll_metadata_ops], [hdf5]) + AC_CHECK_FUNCS([H5Pset_all_coll_metadata_ops]) ]) + + # HDFS support AC_ARG_WITH([hdfs], [AS_HELP_STRING([--with-hdfs], diff --git a/doc/sphinx/userDoc/options.rst b/doc/sphinx/userDoc/options.rst index bb7eb921..8b441269 100644 --- a/doc/sphinx/userDoc/options.rst +++ b/doc/sphinx/userDoc/options.rst @@ -289,6 +289,9 @@ HDF5-ONLY * setAlignment - HDF5 alignment in bytes (e.g.: 8, 4k, 2m, 1g) [1] + * collectiveMetadata - enable HDF5 collective metadata (available since + HDF5-1.10.0) + MPIIO-, HDF5-, AND NCMPI-ONLY ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ * collective - uses collective operations for access [0=FALSE] diff --git a/src/aiori-HDF5.c b/src/aiori-HDF5.c index ffcda2a1..173abb4b 100755 --- a/src/aiori-HDF5.c +++ b/src/aiori-HDF5.c @@ -228,6 +228,17 @@ static void *HDF5_Open(char *testFileName, IOR_param_t * param) param->setAlignment), "cannot set alignment"); +#ifdef HAVE_H5PSET_ALL_COLL_METADATA_OPS + if (param->collective_md) { + /* more scalable metadata */ + + HDF5_CHECK(H5Pset_all_coll_metadata_ops(accessPropList, 1), + "cannot set collective md read"); + HDF5_CHECK(H5Pset_coll_metadata_write(accessPropList, 1), + "cannot set collective md write"); + } +#endif + /* open file */ if (param->open == WRITE) { /* WRITE */ *fd = H5Fcreate(testFileName, fd_mode, diff --git a/src/ior.h b/src/ior.h index c283be79..b75299db 100755 --- a/src/ior.h +++ b/src/ior.h @@ -175,6 +175,7 @@ typedef struct char* URI; /* "path" to target object */ size_t part_number; /* multi-part upload increment (PER-RANK!) */ char* UploadId; /* key for multi-part-uploads */ + int collective_md; /* use collective metatata optimization */ /* RADOS variables */ rados_t rados_cluster; /* RADOS cluster handle */ diff --git a/src/parse_options.c b/src/parse_options.c index a8f81b02..710447a0 100755 --- a/src/parse_options.c +++ b/src/parse_options.c @@ -300,6 +300,8 @@ void DecodeDirective(char *line, IOR_param_t *params) params->numTasks = atoi(value); } else if (strcasecmp(option, "summaryalways") == 0) { params->summary_every_test = atoi(value); + } else if (strcasecmp(option, "collectiveMetadata") == 0) { + params->collective_md = atoi(value); } else { if (rank == 0) fprintf(out_logfile, "Unrecognized parameter \"%s\"\n", From f03e7c6778956d96e51a1ff69a87363d076488ad Mon Sep 17 00:00:00 2001 From: Jean-Yves VET Date: Thu, 20 Sep 2018 11:25:34 +0200 Subject: [PATCH 016/206] Make write verification work even without read test Context: IOR outputs errors when the '-w -W' flags are used without '-r'. Write a file using with check write option should be possible even without setting read. This patch fixes a condition which was introduced for HDFS to remove RDWR flag in some particular cases. Write check was set with the write only flag but it requieres the read flag. --- src/parse_options.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/parse_options.c b/src/parse_options.c index a8f81b02..7830f0d3 100755 --- a/src/parse_options.c +++ b/src/parse_options.c @@ -88,9 +88,8 @@ static void CheckRunSettings(IOR_test_t *tests) * (We assume int-valued params are exclusively 0 or 1.) */ if ((params->openFlags & IOR_RDWR) - && ((params->readFile | params->checkRead) - ^ (params->writeFile | params->checkWrite)) - && (params->openFlags & IOR_RDWR)) { + && ((params->readFile | params->checkRead | params->checkWrite) + ^ params->writeFile)) { params->openFlags &= ~(IOR_RDWR); if (params->readFile | params->checkRead) { @@ -100,7 +99,6 @@ static void CheckRunSettings(IOR_test_t *tests) else params->openFlags |= IOR_WRONLY; } - } } From 87091dde4cebf4c1883e3c0442fd7b5afe2dbee2 Mon Sep 17 00:00:00 2001 From: "Glenn K. Lockwood" Date: Wed, 19 Sep 2018 20:56:51 -0400 Subject: [PATCH 017/206] updated metadata for 3.2 --- ChangeLog | 137 ------------------------------------- META | 2 +- NEWS | 198 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 195 insertions(+), 142 deletions(-) delete mode 100755 ChangeLog diff --git a/ChangeLog b/ChangeLog deleted file mode 100755 index f30050a3..00000000 --- a/ChangeLog +++ /dev/null @@ -1,137 +0,0 @@ -Changes in IOR-3.0.0 - - * Reorganization of the build system. Now uses autoconf/automake. - N.B. Windows suport is not included. Patches welcome. - * Much code refactoring. - * Final summary table is printed after all tests have finished. - * Error messages significantly improved. - * Drop all "undocumented changes". If they are worth having, they - need to be implemented well and documented. - -Changes in IOR-2.10.3 - * bug 2962326 "Segmentation Fault When Summarizing Results" fixed - * bug 2786285 "-Wrong number of parameters to function H5Dcreate" fixed - (NOTE: to compile for HDF5 1.6 libs use "-D H5_USE_16_API") - * bug 1992514 "delay (-d) doesn't work" fixed -Contributed by demyn@users.sourceforge.net - * Ported to Windows. Required changes related to 'long' types, which on Windows - are always 32-bits, even on 64-bit systems. Missing system headers and - functions acount for most of the remaining changes. - New files for Windows: - - IOR/ior.vcproj - Visual C project file - - IOR/src/C/win/getopt.{h,c} - GNU getopt() support - See updates in the USER_GUIDE for build instructions on Windows. - * Fixed bug in incrementing transferCount - * Fixed bugs in SummarizeResults with mismatched format specifiers - * Fixed inconsistencies between option names, -h output, and the USER_GUIDE. - -Changes in IOR-2.10.2: -Hodson, 8/18/2008: - - * extend existing random I/O capabilities and enhance performance - output statistics. - -Fixes in IOR-2.10.1: - * added '-J' setAlignment option for HDF5 alignment in bytes; default value - is 1, which does not set alignment - * changed how HDF5 and PnetCDF calculate performance -- formerly each used - the size of the stat()'ed file; changed it to be number of data bytes - transferred. these library-generated files can have large headers and - filler as well as sparse file content - * found potential overflow error in cast -- using IOR_offset_t, not int now - * replaced HAVE_HDF5_NO_FILL macro to instead directly check if H5_VERS_MAJOR - H5_VERS_MINOR are defined; if so, then must be HDF5-1.6.x or higher for - no-fill usage. - * corrected IOR_GetFileSize() function to point to HDF5 and NCMPI versions of - IOR_GetFileSize() calls - * changed the netcdf dataset from 1D array to 4D array, where the 4 dimensions - are: [segmentCount][numTasksWorld][numTransfers][transferSize] - This patch from Wei-keng Liao allows for file sizes > 4GB (provided no - single dimension is > 4GB). - * finalized random-capability release - * changed statvfs() to be for __sun (SunOS) only - * retired Python GUI - -Fixes in IOR-2.10.0.1: - * Cleaned up WriteOrRead(), reducing much to smaller functions. - * Added random capability for transfer offset. - * modified strtok(NULL, " \t\r\n") in ExtractHints() so no trailing characters - * added capability to set hints in NCMPI - -Fixes in IOR-2.9.6.1: - * for 'pvfs2:' filename prefix, now skips DisplayFreeSpace(); formerly this - caused a problem with statvfs() - * changed gethostname() to MPI_Get_processor_name() so for certain cases - when gethostname() would only return the frontend node name - * added SunOS compiler settings for makefile - * updated O_DIRECT usage for SunOS compliance - * changed statfs() to instead use statvfs() for SunOS compliance - * renamed compiler directive _USE_LUSTRE to _MANUALLY_SET_LUSTRE_STRIPING - -Fixes in IOR-2.9.5: - * Wall clock deviation time relabeled to be "Start time skew across all tasks". - * Added notification for "Using reorderTasks '-C' (expecting block, not cyclic, - task assignment)" - * Corrected bug with read performance with stonewalling (was using full size, - stat'ed file instead of bytes transfered). - -Fixes in IOR-2.9.4: - * Now using IOR_offset_t instead of int for tmpOffset in IOR.c:WriteOrRead(). - Formerly, this would cause error in file(s) > 2GB for ReadCheck. The - more-commonly-used WriteCheck option was not affected by this. - -Fixes in IOR-2.9.3: - * Changed FILE_DELIMITER from ':' to '@'. - * Any time skew between nodes is automatically adjusted so that all nodes - are calibrated to the root node's time. - * Wall clock deviation time is still reported, but have removed the warning - message that was generated for this. (Didn't add much new information, - just annoyed folks.) - * The '-j' outlierThreshold option now is set to 0 (off) as default. To use - this, set to a positive integer N, and any task who's time (for open, - access, etc.) is not within N seconds of the mean of all the tasks will show - up as an outlier. - -Fixes in IOR-2.9.2: - * Simple cleanup of error messages, format, etc. - * Changed error reporting so that with VERBOSITY=2 (-vv) any error encountered - is displayed. Previously, this was with VERBOSITY=3, along with full test - parameters, environment, and all access times of operations. - * Added deadlineForStonewalling option (-D). This option is used for measuring - the amount of data moved in a fixed time. After the barrier, each task - starts its own timer, begins moving data, and the stops moving data at a - prearranged time. Instead of measuring the amount of time to move a fixed - amount of data, this option measures the amount of data moved in a fixed - amount of time. The objective is to prevent tasks slow to complete from - skewing the performance. - -Fixes in IOR-2.9.1: - * Updated test script to run through file1:file2 cases. - * Program exit code is now total numbers of errors (both writecheck and - readcheck for all iterations), unless quitOnError (-q) is set. - * For various failure situations, replace abort with warning, including: - - failed uname() for platform name now gives warning - - failed unlink() of data file now gives warning - - failed fsync() of data file now gives warning - - failed open() of nonexistent script file now gives warning - * Changed labelling for error checking output to be (hopefully) more - clear in details on errors for diagnostics. - * Another fix for -o file1:file2 option. - * Corrected bug in GetTestFileName() -- now possible to handle -o file1:file2 - cases for file-per-proc. - -Fixes in IOR-2.9.0: - * Improved checkRead option to reread data from different node (to avoid - cache) and then compare both reads. - * Added outlierThreshold (-j) option to give warning if any task is more than - this number of seconds from the mean of all participating tasks. If so, the - task is identified, its time (start, elapsed create, elapsed transfer, - elapsed close, or end) is reported, as is the mean and standard deviation for - all tasks. The default for this is 5, i.e. any task not within 5 seconds of - the mean for those times is displayed. This value can be set with - outlierThreshold= or -j . - * Correct for clock skew between nodes - if skew greater than wallclock - deviation threshold (WC_DEV_THRESHOLD) in seconds, then broadcast root - node's timestamp and adjust by the difference. WC_DEV_THRESHOLD currently - set to 5. - * Added a Users Guide. diff --git a/META b/META index e7b78ff2..a4ea8733 100755 --- a/META +++ b/META @@ -1,3 +1,3 @@ Package: ior -Version: 3.1.0 +Version: 3.2.0 Release: 0 diff --git a/NEWS b/NEWS index 368697aa..dedb82a9 100644 --- a/NEWS +++ b/NEWS @@ -1,9 +1,199 @@ IOR NEWS -======== +================================================================================ -Last updated 2017-06 +Last updated 2018-08 -3.0.2 +Version 3.2 +-------------------------------------------------------------------------------- +New major features: + +- mdtest now included as a frontend for the IOR aiori backend (Nathan Hjelm, + LANL) +- Added mmap AIORI (Li Dongyang, DDN) +- Added RADOS AIORI (Shane Snyder, ANL) +- Added IME AIORI (Jean-Yves Vet, DDN) +- Added stonewalling for mdtest (Julian Kunkel, U Reading) + +New minor features: + +- Dropped support for PLFS AIORI (John Bent) +- Added stoneWallingWearOut functionality to allow stonewalling to ensure that + each MPI rank writes the same amount of data and captures the effects of slow + processes (Julian Kunkel, DKRZ) +- Added support for JSON output (Enno Zickler, U Hamburg; Julian Kunkel, U + Reading) +- Added dummy AIORI (Julian Kunkel, U Reading) +- 0870ad78b3b022bcd30392a8990ba4e5090d6e34 + +General user improvements: + +- BeeGFS parameter support (Oliver Steffen, ThinkParQ) +- Semantics of `-R` now compares to expected signature (`-G`) (Julian Kunkel, + DKRZ) +- Improved macOS support for ncmpi (Vinson Leung) +- Added more complete documentation (Enno Zickler, U Hamburg) +- Assorted bugfixes and code refactoring (Adam Moody, LLNL; Julian Kunkel, U + Reading; Enno Zickler, U Hamburg; Nathan Hjelm, LANL; Rob Latham, ANL; + Jean-Yves Vet, DDN) +- More robust support for non-POSIX-backed AIORIs (Shane Snyder, ANL) + +General developer improvements: + +- Improvements to build process (Nathan Hjelm, LANL; Ian Kirker, UCL) +- Added continuous integration support (Enno Zickler, U Hamburg) +- Added better support for automated testing (Julian Kunkel, U Reading) + +Version 3.0.2 +-------------------------------------------------------------------------------- - IOR and mdtest now share a common codebase. This will make it easier -run performance benchmarks on new hardware. + run performance benchmarks on new hardware. +- Note: this version was never properly released + +Version 3.0.0 +-------------------------------------------------------------------------------- +- Reorganization of the build system. Now uses autoconf/automake. N.B. Windows + support is not included. Patches welcome. +- Much code refactoring. +- Final summary table is printed after all tests have finished. +- Error messages significantly improved. +- Drop all "undocumented changes". If they are worth having, they need to be + implemented well and documented. + +Version 2.10.3 +-------------------------------------------------------------------------------- +- bug 2962326 "Segmentation Fault When Summarizing Results" fixed +- bug 2786285 "-Wrong number of parameters to function H5Dcreate" fixed + (NOTE: to compile for HDF5 1.6 libs use "-D H5_USE_16_API") +- bug 1992514 "delay (-d) doesn't work" fixed + +Contributed by demyn@users.sourceforge.net +- Ported to Windows. Required changes related to 'long' types, which on Windows + are always 32-bits, even on 64-bit systems. Missing system headers and + functions acount for most of the remaining changes. + New files for Windows: + - IOR/ior.vcproj - Visual C project file + - IOR/src/C/win/getopt.{h,c} - GNU getopt() support + See updates in the USER_GUIDE for build instructions on Windows. +- Fixed bug in incrementing transferCount +- Fixed bugs in SummarizeResults with mismatched format specifiers +- Fixed inconsistencies between option names, -h output, and the USER_GUIDE. + +Version 2.10.2 +-------------------------------------------------------------------------------- +- Extend existing random I/O capabilities and enhance performance + output statistics. (Hodson, 8/18/2008) + +Version 2.10.1 +-------------------------------------------------------------------------------- +- Added '-J' setAlignment option for HDF5 alignment in bytes; default value + is 1, which does not set alignment +- Changed how HDF5 and PnetCDF calculate performance -- formerly each used + the size of the stat()'ed file; changed it to be number of data bytes + transferred. these library-generated files can have large headers and + filler as well as sparse file content +- Found potential overflow error in cast -- using IOR_offset_t, not int now +- Replaced HAVE_HDF5_NO_FILL macro to instead directly check if H5_VERS_MAJOR + H5_VERS_MINOR are defined; if so, then must be HDF5-1.6.x or higher for + no-fill usage. +- Corrected IOR_GetFileSize() function to point to HDF5 and NCMPI versions of + IOR_GetFileSize() calls +- Changed the netcdf dataset from 1D array to 4D array, where the 4 dimensions + are: [segmentCount][numTasksWorld][numTransfers][transferSize] + This patch from Wei-keng Liao allows for file sizes > 4GB (provided no + single dimension is > 4GB). +- Finalized random-capability release +- Changed statvfs() to be for __sun (SunOS) only +- Retired Python GUI + +Version 2.10.0.1 +-------------------------------------------------------------------------------- +- Cleaned up WriteOrRead(), reducing much to smaller functions. +- Added random capability for transfer offset. +- Modified strtok(NULL, " \t\r\n") in ExtractHints() so no trailing characters +- Added capability to set hints in NCMPI + +Version 2.9.6.1 +-------------------------------------------------------------------------------- +- For 'pvfs2:' filename prefix, now skips DisplayFreeSpace(); formerly this + caused a problem with statvfs() +- Changed gethostname() to MPI_Get_processor_name() so for certain cases + when gethostname() would only return the frontend node name +- Added SunOS compiler settings for makefile +- Updated O_DIRECT usage for SunOS compliance +- Changed statfs() to instead use statvfs() for SunOS compliance +- Renamed compiler directive _USE_LUSTRE to _MANUALLY_SET_LUSTRE_STRIPING + +Version 2.9.5 +-------------------------------------------------------------------------------- +- Wall clock deviation time relabeled to be "Start time skew across all tasks". +- Added notification for "Using reorderTasks '-C' (expecting block, not cyclic, + task assignment)" +- Corrected bug with read performance with stonewalling (was using full size, + stat'ed file instead of bytes transfered). + +Version 2.9.4 +-------------------------------------------------------------------------------- +- Now using IOR_offset_t instead of int for tmpOffset in IOR.c:WriteOrRead(). + Formerly, this would cause error in file(s) > 2GB for ReadCheck. The + more-commonly-used WriteCheck option was not affected by this. + +Version 2.9.3 +-------------------------------------------------------------------------------- +- Changed FILE_DELIMITER from ':' to '@'. +- Any time skew between nodes is automatically adjusted so that all nodes + are calibrated to the root node's time. +- Wall clock deviation time is still reported, but have removed the warning + message that was generated for this. (Didn't add much new information, + just annoyed folks.) +- The '-j' outlierThreshold option now is set to 0 (off) as default. To use + this, set to a positive integer N, and any task who's time (for open, + access, etc.) is not within N seconds of the mean of all the tasks will show + up as an outlier. + +Version 2.9.2 +-------------------------------------------------------------------------------- +- Simple cleanup of error messages, format, etc. +- Changed error reporting so that with VERBOSITY=2 (-vv) any error encountered + is displayed. Previously, this was with VERBOSITY=3, along with full test + parameters, environment, and all access times of operations. +- Added deadlineForStonewalling option (-D). This option is used for measuring + the amount of data moved in a fixed time. After the barrier, each task + starts its own timer, begins moving data, and the stops moving data at a + prearranged time. Instead of measuring the amount of time to move a fixed + amount of data, this option measures the amount of data moved in a fixed + amount of time. The objective is to prevent tasks slow to complete from + skewing the performance. + +Version 2.9.1 +-------------------------------------------------------------------------------- +- Updated test script to run through file1:file2 cases. +- Program exit code is now total numbers of errors (both writecheck and + readcheck for all iterations), unless quitOnError (-q) is set. +- For various failure situations, replace abort with warning, including: + - failed uname() for platform name now gives warning + - failed unlink() of data file now gives warning + - failed fsync() of data file now gives warning + - failed open() of nonexistent script file now gives warning +- Changed labelling for error checking output to be (hopefully) more + clear in details on errors for diagnostics. +- Another fix for -o file1:file2 option. +- Corrected bug in GetTestFileName() -- now possible to handle -o file1:file2 + cases for file-per-proc. + +Version 2.9.0 +-------------------------------------------------------------------------------- +- Improved checkRead option to reread data from different node (to avoid cache) + and then compare both reads. +- Added outlierThreshold (-j) option to give warning if any task is more than + this number of seconds from the mean of all participating tasks. If so, the + task is identified, its time (start, elapsed create, elapsed transfer, + elapsed close, or end) is reported, as is the mean and standard deviation for + all tasks. The default for this is 5, i.e. any task not within 5 seconds of + the mean for those times is displayed. This value can be set with + outlierThreshold= or -j . +- Correct for clock skew between nodes - if skew greater than wallclock + deviation threshold (WC_DEV_THRESHOLD) in seconds, then broadcast root + node's timestamp and adjust by the difference. WC_DEV_THRESHOLD currently + set to 5. +- Added a Users Guide. From 8c727fa99c0ff4b5dbc128e7c29449d8d03c6615 Mon Sep 17 00:00:00 2001 From: Jean-Yves VET Date: Wed, 19 Sep 2018 19:06:05 +0200 Subject: [PATCH 018/206] Simplify and factorize a few functions This patch factorizes some duplicated code. It also simplifies the way times are stored to prepare next patch. --- src/ior-output.c | 122 +++++++++------------ src/ior.c | 271 +++++++++++++++++++++-------------------------- src/iordef.h | 1 - 3 files changed, 169 insertions(+), 225 deletions(-) diff --git a/src/ior-output.c b/src/ior-output.c index 385e63a7..64bc7303 100644 --- a/src/ior-output.c +++ b/src/ior-output.c @@ -11,8 +11,6 @@ extern char **environ; -static struct results *bw_values(int reps, IOR_results_t * measured, int offset, double *vals); -static struct results *ops_values(int reps, IOR_results_t * measured, int offset, IOR_offset_t transfer_size, double *vals); static double mean_of_array_of_doubles(double *values, int len); static void PPDouble(int leftjustify, double number, char *append); static void PrintNextToken(); @@ -456,7 +454,55 @@ void ShowSetup(IOR_param_t *params) fflush(out_resultfile); } +static struct results *bw_ops_values(const int reps, IOR_results_t *measured, + const int offset, IOR_offset_t transfer_size, + const double *vals) +{ + struct results *r; + int i; + r = (struct results *)malloc(sizeof(struct results) + + (reps * sizeof(double))); + if (r == NULL) + ERR("malloc failed"); + r->val = (double *)&r[1]; + + for (i = 0; i < reps; i++, measured++) { + r->val[i] = (double) *((IOR_offset_t*) ((char*)measured + offset)) + / transfer_size / vals[i]; + + if (i == 0) { + r->min = r->val[i]; + r->max = r->val[i]; + r->sum = 0.0; + } + r->min = MIN(r->min, r->val[i]); + r->max = MAX(r->max, r->val[i]); + r->sum += r->val[i]; + } + r->mean = r->sum / reps; + r->var = 0.0; + for (i = 0; i < reps; i++) { + r->var += pow((r->mean - r->val[i]), 2); + } + r->var = r->var / reps; + r->sd = sqrt(r->var); + + return r; +} + +static struct results *bw_values(const int reps, IOR_results_t *measured, + const int offset, const double *vals) +{ + return bw_ops_values(reps, measured, offset, 1, vals); +} + +static struct results *ops_values(const int reps, IOR_results_t *measured, + const int offset, IOR_offset_t transfer_size, + const double *vals) +{ + return bw_ops_values(reps, measured, offset, transfer_size, vals); +} /* * Summarize results @@ -735,78 +781,6 @@ static void PPDouble(int leftjustify, double number, char *append) fprintf(out_resultfile, format, number, append); } - - -static struct results *bw_values(int reps, IOR_results_t * measured, int offset, double *vals) -{ - struct results *r; - int i; - - r = (struct results *) malloc(sizeof(struct results) + (reps * sizeof(double))); - if (r == NULL) - ERR("malloc failed"); - r->val = (double *)&r[1]; - - for (i = 0; i < reps; i++, measured++) { - - r->val[i] = (double) *((IOR_offset_t*) ((char*)measured + offset)) / vals[i]; - if (i == 0) { - r->min = r->val[i]; - r->max = r->val[i]; - r->sum = 0.0; - } - r->min = MIN(r->min, r->val[i]); - r->max = MAX(r->max, r->val[i]); - r->sum += r->val[i]; - } - r->mean = r->sum / reps; - r->var = 0.0; - for (i = 0; i < reps; i++) { - r->var += pow((r->mean - r->val[i]), 2); - } - r->var = r->var / reps; - r->sd = sqrt(r->var); - - return r; -} - -static struct results *ops_values(int reps, IOR_results_t * measured, int offset, - IOR_offset_t transfer_size, - double *vals) -{ - struct results *r; - int i; - - r = (struct results *)malloc(sizeof(struct results) - + (reps * sizeof(double))); - if (r == NULL) - ERR("malloc failed"); - r->val = (double *)&r[1]; - - for (i = 0; i < reps; i++, measured++) { - r->val[i] = (double) *((IOR_offset_t*) ((char*)measured + offset)) - / transfer_size / vals[i]; - if (i == 0) { - r->min = r->val[i]; - r->max = r->val[i]; - r->sum = 0.0; - } - r->min = MIN(r->min, r->val[i]); - r->max = MAX(r->max, r->val[i]); - r->sum += r->val[i]; - } - r->mean = r->sum / reps; - r->var = 0.0; - for (i = 0; i < reps; i++) { - r->var += pow((r->mean - r->val[i]), 2); - } - r->var = r->var / reps; - r->sd = sqrt(r->var); - - return r; -} - - static double mean_of_array_of_doubles(double *values, int len) { double tot = 0.0; diff --git a/src/ior.c b/src/ior.c index cac5a002..edfb8065 100755 --- a/src/ior.c +++ b/src/ior.c @@ -36,6 +36,7 @@ #include "utilities.h" #include "parse_options.h" +#define IOR_NB_TIMERS 6 /* file scope globals */ extern char **environ; @@ -48,8 +49,9 @@ static char **ParseFileName(char *, int *); static void InitTests(IOR_test_t * , MPI_Comm); static void TestIoSys(IOR_test_t *); static void ValidateTests(IOR_param_t *); -static IOR_offset_t WriteOrRead(IOR_param_t * test, IOR_results_t * results, void *fd, int access, IOR_io_buffers* ioBuffers); -static void WriteTimes(IOR_param_t *, double **, int, int); +static IOR_offset_t WriteOrRead(IOR_param_t *test, IOR_results_t *results, + void *fd, const int access, + IOR_io_buffers *ioBuffers); IOR_test_t * ior_run(int argc, char **argv, MPI_Comm world_com, FILE * world_out){ IOR_test_t *tests_head; @@ -256,32 +258,23 @@ DisplayOutliers(int numTasks, /* * Check for outliers in start/end times and elapsed create/xfer/close times. */ -static void CheckForOutliers(IOR_param_t * test, double **timer, int rep, - int access) +static void +CheckForOutliers(IOR_param_t *test, const double *timer, const int access) { - int shift; - - if (access == WRITE) { - shift = 0; - } else { /* READ */ - shift = 6; - } - - DisplayOutliers(test->numTasks, timer[shift + 0][rep], + DisplayOutliers(test->numTasks, timer[0], "start time", access, test->outlierThreshold); DisplayOutliers(test->numTasks, - timer[shift + 1][rep] - timer[shift + 0][rep], + timer[1] - timer[0], "elapsed create time", access, test->outlierThreshold); DisplayOutliers(test->numTasks, - timer[shift + 3][rep] - timer[shift + 2][rep], + timer[3] - timer[2], "elapsed transfer time", access, test->outlierThreshold); DisplayOutliers(test->numTasks, - timer[shift + 5][rep] - timer[shift + 4][rep], + timer[5] - timer[4], "elapsed close time", access, test->outlierThreshold); - DisplayOutliers(test->numTasks, timer[shift + 5][rep], "end time", + DisplayOutliers(test->numTasks, timer[5], "end time", access, test->outlierThreshold); - } /* @@ -848,54 +841,46 @@ static char *PrependDir(IOR_param_t * test, char *rootDir) /* * Reduce test results, and show if verbose set. */ - -static void ReduceIterResults(IOR_test_t *test, double **timer, int rep, - int access) +static void +ReduceIterResults(IOR_test_t *test, double *timer, const int rep, const int access) { - double reduced[12] = { 0 }; - double diff[6]; - double *diff_subset; - double totalTime; - double bw; - int i; - MPI_Op op; + double reduced[IOR_NB_TIMERS] = { 0 }; + double diff[IOR_NB_TIMERS / 2 + 1]; + double totalTime; + double bw; + int i; + MPI_Op op; - assert(access == WRITE || access == READ); + assert(access == WRITE || access == READ); /* Find the minimum start time of the even numbered timers, and the maximum finish time for the odd numbered timers */ - for (i = 0; i < 12; i++) { + for (i = 0; i < IOR_NB_TIMERS; i++) { op = i % 2 ? MPI_MAX : MPI_MIN; - MPI_CHECK(MPI_Reduce(&timer[i][rep], &reduced[i], 1, MPI_DOUBLE, + MPI_CHECK(MPI_Reduce(&timer[i], &reduced[i], 1, MPI_DOUBLE, op, 0, testComm), "MPI_Reduce()"); } - if (rank != 0) { - /* Only rank 0 tallies and prints the results. */ - return; - } + /* Only rank 0 tallies and prints the results. */ + if (rank != 0) + return; - /* Calculate elapsed times and throughput numbers */ - for (i = 0; i < 6; i++) { - diff[i] = reduced[2 * i + 1] - reduced[2 * i]; - } - if (access == WRITE) { - totalTime = reduced[5] - reduced[0]; - test->results[rep].writeTime = totalTime; - diff_subset = &diff[0]; - } else { /* READ */ - totalTime = reduced[11] - reduced[6]; - test->results[rep].readTime = totalTime; - diff_subset = &diff[3]; - } + /* Calculate elapsed times and throughput numbers */ + for (i = 0; i < IOR_NB_TIMERS / 2; i++) + diff[i] = reduced[2 * i + 1] - reduced[2 * i]; - if (verbose < VERBOSE_0) { - return; - } + totalTime = reduced[5] - reduced[0]; - bw = (double)test->results[rep].aggFileSizeForBW / totalTime; + double *time = (access == WRITE) ? &test->results[rep].writeTime : + &test->results[rep].readTime; - PrintReducedResult(test, access, bw, diff_subset, totalTime, rep); + *time = totalTime; + + if (verbose < VERBOSE_0) + return; + + bw = (double)test->results[rep].aggFileSizeForBW / totalTime; + PrintReducedResult(test, access, bw, diff, totalTime, rep); } /* @@ -1116,7 +1101,72 @@ static void *HogMemory(IOR_param_t *params) return buf; } +/* + * Write times taken during each iteration of the test. + */ +static void +WriteTimes(IOR_param_t *test, const double *timer, const int iteration, + const int access) +{ + char timerName[MAX_STR]; + for (int i = 0; i < IOR_NB_TIMERS; i++) { + + if (access == WRITE) { + switch (i) { + case 0: + strcpy(timerName, "write open start"); + break; + case 1: + strcpy(timerName, "write open stop"); + break; + case 2: + strcpy(timerName, "write start"); + break; + case 3: + strcpy(timerName, "write stop"); + break; + case 4: + strcpy(timerName, "write close start"); + break; + case 5: + strcpy(timerName, "write close stop"); + break; + default: + strcpy(timerName, "invalid timer"); + break; + } + } + else { + switch (i) { + case 0: + strcpy(timerName, "read open start"); + break; + case 1: + strcpy(timerName, "read open stop"); + break; + case 2: + strcpy(timerName, "read start"); + break; + case 3: + strcpy(timerName, "read stop"); + break; + case 4: + strcpy(timerName, "read close start"); + break; + case 5: + strcpy(timerName, "read close stop"); + break; + default: + strcpy(timerName, "invalid timer"); + break; + } + } + fprintf(out_logfile, "Test %d: Iter=%d, Task=%d, Time=%f, %s\n", + test->id, iteration, (int)rank, timer[i], + timerName); + } +} /* * Using the test parameters, run iteration(s) of single test. */ @@ -1125,10 +1175,10 @@ static void TestIoSys(IOR_test_t *test) IOR_param_t *params = &test->params; IOR_results_t *results = test->results; char testFileName[MAX_STR]; - double *timer[12]; + double timer[IOR_NB_TIMERS]; double startTime; int pretendRank; - int i, rep; + int rep; void *fd; MPI_Group orig_group, new_group; int range[3]; @@ -1175,13 +1225,6 @@ static void TestIoSys(IOR_test_t *test) } params->tasksPerNode = CountTasksPerNode(testComm); - /* setup timers */ - for (i = 0; i < 12; i++) { - timer[i] = (double *)malloc(params->repetitions * sizeof(double)); - if (timer[i] == NULL) - ERR("malloc failed"); - } - /* bind I/O calls to specific API */ backend = aiori_select(params->api); @@ -1257,9 +1300,9 @@ static void TestIoSys(IOR_test_t *test) params->stoneWallingWearOutIterations = params_saved_wearout; MPI_CHECK(MPI_Barrier(testComm), "barrier error"); params->open = WRITE; - timer[0][rep] = GetTimeStamp(); + timer[0] = GetTimeStamp(); fd = backend->create(testFileName, params); - timer[1][rep] = GetTimeStamp(); + timer[1] = GetTimeStamp(); if (params->intraTestBarriers) MPI_CHECK(MPI_Barrier(testComm), "barrier error"); @@ -1268,20 +1311,20 @@ static void TestIoSys(IOR_test_t *test) "Commencing write performance test: %s", CurrentTimeString()); } - timer[2][rep] = GetTimeStamp(); + timer[2] = GetTimeStamp(); dataMoved = WriteOrRead(params, & results[rep], fd, WRITE, &ioBuffers); if (params->verbose >= VERBOSE_4) { fprintf(out_logfile, "* data moved = %llu\n", dataMoved); fflush(out_logfile); } - timer[3][rep] = GetTimeStamp(); + timer[3] = GetTimeStamp(); if (params->intraTestBarriers) MPI_CHECK(MPI_Barrier(testComm), "barrier error"); - timer[4][rep] = GetTimeStamp(); + timer[4] = GetTimeStamp(); backend->close(fd, params); - timer[5][rep] = GetTimeStamp(); + timer[5] = GetTimeStamp(); MPI_CHECK(MPI_Barrier(testComm), "barrier error"); /* get the size of the file just written */ @@ -1296,7 +1339,7 @@ static void TestIoSys(IOR_test_t *test) WriteTimes(params, timer, rep, WRITE); ReduceIterResults(test, timer, rep, WRITE); if (params->outlierThreshold) { - CheckForOutliers(params, timer, rep, WRITE); + CheckForOutliers(params, timer, WRITE); } /* check if in this round we run write with stonewalling */ @@ -1396,9 +1439,9 @@ static void TestIoSys(IOR_test_t *test) DelaySecs(params->interTestDelay); MPI_CHECK(MPI_Barrier(testComm), "barrier error"); params->open = READ; - timer[6][rep] = GetTimeStamp(); + timer[0] = GetTimeStamp(); fd = backend->open(testFileName, params); - timer[7][rep] = GetTimeStamp(); + timer[1] = GetTimeStamp(); if (params->intraTestBarriers) MPI_CHECK(MPI_Barrier(testComm), "barrier error"); @@ -1407,15 +1450,15 @@ static void TestIoSys(IOR_test_t *test) "Commencing read performance test: %s", CurrentTimeString()); } - timer[8][rep] = GetTimeStamp(); + timer[2] = GetTimeStamp(); dataMoved = WriteOrRead(params, & results[rep], fd, operation_flag, &ioBuffers); - timer[9][rep] = GetTimeStamp(); + timer[3] = GetTimeStamp(); if (params->intraTestBarriers) MPI_CHECK(MPI_Barrier(testComm), "barrier error"); - timer[10][rep] = GetTimeStamp(); + timer[4] = GetTimeStamp(); backend->close(fd, params); - timer[11][rep] = GetTimeStamp(); + timer[5] = GetTimeStamp(); /* get the size of the file just read */ results[rep].aggFileSizeFromStat = @@ -1430,7 +1473,7 @@ static void TestIoSys(IOR_test_t *test) WriteTimes(params, timer, rep, READ); ReduceIterResults(test, timer, rep, READ); if (params->outlierThreshold) { - CheckForOutliers(params, timer, rep, READ); + CheckForOutliers(params, timer, READ); } } @@ -1465,9 +1508,6 @@ static void TestIoSys(IOR_test_t *test) if (hog_buf != NULL) free(hog_buf); - for (i = 0; i < 12; i++) { - free(timer[i]); - } /* Sync with the tasks that did not participate in this test */ MPI_CHECK(MPI_Barrier(mpi_comm_world), "barrier error"); @@ -1831,7 +1871,8 @@ static IOR_offset_t WriteOrReadSingle(IOR_offset_t pairCnt, IOR_offset_t *offset * Write or Read data to file(s). This loops through the strides, writing * out the data to each block in transfer sizes, until the remainder left is 0. */ -static IOR_offset_t WriteOrRead(IOR_param_t * test, IOR_results_t * results, void *fd, int access, IOR_io_buffers* ioBuffers) +static IOR_offset_t WriteOrRead(IOR_param_t *test, IOR_results_t *results, + void *fd, const int access, IOR_io_buffers *ioBuffers) { int errors = 0; IOR_offset_t transferCount = 0; @@ -1910,73 +1951,3 @@ static IOR_offset_t WriteOrRead(IOR_param_t * test, IOR_results_t * results, voi } return (dataMoved); } - -/* - * Write times taken during each iteration of the test. - */ -static void -WriteTimes(IOR_param_t * test, double **timer, int iteration, int writeOrRead) -{ - char accessType[MAX_STR]; - char timerName[MAX_STR]; - int i, start = 0, stop = 0; - - if (writeOrRead == WRITE) { - start = 0; - stop = 6; - strcpy(accessType, "WRITE"); - } else if (writeOrRead == READ) { - start = 6; - stop = 12; - strcpy(accessType, "READ"); - } else { - ERR("incorrect WRITE/READ option"); - } - - for (i = start; i < stop; i++) { - switch (i) { - case 0: - strcpy(timerName, "write open start"); - break; - case 1: - strcpy(timerName, "write open stop"); - break; - case 2: - strcpy(timerName, "write start"); - break; - case 3: - strcpy(timerName, "write stop"); - break; - case 4: - strcpy(timerName, "write close start"); - break; - case 5: - strcpy(timerName, "write close stop"); - break; - case 6: - strcpy(timerName, "read open start"); - break; - case 7: - strcpy(timerName, "read open stop"); - break; - case 8: - strcpy(timerName, "read start"); - break; - case 9: - strcpy(timerName, "read stop"); - break; - case 10: - strcpy(timerName, "read close start"); - break; - case 11: - strcpy(timerName, "read close stop"); - break; - default: - strcpy(timerName, "invalid timer"); - break; - } - fprintf(out_logfile, "Test %d: Iter=%d, Task=%d, Time=%f, %s\n", - test->id, iteration, (int)rank, timer[i][iteration], - timerName); - } -} diff --git a/src/iordef.h b/src/iordef.h index 23febd5a..0eea3114 100755 --- a/src/iordef.h +++ b/src/iordef.h @@ -96,7 +96,6 @@ enum OutputFormat_t{ #define WRITECHECK 1 #define READ 2 #define READCHECK 3 -#define CHECK 4 /* verbosity settings */ #define VERBOSE_0 0 From 7a7655e95916c9211e95769b2cebe27f08c7f98e Mon Sep 17 00:00:00 2001 From: Jean-Yves VET Date: Wed, 19 Sep 2018 23:39:25 +0200 Subject: [PATCH 019/206] Fix erroneous write bandwidth with stonewalling Context: write and read results from the same iteration use the same length value in Bytes. When stonewalling is used the size variates depending on the performance of the access. This leads to wrong max bandwidths reported for writes as shown in the following example: write 10052 ... read 9910 ... write 10022 ... read 9880 ... write 10052 ... read 9894 ... Max Write: 9371.43 MiB/sec (9826.66 MB/sec) Max Read: 9910.48 MiB/sec (10391.89 MB/sec) This patch makes IOR separate variables used for read and write tests. --- src/ior-output.c | 57 ++++++++++++++++++++++----------------- src/ior.c | 70 ++++++++++++++++++++++++++---------------------- src/ior.h | 16 ++++++----- 3 files changed, 80 insertions(+), 63 deletions(-) diff --git a/src/ior-output.c b/src/ior-output.c index 64bc7303..a6b8d9c0 100644 --- a/src/ior-output.c +++ b/src/ior-output.c @@ -385,10 +385,12 @@ void ShowTestStart(IOR_param_t *test) void ShowTestEnd(IOR_test_t *tptr){ if(rank == 0 && tptr->params.stoneWallingWearOut){ + + size_t pairs_accessed = tptr->results->write.pairs_accessed; if (tptr->params.stoneWallingStatusFile){ - StoreStoneWallingIterations(tptr->params.stoneWallingStatusFile, tptr->results->pairs_accessed); + StoreStoneWallingIterations(tptr->params.stoneWallingStatusFile, pairs_accessed); }else{ - fprintf(out_logfile, "Pairs deadlineForStonewallingaccessed: %lld\n", (long long) tptr->results->pairs_accessed); + fprintf(out_logfile, "Pairs deadlineForStonewallingaccessed: %ld\n", pairs_accessed); } } PrintEndSection(); @@ -455,8 +457,8 @@ void ShowSetup(IOR_param_t *params) } static struct results *bw_ops_values(const int reps, IOR_results_t *measured, - const int offset, IOR_offset_t transfer_size, - const double *vals) + IOR_offset_t transfer_size, + const double *vals, const int access) { struct results *r; int i; @@ -468,7 +470,10 @@ static struct results *bw_ops_values(const int reps, IOR_results_t *measured, r->val = (double *)&r[1]; for (i = 0; i < reps; i++, measured++) { - r->val[i] = (double) *((IOR_offset_t*) ((char*)measured + offset)) + IOR_point_t *point = (access == WRITE) ? &measured->write : + &measured->read; + + r->val[i] = ((double) (point->aggFileSizeForBW)) / transfer_size / vals[i]; if (i == 0) { @@ -492,24 +497,22 @@ static struct results *bw_ops_values(const int reps, IOR_results_t *measured, } static struct results *bw_values(const int reps, IOR_results_t *measured, - const int offset, const double *vals) + const double *vals, const int access) { - return bw_ops_values(reps, measured, offset, 1, vals); + return bw_ops_values(reps, measured, 1, vals, access); } static struct results *ops_values(const int reps, IOR_results_t *measured, - const int offset, IOR_offset_t transfer_size, - const double *vals) + IOR_offset_t transfer_size, + const double *vals, const int access) { - return bw_ops_values(reps, measured, offset, transfer_size, vals); + return bw_ops_values(reps, measured, transfer_size, vals, access); } /* * Summarize results - * - * operation is typically "write" or "read" */ -static void PrintLongSummaryOneOperation(IOR_test_t *test, int times_offset, char *operation) +static void PrintLongSummaryOneOperation(IOR_test_t *test, const int access) { IOR_param_t *params = &test->params; IOR_results_t *results = test->results; @@ -524,14 +527,20 @@ static void PrintLongSummaryOneOperation(IOR_test_t *test, int times_offset, cha double * times = malloc(sizeof(double)* reps); for(int i=0; i < reps; i++){ - times[i] = *(double*)((char*) & results[i] + times_offset); + IOR_point_t *point = (access == WRITE) ? &results[i].write : + &results[i].read; + times[i] = point->time; } - bw = bw_values(reps, results, offsetof(IOR_results_t, aggFileSizeForBW), times); - ops = ops_values(reps, results, offsetof(IOR_results_t, aggFileSizeForBW), params->transferSize, times); + bw = bw_values(reps, results, times, access); + ops = ops_values(reps, results, params->transferSize, times, access); + + IOR_point_t *point = (access == WRITE) ? &results[0].write : + &results[0].read; + if(outputFormat == OUTPUT_DEFAULT){ - fprintf(out_resultfile, "%-9s ", operation); + fprintf(out_resultfile, "%-9s ", access == WRITE ? "write" : "read"); fprintf(out_resultfile, "%10.2f ", bw->max / MEBIBYTE); fprintf(out_resultfile, "%10.2f ", bw->min / MEBIBYTE); fprintf(out_resultfile, "%10.2f ", bw->mean / MEBIBYTE); @@ -553,13 +562,13 @@ static void PrintLongSummaryOneOperation(IOR_test_t *test, int times_offset, cha fprintf(out_resultfile, "%6lld ", params->segmentCount); fprintf(out_resultfile, "%8lld ", params->blockSize); fprintf(out_resultfile, "%8lld ", params->transferSize); - fprintf(out_resultfile, "%9.1f ", (float)results[0].aggFileSizeForBW / MEBIBYTE); + fprintf(out_resultfile, "%9.1f ", (float)point->aggFileSizeForBW / MEBIBYTE); fprintf(out_resultfile, "%3s ", params->api); fprintf(out_resultfile, "%6d", params->referenceNumber); fprintf(out_resultfile, "\n"); }else if (outputFormat == OUTPUT_JSON){ PrintStartSection(); - PrintKeyVal("operation", operation); + PrintKeyVal("operation", access == WRITE ? "write" : "read"); PrintKeyVal("API", params->api); PrintKeyValInt("TestID", params->id); PrintKeyValInt("ReferenceNumber", params->referenceNumber); @@ -586,7 +595,7 @@ static void PrintLongSummaryOneOperation(IOR_test_t *test, int times_offset, cha PrintKeyValDouble("OPsMean", ops->mean); PrintKeyValDouble("OPsSD", ops->sd); PrintKeyValDouble("MeanTime", mean_of_array_of_doubles(times, reps)); - PrintKeyValDouble("xsizeMiB", (double) results[0].aggFileSizeForBW / MEBIBYTE); + PrintKeyValDouble("xsizeMiB", (double) point->aggFileSizeForBW / MEBIBYTE); PrintEndSection(); }else if (outputFormat == OUTPUT_CSV){ @@ -604,9 +613,9 @@ void PrintLongSummaryOneTest(IOR_test_t *test) IOR_param_t *params = &test->params; if (params->writeFile) - PrintLongSummaryOneOperation(test, offsetof(IOR_results_t, writeTime), "write"); + PrintLongSummaryOneOperation(test, WRITE); if (params->readFile) - PrintLongSummaryOneOperation(test, offsetof(IOR_results_t, readTime), "read"); + PrintLongSummaryOneOperation(test, READ); } void PrintLongSummaryHeader() @@ -672,9 +681,9 @@ void PrintShortSummary(IOR_test_t * test) reps = params->repetitions; for (i = 0; i < reps; i++) { - bw = (double)results[i].aggFileSizeForBW / results[i].writeTime; + bw = (double)results[i].write.aggFileSizeForBW / results[i].write.time; max_write_bw = MAX(bw, max_write_bw); - bw = (double)results[i].aggFileSizeForBW / results[i].readTime; + bw = (double)results[i].read.aggFileSizeForBW / results[i].read.time; max_read_bw = MAX(bw, max_read_bw); } diff --git a/src/ior.c b/src/ior.c index edfb8065..e8c477fc 100755 --- a/src/ior.c +++ b/src/ior.c @@ -281,30 +281,33 @@ CheckForOutliers(IOR_param_t *test, const double *timer, const int access) * Check if actual file size equals expected size; if not use actual for * calculating performance rate. */ -static void CheckFileSize(IOR_test_t *test, IOR_offset_t dataMoved, int rep) +static void CheckFileSize(IOR_test_t *test, IOR_offset_t dataMoved, int rep, + const int access) { IOR_param_t *params = &test->params; IOR_results_t *results = test->results; + IOR_point_t *point = (access == WRITE) ? &results[rep].write : + &results[rep].read; - MPI_CHECK(MPI_Allreduce(&dataMoved, & results[rep].aggFileSizeFromXfer, + MPI_CHECK(MPI_Allreduce(&dataMoved, &point->aggFileSizeFromXfer, 1, MPI_LONG_LONG_INT, MPI_SUM, testComm), "cannot total data moved"); if (strcasecmp(params->api, "HDF5") != 0 && strcasecmp(params->api, "NCMPI") != 0) { if (verbose >= VERBOSE_0 && rank == 0) { if ((params->expectedAggFileSize - != results[rep].aggFileSizeFromXfer) - || (results[rep].aggFileSizeFromStat - != results[rep].aggFileSizeFromXfer)) { + != point->aggFileSizeFromXfer) + || (point->aggFileSizeFromStat + != point->aggFileSizeFromXfer)) { fprintf(out_logfile, "WARNING: Expected aggregate file size = %lld.\n", (long long) params->expectedAggFileSize); fprintf(out_logfile, "WARNING: Stat() of aggregate file size = %lld.\n", - (long long) results[rep].aggFileSizeFromStat); + (long long) point->aggFileSizeFromStat); fprintf(out_logfile, "WARNING: Using actual aggregate bytes moved = %lld.\n", - (long long) results[rep].aggFileSizeFromXfer); + (long long) point->aggFileSizeFromXfer); if(params->deadlineForStonewalling){ fprintf(out_logfile, "WARNING: maybe caused by deadlineForStonewalling\n"); @@ -312,7 +315,8 @@ static void CheckFileSize(IOR_test_t *test, IOR_offset_t dataMoved, int rep) } } } - results[rep].aggFileSizeForBW = results[rep].aggFileSizeFromXfer; + + point->aggFileSizeForBW = point->aggFileSizeFromXfer; } /* @@ -871,15 +875,15 @@ ReduceIterResults(IOR_test_t *test, double *timer, const int rep, const int acce totalTime = reduced[5] - reduced[0]; - double *time = (access == WRITE) ? &test->results[rep].writeTime : - &test->results[rep].readTime; + IOR_point_t *point = (access == WRITE) ? &test->results[rep].write : + &test->results[rep].read; - *time = totalTime; + point->time = totalTime; if (verbose < VERBOSE_0) return; - bw = (double)test->results[rep].aggFileSizeForBW / totalTime; + bw = (double)point->aggFileSizeForBW / totalTime; PrintReducedResult(test, access, bw, diff, totalTime, rep); } @@ -1312,7 +1316,7 @@ static void TestIoSys(IOR_test_t *test) CurrentTimeString()); } timer[2] = GetTimeStamp(); - dataMoved = WriteOrRead(params, & results[rep], fd, WRITE, &ioBuffers); + dataMoved = WriteOrRead(params, &results[rep], fd, WRITE, &ioBuffers); if (params->verbose >= VERBOSE_4) { fprintf(out_logfile, "* data moved = %llu\n", dataMoved); fflush(out_logfile); @@ -1328,12 +1332,12 @@ static void TestIoSys(IOR_test_t *test) MPI_CHECK(MPI_Barrier(testComm), "barrier error"); /* get the size of the file just written */ - results[rep].aggFileSizeFromStat = + results[rep].write.aggFileSizeFromStat = backend->get_file_size(params, testComm, testFileName); /* check if stat() of file doesn't equal expected file size, use actual amount of byte moved */ - CheckFileSize(test, dataMoved, rep); + CheckFileSize(test, dataMoved, rep, WRITE); if (verbose >= VERBOSE_3) WriteTimes(params, timer, rep, WRITE); @@ -1344,7 +1348,7 @@ static void TestIoSys(IOR_test_t *test) /* check if in this round we run write with stonewalling */ if(params->deadlineForStonewalling > 0){ - params->stoneWallingWearOutIterations = results[rep].pairs_accessed; + params->stoneWallingWearOutIterations = results[rep].write.pairs_accessed; } } @@ -1372,7 +1376,7 @@ static void TestIoSys(IOR_test_t *test) GetTestFileName(testFileName, params); params->open = WRITECHECK; fd = backend->open(testFileName, params); - dataMoved = WriteOrRead(params, & results[rep], fd, WRITECHECK, &ioBuffers); + dataMoved = WriteOrRead(params, &results[rep], fd, WRITECHECK, &ioBuffers); backend->close(fd, params); rankOffset = 0; } @@ -1451,7 +1455,7 @@ static void TestIoSys(IOR_test_t *test) CurrentTimeString()); } timer[2] = GetTimeStamp(); - dataMoved = WriteOrRead(params, & results[rep], fd, operation_flag, &ioBuffers); + dataMoved = WriteOrRead(params, &results[rep], fd, operation_flag, &ioBuffers); timer[3] = GetTimeStamp(); if (params->intraTestBarriers) MPI_CHECK(MPI_Barrier(testComm), @@ -1461,13 +1465,13 @@ static void TestIoSys(IOR_test_t *test) timer[5] = GetTimeStamp(); /* get the size of the file just read */ - results[rep].aggFileSizeFromStat = + results[rep].read.aggFileSizeFromStat = backend->get_file_size(params, testComm, testFileName); /* check if stat() of file doesn't equal expected file size, use actual amount of byte moved */ - CheckFileSize(test, dataMoved, rep); + CheckFileSize(test, dataMoved, rep, READ); if (verbose >= VERBOSE_3) WriteTimes(params, timer, rep, READ); @@ -1882,6 +1886,8 @@ static IOR_offset_t WriteOrRead(IOR_param_t *test, IOR_results_t *results, IOR_offset_t dataMoved = 0; /* for data rate calculation */ double startForStonewall; int hitStonewall; + IOR_point_t *point = ((access == WRITE) || (access == WRITECHECK)) ? + &results->write : &results->read; /* initialize values */ pretendRank = (rank + rankOffset) % test->numTasks; @@ -1910,35 +1916,35 @@ static IOR_offset_t WriteOrRead(IOR_param_t *test, IOR_results_t *results, } long long data_moved_ll = (long long) dataMoved; long long pairs_accessed_min = 0; - MPI_CHECK(MPI_Allreduce(& pairCnt, &results->pairs_accessed, + MPI_CHECK(MPI_Allreduce(& pairCnt, &point->pairs_accessed, 1, MPI_LONG_LONG_INT, MPI_MAX, testComm), "cannot reduce pairs moved"); double stonewall_runtime = GetTimeStamp() - startForStonewall; - results->stonewall_time = stonewall_runtime; + point->stonewall_time = stonewall_runtime; MPI_CHECK(MPI_Reduce(& pairCnt, & pairs_accessed_min, 1, MPI_LONG_LONG_INT, MPI_MIN, 0, testComm), "cannot reduce pairs moved"); - MPI_CHECK(MPI_Reduce(& data_moved_ll, & results->stonewall_min_data_accessed, + MPI_CHECK(MPI_Reduce(& data_moved_ll, &point->stonewall_min_data_accessed, 1, MPI_LONG_LONG_INT, MPI_MIN, 0, testComm), "cannot reduce pairs moved"); - MPI_CHECK(MPI_Reduce(& data_moved_ll, & results->stonewall_avg_data_accessed, + MPI_CHECK(MPI_Reduce(& data_moved_ll, &point->stonewall_avg_data_accessed, 1, MPI_LONG_LONG_INT, MPI_SUM, 0, testComm), "cannot reduce pairs moved"); if(rank == 0){ fprintf(out_logfile, "stonewalling pairs accessed min: %lld max: %zu -- min data: %.1f GiB mean data: %.1f GiB time: %.1fs\n", - pairs_accessed_min, results->pairs_accessed, - results->stonewall_min_data_accessed /1024.0 / 1024 / 1024, results->stonewall_avg_data_accessed / 1024.0 / 1024 / 1024 / test->numTasks , results->stonewall_time); - results->stonewall_min_data_accessed *= test->numTasks; + pairs_accessed_min, point->pairs_accessed, + point->stonewall_min_data_accessed /1024.0 / 1024 / 1024, point->stonewall_avg_data_accessed / 1024.0 / 1024 / 1024 / test->numTasks , point->stonewall_time); + point->stonewall_min_data_accessed *= test->numTasks; } if(pairs_accessed_min == pairCnt){ - results->stonewall_min_data_accessed = 0; - results->stonewall_avg_data_accessed = 0; + point->stonewall_min_data_accessed = 0; + point->stonewall_avg_data_accessed = 0; } - if(pairCnt != results->pairs_accessed){ + if(pairCnt != point->pairs_accessed){ // some work needs still to be done ! - for(; pairCnt < results->pairs_accessed; pairCnt++ ) { + for(; pairCnt < point->pairs_accessed; pairCnt++ ) { dataMoved += WriteOrReadSingle(pairCnt, offsetArray, pretendRank, & transferCount, & errors, test, fd, ioBuffers, access); } } }else{ - results->pairs_accessed = pairCnt; + point->pairs_accessed = pairCnt; } diff --git a/src/ior.h b/src/ior.h index 67198c38..e76a64db 100755 --- a/src/ior.h +++ b/src/ior.h @@ -204,12 +204,9 @@ typedef struct int intraTestBarriers; /* barriers between open/op and op/close */ } IOR_param_t; -/* each pointer is to an array, each of length equal to the number of - repetitions in the test */ +/* each pointer for a single test */ typedef struct { - double writeTime; - double readTime; - int errors; + double time; size_t pairs_accessed; // number of I/Os done, useful for deadlineForStonewalling double stonewall_time; @@ -219,16 +216,21 @@ typedef struct { IOR_offset_t aggFileSizeFromStat; IOR_offset_t aggFileSizeFromXfer; IOR_offset_t aggFileSizeForBW; +} IOR_point_t; + +typedef struct { + int errors; + IOR_point_t write; + IOR_point_t read; } IOR_results_t; /* define the queuing structure for the test parameters */ typedef struct IOR_test_t { IOR_param_t params; - IOR_results_t *results; /* This is an array of reps times IOR_results_t */ + IOR_results_t *results; struct IOR_test_t *next; } IOR_test_t; - IOR_test_t *CreateTest(IOR_param_t *init_params, int test_num); void AllocResults(IOR_test_t *test); From e448ff266ddc1fd68113b82940b51b92326d051d Mon Sep 17 00:00:00 2001 From: "Glenn K. Lockwood" Date: Fri, 21 Sep 2018 18:35:48 -0400 Subject: [PATCH 020/206] fix various issues related to release packages - travis now tests the packaged source to detect missing source/headers - basic tests are less sensitive to the directory from where they are run - fixed some missing files from the `make dist` manifest - updated the format of NEWS to work with `make dist` --- .gitignore | 1 + .travis.yml | 9 +------- Makefile.am | 10 ++++++++- NEWS | 7 +----- src/Makefile.am | 2 +- testing/basic-tests.sh | 2 +- testing/test-lib.sh | 21 ++++++++++-------- travis-build.sh | 26 +++++++++++++++++++++++ travis-test.sh | 48 ++++++++++++++++++++++++++++++++++++++++++ 9 files changed, 100 insertions(+), 26 deletions(-) create mode 100755 travis-build.sh create mode 100755 travis-test.sh diff --git a/.gitignore b/.gitignore index 73dd929e..d065f5b7 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ config/config.sub config/depcomp config/install-sh config/missing +config/test-driver configure contrib/.deps/ contrib/Makefile diff --git a/.travis.yml b/.travis.yml index 49ae8f76..29e091c6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,11 +28,4 @@ install: # aiori-S3.c to achive this. # GPFS # NOTE: Think GPFS need a license and is therefore not testable with travis. -before_script: ./bootstrap -script: mkdir build && cd build && ../configure --with-hdf5 && make && cd .. && ./testing/basic-tests.sh - - -# notifications: -# email: -# on_success: change # default: change -# on_failure: always # default: always +script: ./travis-build.sh && CONFIGURE_OPTS="--with-hdf5" ./travis-test.sh diff --git a/Makefile.am b/Makefile.am index f01f3628..d874a90e 100755 --- a/Makefile.am +++ b/Makefile.am @@ -1,5 +1,13 @@ MAKEFLAGS = --no-print-directory SUBDIRS = src doc contrib -EXTRA_DIST = META COPYRIGHT README.md ChangeLog +EXTRA_DIST = META COPYRIGHT README.md NEWS testing + # ACLOCAL_AMFLAGS needed for autoconf < 2.69 ACLOCAL_AMFLAGS = -I config + +# The basic-tests.sh scripts run MPI versions of IOR/mdtest and are therefore +# too complicated to run in the context of distclean. As such we reserve +# `make dist` and `make test` for simple test binaries that do not require any +# special environment. +#TESTS = testing/basic-tests.sh +#DISTCLEANFILES = -r test test_out diff --git a/NEWS b/NEWS index dedb82a9..4ff935d5 100644 --- a/NEWS +++ b/NEWS @@ -1,9 +1,4 @@ -IOR NEWS -================================================================================ - -Last updated 2018-08 - -Version 3.2 +Version 3.2.0 -------------------------------------------------------------------------------- New major features: diff --git a/src/Makefile.am b/src/Makefile.am index 0e0b9164..51fb8734 100755 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -5,7 +5,7 @@ if USE_CAPS bin_PROGRAMS += IOR MDTEST endif -noinst_HEADERS = ior.h utilities.h parse_options.h aiori.h iordef.h ior-internal.h option.h +noinst_HEADERS = ior.h utilities.h parse_options.h aiori.h iordef.h ior-internal.h option.h mdtest.h lib_LIBRARIES = libaiori.a libaiori_a_SOURCES = ior.c mdtest.c utilities.c parse_options.c ior-output.c option.c diff --git a/testing/basic-tests.sh b/testing/basic-tests.sh index c81b1291..8dd3befe 100755 --- a/testing/basic-tests.sh +++ b/testing/basic-tests.sh @@ -6,7 +6,7 @@ # You can override the defaults by setting the variables before invoking the script, or simply set them here... # Example: export IOR_EXTRA="-v -v -v" -ROOT=${0%/*} +ROOT="$(dirname ${BASH_SOURCE[0]})" source $ROOT/test-lib.sh diff --git a/testing/test-lib.sh b/testing/test-lib.sh index 54ab0b21..965c92a5 100644 --- a/testing/test-lib.sh +++ b/testing/test-lib.sh @@ -1,18 +1,21 @@ # Test script for basic IOR functionality testing various patterns -# It is kept as simple as possible and outputs the parameters used such that any test can be rerun easily. +# It is kept as simple as possible and outputs the parameters used such that any +# test can be rerun easily. -# You can override the defaults by setting the variables before invoking the script, or simply set them here... +# You can override the defaults by setting the variables before invoking the +# script, or simply set them here... # Example: export IOR_EXTRA="-v -v -v" IOR_MPIRUN=${IOR_MPIRUN:-mpiexec -np} -IOR_BIN_DIR=${IOR_BIN_DIR:-./build/src} -IOR_OUT=${IOR_OUT:-./build/test} +IOR_BIN_DIR=${IOR_BIN_DIR:-./src} +IOR_OUT=${IOR_OUT:-./test_logs} +IOR_TMP=${IOR_TMP:-/dev/shm} IOR_EXTRA=${IOR_EXTRA:-} # Add global options like verbosity MDTEST_EXTRA=${MDTEST_EXTRA:-} ################################################################################ mkdir -p ${IOR_OUT} -mkdir -p /dev/shm/mdest +mkdir -p ${IOR_TMP}/mdest ## Sanity check @@ -36,8 +39,8 @@ I=0 function IOR(){ RANKS=$1 shift - WHAT="${IOR_MPIRUN} $RANKS ${IOR_BIN_DIR}/ior ${@} ${IOR_EXTRA} -o /dev/shm/ior" - $WHAT 1>${IOR_OUT}/$I 2>&1 + WHAT="${IOR_MPIRUN} $RANKS ${IOR_BIN_DIR}/ior ${@} ${IOR_EXTRA} -o ${IOR_TMP}/ior" + $WHAT 1>"${IOR_OUT}/test_out.$I" 2>&1 if [[ $? != 0 ]]; then echo -n "ERR" ERRORS=$(($ERRORS + 1)) @@ -51,8 +54,8 @@ function IOR(){ function MDTEST(){ RANKS=$1 shift - WHAT="${IOR_MPIRUN} $RANKS ${IOR_BIN_DIR}/mdtest ${@} ${MDTEST_EXTRA} -d /dev/shm/mdest" - $WHAT 1>${IOR_OUT}/$I 2>&1 + WHAT="${IOR_MPIRUN} $RANKS ${IOR_BIN_DIR}/mdtest ${@} ${MDTEST_EXTRA} -d ${IOR_TMP}/mdest" + $WHAT 1>"${IOR_OUT}/test_out.$I" 2>&1 if [[ $? != 0 ]]; then echo -n "ERR" ERRORS=$(($ERRORS + 1)) diff --git a/travis-build.sh b/travis-build.sh new file mode 100755 index 00000000..6538d4f5 --- /dev/null +++ b/travis-build.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# +# Build the IOR source package. Returns the path to the built artifact. +# + +BASE_DIR="$(cd "${0%/*}" && pwd)" +if [ -z "$BASE_DIR" -o ! -d "$BASE_DIR" ]; then + echo "Cannot determine BASE_DIR (${BASE_DIR})" >&2 + exit 2 +fi +BUILD_DIR="${BASE_DIR}/build" + +PACKAGE="$(awk '/^Package/ {print $2}' $BASE_DIR/META)" +VERSION="$(awk '/^Version/ {print $2}' $BASE_DIR/META)" +DIST_TGZ="${PACKAGE}-${VERSION}.tar.gz" + +# Build the distribution +set -e +./bootstrap +test -d "$BUILD_DIR" && rm -rf "$BUILD_DIR" +mkdir -p "$BUILD_DIR" +cd "$BUILD_DIR" +$BASE_DIR/configure +set +e + +make dist && mv -v "${BUILD_DIR}/${DIST_TGZ}" "${BASE_DIR}/${DIST_TGZ}" diff --git a/travis-test.sh b/travis-test.sh new file mode 100755 index 00000000..dfc12357 --- /dev/null +++ b/travis-test.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# +# Test the IOR source package. This is a complicated alternative to +# the `make distcheck` option. +# + +# These options will be passed directly to the autoconf configure script +CONFIGURE_OPTS="${CONFIGURE_OPTS:-""}" + +BASE_DIR="$(cd "${0%/*}" && pwd)" +if [ -z "$BASE_DIR" -o ! -d "$BASE_DIR" ]; then + echo "Cannot determine BASE_DIR (${BASE_DIR})" >&2 + exit 2 +fi +PACKAGE="$(awk '/^Package/ {print $2}' $BASE_DIR/META)" +VERSION="$(awk '/^Version/ {print $2}' $BASE_DIR/META)" +DIST_TGZ="${BASE_DIR}/${PACKAGE}-${VERSION}.tar.gz" + +TEST_DIR="${BASE_DIR}/test" +INSTALL_DIR="${TEST_DIR}/_inst" + +if [ -z "$DIST_TGZ" -o ! -f "$DIST_TGZ" ]; then + echo "Cannot find DIST_TGZ ($DIST_TGZ)" >&2 + exit 1 +fi + +test -d "$TEST_DIR" && rm -rf "$TEST_DIR" +mkdir -p "$TEST_DIR" + +tar -C "$TEST_DIR" -zxf "${DIST_TGZ}" + +# Configure, make, and install from the source distribution +set -e +cd "$TEST_DIR/${PACKAGE}-${VERSION}" +./configure $CONFIGURE_OPTS "--prefix=$INSTALL_DIR" +make install +set +e + +# Run the MPI tests +export IOR_BIN_DIR="${INSTALL_DIR}/bin" +export IOR_OUT="${TEST_DIR}/test_logs" +export IOR_TMP="$(mktemp -d)" +source "${TEST_DIR}/${PACKAGE}-${VERSION}/testing/basic-tests.sh" + +# Clean up residual temporary directories (if this isn't running as root) +if [ -d "$IOR_TMP" -a "$(id -u)" -ne 0 -a ! -z "$IOR_TMP" ]; then + rm -rvf "$IOR_TMP" +fi From f472162784541e7e2f7876e876ef3b67ddec0481 Mon Sep 17 00:00:00 2001 From: Jean-Yves VET Date: Mon, 10 Sep 2018 18:43:12 +0200 Subject: [PATCH 021/206] Support old legacy name in backends Context: Some backends may have used different names in the past (like IME backend use to be IM). Legacy scripts may break. This patch adds a legacy name option in the aiori structure. Both name and legacy name work to select the interface. But the following warning is printed if the legacy name is used: ior WARNING: [legacy name] backend is deprecated use [name] instead. --- src/aiori-DUMMY.c | 1 + src/aiori-HDF5.c | 1 + src/aiori-HDFS.c | 21 ++++++------ src/aiori-IME.c | 1 + src/aiori-MPIIO.c | 1 + src/aiori-NCMPI.c | 1 + src/aiori-POSIX.c | 1 + src/aiori-RADOS.c | 1 + src/aiori-S3.c | 1 + src/aiori.c | 80 ++++++++++++++++++++++++++------------------- src/aiori.h | 3 +- src/mdtest.c | 6 ++-- src/parse_options.c | 3 +- 13 files changed, 74 insertions(+), 47 deletions(-) diff --git a/src/aiori-DUMMY.c b/src/aiori-DUMMY.c index 8e129c79..8cc0a571 100755 --- a/src/aiori-DUMMY.c +++ b/src/aiori-DUMMY.c @@ -136,6 +136,7 @@ static int DUMMY_stat (const char *path, struct stat *buf, IOR_param_t * param){ ior_aiori_t dummy_aiori = { "DUMMY", + NULL, DUMMY_Create, DUMMY_Open, DUMMY_Xfer, diff --git a/src/aiori-HDF5.c b/src/aiori-HDF5.c index ffcda2a1..ad8365a4 100755 --- a/src/aiori-HDF5.c +++ b/src/aiori-HDF5.c @@ -98,6 +98,7 @@ static int HDF5_Access(const char *, int, IOR_param_t *); ior_aiori_t hdf5_aiori = { .name = "HDF5", + .name_legacy = NULL, .create = HDF5_Create, .open = HDF5_Open, .xfer = HDF5_Xfer, diff --git a/src/aiori-HDFS.c b/src/aiori-HDFS.c index 8939fe5f..2d4dcb1b 100755 --- a/src/aiori-HDFS.c +++ b/src/aiori-HDFS.c @@ -115,6 +115,7 @@ static IOR_offset_t HDFS_GetFileSize(IOR_param_t *, MPI_Comm, char *); ior_aiori_t hdfs_aiori = { .name = "HDFS", + .name_legacy = NULL, .create = HDFS_Create, .open = HDFS_Open, .xfer = HDFS_Xfer, @@ -289,9 +290,9 @@ static void *HDFS_Create_Or_Open( char *testFileName, IOR_param_t *param, unsign * truncate each other's writes */ - if (( param->openFlags & IOR_WRONLY ) && - ( !param->filePerProc ) && - ( rank != 0 )) { + if (( param->openFlags & IOR_WRONLY ) && + ( !param->filePerProc ) && + ( rank != 0 )) { MPI_CHECK(MPI_Barrier(testComm), "barrier error"); } @@ -308,7 +309,7 @@ static void *HDFS_Create_Or_Open( char *testFileName, IOR_param_t *param, unsign param->transferSize, param->hdfs_replicas, param->hdfs_block_size); - } + } hdfs_file = hdfsOpenFile( param->hdfs_fs, testFileName, fd_oflags, @@ -323,12 +324,12 @@ static void *HDFS_Create_Or_Open( char *testFileName, IOR_param_t *param, unsign * For N-1 write, Rank 0 waits for the other ranks to open the file after it has. */ - if (( param->openFlags & IOR_WRONLY ) && - ( !param->filePerProc ) && - ( rank == 0 )) { + if (( param->openFlags & IOR_WRONLY ) && + ( !param->filePerProc ) && + ( rank == 0 )) { MPI_CHECK(MPI_Barrier(testComm), "barrier error"); - } + } if (param->verbose >= VERBOSE_4) { printf("<- HDFS_Create_Or_Open\n"); @@ -404,7 +405,7 @@ static IOR_offset_t HDFS_Xfer(int access, void *file, IOR_size_t * buffer, } if (param->verbose >= VERBOSE_4) { - printf("\thdfsWrite( 0x%llx, 0x%llx, 0x%llx, %lld)\n", + printf("\thdfsWrite( 0x%llx, 0x%llx, 0x%llx, %lld)\n", hdfs_fs, hdfs_file, ptr, remaining ); /* DEBUGGING */ } rc = hdfsWrite( hdfs_fs, hdfs_file, ptr, remaining ); @@ -426,7 +427,7 @@ static IOR_offset_t HDFS_Xfer(int access, void *file, IOR_size_t * buffer, } if (param->verbose >= VERBOSE_4) { - printf("\thdfsRead( 0x%llx, 0x%llx, 0x%llx, %lld)\n", + printf("\thdfsRead( 0x%llx, 0x%llx, 0x%llx, %lld)\n", hdfs_fs, hdfs_file, ptr, remaining ); /* DEBUGGING */ } rc = hdfsRead( hdfs_fs, hdfs_file, ptr, remaining ); diff --git a/src/aiori-IME.c b/src/aiori-IME.c index c8291b3e..08624951 100755 --- a/src/aiori-IME.c +++ b/src/aiori-IME.c @@ -63,6 +63,7 @@ extern MPI_Comm testComm; ior_aiori_t ime_aiori = { .name = "IME", + .name_legacy = "IM", .create = IME_Create, .open = IME_Open, .xfer = IME_Xfer, diff --git a/src/aiori-MPIIO.c b/src/aiori-MPIIO.c index 2ffdc141..5b817365 100755 --- a/src/aiori-MPIIO.c +++ b/src/aiori-MPIIO.c @@ -46,6 +46,7 @@ static void MPIIO_Fsync(void *, IOR_param_t *); ior_aiori_t mpiio_aiori = { .name = "MPIIO", + .name_legacy = NULL, .create = MPIIO_Create, .open = MPIIO_Open, .xfer = MPIIO_Xfer, diff --git a/src/aiori-NCMPI.c b/src/aiori-NCMPI.c index b096c508..0f391dcf 100755 --- a/src/aiori-NCMPI.c +++ b/src/aiori-NCMPI.c @@ -62,6 +62,7 @@ static int NCMPI_Access(const char *, int, IOR_param_t *); ior_aiori_t ncmpi_aiori = { .name = "NCMPI", + .name_legacy = NULL, .create = NCMPI_Create, .open = NCMPI_Open, .xfer = NCMPI_Xfer, diff --git a/src/aiori-POSIX.c b/src/aiori-POSIX.c index 0dd712e0..ef46ed2e 100755 --- a/src/aiori-POSIX.c +++ b/src/aiori-POSIX.c @@ -77,6 +77,7 @@ static void POSIX_Fsync(void *, IOR_param_t *); ior_aiori_t posix_aiori = { .name = "POSIX", + .name_legacy = NULL, .create = POSIX_Create, .open = POSIX_Open, .xfer = POSIX_Xfer, diff --git a/src/aiori-RADOS.c b/src/aiori-RADOS.c index ad4207d3..fdbba6b2 100755 --- a/src/aiori-RADOS.c +++ b/src/aiori-RADOS.c @@ -67,6 +67,7 @@ static option_help * RADOS_options(); /************************** D E C L A R A T I O N S ***************************/ ior_aiori_t rados_aiori = { .name = "RADOS", + .name_legacy = NULL, .create = RADOS_Create, .open = RADOS_Open, .xfer = RADOS_Xfer, diff --git a/src/aiori-S3.c b/src/aiori-S3.c index 326b5bca..3fc12081 100755 --- a/src/aiori-S3.c +++ b/src/aiori-S3.c @@ -167,6 +167,7 @@ static void S3_finalize(); // N:N fails if "transfer-size" != "block-size" (because that requires "append") ior_aiori_t s3_aiori = { .name = "S3", + .name_legacy = NULL, .create = S3_Create, .open = S3_Open, .xfer = S3_Xfer, diff --git a/src/aiori.c b/src/aiori.c index 292ef190..333f2393 100644 --- a/src/aiori.c +++ b/src/aiori.c @@ -63,13 +63,15 @@ ior_aiori_t *available_aiori[] = { NULL }; -void aiori_supported_apis(char * APIs){ +void aiori_supported_apis(char * APIs, char * APIs_legacy){ ior_aiori_t **tmp = available_aiori; if(*tmp != NULL){ APIs += sprintf(APIs, "%s", (*tmp)->name); tmp++; for (; *tmp != NULL; ++tmp) { APIs += sprintf(APIs, "|%s", (*tmp)->name); + if ((*tmp)->name_legacy != NULL) + APIs_legacy += sprintf(APIs_legacy, "|%s", (*tmp)->name_legacy); } } } @@ -223,39 +225,51 @@ const ior_aiori_t *aiori_select (const char *api) { char warn_str[256] = {0}; for (ior_aiori_t **tmp = available_aiori ; *tmp != NULL; ++tmp) { - if (NULL == api || strcasecmp(api, (*tmp)->name) == 0) { - if (NULL == (*tmp)->statfs) { - (*tmp)->statfs = aiori_posix_statfs; - snprintf(warn_str, 256, "assuming POSIX-based backend for" - " %s statfs call", api); - WARN(warn_str); - } - if (NULL == (*tmp)->mkdir) { - (*tmp)->mkdir = aiori_posix_mkdir; - snprintf(warn_str, 256, "assuming POSIX-based backend for" - " %s mkdir call", api); - WARN(warn_str); - } - if (NULL == (*tmp)->rmdir) { - (*tmp)->rmdir = aiori_posix_rmdir; - snprintf(warn_str, 256, "assuming POSIX-based backend for" - " %s rmdir call", api); - WARN(warn_str); - } - if (NULL == (*tmp)->access) { - (*tmp)->access = aiori_posix_access; - snprintf(warn_str, 256, "assuming POSIX-based backend for" - " %s access call", api); - WARN(warn_str); - } - if (NULL == (*tmp)->stat) { - (*tmp)->stat = aiori_posix_stat; - snprintf(warn_str, 256, "assuming POSIX-based backend for" - " %s stat call", api); - WARN(warn_str); - } - return *tmp; + char *name_leg = (*tmp)->name_legacy; + if (NULL != api && + (strcasecmp(api, (*tmp)->name) != 0) && + (name_leg == NULL || strcasecmp(api, name_leg) != 0)) + continue; + + if (name_leg != NULL && strcasecmp(api, name_leg) == 0) + { + snprintf(warn_str, 256, "%s backend is deprecated use %s" + " instead", api, (*tmp)->name); + WARN(warn_str); } + + if (NULL == (*tmp)->statfs) { + (*tmp)->statfs = aiori_posix_statfs; + snprintf(warn_str, 256, "assuming POSIX-based backend for" + " %s statfs call", api); + WARN(warn_str); + } + if (NULL == (*tmp)->mkdir) { + (*tmp)->mkdir = aiori_posix_mkdir; + snprintf(warn_str, 256, "assuming POSIX-based backend for" + " %s mkdir call", api); + WARN(warn_str); + } + if (NULL == (*tmp)->rmdir) { + (*tmp)->rmdir = aiori_posix_rmdir; + snprintf(warn_str, 256, "assuming POSIX-based backend for" + " %s rmdir call", api); + WARN(warn_str); + } + if (NULL == (*tmp)->access) { + (*tmp)->access = aiori_posix_access; + snprintf(warn_str, 256, "assuming POSIX-based backend for" + " %s access call", api); + WARN(warn_str); + } + if (NULL == (*tmp)->stat) { + (*tmp)->stat = aiori_posix_stat; + snprintf(warn_str, 256, "assuming POSIX-based backend for" + " %s stat call", api); + WARN(warn_str); + } + + return *tmp; } return NULL; diff --git a/src/aiori.h b/src/aiori.h index feb19b1a..8fab7675 100755 --- a/src/aiori.h +++ b/src/aiori.h @@ -65,6 +65,7 @@ typedef struct ior_aiori_statfs { typedef struct ior_aiori { char *name; + char *name_legacy; void *(*create)(char *, IOR_param_t *); void *(*open)(char *, IOR_param_t *); IOR_offset_t (*xfer)(int, void *, IOR_size_t *, @@ -101,7 +102,7 @@ void aiori_initialize(IOR_test_t * tests); void aiori_finalize(IOR_test_t * tests); const ior_aiori_t *aiori_select (const char *api); int aiori_count (void); -void aiori_supported_apis(char * APIs); +void aiori_supported_apis(char * APIs, char * APIs_legacy); const char *aiori_default (void); /* some generic POSIX-based backend calls */ diff --git a/src/mdtest.c b/src/mdtest.c index 540418c5..9d732a8e 100644 --- a/src/mdtest.c +++ b/src/mdtest.c @@ -1303,7 +1303,8 @@ void print_help (void) { int j; char APIs[1024]; - aiori_supported_apis(APIs); + char APIs_legacy[1024]; + aiori_supported_apis(APIs, APIs_legacy); char apiStr[1024]; sprintf(apiStr, "API for I/O [%s]", APIs); @@ -2170,7 +2171,8 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * char * path = "./out"; int randomize = 0; char APIs[1024]; - aiori_supported_apis(APIs); + char APIs_legacy[1024]; + aiori_supported_apis(APIs, APIs_legacy); char apiStr[1024]; sprintf(apiStr, "API for I/O [%s]", APIs); diff --git a/src/parse_options.c b/src/parse_options.c index a8f81b02..74038fa0 100755 --- a/src/parse_options.c +++ b/src/parse_options.c @@ -441,7 +441,8 @@ IOR_test_t *ParseCommandLine(int argc, char **argv) parameters = & initialTestParams; char APIs[1024]; - aiori_supported_apis(APIs); + char APIs_legacy[1024]; + aiori_supported_apis(APIs, APIs_legacy); char apiStr[1024]; sprintf(apiStr, "API for I/O [%s]", APIs); From 1f1e56c3638169ecc4743cb73db53ea3bd07ee81 Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Sun, 30 Sep 2018 10:01:21 +0100 Subject: [PATCH 022/206] Usleep feature added according to Nathan's suggestions. --- doc/USER_GUIDE | 2 ++ src/ior.c | 6 ++++++ src/ior.h | 1 + src/parse_options.c | 2 ++ 4 files changed, 11 insertions(+) diff --git a/doc/USER_GUIDE b/doc/USER_GUIDE index 7ea6e49a..6a6cf188 100755 --- a/doc/USER_GUIDE +++ b/doc/USER_GUIDE @@ -170,6 +170,8 @@ GENERAL: NOTE: it does not delay before a check write or check read + * interIODelay - this time in us (microseconds) after each I/O simulates computing time. + * outlierThreshold - gives warning if any task is more than this number of seconds from the mean of all participating tasks. If so, the task is identified, its time (start, diff --git a/src/ior.c b/src/ior.c index e8c477fc..9a720808 100755 --- a/src/ior.c +++ b/src/ior.c @@ -1839,11 +1839,17 @@ static IOR_offset_t WriteOrReadSingle(IOR_offset_t pairCnt, IOR_offset_t *offset backend->xfer(access, fd, buffer, transfer, test); if (amtXferred != transfer) ERR("cannot write to file"); + if (test->interIODelay > 0){ + usleep(test->interIODelay); + } } else if (access == READ) { amtXferred = backend->xfer(access, fd, buffer, transfer, test); if (amtXferred != transfer) ERR("cannot read from file"); + if (test->interIODelay > 0){ + usleep(test->interIODelay); + } } else if (access == WRITECHECK) { memset(checkBuffer, 'a', transfer); diff --git a/src/ior.h b/src/ior.h index e76a64db..5672c9c2 100755 --- a/src/ior.h +++ b/src/ior.h @@ -100,6 +100,7 @@ typedef struct int repCounter; /* rep counter */ int multiFile; /* multiple files */ int interTestDelay; /* delay between reps in seconds */ + int interIODelay; /* delay after each I/O in us */ int open; /* flag for writing or reading */ int readFile; /* read of existing file */ int writeFile; /* write of file */ diff --git a/src/parse_options.c b/src/parse_options.c index 74038fa0..be6b7323 100755 --- a/src/parse_options.c +++ b/src/parse_options.c @@ -167,6 +167,8 @@ void DecodeDirective(char *line, IOR_param_t *params) params->repetitions = atoi(value); } else if (strcasecmp(option, "intertestdelay") == 0) { params->interTestDelay = atoi(value); + } else if (strcasecmp(option, "interiodelay") == 0) { + params->interIODelay = atoi(value); } else if (strcasecmp(option, "readfile") == 0) { params->readFile = atoi(value); } else if (strcasecmp(option, "writefile") == 0) { From 3aa3ff10f5107b540d57ae320e3b2dd89364709c Mon Sep 17 00:00:00 2001 From: "Glenn K. Lockwood" Date: Fri, 5 Oct 2018 22:20:18 -0700 Subject: [PATCH 023/206] fixes #96 Call MPI_Abort() if MPI is initialized; otherwise just exit() --- src/parse_options.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/parse_options.c b/src/parse_options.c index 7e983597..ecbd2b57 100755 --- a/src/parse_options.c +++ b/src/parse_options.c @@ -112,12 +112,17 @@ void DecodeDirective(char *line, IOR_param_t *params) char option[MAX_STR]; char value[MAX_STR]; int rc; + int initialized; rc = sscanf(line, " %[^=# \t\r\n] = %[^# \t\r\n] ", option, value); if (rc != 2 && rank == 0) { fprintf(out_logfile, "Syntax error in configuration options: %s\n", line); - MPI_CHECK(MPI_Abort(MPI_COMM_WORLD, -1), "MPI_Abort() error"); + MPI_CHECK(MPI_Initialized(&initialized), "MPI_Initialized() error"); + if (initialized) + MPI_CHECK(MPI_Abort(MPI_COMM_WORLD, -1), "MPI_Abort() error"); + else + exit(-1); } if (strcasecmp(option, "api") == 0) { params->api = strdup(value); @@ -308,7 +313,11 @@ void DecodeDirective(char *line, IOR_param_t *params) if (rank == 0) fprintf(out_logfile, "Unrecognized parameter \"%s\"\n", option); - MPI_CHECK(MPI_Abort(MPI_COMM_WORLD, -1), "MPI_Abort() error"); + MPI_CHECK(MPI_Initialized(&initialized), "MPI_Initialized() error"); + if (initialized) + MPI_CHECK(MPI_Abort(MPI_COMM_WORLD, -1), "MPI_Abort() error"); + else + exit(-1); } } From d8d47425236f42a26aae5df2b287994b96e657e2 Mon Sep 17 00:00:00 2001 From: "Glenn K. Lockwood" Date: Fri, 5 Oct 2018 22:50:34 -0700 Subject: [PATCH 024/206] fixes #87 Leading whitespace is stripped from each line of the ior input file. This allows indented comments to be treated as comments. However it does NOT allow one to specify excessive whitespace inside of the `ior start` and `ior stop` magic phrases. Also added a test to catch regressions in this functionality. --- src/parse_options.c | 20 ++++++++++++++------ testing/basic-tests.sh | 3 +++ testing/test_comments.ior | 17 +++++++++++++++++ 3 files changed, 34 insertions(+), 6 deletions(-) create mode 100644 testing/test_comments.ior diff --git a/src/parse_options.c b/src/parse_options.c index 7e983597..abef55c6 100755 --- a/src/parse_options.c +++ b/src/parse_options.c @@ -365,6 +365,7 @@ IOR_test_t *ReadConfigScript(char *scriptName) int runflag = 0; char linebuf[MAX_STR]; char empty[MAX_STR]; + char *ptr; FILE *file; IOR_test_t *head = NULL; IOR_test_t *tail = NULL; @@ -387,15 +388,22 @@ IOR_test_t *ReadConfigScript(char *scriptName) /* Iterate over a block of IOR commands */ while (fgets(linebuf, MAX_STR, file) != NULL) { + /* skip over leading whitespace */ + ptr = linebuf; + while (isspace(*ptr)) + ptr++; + /* skip empty lines */ - if (sscanf(linebuf, "%s", empty) == -1) + if (sscanf(ptr, "%s", empty) == -1) continue; + /* skip lines containing only comments */ - if (sscanf(linebuf, " #%s", empty) == 1) + if (sscanf(ptr, " #%s", empty) == 1) continue; - if (contains_only(linebuf, "ior stop")) { + + if (contains_only(ptr, "ior stop")) { break; - } else if (contains_only(linebuf, "run")) { + } else if (contains_only(ptr, "run")) { if (runflag) { /* previous line was a "run" as well create duplicate test */ @@ -411,9 +419,9 @@ IOR_test_t *ReadConfigScript(char *scriptName) tail->next = CreateTest(&tail->params, test_num++); AllocResults(tail); tail = tail->next; - ParseLine(linebuf, &tail->params); + ParseLine(ptr, &tail->params); } else { - ParseLine(linebuf, &tail->params); + ParseLine(ptr, &tail->params); } } diff --git a/testing/basic-tests.sh b/testing/basic-tests.sh index 8dd3befe..2e546531 100755 --- a/testing/basic-tests.sh +++ b/testing/basic-tests.sh @@ -23,4 +23,7 @@ IOR 2 -a POSIX -r -z -Z -Q 2 -F -k -e -i1 -m -t 100k -b 100k IOR 2 -a POSIX -r -z -Z -Q 3 -X 13 -F -k -e -i1 -m -t 100k -b 100k IOR 2 -a POSIX -w -z -Z -Q 1 -X -13 -F -e -i1 -m -t 100k -b 100k + +IOR 2 -f "$ROOT/test_comments.ior" + END diff --git a/testing/test_comments.ior b/testing/test_comments.ior new file mode 100644 index 00000000..b48b36fe --- /dev/null +++ b/testing/test_comments.ior @@ -0,0 +1,17 @@ +# test to ensure that leading whitespace is ignored +IOR START +api=posix +writeFile =1 + randomOffset=1 +reorderTasks=1 + filePerProc=1 + keepFile=1 +fsync=1 + repetitions=1 +multiFile=1 + # tab-prefixed comment +transferSize=100k +blockSize=100k + # space-prefixed comment +run + ior stop From a0ef65e61f1461138beb0f2e2c2fd7d335166f18 Mon Sep 17 00:00:00 2001 From: "Glenn K. Lockwood" Date: Fri, 5 Oct 2018 23:29:21 -0700 Subject: [PATCH 025/206] stop the segfault when an invalid api is specified in configfile --- src/ior.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/ior.c b/src/ior.c index 9a720808..048bc180 100755 --- a/src/ior.c +++ b/src/ior.c @@ -1231,6 +1231,9 @@ static void TestIoSys(IOR_test_t *test) /* bind I/O calls to specific API */ backend = aiori_select(params->api); + if (backend == NULL) + ERR_SIMPLE("unrecognized I/O API"); + /* show test setup */ if (rank == 0 && verbose >= VERBOSE_0) From 7045295a8678ac6cdbb68401d4f6c7f1a9120a98 Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Sat, 6 Oct 2018 17:30:00 +0100 Subject: [PATCH 026/206] Replaced usleep to nanosleep. --- src/aiori-DUMMY.c | 7 +++++-- src/ior.c | 6 ++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/aiori-DUMMY.c b/src/aiori-DUMMY.c index 8cc0a571..cfd920bc 100755 --- a/src/aiori-DUMMY.c +++ b/src/aiori-DUMMY.c @@ -9,6 +9,7 @@ #include #include #include +#include #include "ior.h" #include "aiori.h" @@ -48,7 +49,8 @@ static void *DUMMY_Create(char *testFileName, IOR_param_t * param) } if (o.delay_creates){ if (! o.delay_rank_0_only || (o.delay_rank_0_only && rank == 0)){ - usleep(o.delay_creates); + struct timespec wait = { o.delay_creates / 1000 / 1000, 1000l * (o.delay_creates % 1000000)}; + nanosleep( & wait, NULL); } } return current++; @@ -102,7 +104,8 @@ static IOR_offset_t DUMMY_Xfer(int access, void *file, IOR_size_t * buffer, IOR_ } if (o.delay_xfer){ if (! o.delay_rank_0_only || (o.delay_rank_0_only && rank == 0)){ - usleep(o.delay_xfer); + struct timespec wait = {o.delay_xfer / 1000 / 1000, 1000l * (o.delay_xfer % 1000000)}; + nanosleep( & wait, NULL); } } return length; diff --git a/src/ior.c b/src/ior.c index 048bc180..66cca769 100755 --- a/src/ior.c +++ b/src/ior.c @@ -1843,7 +1843,8 @@ static IOR_offset_t WriteOrReadSingle(IOR_offset_t pairCnt, IOR_offset_t *offset if (amtXferred != transfer) ERR("cannot write to file"); if (test->interIODelay > 0){ - usleep(test->interIODelay); + struct timespec wait = {test->interIODelay / 1000 / 1000, 1000l * (test->interIODelay % 1000000)}; + nanosleep( & wait, NULL); } } else if (access == READ) { amtXferred = @@ -1851,7 +1852,8 @@ static IOR_offset_t WriteOrReadSingle(IOR_offset_t pairCnt, IOR_offset_t *offset if (amtXferred != transfer) ERR("cannot read from file"); if (test->interIODelay > 0){ - usleep(test->interIODelay); + struct timespec wait = {test->interIODelay / 1000 / 1000, 1000l * (test->interIODelay % 1000000)}; + nanosleep( & wait, NULL); } } else if (access == WRITECHECK) { memset(checkBuffer, 'a', transfer); From bd53ce959d6d3be2ed52f0f02d73e951df961907 Mon Sep 17 00:00:00 2001 From: "Glenn K. Lockwood" Date: Sat, 6 Oct 2018 09:27:41 -0700 Subject: [PATCH 027/206] update known issues of 3.2.0 release --- NEWS | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/NEWS b/NEWS index 4ff935d5..670df56a 100644 --- a/NEWS +++ b/NEWS @@ -39,6 +39,14 @@ General developer improvements: - Added continuous integration support (Enno Zickler, U Hamburg) - Added better support for automated testing (Julian Kunkel, U Reading) +Known issues: + +- `api=RADOS` cannot work correctly if specified in a config file (via `-f`) + because `-u`/`-c`/`-p` cannot be specified (issue #98) +- `writeCheck` cannot be enabled for write-only tests using some AIORIs such as + MPI-IO (pull request #89) + + Version 3.0.2 -------------------------------------------------------------------------------- - IOR and mdtest now share a common codebase. This will make it easier From 87c5c9ef0463e1288059b5f7bfc7dd4100cee2e9 Mon Sep 17 00:00:00 2001 From: "Glenn K. Lockwood" Date: Mon, 8 Oct 2018 13:47:28 -0700 Subject: [PATCH 028/206] compile correctly with -std=c99 This patch enables correct compilation on both MacOS and Linux using only POSIX.1-2008-compliant C with XSI extensions. Specifically, POSIX.1-2008 is the minimum version because we use strdup(3); explicit XSI is required to expose putenv from glibc. --- configure.ac | 11 +++++++++++ src/aiori-DUMMY.c | 2 ++ src/aiori-MMAP.c | 2 ++ src/aiori.c | 11 +++++++++++ src/ior.c | 29 ++++++++++++++++++++--------- src/mdtest.c | 8 +++++++- src/option.c | 1 + src/parse_options.c | 5 +++++ travis-test.sh | 2 +- 9 files changed, 60 insertions(+), 11 deletions(-) diff --git a/configure.ac b/configure.ac index a3c41b03..c28fb718 100755 --- a/configure.ac +++ b/configure.ac @@ -19,6 +19,17 @@ AM_INIT_AUTOMAKE([check-news dist-bzip2 gnu no-define foreign subdir-objects]) m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) AM_MAINTAINER_MODE +# Check for system-specific stuff +case "${host_os}" in + *linux*) + ;; + *darwin*) + CPPFLAGS="${CPPFLAGS} -D_DARWIN_C_SOURCE" + ;; + *) + ;; +esac + # Checks for programs # We can't do anything without a working MPI diff --git a/src/aiori-DUMMY.c b/src/aiori-DUMMY.c index cfd920bc..fd3b087a 100755 --- a/src/aiori-DUMMY.c +++ b/src/aiori-DUMMY.c @@ -6,6 +6,8 @@ # include "config.h" #endif +#define _XOPEN_SOURCE 700 + #include #include #include diff --git a/src/aiori-MMAP.c b/src/aiori-MMAP.c index f812bddd..ed043573 100644 --- a/src/aiori-MMAP.c +++ b/src/aiori-MMAP.c @@ -11,6 +11,8 @@ # include "config.h" #endif +#define _XOPEN_SOURCE 700 + #include #include diff --git a/src/aiori.c b/src/aiori.c index 333f2393..9c27386c 100644 --- a/src/aiori.c +++ b/src/aiori.c @@ -12,8 +12,19 @@ * \******************************************************************************/ +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#define _XOPEN_SOURCE 700 + #include #include + +#if defined(HAVE_STRINGS_H) +#include +#endif + #include "aiori.h" #if defined(HAVE_SYS_STATVFS_H) diff --git a/src/ior.c b/src/ior.c index 66cca769..d383eb95 100755 --- a/src/ior.c +++ b/src/ior.c @@ -12,6 +12,8 @@ # include "config.h" #endif +#define _XOPEN_SOURCE 700 + #include #include #include @@ -20,6 +22,11 @@ #include #include #include + +#if defined(HAVE_STRINGS_H) +#include +#endif + #include /* struct stat */ #include @@ -458,12 +465,16 @@ static int CountErrors(IOR_param_t * test, int access, int errors) */ static void *aligned_buffer_alloc(size_t size) { - size_t pageSize; size_t pageMask; char *buf, *tmp; char *aligned; - pageSize = getpagesize(); +#ifdef HAVE_GETPAGESIZE + size_t pageSize = getpagesize(); +#else + long pageSize = sysconf(_SC_PAGESIZE); +#endif + pageMask = pageSize - 1; buf = malloc(size + pageSize + sizeof(void *)); if (buf == NULL) @@ -1410,7 +1421,7 @@ static void TestIoSys(IOR_test_t *test) /* random process offset reading */ if (params->reorderTasksRandom) { /* this should not intefere with randomOffset within a file because GetOffsetArrayRandom */ - /* seeds every random() call */ + /* seeds every rand() call */ int nodeoffset; unsigned int iseed0; nodeoffset = params->taskPerNodeOffset; @@ -1762,11 +1773,11 @@ static IOR_offset_t *GetOffsetArrayRandom(IOR_param_t * test, int pretendRank, /* set up seed for random() */ if (access == WRITE || access == READ) { - test->randomSeed = seed = random(); + test->randomSeed = seed = rand(); } else { seed = test->randomSeed; } - srandom(seed); + srand(seed); fileSize = test->blockSize * test->segmentCount; if (test->filePerProc == FALSE) { @@ -1778,7 +1789,7 @@ static IOR_offset_t *GetOffsetArrayRandom(IOR_param_t * test, int pretendRank, if (test->filePerProc == FALSE) { // this counts which process get how many transferes in // a shared file - if ((random() % test->numTasks) == pretendRank) { + if ((rand() % test->numTasks) == pretendRank) { offsets++; } } else { @@ -1800,9 +1811,9 @@ static IOR_offset_t *GetOffsetArrayRandom(IOR_param_t * test, int pretendRank, } } else { /* fill with offsets (pass 2) */ - srandom(seed); /* need same seed to get same transfers as counted in the beginning*/ + srand(seed); /* need same seed to get same transfers as counted in the beginning*/ for (i = 0; i < fileSize; i += test->transferSize) { - if ((random() % test->numTasks) == pretendRank) { + if ((rand() % test->numTasks) == pretendRank) { offsetArray[offsetCnt] = i; offsetCnt++; } @@ -1810,7 +1821,7 @@ static IOR_offset_t *GetOffsetArrayRandom(IOR_param_t * test, int pretendRank, } /* reorder array */ for (i = 0; i < offsets; i++) { - value = random() % offsets; + value = rand() % offsets; tmp = offsetArray[value]; offsetArray[value] = offsetArray[i]; offsetArray[i] = tmp; diff --git a/src/mdtest.c b/src/mdtest.c index 9d732a8e..84f40c8d 100644 --- a/src/mdtest.c +++ b/src/mdtest.c @@ -28,6 +28,7 @@ * $Date: 2013/11/27 17:05:31 $ * $Author: brettkettering $ */ +#define _XOPEN_SOURCE 700 #include #include @@ -59,6 +60,11 @@ #include #include + +#if HAVE_STRINGS_H +#include +#endif + #include #include #include @@ -1616,7 +1622,7 @@ void valid_tests() { FAIL("-c not compatible with -B"); } - if ( strcasecmp(backend_name, "POSIX") != 0 && strcasecmp(backend_name, "DUMMY") != 0) { + if (strcasecmp(backend_name, "POSIX") != 0 && strcasecmp(backend_name, "DUMMY") != 0) { FAIL("-a only supported interface is POSIX (and DUMMY) right now!"); } diff --git a/src/option.c b/src/option.c index 1e646af2..5bd19fe7 100644 --- a/src/option.c +++ b/src/option.c @@ -1,3 +1,4 @@ +#define _XOPEN_SOURCE 700 #include #include #include diff --git a/src/parse_options.c b/src/parse_options.c index 460dd520..546d0dc5 100755 --- a/src/parse_options.c +++ b/src/parse_options.c @@ -16,11 +16,16 @@ #include "config.h" #endif +#define _XOPEN_SOURCE 700 + #include #include #include #include +#if defined(HAVE_STRINGS_H) +#include +#endif #include "utilities.h" #include "ior.h" diff --git a/travis-test.sh b/travis-test.sh index dfc12357..7bd84a0d 100755 --- a/travis-test.sh +++ b/travis-test.sh @@ -5,7 +5,7 @@ # # These options will be passed directly to the autoconf configure script -CONFIGURE_OPTS="${CONFIGURE_OPTS:-""}" +CONFIGURE_OPTS="${CONFIGURE_OPTS:-"CFLAGS=-std=c99 --disable-silent-rules"}" BASE_DIR="$(cd "${0%/*}" && pwd)" if [ -z "$BASE_DIR" -o ! -d "$BASE_DIR" ]; then From e6bd9ad3b31acf04372b1d442a753260caacfac8 Mon Sep 17 00:00:00 2001 From: Sylvain Didelot Date: Tue, 9 Oct 2018 12:14:30 +0200 Subject: [PATCH 029/206] common: call mkdir/access syscalls defined in backends mkdir/access syscalls used in PrependDir() should call their equivalent implemented by the backend. --- src/ior.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/ior.c b/src/ior.c index 66cca769..bf5faa7a 100755 --- a/src/ior.c +++ b/src/ior.c @@ -823,14 +823,15 @@ static char *PrependDir(IOR_param_t * test, char *rootDir) sprintf(dir, "%s%d", dir, (rank + rankOffset) % test->numTasks); /* dir doesn't exist, so create */ - if (access(dir, F_OK) != 0) { - if (mkdir(dir, S_IRWXU) < 0) { + if (backend->access(dir, F_OK, test) != 0) { + if (backend->mkdir(dir, S_IRWXU, test) < 0) { ERR("cannot create directory"); } /* check if correct permissions */ - } else if (access(dir, R_OK) != 0 || access(dir, W_OK) != 0 || - access(dir, X_OK) != 0) { + } else if (backend->access(dir, R_OK, test) != 0 || + backend->access(dir, W_OK, test) != 0 || + backend->access(dir, X_OK, test) != 0) { ERR("invalid directory permissions"); } From 912d93e94f432589e73c79efd959825090e71411 Mon Sep 17 00:00:00 2001 From: Sylvain Didelot Date: Wed, 3 Oct 2018 18:44:38 +0200 Subject: [PATCH 030/206] aiori-IME: add support of mkdir/rmdir to the backup ime_native_mkdir() and ime_native_rmdir() are available starting from IME native API version 1.3. --- src/aiori-IME.c | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/src/aiori-IME.c b/src/aiori-IME.c index 08624951..107b3210 100755 --- a/src/aiori-IME.c +++ b/src/aiori-IME.c @@ -272,10 +272,10 @@ static char *IME_GetVersion() /* * XXX: statfs call is currently not exposed by IME native interface. */ -static int IME_StatFS(const char *oid, ior_aiori_statfs_t *stat_buf, +static int IME_StatFS(const char *path, ior_aiori_statfs_t *stat_buf, IOR_param_t *param) { - (void)oid; + (void)path; (void)stat_buf; (void)param; @@ -283,29 +283,33 @@ static int IME_StatFS(const char *oid, ior_aiori_statfs_t *stat_buf, return -1; } -/* - * XXX: mkdir call is currently not exposed by IME native interface. - */ -static int IME_MkDir(const char *oid, mode_t mode, IOR_param_t *param) +static int IME_MkDir(const char *path, mode_t mode, IOR_param_t *param) { - (void)oid; - (void)mode; (void)param; - WARN("mkdir is currently not supported in IME backend!"); +#if (IME_NATIVE_API_VERSION >= 130) + return ime_native_mkdir(path, mode); +#else + (void)path; + (void)mode; + + WARN("mkdir not supported in IME backend!"); return -1; +#endif } -/* - * XXX: rmdir call is curretly not exposed by IME native interface. - */ -static int IME_RmDir(const char *oid, IOR_param_t *param) +static int IME_RmDir(const char *path, IOR_param_t *param) { - (void)oid; (void)param; - WARN("rmdir is currently not supported in IME backend!"); +#if (IME_NATIVE_API_VERSION >= 130) + return ime_native_rmdir(path); +#else + (void)path; + + WARN("rmdir not supported in IME backend!"); return -1; +#endif } /* From 6f7576aa8a2b13d1383eb4f74eef8106fec54647 Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Tue, 2 Oct 2018 16:46:29 +0100 Subject: [PATCH 031/206] Changed the parser. The parser now supports concurrent parsing of all plugin options. Moved HDF5 collective_md option into the backend as an example. Example: ./src/ior -a dummy --dummy.delay-xfer=50000 --- doc/sphinx/userDoc/options.rst | 5 +- src/aiori-DUMMY.c | 6 +- src/aiori-HDF5.c | 23 +++- src/aiori-RADOS.c | 6 +- src/aiori.c | 20 +++ src/aiori.h | 1 + src/ior.h | 1 - src/mdtest.c | 24 +--- src/option.c | 236 ++++++++++++++++----------------- src/option.h | 15 ++- src/parse_options.c | 26 +--- 11 files changed, 177 insertions(+), 186 deletions(-) diff --git a/doc/sphinx/userDoc/options.rst b/doc/sphinx/userDoc/options.rst index 8b441269..5af1802a 100644 --- a/doc/sphinx/userDoc/options.rst +++ b/doc/sphinx/userDoc/options.rst @@ -289,8 +289,9 @@ HDF5-ONLY * setAlignment - HDF5 alignment in bytes (e.g.: 8, 4k, 2m, 1g) [1] - * collectiveMetadata - enable HDF5 collective metadata (available since - HDF5-1.10.0) +HDF5-backend options +^^^^^^^^^^^^^^^^^^^^ + * -c | --collectiveMetadata - enable HDF5 collective metadata (available since HDF5-1.10.0) MPIIO-, HDF5-, AND NCMPI-ONLY ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/src/aiori-DUMMY.c b/src/aiori-DUMMY.c index 8cc0a571..a974dc57 100755 --- a/src/aiori-DUMMY.c +++ b/src/aiori-DUMMY.c @@ -29,9 +29,9 @@ static struct dummy_options o = { }; static option_help options [] = { - {'c', "delay-create", "Delay per create in usec", OPTION_OPTIONAL_ARGUMENT, 'l', & o.delay_creates}, - {'x', "delay-xfer", "Delay per xfer in usec", OPTION_OPTIONAL_ARGUMENT, 'l', & o.delay_xfer}, - {'z', "delay-only-rank0", "Delay only Rank0", OPTION_FLAG, 'd', & o.delay_rank_0_only}, + {0, "dummy.delay-create", "Delay per create in usec", OPTION_OPTIONAL_ARGUMENT, 'l', & o.delay_creates}, + {0, "dummy.delay-xfer", "Delay per xfer in usec", OPTION_OPTIONAL_ARGUMENT, 'l', & o.delay_xfer}, + {0, "dummy.delay-only-rank0", "Delay only Rank0", OPTION_FLAG, 'd', & o.delay_rank_0_only}, LAST_OPTION }; diff --git a/src/aiori-HDF5.c b/src/aiori-HDF5.c index 18276270..d1e6eabe 100755 --- a/src/aiori-HDF5.c +++ b/src/aiori-HDF5.c @@ -93,6 +93,7 @@ static char* HDF5_GetVersion(); static void HDF5_Fsync(void *, IOR_param_t *); static IOR_offset_t HDF5_GetFileSize(IOR_param_t *, MPI_Comm, char *); static int HDF5_Access(const char *, int, IOR_param_t *); +static option_help * HDF5_get_options(); /************************** D E C L A R A T I O N S ***************************/ @@ -112,6 +113,7 @@ ior_aiori_t hdf5_aiori = { .rmdir = aiori_posix_rmdir, .access = HDF5_Access, .stat = aiori_posix_stat, + .get_options = HDF5_get_options }; static hid_t xferPropList; /* xfer property list */ @@ -121,8 +123,27 @@ hid_t fileDataSpace; /* file data space id */ hid_t memDataSpace; /* memory data space id */ int newlyOpenedFile; /* newly opened file */ +/************************** O P T I O N S *****************************/ +struct HDF5_options{ + int collective_md; +}; /***************************** F U N C T I O N S ******************************/ + +static struct HDF5_options o = { + .collective_md = 0 +}; + +static option_help options [] = { + {0, "hdf5.collectiveMetadata", "Use collectiveMetadata (available since HDF5-1.10.0)", OPTION_FLAG, 'd', & o.collective_md}, + LAST_OPTION +}; + +static option_help * HDF5_get_options(){ + return options; +} + + /* * Create and open a file through the HDF5 interface. */ @@ -230,7 +251,7 @@ static void *HDF5_Open(char *testFileName, IOR_param_t * param) "cannot set alignment"); #ifdef HAVE_H5PSET_ALL_COLL_METADATA_OPS - if (param->collective_md) { + if (o.collective_md) { /* more scalable metadata */ HDF5_CHECK(H5Pset_all_coll_metadata_ops(accessPropList, 1), diff --git a/src/aiori-RADOS.c b/src/aiori-RADOS.c index fdbba6b2..090414d2 100755 --- a/src/aiori-RADOS.c +++ b/src/aiori-RADOS.c @@ -41,9 +41,9 @@ static struct rados_options o = { }; static option_help options [] = { - {'u', "user", "Username for the RADOS cluster", OPTION_REQUIRED_ARGUMENT, 's', & o.user}, - {'c', "conf", "Config file for the RADOS cluster", OPTION_REQUIRED_ARGUMENT, 's', & o.conf}, - {'p', "pool", "RADOS pool to use for I/O", OPTION_REQUIRED_ARGUMENT, 's', & o.pool}, + {0, "rados.user", "Username for the RADOS cluster", OPTION_REQUIRED_ARGUMENT, 's', & o.user}, + {0, "rados.conf", "Config file for the RADOS cluster", OPTION_REQUIRED_ARGUMENT, 's', & o.conf}, + {0, "rados.pool", "RADOS pool to use for I/O", OPTION_REQUIRED_ARGUMENT, 's', & o.pool}, LAST_OPTION }; diff --git a/src/aiori.c b/src/aiori.c index 333f2393..09a1a6d6 100644 --- a/src/aiori.c +++ b/src/aiori.c @@ -63,6 +63,26 @@ ior_aiori_t *available_aiori[] = { NULL }; +void airoi_parse_options(int argc, char ** argv, option_help * global_options){ + int airoi_c = aiori_count(); + options_all opt; + opt.module_count = airoi_c + 1; + opt.modules = malloc(sizeof(option_module) * (airoi_c + 1)); + opt.modules[0].prefix = NULL; + opt.modules[0].options = global_options; + ior_aiori_t **tmp = available_aiori; + for (int i=1; *tmp != NULL; ++tmp, i++) { + opt.modules[i].prefix = (*tmp)->name; + if((*tmp)->get_options != NULL){ + opt.modules[i].options = (*tmp)->get_options(); + }else{ + opt.modules[i].options = NULL; + } + } + option_parse(argc, argv, &opt); + free(opt.modules); +} + void aiori_supported_apis(char * APIs, char * APIs_legacy){ ior_aiori_t **tmp = available_aiori; if(*tmp != NULL){ diff --git a/src/aiori.h b/src/aiori.h index 8fab7675..7fb013da 100755 --- a/src/aiori.h +++ b/src/aiori.h @@ -103,6 +103,7 @@ void aiori_finalize(IOR_test_t * tests); const ior_aiori_t *aiori_select (const char *api); int aiori_count (void); void aiori_supported_apis(char * APIs, char * APIs_legacy); +void airoi_parse_options(int argc, char ** argv, option_help * global_options); const char *aiori_default (void); /* some generic POSIX-based backend calls */ diff --git a/src/ior.h b/src/ior.h index 871346c4..5672c9c2 100755 --- a/src/ior.h +++ b/src/ior.h @@ -178,7 +178,6 @@ typedef struct char* URI; /* "path" to target object */ size_t part_number; /* multi-part upload increment (PER-RANK!) */ char* UploadId; /* key for multi-part-uploads */ - int collective_md; /* use collective metatata optimization */ /* RADOS variables */ rados_t rados_cluster; /* RADOS cluster handle */ diff --git a/src/mdtest.c b/src/mdtest.c index 9d732a8e..b1cf30ed 100644 --- a/src/mdtest.c +++ b/src/mdtest.c @@ -2213,35 +2213,13 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * {'Z', NULL, "print time instead of rate", OPTION_FLAG, 'd', & print_time}, LAST_OPTION }; - int printhelp = 0; - int parsed_options = option_parse(argc, argv, options, & printhelp); + airoi_parse_options(argc, argv, options); backend = aiori_select(backend_name); if (NULL == backend) { FAIL("Could not find suitable backend to use"); } - if(backend->get_options != NULL){ - option_parse(argc - parsed_options, argv + parsed_options, backend->get_options(), & printhelp); - } - - if(printhelp != 0){ - printf("Usage: %s ", argv[0]); - - option_print_help(options, 0); - - if(backend->get_options != NULL){ - printf("\nPlugin options for backend %s\n", backend_name); - option_print_help(backend->get_options(), 1); - } - if(printhelp == 1){ - exit(0); - }else{ - exit(1); - } - } - - MPI_Comm_rank(testComm, &rank); MPI_Comm_size(testComm, &size); diff --git a/src/option.c b/src/option.c index 1e646af2..4e880555 100644 --- a/src/option.c +++ b/src/option.c @@ -135,40 +135,7 @@ static void print_help_section(option_help * args, option_value_type type, char } } -void option_print_help(option_help * args, int is_plugin){ - option_help * o; - int optionalArgs = 0; - for(o = args; o->shortVar != 0 || o->longVar != 0 ; o++){ - if(o->arg != OPTION_REQUIRED_ARGUMENT){ - optionalArgs = 1; - } - - switch(o->arg){ - case (OPTION_OPTIONAL_ARGUMENT): - case (OPTION_FLAG):{ - if(o->shortVar != 0){ - printf("[-%c] ", o->shortVar); - }else if(o->longVar != 0){ - printf("[--%s] ", o->longVar); - } - break; - }case (OPTION_REQUIRED_ARGUMENT):{ - if(o->shortVar != 0){ - printf("-%c ", o->shortVar); - }else if(o->longVar != 0){ - printf("--%s ", o->longVar); - } - break; - } - } - } - if (optionalArgs){ - //printf(" [Optional Args]"); - } - if (! is_plugin){ - printf(" -- \n"); - } - +void option_print_help(option_help * args){ print_help_section(args, OPTION_REQUIRED_ARGUMENT, "Required arguments"); print_help_section(args, OPTION_FLAG, "Flags"); print_help_section(args, OPTION_OPTIONAL_ARGUMENT, "Optional arguments"); @@ -253,17 +220,23 @@ void option_print_current(option_help * args){ print_current_option_section(args, OPTION_FLAG); } -int option_parse(int argc, char ** argv, option_help * args, int * printhelp){ +int option_parse(int argc, char ** argv, options_all * opt_all){ int error = 0; int requiredArgsSeen = 0; int requiredArgsNeeded = 0; int i; - - for(option_help * o = args; o->shortVar != 0 || o->longVar != 0 ; o++ ){ - if(o->arg == OPTION_REQUIRED_ARGUMENT){ - requiredArgsNeeded++; + int printhelp = 0; + + for(int m = 0; m < opt_all->module_count; m++ ){ + option_help * args = opt_all->modules[m].options; + if(args == NULL) continue; + for(option_help * o = args; o->shortVar != 0 || o->longVar != 0 ; o++ ){ + if(o->arg == OPTION_REQUIRED_ARGUMENT){ + requiredArgsNeeded++; + } } } + for(i=1; i < argc; i++){ char * txt = argv[i]; int foundOption = 0; @@ -274,105 +247,104 @@ int option_parse(int argc, char ** argv, option_help * args, int * printhelp){ arg++; replaced_equal = 1; } - if(strcmp(txt, "--") == 0){ - // we found plugin options - break; - } - - // try to find matching option help - for(option_help * o = args; o->shortVar != 0 || o->longVar != 0 || o->help != NULL ; o++ ){ - if( o->shortVar == 0 && o->longVar == 0 ){ - // section - continue; - } - - if ( (txt[0] == '-' && o->shortVar == txt[1]) || (strlen(txt) > 2 && txt[0] == '-' && txt[1] == '-' && o->longVar != NULL && strcmp(txt + 2, o->longVar) == 0)){ - foundOption = 1; - - // now process the option. - switch(o->arg){ - case (OPTION_FLAG):{ - assert(o->type == 'd'); - (*(int*) o->variable)++; - break; - } - case (OPTION_OPTIONAL_ARGUMENT): - case (OPTION_REQUIRED_ARGUMENT):{ - // check if next is an argument - if(arg == NULL){ - if(o->shortVar == txt[1] && txt[2] != 0){ - arg = & txt[2]; - }else{ - // simply take the next value as argument - i++; - arg = argv[i]; - } - } - if(arg == NULL){ - const char str[] = {o->shortVar, 0}; - printf("Error, argument missing for option %s\n", (o->longVar != NULL) ? o->longVar : str); - exit(1); + for(int m = 0; m < opt_all->module_count; m++ ){ + option_help * args = opt_all->modules[m].options; + if(args == NULL) continue; + // try to find matching option help + for(option_help * o = args; o->shortVar != 0 || o->longVar != 0 || o->help != NULL ; o++ ){ + if( o->shortVar == 0 && o->longVar == 0 ){ + // section + continue; + } + if ( (txt[0] == '-' && o->shortVar == txt[1]) || (strlen(txt) > 2 && txt[0] == '-' && txt[1] == '-' && o->longVar != NULL && strcmp(txt + 2, o->longVar) == 0)){ + foundOption = 1; + + // now process the option. + switch(o->arg){ + case (OPTION_FLAG):{ + assert(o->type == 'd'); + (*(int*) o->variable)++; + break; } - - switch(o->type){ - case('p'):{ - // call the function in the variable - void(*fp)() = o->variable; - fp(arg); - break; - } - case('F'):{ - *(double*) o->variable = atof(arg); - break; - } - case('f'):{ - *(float*) o->variable = atof(arg); - break; - } - case('d'):{ - int64_t val = string_to_bytes(arg); - if (val > INT_MAX || val < INT_MIN){ - printf("WARNING: parsing the number %s to integer, this produced an overflow!\n", arg); + case (OPTION_OPTIONAL_ARGUMENT): + case (OPTION_REQUIRED_ARGUMENT):{ + // check if next is an argument + if(arg == NULL){ + if(o->shortVar == txt[1] && txt[2] != 0){ + arg = & txt[2]; + }else{ + // simply take the next value as argument + i++; + arg = argv[i]; } - *(int*) o->variable = val; - break; } - case('H'): - case('s'):{ - (*(char **) o->variable) = strdup(arg); - break; + + if(arg == NULL){ + const char str[] = {o->shortVar, 0}; + printf("Error, argument missing for option %s\n", (o->longVar != NULL) ? o->longVar : str); + exit(1); } - case('c'):{ - (*(char *)o->variable) = arg[0]; - if(strlen(arg) > 1){ - printf("Error, ignoring remainder of string for option %c (%s).\n", o->shortVar, o->longVar); + + switch(o->type){ + case('p'):{ + // call the function in the variable + void(*fp)() = o->variable; + fp(arg); + break; } - break; - } - case('l'):{ - *(long long*) o->variable = string_to_bytes(arg); - break; + case('F'):{ + *(double*) o->variable = atof(arg); + break; + } + case('f'):{ + *(float*) o->variable = atof(arg); + break; + } + case('d'):{ + int64_t val = string_to_bytes(arg); + if (val > INT_MAX || val < INT_MIN){ + printf("WARNING: parsing the number %s to integer, this produced an overflow!\n", arg); + } + *(int*) o->variable = val; + break; + } + case('H'): + case('s'):{ + (*(char **) o->variable) = strdup(arg); + break; + } + case('c'):{ + (*(char *)o->variable) = arg[0]; + if(strlen(arg) > 1){ + printf("Error, ignoring remainder of string for option %c (%s).\n", o->shortVar, o->longVar); + } + break; + } + case('l'):{ + *(long long*) o->variable = string_to_bytes(arg); + break; + } + default: + printf("ERROR: Unknown option type %c\n", o->type); } - default: - printf("ERROR: Unknown option type %c\n", o->type); } } - } - if(replaced_equal){ - arg[-1] = '='; - } + if(replaced_equal){ + arg[-1] = '='; + } - if(o->arg == OPTION_REQUIRED_ARGUMENT){ - requiredArgsSeen++; - } + if(o->arg == OPTION_REQUIRED_ARGUMENT){ + requiredArgsSeen++; + } - break; + break; + } } } if (! foundOption){ if(strcmp(txt, "-h") == 0 || strcmp(txt, "--help") == 0){ - *printhelp=1; + printhelp = 1; }else{ printf("Error invalid argument: %s\n", txt); error = 1; @@ -382,12 +354,26 @@ int option_parse(int argc, char ** argv, option_help * args, int * printhelp){ if( requiredArgsSeen != requiredArgsNeeded ){ printf("Error: Missing some required arguments\n\n"); - *printhelp = -1; + printhelp = -1; } if(error != 0){ printf("Invalid options\n"); - *printhelp = -1; + printhelp = -1; + } + + if(printhelp == 1){ + printf("Synopsis %s\n", argv[0]); + for(int m = 0; m < opt_all->module_count; m++ ){ + option_help * args = opt_all->modules[m].options; + if(args == NULL) continue; + char * prefix = opt_all->modules[m].prefix; + if(prefix != NULL){ + printf("\n\nModule %s\n", prefix); + } + option_print_help(args); + } + exit(0); } return i; diff --git a/src/option.h b/src/option.h index 8337d8da..53a3732b 100644 --- a/src/option.h +++ b/src/option.h @@ -4,7 +4,7 @@ #include /* - * Initial revision by JK + * Initial version by JK */ typedef enum{ @@ -23,13 +23,22 @@ typedef struct{ void * variable; } option_help; +typedef struct{ + char * prefix; // may be NULL to include it in the standard name + option_help * options; +} option_module; + +typedef struct{ + int module_count; + option_module * modules; +} options_all; + #define LAST_OPTION {0, 0, 0, (option_value_type) 0, 0, NULL} int64_t string_to_bytes(char *size_str); -void option_print_help(option_help * args, int is_plugin); void option_print_current(option_help * args); //@return the number of parsed arguments -int option_parse(int argc, char ** argv, option_help * args, int * print_help); +int option_parse(int argc, char ** argv, options_all * args); #endif diff --git a/src/parse_options.c b/src/parse_options.c index 7e983597..6c32a1ba 100755 --- a/src/parse_options.c +++ b/src/parse_options.c @@ -302,8 +302,6 @@ void DecodeDirective(char *line, IOR_param_t *params) params->numTasks = atoi(value); } else if (strcasecmp(option, "summaryalways") == 0) { params->summary_every_test = atoi(value); - } else if (strcasecmp(option, "collectiveMetadata") == 0) { - params->collective_md = atoi(value); } else { if (rank == 0) fprintf(out_logfile, "Unrecognized parameter \"%s\"\n", @@ -515,8 +513,7 @@ IOR_test_t *ParseCommandLine(int argc, char **argv) IOR_test_t *tests = NULL; GetPlatformName(initialTestParams.platform); - int printhelp = 0; - int parsed_options = option_parse(argc, argv, options, & printhelp); + airoi_parse_options(argc, argv, options); if (toggleG){ initialTestParams.setTimeStampSignature = toggleG; @@ -544,7 +541,6 @@ IOR_test_t *ParseCommandLine(int argc, char **argv) if (memoryPerNode){ initialTestParams.memoryPerNode = NodeMemoryStringToBytes(optarg); } - const ior_aiori_t * backend = aiori_select(initialTestParams.api); if (backend == NULL) ERR_SIMPLE("unrecognized I/O API"); @@ -552,26 +548,6 @@ IOR_test_t *ParseCommandLine(int argc, char **argv) initialTestParams.backend = backend; initialTestParams.apiVersion = backend->get_version(); - if(backend->get_options != NULL){ - option_parse(argc - parsed_options, argv + parsed_options, backend->get_options(), & printhelp); - } - - if(printhelp != 0){ - printf("Usage: %s ", argv[0]); - - option_print_help(options, 0); - - if(backend->get_options != NULL){ - printf("\nPlugin options for backend %s (%s)\n", initialTestParams.api, backend->get_version()); - option_print_help(backend->get_options(), 1); - } - if(printhelp == 1){ - exit(0); - }else{ - exit(1); - } - } - if (testscripts){ tests = ReadConfigScript(testscripts); }else{ From afb65228aa0125434db9264b0657aada13d14fcc Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Thu, 11 Oct 2018 18:35:18 +0100 Subject: [PATCH 032/206] Updated docu for the changed behavior. --- doc/sphinx/userDoc/options.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx/userDoc/options.rst b/doc/sphinx/userDoc/options.rst index 5af1802a..b46bee46 100644 --- a/doc/sphinx/userDoc/options.rst +++ b/doc/sphinx/userDoc/options.rst @@ -291,7 +291,7 @@ HDF5-ONLY HDF5-backend options ^^^^^^^^^^^^^^^^^^^^ - * -c | --collectiveMetadata - enable HDF5 collective metadata (available since HDF5-1.10.0) + * --hdf5.collectiveMetadata - enable HDF5 collective metadata (available since HDF5-1.10.0) MPIIO-, HDF5-, AND NCMPI-ONLY ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ From fa38cb7992d868b0ca6e662384948b1a3e896868 Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Thu, 11 Oct 2018 18:46:42 +0100 Subject: [PATCH 033/206] Bugfix JSON header (allows parsing at line 0). --- src/ior-output.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/ior-output.c b/src/ior-output.c index a6b8d9c0..da4f5f1a 100644 --- a/src/ior-output.c +++ b/src/ior-output.c @@ -241,8 +241,12 @@ void PrintEarlyHeader() if (rank != 0) return; - fprintf(out_resultfile, "IOR-" META_VERSION ": MPI Coordinated Test of Parallel I/O\n"); - fflush(out_resultfile); + PrintStartSection(); + if (outputFormat != OUTPUT_DEFAULT){ + PrintKeyVal("Version", META_VERSION); + }else{ + printf("IOR-" META_VERSION ": MPI Coordinated Test of Parallel I/O\n"); + } } void PrintHeader(int argc, char **argv) @@ -252,7 +256,6 @@ void PrintHeader(int argc, char **argv) if (rank != 0) return; - PrintStartSection(); PrintKeyVal("Began", CurrentTimeString()); PrintKeyValStart("Command line"); From 3b5f4fc00271146e3a103e42a8e6bd9e6c1eabf2 Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Thu, 11 Oct 2018 18:59:12 +0100 Subject: [PATCH 034/206] Bugfix JSON output for multiple processes --- src/ior-internal.h | 1 - src/ior-output.c | 34 ++++++++++++++++++---------------- src/ior.c | 3 --- 3 files changed, 18 insertions(+), 20 deletions(-) diff --git a/src/ior-internal.h b/src/ior-internal.h index 11bbb8d1..6b89af10 100644 --- a/src/ior-internal.h +++ b/src/ior-internal.h @@ -6,7 +6,6 @@ #define _IOR_INTERNAL_H /* Part of ior-output.c */ -void PrintEarlyHeader(); void PrintHeader(int argc, char **argv); void ShowTestStart(IOR_param_t *params); void ShowTestEnd(IOR_test_t *tptr); diff --git a/src/ior-output.c b/src/ior-output.c index da4f5f1a..38dc6e63 100644 --- a/src/ior-output.c +++ b/src/ior-output.c @@ -151,6 +151,9 @@ static void PrintNamedArrayStart(char * key){ } static void PrintEndSection(){ + if (rank != 0) + return; + indent--; if(outputFormat == OUTPUT_JSON){ fprintf(out_resultfile, "\n"); @@ -161,6 +164,8 @@ static void PrintEndSection(){ } static void PrintArrayStart(){ + if (rank != 0) + return; PrintNextToken(); needNextToken = 0; if(outputFormat == OUTPUT_JSON){ @@ -169,6 +174,8 @@ static void PrintArrayStart(){ } static void PrintArrayNamedStart(char * key){ + if (rank != 0) + return; PrintNextToken(); needNextToken = 0; if(outputFormat == OUTPUT_JSON){ @@ -177,6 +184,9 @@ static void PrintArrayNamedStart(char * key){ } static void PrintArrayEnd(){ + if (rank != 0) + return; + indent--; if(outputFormat == OUTPUT_JSON){ fprintf(out_resultfile, "]\n"); @@ -185,10 +195,14 @@ static void PrintArrayEnd(){ } void PrintRepeatEnd(){ + if (rank != 0) + return; PrintArrayEnd(); } void PrintRepeatStart(){ + if (rank != 0) + return; if( outputFormat == OUTPUT_DEFAULT){ return; } @@ -231,13 +245,11 @@ void PrintReducedResult(IOR_test_t *test, int access, double bw, double *diff_su fflush(out_resultfile); } - -/* - * Message to print immediately after MPI_Init so we know that - * ior has started. - */ -void PrintEarlyHeader() +void PrintHeader(int argc, char **argv) { + struct utsname unamebuf; + int i; + if (rank != 0) return; @@ -247,16 +259,6 @@ void PrintEarlyHeader() }else{ printf("IOR-" META_VERSION ": MPI Coordinated Test of Parallel I/O\n"); } -} - -void PrintHeader(int argc, char **argv) -{ - struct utsname unamebuf; - int i; - - if (rank != 0) - return; - PrintKeyVal("Began", CurrentTimeString()); PrintKeyValStart("Command line"); fprintf(out_resultfile, "%s", argv[0]); diff --git a/src/ior.c b/src/ior.c index 66cca769..b84f01ad 100755 --- a/src/ior.c +++ b/src/ior.c @@ -62,7 +62,6 @@ IOR_test_t * ior_run(int argc, char **argv, MPI_Comm world_com, FILE * world_out MPI_CHECK(MPI_Comm_size(mpi_comm_world, &numTasksWorld), "cannot get number of tasks"); MPI_CHECK(MPI_Comm_rank(mpi_comm_world, &rank), "cannot get rank"); - PrintEarlyHeader(); /* setup tests, and validate parameters */ tests_head = ParseCommandLine(argc, argv); @@ -113,8 +112,6 @@ int ior_main(int argc, char **argv) "cannot get number of tasks"); MPI_CHECK(MPI_Comm_rank(mpi_comm_world, &rank), "cannot get rank"); - PrintEarlyHeader(); - /* set error-handling */ /*MPI_CHECK(MPI_Errhandler_set(mpi_comm_world, MPI_ERRORS_RETURN), "cannot set errhandler"); */ From 6c5952906b26689a369f53975df3f080b4368764 Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Thu, 11 Oct 2018 19:58:30 +0100 Subject: [PATCH 035/206] Dry-run option to test the configuration / scripts. --- src/aiori-HDF5.c | 34 ++++++++++++++++++++++++---------- src/aiori-MPIIO.c | 41 +++++++++++++++++++++++++++++++---------- src/aiori-POSIX.c | 25 +++++++++++++++++++++---- src/ior-output.c | 4 ++++ src/ior.h | 1 + src/parse_options.c | 1 + 6 files changed, 82 insertions(+), 24 deletions(-) diff --git a/src/aiori-HDF5.c b/src/aiori-HDF5.c index 18276270..250f93d1 100755 --- a/src/aiori-HDF5.c +++ b/src/aiori-HDF5.c @@ -241,13 +241,15 @@ static void *HDF5_Open(char *testFileName, IOR_param_t * param) #endif /* open file */ - if (param->open == WRITE) { /* WRITE */ - *fd = H5Fcreate(testFileName, fd_mode, - createPropList, accessPropList); - HDF5_CHECK(*fd, "cannot create file"); - } else { /* READ or CHECK */ - *fd = H5Fopen(testFileName, fd_mode, accessPropList); - HDF5_CHECK(*fd, "cannot open file"); + if(! param->dryRun){ + if (param->open == WRITE) { /* WRITE */ + *fd = H5Fcreate(testFileName, fd_mode, + createPropList, accessPropList); + HDF5_CHECK(*fd, "cannot create file"); + } else { /* READ or CHECK */ + *fd = H5Fopen(testFileName, fd_mode, accessPropList); + HDF5_CHECK(*fd, "cannot open file"); + } } /* show hints actually attached to file handle */ @@ -395,6 +397,9 @@ static IOR_offset_t HDF5_Xfer(int access, void *fd, IOR_size_t * buffer, } } + if(param->dryRun) + return length; + /* create new data set */ if (startNewDataSet == TRUE) { /* if just opened this file, no data set to close yet */ @@ -440,6 +445,8 @@ static void HDF5_Fsync(void *fd, IOR_param_t * param) */ static void HDF5_Close(void *fd, IOR_param_t * param) { + if(param->dryRun) + return; if (param->fd_fppReadCheck == NULL) { HDF5_CHECK(H5Dclose(dataSet), "cannot close data set"); HDF5_CHECK(H5Sclose(dataSpace), "cannot close data space"); @@ -459,7 +466,10 @@ static void HDF5_Close(void *fd, IOR_param_t * param) */ static void HDF5_Delete(char *testFileName, IOR_param_t * param) { - return(MPIIO_Delete(testFileName, param)); + if(param->dryRun) + return + MPIIO_Delete(testFileName, param); + return; } /* @@ -591,7 +601,9 @@ static void SetupDataSet(void *fd, IOR_param_t * param) static IOR_offset_t HDF5_GetFileSize(IOR_param_t * test, MPI_Comm testComm, char *testFileName) { - return(MPIIO_GetFileSize(test, testComm, testFileName)); + if(test->dryRun) + return 0; + return(MPIIO_GetFileSize(test, testComm, testFileName)); } /* @@ -599,5 +611,7 @@ HDF5_GetFileSize(IOR_param_t * test, MPI_Comm testComm, char *testFileName) */ static int HDF5_Access(const char *path, int mode, IOR_param_t *param) { - return(MPIIO_Access(path, mode, param)); + if(param->dryRun) + return 0; + return(MPIIO_Access(path, mode, param)); } diff --git a/src/aiori-MPIIO.c b/src/aiori-MPIIO.c index 5b817365..6bf991bd 100755 --- a/src/aiori-MPIIO.c +++ b/src/aiori-MPIIO.c @@ -69,6 +69,9 @@ ior_aiori_t mpiio_aiori = { */ int MPIIO_Access(const char *path, int mode, IOR_param_t *param) { + if(param->dryRun){ + return MPI_SUCCESS; + } MPI_File fd; int mpi_mode = MPI_MODE_UNIQUE_OPEN; @@ -93,7 +96,10 @@ int MPIIO_Access(const char *path, int mode, IOR_param_t *param) */ static void *MPIIO_Create(char *testFileName, IOR_param_t * param) { - return MPIIO_Open(testFileName, param); + if(param->dryRun){ + return 0; + } + return MPIIO_Open(testFileName, param); } /* @@ -171,11 +177,13 @@ static void *MPIIO_Open(char *testFileName, IOR_param_t * param) ShowHints(&mpiHints); fprintf(stdout, "}\n"); } - MPI_CHECK(MPI_File_open(comm, testFileName, fd_mode, mpiHints, fd), + if(! param->dryRun){ + MPI_CHECK(MPI_File_open(comm, testFileName, fd_mode, mpiHints, fd), "cannot open file"); + } /* show hints actually attached to file handle */ - if (rank == 0 && param->showHints) { + if (rank == 0 && param->showHints && ! param->dryRun) { if (mpiHints != MPI_INFO_NULL) MPI_CHECK(MPI_Info_free(&mpiHints), "MPI_Info_free failed"); MPI_CHECK(MPI_File_get_info(*fd, &mpiHints), @@ -186,7 +194,7 @@ static void *MPIIO_Open(char *testFileName, IOR_param_t * param) } /* preallocate space for file */ - if (param->preallocate && param->open == WRITE) { + if (param->preallocate && param->open == WRITE && ! param->dryRun) { MPI_CHECK(MPI_File_preallocate(*fd, (MPI_Offset) (param->segmentCount * @@ -232,11 +240,13 @@ static void *MPIIO_Open(char *testFileName, IOR_param_t * param) MPI_CHECK(MPI_Type_commit(¶m->fileType), "cannot commit datatype"); - MPI_CHECK(MPI_File_set_view(*fd, (MPI_Offset) 0, + if(! param->dryRun){ + MPI_CHECK(MPI_File_set_view(*fd, (MPI_Offset) 0, param->transferType, param->fileType, "native", (MPI_Info) MPI_INFO_NULL), "cannot set file view"); + } } if (mpiHints != MPI_INFO_NULL) MPI_CHECK(MPI_Info_free(&mpiHints), "MPI_Info_free failed"); @@ -254,6 +264,9 @@ static IOR_offset_t MPIIO_Xfer(int access, void *fd, IOR_size_t * buffer, will get "assignment from incompatible pointer-type" warnings, if we only use this one set of signatures. */ + if(param->dryRun) + return length; + int (MPIAPI * Access) (MPI_File, void *, int, MPI_Datatype, MPI_Status *); int (MPIAPI * Access_at) (MPI_File, MPI_Offset, void *, int, @@ -382,8 +395,10 @@ static IOR_offset_t MPIIO_Xfer(int access, void *fd, IOR_size_t * buffer, */ static void MPIIO_Fsync(void *fdp, IOR_param_t * param) { - if (MPI_File_sync(*(MPI_File *)fdp) != MPI_SUCCESS) - EWARN("fsync() failed"); + if(param->dryRun) + return; + if (MPI_File_sync(*(MPI_File *)fdp) != MPI_SUCCESS) + EWARN("fsync() failed"); } /* @@ -391,7 +406,9 @@ static void MPIIO_Fsync(void *fdp, IOR_param_t * param) */ static void MPIIO_Close(void *fd, IOR_param_t * param) { - MPI_CHECK(MPI_File_close((MPI_File *) fd), "cannot close file"); + if(! param->dryRun){ + MPI_CHECK(MPI_File_close((MPI_File *) fd), "cannot close file"); + } if ((param->useFileView == TRUE) && (param->fd_fppReadCheck == NULL)) { /* * need to free the datatype, so done in the close process @@ -409,8 +426,10 @@ static void MPIIO_Close(void *fd, IOR_param_t * param) */ void MPIIO_Delete(char *testFileName, IOR_param_t * param) { - MPI_CHECK(MPI_File_delete(testFileName, (MPI_Info) MPI_INFO_NULL), - "cannot delete file"); + if(param->dryRun) + return; + MPI_CHECK(MPI_File_delete(testFileName, (MPI_Info) MPI_INFO_NULL), + "cannot delete file"); } /* @@ -473,6 +492,8 @@ static IOR_offset_t SeekOffset(MPI_File fd, IOR_offset_t offset, IOR_offset_t MPIIO_GetFileSize(IOR_param_t * test, MPI_Comm testComm, char *testFileName) { + if(test->dryRun) + return 0; IOR_offset_t aggFileSizeFromStat, tmpMin, tmpMax, tmpSum; MPI_File fd; MPI_Comm comm; diff --git a/src/aiori-POSIX.c b/src/aiori-POSIX.c index ef46ed2e..cb872971 100755 --- a/src/aiori-POSIX.c +++ b/src/aiori-POSIX.c @@ -278,6 +278,9 @@ void *POSIX_Create(char *testFileName, IOR_param_t * param) if (param->useO_DIRECT == TRUE) set_o_direct_flag(&fd_oflag); + if(param->dryRun) + return 0; + #ifdef HAVE_LUSTRE_LUSTRE_USER_H if (param->lustre_set_striping) { /* In the single-shared-file case, task 0 has to creat the @@ -383,6 +386,10 @@ void *POSIX_Open(char *testFileName, IOR_param_t * param) set_o_direct_flag(&fd_oflag); fd_oflag |= O_RDWR; + + if(param->dryRun) + return 0; + *fd = open64(testFileName, fd_oflag); if (*fd < 0) ERR("open64 failed"); @@ -419,6 +426,9 @@ static IOR_offset_t POSIX_Xfer(int access, void *file, IOR_size_t * buffer, long long rc; int fd; + if(param->dryRun) + return length; + fd = *(int *)file; #ifdef HAVE_GPFS_FCNTL_H @@ -500,6 +510,8 @@ static void POSIX_Fsync(void *fd, IOR_param_t * param) */ void POSIX_Close(void *fd, IOR_param_t * param) { + if(param->dryRun) + return; if (close(*(int *)fd) != 0) ERR("close() failed"); free(fd); @@ -510,11 +522,14 @@ void POSIX_Close(void *fd, IOR_param_t * param) */ void POSIX_Delete(char *testFileName, IOR_param_t * param) { - char errmsg[256]; - sprintf(errmsg, "[RANK %03d]: unlink() of file \"%s\" failed\n", - rank, testFileName); - if (unlink(testFileName) != 0) + if(param->dryRun) + return; + if (unlink(testFileName) != 0){ + char errmsg[256]; + sprintf(errmsg, "[RANK %03d]: unlink() of file \"%s\" failed\n", + rank, testFileName); EWARN(errmsg); + } } /* @@ -523,6 +538,8 @@ void POSIX_Delete(char *testFileName, IOR_param_t * param) IOR_offset_t POSIX_GetFileSize(IOR_param_t * test, MPI_Comm testComm, char *testFileName) { + if(test->dryRun) + return 0; struct stat stat_buf; IOR_offset_t aggFileSizeFromStat, tmpMin, tmpMax, tmpSum; diff --git a/src/ior-output.c b/src/ior-output.c index 38dc6e63..32a60549 100644 --- a/src/ior-output.c +++ b/src/ior-output.c @@ -338,6 +338,7 @@ void ShowTestStart(IOR_param_t *test) PrintKeyValInt("outlierThreshold", test->outlierThreshold); PrintKeyVal("options", test->options); + PrintKeyValInt("dryRun", test->dryRun); PrintKeyValInt("nodes", test->nodes); PrintKeyValInt("memoryPerTask", (unsigned long) test->memoryPerTask); PrintKeyValInt("memoryPerNode", (unsigned long) test->memoryPerNode); @@ -442,6 +443,9 @@ void ShowSetup(IOR_param_t *params) PrintKeyVal("xfersize", HumanReadable(params->transferSize, BASE_TWO)); PrintKeyVal("blocksize", HumanReadable(params->blockSize, BASE_TWO)); PrintKeyVal("aggregate filesize", HumanReadable(params->expectedAggFileSize, BASE_TWO)); + if(params->dryRun){ + PrintKeyValInt("dryRun", params->dryRun); + } #ifdef HAVE_LUSTRE_LUSTRE_USER_H if (params->lustre_set_striping) { diff --git a/src/ior.h b/src/ior.h index 871346c4..202cd7fd 100755 --- a/src/ior.h +++ b/src/ior.h @@ -93,6 +93,7 @@ typedef struct char * testFileName_fppReadCheck;/* filename for fpp read check */ char * hintsFileName; /* full name for hints file */ char * options; /* options string */ + int dryRun; /* do not perform any I/Os just run evtl. inputs print dummy output */ int numTasks; /* number of tasks for test */ int nodes; /* number of nodes for test */ int tasksPerNode; /* number of tasks per node */ diff --git a/src/parse_options.c b/src/parse_options.c index 460dd520..b48b6c1d 100755 --- a/src/parse_options.c +++ b/src/parse_options.c @@ -524,6 +524,7 @@ IOR_test_t *ParseCommandLine(int argc, char **argv) {'Z', NULL, "reorderTasksRandom -- changes task ordering to random ordering for readback", OPTION_FLAG, 'd', & initialTestParams.reorderTasksRandom}, {.help=" -O summaryFile=FILE -- store result data into this file", .arg = OPTION_OPTIONAL_ARGUMENT}, {.help=" -O summaryFormat=[default,JSON,CSV] -- use the format for outputing the summary", .arg = OPTION_OPTIONAL_ARGUMENT}, + {0, "dryRun", "do not perform any I/Os just run evtl. inputs print dummy output", OPTION_FLAG, 'd', & initialTestParams.dryRun}, LAST_OPTION, }; From f4afa63ebffa7611044a307690223d4b23170a24 Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Wed, 17 Oct 2018 13:50:04 +0100 Subject: [PATCH 036/206] Bugfix #106 mostly. --- src/mdtest.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/mdtest.c b/src/mdtest.c index 9d732a8e..aa697f52 100644 --- a/src/mdtest.c +++ b/src/mdtest.c @@ -2332,12 +2332,11 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * } else if (branch_factor == 1) { num_dirs_in_tree = depth + 1; } else { - num_dirs_in_tree = - (1 - pow(branch_factor, depth+1)) / (1 - branch_factor); + num_dirs_in_tree = (pow(branch_factor, depth+1) - 1) / (branch_factor - 1); } } if (items_per_dir > 0) { - if(unique_dir_per_task){ + if(items == 0){ items = items_per_dir * num_dirs_in_tree; } } else { From 093113313d691c4a6129733c17e00f1c8a244ddd Mon Sep 17 00:00:00 2001 From: Marc Vef Date: Wed, 17 Oct 2018 16:38:35 +0200 Subject: [PATCH 037/206] mdtest: Use GetTimeStamp() instead of MPI_Wtime() IOR uses the GetTimeStamp() wrapper to allow gettimeofday() to be used for timings instead of MPI_Wtime() to calculate throughput. This commits adds the same logic to mdtest timings. --- src/mdtest.c | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/mdtest.c b/src/mdtest.c index 9d732a8e..7d8cbf56 100644 --- a/src/mdtest.c +++ b/src/mdtest.c @@ -179,7 +179,7 @@ void offset_timers(double * t, int tcount) { fflush( out_logfile ); } - toffset = MPI_Wtime() - t[tcount]; + toffset = GetTimeStamp() - t[tcount]; for (i = 0; i < tcount+1; i++) { t[i] += toffset; } @@ -845,7 +845,7 @@ void directory_test(const int iteration, const int ntasks, const char *path, ran } MPI_Barrier(testComm); - t[0] = MPI_Wtime(); + t[0] = GetTimeStamp(); /* create phase */ if(create_only) { @@ -880,7 +880,7 @@ void directory_test(const int iteration, const int ntasks, const char *path, ran if (barriers) { MPI_Barrier(testComm); } - t[1] = MPI_Wtime(); + t[1] = GetTimeStamp(); /* stat phase */ if (stat_only) { @@ -912,7 +912,7 @@ void directory_test(const int iteration, const int ntasks, const char *path, ran if (barriers) { MPI_Barrier(testComm); } - t[2] = MPI_Wtime(); + t[2] = GetTimeStamp(); /* read phase */ if (read_only) { @@ -944,7 +944,7 @@ void directory_test(const int iteration, const int ntasks, const char *path, ran if (barriers) { MPI_Barrier(testComm); } - t[3] = MPI_Wtime(); + t[3] = GetTimeStamp(); if (remove_only) { for (int dir_iter = 0; dir_iter < directory_loops; dir_iter ++){ @@ -977,7 +977,7 @@ void directory_test(const int iteration, const int ntasks, const char *path, ran if (barriers) { MPI_Barrier(testComm); } - t[4] = MPI_Wtime(); + t[4] = GetTimeStamp(); if (remove_only) { if (unique_dir_per_task) { @@ -1047,7 +1047,7 @@ int updateStoneWallIterations(int iteration, rank_progress_t * progress, double uint64_t done = progress->items_done; long long unsigned max_iter = 0; MPI_Allreduce(& progress->items_done, & max_iter, 1, MPI_LONG_LONG_INT, MPI_MAX, testComm); - summary_table[iteration].stonewall_time[MDTEST_FILE_CREATE_NUM] = MPI_Wtime() - tstart; + summary_table[iteration].stonewall_time[MDTEST_FILE_CREATE_NUM] = GetTimeStamp() - tstart; // continue to the maximum... long long min_accessed = 0; @@ -1082,7 +1082,7 @@ void file_test(const int iteration, const int ntasks, const char *path, rank_pro } MPI_Barrier(testComm); - t[0] = MPI_Wtime(); + t[0] = GetTimeStamp(); /* create phase */ if (create_only ) { @@ -1150,7 +1150,7 @@ void file_test(const int iteration, const int ntasks, const char *path, rank_pro if (barriers) { MPI_Barrier(testComm); } - t[1] = MPI_Wtime(); + t[1] = GetTimeStamp(); /* stat phase */ if (stat_only ) { @@ -1178,7 +1178,7 @@ void file_test(const int iteration, const int ntasks, const char *path, rank_pro if (barriers) { MPI_Barrier(testComm); } - t[2] = MPI_Wtime(); + t[2] = GetTimeStamp(); /* read phase */ if (read_only ) { @@ -1210,7 +1210,7 @@ void file_test(const int iteration, const int ntasks, const char *path, rank_pro if (barriers) { MPI_Barrier(testComm); } - t[3] = MPI_Wtime(); + t[3] = GetTimeStamp(); if (remove_only) { for (int dir_iter = 0; dir_iter < directory_loops; dir_iter ++){ @@ -1242,7 +1242,7 @@ void file_test(const int iteration, const int ntasks, const char *path, rank_pro if (barriers) { MPI_Barrier(testComm); } - t[4] = MPI_Wtime(); + t[4] = GetTimeStamp(); if (remove_only) { if (unique_dir_per_task) { unique_dir_access(RM_UNI_DIR, temp_path); @@ -1888,7 +1888,7 @@ static void mdtest_iteration(int i, int j, MPI_Group testgroup, mdtest_results_t /* create hierarchical directory structure */ MPI_Barrier(testComm); - startCreate = MPI_Wtime(); + startCreate = GetTimeStamp(); for (int dir_iter = 0; dir_iter < directory_loops; dir_iter ++){ prep_testdir(j, dir_iter); @@ -1950,7 +1950,7 @@ static void mdtest_iteration(int i, int j, MPI_Group testgroup, mdtest_results_t } } MPI_Barrier(testComm); - endCreate = MPI_Wtime(); + endCreate = GetTimeStamp(); summary_table->rate[8] = num_dirs_in_tree / (endCreate - startCreate); summary_table->time[8] = (endCreate - startCreate); @@ -2021,7 +2021,7 @@ static void mdtest_iteration(int i, int j, MPI_Group testgroup, mdtest_results_t MPI_Barrier(testComm); if (remove_only) { - startCreate = MPI_Wtime(); + startCreate = GetTimeStamp(); for (int dir_iter = 0; dir_iter < directory_loops; dir_iter ++){ prep_testdir(j, dir_iter); if (unique_dir_per_task) { @@ -2083,7 +2083,7 @@ static void mdtest_iteration(int i, int j, MPI_Group testgroup, mdtest_results_t } MPI_Barrier(testComm); - endCreate = MPI_Wtime(); + endCreate = GetTimeStamp(); summary_table->rate[9] = num_dirs_in_tree / (endCreate - startCreate); summary_table->time[9] = endCreate - startCreate; summary_table->items[9] = num_dirs_in_tree; From 6b0b1a9830d76821ddbfc90330a3e4e969265e5b Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Wed, 17 Oct 2018 16:04:49 +0100 Subject: [PATCH 038/206] Fix computation when using branching factor with -n and -i. --- src/mdtest.c | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/mdtest.c b/src/mdtest.c index aa697f52..760446c4 100644 --- a/src/mdtest.c +++ b/src/mdtest.c @@ -123,6 +123,7 @@ static uint64_t num_dirs_in_tree; */ static uint64_t items; static uint64_t items_per_dir; +static uint64_t num_dirs_in_tree_calc; /* this is a workaround until the overal code is refactored */ static int directory_loops; static int print_time; static int random_seed; @@ -1260,6 +1261,10 @@ void file_test(const int iteration, const int ntasks, const char *path, rank_pro offset_timers(t, 4); } + if(num_dirs_in_tree_calc){ /* this is temporary fix needed when using -n and -i together */ + items *= num_dirs_in_tree_calc; + } + /* calculate times */ if (create_only) { summary_table[iteration].rate[4] = items*size/(t[1] - t[0]); @@ -2338,6 +2343,8 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * if (items_per_dir > 0) { if(items == 0){ items = items_per_dir * num_dirs_in_tree; + }else{ + num_dirs_in_tree_calc = num_dirs_in_tree; } } else { if (leaf_only) { @@ -2481,17 +2488,21 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * MPI_Group_range_incl(worldgroup, 1, (void *)&range, &testgroup); MPI_Comm_create(testComm, testgroup, &testComm); if (rank == 0) { + uint64_t items_all = i * items; + if(num_dirs_in_tree_calc){ + items_all *= num_dirs_in_tree_calc; + } if (files_only && dirs_only) { - fprintf(out_logfile, "\n%d tasks, "LLU" files/directories\n", i, i * items); + fprintf(out_logfile, "\n%d tasks, "LLU" files/directories\n", i, items_all); } else if (files_only) { if (!shared_file) { - fprintf(out_logfile, "\n%d tasks, "LLU" files\n", i, i * items); + fprintf(out_logfile, "\n%d tasks, "LLU" files\n", i, items_all); } else { fprintf(out_logfile, "\n%d tasks, 1 file\n", i); } } else if (dirs_only) { - fprintf(out_logfile, "\n%d tasks, "LLU" directories\n", i, i * items); + fprintf(out_logfile, "\n%d tasks, "LLU" directories\n", i, items_all); } } if (rank == 0 && verbose >= 1) { From ea3aeea964eebe5db0dd136b80bafec7392efb00 Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Wed, 17 Oct 2018 16:16:46 +0100 Subject: [PATCH 039/206] Bugfix initialize the variable to zero for proper API usage. --- src/mdtest.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/mdtest.c b/src/mdtest.c index 760446c4..4ad425ad 100644 --- a/src/mdtest.c +++ b/src/mdtest.c @@ -2140,6 +2140,7 @@ void mdtest_init_args(){ unique_dir_per_task = 0; time_unique_dir_overhead = 0; items = 0; + num_dirs_in_tree_calc = 0; collective_creates = 0; write_bytes = 0; stone_wall_timer_seconds = 0; From 5a8071fbd8b7f1e4b6695d332c3c6a774e6c55a6 Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Sun, 28 Oct 2018 18:34:22 +0000 Subject: [PATCH 040/206] GNU99 extracted _XOPEN_SOURCE to config.h. --- configure.ac | 10 +++------- src/aiori-DUMMY.c | 2 -- src/aiori-MMAP.c | 2 -- src/aiori.c | 2 -- src/ior.c | 8 +++----- src/iordef.h | 2 +- src/mdtest.c | 2 -- src/option.c | 1 - src/parse_options.c | 7 +++++-- 9 files changed, 12 insertions(+), 24 deletions(-) diff --git a/configure.ac b/configure.ac index c28fb718..002b2dfd 100755 --- a/configure.ac +++ b/configure.ac @@ -50,7 +50,7 @@ AC_CHECK_HEADERS([fcntl.h libintl.h stdlib.h string.h strings.h sys/ioctl.h sys/ AC_TYPE_SIZE_T # Checks for library functions. -AC_CHECK_FUNCS([getpagesize gettimeofday memset mkdir pow putenv realpath regcomp sqrt strcasecmp strchr strerror strncasecmp strstr uname statfs statvfs]) +AC_CHECK_FUNCS([sysconf gettimeofday memset mkdir pow putenv realpath regcomp sqrt strcasecmp strchr strerror strncasecmp strstr uname statfs statvfs]) AC_SEARCH_LIBS([sqrt], [m], [], [AC_MSG_ERROR([Math library not found])]) @@ -76,6 +76,8 @@ AS_IF([test "$ac_cv_header_gpfs_h" = "yes" -o "$ac_cv_header_gpfs_fcntl_h" = "ye # Check for system capabilities AC_SYS_LARGEFILE +AC_DEFINE([_XOPEN_SOURCE], [700], [C99 compatibility]) + # Check for lustre availability AC_ARG_WITH([lustre], [AS_HELP_STRING([--with-lustre], @@ -254,12 +256,6 @@ Consider --with-aws4c=, CPPFLAGS, LDFLAGS, etc]) ]) - - - - - - # Enable building "IOR", in all capitals AC_ARG_ENABLE([caps], [AS_HELP_STRING([--enable-caps], diff --git a/src/aiori-DUMMY.c b/src/aiori-DUMMY.c index fd3b087a..cfd920bc 100755 --- a/src/aiori-DUMMY.c +++ b/src/aiori-DUMMY.c @@ -6,8 +6,6 @@ # include "config.h" #endif -#define _XOPEN_SOURCE 700 - #include #include #include diff --git a/src/aiori-MMAP.c b/src/aiori-MMAP.c index ed043573..f812bddd 100644 --- a/src/aiori-MMAP.c +++ b/src/aiori-MMAP.c @@ -11,8 +11,6 @@ # include "config.h" #endif -#define _XOPEN_SOURCE 700 - #include #include diff --git a/src/aiori.c b/src/aiori.c index 9c27386c..1c3d6b86 100644 --- a/src/aiori.c +++ b/src/aiori.c @@ -16,8 +16,6 @@ # include "config.h" #endif -#define _XOPEN_SOURCE 700 - #include #include diff --git a/src/ior.c b/src/ior.c index d29ef3b2..12b93906 100755 --- a/src/ior.c +++ b/src/ior.c @@ -12,8 +12,6 @@ # include "config.h" #endif -#define _XOPEN_SOURCE 700 - #include #include #include @@ -466,10 +464,10 @@ static void *aligned_buffer_alloc(size_t size) char *buf, *tmp; char *aligned; -#ifdef HAVE_GETPAGESIZE - size_t pageSize = getpagesize(); -#else +#ifdef HAVE_SYSCONF long pageSize = sysconf(_SC_PAGESIZE); +#else + size_t pageSize = getpagesize(); #endif pageMask = pageSize - 1; diff --git a/src/iordef.h b/src/iordef.h index 0eea3114..1d1f2162 100755 --- a/src/iordef.h +++ b/src/iordef.h @@ -45,7 +45,7 @@ # define srandom srand # define random() (rand() * (RAND_MAX+1) + rand()) /* Note: only 30 bits */ # define sleep(X) Sleep((X)*1000) -# define getpagesize() 4096 +# define sysconf(X) 4096 #else # include /* MAXPATHLEN */ # include diff --git a/src/mdtest.c b/src/mdtest.c index 147cf840..f7c42643 100644 --- a/src/mdtest.c +++ b/src/mdtest.c @@ -28,8 +28,6 @@ * $Date: 2013/11/27 17:05:31 $ * $Author: brettkettering $ */ -#define _XOPEN_SOURCE 700 - #include #include #include diff --git a/src/option.c b/src/option.c index 5bd19fe7..1e646af2 100644 --- a/src/option.c +++ b/src/option.c @@ -1,4 +1,3 @@ -#define _XOPEN_SOURCE 700 #include #include #include diff --git a/src/parse_options.c b/src/parse_options.c index 546d0dc5..f0db71e3 100755 --- a/src/parse_options.c +++ b/src/parse_options.c @@ -16,8 +16,6 @@ #include "config.h" #endif -#define _XOPEN_SOURCE 700 - #include #include #include @@ -53,7 +51,12 @@ static size_t NodeMemoryStringToBytes(char *size_str) if (percent > 100 || percent < 0) ERR("percentage must be between 0 and 100"); +#ifdef HAVE_SYSCONF page_size = sysconf(_SC_PAGESIZE); +#else + page_size = getpagesize(); +#endif + #ifdef _SC_PHYS_PAGES num_pages = sysconf(_SC_PHYS_PAGES); if (num_pages == -1) From 8902bc8722da89d094e44338da81c8ff61b3e039 Mon Sep 17 00:00:00 2001 From: Wang Shilong Date: Thu, 22 Nov 2018 19:54:49 +0800 Subject: [PATCH 041/206] mdtest: fix calculation for barriers Currently, calculation for barriers case looks a bit suprising, for example, Min stats is from min of Max value of number of mpi proc, this makes Min and Max very similar in our testing thus we get a small Std value too. I am not a MPI expert, but question is why we don't use more nature calcuation which calculate all results from different iterations and MPI proc? Signed-off-by: Wang Shilong --- src/mdtest.c | 64 +--------------------------------------------------- 1 file changed, 1 insertion(+), 63 deletions(-) diff --git a/src/mdtest.c b/src/mdtest.c index 147cf840..f1c62a96 100644 --- a/src/mdtest.c +++ b/src/mdtest.c @@ -1420,68 +1420,7 @@ void summarize_results(int iterations) { start = stop = 0; } - /* calculate aggregates */ - if (barriers) { - double maxes[iterations]; - - - /* Because each proc times itself, in the case of barriers we - * have to backwards calculate the time to simulate the use - * of barriers. - */ - for (i = start; i < stop; i++) { - for (j=0; j maxes[j]) { - min = maxes[j]; - } - if (max < maxes[j]) { - max = maxes[j]; - } - sum += maxes[j]; - } - mean = sum / iterations; - for (j=0; j Date: Tue, 4 Dec 2018 19:16:41 +0000 Subject: [PATCH 042/206] Reverted the change to HDF5 module back. --- doc/sphinx/userDoc/options.rst | 5 ++--- src/aiori-HDF5.c | 23 +---------------------- 2 files changed, 3 insertions(+), 25 deletions(-) diff --git a/doc/sphinx/userDoc/options.rst b/doc/sphinx/userDoc/options.rst index b46bee46..8b441269 100644 --- a/doc/sphinx/userDoc/options.rst +++ b/doc/sphinx/userDoc/options.rst @@ -289,9 +289,8 @@ HDF5-ONLY * setAlignment - HDF5 alignment in bytes (e.g.: 8, 4k, 2m, 1g) [1] -HDF5-backend options -^^^^^^^^^^^^^^^^^^^^ - * --hdf5.collectiveMetadata - enable HDF5 collective metadata (available since HDF5-1.10.0) + * collectiveMetadata - enable HDF5 collective metadata (available since + HDF5-1.10.0) MPIIO-, HDF5-, AND NCMPI-ONLY ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/src/aiori-HDF5.c b/src/aiori-HDF5.c index d1e6eabe..18276270 100755 --- a/src/aiori-HDF5.c +++ b/src/aiori-HDF5.c @@ -93,7 +93,6 @@ static char* HDF5_GetVersion(); static void HDF5_Fsync(void *, IOR_param_t *); static IOR_offset_t HDF5_GetFileSize(IOR_param_t *, MPI_Comm, char *); static int HDF5_Access(const char *, int, IOR_param_t *); -static option_help * HDF5_get_options(); /************************** D E C L A R A T I O N S ***************************/ @@ -113,7 +112,6 @@ ior_aiori_t hdf5_aiori = { .rmdir = aiori_posix_rmdir, .access = HDF5_Access, .stat = aiori_posix_stat, - .get_options = HDF5_get_options }; static hid_t xferPropList; /* xfer property list */ @@ -123,27 +121,8 @@ hid_t fileDataSpace; /* file data space id */ hid_t memDataSpace; /* memory data space id */ int newlyOpenedFile; /* newly opened file */ -/************************** O P T I O N S *****************************/ -struct HDF5_options{ - int collective_md; -}; /***************************** F U N C T I O N S ******************************/ - -static struct HDF5_options o = { - .collective_md = 0 -}; - -static option_help options [] = { - {0, "hdf5.collectiveMetadata", "Use collectiveMetadata (available since HDF5-1.10.0)", OPTION_FLAG, 'd', & o.collective_md}, - LAST_OPTION -}; - -static option_help * HDF5_get_options(){ - return options; -} - - /* * Create and open a file through the HDF5 interface. */ @@ -251,7 +230,7 @@ static void *HDF5_Open(char *testFileName, IOR_param_t * param) "cannot set alignment"); #ifdef HAVE_H5PSET_ALL_COLL_METADATA_OPS - if (o.collective_md) { + if (param->collective_md) { /* more scalable metadata */ HDF5_CHECK(H5Pset_all_coll_metadata_ops(accessPropList, 1), From 72c487013a096b98483ad91459b1432d699c10b8 Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Tue, 4 Dec 2018 22:59:15 +0000 Subject: [PATCH 043/206] rebase with latest DAOS Signed-off-by: Mohamad Chaarawi --- src/aiori-DAOS.c | 26 ++++++++++++++++---------- src/aiori-DFS.c | 10 +++++----- 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/src/aiori-DAOS.c b/src/aiori-DAOS.c index 657bc1d2..e498b8f6 100644 --- a/src/aiori-DAOS.c +++ b/src/aiori-DAOS.c @@ -518,8 +518,14 @@ static void ObjectClassParse(const char *string) objectClass = DAOS_OC_SMALL_RW; else if (strcasecmp(string, "large") == 0) objectClass = DAOS_OC_LARGE_RW; - else if (strcasecmp(string, "echo") == 0) - objectClass = DAOS_OC_ECHO_RW; + else if (strcasecmp(string, "echo_tiny") == 0) + objectClass = DAOS_OC_ECHO_TINY_RW; + else if (strcasecmp(string, "echo_R2S") == 0) + objectClass = DAOS_OC_ECHO_R2S_RW; + else if (strcasecmp(string, "echo_R3S") == 0) + objectClass = DAOS_OC_ECHO_R3S_RW; + else if (strcasecmp(string, "echo_R4S") == 0) + objectClass = DAOS_OC_ECHO_R4S_RW; else if (strcasecmp(string, "R2") == 0) objectClass = DAOS_OC_R2_RW; else if (strcasecmp(string, "R2S") == 0) @@ -597,7 +603,7 @@ static void DAOS_Init(IOR_param_t *param) if (rank == 0) { uuid_t uuid; - d_rank_t rank[13]; + d_rank_t d_rank[13]; d_rank_list_t ranks; if (o.daosPool == NULL) @@ -610,8 +616,8 @@ static void DAOS_Init(IOR_param_t *param) rc = uuid_parse(o.daosPool, uuid); DCHECK(rc, "Failed to parse 'daosPool': %s", o.daosPool); - ranks.rl_ranks = rank; - ParseService(param, sizeof(rank) / sizeof(rank[0]), &ranks); + ranks.rl_ranks = d_rank; + ParseService(param, sizeof(d_rank) / sizeof(d_rank[0]), &ranks); rc = daos_pool_connect(uuid, o.daosGroup, &ranks, DAOS_PC_RW, &pool, &poolInfo, @@ -706,7 +712,7 @@ static void kill_daos_server(IOR_param_t *param) { daos_pool_info_t info; - d_rank_t rank, svc_ranks[13]; + d_rank_t d_rank, svc_ranks[13]; d_rank_list_t svc, targets; uuid_t uuid; char *s; @@ -718,21 +724,21 @@ kill_daos_server(IOR_param_t *param) if (info.pi_ntargets - info.pi_ndisabled <= 1) return; /* choose the last alive one */ - rank = info.pi_ntargets - 1 - info.pi_ndisabled; + d_rank = info.pi_ntargets - 1 - info.pi_ndisabled; rc = uuid_parse(o.daosPool, uuid); DCHECK(rc, "Failed to parse 'daosPool': %s", o.daosPool); if (rc != 0) printf("Killing tgt rank: %d (total of %d of %d already disabled)\n", - rank, info.pi_ndisabled, info.pi_ntargets); + d_rank, info.pi_ndisabled, info.pi_ntargets); fflush(stdout); - rc = daos_mgmt_svc_rip(o.daosGroup, rank, true, NULL); + rc = daos_mgmt_svc_rip(o.daosGroup, d_rank, true, NULL); DCHECK(rc, "Error in killing server\n"); targets.rl_nr = 1; - targets.rl_ranks = &rank; + targets.rl_ranks = &d_rank; svc.rl_ranks = svc_ranks; ParseService(param, sizeof(svc_ranks)/ sizeof(svc_ranks[0]), &svc); diff --git a/src/aiori-DFS.c b/src/aiori-DFS.c index 46edeb63..06d452bf 100755 --- a/src/aiori-DFS.c +++ b/src/aiori-DFS.c @@ -39,8 +39,8 @@ #include #include "ior.h" -#include "aiori.h" #include "iordef.h" +#include "aiori.h" #include "utilities.h" dfs_t *dfs; @@ -491,7 +491,7 @@ static char* DFS_GetVersion() * Use DFS stat() to return aggregate file size. */ static IOR_offset_t -DFS_GetFileSize(IOR_param_t * test, MPI_Comm testComm, char *testFileName) +DFS_GetFileSize(IOR_param_t * test, MPI_Comm comm, char *testFileName) { dfs_obj_t *obj; daos_size_t fsize, tmpMin, tmpMax, tmpSum; @@ -511,15 +511,15 @@ DFS_GetFileSize(IOR_param_t * test, MPI_Comm testComm, char *testFileName) if (test->filePerProc == TRUE) { MPI_CHECK(MPI_Allreduce(&fsize, &tmpSum, 1, - MPI_LONG_LONG_INT, MPI_SUM, testComm), + MPI_LONG_LONG_INT, MPI_SUM, comm), "cannot total data moved"); fsize = tmpSum; } else { MPI_CHECK(MPI_Allreduce(&fsize, &tmpMin, 1, - MPI_LONG_LONG_INT, MPI_MIN, testComm), + MPI_LONG_LONG_INT, MPI_MIN, comm), "cannot total data moved"); MPI_CHECK(MPI_Allreduce(&fsize, &tmpMax, 1, - MPI_LONG_LONG_INT, MPI_MAX, testComm), + MPI_LONG_LONG_INT, MPI_MAX, comm), "cannot total data moved"); if (tmpMin != tmpMax) { if (rank == 0) { From 1f217ae48c63db1cc03660425c2862a9e1a0b1ee Mon Sep 17 00:00:00 2001 From: Andreas Dilger Date: Fri, 14 Dec 2018 16:26:16 -0700 Subject: [PATCH 044/206] lustre: fix checks for lustre_user.h header Fix the configure check if --with-lustre is specified, but the linux/lustre/lustre_user.h header is not present. Only one of the headers needs to be included if both are found. In some cases, FASYNC is not defined, but forms part of the O_LOV_DELAY_CREATE value, add a #define in that case. Fixes #115. Fixes: cb88c4c19a831d94b864c49a162e2635730540e5 Signed-off-by: Andreas Dilger --- configure.ac | 6 ++++-- src/aiori-POSIX.c | 14 ++++++++++---- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/configure.ac b/configure.ac index 002b2dfd..80e8533f 100755 --- a/configure.ac +++ b/configure.ac @@ -34,7 +34,7 @@ esac # We can't do anything without a working MPI AX_PROG_CC_MPI(,,[ - AC_MSG_FAILURE([MPI compiler requested, but couldn't use MPI.]) + AC_MSG_FAILURE([MPI compiler requested, but could not use MPI.]) ]) AC_PROG_RANLIB @@ -85,7 +85,9 @@ AC_ARG_WITH([lustre], [], [with_lustre=check]) AS_IF([test "x$with_lustre" != xno], [ AC_CHECK_HEADERS([linux/lustre/lustre_user.h lustre/lustre_user.h], break, [ - if test "x$with_lustre" != xcheck; then + if test "x$with_lustre" != xcheck -a \ + "x$ac_cv_header_linux_lustre_lustre_user_h" = "xno" -a \ + "x$ac_cv_header_lustre_lustre_user_h" = "xno" ; then AC_MSG_FAILURE([--with-lustre was given, not found]) fi ]) diff --git a/src/aiori-POSIX.c b/src/aiori-POSIX.c index cb872971..94d0a7e4 100755 --- a/src/aiori-POSIX.c +++ b/src/aiori-POSIX.c @@ -32,11 +32,10 @@ #include -#ifdef HAVE_LUSTRE_LUSTRE_USER_H -# include -#endif #ifdef HAVE_LINUX_LUSTRE_LUSTRE_USER_H # include +#elif defined(HAVE_LUSTRE_LUSTRE_USER_H) +# include #endif #ifdef HAVE_GPFS_H # include @@ -282,6 +281,12 @@ void *POSIX_Create(char *testFileName, IOR_param_t * param) return 0; #ifdef HAVE_LUSTRE_LUSTRE_USER_H +/* Add a #define for FASYNC if not available, as it forms part of + * the Lustre O_LOV_DELAY_CREATE definition. */ +#ifndef FASYNC +#define FASYNC 00020000 /* fcntl, for BSD compatibility */ +#endif + if (param->lustre_set_striping) { /* In the single-shared-file case, task 0 has to creat the file with the Lustre striping options before any other processes @@ -302,7 +307,8 @@ void *POSIX_Create(char *testFileName, IOR_param_t * param) opts.lmm_stripe_count = param->lustre_stripe_count; /* File needs to be opened O_EXCL because we cannot set - Lustre striping information on a pre-existing file. */ + * Lustre striping information on a pre-existing file.*/ + fd_oflag |= O_CREAT | O_EXCL | O_RDWR | O_LOV_DELAY_CREATE; *fd = open64(testFileName, fd_oflag, 0664); From 4297ccd7f2130887417402e299acca7d2cb1ef9c Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Wed, 19 Dec 2018 20:29:47 +0000 Subject: [PATCH 045/206] Bugfix for stonewall computation. --- src/mdtest.c | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/src/mdtest.c b/src/mdtest.c index 8c2e1e98..3fb9f00f 100644 --- a/src/mdtest.c +++ b/src/mdtest.c @@ -324,7 +324,7 @@ static void create_file (const char *path, uint64_t itemNum) { //create files sprintf(curr_item, "%s/file.%s"LLU"", path, mk_name, itemNum); - if (rank == 0 && verbose >= 3) { + if ((rank == 0 && verbose >= 3) || verbose >= 5) { fprintf(out_logfile, "V-3: create_remove_items_helper (non-dirs create): curr_item is \"%s\"\n", curr_item); fflush(out_logfile); } @@ -405,6 +405,10 @@ void create_remove_items_helper(const int dirs, const int create, const char *pa create_remove_dirs (path, create, itemNum + i); } if(CHECK_STONE_WALL(progress)){ + if(progress->items_done != 0){ + printf("Error, this is an invalid configuration with stonewall!\n"); + exit(1); + } progress->items_done = i + 1; return; } @@ -1060,13 +1064,13 @@ int updateStoneWallIterations(int iteration, rank_progress_t * progress, double long long sum_accessed = 0; MPI_Reduce(& progress->items_done, & sum_accessed, 1, MPI_LONG_LONG_INT, MPI_SUM, 0, testComm); - if(items != (sum_accessed / size) && rank == 0){ + if(items != (sum_accessed / size)){ summary_table[iteration].stonewall_item_sum[MDTEST_FILE_CREATE_NUM] = sum_accessed; summary_table[iteration].stonewall_item_min[MDTEST_FILE_CREATE_NUM] = min_accessed * size; - fprintf( out_logfile, "Continue stonewall hit min: %lld max: %lld avg: %.1f \n", min_accessed, max_iter, ((double) sum_accessed) / size); - fflush( out_logfile ); - } - if( done != max_iter ){ + if (rank == 0){ + fprintf( out_logfile, "Continue stonewall hit min: %lld max: %lld avg: %.1f \n", min_accessed, max_iter, ((double) sum_accessed) / size); + fflush( out_logfile ); + } hit = 1; } progress->items_start = done; @@ -1123,10 +1127,13 @@ void file_test(const int iteration, const int ntasks, const char *path, rank_pro if (hit){ progress->stone_wall_timer_seconds = 0; - printf("stonewall rank %d: %lld of %lld \n", rank, (long long) progress->items_start, (long long) progress->items_per_dir); + if (verbose > 1){ + printf("stonewall rank %d: %lld of %lld \n", rank, (long long) progress->items_start, (long long) progress->items_per_dir); + } create_remove_items(0, 0, 1, 0, temp_path, 0, progress); // now reset the values progress->stone_wall_timer_seconds = stone_wall_timer_seconds; + items = progress->items_done; } if (stoneWallingStatusFile){ StoreStoneWallingIterations(stoneWallingStatusFile, progress->items_done); From 7b603bc5a99b1164457616fc1fc21947450658de Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Wed, 19 Dec 2018 21:37:37 +0000 Subject: [PATCH 046/206] Reset stonewall timer to make it work again with running all phases in one execution; i.e. not using multiple runs specifying: -C, -r --- src/mdtest.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/mdtest.c b/src/mdtest.c index 3fb9f00f..6cacb5de 100644 --- a/src/mdtest.c +++ b/src/mdtest.c @@ -304,7 +304,6 @@ static void remove_file (const char *path, uint64_t itemNum) { fprintf(out_logfile, "V-3: create_remove_items_helper (non-dirs remove): curr_item is \"%s\"\n", curr_item); fflush(out_logfile); } - if (!(shared_file && rank != 0)) { backend->delete (curr_item, ¶m); } @@ -405,11 +404,9 @@ void create_remove_items_helper(const int dirs, const int create, const char *pa create_remove_dirs (path, create, itemNum + i); } if(CHECK_STONE_WALL(progress)){ - if(progress->items_done != 0){ - printf("Error, this is an invalid configuration with stonewall!\n"); - exit(1); + if(progress->items_done == 0){ + progress->items_done = i + 1; } - progress->items_done = i + 1; return; } } @@ -1128,7 +1125,7 @@ void file_test(const int iteration, const int ntasks, const char *path, rank_pro if (hit){ progress->stone_wall_timer_seconds = 0; if (verbose > 1){ - printf("stonewall rank %d: %lld of %lld \n", rank, (long long) progress->items_start, (long long) progress->items_per_dir); + printf("stonewall rank %d: %lld of %lld \n", rank, (long long) progress->items_start, (long long) progress->items_per_dir); } create_remove_items(0, 0, 1, 0, temp_path, 0, progress); // now reset the values @@ -1138,6 +1135,8 @@ void file_test(const int iteration, const int ntasks, const char *path, rank_pro if (stoneWallingStatusFile){ StoreStoneWallingIterations(stoneWallingStatusFile, progress->items_done); } + // reset stone wall timer to allow proper cleanup + progress->stone_wall_timer_seconds = 0; } } }else{ @@ -1225,6 +1224,8 @@ void file_test(const int iteration, const int ntasks, const char *path, rank_pro t[3] = GetTimeStamp(); if (remove_only) { + progress->items_start = 0; + for (int dir_iter = 0; dir_iter < directory_loops; dir_iter ++){ prep_testdir(iteration, dir_iter); if (unique_dir_per_task) { @@ -1975,6 +1976,7 @@ static void mdtest_iteration(int i, int j, MPI_Group testgroup, mdtest_results_t MPI_Barrier(testComm); if (remove_only) { + progress->items_start = 0; startCreate = GetTimeStamp(); for (int dir_iter = 0; dir_iter < directory_loops; dir_iter ++){ prep_testdir(j, dir_iter); From dced60b060bb1f2f59e6a27e6018529c5fbc1fa6 Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Thu, 20 Dec 2018 20:40:53 +0000 Subject: [PATCH 047/206] Added test for bug #112. --- testing/basic-tests.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/testing/basic-tests.sh b/testing/basic-tests.sh index 2e546531..3e54a2ad 100755 --- a/testing/basic-tests.sh +++ b/testing/basic-tests.sh @@ -12,6 +12,8 @@ source $ROOT/test-lib.sh MDTEST 1 -a POSIX MDTEST 2 -a POSIX -W 2 +MDTEST 1 -C -T -F -I 1 -z 1 -b 1 -L -u + IOR 1 -a POSIX -w -z -F -Y -e -i1 -m -t 100k -b 1000k IOR 1 -a POSIX -w -z -F -k -e -i2 -m -t 100k -b 100k From ed421bcc7bbcc1de1b78d27b792d982c4cca35f7 Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Thu, 20 Dec 2018 21:08:55 +0000 Subject: [PATCH 048/206] Fixes that issue. Hopefully works for all combinations. --- src/mdtest.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mdtest.c b/src/mdtest.c index 6cacb5de..afc4fc67 100644 --- a/src/mdtest.c +++ b/src/mdtest.c @@ -549,7 +549,7 @@ void mdtest_stat(const int random, const int dirs, const long dir_iter, const ch uint64_t stop_items = items; - if( directory_loops != 1 ){ + if( directory_loops != 1 || leaf_only ){ stop_items = items_per_dir; } From 4d3d06da7a778d7eb738b486f9534076ea105ea0 Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Thu, 20 Dec 2018 21:09:34 +0000 Subject: [PATCH 049/206] New testing framework for mdtest. The framework compares the observed with the expected pattern for the runs. For a new test, the pattern is recorded. Note that the tests are enumerated. --- testing/basic-tests.sh | 5 +- testing/complex-tests.sh | 1 + testing/mdtest-patterns/advanced/3.txt | 93 ++++++++++++++++++++++++++ testing/mdtest-patterns/advanced/4.txt | 50 ++++++++++++++ testing/mdtest-patterns/advanced/5.txt | 77 +++++++++++++++++++++ testing/mdtest-patterns/basic/0.txt | 24 +++++++ testing/mdtest-patterns/basic/1.txt | 24 +++++++ testing/mdtest-patterns/basic/2.txt | 25 +++++++ testing/mdtest-patterns/basic/3.txt | 31 +++++++++ testing/test-lib.sh | 18 ++++- 10 files changed, 345 insertions(+), 3 deletions(-) create mode 100644 testing/mdtest-patterns/advanced/3.txt create mode 100644 testing/mdtest-patterns/advanced/4.txt create mode 100644 testing/mdtest-patterns/advanced/5.txt create mode 100644 testing/mdtest-patterns/basic/0.txt create mode 100644 testing/mdtest-patterns/basic/1.txt create mode 100644 testing/mdtest-patterns/basic/2.txt create mode 100644 testing/mdtest-patterns/basic/3.txt diff --git a/testing/basic-tests.sh b/testing/basic-tests.sh index 3e54a2ad..91dba4b1 100755 --- a/testing/basic-tests.sh +++ b/testing/basic-tests.sh @@ -7,13 +7,14 @@ # Example: export IOR_EXTRA="-v -v -v" ROOT="$(dirname ${BASH_SOURCE[0]})" +TYPE="basic" source $ROOT/test-lib.sh MDTEST 1 -a POSIX MDTEST 2 -a POSIX -W 2 -MDTEST 1 -C -T -F -I 1 -z 1 -b 1 -L -u - +MDTEST 1 -C -T -r -F -I 1 -z 1 -b 1 -L -u +MDTEST 1 -C -T -I 1 -z 1 -b 1 -u IOR 1 -a POSIX -w -z -F -Y -e -i1 -m -t 100k -b 1000k IOR 1 -a POSIX -w -z -F -k -e -i2 -m -t 100k -b 100k diff --git a/testing/complex-tests.sh b/testing/complex-tests.sh index 951dc32f..c314cf95 100755 --- a/testing/complex-tests.sh +++ b/testing/complex-tests.sh @@ -5,6 +5,7 @@ # Example: export IOR_EXTRA="-v -v -v" ROOT=${0%/*} +TYPE="advanced" source $ROOT/test-lib.sh diff --git a/testing/mdtest-patterns/advanced/3.txt b/testing/mdtest-patterns/advanced/3.txt new file mode 100644 index 00000000..a51000ac --- /dev/null +++ b/testing/mdtest-patterns/advanced/3.txt @@ -0,0 +1,93 @@ +V-3: main (before display_freespace): testdirpath is "/dev/shm/mdest" +V-3: testdirpath is "/dev/shm/mdest" +V-3: Before show_file_system_size, dirpath is "/dev/shm" +V-3: After show_file_system_size, dirpath is "/dev/shm" +V-3: main (after display_freespace): testdirpath is "/dev/shm/mdest" +V-3: main (create hierarchical directory loop-!unque_dir_per_task): Calling create_remove_directory_tree with "/dev/shm/mdest/#test-dir.0-0" +V-3: main: Using unique_mk_dir, "mdtest_tree.0" +V-3: main: Copied unique_mk_dir, "mdtest_tree.0", to topdir +V-3: directory_test: create path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" +V-3: create_remove_items (start): temp_path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" +V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.0" +V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.1" +V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.2" +V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.3" +V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.4" +V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.5" +V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.6" +V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.7" +V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.8" +V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.9" +V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.10" +V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.11" +V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.12" +V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.13" +V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.14" +V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.15" +V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.16" +V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.17" +V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.18" +V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.19" +V-3: file_test: create path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" +V-3: create_remove_items (start): temp_path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" +V-3: create_remove_items_helper (non-dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.0" +V-3: create_remove_items_helper (non-collective, shared): open... +V-3: create_remove_items_helper: close... +V-3: create_remove_items_helper (non-dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.1" +V-3: create_remove_items_helper (non-collective, shared): open... +V-3: create_remove_items_helper: close... +V-3: create_remove_items_helper (non-dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.2" +V-3: create_remove_items_helper (non-collective, shared): open... +V-3: create_remove_items_helper: close... +V-3: create_remove_items_helper (non-dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.3" +V-3: create_remove_items_helper (non-collective, shared): open... +V-3: create_remove_items_helper: close... +V-3: create_remove_items_helper (non-dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.4" +V-3: create_remove_items_helper (non-collective, shared): open... +V-3: create_remove_items_helper: close... +V-3: create_remove_items_helper (non-dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.5" +V-3: create_remove_items_helper (non-collective, shared): open... +V-3: create_remove_items_helper: close... +V-3: create_remove_items_helper (non-dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.6" +V-3: create_remove_items_helper (non-collective, shared): open... +V-3: create_remove_items_helper: close... +V-3: create_remove_items_helper (non-dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.7" +V-3: create_remove_items_helper (non-collective, shared): open... +V-3: create_remove_items_helper: close... +V-3: create_remove_items_helper (non-dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.8" +V-3: create_remove_items_helper (non-collective, shared): open... +V-3: create_remove_items_helper: close... +V-3: create_remove_items_helper (non-dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.9" +V-3: create_remove_items_helper (non-collective, shared): open... +V-3: create_remove_items_helper: close... +V-3: create_remove_items_helper (non-dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.10" +V-3: create_remove_items_helper (non-collective, shared): open... +V-3: create_remove_items_helper: close... +V-3: create_remove_items_helper (non-dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.11" +V-3: create_remove_items_helper (non-collective, shared): open... +V-3: create_remove_items_helper: close... +V-3: create_remove_items_helper (non-dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.12" +V-3: create_remove_items_helper (non-collective, shared): open... +V-3: create_remove_items_helper: close... +V-3: create_remove_items_helper (non-dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.13" +V-3: create_remove_items_helper (non-collective, shared): open... +V-3: create_remove_items_helper: close... +V-3: create_remove_items_helper (non-dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.14" +V-3: create_remove_items_helper (non-collective, shared): open... +V-3: create_remove_items_helper: close... +V-3: create_remove_items_helper (non-dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.15" +V-3: create_remove_items_helper (non-collective, shared): open... +V-3: create_remove_items_helper: close... +V-3: create_remove_items_helper (non-dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.16" +V-3: create_remove_items_helper (non-collective, shared): open... +V-3: create_remove_items_helper: close... +V-3: create_remove_items_helper (non-dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.17" +V-3: create_remove_items_helper (non-collective, shared): open... +V-3: create_remove_items_helper: close... +V-3: create_remove_items_helper (non-dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.18" +V-3: create_remove_items_helper (non-collective, shared): open... +V-3: create_remove_items_helper: close... +V-3: create_remove_items_helper (non-dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.19" +V-3: create_remove_items_helper (non-collective, shared): open... +V-3: create_remove_items_helper: close... +V-3: main: Using testdir, "/dev/shm/mdest/#test-dir.0-0" diff --git a/testing/mdtest-patterns/advanced/4.txt b/testing/mdtest-patterns/advanced/4.txt new file mode 100644 index 00000000..8df166a4 --- /dev/null +++ b/testing/mdtest-patterns/advanced/4.txt @@ -0,0 +1,50 @@ +V-3: main (before display_freespace): testdirpath is "/dev/shm/mdest" +V-3: testdirpath is "/dev/shm/mdest" +V-3: Before show_file_system_size, dirpath is "/dev/shm" +V-3: After show_file_system_size, dirpath is "/dev/shm" +V-3: main (after display_freespace): testdirpath is "/dev/shm/mdest" +V-3: main: Using unique_mk_dir, "mdtest_tree.0" +V-3: main: Copied unique_mk_dir, "mdtest_tree.0", to topdir +V-3: directory_test: stat path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" +V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.0 +V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.1 +V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.2 +V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.3 +V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.4 +V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.5 +V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.6 +V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.7 +V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.8 +V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.9 +V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.10 +V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.11 +V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.12 +V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.13 +V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.14 +V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.15 +V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.16 +V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.17 +V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.18 +V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.19 +V-3: file_test: stat path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" +V-3: mdtest_stat file: /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.0 +V-3: mdtest_stat file: /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.1 +V-3: mdtest_stat file: /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.2 +V-3: mdtest_stat file: /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.3 +V-3: mdtest_stat file: /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.4 +V-3: mdtest_stat file: /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.5 +V-3: mdtest_stat file: /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.6 +V-3: mdtest_stat file: /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.7 +V-3: mdtest_stat file: /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.8 +V-3: mdtest_stat file: /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.9 +V-3: mdtest_stat file: /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.10 +V-3: mdtest_stat file: /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.11 +V-3: mdtest_stat file: /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.12 +V-3: mdtest_stat file: /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.13 +V-3: mdtest_stat file: /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.14 +V-3: mdtest_stat file: /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.15 +V-3: mdtest_stat file: /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.16 +V-3: mdtest_stat file: /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.17 +V-3: mdtest_stat file: /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.18 +V-3: mdtest_stat file: /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.19 +V-3: main: Using testdir, "/dev/shm/mdest/#test-dir.0-0" diff --git a/testing/mdtest-patterns/advanced/5.txt b/testing/mdtest-patterns/advanced/5.txt new file mode 100644 index 00000000..300bce63 --- /dev/null +++ b/testing/mdtest-patterns/advanced/5.txt @@ -0,0 +1,77 @@ +V-3: main (before display_freespace): testdirpath is "/dev/shm/mdest" +V-3: testdirpath is "/dev/shm/mdest" +V-3: Before show_file_system_size, dirpath is "/dev/shm" +V-3: After show_file_system_size, dirpath is "/dev/shm" +V-3: main (after display_freespace): testdirpath is "/dev/shm/mdest" +V-3: main (create hierarchical directory loop-!unque_dir_per_task): Calling create_remove_directory_tree with "/dev/shm/mdest/#test-dir.0-0" +V-3: main: Using unique_mk_dir, "mdtest_tree.0" +V-3: main: Copied unique_mk_dir, "mdtest_tree.0", to topdir +V-3: directory_test: create path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" +V-3: create_remove_items (start): temp_path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" +V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.0" +V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.1" +V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.2" +V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.3" +V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.4" +V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.5" +V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.6" +V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.7" +V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.8" +V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.9" +V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.10" +V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.11" +V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.12" +V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.13" +V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.14" +V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.15" +V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.16" +V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.17" +V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.18" +V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.19" +V-3: directory_test: stat path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" +V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.0 +V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.1 +V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.2 +V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.3 +V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.4 +V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.5 +V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.6 +V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.7 +V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.8 +V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.9 +V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.10 +V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.11 +V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.12 +V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.13 +V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.14 +V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.15 +V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.16 +V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.17 +V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.18 +V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.19 +V-3: directory_test: read path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" +V-3: directory_test: remove directories path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" +V-3: create_remove_items (start): temp_path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" +V-3: create_remove_items_helper (dirs remove): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.0" +V-3: create_remove_items_helper (dirs remove): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.1" +V-3: create_remove_items_helper (dirs remove): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.2" +V-3: create_remove_items_helper (dirs remove): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.3" +V-3: create_remove_items_helper (dirs remove): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.4" +V-3: create_remove_items_helper (dirs remove): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.5" +V-3: create_remove_items_helper (dirs remove): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.6" +V-3: create_remove_items_helper (dirs remove): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.7" +V-3: create_remove_items_helper (dirs remove): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.8" +V-3: create_remove_items_helper (dirs remove): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.9" +V-3: create_remove_items_helper (dirs remove): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.10" +V-3: create_remove_items_helper (dirs remove): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.11" +V-3: create_remove_items_helper (dirs remove): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.12" +V-3: create_remove_items_helper (dirs remove): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.13" +V-3: create_remove_items_helper (dirs remove): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.14" +V-3: create_remove_items_helper (dirs remove): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.15" +V-3: create_remove_items_helper (dirs remove): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.16" +V-3: create_remove_items_helper (dirs remove): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.17" +V-3: create_remove_items_helper (dirs remove): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.18" +V-3: create_remove_items_helper (dirs remove): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.19" +V-3: directory_test: remove unique directories path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" +V-3: main: Using testdir, "/dev/shm/mdest/#test-dir.0-0" +V-3: main (remove hierarchical directory loop-!unique_dir_per_task): Calling create_remove_directory_tree with "/dev/shm/mdest/#test-dir.0-0" diff --git a/testing/mdtest-patterns/basic/0.txt b/testing/mdtest-patterns/basic/0.txt new file mode 100644 index 00000000..7ff10fc8 --- /dev/null +++ b/testing/mdtest-patterns/basic/0.txt @@ -0,0 +1,24 @@ +V-3: main (before display_freespace): testdirpath is "/dev/shm/mdest" +V-3: testdirpath is "/dev/shm/mdest" +V-3: Before show_file_system_size, dirpath is "/dev/shm" +V-3: After show_file_system_size, dirpath is "/dev/shm" +V-3: main (after display_freespace): testdirpath is "/dev/shm/mdest" +V-3: main (create hierarchical directory loop-!unque_dir_per_task): Calling create_remove_directory_tree with "/dev/shm/mdest/#test-dir.0-0" +V-3: main: Using unique_mk_dir, "mdtest_tree.0" +V-3: main: Copied unique_mk_dir, "mdtest_tree.0", to topdir +V-3: directory_test: create path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" +V-3: create_remove_items (start): temp_path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" +V-3: directory_test: stat path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" +V-3: directory_test: read path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" +V-3: directory_test: remove directories path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" +V-3: create_remove_items (start): temp_path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" +V-3: directory_test: remove unique directories path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" +V-3: file_test: create path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" +V-3: create_remove_items (start): temp_path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" +V-3: file_test: stat path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" +V-3: file_test: read path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" +V-3: file_test: rm directories path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" +V-3: create_remove_items (start): temp_path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" +V-3: file_test: rm unique directories path is "mdtest_tree.0" +V-3: main: Using testdir, "/dev/shm/mdest/#test-dir.0-0" +V-3: main (remove hierarchical directory loop-!unique_dir_per_task): Calling create_remove_directory_tree with "/dev/shm/mdest/#test-dir.0-0" diff --git a/testing/mdtest-patterns/basic/1.txt b/testing/mdtest-patterns/basic/1.txt new file mode 100644 index 00000000..7ff10fc8 --- /dev/null +++ b/testing/mdtest-patterns/basic/1.txt @@ -0,0 +1,24 @@ +V-3: main (before display_freespace): testdirpath is "/dev/shm/mdest" +V-3: testdirpath is "/dev/shm/mdest" +V-3: Before show_file_system_size, dirpath is "/dev/shm" +V-3: After show_file_system_size, dirpath is "/dev/shm" +V-3: main (after display_freespace): testdirpath is "/dev/shm/mdest" +V-3: main (create hierarchical directory loop-!unque_dir_per_task): Calling create_remove_directory_tree with "/dev/shm/mdest/#test-dir.0-0" +V-3: main: Using unique_mk_dir, "mdtest_tree.0" +V-3: main: Copied unique_mk_dir, "mdtest_tree.0", to topdir +V-3: directory_test: create path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" +V-3: create_remove_items (start): temp_path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" +V-3: directory_test: stat path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" +V-3: directory_test: read path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" +V-3: directory_test: remove directories path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" +V-3: create_remove_items (start): temp_path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" +V-3: directory_test: remove unique directories path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" +V-3: file_test: create path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" +V-3: create_remove_items (start): temp_path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" +V-3: file_test: stat path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" +V-3: file_test: read path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" +V-3: file_test: rm directories path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" +V-3: create_remove_items (start): temp_path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" +V-3: file_test: rm unique directories path is "mdtest_tree.0" +V-3: main: Using testdir, "/dev/shm/mdest/#test-dir.0-0" +V-3: main (remove hierarchical directory loop-!unique_dir_per_task): Calling create_remove_directory_tree with "/dev/shm/mdest/#test-dir.0-0" diff --git a/testing/mdtest-patterns/basic/2.txt b/testing/mdtest-patterns/basic/2.txt new file mode 100644 index 00000000..a21f5a34 --- /dev/null +++ b/testing/mdtest-patterns/basic/2.txt @@ -0,0 +1,25 @@ +V-3: main (before display_freespace): testdirpath is "/dev/shm/mdest" +V-3: testdirpath is "/dev/shm/mdest" +V-3: Before show_file_system_size, dirpath is "/dev/shm" +V-3: After show_file_system_size, dirpath is "/dev/shm" +V-3: main (after display_freespace): testdirpath is "/dev/shm/mdest" +V-3: main (create hierarchical directory loop-!collective_creates): Calling create_remove_directory_tree with "/dev/shm/mdest/#test-dir.0-0" +V-3: main: Copied unique_mk_dir, "mdtest_tree.0.0", to topdir +V-3: file_test: create path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0" +V-3: create_remove_items (start): temp_path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0" +V-3: create_remove_items (start): temp_path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0" +V-3: create_remove_items (for loop): temp_path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0/mdtest_tree.0.1/" +V-3: create_remove_items_helper (non-dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0/mdtest_tree.0.1//file.mdtest.0.1" +V-3: create_remove_items_helper (non-collective, shared): open... +V-3: create_remove_items_helper: close... +V-3: create_remove_items (start): temp_path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0/mdtest_tree.0.1/" +V-3: file_test: stat path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0" +V-3: mdtest_stat file: /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0/mdtest_tree.0.1/file.mdtest.0.1 +V-3: file_test: rm directories path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0" +V-3: create_remove_items (start): temp_path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0" +V-3: create_remove_items (start): temp_path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0" +V-3: create_remove_items (for loop): temp_path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0/mdtest_tree.0.1/" +V-3: create_remove_items_helper (non-dirs remove): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0/mdtest_tree.0.1//file.mdtest.0.1" +V-3: create_remove_items (start): temp_path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0/mdtest_tree.0.1/" +V-3: file_test: rm unique directories path is "/dev/shm/mdest/#test-dir.0-0/" +V-3: main (remove hierarchical directory loop-!collective): Calling create_remove_directory_tree with "/dev/shm/mdest/#test-dir.0-0" diff --git a/testing/mdtest-patterns/basic/3.txt b/testing/mdtest-patterns/basic/3.txt new file mode 100644 index 00000000..3e4aa787 --- /dev/null +++ b/testing/mdtest-patterns/basic/3.txt @@ -0,0 +1,31 @@ +V-3: main (before display_freespace): testdirpath is "/dev/shm/mdest" +V-3: testdirpath is "/dev/shm/mdest" +V-3: Before show_file_system_size, dirpath is "/dev/shm" +V-3: After show_file_system_size, dirpath is "/dev/shm" +V-3: main (after display_freespace): testdirpath is "/dev/shm/mdest" +V-3: main (create hierarchical directory loop-!collective_creates): Calling create_remove_directory_tree with "/dev/shm/mdest/#test-dir.0-0" +V-3: main: Copied unique_mk_dir, "mdtest_tree.0.0", to topdir +V-3: directory_test: create path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0" +V-3: create_remove_items (start): temp_path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0" +V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0/dir.mdtest.0.0" +V-3: create_remove_items (start): temp_path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0" +V-3: create_remove_items (for loop): temp_path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0/mdtest_tree.0.1/" +V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0/mdtest_tree.0.1//dir.mdtest.0.1" +V-3: create_remove_items (start): temp_path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0/mdtest_tree.0.1/" +V-3: directory_test: stat path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0" +V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0/dir.mdtest.0.0 +V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0/mdtest_tree.0.1/dir.mdtest.0.1 +V-3: file_test: create path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0" +V-3: create_remove_items (start): temp_path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0" +V-3: create_remove_items_helper (non-dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0/file.mdtest.0.0" +V-3: create_remove_items_helper (non-collective, shared): open... +V-3: create_remove_items_helper: close... +V-3: create_remove_items (start): temp_path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0" +V-3: create_remove_items (for loop): temp_path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0/mdtest_tree.0.1/" +V-3: create_remove_items_helper (non-dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0/mdtest_tree.0.1//file.mdtest.0.1" +V-3: create_remove_items_helper (non-collective, shared): open... +V-3: create_remove_items_helper: close... +V-3: create_remove_items (start): temp_path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0/mdtest_tree.0.1/" +V-3: file_test: stat path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0" +V-3: mdtest_stat file: /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0/file.mdtest.0.0 +V-3: mdtest_stat file: /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0/mdtest_tree.0.1/file.mdtest.0.1 diff --git a/testing/test-lib.sh b/testing/test-lib.sh index 965c92a5..444873d2 100644 --- a/testing/test-lib.sh +++ b/testing/test-lib.sh @@ -12,6 +12,7 @@ IOR_OUT=${IOR_OUT:-./test_logs} IOR_TMP=${IOR_TMP:-/dev/shm} IOR_EXTRA=${IOR_EXTRA:-} # Add global options like verbosity MDTEST_EXTRA=${MDTEST_EXTRA:-} +MDTEST_TEST_PATTERNS=${MDTEST_TEST_PATTERNS:-../testing/mdtest-patterns/$TYPE} ################################################################################ mkdir -p ${IOR_OUT} @@ -54,12 +55,27 @@ function IOR(){ function MDTEST(){ RANKS=$1 shift - WHAT="${IOR_MPIRUN} $RANKS ${IOR_BIN_DIR}/mdtest ${@} ${MDTEST_EXTRA} -d ${IOR_TMP}/mdest" + rm -rf ${IOR_TMP}/mdest + WHAT="${IOR_MPIRUN} $RANKS ${IOR_BIN_DIR}/mdtest ${@} ${MDTEST_EXTRA} -d ${IOR_TMP}/mdest -V=4" $WHAT 1>"${IOR_OUT}/test_out.$I" 2>&1 if [[ $? != 0 ]]; then echo -n "ERR" ERRORS=$(($ERRORS + 1)) else + # compare basic pattern + if [[ -r ${MDTEST_TEST_PATTERNS}/$I.txt ]] ; then + grep "V-3" "${IOR_OUT}/test_out.$I" > "${IOR_OUT}/tmp" + cmp -s "${IOR_OUT}/tmp" ${MDTEST_TEST_PATTERNS}/$I.txt + if [[ $? != 0 ]]; then + mv "${IOR_OUT}/tmp" ${IOR_OUT}/tmp.$I + echo -n "Pattern differs! check: diff -u ${MDTEST_TEST_PATTERNS}/$I.txt ${IOR_OUT}/tmp.$I " + fi + else + if [[ ! -e ${MDTEST_TEST_PATTERNS} ]] ; then + mkdir -p ${MDTEST_TEST_PATTERNS} + fi + grep "V-3" "${IOR_OUT}/test_out.$I" > ${MDTEST_TEST_PATTERNS}/$I.txt + fi echo -n "OK " fi echo " $WHAT" From 1dbca5c293f95074f9887ddb2043fa984670fb4d Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Sun, 23 Dec 2018 13:06:03 +0000 Subject: [PATCH 050/206] Bugfix for HDF5 plugin; readded collective_md. --- src/ior.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/ior.h b/src/ior.h index 015d1111..ad543ffe 100755 --- a/src/ior.h +++ b/src/ior.h @@ -179,6 +179,8 @@ typedef struct char* URI; /* "path" to target object */ size_t part_number; /* multi-part upload increment (PER-RANK!) */ char* UploadId; /* key for multi-part-uploads */ + int collective_md; /* use collective metatata optimization */ + /* RADOS variables */ rados_t rados_cluster; /* RADOS cluster handle */ From d2a0023765bcffa2b3b2afe605f436dc5ce7f937 Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Mon, 7 Jan 2019 18:28:59 +0000 Subject: [PATCH 051/206] remove epoch usage and rebase with latest DAOS API that switches to transactions. Signed-off-by: Mohamad Chaarawi --- src/aiori-DAOS.c | 150 +++++------------------------------------------ src/aiori-DFS.c | 2 +- 2 files changed, 17 insertions(+), 135 deletions(-) diff --git a/src/aiori-DAOS.c b/src/aiori-DAOS.c index e498b8f6..6846f1c9 100644 --- a/src/aiori-DAOS.c +++ b/src/aiori-DAOS.c @@ -46,8 +46,6 @@ struct daos_options{ uint64_t daosStripeMax; /* max length of a stripe */ int daosAios; /* max number of concurrent async I/Os */ int daosWriteOnly; /* write only, no flush and commit */ - uint64_t daosEpoch; /* epoch to access */ - uint64_t daosWait; /* epoch to wait for before reading */ int daosKill; /* kill a target while running IOR */ char *daosObjectClass; /* object class */ }; @@ -62,8 +60,6 @@ static struct daos_options o = { .daosStripeMax = 0, .daosAios = 1, .daosWriteOnly = 0, - .daosEpoch = 0, - .daosWait = 0, .daosKill = 0, .daosObjectClass = NULL, }; @@ -78,8 +74,6 @@ static option_help options [] = { {'m', "daosStripeMax", "Max Stripe",OPTION_OPTIONAL_ARGUMENT, 'u', &o.daosStripeMax}, {'a', "daosAios", "Concurrent Async IOs",OPTION_OPTIONAL_ARGUMENT, 'd', &o.daosAios}, {'w', "daosWriteOnly", "Write Only, no commit",OPTION_FLAG, 'd', &o.daosWriteOnly}, - {'e', "daosEpoch", "Epoch Number to Access",OPTION_OPTIONAL_ARGUMENT, 'u', &o.daosEpoch}, - {'t', "daosWait", "Epoch to wait for before read",OPTION_OPTIONAL_ARGUMENT, 'u', &o.daosWait}, {'k', "daosKill", "Kill target while running",OPTION_FLAG, 'd', &o.daosKill}, {'o', "daosObjectClass", "object class", OPTION_OPTIONAL_ARGUMENT, 's', &o.daosObjectClass}, LAST_OPTION @@ -126,7 +120,6 @@ struct fileDescriptor { daos_handle_t container; daos_cont_info_t containerInfo; daos_handle_t object; - daos_epoch_t epoch; }; struct aio { @@ -136,7 +129,6 @@ struct aio { daos_recx_t a_recx; unsigned char a_csumBuf[32]; daos_csum_buf_t a_csum; - daos_epoch_range_t a_epochRange; daos_iod_t a_iod; daos_iov_t a_iov; daos_sg_list_t a_sgl; @@ -270,56 +262,6 @@ static void ContainerOpen(char *testFileName, IOR_param_t *param, rc = daos_cont_open(pool, uuid, dFlags, container, info, NULL /* ev */); DCHECK(rc, "Failed to open container %s", testFileName); - - INFO(VERBOSE_2, param, "Container epoch state:"); - INFO(VERBOSE_2, param, " HCE: %lu", - info->ci_epoch_state.es_hce); - INFO(VERBOSE_2, param, " LRE: %lu", - info->ci_epoch_state.es_lre); - INFO(VERBOSE_2, param, " LHE: %lu (%lx)", - info->ci_epoch_state.es_lhe, info->ci_epoch_state.es_lhe); - INFO(VERBOSE_2, param, " GHCE: %lu", - info->ci_epoch_state.es_ghce); - INFO(VERBOSE_2, param, " GLRE: %lu", - info->ci_epoch_state.es_glre); - INFO(VERBOSE_2, param, " GHPCE: %lu", - info->ci_epoch_state.es_ghpce); - -#if 0 - if (param->open != WRITE && o.daosWait != 0) { - daos_epoch_t e; - - e = o.daosWait; - - INFO(VERBOSE_2, param, "Waiting for epoch %lu", e); - - rc = daos_epoch_wait(*container, &e, - NULL /* ignore HLE */, - NULL /* synchronous */); - DCHECK(rc, "Failed to wait for epoch %lu", - o.daosWait); - } - - if (param->open == WRITE && - param->useExistingTestFile == FALSE) { - daos_oclass_attr_t attr = { - .ca_schema = DAOS_OS_STRIPED, - .ca_resil_degree = 0, - .ca_resil = DAOS_RES_REPL, - .ca_grp_nr = 4, - .u.repl = { - .r_method = 0, - .r_num = 2 - } - }; - - INFO(VERBOSE_2, param, "Registering object class"); - - rc = daos_oclass_register(container, objectClass, &attr, - NULL /* ev */); - DCHECK(rc, "Failed to register object class"); - } -#endif } HandleDistribute(container, CONTAINER_HANDLE, param); @@ -348,7 +290,7 @@ static void ContainerClose(daos_handle_t container, IOR_param_t *param) } static void ObjectOpen(daos_handle_t container, daos_handle_t *object, - daos_epoch_t epoch, IOR_param_t *param) + IOR_param_t *param) { daos_obj_id_t oid; unsigned int flags; @@ -356,29 +298,14 @@ static void ObjectOpen(daos_handle_t container, daos_handle_t *object, oid.hi = 0; oid.lo = 1; - daos_obj_id_generate(&oid, 0, objectClass); - -#if 0 - /** declaring object not implemented commenting it */ - if (rank == 0 && param->open == WRITE && - param->useExistingTestFile == FALSE) { - INFO(VERBOSE_2, param, "Declaring object"); - - rc = daos_obj_declare(container, oid, epoch, NULL /* oa */, - NULL /* ev */); - DCHECK(rc, "Failed to declare object"); - } -#endif - /* An MPI_Bcast() call would probably be more efficient. */ - MPI_CHECK(MPI_Barrier(param->testComm), - "Failed to synchronize processes"); + daos_obj_generate_id(&oid, 0, objectClass); if (param->open == WRITE) flags = DAOS_OO_RW; else flags = DAOS_OO_RO; - rc = daos_obj_open(container, oid, epoch, flags, object, NULL /* ev */); + rc = daos_obj_open(container, oid, flags, object, NULL /* ev */); DCHECK(rc, "Failed to open object"); } @@ -416,8 +343,6 @@ static void AIOInit(IOR_param_t *param) aio->a_csum.cs_buf_len = sizeof aio->a_csumBuf; aio->a_csum.cs_len = aio->a_csum.cs_buf_len; - aio->a_epochRange.epr_hi = DAOS_EPOCH_MAX; - aio->a_iod.iod_name.iov_buf = "data"; aio->a_iod.iod_name.iov_buf_len = strlen(aio->a_iod.iod_name.iov_buf) + 1; @@ -426,7 +351,7 @@ static void AIOInit(IOR_param_t *param) aio->a_iod.iod_type = DAOS_IOD_ARRAY; aio->a_iod.iod_recxs = &aio->a_recx; aio->a_iod.iod_csums = &aio->a_csum; - aio->a_iod.iod_eprs = &aio->a_epochRange; + aio->a_iod.iod_eprs = NULL; aio->a_iod.iod_size = param->transferSize; aio->a_iov.iov_buf = buffers + param->transferSize * i; @@ -657,52 +582,13 @@ static void *DAOS_Create(char *testFileName, IOR_param_t *param) static void *DAOS_Open(char *testFileName, IOR_param_t *param) { struct fileDescriptor *fd; - daos_epoch_t ghce; fd = malloc(sizeof *fd); if (fd == NULL) ERR("Failed to allocate fd"); ContainerOpen(testFileName, param, &fd->container, &fd->containerInfo); - - ghce = fd->containerInfo.ci_epoch_state.es_ghce; - if (param->open == WRITE) { - if (o.daosEpoch == 0) - fd->epoch = ghce + 1; - else if (o.daosEpoch <= ghce) - GERR("Can't modify committed epoch\n"); - else - fd->epoch = o.daosEpoch; - } else { - if (o.daosEpoch == 0) { - if (o.daosWait == 0) - fd->epoch = ghce; - else - fd->epoch = o.daosWait; - } else if (o.daosEpoch > ghce) { - GERR("Can't read uncommitted epoch\n"); - } else { - fd->epoch = o.daosEpoch; - } - } - - if (rank == 0) - INFO(VERBOSE_2, param, "Accessing epoch %lu", fd->epoch); - - if (rank == 0 && param->open == WRITE) { - daos_epoch_t e = fd->epoch; - int rc; - - INFO(VERBOSE_2, param, "Holding epoch %lu", fd->epoch); - - rc = daos_epoch_hold(fd->container, &fd->epoch, - NULL /* state */, NULL /* ev */); - DCHECK(rc, "Failed to hold epoch"); - assert(fd->epoch == e); - } - - ObjectOpen(fd->container, &fd->object, fd->epoch, param); - + ObjectOpen(fd->container, &fd->object, param); AIOInit(param); return fd; @@ -822,7 +708,6 @@ static IOR_offset_t DAOS_Xfer(int access, void *file, IOR_size_t *buffer, if (o.daosStripeMax != 0) stripeOffset %= o.daosStripeMax; aio->a_recx.rx_idx = stripeOffset / o.daosRecordSize; - aio->a_epochRange.epr_lo = fd->epoch; /* * If the data written will be checked later, we have to copy in valid @@ -843,12 +728,12 @@ static IOR_offset_t DAOS_Xfer(int access, void *file, IOR_size_t *buffer, (unsigned long long) aio->a_sgl.sg_iovs->iov_buf_len); if (access == WRITE) { - rc = daos_obj_update(fd->object, fd->epoch, &aio->a_dkey, + rc = daos_obj_update(fd->object, DAOS_TX_NONE, &aio->a_dkey, 1 /* nr */, &aio->a_iod, &aio->a_sgl, &aio->a_event); DCHECK(rc, "Failed to start update operation"); } else { - rc = daos_obj_fetch(fd->object, fd->epoch, &aio->a_dkey, + rc = daos_obj_fetch(fd->object, DAOS_TX_NONE, &aio->a_dkey, 1 /* nr */, &aio->a_iod, &aio->a_sgl, NULL /* maps */, &aio->a_event); DCHECK(rc, "Failed to start fetch operation"); @@ -884,19 +769,16 @@ static void DAOS_Close(void *file, IOR_param_t *param) MPI_CHECK(MPI_Barrier(param->testComm), "Failed to synchronize processes"); + /* MSC - temp hack to commit since close will rollback */ if (rank == 0) { - INFO(VERBOSE_2, param, "Flushing epoch %lu", fd->epoch); - - rc = daos_epoch_flush(fd->container, fd->epoch, - NULL /* state */, NULL /* ev */); - DCHECK(rc, "Failed to flush epoch"); - - INFO(VERBOSE_2, param, "Committing epoch %lu", - fd->epoch); - - rc = daos_epoch_commit(fd->container, fd->epoch, - NULL /* state */, NULL /* ev */); - DCHECK(rc, "Failed to commit object write"); + daos_handle_t th; + + rc = daos_tx_open(fd->container, &th, NULL); + DCHECK(rc, "Failed sync"); + rc = daos_tx_commit(th, NULL); + DCHECK(rc, "Failed sync"); + rc = daos_tx_close(th, NULL); + DCHECK(rc, "Failed sync"); } } diff --git a/src/aiori-DFS.c b/src/aiori-DFS.c index 06d452bf..ea24bc8a 100755 --- a/src/aiori-DFS.c +++ b/src/aiori-DFS.c @@ -279,7 +279,7 @@ DFS_Finalize(IOR_param_t *param) { int rc; - rc = dfs_umount(dfs, true); + rc = dfs_umount(dfs); DCHECK(rc, "Failed to umount DFS namespace"); rc = daos_cont_close(coh, NULL); From 334bf737974e5dffcfd92d647f9e6967358a4449 Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Thu, 17 Jan 2019 20:58:58 +0000 Subject: [PATCH 052/206] remove transaction commit and options to do that. Signed-off-by: Mohamad Chaarawi --- src/aiori-DAOS.c | 24 ++---------------------- 1 file changed, 2 insertions(+), 22 deletions(-) diff --git a/src/aiori-DAOS.c b/src/aiori-DAOS.c index 6846f1c9..d790b9b0 100644 --- a/src/aiori-DAOS.c +++ b/src/aiori-DAOS.c @@ -45,7 +45,6 @@ struct daos_options{ uint64_t daosStripeCount; uint64_t daosStripeMax; /* max length of a stripe */ int daosAios; /* max number of concurrent async I/Os */ - int daosWriteOnly; /* write only, no flush and commit */ int daosKill; /* kill a target while running IOR */ char *daosObjectClass; /* object class */ }; @@ -59,7 +58,6 @@ static struct daos_options o = { .daosStripeCount = -1, .daosStripeMax = 0, .daosAios = 1, - .daosWriteOnly = 0, .daosKill = 0, .daosObjectClass = NULL, }; @@ -73,7 +71,6 @@ static option_help options [] = { {'c', "daosStripeCount", "Stripe Count", OPTION_OPTIONAL_ARGUMENT, 'u', &o.daosStripeCount}, {'m', "daosStripeMax", "Max Stripe",OPTION_OPTIONAL_ARGUMENT, 'u', &o.daosStripeMax}, {'a', "daosAios", "Concurrent Async IOs",OPTION_OPTIONAL_ARGUMENT, 'd', &o.daosAios}, - {'w', "daosWriteOnly", "Write Only, no commit",OPTION_FLAG, 'd', &o.daosWriteOnly}, {'k', "daosKill", "Kill target while running",OPTION_FLAG, 'd', &o.daosKill}, {'o', "daosObjectClass", "object class", OPTION_OPTIONAL_ARGUMENT, 's', &o.daosObjectClass}, LAST_OPTION @@ -521,7 +518,8 @@ static void DAOS_Init(IOR_param_t *param) INFO(VERBOSE_0, param, "WARNING: USING daosStripeMax CAUSES READS TO RETURN INVALID DATA"); rc = daos_init(); - DCHECK(rc, "Failed to initialize daos"); + if (rc != -DER_ALREADY) + DCHECK(rc, "Failed to initialize daos"); rc = daos_eq_create(&eventQueue); DCHECK(rc, "Failed to create event queue"); @@ -764,24 +762,6 @@ static void DAOS_Close(void *file, IOR_param_t *param) ObjectClose(fd->object); - if (param->open == WRITE && !o.daosWriteOnly) { - /* Wait for everybody for to complete the writes. */ - MPI_CHECK(MPI_Barrier(param->testComm), - "Failed to synchronize processes"); - - /* MSC - temp hack to commit since close will rollback */ - if (rank == 0) { - daos_handle_t th; - - rc = daos_tx_open(fd->container, &th, NULL); - DCHECK(rc, "Failed sync"); - rc = daos_tx_commit(th, NULL); - DCHECK(rc, "Failed sync"); - rc = daos_tx_close(th, NULL); - DCHECK(rc, "Failed sync"); - } - } - ContainerClose(fd->container, param); free(fd); From 20bb99e1b76aacb43ac4f59e255e7c9fc3291c9d Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Sat, 19 Jan 2019 17:00:49 +0000 Subject: [PATCH 053/206] Potential fix for #127 => Segmentation fault due to memory protection. --- src/mdtest.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/mdtest.c b/src/mdtest.c index afc4fc67..f1eb08d0 100644 --- a/src/mdtest.c +++ b/src/mdtest.c @@ -215,6 +215,8 @@ void parse_dirpath(char *dirpath_arg) { FAIL("out of memory"); } + // prevent changes to the original dirpath_arg + dirpath_arg = strdup(dirpath_arg); token = strtok(dirpath_arg, delimiter_string); while (token != NULL) { filenames[i] = token; From 4c1c73ecae16cb7ae03d3cbd0a3567502ad03b84 Mon Sep 17 00:00:00 2001 From: "Glenn K. Lockwood" Date: Wed, 23 Jan 2019 16:02:58 -0800 Subject: [PATCH 054/206] add malloc check to fix --- src/mdtest.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/mdtest.c b/src/mdtest.c index f1eb08d0..6ae6838c 100644 --- a/src/mdtest.c +++ b/src/mdtest.c @@ -210,13 +210,13 @@ void parse_dirpath(char *dirpath_arg) { } tmp++; } + // prevent changes to the original dirpath_arg + dirpath_arg = strdup(dirpath_arg); filenames = (char **)malloc(path_count * sizeof(char **)); - if (filenames == NULL) { + if (filenames == NULL || dirpath_arg == NULL) { FAIL("out of memory"); } - // prevent changes to the original dirpath_arg - dirpath_arg = strdup(dirpath_arg); token = strtok(dirpath_arg, delimiter_string); while (token != NULL) { filenames[i] = token; From 29090df2833f5653748b902a3dc4edeb916eda90 Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Fri, 8 Feb 2019 18:00:33 +0000 Subject: [PATCH 055/206] update DAOS and DFS drivers with new DAOS API changes. Signed-off-by: Mohamad Chaarawi --- src/aiori-DAOS.c | 18 ++++++++++-------- src/aiori-DFS.c | 2 +- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/aiori-DAOS.c b/src/aiori-DAOS.c index 4b358817..90356001 100644 --- a/src/aiori-DAOS.c +++ b/src/aiori-DAOS.c @@ -244,7 +244,7 @@ static void ContainerOpen(char *testFileName, IOR_param_t *param, param->useExistingTestFile == FALSE) { INFO(VERBOSE_2, "Creating container %s", testFileName); - rc = daos_cont_create(pool, uuid, NULL /* ev */); + rc = daos_cont_create(pool, uuid, NULL, NULL); DCHECK(rc, "Failed to create container %s", testFileName); } @@ -594,12 +594,14 @@ kill_daos_server(IOR_param_t *param) { daos_pool_info_t info; d_rank_t d_rank, svc_ranks[13]; - d_rank_list_t svc, targets; + d_rank_list_t svc; + struct d_tgt_list targets; + int tgt_idx = -1; uuid_t uuid; char *s; int rc; - rc = daos_pool_query(pool, NULL, &info, NULL); + rc = daos_pool_query(pool, NULL, &info, NULL, NULL); DCHECK(rc, "Error in querying pool\n"); if (info.pi_ntargets - info.pi_ndisabled <= 1) @@ -618,16 +620,16 @@ kill_daos_server(IOR_param_t *param) rc = daos_mgmt_svc_rip(o.daosGroup, d_rank, true, NULL); DCHECK(rc, "Error in killing server\n"); - targets.rl_nr = 1; - targets.rl_ranks = &d_rank; - + targets.tl_nr = 1; + targets.tl_ranks = &d_rank; + targets.tl_tgts = &tgt_idx; svc.rl_ranks = svc_ranks; ParseService(sizeof(svc_ranks)/ sizeof(svc_ranks[0]), &svc); - rc = daos_pool_exclude(uuid, NULL, &svc, &targets, NULL); + rc = daos_pool_tgt_exclude(uuid, NULL, &svc, &targets, NULL); DCHECK(rc, "Error in excluding pool from poolmap\n"); - rc = daos_pool_query(pool, NULL, &info, NULL); + rc = daos_pool_query(pool, NULL, &info, NULL, NULL); DCHECK(rc, "Error in querying pool\n"); printf("%d targets succesfully disabled\n", diff --git a/src/aiori-DFS.c b/src/aiori-DFS.c index e51d045f..9b8f60f9 100755 --- a/src/aiori-DFS.c +++ b/src/aiori-DFS.c @@ -261,7 +261,7 @@ DFS_Init() { if (rc == -DER_NONEXIST) { if (verbose >= VERBOSE_1) printf("Creating DFS Container ...\n"); - rc = daos_cont_create(poh, co_uuid, NULL); + rc = daos_cont_create(poh, co_uuid, NULL, NULL); if (rc == 0) { cont_created = true; rc = daos_cont_open(poh, co_uuid, DAOS_COO_RW, &coh, From c6abf85378ede3c3fd61363fc7166909b7d2251f Mon Sep 17 00:00:00 2001 From: Jean-Yves VET Date: Mon, 11 Feb 2019 14:42:30 +0100 Subject: [PATCH 056/206] aiori-IME: Add support of statfs in IME backend This patch adds the support of statfs in the IME backend. It uses ime_native_statvfs call to fill the ior_aiori_statfs_t structure. --- src/aiori-IME.c | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/src/aiori-IME.c b/src/aiori-IME.c index 107b3210..95c48f2c 100755 --- a/src/aiori-IME.c +++ b/src/aiori-IME.c @@ -269,20 +269,35 @@ static char *IME_GetVersion() return ver; } -/* - * XXX: statfs call is currently not exposed by IME native interface. - */ static int IME_StatFS(const char *path, ior_aiori_statfs_t *stat_buf, IOR_param_t *param) { + (void)param; + +#if (IME_NATIVE_API_VERSION >= 130) + struct statvfs statfs_buf; + + int ret = ime_native_statvfs(path, &statfs_buf); + if (ret) + return ret; + + stat_buf->f_bsize = statfs_buf.f_bsize; + stat_buf->f_blocks = statfs_buf.f_blocks; + stat_buf->f_bfree = statfs_buf.f_bfree; + stat_buf->f_files = statfs_buf.f_files; + stat_buf->f_ffree = statfs_buf.f_ffree; + + return 0; +#else (void)path; (void)stat_buf; - (void)param; WARN("statfs is currently not supported in IME backend!"); return -1; +#endif } + static int IME_MkDir(const char *path, mode_t mode, IOR_param_t *param) { (void)param; From 110153cf3ab88b034c8b1d7b05121d38b4616a0a Mon Sep 17 00:00:00 2001 From: Jean-Yves VET Date: Mon, 11 Feb 2019 14:49:14 +0100 Subject: [PATCH 057/206] mdtest: Enable IME native interface This patch makes mdtest allow IME native interface. It also defines a generic way to enable mdtest with each backend. --- src/aiori-DUMMY.c | 37 +++++++++++++++++++------------------ src/aiori-IME.c | 1 + src/aiori-POSIX.c | 1 + src/aiori.c | 37 ++++++++++++++++++++++++++----------- src/aiori.h | 9 ++++++++- src/mdtest.c | 10 +++------- src/parse_options.c | 2 +- 7 files changed, 59 insertions(+), 38 deletions(-) diff --git a/src/aiori-DUMMY.c b/src/aiori-DUMMY.c index a378c2c9..675a91be 100755 --- a/src/aiori-DUMMY.c +++ b/src/aiori-DUMMY.c @@ -138,22 +138,23 @@ static int DUMMY_stat (const char *path, struct stat *buf, IOR_param_t * param){ } ior_aiori_t dummy_aiori = { - "DUMMY", - NULL, - DUMMY_Create, - DUMMY_Open, - DUMMY_Xfer, - DUMMY_Close, - DUMMY_Delete, - DUMMY_getVersion, - DUMMY_Fsync, - DUMMY_GetFileSize, - DUMMY_statfs, - DUMMY_mkdir, - DUMMY_rmdir, - DUMMY_access, - DUMMY_stat, - NULL, - NULL, - DUMMY_options + .name = "DUMMY", + .name_legacy = NULL, + .create = DUMMY_Create, + .open = DUMMY_Open, + .xfer = DUMMY_Xfer, + .close = DUMMY_Close, + .delete = DUMMY_Delete, + .get_version = DUMMY_getVersion, + .fsync = DUMMY_Fsync, + .get_file_size = DUMMY_GetFileSize, + .statfs = DUMMY_statfs, + .mkdir = DUMMY_mkdir, + .rmdir = DUMMY_rmdir, + .access = DUMMY_access, + .stat = DUMMY_stat, + .initialize = NULL, + .finalize = NULL, + .get_options = DUMMY_options, + .enable_mdtest = true, }; diff --git a/src/aiori-IME.c b/src/aiori-IME.c index 95c48f2c..ef7b3cfb 100755 --- a/src/aiori-IME.c +++ b/src/aiori-IME.c @@ -79,6 +79,7 @@ ior_aiori_t ime_aiori = { .stat = IME_Stat, .initialize = IME_Initialize, .finalize = IME_Finalize, + .enable_mdtest = true, }; /***************************** F U N C T I O N S ******************************/ diff --git a/src/aiori-POSIX.c b/src/aiori-POSIX.c index 94d0a7e4..ca7c4a29 100755 --- a/src/aiori-POSIX.c +++ b/src/aiori-POSIX.c @@ -90,6 +90,7 @@ ior_aiori_t posix_aiori = { .rmdir = aiori_posix_rmdir, .access = aiori_posix_access, .stat = aiori_posix_stat, + .enable_mdtest = true, }; /***************************** F U N C T I O N S ******************************/ diff --git a/src/aiori.c b/src/aiori.c index f4834356..11f11b4a 100644 --- a/src/aiori.c +++ b/src/aiori.c @@ -92,17 +92,32 @@ void airoi_parse_options(int argc, char ** argv, option_help * global_options){ free(opt.modules); } -void aiori_supported_apis(char * APIs, char * APIs_legacy){ - ior_aiori_t **tmp = available_aiori; - if(*tmp != NULL){ - APIs += sprintf(APIs, "%s", (*tmp)->name); - tmp++; - for (; *tmp != NULL; ++tmp) { - APIs += sprintf(APIs, "|%s", (*tmp)->name); - if ((*tmp)->name_legacy != NULL) - APIs_legacy += sprintf(APIs_legacy, "|%s", (*tmp)->name_legacy); - } - } +void aiori_supported_apis(char * APIs, char * APIs_legacy, enum bench_type type) +{ + ior_aiori_t **tmp = available_aiori; + char delimiter = ' '; + + while (*tmp != NULL) + { + if ((type == MDTEST) && !(*tmp)->enable_mdtest) + { + tmp++; + continue; + } + + if (delimiter == ' ') + { + APIs += sprintf(APIs, "%s", (*tmp)->name); + delimiter = '|'; + } + else + APIs += sprintf(APIs, "%c%s", delimiter, (*tmp)->name); + + if ((*tmp)->name_legacy != NULL) + APIs_legacy += sprintf(APIs_legacy, "%c%s", + delimiter, (*tmp)->name_legacy); + tmp++; + } } /** diff --git a/src/aiori.h b/src/aiori.h index 7fb013da..727af58c 100755 --- a/src/aiori.h +++ b/src/aiori.h @@ -22,6 +22,7 @@ #endif /* not MPI_FILE_NULL */ #include +#include #include "ior.h" #include "iordef.h" /* IOR Definitions */ @@ -83,8 +84,14 @@ typedef struct ior_aiori { void (*initialize)(); /* called once per program before MPI is started */ void (*finalize)(); /* called once per program after MPI is shutdown */ option_help * (*get_options)(); + bool enable_mdtest; } ior_aiori_t; +enum bench_type { + IOR, + MDTEST +}; + extern ior_aiori_t dummy_aiori; extern ior_aiori_t hdf5_aiori; extern ior_aiori_t hdfs_aiori; @@ -102,7 +109,7 @@ void aiori_initialize(IOR_test_t * tests); void aiori_finalize(IOR_test_t * tests); const ior_aiori_t *aiori_select (const char *api); int aiori_count (void); -void aiori_supported_apis(char * APIs, char * APIs_legacy); +void aiori_supported_apis(char * APIs, char * APIs_legacy, enum bench_type type); void airoi_parse_options(int argc, char ** argv, option_help * global_options); const char *aiori_default (void); diff --git a/src/mdtest.c b/src/mdtest.c index 6ae6838c..8a819796 100644 --- a/src/mdtest.c +++ b/src/mdtest.c @@ -150,7 +150,7 @@ static mdtest_results_t * summary_table; static pid_t pid; static uid_t uid; -/* just use the POSIX backend for now */ +/* Use the POSIX backend by default */ static const char *backend_name = "POSIX"; static const ior_aiori_t *backend; @@ -1323,7 +1323,7 @@ void print_help (void) { char APIs[1024]; char APIs_legacy[1024]; - aiori_supported_apis(APIs, APIs_legacy); + aiori_supported_apis(APIs, APIs_legacy, MDTEST); char apiStr[1024]; sprintf(apiStr, "API for I/O [%s]", APIs); @@ -1573,10 +1573,6 @@ void valid_tests() { FAIL("-c not compatible with -B"); } - if (strcasecmp(backend_name, "POSIX") != 0 && strcasecmp(backend_name, "DUMMY") != 0) { - FAIL("-a only supported interface is POSIX (and DUMMY) right now!"); - } - /* check for shared file incompatibilities */ if (unique_dir_per_task && shared_file && rank == 0) { FAIL("-u not compatible with -S"); @@ -2131,7 +2127,7 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * int randomize = 0; char APIs[1024]; char APIs_legacy[1024]; - aiori_supported_apis(APIs, APIs_legacy); + aiori_supported_apis(APIs, APIs_legacy, MDTEST); char apiStr[1024]; sprintf(apiStr, "API for I/O [%s]", APIs); diff --git a/src/parse_options.c b/src/parse_options.c index b265953b..5a093b83 100755 --- a/src/parse_options.c +++ b/src/parse_options.c @@ -467,7 +467,7 @@ IOR_test_t *ParseCommandLine(int argc, char **argv) char APIs[1024]; char APIs_legacy[1024]; - aiori_supported_apis(APIs, APIs_legacy); + aiori_supported_apis(APIs, APIs_legacy, IOR); char apiStr[1024]; sprintf(apiStr, "API for I/O [%s]", APIs); From db19141c1998c49083bcf7dbdbe66a3c916a042f Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Fri, 1 Mar 2019 20:41:01 +0000 Subject: [PATCH 058/206] remove usage of GURT's D_GOTO from DFS plugin. Signed-off-by: Mohamad Chaarawi --- src/aiori-DFS.c | 40 +++++++++++++++++++++++++++------------- 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/src/aiori-DFS.c b/src/aiori-DFS.c index 9b8f60f9..dd9bc7f7 100755 --- a/src/aiori-DFS.c +++ b/src/aiori-DFS.c @@ -25,6 +25,7 @@ #endif #include +#include #include #include #include @@ -160,12 +161,16 @@ parse_filename(const char *path, char **_obj_name, char **_cont_name) } f1 = strdup(path); - if (f1 == NULL) - D_GOTO(out, rc = -ENOMEM); + if (f1 == NULL) { + rc = -ENOMEM; + goto out; + } f2 = strdup(path); - if (f2 == NULL) - D_GOTO(out, rc = -ENOMEM); + if (f2 == NULL) { + rc = -ENOMEM; + goto out; + } fname = basename(f1); cont_name = dirname(f2); @@ -173,18 +178,24 @@ parse_filename(const char *path, char **_obj_name, char **_cont_name) if (cont_name[0] == '.' || cont_name[0] != '/') { char cwd[1024]; - if (getcwd(cwd, 1024) == NULL) - D_GOTO(out, rc = -ENOMEM); + if (getcwd(cwd, 1024) == NULL) { + rc = -ENOMEM; + goto out; + } if (strcmp(cont_name, ".") == 0) { cont_name = strdup(cwd); - if (cont_name == NULL) - D_GOTO(out, rc = -ENOMEM); + if (cont_name == NULL) { + rc = -ENOMEM; + goto out; + } } else { char *new_dir = calloc(strlen(cwd) + strlen(cont_name) + 1, sizeof(char)); - if (new_dir == NULL) - D_GOTO(out, rc = -ENOMEM); + if (new_dir == NULL) { + rc = -ENOMEM; + goto out; + } strcpy(new_dir, cwd); if (cont_name[0] == '.') { @@ -198,15 +209,18 @@ parse_filename(const char *path, char **_obj_name, char **_cont_name) *_cont_name = cont_name; } else { *_cont_name = strdup(cont_name); - if (*_cont_name == NULL) - D_GOTO(out, rc = -ENOMEM); + if (*_cont_name == NULL) { + rc = -ENOMEM; + goto out; + } } *_obj_name = strdup(fname); if (*_obj_name == NULL) { free(*_cont_name); *_cont_name = NULL; - D_GOTO(out, rc = -ENOMEM); + rc = -ENOMEM; + goto out; } out: From c35f64237ffb8ef151abeff1b9669bc754ea160e Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Thu, 14 Mar 2019 16:55:54 +0000 Subject: [PATCH 059/206] initialize data and akey checksum to NULL since they are not being used. Signed-off-by: Mohamad Chaarawi --- src/aiori-DAOS.c | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/src/aiori-DAOS.c b/src/aiori-DAOS.c index 90356001..f06c9175 100644 --- a/src/aiori-DAOS.c +++ b/src/aiori-DAOS.c @@ -336,18 +336,12 @@ static void AIOInit(IOR_param_t *param) aio->a_recx.rx_nr = 1; - aio->a_csum.cs_csum = &aio->a_csumBuf; - aio->a_csum.cs_buf_len = sizeof aio->a_csumBuf; - aio->a_csum.cs_len = aio->a_csum.cs_buf_len; - - aio->a_iod.iod_name.iov_buf = "data"; - aio->a_iod.iod_name.iov_buf_len = - strlen(aio->a_iod.iod_name.iov_buf) + 1; - aio->a_iod.iod_name.iov_len = aio->a_iod.iod_name.iov_buf_len; + daos_iov_set(&aio->a_iod.iod_name, "data", strlen("data")); + daos_csum_set(&aio->a_iod.iod_kcsum, NULL, 0); aio->a_iod.iod_nr = 1; aio->a_iod.iod_type = DAOS_IOD_ARRAY; aio->a_iod.iod_recxs = &aio->a_recx; - aio->a_iod.iod_csums = &aio->a_csum; + aio->a_iod.iod_csums = NULL; aio->a_iod.iod_eprs = NULL; aio->a_iod.iod_size = param->transferSize; From fabb1f4d3bca62dff5dcb930f7cf66bba03f96fb Mon Sep 17 00:00:00 2001 From: Sylvain Didelot Date: Thu, 14 Mar 2019 18:37:00 +0100 Subject: [PATCH 060/206] Fix segfault with option memoryPerNode optarg contains a NULL address, which causes a segfault in NodeMemoryStringToBytes(). --- src/parse_options.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/parse_options.c b/src/parse_options.c index a8f81b02..45be0bad 100755 --- a/src/parse_options.c +++ b/src/parse_options.c @@ -537,7 +537,7 @@ IOR_test_t *ParseCommandLine(int argc, char **argv) } } if (memoryPerNode){ - initialTestParams.memoryPerNode = NodeMemoryStringToBytes(optarg); + initialTestParams.memoryPerNode = NodeMemoryStringToBytes(memoryPerNode); } const ior_aiori_t * backend = aiori_select(initialTestParams.api); From 39eca1bb088c960299c8a14a90d25640ff0e3cfc Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Thu, 14 Mar 2019 21:17:37 +0000 Subject: [PATCH 061/206] add DAOS_Access routine to check if a container exists before deleting it. Signed-off-by: Mohamad Chaarawi --- src/aiori-DAOS.c | 26 ++++++++++++++++++++++++-- src/ior.c | 6 ++---- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/src/aiori-DAOS.c b/src/aiori-DAOS.c index f06c9175..1555c57a 100644 --- a/src/aiori-DAOS.c +++ b/src/aiori-DAOS.c @@ -83,6 +83,7 @@ static void DAOS_Init(); static void DAOS_Fini(); static void *DAOS_Create(char *, IOR_param_t *); static void *DAOS_Open(char *, IOR_param_t *); +static int DAOS_Access(const char *, int, IOR_param_t *); static IOR_offset_t DAOS_Xfer(int, void *, IOR_size_t *, IOR_offset_t, IOR_param_t *); static void DAOS_Close(void *, IOR_param_t *); @@ -98,6 +99,7 @@ ior_aiori_t daos_aiori = { .name = "DAOS", .create = DAOS_Create, .open = DAOS_Open, + .access = DAOS_Access, .xfer = DAOS_Xfer, .close = DAOS_Close, .delete = DAOS_Delete, @@ -568,6 +570,26 @@ static void *DAOS_Create(char *testFileName, IOR_param_t *param) return DAOS_Open(testFileName, param); } +static int +DAOS_Access(const char *testFileName, int mode, IOR_param_t * param) +{ + uuid_t uuid; + unsigned int dFlags; + daos_handle_t coh; + daos_cont_info_t info; + int rc; + + rc = uuid_parse(testFileName, uuid); + DCHECK(rc, "Failed to parse 'testFile': %s", testFileName); + + rc = daos_cont_open(pool, uuid, DAOS_COO_RO, &coh, &info, NULL); + if (rc) + return rc; + + rc = daos_cont_close(coh, NULL); + return rc; +} + static void *DAOS_Open(char *testFileName, IOR_param_t *param) { struct fileDescriptor *fd; @@ -785,8 +807,8 @@ static void DAOS_Delete(char *testFileName, IOR_param_t *param) DCHECK(rc, "Failed to parse 'testFile': %s", testFileName); rc = daos_cont_destroy(pool, uuid, 1 /* force */, NULL /* ev */); - if (rc != -DER_NONEXIST) - DCHECK(rc, "Failed to destroy container %s", testFileName); + if (rc) + DCHECK(rc, "Failed to destroy container %s (%d)", testFileName, rc); } static char* DAOS_GetVersion() diff --git a/src/ior.c b/src/ior.c index 2a5532a9..8dba1562 100755 --- a/src/ior.c +++ b/src/ior.c @@ -910,8 +910,7 @@ static void RemoveFile(char *testFileName, int filePerProc, IOR_param_t * test) rankOffset = 0; GetTestFileName(testFileName, test); } - if (backend->access(testFileName, F_OK, test) == 0 || - strcasecmp(test->api, "DAOS") == 0) { + if (backend->access(testFileName, F_OK, test) == 0) { backend->delete(testFileName, test); } if (test->reorderTasksRandom == TRUE) { @@ -919,8 +918,7 @@ static void RemoveFile(char *testFileName, int filePerProc, IOR_param_t * test) GetTestFileName(testFileName, test); } } else { - if ((rank == 0) && (backend->access(testFileName, F_OK, test) == 0 || - strcasecmp(test->api, "DAOS") == 0)) { + if (rank == 0 && backend->access(testFileName, F_OK, test) == 0) { backend->delete(testFileName, test); } } From 8b23c5038900f09661b6be0af048bee95955151f Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Fri, 22 Mar 2019 16:53:34 +0000 Subject: [PATCH 062/206] update dfs_open for API change to expose chunk size selection Signed-off-by: Mohamad Chaarawi --- src/aiori-DFS.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/aiori-DFS.c b/src/aiori-DFS.c index dd9bc7f7..86bd1759 100755 --- a/src/aiori-DFS.c +++ b/src/aiori-DFS.c @@ -333,7 +333,7 @@ DFS_Create(char *testFileName, IOR_param_t *param) DERR(rc, "dfs_lookup() of %s Failed", dir_name); rc = dfs_open(dfs, parent, name, mode, fd_oflag, DAOS_OC_LARGE_RW, - NULL, &obj); + 0, NULL, &obj); DERR(rc, "dfs_open() of %s Failed", name); out: @@ -371,7 +371,7 @@ DFS_Open(char *testFileName, IOR_param_t *param) rc = dfs_lookup(dfs, dir_name, O_RDWR, &parent, &pmode); DERR(rc, "dfs_lookup() of %s Failed", dir_name); - rc = dfs_open(dfs, parent, name, mode, fd_oflag, 0, NULL, &obj); + rc = dfs_open(dfs, parent, name, mode, fd_oflag, 0, 0, NULL, &obj); DERR(rc, "dfs_open() of %s Failed", name); out: From 20e960d0205acb4d8bda8ea50bc4f9f65b16494a Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Wed, 27 Mar 2019 20:04:48 +0000 Subject: [PATCH 063/206] Support the setting of module-specific options per test. --- src/aiori-DUMMY.c | 50 ++++---- src/aiori-POSIX.c | 36 +++++- src/aiori.c | 47 ++++--- src/aiori.h | 7 +- src/ior-output.c | 1 - src/ior.c | 21 +--- src/ior.h | 12 +- src/mdtest.c | 16 +-- src/option.c | 247 ++++++++++++++++++++----------------- src/option.h | 8 +- src/parse_options.c | 295 +++++++++++++++++++------------------------- src/utilities.c | 77 ++++++++++++ src/utilities.h | 3 + 13 files changed, 463 insertions(+), 357 deletions(-) diff --git a/src/aiori-DUMMY.c b/src/aiori-DUMMY.c index 675a91be..0494cb4d 100755 --- a/src/aiori-DUMMY.c +++ b/src/aiori-DUMMY.c @@ -17,29 +17,33 @@ /************************** O P T I O N S *****************************/ -struct dummy_options{ +typedef struct { uint64_t delay_creates; uint64_t delay_xfer; int delay_rank_0_only; -}; +} dummy_options_t; -static struct dummy_options o = { - .delay_creates = 0, - .delay_xfer = 0, - .delay_rank_0_only = 0, -}; +static char * current = (char*) 1; -static option_help options [] = { - {0, "dummy.delay-create", "Delay per create in usec", OPTION_OPTIONAL_ARGUMENT, 'l', & o.delay_creates}, - {0, "dummy.delay-xfer", "Delay per xfer in usec", OPTION_OPTIONAL_ARGUMENT, 'l', & o.delay_xfer}, - {0, "dummy.delay-only-rank0", "Delay only Rank0", OPTION_FLAG, 'd', & o.delay_rank_0_only}, - LAST_OPTION -}; +static option_help * DUMMY_options(void ** init_backend_options, void * init_values){ + dummy_options_t * o = malloc(sizeof(dummy_options_t)); + if (init_values != NULL){ + memcpy(o, init_values, sizeof(dummy_options_t)); + }else{ + memset(o, 0, sizeof(dummy_options_t)); + } -static char * current = (char*) 1; + *init_backend_options = o; -static option_help * DUMMY_options(){ - return options; + option_help h [] = { + {0, "dummy.delay-create", "Delay per create in usec", OPTION_OPTIONAL_ARGUMENT, 'l', & o->delay_creates}, + {0, "dummy.delay-xfer", "Delay per xfer in usec", OPTION_OPTIONAL_ARGUMENT, 'l', & o->delay_xfer}, + {0, "dummy.delay-only-rank0", "Delay only Rank0", OPTION_FLAG, 'd', & o->delay_rank_0_only}, + LAST_OPTION + }; + option_help * help = malloc(sizeof(h)); + memcpy(help, h, sizeof(h)); + return help; } static void *DUMMY_Create(char *testFileName, IOR_param_t * param) @@ -47,9 +51,10 @@ static void *DUMMY_Create(char *testFileName, IOR_param_t * param) if(verbose > 4){ fprintf(out_logfile, "DUMMY create: %s = %p\n", testFileName, current); } - if (o.delay_creates){ - if (! o.delay_rank_0_only || (o.delay_rank_0_only && rank == 0)){ - struct timespec wait = { o.delay_creates / 1000 / 1000, 1000l * (o.delay_creates % 1000000)}; + dummy_options_t * o = (dummy_options_t*) param->backend_options; + if (o->delay_creates){ + if (! o->delay_rank_0_only || (o->delay_rank_0_only && rank == 0)){ + struct timespec wait = { o->delay_creates / 1000 / 1000, 1000l * (o->delay_creates % 1000000)}; nanosleep( & wait, NULL); } } @@ -102,9 +107,10 @@ static IOR_offset_t DUMMY_Xfer(int access, void *file, IOR_size_t * buffer, IOR_ if(verbose > 4){ fprintf(out_logfile, "DUMMY xfer: %p\n", file); } - if (o.delay_xfer){ - if (! o.delay_rank_0_only || (o.delay_rank_0_only && rank == 0)){ - struct timespec wait = {o.delay_xfer / 1000 / 1000, 1000l * (o.delay_xfer % 1000000)}; + dummy_options_t * o = (dummy_options_t*) param->backend_options; + if (o->delay_xfer){ + if (! o->delay_rank_0_only || (o->delay_rank_0_only && rank == 0)){ + struct timespec wait = {o->delay_xfer / 1000 / 1000, 1000l * (o->delay_xfer % 1000000)}; nanosleep( & wait, NULL); } } diff --git a/src/aiori-POSIX.c b/src/aiori-POSIX.c index ca7c4a29..1d41c640 100755 --- a/src/aiori-POSIX.c +++ b/src/aiori-POSIX.c @@ -72,8 +72,36 @@ static IOR_offset_t POSIX_Xfer(int, void *, IOR_size_t *, IOR_offset_t, IOR_param_t *); static void POSIX_Fsync(void *, IOR_param_t *); +/************************** O P T I O N S *****************************/ +typedef struct{ + int direct_io; +} posix_options_t; + + +static option_help * posix_options(void ** init_backend_options, void * init_values){ + posix_options_t * o = malloc(sizeof(posix_options_t)); + + if (init_values != NULL){ + memcpy(o, init_values, sizeof(posix_options_t)); + }else{ + o->direct_io = 0; + } + + *init_backend_options = o; + + option_help h [] = { + {0, "posix.odirect", "Direct I/O Mode", OPTION_FLAG, 'd', & o->direct_io}, + LAST_OPTION + }; + option_help * help = malloc(sizeof(h)); + memcpy(help, h, sizeof(h)); + return help; +} + + /************************** D E C L A R A T I O N S ***************************/ + ior_aiori_t posix_aiori = { .name = "POSIX", .name_legacy = NULL, @@ -90,6 +118,7 @@ ior_aiori_t posix_aiori = { .rmdir = aiori_posix_rmdir, .access = aiori_posix_access, .stat = aiori_posix_stat, + .get_options = posix_options, .enable_mdtest = true, }; @@ -274,8 +303,8 @@ void *POSIX_Create(char *testFileName, IOR_param_t * param) fd = (int *)malloc(sizeof(int)); if (fd == NULL) ERR("Unable to malloc file descriptor"); - - if (param->useO_DIRECT == TRUE) + posix_options_t * o = (posix_options_t*) param->backend_options; + if (o->direct_io == TRUE) set_o_direct_flag(&fd_oflag); if(param->dryRun) @@ -389,7 +418,8 @@ void *POSIX_Open(char *testFileName, IOR_param_t * param) if (fd == NULL) ERR("Unable to malloc file descriptor"); - if (param->useO_DIRECT == TRUE) + posix_options_t * o = (posix_options_t*) param->backend_options; + if (o->direct_io == TRUE) set_o_direct_flag(&fd_oflag); fd_oflag |= O_RDWR; diff --git a/src/aiori.c b/src/aiori.c index 11f11b4a..52001c8a 100644 --- a/src/aiori.c +++ b/src/aiori.c @@ -72,24 +72,37 @@ ior_aiori_t *available_aiori[] = { NULL }; -void airoi_parse_options(int argc, char ** argv, option_help * global_options){ - int airoi_c = aiori_count(); - options_all opt; - opt.module_count = airoi_c + 1; - opt.modules = malloc(sizeof(option_module) * (airoi_c + 1)); - opt.modules[0].prefix = NULL; - opt.modules[0].options = global_options; - ior_aiori_t **tmp = available_aiori; - for (int i=1; *tmp != NULL; ++tmp, i++) { - opt.modules[i].prefix = (*tmp)->name; - if((*tmp)->get_options != NULL){ - opt.modules[i].options = (*tmp)->get_options(); - }else{ - opt.modules[i].options = NULL; - } +void * airoi_update_module_options(const ior_aiori_t * backend, options_all_t * opt){ + if (backend->get_options == NULL) + return NULL; + char * name = backend->name; + ior_aiori_t **tmp = available_aiori; + for (int i=1; *tmp != NULL; ++tmp, i++) { + if (strcmp(opt->modules[i].prefix, name) == 0){ + opt->modules[i].options = (*tmp)->get_options(& opt->modules[i].defaults, opt->modules[i].defaults); + return opt->modules[i].defaults; } - option_parse(argc, argv, &opt); - free(opt.modules); + } + return NULL; +} + +options_all_t * airoi_create_all_module_options(option_help * global_options){ + int airoi_c = aiori_count(); + options_all_t * opt = malloc(sizeof(options_all_t)); + opt->module_count = airoi_c + 1; + opt->modules = malloc(sizeof(option_module) * (airoi_c + 1)); + opt->modules[0].prefix = NULL; + opt->modules[0].options = global_options; + ior_aiori_t **tmp = available_aiori; + for (int i=1; *tmp != NULL; ++tmp, i++) { + opt->modules[i].prefix = (*tmp)->name; + if((*tmp)->get_options != NULL){ + opt->modules[i].options = (*tmp)->get_options(& opt->modules[i].defaults, NULL); + }else{ + opt->modules[i].options = NULL; + } + } + return opt; } void aiori_supported_apis(char * APIs, char * APIs_legacy, enum bench_type type) diff --git a/src/aiori.h b/src/aiori.h index 727af58c..782d1ac4 100755 --- a/src/aiori.h +++ b/src/aiori.h @@ -83,7 +83,7 @@ typedef struct ior_aiori { int (*stat) (const char *path, struct stat *buf, IOR_param_t * param); void (*initialize)(); /* called once per program before MPI is started */ void (*finalize)(); /* called once per program after MPI is shutdown */ - option_help * (*get_options)(); + option_help * (*get_options)(void ** init_backend_options, void* init_values); /* initializes the backend options as well and returns the pointer to the option help structure */ bool enable_mdtest; } ior_aiori_t; @@ -110,7 +110,10 @@ void aiori_finalize(IOR_test_t * tests); const ior_aiori_t *aiori_select (const char *api); int aiori_count (void); void aiori_supported_apis(char * APIs, char * APIs_legacy, enum bench_type type); -void airoi_parse_options(int argc, char ** argv, option_help * global_options); +options_all_t * airoi_create_all_module_options(option_help * global_options); + +void * airoi_update_module_options(const ior_aiori_t * backend, options_all_t * module_defaults); + const char *aiori_default (void); /* some generic POSIX-based backend calls */ diff --git a/src/ior-output.c b/src/ior-output.c index 32a60549..40764fff 100644 --- a/src/ior-output.c +++ b/src/ior-output.c @@ -367,7 +367,6 @@ void ShowTestStart(IOR_param_t *test) PrintKeyValInt("setAlignment", test->setAlignment); PrintKeyValInt("storeFileOffset", test->storeFileOffset); PrintKeyValInt("useSharedFilePointer", test->useSharedFilePointer); - PrintKeyValInt("useO_DIRECT", test->useO_DIRECT); PrintKeyValInt("useStridedDatatype", test->useStridedDatatype); PrintKeyValInt("keepFile", test->keepFile); PrintKeyValInt("keepFileWithError", test->keepFileWithError); diff --git a/src/ior.c b/src/ior.c index 12b93906..a1dc321a 100755 --- a/src/ior.c +++ b/src/ior.c @@ -493,20 +493,13 @@ static void aligned_buffer_free(void *buf) free(*(void **)((char *)buf - sizeof(char *))); } -static void* safeMalloc(uint64_t size){ - void * d = malloc(size); - if (d == NULL){ - ERR("Could not malloc an array"); - } - memset(d, 0, size); - return d; -} - void AllocResults(IOR_test_t *test) { int reps; if (test->results != NULL) - return; + return; + + IOR_param_t * params = & test->params; reps = test->params.repetitions; test->results = (IOR_results_t *) safeMalloc(sizeof(IOR_results_t) * reps); @@ -1235,13 +1228,7 @@ static void TestIoSys(IOR_test_t *test) fflush(out_logfile); } params->tasksPerNode = CountTasksPerNode(testComm); - - /* bind I/O calls to specific API */ - backend = aiori_select(params->api); - if (backend == NULL) - ERR_SIMPLE("unrecognized I/O API"); - - + backend = params->backend; /* show test setup */ if (rank == 0 && verbose >= VERBOSE_0) ShowSetup(params); diff --git a/src/ior.h b/src/ior.h index ad543ffe..acf3a7db 100755 --- a/src/ior.h +++ b/src/ior.h @@ -36,6 +36,8 @@ typedef void *rados_ioctx_t; #endif +#include "option.h" + #include "iordef.h" /******************** DATA Packet Type ***************************************/ /* Holds the types of data packets: generic, offset, timestamp, incompressible */ @@ -93,6 +95,7 @@ typedef struct char * testFileName_fppReadCheck;/* filename for fpp read check */ char * hintsFileName; /* full name for hints file */ char * options; /* options string */ + // intermediate options int dryRun; /* do not perform any I/Os just run evtl. inputs print dummy output */ int numTasks; /* number of tasks for test */ int nodes; /* number of nodes for test */ @@ -126,7 +129,6 @@ typedef struct int useFileView; /* use MPI_File_set_view */ int useSharedFilePointer; /* use shared file pointer */ int useStridedDatatype; /* put strided access into datatype */ - int useO_DIRECT; /* use O_DIRECT, bypassing I/O buffers */ int showHints; /* show hints */ int summary_every_test; /* flag to print summary every test, not just at end */ int uniqueDir; /* use unique directory for each fpp */ @@ -148,8 +150,12 @@ typedef struct int randomOffset; /* access is to random offsets */ size_t memoryPerTask; /* additional memory used per task */ size_t memoryPerNode; /* additional memory used per node */ - enum PACKET_TYPE dataPacketType; /* The type of data packet. */ + char * memoryPerNodeStr; /* for parsing */ + char * testscripts; /* for parsing */ + char * buffer_type; /* for parsing */ + enum PACKET_TYPE dataPacketType; /* The type of data packet. */ + void * backend_options; /* Backend-specific options */ /* POSIX variables */ int singleXferAttempt; /* do not retry transfer if incomplete */ @@ -180,7 +186,7 @@ typedef struct size_t part_number; /* multi-part upload increment (PER-RANK!) */ char* UploadId; /* key for multi-part-uploads */ int collective_md; /* use collective metatata optimization */ - + /* RADOS variables */ rados_t rados_cluster; /* RADOS cluster handle */ diff --git a/src/mdtest.c b/src/mdtest.c index 8a819796..387bd863 100644 --- a/src/mdtest.c +++ b/src/mdtest.c @@ -151,7 +151,6 @@ static pid_t pid; static uid_t uid; /* Use the POSIX backend by default */ -static const char *backend_name = "POSIX"; static const ior_aiori_t *backend; static IOR_param_t param; @@ -2099,7 +2098,6 @@ void mdtest_init_args(){ nstride = 0; } - mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * world_out) { testComm = world_com; out_logfile = world_out; @@ -2132,7 +2130,7 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * sprintf(apiStr, "API for I/O [%s]", APIs); option_help options [] = { - {'a', NULL, apiStr, OPTION_OPTIONAL_ARGUMENT, 's', & backend_name}, + {'a', NULL, apiStr, OPTION_OPTIONAL_ARGUMENT, 's', & param.api}, {'b', NULL, "branching factor of hierarchical directory structure", OPTION_OPTIONAL_ARGUMENT, 'd', & branch_factor}, {'d', NULL, "the directory in which the tests will run", OPTION_OPTIONAL_ARGUMENT, 's', & path}, {'B', NULL, "no barriers between phases", OPTION_OPTIONAL_ARGUMENT, 'd', & no_barriers}, @@ -2168,12 +2166,10 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * {'Z', NULL, "print time instead of rate", OPTION_FLAG, 'd', & print_time}, LAST_OPTION }; - airoi_parse_options(argc, argv, options); - - backend = aiori_select(backend_name); - if (NULL == backend) { - FAIL("Could not find suitable backend to use"); - } + options_all_t * global_options = airoi_create_all_module_options(options); + option_parse(argc, argv, global_options); + updateParsedOptions(& param, global_options); + backend = param.backend; MPI_Comm_rank(testComm, &rank); MPI_Comm_size(testComm, &size); @@ -2222,7 +2218,7 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * if (( rank == 0 ) && ( verbose >= 1 )) { // option_print_current(options); - fprintf (out_logfile, "api : %s\n", backend_name); + fprintf (out_logfile, "api : %s\n", param.api); fprintf( out_logfile, "barriers : %s\n", ( barriers ? "True" : "False" )); fprintf( out_logfile, "collective_creates : %s\n", ( collective_creates ? "True" : "False" )); fprintf( out_logfile, "create_only : %s\n", ( create_only ? "True" : "False" )); diff --git a/src/option.c b/src/option.c index 4e880555..08833dac 100644 --- a/src/option.c +++ b/src/option.c @@ -220,135 +220,160 @@ void option_print_current(option_help * args){ print_current_option_section(args, OPTION_FLAG); } -int option_parse(int argc, char ** argv, options_all * opt_all){ - int error = 0; - int requiredArgsSeen = 0; - int requiredArgsNeeded = 0; - int i; - int printhelp = 0; +static void option_parse_token(char ** argv, int * flag_parsed_next, int * requiredArgsSeen, options_all_t * opt_all, int * error, int * print_help){ + int foundOption = 0; + char * txt = argv[0]; + char * arg = strstr(txt, "="); + int replaced_equal = 0; + int i = 0; + if(arg != NULL){ + arg[0] = 0; + arg++; + replaced_equal = 1; + } + *flag_parsed_next = 0; for(int m = 0; m < opt_all->module_count; m++ ){ option_help * args = opt_all->modules[m].options; if(args == NULL) continue; - for(option_help * o = args; o->shortVar != 0 || o->longVar != 0 ; o++ ){ - if(o->arg == OPTION_REQUIRED_ARGUMENT){ - requiredArgsNeeded++; + // try to find matching option help + for(option_help * o = args; o->shortVar != 0 || o->longVar != 0 || o->help != NULL ; o++ ){ + if( o->shortVar == 0 && o->longVar == 0 ){ + // section + continue; } - } - } - - for(i=1; i < argc; i++){ - char * txt = argv[i]; - int foundOption = 0; - char * arg = strstr(txt, "="); - int replaced_equal = 0; - if(arg != NULL){ - arg[0] = 0; - arg++; - replaced_equal = 1; - } + if ( (txt[0] == '-' && o->shortVar == txt[1]) || (strlen(txt) > 2 && txt[0] == '-' && txt[1] == '-' && o->longVar != NULL && strcmp(txt + 2, o->longVar) == 0)){ + foundOption = 1; + + // now process the option. + switch(o->arg){ + case (OPTION_FLAG):{ + assert(o->type == 'd'); + (*(int*) o->variable)++; + break; + } + case (OPTION_OPTIONAL_ARGUMENT): + case (OPTION_REQUIRED_ARGUMENT):{ + // check if next is an argument + if(arg == NULL){ + if(o->shortVar == txt[1] && txt[2] != 0){ + arg = & txt[2]; + }else{ + // simply take the next value as argument + i++; + arg = argv[1]; + *flag_parsed_next = 1; + } + } - for(int m = 0; m < opt_all->module_count; m++ ){ - option_help * args = opt_all->modules[m].options; - if(args == NULL) continue; - // try to find matching option help - for(option_help * o = args; o->shortVar != 0 || o->longVar != 0 || o->help != NULL ; o++ ){ - if( o->shortVar == 0 && o->longVar == 0 ){ - // section - continue; - } - if ( (txt[0] == '-' && o->shortVar == txt[1]) || (strlen(txt) > 2 && txt[0] == '-' && txt[1] == '-' && o->longVar != NULL && strcmp(txt + 2, o->longVar) == 0)){ - foundOption = 1; - - // now process the option. - switch(o->arg){ - case (OPTION_FLAG):{ - assert(o->type == 'd'); - (*(int*) o->variable)++; - break; + if(arg == NULL){ + const char str[] = {o->shortVar, 0}; + printf("Error, argument missing for option %s\n", (o->longVar != NULL) ? o->longVar : str); + exit(1); } - case (OPTION_OPTIONAL_ARGUMENT): - case (OPTION_REQUIRED_ARGUMENT):{ - // check if next is an argument - if(arg == NULL){ - if(o->shortVar == txt[1] && txt[2] != 0){ - arg = & txt[2]; - }else{ - // simply take the next value as argument - i++; - arg = argv[i]; - } - } - if(arg == NULL){ - const char str[] = {o->shortVar, 0}; - printf("Error, argument missing for option %s\n", (o->longVar != NULL) ? o->longVar : str); - exit(1); + switch(o->type){ + case('p'):{ + // call the function in the variable + void(*fp)() = o->variable; + fp(arg); + break; } - - switch(o->type){ - case('p'):{ - // call the function in the variable - void(*fp)() = o->variable; - fp(arg); - break; - } - case('F'):{ - *(double*) o->variable = atof(arg); - break; - } - case('f'):{ - *(float*) o->variable = atof(arg); - break; - } - case('d'):{ - int64_t val = string_to_bytes(arg); - if (val > INT_MAX || val < INT_MIN){ - printf("WARNING: parsing the number %s to integer, this produced an overflow!\n", arg); - } - *(int*) o->variable = val; - break; - } - case('H'): - case('s'):{ - (*(char **) o->variable) = strdup(arg); - break; - } - case('c'):{ - (*(char *)o->variable) = arg[0]; - if(strlen(arg) > 1){ - printf("Error, ignoring remainder of string for option %c (%s).\n", o->shortVar, o->longVar); - } - break; + case('F'):{ + *(double*) o->variable = atof(arg); + break; + } + case('f'):{ + *(float*) o->variable = atof(arg); + break; + } + case('d'):{ + int64_t val = string_to_bytes(arg); + if (val > INT_MAX || val < INT_MIN){ + printf("WARNING: parsing the number %s to integer, this produced an overflow!\n", arg); } - case('l'):{ - *(long long*) o->variable = string_to_bytes(arg); - break; + *(int*) o->variable = val; + break; + } + case('H'): + case('s'):{ + (*(char **) o->variable) = strdup(arg); + break; + } + case('c'):{ + (*(char *)o->variable) = arg[0]; + if(strlen(arg) > 1){ + printf("Error, ignoring remainder of string for option %c (%s).\n", o->shortVar, o->longVar); } - default: - printf("ERROR: Unknown option type %c\n", o->type); + break; + } + case('l'):{ + *(long long*) o->variable = string_to_bytes(arg); + break; } + default: + printf("ERROR: Unknown option type %c\n", o->type); } } - if(replaced_equal){ - arg[-1] = '='; - } - - if(o->arg == OPTION_REQUIRED_ARGUMENT){ - requiredArgsSeen++; - } + } + if(replaced_equal){ + arg[-1] = '='; + } - break; + if(o->arg == OPTION_REQUIRED_ARGUMENT){ + (*requiredArgsSeen)++; } + + break; } } - if (! foundOption){ - if(strcmp(txt, "-h") == 0 || strcmp(txt, "--help") == 0){ - printhelp = 1; - }else{ - printf("Error invalid argument: %s\n", txt); - error = 1; - } + } + + + if (! foundOption){ + if(strcmp(txt, "-h") == 0 || strcmp(txt, "--help") == 0){ + *print_help = 1; + }else{ + *error = 1; + } + } +} + +int option_parse_key_value(char * key, char * value, options_all_t * opt_all){ + int flag_parsed_next; + char * argv[2] = {key, value}; + int error = 0; + int requiredArgsSeen = 0; + int print_help = 0; + option_parse_token(argv, & flag_parsed_next, & requiredArgsSeen, opt_all, & error, & print_help); + return error; +} + +int option_parse(int argc, char ** argv, options_all_t * opt_all){ + int error = 0; + int requiredArgsSeen = 0; + int requiredArgsNeeded = 0; + int i; + int printhelp = 0; + + for(int m = 0; m < opt_all->module_count; m++ ){ + option_help * args = opt_all->modules[m].options; + if(args == NULL) continue; + for(option_help * o = args; o->shortVar != 0 || o->longVar != 0 ; o++ ){ + if(o->arg == OPTION_REQUIRED_ARGUMENT){ + requiredArgsNeeded++; + } + } + } + + for(i=1; i < argc; i++){ + int flag_parsed_next; + option_parse_token(& argv[i], & flag_parsed_next, & requiredArgsSeen, opt_all, & error, & printhelp); + if (flag_parsed_next){ + i++; + } + if(error){ + printf("Error invalid argument: %s\n", argv[i]); } } diff --git a/src/option.h b/src/option.h index 53a3732b..72259216 100644 --- a/src/option.h +++ b/src/option.h @@ -26,12 +26,13 @@ typedef struct{ typedef struct{ char * prefix; // may be NULL to include it in the standard name option_help * options; + void * defaults; // these default values are taken from the command line } option_module; typedef struct{ int module_count; option_module * modules; -} options_all; +} options_all_t; #define LAST_OPTION {0, 0, 0, (option_value_type) 0, 0, NULL} @@ -39,6 +40,9 @@ int64_t string_to_bytes(char *size_str); void option_print_current(option_help * args); //@return the number of parsed arguments -int option_parse(int argc, char ** argv, options_all * args); +int option_parse(int argc, char ** argv, options_all_t * args); + +/* Parse a single line */ +int option_parse_key_value(char * key, char * value, options_all_t * opt_all); #endif diff --git a/src/parse_options.c b/src/parse_options.c index ea6dd63d..f92e2399 100755 --- a/src/parse_options.c +++ b/src/parse_options.c @@ -36,38 +36,11 @@ IOR_param_t initialTestParams; +option_help * createGlobalOptions(IOR_param_t * params); -static size_t NodeMemoryStringToBytes(char *size_str) -{ - int percent; - int rc; - long page_size; - long num_pages; - long long mem; - - rc = sscanf(size_str, " %d %% ", &percent); - if (rc == 0) - return (size_t) string_to_bytes(size_str); - if (percent > 100 || percent < 0) - ERR("percentage must be between 0 and 100"); - -#ifdef HAVE_SYSCONF - page_size = sysconf(_SC_PAGESIZE); -#else - page_size = getpagesize(); -#endif - -#ifdef _SC_PHYS_PAGES - num_pages = sysconf(_SC_PHYS_PAGES); - if (num_pages == -1) - ERR("sysconf(_SC_PHYS_PAGES) is not supported"); -#else - ERR("sysconf(_SC_PHYS_PAGES) is not supported"); -#endif - mem = page_size * num_pages; - return mem / 100 * percent; -} +static IOR_param_t * parameters; +static options_all_t * global_options; /* @@ -113,7 +86,7 @@ static void CheckRunSettings(IOR_test_t *tests) /* * Set flags from commandline string/value pairs. */ -void DecodeDirective(char *line, IOR_param_t *params) +void DecodeDirective(char *line, IOR_param_t *params, options_all_t * module_options) { char option[MAX_STR]; char value[MAX_STR]; @@ -132,6 +105,14 @@ void DecodeDirective(char *line, IOR_param_t *params) } if (strcasecmp(option, "api") == 0) { params->api = strdup(value); + + params->backend = aiori_select(params->api); + if (params->backend == NULL){ + fprintf(out_logfile, "Could not load backend API %s\n", params->api); + exit(-1); + } + /* copy the actual module options into the test */ + params->backend_options = airoi_update_module_options(params->backend, global_options); } else if (strcasecmp(option, "summaryFile") == 0) { if (rank == 0){ out_resultfile = fopen(value, "w"); @@ -240,8 +221,6 @@ void DecodeDirective(char *line, IOR_param_t *params) params->useFileView = atoi(value); } else if (strcasecmp(option, "usesharedfilepointer") == 0) { params->useSharedFilePointer = atoi(value); - } else if (strcasecmp(option, "useo_direct") == 0) { - params->useO_DIRECT = atoi(value); } else if (strcasecmp(option, "usestrideddatatype") == 0) { params->useStridedDatatype = atoi(value); } else if (strcasecmp(option, "showhints") == 0) { @@ -314,21 +293,28 @@ void DecodeDirective(char *line, IOR_param_t *params) } else if (strcasecmp(option, "summaryalways") == 0) { params->summary_every_test = atoi(value); } else { - if (rank == 0) - fprintf(out_logfile, "Unrecognized parameter \"%s\"\n", - option); - MPI_CHECK(MPI_Initialized(&initialized), "MPI_Initialized() error"); - if (initialized) - MPI_CHECK(MPI_Abort(MPI_COMM_WORLD, -1), "MPI_Abort() error"); - else - exit(-1); + int parsing_error = option_parse_key_value(option, value, module_options); + if(parsing_error){ + if (rank == 0) + fprintf(out_logfile, "Unrecognized parameter \"%s\"\n", + option); + MPI_CHECK(MPI_Initialized(&initialized), "MPI_Initialized() error"); + if (initialized) + MPI_CHECK(MPI_Abort(MPI_COMM_WORLD, -1), "MPI_Abort() error"); + else + exit(-1); + } } } +static void decodeDirectiveWrapper(char *line){ + DecodeDirective(line, parameters, global_options); +} + /* * Parse a single line, which may contain multiple comma-seperated directives */ -void ParseLine(char *line, IOR_param_t * test) +void ParseLine(char *line, IOR_param_t * test, options_all_t * module_options) { char *start, *end; @@ -337,7 +323,7 @@ void ParseLine(char *line, IOR_param_t * test) end = strchr(start, ','); if (end != NULL) *end = '\0'; - DecodeDirective(start, test); + DecodeDirective(start, test, module_options); start = end + 1; } while (end != NULL); } @@ -383,9 +369,12 @@ IOR_test_t *ReadConfigScript(char *scriptName) IOR_test_t *head = NULL; IOR_test_t *tail = NULL; + option_help ** option_p = & global_options->modules[0].options; + /* Initialize the first test */ - head = CreateTest(&initialTestParams, test_num++); + head = CreateTest(& initialTestParams, test_num++); tail = head; + *option_p = createGlobalOptions(& ((IOR_test_t*) head)->params); /* The current options */ /* open the script */ file = fopen(scriptName, "r"); @@ -423,6 +412,7 @@ IOR_test_t *ReadConfigScript(char *scriptName) tail->next = CreateTest(&tail->params, test_num++); AllocResults(tail); tail = tail->next; + *option_p = createGlobalOptions(& ((IOR_test_t*) tail->next)->params); } runflag = 1; } else if (runflag) { @@ -430,11 +420,12 @@ IOR_test_t *ReadConfigScript(char *scriptName) create and initialize a new test structure */ runflag = 0; tail->next = CreateTest(&tail->params, test_num++); + *option_p = createGlobalOptions(& ((IOR_test_t*) tail->next)->params); AllocResults(tail); tail = tail->next; - ParseLine(ptr, &tail->params); + ParseLine(ptr, &tail->params, global_options); } else { - ParseLine(ptr, &tail->params); + ParseLine(ptr, &tail->params, global_options); } } @@ -447,139 +438,105 @@ IOR_test_t *ReadConfigScript(char *scriptName) } -static IOR_param_t * parameters; - -static void decodeDirectiveWrapper(char *line){ - DecodeDirective(line, parameters); +option_help * createGlobalOptions(IOR_param_t * params){ + char APIs[1024]; + char APIs_legacy[1024]; + aiori_supported_apis(APIs, APIs_legacy, IOR); + char apiStr[1024]; + sprintf(apiStr, "API for I/O [%s]", APIs); + + option_help o [] = { + {'a', NULL, apiStr, OPTION_OPTIONAL_ARGUMENT, 's', & params->api}, + {'A', NULL, "refNum -- user supplied reference number to include in the summary", OPTION_OPTIONAL_ARGUMENT, 'd', & params->referenceNumber}, + {'b', NULL, "blockSize -- contiguous bytes to write per task (e.g.: 8, 4k, 2m, 1g)", OPTION_OPTIONAL_ARGUMENT, 'l', & params->blockSize}, + {'c', NULL, "collective -- collective I/O", OPTION_FLAG, 'd', & params->collective}, + {'C', NULL, "reorderTasks -- changes task ordering to n+1 ordering for readback", OPTION_FLAG, 'd', & params->reorderTasks}, + {'d', NULL, "interTestDelay -- delay between reps in seconds", OPTION_OPTIONAL_ARGUMENT, 'd', & params->interTestDelay}, + {'D', NULL, "deadlineForStonewalling -- seconds before stopping write or read phase", OPTION_OPTIONAL_ARGUMENT, 'd', & params->deadlineForStonewalling}, + {.help=" -O stoneWallingWearOut=1 -- once the stonewalling timout is over, all process finish to access the amount of data", .arg = OPTION_OPTIONAL_ARGUMENT}, + {.help=" -O stoneWallingWearOutIterations=N -- stop after processing this number of iterations, needed for reading data back written with stoneWallingWearOut", .arg = OPTION_OPTIONAL_ARGUMENT}, + {.help=" -O stoneWallingStatusFile=FILE -- this file keeps the number of iterations from stonewalling during write and allows to use them for read", .arg = OPTION_OPTIONAL_ARGUMENT}, + {'e', NULL, "fsync -- perform sync operation after each block write", OPTION_FLAG, 'd', & params->fsync}, + {'E', NULL, "useExistingTestFile -- do not remove test file before write access", OPTION_FLAG, 'd', & params->useExistingTestFile}, + {'f', NULL, "scriptFile -- test script name", OPTION_OPTIONAL_ARGUMENT, 's', & params->testscripts}, + {'F', NULL, "filePerProc -- file-per-process", OPTION_FLAG, 'd', & params->filePerProc}, + {'g', NULL, "intraTestBarriers -- use barriers between open, write/read, and close", OPTION_FLAG, 'd', & params->intraTestBarriers}, + /* This option toggles between Incompressible Seed and Time stamp sig based on -l, + * so we'll toss the value in both for now, and sort it out in initialization + * after all the arguments are in and we know which it keep. + */ + {'G', NULL, "setTimeStampSignature -- set value for time stamp signature/random seed", OPTION_OPTIONAL_ARGUMENT, 'd', & params->setTimeStampSignature}, + {'H', NULL, "showHints -- show hints", OPTION_FLAG, 'd', & params->showHints}, + {'i', NULL, "repetitions -- number of repetitions of test", OPTION_OPTIONAL_ARGUMENT, 'd', & params->repetitions}, + {'I', NULL, "individualDataSets -- datasets not shared by all procs [not working]", OPTION_FLAG, 'd', & params->individualDataSets}, + {'j', NULL, "outlierThreshold -- warn on outlier N seconds from mean", OPTION_OPTIONAL_ARGUMENT, 'd', & params->outlierThreshold}, + {'J', NULL, "setAlignment -- HDF5 alignment in bytes (e.g.: 8, 4k, 2m, 1g)", OPTION_OPTIONAL_ARGUMENT, 'd', & params->setAlignment}, + {'k', NULL, "keepFile -- don't remove the test file(s) on program exit", OPTION_FLAG, 'd', & params->keepFile}, + {'K', NULL, "keepFileWithError -- keep error-filled file(s) after data-checking", OPTION_FLAG, 'd', & params->keepFileWithError}, + {'l', NULL, "datapacket type-- type of packet that will be created [offset|incompressible|timestamp|o|i|t]", OPTION_OPTIONAL_ARGUMENT, 's', & params->buffer_type}, + {'m', NULL, "multiFile -- use number of reps (-i) for multiple file count", OPTION_FLAG, 'd', & params->multiFile}, + {'M', NULL, "memoryPerNode -- hog memory on the node (e.g.: 2g, 75%)", OPTION_OPTIONAL_ARGUMENT, 's', & params->memoryPerNodeStr}, + {'n', NULL, "noFill -- no fill in HDF5 file creation", OPTION_FLAG, 'd', & params->noFill}, + {'N', NULL, "numTasks -- number of tasks that should participate in the test", OPTION_OPTIONAL_ARGUMENT, 'd', & params->numTasks}, + {'o', NULL, "testFile -- full name for test", OPTION_OPTIONAL_ARGUMENT, 's', & params->testFileName}, + {'O', NULL, "string of IOR directives (e.g. -O checkRead=1,lustreStripeCount=32)", OPTION_OPTIONAL_ARGUMENT, 'p', & decodeDirectiveWrapper}, + {'p', NULL, "preallocate -- preallocate file size", OPTION_FLAG, 'd', & params->preallocate}, + {'P', NULL, "useSharedFilePointer -- use shared file pointer [not working]", OPTION_FLAG, 'd', & params->useSharedFilePointer}, + {'q', NULL, "quitOnError -- during file error-checking, abort on error", OPTION_FLAG, 'd', & params->quitOnError}, + {'Q', NULL, "taskPerNodeOffset for read tests use with -C & -Z options (-C constant N, -Z at least N)", OPTION_OPTIONAL_ARGUMENT, 'd', & params->taskPerNodeOffset}, + {'r', NULL, "readFile -- read existing file", OPTION_FLAG, 'd', & params->readFile}, + {'R', NULL, "checkRead -- verify that the output of read matches the expected signature (used with -G)", OPTION_FLAG, 'd', & params->checkRead}, + {'s', NULL, "segmentCount -- number of segments", OPTION_OPTIONAL_ARGUMENT, 'd', & params->segmentCount}, + {'S', NULL, "useStridedDatatype -- put strided access into datatype [not working]", OPTION_FLAG, 'd', & params->useStridedDatatype}, + {'t', NULL, "transferSize -- size of transfer in bytes (e.g.: 8, 4k, 2m, 1g)", OPTION_OPTIONAL_ARGUMENT, 'l', & params->transferSize}, + {'T', NULL, "maxTimeDuration -- max time in minutes executing repeated test; it aborts only between iterations and not within a test!", OPTION_OPTIONAL_ARGUMENT, 'd', & params->maxTimeDuration}, + {'u', NULL, "uniqueDir -- use unique directory name for each file-per-process", OPTION_FLAG, 'd', & params->uniqueDir}, + {'U', NULL, "hintsFileName -- full name for hints file", OPTION_OPTIONAL_ARGUMENT, 's', & params->hintsFileName}, + {'v', NULL, "verbose -- output information (repeating flag increases level)", OPTION_FLAG, 'd', & params->verbose}, + {'V', NULL, "useFileView -- use MPI_File_set_view", OPTION_FLAG, 'd', & params->useFileView}, + {'w', NULL, "writeFile -- write file", OPTION_FLAG, 'd', & params->writeFile}, + {'W', NULL, "checkWrite -- check read after write", OPTION_FLAG, 'd', & params->checkWrite}, + {'x', NULL, "singleXferAttempt -- do not retry transfer if incomplete", OPTION_FLAG, 'd', & params->singleXferAttempt}, + {'X', NULL, "reorderTasksRandomSeed -- random seed for -Z option", OPTION_OPTIONAL_ARGUMENT, 'd', & params->reorderTasksRandomSeed}, + {'Y', NULL, "fsyncPerWrite -- perform sync operation after every write operation", OPTION_FLAG, 'd', & params->fsyncPerWrite}, + {'z', NULL, "randomOffset -- access is to random, not sequential, offsets within a file", OPTION_FLAG, 'd', & params->randomOffset}, + {'Z', NULL, "reorderTasksRandom -- changes task ordering to random ordering for readback", OPTION_FLAG, 'd', & params->reorderTasksRandom}, + {.help=" -O summaryFile=FILE -- store result data into this file", .arg = OPTION_OPTIONAL_ARGUMENT}, + {.help=" -O summaryFormat=[default,JSON,CSV] -- use the format for outputing the summary", .arg = OPTION_OPTIONAL_ARGUMENT}, + {0, "dryRun", "do not perform any I/Os just run evtl. inputs print dummy output", OPTION_FLAG, 'd', & params->dryRun}, + LAST_OPTION, + }; + option_help * options = malloc(sizeof(o)); + memcpy(options, & o, sizeof(o)); + return options; } + /* * Parse Commandline. */ IOR_test_t *ParseCommandLine(int argc, char **argv) { - char * testscripts = NULL; - int toggleG = FALSE; - char * buffer_type = ""; - char * memoryPerNode = NULL; init_IOR_Param_t(& initialTestParams); - parameters = & initialTestParams; - char APIs[1024]; - char APIs_legacy[1024]; - aiori_supported_apis(APIs, APIs_legacy, IOR); - char apiStr[1024]; - sprintf(apiStr, "API for I/O [%s]", APIs); - - option_help options [] = { - {'a', NULL, apiStr, OPTION_OPTIONAL_ARGUMENT, 's', & initialTestParams.api}, - {'A', NULL, "refNum -- user supplied reference number to include in the summary", OPTION_OPTIONAL_ARGUMENT, 'd', & initialTestParams.referenceNumber}, - {'b', NULL, "blockSize -- contiguous bytes to write per task (e.g.: 8, 4k, 2m, 1g)", OPTION_OPTIONAL_ARGUMENT, 'l', & initialTestParams.blockSize}, - {'B', NULL, "useO_DIRECT -- uses O_DIRECT for POSIX, bypassing I/O buffers", OPTION_FLAG, 'd', & initialTestParams.useO_DIRECT}, - {'c', NULL, "collective -- collective I/O", OPTION_FLAG, 'd', & initialTestParams.collective}, - {'C', NULL, "reorderTasks -- changes task ordering to n+1 ordering for readback", OPTION_FLAG, 'd', & initialTestParams.reorderTasks}, - {'d', NULL, "interTestDelay -- delay between reps in seconds", OPTION_OPTIONAL_ARGUMENT, 'd', & initialTestParams.interTestDelay}, - {'D', NULL, "deadlineForStonewalling -- seconds before stopping write or read phase", OPTION_OPTIONAL_ARGUMENT, 'd', & initialTestParams.deadlineForStonewalling}, - {.help=" -O stoneWallingWearOut=1 -- once the stonewalling timout is over, all process finish to access the amount of data", .arg = OPTION_OPTIONAL_ARGUMENT}, - {.help=" -O stoneWallingWearOutIterations=N -- stop after processing this number of iterations, needed for reading data back written with stoneWallingWearOut", .arg = OPTION_OPTIONAL_ARGUMENT}, - {.help=" -O stoneWallingStatusFile=FILE -- this file keeps the number of iterations from stonewalling during write and allows to use them for read", .arg = OPTION_OPTIONAL_ARGUMENT}, - {'e', NULL, "fsync -- perform sync operation after each block write", OPTION_FLAG, 'd', & initialTestParams.fsync}, - {'E', NULL, "useExistingTestFile -- do not remove test file before write access", OPTION_FLAG, 'd', & initialTestParams.useExistingTestFile}, - {'f', NULL, "scriptFile -- test script name", OPTION_OPTIONAL_ARGUMENT, 's', & testscripts}, - {'F', NULL, "filePerProc -- file-per-process", OPTION_FLAG, 'd', & initialTestParams.filePerProc}, - {'g', NULL, "intraTestBarriers -- use barriers between open, write/read, and close", OPTION_FLAG, 'd', & initialTestParams.intraTestBarriers}, - /* This option toggles between Incompressible Seed and Time stamp sig based on -l, - * so we'll toss the value in both for now, and sort it out in initialization - * after all the arguments are in and we know which it keep. - */ - {'G', NULL, "setTimeStampSignature -- set value for time stamp signature/random seed", OPTION_OPTIONAL_ARGUMENT, 'd', & toggleG}, - {'H', NULL, "showHints -- show hints", OPTION_FLAG, 'd', & initialTestParams.showHints}, - {'i', NULL, "repetitions -- number of repetitions of test", OPTION_OPTIONAL_ARGUMENT, 'd', & initialTestParams.repetitions}, - {'I', NULL, "individualDataSets -- datasets not shared by all procs [not working]", OPTION_FLAG, 'd', & initialTestParams.individualDataSets}, - {'j', NULL, "outlierThreshold -- warn on outlier N seconds from mean", OPTION_OPTIONAL_ARGUMENT, 'd', & initialTestParams.outlierThreshold}, - {'J', NULL, "setAlignment -- HDF5 alignment in bytes (e.g.: 8, 4k, 2m, 1g)", OPTION_OPTIONAL_ARGUMENT, 'd', & initialTestParams.setAlignment}, - {'k', NULL, "keepFile -- don't remove the test file(s) on program exit", OPTION_FLAG, 'd', & initialTestParams.keepFile}, - {'K', NULL, "keepFileWithError -- keep error-filled file(s) after data-checking", OPTION_FLAG, 'd', & initialTestParams.keepFileWithError}, - {'l', NULL, "datapacket type-- type of packet that will be created [offset|incompressible|timestamp|o|i|t]", OPTION_OPTIONAL_ARGUMENT, 's', & buffer_type}, - {'m', NULL, "multiFile -- use number of reps (-i) for multiple file count", OPTION_FLAG, 'd', & initialTestParams.multiFile}, - {'M', NULL, "memoryPerNode -- hog memory on the node (e.g.: 2g, 75%)", OPTION_OPTIONAL_ARGUMENT, 's', & memoryPerNode}, - {'n', NULL, "noFill -- no fill in HDF5 file creation", OPTION_FLAG, 'd', & initialTestParams.noFill}, - {'N', NULL, "numTasks -- number of tasks that should participate in the test", OPTION_OPTIONAL_ARGUMENT, 'd', & initialTestParams.numTasks}, - {'o', NULL, "testFile -- full name for test", OPTION_OPTIONAL_ARGUMENT, 's', & initialTestParams.testFileName}, - {'O', NULL, "string of IOR directives (e.g. -O checkRead=1,lustreStripeCount=32)", OPTION_OPTIONAL_ARGUMENT, 'p', & decodeDirectiveWrapper}, - {'p', NULL, "preallocate -- preallocate file size", OPTION_FLAG, 'd', & initialTestParams.preallocate}, - {'P', NULL, "useSharedFilePointer -- use shared file pointer [not working]", OPTION_FLAG, 'd', & initialTestParams.useSharedFilePointer}, - {'q', NULL, "quitOnError -- during file error-checking, abort on error", OPTION_FLAG, 'd', & initialTestParams.quitOnError}, - {'Q', NULL, "taskPerNodeOffset for read tests use with -C & -Z options (-C constant N, -Z at least N)", OPTION_OPTIONAL_ARGUMENT, 'd', & initialTestParams.taskPerNodeOffset}, - {'r', NULL, "readFile -- read existing file", OPTION_FLAG, 'd', & initialTestParams.readFile}, - {'R', NULL, "checkRead -- verify that the output of read matches the expected signature (used with -G)", OPTION_FLAG, 'd', & initialTestParams.checkRead}, - {'s', NULL, "segmentCount -- number of segments", OPTION_OPTIONAL_ARGUMENT, 'd', & initialTestParams.segmentCount}, - {'S', NULL, "useStridedDatatype -- put strided access into datatype [not working]", OPTION_FLAG, 'd', & initialTestParams.useStridedDatatype}, - {'t', NULL, "transferSize -- size of transfer in bytes (e.g.: 8, 4k, 2m, 1g)", OPTION_OPTIONAL_ARGUMENT, 'l', & initialTestParams.transferSize}, - {'T', NULL, "maxTimeDuration -- max time in minutes executing repeated test; it aborts only between iterations and not within a test!", OPTION_OPTIONAL_ARGUMENT, 'd', & initialTestParams.maxTimeDuration}, - {'u', NULL, "uniqueDir -- use unique directory name for each file-per-process", OPTION_FLAG, 'd', & initialTestParams.uniqueDir}, - {'U', NULL, "hintsFileName -- full name for hints file", OPTION_OPTIONAL_ARGUMENT, 's', & initialTestParams.hintsFileName}, - {'v', NULL, "verbose -- output information (repeating flag increases level)", OPTION_FLAG, 'd', & initialTestParams.verbose}, - {'V', NULL, "useFileView -- use MPI_File_set_view", OPTION_FLAG, 'd', & initialTestParams.useFileView}, - {'w', NULL, "writeFile -- write file", OPTION_FLAG, 'd', & initialTestParams.writeFile}, - {'W', NULL, "checkWrite -- check read after write", OPTION_FLAG, 'd', & initialTestParams.checkWrite}, - {'x', NULL, "singleXferAttempt -- do not retry transfer if incomplete", OPTION_FLAG, 'd', & initialTestParams.singleXferAttempt}, - {'X', NULL, "reorderTasksRandomSeed -- random seed for -Z option", OPTION_OPTIONAL_ARGUMENT, 'd', & initialTestParams.reorderTasksRandomSeed}, - {'Y', NULL, "fsyncPerWrite -- perform sync operation after every write operation", OPTION_FLAG, 'd', & initialTestParams.fsyncPerWrite}, - {'z', NULL, "randomOffset -- access is to random, not sequential, offsets within a file", OPTION_FLAG, 'd', & initialTestParams.randomOffset}, - {'Z', NULL, "reorderTasksRandom -- changes task ordering to random ordering for readback", OPTION_FLAG, 'd', & initialTestParams.reorderTasksRandom}, - {.help=" -O summaryFile=FILE -- store result data into this file", .arg = OPTION_OPTIONAL_ARGUMENT}, - {.help=" -O summaryFormat=[default,JSON,CSV] -- use the format for outputing the summary", .arg = OPTION_OPTIONAL_ARGUMENT}, - {0, "dryRun", "do not perform any I/Os just run evtl. inputs print dummy output", OPTION_FLAG, 'd', & initialTestParams.dryRun}, - LAST_OPTION, - }; - - IOR_test_t *tests = NULL; - - GetPlatformName(initialTestParams.platform); - airoi_parse_options(argc, argv, options); - - if (toggleG){ - initialTestParams.setTimeStampSignature = toggleG; - initialTestParams.incompressibleSeed = toggleG; - } + IOR_test_t *tests = NULL; - if (buffer_type[0] != 0){ - switch(buffer_type[0]) { - case 'i': /* Incompressible */ - initialTestParams.dataPacketType = incompressible; - break; - case 't': /* timestamp */ - initialTestParams.dataPacketType = timestamp; - break; - case 'o': /* offset packet */ - initialTestParams.storeFileOffset = TRUE; - initialTestParams.dataPacketType = offset; - break; - default: - fprintf(out_logfile, - "Unknown arguement for -l %s; generic assumed\n", buffer_type); - break; - } - } - if (memoryPerNode){ - initialTestParams.memoryPerNode = NodeMemoryStringToBytes(memoryPerNode); - } - const ior_aiori_t * backend = aiori_select(initialTestParams.api); - if (backend == NULL) - ERR_SIMPLE("unrecognized I/O API"); - - initialTestParams.backend = backend; - initialTestParams.apiVersion = backend->get_version(); - - if (testscripts){ - tests = ReadConfigScript(testscripts); - }else{ - tests = CreateTest(&initialTestParams, 0); - AllocResults(tests); - } + GetPlatformName(initialTestParams.platform); + + option_help * options = createGlobalOptions( & initialTestParams); + parameters = & initialTestParams; + global_options = airoi_create_all_module_options(options); + option_parse(argc, argv, global_options); + updateParsedOptions(& initialTestParams, global_options); + + if (initialTestParams.testscripts){ + tests = ReadConfigScript(initialTestParams.testscripts); + }else{ + tests = CreateTest(&initialTestParams, 0); + AllocResults(tests); + } - CheckRunSettings(tests); + CheckRunSettings(tests); - return (tests); + return (tests); } diff --git a/src/utilities.c b/src/utilities.c index b71ce41d..cc0918c3 100755 --- a/src/utilities.c +++ b/src/utilities.c @@ -66,6 +66,83 @@ enum OutputFormat_t outputFormat; /***************************** F U N C T I O N S ******************************/ +void* safeMalloc(uint64_t size){ + void * d = malloc(size); + if (d == NULL){ + ERR("Could not malloc an array"); + } + memset(d, 0, size); + return d; +} + + +size_t NodeMemoryStringToBytes(char *size_str) +{ + int percent; + int rc; + long page_size; + long num_pages; + long long mem; + + rc = sscanf(size_str, " %d %% ", &percent); + if (rc == 0) + return (size_t) string_to_bytes(size_str); + if (percent > 100 || percent < 0) + ERR("percentage must be between 0 and 100"); + +#ifdef HAVE_SYSCONF + page_size = sysconf(_SC_PAGESIZE); +#else + page_size = getpagesize(); +#endif + +#ifdef _SC_PHYS_PAGES + num_pages = sysconf(_SC_PHYS_PAGES); + if (num_pages == -1) + ERR("sysconf(_SC_PHYS_PAGES) is not supported"); +#else + ERR("sysconf(_SC_PHYS_PAGES) is not supported"); +#endif + mem = page_size * num_pages; + + return mem / 100 * percent; +} + +void updateParsedOptions(IOR_param_t * options, options_all_t * global_options){ + if (options->setTimeStampSignature){ + options->incompressibleSeed = options->setTimeStampSignature; + } + + if (options->buffer_type && options->buffer_type[0] != 0){ + switch(options->buffer_type[0]) { + case 'i': /* Incompressible */ + options->dataPacketType = incompressible; + break; + case 't': /* timestamp */ + options->dataPacketType = timestamp; + break; + case 'o': /* offset packet */ + options->storeFileOffset = TRUE; + options->dataPacketType = offset; + break; + default: + fprintf(out_logfile, + "Unknown argument for -l %s; generic assumed\n", options->buffer_type); + break; + } + } + if (options->memoryPerNodeStr){ + options->memoryPerNode = NodeMemoryStringToBytes(options->memoryPerNodeStr); + } + const ior_aiori_t * backend = aiori_select(options->api); + if (backend == NULL) + ERR_SIMPLE("unrecognized I/O API"); + + options->backend = backend; + /* copy the actual module options into the test */ + options->backend_options = airoi_update_module_options(backend, global_options); + options->apiVersion = backend->get_version(); +} /* Used in aiori-POSIX.c and aiori-PLFS.c */ diff --git a/src/utilities.h b/src/utilities.h index 7933cbc4..b6461744 100755 --- a/src/utilities.h +++ b/src/utilities.h @@ -53,6 +53,7 @@ extern enum OutputFormat_t outputFormat; /* format of the output */ } while(0) #endif +void* safeMalloc(uint64_t size); void set_o_direct_flag(int *fd); char *CurrentTimeString(void); @@ -65,6 +66,8 @@ void ShowHints (MPI_Info *); char *HumanReadable(IOR_offset_t value, int base); int CountTasksPerNode(MPI_Comm comm); void DelaySecs(int delay); +void updateParsedOptions(IOR_param_t * options, options_all_t * global_options); +size_t NodeMemoryStringToBytes(char *size_str); /* Returns -1, if cannot be read */ int64_t ReadStoneWallingIterations(char * const filename); From 07ca5247d7255d50fa2ec9730e56cf3e4df81799 Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Wed, 27 Mar 2019 20:37:46 +0000 Subject: [PATCH 064/206] Checked proper IOR behavior for -f option. --- src/aiori-POSIX.c | 5 +++-- src/option.c | 16 +++++----------- src/parse_options.c | 9 ++++++--- testing/test_comments.ior | 2 ++ 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/aiori-POSIX.c b/src/aiori-POSIX.c index 1d41c640..a4464592 100755 --- a/src/aiori-POSIX.c +++ b/src/aiori-POSIX.c @@ -304,8 +304,9 @@ void *POSIX_Create(char *testFileName, IOR_param_t * param) if (fd == NULL) ERR("Unable to malloc file descriptor"); posix_options_t * o = (posix_options_t*) param->backend_options; - if (o->direct_io == TRUE) - set_o_direct_flag(&fd_oflag); + if (o->direct_io == TRUE){ + set_o_direct_flag(&fd_oflag); + } if(param->dryRun) return 0; diff --git a/src/option.c b/src/option.c index 08833dac..f940cee1 100644 --- a/src/option.c +++ b/src/option.c @@ -221,7 +221,6 @@ void option_print_current(option_help * args){ } static void option_parse_token(char ** argv, int * flag_parsed_next, int * requiredArgsSeen, options_all_t * opt_all, int * error, int * print_help){ - int foundOption = 0; char * txt = argv[0]; char * arg = strstr(txt, "="); int replaced_equal = 0; @@ -243,8 +242,6 @@ static void option_parse_token(char ** argv, int * flag_parsed_next, int * requi continue; } if ( (txt[0] == '-' && o->shortVar == txt[1]) || (strlen(txt) > 2 && txt[0] == '-' && txt[1] == '-' && o->longVar != NULL && strcmp(txt + 2, o->longVar) == 0)){ - foundOption = 1; - // now process the option. switch(o->arg){ case (OPTION_FLAG):{ @@ -324,18 +321,15 @@ static void option_parse_token(char ** argv, int * flag_parsed_next, int * requi (*requiredArgsSeen)++; } - break; + return; } } } - - if (! foundOption){ - if(strcmp(txt, "-h") == 0 || strcmp(txt, "--help") == 0){ - *print_help = 1; - }else{ - *error = 1; - } + if(strcmp(txt, "-h") == 0 || strcmp(txt, "--help") == 0){ + *print_help = 1; + }else{ + *error = 1; } } diff --git a/src/parse_options.c b/src/parse_options.c index f92e2399..dd6e68bb 100755 --- a/src/parse_options.c +++ b/src/parse_options.c @@ -111,8 +111,6 @@ void DecodeDirective(char *line, IOR_param_t *params, options_all_t * module_opt fprintf(out_logfile, "Could not load backend API %s\n", params->api); exit(-1); } - /* copy the actual module options into the test */ - params->backend_options = airoi_update_module_options(params->backend, global_options); } else if (strcasecmp(option, "summaryFile") == 0) { if (rank == 0){ out_resultfile = fopen(value, "w"); @@ -411,6 +409,8 @@ IOR_test_t *ReadConfigScript(char *scriptName) create duplicate test */ tail->next = CreateTest(&tail->params, test_num++); AllocResults(tail); + ((IOR_test_t*) tail)->params.backend_options = airoi_update_module_options(((IOR_test_t*) tail)->params.backend, global_options); + tail = tail->next; *option_p = createGlobalOptions(& ((IOR_test_t*) tail->next)->params); } @@ -422,6 +422,8 @@ IOR_test_t *ReadConfigScript(char *scriptName) tail->next = CreateTest(&tail->params, test_num++); *option_p = createGlobalOptions(& ((IOR_test_t*) tail->next)->params); AllocResults(tail); + ((IOR_test_t*) tail)->params.backend_options = airoi_update_module_options(((IOR_test_t*) tail)->params.backend, global_options); + tail = tail->next; ParseLine(ptr, &tail->params, global_options); } else { @@ -432,7 +434,8 @@ IOR_test_t *ReadConfigScript(char *scriptName) /* close the script */ if (fclose(file) != 0) ERR("fclose() of script file failed"); - AllocResults(tail); + AllocResults(tail); /* copy the actual module options into the test */ + ((IOR_test_t*) tail)->params.backend_options = airoi_update_module_options(((IOR_test_t*) tail)->params.backend, global_options); return head; } diff --git a/testing/test_comments.ior b/testing/test_comments.ior index b48b36fe..bd2efa89 100644 --- a/testing/test_comments.ior +++ b/testing/test_comments.ior @@ -14,4 +14,6 @@ transferSize=100k blockSize=100k # space-prefixed comment run +--dummy.delay-create=1000 +api=dummy ior stop From b4fb80a69e33af50ee6306e2df7d8a48f581c4af Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Wed, 27 Mar 2019 22:32:38 +0000 Subject: [PATCH 065/206] Allowed option parser to reset flags (needed in scriptfiles). --- src/option.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/option.c b/src/option.c index f940cee1..d5b76c3c 100644 --- a/src/option.c +++ b/src/option.c @@ -246,7 +246,12 @@ static void option_parse_token(char ** argv, int * flag_parsed_next, int * requi switch(o->arg){ case (OPTION_FLAG):{ assert(o->type == 'd'); - (*(int*) o->variable)++; + if(arg != NULL){ + int val = atoi(arg); + (*(int*) o->variable) = (val < 0) ? 0 : val; + }else{ + (*(int*) o->variable)++; + } break; } case (OPTION_OPTIONAL_ARGUMENT): From 21405ed92434b9687235e6a01083b434a101a4af Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Wed, 27 Mar 2019 22:32:59 +0000 Subject: [PATCH 066/206] Updated MMap module to option parser. --- src/aiori-MMAP.c | 65 +++++++++++++++++++++++++++++++++++++---------- src/aiori-POSIX.c | 5 ++-- src/aiori.h | 2 ++ src/ior.c | 2 -- src/ior.h | 2 -- 5 files changed, 57 insertions(+), 19 deletions(-) diff --git a/src/aiori-MMAP.c b/src/aiori-MMAP.c index f812bddd..dd6c6490 100644 --- a/src/aiori-MMAP.c +++ b/src/aiori-MMAP.c @@ -32,6 +32,7 @@ static IOR_offset_t MMAP_Xfer(int, void *, IOR_size_t *, IOR_offset_t, IOR_param_t *); static void MMAP_Close(void *, IOR_param_t *); static void MMAP_Fsync(void *, IOR_param_t *); +static option_help * MMAP_options(void ** init_backend_options, void * init_values); /************************** D E C L A R A T I O N S ***************************/ @@ -45,9 +46,38 @@ ior_aiori_t mmap_aiori = { .get_version = aiori_get_version, .fsync = MMAP_Fsync, .get_file_size = POSIX_GetFileSize, + .get_options = MMAP_options, }; /***************************** F U N C T I O N S ******************************/ +typedef struct{ + int direct_io_ignored; /* this option is ignored */ + void* mmap_ptr; /* for internal usage */ + + int madv_dont_need; + int madv_pattern; +} mmap_options_t; + +static option_help * MMAP_options(void ** init_backend_options, void * init_values){ + mmap_options_t * o = malloc(sizeof(mmap_options_t)); + + if (init_values != NULL){ + memcpy(o, init_values, sizeof(mmap_options_t)); + }else{ + o->direct_io_ignored = 0; + } + + *init_backend_options = o; + + option_help h [] = { + {0, "mmap.madv_dont_need", "Use advise don't need", OPTION_FLAG, 'd', & o->madv_dont_need}, + {0, "mmap.madv_pattern", "Use advise to indicate the pattern random/sequential", OPTION_FLAG, 'd', & o->madv_pattern}, + LAST_OPTION + }; + option_help * help = malloc(sizeof(h)); + memcpy(help, h, sizeof(h)); + return help; +} static void ior_mmap_file(int *file, IOR_param_t *param) { @@ -56,21 +86,27 @@ static void ior_mmap_file(int *file, IOR_param_t *param) if (param->open == WRITE) flags |= PROT_WRITE; + mmap_options_t *o = (mmap_options_t*) param->backend_options; - param->mmap_ptr = mmap(NULL, size, flags, MAP_SHARED, + o->mmap_ptr = mmap(NULL, size, flags, MAP_SHARED, *file, 0); - if (param->mmap_ptr == MAP_FAILED) + if (o->mmap_ptr == MAP_FAILED) ERR("mmap() failed"); if (param->randomOffset) flags = POSIX_MADV_RANDOM; else flags = POSIX_MADV_SEQUENTIAL; - if (posix_madvise(param->mmap_ptr, size, flags) != 0) - ERR("madvise() failed"); + + if(o->madv_pattern){ + if (posix_madvise(o->mmap_ptr, size, flags) != 0) + ERR("madvise() failed"); + } - if (posix_madvise(param->mmap_ptr, size, POSIX_MADV_DONTNEED) != 0) - ERR("madvise() failed"); + if (o->madv_dont_need){ + if (posix_madvise(o->mmap_ptr, size, POSIX_MADV_DONTNEED) != 0) + ERR("madvise() failed"); + } return; } @@ -107,16 +143,17 @@ static void *MMAP_Open(char *testFileName, IOR_param_t * param) static IOR_offset_t MMAP_Xfer(int access, void *file, IOR_size_t * buffer, IOR_offset_t length, IOR_param_t * param) { + mmap_options_t *o = (mmap_options_t*) param->backend_options; if (access == WRITE) { - memcpy(param->mmap_ptr + param->offset, buffer, length); + memcpy(o->mmap_ptr + param->offset, buffer, length); } else { - memcpy(buffer, param->mmap_ptr + param->offset, length); + memcpy(buffer, o->mmap_ptr + param->offset, length); } if (param->fsyncPerWrite == TRUE) { - if (msync(param->mmap_ptr + param->offset, length, MS_SYNC) != 0) + if (msync(o->mmap_ptr + param->offset, length, MS_SYNC) != 0) ERR("msync() failed"); - if (posix_madvise(param->mmap_ptr + param->offset, length, + if (posix_madvise(o->mmap_ptr + param->offset, length, POSIX_MADV_DONTNEED) != 0) ERR("madvise() failed"); } @@ -128,7 +165,8 @@ static IOR_offset_t MMAP_Xfer(int access, void *file, IOR_size_t * buffer, */ static void MMAP_Fsync(void *fd, IOR_param_t * param) { - if (msync(param->mmap_ptr, param->expectedAggFileSize, MS_SYNC) != 0) + mmap_options_t *o = (mmap_options_t*) param->backend_options; + if (msync(o->mmap_ptr, param->expectedAggFileSize, MS_SYNC) != 0) EWARN("msync() failed"); } @@ -137,8 +175,9 @@ static void MMAP_Fsync(void *fd, IOR_param_t * param) */ static void MMAP_Close(void *fd, IOR_param_t * param) { - if (munmap(param->mmap_ptr, param->expectedAggFileSize) != 0) + mmap_options_t *o = (mmap_options_t*) param->backend_options; + if (munmap(o->mmap_ptr, param->expectedAggFileSize) != 0) ERR("munmap failed"); - param->mmap_ptr = NULL; + o->mmap_ptr = NULL; POSIX_Close(fd, param); } diff --git a/src/aiori-POSIX.c b/src/aiori-POSIX.c index a4464592..9e85fa79 100755 --- a/src/aiori-POSIX.c +++ b/src/aiori-POSIX.c @@ -74,11 +74,12 @@ static void POSIX_Fsync(void *, IOR_param_t *); /************************** O P T I O N S *****************************/ typedef struct{ + /* in case of a change, please update depending MMAP module too */ int direct_io; } posix_options_t; -static option_help * posix_options(void ** init_backend_options, void * init_values){ +option_help * POSIX_options(void ** init_backend_options, void * init_values){ posix_options_t * o = malloc(sizeof(posix_options_t)); if (init_values != NULL){ @@ -118,7 +119,7 @@ ior_aiori_t posix_aiori = { .rmdir = aiori_posix_rmdir, .access = aiori_posix_access, .stat = aiori_posix_stat, - .get_options = posix_options, + .get_options = POSIX_options, .enable_mdtest = true, }; diff --git a/src/aiori.h b/src/aiori.h index 782d1ac4..d8cabb5d 100755 --- a/src/aiori.h +++ b/src/aiori.h @@ -129,6 +129,8 @@ void *POSIX_Open(char *testFileName, IOR_param_t * param); IOR_offset_t POSIX_GetFileSize(IOR_param_t * test, MPI_Comm testComm, char *testFileName); void POSIX_Delete(char *testFileName, IOR_param_t * param); void POSIX_Close(void *fd, IOR_param_t * param); +option_help * POSIX_options(void ** init_backend_options, void * init_values); + /* NOTE: these 3 MPI-IO functions are exported for reuse by HDF5/PNetCDF */ void MPIIO_Delete(char *testFileName, IOR_param_t * param); diff --git a/src/ior.c b/src/ior.c index a1dc321a..3ef411cb 100755 --- a/src/ior.c +++ b/src/ior.c @@ -218,8 +218,6 @@ void init_IOR_Param_t(IOR_param_t * p) p->beegfs_numTargets = -1; p->beegfs_chunkSize = -1; - - p->mmap_ptr = NULL; } static void diff --git a/src/ior.h b/src/ior.h index acf3a7db..da4e5c18 100755 --- a/src/ior.h +++ b/src/ior.h @@ -162,8 +162,6 @@ typedef struct int fsyncPerWrite; /* fsync() after each write */ int fsync; /* fsync() after write */ - void* mmap_ptr; - /* MPI variables */ MPI_Comm testComm; /* MPI communicator */ MPI_Datatype transferType; /* datatype for transfer */ From 4a788a0c2374e8cff3404eeee5114b19e6ce9e0e Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Sat, 30 Mar 2019 19:11:44 +0000 Subject: [PATCH 067/206] re-write the DAOS IOR driver to use the Array API Signed-off-by: Mohamad Chaarawi --- README_DAOS | 50 +-- configure.ac | 15 +- src/aiori-DAOS.c | 822 ++++++++++++++++------------------------------- 3 files changed, 311 insertions(+), 576 deletions(-) diff --git a/README_DAOS b/README_DAOS index eea57a8f..33d94648 100644 --- a/README_DAOS +++ b/README_DAOS @@ -4,14 +4,16 @@ Building The DAOS library must be installed on the system. ./bootsrap -./configure --prefix=iorInstallDir --with-daos=DIR +./configure --prefix=iorInstallDir --with-daos=DIR --with-cart=DIR -One must specify "--with-daos=/path/to/daos/install". When that is specified -the DAOS and DFS driver will be built. +One must specify "--with-daos=/path/to/daos/install and --with-cart". When that +is specified the DAOS and DFS driver will be built. -The DAOS driver uses the DAOS API to create a container (file) and update/fetch -an object to the container using the daos object API. The DAOS driver works with -IOR only (no mdtest support yet). +The DAOS driver uses the DAOS API to open a container (or create it if it +doesn't exist first) then create an array object in that container (file) and +read/write to the array object using the daos Array API. The DAOS driver works +with IOR only (no mdtest support yet). The file name used by IOR (passed by -o +option) is hashed to an object ID that is used as the array oid. The DFS (DAOS File System) driver creates an encapsulated namespace and emulates the POSIX driver using the DFS API directly on top of DAOS. The DFS driver works @@ -33,40 +35,22 @@ The DAOS options include: Required Options: --daos.pool : pool uuid to connect to (has to be created beforehand) --daos.svcl : pool svcl list (: separated) +--daos.cont : container for the IOR files/objects (can use `uuidgen`) Optional Options: --daos.group : group name of servers with the pool ---daos.recordSize : object record size for IO ---daos.stripeSize ---daos.stripeCount ---daos.stripeMax ---daos.aios : number of concurrent async IOs ---daos.kill flag to kill a rank during IO ---daos.objectClass : specific object class - -When benchmarking write performance, one likely does not want "-W", which causes -the write phase to do one additional memory copy for every I/O. This is due to -IOR's assumption that when a DAOS_Xfer() call returns the buffer may be -released. Therefore, random data is written when "-W" is absent, while data is -copied from IOR buffers when "-W" is present. - -See doc/USER_GUIDE for all options and directives. Note that not all -combinations of options are supported. For example specifying an epoch to access -and running ior with multiple iterations would cause all iterations other than -first one to fail because the epoch will be committed in the first iteration. In -that case, the epoch should not be specified and the DAOS driver would choose -the epoch to access automatically on each iteration. +--daos.chunk_size : Chunk size of the array object controlling striping over DKEYs +--daos.destroy flag to destory the container on finalize +--daos.oclass : specific object class for array object Examples that should work include: - - "ior -a DAOS -w -W -o --daos.pool --daos.svcl " - writes into a new container and verifies the data, using default - daosRecordSize, transferSize, daosStripeSize, blockSize, daosAios, etc. + - "ior -a DAOS -w -W -o file_name --daos.pool --daos.svcl \ + --daos.cont " - - "ior -a DAOS -w -W -r -R -o -b 1g -t 4m -C \ - --daos.pool --daos.svcl --daos.recordSize 1m --daos.stripeSize 4m\ - --daos.stripeCount 256 --daos.aios 8 - does all IOR tests and shifts ranks during checkWrite and checkRead. + - "ior -a DAOS -w -W -r -R -o file_name -b 1g -t 4m \ + --daos.pool --daos.svcl --daos.cont \ + --daos.chunk_size 1024 --daos.oclass R2" Running with DFS API --------------------- diff --git a/configure.ac b/configure.ac index 395c0ab6..d38b2a33 100755 --- a/configure.ac +++ b/configure.ac @@ -185,7 +185,20 @@ AM_COND_IF([USE_RADOS_AIORI],[ AC_DEFINE([USE_RADOS_AIORI], [], [Build RADOS backend AIORI]) ]) -# DAOS Backends (DAOS and DFS) IO support +# DAOS Backends (DAOS and DFS) IO support require DAOS and CART/GURT +AC_ARG_WITH([cart], + [AS_HELP_STRING([--with-cart], + [support IO with DAOS backends @<:@default=no@:>@])], + [], + [with_daos=no]) + +AS_IF([test "x$with_cart" != xno], + CART="yes" + LDFLAGS="$LDFLAGS -L$with_cart/lib" + CPPFLAGS="$CPPFLAGS -I$with_cart/include/" + AC_CHECK_HEADERS(gurt/common.h,, [unset CART]) + AC_CHECK_LIB([gurt], [d_hash_murmur64],, [unset CART])) + AC_ARG_WITH([daos], [AS_HELP_STRING([--with-daos], [support IO with DAOS backends @<:@default=no@:>@])], diff --git a/src/aiori-DAOS.c b/src/aiori-DAOS.c index 1555c57a..3e7f9c7c 100644 --- a/src/aiori-DAOS.c +++ b/src/aiori-DAOS.c @@ -28,8 +28,10 @@ #include #include #include +#include #include #include +#include #include "ior.h" #include "aiori.h" @@ -38,42 +40,33 @@ /************************** O P T I O N S *****************************/ struct daos_options{ - char *daosPool; - char *daosPoolSvc; - char *daosGroup; - int daosRecordSize; - int daosStripeSize; - uint64_t daosStripeCount; - uint64_t daosStripeMax; /* max length of a stripe */ - int daosAios; /* max number of concurrent async I/Os */ - int daosKill; /* kill a target while running IOR */ - char *daosObjectClass; /* object class */ + char *pool; + char *svcl; + char *group; + char *cont; + int chunk_size; + int destroy; + char *oclass; }; static struct daos_options o = { - .daosPool = NULL, - .daosPoolSvc = NULL, - .daosGroup = NULL, - .daosRecordSize = 262144, - .daosStripeSize = 524288, - .daosStripeCount = -1, - .daosStripeMax = 0, - .daosAios = 1, - .daosKill = 0, - .daosObjectClass = NULL, + .pool = NULL, + .svcl = NULL, + .group = NULL, + .cont = NULL, + .chunk_size = 1048576, + .destroy = 0, + .oclass = NULL, }; static option_help options [] = { - {0, "daos.pool", "pool uuid", OPTION_REQUIRED_ARGUMENT, 's', &o.daosPool}, - {0, "daos.svcl", "pool SVCL", OPTION_REQUIRED_ARGUMENT, 's', &o.daosPoolSvc}, - {0, "daos.group", "server group", OPTION_OPTIONAL_ARGUMENT, 's', &o.daosGroup}, - {0, "daos.recordSize", "Record Size", OPTION_OPTIONAL_ARGUMENT, 'd', &o.daosRecordSize}, - {0, "daos.stripeSize", "Stripe Size", OPTION_OPTIONAL_ARGUMENT, 'd', &o.daosStripeSize}, - {0, "daos.stripeCount", "Stripe Count", OPTION_OPTIONAL_ARGUMENT, 'u', &o.daosStripeCount}, - {0, "daos.stripeMax", "Max Stripe",OPTION_OPTIONAL_ARGUMENT, 'u', &o.daosStripeMax}, - {0, "daos.aios", "Concurrent Async IOs",OPTION_OPTIONAL_ARGUMENT, 'd', &o.daosAios}, - {0, "daos.kill", "Kill target while running",OPTION_FLAG, 'd', &o.daosKill}, - {0, "daos.objectClass", "object class", OPTION_OPTIONAL_ARGUMENT, 's', &o.daosObjectClass}, + {0, "daos.pool", "pool uuid", OPTION_REQUIRED_ARGUMENT, 's', &o.pool}, + {0, "daos.svcl", "pool SVCL", OPTION_REQUIRED_ARGUMENT, 's', &o.svcl}, + {0, "daos.group", "server group", OPTION_OPTIONAL_ARGUMENT, 's', &o.group}, + {0, "daos.cont", "container uuid", OPTION_REQUIRED_ARGUMENT, 's', &o.cont}, + {0, "daos.chunk_size", "chunk size", OPTION_OPTIONAL_ARGUMENT, 'd', &o.chunk_size}, + {0, "daos.destroy", "Destroy Container", OPTION_FLAG, 'd', &o.destroy}, + {0, "daos.oclass", "object class", OPTION_OPTIONAL_ARGUMENT, 's', &o.oclass}, LAST_OPTION }; @@ -111,40 +104,19 @@ ior_aiori_t daos_aiori = { .get_options = DAOS_options, }; +#define IOR_DAOS_MUR_SEED 0xDEAD10CC + enum handleType { POOL_HANDLE, - CONTAINER_HANDLE -}; - -struct fileDescriptor { - daos_handle_t container; - daos_cont_info_t containerInfo; - daos_handle_t object; -}; - -struct aio { - cfs_list_t a_list; - char a_dkeyBuf[32]; - daos_key_t a_dkey; - daos_recx_t a_recx; - unsigned char a_csumBuf[32]; - daos_csum_buf_t a_csum; - daos_iod_t a_iod; - daos_iov_t a_iov; - daos_sg_list_t a_sgl; - struct daos_event a_event; + CONT_HANDLE, + ARRAY_HANDLE }; -static daos_handle_t eventQueue; -static struct daos_event **events; -static unsigned char *buffers; -static int nAios; -static daos_handle_t pool; -static daos_pool_info_t poolInfo; -static daos_oclass_id_t objectClass = DAOS_OC_LARGE_RW; -static CFS_LIST_HEAD(aios); -static IOR_offset_t total_size; -static bool daos_initialized = false; +static daos_handle_t poh; +static daos_handle_t coh; +static daos_handle_t aoh; +static daos_oclass_id_t objectClass = DAOS_OC_LARGE_RW; +static bool daos_initialized = false; /***************************** F U N C T I O N S ******************************/ @@ -176,13 +148,12 @@ do { \ } while (0) /* Distribute process 0's pool or container handle to others. */ -static void HandleDistribute(daos_handle_t *handle, enum handleType type) +static void +HandleDistribute(daos_handle_t *handle, enum handleType type) { daos_iov_t global; int rc; - assert(type == POOL_HANDLE || !daos_handle_is_inval(pool)); - global.iov_buf = NULL; global.iov_buf_len = 0; global.iov_len = 0; @@ -191,8 +162,10 @@ static void HandleDistribute(daos_handle_t *handle, enum handleType type) /* Get the global handle size. */ if (type == POOL_HANDLE) rc = daos_pool_local2global(*handle, &global); - else + else if (type == CONT_HANDLE) rc = daos_cont_local2global(*handle, &global); + else + rc = daos_array_local2global(*handle, &global); DCHECK(rc, "Failed to get global handle size"); } @@ -207,8 +180,10 @@ static void HandleDistribute(daos_handle_t *handle, enum handleType type) if (rank == 0) { if (type == POOL_HANDLE) rc = daos_pool_local2global(*handle, &global); - else + else if (type == CONT_HANDLE) rc = daos_cont_local2global(*handle, &global); + else + rc = daos_array_local2global(*handle, &global); DCHECK(rc, "Failed to create global handle"); } @@ -222,213 +197,18 @@ static void HandleDistribute(daos_handle_t *handle, enum handleType type) if (type == POOL_HANDLE) rc = daos_pool_global2local(global, handle); - else - rc = daos_cont_global2local(pool, global, handle); + else if (type == CONT_HANDLE) + rc = daos_cont_global2local(poh, global, handle); + else + rc = daos_array_global2local(coh, global, handle); DCHECK(rc, "Failed to get local handle"); } free(global.iov_buf); } -static void ContainerOpen(char *testFileName, IOR_param_t *param, - daos_handle_t *container, daos_cont_info_t *info) -{ - int rc; - - if (rank == 0) { - uuid_t uuid; - unsigned int dFlags; - - rc = uuid_parse(testFileName, uuid); - DCHECK(rc, "Failed to parse 'testFile': %s", testFileName); - - if (param->open == WRITE && - param->useExistingTestFile == FALSE) { - INFO(VERBOSE_2, "Creating container %s", testFileName); - - rc = daos_cont_create(pool, uuid, NULL, NULL); - DCHECK(rc, "Failed to create container %s", - testFileName); - } - - INFO(VERBOSE_2, "Opening container %s", testFileName); - - if (param->open == WRITE) - dFlags = DAOS_COO_RW; - else - dFlags = DAOS_COO_RO; - - rc = daos_cont_open(pool, uuid, dFlags, container, info, - NULL /* ev */); - DCHECK(rc, "Failed to open container %s", testFileName); - } - - HandleDistribute(container, CONTAINER_HANDLE); - - MPI_CHECK(MPI_Bcast(info, sizeof *info, MPI_BYTE, 0, param->testComm), - "Failed to broadcast container info"); -} - -static void ContainerClose(daos_handle_t container, IOR_param_t *param) -{ - int rc; - - if (rank != 0) { - rc = daos_cont_close(container, NULL /* ev */); - DCHECK(rc, "Failed to close container"); - } - - /* An MPI_Gather() call would probably be more efficient. */ - MPI_CHECK(MPI_Barrier(param->testComm), - "Failed to synchronize processes"); - - if (rank == 0) { - rc = daos_cont_close(container, NULL /* ev */); - DCHECK(rc, "Failed to close container"); - } -} - -static void ObjectOpen(daos_handle_t container, daos_handle_t *object, - IOR_param_t *param) -{ - daos_obj_id_t oid; - unsigned int flags; - int rc; - - oid.hi = 0; - oid.lo = 1; - daos_obj_generate_id(&oid, 0, objectClass); - - if (param->open == WRITE) - flags = DAOS_OO_RW; - else - flags = DAOS_OO_RO; - - rc = daos_obj_open(container, oid, flags, object, NULL /* ev */); - DCHECK(rc, "Failed to open object"); -} - -static void ObjectClose(daos_handle_t object) -{ - int rc; - - rc = daos_obj_close(object, NULL /* ev */); - DCHECK(rc, "Failed to close object"); -} - -static void AIOInit(IOR_param_t *param) -{ - struct aio *aio; - int i; - int rc; - - rc = posix_memalign((void **) &buffers, sysconf(_SC_PAGESIZE), - param->transferSize * o.daosAios); - DCHECK(rc, "Failed to allocate buffer array"); - - for (i = 0; i < o.daosAios; i++) { - aio = malloc(sizeof *aio); - if (aio == NULL) - ERR("Failed to allocate aio array"); - - memset(aio, 0, sizeof *aio); - - aio->a_dkey.iov_buf = aio->a_dkeyBuf; - aio->a_dkey.iov_buf_len = sizeof aio->a_dkeyBuf; - - aio->a_recx.rx_nr = 1; - - daos_iov_set(&aio->a_iod.iod_name, "data", strlen("data")); - daos_csum_set(&aio->a_iod.iod_kcsum, NULL, 0); - aio->a_iod.iod_nr = 1; - aio->a_iod.iod_type = DAOS_IOD_ARRAY; - aio->a_iod.iod_recxs = &aio->a_recx; - aio->a_iod.iod_csums = NULL; - aio->a_iod.iod_eprs = NULL; - aio->a_iod.iod_size = param->transferSize; - - aio->a_iov.iov_buf = buffers + param->transferSize * i; - aio->a_iov.iov_buf_len = param->transferSize; - aio->a_iov.iov_len = aio->a_iov.iov_buf_len; - - aio->a_sgl.sg_nr = 1; - aio->a_sgl.sg_iovs = &aio->a_iov; - - rc = daos_event_init(&aio->a_event, eventQueue, - NULL /* parent */); - DCHECK(rc, "Failed to initialize event for aio[%d]", i); - - cfs_list_add(&aio->a_list, &aios); - - INFO(VERBOSE_3, "Allocated AIO %p: buffer %p", aio, - aio->a_iov.iov_buf); - } - - nAios = o.daosAios; - - events = malloc((sizeof *events) * o.daosAios); - if (events == NULL) - ERR("Failed to allocate events array"); -} - -static void AIOFini(IOR_param_t *param) -{ - struct aio *aio; - struct aio *tmp; - - free(events); - - cfs_list_for_each_entry_safe(aio, tmp, &aios, a_list) { - INFO(VERBOSE_3, "Freeing AIO %p: buffer %p", aio, - aio->a_iov.iov_buf); - cfs_list_del_init(&aio->a_list); - daos_event_fini(&aio->a_event); - free(aio); - } - - free(buffers); -} - -static void AIOWait(IOR_param_t *param) -{ - struct aio *aio; - int i; - int rc; - - rc = daos_eq_poll(eventQueue, 0, DAOS_EQ_WAIT, o.daosAios, - events); - DCHECK(rc, "Failed to poll event queue"); - assert(rc <= o.daosAios - nAios); - - for (i = 0; i < rc; i++) { - int ret; - - aio = (struct aio *) - ((char *) events[i] - - (char *) (&((struct aio *) 0)->a_event)); - - DCHECK(aio->a_event.ev_error, "Failed to transfer (%lu, %lu)", - aio->a_iod.iod_recxs->rx_idx, - aio->a_iod.iod_recxs->rx_nr); - - daos_event_fini(&aio->a_event); - ret = daos_event_init(&aio->a_event, eventQueue, - NULL /* parent */); - DCHECK(ret, "Failed to reinitialize event for AIO %p", aio); - - cfs_list_move(&aio->a_list, &aios); - nAios++; - - if (param->verbose >= VERBOSE_3) - INFO(VERBOSE_3, "Completed AIO %p: buffer %p", aio, - aio->a_iov.iov_buf); - } - - INFO(VERBOSE_3, "Found %d completed AIOs (%d free %d busy)", rc, - nAios, o.daosAios - nAios); -} - -static void ObjectClassParse(const char *string) +static void +ObjectClassParse(const char *string) { if (strcasecmp(string, "tiny") == 0) objectClass = DAOS_OC_TINY_RW; @@ -459,105 +239,107 @@ static void ObjectClassParse(const char *string) else if (strcasecmp(string, "repl_max") == 0) objectClass = DAOS_OC_REPL_MAX_RW; else - GERR("Invalid 'daosObjectClass' argument: '%s'", string); + GERR("Invalid 'oclass' argument: '%s'", string); } -static void ParseService(int max, d_rank_list_t *ranks) +static option_help * +DAOS_options() { - char *s; - - s = strdup(o.daosPoolSvc); - if (s == NULL) - GERR("failed to duplicate argument"); - ranks->rl_nr = 0; - while ((s = strtok(s, ":")) != NULL) { - if (ranks->rl_nr >= max) { - free(s); - GERR("at most %d pool service replicas supported", max); - } - ranks->rl_ranks[ranks->rl_nr] = atoi(s); - ranks->rl_nr++; - s = NULL; - } - free(s); -} - -static option_help * DAOS_options(){ return options; } -static void DAOS_Init() +static void +DAOS_Init() { int rc; if (daos_initialized) return; - if (o.daosPool == NULL || o.daosPoolSvc == NULL) + + if (o.pool == NULL || o.svcl == NULL || o.cont == NULL) return; - if (o.daosObjectClass) - ObjectClassParse(o.daosObjectClass); - if (o.daosStripeMax % o.daosStripeSize != 0) - GERR("'daosStripeMax' must be a multiple of 'daosStripeSize'"); - if (o.daosKill && ((objectClass != DAOS_OC_R2_RW) || - (objectClass != DAOS_OC_R3_RW) || - (objectClass != DAOS_OC_R4_RW) || - (objectClass != DAOS_OC_R2S_RW) || - (objectClass != DAOS_OC_R3S_RW) || - (objectClass != DAOS_OC_R4S_RW) || - (objectClass != DAOS_OC_REPL_MAX_RW))) - GERR("'daosKill' only makes sense with 'daosObjectClass=repl'"); - - if (rank == 0) - INFO(VERBOSE_0, "WARNING: USING daosStripeMax CAUSES READS TO RETURN INVALID DATA"); + if (o.oclass) + ObjectClassParse(o.oclass); rc = daos_init(); - if (rc != -DER_ALREADY) + if (rc) DCHECK(rc, "Failed to initialize daos"); - rc = daos_eq_create(&eventQueue); - DCHECK(rc, "Failed to create event queue"); - if (rank == 0) { - uuid_t uuid; - d_rank_t d_rank[13]; - d_rank_list_t ranks; - - INFO(VERBOSE_2, "Connecting to pool %s %s", o.daosPool, o.daosPoolSvc); - - rc = uuid_parse(o.daosPool, uuid); - DCHECK(rc, "Failed to parse 'daosPool': %s", o.daosPool); - ranks.rl_ranks = d_rank; - ParseService(sizeof(d_rank) / sizeof(d_rank[0]), &ranks); - - rc = daos_pool_connect(uuid, o.daosGroup, &ranks, - DAOS_PC_RW, &pool, &poolInfo, - NULL /* ev */); - DCHECK(rc, "Failed to connect to pool %s", o.daosPool); + uuid_t uuid; + d_rank_list_t *svcl = NULL; + d_rank_list_t ranks; + static daos_pool_info_t po_info; + static daos_cont_info_t co_info; + + INFO(VERBOSE_1, "Connecting to pool %s", o.pool); + + rc = uuid_parse(o.pool, uuid); + DCHECK(rc, "Failed to parse 'pool': %s", o.pool); + + svcl = daos_rank_list_parse(o.svcl, ":"); + if (svcl == NULL) + ERR("Failed to allocate svcl"); + + rc = daos_pool_connect(uuid, o.group, svcl, DAOS_PC_RW, + &poh, &po_info, NULL); + daos_rank_list_free(svcl); + DCHECK(rc, "Failed to connect to pool %s", o.pool); + + INFO(VERBOSE_1, "Create/Open Container %s", o.cont); + + uuid_clear(uuid); + rc = uuid_parse(o.cont, uuid); + DCHECK(rc, "Failed to parse 'cont': %s", o.cont); + + rc = daos_cont_open(poh, uuid, DAOS_COO_RW, &coh, &co_info, + NULL); + /* If NOEXIST we create it */ + if (rc == -DER_NONEXIST) { + INFO(VERBOSE_2, "Creating DAOS Container...\n"); + rc = daos_cont_create(poh, uuid, NULL, NULL); + if (rc == 0) + rc = daos_cont_open(poh, uuid, DAOS_COO_RW, + &coh, &co_info, NULL); + } + DCHECK(rc, "Failed to create container"); } - HandleDistribute(&pool, POOL_HANDLE); - - MPI_CHECK(MPI_Bcast(&poolInfo, sizeof poolInfo, MPI_BYTE, 0, MPI_COMM_WORLD), - "Failed to bcast pool info"); - - if (o.daosStripeCount == -1) - o.daosStripeCount = poolInfo.pi_ntargets * 64UL; + HandleDistribute(&poh, POOL_HANDLE); + HandleDistribute(&coh, CONT_HANDLE); + aoh.cookie = 0; daos_initialized = true; } -static void DAOS_Fini() +static void +DAOS_Fini() { int rc; if (!daos_initialized) return; - rc = daos_pool_disconnect(pool, NULL /* ev */); - DCHECK(rc, "Failed to disconnect from pool %s", o.daosPool); + rc = daos_cont_close(coh, NULL); + DCHECK(rc, "Failed to close container\n"); + + if (o.destroy) { + if (rank == 0) { + uuid_t uuid; + + INFO(VERBOSE_1, "Destroying Container %s", o.cont); + uuid_parse(o.cont, uuid); + rc = daos_cont_destroy(poh, o.cont, 1, NULL); + } - rc = daos_eq_destroy(eventQueue, 0 /* flags */); - DCHECK(rc, "Failed to destroy event queue"); + MPI_Bcast(&rc, 1, MPI_INT, 0, MPI_COMM_WORLD); + if (rc) + DCHECK(rc, "Failed to destroy container %s (%d)", + o.cont, rc); + } + + rc = daos_pool_disconnect(poh, NULL); + DCHECK(rc, "Failed to disconnect from pool %s", o.pool); rc = daos_fini(); DCHECK(rc, "Failed to finalize daos"); @@ -565,253 +347,180 @@ static void DAOS_Fini() daos_initialized = false; } -static void *DAOS_Create(char *testFileName, IOR_param_t *param) +static void +gen_oid(const char *name, daos_obj_id_t *oid) { - return DAOS_Open(testFileName, param); -} + daos_ofeat_t feat = 0; -static int -DAOS_Access(const char *testFileName, int mode, IOR_param_t * param) -{ - uuid_t uuid; - unsigned int dFlags; - daos_handle_t coh; - daos_cont_info_t info; - int rc; + oid->lo = d_hash_murmur64(name, strlen(name), IOR_DAOS_MUR_SEED); + oid->hi = 0; - rc = uuid_parse(testFileName, uuid); - DCHECK(rc, "Failed to parse 'testFile': %s", testFileName); - - rc = daos_cont_open(pool, uuid, DAOS_COO_RO, &coh, &info, NULL); - if (rc) - return rc; - - rc = daos_cont_close(coh, NULL); - return rc; + feat = DAOS_OF_DKEY_UINT64 | DAOS_OF_AKEY_HASHED; + daos_obj_generate_id(oid, feat, objectClass); } -static void *DAOS_Open(char *testFileName, IOR_param_t *param) +static void * +DAOS_Create(char *testFileName, IOR_param_t *param) { - struct fileDescriptor *fd; + daos_obj_id_t oid; + int rc; - fd = malloc(sizeof *fd); - if (fd == NULL) - ERR("Failed to allocate fd"); + /** Convert file name into object ID */ + gen_oid(testFileName, &oid); - ContainerOpen(testFileName, param, &fd->container, &fd->containerInfo); - ObjectOpen(fd->container, &fd->object, param); - AIOInit(param); + /** Create the array */ + if (param->filePerProc || rank == 0) { + printf("Chunk size = %zu\n", o.chunk_size); + rc = daos_array_create(coh, oid, DAOS_TX_NONE, 1, o.chunk_size, + &aoh, NULL); + DCHECK(rc, "Failed to create array object\n"); + } - return fd; + /** Distribute the array handle if not FPP */ + if (!param->filePerProc) + HandleDistribute(&aoh, ARRAY_HANDLE); } -static void -kill_daos_server(IOR_param_t *param) +static int +DAOS_Access(const char *testFileName, int mode, IOR_param_t * param) { - daos_pool_info_t info; - d_rank_t d_rank, svc_ranks[13]; - d_rank_list_t svc; - struct d_tgt_list targets; - int tgt_idx = -1; - uuid_t uuid; - char *s; - int rc; - - rc = daos_pool_query(pool, NULL, &info, NULL, NULL); - DCHECK(rc, "Error in querying pool\n"); - - if (info.pi_ntargets - info.pi_ndisabled <= 1) - return; - /* choose the last alive one */ - d_rank = info.pi_ntargets - 1 - info.pi_ndisabled; - - rc = uuid_parse(o.daosPool, uuid); - DCHECK(rc, "Failed to parse 'daosPool': %s", o.daosPool); + daos_obj_id_t oid; + int rc; - if (rc != 0) - printf("Killing tgt rank: %d (total of %d of %d already disabled)\n", - d_rank, info.pi_ndisabled, info.pi_ntargets); - fflush(stdout); + /** Convert file name into object ID */ + gen_oid(testFileName, &oid); - rc = daos_mgmt_svc_rip(o.daosGroup, d_rank, true, NULL); - DCHECK(rc, "Error in killing server\n"); + /** open the array to verify it exists */ + if (param->filePerProc || rank == 0) { + daos_size_t cell_size, chunk_size; - targets.tl_nr = 1; - targets.tl_ranks = &d_rank; - targets.tl_tgts = &tgt_idx; - svc.rl_ranks = svc_ranks; - ParseService(sizeof(svc_ranks)/ sizeof(svc_ranks[0]), &svc); + rc = daos_array_open(coh, oid, DAOS_TX_NONE, DAOS_OO_RO, + &cell_size, &chunk_size, &aoh, NULL); + if (rc) + return rc; - rc = daos_pool_tgt_exclude(uuid, NULL, &svc, &targets, NULL); - DCHECK(rc, "Error in excluding pool from poolmap\n"); + if (cell_size != 1) + GERR("Invalid DAOS Array object.\n"); - rc = daos_pool_query(pool, NULL, &info, NULL, NULL); - DCHECK(rc, "Error in querying pool\n"); + rc = daos_array_close(aoh, NULL); + aoh.cookie = 0; + } - printf("%d targets succesfully disabled\n", - info.pi_ndisabled); + if (!param->filePerProc) + MPI_Bcast(&rc, 1, MPI_INT, 0, MPI_COMM_WORLD); + return rc; } -static void -kill_and_sync(IOR_param_t *param) +static void * +DAOS_Open(char *testFileName, IOR_param_t *param) { - double start, end; + daos_obj_id_t oid; - start = MPI_Wtime(); - if (rank == 0) - kill_daos_server(param); + /** Convert file name into object ID */ + gen_oid(testFileName, &oid); - if (rank == 0) - printf("Done killing and excluding\n"); + /** Open the array */ + if (param->filePerProc || rank == 0) { + daos_size_t cell_size, chunk_size; + int rc; - MPI_CHECK(MPI_Barrier(param->testComm), - "Failed to synchronize processes"); + rc = daos_array_open(coh, oid, DAOS_TX_NONE, DAOS_OO_RW, + &cell_size, &chunk_size, &aoh, NULL); + DCHECK(rc, "Failed to create array object\n"); - end = MPI_Wtime(); - if (rank == 0) - printf("Time spent inducing failure: %lf\n", (end - start)); + if (cell_size != 1) + GERR("Invalid DAOS Array object.\n"); + } + + /** Distribute the array handle if not FPP */ + if (!param->filePerProc) + HandleDistribute(&aoh, ARRAY_HANDLE); } -static IOR_offset_t DAOS_Xfer(int access, void *file, IOR_size_t *buffer, - IOR_offset_t length, IOR_param_t *param) +static IOR_offset_t +DAOS_Xfer(int access, void *file, IOR_size_t *buffer, + IOR_offset_t length, IOR_param_t *param) { - struct fileDescriptor *fd = file; - struct aio *aio; - uint64_t stripe; - IOR_offset_t stripeOffset; - uint64_t round; - int rc; - - if (!daos_initialized) - GERR("DAOS is not initialized!"); - if (param->filePerProc) - GERR("'filePerProc' not yet supported"); - if (o.daosStripeSize % param->transferSize != 0) - GERR("'daosStripeSize' must be a multiple of 'transferSize'"); - if (param->transferSize % o.daosRecordSize != 0) - GERR("'transferSize' must be a multiple of 'daosRecordSize'"); - - assert(length == param->transferSize); - assert(param->offset % length == 0); - - /** - * Currently killing only during writes - * Kills once when 1/2 of blocksize is - * written - **/ - total_size += length; - if (o.daosKill && (access == WRITE) && - ((param->blockSize)/2) == total_size) { - /** More than half written lets kill */ - if (rank == 0) - printf("Killing and Syncing\n", rank); - kill_and_sync(param); - o.daosKill = 0; - } - - /* - * Find an available AIO descriptor. If none, wait for one. - */ - while (nAios == 0) - AIOWait(param); - aio = cfs_list_entry(aios.next, struct aio, a_list); - cfs_list_move_tail(&aio->a_list, &aios); - nAios--; - - stripe = (param->offset / o.daosStripeSize) % - o.daosStripeCount; - rc = snprintf(aio->a_dkeyBuf, sizeof aio->a_dkeyBuf, "%lu", stripe); - assert(rc < sizeof aio->a_dkeyBuf); - aio->a_dkey.iov_len = strlen(aio->a_dkeyBuf) + 1; - round = param->offset / (o.daosStripeSize * o.daosStripeCount); - stripeOffset = o.daosStripeSize * round + - param->offset % o.daosStripeSize; - if (o.daosStripeMax != 0) - stripeOffset %= o.daosStripeMax; - aio->a_recx.rx_idx = stripeOffset / o.daosRecordSize; - - /* - * If the data written will be checked later, we have to copy in valid - * data instead of writing random bytes. If the data being read is for - * checking purposes, poison the buffer first. - */ - if (access == WRITE && param->checkWrite) - memcpy(aio->a_iov.iov_buf, buffer, length); - else if (access == WRITECHECK || access == READCHECK) - memset(aio->a_iov.iov_buf, '#', length); - - INFO(VERBOSE_3, "Starting AIO %p (%d free %d busy): access %d " - "dkey '%s' iod <%llu, %llu> sgl <%p, %lu>", aio, nAios, - o.daosAios - nAios, access, (char *) aio->a_dkey.iov_buf, - (unsigned long long) aio->a_iod.iod_recxs->rx_idx, - (unsigned long long) aio->a_iod.iod_recxs->rx_nr, - aio->a_sgl.sg_iovs->iov_buf, - (unsigned long long) aio->a_sgl.sg_iovs->iov_buf_len); + daos_array_iod_t iod; + daos_range_t rg; + daos_sg_list_t sgl; + daos_iov_t iov; + int rc; + + /** set array location */ + iod.arr_nr = 1; + rg.rg_len = length; + rg.rg_idx = param->offset; + iod.arr_rgs = &rg; + + /** set memory location */ + sgl.sg_nr = 1; + daos_iov_set(&iov, buffer, length); + sgl.sg_iovs = &iov; if (access == WRITE) { - rc = daos_obj_update(fd->object, DAOS_TX_NONE, &aio->a_dkey, - 1 /* nr */, &aio->a_iod, &aio->a_sgl, - &aio->a_event); - DCHECK(rc, "Failed to start update operation"); - } else { - rc = daos_obj_fetch(fd->object, DAOS_TX_NONE, &aio->a_dkey, - 1 /* nr */, &aio->a_iod, &aio->a_sgl, - NULL /* maps */, &aio->a_event); - DCHECK(rc, "Failed to start fetch operation"); - } - - /* - * If this is a WRITECHECK or READCHECK, we are expected to fill data - * into the buffer before returning. Note that if this is a READ, we - * don't have to return valid data as WriteOrRead() doesn't care. - */ - if (access == WRITECHECK || access == READCHECK) { - while (o.daosAios - nAios > 0) - AIOWait(param); - memcpy(buffer, aio->a_sgl.sg_iovs->iov_buf, length); - } - - return length; + rc = daos_array_write(aoh, DAOS_TX_NONE, &iod, &sgl, NULL, NULL); + DCHECK(rc, "daos_array_write() failed (%d).", rc); + } else { + rc = daos_array_read(aoh, DAOS_TX_NONE, &iod, &sgl, NULL, NULL); + DCHECK(rc, "daos_array_read() failed (%d).", rc); + } + + return length; } -static void DAOS_Close(void *file, IOR_param_t *param) +static void +DAOS_Close(void *file, IOR_param_t *param) { - struct fileDescriptor *fd = file; - int rc; + int rc; if (!daos_initialized) - return; - while (o.daosAios - nAios > 0) - AIOWait(param); - AIOFini(param); - - ObjectClose(fd->object); + GERR("DAOS is not initialized!"); - ContainerClose(fd->container, param); + rc = daos_array_close(aoh, NULL); + DCHECK(rc, "daos_array_close() failed (%d).", rc); - free(fd); + aoh.cookie = 0; } -static void DAOS_Delete(char *testFileName, IOR_param_t *param) +static void +DAOS_Delete(char *testFileName, IOR_param_t *param) { - uuid_t uuid; - int rc; + daos_obj_id_t oid; + int rc; if (!daos_initialized) GERR("DAOS is not initialized!"); - INFO(VERBOSE_2, "Deleting container %s", testFileName); + /** Convert file name into object ID */ + gen_oid(testFileName, &oid); + + /** open the array to verify it exists */ + if (param->filePerProc || rank == 0) { + daos_size_t cell_size, chunk_size; + + rc = daos_array_open(coh, oid, DAOS_TX_NONE, DAOS_OO_RW, + &cell_size, &chunk_size, &aoh, NULL); + DCHECK(rc, "daos_array_open() failed (%d).", rc); + + if (cell_size != 1) + GERR("Invalid DAOS Array object.\n"); - rc = uuid_parse(testFileName, uuid); - DCHECK(rc, "Failed to parse 'testFile': %s", testFileName); + rc = daos_array_destroy(aoh, DAOS_TX_NONE, NULL); + DCHECK(rc, "daos_array_destroy() failed (%d).", rc); - rc = daos_cont_destroy(pool, uuid, 1 /* force */, NULL /* ev */); - if (rc) - DCHECK(rc, "Failed to destroy container %s (%d)", testFileName, rc); + rc = daos_array_close(aoh, NULL); + DCHECK(rc, "daos_array_close() failed (%d).", rc); + aoh.cookie = 0; + } + + if (!param->filePerProc) + MPI_Bcast(&rc, 1, MPI_INT, 0, MPI_COMM_WORLD); } -static char* DAOS_GetVersion() +static char * +DAOS_GetVersion() { static char ver[1024] = {}; @@ -819,17 +528,46 @@ static char* DAOS_GetVersion() return ver; } -static void DAOS_Fsync(void *file, IOR_param_t *param) +static void +DAOS_Fsync(void *file, IOR_param_t *param) { - while (o.daosAios - nAios > 0) - AIOWait(param); + return; } -static IOR_offset_t DAOS_GetFileSize(IOR_param_t *test, MPI_Comm testComm, - char *testFileName) +static IOR_offset_t +DAOS_GetFileSize(IOR_param_t *param, MPI_Comm testComm, char *testFileName) { - /* - * Sizes are inapplicable to containers at the moment. - */ - return 0; + daos_obj_id_t oid; + daos_size_t size; + int rc; + + if (!daos_initialized) + GERR("DAOS is not initialized!"); + + /** Convert file name into object ID */ + gen_oid(testFileName, &oid); + + /** open the array to verify it exists */ + if (param->filePerProc || rank == 0) { + daos_size_t cell_size, chunk_size; + + rc = daos_array_open(coh, oid, DAOS_TX_NONE, DAOS_OO_RO, + &cell_size, &chunk_size, &aoh, NULL); + DCHECK(rc, "daos_array_open() failed (%d).", rc); + + if (cell_size != 1) + GERR("Invalid DAOS Array object.\n"); + + rc = daos_array_get_size(aoh, DAOS_TX_NONE, &size, NULL); + DCHECK(rc, "daos_array_get_size() failed (%d).", rc); + + rc = daos_array_close(aoh, NULL); + DCHECK(rc, "daos_array_close() failed (%d).", rc); + aoh.cookie = 0; + } + + if (!param->filePerProc) + MPI_Bcast(&size, 1, MPI_LONG, 0, MPI_COMM_WORLD); + + return size; } From e295a4e45f4d161d32a55164769c63e9cdf0f19d Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Mon, 1 Apr 2019 22:58:00 +0000 Subject: [PATCH 068/206] remove printf of chunk size. Signed-off-by: Mohamad Chaarawi --- src/aiori-DAOS.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/aiori-DAOS.c b/src/aiori-DAOS.c index 3e7f9c7c..4fc7cc37 100644 --- a/src/aiori-DAOS.c +++ b/src/aiori-DAOS.c @@ -370,7 +370,6 @@ DAOS_Create(char *testFileName, IOR_param_t *param) /** Create the array */ if (param->filePerProc || rank == 0) { - printf("Chunk size = %zu\n", o.chunk_size); rc = daos_array_create(coh, oid, DAOS_TX_NONE, 1, o.chunk_size, &aoh, NULL); DCHECK(rc, "Failed to create array object\n"); From 07217c93aabc42d0a81d03af00b1ae7e39c3d2ed Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Tue, 2 Apr 2019 12:08:31 +0100 Subject: [PATCH 069/206] Bugfix for the parsing of flags. Backwards compatibility for direct IO. --- src/aiori-POSIX.c | 1 + src/option.c | 7 +++++-- src/parse_options.c | 6 +++++- testing/test_comments.ior | 2 ++ 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/aiori-POSIX.c b/src/aiori-POSIX.c index 9e85fa79..6f865b37 100755 --- a/src/aiori-POSIX.c +++ b/src/aiori-POSIX.c @@ -306,6 +306,7 @@ void *POSIX_Create(char *testFileName, IOR_param_t * param) ERR("Unable to malloc file descriptor"); posix_options_t * o = (posix_options_t*) param->backend_options; if (o->direct_io == TRUE){ + printf("ODIREC\n"); set_o_direct_flag(&fd_oflag); } diff --git a/src/option.c b/src/option.c index d5b76c3c..99f696cc 100644 --- a/src/option.c +++ b/src/option.c @@ -223,6 +223,7 @@ void option_print_current(option_help * args){ static void option_parse_token(char ** argv, int * flag_parsed_next, int * requiredArgsSeen, options_all_t * opt_all, int * error, int * print_help){ char * txt = argv[0]; char * arg = strstr(txt, "="); + int replaced_equal = 0; int i = 0; if(arg != NULL){ @@ -338,12 +339,14 @@ static void option_parse_token(char ** argv, int * flag_parsed_next, int * requi } } -int option_parse_key_value(char * key, char * value, options_all_t * opt_all){ +int option_parse_key_value(char * key, char *val, options_all_t * opt_all){ int flag_parsed_next; - char * argv[2] = {key, value}; int error = 0; int requiredArgsSeen = 0; int print_help = 0; + char value[1024]; + sprintf(value, "%s=%s", key, val); + char * argv[2] = {value, NULL}; option_parse_token(argv, & flag_parsed_next, & requiredArgsSeen, opt_all, & error, & print_help); return error; } diff --git a/src/parse_options.c b/src/parse_options.c index dd6e68bb..71af8e59 100755 --- a/src/parse_options.c +++ b/src/parse_options.c @@ -291,6 +291,10 @@ void DecodeDirective(char *line, IOR_param_t *params, options_all_t * module_opt } else if (strcasecmp(option, "summaryalways") == 0) { params->summary_every_test = atoi(value); } else { + // backward compatibility for now + if (strcasecmp(option, "useo_direct") == 0) { + strcpy(option, "--posix.odirect"); + } int parsing_error = option_parse_key_value(option, value, module_options); if(parsing_error){ if (rank == 0) @@ -410,7 +414,7 @@ IOR_test_t *ReadConfigScript(char *scriptName) tail->next = CreateTest(&tail->params, test_num++); AllocResults(tail); ((IOR_test_t*) tail)->params.backend_options = airoi_update_module_options(((IOR_test_t*) tail)->params.backend, global_options); - + tail = tail->next; *option_p = createGlobalOptions(& ((IOR_test_t*) tail->next)->params); } diff --git a/testing/test_comments.ior b/testing/test_comments.ior index bd2efa89..eaf7997e 100644 --- a/testing/test_comments.ior +++ b/testing/test_comments.ior @@ -15,5 +15,7 @@ blockSize=100k # space-prefixed comment run --dummy.delay-create=1000 +useo_direct=0 +#--posix.odirect=0 api=dummy ior stop From 27ac1b88af7a282ea25c07581447e1887a4c31b5 Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Tue, 2 Apr 2019 12:34:35 +0100 Subject: [PATCH 070/206] Removed verbose output. --- src/aiori-POSIX.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/aiori-POSIX.c b/src/aiori-POSIX.c index 6f865b37..9e85fa79 100755 --- a/src/aiori-POSIX.c +++ b/src/aiori-POSIX.c @@ -306,7 +306,6 @@ void *POSIX_Create(char *testFileName, IOR_param_t * param) ERR("Unable to malloc file descriptor"); posix_options_t * o = (posix_options_t*) param->backend_options; if (o->direct_io == TRUE){ - printf("ODIREC\n"); set_o_direct_flag(&fd_oflag); } From 5797624d56c45d1b14f35807f8e0ecaa78b297a0 Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Mon, 15 Apr 2019 22:14:33 +0000 Subject: [PATCH 071/206] fix bug in file delete and access where only rank 0 is calling those functions. remove the bcasts for the error code since the other ranks are not calling it. Signed-off-by: Mohamad Chaarawi --- src/aiori-DAOS.c | 58 ++++++++++++++++++------------------------------ 1 file changed, 21 insertions(+), 37 deletions(-) diff --git a/src/aiori-DAOS.c b/src/aiori-DAOS.c index 4fc7cc37..751e5a72 100644 --- a/src/aiori-DAOS.c +++ b/src/aiori-DAOS.c @@ -173,6 +173,7 @@ HandleDistribute(daos_handle_t *handle, enum handleType type) MPI_COMM_WORLD), "Failed to bcast global handle buffer size"); + global.iov_len = global.iov_buf_len; global.iov_buf = malloc(global.iov_buf_len); if (global.iov_buf == NULL) ERR("Failed to allocate global handle buffer"); @@ -192,9 +193,6 @@ HandleDistribute(daos_handle_t *handle, enum handleType type) "Failed to bcast global pool handle"); if (rank != 0) { - /* A larger-than-actual length works just fine. */ - global.iov_len = global.iov_buf_len; - if (type == POOL_HANDLE) rc = daos_pool_global2local(global, handle); else if (type == CONT_HANDLE) @@ -384,30 +382,22 @@ static int DAOS_Access(const char *testFileName, int mode, IOR_param_t * param) { daos_obj_id_t oid; + daos_size_t cell_size, chunk_size; int rc; /** Convert file name into object ID */ gen_oid(testFileName, &oid); - /** open the array to verify it exists */ - if (param->filePerProc || rank == 0) { - daos_size_t cell_size, chunk_size; - - rc = daos_array_open(coh, oid, DAOS_TX_NONE, DAOS_OO_RO, - &cell_size, &chunk_size, &aoh, NULL); - if (rc) - return rc; - - if (cell_size != 1) - GERR("Invalid DAOS Array object.\n"); - - rc = daos_array_close(aoh, NULL); - aoh.cookie = 0; - } + rc = daos_array_open(coh, oid, DAOS_TX_NONE, DAOS_OO_RO, + &cell_size, &chunk_size, &aoh, NULL); + if (rc) + return rc; - if (!param->filePerProc) - MPI_Bcast(&rc, 1, MPI_INT, 0, MPI_COMM_WORLD); + if (cell_size != 1) + GERR("Invalid DAOS Array object.\n"); + rc = daos_array_close(aoh, NULL); + aoh.cookie = 0; return rc; } @@ -487,6 +477,7 @@ static void DAOS_Delete(char *testFileName, IOR_param_t *param) { daos_obj_id_t oid; + daos_size_t cell_size, chunk_size; int rc; if (!daos_initialized) @@ -496,26 +487,19 @@ DAOS_Delete(char *testFileName, IOR_param_t *param) gen_oid(testFileName, &oid); /** open the array to verify it exists */ - if (param->filePerProc || rank == 0) { - daos_size_t cell_size, chunk_size; - - rc = daos_array_open(coh, oid, DAOS_TX_NONE, DAOS_OO_RW, - &cell_size, &chunk_size, &aoh, NULL); - DCHECK(rc, "daos_array_open() failed (%d).", rc); + rc = daos_array_open(coh, oid, DAOS_TX_NONE, DAOS_OO_RW, + &cell_size, &chunk_size, &aoh, NULL); + DCHECK(rc, "daos_array_open() failed (%d).", rc); - if (cell_size != 1) - GERR("Invalid DAOS Array object.\n"); + if (cell_size != 1) + GERR("Invalid DAOS Array object.\n"); - rc = daos_array_destroy(aoh, DAOS_TX_NONE, NULL); - DCHECK(rc, "daos_array_destroy() failed (%d).", rc); + rc = daos_array_destroy(aoh, DAOS_TX_NONE, NULL); + DCHECK(rc, "daos_array_destroy() failed (%d).", rc); - rc = daos_array_close(aoh, NULL); - DCHECK(rc, "daos_array_close() failed (%d).", rc); - aoh.cookie = 0; - } - - if (!param->filePerProc) - MPI_Bcast(&rc, 1, MPI_INT, 0, MPI_COMM_WORLD); + rc = daos_array_close(aoh, NULL); + DCHECK(rc, "daos_array_close() failed (%d).", rc); + aoh.cookie = 0; } static char * From 8fd755147a17512f09b98a837a2f0a6997c10b6b Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Fri, 19 Apr 2019 10:20:04 +0100 Subject: [PATCH 072/206] Fix for #144 --- src/parse_options.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/parse_options.c b/src/parse_options.c index 71af8e59..8a569a57 100755 --- a/src/parse_options.c +++ b/src/parse_options.c @@ -322,9 +322,12 @@ void ParseLine(char *line, IOR_param_t * test, options_all_t * module_options) start = line; do { + end = strchr(start, '#'); + if (end != NULL) + *end = '\0'; end = strchr(start, ','); if (end != NULL) - *end = '\0'; + *end = '\0'; DecodeDirective(start, test, module_options); start = end + 1; } while (end != NULL); From e3afdb5ed55aa27d5cfffa4d2f0163d577274855 Mon Sep 17 00:00:00 2001 From: Oliver Steffen Date: Mon, 29 Apr 2019 15:49:57 +0200 Subject: [PATCH 073/206] beegfs: don't fail if output exists Use existing output file if it has the desired stripe settings (number of targets, chunk size) --- src/aiori-POSIX.c | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/src/aiori-POSIX.c b/src/aiori-POSIX.c index ca7c4a29..7c8c9ea7 100755 --- a/src/aiori-POSIX.c +++ b/src/aiori-POSIX.c @@ -201,6 +201,24 @@ bool beegfs_isOptionSet(int opt) { return opt != -1; } +bool beegfs_compatibleFileExists(char* filepath, int numTargets, int chunkSize) +{ + int fd = open(filepath, O_RDWR); + + if (fd == -1) + return false; + + unsigned read_stripePattern = 0; + u_int16_t read_numTargets = 0; + int read_chunkSize = 0; + + bool retVal = beegfs_getStripeInfo(fd, &read_stripePattern, &read_chunkSize, &read_numTargets); + + close(fd); + + return retVal && read_numTargets == numTargets && read_chunkSize == chunkSize; +} + /* * Create a file on a BeeGFS file system with striping parameters */ @@ -247,8 +265,9 @@ bool beegfs_createFilePath(char* filepath, mode_t mode, int numTargets, int chun char* filenameTmp = strdup(filepath); char* filename = basename(filepath); - bool isFileCreated = beegfs_createFile(parentDirFd, filename, - mode, numTargets, chunkSize); + bool isFileCreated = beegfs_compatibleFileExists(filepath, numTargets, chunkSize) + || beegfs_createFile(parentDirFd, filename, + mode, numTargets, chunkSize); if (!isFileCreated) ERR("Could not create file"); retVal = true; From a9b0bf35bca95637244d9759f28de704c0c9b78f Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Thu, 2 May 2019 09:58:35 +0100 Subject: [PATCH 074/206] Support MDTest verbosity >=5 prints behavior for all ranks instead of just rank 0 (which is enabled with verbosity level 3 as before) --- src/mdtest.c | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/src/mdtest.c b/src/mdtest.c index 8a819796..6592e5d1 100644 --- a/src/mdtest.c +++ b/src/mdtest.c @@ -273,7 +273,7 @@ static void create_remove_dirs (const char *path, bool create, uint64_t itemNum) //create dirs sprintf(curr_item, "%s/dir.%s%" PRIu64, path, create ? mk_name : rm_name, itemNum); - if (rank == 0 && verbose >= 3) { + if (verbose >= 5 || (rank == 0 && verbose >= 3)) { fprintf(out_logfile, "V-3: create_remove_items_helper (dirs %s): curr_item is \"%s\"\n", operation, curr_item); fflush(out_logfile); } @@ -302,7 +302,7 @@ static void remove_file (const char *path, uint64_t itemNum) { //remove files sprintf(curr_item, "%s/file.%s"LLU"", path, rm_name, itemNum); - if (rank == 0 && verbose >= 3) { + if (verbose >= 5 || (rank == 0 && verbose >= 3)) { fprintf(out_logfile, "V-3: create_remove_items_helper (non-dirs remove): curr_item is \"%s\"\n", curr_item); fflush(out_logfile); } @@ -333,7 +333,7 @@ static void create_file (const char *path, uint64_t itemNum) { if (collective_creates) { param.openFlags = IOR_WRONLY; - if (rank == 0 && verbose >= 3) { + if (verbose >= 5 || (rank == 0 && verbose >= 3)) { fprintf(out_logfile, "V-3: create_remove_items_helper (collective): open...\n" ); fflush( out_logfile ); } @@ -350,7 +350,7 @@ static void create_file (const char *path, uint64_t itemNum) { param.openFlags = IOR_CREAT | IOR_WRONLY; param.filePerProc = !shared_file; - if (rank == 0 && verbose >= 3) { + if (verbose >= 5 || (rank == 0 && verbose >= 3)) { fprintf(out_logfile, "V-3: create_remove_items_helper (non-collective, shared): open...\n" ); fflush( out_logfile ); } @@ -362,7 +362,7 @@ static void create_file (const char *path, uint64_t itemNum) { } if (write_bytes > 0) { - if (rank == 0 && verbose >= 3) { + if (verbose >= 5 || (rank == 0 && verbose >= 3)) { fprintf(out_logfile, "V-3: create_remove_items_helper: write...\n" ); fflush( out_logfile ); } @@ -378,7 +378,7 @@ static void create_file (const char *path, uint64_t itemNum) { } } - if (rank == 0 && verbose >= 3) { + if (verbose >= 5 || (rank == 0 && verbose >= 3)) { fprintf(out_logfile, "V-3: create_remove_items_helper: close...\n" ); fflush( out_logfile ); } @@ -430,7 +430,7 @@ void collective_helper(const int dirs, const int create, const char* path, uint6 } sprintf(curr_item, "%s/file.%s"LLU"", path, create ? mk_name : rm_name, itemNum+i); - if (rank == 0 && verbose >= 3) { + if (verbose >= 5 || (rank == 0 && verbose >= 3)) { fprintf(out_logfile, "V-3: create file: %s\n", curr_item); fflush(out_logfile); } @@ -476,7 +476,7 @@ void create_remove_items(int currDepth, const int dirs, const int create, const memset(dir, 0, MAX_PATHLEN); strcpy(temp_path, path); - if (rank == 0 && verbose >= 3) { + if (verbose >= 5 || (rank == 0 && verbose >= 3)) { fprintf(out_logfile, "V-3: create_remove_items (start): temp_path is \"%s\"\n", temp_path ); fflush(out_logfile); } @@ -505,7 +505,7 @@ void create_remove_items(int currDepth, const int dirs, const int create, const strcat(temp_path, "/"); strcat(temp_path, dir); - if (rank == 0 && verbose >= 3) { + if (verbose >= 5 || (rank == 0 && verbose >= 3)) { fprintf(out_logfile, "V-3: create_remove_items (for loop): temp_path is \"%s\"\n", temp_path ); fflush(out_logfile); } @@ -583,13 +583,13 @@ void mdtest_stat(const int random, const int dirs, const long dir_iter, const ch /* create name of file/dir to stat */ if (dirs) { - if (rank == 0 && verbose >= 3 && (i%ITEM_COUNT == 0) && (i != 0)) { + if ((verbose >= 5 || (rank == 0 && verbose >= 3)) && (i%ITEM_COUNT == 0) && (i != 0)) { fprintf(out_logfile, "V-3: stat dir: "LLU"\n", i); fflush(out_logfile); } sprintf(item, "dir.%s"LLU"", stat_name, item_num); } else { - if (rank == 0 && verbose >= 3 && (i%ITEM_COUNT == 0) && (i != 0)) { + if ((verbose >= 5 || (rank == 0 && verbose >= 3)) && (i%ITEM_COUNT == 0) && (i != 0)) { fprintf(out_logfile, "V-3: stat file: "LLU"\n", i); fflush(out_logfile); } @@ -618,7 +618,7 @@ void mdtest_stat(const int random, const int dirs, const long dir_iter, const ch strcpy( item, temp ); /* below temp used to be hiername */ - if (rank == 0 && verbose >= 3) { + if (verbose >= 5 || (rank == 0 && verbose >= 3)) { if (dirs) { fprintf(out_logfile, "V-3: mdtest_stat dir : %s\n", item); } else { @@ -700,7 +700,7 @@ void mdtest_read(int random, int dirs, const long dir_iter, char *path) { /* create name of file to read */ if (!dirs) { - if (rank == 0 && verbose >= 3 && (i%ITEM_COUNT == 0) && (i != 0)) { + if ((verbose >= 5 || (rank == 0 && verbose >= 3)) && (i%ITEM_COUNT == 0) && (i != 0)) { fprintf(out_logfile, "V-3: read file: "LLU"\n", i); fflush(out_logfile); } @@ -729,7 +729,7 @@ void mdtest_read(int random, int dirs, const long dir_iter, char *path) { strcpy( item, temp ); /* below temp used to be hiername */ - if (rank == 0 && verbose >= 3) { + if (verbose >= 5 || (rank == 0 && verbose >= 3)) { if (!dirs) { fprintf(out_logfile, "V-3: mdtest_read file: %s\n", item); } @@ -805,7 +805,7 @@ void collective_create_remove(const int create, const int dirs, const int ntasks } /* Now that everything is set up as it should be, do the create or remove */ - if (rank == 0 && verbose >= 3) { + if (verbose >= 5 || (rank == 0 && verbose >= 3)) { fprintf(out_logfile, "V-3: collective_create_remove (create_remove_items): temp is \"%s\"\n", temp); fflush( out_logfile ); } From 54522315ec92f6f2228df1af853eebab06a61a0c Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Wed, 8 May 2019 14:02:40 +0100 Subject: [PATCH 075/206] Fix #146; handling of hints. Why are GPFS hints special, remains unclear. --- src/utilities.c | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/utilities.c b/src/utilities.c index cc0918c3..c0c7c614 100755 --- a/src/utilities.c +++ b/src/utilities.c @@ -292,21 +292,23 @@ int CountTasksPerNode(MPI_Comm comm) { */ void ExtractHint(char *settingVal, char *valueVal, char *hintString) { - char *settingPtr, *valuePtr, *tmpPtr1, *tmpPtr2; + char *settingPtr, *valuePtr, *tmpPtr2; + /* find the value */ settingPtr = (char *)strtok(hintString, " ="); valuePtr = (char *)strtok(NULL, " =\t\r\n"); - tmpPtr1 = settingPtr; - tmpPtr2 = (char *)strstr(settingPtr, "IOR_HINT__MPI__"); - if (tmpPtr1 == tmpPtr2) { + /* is this an MPI hint? */ + tmpPtr2 = (char *) strstr(settingPtr, "IOR_HINT__MPI__"); + if (settingPtr == tmpPtr2) { settingPtr += strlen("IOR_HINT__MPI__"); - } else { - tmpPtr2 = (char *)strstr(settingPtr, "IOR_HINT__GPFS__"); - if (tmpPtr1 == tmpPtr2) { - settingPtr += strlen("IOR_HINT__GPFS__"); - fprintf(out_logfile, - "WARNING: Unable to set GPFS hints (not implemented.)\n"); + tmpPtr2 = (char *) strstr(hintString, "IOR_HINT__GPFS__"); + /* is it an GPFS hint? */ + if (settingPtr == tmpPtr2) { + settingPtr += strlen("IOR_HINT__GPFS__"); + }else{ + fprintf(out_logfile, "WARNING: Unable to set unknown hint type (not implemented.)\n"); + return; } } strcpy(settingVal, settingPtr); From 30a1ca187bc0373ca3e90b87f97b7b34a49e2250 Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Tue, 14 May 2019 14:55:11 +0100 Subject: [PATCH 076/206] Bugfix by rmn1 for #147 and #112. --- src/mdtest.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/mdtest.c b/src/mdtest.c index 6592e5d1..263b18ac 100644 --- a/src/mdtest.c +++ b/src/mdtest.c @@ -551,7 +551,7 @@ void mdtest_stat(const int random, const int dirs, const long dir_iter, const ch uint64_t stop_items = items; - if( directory_loops != 1 || leaf_only ){ + if( directory_loops != 1 ){ stop_items = items_per_dir; } @@ -2270,7 +2270,11 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * } if (items_per_dir > 0) { if(items == 0){ - items = items_per_dir * num_dirs_in_tree; + if (leaf_only) { + items = items_per_dir * (uint64_t) pow(branch_factor, depth); + } else { + items = items_per_dir * num_dirs_in_tree; + } }else{ num_dirs_in_tree_calc = num_dirs_in_tree; } From 6abdcc363a3114fcfca40ee40d0a64adde12e7b7 Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Thu, 16 May 2019 12:56:56 +0100 Subject: [PATCH 077/206] Trivial fix for #149 --- src/parse_options.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/parse_options.c b/src/parse_options.c index 8a569a57..153fef8e 100755 --- a/src/parse_options.c +++ b/src/parse_options.c @@ -92,7 +92,7 @@ void DecodeDirective(char *line, IOR_param_t *params, options_all_t * module_opt char value[MAX_STR]; int rc; int initialized; - + rc = sscanf(line, " %[^=# \t\r\n] = %[^# \t\r\n] ", option, value); if (rc != 2 && rank == 0) { fprintf(out_logfile, "Syntax error in configuration options: %s\n", @@ -309,9 +309,6 @@ void DecodeDirective(char *line, IOR_param_t *params, options_all_t * module_opt } } -static void decodeDirectiveWrapper(char *line){ - DecodeDirective(line, parameters, global_options); -} /* * Parse a single line, which may contain multiple comma-seperated directives @@ -333,6 +330,11 @@ void ParseLine(char *line, IOR_param_t * test, options_all_t * module_options) } while (end != NULL); } + +static void decodeDirectiveWrapper(char *line){ + ParseLine(line, parameters, global_options); +} + /* * Determines if the string "haystack" contains only the string "needle", and * possibly whitespace before and after needle. Function is case insensitive. From 3a873ea2b0702967f99067b82c05f887ac0757b1 Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Thu, 16 May 2019 19:35:00 +0100 Subject: [PATCH 078/206] Fix #150. --- src/ior-output.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ior-output.c b/src/ior-output.c index 40764fff..eaf5bab6 100644 --- a/src/ior-output.c +++ b/src/ior-output.c @@ -450,7 +450,7 @@ void ShowSetup(IOR_param_t *params) if (params->lustre_set_striping) { PrintKeyVal("Lustre stripe size", ((params->lustre_stripe_size == 0) ? "Use default" : HumanReadable(params->lustre_stripe_size, BASE_TWO))); - PrintKeyVal("stripe count", (params->lustre_stripe_count == 0 ? "Use default" : HumanReadable(params->lustre_stripe_count, BASE_TWO))); + PrintKeyVal("stripe count", (params->lustre_stripe_count == 0 ? "Use default" : params->lustre_stripe_count) ); } #endif /* HAVE_LUSTRE_LUSTRE_USER_H */ if (params->deadlineForStonewalling > 0) { From 13a17352ec0d5dabb64c0f523f7e7292449abd07 Mon Sep 17 00:00:00 2001 From: Vaclav Hapla Date: Thu, 16 May 2019 20:48:28 +0200 Subject: [PATCH 079/206] Update tutorial.rst "Defeating Page Cache" should be a heading I guess. --- doc/sphinx/userDoc/tutorial.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/sphinx/userDoc/tutorial.rst b/doc/sphinx/userDoc/tutorial.rst index 9556cebe..5fa68141 100644 --- a/doc/sphinx/userDoc/tutorial.rst +++ b/doc/sphinx/userDoc/tutorial.rst @@ -181,6 +181,7 @@ again, using this option changes our performance measurement quite a bit:: and we finally have a believable bandwidth measurement for our file system. Defeating Page Cache +-------------------- Since IOR is specifically designed to benchmark I/O, it provides these options that make it as easy as possible to ensure that you are actually measuring the performance of your file system and not your compute nodes' memory. That being From c5c43c4f3cc89cf9f08c6c3d16b4f7ff58f583ea Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Thu, 16 May 2019 21:25:34 +0100 Subject: [PATCH 080/206] Changed the version name to alpha to differentiate. Now mdtest has the version number from IOR. --- META | 2 +- src/mdtest.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/META b/META index a4ea8733..af848f61 100755 --- a/META +++ b/META @@ -1,3 +1,3 @@ Package: ior -Version: 3.2.0 +Version: 3.3alpha1 Release: 0 diff --git a/src/mdtest.c b/src/mdtest.c index ef5db1e6..d65d6f94 100644 --- a/src/mdtest.c +++ b/src/mdtest.c @@ -77,7 +77,7 @@ #define FILEMODE S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH #define DIRMODE S_IRUSR|S_IWUSR|S_IXUSR|S_IRGRP|S_IWGRP|S_IXGRP|S_IROTH|S_IXOTH -#define RELEASE_VERS "1.9.3" +#define RELEASE_VERS META_VERSION #define TEST_DIR "#test-dir" #define ITEM_COUNT 25000 From eb2f4c6a017697ec93db68d65fc1e56aba9fc6e2 Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Thu, 16 May 2019 21:36:26 +0100 Subject: [PATCH 081/206] Added NEWS stub for new version. --- NEWS | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/NEWS b/NEWS index 00cbf439..a420ded6 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,13 @@ +Version 3.3 +-------------------------------------------------------------------------------- + +New major features: + +New minor features: + +Bugfixes: + + Version 3.2.0 -------------------------------------------------------------------------------- From c21744d893e4d0b93e5d8327b191b505fe6cbe70 Mon Sep 17 00:00:00 2001 From: "Glenn K. Lockwood" Date: Sun, 19 May 2019 09:35:53 -0700 Subject: [PATCH 082/206] updated version after 3.2 was frozen this should've been done when we did the 3.2 feature freeze --- META | 2 +- NEWS | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/META b/META index af848f61..782f8466 100755 --- a/META +++ b/META @@ -1,3 +1,3 @@ Package: ior -Version: 3.3alpha1 +Version: 3.3.0+dev Release: 0 diff --git a/NEWS b/NEWS index a420ded6..1f2cc74d 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,4 @@ -Version 3.3 +Version 3.3.0+dev -------------------------------------------------------------------------------- New major features: @@ -7,6 +7,12 @@ New minor features: Bugfixes: +Version 3.2.1 +-------------------------------------------------------------------------------- + +- Fixed a memory protection bug in mdtest (Julian Kunkel) +- Fixed correctness bugs in mdtest leaf-only mode (#147) (rmn1) +- Fixed bug where mdtest attempted to stat uncreated files (Julian Kunkel) Version 3.2.0 -------------------------------------------------------------------------------- From f1b8174e4ba3474b82dd07a99253601ffbb1b84d Mon Sep 17 00:00:00 2001 From: "Glenn K. Lockwood" Date: Fri, 21 Sep 2018 19:04:31 -0400 Subject: [PATCH 083/206] cleaned up README to reflect the current state of the repo --- README.md | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 81a1fa32..c1c73a8a 100755 --- a/README.md +++ b/README.md @@ -1,31 +1,31 @@ # HPC IO Benchmark Repository [![Build Status](https://travis-ci.org/hpc/ior.svg?branch=master)](https://travis-ci.org/hpc/ior) -This repo now contains both IOR and mdtest. -See also NOTES.txt +This repository contains the IOR and mdtest parallel I/O benchmarks. The +[official IOR/mdtest documention][] can be found in the `docs/` subdirectory or +on Read the Docs. -# Building +## Building -0. If "configure" is missing from the top level directory, you - probably retrieved this code directly from the repository. - Run "./bootstrap". +1. If `configure` is missing from the top level directory, you probably + retrieved this code directly from the repository. Run `./bootstrap` + to generate the configure script. Alternatively, download an + [official IOR release][] which includes the configure script. - If your versions of the autotools are not new enough to run - this script, download and official tarball in which the - configure script is already provided. +1. Run `./configure`. For a full list of configuration options, use + `./configure --help`. -1. Run "./configure" +2. Run `make` - See "./configure --help" for configuration options. +3. Optionally, run `make install`. The installation prefix + can be changed via `./configure --prefix=...`. -2. Run "make" +## Testing -3. Optionally, run "make install". The installation prefix - can be changed as an option to the "configure" script. +* Run `make check` to invoke the unit tests. +* More comprehensive functionality tests are included in `testing/`. These + scripts will launch IOR and mdtest via MPI. +* Docker scripts are also provided in `testing/docker/` to test various + distributions at once. -# Testing - - Run "make check" to invoke the unit test framework of Automake. - - * To run basic functionality tests that we use for continuous integration, see ./testing/ - * There are docker scripts provided to test various distributions at once. - * See ./testing/docker/ +[official IOR release]: https://github.com/hpc/ior/releases +[official IOR/mdtest documention]: http://ior.readthedocs.org/ From 13fd0c6c167c32920d9e4be635bdc05dc7e14334 Mon Sep 17 00:00:00 2001 From: "Glenn K. Lockwood" Date: Fri, 21 Sep 2018 19:23:59 -0400 Subject: [PATCH 084/206] populate sphinx doc versions automatically --- doc/sphinx/conf.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/doc/sphinx/conf.py b/doc/sphinx/conf.py index 05e58074..8bee2a46 100644 --- a/doc/sphinx/conf.py +++ b/doc/sphinx/conf.py @@ -67,15 +67,27 @@ project = u'IOR' copyright = u'2017, IOR' author = u'IOR' + # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = u'3.1.0' +version = u'3.2' # The full version, including alpha/beta/rc tags. -release = u'0' +release = u'3.2.0' + +for line in open(os.path.join('..', '..', 'META', 'r')): + key, value = line.split(None, 1) + if key == "Package:": + project = value + elif key == "Version:": + version = value.rsplit('.', 1)[0] + release = value +# We don't use the release field +# elif key == "Release:": +# release = value # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. From 32f256f4ef9dba1cd2cab9b5a79457925d537aa8 Mon Sep 17 00:00:00 2001 From: "Glenn K. Lockwood" Date: Fri, 21 Sep 2018 19:28:50 -0400 Subject: [PATCH 085/206] bugfix --- doc/sphinx/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx/conf.py b/doc/sphinx/conf.py index 8bee2a46..9def583d 100644 --- a/doc/sphinx/conf.py +++ b/doc/sphinx/conf.py @@ -78,7 +78,7 @@ # The full version, including alpha/beta/rc tags. release = u'3.2.0' -for line in open(os.path.join('..', '..', 'META', 'r')): +for line in open(os.path.join('..', '..', 'META'), 'r'): key, value = line.split(None, 1) if key == "Package:": project = value From 11d63f067ae5156f8f571116624638a0a40faaa8 Mon Sep 17 00:00:00 2001 From: "Glenn K. Lockwood" Date: Fri, 21 Sep 2018 21:09:22 -0400 Subject: [PATCH 086/206] updated formatting --- doc/sphinx/conf.py | 4 +- doc/sphinx/userDoc/compatibility.rst | 9 +- doc/sphinx/userDoc/install.rst | 2 + doc/sphinx/userDoc/options.rst | 428 +++++++++++++-------------- doc/sphinx/userDoc/scripts.rst | 82 +++++ doc/sphinx/userDoc/skripts.rst | 72 ----- 6 files changed, 297 insertions(+), 300 deletions(-) create mode 100644 doc/sphinx/userDoc/scripts.rst delete mode 100644 doc/sphinx/userDoc/skripts.rst diff --git a/doc/sphinx/conf.py b/doc/sphinx/conf.py index 9def583d..90628fd9 100644 --- a/doc/sphinx/conf.py +++ b/doc/sphinx/conf.py @@ -74,7 +74,7 @@ # built documents. # # The short X.Y version. -version = u'3.2' +version = u'3.2.0' # The full version, including alpha/beta/rc tags. release = u'3.2.0' @@ -83,7 +83,7 @@ if key == "Package:": project = value elif key == "Version:": - version = value.rsplit('.', 1)[0] + version = value release = value # We don't use the release field # elif key == "Release:": diff --git a/doc/sphinx/userDoc/compatibility.rst b/doc/sphinx/userDoc/compatibility.rst index 6b87d680..0c925e2a 100644 --- a/doc/sphinx/userDoc/compatibility.rst +++ b/doc/sphinx/userDoc/compatibility.rst @@ -1,10 +1,11 @@ .. _compatibility: -Compatibility -============= +Version Compatibility +===================== -IOR has a long history. Here are some hints about compatibility with older -versions. +IOR has a long history and only IOR version 3 is currently supported. However, +there are many forks of IOR based on earlier versions, and the following +incompatibilities are known to exist between major versions. 1) IOR version 1 (c. 1996-2002) and IOR version 2 (c. 2003-present) are incompatible. Input decks from one will not work on the other. As version diff --git a/doc/sphinx/userDoc/install.rst b/doc/sphinx/userDoc/install.rst index 48948ac0..4bfa684b 100644 --- a/doc/sphinx/userDoc/install.rst +++ b/doc/sphinx/userDoc/install.rst @@ -1,3 +1,5 @@ +.. _install: + Install ======= diff --git a/doc/sphinx/userDoc/options.rst b/doc/sphinx/userDoc/options.rst index 8b441269..6d307b69 100644 --- a/doc/sphinx/userDoc/options.rst +++ b/doc/sphinx/userDoc/options.rst @@ -16,7 +16,9 @@ normal parameters override each other, so the last one executed. Command line options -------------------- -These options are to be used on the command line. E.g., 'IOR -a POSIX -b 4K'. + +These options are to be used on the command line (e.g., ``./ior -a POSIX -b 4K``). + -a S api -- API for I/O [POSIX|MPIIO|HDF5|HDFS|S3|S3_EMC|NCMPI|RADOS] -A N refNum -- user reference number to include in long summary -b N blockSize -- contiguous bytes to write per task (e.g.: 8, 4k, 2m, 1g) @@ -69,283 +71,265 @@ These options are to be used on the command line. E.g., 'IOR -a POSIX -b 4K'. -Z reorderTasksRandom -- changes task ordering to random ordering for readback -NOTES: * S is a string, N is an integer number. - * For transfer and block sizes, the case-insensitive K, M, and G - suffices are recognized. I.e., '4k' or '4K' is accepted as 4096. +* S is a string, N is an integer number. + +* For transfer and block sizes, the case-insensitive K, M, and G + suffices are recognized. I.e., '4k' or '4K' is accepted as 4096. Directive Options ------------------ -For each of the general settings, note the default is shown in brackets. -IMPORTANT NOTE: For all true/false options below [1]=true, [0]=false -IMPORTANT NOTE: Contrary to appearance, the script options below are NOT case sensitive +For all true/false options below, [1]=true, [0]=false. All options are case-insensitive. -GENERAL: +GENERAL ^^^^^^^^^^^^^^ - * refNum - user supplied reference number, included in - long summary [0] - * api - must be set to one of POSIX, MPIIO, HDF5, HDFS, S3, - S3_EMC, or NCMPI, depending on test [POSIX] + * ``refNum`` - user supplied reference number, included in long summary + (default: 0) + + * ``api`` - must be set to one of POSIX, MPIIO, HDF5, HDFS, S3, S3_EMC, NCMPI, + IME, MMAP, or RAODS depending on test (default: ``POSIX``) + + * ``testFile`` - name of the output file [testFile]. With ``filePerProc`` set, + the tasks can round robin across multiple file names via ``-o S@S@S``. + If only a single file name is specified in this case, IOR appends the MPI + rank to the end of each file generated (e.g., ``testFile.00000059``) + (default: ``testFile``) + + * ``hintsFileName`` - name of the hints file (default: none) + + * ``repetitions`` - number of times to run each test (default: 1) + + * ``multiFile`` - creates multiple files for single-shared-file or + file-per-process modes for each iteration (default: 0) + + * ``reorderTasksConstant`` - reorders tasks by a constant node offset for + writing/reading neighbor's data from different nodes (default: 0) + + * ``taskPerNodeOffset`` - for read tests. Use with ``-C`` and ``-Z`` options. + With ``reorderTasks``, constant N. With ``reordertasksrandom``, >= N + (default: 1) + + * ``reorderTasksRandom`` - reorders tasks to random ordering for read tests + (default: 0) + + * ``reorderTasksRandomSeed`` - random seed for ``reordertasksrandom`` option. (default: 0) + * When > 0, use the same seed for all iterations + * When < 0, different seed for each iteration + + * ``quitOnError`` - upon error encountered on ``checkWrite`` or ``checkRead``, + display current error and then stop execution. Otherwise, count errors and + continue (default: 0) + + * ``numTasks`` - number of tasks that should participate in the test. 0 + denotes all tasks. (default: 0) + + * ``interTestDelay`` - time (in seconds) to delay before beginning a write or + read phase in a series of tests This does not delay before check-write or + check-read phases. (default: 0) + + * ``outlierThreshold`` - gives warning if any task is more than this number of + seconds from the mean of all participating tasks. The warning includes the + offending task, its timers (start, elapsed create, elapsed transfer, elapsed + close, end), and the mean and standard deviation for all tasks. When zero, + disable this feature. (default: 0) + + * ``intraTestBarriers`` - use barrier between open, write/read, and close + phases (default: 0) + + * ``uniqueDir`` - create and use unique directory for each file-per-process + (default: 0) + + * ``writeFile`` - write file(s), first deleting any existing file. + The defaults for ``writeFile`` and ``readFile`` are set such that if there + is not at least one of ``-w``, ``-r``, ``-W``, or ``-R``, ``-w`` and ``-r`` + are enabled. If either ``writeFile`` or ``readFile`` are explicitly + enabled, though, its complement is *not* also implicitly enabled. + + * ``readFile`` - reads existing file(s) as specified by the ``testFile`` + option. The defaults for ``writeFile`` and ``readFile`` are set such that + if there is not at least one of ``-w``, ``-r``, ``-W``, or ``-R``, ``-w`` + and ``-r`` are enabled. If either ``writeFile`` or ``readFile`` are + explicitly enabled, though, its complement is *not* also implicitly enabled. - * testFile - name of the output file [testFile] - NOTE: with filePerProc set, the tasks can round - robin across multiple file names '-o S@S@S' + * ``filePerProc`` - have each MPI process perform I/O to a unique file + (default: 0) - * hintsFileName - name of the hints file [] + * ``checkWrite`` - read data back and check for errors against known pattern. + Can be used independently of ``writeFile``. Data checking is not timed and + does not affect other performance timings. All errors detected are tallied + and returned as the program exit code unless ``quitOnError`` is set. + (default: 0) - * repetitions - number of times to run each test [1] + * ``checkRead`` - re-read data and check for errors between reads. Can be + used independently of ``readFile``. Data checking is not timed and does not + affect other performance timings. All errors detected are tallied and + returned as the program exit code unless ``quitOnError`` is set. + (default: 0) - * multiFile - creates multiple files for single-shared-file or - file-per-process modes; i.e. each iteration creates - a new file [0=FALSE] - - * reorderTasksConstant - reorders tasks by a constant node offset for writing/reading neighbor's - data from different nodes [0=FALSE] + * ``keepFile`` - do not remove test file(s) on program exit (default: 0) - * taskPerNodeOffset - for read tests. Use with -C & -Z options. [1] - With reorderTasks, constant N. With reordertasksrandom, >= N - - * reorderTasksRandom - reorders tasks to random ordering for readback [0=FALSE] + * ``keepFileWithError`` - do not delete any files containing errors if + detected during read-check or write-check phases. (default: 0) - * reorderTasksRandomSeed - random seed for reordertasksrandom option. [0] - >0, same seed for all iterations. <0, different seed for each iteration - - * quitOnError - upon error encountered on checkWrite or checkRead, - display current error and then stop execution; - if not set, count errors and continue [0=FALSE] - - * numTasks - number of tasks that should participate in the test - [0] - NOTE: 0 denotes all tasks - - * interTestDelay - this is the time in seconds to delay before - beginning a write or read in a series of tests [0] - NOTE: it does not delay before a check write or - check read - - * outlierThreshold - gives warning if any task is more than this number - of seconds from the mean of all participating tasks. - If so, the task is identified, its time (start, - elapsed create, elapsed transfer, elapsed close, or - end) is reported, as is the mean and standard - deviation for all tasks. The default for this is 0, - which turns it off. If set to a positive value, for - example 3, any task not within 3 seconds of the mean - displays its times. [0] - - * intraTestBarriers - use barrier between open, write/read, and close [0=FALSE] - - * uniqueDir - create and use unique directory for each - file-per-process [0=FALSE] - - * writeFile - writes file(s), first deleting any existing file [1=TRUE] - NOTE: the defaults for writeFile and readFile are - set such that if there is not at least one of - the following -w, -r, -W, or -R, it is assumed - that -w and -r are expected and are - consequently used -- this is only true with - the command line, and may be overridden in - a script - - * readFile - reads existing file(s) (from current or previous - run) [1=TRUE] - NOTE: see writeFile notes - - * filePerProc - accesses a single file for each processor; default - is a single file accessed by all processors [0=FALSE] - - * checkWrite - read data back and check for errors against known - pattern; can be used independently of writeFile [0=FALSE] - NOTES: - data checking is not timed and does not - affect other performance timings - - all errors tallied and returned as program - exit code, unless quitOnError set - - * checkRead - reread data and check for errors between reads; can - be used independently of readFile [0=FALSE] - NOTE: see checkWrite notes - - * keepFile - stops removal of test file(s) on program exit [0=FALSE] - - * keepFileWithError - ensures that with any error found in data-checking, - the error-filled file(s) will not be deleted [0=FALSE] - - * useExistingTestFile - do not remove test file before write access [0=FALSE] - - * segmentCount - number of segments in file [1] - NOTES: - a segment is a contiguous chunk of data - accessed by multiple clients each writing/ - reading their own contiguous data; - comprised of blocks accessed by multiple - clients - - with HDF5 this repeats the pattern of an - entire shared dataset - - * blockSize - size (in bytes) of a contiguous chunk of data - accessed by a single client; it is comprised of one - or more transfers [1048576] - - * transferSize - size (in bytes) of a single data buffer to be - transferred in a single I/O call [262144] - - * verbose - output information [0] - NOTE: this can be set to levels 0-5 on the command - line; repeating the -v flag will increase - verbosity level - - * setTimeStampSignature - set value for time stamp signature [0] - NOTE: used to rerun tests with the exact data - pattern by setting data signature to contain - positive integer value as timestamp to be - written in data file; if set to 0, is - disabled - - * showHelp - display options and help [0=FALSE] - - * storeFileOffset - use file offset as stored signature when writing - file [0=FALSE] - NOTE: this will affect performance measurements - - * memoryPerNode - Allocate memory on each node to simulate real - application memory usage. Accepts a percentage of - node memory (e.g. "50%") on machines that support - sysconf(_SC_PHYS_PAGES) or a size. Allocation will - be split between tasks that share the node. - - * memoryPerTask - Allocate secified amount of memory per task to - simulate real application memory usage. - - * maxTimeDuration - max time in minutes to run tests [0] - NOTES: * setting this to zero (0) unsets this option - * this option allows the current read/write - to complete without interruption - - * deadlineForStonewalling - seconds before stopping write or read phase [0] - NOTES: - used for measuring the amount of data moved - in a fixed time. After the barrier, each - task starts its own timer, begins moving - data, and the stops moving data at a pre- - arranged time. Instead of measuring the - amount of time to move a fixed amount of - data, this option measures the amount of - data moved in a fixed amount of time. The - objective is to prevent tasks slow to - complete from skewing the performance. - - setting this to zero (0) unsets this option - - this option is incompatible w/data checking - - * randomOffset - access is to random, not sequential, offsets within a file [0=FALSE] - NOTES: - this option is currently incompatible with: - -checkRead - -storeFileOffset - -MPIIO collective or useFileView - -HDF5 or NCMPI - * summaryAlways - Always print the long summary for each test. - Useful for long runs that may be interrupted, preventing - the final long summary for ALL tests to be printed. + * ``useExistingTestFile`` - do not remove test file(s) before write phase + (default: 0) + * ``segmentCount`` - number of segments in file, where a segment is a + contiguous chunk of data accessed by multiple clients each writing/reading + their own contiguous data (blocks). The exact semantics of segments + depend on the API used; for example, HDF5 repeats the pattern of an entire + shared dataset. (default: 1) + + * ``blockSize`` - size (in bytes) of a contiguous chunk of data accessed by a + single client. It is comprised of one or more transfers (default: 1048576) + + * ``transferSize`` - size (in bytes) of a single data buffer to be transferred + in a single I/O call (default: 262144) + + * ``verbose`` - output more information about what IOR is doing. Can be set + to levels 0-5; repeating the -v flag will increase verbosity level. + (default: 0) + + * ``setTimeStampSignature`` - Value to use for the time stamp signature. Used + to rerun tests with the exact data pattern by setting data signature to + contain positive integer value as timestamp to be written in data file; if + set to 0, is disabled (default: 0) + + * ``showHelp`` - display options and help (default: 0) + + * ``storeFileOffset`` - use file offset as stored signature when writing file. + This will affect performance measurements (default: 0) + + * ``memoryPerNode`` - allocate memory on each node to simulate real + application memory usage or restrict page cache size. Accepts a percentage + of node memory (e.g. ``50%``) on systems that support + ``sysconf(_SC_PHYS_PAGES)`` or a size. Allocation will be split between + tasks that share the node. (default: 0) + + * ``memoryPerTask`` - allocate specified amount of memory (in bytes) per task + to simulate real application memory usage. (default: 0) + + * ``maxTimeDuration`` - max time (in minutes) to run all tests. Any current + read/write phase is not interrupted; only future I/O phases are cancelled + once this time is exceeded. Value of zero unsets disables. (default: 0) + + * ``deadlineForStonewalling`` - seconds before stopping write or read phase. + Used for measuring the amount of data moved in a fixed time. After the + barrier, each task starts its own timer, begins moving data, and the stops + moving data at a pre-arranged time. Instead of measuring the amount of time + to move a fixed amount of data, this option measures the amount of data + moved in a fixed amount of time. The objective is to prevent straggling + tasks slow from skewing the performance. This option is incompatible with + read-check and write-check modes. Value of zero unsets this option. + (default: 0) + + * ``randomOffset`` - randomize access offsets within test file(s). Currently + incompatible with ``checkRead``, ``storeFileOffset``, MPIIO ``collective`` + and ``useFileView``, and HDF5 and NCMPI APIs. (default: 0) + + * ``summaryAlways`` - Always print the long summary for each test even if the job is interrupted. (default: 0) POSIX-ONLY ^^^^^^^^^^ - * useO_DIRECT - use O_DIRECT for POSIX, bypassing I/O buffers [0] - * singleXferAttempt - will not continue to retry transfer entire buffer - until it is transferred [0=FALSE] - NOTE: when performing a write() or read() in POSIX, - there is no guarantee that the entire - requested size of the buffer will be - transferred; this flag keeps the retrying a - single transfer until it completes or returns - an error + * ``useO_DIRECT`` - use direct I/ for POSIX, bypassing I/O buffers (default: 0) - * fsyncPerWrite - perform fsync after each POSIX write [0=FALSE] - * fsync - perform fsync after POSIX write close [0=FALSE] + * ``singleXferAttempt`` - do not continue to retry transfer entire buffer + until it is transferred. When performing a write() or read() in POSIX, + there is no guarantee that the entire requested size of the buffer will be + transferred; this flag keeps the retrying a single transfer until it + completes or returns an error (default: 0) + + * ``fsyncPerWrite`` - perform fsync after each POSIX write (default: 0) + + * ``fsync`` - perform fsync after POSIX file close (default: 0) MPIIO-ONLY ^^^^^^^^^^ - * preallocate - preallocate the entire file before writing [0=FALSE] - * useFileView - use an MPI datatype for setting the file view option - to use individual file pointer [0=FALSE] - NOTE: default IOR uses explicit file pointers + * ``preallocate`` - preallocate the entire file before writing (default: 0) + + * ``useFileView`` - use an MPI datatype for setting the file view option to + use individual file pointer. Default IOR uses explicit file pointers. + (default: 0) - * useSharedFilePointer - use a shared file pointer [0=FALSE] (not working) - NOTE: default IOR uses explicit file pointers + * ``useSharedFilePointer`` - use a shared file pointer. Default IOR uses + explicit file pointers. (default: 0) - * useStridedDatatype - create a datatype (max=2GB) for strided access; akin - to MULTIBLOCK_REGION_SIZE [0] (not working) + * ``useStridedDatatype`` - create a datatype (max=2GB) for strided access; + akin to ``MULTIBLOCK_REGION_SIZE`` (default: 0) HDF5-ONLY ^^^^^^^^^ - * individualDataSets - within a single file each task will access its own - dataset [0=FALSE] (not working) - NOTE: default IOR creates a dataset the size of - numTasks * blockSize to be accessed by all - tasks - * noFill - no pre-filling of data in HDF5 file creation [0=FALSE] + * ``individualDataSets`` - within a single file, each task will access its own + dataset. Default IOR creates a dataset the size of ``numTasks * blockSize`` + to be accessed by all tasks (default: 0) - * setAlignment - HDF5 alignment in bytes (e.g.: 8, 4k, 2m, 1g) [1] + * ``noFill`` - do not pre-fill data in HDF5 file creation (default: 0) + + * ``setAlignment`` - set the HDF5 alignment in bytes (e.g.: 8, 4k, 2m, 1g) (default: 1) * collectiveMetadata - enable HDF5 collective metadata (available since HDF5-1.10.0) MPIIO-, HDF5-, AND NCMPI-ONLY ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - * collective - uses collective operations for access [0=FALSE] - * showHints - show hint/value pairs attached to open file [0=FALSE] - NOTE: not available in NCMPI + * ``collective`` - uses collective operations for access (default: 0) + + * ``showHints`` - show hint/value pairs attached to open file. Not available + for NCMPI. (default: 0) LUSTRE-SPECIFIC ^^^^^^^^^^^^^^^^^ - * lustreStripeCount - set the lustre stripe count for the test file(s) [0] - * lustreStripeSize - set the lustre stripe size for the test file(s) [0] + * ``lustreStripeCount`` - set the Lustre stripe count for the test file(s) (default: 0) - * lustreStartOST - set the starting OST for the test file(s) [-1] + * ``lustreStripeSize`` - set the Lustre stripe size for the test file(s) (default: 0) - * lustreIgnoreLocks - disable lustre range locking [0] + * ``lustreStartOST`` - set the starting OST for the test file(s) (default: -1) + + * ``lustreIgnoreLocks`` - disable Lustre range locking (default: 0) GPFS-SPECIFIC ^^^^^^^^^^^^^^ - * gpfsHintAccess - use gpfs_fcntl hints to pre-declare accesses - - * gpfsReleaseToken - immediately after opening or creating file, release - all locks. Might help mitigate lock-revocation - traffic when many proceses write/read to same file. + * ``gpfsHintAccess`` - use ``gpfs_fcntl`` hints to pre-declare accesses (default: 0) + * ``gpfsReleaseToken`` - release all locks immediately after opening or + creating file. Might help mitigate lock-revocation traffic when many + proceses write/read to same file. (default: 0) Verbosity levels ---------------------- -The verbosity of output for IOR can be set with -v. Increasing the number of --v instances on a command line sets the verbosity higher. +---------------- + +The verbosity of output for IOR can be set with ``-v``. Increasing the number +of ``-v`` instances on a command line sets the verbosity higher. Here is an overview of the information shown for different verbosity levels: -0) default; only bare essentials shown -1) max clock deviation, participating tasks, free space, access pattern, - commence/verify access notification w/time -2) rank/hostname, machine name, timer used, individual repetition - performance results, timestamp used for data signature -3) full test details, transfer block/offset compared, individual data - checking errors, environment variables, task writing/reading file name, - all test operation times -4) task id and offset for each transfer -5) each 8-byte data signature comparison (WARNING: more data to STDOUT - than stored in file, use carefully) +====== =================================== +Level Behavior +====== =================================== + 0 default; only bare essentials shown + 1 max clock deviation, participating tasks, free space, access pattern, commence/verify access notification with time + 2 rank/hostname, machine name, timer used, individual repetition performance results, timestamp used for data signature + 3 full test details, transfer block/offset compared, individual data checking errors, environment variables, task writing/reading file name, all test operation times + 4 task id and offset for each transfer + 5 each 8-byte data signature comparison (WARNING: more data to STDOUT than stored in file, use carefully) +====== =================================== Incompressible notes -------------------------- +-------------------- Please note that incompressibility is a factor of how large a block compression -algorithm uses. The incompressible buffer is filled only once before write times, -so if the compression algorithm takes in blocks larger than the transfer size, -there will be compression. Below are some baselines that I established for -zip, gzip, and bzip. +algorithm uses. The incompressible buffer is filled only once before write +times, so if the compression algorithm takes in blocks larger than the transfer +size, there will be compression. Below are some baselines for zip, gzip, and +bzip. 1) zip: For zipped files, a transfer size of 1k is sufficient. @@ -355,5 +339,5 @@ zip, gzip, and bzip. To avoid compression a transfer size of greater than the bzip block size is required (default = 900KB). I suggest a transfer size of greather than 1MB to avoid bzip2 compression. -Be aware of the block size your compression algorithm will look at, and adjust the transfer size -accordingly. +Be aware of the block size your compression algorithm will look at, and adjust +the transfer size accordingly. diff --git a/doc/sphinx/userDoc/scripts.rst b/doc/sphinx/userDoc/scripts.rst new file mode 100644 index 00000000..14e00e05 --- /dev/null +++ b/doc/sphinx/userDoc/scripts.rst @@ -0,0 +1,82 @@ +Scripting +========= + +IOR can use an input script with the command line using the ``-f`` option. +**Any options on the command line set before the '-f' option is given will be +considered the default settings for running the script.** For example, :: + + mpirun ./ior -W -f script + +will run all tests in the script with an implicit ``-W``. The script itself can +override these settings and may be set to run many different tests of IOR under +a single execution, and it is important to note that **any command-line options +specified after ``-f`` will not be applied to the runs dictated by the script.** +For example, :: + + mpirun ./ior -f script -W + +will *not* run any tests with the implicit ``-W`` since that argument does not +get applied until after the ``-f`` option (and its constituent runs) are complete. + +Input scripts are specified using the long-form option names that correspond to +each command-line option. In addition to long-form options, + + * ``IOR START`` and ``IOR END`` mark the beginning and end of the script + * ``RUN`` dispatches the test using all of the options specified before it + * All previous set parameter stay set for the next test. They are not reset + to the default! For default the must be rest manually. + * White space is ignored in script, as are comments starting with ``#``. + * Not all test parameters need be set. + +An example of a script: :: + + IOR START + api=[POSIX|MPIIO|HDF5|HDFS|S3|S3_EMC|NCMPI|RADOS] + testFile=testFile + hintsFileName=hintsFile + repetitions=8 + multiFile=0 + interTestDelay=5 + readFile=1 + writeFile=1 + filePerProc=0 + checkWrite=0 + checkRead=0 + keepFile=1 + quitOnError=0 + segmentCount=1 + blockSize=32k + outlierThreshold=0 + setAlignment=1 + transferSize=32 + singleXferAttempt=0 + individualDataSets=0 + verbose=0 + numTasks=32 + collective=1 + preallocate=0 + useFileView=0 + keepFileWithError=0 + setTimeStampSignature=0 + useSharedFilePointer=0 + useStridedDatatype=0 + uniqueDir=0 + fsync=0 + storeFileOffset=0 + maxTimeDuration=60 + deadlineForStonewalling=0 + useExistingTestFile=0 + useO_DIRECT=0 + showHints=0 + showHelp=0 + RUN + # additional tests are optional + transferSize=64 + blockSize=64k + segmentcount=2 + RUN + transferSize=4K + blockSize=1M + segmentcount=1024 + RUN + IOR STOP diff --git a/doc/sphinx/userDoc/skripts.rst b/doc/sphinx/userDoc/skripts.rst deleted file mode 100644 index 964f1acc..00000000 --- a/doc/sphinx/userDoc/skripts.rst +++ /dev/null @@ -1,72 +0,0 @@ -Scripting -========= - -IOR can use a script with the command line. Any options on the command line set -before the script will be considered the default settings for running the script. -(I.e.,'$ ./IOR -W -f script' will have all tests in the script run with the -W -option as default.) -The script itself can override these settings and may be set to run -run many different tests of IOR under a single execution. -The command line is: :: - - ./IOR -f script - -In IOR/scripts, there are scripts of test cases for simulating I/O behavior of -various application codes. Details are included in each script as necessary. - -Syntax: - * IOR START / IOR END: marks the beginning and end of the script - * RUN: Delimiter for next Test - * All previous set parameter stay set for the next test. They are not reset - to the default! For default the musst be rest manually. - * White space is ignored in script, as are comments starting with '#'. - * Not all test parameters need be set. - -An example of a script: :: - - IOR START - api=[POSIX|MPIIO|HDF5|HDFS|S3|S3_EMC|NCMPI|RADOS] - testFile=testFile - hintsFileName=hintsFile - repetitions=8 - multiFile=0 - interTestDelay=5 - readFile=1 - writeFile=1 - filePerProc=0 - checkWrite=0 - checkRead=0 - keepFile=1 - quitOnError=0 - segmentCount=1 - blockSize=32k - outlierThreshold=0 - setAlignment=1 - transferSize=32 - singleXferAttempt=0 - individualDataSets=0 - verbose=0 - numTasks=32 - collective=1 - preallocate=0 - useFileView=0 - keepFileWithError=0 - setTimeStampSignature=0 - useSharedFilePointer=0 - useStridedDatatype=0 - uniqueDir=0 - fsync=0 - storeFileOffset=0 - maxTimeDuration=60 - deadlineForStonewalling=0 - useExistingTestFile=0 - useO_DIRECT=0 - showHints=0 - showHelp=0 - RUN - # additional tests are optional - - RUN - - RUN - IOR STOP From 2333c23bedf4b57e1f105022ef35008097c59e6f Mon Sep 17 00:00:00 2001 From: "Glenn K. Lockwood" Date: Fri, 21 Sep 2018 21:09:39 -0400 Subject: [PATCH 087/206] clean up intro --- doc/sphinx/intro.rst | 34 +++++++++++++++------------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/doc/sphinx/intro.rst b/doc/sphinx/intro.rst index 3964bbc3..11cf138f 100644 --- a/doc/sphinx/intro.rst +++ b/doc/sphinx/intro.rst @@ -1,26 +1,22 @@ Introduction ============ -Welcome to the IOR documentation. +IOR is a parallel IO benchmark that can be used to test the performance of +parallel storage systems using various interfaces and access patterns. The +IOR repository also includes the mdtest benchmark which specifically tests +the peak metadata rates of storage systems under different directory +structures. Both benchmarks use a common parallel I/O abstraction backend +and rely on MPI for synchronization. -**I**\ nterleaved **o**\ r **R**\ andom is a parallel IO benchmark. -IOR can be used for testing performance of parallel file systems using various -interfaces and access patterns. IOR uses MPI for process synchronization. -This documentation provides information for versions 3 and higher, for other -versions check :ref:`compatibility` +This documentation consists of two parts. -This documentation consists of tow parts. +**User documentation** includes installation instructions (:ref:`install`), a +beginner's tutorial (:ref:`first-steps`), and information about IOR's +runtime :ref:`options`. -The first part is a user documentation were you find instructions on compilation, a -beginners tutorial (:ref:`first-steps`) as well as information about all -available :ref:`options`. +**Developer documentation** consists of code documentation generated with +Doxygen and some notes about the contiguous integration with Travis. -The second part is the developer documentation. It currently only consists of a -auto generated Doxygen and some notes about the contiguous integration with travis. -As there are quite some people how needs to modify or extend IOR to there needs -it would be great to have documentation on what and how to alter IOR without -breaking other stuff. Currently there is neither a documentation on the overall -concept of the code nor on implementation details. If you are getting your -hands dirty in code anyways or have deeper understanding of IOR, you are more -then welcome to comment the code directly, which will result in better Doxygen -output or add your insight to this sphinx documentation. +Many aspects of both IOR/mdtest user and developer documentation are incomplete, +and contributors are encouraged to comment the code directly or expand upon this +documentation. From d0237d6d6658ce97f47f94d0f3695f205bb18412 Mon Sep 17 00:00:00 2001 From: "Glenn K. Lockwood" Date: Fri, 21 Sep 2018 21:09:57 -0400 Subject: [PATCH 088/206] skript -> script --- doc/sphinx/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx/index.rst b/doc/sphinx/index.rst index de693ac5..b2532bf0 100644 --- a/doc/sphinx/index.rst +++ b/doc/sphinx/index.rst @@ -13,7 +13,7 @@ userDoc/install First Steps userDoc/options - userDoc/skripts + userDoc/scripts userDoc/compatibility FAQ From 57f47db37f975e4b7f6786581de3ff0fa63c629a Mon Sep 17 00:00:00 2001 From: "Glenn K. Lockwood" Date: Fri, 21 Sep 2018 21:10:52 -0400 Subject: [PATCH 089/206] odds and ends --- doc/sphinx/README.md | 5 +++++ doc/sphinx/changes.rst | 4 ++-- doc/sphinx/conf.py | 8 ++++---- 3 files changed, 11 insertions(+), 6 deletions(-) create mode 100644 doc/sphinx/README.md diff --git a/doc/sphinx/README.md b/doc/sphinx/README.md new file mode 100644 index 00000000..2ec8323a --- /dev/null +++ b/doc/sphinx/README.md @@ -0,0 +1,5 @@ +To build the documentation in this directory, + + sphinx-build . _build_html + +The output will be saved as html in `_build_html/`. diff --git a/doc/sphinx/changes.rst b/doc/sphinx/changes.rst index 3632a7d8..c224e64b 100644 --- a/doc/sphinx/changes.rst +++ b/doc/sphinx/changes.rst @@ -1,4 +1,4 @@ Changes in IOR -***************** +************** -.. .. include:: ../../ChangeLog +.. .. include:: ../../NEWS diff --git a/doc/sphinx/conf.py b/doc/sphinx/conf.py index 90628fd9..688303ec 100644 --- a/doc/sphinx/conf.py +++ b/doc/sphinx/conf.py @@ -64,9 +64,9 @@ master_doc = 'index' # General information about the project. -project = u'IOR' -copyright = u'2017, IOR' -author = u'IOR' +project = u'ior' +copyright = u'2018, The Regents of the University of California' +author = u'The IOR Developer Community' # The version info for the project you're documenting, acts as replacement for @@ -197,7 +197,7 @@ # dir menu entry, description, category) texinfo_documents = [ (master_doc, 'IOR', u'IOR Documentation', - author, 'IOR', 'One line description of project.', + author, 'IOR', 'IOR/mdtest Parallel I/O Benchmarks', 'Miscellaneous'), ] From aa3f045c92d8bd887a78f5b4b5261b2b17b50317 Mon Sep 17 00:00:00 2001 From: "Glenn K. Lockwood" Date: Fri, 21 Sep 2018 21:12:23 -0400 Subject: [PATCH 090/206] fix spelling mistake --- doc/sphinx/devDoc/CI.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/sphinx/devDoc/CI.rst b/doc/sphinx/devDoc/CI.rst index 39cbf9e8..43f1b5c3 100644 --- a/doc/sphinx/devDoc/CI.rst +++ b/doc/sphinx/devDoc/CI.rst @@ -1,8 +1,8 @@ -Continues Integration -===================== +Continuous Integration +====================== -Continues Integration is used for basic sanity checking. Travis-CI provides free -CI for open source github projects and is configured via a .travis.yml. +Continuous Integration is used for basic sanity checking. Travis-CI provides free +CI for open source github projects and is configured via a ``.travis.yml``. For now this is set up to compile IOR on a ubuntu 14.04 machine with gcc 4.8, openmpi and hdf5 for the backends. This is a pretty basic check and should be From 2241f77a5d8edbabbd5aa95942765076974317d0 Mon Sep 17 00:00:00 2001 From: "Glenn K. Lockwood" Date: Fri, 21 Sep 2018 21:18:28 -0400 Subject: [PATCH 091/206] got NEWS importing correctly --- doc/sphinx/changes.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx/changes.rst b/doc/sphinx/changes.rst index c224e64b..c11d45e1 100644 --- a/doc/sphinx/changes.rst +++ b/doc/sphinx/changes.rst @@ -1,4 +1,4 @@ Changes in IOR ************** -.. .. include:: ../../NEWS +.. include:: ../../NEWS From 0bb96e96841484e3cd9226aa9337c75264b282b8 Mon Sep 17 00:00:00 2001 From: "Glenn K. Lockwood" Date: Sat, 22 Dec 2018 20:57:44 -0800 Subject: [PATCH 092/206] added docs for release process --- doc/sphinx/devDoc/release.rst | 37 +++++++++++++++++++++++++++++++++++ doc/sphinx/index.rst | 1 + 2 files changed, 38 insertions(+) create mode 100644 doc/sphinx/devDoc/release.rst diff --git a/doc/sphinx/devDoc/release.rst b/doc/sphinx/devDoc/release.rst new file mode 100644 index 00000000..3abe6d85 --- /dev/null +++ b/doc/sphinx/devDoc/release.rst @@ -0,0 +1,37 @@ +Release Process +=============== + +To build a new version of IOR, + + $ docker run -it ubuntu bash + $ apt-get update + $ apt-get install -y git automake autoconf make gcc mpich + $ git clone -b rc https://github.com/hpc/ior + $ cd ior + $ ./travis-build.sh + +To create a new release candidate from RC, + +1. Disable the `check-news` option in `AM_INIT_AUTOMAKE` inside configure.ac +2. Append "rcX" to the `Version:` field in META +3. Build a release package as described above + +To create a new minor release of IOR, + +1. Build the rc branch as described above +2. Create a release on GitHub which creates the appropriate tag +3. Upload the source distributions generated by travis-build.sh + +To create a micro branch of IOR (e.g., if a release needs a hotfix), + +1. Check out the relevant release tagged in the rc branch (e.g., `3.2.0`) +2. Create a branch with the major.minor name (e.g., `v3.2`) from that tag +3. Update the `Version:` in META +4. Apply hotfix(es) to that major.minor branch +5. Create the major.minor.micro release on GitHub + +To initiate a feature freeze, + +1. Merge the `master` branch into the `rc` branch +2. Update the `Version:` field in META _of the master branch_ to be the _next_ + release version, not the one whose features have just been frozen diff --git a/doc/sphinx/index.rst b/doc/sphinx/index.rst index b2532bf0..fe3abb78 100644 --- a/doc/sphinx/index.rst +++ b/doc/sphinx/index.rst @@ -24,6 +24,7 @@ devDoc/doxygen devDoc/CI + devDoc/release .. toctree:: :hidden: From 23106d17dbb74674da297d8ecda35c67d6ac77f8 Mon Sep 17 00:00:00 2001 From: "Glenn K. Lockwood" Date: Sat, 22 Dec 2018 21:09:04 -0800 Subject: [PATCH 093/206] forgot to RST-ify this --- doc/sphinx/devDoc/release.rst | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/doc/sphinx/devDoc/release.rst b/doc/sphinx/devDoc/release.rst index 3abe6d85..1e39a35e 100644 --- a/doc/sphinx/devDoc/release.rst +++ b/doc/sphinx/devDoc/release.rst @@ -1,7 +1,7 @@ Release Process =============== -To build a new version of IOR, +To build a new version of IOR:: $ docker run -it ubuntu bash $ apt-get update @@ -12,8 +12,9 @@ To build a new version of IOR, To create a new release candidate from RC, -1. Disable the `check-news` option in `AM_INIT_AUTOMAKE` inside configure.ac -2. Append "rcX" to the `Version:` field in META +1. Disable the ``check-news`` option in ``AM_INIT_AUTOMAKE`` inside configure.ac +2. Append "rcX" to the ``Version:`` field in META where X is the release + candidate number 3. Build a release package as described above To create a new minor release of IOR, @@ -24,14 +25,14 @@ To create a new minor release of IOR, To create a micro branch of IOR (e.g., if a release needs a hotfix), -1. Check out the relevant release tagged in the rc branch (e.g., `3.2.0`) -2. Create a branch with the major.minor name (e.g., `v3.2`) from that tag -3. Update the `Version:` in META +1. Check out the relevant release tagged in the rc branch (e.g., ``3.2.0``) +2. Create a branch with the major.minor name (e.g., ``3.2``) from that tag +3. Update the ``Version:`` in META 4. Apply hotfix(es) to that major.minor branch 5. Create the major.minor.micro release on GitHub To initiate a feature freeze, -1. Merge the `master` branch into the `rc` branch -2. Update the `Version:` field in META _of the master branch_ to be the _next_ +1. Merge the master branch into the rc branch +2. Update the ``Version:`` field in META `of the master branch` to be the `next` release version, not the one whose features have just been frozen From cf197d5e381f2e1ee4074373399a85c6dc06f083 Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Wed, 22 May 2019 10:21:18 +0100 Subject: [PATCH 094/206] Moved broken collectiveMetadata option into HDF5 backend module. #154 --- doc/sphinx/userDoc/options.rst | 3 +-- src/aiori-HDF5.c | 32 +++++++++++++++++++++++++++++++- src/ior.h | 2 -- 3 files changed, 32 insertions(+), 5 deletions(-) diff --git a/doc/sphinx/userDoc/options.rst b/doc/sphinx/userDoc/options.rst index 8b441269..22728306 100644 --- a/doc/sphinx/userDoc/options.rst +++ b/doc/sphinx/userDoc/options.rst @@ -289,8 +289,7 @@ HDF5-ONLY * setAlignment - HDF5 alignment in bytes (e.g.: 8, 4k, 2m, 1g) [1] - * collectiveMetadata - enable HDF5 collective metadata (available since - HDF5-1.10.0) + * hdf5.collectiveMetadata - enable HDF5 collective metadata (available since HDF5-1.10.0) MPIIO-, HDF5-, AND NCMPI-ONLY ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/src/aiori-HDF5.c b/src/aiori-HDF5.c index 250f93d1..ab329dbb 100755 --- a/src/aiori-HDF5.c +++ b/src/aiori-HDF5.c @@ -94,6 +94,34 @@ static void HDF5_Fsync(void *, IOR_param_t *); static IOR_offset_t HDF5_GetFileSize(IOR_param_t *, MPI_Comm, char *); static int HDF5_Access(const char *, int, IOR_param_t *); +/************************** O P T I O N S *****************************/ +typedef struct{ + int collective_md; +} HDF5_options_t; +/***************************** F U N C T I O N S ******************************/ + +static option_help * HDF5_options(void ** init_backend_options, void * init_values){ + HDF5_options_t * o = malloc(sizeof(HDF5_options_t)); + + if (init_values != NULL){ + memcpy(o, init_values, sizeof(HDF5_options_t)); + }else{ + /* initialize the options properly */ + o->collective_md = 0; + } + + *init_backend_options = o; + + option_help h [] = { + {0, "hdf5.collectiveMetadata", "Use collectiveMetadata (available since HDF5-1.10.0)", OPTION_FLAG, 'd', & o->collective_md}, + LAST_OPTION + }; + option_help * help = malloc(sizeof(h)); + memcpy(help, h, sizeof(h)); + return help; +} + + /************************** D E C L A R A T I O N S ***************************/ ior_aiori_t hdf5_aiori = { @@ -112,6 +140,7 @@ ior_aiori_t hdf5_aiori = { .rmdir = aiori_posix_rmdir, .access = HDF5_Access, .stat = aiori_posix_stat, + .get_options = HDF5_options }; static hid_t xferPropList; /* xfer property list */ @@ -230,7 +259,8 @@ static void *HDF5_Open(char *testFileName, IOR_param_t * param) "cannot set alignment"); #ifdef HAVE_H5PSET_ALL_COLL_METADATA_OPS - if (param->collective_md) { + HDF5_options_t *o = (HDF5_options_t*) param->backend_options; + if (o->collective_md) { /* more scalable metadata */ HDF5_CHECK(H5Pset_all_coll_metadata_ops(accessPropList, 1), diff --git a/src/ior.h b/src/ior.h index da4e5c18..5c46efe7 100755 --- a/src/ior.h +++ b/src/ior.h @@ -183,8 +183,6 @@ typedef struct char* URI; /* "path" to target object */ size_t part_number; /* multi-part upload increment (PER-RANK!) */ char* UploadId; /* key for multi-part-uploads */ - int collective_md; /* use collective metatata optimization */ - /* RADOS variables */ rados_t rados_cluster; /* RADOS cluster handle */ From 3d8893e507ef0c0e129bf6bd517fbb985330469e Mon Sep 17 00:00:00 2001 From: Vaclav Hapla Date: Wed, 22 May 2019 15:25:21 +0200 Subject: [PATCH 095/206] Fix #155. Fixup 3a873ea fixing #150. --- src/ior-output.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ior-output.c b/src/ior-output.c index eaf5bab6..4bdb37dc 100644 --- a/src/ior-output.c +++ b/src/ior-output.c @@ -450,7 +450,7 @@ void ShowSetup(IOR_param_t *params) if (params->lustre_set_striping) { PrintKeyVal("Lustre stripe size", ((params->lustre_stripe_size == 0) ? "Use default" : HumanReadable(params->lustre_stripe_size, BASE_TWO))); - PrintKeyVal("stripe count", (params->lustre_stripe_count == 0 ? "Use default" : params->lustre_stripe_count) ); + PrintKeyValInt("Lustre stripe count", params->lustre_stripe_count); } #endif /* HAVE_LUSTRE_LUSTRE_USER_H */ if (params->deadlineForStonewalling > 0) { From a6f8388f9f8353efec493a69dfb2ba93143c0fb5 Mon Sep 17 00:00:00 2001 From: Gu Zheng Date: Mon, 8 Oct 2018 20:30:47 +0800 Subject: [PATCH 096/206] add mknod support Add new option '-k' to support file creation test with mknod, which is widely used in lustre. Signed-off-by: Gu Zheng --- src/aiori-POSIX.c | 19 +++++++++++++++++++ src/aiori.h | 2 ++ src/mdtest.c | 25 ++++++++++++++++++++----- 3 files changed, 41 insertions(+), 5 deletions(-) diff --git a/src/aiori-POSIX.c b/src/aiori-POSIX.c index 5d38accf..d78cd40a 100755 --- a/src/aiori-POSIX.c +++ b/src/aiori-POSIX.c @@ -107,6 +107,7 @@ ior_aiori_t posix_aiori = { .name = "POSIX", .name_legacy = NULL, .create = POSIX_Create, + .mknod = POSIX_Mknod, .open = POSIX_Open, .xfer = POSIX_Xfer, .close = POSIX_Close, @@ -427,6 +428,24 @@ void *POSIX_Create(char *testFileName, IOR_param_t * param) return ((void *)fd); } +/* + * Creat a file through mknod interface. + */ +void *POSIX_Mknod(char *testFileName) +{ + int *fd; + + fd = (int *)malloc(sizeof(int)); + if (fd == NULL) + ERR("Unable to malloc file descriptor"); + + *fd = mknod(testFileName, S_IFREG | S_IRUSR, 0); + if (*fd < 0) + ERR("mknod failed"); + + return ((void *)fd); +} + /* * Open a file through the POSIX interface. */ diff --git a/src/aiori.h b/src/aiori.h index d8cabb5d..c2074c2f 100755 --- a/src/aiori.h +++ b/src/aiori.h @@ -68,6 +68,7 @@ typedef struct ior_aiori { char *name; char *name_legacy; void *(*create)(char *, IOR_param_t *); + void *(*mknod)(char *); void *(*open)(char *, IOR_param_t *); IOR_offset_t (*xfer)(int, void *, IOR_size_t *, IOR_offset_t, IOR_param_t *); @@ -125,6 +126,7 @@ int aiori_posix_access (const char *path, int mode, IOR_param_t * param); int aiori_posix_stat (const char *path, struct stat *buf, IOR_param_t * param); void *POSIX_Create(char *testFileName, IOR_param_t * param); +void *POSIX_Mknod(char *testFileName); void *POSIX_Open(char *testFileName, IOR_param_t * param); IOR_offset_t POSIX_GetFileSize(IOR_param_t * test, MPI_Comm testComm, char *testFileName); void POSIX_Delete(char *testFileName, IOR_param_t * param); diff --git a/src/mdtest.c b/src/mdtest.c index d65d6f94..8613e3b0 100644 --- a/src/mdtest.c +++ b/src/mdtest.c @@ -145,6 +145,7 @@ static size_t read_bytes; static int sync_file; static int path_count; static int nstride; /* neighbor stride */ +static int make_node = 0; static mdtest_results_t * summary_table; static pid_t pid; @@ -337,7 +338,10 @@ static void create_file (const char *path, uint64_t itemNum) { fflush( out_logfile ); } - aiori_fh = backend->open (curr_item, ¶m); + if (make_node) + aiori_fh = backend->mknod (curr_item); + else + aiori_fh = backend->open (curr_item, ¶m); if (NULL == aiori_fh) { FAIL("unable to open file"); } @@ -354,7 +358,10 @@ static void create_file (const char *path, uint64_t itemNum) { fflush( out_logfile ); } - aiori_fh = backend->create (curr_item, ¶m); + if (make_node) + aiori_fh = backend->mknod (curr_item); + else + aiori_fh = backend->create (curr_item, ¶m); if (NULL == aiori_fh) { FAIL("unable to create file"); } @@ -382,7 +389,8 @@ static void create_file (const char *path, uint64_t itemNum) { fflush( out_logfile ); } - backend->close (aiori_fh, ¶m); + if (!make_node) + backend->close (aiori_fh, ¶m); } /* helper for creating/removing items */ @@ -1328,7 +1336,7 @@ void print_help (void) { fprintf(out_logfile, "Usage: mdtest [-b branching_factor] [-B] [-c] [-C] [-d testdir] [-D] [-e number_of_bytes_to_read]\n" - " [-E] [-f first] [-F] [-h] [-i iterations] [-I items_per_dir] [-l last] [-L]\n" + " [-E] [-f first] [-F] [-h] [-i iterations] [-I items_per_dir] [-k] [-l last] [-L]\n" " [-n number_of_items] [-N stride_length] [-p seconds] [-r]\n" " [-R[seed]] [-s stride] [-S] [-t] [-T] [-u] [-v] [-a API]\n" " [-V verbosity_value] [-w number_of_bytes_to_write] [-W seconds] [-y] [-z depth] -Z\n" @@ -1346,6 +1354,7 @@ void print_help (void) { "\t-h: prints this help message\n" "\t-i: number of iterations the test will run\n" "\t-I: number of items per directory in tree\n" + "\t-k: use mknod\n" "\t-l: last number of tasks on which the test will run\n" "\t-L: files only at leaf level of tree\n" "\t-n: every process will creat/stat/read/remove # directories and files\n" @@ -1610,7 +1619,10 @@ void valid_tests() { FAIL("items + items_per_dir can only be set without stonewalling"); } } - + /* check for using mknod */ + if (write_bytes > 0 && make_node) { + FAIL("-k not compatible with -w"); + } } void show_file_system_size(char *file_system) { @@ -2096,6 +2108,7 @@ void mdtest_init_args(){ sync_file = 0; path_count = 0; nstride = 0; + make_node = 0; } mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * world_out) { @@ -2144,6 +2157,7 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * {'F', NULL, "perform test on files only (no directories)", OPTION_FLAG, 'd', & files_only}, {'i', NULL, "number of iterations the test will run", OPTION_OPTIONAL_ARGUMENT, 'd', & iterations}, {'I', NULL, "number of items per directory in tree", OPTION_OPTIONAL_ARGUMENT, 'l', & items_per_dir}, + {'k', NULL, "use mknod to create file", OPTION_FLAG, 'd', & make_node}, {'l', NULL, "last number of tasks on which the test will run", OPTION_OPTIONAL_ARGUMENT, 'd', & last}, {'L', NULL, "files only at leaf level of tree", OPTION_FLAG, 'd', & leaf_only}, {'n', NULL, "every process will creat/stat/read/remove # directories and files", OPTION_OPTIONAL_ARGUMENT, 'l', & items}, @@ -2249,6 +2263,7 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * fprintf( out_logfile, "write_bytes : "LLU"\n", write_bytes ); fprintf( out_logfile, "sync_file : %s\n", ( sync_file ? "True" : "False" )); fprintf( out_logfile, "depth : %d\n", depth ); + fprintf( out_logfile, "make_node : %d\n", make_node ); fflush( out_logfile ); } From ed4c1d88d0064694783695c0b37b4c77264b828d Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Thu, 23 May 2019 09:31:05 +0100 Subject: [PATCH 097/206] Fix missing initialization in MMAP options. --- src/aiori-MMAP.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aiori-MMAP.c b/src/aiori-MMAP.c index dd6c6490..7be860a6 100644 --- a/src/aiori-MMAP.c +++ b/src/aiori-MMAP.c @@ -64,7 +64,7 @@ static option_help * MMAP_options(void ** init_backend_options, void * init_valu if (init_values != NULL){ memcpy(o, init_values, sizeof(mmap_options_t)); }else{ - o->direct_io_ignored = 0; + memset(o, 0, sizeof(mmap_options_t)); } *init_backend_options = o; From 1b3182904d9840b9542ab6ed9dbc6d9733078562 Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Thu, 23 May 2019 09:53:51 +0100 Subject: [PATCH 098/206] Moving IME ODirect option into IME. #157 --- src/aiori-IME.c | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/src/aiori-IME.c b/src/aiori-IME.c index ef7b3cfb..b6cef34d 100755 --- a/src/aiori-IME.c +++ b/src/aiori-IME.c @@ -54,6 +54,33 @@ static int IME_Stat(const char *, struct stat *, IOR_param_t *); static void IME_Initialize(); static void IME_Finalize(); + +/************************** O P T I O N S *****************************/ +typedef struct{ + int direct_io; +} ime_options_t; + + +option_help * IME_options(void ** init_backend_options, void * init_values){ + ime_options_t * o = malloc(sizeof(ime_options_t)); + + if (init_values != NULL){ + memcpy(o, init_values, sizeof(ime_options_t)); + }else{ + o->direct_io = 0; + } + + *init_backend_options = o; + + option_help h [] = { + {0, "ime.odirect", "Direct I/O Mode", OPTION_FLAG, 'd', & o->direct_io}, + LAST_OPTION + }; + option_help * help = malloc(sizeof(h)); + memcpy(help, h, sizeof(h)); + return help; +} + /************************** D E C L A R A T I O N S ***************************/ extern int rank; @@ -79,6 +106,7 @@ ior_aiori_t ime_aiori = { .stat = IME_Stat, .initialize = IME_Initialize, .finalize = IME_Finalize, + .get_options = IME_options, .enable_mdtest = true, }; @@ -130,8 +158,10 @@ static void *IME_Open(char *testFileName, IOR_param_t *param) if (fd == NULL) ERR("Unable to malloc file descriptor"); - if (param->useO_DIRECT) - set_o_direct_flag(&fd_oflag); + ime_options_t * o = (ime_options_t*) param->backend_options; + if (o->direct_io == TRUE){ + set_o_direct_flag(&fd_oflag); + } if (param->openFlags & IOR_RDONLY) fd_oflag |= O_RDONLY; From e4969b5af71516c0e8059ae82b05b7703d5ab23f Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Thu, 23 May 2019 12:47:43 +0100 Subject: [PATCH 099/206] Fix for #159 (also slightly improved parsing for options, for portability between scripts also allow comment at the end of an option). --- src/parse_options.c | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/parse_options.c b/src/parse_options.c index 153fef8e..ba72413c 100755 --- a/src/parse_options.c +++ b/src/parse_options.c @@ -92,7 +92,7 @@ void DecodeDirective(char *line, IOR_param_t *params, options_all_t * module_opt char value[MAX_STR]; int rc; int initialized; - + rc = sscanf(line, " %[^=# \t\r\n] = %[^# \t\r\n] ", option, value); if (rc != 2 && rank == 0) { fprintf(out_logfile, "Syntax error in configuration options: %s\n", @@ -317,17 +317,26 @@ void ParseLine(char *line, IOR_param_t * test, options_all_t * module_options) { char *start, *end; - start = line; + char * newline = strdup(line); + start = newline; do { end = strchr(start, '#'); - if (end != NULL) + if (end != NULL){ *end = '\0'; + end = NULL; // stop parsing after comment + } end = strchr(start, ','); - if (end != NULL) - *end = '\0'; + if (end != NULL){ + *end = '\0'; + } + if(strlen(start) < 3){ + fprintf(out_logfile, "Invalid option substring string: \"%s\" in \"%s\"\n", start, line); + exit(1); + } DecodeDirective(start, test, module_options); start = end + 1; } while (end != NULL); + free(newline); } From af1aa627ebe1ed8581b8ddb3472ff51689e73fd4 Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Fri, 7 Jun 2019 16:59:11 +0000 Subject: [PATCH 100/206] use d_ version of function instead of daos_ since some of those are going to be removed from the DAOS API. Signed-off-by: Mohamad Chaarawi --- src/aiori-DAOS.c | 10 +++++----- src/aiori-DFS.c | 9 +++++---- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/aiori-DAOS.c b/src/aiori-DAOS.c index 751e5a72..922975d8 100644 --- a/src/aiori-DAOS.c +++ b/src/aiori-DAOS.c @@ -151,7 +151,7 @@ do { \ static void HandleDistribute(daos_handle_t *handle, enum handleType type) { - daos_iov_t global; + d_iov_t global; int rc; global.iov_buf = NULL; @@ -281,7 +281,7 @@ DAOS_Init() rc = daos_pool_connect(uuid, o.group, svcl, DAOS_PC_RW, &poh, &po_info, NULL); - daos_rank_list_free(svcl); + d_rank_list_free(svcl); DCHECK(rc, "Failed to connect to pool %s", o.pool); INFO(VERBOSE_1, "Create/Open Container %s", o.cont); @@ -433,8 +433,8 @@ DAOS_Xfer(int access, void *file, IOR_size_t *buffer, { daos_array_iod_t iod; daos_range_t rg; - daos_sg_list_t sgl; - daos_iov_t iov; + d_sg_list_t sgl; + d_iov_t iov; int rc; /** set array location */ @@ -445,7 +445,7 @@ DAOS_Xfer(int access, void *file, IOR_size_t *buffer, /** set memory location */ sgl.sg_nr = 1; - daos_iov_set(&iov, buffer, length); + d_iov_set(&iov, buffer, length); sgl.sg_iovs = &iov; if (access == WRITE) { diff --git a/src/aiori-DFS.c b/src/aiori-DFS.c index 86bd1759..f9f945ca 100755 --- a/src/aiori-DFS.c +++ b/src/aiori-DFS.c @@ -35,6 +35,7 @@ #include #include +#include #include #include #include @@ -269,7 +270,7 @@ DFS_Init() { rc = daos_pool_connect(pool_uuid, o.group, svcl, DAOS_PC_RW, &poh, &pool_info, NULL); DCHECK(rc, "Failed to connect to pool"); - + d_rank_list_free(svcl); rc = daos_cont_open(poh, co_uuid, DAOS_COO_RW, &coh, &co_info, NULL); /* If NOEXIST we create it */ if (rc == -DER_NONEXIST) { @@ -402,13 +403,13 @@ DFS_Xfer(int access, void *file, IOR_size_t *buffer, IOR_offset_t length, obj = (dfs_obj_t *)file; while (remaining > 0) { - daos_iov_t iov; - daos_sg_list_t sgl; + d_iov_t iov; + d_sg_list_t sgl; /** set memory location */ sgl.sg_nr = 1; sgl.sg_nr_out = 0; - daos_iov_set(&iov, (void *)ptr, remaining); + d_iov_set(&iov, (void *)ptr, remaining); sgl.sg_iovs = &iov; /* write/read file */ From 915283bae85813984da7247200b68f0fe725fed3 Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Mon, 17 Jun 2019 20:38:02 +0000 Subject: [PATCH 101/206] update array_g2l usage with DAOS API change Signed-off-by: Mohamad Chaarawi --- src/aiori-DAOS.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aiori-DAOS.c b/src/aiori-DAOS.c index 922975d8..6557d133 100644 --- a/src/aiori-DAOS.c +++ b/src/aiori-DAOS.c @@ -198,7 +198,7 @@ HandleDistribute(daos_handle_t *handle, enum handleType type) else if (type == CONT_HANDLE) rc = daos_cont_global2local(poh, global, handle); else - rc = daos_array_global2local(coh, global, handle); + rc = daos_array_global2local(coh, global, 0, handle); DCHECK(rc, "Failed to get local handle"); } From 6cae44b512ce9eae77bccf2e1b5a24e45ce80637 Mon Sep 17 00:00:00 2001 From: Osamu Tatebe Date: Fri, 21 Jun 2019 21:05:41 +0900 Subject: [PATCH 102/206] abstract I/O interface for Gfarm file system --- configure.ac | 20 ++- src/Makefile.am | 5 + src/aiori-Gfarm.c | 316 ++++++++++++++++++++++++++++++++++++++++++++++ src/aiori.c | 3 + src/aiori.h | 1 + 5 files changed, 344 insertions(+), 1 deletion(-) create mode 100644 src/aiori-Gfarm.c diff --git a/configure.ac b/configure.ac index 80e8533f..7d8e4a74 100755 --- a/configure.ac +++ b/configure.ac @@ -185,7 +185,25 @@ AM_COND_IF([USE_RADOS_AIORI],[ AC_DEFINE([USE_RADOS_AIORI], [], [Build RADOS backend AIORI]) ]) - +# Gfarm support +AC_MSG_CHECKING([for Gfarm file system]) +AC_ARG_WITH([gfarm], + [AS_HELP_STRING([--with-gfarm=GFARM_ROOT], + [support IO with libgfarm backend @<:@default=no@:>@])], + [], [with_gfarm=no]) +AC_MSG_RESULT([$with_gfarm]) +AM_CONDITIONAL([USE_GFARM_AIORI], [test x$with_gfarm != xno]) +if test x$with_gfarm != xno; then + AC_DEFINE([USE_GFARM_AIORI], [], [Build Gfarm backend AIORI]) + case x$with_gfarm in + xyes) ;; + *) + CPPFLAGS="$CPPFLAGS -I$with_gfarm/include" + LDFLAGS="$LDFLAGS -L$with_gfarm/lib" ;; + esac + AC_CHECK_LIB([gfarm], [gfarm_initialize],, [AC_MSG_ERROR([libgfarm not found])]) + AC_CHECK_MEMBERS([struct stat.st_mtim.tv_nsec]) +fi # aws4c is needed for the S3 backend (see --with-S3, below). # Version 0.5.2 of aws4c is available at https://github.com/jti-lanl/aws4c.git diff --git a/src/Makefile.am b/src/Makefile.am index 51fb8734..74dcd31a 100755 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -70,6 +70,11 @@ extraSOURCES += aiori-RADOS.c extraLDADD += -lrados endif +if USE_GFARM_AIORI +extraSOURCES += aiori-Gfarm.c +extraLDADD += -lgfarm +endif + if USE_S3_AIORI extraSOURCES += aiori-S3.c if AWS4C_DIR diff --git a/src/aiori-Gfarm.c b/src/aiori-Gfarm.c new file mode 100644 index 00000000..cc835f1a --- /dev/null +++ b/src/aiori-Gfarm.c @@ -0,0 +1,316 @@ +#include +#include +#include +#include +#include +#undef PACKAGE_NAME +#undef PACKAGE_STRING +#undef PACKAGE_TARNAME +#undef PACKAGE_VERSION +#include "ior.h" +#include "aiori.h" + +struct gfarm_file { + GFS_File gf; +}; + +void +Gfarm_initialize() +{ + gfarm_initialize(NULL, NULL); +} + +void +Gfarm_finalize() +{ + gfarm_terminate(); +} + +void * +Gfarm_create(char *fn, IOR_param_t *param) +{ + GFS_File gf; + struct gfarm_file *fp; + gfarm_error_t e; + + if (param->dryRun) + return (NULL); + + e = gfs_pio_create(fn, GFARM_FILE_RDWR, 0664, &gf); + if (e != GFARM_ERR_NO_ERROR) + ERR("gfs_pio_create failed"); + GFARM_MALLOC(fp); + if (fp == NULL) + ERR("no memory"); + fp->gf = gf; + return (fp); +} + +void * +Gfarm_open(char *fn, IOR_param_t *param) +{ + GFS_File gf; + struct gfarm_file *fp; + gfarm_error_t e; + + if (param->dryRun) + return (NULL); + + e = gfs_pio_open(fn, GFARM_FILE_RDWR, &gf); + if (e != GFARM_ERR_NO_ERROR) + ERR("gfs_pio_open failed"); + GFARM_MALLOC(fp); + if (fp == NULL) + ERR("no memory"); + fp->gf = gf; + return (fp); +} + +IOR_offset_t +Gfarm_xfer(int access, void *fd, IOR_size_t *buffer, IOR_offset_t len, + IOR_param_t *param) +{ + struct gfarm_file *fp = fd; + IOR_offset_t rem = len; + gfarm_off_t off; + gfarm_error_t e; +#define MAX_SZ (1024 * 1024 * 1024) + int sz, n; + char *buf = (char *)buffer; + + if (param->dryRun) + return (len); + + if (len > MAX_SZ) + sz = MAX_SZ; + else + sz = len; + + e = gfs_pio_seek(fp->gf, param->offset, GFARM_SEEK_SET, &off); + if (e != GFARM_ERR_NO_ERROR) + ERR("gfs_pio_seek failed"); + while (rem > 0) { + if (access == WRITE) + e = gfs_pio_write(fp->gf, buf, sz, &n); + else + e = gfs_pio_read(fp->gf, buf, sz, &n); + if (e != GFARM_ERR_NO_ERROR) + ERR("xfer failed"); + if (n == 0) + ERR("EOF encountered"); + rem -= n; + buf += n; + } + return (len); +} + +void +Gfarm_close(void *fd, IOR_param_t *param) +{ + struct gfarm_file *fp = fd; + + if(param->dryRun) + return; + + if (gfs_pio_close(fp->gf) != GFARM_ERR_NO_ERROR) + ERR("gfs_pio_close failed"); + free(fp); +} + +void +Gfarm_delete(char *fn, IOR_param_t *param) +{ + gfarm_error_t e; + + if (param->dryRun) + return; + + e = gfs_unlink(fn); + if (e != GFARM_ERR_NO_ERROR) + errno = gfarm_error_to_errno(e); +} + +char * +Gfarm_version() +{ + return ((char *)gfarm_version()); +} + +void +Gfarm_fsync(void *fd, IOR_param_t *param) +{ + struct gfarm_file *fp = fd; + + if(param->dryRun) + return; + + if (gfs_pio_sync(fp->gf) != GFARM_ERR_NO_ERROR) + ERR("gfs_pio_sync failed"); +} + +IOR_offset_t +Gfarm_get_file_size(IOR_param_t *param, MPI_Comm comm, char *fn) +{ + struct gfs_stat st; + IOR_offset_t size, sum, min, max; + + if (param->dryRun) + return (0); + + if (gfs_stat(fn, &st) != GFARM_ERR_NO_ERROR) + ERR("gfs_stat failed"); + size = st.st_size; + gfs_stat_free(&st); + + if (param->filePerProc == TRUE) { + MPI_CHECK(MPI_Allreduce(&size, &sum, 1, MPI_LONG_LONG_INT, + MPI_SUM, comm), "cannot total data moved"); + size = sum; + } else { + MPI_CHECK(MPI_Allreduce(&size, &min, 1, MPI_LONG_LONG_INT, + MPI_MIN, comm), "cannot total data moved"); + MPI_CHECK(MPI_Allreduce(&size, &max, 1, MPI_LONG_LONG_INT, + MPI_MAX, comm), "cannot total data moved"); + if (min != max) { + if (rank == 0) + WARN("inconsistent file size by different " + "tasks"); + /* incorrect, but now consistent across tasks */ + size = min; + } + } + return (size); +} + +int +Gfarm_statfs(const char *fn, ior_aiori_statfs_t *st, IOR_param_t *param) +{ + gfarm_off_t used, avail, files; + gfarm_error_t e; + int bsize = 4096; + + if (param->dryRun) + return (0); + + e = gfs_statfs_by_path(fn, &used, &avail, &files); + if (e != GFARM_ERR_NO_ERROR) { + errno = gfarm_error_to_errno(e); + return (-1); + } + st->f_bsize = bsize; + st->f_blocks = (used + avail) / bsize; + st->f_bfree = avail / bsize; + st->f_files = 2 * files; /* XXX */ + st->f_ffree = files; /* XXX */ + return (0); +} + +int +Gfarm_mkdir(const char *fn, mode_t mode, IOR_param_t *param) +{ + gfarm_error_t e; + + if (param->dryRun) + return (0); + + e = gfs_mkdir(fn, mode); + if (e == GFARM_ERR_NO_ERROR) + return (0); + errno = gfarm_error_to_errno(e); + return (-1); +} + +int +Gfarm_rmdir(const char *fn, IOR_param_t *param) +{ + gfarm_error_t e; + + if (param->dryRun) + return (0); + + e = gfs_rmdir(fn); + if (e == GFARM_ERR_NO_ERROR) + return (0); + errno = gfarm_error_to_errno(e); + return (-1); +} + +int +Gfarm_access(const char *fn, int mode, IOR_param_t *param) +{ + struct gfs_stat st; + gfarm_error_t e; + + if (param->dryRun) + return (0); + + e = gfs_stat(fn, &st); + if (e != GFARM_ERR_NO_ERROR) { + errno = gfarm_error_to_errno(e); + return (-1); + } + gfs_stat_free(&st); + return (0); +} + +/* XXX FIXME */ +#define GFS_DEV ((dev_t)-1) +#define GFS_BLKSIZE 8192 +#define STAT_BLKSIZ 512 /* for st_blocks */ + +int +Gfarm_stat(const char *fn, struct stat *buf, IOR_param_t *param) +{ + struct gfs_stat st; + gfarm_error_t e; + + if (param->dryRun) + return (0); + + e = gfs_stat(fn, &st); + if (e != GFARM_ERR_NO_ERROR) { + errno = gfarm_error_to_errno(e); + return (-1); + } + buf->st_dev = GFS_DEV; + buf->st_ino = st.st_ino; + buf->st_mode = st.st_mode; + buf->st_nlink = st.st_nlink; + buf->st_uid = getuid(); /* XXX */ + buf->st_gid = getgid(); /* XXX */ + buf->st_size = st.st_size; + buf->st_blksize = GFS_BLKSIZE; + buf->st_blocks = (st.st_size + STAT_BLKSIZ - 1) / STAT_BLKSIZ; + buf->st_atime = st.st_atimespec.tv_sec; + buf->st_mtime = st.st_mtimespec.tv_sec; + buf->st_ctime = st.st_ctimespec.tv_sec; +#if defined(HAVE_STRUCT_STAT_ST_MTIM_TV_NSEC) + buf->st_atim.tv_nsec = st.st_atimespec.tv_nsec; + buf->st_mtim.tv_nsec = st.st_mtimespec.tv_nsec; + buf->st_ctim.tv_nsec = st.st_ctimespec.tv_nsec; +#endif + gfs_stat_free(&st); + return (0); +} + +ior_aiori_t gfarm_aiori = { + .name = "Gfarm", + .name_legacy = NULL, + .create = Gfarm_create, + .open = Gfarm_open, + .xfer = Gfarm_xfer, + .close = Gfarm_close, + .delete = Gfarm_delete, + .get_version = Gfarm_version, + .fsync = Gfarm_fsync, + .get_file_size = Gfarm_get_file_size, + .statfs = Gfarm_statfs, + .mkdir = Gfarm_mkdir, + .rmdir = Gfarm_rmdir, + .access = Gfarm_access, + .stat = Gfarm_stat, + .initialize = Gfarm_initialize, + .finalize = Gfarm_finalize, + .get_options = NULL, + .enable_mdtest = true, +}; diff --git a/src/aiori.c b/src/aiori.c index 52001c8a..41773d98 100644 --- a/src/aiori.c +++ b/src/aiori.c @@ -68,6 +68,9 @@ ior_aiori_t *available_aiori[] = { #endif #ifdef USE_RADOS_AIORI &rados_aiori, +#endif +#ifdef USE_GFARM_AIORI + &gfarm_aiori, #endif NULL }; diff --git a/src/aiori.h b/src/aiori.h index c2074c2f..56717fe1 100755 --- a/src/aiori.h +++ b/src/aiori.h @@ -105,6 +105,7 @@ extern ior_aiori_t s3_aiori; extern ior_aiori_t s3_plus_aiori; extern ior_aiori_t s3_emc_aiori; extern ior_aiori_t rados_aiori; +extern ior_aiori_t gfarm_aiori; void aiori_initialize(IOR_test_t * tests); void aiori_finalize(IOR_test_t * tests); From a822363ee0d92beba47f359271f370608163cd07 Mon Sep 17 00:00:00 2001 From: Osamu Tatebe Date: Sat, 22 Jun 2019 10:48:41 +0900 Subject: [PATCH 103/206] style fix --- configure.ac | 4 ++-- src/aiori-Gfarm.c | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/configure.ac b/configure.ac index 7d8e4a74..af400abb 100755 --- a/configure.ac +++ b/configure.ac @@ -189,8 +189,8 @@ AM_COND_IF([USE_RADOS_AIORI],[ AC_MSG_CHECKING([for Gfarm file system]) AC_ARG_WITH([gfarm], [AS_HELP_STRING([--with-gfarm=GFARM_ROOT], - [support IO with libgfarm backend @<:@default=no@:>@])], - [], [with_gfarm=no]) + [support IO with Gfarm backend @<:@default=no@:>@])], + [], [with_gfarm=no]) AC_MSG_RESULT([$with_gfarm]) AM_CONDITIONAL([USE_GFARM_AIORI], [test x$with_gfarm != xno]) if test x$with_gfarm != xno; then diff --git a/src/aiori-Gfarm.c b/src/aiori-Gfarm.c index cc835f1a..a7af0ea8 100644 --- a/src/aiori-Gfarm.c +++ b/src/aiori-Gfarm.c @@ -77,7 +77,7 @@ Gfarm_xfer(int access, void *fd, IOR_size_t *buffer, IOR_offset_t len, #define MAX_SZ (1024 * 1024 * 1024) int sz, n; char *buf = (char *)buffer; - + if (param->dryRun) return (len); @@ -109,7 +109,7 @@ Gfarm_close(void *fd, IOR_param_t *param) { struct gfarm_file *fp = fd; - if(param->dryRun) + if (param->dryRun) return; if (gfs_pio_close(fp->gf) != GFARM_ERR_NO_ERROR) @@ -141,7 +141,7 @@ Gfarm_fsync(void *fd, IOR_param_t *param) { struct gfarm_file *fp = fd; - if(param->dryRun) + if (param->dryRun) return; if (gfs_pio_sync(fp->gf) != GFARM_ERR_NO_ERROR) @@ -213,7 +213,7 @@ Gfarm_mkdir(const char *fn, mode_t mode, IOR_param_t *param) if (param->dryRun) return (0); - e = gfs_mkdir(fn, mode); + e = gfs_mkdir(fn, mode); if (e == GFARM_ERR_NO_ERROR) return (0); errno = gfarm_error_to_errno(e); @@ -228,7 +228,7 @@ Gfarm_rmdir(const char *fn, IOR_param_t *param) if (param->dryRun) return (0); - e = gfs_rmdir(fn); + e = gfs_rmdir(fn); if (e == GFARM_ERR_NO_ERROR) return (0); errno = gfarm_error_to_errno(e); From 960005351d06c22faf47ea9d8432b6fb2967d160 Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Tue, 25 Jun 2019 22:41:30 +0000 Subject: [PATCH 104/206] update DAOS and DFS markings. Signed-off-by: Mohamad Chaarawi --- src/aiori-DAOS.c | 14 +++++++------- src/aiori-DFS.c | 32 ++++++++++++++------------------ 2 files changed, 21 insertions(+), 25 deletions(-) diff --git a/src/aiori-DAOS.c b/src/aiori-DAOS.c index 7210ae39..c4128d12 100644 --- a/src/aiori-DAOS.c +++ b/src/aiori-DAOS.c @@ -3,18 +3,18 @@ * vim:expandtab:shiftwidth=8:tabstop=8: */ /* - * SPECIAL LICENSE RIGHTS-OPEN SOURCE SOFTWARE + * Copyright (C) 2018-2019 Intel Corporation + * + * GOVERNMENT LICENSE RIGHTS-OPEN SOURCE SOFTWARE * The Government's rights to use, modify, reproduce, release, perform, display, - * or disclose this software are subject to the terms of Contract No. B599860, - * and the terms of the GNU General Public License version 2. + * or disclose this software are subject to the terms of the Apache License as + * provided in Contract No. 8F-30005. * Any reproduction of computer software, computer software documentation, or * portions thereof marked with this legend must also reproduce the markings. */ + /* - * Copyright (c) 2013, 2016 Intel Corporation. - */ -/* - * This file implements the abstract I/O interface for DAOS. + * This file implements the abstract I/O interface for DAOS Array API. */ #ifdef HAVE_CONFIG_H diff --git a/src/aiori-DFS.c b/src/aiori-DFS.c index 0a400b96..fe27d0b9 100755 --- a/src/aiori-DFS.c +++ b/src/aiori-DFS.c @@ -1,24 +1,20 @@ /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- * vim:expandtab:shiftwidth=8:tabstop=8: */ -/******************************************************************************\ -* * -* Copyright (c) 2003, The Regents of the University of California * -* See the file COPYRIGHT for a complete copyright notice and license. * -* * -* Copyright (C) 2018 Intel Corporation -* -* GOVERNMENT LICENSE RIGHTS-OPEN SOURCE SOFTWARE -* The Government's rights to use, modify, reproduce, release, perform, display, -* or disclose this software are subject to the terms of the Apache License as -* provided in Contract No. 8F-30005. -* Any reproduction of computer software, computer software documentation, or -* portions thereof marked with this legend must also reproduce the markings. -******************************************************************************** -* -* Implement of abstract I/O interface for DFS. -* -\******************************************************************************/ +/* + * Copyright (C) 2018-2019 Intel Corporation + * + * GOVERNMENT LICENSE RIGHTS-OPEN SOURCE SOFTWARE + * The Government's rights to use, modify, reproduce, release, perform, display, + * or disclose this software are subject to the terms of the Apache License as + * provided in Contract No. 8F-30005. + * Any reproduction of computer software, computer software documentation, or + * portions thereof marked with this legend must also reproduce the markings. + */ + +/* + * This file implements the abstract I/O interface for DAOS FS API. + */ #ifdef HAVE_CONFIG_H #include "config.h" From 2ae7c29de21490b1393856f4a405ac1cea2c9324 Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Thu, 27 Jun 2019 21:31:07 +0000 Subject: [PATCH 105/206] cache the parent dir in a hashtable so we don't have to do a lookup on every access. Signed-off-by: Mohamad Chaarawi --- src/aiori-DFS.c | 141 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 102 insertions(+), 39 deletions(-) diff --git a/src/aiori-DFS.c b/src/aiori-DFS.c index fe27d0b9..d319ebd2 100755 --- a/src/aiori-DFS.c +++ b/src/aiori-DFS.c @@ -32,6 +32,7 @@ #include #include +#include #include #include #include @@ -43,6 +44,13 @@ dfs_t *dfs; daos_handle_t poh, coh; +static struct d_hash_table *dir_hash; + +struct aiori_dir_hdl { + d_list_t entry; + dfs_obj_t *oh; + char name[PATH_MAX]; +}; enum handleType { POOL_HANDLE, @@ -146,6 +154,36 @@ do { \ } \ } while (0) +static inline struct aiori_dir_hdl * +hdl_obj(d_list_t *rlink) +{ + return container_of(rlink, struct aiori_dir_hdl, entry); +} + +static bool +key_cmp(struct d_hash_table *htable, d_list_t *rlink, + const void *key, unsigned int ksize) +{ + struct aiori_dir_hdl *hdl = hdl_obj(rlink); + + return (strcmp(hdl->name, (const char *)key) == 0); +} + +static void +rec_free(struct d_hash_table *htable, d_list_t *rlink) +{ + struct aiori_dir_hdl *hdl = hdl_obj(rlink); + + assert(d_hash_rec_unlinked(&hdl->entry)); + dfs_release(hdl->oh); + free(hdl); +} + +static d_hash_table_ops_t hdl_hash_ops = { + .hop_key_cmp = key_cmp, + .hop_rec_free = rec_free +}; + /* Distribute process 0's pool or container handle to others. */ static void HandleDistribute(daos_handle_t *handle, enum handleType type) @@ -290,6 +328,39 @@ parse_filename(const char *path, char **_obj_name, char **_cont_name) return rc; } +static dfs_obj_t * +lookup_insert_dir(const char *name) +{ + struct aiori_dir_hdl *hdl; + d_list_t *rlink; + mode_t mode; + int rc; + + rlink = d_hash_rec_find(dir_hash, name, strlen(name)); + if (rlink != NULL) { + hdl = hdl_obj(rlink); + return hdl->oh; + } + + hdl = calloc(1, sizeof(struct aiori_dir_hdl)); + if (hdl == NULL) + DERR(ENOMEM, "failed to alloc dir handle"); + + strncpy(hdl->name, name, PATH_MAX-1); + hdl->name[PATH_MAX-1] = '\0'; + + rc = dfs_lookup(dfs, name, O_RDWR, &hdl->oh, &mode); + DERR(rc, "dfs_lookup() of %s Failed", name); + + rc = d_hash_rec_insert(dir_hash, hdl->name, strlen(hdl->name), + &hdl->entry, true); + DERR(rc, "Failed to insert dir handle in hashtable"); + + return hdl->oh; +out: + return NULL; +} + static option_help * DFS_options(){ return options; } @@ -304,6 +375,9 @@ DFS_Init() { rc = daos_init(); DCHECK(rc, "Failed to initialize daos"); + rc = d_hash_table_create(0, 16, NULL, &hdl_hash_ops, &dir_hash); + DCHECK(rc, "Failed to initialize dir hashtable"); + if (rank == 0) { uuid_t pool_uuid, co_uuid; d_rank_list_t *svcl = NULL; @@ -359,6 +433,8 @@ DFS_Finalize() { int rc; + d_hash_table_destroy(dir_hash, true /* force */); + rc = dfs_umount(dfs); DCHECK(rc, "Failed to umount DFS namespace"); @@ -393,22 +469,22 @@ DFS_Create(char *testFileName, IOR_param_t *param) { char *name = NULL, *dir_name = NULL; dfs_obj_t *obj = NULL, *parent = NULL; - mode_t pmode, mode; + mode_t mode; int fd_oflag = 0; int rc; assert(param); - rc = parse_filename(testFileName, &name, &dir_name); + rc = parse_filename(testFileName, &name, &dir_name); DERR(rc, "Failed to parse path %s", testFileName); + assert(dir_name); + assert(name); - assert(dir_name); - assert(name); + parent = lookup_insert_dir(dir_name); + if (parent == NULL) + DERR(rc, "Failed to lookup parent dir"); - rc = dfs_lookup(dfs, dir_name, O_RDWR, &parent, &pmode); - DERR(rc, "dfs_lookup() of %s Failed", dir_name); mode = S_IFREG | param->mode; - if (param->filePerProc || rank == 0) { fd_oflag |= O_CREAT | O_RDWR | O_EXCL; @@ -431,8 +507,6 @@ DFS_Create(char *testFileName, IOR_param_t *param) free(name); if (dir_name) free(dir_name); - if (parent) - dfs_release(parent); return ((void *)obj); } @@ -445,7 +519,7 @@ DFS_Open(char *testFileName, IOR_param_t *param) { char *name = NULL, *dir_name = NULL; dfs_obj_t *obj = NULL, *parent = NULL; - mode_t pmode, mode; + mode_t mode; int rc; int fd_oflag = 0; @@ -458,8 +532,9 @@ DFS_Open(char *testFileName, IOR_param_t *param) assert(dir_name); assert(name); - rc = dfs_lookup(dfs, dir_name, O_RDWR, &parent, &pmode); - DERR(rc, "dfs_lookup() of %s Failed", dir_name); + parent = lookup_insert_dir(dir_name); + if (parent == NULL) + DERR(rc, "Failed to lookup parent dir"); rc = dfs_open(dfs, parent, name, mode, fd_oflag, 0, 0, NULL, &obj); DERR(rc, "dfs_open() of %s Failed", name); @@ -469,8 +544,6 @@ DFS_Open(char *testFileName, IOR_param_t *param) free(name); if (dir_name) free(dir_name); - if (parent) - dfs_release(parent); return ((void *)obj); } @@ -559,7 +632,6 @@ DFS_Delete(char *testFileName, IOR_param_t * param) { char *name = NULL, *dir_name = NULL; dfs_obj_t *parent = NULL; - mode_t pmode; int rc; rc = parse_filename(testFileName, &name, &dir_name); @@ -568,8 +640,9 @@ DFS_Delete(char *testFileName, IOR_param_t * param) assert(dir_name); assert(name); - rc = dfs_lookup(dfs, dir_name, O_RDWR, &parent, &pmode); - DERR(rc, "dfs_lookup() of %s Failed", dir_name); + parent = lookup_insert_dir(dir_name); + if (parent == NULL) + DERR(rc, "Failed to lookup parent dir"); rc = dfs_remove(dfs, parent, name, false); DERR(rc, "dfs_remove() of %s Failed", name); @@ -579,8 +652,6 @@ DFS_Delete(char *testFileName, IOR_param_t * param) free(name); if (dir_name) free(dir_name); - if (parent) - dfs_release(parent); } static char* DFS_GetVersion() @@ -647,7 +718,6 @@ static int DFS_Mkdir(const char *path, mode_t mode, IOR_param_t * param) { dfs_obj_t *parent = NULL; - mode_t pmode; char *name = NULL, *dir_name = NULL; int rc; @@ -657,8 +727,9 @@ DFS_Mkdir(const char *path, mode_t mode, IOR_param_t * param) assert(dir_name); assert(name); - rc = dfs_lookup(dfs, dir_name, O_RDWR, &parent, &pmode); - DERR(rc, "dfs_lookup() of %s Failed", dir_name); + parent = lookup_insert_dir(dir_name); + if (parent == NULL) + DERR(rc, "Failed to lookup parent dir"); rc = dfs_mkdir(dfs, parent, name, mode); DERR(rc, "dfs_mkdir() of %s Failed", name); @@ -668,8 +739,6 @@ DFS_Mkdir(const char *path, mode_t mode, IOR_param_t * param) free(name); if (dir_name) free(dir_name); - if (parent) - dfs_release(parent); if (rc) return -1; return rc; @@ -679,7 +748,6 @@ static int DFS_Rmdir(const char *path, IOR_param_t * param) { dfs_obj_t *parent = NULL; - mode_t pmode; char *name = NULL, *dir_name = NULL; int rc; @@ -689,8 +757,9 @@ DFS_Rmdir(const char *path, IOR_param_t * param) assert(dir_name); assert(name); - rc = dfs_lookup(dfs, dir_name, O_RDWR, &parent, &pmode); - DERR(rc, "dfs_lookup() of %s Failed", dir_name); + parent = lookup_insert_dir(dir_name); + if (parent == NULL) + DERR(rc, "Failed to lookup parent dir"); rc = dfs_remove(dfs, parent, name, false); DERR(rc, "dfs_remove() of %s Failed", name); @@ -700,8 +769,6 @@ DFS_Rmdir(const char *path, IOR_param_t * param) free(name); if (dir_name) free(dir_name); - if (parent) - dfs_release(parent); if (rc) return -1; return rc; @@ -711,7 +778,6 @@ static int DFS_Access(const char *path, int mode, IOR_param_t * param) { dfs_obj_t *parent = NULL; - mode_t pmode; char *name = NULL, *dir_name = NULL; struct stat stbuf; int rc; @@ -721,8 +787,9 @@ DFS_Access(const char *path, int mode, IOR_param_t * param) assert(dir_name); - rc = dfs_lookup(dfs, dir_name, O_RDWR, &parent, &pmode); - DERR(rc, "dfs_lookup() of %s Failed", dir_name); + parent = lookup_insert_dir(dir_name); + if (parent == NULL) + DERR(rc, "Failed to lookup parent dir"); if (name && strcmp(name, ".") == 0) { free(name); @@ -735,8 +802,6 @@ DFS_Access(const char *path, int mode, IOR_param_t * param) free(name); if (dir_name) free(dir_name); - if (parent) - dfs_release(parent); if (rc) return -1; return rc; @@ -746,7 +811,6 @@ static int DFS_Stat(const char *path, struct stat *buf, IOR_param_t * param) { dfs_obj_t *parent = NULL; - mode_t pmode; char *name = NULL, *dir_name = NULL; int rc; @@ -756,8 +820,9 @@ DFS_Stat(const char *path, struct stat *buf, IOR_param_t * param) assert(dir_name); assert(name); - rc = dfs_lookup(dfs, dir_name, O_RDONLY, &parent, &pmode); - DERR(rc, "dfs_lookup() of %s Failed", dir_name); + parent = lookup_insert_dir(dir_name); + if (parent == NULL) + DERR(rc, "Failed to lookup parent dir"); rc = dfs_stat(dfs, parent, name, buf); DERR(rc, "dfs_stat() of %s Failed", name); @@ -767,8 +832,6 @@ DFS_Stat(const char *path, struct stat *buf, IOR_param_t * param) free(name); if (dir_name) free(dir_name); - if (parent) - dfs_release(parent); if (rc) return -1; return rc; From 9f9730d6fddd6359acbd9e1eb390a5f8ddfc5996 Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Fri, 28 Jun 2019 16:20:56 +0000 Subject: [PATCH 106/206] should abort on daos init if pool/cont/svcl not specified. Signed-off-by: Mohamad Chaarawi --- src/aiori-DAOS.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/aiori-DAOS.c b/src/aiori-DAOS.c index c4128d12..3b5c80c4 100644 --- a/src/aiori-DAOS.c +++ b/src/aiori-DAOS.c @@ -258,8 +258,11 @@ DAOS_Init() if (daos_initialized) return; - if (o.pool == NULL || o.svcl == NULL || o.cont == NULL) + if (o.pool == NULL || o.svcl == NULL || o.cont == NULL) { + GERR("Invalid DAOS pool/cont\n"); return; + } + if (o.oclass) ObjectClassParse(o.oclass); From 3eb0f86640d0fd18def2cd49d4c6c148bd4f7c0f Mon Sep 17 00:00:00 2001 From: Wei-keng Liao Date: Fri, 28 Jun 2019 17:40:35 -0500 Subject: [PATCH 107/206] Function ncmpi_get_file_info was first introduced in PnetCDF version 1.2.0. Check PnetCDF library version, so printing MPI-IO hints can be enabled for NCMPI. --- doc/USER_GUIDE | 2 +- src/aiori-NCMPI.c | 91 +++++++++++++++++++++-------------------------- 2 files changed, 42 insertions(+), 51 deletions(-) diff --git a/doc/USER_GUIDE b/doc/USER_GUIDE index 6a6cf188..b8888b9f 100755 --- a/doc/USER_GUIDE +++ b/doc/USER_GUIDE @@ -342,7 +342,7 @@ MPIIO-, HDF5-, AND NCMPI-ONLY: * collective - uses collective operations for access [0=FALSE] * showHints - show hint/value pairs attached to open file [0=FALSE] - NOTE: not available in NCMPI + NOTE: available in NCMPI only if PnetCDF is 1.2.0 or later LUSTRE-SPECIFIC: ================ diff --git a/src/aiori-NCMPI.c b/src/aiori-NCMPI.c index 0f391dcf..36074669 100755 --- a/src/aiori-NCMPI.c +++ b/src/aiori-NCMPI.c @@ -89,7 +89,7 @@ static void *NCMPI_Create(char *testFileName, IOR_param_t * param) int fd_mode; MPI_Info mpiHints = MPI_INFO_NULL; - /* Wei-keng Liao: read and set MPI file hints from hintsFile */ + /* read and set MPI file hints from hintsFile */ SetHints(&mpiHints, param->hintsFileName); if (rank == 0 && param->showHints) { fprintf(stdout, "\nhints passed to MPI_File_open() {\n"); @@ -105,22 +105,23 @@ static void *NCMPI_Create(char *testFileName, IOR_param_t * param) NCMPI_CHECK(ncmpi_create(testComm, testFileName, fd_mode, mpiHints, fd), "cannot create file"); - /* Wei-keng Liao: print the MPI file hints currently used */ -/* WEL - add when ncmpi_get_file_info() is in current parallel-netcdf release - if (rank == 0 && param->showHints) { - MPI_CHECK(ncmpi_get_file_info(*fd, &mpiHints), - "cannot get file info"); - fprintf(stdout, "\nhints returned from opened file {\n"); - ShowHints(&mpiHints); - fprintf(stdout, "}\n"); - } -*/ - - /* Wei-keng Liao: free up the mpiHints object */ -/* WEL - this needs future fix from next release of PnetCDF - if (mpiHints != MPI_INFO_NULL) - MPI_CHECK(MPI_Info_free(&mpiHints), "cannot free file info"); -*/ + /* free up the mpiHints object */ + if (mpiHints != MPI_INFO_NULL) + MPI_CHECK(MPI_Info_free(&mpiHints), "cannot free file info"); + +#if defined(PNETCDF_VERSION_MAJOR) && (PNETCDF_VERSION_MAJOR > 1 || PNETCDF_VERSION_MINOR >= 2) + /* ncmpi_get_file_info is first available in 1.2.0 */ + if (rank == 0 && param->showHints) { + MPI_Info info_used; + MPI_CHECK(ncmpi_get_file_info(*fd, &info_used), + "cannot inquire file info"); + /* print the MPI file hints currently used */ + fprintf(stdout, "\nhints returned from opened file {\n"); + ShowHints(&info_used); + fprintf(stdout, "}\n"); + MPI_CHECK(MPI_Info_free(&info_used), "cannot free file info"); + } +#endif return (fd); } @@ -134,7 +135,7 @@ static void *NCMPI_Open(char *testFileName, IOR_param_t * param) int fd_mode; MPI_Info mpiHints = MPI_INFO_NULL; - /* Wei-keng Liao: read and set MPI file hints from hintsFile */ + /* read and set MPI file hints from hintsFile */ SetHints(&mpiHints, param->hintsFileName); if (rank == 0 && param->showHints) { fprintf(stdout, "\nhints passed to MPI_File_open() {\n"); @@ -150,22 +151,23 @@ static void *NCMPI_Open(char *testFileName, IOR_param_t * param) NCMPI_CHECK(ncmpi_open(testComm, testFileName, fd_mode, mpiHints, fd), "cannot open file"); - /* Wei-keng Liao: print the MPI file hints currently used */ -/* WEL - add when ncmpi_get_file_info() is in current parallel-netcdf release - if (rank == 0 && param->showHints) { - MPI_CHECK(ncmpi_get_file_info(*fd, &mpiHints), - "cannot get file info"); - fprintf(stdout, "\nhints returned from opened file {\n"); - ShowHints(&mpiHints); - fprintf(stdout, "}\n"); - } -*/ - - /* Wei-keng Liao: free up the mpiHints object */ -/* WEL - this needs future fix from next release of PnetCDF - if (mpiHints != MPI_INFO_NULL) - MPI_CHECK(MPI_Info_free(&mpiHints), "cannot free file info"); -*/ + /* free up the mpiHints object */ + if (mpiHints != MPI_INFO_NULL) + MPI_CHECK(MPI_Info_free(&mpiHints), "cannot free file info"); + +#if defined(PNETCDF_VERSION_MAJOR) && (PNETCDF_VERSION_MAJOR > 1 || PNETCDF_VERSION_MINOR >= 2) + /* ncmpi_get_file_info is first available in 1.2.0 */ + if (rank == 0 && param->showHints) { + MPI_Info info_used; + MPI_CHECK(ncmpi_get_file_info(*fd, &info_used), + "cannot inquire file info"); + /* print the MPI file hints currently used */ + fprintf(stdout, "\nhints returned from opened file {\n"); + ShowHints(&info_used); + fprintf(stdout, "}\n"); + MPI_CHECK(MPI_Info_free(&info_used), "cannot free file info"); + } +#endif return (fd); } @@ -183,17 +185,6 @@ static IOR_offset_t NCMPI_Xfer(int access, void *fd, IOR_size_t * buffer, IOR_offset_t segmentPosition; int segmentNum, transferNum; - /* Wei-keng Liao: In ior.c line 1979 says "block size must be a multiple - of transfer size." Hence, length should always == param->transferSize - below. I leave it here to double check. - */ - if (length != param->transferSize) { - char errMsg[256]; - sprintf(errMsg, "length(%lld) != param->transferSize(%lld)\n", - length, param->transferSize); - NCMPI_CHECK(-1, errMsg); - } - /* determine by offset if need to start data set */ if (param->filePerProc == TRUE) { segmentPosition = (IOR_offset_t) 0; @@ -224,7 +215,7 @@ static IOR_offset_t NCMPI_Xfer(int access, void *fd, IOR_size_t * buffer, int numTransfers = param->blockSize / param->transferSize; - /* Wei-keng Liao: change 1D array to 3D array of dimensions: + /* reshape 1D array to 3D array: [segmentCount*numTasksWorld][numTransfers][transferSize] Requirement: none of these dimensions should be > 4G, */ @@ -264,13 +255,13 @@ static IOR_offset_t NCMPI_Xfer(int access, void *fd, IOR_size_t * buffer, var_id = param->var_id; - /* Wei-keng Liao: calculate the segment number */ + /* calculate the segment number */ segmentNum = param->offset / (param->numTasks * param->blockSize); - /* Wei-keng Liao: calculate the transfer number in each block */ + /* calculate the transfer number in each block */ transferNum = param->offset % param->blockSize / param->transferSize; - /* Wei-keng Liao: read/write the 3rd dim of the dataset, each is of + /* read/write the 3rd dim of the dataset, each is of amount param->transferSize */ bufSize[0] = 1; bufSize[1] = 1; @@ -381,7 +372,7 @@ static int GetFileMode(IOR_param_t * param) fprintf(stdout, "O_DIRECT not implemented in NCMPI\n"); } - /* Wei-keng Liao: to enable > 4GB file size */ + /* to enable > 4GB file size */ fd_mode |= NC_64BIT_OFFSET; return (fd_mode); From 52ed766df43114caa1a5bb6c59505ebe295a71a3 Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Mon, 8 Jul 2019 15:33:06 +0000 Subject: [PATCH 108/206] update DAOS and DFS drivers to new DAOS API changes. Signed-off-by: Mohamad Chaarawi --- src/aiori-DAOS.c | 97 +++++++++++++++++++++++++++++++----------------- src/aiori-DFS.c | 10 ++--- 2 files changed, 67 insertions(+), 40 deletions(-) diff --git a/src/aiori-DAOS.c b/src/aiori-DAOS.c index 3b5c80c4..6b03c44e 100644 --- a/src/aiori-DAOS.c +++ b/src/aiori-DAOS.c @@ -28,9 +28,9 @@ #include #include #include + #include #include -#include #include #include "ior.h" @@ -119,7 +119,7 @@ enum handleType { static daos_handle_t poh; static daos_handle_t coh; static daos_handle_t aoh; -static daos_oclass_id_t objectClass = DAOS_OC_LARGE_RW; +static daos_oclass_id_t objectClass = OC_SX; static bool daos_initialized = false; /***************************** F U N C T I O N S ******************************/ @@ -212,34 +212,58 @@ HandleDistribute(daos_handle_t *handle, enum handleType type) static void ObjectClassParse(const char *string) { - if (strcasecmp(string, "tiny") == 0) - objectClass = DAOS_OC_TINY_RW; - else if (strcasecmp(string, "small") == 0) - objectClass = DAOS_OC_SMALL_RW; - else if (strcasecmp(string, "large") == 0) - objectClass = DAOS_OC_LARGE_RW; - else if (strcasecmp(string, "echo_tiny") == 0) - objectClass = DAOS_OC_ECHO_TINY_RW; - else if (strcasecmp(string, "echo_R2S") == 0) - objectClass = DAOS_OC_ECHO_R2S_RW; - else if (strcasecmp(string, "echo_R3S") == 0) - objectClass = DAOS_OC_ECHO_R3S_RW; - else if (strcasecmp(string, "echo_R4S") == 0) - objectClass = DAOS_OC_ECHO_R4S_RW; - else if (strcasecmp(string, "R2") == 0) - objectClass = DAOS_OC_R2_RW; - else if (strcasecmp(string, "R2S") == 0) - objectClass = DAOS_OC_R2S_RW; - else if (strcasecmp(string, "R3S") == 0) - objectClass = DAOS_OC_R3S_RW; - else if (strcasecmp(string, "R3") == 0) - objectClass = DAOS_OC_R3_RW; - else if (strcasecmp(string, "R4") == 0) - objectClass = DAOS_OC_R4_RW; - else if (strcasecmp(string, "R4S") == 0) - objectClass = DAOS_OC_R4S_RW; - else if (strcasecmp(string, "repl_max") == 0) - objectClass = DAOS_OC_REPL_MAX_RW; + if (strcasecmp(string, "oc_s1") == 0) + objectClass = OC_S1; + else if (strcasecmp(string, "oc_s2") == 0) + objectClass = OC_S2; + else if (strcasecmp(string, "oc_s4") == 0) + objectClass = OC_S4; + else if (strcasecmp(string, "oc_sx") == 0) + objectClass = OC_SX; + else if (strcasecmp(string, "oc_tiny") == 0) + objectClass = OC_TINY; + else if (strcasecmp(string, "oc_small") == 0) + objectClass = OC_SMALL; + else if (strcasecmp(string, "oc_large") == 0) + objectClass = OC_LARGE; + else if (strcasecmp(string, "oc_max") == 0) + objectClass = OC_MAX; + else if (strcasecmp(string, "oc_rp_tiny") == 0) + objectClass = OC_RP_TINY; + else if (strcasecmp(string, "oc_rp_small") == 0) + objectClass = OC_RP_SMALL; + else if (strcasecmp(string, "oc_rp_large") == 0) + objectClass = OC_RP_LARGE; + else if (strcasecmp(string, "oc_rp_max") == 0) + objectClass = OC_RP_MAX; + else if (strcasecmp(string, "oc_rp_sf_tiny") == 0) + objectClass = OC_RP_SF_TINY; + else if (strcasecmp(string, "oc_rp_sf_small") == 0) + objectClass = OC_RP_SF_SMALL; + else if (strcasecmp(string, "oc_rp_sf_large") == 0) + objectClass = OC_RP_SF_LARGE; + else if (strcasecmp(string, "oc_rp_sf_max") == 0) + objectClass = OC_RP_SF_MAX; + else if (strcasecmp(string, "oc_ec_tiny") == 0) + objectClass = OC_EC_TINY; + else if (strcasecmp(string, "oc_ec_small") == 0) + objectClass = OC_EC_SMALL; + else if (strcasecmp(string, "oc_ec_large") == 0) + objectClass = OC_EC_LARGE; + else if (strcasecmp(string, "oc_ec_max") == 0) + objectClass = OC_EC_MAX; + else if (strcasecmp(string, "oc_rp_2g1") == 0) + objectClass = OC_RP_2G1; + else if (strcasecmp(string, "oc_rp_2g4") == 0) + objectClass = OC_RP_2G4; + else if (strcasecmp(string, "oc_rp_2gx") == 0) + objectClass = OC_RP_2GX; + else if (strcasecmp(string, "oc_rp_3g1") == 0) + objectClass = OC_RP_3G1; + else if (strcasecmp(string, "oc_rp_3g4") == 0) + objectClass = OC_RP_3G4; + else if (strcasecmp(string, "oc_rp_3gx") == 0) + objectClass = OC_RP_3GX; else GERR("Invalid 'oclass' argument: '%s'", string); } @@ -338,9 +362,12 @@ DAOS_Fini() } MPI_Bcast(&rc, 1, MPI_INT, 0, MPI_COMM_WORLD); - if (rc) - DCHECK(rc, "Failed to destroy container %s (%d)", - o.cont, rc); + if (rc) { + if (rank == 0) + DCHECK(rc, "Failed to destroy container %s (%d)", + o.cont, rc); + MPI_Abort(MPI_COMM_WORLD, -1); + } } rc = daos_pool_disconnect(poh, NULL); @@ -360,8 +387,8 @@ gen_oid(const char *name, daos_obj_id_t *oid) oid->lo = d_hash_murmur64(name, strlen(name), IOR_DAOS_MUR_SEED); oid->hi = 0; - feat = DAOS_OF_DKEY_UINT64 | DAOS_OF_AKEY_HASHED; - daos_obj_generate_id(oid, feat, objectClass); + feat = DAOS_OF_DKEY_UINT64; + daos_obj_generate_id(oid, feat, objectClass, 0); } static void * diff --git a/src/aiori-DFS.c b/src/aiori-DFS.c index d319ebd2..cd66ee02 100755 --- a/src/aiori-DFS.c +++ b/src/aiori-DFS.c @@ -33,8 +33,7 @@ #include #include -#include -#include +#include #include #include "ior.h" @@ -489,7 +488,7 @@ DFS_Create(char *testFileName, IOR_param_t *param) fd_oflag |= O_CREAT | O_RDWR | O_EXCL; rc = dfs_open(dfs, parent, name, mode, fd_oflag, - DAOS_OC_LARGE_RW, 0, NULL, &obj); + OC_SX, 0, NULL, &obj); DERR(rc, "dfs_open() of %s Failed", name); MPI_CHECK(MPI_Barrier(testComm), "barrier error"); @@ -498,7 +497,7 @@ DFS_Create(char *testFileName, IOR_param_t *param) fd_oflag |= O_RDWR; rc = dfs_open(dfs, parent, name, mode, fd_oflag, - DAOS_OC_LARGE_RW, 0, NULL, &obj); + OC_SX, 0, NULL, &obj); DERR(rc, "dfs_open() of %s Failed", name); } @@ -725,7 +724,8 @@ DFS_Mkdir(const char *path, mode_t mode, IOR_param_t * param) DERR(rc, "Failed to parse path %s", path); assert(dir_name); - assert(name); + if (!name) + return 0; parent = lookup_insert_dir(dir_name); if (parent == NULL) From 04e3df0d42eb43dc534f550e8834baf418c33595 Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Mon, 8 Jul 2019 17:02:30 +0000 Subject: [PATCH 109/206] use the daos provided function for name to object class Signed-off-by: Mohamad Chaarawi --- src/aiori-DAOS.c | 61 +----------------------------------------------- 1 file changed, 1 insertion(+), 60 deletions(-) diff --git a/src/aiori-DAOS.c b/src/aiori-DAOS.c index 6b03c44e..8d80a69b 100644 --- a/src/aiori-DAOS.c +++ b/src/aiori-DAOS.c @@ -209,65 +209,6 @@ HandleDistribute(daos_handle_t *handle, enum handleType type) free(global.iov_buf); } -static void -ObjectClassParse(const char *string) -{ - if (strcasecmp(string, "oc_s1") == 0) - objectClass = OC_S1; - else if (strcasecmp(string, "oc_s2") == 0) - objectClass = OC_S2; - else if (strcasecmp(string, "oc_s4") == 0) - objectClass = OC_S4; - else if (strcasecmp(string, "oc_sx") == 0) - objectClass = OC_SX; - else if (strcasecmp(string, "oc_tiny") == 0) - objectClass = OC_TINY; - else if (strcasecmp(string, "oc_small") == 0) - objectClass = OC_SMALL; - else if (strcasecmp(string, "oc_large") == 0) - objectClass = OC_LARGE; - else if (strcasecmp(string, "oc_max") == 0) - objectClass = OC_MAX; - else if (strcasecmp(string, "oc_rp_tiny") == 0) - objectClass = OC_RP_TINY; - else if (strcasecmp(string, "oc_rp_small") == 0) - objectClass = OC_RP_SMALL; - else if (strcasecmp(string, "oc_rp_large") == 0) - objectClass = OC_RP_LARGE; - else if (strcasecmp(string, "oc_rp_max") == 0) - objectClass = OC_RP_MAX; - else if (strcasecmp(string, "oc_rp_sf_tiny") == 0) - objectClass = OC_RP_SF_TINY; - else if (strcasecmp(string, "oc_rp_sf_small") == 0) - objectClass = OC_RP_SF_SMALL; - else if (strcasecmp(string, "oc_rp_sf_large") == 0) - objectClass = OC_RP_SF_LARGE; - else if (strcasecmp(string, "oc_rp_sf_max") == 0) - objectClass = OC_RP_SF_MAX; - else if (strcasecmp(string, "oc_ec_tiny") == 0) - objectClass = OC_EC_TINY; - else if (strcasecmp(string, "oc_ec_small") == 0) - objectClass = OC_EC_SMALL; - else if (strcasecmp(string, "oc_ec_large") == 0) - objectClass = OC_EC_LARGE; - else if (strcasecmp(string, "oc_ec_max") == 0) - objectClass = OC_EC_MAX; - else if (strcasecmp(string, "oc_rp_2g1") == 0) - objectClass = OC_RP_2G1; - else if (strcasecmp(string, "oc_rp_2g4") == 0) - objectClass = OC_RP_2G4; - else if (strcasecmp(string, "oc_rp_2gx") == 0) - objectClass = OC_RP_2GX; - else if (strcasecmp(string, "oc_rp_3g1") == 0) - objectClass = OC_RP_3G1; - else if (strcasecmp(string, "oc_rp_3g4") == 0) - objectClass = OC_RP_3G4; - else if (strcasecmp(string, "oc_rp_3gx") == 0) - objectClass = OC_RP_3GX; - else - GERR("Invalid 'oclass' argument: '%s'", string); -} - static option_help * DAOS_options() { @@ -288,7 +229,7 @@ DAOS_Init() } if (o.oclass) - ObjectClassParse(o.oclass); + objectClass = daos_oclass_name2id(o.oclass); rc = daos_init(); if (rc) From ad61fdd89c364888cdd645fee68ec86008135e18 Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Mon, 8 Jul 2019 18:18:24 +0000 Subject: [PATCH 110/206] add proper check for DAOS object class and exit if unknown. Signed-off-by: Mohamad Chaarawi --- src/aiori-DAOS.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/aiori-DAOS.c b/src/aiori-DAOS.c index 8d80a69b..cce5f53a 100644 --- a/src/aiori-DAOS.c +++ b/src/aiori-DAOS.c @@ -228,8 +228,11 @@ DAOS_Init() return; } - if (o.oclass) + if (o.oclass) { objectClass = daos_oclass_name2id(o.oclass); + if (objectClass == OC_UNKNOWN) + GERR("Invalid DAOS Object class %s\n", o.oclass); + } rc = daos_init(); if (rc) From 1c640716c7e821c5be63d54a4abd697aaa69c735 Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Mon, 8 Jul 2019 21:31:00 +0000 Subject: [PATCH 111/206] Use MPI Info object for all MPI_File_open calls. Signed-off-by: Mohamad Chaarawi --- src/aiori-MPIIO.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/aiori-MPIIO.c b/src/aiori-MPIIO.c index 6bf991bd..46f62477 100755 --- a/src/aiori-MPIIO.c +++ b/src/aiori-MPIIO.c @@ -74,6 +74,7 @@ int MPIIO_Access(const char *path, int mode, IOR_param_t *param) } MPI_File fd; int mpi_mode = MPI_MODE_UNIQUE_OPEN; + MPI_Info mpiHints = MPI_INFO_NULL; if ((mode & W_OK) && (mode & R_OK)) mpi_mode |= MPI_MODE_RDWR; @@ -82,12 +83,15 @@ int MPIIO_Access(const char *path, int mode, IOR_param_t *param) else mpi_mode |= MPI_MODE_RDONLY; - int ret = MPI_File_open(MPI_COMM_SELF, path, mpi_mode, - MPI_INFO_NULL, &fd); + SetHints(&mpiHints, param->hintsFileName); + + int ret = MPI_File_open(MPI_COMM_SELF, path, mpi_mode, mpiHints, &fd); if (!ret) MPI_File_close(&fd); + if (mpiHints != MPI_INFO_NULL) + MPI_CHECK(MPI_Info_free(&mpiHints), "MPI_Info_free failed"); return ret; } @@ -497,6 +501,7 @@ IOR_offset_t MPIIO_GetFileSize(IOR_param_t * test, MPI_Comm testComm, IOR_offset_t aggFileSizeFromStat, tmpMin, tmpMax, tmpSum; MPI_File fd; MPI_Comm comm; + MPI_Info mpiHints = MPI_INFO_NULL; if (test->filePerProc == TRUE) { comm = MPI_COMM_SELF; @@ -504,12 +509,15 @@ IOR_offset_t MPIIO_GetFileSize(IOR_param_t * test, MPI_Comm testComm, comm = testComm; } + SetHints(&mpiHints, test->hintsFileName); MPI_CHECK(MPI_File_open(comm, testFileName, MPI_MODE_RDONLY, - MPI_INFO_NULL, &fd), + mpiHints, &fd), "cannot open file to get file size"); MPI_CHECK(MPI_File_get_size(fd, (MPI_Offset *) & aggFileSizeFromStat), "cannot get file size"); MPI_CHECK(MPI_File_close(&fd), "cannot close file"); + if (mpiHints != MPI_INFO_NULL) + MPI_CHECK(MPI_Info_free(&mpiHints), "MPI_Info_free failed"); if (test->filePerProc == TRUE) { MPI_CHECK(MPI_Allreduce(&aggFileSizeFromStat, &tmpSum, 1, From 76f6bb6e421275f1a42b8e804a21c2f1ee78a7c1 Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Wed, 24 Jul 2019 00:29:58 +0100 Subject: [PATCH 112/206] Abort when invalid options are set. --- src/option.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/option.c b/src/option.c index 99f696cc..b1833c18 100644 --- a/src/option.c +++ b/src/option.c @@ -381,12 +381,12 @@ int option_parse(int argc, char ** argv, options_all_t * opt_all){ if( requiredArgsSeen != requiredArgsNeeded ){ printf("Error: Missing some required arguments\n\n"); - printhelp = -1; + printhelp = 1; } if(error != 0){ printf("Invalid options\n"); - printhelp = -1; + printhelp = 1; } if(printhelp == 1){ From 7981fc70b12cc21b63dfba99b6b751596df0f49e Mon Sep 17 00:00:00 2001 From: John Bent Date: Thu, 25 Jul 2019 19:18:52 -0600 Subject: [PATCH 113/206] Changed the behavior of shift in mdtest to mimic that in IOR so that it shifts by _entire nodes_ not by ranks. --- src/mdtest.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/mdtest.c b/src/mdtest.c index 8613e3b0..2d15c9f5 100644 --- a/src/mdtest.c +++ b/src/mdtest.c @@ -1362,7 +1362,7 @@ void print_help (void) { "\t-p: pre-iteration delay (in seconds)\n" "\t-r: only remove files or directories left behind by previous runs\n" "\t-R: randomly stat files (optional argument for random seed)\n" - "\t-s: stride between the number of tasks for each test\n" + "\t-s: stride between the number of nodes for each test\n" "\t-S: shared file access (file only, no directories)\n" "\t-t: time unique working directory overhead\n" "\t-T: only stat files/dirs\n" @@ -2191,7 +2191,8 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * pid = getpid(); uid = getuid(); - nodeCount = size / CountTasksPerNode(testComm); + tasksPerNode = CountTasksPerNode(testComm); + nodeCount = size / tasksPerNode; if (rank == 0) { fprintf(out_logfile, "-- started at %s --\n\n", PrintTimestamp()); @@ -2267,6 +2268,11 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * fflush( out_logfile ); } + /* set the shift to mimic IOR and shift by procs per node */ + if (nstride > 0) { + nstride *= tasksPerNode; + } + /* setup total number of items and number of items per dir */ if (depth <= 0) { num_dirs_in_tree = 1; From 524d053be1dd789cd4ba241011f9039def364d96 Mon Sep 17 00:00:00 2001 From: John Bent Date: Fri, 26 Jul 2019 08:55:24 -0600 Subject: [PATCH 114/206] Making shift work in mdtest as well as it works in IOR and on a per-node basis. Also added printing the nodemap so we can check the allocation. --- src/mdtest.c | 16 +++++++++++----- src/utilities.c | 38 ++++++++++++++++++++++++++++++++++++++ src/utilities.h | 1 + 3 files changed, 50 insertions(+), 5 deletions(-) diff --git a/src/mdtest.c b/src/mdtest.c index 2d15c9f5..cbb0de89 100644 --- a/src/mdtest.c +++ b/src/mdtest.c @@ -2268,11 +2268,6 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * fflush( out_logfile ); } - /* set the shift to mimic IOR and shift by procs per node */ - if (nstride > 0) { - nstride *= tasksPerNode; - } - /* setup total number of items and number of items per dir */ if (depth <= 0) { num_dirs_in_tree = 1; @@ -2381,6 +2376,17 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * } if (rank == 0) display_freespace(testdirpath); + int packedByNode = QueryNodeMapping(testComm); + + /* set the shift to mimic IOR and shift by procs per node */ + if (nstride > 0) { + if ( packedByNode ) { + nstride *= tasksPerNode; + } + if (rank == 0) { + fprintf(out_logfile, "Shifting ranks by %d for each phase.\n", nstride); + } + } if (verbose >= 3 && rank == 0) { fprintf(out_logfile, "V-3: main (after display_freespace): testdirpath is \"%s\"\n", testdirpath ); diff --git a/src/utilities.c b/src/utilities.c index c0c7c614..493eb891 100755 --- a/src/utilities.c +++ b/src/utilities.c @@ -213,12 +213,50 @@ void DumpBuffer(void *buffer, return; } /* DumpBuffer() */ +/* a function that prints an int array where each index corresponds to a rank + and the value is whether that rank is on the same host as root. + Also returns 1 if rank 1 is on same host and 0 otherwise +*/ +int QueryNodeMapping(MPI_Comm comm) { + char localhost[MAX_PATHLEN], roothost[MAX_PATHLEN]; + int num_ranks; + MPI_Comm_size(comm, &num_ranks); + int *node_map = (int*)malloc(sizeof(int) * num_ranks); + if ( ! node_map ) { + FAIL("malloc"); + } + if (gethostname(localhost, MAX_PATHLEN) != 0) { + FAIL("gethostname()"); + } + if (rank==0) { + strncpy(roothost,localhost,MAX_PATHLEN); + } + + /* have rank 0 broadcast out its hostname */ + MPI_Bcast(roothost, MAX_PATHLEN, MPI_CHAR, 0, comm); + //printf("Rank %d received root host as %s\n", rank, roothost); + /* then every rank figures out whether it is same host as root and then gathers that */ + int same_as_root = strcmp(roothost,localhost) == 0; + MPI_Gather( &same_as_root, 1, MPI_INT, node_map, 1, MPI_INT, 0, comm); + if (rank==0) { + fprintf( out_logfile, "Nodemap: " ); + for ( int i = 0; i < num_ranks; i++ ) { + fprintf( out_logfile, "%d", node_map[i] ); + } + fprintf( out_logfile, "\n" ); + } + int ret = node_map[1] == 1; + free(node_map); + return ret; +} + #if MPI_VERSION >= 3 int CountTasksPerNode(MPI_Comm comm) { /* modern MPI provides a simple way to get the local process count */ MPI_Comm shared_comm; int count; + MPI_Comm_split_type (comm, MPI_COMM_TYPE_SHARED, 0, MPI_INFO_NULL, &shared_comm); MPI_Comm_size (shared_comm, &count); MPI_Comm_free (&shared_comm); diff --git a/src/utilities.h b/src/utilities.h index b6461744..d811261c 100755 --- a/src/utilities.h +++ b/src/utilities.h @@ -65,6 +65,7 @@ void SetHints (MPI_Info *, char *); void ShowHints (MPI_Info *); char *HumanReadable(IOR_offset_t value, int base); int CountTasksPerNode(MPI_Comm comm); +int QueryNodeMapping(MPI_Comm comm); void DelaySecs(int delay); void updateParsedOptions(IOR_param_t * options, options_all_t * global_options); size_t NodeMemoryStringToBytes(char *size_str); From 945487f7437f56a6e85c9b726a744af1be83c789 Mon Sep 17 00:00:00 2001 From: John Bent Date: Sat, 27 Jul 2019 09:20:20 -0600 Subject: [PATCH 115/206] Better debug message when stat fails --- src/mdtest.c | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/src/mdtest.c b/src/mdtest.c index cbb0de89..d9f28db0 100644 --- a/src/mdtest.c +++ b/src/mdtest.c @@ -635,19 +635,13 @@ void mdtest_stat(const int random, const int dirs, const long dir_iter, const ch } if (-1 == backend->stat (item, &buf, ¶m)) { - if (dirs) { - if ( verbose >= 3 ) { - fprintf( out_logfile, "V-3: Stat'ing directory \"%s\"\n", item ); - fflush( out_logfile ); - } - FAIL("unable to stat directory"); - } else { - if ( verbose >= 3 ) { - fprintf( out_logfile, "V-3: Stat'ing file \"%s\"\n", item ); - fflush( out_logfile ); - } - FAIL("unable to stat file"); + char msg_buf[4096]; + if ( verbose >=3 ) { + fprintf( out_logfile, "V-3: Stat'ing %s \"%s\"\n", dirs ? "directory" : "file", item ); + fflush( out_logfile ); } + snprintf(msg_buf, 4096, "unable to stat %s %s", dirs ? "directory" : "file", item); + FAIL("unable to stat directory"); } } } From 9d6480d46e5658a269b6bf7e7022f5ad55fba203 Mon Sep 17 00:00:00 2001 From: John Bent Date: Sat, 27 Jul 2019 11:30:07 -0600 Subject: [PATCH 116/206] Fixed bug in the nstride calculation where only 0 was computing it correctly. Had to bcast it out --- src/mdtest.c | 16 +++++++++++++++- src/utilities.c | 1 + 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/mdtest.c b/src/mdtest.c index d9f28db0..072dc29e 100644 --- a/src/mdtest.c +++ b/src/mdtest.c @@ -494,6 +494,7 @@ void create_remove_items(int currDepth, const int dirs, const int create, const if (collective) { collective_helper(dirs, create, temp_path, 0, progress); } else { + printf("DEBUG %5d rank %d gonna create %s\n", __LINE__, rank, temp_path); create_remove_items_helper(dirs, create, temp_path, 0, progress); } } @@ -522,6 +523,7 @@ void create_remove_items(int currDepth, const int dirs, const int create, const if (collective) { collective_helper(dirs, create, temp_path, currDir*items_per_dir, progress); } else { + printf("DEBUG %5d rank %d gonna create %s\n", __LINE__, rank, temp_path); create_remove_items_helper(dirs, create, temp_path, currDir*items_per_dir, progress); } } @@ -641,7 +643,7 @@ void mdtest_stat(const int random, const int dirs, const long dir_iter, const ch fflush( out_logfile ); } snprintf(msg_buf, 4096, "unable to stat %s %s", dirs ? "directory" : "file", item); - FAIL("unable to stat directory"); + FAIL(msg_buf); } } } @@ -792,6 +794,7 @@ void collective_create_remove(const int create, const int dirs, const int ntasks sprintf(rm_name, "mdtest.%d.", (i+(3*nstride))%ntasks); } if (unique_dir_per_task) { + printf("DEBUG %5d Rank %d i %d nstride %d ntasks %d", __LINE__, rank, i, nstride, ntasks); sprintf(unique_mk_dir, "%s/mdtest_tree.%d.0", testdir, (i+(0*nstride))%ntasks); sprintf(unique_chdir_dir, "%s/mdtest_tree.%d.0", testdir, @@ -1085,6 +1088,8 @@ void file_test(const int iteration, const int ntasks, const char *path, rank_pro char temp_path[MAX_PATHLEN]; MPI_Comm_size(testComm, &size); + printf("DEBUG %5d Rank %d file_test on %s\n", __LINE__, rank, path); + if (( rank == 0 ) && ( verbose >= 1 )) { fprintf( out_logfile, "V-1: Entering file_test...\n" ); fflush( out_logfile ); @@ -1099,7 +1104,9 @@ void file_test(const int iteration, const int ntasks, const char *path, rank_pro prep_testdir(iteration, dir_iter); if (unique_dir_per_task) { + printf("DEBUG %5d Rank %d operating on %s\n", __LINE__, rank, temp_path); unique_dir_access(MK_UNI_DIR, temp_path); + printf("DEBUG %5d Rank %d operating on %s\n", __LINE__, rank, temp_path); if (!time_unique_dir_overhead) { offset_timers(t, 0); } @@ -1107,6 +1114,8 @@ void file_test(const int iteration, const int ntasks, const char *path, rank_pro sprintf( temp_path, "%s/%s", testdir, path ); } + + if (verbose >= 3 && rank == 0) { fprintf(out_logfile, "V-3: file_test: create path is \"%s\"\n", temp_path ); fflush( out_logfile ); @@ -1250,6 +1259,7 @@ void file_test(const int iteration, const int ntasks, const char *path, rank_pro collective_create_remove(0, 0, ntasks, temp_path, progress); } } else { + printf("DEBUG %5d rank %d gonna create %s\n", __LINE__, rank, temp_path); create_remove_items(0, 0, 0, 0, temp_path, 0, progress); } } @@ -1942,12 +1952,15 @@ static void mdtest_iteration(int i, int j, MPI_Group testgroup, mdtest_results_t sprintf(rm_name, "mdtest.%d.", (rank+(3*nstride))%i); } if (unique_dir_per_task) { + printf("DEBUG %5d Rank %d i %d nstride %d\n", __LINE__, rank, i, nstride); sprintf(unique_mk_dir, "mdtest_tree.%d.0", (rank+(0*nstride))%i); sprintf(unique_chdir_dir, "mdtest_tree.%d.0", (rank+(1*nstride))%i); sprintf(unique_stat_dir, "mdtest_tree.%d.0", (rank+(2*nstride))%i); sprintf(unique_read_dir, "mdtest_tree.%d.0", (rank+(3*nstride))%i); sprintf(unique_rm_dir, "mdtest_tree.%d.0", (rank+(4*nstride))%i); unique_rm_uni_dir[0] = 0; + printf("DEBUG %5d Rank %d mk_dir %s chdir %s stat_dir %s read_dir %s rm_dir %s\n", __LINE__, + rank, unique_mk_dir,unique_chdir_dir,unique_stat_dir,unique_read_dir,unique_rm_dir); } if (verbose >= 3 && rank == 0) { @@ -1965,6 +1978,7 @@ static void mdtest_iteration(int i, int j, MPI_Group testgroup, mdtest_results_t if (pre_delay) { DelaySecs(pre_delay); } + printf("DEBUG %5d Rank %d gonna file_test on %s\n", __LINE__, rank, unique_mk_dir); file_test(j, i, unique_mk_dir, progress); } } diff --git a/src/utilities.c b/src/utilities.c index 493eb891..5a3168c3 100755 --- a/src/utilities.c +++ b/src/utilities.c @@ -246,6 +246,7 @@ int QueryNodeMapping(MPI_Comm comm) { fprintf( out_logfile, "\n" ); } int ret = node_map[1] == 1; + MPI_Bcast(&ret, 1, MPI_INT, 0, comm); free(node_map); return ret; } From f6491fcd37d9365fd0b2d502d4d3a79b5af755a0 Mon Sep 17 00:00:00 2001 From: John Bent Date: Sat, 27 Jul 2019 13:22:15 -0600 Subject: [PATCH 117/206] Cleaned up the verbose messages by creating a macro and a function --- src/mdtest.c | 548 ++++++++++++++------------------------------------- 1 file changed, 148 insertions(+), 400 deletions(-) diff --git a/src/mdtest.c b/src/mdtest.c index 072dc29e..bfeeda1d 100644 --- a/src/mdtest.c +++ b/src/mdtest.c @@ -36,6 +36,7 @@ #include #include #include +#include #include "option.h" #include "utilities.h" @@ -173,16 +174,30 @@ typedef struct{ /* for making/removing unique directory && stating/deleting subdirectory */ enum {MK_UNI_DIR, STAT_SUB_DIR, READ_SUB_DIR, RM_SUB_DIR, RM_UNI_DIR}; +/* a helper function for passing debug and verbose messages. + use the MACRO as it will insert __LINE__ for you. + Pass the verbose level for root to print, then the verbose level for anyone to print. + Pass -1 to suppress the print for anyone. + Then do the standard printf stuff. This function adds the newline for you. +*/ +#define VERBOSE(root,any,...) VerboseMessage(root,any,__LINE__,__VA_ARGS__) +void VerboseMessage (int root_level, int any_level, int line, char * format, ...) { + if ((rank==0 && verbose >= root_level) || (any_level > 0 && verbose >= any_level)) { + char buffer[1024]; + va_list args; + va_start (args, format); + vsnprintf (buffer, 1024, format, args); + va_end (args); + fprintf( out_logfile, "V-%d: Rank %3d Line %5d %s\n", root_level, rank, line, buffer ); + } +} void offset_timers(double * t, int tcount) { double toffset; int i; - if (( rank == 0 ) && ( verbose >= 1 )) { - fprintf( out_logfile, "V-1: Entering offset_timers...\n" ); - fflush( out_logfile ); - } + VERBOSE(1,-1,"V-1: Entering offset_timers..." ); toffset = GetTimeStamp() - t[tcount]; for (i = 0; i < tcount+1; i++) { @@ -196,10 +211,7 @@ void parse_dirpath(char *dirpath_arg) { int i = 0; - if (( rank == 0 ) && ( verbose >= 1 )) { - fprintf( out_logfile, "V-1: Entering parse_dirpath...\n" ); - fflush( out_logfile ); - } + VERBOSE(1,-1, "Entering parse_dirpath..." ); tmp = dirpath_arg; @@ -240,10 +252,7 @@ static void prep_testdir(int j, int dir_iter){ */ void unique_dir_access(int opt, char *to) { - if (( rank == 0 ) && ( verbose >= 1 )) { - fprintf( out_logfile, "V-1: Entering unique_dir_access...\n" ); - fflush( out_logfile ); - } + VERBOSE(1,-1,"Entering unique_dir_access..." ); if (opt == MK_UNI_DIR) { MPI_Barrier(testComm); @@ -263,20 +272,13 @@ static void create_remove_dirs (const char *path, bool create, uint64_t itemNum) char curr_item[MAX_PATHLEN]; const char *operation = create ? "create" : "remove"; - if (( rank == 0 ) && - ( verbose >= 3 ) && - (itemNum % ITEM_COUNT==0 && (itemNum != 0))) { - - fprintf(out_logfile, "V-3: %s dir: "LLU"\n", operation, itemNum); - fflush(out_logfile); + if ( (itemNum % ITEM_COUNT==0 && (itemNum != 0))) { + VERBOSE(3,5,"dir: "LLU"", operation, itemNum); } //create dirs sprintf(curr_item, "%s/dir.%s%" PRIu64, path, create ? mk_name : rm_name, itemNum); - if (verbose >= 5 || (rank == 0 && verbose >= 3)) { - fprintf(out_logfile, "V-3: create_remove_items_helper (dirs %s): curr_item is \"%s\"\n", operation, curr_item); - fflush(out_logfile); - } + VERBOSE(3,5,"create_remove_items_helper (dirs %s): curr_item is \"%s\"", operation, curr_item); if (create) { if (backend->mkdir(curr_item, DIRMODE, ¶m) == -1) { @@ -292,20 +294,13 @@ static void create_remove_dirs (const char *path, bool create, uint64_t itemNum) static void remove_file (const char *path, uint64_t itemNum) { char curr_item[MAX_PATHLEN]; - if (( rank == 0 ) && - ( verbose >= 3 ) && - (itemNum % ITEM_COUNT==0 && (itemNum != 0))) { - - fprintf(out_logfile, "V-3: remove file: "LLU"\n", itemNum); - fflush(out_logfile); + if ( (itemNum % ITEM_COUNT==0 && (itemNum != 0))) { + VERBOSE(3,5,"remove file: "LLU"\n", itemNum); } //remove files sprintf(curr_item, "%s/file.%s"LLU"", path, rm_name, itemNum); - if (verbose >= 5 || (rank == 0 && verbose >= 3)) { - fprintf(out_logfile, "V-3: create_remove_items_helper (non-dirs remove): curr_item is \"%s\"\n", curr_item); - fflush(out_logfile); - } + VERBOSE(3,5,"create_remove_items_helper (non-dirs remove): curr_item is \"%s\"", curr_item); if (!(shared_file && rank != 0)) { backend->delete (curr_item, ¶m); } @@ -315,28 +310,18 @@ static void create_file (const char *path, uint64_t itemNum) { char curr_item[MAX_PATHLEN]; void *aiori_fh; - if (( rank == 0 ) && - ( verbose >= 3 ) && - (itemNum % ITEM_COUNT==0 && (itemNum != 0))) { - - fprintf(out_logfile, "V-3: create file: "LLU"\n", itemNum); - fflush(out_logfile); + if ( (itemNum % ITEM_COUNT==0 && (itemNum != 0))) { + VERBOSE(3,5,"create file: "LLU"", itemNum); } //create files sprintf(curr_item, "%s/file.%s"LLU"", path, mk_name, itemNum); - if ((rank == 0 && verbose >= 3) || verbose >= 5) { - fprintf(out_logfile, "V-3: create_remove_items_helper (non-dirs create): curr_item is \"%s\"\n", curr_item); - fflush(out_logfile); - } + VERBOSE(3,5,"create_remove_items_helper (non-dirs create): curr_item is \"%s\"", curr_item); if (collective_creates) { param.openFlags = IOR_WRONLY; - if (verbose >= 5 || (rank == 0 && verbose >= 3)) { - fprintf(out_logfile, "V-3: create_remove_items_helper (collective): open...\n" ); - fflush( out_logfile ); - } + VERBOSE(3,5,"create_remove_items_helper (collective): open..." ); if (make_node) aiori_fh = backend->mknod (curr_item); @@ -353,10 +338,7 @@ static void create_file (const char *path, uint64_t itemNum) { param.openFlags = IOR_CREAT | IOR_WRONLY; param.filePerProc = !shared_file; - if (verbose >= 5 || (rank == 0 && verbose >= 3)) { - fprintf(out_logfile, "V-3: create_remove_items_helper (non-collective, shared): open...\n" ); - fflush( out_logfile ); - } + VERBOSE(3,5,"create_remove_items_helper (non-collective, shared): open..." ); if (make_node) aiori_fh = backend->mknod (curr_item); @@ -368,10 +350,7 @@ static void create_file (const char *path, uint64_t itemNum) { } if (write_bytes > 0) { - if (verbose >= 5 || (rank == 0 && verbose >= 3)) { - fprintf(out_logfile, "V-3: create_remove_items_helper: write...\n" ); - fflush( out_logfile ); - } + VERBOSE(3,5,"create_remove_items_helper: write..." ); /* * According to Bill Loewe, writes are only done one time, so they are always at @@ -384,10 +363,7 @@ static void create_file (const char *path, uint64_t itemNum) { } } - if (verbose >= 5 || (rank == 0 && verbose >= 3)) { - fprintf(out_logfile, "V-3: create_remove_items_helper: close...\n" ); - fflush( out_logfile ); - } + VERBOSE(3,5,"create_remove_items_helper: close..." ); if (!make_node) backend->close (aiori_fh, ¶m); @@ -397,10 +373,7 @@ static void create_file (const char *path, uint64_t itemNum) { void create_remove_items_helper(const int dirs, const int create, const char *path, uint64_t itemNum, rank_progress_t * progress) { - if (( rank == 0 ) && ( verbose >= 1 )) { - fprintf( out_logfile, "V-1: Entering create_remove_items_helper...\n" ); - fflush( out_logfile ); - } + VERBOSE(1,-1,"Entering create_remove_items_helper..." ); for (uint64_t i = progress->items_start; i < progress->items_per_dir ; ++i) { if (!dirs) { @@ -426,10 +399,7 @@ void create_remove_items_helper(const int dirs, const int create, const char *pa void collective_helper(const int dirs, const int create, const char* path, uint64_t itemNum, rank_progress_t * progress) { char curr_item[MAX_PATHLEN]; - if (( rank == 0 ) && ( verbose >= 1 )) { - fprintf( out_logfile, "V-1: Entering collective_helper...\n" ); - fflush( out_logfile ); - } + VERBOSE(1,-1,"Entering collective_helper..." ); for (uint64_t i = progress->items_start ; i < progress->items_per_dir ; ++i) { if (dirs) { create_remove_dirs (path, create, itemNum + i); @@ -437,10 +407,7 @@ void collective_helper(const int dirs, const int create, const char* path, uint6 } sprintf(curr_item, "%s/file.%s"LLU"", path, create ? mk_name : rm_name, itemNum+i); - if (verbose >= 5 || (rank == 0 && verbose >= 3)) { - fprintf(out_logfile, "V-3: create file: %s\n", curr_item); - fflush(out_logfile); - } + VERBOSE(3,5,"create file: %s", curr_item); if (create) { void *aiori_fh; @@ -474,19 +441,13 @@ void create_remove_items(int currDepth, const int dirs, const int create, const unsigned long long currDir = dirNum; - if (( rank == 0 ) && ( verbose >= 1 )) { - fprintf( out_logfile, "V-1: Entering create_remove_items, currDepth = %d...\n", currDepth ); - fflush( out_logfile ); - } + VERBOSE(1,-1,"Entering create_remove_items, currDepth = %d...", currDepth ); memset(dir, 0, MAX_PATHLEN); strcpy(temp_path, path); - if (verbose >= 5 || (rank == 0 && verbose >= 3)) { - fprintf(out_logfile, "V-3: create_remove_items (start): temp_path is \"%s\"\n", temp_path ); - fflush(out_logfile); - } + VERBOSE(3,5,"create_remove_items (start): temp_path is \"%s\"", temp_path ); if (currDepth == 0) { /* create items at this depth */ @@ -513,10 +474,7 @@ void create_remove_items(int currDepth, const int dirs, const int create, const strcat(temp_path, "/"); strcat(temp_path, dir); - if (verbose >= 5 || (rank == 0 && verbose >= 3)) { - fprintf(out_logfile, "V-3: create_remove_items (for loop): temp_path is \"%s\"\n", temp_path ); - fflush(out_logfile); - } + VERBOSE(3,5,"create_remove_items (for loop): temp_path is \"%s\"", temp_path ); /* create the items in this branch */ if (!leaf_only || (leaf_only && currDepth == depth)) { @@ -553,10 +511,7 @@ void mdtest_stat(const int random, const int dirs, const long dir_iter, const ch uint64_t parent_dir, item_num = 0; char item[MAX_PATHLEN], temp[MAX_PATHLEN]; - if (( rank == 0 ) && ( verbose >= 1 )) { - fprintf( out_logfile, "V-1: Entering mdtest_stat...\n" ); - fflush( out_logfile ); - } + VERBOSE(1,-1,"Entering mdtest_stat..." ); uint64_t stop_items = items; @@ -592,15 +547,13 @@ void mdtest_stat(const int random, const int dirs, const long dir_iter, const ch /* create name of file/dir to stat */ if (dirs) { - if ((verbose >= 5 || (rank == 0 && verbose >= 3)) && (i%ITEM_COUNT == 0) && (i != 0)) { - fprintf(out_logfile, "V-3: stat dir: "LLU"\n", i); - fflush(out_logfile); + if ((i%ITEM_COUNT == 0) && (i != 0)) { + VERBOSE(3,5,"stat dir: "LLU"", i); } sprintf(item, "dir.%s"LLU"", stat_name, item_num); } else { - if ((verbose >= 5 || (rank == 0 && verbose >= 3)) && (i%ITEM_COUNT == 0) && (i != 0)) { - fprintf(out_logfile, "V-3: stat file: "LLU"\n", i); - fflush(out_logfile); + if ((i%ITEM_COUNT == 0) && (i != 0)) { + VERBOSE(3,5,"stat file: "LLU"", i); } sprintf(item, "file.%s"LLU"", stat_name, item_num); } @@ -627,21 +580,9 @@ void mdtest_stat(const int random, const int dirs, const long dir_iter, const ch strcpy( item, temp ); /* below temp used to be hiername */ - if (verbose >= 5 || (rank == 0 && verbose >= 3)) { - if (dirs) { - fprintf(out_logfile, "V-3: mdtest_stat dir : %s\n", item); - } else { - fprintf(out_logfile, "V-3: mdtest_stat file: %s\n", item); - } - fflush(out_logfile); - } - + VERBOSE(3,5,"mdtest_stat %4s: %s", (dirs ? "dir" : "file"), item); if (-1 == backend->stat (item, &buf, ¶m)) { char msg_buf[4096]; - if ( verbose >=3 ) { - fprintf( out_logfile, "V-3: Stat'ing %s \"%s\"\n", dirs ? "directory" : "file", item ); - fflush( out_logfile ); - } snprintf(msg_buf, 4096, "unable to stat %s %s", dirs ? "directory" : "file", item); FAIL(msg_buf); } @@ -655,10 +596,7 @@ void mdtest_read(int random, int dirs, const long dir_iter, char *path) { char item[MAX_PATHLEN], temp[MAX_PATHLEN]; void *aiori_fh; - if (( rank == 0 ) && ( verbose >= 1 )) { - fprintf( out_logfile, "V-1: Entering mdtest_read...\n" ); - fflush( out_logfile ); - } + VERBOSE(1,-1,"Entering mdtest_read..." ); /* allocate read buffer */ if (read_bytes > 0) { @@ -703,9 +641,8 @@ void mdtest_read(int random, int dirs, const long dir_iter, char *path) { /* create name of file to read */ if (!dirs) { - if ((verbose >= 5 || (rank == 0 && verbose >= 3)) && (i%ITEM_COUNT == 0) && (i != 0)) { - fprintf(out_logfile, "V-3: read file: "LLU"\n", i); - fflush(out_logfile); + if ((i%ITEM_COUNT == 0) && (i != 0)) { + VERBOSE(3,5,"read file: "LLU"", i); } sprintf(item, "file.%s"LLU"", read_name, item_num); } @@ -732,12 +669,7 @@ void mdtest_read(int random, int dirs, const long dir_iter, char *path) { strcpy( item, temp ); /* below temp used to be hiername */ - if (verbose >= 5 || (rank == 0 && verbose >= 3)) { - if (!dirs) { - fprintf(out_logfile, "V-3: mdtest_read file: %s\n", item); - } - fflush(out_logfile); - } + VERBOSE(3,5,"mdtest_read file: %s", item); /* open file for reading */ param.openFlags = O_RDONLY; @@ -763,10 +695,7 @@ void mdtest_read(int random, int dirs, const long dir_iter, char *path) { void collective_create_remove(const int create, const int dirs, const int ntasks, const char *path, rank_progress_t * progress) { char temp[MAX_PATHLEN]; - if (( rank == 0 ) && ( verbose >= 1 )) { - fprintf( out_logfile, "V-1: Entering collective_create_remove...\n" ); - fflush( out_logfile ); - } + VERBOSE(1,-1,"Entering collective_create_remove on %s", path ); /* rank 0 does all of the creates and removes for all of the ranks */ for (int i = 0 ; i < ntasks ; ++i) { @@ -794,7 +723,7 @@ void collective_create_remove(const int create, const int dirs, const int ntasks sprintf(rm_name, "mdtest.%d.", (i+(3*nstride))%ntasks); } if (unique_dir_per_task) { - printf("DEBUG %5d Rank %d i %d nstride %d ntasks %d", __LINE__, rank, i, nstride, ntasks); + VERBOSE(3,5,"i %d nstride %d ntasks %d", i, nstride, ntasks); sprintf(unique_mk_dir, "%s/mdtest_tree.%d.0", testdir, (i+(0*nstride))%ntasks); sprintf(unique_chdir_dir, "%s/mdtest_tree.%d.0", testdir, @@ -809,10 +738,7 @@ void collective_create_remove(const int create, const int dirs, const int ntasks } /* Now that everything is set up as it should be, do the create or remove */ - if (verbose >= 5 || (rank == 0 && verbose >= 3)) { - fprintf(out_logfile, "V-3: collective_create_remove (create_remove_items): temp is \"%s\"\n", temp); - fflush( out_logfile ); - } + VERBOSE(3,5,"collective_create_remove (create_remove_items): temp is \"%s\"", temp); create_remove_items(0, dirs, create, 1, temp, 0, progress); } @@ -851,10 +777,7 @@ void directory_test(const int iteration, const int ntasks, const char *path, ran MPI_Comm_size(testComm, &size); - if (( rank == 0 ) && ( verbose >= 1 )) { - fprintf( out_logfile, "V-1: Entering directory_test...\n" ); - fflush( out_logfile ); - } + VERBOSE(1,-1,"Entering directory_test on %s", path ); MPI_Barrier(testComm); t[0] = GetTimeStamp(); @@ -872,10 +795,7 @@ void directory_test(const int iteration, const int ntasks, const char *path, ran sprintf( temp_path, "%s/%s", testdir, path ); } - if (verbose >= 3 && rank == 0) { - fprintf(out_logfile, "V-3: directory_test: create path is \"%s\"\n", temp_path ); - fflush( out_logfile ); - } + VERBOSE(3,-1,"directory_test: create path is \"%s\"", temp_path ); /* "touch" the files */ if (collective_creates) { @@ -907,10 +827,7 @@ void directory_test(const int iteration, const int ntasks, const char *path, ran sprintf( temp_path, "%s/%s", testdir, path ); } - if (verbose >= 3 && rank == 0) { - fprintf(out_logfile, "V-3: directory_test: stat path is \"%s\"\n", temp_path ); - fflush( out_logfile ); - } + VERBOSE(3,5,"stat path is \"%s\"", temp_path ); /* stat directories */ if (random_seed > 0) { @@ -939,10 +856,7 @@ void directory_test(const int iteration, const int ntasks, const char *path, ran sprintf( temp_path, "%s/%s", testdir, path ); } - if (verbose >= 3 && rank == 0) { - fprintf(out_logfile, "V-3: directory_test: read path is \"%s\"\n", temp_path ); - fflush( out_logfile ); - } + VERBOSE(3,5,"directory_test: read path is \"%s\"", temp_path ); /* read directories */ if (random_seed > 0) { @@ -970,10 +884,7 @@ void directory_test(const int iteration, const int ntasks, const char *path, ran sprintf( temp_path, "%s/%s", testdir, path ); } - if (verbose >= 3 && rank == 0) { - fprintf(out_logfile, "V-3: directory_test: remove directories path is \"%s\"\n", temp_path ); - fflush( out_logfile ); - } + VERBOSE(3,5,"directory_test: remove directories path is \"%s\"", temp_path ); /* remove directories */ if (collective_creates) { @@ -998,10 +909,7 @@ void directory_test(const int iteration, const int ntasks, const char *path, ran sprintf( temp_path, "%s/%s", testdir, path ); } - if (verbose >= 3 && rank == 0) { - fprintf(out_logfile, "V-3: directory_test: remove unique directories path is \"%s\"\n", temp_path ); - fflush( out_logfile ); - } + VERBOSE(3,5,"directory_test: remove unique directories path is \"%s\"\n", temp_path ); } if (unique_dir_per_task && !time_unique_dir_overhead) { @@ -1034,30 +942,21 @@ void directory_test(const int iteration, const int ntasks, const char *path, ran summary_table[iteration].stonewall_last_item[3] = items; } - if (verbose >= 1 && rank == 0) { - fprintf(out_logfile, "V-1: Directory creation: %14.3f sec, %14.3f ops/sec\n", - t[1] - t[0], summary_table[iteration].rate[0]); - fprintf(out_logfile, "V-1: Directory stat : %14.3f sec, %14.3f ops/sec\n", - t[2] - t[1], summary_table[iteration].rate[1]); -/* N/A - fprintf(out_logfile, "V-1: Directory read : %14.3f sec, %14.3f ops/sec\n", - t[3] - t[2], summary_table[iteration].rate[2]); -*/ - fprintf(out_logfile, "V-1: Directory removal : %14.3f sec, %14.3f ops/sec\n", - t[4] - t[3], summary_table[iteration].rate[3]); - fflush(out_logfile); - } + VERBOSE(1,-1," Directory creation: %14.3f sec, %14.3f ops/sec\n", t[1] - t[0], summary_table[iteration].rate[0]); + VERBOSE(1,-1," Directory stat : %14.3f sec, %14.3f ops/sec\n", t[2] - t[1], summary_table[iteration].rate[1]); + /* N/A + VERBOSE(1,-1," Directory read : %14.3f sec, %14.3f ops/sec\n", t[3] - t[2], summary_table[iteration].rate[2]); + */ + VERBOSE(1,-1," Directory removal : %14.3f sec, %14.3f ops/sec\n", t[4] - t[3], summary_table[iteration].rate[3]); } /* Returns if the stonewall was hit */ int updateStoneWallIterations(int iteration, rank_progress_t * progress, double tstart){ int hit = 0; - if (verbose >= 1 ) { - fprintf( out_logfile, "V-1: rank %d stonewall hit with %lld items\n", rank, (long long) progress->items_done ); - fflush( out_logfile ); - } uint64_t done = progress->items_done; long long unsigned max_iter = 0; + + VERBOSE(1,1,"stonewall hit with %lld items", (long long) progress->items_done ); MPI_Allreduce(& progress->items_done, & max_iter, 1, MPI_LONG_LONG_INT, MPI_MAX, testComm); summary_table[iteration].stonewall_time[MDTEST_FILE_CREATE_NUM] = GetTimeStamp() - tstart; @@ -1088,12 +987,7 @@ void file_test(const int iteration, const int ntasks, const char *path, rank_pro char temp_path[MAX_PATHLEN]; MPI_Comm_size(testComm, &size); - printf("DEBUG %5d Rank %d file_test on %s\n", __LINE__, rank, path); - - if (( rank == 0 ) && ( verbose >= 1 )) { - fprintf( out_logfile, "V-1: Entering file_test...\n" ); - fflush( out_logfile ); - } + VERBOSE(3,5,"Entering file_test on %s", path); MPI_Barrier(testComm); t[0] = GetTimeStamp(); @@ -1104,9 +998,8 @@ void file_test(const int iteration, const int ntasks, const char *path, rank_pro prep_testdir(iteration, dir_iter); if (unique_dir_per_task) { - printf("DEBUG %5d Rank %d operating on %s\n", __LINE__, rank, temp_path); unique_dir_access(MK_UNI_DIR, temp_path); - printf("DEBUG %5d Rank %d operating on %s\n", __LINE__, rank, temp_path); + VERBOSE(5,5,"operating on %s", temp_path); if (!time_unique_dir_overhead) { offset_timers(t, 0); } @@ -1116,10 +1009,7 @@ void file_test(const int iteration, const int ntasks, const char *path, rank_pro - if (verbose >= 3 && rank == 0) { - fprintf(out_logfile, "V-3: file_test: create path is \"%s\"\n", temp_path ); - fflush( out_logfile ); - } + VERBOSE(3,-1,"file_test: create path is \"%s\"", temp_path ); /* "touch" the files */ if (collective_creates) { @@ -1136,9 +1026,7 @@ void file_test(const int iteration, const int ntasks, const char *path, rank_pro if (hit){ progress->stone_wall_timer_seconds = 0; - if (verbose > 1){ - printf("stonewall rank %d: %lld of %lld \n", rank, (long long) progress->items_start, (long long) progress->items_per_dir); - } + VERBOSE(1,1,"stonewall: %lld of %lld", (long long) progress->items_start, (long long) progress->items_per_dir); create_remove_items(0, 0, 1, 0, temp_path, 0, progress); // now reset the values progress->stone_wall_timer_seconds = stone_wall_timer_seconds; @@ -1163,8 +1051,8 @@ void file_test(const int iteration, const int ntasks, const char *path, rank_pro if (rank == 0) { if(expected_items == -1){ fprintf(out_logfile, "WARNING: could not read stonewall status file\n"); - }else if(verbose >= 1){ - fprintf(out_logfile, "Read stonewall status; items: "LLU"\n", items); + }else { + VERBOSE(1,1, "Read stonewall status; items: "LLU"\n", items); } } } @@ -1188,10 +1076,7 @@ void file_test(const int iteration, const int ntasks, const char *path, rank_pro sprintf( temp_path, "%s/%s", testdir, path ); } - if (verbose >= 3 && rank == 0) { - fprintf(out_logfile, "V-3: file_test: stat path is \"%s\"\n", temp_path ); - fflush( out_logfile ); - } + VERBOSE(3,5,"file_test: stat path is \"%s\"", temp_path ); /* stat files */ mdtest_stat((random_seed > 0 ? 1 : 0), 0, dir_iter, temp_path, progress); @@ -1216,10 +1101,7 @@ void file_test(const int iteration, const int ntasks, const char *path, rank_pro sprintf( temp_path, "%s/%s", testdir, path ); } - if (verbose >= 3 && rank == 0) { - fprintf(out_logfile, "V-3: file_test: read path is \"%s\"\n", temp_path ); - fflush( out_logfile ); - } + VERBOSE(3,5,"file_test: read path is \"%s\"", temp_path ); /* read files */ if (random_seed > 0) { @@ -1249,17 +1131,14 @@ void file_test(const int iteration, const int ntasks, const char *path, rank_pro sprintf( temp_path, "%s/%s", testdir, path ); } - if (verbose >= 3 && rank == 0) { - fprintf(out_logfile, "V-3: file_test: rm directories path is \"%s\"\n", temp_path ); - fflush( out_logfile ); - } + VERBOSE(3,5,"file_test: rm directories path is \"%s\"", temp_path ); if (collective_creates) { if (rank == 0) { collective_create_remove(0, 0, ntasks, temp_path, progress); } } else { - printf("DEBUG %5d rank %d gonna create %s\n", __LINE__, rank, temp_path); + VERBOSE(3,5,"gonna create %s", temp_path); create_remove_items(0, 0, 0, 0, temp_path, 0, progress); } } @@ -1276,10 +1155,7 @@ void file_test(const int iteration, const int ntasks, const char *path, rank_pro strcpy( temp_path, path ); } - if (verbose >= 3 && rank == 0) { - fprintf(out_logfile, "V-3: file_test: rm unique directories path is \"%s\"\n", temp_path ); - fflush( out_logfile ); - } + VERBOSE(3,5,"file_test: rm unique directories path is \"%s\"", temp_path ); } if (unique_dir_per_task && !time_unique_dir_overhead) { @@ -1316,17 +1192,10 @@ void file_test(const int iteration, const int ntasks, const char *path, rank_pro summary_table[iteration].stonewall_last_item[7] = items; } - if (verbose >= 1 && rank == 0) { - fprintf(out_logfile, "V-1: File creation : %14.3f sec, %14.3f ops/sec\n", - t[1] - t[0], summary_table[iteration].rate[4]); - fprintf(out_logfile, "V-1: File stat : %14.3f sec, %14.3f ops/sec\n", - t[2] - t[1], summary_table[iteration].rate[5]); - fprintf(out_logfile, "V-1: File read : %14.3f sec, %14.3f ops/sec\n", - t[3] - t[2], summary_table[iteration].rate[6]); - fprintf(out_logfile, "V-1: File removal : %14.3f sec, %14.3f ops/sec\n", - t[4] - t[3], summary_table[iteration].rate[7]); - fflush(out_logfile); - } + VERBOSE(1,-1," File creation : %14.3f sec, %14.3f ops/sec", t[1] - t[0], summary_table[iteration].rate[4]); + VERBOSE(1,-1," File stat : %14.3f sec, %14.3f ops/sec", t[2] - t[1], summary_table[iteration].rate[5]); + VERBOSE(1,-1," File read : %14.3f sec, %14.3f ops/sec", t[3] - t[2], summary_table[iteration].rate[6]); + VERBOSE(1,-1," File removal : %14.3f sec, %14.3f ops/sec", t[4] - t[3], summary_table[iteration].rate[7]); } void print_help (void) { @@ -1398,10 +1267,7 @@ void summarize_results(int iterations) { double all[iterations * size * tableSize]; - if (( rank == 0 ) && ( verbose >= 1 )) { - fprintf( out_logfile, "V-1: Entering summarize_results...\n" ); - fflush( out_logfile ); - } + VERBOSE(1,-1,"Entering summarize_results..." ); MPI_Barrier(testComm); for(int i=0; i < iterations; i++){ @@ -1544,16 +1410,10 @@ void valid_tests() { if (!create_only && !stat_only && !read_only && !remove_only) { create_only = stat_only = read_only = remove_only = 1; - if (( rank == 0 ) && ( verbose >= 1 )) { - fprintf( out_logfile, "V-1: main: Setting create/stat/read/remove_only to True\n" ); - fflush( out_logfile ); - } + VERBOSE(1,-1,"main: Setting create/stat/read/remove_only to True" ); } - if (( rank == 0 ) && ( verbose >= 1 )) { - fprintf( out_logfile, "V-1: Entering valid_tests...\n" ); - fflush( out_logfile ); - } + VERBOSE(1,-1,"Entering valid_tests..." ); /* if dirs_only and files_only were both left unset, set both now */ if (!dirs_only && !files_only) { @@ -1645,10 +1505,7 @@ void show_file_system_size(char *file_system) { ior_aiori_statfs_t stat_buf; int ret; - if (( rank == 0 ) && ( verbose >= 1 )) { - fprintf( out_logfile, "V-1: Entering show_file_system_size...\n" ); - fflush( out_logfile ); - } + VERBOSE(1,-1,"Entering show_file_system_size..." ); ret = backend->statfs (file_system, &stat_buf, ¶m); if (0 != ret) { @@ -1699,15 +1556,7 @@ void display_freespace(char *testdirpath) int directoryFound = 0; - if (( rank == 0 ) && ( verbose >= 1 )) { - fprintf( out_logfile, "V-1: Entering display_freespace...\n" ); - fflush( out_logfile ); - } - - if (verbose >= 3 && rank == 0) { - fprintf(out_logfile, "V-3: testdirpath is \"%s\"\n", testdirpath ); - fflush( out_logfile ); - } + VERBOSE(3,5,"Entering display_freespace on %s...", testdirpath ); strcpy(dirpath, testdirpath); @@ -1726,17 +1575,9 @@ void display_freespace(char *testdirpath) strcpy(dirpath, "."); } - if (verbose >= 3 && rank == 0) { - fprintf(out_logfile, "V-3: Before show_file_system_size, dirpath is \"%s\"\n", dirpath ); - fflush( out_logfile ); - } - + VERBOSE(3,5,"Before show_file_system_size, dirpath is \"%s\"", dirpath ); show_file_system_size(dirpath); - - if (verbose >= 3 && rank == 0) { - fprintf(out_logfile, "V-3: After show_file_system_size, dirpath is \"%s\"\n", dirpath ); - fflush( out_logfile ); - } + VERBOSE(3,5, "After show_file_system_size, dirpath is \"%s\"\n", dirpath ); return; } @@ -1748,20 +1589,13 @@ void create_remove_directory_tree(int create, char dir[MAX_PATHLEN]; - if (( rank == 0 ) && ( verbose >= 1 )) { - fprintf( out_logfile, "V-1: Entering create_remove_directory_tree, currDepth = %d...\n", currDepth ); - fflush( out_logfile ); - } + VERBOSE(1,5,"Entering create_remove_directory_tree on %s, currDepth = %d...", path, currDepth ); if (currDepth == 0) { sprintf(dir, "%s/%s.%d/", path, base_tree_name, dirNum); if (create) { - if (rank == 0 && verbose >= 2) { - fprintf(out_logfile, "V-2: Making directory \"%s\"\n", dir); - fflush(out_logfile); - } - + VERBOSE(2,5,"Making directory \"%s\"", dir); if (-1 == backend->mkdir (dir, DIRMODE, ¶m)) { fprintf(out_logfile, "error could not create directory \"%s\"\n", dir); } @@ -1770,11 +1604,7 @@ void create_remove_directory_tree(int create, create_remove_directory_tree(create, ++currDepth, dir, ++dirNum, progress); if (!create) { - if (rank == 0 && verbose >= 2) { - fprintf(out_logfile, "V-2: Remove directory \"%s\"\n", dir); - fflush(out_logfile); - } - + VERBOSE(2,5,"Remove directory \"%s\"", dir); if (-1 == backend->rmdir(dir, ¶m)) { FAIL("Unable to remove directory"); } @@ -1790,11 +1620,7 @@ void create_remove_directory_tree(int create, strcat(temp_path, dir); if (create) { - if (rank == 0 && verbose >= 2) { - fprintf(out_logfile, "V-2: Making directory \"%s\"\n", temp_path); - fflush(out_logfile); - } - + VERBOSE(2,5,"Making directory \"%s\"", temp_path); if (-1 == backend->mkdir(temp_path, DIRMODE, ¶m)) { FAIL("Unable to create directory"); } @@ -1805,11 +1631,7 @@ void create_remove_directory_tree(int create, currDepth--; if (!create) { - if (rank == 0 && verbose >= 2) { - fprintf(out_logfile, "V-2: Remove directory \"%s\"\n", temp_path); - fflush(out_logfile); - } - + VERBOSE(2,5,"Remove directory \"%s\"", temp_path); if (-1 == backend->rmdir(temp_path, ¶m)) { FAIL("Unable to remove directory"); } @@ -1833,18 +1655,12 @@ static void mdtest_iteration(int i, int j, MPI_Group testgroup, mdtest_results_t double startCreate, endCreate; int k; - if (rank == 0 && verbose >= 1) { - fprintf(out_logfile, "V-1: main: * iteration %d *\n", j+1); - fflush(out_logfile); - } + VERBOSE(1,-1,"main: * iteration %d *", j+1); for (int dir_iter = 0; dir_iter < directory_loops; dir_iter ++){ prep_testdir(j, dir_iter); - if (verbose >= 2 && rank == 0) { - fprintf(out_logfile, "V-2: main (for j loop): making testdir, \"%s\"\n", testdir ); - fflush( out_logfile ); - } + VERBOSE(2,5,"main (for j loop): making testdir, \"%s\"", testdir ); if ((rank < path_count) && backend->access(testdir, F_OK, ¶m) != 0) { if (backend->mkdir(testdir, DIRMODE, ¶m) != 0) { FAIL("Unable to create test directory"); @@ -1869,13 +1685,7 @@ static void mdtest_iteration(int i, int j, MPI_Group testgroup, mdtest_results_t for (k=0; k= 3 && rank == 0) { - fprintf(out_logfile, - "V-3: main (create hierarchical directory loop-collective): Calling create_remove_directory_tree with \"%s\"\n", - testdir ); - fflush( out_logfile ); - } - + VERBOSE(3,5,"main (create hierarchical directory loop-collective): Calling create_remove_directory_tree with \"%s\"", testdir ); /* * Let's pass in the path to the directory we most recently made so that we can use * full paths in the other calls. @@ -1887,13 +1697,7 @@ static void mdtest_iteration(int i, int j, MPI_Group testgroup, mdtest_results_t } } } else if (!collective_creates) { - if (verbose >= 3 && rank == 0) { - fprintf(out_logfile, - "V-3: main (create hierarchical directory loop-!collective_creates): Calling create_remove_directory_tree with \"%s\"\n", - testdir ); - fflush( out_logfile ); - } - + VERBOSE(3,5,"main (create hierarchical directory loop-!collective_creates): Calling create_remove_directory_tree with \"%s\"", testdir ); /* * Let's pass in the path to the directory we most recently made so that we can use * full paths in the other calls. @@ -1902,12 +1706,7 @@ static void mdtest_iteration(int i, int j, MPI_Group testgroup, mdtest_results_t } } else { if (rank == 0) { - if (verbose >= 3 && rank == 0) { - fprintf(out_logfile, - "V-3: main (create hierarchical directory loop-!unque_dir_per_task): Calling create_remove_directory_tree with \"%s\"\n", - testdir ); - fflush( out_logfile ); - } + VERBOSE(3,5,"main (create hierarchical directory loop-!unque_dir_per_task): Calling create_remove_directory_tree with \"%s\"", testdir ); /* * Let's pass in the path to the directory we most recently made so that we can use @@ -1924,11 +1723,7 @@ static void mdtest_iteration(int i, int j, MPI_Group testgroup, mdtest_results_t summary_table->time[8] = (endCreate - startCreate); summary_table->items[8] = num_dirs_in_tree; summary_table->stonewall_last_item[8] = num_dirs_in_tree; - if (verbose >= 1 && rank == 0) { - fprintf(out_logfile, "V-1: main: Tree creation : %14.3f sec, %14.3f ops/sec\n", - (endCreate - startCreate), summary_table->rate[8]); - fflush(out_logfile); - } + VERBOSE(1,-1,"V-1: main: Tree creation : %14.3f sec, %14.3f ops/sec", (endCreate - startCreate), summary_table->rate[8]); } sprintf(unique_mk_dir, "%s.0", base_tree_name); sprintf(unique_chdir_dir, "%s.0", base_tree_name); @@ -1938,10 +1733,7 @@ static void mdtest_iteration(int i, int j, MPI_Group testgroup, mdtest_results_t unique_rm_uni_dir[0] = 0; if (!unique_dir_per_task) { - if (verbose >= 3 && rank == 0) { - fprintf(out_logfile, "V-3: main: Using unique_mk_dir, \"%s\"\n", unique_mk_dir ); - fflush( out_logfile ); - } + VERBOSE(3,-1,"V-3: main: Using unique_mk_dir, \"%s\"", unique_mk_dir ); } if (rank < i) { @@ -1963,10 +1755,7 @@ static void mdtest_iteration(int i, int j, MPI_Group testgroup, mdtest_results_t rank, unique_mk_dir,unique_chdir_dir,unique_stat_dir,unique_read_dir,unique_rm_dir); } - if (verbose >= 3 && rank == 0) { - fprintf(out_logfile, "V-3: main: Copied unique_mk_dir, \"%s\", to topdir\n", unique_mk_dir ); - fflush( out_logfile ); - } + VERBOSE(3,-1,"V-3: main: Copied unique_mk_dir, \"%s\", to topdir", unique_mk_dir ); if (dirs_only && !shared_file) { if (pre_delay) { @@ -1985,10 +1774,7 @@ static void mdtest_iteration(int i, int j, MPI_Group testgroup, mdtest_results_t /* remove directory structure */ if (!unique_dir_per_task) { - if (verbose >= 3 && rank == 0) { - fprintf(out_logfile, "V-3: main: Using testdir, \"%s\"\n", testdir ); - fflush( out_logfile ); - } + VERBOSE(3,-1,"main: Using testdir, \"%s\"", testdir ); } MPI_Barrier(testComm); @@ -2006,12 +1792,7 @@ static void mdtest_iteration(int i, int j, MPI_Group testgroup, mdtest_results_t for (k=0; k= 3 && rank == 0) { - fprintf(out_logfile, - "V-3: main (remove hierarchical directory loop-collective): Calling create_remove_directory_tree with \"%s\"\n", - testdir ); - fflush( out_logfile ); - } + VERBOSE(3,-1,"main (remove hierarchical directory loop-collective): Calling create_remove_directory_tree with \"%s\"", testdir ); /* * Let's pass in the path to the directory we most recently made so that we can use @@ -2024,12 +1805,7 @@ static void mdtest_iteration(int i, int j, MPI_Group testgroup, mdtest_results_t } } } else if (!collective_creates) { - if (verbose >= 3 && rank == 0) { - fprintf(out_logfile, - "V-3: main (remove hierarchical directory loop-!collective): Calling create_remove_directory_tree with \"%s\"\n", - testdir ); - fflush( out_logfile ); - } + VERBOSE(3,-1,"main (remove hierarchical directory loop-!collective): Calling create_remove_directory_tree with \"%s\"", testdir ); /* * Let's pass in the path to the directory we most recently made so that we can use @@ -2039,12 +1815,7 @@ static void mdtest_iteration(int i, int j, MPI_Group testgroup, mdtest_results_t } } else { if (rank == 0) { - if (verbose >= 3 && rank == 0) { - fprintf(out_logfile, - "V-3: main (remove hierarchical directory loop-!unique_dir_per_task): Calling create_remove_directory_tree with \"%s\"\n", - testdir ); - fflush( out_logfile ); - } + VERBOSE(3,-1,"V-3: main (remove hierarchical directory loop-!unique_dir_per_task): Calling create_remove_directory_tree with \"%s\"", testdir ); /* * Let's pass in the path to the directory we most recently made so that we can use @@ -2061,16 +1832,8 @@ static void mdtest_iteration(int i, int j, MPI_Group testgroup, mdtest_results_t summary_table->time[9] = endCreate - startCreate; summary_table->items[9] = num_dirs_in_tree; summary_table->stonewall_last_item[8] = num_dirs_in_tree; - if (verbose >= 1 && rank == 0) { - fprintf(out_logfile, "V-1: main Tree removal : %14.3f sec, %14.3f ops/sec\n", - (endCreate - startCreate), summary_table->rate[9]); - fflush(out_logfile); - } - - if (( rank == 0 ) && ( verbose >=2 )) { - fprintf( out_logfile, "V-2: main (at end of for j loop): Removing testdir of \"%s\"\n", testdir ); - fflush( out_logfile ); - } + VERBOSE(1,-1,"main Tree removal : %14.3f sec, %14.3f ops/sec", (endCreate - startCreate), summary_table->rate[9]); + VERBOSE(2,-1,"main (at end of for j loop): Removing testdir of \"%s\"\n", testdir ); for (int dir_iter = 0; dir_iter < directory_loops; dir_iter ++){ prep_testdir(j, dir_iter); @@ -2204,8 +1967,7 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * if (rank == 0) { fprintf(out_logfile, "-- started at %s --\n\n", PrintTimestamp()); - fprintf(out_logfile, "mdtest-%s was launched with %d total task(s) on %d node(s)\n", - RELEASE_VERS, size, nodeCount); + fprintf(out_logfile, "mdtest-%s was launched with %d total task(s) on %d node(s)\n", RELEASE_VERS, size, nodeCount); fflush(out_logfile); } @@ -2239,42 +2001,39 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * } valid_tests(); - if (( rank == 0 ) && ( verbose >= 1 )) { - // option_print_current(options); - fprintf (out_logfile, "api : %s\n", param.api); - fprintf( out_logfile, "barriers : %s\n", ( barriers ? "True" : "False" )); - fprintf( out_logfile, "collective_creates : %s\n", ( collective_creates ? "True" : "False" )); - fprintf( out_logfile, "create_only : %s\n", ( create_only ? "True" : "False" )); - fprintf( out_logfile, "dirpath(s):\n" ); - for ( i = 0; i < path_count; i++ ) { - fprintf( out_logfile, "\t%s\n", filenames[i] ); - } - fprintf( out_logfile, "dirs_only : %s\n", ( dirs_only ? "True" : "False" )); - fprintf( out_logfile, "read_bytes : "LLU"\n", read_bytes ); - fprintf( out_logfile, "read_only : %s\n", ( read_only ? "True" : "False" )); - fprintf( out_logfile, "first : %d\n", first ); - fprintf( out_logfile, "files_only : %s\n", ( files_only ? "True" : "False" )); - fprintf( out_logfile, "iterations : %d\n", iterations ); - fprintf( out_logfile, "items_per_dir : "LLU"\n", items_per_dir ); - fprintf( out_logfile, "last : %d\n", last ); - fprintf( out_logfile, "leaf_only : %s\n", ( leaf_only ? "True" : "False" )); - fprintf( out_logfile, "items : "LLU"\n", items ); - fprintf( out_logfile, "nstride : %d\n", nstride ); - fprintf( out_logfile, "pre_delay : %d\n", pre_delay ); - fprintf( out_logfile, "remove_only : %s\n", ( leaf_only ? "True" : "False" )); - fprintf( out_logfile, "random_seed : %d\n", random_seed ); - fprintf( out_logfile, "stride : %d\n", stride ); - fprintf( out_logfile, "shared_file : %s\n", ( shared_file ? "True" : "False" )); - fprintf( out_logfile, "time_unique_dir_overhead: %s\n", ( time_unique_dir_overhead ? "True" : "False" )); - fprintf( out_logfile, "stone_wall_timer_seconds: %d\n", stone_wall_timer_seconds); - fprintf( out_logfile, "stat_only : %s\n", ( stat_only ? "True" : "False" )); - fprintf( out_logfile, "unique_dir_per_task : %s\n", ( unique_dir_per_task ? "True" : "False" )); - fprintf( out_logfile, "write_bytes : "LLU"\n", write_bytes ); - fprintf( out_logfile, "sync_file : %s\n", ( sync_file ? "True" : "False" )); - fprintf( out_logfile, "depth : %d\n", depth ); - fprintf( out_logfile, "make_node : %d\n", make_node ); - fflush( out_logfile ); - } + // option_print_current(options); + VERBOSE(1,-1, "api : %s", param.api); + VERBOSE(1,-1, "barriers : %s", ( barriers ? "True" : "False" )); + VERBOSE(1,-1, "collective_creates : %s", ( collective_creates ? "True" : "False" )); + VERBOSE(1,-1, "create_only : %s", ( create_only ? "True" : "False" )); + VERBOSE(1,-1, "dirpath(s):" ); + for ( i = 0; i < path_count; i++ ) { + VERBOSE(1,-1, "\t%s", filenames[i] ); + } + VERBOSE(1,-1, "dirs_only : %s", ( dirs_only ? "True" : "False" )); + VERBOSE(1,-1, "read_bytes : "LLU"", read_bytes ); + VERBOSE(1,-1, "read_only : %s", ( read_only ? "True" : "False" )); + VERBOSE(1,-1, "first : %d", first ); + VERBOSE(1,-1, "files_only : %s", ( files_only ? "True" : "False" )); + VERBOSE(1,-1, "iterations : %d", iterations ); + VERBOSE(1,-1, "items_per_dir : "LLU"", items_per_dir ); + VERBOSE(1,-1, "last : %d", last ); + VERBOSE(1,-1, "leaf_only : %s", ( leaf_only ? "True" : "False" )); + VERBOSE(1,-1, "items : "LLU"", items ); + VERBOSE(1,-1, "nstride : %d", nstride ); + VERBOSE(1,-1, "pre_delay : %d", pre_delay ); + VERBOSE(1,-1, "remove_only : %s", ( leaf_only ? "True" : "False" )); + VERBOSE(1,-1, "random_seed : %d", random_seed ); + VERBOSE(1,-1, "stride : %d", stride ); + VERBOSE(1,-1, "shared_file : %s", ( shared_file ? "True" : "False" )); + VERBOSE(1,-1, "time_unique_dir_overhead: %s", ( time_unique_dir_overhead ? "True" : "False" )); + VERBOSE(1,-1, "stone_wall_timer_seconds: %d", stone_wall_timer_seconds); + VERBOSE(1,-1, "stat_only : %s", ( stat_only ? "True" : "False" )); + VERBOSE(1,-1, "unique_dir_per_task : %s", ( unique_dir_per_task ? "True" : "False" )); + VERBOSE(1,-1, "write_bytes : "LLU"", write_bytes ); + VERBOSE(1,-1, "sync_file : %s", ( sync_file ? "True" : "False" )); + VERBOSE(1,-1, "depth : %d", depth ); + VERBOSE(1,-1, "make_node : %d", make_node ); /* setup total number of items and number of items per dir */ if (depth <= 0) { @@ -2378,10 +2137,7 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * } /* display disk usage */ - if (verbose >= 3 && rank == 0) { - fprintf(out_logfile, "V-3: main (before display_freespace): testdirpath is \"%s\"\n", testdirpath ); - fflush( out_logfile ); - } + VERBOSE(3,-1,"main (before display_freespace): testdirpath is \"%s\"", testdirpath ); if (rank == 0) display_freespace(testdirpath); int packedByNode = QueryNodeMapping(testComm); @@ -2396,10 +2152,7 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * } } - if (verbose >= 3 && rank == 0) { - fprintf(out_logfile, "V-3: main (after display_freespace): testdirpath is \"%s\"\n", testdirpath ); - fflush( out_logfile ); - } + VERBOSE(3,-1,"main (after display_freespace): testdirpath is \"%s\"", testdirpath ); if (rank == 0) { if (random_seed > 0) { @@ -2468,11 +2221,9 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * fprintf(out_logfile, "\n%d tasks, "LLU" directories\n", i, items_all); } } - if (rank == 0 && verbose >= 1) { - fprintf(out_logfile, "\n"); - fprintf(out_logfile, " Operation Duration Rate\n"); - fprintf(out_logfile, " --------- -------- ----\n"); - } + VERBOSE(1,-1,""); + VERBOSE(1,-1," Operation Duration Rate"); + VERBOSE(1,-1," --------- -------- ----"); for (j = 0; j < iterations; j++) { // keep track of the current status for stonewalling @@ -2484,10 +2235,7 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * } } - if (rank == 0) { - fprintf(out_logfile, "\n-- finished at %s --\n", PrintTimestamp()); - fflush(out_logfile); - } + VERBOSE(0,-1,"-- finished at %s --\n", PrintTimestamp()); if (random_seed > 0) { free(rand_array); From e767ef3de980698677450452bd4933a621d4c1c0 Mon Sep 17 00:00:00 2001 From: John Bent Date: Sat, 27 Jul 2019 13:26:39 -0600 Subject: [PATCH 118/206] Remove extraneous print_help function that was causing people to have to edit the same string in two different locations --- src/mdtest.c | 62 ---------------------------------------------------- 1 file changed, 62 deletions(-) diff --git a/src/mdtest.c b/src/mdtest.c index bfeeda1d..368acf1a 100644 --- a/src/mdtest.c +++ b/src/mdtest.c @@ -455,7 +455,6 @@ void create_remove_items(int currDepth, const int dirs, const int create, const if (collective) { collective_helper(dirs, create, temp_path, 0, progress); } else { - printf("DEBUG %5d rank %d gonna create %s\n", __LINE__, rank, temp_path); create_remove_items_helper(dirs, create, temp_path, 0, progress); } } @@ -481,7 +480,6 @@ void create_remove_items(int currDepth, const int dirs, const int create, const if (collective) { collective_helper(dirs, create, temp_path, currDir*items_per_dir, progress); } else { - printf("DEBUG %5d rank %d gonna create %s\n", __LINE__, rank, temp_path); create_remove_items_helper(dirs, create, temp_path, currDir*items_per_dir, progress); } } @@ -1198,66 +1196,6 @@ void file_test(const int iteration, const int ntasks, const char *path, rank_pro VERBOSE(1,-1," File removal : %14.3f sec, %14.3f ops/sec", t[4] - t[3], summary_table[iteration].rate[7]); } -void print_help (void) { - int j; - - char APIs[1024]; - char APIs_legacy[1024]; - aiori_supported_apis(APIs, APIs_legacy, MDTEST); - char apiStr[1024]; - sprintf(apiStr, "API for I/O [%s]", APIs); - - fprintf(out_logfile, - "Usage: mdtest [-b branching_factor] [-B] [-c] [-C] [-d testdir] [-D] [-e number_of_bytes_to_read]\n" - " [-E] [-f first] [-F] [-h] [-i iterations] [-I items_per_dir] [-k] [-l last] [-L]\n" - " [-n number_of_items] [-N stride_length] [-p seconds] [-r]\n" - " [-R[seed]] [-s stride] [-S] [-t] [-T] [-u] [-v] [-a API]\n" - " [-V verbosity_value] [-w number_of_bytes_to_write] [-W seconds] [-y] [-z depth] -Z\n" - "\t-a: %s\n" - "\t-b: branching factor of hierarchical directory structure\n" - "\t-B: no barriers between phases\n" - "\t-c: collective creates: task 0 does all creates\n" - "\t-C: only create files/dirs\n" - "\t-d: the directory in which the tests will run\n" - "\t-D: perform test on directories only (no files)\n" - "\t-e: bytes to read from each file\n" - "\t-E: only read files/dir\n" - "\t-f: first number of tasks on which the test will run\n" - "\t-F: perform test on files only (no directories)\n" - "\t-h: prints this help message\n" - "\t-i: number of iterations the test will run\n" - "\t-I: number of items per directory in tree\n" - "\t-k: use mknod\n" - "\t-l: last number of tasks on which the test will run\n" - "\t-L: files only at leaf level of tree\n" - "\t-n: every process will creat/stat/read/remove # directories and files\n" - "\t-N: stride # between neighbor tasks for file/dir operation (local=0)\n" - "\t-p: pre-iteration delay (in seconds)\n" - "\t-r: only remove files or directories left behind by previous runs\n" - "\t-R: randomly stat files (optional argument for random seed)\n" - "\t-s: stride between the number of nodes for each test\n" - "\t-S: shared file access (file only, no directories)\n" - "\t-t: time unique working directory overhead\n" - "\t-T: only stat files/dirs\n" - "\t-u: unique working directory for each task\n" - "\t-v: verbosity (each instance of option increments by one)\n" - "\t-V: verbosity value\n" - "\t-w: bytes to write to each file after it is created\n" - "\t-W: number in seconds; stonewall timer, write as many seconds and ensure all processes did the same number of operations (currently only stops during create phase)\n" - "\t-x: StoneWallingStatusFile; contains the number of iterations of the creation phase, can be used to split phases across runs\n" - "\t-y: sync file after writing\n" - "\t-z: depth of hierarchical directory structure\n" - "\t-Z: print time instead of rate\n", - apiStr - ); - - MPI_Initialized(&j); - if (j) { - MPI_Finalize(); - } - exit(0); -} - void summarize_results(int iterations) { char access[MAX_PATHLEN]; int i, j, k; From d69957e55b52137d62692872596c755d7fbe727f Mon Sep 17 00:00:00 2001 From: John Bent Date: Sat, 27 Jul 2019 14:31:49 -0600 Subject: [PATCH 119/206] Final changes cleaning up the output messages --- src/mdtest.c | 105 ++++++++++++++++++++++----------------------------- 1 file changed, 46 insertions(+), 59 deletions(-) diff --git a/src/mdtest.c b/src/mdtest.c index 368acf1a..75558c1f 100644 --- a/src/mdtest.c +++ b/src/mdtest.c @@ -188,7 +188,14 @@ void VerboseMessage (int root_level, int any_level, int line, char * format, ... va_start (args, format); vsnprintf (buffer, 1024, format, args); va_end (args); - fprintf( out_logfile, "V-%d: Rank %3d Line %5d %s\n", root_level, rank, line, buffer ); + if (root_level == 0 && any_level == -1) { + /* No header when it is just the standard output */ + fprintf( out_logfile, "%s\n", buffer ); + } else { + /* add a header when the verbose is greater than 0 */ + fprintf( out_logfile, "V-%d: Rank %3d Line %5d %s\n", root_level, rank, line, buffer ); + } + fflush(out_logfile); } } @@ -211,7 +218,7 @@ void parse_dirpath(char *dirpath_arg) { int i = 0; - VERBOSE(1,-1, "Entering parse_dirpath..." ); + VERBOSE(1,-1, "Entering parse_dirpath on %s...", dirpath_arg ); tmp = dirpath_arg; @@ -252,8 +259,6 @@ static void prep_testdir(int j, int dir_iter){ */ void unique_dir_access(int opt, char *to) { - VERBOSE(1,-1,"Entering unique_dir_access..." ); - if (opt == MK_UNI_DIR) { MPI_Barrier(testComm); sprintf( to, "%s/%s", testdir, unique_chdir_dir ); @@ -266,6 +271,7 @@ void unique_dir_access(int opt, char *to) { } else if (opt == RM_UNI_DIR) { sprintf( to, "%s/%s", testdir, unique_rm_uni_dir ); } + VERBOSE(1,-1,"Entering unique_dir_access, set it to %s", to ); } static void create_remove_dirs (const char *path, bool create, uint64_t itemNum) { @@ -373,7 +379,7 @@ static void create_file (const char *path, uint64_t itemNum) { void create_remove_items_helper(const int dirs, const int create, const char *path, uint64_t itemNum, rank_progress_t * progress) { - VERBOSE(1,-1,"Entering create_remove_items_helper..." ); + VERBOSE(1,-1,"Entering create_remove_items_helper on %s", path ); for (uint64_t i = progress->items_start; i < progress->items_per_dir ; ++i) { if (!dirs) { @@ -399,7 +405,7 @@ void create_remove_items_helper(const int dirs, const int create, const char *pa void collective_helper(const int dirs, const int create, const char* path, uint64_t itemNum, rank_progress_t * progress) { char curr_item[MAX_PATHLEN]; - VERBOSE(1,-1,"Entering collective_helper..." ); + VERBOSE(1,-1,"Entering collective_helper on %s", path ); for (uint64_t i = progress->items_start ; i < progress->items_per_dir ; ++i) { if (dirs) { create_remove_dirs (path, create, itemNum + i); @@ -441,7 +447,7 @@ void create_remove_items(int currDepth, const int dirs, const int create, const unsigned long long currDir = dirNum; - VERBOSE(1,-1,"Entering create_remove_items, currDepth = %d...", currDepth ); + VERBOSE(1,-1,"Entering create_remove_items on %s, currDepth = %d...", path, currDepth ); memset(dir, 0, MAX_PATHLEN); @@ -509,7 +515,7 @@ void mdtest_stat(const int random, const int dirs, const long dir_iter, const ch uint64_t parent_dir, item_num = 0; char item[MAX_PATHLEN], temp[MAX_PATHLEN]; - VERBOSE(1,-1,"Entering mdtest_stat..." ); + VERBOSE(1,-1,"Entering mdtest_stat on %s", path ); uint64_t stop_items = items; @@ -594,7 +600,7 @@ void mdtest_read(int random, int dirs, const long dir_iter, char *path) { char item[MAX_PATHLEN], temp[MAX_PATHLEN]; void *aiori_fh; - VERBOSE(1,-1,"Entering mdtest_read..." ); + VERBOSE(1,-1,"Entering mdtest_read on %s", path ); /* allocate read buffer */ if (read_bytes > 0) { @@ -940,12 +946,12 @@ void directory_test(const int iteration, const int ntasks, const char *path, ran summary_table[iteration].stonewall_last_item[3] = items; } - VERBOSE(1,-1," Directory creation: %14.3f sec, %14.3f ops/sec\n", t[1] - t[0], summary_table[iteration].rate[0]); - VERBOSE(1,-1," Directory stat : %14.3f sec, %14.3f ops/sec\n", t[2] - t[1], summary_table[iteration].rate[1]); + VERBOSE(1,-1," Directory creation: %14.3f sec, %14.3f ops/sec", t[1] - t[0], summary_table[iteration].rate[0]); + VERBOSE(1,-1," Directory stat : %14.3f sec, %14.3f ops/sec", t[2] - t[1], summary_table[iteration].rate[1]); /* N/A - VERBOSE(1,-1," Directory read : %14.3f sec, %14.3f ops/sec\n", t[3] - t[2], summary_table[iteration].rate[2]); + VERBOSE(1,-1," Directory read : %14.3f sec, %14.3f ops/sec", t[3] - t[2], summary_table[iteration].rate[2]); */ - VERBOSE(1,-1," Directory removal : %14.3f sec, %14.3f ops/sec\n", t[4] - t[3], summary_table[iteration].rate[3]); + VERBOSE(1,-1," Directory removal : %14.3f sec, %14.3f ops/sec", t[4] - t[3], summary_table[iteration].rate[3]); } /* Returns if the stonewall was hit */ @@ -967,10 +973,7 @@ int updateStoneWallIterations(int iteration, rank_progress_t * progress, double if(items != (sum_accessed / size)){ summary_table[iteration].stonewall_item_sum[MDTEST_FILE_CREATE_NUM] = sum_accessed; summary_table[iteration].stonewall_item_min[MDTEST_FILE_CREATE_NUM] = min_accessed * size; - if (rank == 0){ - fprintf( out_logfile, "Continue stonewall hit min: %lld max: %lld avg: %.1f \n", min_accessed, max_iter, ((double) sum_accessed) / size); - fflush( out_logfile ); - } + VERBOSE(0,-1, "Continue stonewall hit min: %lld max: %lld avg: %.1f \n", min_accessed, max_iter, ((double) sum_accessed) / size); hit = 1; } progress->items_start = done; @@ -1218,12 +1221,9 @@ void summarize_results(int iterations) { if (rank == 0) { - fprintf(out_logfile, "\nSUMMARY %s: (of %d iterations)\n", print_time ? "time": "rate", iterations); - fprintf(out_logfile, - " Operation Max Min Mean Std Dev\n"); - fprintf(out_logfile, - " --------- --- --- ---- -------\n"); - fflush(out_logfile); + VERBOSE(0,-1,"\nSUMMARY %s: (of %d iterations)", print_time ? "time": "rate", iterations); + VERBOSE(0,-1," Operation Max Min Mean Std Dev"); + VERBOSE(0,-1," --------- --- --- ---- -------"); /* if files only access, skip entries 0-3 (the dir tests) */ if (files_only && !dirs_only) { @@ -1342,8 +1342,7 @@ void summarize_results(int iterations) { void valid_tests() { if (((stone_wall_timer_seconds > 0) && (branch_factor > 1)) || ! barriers) { - fprintf(out_logfile, "Error, stone wall timer does only work with a branch factor <= 1 and with barriers\n"); - MPI_Abort(testComm, 1); + FAIL( "Error, stone wall timer does only work with a branch factor <= 1 and with barriers\n"); } if (!create_only && !stat_only && !read_only && !remove_only) { @@ -1443,7 +1442,7 @@ void show_file_system_size(char *file_system) { ior_aiori_statfs_t stat_buf; int ret; - VERBOSE(1,-1,"Entering show_file_system_size..." ); + VERBOSE(1,-1,"Entering show_file_system_size on %s", file_system ); ret = backend->statfs (file_system, &stat_buf, ¶m); if (0 != ret) { @@ -1475,14 +1474,10 @@ void show_file_system_size(char *file_system) { /* show results */ - fprintf(out_logfile, "Path: %s\n", real_path); - fprintf(out_logfile, "FS: %.1f %s Used FS: %2.1f%% ", - total_file_system_size_hr, file_system_unit_str, - used_file_system_percentage); - fprintf(out_logfile, "Inodes: %.1f %s Used Inodes: %2.1f%%\n", - (double)total_inodes / (double)inode_unit_val, - inode_unit_str, used_inode_percentage); - fflush(out_logfile); + VERBOSE(0,-1,"Path: %s", real_path); + VERBOSE(0,-1,"FS: %.1f %s Used FS: %2.1f%% Inodes: %.1f %s Used Inodes: %2.1f%%\n", + total_file_system_size_hr, file_system_unit_str, used_file_system_percentage, + (double)total_inodes / (double)inode_unit_val, inode_unit_str, used_inode_percentage); return; } @@ -1682,15 +1677,14 @@ static void mdtest_iteration(int i, int j, MPI_Group testgroup, mdtest_results_t sprintf(rm_name, "mdtest.%d.", (rank+(3*nstride))%i); } if (unique_dir_per_task) { - printf("DEBUG %5d Rank %d i %d nstride %d\n", __LINE__, rank, i, nstride); + VERBOSE(3,5,"i %d nstride %d", i, nstride); sprintf(unique_mk_dir, "mdtest_tree.%d.0", (rank+(0*nstride))%i); sprintf(unique_chdir_dir, "mdtest_tree.%d.0", (rank+(1*nstride))%i); sprintf(unique_stat_dir, "mdtest_tree.%d.0", (rank+(2*nstride))%i); sprintf(unique_read_dir, "mdtest_tree.%d.0", (rank+(3*nstride))%i); sprintf(unique_rm_dir, "mdtest_tree.%d.0", (rank+(4*nstride))%i); unique_rm_uni_dir[0] = 0; - printf("DEBUG %5d Rank %d mk_dir %s chdir %s stat_dir %s read_dir %s rm_dir %s\n", __LINE__, - rank, unique_mk_dir,unique_chdir_dir,unique_stat_dir,unique_read_dir,unique_rm_dir); + VERBOSE(5,5,"mk_dir %s chdir %s stat_dir %s read_dir %s rm_dir %s\n", unique_mk_dir,unique_chdir_dir,unique_stat_dir,unique_read_dir,unique_rm_dir); } VERBOSE(3,-1,"V-3: main: Copied unique_mk_dir, \"%s\", to topdir", unique_mk_dir ); @@ -1705,7 +1699,7 @@ static void mdtest_iteration(int i, int j, MPI_Group testgroup, mdtest_results_t if (pre_delay) { DelaySecs(pre_delay); } - printf("DEBUG %5d Rank %d gonna file_test on %s\n", __LINE__, rank, unique_mk_dir); + VERBOSE(3,5,"will file_test on %s", unique_mk_dir); file_test(j, i, unique_mk_dir, progress); } } @@ -1903,20 +1897,15 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * tasksPerNode = CountTasksPerNode(testComm); nodeCount = size / tasksPerNode; - if (rank == 0) { - fprintf(out_logfile, "-- started at %s --\n\n", PrintTimestamp()); - fprintf(out_logfile, "mdtest-%s was launched with %d total task(s) on %d node(s)\n", RELEASE_VERS, size, nodeCount); - fflush(out_logfile); + char cmd_buffer[4096]; + strncpy(cmd_buffer, argv[0], 4096); + for (i = 1; i < argc; i++) { + snprintf(&cmd_buffer[strlen(cmd_buffer)], 4096-strlen(cmd_buffer), " \"%s\"", argv[i]); } - if (rank == 0) { - fprintf(out_logfile, "Command line used: %s", argv[0]); - for (i = 1; i < argc; i++) { - fprintf(out_logfile, " \"%s\"", argv[i]); - } - fprintf(out_logfile, "\n"); - fflush(out_logfile); - } + VERBOSE(0,-1,"-- started at %s --\n", PrintTimestamp()); + VERBOSE(0,-1,"mdtest-%s was launched with %d total task(s) on %d node(s)", RELEASE_VERS, size, nodeCount); + VERBOSE(0,-1,"Command line used: %s", cmd_buffer); /* adjust special variables */ barriers = ! no_barriers; @@ -2082,19 +2071,17 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * /* set the shift to mimic IOR and shift by procs per node */ if (nstride > 0) { - if ( packedByNode ) { + if ( nodeCount > 1 && packedByNode ) { nstride *= tasksPerNode; } - if (rank == 0) { - fprintf(out_logfile, "Shifting ranks by %d for each phase.\n", nstride); - } + VERBOSE(0,5,"Shifting ranks by %d for each phase.", nstride); } VERBOSE(3,-1,"main (after display_freespace): testdirpath is \"%s\"", testdirpath ); if (rank == 0) { if (random_seed > 0) { - fprintf(out_logfile, "random seed: %d\n", random_seed); + VERBOSE(0,-1,"random seed: %d", random_seed); } } @@ -2147,16 +2134,16 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * items_all *= num_dirs_in_tree_calc; } if (files_only && dirs_only) { - fprintf(out_logfile, "\n%d tasks, "LLU" files/directories\n", i, items_all); + VERBOSE(0,-1,"%d tasks, "LLU" files/directories", i, items_all); } else if (files_only) { if (!shared_file) { - fprintf(out_logfile, "\n%d tasks, "LLU" files\n", i, items_all); + VERBOSE(0,-1,"%d tasks, "LLU" files", i, items_all); } else { - fprintf(out_logfile, "\n%d tasks, 1 file\n", i); + VERBOSE(0,-1,"%d tasks, 1 file", i); } } else if (dirs_only) { - fprintf(out_logfile, "\n%d tasks, "LLU" directories\n", i, items_all); + VERBOSE(0,-1,"%d tasks, "LLU" directories", i, items_all); } } VERBOSE(1,-1,""); From 629ff810b70cd98f8eaec51167154af70dcc3386 Mon Sep 17 00:00:00 2001 From: John Bent Date: Sat, 27 Jul 2019 15:27:20 -0600 Subject: [PATCH 120/206] Got IOR shifting to work regardless of whether node/task mapping is round-robin or contiguous --- src/ior.c | 20 ++++++++++++++++---- src/ior.h | 1 + 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/ior.c b/src/ior.c index 3ef411cb..bb2b4c27 100755 --- a/src/ior.c +++ b/src/ior.c @@ -939,6 +939,7 @@ static void InitTests(IOR_test_t *tests, MPI_Comm com) params->testComm = com; params->nodes = params->numTasks / tasksPerNode; params->tasksPerNode = tasksPerNode; + params->packedTasks = QueryNodeMapping(com); if (params->numTasks == 0) { params->numTasks = size; } @@ -1360,7 +1361,13 @@ static void TestIoSys(IOR_test_t *test) } if (params->reorderTasks) { /* move two nodes away from writing node */ - rankOffset = (2 * params->tasksPerNode) % params->numTasks; + int shift = 1; + if (params->packedTasks) { + shift = params->tasksPerNode; + } else { + shift = 1; + } + rankOffset = (2 * shift) % params->numTasks; } // update the check buffer @@ -1395,9 +1402,14 @@ static void TestIoSys(IOR_test_t *test) /* Get rankOffset [file offset] for this process to read, based on -C,-Z,-Q,-X options */ /* Constant process offset reading */ if (params->reorderTasks) { - /* move taskPerNodeOffset nodes[1==default] away from writing node */ - rankOffset = (params->taskPerNodeOffset * - params->tasksPerNode) % params->numTasks; + /* move one node away from writing node */ + int shift = 1; + if (params->packedTasks) { + shift=params->tasksPerNode; + } else { + shift=1; + } + rankOffset = (params->taskPerNodeOffset * shift) % params->numTasks; } /* random process offset reading */ if (params->reorderTasksRandom) { diff --git a/src/ior.h b/src/ior.h index 5c46efe7..2ea6c051 100755 --- a/src/ior.h +++ b/src/ior.h @@ -100,6 +100,7 @@ typedef struct int numTasks; /* number of tasks for test */ int nodes; /* number of nodes for test */ int tasksPerNode; /* number of tasks per node */ + int packedTasks; /* are the tasks round-robin across nodes or are they packed tightly*/ int repetitions; /* number of repetitions of test */ int repCounter; /* rep counter */ int multiFile; /* multiple files */ From 168a4077938e29cd46f627612e13b6dc6363ddeb Mon Sep 17 00:00:00 2001 From: John Bent Date: Sun, 28 Jul 2019 09:55:00 -0600 Subject: [PATCH 121/206] Fixed inconsistent spacing that Andreas commented upon --- src/mdtest.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mdtest.c b/src/mdtest.c index 75558c1f..074463e4 100644 --- a/src/mdtest.c +++ b/src/mdtest.c @@ -551,12 +551,12 @@ void mdtest_stat(const int random, const int dirs, const long dir_iter, const ch /* create name of file/dir to stat */ if (dirs) { - if ((i%ITEM_COUNT == 0) && (i != 0)) { + if ( (i % ITEM_COUNT == 0) && (i != 0)) { VERBOSE(3,5,"stat dir: "LLU"", i); } sprintf(item, "dir.%s"LLU"", stat_name, item_num); } else { - if ((i%ITEM_COUNT == 0) && (i != 0)) { + if ( (i % ITEM_COUNT == 0) && (i != 0)) { VERBOSE(3,5,"stat file: "LLU"", i); } sprintf(item, "file.%s"LLU"", stat_name, item_num); From b2d486f749c16fe3375dfc4389732bfae08db5f9 Mon Sep 17 00:00:00 2001 From: John Bent Date: Sun, 28 Jul 2019 10:07:03 -0600 Subject: [PATCH 122/206] Followed Andreas suggestion to replace escape double quotes within printf's with single quotes --- src/mdtest.c | 74 ++++++++++++++++++++++++++-------------------------- 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/src/mdtest.c b/src/mdtest.c index 074463e4..e35a20e2 100644 --- a/src/mdtest.c +++ b/src/mdtest.c @@ -284,7 +284,7 @@ static void create_remove_dirs (const char *path, bool create, uint64_t itemNum) //create dirs sprintf(curr_item, "%s/dir.%s%" PRIu64, path, create ? mk_name : rm_name, itemNum); - VERBOSE(3,5,"create_remove_items_helper (dirs %s): curr_item is \"%s\"", operation, curr_item); + VERBOSE(3,5,"create_remove_items_helper (dirs %s): curr_item is '%s'", operation, curr_item); if (create) { if (backend->mkdir(curr_item, DIRMODE, ¶m) == -1) { @@ -306,7 +306,7 @@ static void remove_file (const char *path, uint64_t itemNum) { //remove files sprintf(curr_item, "%s/file.%s"LLU"", path, rm_name, itemNum); - VERBOSE(3,5,"create_remove_items_helper (non-dirs remove): curr_item is \"%s\"", curr_item); + VERBOSE(3,5,"create_remove_items_helper (non-dirs remove): curr_item is '%s'", curr_item); if (!(shared_file && rank != 0)) { backend->delete (curr_item, ¶m); } @@ -322,7 +322,7 @@ static void create_file (const char *path, uint64_t itemNum) { //create files sprintf(curr_item, "%s/file.%s"LLU"", path, mk_name, itemNum); - VERBOSE(3,5,"create_remove_items_helper (non-dirs create): curr_item is \"%s\"", curr_item); + VERBOSE(3,5,"create_remove_items_helper (non-dirs create): curr_item is '%s'", curr_item); if (collective_creates) { param.openFlags = IOR_WRONLY; @@ -453,7 +453,7 @@ void create_remove_items(int currDepth, const int dirs, const int create, const memset(dir, 0, MAX_PATHLEN); strcpy(temp_path, path); - VERBOSE(3,5,"create_remove_items (start): temp_path is \"%s\"", temp_path ); + VERBOSE(3,5,"create_remove_items (start): temp_path is '%s'", temp_path ); if (currDepth == 0) { /* create items at this depth */ @@ -479,7 +479,7 @@ void create_remove_items(int currDepth, const int dirs, const int create, const strcat(temp_path, "/"); strcat(temp_path, dir); - VERBOSE(3,5,"create_remove_items (for loop): temp_path is \"%s\"", temp_path ); + VERBOSE(3,5,"create_remove_items (for loop): temp_path is '%s'", temp_path ); /* create the items in this branch */ if (!leaf_only || (leaf_only && currDepth == depth)) { @@ -742,7 +742,7 @@ void collective_create_remove(const int create, const int dirs, const int ntasks } /* Now that everything is set up as it should be, do the create or remove */ - VERBOSE(3,5,"collective_create_remove (create_remove_items): temp is \"%s\"", temp); + VERBOSE(3,5,"collective_create_remove (create_remove_items): temp is '%s'", temp); create_remove_items(0, dirs, create, 1, temp, 0, progress); } @@ -799,7 +799,7 @@ void directory_test(const int iteration, const int ntasks, const char *path, ran sprintf( temp_path, "%s/%s", testdir, path ); } - VERBOSE(3,-1,"directory_test: create path is \"%s\"", temp_path ); + VERBOSE(3,-1,"directory_test: create path is '%s'", temp_path ); /* "touch" the files */ if (collective_creates) { @@ -831,7 +831,7 @@ void directory_test(const int iteration, const int ntasks, const char *path, ran sprintf( temp_path, "%s/%s", testdir, path ); } - VERBOSE(3,5,"stat path is \"%s\"", temp_path ); + VERBOSE(3,5,"stat path is '%s'", temp_path ); /* stat directories */ if (random_seed > 0) { @@ -860,7 +860,7 @@ void directory_test(const int iteration, const int ntasks, const char *path, ran sprintf( temp_path, "%s/%s", testdir, path ); } - VERBOSE(3,5,"directory_test: read path is \"%s\"", temp_path ); + VERBOSE(3,5,"directory_test: read path is '%s'", temp_path ); /* read directories */ if (random_seed > 0) { @@ -888,7 +888,7 @@ void directory_test(const int iteration, const int ntasks, const char *path, ran sprintf( temp_path, "%s/%s", testdir, path ); } - VERBOSE(3,5,"directory_test: remove directories path is \"%s\"", temp_path ); + VERBOSE(3,5,"directory_test: remove directories path is '%s'", temp_path ); /* remove directories */ if (collective_creates) { @@ -913,7 +913,7 @@ void directory_test(const int iteration, const int ntasks, const char *path, ran sprintf( temp_path, "%s/%s", testdir, path ); } - VERBOSE(3,5,"directory_test: remove unique directories path is \"%s\"\n", temp_path ); + VERBOSE(3,5,"directory_test: remove unique directories path is '%s'\n", temp_path ); } if (unique_dir_per_task && !time_unique_dir_overhead) { @@ -1010,7 +1010,7 @@ void file_test(const int iteration, const int ntasks, const char *path, rank_pro - VERBOSE(3,-1,"file_test: create path is \"%s\"", temp_path ); + VERBOSE(3,-1,"file_test: create path is '%s'", temp_path ); /* "touch" the files */ if (collective_creates) { @@ -1077,7 +1077,7 @@ void file_test(const int iteration, const int ntasks, const char *path, rank_pro sprintf( temp_path, "%s/%s", testdir, path ); } - VERBOSE(3,5,"file_test: stat path is \"%s\"", temp_path ); + VERBOSE(3,5,"file_test: stat path is '%s'", temp_path ); /* stat files */ mdtest_stat((random_seed > 0 ? 1 : 0), 0, dir_iter, temp_path, progress); @@ -1102,7 +1102,7 @@ void file_test(const int iteration, const int ntasks, const char *path, rank_pro sprintf( temp_path, "%s/%s", testdir, path ); } - VERBOSE(3,5,"file_test: read path is \"%s\"", temp_path ); + VERBOSE(3,5,"file_test: read path is '%s'", temp_path ); /* read files */ if (random_seed > 0) { @@ -1132,7 +1132,7 @@ void file_test(const int iteration, const int ntasks, const char *path, rank_pro sprintf( temp_path, "%s/%s", testdir, path ); } - VERBOSE(3,5,"file_test: rm directories path is \"%s\"", temp_path ); + VERBOSE(3,5,"file_test: rm directories path is '%s'", temp_path ); if (collective_creates) { if (rank == 0) { @@ -1156,7 +1156,7 @@ void file_test(const int iteration, const int ntasks, const char *path, rank_pro strcpy( temp_path, path ); } - VERBOSE(3,5,"file_test: rm unique directories path is \"%s\"", temp_path ); + VERBOSE(3,5,"file_test: rm unique directories path is '%s'", temp_path ); } if (unique_dir_per_task && !time_unique_dir_overhead) { @@ -1508,9 +1508,9 @@ void display_freespace(char *testdirpath) strcpy(dirpath, "."); } - VERBOSE(3,5,"Before show_file_system_size, dirpath is \"%s\"", dirpath ); + VERBOSE(3,5,"Before show_file_system_size, dirpath is '%s'", dirpath ); show_file_system_size(dirpath); - VERBOSE(3,5, "After show_file_system_size, dirpath is \"%s\"\n", dirpath ); + VERBOSE(3,5, "After show_file_system_size, dirpath is '%s'\n", dirpath ); return; } @@ -1528,16 +1528,16 @@ void create_remove_directory_tree(int create, sprintf(dir, "%s/%s.%d/", path, base_tree_name, dirNum); if (create) { - VERBOSE(2,5,"Making directory \"%s\"", dir); + VERBOSE(2,5,"Making directory '%s'", dir); if (-1 == backend->mkdir (dir, DIRMODE, ¶m)) { - fprintf(out_logfile, "error could not create directory \"%s\"\n", dir); + fprintf(out_logfile, "error could not create directory '%s'\n", dir); } } create_remove_directory_tree(create, ++currDepth, dir, ++dirNum, progress); if (!create) { - VERBOSE(2,5,"Remove directory \"%s\"", dir); + VERBOSE(2,5,"Remove directory '%s'", dir); if (-1 == backend->rmdir(dir, ¶m)) { FAIL("Unable to remove directory"); } @@ -1553,7 +1553,7 @@ void create_remove_directory_tree(int create, strcat(temp_path, dir); if (create) { - VERBOSE(2,5,"Making directory \"%s\"", temp_path); + VERBOSE(2,5,"Making directory '%s'", temp_path); if (-1 == backend->mkdir(temp_path, DIRMODE, ¶m)) { FAIL("Unable to create directory"); } @@ -1564,7 +1564,7 @@ void create_remove_directory_tree(int create, currDepth--; if (!create) { - VERBOSE(2,5,"Remove directory \"%s\"", temp_path); + VERBOSE(2,5,"Remove directory '%s'", temp_path); if (-1 == backend->rmdir(temp_path, ¶m)) { FAIL("Unable to remove directory"); } @@ -1593,7 +1593,7 @@ static void mdtest_iteration(int i, int j, MPI_Group testgroup, mdtest_results_t for (int dir_iter = 0; dir_iter < directory_loops; dir_iter ++){ prep_testdir(j, dir_iter); - VERBOSE(2,5,"main (for j loop): making testdir, \"%s\"", testdir ); + VERBOSE(2,5,"main (for j loop): making testdir, '%s'", testdir ); if ((rank < path_count) && backend->access(testdir, F_OK, ¶m) != 0) { if (backend->mkdir(testdir, DIRMODE, ¶m) != 0) { FAIL("Unable to create test directory"); @@ -1618,7 +1618,7 @@ static void mdtest_iteration(int i, int j, MPI_Group testgroup, mdtest_results_t for (k=0; kitems[9] = num_dirs_in_tree; summary_table->stonewall_last_item[8] = num_dirs_in_tree; VERBOSE(1,-1,"main Tree removal : %14.3f sec, %14.3f ops/sec", (endCreate - startCreate), summary_table->rate[9]); - VERBOSE(2,-1,"main (at end of for j loop): Removing testdir of \"%s\"\n", testdir ); + VERBOSE(2,-1,"main (at end of for j loop): Removing testdir of '%s'\n", testdir ); for (int dir_iter = 0; dir_iter < directory_loops; dir_iter ++){ prep_testdir(j, dir_iter); @@ -1900,7 +1900,7 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * char cmd_buffer[4096]; strncpy(cmd_buffer, argv[0], 4096); for (i = 1; i < argc; i++) { - snprintf(&cmd_buffer[strlen(cmd_buffer)], 4096-strlen(cmd_buffer), " \"%s\"", argv[i]); + snprintf(&cmd_buffer[strlen(cmd_buffer)], 4096-strlen(cmd_buffer), " '%s'", argv[i]); } VERBOSE(0,-1,"-- started at %s --\n", PrintTimestamp()); @@ -2064,7 +2064,7 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * } /* display disk usage */ - VERBOSE(3,-1,"main (before display_freespace): testdirpath is \"%s\"", testdirpath ); + VERBOSE(3,-1,"main (before display_freespace): testdirpath is '%s'", testdirpath ); if (rank == 0) display_freespace(testdirpath); int packedByNode = QueryNodeMapping(testComm); @@ -2077,7 +2077,7 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * VERBOSE(0,5,"Shifting ranks by %d for each phase.", nstride); } - VERBOSE(3,-1,"main (after display_freespace): testdirpath is \"%s\"", testdirpath ); + VERBOSE(3,-1,"main (after display_freespace): testdirpath is '%s'", testdirpath ); if (rank == 0) { if (random_seed > 0) { From 0ffec67d2b35f60cd99cfb8307a2f342414c3bcc Mon Sep 17 00:00:00 2001 From: John Bent Date: Sun, 28 Jul 2019 10:25:42 -0600 Subject: [PATCH 123/206] Following Julian's suggestion about better naming --- src/ior.c | 18 +++++++----------- src/ior.h | 2 +- src/mdtest.c | 6 ++++-- 3 files changed, 12 insertions(+), 14 deletions(-) diff --git a/src/ior.c b/src/ior.c index bb2b4c27..64530fdd 100755 --- a/src/ior.c +++ b/src/ior.c @@ -939,7 +939,7 @@ static void InitTests(IOR_test_t *tests, MPI_Comm com) params->testComm = com; params->nodes = params->numTasks / tasksPerNode; params->tasksPerNode = tasksPerNode; - params->packedTasks = QueryNodeMapping(com); + params->tasksBlockMapping = QueryNodeMapping(com); if (params->numTasks == 0) { params->numTasks = size; } @@ -1361,11 +1361,9 @@ static void TestIoSys(IOR_test_t *test) } if (params->reorderTasks) { /* move two nodes away from writing node */ - int shift = 1; - if (params->packedTasks) { - shift = params->tasksPerNode; - } else { - shift = 1; + int shift = 1; /* assume a by-node (round-robin) mapping of tasks to nodes */ + if (params->tasksBlockMapping) { + shift = params->tasksPerNode; /* switch to by-slot (contiguous block) mapping */ } rankOffset = (2 * shift) % params->numTasks; } @@ -1403,11 +1401,9 @@ static void TestIoSys(IOR_test_t *test) /* Constant process offset reading */ if (params->reorderTasks) { /* move one node away from writing node */ - int shift = 1; - if (params->packedTasks) { - shift=params->tasksPerNode; - } else { - shift=1; + int shift = 1; /* assume a by-node (round-robin) mapping of tasks to nodes */ + if (params->tasksBlockMapping) { + shift=params->tasksPerNode; /* switch to a by-slot (contiguous block) mapping */ } rankOffset = (params->taskPerNodeOffset * shift) % params->numTasks; } diff --git a/src/ior.h b/src/ior.h index 2ea6c051..e245b083 100755 --- a/src/ior.h +++ b/src/ior.h @@ -100,7 +100,7 @@ typedef struct int numTasks; /* number of tasks for test */ int nodes; /* number of nodes for test */ int tasksPerNode; /* number of tasks per node */ - int packedTasks; /* are the tasks round-robin across nodes or are they packed tightly*/ + int tasksBlockMapping; /* are the tasks in contiguous blocks across nodes or round-robin */ int repetitions; /* number of repetitions of test */ int repCounter; /* rep counter */ int multiFile; /* multiple files */ diff --git a/src/mdtest.c b/src/mdtest.c index e35a20e2..0005c0a0 100644 --- a/src/mdtest.c +++ b/src/mdtest.c @@ -2067,11 +2067,13 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * VERBOSE(3,-1,"main (before display_freespace): testdirpath is '%s'", testdirpath ); if (rank == 0) display_freespace(testdirpath); - int packedByNode = QueryNodeMapping(testComm); + int tasksBlockMapping = QueryNodeMapping(testComm); /* set the shift to mimic IOR and shift by procs per node */ if (nstride > 0) { - if ( nodeCount > 1 && packedByNode ) { + if ( nodeCount > 1 && tasksBlockMapping ) { + /* the user set the stride presumably to get the consumer tasks on a different node than the producer tasks + however, if the mpirun scheduler placed the tasks by-slot (in a contiguous block) then we need to adjust the shift by ppn */ nstride *= tasksPerNode; } VERBOSE(0,5,"Shifting ranks by %d for each phase.", nstride); From a3c37808dadebc12c21ed4bad1ac55de5d25fa31 Mon Sep 17 00:00:00 2001 From: John Bent Date: Sun, 28 Jul 2019 11:17:11 -0600 Subject: [PATCH 124/206] Made FAIL take variable args so we can pass printf like args to it --- src/mdtest.c | 40 +++++++++++++++++++--------------------- src/utilities.c | 12 ++++++++++++ src/utilities.h | 19 +++++-------------- 3 files changed, 36 insertions(+), 35 deletions(-) diff --git a/src/mdtest.c b/src/mdtest.c index 0005c0a0..127aa5f9 100644 --- a/src/mdtest.c +++ b/src/mdtest.c @@ -288,11 +288,11 @@ static void create_remove_dirs (const char *path, bool create, uint64_t itemNum) if (create) { if (backend->mkdir(curr_item, DIRMODE, ¶m) == -1) { - FAIL("unable to create directory"); + FAIL("unable to create directory %s", curr_item); } } else { if (backend->rmdir(curr_item, ¶m) == -1) { - FAIL("unable to remove directory"); + FAIL("unable to remove directory %s", curr_item); } } } @@ -334,7 +334,7 @@ static void create_file (const char *path, uint64_t itemNum) { else aiori_fh = backend->open (curr_item, ¶m); if (NULL == aiori_fh) { - FAIL("unable to open file"); + FAIL("unable to open file %s", curr_item); } /* @@ -351,7 +351,7 @@ static void create_file (const char *path, uint64_t itemNum) { else aiori_fh = backend->create (curr_item, ¶m); if (NULL == aiori_fh) { - FAIL("unable to create file"); + FAIL("unable to create file %s", curr_item); } } @@ -365,7 +365,7 @@ static void create_file (const char *path, uint64_t itemNum) { param.offset = 0; param.fsyncPerWrite = sync_file; if ( write_bytes != (size_t) backend->xfer (WRITE, aiori_fh, (IOR_size_t *) write_buffer, write_bytes, ¶m)) { - FAIL("unable to write file"); + FAIL("unable to write file %s", curr_item); } } @@ -422,7 +422,7 @@ void collective_helper(const int dirs, const int create, const char* path, uint6 param.openFlags = IOR_WRONLY | IOR_CREAT; aiori_fh = backend->create (curr_item, ¶m); if (NULL == aiori_fh) { - FAIL("unable to create file"); + FAIL("unable to create file %s", curr_item); } backend->close (aiori_fh, ¶m); @@ -586,9 +586,7 @@ void mdtest_stat(const int random, const int dirs, const long dir_iter, const ch /* below temp used to be hiername */ VERBOSE(3,5,"mdtest_stat %4s: %s", (dirs ? "dir" : "file"), item); if (-1 == backend->stat (item, &buf, ¶m)) { - char msg_buf[4096]; - snprintf(msg_buf, 4096, "unable to stat %s %s", dirs ? "directory" : "file", item); - FAIL(msg_buf); + FAIL("unable to stat %s %s", dirs ? "directory" : "file", item); } } } @@ -679,13 +677,13 @@ void mdtest_read(int random, int dirs, const long dir_iter, char *path) { param.openFlags = O_RDONLY; aiori_fh = backend->open (item, ¶m); if (NULL == aiori_fh) { - FAIL("unable to open file"); + FAIL("unable to open file %s", item); } /* read file */ if (read_bytes > 0) { if (read_bytes != (size_t) backend->xfer (READ, aiori_fh, (IOR_size_t *) read_buffer, read_bytes, ¶m)) { - FAIL("unable to read file"); + FAIL("unable to read file %s", item); } } @@ -1342,7 +1340,7 @@ void summarize_results(int iterations) { void valid_tests() { if (((stone_wall_timer_seconds > 0) && (branch_factor > 1)) || ! barriers) { - FAIL( "Error, stone wall timer does only work with a branch factor <= 1 and with barriers\n"); + FAIL( "Error, stone wall timer does only work with a branch factor <= 1 (current is %d) and with barriers\n", branch_factor); } if (!create_only && !stat_only && !read_only && !remove_only) { @@ -1446,7 +1444,7 @@ void show_file_system_size(char *file_system) { ret = backend->statfs (file_system, &stat_buf, ¶m); if (0 != ret) { - FAIL("unable to stat file system"); + FAIL("unable to stat file system %s", file_system); } total_file_system_size = stat_buf.f_blocks * stat_buf.f_bsize; @@ -1469,7 +1467,7 @@ void show_file_system_size(char *file_system) { * 100; if (realpath(file_system, real_path) == NULL) { - FAIL("unable to use realpath()"); + FAIL("unable to use realpath() on file system %s", file_system); } @@ -1539,7 +1537,7 @@ void create_remove_directory_tree(int create, if (!create) { VERBOSE(2,5,"Remove directory '%s'", dir); if (-1 == backend->rmdir(dir, ¶m)) { - FAIL("Unable to remove directory"); + FAIL("Unable to remove directory %s", dir); } } } else if (currDepth <= depth) { @@ -1555,7 +1553,7 @@ void create_remove_directory_tree(int create, if (create) { VERBOSE(2,5,"Making directory '%s'", temp_path); if (-1 == backend->mkdir(temp_path, DIRMODE, ¶m)) { - FAIL("Unable to create directory"); + FAIL("Unable to create directory %s", temp_path); } } @@ -1566,7 +1564,7 @@ void create_remove_directory_tree(int create, if (!create) { VERBOSE(2,5,"Remove directory '%s'", temp_path); if (-1 == backend->rmdir(temp_path, ¶m)) { - FAIL("Unable to remove directory"); + FAIL("Unable to remove directory %s", temp_path); } } @@ -1596,7 +1594,7 @@ static void mdtest_iteration(int i, int j, MPI_Group testgroup, mdtest_results_t VERBOSE(2,5,"main (for j loop): making testdir, '%s'", testdir ); if ((rank < path_count) && backend->access(testdir, F_OK, ¶m) != 0) { if (backend->mkdir(testdir, DIRMODE, ¶m) != 0) { - FAIL("Unable to create test directory"); + FAIL("Unable to create test directory %s", testdir); } } } @@ -1772,7 +1770,7 @@ static void mdtest_iteration(int i, int j, MPI_Group testgroup, mdtest_results_t if ((rank < path_count) && backend->access(testdir, F_OK, ¶m) == 0) { //if (( rank == 0 ) && access(testdir, F_OK) == 0) { if (backend->rmdir(testdir, ¶m) == -1) { - FAIL("unable to remove directory"); + FAIL("unable to remove directory %s", testdir); } } } @@ -2049,7 +2047,7 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * if (path_count == 0) { /* special case where no directory path provided with '-d' option */ char *ret = getcwd(testdirpath, MAX_PATHLEN); if (ret == NULL) { - FAIL("Unable to get current working directory"); + FAIL("Unable to get current working directory on %s", testdirpath); } path_count = 1; } else { @@ -2059,7 +2057,7 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * /* if directory does not exist, create it */ if ((rank < path_count) && backend->access(testdirpath, F_OK, ¶m) != 0) { if (backend->mkdir(testdirpath, DIRMODE, ¶m) != 0) { - FAIL("Unable to create test directory path"); + FAIL("Unable to create test directory path %s", testdirpath); } } diff --git a/src/utilities.c b/src/utilities.c index 5a3168c3..8eb44850 100755 --- a/src/utilities.c +++ b/src/utilities.c @@ -20,6 +20,7 @@ # define _GNU_SOURCE /* Needed for O_DIRECT in fcntl */ #endif /* __linux__ */ +#include #include #include #include @@ -75,6 +76,17 @@ void* safeMalloc(uint64_t size){ return d; } +void FailMessage(int rank, const char *location, char *format, ...) { + char msg[4096]; + va_list args; + va_start(args, format); + vsnprintf(msg, 4096, format, args); + va_end(args); + fprintf(out_logfile, "%s: Process %d: FAILED in %s, %s: %s\n", + PrintTimestamp(), rank, location, msg, strerror(errno)); + fflush(out_logfile); + MPI_Abort(testComm, 1); +} size_t NodeMemoryStringToBytes(char *size_str) { diff --git a/src/utilities.h b/src/utilities.h index d811261c..09641691 100755 --- a/src/utilities.h +++ b/src/utilities.h @@ -36,23 +36,14 @@ extern enum OutputFormat_t outputFormat; /* format of the output */ #ifdef __linux__ -#define FAIL(msg) do { \ - fprintf(out_logfile, "%s: Process %d: FAILED in %s, %s: %s\n", \ - PrintTimestamp(), rank, __func__, \ - msg, strerror(errno)); \ - fflush(out_logfile); \ - MPI_Abort(testComm, 1); \ - } while(0) +#define ERROR_LOCATION __func__ #else -#define FAIL(msg) do { \ - fprintf(out_logfile, "%s: Process %d: FAILED at %d, %s: %s\n", \ - PrintTimestamp(), rank, __LINE__, \ - msg, strerror(errno)); \ - fflush(out_logfile); \ - MPI_Abort(testComm, 1); \ - } while(0) +#define ERROR_LOCATION __LINE__ #endif +#define FAIL(...) FailMessage(rank, ERROR_LOCATION, __VA_ARGS__) +void FailMessage(int rank, const char *location, char *format, ...); + void* safeMalloc(uint64_t size); void set_o_direct_flag(int *fd); From 92939e4fbdf33485627b9bae1ddcc6bec355ff60 Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Wed, 31 Jul 2019 17:22:20 +0000 Subject: [PATCH 125/206] update for DAOS API changes Signed-off-by: Mohamad Chaarawi --- src/aiori-DAOS.c | 5 +---- src/aiori-DFS.c | 4 ---- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/src/aiori-DAOS.c b/src/aiori-DAOS.c index cce5f53a..71aff90d 100644 --- a/src/aiori-DAOS.c +++ b/src/aiori-DAOS.c @@ -31,7 +31,6 @@ #include #include -#include #include "ior.h" #include "aiori.h" @@ -326,13 +325,11 @@ DAOS_Fini() static void gen_oid(const char *name, daos_obj_id_t *oid) { - daos_ofeat_t feat = 0; oid->lo = d_hash_murmur64(name, strlen(name), IOR_DAOS_MUR_SEED); oid->hi = 0; - feat = DAOS_OF_DKEY_UINT64; - daos_obj_generate_id(oid, feat, objectClass, 0); + daos_array_generate_id(oid, objectClass, true, 0); } static void * diff --git a/src/aiori-DFS.c b/src/aiori-DFS.c index cd66ee02..c88b80e3 100755 --- a/src/aiori-DFS.c +++ b/src/aiori-DFS.c @@ -490,11 +490,7 @@ DFS_Create(char *testFileName, IOR_param_t *param) rc = dfs_open(dfs, parent, name, mode, fd_oflag, OC_SX, 0, NULL, &obj); DERR(rc, "dfs_open() of %s Failed", name); - - MPI_CHECK(MPI_Barrier(testComm), "barrier error"); } else { - MPI_CHECK(MPI_Barrier(testComm), "barrier error"); - fd_oflag |= O_RDWR; rc = dfs_open(dfs, parent, name, mode, fd_oflag, OC_SX, 0, NULL, &obj); From 3890b71b54f6bc3b8d931173bb6fb27090ddc652 Mon Sep 17 00:00:00 2001 From: John Bent Date: Thu, 1 Aug 2019 09:42:03 +0900 Subject: [PATCH 126/206] Fixed issues and followed suggestions from Glenn's review of the PR --- src/ior.c | 4 ++-- src/mdtest.c | 4 ++-- src/parse_options.c | 2 +- src/utilities.c | 4 ++-- src/utilities.h | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/ior.c b/src/ior.c index 64530fdd..15b1d07c 100755 --- a/src/ior.c +++ b/src/ior.c @@ -939,7 +939,7 @@ static void InitTests(IOR_test_t *tests, MPI_Comm com) params->testComm = com; params->nodes = params->numTasks / tasksPerNode; params->tasksPerNode = tasksPerNode; - params->tasksBlockMapping = QueryNodeMapping(com); + params->tasksBlockMapping = QueryNodeMapping(com,false); if (params->numTasks == 0) { params->numTasks = size; } @@ -1223,7 +1223,7 @@ static void TestIoSys(IOR_test_t *test) } if (rank == 0 && params->reorderTasks == TRUE && verbose >= VERBOSE_1) { fprintf(out_logfile, - "Using reorderTasks '-C' (expecting block, not cyclic, task assignment)\n"); + "Using reorderTasks '-C' (useful to avoid read cache in client)\n"); fflush(out_logfile); } params->tasksPerNode = CountTasksPerNode(testComm); diff --git a/src/mdtest.c b/src/mdtest.c index 127aa5f9..f65ea7f6 100644 --- a/src/mdtest.c +++ b/src/mdtest.c @@ -1862,7 +1862,7 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * {'l', NULL, "last number of tasks on which the test will run", OPTION_OPTIONAL_ARGUMENT, 'd', & last}, {'L', NULL, "files only at leaf level of tree", OPTION_FLAG, 'd', & leaf_only}, {'n', NULL, "every process will creat/stat/read/remove # directories and files", OPTION_OPTIONAL_ARGUMENT, 'l', & items}, - {'N', NULL, "stride # between neighbor tasks for file/dir operation (local=0)", OPTION_OPTIONAL_ARGUMENT, 'd', & nstride}, + {'N', NULL, "stride # between tasks for file/dir operation (local=0; set to 1 to avoid client cache)", OPTION_OPTIONAL_ARGUMENT, 'd', & nstride}, {'p', NULL, "pre-iteration delay (in seconds)", OPTION_OPTIONAL_ARGUMENT, 'd', & pre_delay}, {'R', NULL, "random access to files (only for stat)", OPTION_FLAG, 'd', & randomize}, {0, "random-seed", "random seed for -R", OPTION_OPTIONAL_ARGUMENT, 'd', & random_seed}, @@ -2065,7 +2065,7 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * VERBOSE(3,-1,"main (before display_freespace): testdirpath is '%s'", testdirpath ); if (rank == 0) display_freespace(testdirpath); - int tasksBlockMapping = QueryNodeMapping(testComm); + int tasksBlockMapping = QueryNodeMapping(testComm, true); /* set the shift to mimic IOR and shift by procs per node */ if (nstride > 0) { diff --git a/src/parse_options.c b/src/parse_options.c index ba72413c..af30c36e 100755 --- a/src/parse_options.c +++ b/src/parse_options.c @@ -471,7 +471,7 @@ option_help * createGlobalOptions(IOR_param_t * params){ {'A', NULL, "refNum -- user supplied reference number to include in the summary", OPTION_OPTIONAL_ARGUMENT, 'd', & params->referenceNumber}, {'b', NULL, "blockSize -- contiguous bytes to write per task (e.g.: 8, 4k, 2m, 1g)", OPTION_OPTIONAL_ARGUMENT, 'l', & params->blockSize}, {'c', NULL, "collective -- collective I/O", OPTION_FLAG, 'd', & params->collective}, - {'C', NULL, "reorderTasks -- changes task ordering to n+1 ordering for readback", OPTION_FLAG, 'd', & params->reorderTasks}, + {'C', NULL, "reorderTasks -- changes task ordering for readback (useful to avoid client cache)", OPTION_FLAG, 'd', & params->reorderTasks}, {'d', NULL, "interTestDelay -- delay between reps in seconds", OPTION_OPTIONAL_ARGUMENT, 'd', & params->interTestDelay}, {'D', NULL, "deadlineForStonewalling -- seconds before stopping write or read phase", OPTION_OPTIONAL_ARGUMENT, 'd', & params->deadlineForStonewalling}, {.help=" -O stoneWallingWearOut=1 -- once the stonewalling timout is over, all process finish to access the amount of data", .arg = OPTION_OPTIONAL_ARGUMENT}, diff --git a/src/utilities.c b/src/utilities.c index 8eb44850..99d26e76 100755 --- a/src/utilities.c +++ b/src/utilities.c @@ -229,7 +229,7 @@ void DumpBuffer(void *buffer, and the value is whether that rank is on the same host as root. Also returns 1 if rank 1 is on same host and 0 otherwise */ -int QueryNodeMapping(MPI_Comm comm) { +int QueryNodeMapping(MPI_Comm comm, int print_nodemap) { char localhost[MAX_PATHLEN], roothost[MAX_PATHLEN]; int num_ranks; MPI_Comm_size(comm, &num_ranks); @@ -250,7 +250,7 @@ int QueryNodeMapping(MPI_Comm comm) { /* then every rank figures out whether it is same host as root and then gathers that */ int same_as_root = strcmp(roothost,localhost) == 0; MPI_Gather( &same_as_root, 1, MPI_INT, node_map, 1, MPI_INT, 0, comm); - if (rank==0) { + if ( print_nodemap && rank==0) { fprintf( out_logfile, "Nodemap: " ); for ( int i = 0; i < num_ranks; i++ ) { fprintf( out_logfile, "%d", node_map[i] ); diff --git a/src/utilities.h b/src/utilities.h index 09641691..d2c9962a 100755 --- a/src/utilities.h +++ b/src/utilities.h @@ -56,7 +56,7 @@ void SetHints (MPI_Info *, char *); void ShowHints (MPI_Info *); char *HumanReadable(IOR_offset_t value, int base); int CountTasksPerNode(MPI_Comm comm); -int QueryNodeMapping(MPI_Comm comm); +int QueryNodeMapping(MPI_Comm comm, int print_nodemap); void DelaySecs(int delay); void updateParsedOptions(IOR_param_t * options, options_all_t * global_options); size_t NodeMemoryStringToBytes(char *size_str); From 6c0fadc2a9f82623bbd6016194802e35245daea6 Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Thu, 1 Aug 2019 17:20:01 +0100 Subject: [PATCH 127/206] Include performance when stonewall is hit to output. --- src/ior-output.c | 23 +++++++++++++++++++---- src/ior.c | 6 +----- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/src/ior-output.c b/src/ior-output.c index 4bdb37dc..7049a971 100644 --- a/src/ior-output.c +++ b/src/ior-output.c @@ -18,8 +18,8 @@ static void PrintNextToken(); void PrintTableHeader(){ if (outputFormat == OUTPUT_DEFAULT){ fprintf(out_resultfile, "\n"); - fprintf(out_resultfile, "access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter\n"); - fprintf(out_resultfile, "------ --------- ---------- --------- -------- -------- -------- -------- ----\n"); + fprintf(out_resultfile, "access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter\n"); + fprintf(out_resultfile, "------ --------- ---------- --------- -------- -------- -------- -------- ----\n"); } } @@ -534,10 +534,14 @@ static void PrintLongSummaryOneOperation(IOR_test_t *test, const int access) reps = params->repetitions; double * times = malloc(sizeof(double)* reps); + long long stonewall_avg_data_accessed = 0; + double stonewall_time = 0; for(int i=0; i < reps; i++){ IOR_point_t *point = (access == WRITE) ? &results[i].write : &results[i].read; times[i] = point->time; + stonewall_time += point->stonewall_time; + stonewall_avg_data_accessed += point->stonewall_avg_data_accessed; } bw = bw_values(reps, results, times, access); @@ -558,6 +562,13 @@ static void PrintLongSummaryOneOperation(IOR_test_t *test, const int access) fprintf(out_resultfile, "%10.2f ", ops->mean); fprintf(out_resultfile, "%10.2f ", ops->sd); fprintf(out_resultfile, "%10.5f ", mean_of_array_of_doubles(times, reps)); + if(test->params.stoneWallingWearOut){ + fprintf(out_resultfile, "%10.2f ", stonewall_time / reps); + fprintf(out_resultfile, "%13.2f ", stonewall_avg_data_accessed / stonewall_time / MEBIBYTE); + }else{ + fprintf(out_resultfile, "%10s ", "NA"); + fprintf(out_resultfile, "%13s ", "NA"); + } fprintf(out_resultfile, "%5d ", params->id); fprintf(out_resultfile, "%6d ", params->numTasks); fprintf(out_resultfile, "%3d ", params->tasksPerNode); @@ -603,6 +614,10 @@ static void PrintLongSummaryOneOperation(IOR_test_t *test, const int access) PrintKeyValDouble("OPsMean", ops->mean); PrintKeyValDouble("OPsSD", ops->sd); PrintKeyValDouble("MeanTime", mean_of_array_of_doubles(times, reps)); + if(test->params.stoneWallingWearOut){ + PrintKeyValDouble("StoneWallTime", stonewall_time / reps); + PrintKeyValDouble("StoneWallbwMeanMIB", stonewall_avg_data_accessed / stonewall_time / MEBIBYTE); + } PrintKeyValDouble("xsizeMiB", (double) point->aggFileSizeForBW / MEBIBYTE); PrintEndSection(); }else if (outputFormat == OUTPUT_CSV){ @@ -635,10 +650,10 @@ void PrintLongSummaryHeader() } fprintf(out_resultfile, "\n"); - fprintf(out_resultfile, "%-9s %10s %10s %10s %10s %10s %10s %10s %10s %10s", + fprintf(out_resultfile, "%-9s %10s %10s %10s %10s %10s %10s %10s %10s %10s %10s %13s", "Operation", "Max(MiB)", "Min(MiB)", "Mean(MiB)", "StdDev", "Max(OPs)", "Min(OPs)", "Mean(OPs)", "StdDev", - "Mean(s)"); + "Mean(s)", "Stonewall(s)", "Stonewall(MiB)"); fprintf(out_resultfile, " Test# #Tasks tPN reps fPP reord reordoff reordrand seed" " segcnt "); fprintf(out_resultfile, "%8s %8s %9s %5s", " blksiz", "xsize","aggs(MiB)", "API"); diff --git a/src/ior.c b/src/ior.c index 15b1d07c..4cd55718 100755 --- a/src/ior.c +++ b/src/ior.c @@ -1400,7 +1400,7 @@ static void TestIoSys(IOR_test_t *test) /* Get rankOffset [file offset] for this process to read, based on -C,-Z,-Q,-X options */ /* Constant process offset reading */ if (params->reorderTasks) { - /* move one node away from writing node */ + /* move one node away from writing node */ int shift = 1; /* assume a by-node (round-robin) mapping of tasks to nodes */ if (params->tasksBlockMapping) { shift=params->tasksPerNode; /* switch to a by-slot (contiguous block) mapping */ @@ -1944,10 +1944,6 @@ static IOR_offset_t WriteOrRead(IOR_param_t *test, IOR_results_t *results, point->stonewall_min_data_accessed /1024.0 / 1024 / 1024, point->stonewall_avg_data_accessed / 1024.0 / 1024 / 1024 / test->numTasks , point->stonewall_time); point->stonewall_min_data_accessed *= test->numTasks; } - if(pairs_accessed_min == pairCnt){ - point->stonewall_min_data_accessed = 0; - point->stonewall_avg_data_accessed = 0; - } if(pairCnt != point->pairs_accessed){ // some work needs still to be done ! for(; pairCnt < point->pairs_accessed; pairCnt++ ) { From 061b5a860f6ac010ec6cabab54d83ef2a1cad0d7 Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Thu, 1 Aug 2019 17:54:11 +0100 Subject: [PATCH 128/206] Backmerged: New option: print rate AND time; improves debugging. --- src/mdtest.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/mdtest.c b/src/mdtest.c index f65ea7f6..d2cd0681 100644 --- a/src/mdtest.c +++ b/src/mdtest.c @@ -131,6 +131,7 @@ static uint64_t items_per_dir; static uint64_t num_dirs_in_tree_calc; /* this is a workaround until the overal code is refactored */ static int directory_loops; static int print_time; +static int print_rate_and_time; static int random_seed; static int shared_file; static int files_only; @@ -1197,7 +1198,7 @@ void file_test(const int iteration, const int ntasks, const char *path, rank_pro VERBOSE(1,-1," File removal : %14.3f sec, %14.3f ops/sec", t[4] - t[3], summary_table[iteration].rate[7]); } -void summarize_results(int iterations) { +void summarize_results(int iterations, int print_time) { char access[MAX_PATHLEN]; int i, j, k; int start, stop, tableSize = MDTEST_LAST_NUM; @@ -1794,6 +1795,7 @@ void mdtest_init_args(){ items_per_dir = 0; random_seed = 0; print_time = 0; + print_rate_and_time = 0; shared_file = 0; files_only = 0; dirs_only = 0; @@ -1864,6 +1866,7 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * {'n', NULL, "every process will creat/stat/read/remove # directories and files", OPTION_OPTIONAL_ARGUMENT, 'l', & items}, {'N', NULL, "stride # between tasks for file/dir operation (local=0; set to 1 to avoid client cache)", OPTION_OPTIONAL_ARGUMENT, 'd', & nstride}, {'p', NULL, "pre-iteration delay (in seconds)", OPTION_OPTIONAL_ARGUMENT, 'd', & pre_delay}, + {'P', NULL, "print rate AND time", OPTION_FLAG, 'd', & print_rate_and_time}, {'R', NULL, "random access to files (only for stat)", OPTION_FLAG, 'd', & randomize}, {0, "random-seed", "random seed for -R", OPTION_OPTIONAL_ARGUMENT, 'd', & random_seed}, {'s', NULL, "stride between the number of tasks for each test", OPTION_OPTIONAL_ARGUMENT, 'd', & stride}, @@ -2154,7 +2157,12 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * // keep track of the current status for stonewalling mdtest_iteration(i, j, testgroup, & summary_table[j]); } - summarize_results(iterations); + if (print_rate_and_time){ + summarize_results(iterations, 0); + summarize_results(iterations, 1); + }else{ + summarize_results(iterations, print_time); + } if (i == 1 && stride > 1) { i = 0; } From df8355a9bc755c9c83384fa14d7c128ce6b347b3 Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Thu, 1 Aug 2019 17:57:45 +0100 Subject: [PATCH 129/206] Added output of mdtest stonewall timer. --- src/mdtest.c | 232 +++++++++++++++++++++++++++------------------------ 1 file changed, 123 insertions(+), 109 deletions(-) diff --git a/src/mdtest.c b/src/mdtest.c index d2cd0681..1ff204d3 100644 --- a/src/mdtest.c +++ b/src/mdtest.c @@ -968,10 +968,10 @@ int updateStoneWallIterations(int iteration, rank_progress_t * progress, double MPI_Reduce(& progress->items_done, & min_accessed, 1, MPI_LONG_LONG_INT, MPI_MIN, 0, testComm); long long sum_accessed = 0; MPI_Reduce(& progress->items_done, & sum_accessed, 1, MPI_LONG_LONG_INT, MPI_SUM, 0, testComm); + summary_table[iteration].stonewall_item_sum[MDTEST_FILE_CREATE_NUM] = sum_accessed; + summary_table[iteration].stonewall_item_min[MDTEST_FILE_CREATE_NUM] = min_accessed * size; if(items != (sum_accessed / size)){ - summary_table[iteration].stonewall_item_sum[MDTEST_FILE_CREATE_NUM] = sum_accessed; - summary_table[iteration].stonewall_item_min[MDTEST_FILE_CREATE_NUM] = min_accessed * size; VERBOSE(0,-1, "Continue stonewall hit min: %lld max: %lld avg: %.1f \n", min_accessed, max_iter, ((double) sum_accessed) / size); hit = 1; } @@ -1193,6 +1193,9 @@ void file_test(const int iteration, const int ntasks, const char *path, rank_pro } VERBOSE(1,-1," File creation : %14.3f sec, %14.3f ops/sec", t[1] - t[0], summary_table[iteration].rate[4]); + if(summary_table[iteration].stonewall_time[MDTEST_FILE_CREATE_NUM]){ + VERBOSE(1,-1," File creation (stonewall): %14.3f sec, %14.3f ops/sec", summary_table[iteration].stonewall_time[MDTEST_FILE_CREATE_NUM], summary_table[iteration].stonewall_item_sum[MDTEST_FILE_CREATE_NUM]); + } VERBOSE(1,-1," File stat : %14.3f sec, %14.3f ops/sec", t[2] - t[1], summary_table[iteration].rate[5]); VERBOSE(1,-1," File read : %14.3f sec, %14.3f ops/sec", t[3] - t[2], summary_table[iteration].rate[6]); VERBOSE(1,-1," File removal : %14.3f sec, %14.3f ops/sec", t[4] - t[3], summary_table[iteration].rate[7]); @@ -1211,129 +1214,141 @@ void summarize_results(int iterations, int print_time) { MPI_Barrier(testComm); for(int i=0; i < iterations; i++){ - if(print_time){ - MPI_Gather(& summary_table[i].time[0], tableSize, MPI_DOUBLE, & all[i*tableSize*size], tableSize, MPI_DOUBLE, 0, testComm); - }else{ - MPI_Gather(& summary_table[i].rate[0], tableSize, MPI_DOUBLE, & all[i*tableSize*size], tableSize, MPI_DOUBLE, 0, testComm); - } + MPI_Gather(& summary_table[i].time[0], tableSize, MPI_DOUBLE, & all[i*tableSize*size], tableSize, MPI_DOUBLE, 0, testComm); + MPI_Gather(& summary_table[i].rate[0], tableSize, MPI_DOUBLE, & all[i*tableSize*size], tableSize, MPI_DOUBLE, 0, testComm); } - if (rank == 0) { + if (rank != 0) { + return; + } - VERBOSE(0,-1,"\nSUMMARY %s: (of %d iterations)", print_time ? "time": "rate", iterations); - VERBOSE(0,-1," Operation Max Min Mean Std Dev"); - VERBOSE(0,-1," --------- --- --- ---- -------"); + VERBOSE(0,-1,"\nSUMMARY %s: (of %d iterations)", print_time ? "time": "rate", iterations); + VERBOSE(0,-1," Operation Max Min Mean Std Dev"); + VERBOSE(0,-1," --------- --- --- ---- -------"); - /* if files only access, skip entries 0-3 (the dir tests) */ - if (files_only && !dirs_only) { - start = 4; - } else { - start = 0; - } + /* if files only access, skip entries 0-3 (the dir tests) */ + if (files_only && !dirs_only) { + start = 4; + } else { + start = 0; + } - /* if directories only access, skip entries 4-7 (the file tests) */ - if (dirs_only && !files_only) { - stop = 4; - } else { - stop = 8; - } - - /* special case: if no directory or file tests, skip all */ - if (!dirs_only && !files_only) { - start = stop = 0; - } - - for (i = start; i < stop; i++) { - min = max = all[i]; - for (k=0; k < size; k++) { - for (j = 0; j < iterations; j++) { - curr = all[(k*tableSize*iterations) - + (j*tableSize) + i]; - if (min > curr) { - min = curr; - } - if (max < curr) { - max = curr; - } - sum += curr; - } - } - mean = sum / (iterations * size); - for (k=0; k curr) { - min = curr; - } - if (max < curr) { - max = curr; + for (k=0; k < size; k++) { + for (j = 0; j < iterations; j++) { + curr = all[(k*tableSize*iterations) + + (j*tableSize) + i]; + if (min > curr) { + min = curr; + } + if (max < curr) { + max = curr; + } + sum += curr; } - sum += curr; } - mean = sum / (iterations); - for (j = 0; j < iterations; j++) { - if(print_time){ - curr = summary_table[j].time[i]; - }else{ - curr = summary_table[j].rate[i]; + mean = sum / (iterations * size); + for (k=0; k curr) { + min = curr; + } + if (max < curr) { + max = curr; + } + sum += curr; } + mean = sum / (iterations); + for (j = 0; j < iterations; j++) { + if(print_time){ + curr = summary_table[j].time[i]; + }else{ + curr = summary_table[j].rate[i]; + } + + var += pow((mean - curr), 2); + } + var = var / (iterations); + sd = sqrt(var); + switch (i) { + case 8: strcpy(access, "Tree creation :"); break; + case 9: strcpy(access, "Tree removal :"); break; + default: strcpy(access, "ERR"); break; + } + fprintf(out_logfile, " %s ", access); + fprintf(out_logfile, "%14.3f ", max); + fprintf(out_logfile, "%14.3f ", min); + fprintf(out_logfile, "%14.3f ", mean); + fprintf(out_logfile, "%14.3f\n", sd); + fflush(out_logfile); + sum = var = 0; } } @@ -1474,7 +1489,7 @@ void show_file_system_size(char *file_system) { /* show results */ VERBOSE(0,-1,"Path: %s", real_path); - VERBOSE(0,-1,"FS: %.1f %s Used FS: %2.1f%% Inodes: %.1f %s Used Inodes: %2.1f%%\n", + VERBOSE(0,-1,"FS: %.1f %s Used FS: %2.1f%% Inodes: %.1f %s Used Inodes: %2.1f%%\n", total_file_system_size_hr, file_system_unit_str, used_file_system_percentage, (double)total_inodes / (double)inode_unit_val, inode_unit_str, used_inode_percentage); @@ -1881,7 +1896,6 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * {'x', NULL, "StoneWallingStatusFile; contains the number of iterations of the creation phase, can be used to split phases across runs", OPTION_OPTIONAL_ARGUMENT, 's', & stoneWallingStatusFile}, {'y', NULL, "sync file after writing", OPTION_FLAG, 'd', & sync_file}, {'z', NULL, "depth of hierarchical directory structure", OPTION_OPTIONAL_ARGUMENT, 'd', & depth}, - {'Z', NULL, "print time instead of rate", OPTION_FLAG, 'd', & print_time}, LAST_OPTION }; options_all_t * global_options = airoi_create_all_module_options(options); From ce1ae750f6bbe23c915c1c9f034834fe382b51c1 Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Thu, 1 Aug 2019 18:29:32 +0100 Subject: [PATCH 130/206] MDtest: Support to verify the read operation with a default pattern. --- src/mdtest.c | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/src/mdtest.c b/src/mdtest.c index 1ff204d3..d68c86ba 100644 --- a/src/mdtest.c +++ b/src/mdtest.c @@ -103,6 +103,7 @@ static char unique_rm_dir[MAX_PATHLEN]; static char unique_rm_uni_dir[MAX_PATHLEN]; static char *write_buffer; static char *read_buffer; +static char *verify_read_buffer; static char *stoneWallingStatusFile; @@ -110,6 +111,8 @@ static int barriers; static int create_only; static int stat_only; static int read_only; +static int verify_read; +static int verification_error; static int remove_only; static int leaf_only; static unsigned branch_factor; @@ -200,6 +203,12 @@ void VerboseMessage (int root_level, int any_level, int line, char * format, ... } } +void generate_memory_pattern(char * buffer, size_t bytes){ + for(int i=0; i < bytes; i++){ + buffer[i] = i + 1; + } +} + void offset_timers(double * t, int tcount) { double toffset; int i; @@ -607,6 +616,14 @@ void mdtest_read(int random, int dirs, const long dir_iter, char *path) { if (read_buffer == NULL) { FAIL("out of memory"); } + + if (verify_read > 0) { + verify_read_buffer = (char *)malloc(read_bytes); + if (verify_read_buffer == NULL) { + FAIL("out of memory"); + } + generate_memory_pattern(verify_read_buffer, read_bytes); + } } uint64_t stop_items = items; @@ -686,6 +703,12 @@ void mdtest_read(int random, int dirs, const long dir_iter, char *path) { if (read_bytes != (size_t) backend->xfer (READ, aiori_fh, (IOR_size_t *) read_buffer, read_bytes, ¶m)) { FAIL("unable to read file %s", item); } + if(verify_read){ + if (memcmp(read_buffer, verify_read_buffer, read_bytes) != 0){ + VERBOSE(2, -1, "Error verifying %s", item); + verification_error++; + } + } } /* close file */ @@ -1803,6 +1826,8 @@ void mdtest_init_args(){ create_only = 0; stat_only = 0; read_only = 0; + verify_read = 0; + verification_error = 0; remove_only = 0; leaf_only = 0; depth = 0; @@ -1894,6 +1919,7 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * {'w', NULL, "bytes to write to each file after it is created", OPTION_OPTIONAL_ARGUMENT, 'l', & write_bytes}, {'W', NULL, "number in seconds; stonewall timer, write as many seconds and ensure all processes did the same number of operations (currently only stops during create phase)", OPTION_OPTIONAL_ARGUMENT, 'd', & stone_wall_timer_seconds}, {'x', NULL, "StoneWallingStatusFile; contains the number of iterations of the creation phase, can be used to split phases across runs", OPTION_OPTIONAL_ARGUMENT, 's', & stoneWallingStatusFile}, + {'X', "verify-read", "Verify the data read", OPTION_FLAG, 'd', & verify_read}, {'y', NULL, "sync file after writing", OPTION_FLAG, 'd', & sync_file}, {'z', NULL, "depth of hierarchical directory structure", OPTION_OPTIONAL_ARGUMENT, 'd', & depth}, LAST_OPTION @@ -2057,7 +2083,7 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * if (write_buffer == NULL) { FAIL("out of memory"); } - memset(write_buffer, 0x23, write_bytes); + generate_memory_pattern(write_buffer, write_bytes); } /* setup directory path to work in */ @@ -2182,6 +2208,9 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * } } + if(verification_error){ + VERBOSE(0, -1, "\nERROR: verifying the data read! Take the performance values with care!\n"); + } VERBOSE(0,-1,"-- finished at %s --\n", PrintTimestamp()); if (random_seed > 0) { From cf56715a5a2ab12f8f4b0bfed7700900e9e3f770 Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Thu, 1 Aug 2019 18:33:44 +0100 Subject: [PATCH 131/206] Make sure that each read buffer contains an invalid first byte. --- src/mdtest.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/mdtest.c b/src/mdtest.c index d68c86ba..00eb57c5 100644 --- a/src/mdtest.c +++ b/src/mdtest.c @@ -700,6 +700,7 @@ void mdtest_read(int random, int dirs, const long dir_iter, char *path) { /* read file */ if (read_bytes > 0) { + read_buffer[0] = 42; if (read_bytes != (size_t) backend->xfer (READ, aiori_fh, (IOR_size_t *) read_buffer, read_bytes, ¶m)) { FAIL("unable to read file %s", item); } From b1b66962acc1f1ecfdcd9e89965c5a3a24516f84 Mon Sep 17 00:00:00 2001 From: Osamu Tatebe Date: Fri, 2 Aug 2019 13:03:59 +0900 Subject: [PATCH 132/206] incorrect warning --- src/ior.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ior.c b/src/ior.c index 3ef411cb..1ebcda8b 100755 --- a/src/ior.c +++ b/src/ior.c @@ -1580,6 +1580,7 @@ static void ValidateTests(IOR_param_t * test) && (strcasecmp(test->api, "MPIIO") != 0) && (strcasecmp(test->api, "MMAP") != 0) && (strcasecmp(test->api, "HDFS") != 0) + && (strcasecmp(test->api, "Gfarm") != 0) && (strcasecmp(test->api, "RADOS") != 0)) && test->fsync) WARN_RESET("fsync() not supported in selected backend", test, &defaults, fsync); From f89f33873426fe163661318c1a001e5240590bdf Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 2 Aug 2019 19:30:29 -0500 Subject: [PATCH 133/206] Travis CI: Fix HDF5 Build Fix the CI. --- .travis.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 29e091c6..7d8202d6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,7 @@ install: # Install MPI - sudo apt-get install -y openmpi-bin libopenmpi-dev # Install HDF5 - - sudo apt-get install -y hdf5-tools libhdf5-mpi-dev + - sudo apt-get install -y hdf5-tools libhdf5-openmpi-dev # Install Lustre # TODO: As far as a i can see it needs to be compiled form source with kernel # module as there is no package. @@ -22,10 +22,13 @@ install: # Install HDFS # TODO: Not sure with which c libray hdfs should be used and if it is in # the ubuntu repos - # Probably hadoop needs to be installed an provides nativ api. + # Probably hadoop needs to be installed an provides native API. # Install Amazon S3 # TODO: The needed library needs to be installed. Follow the instructions in # aiori-S3.c to achive this. # GPFS # NOTE: Think GPFS need a license and is therefore not testable with travis. -script: ./travis-build.sh && CONFIGURE_OPTS="--with-hdf5" ./travis-test.sh +script: + - ./travis-build.sh + - export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/hdf5/openmpi:${LD_LIBRARY_PATH} + - CONFIGURE_OPTS="--with-hdf5" CFLAGS="-I/usr/include/hdf5/openmpi -L/usr/lib/x86_64-linux-gnu/hdf5/openmpi" ./travis-test.sh From bfff0df8fd99dc53068dc28ab3523bfbf86f95c9 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 2 Aug 2019 19:23:05 -0500 Subject: [PATCH 134/206] Fix Some Memory Leaks Fixing some memory leaks :) --- src/aiori-POSIX.c | 1 + src/mdtest.c | 3 +++ src/test/lib.c | 5 +++++ 3 files changed, 9 insertions(+) diff --git a/src/aiori-POSIX.c b/src/aiori-POSIX.c index d78cd40a..0eb663b4 100755 --- a/src/aiori-POSIX.c +++ b/src/aiori-POSIX.c @@ -89,6 +89,7 @@ option_help * POSIX_options(void ** init_backend_options, void * init_values){ } *init_backend_options = o; + free(o); option_help h [] = { {0, "posix.odirect", "Direct I/O Mode", OPTION_FLAG, 'd', & o->direct_io}, diff --git a/src/mdtest.c b/src/mdtest.c index 1ff204d3..e187a778 100644 --- a/src/mdtest.c +++ b/src/mdtest.c @@ -1901,6 +1901,9 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * options_all_t * global_options = airoi_create_all_module_options(options); option_parse(argc, argv, global_options); updateParsedOptions(& param, global_options); + + free(global_options->modules); + free(global_options); backend = param.backend; MPI_Comm_rank(testComm, &rank); diff --git a/src/test/lib.c b/src/test/lib.c index 55ca6f00..c163f54b 100644 --- a/src/test/lib.c +++ b/src/test/lib.c @@ -16,6 +16,11 @@ int main(int argc, char ** argv){ fprintf(stderr, "Could not run ior\n"); ret = 1; } + else + { + free(res->params.platform); + free(res); + } } if (rank == 0){ char * param[] = {"./mdtest", "-a", "DUMMY"}; From b72b51be48f52d2deff1e8ad3d111a53d095a90e Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 2 Aug 2019 16:43:55 -0500 Subject: [PATCH 135/206] Fix: Heap Buffer Overflow Fix a memory violation when run in serial. --- src/utilities.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/utilities.c b/src/utilities.c index 99d26e76..cdb090ea 100755 --- a/src/utilities.c +++ b/src/utilities.c @@ -257,7 +257,9 @@ int QueryNodeMapping(MPI_Comm comm, int print_nodemap) { } fprintf( out_logfile, "\n" ); } - int ret = node_map[1] == 1; + int ret = 1; + if(num_ranks>1) + ret = node_map[1] == 1; MPI_Bcast(&ret, 1, MPI_INT, 0, comm); free(node_map); return ret; From f16ef9ace517a471bd59fcfddecaf9d80bde16a1 Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Sat, 3 Aug 2019 05:07:13 +0000 Subject: [PATCH 136/206] update dfs_lookup() call for extra parameter. Signed-off-by: Mohamad Chaarawi --- src/aiori-DFS.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/aiori-DFS.c b/src/aiori-DFS.c index c88b80e3..35870e1f 100755 --- a/src/aiori-DFS.c +++ b/src/aiori-DFS.c @@ -332,7 +332,6 @@ lookup_insert_dir(const char *name) { struct aiori_dir_hdl *hdl; d_list_t *rlink; - mode_t mode; int rc; rlink = d_hash_rec_find(dir_hash, name, strlen(name)); @@ -348,7 +347,7 @@ lookup_insert_dir(const char *name) strncpy(hdl->name, name, PATH_MAX-1); hdl->name[PATH_MAX-1] = '\0'; - rc = dfs_lookup(dfs, name, O_RDWR, &hdl->oh, &mode); + rc = dfs_lookup(dfs, name, O_RDWR, &hdl->oh, NULL, NULL); DERR(rc, "dfs_lookup() of %s Failed", name); rc = d_hash_rec_insert(dir_hash, hdl->name, strlen(hdl->name), @@ -667,7 +666,7 @@ DFS_GetFileSize(IOR_param_t * test, MPI_Comm comm, char *testFileName) daos_size_t fsize, tmpMin, tmpMax, tmpSum; int rc; - rc = dfs_lookup(dfs, testFileName, O_RDONLY, &obj, NULL); + rc = dfs_lookup(dfs, testFileName, O_RDONLY, &obj, NULL, NULL); if (rc) { fprintf(stderr, "dfs_lookup() of %s Failed (%d)", testFileName, rc); return -1; From c4ff3d7c4e35edc4022e81e779d09f38b00f1954 Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Sat, 3 Aug 2019 09:12:48 +0100 Subject: [PATCH 137/206] Trivial fix for #168 --- src/mdtest.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mdtest.c b/src/mdtest.c index 1ff204d3..24f3ec11 100644 --- a/src/mdtest.c +++ b/src/mdtest.c @@ -79,7 +79,7 @@ #define FILEMODE S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH #define DIRMODE S_IRUSR|S_IWUSR|S_IXUSR|S_IRGRP|S_IWGRP|S_IXGRP|S_IROTH|S_IXOTH #define RELEASE_VERS META_VERSION -#define TEST_DIR "#test-dir" +#define TEST_DIR "test-dir" #define ITEM_COUNT 25000 #define LLU "%lu" From 361a3261d1f8be2621bce8be8ed41ee9f6d0bf1f Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Sat, 3 Aug 2019 09:15:34 +0100 Subject: [PATCH 138/206] Updated test patterns --- testing/mdtest-patterns/advanced/3.txt | 188 +++++++++++++------------ testing/mdtest-patterns/advanced/4.txt | 102 +++++++------- testing/mdtest-patterns/advanced/5.txt | 154 ++++++++++---------- testing/mdtest-patterns/basic/0.txt | 51 +++---- testing/mdtest-patterns/basic/1.txt | 51 +++---- testing/mdtest-patterns/basic/2.txt | 54 +++---- testing/mdtest-patterns/basic/3.txt | 65 +++++---- 7 files changed, 341 insertions(+), 324 deletions(-) diff --git a/testing/mdtest-patterns/advanced/3.txt b/testing/mdtest-patterns/advanced/3.txt index a51000ac..4c459416 100644 --- a/testing/mdtest-patterns/advanced/3.txt +++ b/testing/mdtest-patterns/advanced/3.txt @@ -1,93 +1,95 @@ -V-3: main (before display_freespace): testdirpath is "/dev/shm/mdest" -V-3: testdirpath is "/dev/shm/mdest" -V-3: Before show_file_system_size, dirpath is "/dev/shm" -V-3: After show_file_system_size, dirpath is "/dev/shm" -V-3: main (after display_freespace): testdirpath is "/dev/shm/mdest" -V-3: main (create hierarchical directory loop-!unque_dir_per_task): Calling create_remove_directory_tree with "/dev/shm/mdest/#test-dir.0-0" -V-3: main: Using unique_mk_dir, "mdtest_tree.0" -V-3: main: Copied unique_mk_dir, "mdtest_tree.0", to topdir -V-3: directory_test: create path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" -V-3: create_remove_items (start): temp_path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" -V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.0" -V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.1" -V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.2" -V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.3" -V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.4" -V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.5" -V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.6" -V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.7" -V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.8" -V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.9" -V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.10" -V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.11" -V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.12" -V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.13" -V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.14" -V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.15" -V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.16" -V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.17" -V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.18" -V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.19" -V-3: file_test: create path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" -V-3: create_remove_items (start): temp_path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" -V-3: create_remove_items_helper (non-dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.0" -V-3: create_remove_items_helper (non-collective, shared): open... -V-3: create_remove_items_helper: close... -V-3: create_remove_items_helper (non-dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.1" -V-3: create_remove_items_helper (non-collective, shared): open... -V-3: create_remove_items_helper: close... -V-3: create_remove_items_helper (non-dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.2" -V-3: create_remove_items_helper (non-collective, shared): open... -V-3: create_remove_items_helper: close... -V-3: create_remove_items_helper (non-dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.3" -V-3: create_remove_items_helper (non-collective, shared): open... -V-3: create_remove_items_helper: close... -V-3: create_remove_items_helper (non-dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.4" -V-3: create_remove_items_helper (non-collective, shared): open... -V-3: create_remove_items_helper: close... -V-3: create_remove_items_helper (non-dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.5" -V-3: create_remove_items_helper (non-collective, shared): open... -V-3: create_remove_items_helper: close... -V-3: create_remove_items_helper (non-dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.6" -V-3: create_remove_items_helper (non-collective, shared): open... -V-3: create_remove_items_helper: close... -V-3: create_remove_items_helper (non-dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.7" -V-3: create_remove_items_helper (non-collective, shared): open... -V-3: create_remove_items_helper: close... -V-3: create_remove_items_helper (non-dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.8" -V-3: create_remove_items_helper (non-collective, shared): open... -V-3: create_remove_items_helper: close... -V-3: create_remove_items_helper (non-dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.9" -V-3: create_remove_items_helper (non-collective, shared): open... -V-3: create_remove_items_helper: close... -V-3: create_remove_items_helper (non-dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.10" -V-3: create_remove_items_helper (non-collective, shared): open... -V-3: create_remove_items_helper: close... -V-3: create_remove_items_helper (non-dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.11" -V-3: create_remove_items_helper (non-collective, shared): open... -V-3: create_remove_items_helper: close... -V-3: create_remove_items_helper (non-dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.12" -V-3: create_remove_items_helper (non-collective, shared): open... -V-3: create_remove_items_helper: close... -V-3: create_remove_items_helper (non-dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.13" -V-3: create_remove_items_helper (non-collective, shared): open... -V-3: create_remove_items_helper: close... -V-3: create_remove_items_helper (non-dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.14" -V-3: create_remove_items_helper (non-collective, shared): open... -V-3: create_remove_items_helper: close... -V-3: create_remove_items_helper (non-dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.15" -V-3: create_remove_items_helper (non-collective, shared): open... -V-3: create_remove_items_helper: close... -V-3: create_remove_items_helper (non-dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.16" -V-3: create_remove_items_helper (non-collective, shared): open... -V-3: create_remove_items_helper: close... -V-3: create_remove_items_helper (non-dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.17" -V-3: create_remove_items_helper (non-collective, shared): open... -V-3: create_remove_items_helper: close... -V-3: create_remove_items_helper (non-dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.18" -V-3: create_remove_items_helper (non-collective, shared): open... -V-3: create_remove_items_helper: close... -V-3: create_remove_items_helper (non-dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.19" -V-3: create_remove_items_helper (non-collective, shared): open... -V-3: create_remove_items_helper: close... -V-3: main: Using testdir, "/dev/shm/mdest/#test-dir.0-0" +V-3: Rank 0 Line 2082 main (before display_freespace): testdirpath is '/dev/shm/mdest' +V-3: Rank 0 Line 1506 Entering display_freespace on /dev/shm/mdest... +V-3: Rank 0 Line 1525 Before show_file_system_size, dirpath is '/dev/shm' +V-3: Rank 0 Line 1527 After show_file_system_size, dirpath is '/dev/shm' +V-3: Rank 0 Line 2097 main (after display_freespace): testdirpath is '/dev/shm/mdest' +V-3: Rank 0 Line 1656 main (create hierarchical directory loop-!unque_dir_per_task): Calling create_remove_directory_tree with '/dev/shm/mdest/test-dir.0-0' +V-3: Rank 0 Line 1683 V-3: main: Using unique_mk_dir, 'mdtest_tree.0' +V-3: Rank 0 Line 1704 V-3: main: Copied unique_mk_dir, 'mdtest_tree.0', to topdir +V-3: Rank 0 Line 801 directory_test: create path is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0' +V-3: Rank 0 Line 457 create_remove_items (start): temp_path is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0' +V-3: Rank 0 Line 288 create_remove_items_helper (dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.0' +V-3: Rank 0 Line 288 create_remove_items_helper (dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.1' +V-3: Rank 0 Line 288 create_remove_items_helper (dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.2' +V-3: Rank 0 Line 288 create_remove_items_helper (dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.3' +V-3: Rank 0 Line 288 create_remove_items_helper (dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.4' +V-3: Rank 0 Line 288 create_remove_items_helper (dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.5' +V-3: Rank 0 Line 288 create_remove_items_helper (dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.6' +V-3: Rank 0 Line 288 create_remove_items_helper (dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.7' +V-3: Rank 0 Line 288 create_remove_items_helper (dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.8' +V-3: Rank 0 Line 288 create_remove_items_helper (dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.9' +V-3: Rank 0 Line 288 create_remove_items_helper (dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.10' +V-3: Rank 0 Line 288 create_remove_items_helper (dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.11' +V-3: Rank 0 Line 288 create_remove_items_helper (dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.12' +V-3: Rank 0 Line 288 create_remove_items_helper (dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.13' +V-3: Rank 0 Line 288 create_remove_items_helper (dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.14' +V-3: Rank 0 Line 288 create_remove_items_helper (dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.15' +V-3: Rank 0 Line 288 create_remove_items_helper (dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.16' +V-3: Rank 0 Line 288 create_remove_items_helper (dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.17' +V-3: Rank 0 Line 288 create_remove_items_helper (dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.18' +V-3: Rank 0 Line 288 create_remove_items_helper (dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.19' +V-3: Rank 0 Line 1716 will file_test on mdtest_tree.0 +V-3: Rank 0 Line 990 Entering file_test on mdtest_tree.0 +V-3: Rank 0 Line 1012 file_test: create path is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0' +V-3: Rank 0 Line 457 create_remove_items (start): temp_path is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0' +V-3: Rank 0 Line 326 create_remove_items_helper (non-dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/file.mdtest.0.0' +V-3: Rank 0 Line 348 create_remove_items_helper (non-collective, shared): open... +V-3: Rank 0 Line 373 create_remove_items_helper: close... +V-3: Rank 0 Line 326 create_remove_items_helper (non-dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/file.mdtest.0.1' +V-3: Rank 0 Line 348 create_remove_items_helper (non-collective, shared): open... +V-3: Rank 0 Line 373 create_remove_items_helper: close... +V-3: Rank 0 Line 326 create_remove_items_helper (non-dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/file.mdtest.0.2' +V-3: Rank 0 Line 348 create_remove_items_helper (non-collective, shared): open... +V-3: Rank 0 Line 373 create_remove_items_helper: close... +V-3: Rank 0 Line 326 create_remove_items_helper (non-dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/file.mdtest.0.3' +V-3: Rank 0 Line 348 create_remove_items_helper (non-collective, shared): open... +V-3: Rank 0 Line 373 create_remove_items_helper: close... +V-3: Rank 0 Line 326 create_remove_items_helper (non-dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/file.mdtest.0.4' +V-3: Rank 0 Line 348 create_remove_items_helper (non-collective, shared): open... +V-3: Rank 0 Line 373 create_remove_items_helper: close... +V-3: Rank 0 Line 326 create_remove_items_helper (non-dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/file.mdtest.0.5' +V-3: Rank 0 Line 348 create_remove_items_helper (non-collective, shared): open... +V-3: Rank 0 Line 373 create_remove_items_helper: close... +V-3: Rank 0 Line 326 create_remove_items_helper (non-dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/file.mdtest.0.6' +V-3: Rank 0 Line 348 create_remove_items_helper (non-collective, shared): open... +V-3: Rank 0 Line 373 create_remove_items_helper: close... +V-3: Rank 0 Line 326 create_remove_items_helper (non-dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/file.mdtest.0.7' +V-3: Rank 0 Line 348 create_remove_items_helper (non-collective, shared): open... +V-3: Rank 0 Line 373 create_remove_items_helper: close... +V-3: Rank 0 Line 326 create_remove_items_helper (non-dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/file.mdtest.0.8' +V-3: Rank 0 Line 348 create_remove_items_helper (non-collective, shared): open... +V-3: Rank 0 Line 373 create_remove_items_helper: close... +V-3: Rank 0 Line 326 create_remove_items_helper (non-dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/file.mdtest.0.9' +V-3: Rank 0 Line 348 create_remove_items_helper (non-collective, shared): open... +V-3: Rank 0 Line 373 create_remove_items_helper: close... +V-3: Rank 0 Line 326 create_remove_items_helper (non-dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/file.mdtest.0.10' +V-3: Rank 0 Line 348 create_remove_items_helper (non-collective, shared): open... +V-3: Rank 0 Line 373 create_remove_items_helper: close... +V-3: Rank 0 Line 326 create_remove_items_helper (non-dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/file.mdtest.0.11' +V-3: Rank 0 Line 348 create_remove_items_helper (non-collective, shared): open... +V-3: Rank 0 Line 373 create_remove_items_helper: close... +V-3: Rank 0 Line 326 create_remove_items_helper (non-dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/file.mdtest.0.12' +V-3: Rank 0 Line 348 create_remove_items_helper (non-collective, shared): open... +V-3: Rank 0 Line 373 create_remove_items_helper: close... +V-3: Rank 0 Line 326 create_remove_items_helper (non-dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/file.mdtest.0.13' +V-3: Rank 0 Line 348 create_remove_items_helper (non-collective, shared): open... +V-3: Rank 0 Line 373 create_remove_items_helper: close... +V-3: Rank 0 Line 326 create_remove_items_helper (non-dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/file.mdtest.0.14' +V-3: Rank 0 Line 348 create_remove_items_helper (non-collective, shared): open... +V-3: Rank 0 Line 373 create_remove_items_helper: close... +V-3: Rank 0 Line 326 create_remove_items_helper (non-dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/file.mdtest.0.15' +V-3: Rank 0 Line 348 create_remove_items_helper (non-collective, shared): open... +V-3: Rank 0 Line 373 create_remove_items_helper: close... +V-3: Rank 0 Line 326 create_remove_items_helper (non-dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/file.mdtest.0.16' +V-3: Rank 0 Line 348 create_remove_items_helper (non-collective, shared): open... +V-3: Rank 0 Line 373 create_remove_items_helper: close... +V-3: Rank 0 Line 326 create_remove_items_helper (non-dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/file.mdtest.0.17' +V-3: Rank 0 Line 348 create_remove_items_helper (non-collective, shared): open... +V-3: Rank 0 Line 373 create_remove_items_helper: close... +V-3: Rank 0 Line 326 create_remove_items_helper (non-dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/file.mdtest.0.18' +V-3: Rank 0 Line 348 create_remove_items_helper (non-collective, shared): open... +V-3: Rank 0 Line 373 create_remove_items_helper: close... +V-3: Rank 0 Line 326 create_remove_items_helper (non-dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/file.mdtest.0.19' +V-3: Rank 0 Line 348 create_remove_items_helper (non-collective, shared): open... +V-3: Rank 0 Line 373 create_remove_items_helper: close... +V-3: Rank 0 Line 1723 main: Using testdir, '/dev/shm/mdest/test-dir.0-0' diff --git a/testing/mdtest-patterns/advanced/4.txt b/testing/mdtest-patterns/advanced/4.txt index 8df166a4..5d3b7da0 100644 --- a/testing/mdtest-patterns/advanced/4.txt +++ b/testing/mdtest-patterns/advanced/4.txt @@ -1,50 +1,52 @@ -V-3: main (before display_freespace): testdirpath is "/dev/shm/mdest" -V-3: testdirpath is "/dev/shm/mdest" -V-3: Before show_file_system_size, dirpath is "/dev/shm" -V-3: After show_file_system_size, dirpath is "/dev/shm" -V-3: main (after display_freespace): testdirpath is "/dev/shm/mdest" -V-3: main: Using unique_mk_dir, "mdtest_tree.0" -V-3: main: Copied unique_mk_dir, "mdtest_tree.0", to topdir -V-3: directory_test: stat path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" -V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.0 -V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.1 -V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.2 -V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.3 -V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.4 -V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.5 -V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.6 -V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.7 -V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.8 -V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.9 -V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.10 -V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.11 -V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.12 -V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.13 -V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.14 -V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.15 -V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.16 -V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.17 -V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.18 -V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.19 -V-3: file_test: stat path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" -V-3: mdtest_stat file: /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.0 -V-3: mdtest_stat file: /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.1 -V-3: mdtest_stat file: /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.2 -V-3: mdtest_stat file: /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.3 -V-3: mdtest_stat file: /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.4 -V-3: mdtest_stat file: /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.5 -V-3: mdtest_stat file: /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.6 -V-3: mdtest_stat file: /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.7 -V-3: mdtest_stat file: /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.8 -V-3: mdtest_stat file: /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.9 -V-3: mdtest_stat file: /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.10 -V-3: mdtest_stat file: /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.11 -V-3: mdtest_stat file: /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.12 -V-3: mdtest_stat file: /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.13 -V-3: mdtest_stat file: /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.14 -V-3: mdtest_stat file: /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.15 -V-3: mdtest_stat file: /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.16 -V-3: mdtest_stat file: /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.17 -V-3: mdtest_stat file: /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.18 -V-3: mdtest_stat file: /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/file.mdtest.0.19 -V-3: main: Using testdir, "/dev/shm/mdest/#test-dir.0-0" +V-3: Rank 0 Line 2082 main (before display_freespace): testdirpath is '/dev/shm/mdest' +V-3: Rank 0 Line 1506 Entering display_freespace on /dev/shm/mdest... +V-3: Rank 0 Line 1525 Before show_file_system_size, dirpath is '/dev/shm' +V-3: Rank 0 Line 1527 After show_file_system_size, dirpath is '/dev/shm' +V-3: Rank 0 Line 2097 main (after display_freespace): testdirpath is '/dev/shm/mdest' +V-3: Rank 0 Line 1683 V-3: main: Using unique_mk_dir, 'mdtest_tree.0' +V-3: Rank 0 Line 1704 V-3: main: Copied unique_mk_dir, 'mdtest_tree.0', to topdir +V-3: Rank 0 Line 833 stat path is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0' +V-3: Rank 0 Line 588 mdtest_stat dir: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.0 +V-3: Rank 0 Line 588 mdtest_stat dir: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.1 +V-3: Rank 0 Line 588 mdtest_stat dir: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.2 +V-3: Rank 0 Line 588 mdtest_stat dir: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.3 +V-3: Rank 0 Line 588 mdtest_stat dir: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.4 +V-3: Rank 0 Line 588 mdtest_stat dir: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.5 +V-3: Rank 0 Line 588 mdtest_stat dir: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.6 +V-3: Rank 0 Line 588 mdtest_stat dir: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.7 +V-3: Rank 0 Line 588 mdtest_stat dir: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.8 +V-3: Rank 0 Line 588 mdtest_stat dir: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.9 +V-3: Rank 0 Line 588 mdtest_stat dir: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.10 +V-3: Rank 0 Line 588 mdtest_stat dir: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.11 +V-3: Rank 0 Line 588 mdtest_stat dir: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.12 +V-3: Rank 0 Line 588 mdtest_stat dir: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.13 +V-3: Rank 0 Line 588 mdtest_stat dir: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.14 +V-3: Rank 0 Line 588 mdtest_stat dir: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.15 +V-3: Rank 0 Line 588 mdtest_stat dir: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.16 +V-3: Rank 0 Line 588 mdtest_stat dir: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.17 +V-3: Rank 0 Line 588 mdtest_stat dir: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.18 +V-3: Rank 0 Line 588 mdtest_stat dir: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.19 +V-3: Rank 0 Line 1716 will file_test on mdtest_tree.0 +V-3: Rank 0 Line 990 Entering file_test on mdtest_tree.0 +V-3: Rank 0 Line 1079 file_test: stat path is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0' +V-3: Rank 0 Line 588 mdtest_stat file: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0/file.mdtest.0.0 +V-3: Rank 0 Line 588 mdtest_stat file: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0/file.mdtest.0.1 +V-3: Rank 0 Line 588 mdtest_stat file: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0/file.mdtest.0.2 +V-3: Rank 0 Line 588 mdtest_stat file: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0/file.mdtest.0.3 +V-3: Rank 0 Line 588 mdtest_stat file: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0/file.mdtest.0.4 +V-3: Rank 0 Line 588 mdtest_stat file: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0/file.mdtest.0.5 +V-3: Rank 0 Line 588 mdtest_stat file: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0/file.mdtest.0.6 +V-3: Rank 0 Line 588 mdtest_stat file: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0/file.mdtest.0.7 +V-3: Rank 0 Line 588 mdtest_stat file: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0/file.mdtest.0.8 +V-3: Rank 0 Line 588 mdtest_stat file: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0/file.mdtest.0.9 +V-3: Rank 0 Line 588 mdtest_stat file: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0/file.mdtest.0.10 +V-3: Rank 0 Line 588 mdtest_stat file: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0/file.mdtest.0.11 +V-3: Rank 0 Line 588 mdtest_stat file: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0/file.mdtest.0.12 +V-3: Rank 0 Line 588 mdtest_stat file: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0/file.mdtest.0.13 +V-3: Rank 0 Line 588 mdtest_stat file: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0/file.mdtest.0.14 +V-3: Rank 0 Line 588 mdtest_stat file: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0/file.mdtest.0.15 +V-3: Rank 0 Line 588 mdtest_stat file: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0/file.mdtest.0.16 +V-3: Rank 0 Line 588 mdtest_stat file: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0/file.mdtest.0.17 +V-3: Rank 0 Line 588 mdtest_stat file: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0/file.mdtest.0.18 +V-3: Rank 0 Line 588 mdtest_stat file: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0/file.mdtest.0.19 +V-3: Rank 0 Line 1723 main: Using testdir, '/dev/shm/mdest/test-dir.0-0' diff --git a/testing/mdtest-patterns/advanced/5.txt b/testing/mdtest-patterns/advanced/5.txt index 300bce63..e87ae0a5 100644 --- a/testing/mdtest-patterns/advanced/5.txt +++ b/testing/mdtest-patterns/advanced/5.txt @@ -1,77 +1,77 @@ -V-3: main (before display_freespace): testdirpath is "/dev/shm/mdest" -V-3: testdirpath is "/dev/shm/mdest" -V-3: Before show_file_system_size, dirpath is "/dev/shm" -V-3: After show_file_system_size, dirpath is "/dev/shm" -V-3: main (after display_freespace): testdirpath is "/dev/shm/mdest" -V-3: main (create hierarchical directory loop-!unque_dir_per_task): Calling create_remove_directory_tree with "/dev/shm/mdest/#test-dir.0-0" -V-3: main: Using unique_mk_dir, "mdtest_tree.0" -V-3: main: Copied unique_mk_dir, "mdtest_tree.0", to topdir -V-3: directory_test: create path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" -V-3: create_remove_items (start): temp_path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" -V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.0" -V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.1" -V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.2" -V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.3" -V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.4" -V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.5" -V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.6" -V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.7" -V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.8" -V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.9" -V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.10" -V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.11" -V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.12" -V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.13" -V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.14" -V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.15" -V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.16" -V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.17" -V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.18" -V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.19" -V-3: directory_test: stat path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" -V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.0 -V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.1 -V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.2 -V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.3 -V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.4 -V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.5 -V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.6 -V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.7 -V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.8 -V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.9 -V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.10 -V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.11 -V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.12 -V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.13 -V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.14 -V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.15 -V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.16 -V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.17 -V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.18 -V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.19 -V-3: directory_test: read path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" -V-3: directory_test: remove directories path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" -V-3: create_remove_items (start): temp_path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" -V-3: create_remove_items_helper (dirs remove): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.0" -V-3: create_remove_items_helper (dirs remove): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.1" -V-3: create_remove_items_helper (dirs remove): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.2" -V-3: create_remove_items_helper (dirs remove): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.3" -V-3: create_remove_items_helper (dirs remove): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.4" -V-3: create_remove_items_helper (dirs remove): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.5" -V-3: create_remove_items_helper (dirs remove): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.6" -V-3: create_remove_items_helper (dirs remove): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.7" -V-3: create_remove_items_helper (dirs remove): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.8" -V-3: create_remove_items_helper (dirs remove): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.9" -V-3: create_remove_items_helper (dirs remove): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.10" -V-3: create_remove_items_helper (dirs remove): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.11" -V-3: create_remove_items_helper (dirs remove): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.12" -V-3: create_remove_items_helper (dirs remove): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.13" -V-3: create_remove_items_helper (dirs remove): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.14" -V-3: create_remove_items_helper (dirs remove): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.15" -V-3: create_remove_items_helper (dirs remove): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.16" -V-3: create_remove_items_helper (dirs remove): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.17" -V-3: create_remove_items_helper (dirs remove): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.18" -V-3: create_remove_items_helper (dirs remove): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0/dir.mdtest.0.19" -V-3: directory_test: remove unique directories path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" -V-3: main: Using testdir, "/dev/shm/mdest/#test-dir.0-0" -V-3: main (remove hierarchical directory loop-!unique_dir_per_task): Calling create_remove_directory_tree with "/dev/shm/mdest/#test-dir.0-0" +V-3: Rank 0 Line 2082 main (before display_freespace): testdirpath is '/dev/shm/mdest' +V-3: Rank 0 Line 1506 Entering display_freespace on /dev/shm/mdest... +V-3: Rank 0 Line 1525 Before show_file_system_size, dirpath is '/dev/shm' +V-3: Rank 0 Line 1527 After show_file_system_size, dirpath is '/dev/shm' +V-3: Rank 0 Line 2097 main (after display_freespace): testdirpath is '/dev/shm/mdest' +V-3: Rank 0 Line 1656 main (create hierarchical directory loop-!unque_dir_per_task): Calling create_remove_directory_tree with '/dev/shm/mdest/test-dir.0-0' +V-3: Rank 0 Line 1683 V-3: main: Using unique_mk_dir, 'mdtest_tree.0' +V-3: Rank 0 Line 1704 V-3: main: Copied unique_mk_dir, 'mdtest_tree.0', to topdir +V-3: Rank 0 Line 801 directory_test: create path is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0' +V-3: Rank 0 Line 457 create_remove_items (start): temp_path is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0' +V-3: Rank 0 Line 288 create_remove_items_helper (dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.0' +V-3: Rank 0 Line 288 create_remove_items_helper (dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.1' +V-3: Rank 0 Line 288 create_remove_items_helper (dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.2' +V-3: Rank 0 Line 288 create_remove_items_helper (dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.3' +V-3: Rank 0 Line 288 create_remove_items_helper (dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.4' +V-3: Rank 0 Line 288 create_remove_items_helper (dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.5' +V-3: Rank 0 Line 288 create_remove_items_helper (dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.6' +V-3: Rank 0 Line 288 create_remove_items_helper (dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.7' +V-3: Rank 0 Line 288 create_remove_items_helper (dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.8' +V-3: Rank 0 Line 288 create_remove_items_helper (dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.9' +V-3: Rank 0 Line 288 create_remove_items_helper (dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.10' +V-3: Rank 0 Line 288 create_remove_items_helper (dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.11' +V-3: Rank 0 Line 288 create_remove_items_helper (dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.12' +V-3: Rank 0 Line 288 create_remove_items_helper (dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.13' +V-3: Rank 0 Line 288 create_remove_items_helper (dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.14' +V-3: Rank 0 Line 288 create_remove_items_helper (dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.15' +V-3: Rank 0 Line 288 create_remove_items_helper (dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.16' +V-3: Rank 0 Line 288 create_remove_items_helper (dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.17' +V-3: Rank 0 Line 288 create_remove_items_helper (dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.18' +V-3: Rank 0 Line 288 create_remove_items_helper (dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.19' +V-3: Rank 0 Line 833 stat path is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0' +V-3: Rank 0 Line 588 mdtest_stat dir: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.0 +V-3: Rank 0 Line 588 mdtest_stat dir: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.1 +V-3: Rank 0 Line 588 mdtest_stat dir: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.2 +V-3: Rank 0 Line 588 mdtest_stat dir: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.3 +V-3: Rank 0 Line 588 mdtest_stat dir: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.4 +V-3: Rank 0 Line 588 mdtest_stat dir: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.5 +V-3: Rank 0 Line 588 mdtest_stat dir: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.6 +V-3: Rank 0 Line 588 mdtest_stat dir: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.7 +V-3: Rank 0 Line 588 mdtest_stat dir: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.8 +V-3: Rank 0 Line 588 mdtest_stat dir: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.9 +V-3: Rank 0 Line 588 mdtest_stat dir: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.10 +V-3: Rank 0 Line 588 mdtest_stat dir: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.11 +V-3: Rank 0 Line 588 mdtest_stat dir: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.12 +V-3: Rank 0 Line 588 mdtest_stat dir: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.13 +V-3: Rank 0 Line 588 mdtest_stat dir: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.14 +V-3: Rank 0 Line 588 mdtest_stat dir: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.15 +V-3: Rank 0 Line 588 mdtest_stat dir: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.16 +V-3: Rank 0 Line 588 mdtest_stat dir: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.17 +V-3: Rank 0 Line 588 mdtest_stat dir: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.18 +V-3: Rank 0 Line 588 mdtest_stat dir: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.19 +V-3: Rank 0 Line 862 directory_test: read path is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0' +V-3: Rank 0 Line 890 directory_test: remove directories path is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0' +V-3: Rank 0 Line 457 create_remove_items (start): temp_path is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0' +V-3: Rank 0 Line 288 create_remove_items_helper (dirs remove): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.0' +V-3: Rank 0 Line 288 create_remove_items_helper (dirs remove): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.1' +V-3: Rank 0 Line 288 create_remove_items_helper (dirs remove): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.2' +V-3: Rank 0 Line 288 create_remove_items_helper (dirs remove): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.3' +V-3: Rank 0 Line 288 create_remove_items_helper (dirs remove): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.4' +V-3: Rank 0 Line 288 create_remove_items_helper (dirs remove): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.5' +V-3: Rank 0 Line 288 create_remove_items_helper (dirs remove): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.6' +V-3: Rank 0 Line 288 create_remove_items_helper (dirs remove): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.7' +V-3: Rank 0 Line 288 create_remove_items_helper (dirs remove): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.8' +V-3: Rank 0 Line 288 create_remove_items_helper (dirs remove): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.9' +V-3: Rank 0 Line 288 create_remove_items_helper (dirs remove): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.10' +V-3: Rank 0 Line 288 create_remove_items_helper (dirs remove): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.11' +V-3: Rank 0 Line 288 create_remove_items_helper (dirs remove): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.12' +V-3: Rank 0 Line 288 create_remove_items_helper (dirs remove): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.13' +V-3: Rank 0 Line 288 create_remove_items_helper (dirs remove): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.14' +V-3: Rank 0 Line 288 create_remove_items_helper (dirs remove): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.15' +V-3: Rank 0 Line 288 create_remove_items_helper (dirs remove): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.16' +V-3: Rank 0 Line 288 create_remove_items_helper (dirs remove): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.17' +V-3: Rank 0 Line 288 create_remove_items_helper (dirs remove): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.18' +V-3: Rank 0 Line 288 create_remove_items_helper (dirs remove): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0/dir.mdtest.0.19' +V-3: Rank 0 Line 915 directory_test: remove unique directories path is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0' +V-3: Rank 0 Line 1723 main: Using testdir, '/dev/shm/mdest/test-dir.0-0' +V-3: Rank 0 Line 1764 V-3: main (remove hierarchical directory loop-!unique_dir_per_task): Calling create_remove_directory_tree with '/dev/shm/mdest/test-dir.0-0' diff --git a/testing/mdtest-patterns/basic/0.txt b/testing/mdtest-patterns/basic/0.txt index 7ff10fc8..ebe0f144 100644 --- a/testing/mdtest-patterns/basic/0.txt +++ b/testing/mdtest-patterns/basic/0.txt @@ -1,24 +1,27 @@ -V-3: main (before display_freespace): testdirpath is "/dev/shm/mdest" -V-3: testdirpath is "/dev/shm/mdest" -V-3: Before show_file_system_size, dirpath is "/dev/shm" -V-3: After show_file_system_size, dirpath is "/dev/shm" -V-3: main (after display_freespace): testdirpath is "/dev/shm/mdest" -V-3: main (create hierarchical directory loop-!unque_dir_per_task): Calling create_remove_directory_tree with "/dev/shm/mdest/#test-dir.0-0" -V-3: main: Using unique_mk_dir, "mdtest_tree.0" -V-3: main: Copied unique_mk_dir, "mdtest_tree.0", to topdir -V-3: directory_test: create path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" -V-3: create_remove_items (start): temp_path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" -V-3: directory_test: stat path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" -V-3: directory_test: read path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" -V-3: directory_test: remove directories path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" -V-3: create_remove_items (start): temp_path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" -V-3: directory_test: remove unique directories path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" -V-3: file_test: create path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" -V-3: create_remove_items (start): temp_path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" -V-3: file_test: stat path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" -V-3: file_test: read path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" -V-3: file_test: rm directories path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" -V-3: create_remove_items (start): temp_path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" -V-3: file_test: rm unique directories path is "mdtest_tree.0" -V-3: main: Using testdir, "/dev/shm/mdest/#test-dir.0-0" -V-3: main (remove hierarchical directory loop-!unique_dir_per_task): Calling create_remove_directory_tree with "/dev/shm/mdest/#test-dir.0-0" +V-3: Rank 0 Line 2082 main (before display_freespace): testdirpath is '/dev/shm/mdest' +V-3: Rank 0 Line 1506 Entering display_freespace on /dev/shm/mdest... +V-3: Rank 0 Line 1525 Before show_file_system_size, dirpath is '/dev/shm' +V-3: Rank 0 Line 1527 After show_file_system_size, dirpath is '/dev/shm' +V-3: Rank 0 Line 2097 main (after display_freespace): testdirpath is '/dev/shm/mdest' +V-3: Rank 0 Line 1656 main (create hierarchical directory loop-!unque_dir_per_task): Calling create_remove_directory_tree with '/dev/shm/mdest/test-dir.0-0' +V-3: Rank 0 Line 1683 V-3: main: Using unique_mk_dir, 'mdtest_tree.0' +V-3: Rank 0 Line 1704 V-3: main: Copied unique_mk_dir, 'mdtest_tree.0', to topdir +V-3: Rank 0 Line 801 directory_test: create path is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0' +V-3: Rank 0 Line 457 create_remove_items (start): temp_path is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0' +V-3: Rank 0 Line 833 stat path is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0' +V-3: Rank 0 Line 862 directory_test: read path is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0' +V-3: Rank 0 Line 890 directory_test: remove directories path is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0' +V-3: Rank 0 Line 457 create_remove_items (start): temp_path is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0' +V-3: Rank 0 Line 915 directory_test: remove unique directories path is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0' +V-3: Rank 0 Line 1716 will file_test on mdtest_tree.0 +V-3: Rank 0 Line 990 Entering file_test on mdtest_tree.0 +V-3: Rank 0 Line 1012 file_test: create path is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0' +V-3: Rank 0 Line 457 create_remove_items (start): temp_path is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0' +V-3: Rank 0 Line 1079 file_test: stat path is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0' +V-3: Rank 0 Line 1104 file_test: read path is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0' +V-3: Rank 0 Line 1134 file_test: rm directories path is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0' +V-3: Rank 0 Line 1141 gonna create /dev/shm/mdest/test-dir.0-0/mdtest_tree.0 +V-3: Rank 0 Line 457 create_remove_items (start): temp_path is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0' +V-3: Rank 0 Line 1158 file_test: rm unique directories path is 'mdtest_tree.0' +V-3: Rank 0 Line 1723 main: Using testdir, '/dev/shm/mdest/test-dir.0-0' +V-3: Rank 0 Line 1764 V-3: main (remove hierarchical directory loop-!unique_dir_per_task): Calling create_remove_directory_tree with '/dev/shm/mdest/test-dir.0-0' diff --git a/testing/mdtest-patterns/basic/1.txt b/testing/mdtest-patterns/basic/1.txt index 7ff10fc8..ebe0f144 100644 --- a/testing/mdtest-patterns/basic/1.txt +++ b/testing/mdtest-patterns/basic/1.txt @@ -1,24 +1,27 @@ -V-3: main (before display_freespace): testdirpath is "/dev/shm/mdest" -V-3: testdirpath is "/dev/shm/mdest" -V-3: Before show_file_system_size, dirpath is "/dev/shm" -V-3: After show_file_system_size, dirpath is "/dev/shm" -V-3: main (after display_freespace): testdirpath is "/dev/shm/mdest" -V-3: main (create hierarchical directory loop-!unque_dir_per_task): Calling create_remove_directory_tree with "/dev/shm/mdest/#test-dir.0-0" -V-3: main: Using unique_mk_dir, "mdtest_tree.0" -V-3: main: Copied unique_mk_dir, "mdtest_tree.0", to topdir -V-3: directory_test: create path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" -V-3: create_remove_items (start): temp_path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" -V-3: directory_test: stat path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" -V-3: directory_test: read path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" -V-3: directory_test: remove directories path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" -V-3: create_remove_items (start): temp_path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" -V-3: directory_test: remove unique directories path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" -V-3: file_test: create path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" -V-3: create_remove_items (start): temp_path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" -V-3: file_test: stat path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" -V-3: file_test: read path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" -V-3: file_test: rm directories path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" -V-3: create_remove_items (start): temp_path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0" -V-3: file_test: rm unique directories path is "mdtest_tree.0" -V-3: main: Using testdir, "/dev/shm/mdest/#test-dir.0-0" -V-3: main (remove hierarchical directory loop-!unique_dir_per_task): Calling create_remove_directory_tree with "/dev/shm/mdest/#test-dir.0-0" +V-3: Rank 0 Line 2082 main (before display_freespace): testdirpath is '/dev/shm/mdest' +V-3: Rank 0 Line 1506 Entering display_freespace on /dev/shm/mdest... +V-3: Rank 0 Line 1525 Before show_file_system_size, dirpath is '/dev/shm' +V-3: Rank 0 Line 1527 After show_file_system_size, dirpath is '/dev/shm' +V-3: Rank 0 Line 2097 main (after display_freespace): testdirpath is '/dev/shm/mdest' +V-3: Rank 0 Line 1656 main (create hierarchical directory loop-!unque_dir_per_task): Calling create_remove_directory_tree with '/dev/shm/mdest/test-dir.0-0' +V-3: Rank 0 Line 1683 V-3: main: Using unique_mk_dir, 'mdtest_tree.0' +V-3: Rank 0 Line 1704 V-3: main: Copied unique_mk_dir, 'mdtest_tree.0', to topdir +V-3: Rank 0 Line 801 directory_test: create path is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0' +V-3: Rank 0 Line 457 create_remove_items (start): temp_path is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0' +V-3: Rank 0 Line 833 stat path is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0' +V-3: Rank 0 Line 862 directory_test: read path is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0' +V-3: Rank 0 Line 890 directory_test: remove directories path is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0' +V-3: Rank 0 Line 457 create_remove_items (start): temp_path is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0' +V-3: Rank 0 Line 915 directory_test: remove unique directories path is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0' +V-3: Rank 0 Line 1716 will file_test on mdtest_tree.0 +V-3: Rank 0 Line 990 Entering file_test on mdtest_tree.0 +V-3: Rank 0 Line 1012 file_test: create path is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0' +V-3: Rank 0 Line 457 create_remove_items (start): temp_path is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0' +V-3: Rank 0 Line 1079 file_test: stat path is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0' +V-3: Rank 0 Line 1104 file_test: read path is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0' +V-3: Rank 0 Line 1134 file_test: rm directories path is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0' +V-3: Rank 0 Line 1141 gonna create /dev/shm/mdest/test-dir.0-0/mdtest_tree.0 +V-3: Rank 0 Line 457 create_remove_items (start): temp_path is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0' +V-3: Rank 0 Line 1158 file_test: rm unique directories path is 'mdtest_tree.0' +V-3: Rank 0 Line 1723 main: Using testdir, '/dev/shm/mdest/test-dir.0-0' +V-3: Rank 0 Line 1764 V-3: main (remove hierarchical directory loop-!unique_dir_per_task): Calling create_remove_directory_tree with '/dev/shm/mdest/test-dir.0-0' diff --git a/testing/mdtest-patterns/basic/2.txt b/testing/mdtest-patterns/basic/2.txt index a21f5a34..77f5c781 100644 --- a/testing/mdtest-patterns/basic/2.txt +++ b/testing/mdtest-patterns/basic/2.txt @@ -1,25 +1,29 @@ -V-3: main (before display_freespace): testdirpath is "/dev/shm/mdest" -V-3: testdirpath is "/dev/shm/mdest" -V-3: Before show_file_system_size, dirpath is "/dev/shm" -V-3: After show_file_system_size, dirpath is "/dev/shm" -V-3: main (after display_freespace): testdirpath is "/dev/shm/mdest" -V-3: main (create hierarchical directory loop-!collective_creates): Calling create_remove_directory_tree with "/dev/shm/mdest/#test-dir.0-0" -V-3: main: Copied unique_mk_dir, "mdtest_tree.0.0", to topdir -V-3: file_test: create path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0" -V-3: create_remove_items (start): temp_path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0" -V-3: create_remove_items (start): temp_path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0" -V-3: create_remove_items (for loop): temp_path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0/mdtest_tree.0.1/" -V-3: create_remove_items_helper (non-dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0/mdtest_tree.0.1//file.mdtest.0.1" -V-3: create_remove_items_helper (non-collective, shared): open... -V-3: create_remove_items_helper: close... -V-3: create_remove_items (start): temp_path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0/mdtest_tree.0.1/" -V-3: file_test: stat path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0" -V-3: mdtest_stat file: /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0/mdtest_tree.0.1/file.mdtest.0.1 -V-3: file_test: rm directories path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0" -V-3: create_remove_items (start): temp_path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0" -V-3: create_remove_items (start): temp_path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0" -V-3: create_remove_items (for loop): temp_path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0/mdtest_tree.0.1/" -V-3: create_remove_items_helper (non-dirs remove): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0/mdtest_tree.0.1//file.mdtest.0.1" -V-3: create_remove_items (start): temp_path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0/mdtest_tree.0.1/" -V-3: file_test: rm unique directories path is "/dev/shm/mdest/#test-dir.0-0/" -V-3: main (remove hierarchical directory loop-!collective): Calling create_remove_directory_tree with "/dev/shm/mdest/#test-dir.0-0" +V-3: Rank 0 Line 2082 main (before display_freespace): testdirpath is '/dev/shm/mdest' +V-3: Rank 0 Line 1506 Entering display_freespace on /dev/shm/mdest... +V-3: Rank 0 Line 1525 Before show_file_system_size, dirpath is '/dev/shm' +V-3: Rank 0 Line 1527 After show_file_system_size, dirpath is '/dev/shm' +V-3: Rank 0 Line 2097 main (after display_freespace): testdirpath is '/dev/shm/mdest' +V-3: Rank 0 Line 1647 main (create hierarchical directory loop-!collective_creates): Calling create_remove_directory_tree with '/dev/shm/mdest/test-dir.0-0' +V-3: Rank 0 Line 1694 i 1 nstride 0 +V-3: Rank 0 Line 1704 V-3: main: Copied unique_mk_dir, 'mdtest_tree.0.0', to topdir +V-3: Rank 0 Line 1716 will file_test on mdtest_tree.0.0 +V-3: Rank 0 Line 990 Entering file_test on mdtest_tree.0.0 +V-3: Rank 0 Line 1012 file_test: create path is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0.0' +V-3: Rank 0 Line 457 create_remove_items (start): temp_path is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0.0' +V-3: Rank 0 Line 457 create_remove_items (start): temp_path is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0.0' +V-3: Rank 0 Line 483 create_remove_items (for loop): temp_path is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0.0/mdtest_tree.0.1/' +V-3: Rank 0 Line 326 create_remove_items_helper (non-dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0.0/mdtest_tree.0.1//file.mdtest.0.1' +V-3: Rank 0 Line 348 create_remove_items_helper (non-collective, shared): open... +V-3: Rank 0 Line 373 create_remove_items_helper: close... +V-3: Rank 0 Line 457 create_remove_items (start): temp_path is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0.0/mdtest_tree.0.1/' +V-3: Rank 0 Line 1079 file_test: stat path is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0.0' +V-3: Rank 0 Line 588 mdtest_stat file: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0.0/mdtest_tree.0.1/file.mdtest.0.1 +V-3: Rank 0 Line 1134 file_test: rm directories path is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0.0' +V-3: Rank 0 Line 1141 gonna create /dev/shm/mdest/test-dir.0-0/mdtest_tree.0.0 +V-3: Rank 0 Line 457 create_remove_items (start): temp_path is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0.0' +V-3: Rank 0 Line 457 create_remove_items (start): temp_path is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0.0' +V-3: Rank 0 Line 483 create_remove_items (for loop): temp_path is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0.0/mdtest_tree.0.1/' +V-3: Rank 0 Line 310 create_remove_items_helper (non-dirs remove): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0.0/mdtest_tree.0.1//file.mdtest.0.1' +V-3: Rank 0 Line 457 create_remove_items (start): temp_path is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0.0/mdtest_tree.0.1/' +V-3: Rank 0 Line 1158 file_test: rm unique directories path is '/dev/shm/mdest/test-dir.0-0/' +V-3: Rank 0 Line 1754 main (remove hierarchical directory loop-!collective): Calling create_remove_directory_tree with '/dev/shm/mdest/test-dir.0-0' diff --git a/testing/mdtest-patterns/basic/3.txt b/testing/mdtest-patterns/basic/3.txt index 3e4aa787..eafadc1e 100644 --- a/testing/mdtest-patterns/basic/3.txt +++ b/testing/mdtest-patterns/basic/3.txt @@ -1,31 +1,34 @@ -V-3: main (before display_freespace): testdirpath is "/dev/shm/mdest" -V-3: testdirpath is "/dev/shm/mdest" -V-3: Before show_file_system_size, dirpath is "/dev/shm" -V-3: After show_file_system_size, dirpath is "/dev/shm" -V-3: main (after display_freespace): testdirpath is "/dev/shm/mdest" -V-3: main (create hierarchical directory loop-!collective_creates): Calling create_remove_directory_tree with "/dev/shm/mdest/#test-dir.0-0" -V-3: main: Copied unique_mk_dir, "mdtest_tree.0.0", to topdir -V-3: directory_test: create path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0" -V-3: create_remove_items (start): temp_path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0" -V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0/dir.mdtest.0.0" -V-3: create_remove_items (start): temp_path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0" -V-3: create_remove_items (for loop): temp_path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0/mdtest_tree.0.1/" -V-3: create_remove_items_helper (dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0/mdtest_tree.0.1//dir.mdtest.0.1" -V-3: create_remove_items (start): temp_path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0/mdtest_tree.0.1/" -V-3: directory_test: stat path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0" -V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0/dir.mdtest.0.0 -V-3: mdtest_stat dir : /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0/mdtest_tree.0.1/dir.mdtest.0.1 -V-3: file_test: create path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0" -V-3: create_remove_items (start): temp_path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0" -V-3: create_remove_items_helper (non-dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0/file.mdtest.0.0" -V-3: create_remove_items_helper (non-collective, shared): open... -V-3: create_remove_items_helper: close... -V-3: create_remove_items (start): temp_path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0" -V-3: create_remove_items (for loop): temp_path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0/mdtest_tree.0.1/" -V-3: create_remove_items_helper (non-dirs create): curr_item is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0/mdtest_tree.0.1//file.mdtest.0.1" -V-3: create_remove_items_helper (non-collective, shared): open... -V-3: create_remove_items_helper: close... -V-3: create_remove_items (start): temp_path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0/mdtest_tree.0.1/" -V-3: file_test: stat path is "/dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0" -V-3: mdtest_stat file: /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0/file.mdtest.0.0 -V-3: mdtest_stat file: /dev/shm/mdest/#test-dir.0-0/mdtest_tree.0.0/mdtest_tree.0.1/file.mdtest.0.1 +V-3: Rank 0 Line 2082 main (before display_freespace): testdirpath is '/dev/shm/mdest' +V-3: Rank 0 Line 1506 Entering display_freespace on /dev/shm/mdest... +V-3: Rank 0 Line 1525 Before show_file_system_size, dirpath is '/dev/shm' +V-3: Rank 0 Line 1527 After show_file_system_size, dirpath is '/dev/shm' +V-3: Rank 0 Line 2097 main (after display_freespace): testdirpath is '/dev/shm/mdest' +V-3: Rank 0 Line 1647 main (create hierarchical directory loop-!collective_creates): Calling create_remove_directory_tree with '/dev/shm/mdest/test-dir.0-0' +V-3: Rank 0 Line 1694 i 1 nstride 0 +V-3: Rank 0 Line 1704 V-3: main: Copied unique_mk_dir, 'mdtest_tree.0.0', to topdir +V-3: Rank 0 Line 801 directory_test: create path is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0.0' +V-3: Rank 0 Line 457 create_remove_items (start): temp_path is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0.0' +V-3: Rank 0 Line 288 create_remove_items_helper (dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0.0/dir.mdtest.0.0' +V-3: Rank 0 Line 457 create_remove_items (start): temp_path is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0.0' +V-3: Rank 0 Line 483 create_remove_items (for loop): temp_path is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0.0/mdtest_tree.0.1/' +V-3: Rank 0 Line 288 create_remove_items_helper (dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0.0/mdtest_tree.0.1//dir.mdtest.0.1' +V-3: Rank 0 Line 457 create_remove_items (start): temp_path is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0.0/mdtest_tree.0.1/' +V-3: Rank 0 Line 833 stat path is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0.0' +V-3: Rank 0 Line 588 mdtest_stat dir: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0.0/dir.mdtest.0.0 +V-3: Rank 0 Line 588 mdtest_stat dir: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0.0/mdtest_tree.0.1/dir.mdtest.0.1 +V-3: Rank 0 Line 1716 will file_test on mdtest_tree.0.0 +V-3: Rank 0 Line 990 Entering file_test on mdtest_tree.0.0 +V-3: Rank 0 Line 1012 file_test: create path is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0.0' +V-3: Rank 0 Line 457 create_remove_items (start): temp_path is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0.0' +V-3: Rank 0 Line 326 create_remove_items_helper (non-dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0.0/file.mdtest.0.0' +V-3: Rank 0 Line 348 create_remove_items_helper (non-collective, shared): open... +V-3: Rank 0 Line 373 create_remove_items_helper: close... +V-3: Rank 0 Line 457 create_remove_items (start): temp_path is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0.0' +V-3: Rank 0 Line 483 create_remove_items (for loop): temp_path is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0.0/mdtest_tree.0.1/' +V-3: Rank 0 Line 326 create_remove_items_helper (non-dirs create): curr_item is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0.0/mdtest_tree.0.1//file.mdtest.0.1' +V-3: Rank 0 Line 348 create_remove_items_helper (non-collective, shared): open... +V-3: Rank 0 Line 373 create_remove_items_helper: close... +V-3: Rank 0 Line 457 create_remove_items (start): temp_path is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0.0/mdtest_tree.0.1/' +V-3: Rank 0 Line 1079 file_test: stat path is '/dev/shm/mdest/test-dir.0-0/mdtest_tree.0.0' +V-3: Rank 0 Line 588 mdtest_stat file: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0.0/file.mdtest.0.0 +V-3: Rank 0 Line 588 mdtest_stat file: /dev/shm/mdest/test-dir.0-0/mdtest_tree.0.0/mdtest_tree.0.1/file.mdtest.0.1 From 70e8b13d1dc9586113f929ba70619fb2089bc7fb Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Sun, 4 Aug 2019 18:27:20 -0500 Subject: [PATCH 139/206] Fix Last Free --- src/aiori-POSIX.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/aiori-POSIX.c b/src/aiori-POSIX.c index 0eb663b4..d78cd40a 100755 --- a/src/aiori-POSIX.c +++ b/src/aiori-POSIX.c @@ -89,7 +89,6 @@ option_help * POSIX_options(void ** init_backend_options, void * init_values){ } *init_backend_options = o; - free(o); option_help h [] = { {0, "posix.odirect", "Direct I/O Mode", OPTION_FLAG, 'd', & o->direct_io}, From 8cb878507e73f57ebe0809ee623dcf95a5417221 Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Mon, 12 Aug 2019 14:25:05 +0000 Subject: [PATCH 140/206] Add dfs chunk_size and oclass options. update dfs_remove for API change. Signed-off-by: Mohamad Chaarawi --- src/aiori-DAOS.c | 2 +- src/aiori-DFS.c | 64 +++++++++++++++++++++++++++++++----------------- 2 files changed, 42 insertions(+), 24 deletions(-) diff --git a/src/aiori-DAOS.c b/src/aiori-DAOS.c index 71aff90d..fc339cae 100644 --- a/src/aiori-DAOS.c +++ b/src/aiori-DAOS.c @@ -211,7 +211,7 @@ HandleDistribute(daos_handle_t *handle, enum handleType type) static option_help * DAOS_options() { - return options; + return options; } static void diff --git a/src/aiori-DFS.c b/src/aiori-DFS.c index 35870e1f..2acd9bb7 100755 --- a/src/aiori-DFS.c +++ b/src/aiori-DFS.c @@ -42,7 +42,8 @@ #include "utilities.h" dfs_t *dfs; -daos_handle_t poh, coh; +static daos_handle_t poh, coh; +static daos_oclass_id_t objectClass = OC_SX; static struct d_hash_table *dir_hash; struct aiori_dir_hdl { @@ -63,6 +64,8 @@ struct dfs_options{ char *svcl; char *group; char *cont; + int chunk_size; + char *oclass; int destroy; }; @@ -71,14 +74,18 @@ static struct dfs_options o = { .svcl = NULL, .group = NULL, .cont = NULL, + .chunk_size = 1048576, + .oclass = NULL, .destroy = 0, }; static option_help options [] = { - {0, "dfs.pool", "DAOS pool uuid", OPTION_REQUIRED_ARGUMENT, 's', & o.pool}, - {0, "dfs.svcl", "DAOS pool SVCL", OPTION_REQUIRED_ARGUMENT, 's', & o.svcl}, - {0, "dfs.group", "DAOS server group", OPTION_OPTIONAL_ARGUMENT, 's', & o.group}, + {0, "dfs.pool", "pool uuid", OPTION_REQUIRED_ARGUMENT, 's', & o.pool}, + {0, "dfs.svcl", "pool SVCL", OPTION_REQUIRED_ARGUMENT, 's', & o.svcl}, + {0, "dfs.group", "server group", OPTION_OPTIONAL_ARGUMENT, 's', & o.group}, {0, "dfs.cont", "DFS container uuid", OPTION_REQUIRED_ARGUMENT, 's', & o.cont}, + {0, "dfs.chunk_size", "chunk size", OPTION_OPTIONAL_ARGUMENT, 'd', &o.chunk_size}, + {0, "dfs.oclass", "object class", OPTION_OPTIONAL_ARGUMENT, 's', &o.oclass}, {0, "dfs.destroy", "Destroy DFS Container", OPTION_FLAG, 'd', &o.destroy}, LAST_OPTION }; @@ -156,31 +163,31 @@ do { \ static inline struct aiori_dir_hdl * hdl_obj(d_list_t *rlink) { - return container_of(rlink, struct aiori_dir_hdl, entry); + return container_of(rlink, struct aiori_dir_hdl, entry); } static bool key_cmp(struct d_hash_table *htable, d_list_t *rlink, const void *key, unsigned int ksize) { - struct aiori_dir_hdl *hdl = hdl_obj(rlink); + struct aiori_dir_hdl *hdl = hdl_obj(rlink); - return (strcmp(hdl->name, (const char *)key) == 0); + return (strcmp(hdl->name, (const char *)key) == 0); } static void rec_free(struct d_hash_table *htable, d_list_t *rlink) { - struct aiori_dir_hdl *hdl = hdl_obj(rlink); + struct aiori_dir_hdl *hdl = hdl_obj(rlink); - assert(d_hash_rec_unlinked(&hdl->entry)); - dfs_release(hdl->oh); - free(hdl); + assert(d_hash_rec_unlinked(&hdl->entry)); + dfs_release(hdl->oh); + free(hdl); } static d_hash_table_ops_t hdl_hash_ops = { - .hop_key_cmp = key_cmp, - .hop_rec_free = rec_free + .hop_key_cmp = key_cmp, + .hop_rec_free = rec_free }; /* Distribute process 0's pool or container handle to others. */ @@ -360,7 +367,7 @@ lookup_insert_dir(const char *name) } static option_help * DFS_options(){ - return options; + return options; } static void @@ -370,6 +377,12 @@ DFS_Init() { if (o.pool == NULL || o.svcl == NULL || o.cont == NULL) ERR("Invalid pool or container options\n"); + if (o.oclass) { + objectClass = daos_oclass_name2id(o.oclass); + if (objectClass == OC_UNKNOWN) + DCHECK(-1, "Invalid DAOS Object class %s\n", o.oclass); + } + rc = daos_init(); DCHECK(rc, "Failed to initialize daos"); @@ -487,14 +500,18 @@ DFS_Create(char *testFileName, IOR_param_t *param) fd_oflag |= O_CREAT | O_RDWR | O_EXCL; rc = dfs_open(dfs, parent, name, mode, fd_oflag, - OC_SX, 0, NULL, &obj); - DERR(rc, "dfs_open() of %s Failed", name); - } else { - fd_oflag |= O_RDWR; - rc = dfs_open(dfs, parent, name, mode, fd_oflag, - OC_SX, 0, NULL, &obj); + objectClass, o.chunk_size, NULL, &obj); DERR(rc, "dfs_open() of %s Failed", name); } + if (!param->filePerProc) { + MPI_Barrier(MPI_COMM_WORLD); + if (rank != 0) { + fd_oflag |= O_RDWR; + rc = dfs_open(dfs, parent, name, mode, fd_oflag, + objectClass, o.chunk_size, NULL, &obj); + DERR(rc, "dfs_open() of %s Failed", name); + } + } out: if (name) @@ -530,7 +547,8 @@ DFS_Open(char *testFileName, IOR_param_t *param) if (parent == NULL) DERR(rc, "Failed to lookup parent dir"); - rc = dfs_open(dfs, parent, name, mode, fd_oflag, 0, 0, NULL, &obj); + rc = dfs_open(dfs, parent, name, mode, fd_oflag, objectClass, + o.chunk_size, NULL, &obj); DERR(rc, "dfs_open() of %s Failed", name); out: @@ -638,7 +656,7 @@ DFS_Delete(char *testFileName, IOR_param_t * param) if (parent == NULL) DERR(rc, "Failed to lookup parent dir"); - rc = dfs_remove(dfs, parent, name, false); + rc = dfs_remove(dfs, parent, name, false, NULL); DERR(rc, "dfs_remove() of %s Failed", name); out: @@ -756,7 +774,7 @@ DFS_Rmdir(const char *path, IOR_param_t * param) if (parent == NULL) DERR(rc, "Failed to lookup parent dir"); - rc = dfs_remove(dfs, parent, name, false); + rc = dfs_remove(dfs, parent, name, false, NULL); DERR(rc, "dfs_remove() of %s Failed", name); out: From 1320aa279cdb18150d5f01e8838062793dbf0877 Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Mon, 12 Aug 2019 16:04:20 +0000 Subject: [PATCH 141/206] add some barriers before cont close and destroy to make sure all ranks are done. Signed-off-by: Mohamad Chaarawi --- .gitignore | 1 + src/aiori-DAOS.c | 10 +++++++--- src/aiori-DFS.c | 5 ++++- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 8c0ab90d..eb59cf7a 100644 --- a/.gitignore +++ b/.gitignore @@ -42,6 +42,7 @@ src/testlib src/test/.deps/ src/test/.dirstamp src/test/lib.o +build/ doc/doxygen/build doc/sphinx/_*/ diff --git a/src/aiori-DAOS.c b/src/aiori-DAOS.c index fc339cae..9f4270b6 100644 --- a/src/aiori-DAOS.c +++ b/src/aiori-DAOS.c @@ -292,8 +292,13 @@ DAOS_Fini() if (!daos_initialized) return; + MPI_Barrier(MPI_COMM_WORLD); rc = daos_cont_close(coh, NULL); - DCHECK(rc, "Failed to close container\n"); + if (rc) { + DCHECK(rc, "Failed to close container %s (%d)", o.cont, rc); + MPI_Abort(MPI_COMM_WORLD, -1); + } + MPI_Barrier(MPI_COMM_WORLD); if (o.destroy) { if (rank == 0) { @@ -307,8 +312,7 @@ DAOS_Fini() MPI_Bcast(&rc, 1, MPI_INT, 0, MPI_COMM_WORLD); if (rc) { if (rank == 0) - DCHECK(rc, "Failed to destroy container %s (%d)", - o.cont, rc); + DCHECK(rc, "Failed to destroy container %s (%d)", o.cont, rc); MPI_Abort(MPI_COMM_WORLD, -1); } } diff --git a/src/aiori-DFS.c b/src/aiori-DFS.c index 2acd9bb7..9c4ca251 100755 --- a/src/aiori-DFS.c +++ b/src/aiori-DFS.c @@ -444,13 +444,16 @@ DFS_Finalize() { int rc; + MPI_Barrier(MPI_COMM_WORLD); d_hash_table_destroy(dir_hash, true /* force */); rc = dfs_umount(dfs); DCHECK(rc, "Failed to umount DFS namespace"); + MPI_Barrier(MPI_COMM_WORLD); rc = daos_cont_close(coh, NULL); - DCHECK(rc, "Failed to close container"); + DCHECK(rc, "Failed to close container %s (%d)", o.cont, rc); + MPI_Barrier(MPI_COMM_WORLD); if (rank == 0 && o.destroy) { uuid_t uuid; From b3663bd29a7bb15f60d0714faa898945181c11a6 Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Wed, 14 Aug 2019 13:32:51 +0000 Subject: [PATCH 142/206] add sleep depending on MPI rank to avoid all ranks calling daos_fini() at once (issue with PSM2). Signed-off-by: Mohamad Chaarawi --- src/aiori-DAOS.c | 5 +++++ src/aiori-DFS.c | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/src/aiori-DAOS.c b/src/aiori-DAOS.c index 9f4270b6..870aa684 100644 --- a/src/aiori-DAOS.c +++ b/src/aiori-DAOS.c @@ -17,6 +17,8 @@ * This file implements the abstract I/O interface for DAOS Array API. */ +#define _BSD_SOURCE + #ifdef HAVE_CONFIG_H #include "config.h" #endif @@ -320,6 +322,9 @@ DAOS_Fini() rc = daos_pool_disconnect(poh, NULL); DCHECK(rc, "Failed to disconnect from pool %s", o.pool); + MPI_CHECK(MPI_Barrier(MPI_COMM_WORLD), "barrier error"); + usleep(20000 * rank); + rc = daos_fini(); DCHECK(rc, "Failed to finalize daos"); diff --git a/src/aiori-DFS.c b/src/aiori-DFS.c index 9c4ca251..ef89f412 100755 --- a/src/aiori-DFS.c +++ b/src/aiori-DFS.c @@ -16,6 +16,8 @@ * This file implements the abstract I/O interface for DAOS FS API. */ +#define _BSD_SOURCE + #ifdef HAVE_CONFIG_H #include "config.h" #endif @@ -471,6 +473,9 @@ DFS_Finalize() daos_pool_disconnect(poh, NULL); DCHECK(rc, "Failed to disconnect from pool"); + MPI_CHECK(MPI_Barrier(MPI_COMM_WORLD), "barrier error"); + usleep(20000 * rank); + rc = daos_fini(); DCHECK(rc, "Failed to finalize DAOS"); } From de3baf886107f2bff560dbaf3f199dc6be59e2bd Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Thu, 15 Aug 2019 16:21:30 +0100 Subject: [PATCH 143/206] MDTest: Document choice of 42. --- src/mdtest.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mdtest.c b/src/mdtest.c index 6b5ab462..44a210c2 100644 --- a/src/mdtest.c +++ b/src/mdtest.c @@ -700,7 +700,7 @@ void mdtest_read(int random, int dirs, const long dir_iter, char *path) { /* read file */ if (read_bytes > 0) { - read_buffer[0] = 42; + read_buffer[0] = 42; /* use a random value to ensure that the read_buffer is now different from the expected buffer and read isn't sometimes NOOP */ if (read_bytes != (size_t) backend->xfer (READ, aiori_fh, (IOR_size_t *) read_buffer, read_bytes, ¶m)) { FAIL("unable to read file %s", item); } From 0d9f46e980054b9c149448a1203a8196fd8b41ad Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Thu, 15 Aug 2019 16:49:46 +0100 Subject: [PATCH 144/206] MDTest re-added the -Z option for compatibility (for now) and switched back behavior. --- src/mdtest.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/mdtest.c b/src/mdtest.c index 44a210c2..77b27596 100644 --- a/src/mdtest.c +++ b/src/mdtest.c @@ -1238,8 +1238,11 @@ void summarize_results(int iterations, int print_time) { MPI_Barrier(testComm); for(int i=0; i < iterations; i++){ - MPI_Gather(& summary_table[i].time[0], tableSize, MPI_DOUBLE, & all[i*tableSize*size], tableSize, MPI_DOUBLE, 0, testComm); - MPI_Gather(& summary_table[i].rate[0], tableSize, MPI_DOUBLE, & all[i*tableSize*size], tableSize, MPI_DOUBLE, 0, testComm); + if(print_time){ + MPI_Gather(& summary_table[i].time[0], tableSize, MPI_DOUBLE, & all[i*tableSize*size], tableSize, MPI_DOUBLE, 0, testComm); + }else{ + MPI_Gather(& summary_table[i].rate[0], tableSize, MPI_DOUBLE, & all[i*tableSize*size], tableSize, MPI_DOUBLE, 0, testComm); + } } if (rank != 0) { @@ -1923,6 +1926,7 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * {'X', "verify-read", "Verify the data read", OPTION_FLAG, 'd', & verify_read}, {'y', NULL, "sync file after writing", OPTION_FLAG, 'd', & sync_file}, {'z', NULL, "depth of hierarchical directory structure", OPTION_OPTIONAL_ARGUMENT, 'd', & depth}, + {'Z', NULL, "print time instead of rate", OPTION_FLAG, 'd', & print_time}, LAST_OPTION }; options_all_t * global_options = airoi_create_all_module_options(options); From a4068be551c38e327fb9b054a91cbeb778a9bd76 Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Mon, 26 Aug 2019 15:57:13 +0100 Subject: [PATCH 145/206] Improved help for fsync. --- src/parse_options.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/parse_options.c b/src/parse_options.c index af30c36e..47f9920b 100755 --- a/src/parse_options.c +++ b/src/parse_options.c @@ -477,7 +477,7 @@ option_help * createGlobalOptions(IOR_param_t * params){ {.help=" -O stoneWallingWearOut=1 -- once the stonewalling timout is over, all process finish to access the amount of data", .arg = OPTION_OPTIONAL_ARGUMENT}, {.help=" -O stoneWallingWearOutIterations=N -- stop after processing this number of iterations, needed for reading data back written with stoneWallingWearOut", .arg = OPTION_OPTIONAL_ARGUMENT}, {.help=" -O stoneWallingStatusFile=FILE -- this file keeps the number of iterations from stonewalling during write and allows to use them for read", .arg = OPTION_OPTIONAL_ARGUMENT}, - {'e', NULL, "fsync -- perform sync operation after each block write", OPTION_FLAG, 'd', & params->fsync}, + {'e', NULL, "fsync -- perform a fsync() operation at the end of each read/write phase", OPTION_FLAG, 'd', & params->fsync}, {'E', NULL, "useExistingTestFile -- do not remove test file before write access", OPTION_FLAG, 'd', & params->useExistingTestFile}, {'f', NULL, "scriptFile -- test script name", OPTION_OPTIONAL_ARGUMENT, 's', & params->testscripts}, {'F', NULL, "filePerProc -- file-per-process", OPTION_FLAG, 'd', & params->filePerProc}, From 4df051bf283675faefcb8523dc034b7f817419c6 Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Mon, 26 Aug 2019 18:57:14 +0100 Subject: [PATCH 146/206] New option -Y to invoke the sync command. --- src/mdtest.c | 47 ++++++++++++++++++++++------------------------- src/utilities.c | 17 ++++++++++++----- src/utilities.h | 1 + 3 files changed, 35 insertions(+), 30 deletions(-) diff --git a/src/mdtest.c b/src/mdtest.c index 77b27596..e07613b3 100644 --- a/src/mdtest.c +++ b/src/mdtest.c @@ -148,6 +148,7 @@ static size_t write_bytes; static int stone_wall_timer_seconds; static size_t read_bytes; static int sync_file; +static int call_sync; static int path_count; static int nstride; /* neighbor stride */ static int make_node = 0; @@ -263,6 +264,16 @@ static void prep_testdir(int j, int dir_iter){ pos += sprintf(& testdir[pos], ".%d-%d", j, dir_iter); } +static void phase_end(){ + if (call_sync){ + call_sync_cmd(); + } + + if (barriers) { + MPI_Barrier(testComm); + } +} + /* * This function copies the unique directory name for a given option to * the "to" parameter. Some memory must be allocated to the "to" parameter. @@ -836,9 +847,7 @@ void directory_test(const int iteration, const int ntasks, const char *path, ran } } - if (barriers) { - MPI_Barrier(testComm); - } + phase_end(); t[1] = GetTimeStamp(); /* stat phase */ @@ -864,10 +873,7 @@ void directory_test(const int iteration, const int ntasks, const char *path, ran } } } - - if (barriers) { - MPI_Barrier(testComm); - } + phase_end(); t[2] = GetTimeStamp(); /* read phase */ @@ -894,9 +900,7 @@ void directory_test(const int iteration, const int ntasks, const char *path, ran } } - if (barriers) { - MPI_Barrier(testComm); - } + phase_end(); t[3] = GetTimeStamp(); if (remove_only) { @@ -924,9 +928,7 @@ void directory_test(const int iteration, const int ntasks, const char *path, ran } } - if (barriers) { - MPI_Barrier(testComm); - } + phase_end(); t[4] = GetTimeStamp(); if (remove_only) { @@ -1082,9 +1084,7 @@ void file_test(const int iteration, const int ntasks, const char *path, rank_pro } } - if (barriers) { - MPI_Barrier(testComm); - } + phase_end(); t[1] = GetTimeStamp(); /* stat phase */ @@ -1107,9 +1107,7 @@ void file_test(const int iteration, const int ntasks, const char *path, rank_pro } } - if (barriers) { - MPI_Barrier(testComm); - } + phase_end(); t[2] = GetTimeStamp(); /* read phase */ @@ -1136,9 +1134,7 @@ void file_test(const int iteration, const int ntasks, const char *path, rank_pro } } - if (barriers) { - MPI_Barrier(testComm); - } + phase_end(); t[3] = GetTimeStamp(); if (remove_only) { @@ -1168,9 +1164,7 @@ void file_test(const int iteration, const int ntasks, const char *path, rank_pro } } - if (barriers) { - MPI_Barrier(testComm); - } + phase_end(); t[4] = GetTimeStamp(); if (remove_only) { if (unique_dir_per_task) { @@ -1853,6 +1847,7 @@ void mdtest_init_args(){ stone_wall_timer_seconds = 0; read_bytes = 0; sync_file = 0; + call_sync = 0; path_count = 0; nstride = 0; make_node = 0; @@ -1925,6 +1920,7 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * {'x', NULL, "StoneWallingStatusFile; contains the number of iterations of the creation phase, can be used to split phases across runs", OPTION_OPTIONAL_ARGUMENT, 's', & stoneWallingStatusFile}, {'X', "verify-read", "Verify the data read", OPTION_FLAG, 'd', & verify_read}, {'y', NULL, "sync file after writing", OPTION_FLAG, 'd', & sync_file}, + {'Y', NULL, "call the sync command after each phase (included in the timing; note it causes all IO to be flushed from your node)", OPTION_FLAG, 'd', & call_sync}, {'z', NULL, "depth of hierarchical directory structure", OPTION_OPTIONAL_ARGUMENT, 'd', & depth}, {'Z', NULL, "print time instead of rate", OPTION_FLAG, 'd', & print_time}, LAST_OPTION @@ -2008,6 +2004,7 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * VERBOSE(1,-1, "unique_dir_per_task : %s", ( unique_dir_per_task ? "True" : "False" )); VERBOSE(1,-1, "write_bytes : "LLU"", write_bytes ); VERBOSE(1,-1, "sync_file : %s", ( sync_file ? "True" : "False" )); + VERBOSE(1,-1, "call_sync : %s", ( call_sync ? "True" : "False" )); VERBOSE(1,-1, "depth : %d", depth ); VERBOSE(1,-1, "make_node : %d", make_node ); diff --git a/src/utilities.c b/src/utilities.c index cdb090ea..a2e4b0dc 100755 --- a/src/utilities.c +++ b/src/utilities.c @@ -77,15 +77,15 @@ void* safeMalloc(uint64_t size){ } void FailMessage(int rank, const char *location, char *format, ...) { - char msg[4096]; + char msg[4096]; va_list args; va_start(args, format); vsnprintf(msg, 4096, format, args); va_end(args); - fprintf(out_logfile, "%s: Process %d: FAILED in %s, %s: %s\n", - PrintTimestamp(), rank, location, msg, strerror(errno)); - fflush(out_logfile); - MPI_Abort(testComm, 1); + fprintf(out_logfile, "%s: Process %d: FAILED in %s, %s: %s\n", + PrintTimestamp(), rank, location, msg, strerror(errno)); + fflush(out_logfile); + MPI_Abort(testComm, 1); } size_t NodeMemoryStringToBytes(char *size_str) @@ -809,3 +809,10 @@ char *HumanReadable(IOR_offset_t value, int base) } return valueStr; } + +void call_sync_cmd(){ + int ret = system("sync"); + if (ret != 0){ + FAIL("Error executing the sync command, ensure it exists."); + } +} diff --git a/src/utilities.h b/src/utilities.h index d2c9962a..b85f9579 100755 --- a/src/utilities.h +++ b/src/utilities.h @@ -60,6 +60,7 @@ int QueryNodeMapping(MPI_Comm comm, int print_nodemap); void DelaySecs(int delay); void updateParsedOptions(IOR_param_t * options, options_all_t * global_options); size_t NodeMemoryStringToBytes(char *size_str); +void call_sync_cmd(); /* Returns -1, if cannot be read */ int64_t ReadStoneWallingIterations(char * const filename); From 93730771fd599fde7b57fb78ad346d655602cb6a Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Wed, 28 Aug 2019 17:01:51 +0000 Subject: [PATCH 147/206] add some verbose messages on finalize routines for DAOS and DFS drivers. Signed-off-by: Mohamad Chaarawi --- README_DAOS | 2 +- src/aiori-DAOS.c | 8 +++++++- src/aiori-DFS.c | 12 +++++++++--- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/README_DAOS b/README_DAOS index 33d94648..9467989a 100644 --- a/README_DAOS +++ b/README_DAOS @@ -3,7 +3,7 @@ Building The DAOS library must be installed on the system. -./bootsrap +./bootstrap ./configure --prefix=iorInstallDir --with-daos=DIR --with-cart=DIR One must specify "--with-daos=/path/to/daos/install and --with-cart". When that diff --git a/src/aiori-DAOS.c b/src/aiori-DAOS.c index 870aa684..16d386ae 100644 --- a/src/aiori-DAOS.c +++ b/src/aiori-DAOS.c @@ -306,7 +306,7 @@ DAOS_Fini() if (rank == 0) { uuid_t uuid; - INFO(VERBOSE_1, "Destroying Container %s", o.cont); + INFO(VERBOSE_1, "Destroying DAOS Container %s", o.cont); uuid_parse(o.cont, uuid); rc = daos_cont_destroy(poh, uuid, 1, NULL); } @@ -319,12 +319,18 @@ DAOS_Fini() } } + if (rank == 0) + INFO(VERBOSE_1, "Disconnecting from DAOS POOL.."); + rc = daos_pool_disconnect(poh, NULL); DCHECK(rc, "Failed to disconnect from pool %s", o.pool); MPI_CHECK(MPI_Barrier(MPI_COMM_WORLD), "barrier error"); usleep(20000 * rank); + if (rank == 0) + INFO(VERBOSE_1, "Finalizing DAOS.."); + rc = daos_fini(); DCHECK(rc, "Failed to finalize daos"); diff --git a/src/aiori-DFS.c b/src/aiori-DFS.c index ef89f412..17a344db 100755 --- a/src/aiori-DFS.c +++ b/src/aiori-DFS.c @@ -140,7 +140,7 @@ ior_aiori_t dfs_aiori = { do { \ int _rc = (rc); \ \ - if (_rc < 0) { \ + if (_rc != 0) { \ fprintf(stderr, "ERROR (%s:%d): %d: %d: " \ format"\n", __FILE__, __LINE__, rank, _rc, \ ##__VA_ARGS__); \ @@ -153,7 +153,7 @@ do { \ do { \ int _rc = (rc); \ \ - if (_rc < 0) { \ + if (_rc != 0) { \ fprintf(stderr, "ERROR (%s:%d): %d: %d: " \ format"\n", __FILE__, __LINE__, rank, _rc, \ ##__VA_ARGS__); \ @@ -470,12 +470,18 @@ DFS_Finalize() if (rc) DCHECK(rc, "Failed to destroy container %s (%d)", o.cont, rc); - daos_pool_disconnect(poh, NULL); + if (rank == 0 && verbose >= VERBOSE_1) + printf("Disconnecting from DAOS POOL\n"); + + rc = daos_pool_disconnect(poh, NULL); DCHECK(rc, "Failed to disconnect from pool"); MPI_CHECK(MPI_Barrier(MPI_COMM_WORLD), "barrier error"); usleep(20000 * rank); + if (rank == 0 && verbose >= VERBOSE_1) + printf("Finalizing DAOS..\n"); + rc = daos_fini(); DCHECK(rc, "Failed to finalize DAOS"); } From 32db1cd9029df0b69fa8f480d83ef412223dd06c Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Thu, 29 Aug 2019 03:24:48 +0000 Subject: [PATCH 148/206] add timing for container close. Signed-off-by: Mohamad Chaarawi --- src/aiori-DAOS.c | 5 +++++ src/aiori-DFS.c | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/src/aiori-DAOS.c b/src/aiori-DAOS.c index 16d386ae..18dd689e 100644 --- a/src/aiori-DAOS.c +++ b/src/aiori-DAOS.c @@ -305,10 +305,15 @@ DAOS_Fini() if (o.destroy) { if (rank == 0) { uuid_t uuid; + double t1, t2; INFO(VERBOSE_1, "Destroying DAOS Container %s", o.cont); uuid_parse(o.cont, uuid); + t1 = MPI_Wtime(); rc = daos_cont_destroy(poh, uuid, 1, NULL); + t2 = MPI_Wtime(); + if (rc == 0) + INFO(VERBOSE_1, "Container Destroy time = %f secs", t2-t1); } MPI_Bcast(&rc, 1, MPI_INT, 0, MPI_COMM_WORLD); diff --git a/src/aiori-DFS.c b/src/aiori-DFS.c index 17a344db..df956e84 100755 --- a/src/aiori-DFS.c +++ b/src/aiori-DFS.c @@ -459,11 +459,16 @@ DFS_Finalize() if (rank == 0 && o.destroy) { uuid_t uuid; + double t1, t2; if (verbose >= VERBOSE_1) printf("Destorying DFS Container: %s\n", o.cont); uuid_parse(o.cont, uuid); + t1 = MPI_Wtime(); rc = daos_cont_destroy(poh, uuid, 1, NULL); + t2 = MPI_Wtime(); + if (rc == 0 && verbose >= VERBOSE_1) + printf("Container Destroy time = %f secs", t2-t1); } MPI_Bcast(&rc, 1, MPI_INT, 0, MPI_COMM_WORLD); From 0b809b36e292267c0c22bf7077769641e0b7c1fd Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Fri, 30 Aug 2019 20:35:15 +0000 Subject: [PATCH 149/206] fix README_DAOS for DFS plugin Signed-off-by: Mohamad Chaarawi --- README_DAOS | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README_DAOS b/README_DAOS index 9467989a..b4e6dba0 100644 --- a/README_DAOS +++ b/README_DAOS @@ -78,6 +78,6 @@ Examples that should work include: Running mdtest, the user needs to specify a directory with -d where the test tree will be created. Some examples: - - "mdtest -a DFS -n 100 -F -D -d /bla --daos.pool --daos.svcl --daos.cont " - - "mdtest -a DFS -n 1000 -F -C -d /bla --daos.pool --daos.svcl --daos.cont " - - "mdtest -a DFS -I 10 -z 5 -b 2 -L -d /bla --daos.pool --daos.svcl --daos.cont " + - "mdtest -a DFS -n 100 -F -D -d /bla --dfs.pool --dfs.svcl --dfs.cont " + - "mdtest -a DFS -n 1000 -F -C -d /bla --dfs.pool --dfs.svcl --dfs.cont " + - "mdtest -a DFS -I 10 -z 5 -b 2 -L -d /bla --dfs.pool --dfs.svcl --dfs.cont " From 0bd979637e8b8c340fb7475897c84aa8172d7117 Mon Sep 17 00:00:00 2001 From: Josh Schwartz <52082483+jschwartz-cray@users.noreply.github.com> Date: Fri, 30 Aug 2019 15:11:25 -0600 Subject: [PATCH 150/206] Added some extra debug including ERRF, WARNF, and MPI_CHECKF format string macros. --- src/aiori-MPIIO.c | 14 ++++++------ src/aiori-POSIX.c | 54 +++++++++++++++++++++++++---------------------- src/ior.c | 17 +++++++++------ src/iordef.h | 52 +++++++++++++++++++++++++++++++++------------ 4 files changed, 85 insertions(+), 52 deletions(-) diff --git a/src/aiori-MPIIO.c b/src/aiori-MPIIO.c index 6bf991bd..707b3a78 100755 --- a/src/aiori-MPIIO.c +++ b/src/aiori-MPIIO.c @@ -178,8 +178,8 @@ static void *MPIIO_Open(char *testFileName, IOR_param_t * param) fprintf(stdout, "}\n"); } if(! param->dryRun){ - MPI_CHECK(MPI_File_open(comm, testFileName, fd_mode, mpiHints, fd), - "cannot open file"); + MPI_CHECKF(MPI_File_open(comm, testFileName, fd_mode, mpiHints, fd), + "cannot open file: %s", testFileName); } /* show hints actually attached to file handle */ @@ -428,8 +428,8 @@ void MPIIO_Delete(char *testFileName, IOR_param_t * param) { if(param->dryRun) return; - MPI_CHECK(MPI_File_delete(testFileName, (MPI_Info) MPI_INFO_NULL), - "cannot delete file"); + MPI_CHECKF(MPI_File_delete(testFileName, (MPI_Info) MPI_INFO_NULL), + "cannot delete file: %s", testFileName); } /* @@ -504,9 +504,9 @@ IOR_offset_t MPIIO_GetFileSize(IOR_param_t * test, MPI_Comm testComm, comm = testComm; } - MPI_CHECK(MPI_File_open(comm, testFileName, MPI_MODE_RDONLY, - MPI_INFO_NULL, &fd), - "cannot open file to get file size"); + MPI_CHECKF(MPI_File_open(comm, testFileName, MPI_MODE_RDONLY, + MPI_INFO_NULL, &fd), + "cannot open file to get file size: %s", testFileName); MPI_CHECK(MPI_File_get_size(fd, (MPI_Offset *) & aggFileSizeFromStat), "cannot get file size"); MPI_CHECK(MPI_File_close(&fd), "cannot close file"); diff --git a/src/aiori-POSIX.c b/src/aiori-POSIX.c index d78cd40a..00e974c4 100755 --- a/src/aiori-POSIX.c +++ b/src/aiori-POSIX.c @@ -146,7 +146,7 @@ void gpfs_free_all_locks(int fd) rc = gpfs_fcntl(fd, &release_all); if (verbose >= VERBOSE_0 && rc != 0) { - EWARN("gpfs_fcntl release all locks hint failed."); + EWARNF("gpfs_fcntl(%d, ...) release all locks hint failed.", fd); } } void gpfs_access_start(int fd, IOR_offset_t length, IOR_param_t *param, int access) @@ -169,7 +169,7 @@ void gpfs_access_start(int fd, IOR_offset_t length, IOR_param_t *param, int acce rc = gpfs_fcntl(fd, &take_locks); if (verbose >= VERBOSE_2 && rc != 0) { - EWARN("gpfs_fcntl access range hint failed."); + EWARNF("gpfs_fcntl(fd, ...) access range hint failed.", fd); } } @@ -193,7 +193,7 @@ void gpfs_access_end(int fd, IOR_offset_t length, IOR_param_t *param, int access rc = gpfs_fcntl(fd, &free_locks); if (verbose >= VERBOSE_2 && rc != 0) { - EWARN("gpfs_fcntl free range hint failed."); + EWARNF("gpfs_fcntl(fd, ...) free range hint failed.", fd); } } @@ -260,14 +260,14 @@ bool beegfs_createFilePath(char* filepath, mode_t mode, int numTargets, int chun char* dir = dirname(dirTmp); DIR* parentDirS = opendir(dir); if (!parentDirS) { - ERR("Failed to get directory"); + ERRF("Failed to get directory: %s", dir); } else { int parentDirFd = dirfd(parentDirS); if (parentDirFd < 0) { - ERR("Failed to get directory descriptor"); + ERRF("Failed to get directory descriptor: %s", dir); } else { @@ -319,6 +319,7 @@ bool beegfs_createFilePath(char* filepath, mode_t mode, int numTargets, int chun void *POSIX_Create(char *testFileName, IOR_param_t * param) { int fd_oflag = O_BINARY; + int mode = 0664; int *fd; fd = (int *)malloc(sizeof(int)); @@ -346,9 +347,10 @@ void *POSIX_Create(char *testFileName, IOR_param_t * param) if (!param->filePerProc && rank != 0) { MPI_CHECK(MPI_Barrier(testComm), "barrier error"); fd_oflag |= O_RDWR; - *fd = open64(testFileName, fd_oflag, 0664); + *fd = open64(testFileName, fd_oflag, mode); if (*fd < 0) - ERR("open64() failed"); + ERRF("open64(\"%s\", %d, %#o) failed", + testFileName, fd_oflag, mode); } else { struct lov_user_md opts = { 0 }; @@ -363,7 +365,7 @@ void *POSIX_Create(char *testFileName, IOR_param_t * param) fd_oflag |= O_CREAT | O_EXCL | O_RDWR | O_LOV_DELAY_CREATE; - *fd = open64(testFileName, fd_oflag, 0664); + *fd = open64(testFileName, fd_oflag, mode); if (*fd < 0) { fprintf(stdout, "\nUnable to open '%s': %s\n", testFileName, strerror(errno)); @@ -392,7 +394,7 @@ void *POSIX_Create(char *testFileName, IOR_param_t * param) if (beegfs_isOptionSet(param->beegfs_chunkSize) || beegfs_isOptionSet(param->beegfs_numTargets)) { bool result = beegfs_createFilePath(testFileName, - 0664, + mode, param->beegfs_numTargets, param->beegfs_chunkSize); if (result) { @@ -403,9 +405,10 @@ void *POSIX_Create(char *testFileName, IOR_param_t * param) } #endif /* HAVE_BEEGFS_BEEGFS_H */ - *fd = open64(testFileName, fd_oflag, 0664); + *fd = open64(testFileName, fd_oflag, mode); if (*fd < 0) - ERR("open64() failed"); + ERRF("open64(\"%s\", %d, %#o) failed", + testFileName, fd_oflag, mode); #ifdef HAVE_LUSTRE_LUSTRE_USER_H } @@ -413,7 +416,7 @@ void *POSIX_Create(char *testFileName, IOR_param_t * param) if (param->lustre_ignore_locks) { int lustre_ioctl_flags = LL_FILE_IGNORE_LOCK; if (ioctl(*fd, LL_IOC_SETFLAGS, &lustre_ioctl_flags) == -1) - ERR("ioctl(LL_IOC_SETFLAGS) failed"); + ERRF("ioctl(%d, LL_IOC_SETFLAGS, ...) failed", *fd); } #endif /* HAVE_LUSTRE_LUSTRE_USER_H */ @@ -469,7 +472,7 @@ void *POSIX_Open(char *testFileName, IOR_param_t * param) *fd = open64(testFileName, fd_oflag); if (*fd < 0) - ERR("open64 failed"); + ERRF("open64(\"%s\", %d) failed", testFileName, fd_oflag); #ifdef HAVE_LUSTRE_LUSTRE_USER_H if (param->lustre_ignore_locks) { @@ -479,7 +482,7 @@ void *POSIX_Open(char *testFileName, IOR_param_t * param) "** Disabling lustre range locking **\n"); } if (ioctl(*fd, LL_IOC_SETFLAGS, &lustre_ioctl_flags) == -1) - ERR("ioctl(LL_IOC_SETFLAGS) failed"); + ERRF("ioctl(%d, LL_IOC_SETFLAGS, ...) failed", *fd); } #endif /* HAVE_LUSTRE_LUSTRE_USER_H */ @@ -517,7 +520,7 @@ static IOR_offset_t POSIX_Xfer(int access, void *file, IOR_size_t * buffer, /* seek to offset */ if (lseek64(fd, param->offset, SEEK_SET) == -1) - ERR("lseek64() failed"); + ERRF("lseek64(%d, %lld, SEEK_SET) failed", fd, param->offset); while (remaining > 0) { /* write/read file */ @@ -530,7 +533,8 @@ static IOR_offset_t POSIX_Xfer(int access, void *file, IOR_size_t * buffer, } rc = write(fd, ptr, remaining); if (rc == -1) - ERR("write() failed"); + ERRF("write(%d, %p, %lld) failed", + fd, (void*)ptr, remaining); if (param->fsyncPerWrite == TRUE) POSIX_Fsync(&fd, param); } else { /* READ or CHECK */ @@ -542,9 +546,11 @@ static IOR_offset_t POSIX_Xfer(int access, void *file, IOR_size_t * buffer, } rc = read(fd, ptr, remaining); if (rc == 0) - ERR("read() returned EOF prematurely"); + ERRF("read(%d, %p, %lld) returned EOF prematurely", + fd, (void*)ptr, remaining); if (rc == -1) - ERR("read() failed"); + ERRF("read(%d, %p, %lld) failed", + fd, (void*)ptr, remaining); } if (rc < remaining) { fprintf(stdout, @@ -579,7 +585,7 @@ static IOR_offset_t POSIX_Xfer(int access, void *file, IOR_size_t * buffer, static void POSIX_Fsync(void *fd, IOR_param_t * param) { if (fsync(*(int *)fd) != 0) - EWARN("fsync() failed"); + EWARNF("fsync(%d) failed", *(int *)fd); } /* @@ -590,7 +596,7 @@ void POSIX_Close(void *fd, IOR_param_t * param) if(param->dryRun) return; if (close(*(int *)fd) != 0) - ERR("close() failed"); + ERRF("close(%d) failed", *(int *)fd); free(fd); } @@ -602,10 +608,8 @@ void POSIX_Delete(char *testFileName, IOR_param_t * param) if(param->dryRun) return; if (unlink(testFileName) != 0){ - char errmsg[256]; - sprintf(errmsg, "[RANK %03d]: unlink() of file \"%s\" failed\n", - rank, testFileName); - EWARN(errmsg); + EWARNF("[RANK %03d]: unlink() of file \"%s\" failed\n", + rank, testFileName); } } @@ -621,7 +625,7 @@ IOR_offset_t POSIX_GetFileSize(IOR_param_t * test, MPI_Comm testComm, IOR_offset_t aggFileSizeFromStat, tmpMin, tmpMax, tmpSum; if (stat(testFileName, &stat_buf) != 0) { - ERR("stat() failed"); + ERRF("stat(\"%s\", ...) failed", testFileName); } aggFileSizeFromStat = stat_buf.st_size; diff --git a/src/ior.c b/src/ior.c index 4cd55718..f6933214 100755 --- a/src/ior.c +++ b/src/ior.c @@ -822,14 +822,14 @@ static char *PrependDir(IOR_param_t * test, char *rootDir) /* dir doesn't exist, so create */ if (backend->access(dir, F_OK, test) != 0) { if (backend->mkdir(dir, S_IRWXU, test) < 0) { - ERR("cannot create directory"); + ERRF("cannot create directory: %s", dir); } /* check if correct permissions */ } else if (backend->access(dir, R_OK, test) != 0 || backend->access(dir, W_OK, test) != 0 || backend->access(dir, X_OK, test) != 0) { - ERR("invalid directory permissions"); + ERRF("invalid directory permissions: %s", dir); } /* concatenate dir and file names */ @@ -900,6 +900,10 @@ static void RemoveFile(char *testFileName, int filePerProc, IOR_param_t * test) GetTestFileName(testFileName, test); } if (backend->access(testFileName, F_OK, test) == 0) { + if (verbose >= VERBOSE_3) { + fprintf(out_logfile, "task %d removing %s\n", rank, + testFileName); + } backend->delete(testFileName, test); } if (test->reorderTasksRandom == TRUE) { @@ -908,6 +912,10 @@ static void RemoveFile(char *testFileName, int filePerProc, IOR_param_t * test) } } else { if ((rank == 0) && (backend->access(testFileName, F_OK, test) == 0)) { + if (verbose >= VERBOSE_3) { + fprintf(out_logfile, "task %d removing %s\n", rank, + testFileName); + } backend->delete(testFileName, test); } } @@ -1667,11 +1675,8 @@ static void ValidateTests(IOR_param_t * test) #if (H5_VERS_MAJOR > 0 && H5_VERS_MINOR > 5) ; #else - char errorString[MAX_STR]; - sprintf(errorString, - "'no fill' option not available in %s", + ERRF("'no fill' option not available in %s", test->apiVersion); - ERR(errorString); #endif #else WARN("unable to determine HDF5 version for 'no fill' usage"); diff --git a/src/iordef.h b/src/iordef.h index 1d1f2162..78cf1d39 100755 --- a/src/iordef.h +++ b/src/iordef.h @@ -151,28 +151,41 @@ typedef long long int IOR_size_t; fflush(stdout); \ } while (0) -/* warning with errno printed */ -#define EWARN(MSG) do { \ + +/* warning with format string and errno printed */ +#define EWARNF(FORMAT, ...) do { \ if (verbose > VERBOSE_2) { \ - fprintf(stdout, "ior WARNING: %s, errno %d, %s (%s:%d).\n", \ - MSG, errno, strerror(errno), __FILE__, __LINE__); \ + fprintf(stdout, "ior WARNING: " FORMAT ", errno %d, %s (%s:%d).\n", \ + __VA_ARGS__, errno, strerror(errno), __FILE__, __LINE__); \ } else { \ - fprintf(stdout, "ior WARNING: %s, errno %d, %s \n", \ - MSG, errno, strerror(errno)); \ + fprintf(stdout, "ior WARNING: " FORMAT ", errno %d, %s \n", \ + __VA_ARGS__, errno, strerror(errno)); \ } \ fflush(stdout); \ } while (0) -/* display error message and terminate execution */ -#define ERR(MSG) do { \ - fprintf(stdout, "ior ERROR: %s, errno %d, %s (%s:%d)\n", \ - MSG, errno, strerror(errno), __FILE__, __LINE__); \ +/* warning with errno printed */ +#define EWARN(MSG) do { \ + EWARNF("%s", MSG); \ +} while (0) + + +/* display error message with format string and terminate execution */ +#define ERRF(FORMAT, ...) do { \ + fprintf(stdout, "ior ERROR: " FORMAT ", errno %d, %s (%s:%d)\n", \ + __VA_ARGS__, errno, strerror(errno), __FILE__, __LINE__); \ fflush(stdout); \ MPI_Abort(MPI_COMM_WORLD, -1); \ } while (0) +/* display error message and terminate execution */ +#define ERR(MSG) do { \ + ERRF("%s", MSG); \ +} while (0) + + /* display a simple error message (i.e. errno is not set) and terminate execution */ #define ERR_SIMPLE(MSG) do { \ fprintf(stdout, "ior ERROR: %s, (%s:%d)\n", \ @@ -184,24 +197,35 @@ typedef long long int IOR_size_t; /******************************************************************************/ /* - * MPI_CHECK will display a custom error message as well as an error string + * MPI_CHECKF will display a custom format string as well as an error string * from the MPI_STATUS and then exit the program */ -#define MPI_CHECK(MPI_STATUS, MSG) do { \ +#define MPI_CHECKF(MPI_STATUS, FORMAT, ...) do { \ char resultString[MPI_MAX_ERROR_STRING]; \ int resultLength; \ \ if (MPI_STATUS != MPI_SUCCESS) { \ MPI_Error_string(MPI_STATUS, resultString, &resultLength); \ - fprintf(stdout, "ior ERROR: %s, MPI %s, (%s:%d)\n", \ - MSG, resultString, __FILE__, __LINE__); \ + fprintf(stdout, "ior ERROR: " FORMAT ", MPI %s, (%s:%d)\n", \ + __VA_ARGS__, resultString, __FILE__, __LINE__); \ fflush(stdout); \ MPI_Abort(MPI_COMM_WORLD, -1); \ } \ } while(0) +/******************************************************************************/ +/* + * MPI_CHECK will display a custom error message as well as an error string + * from the MPI_STATUS and then exit the program + */ + +#define MPI_CHECK(MPI_STATUS, MSG) do { \ + MPI_CHECKF(MPI_STATUS, "%s", MSG); \ +} while(0) + + /******************************************************************************/ /* * System info for Windows. From 4c3d96bfed7f783acd7b2beb7195eee579487257 Mon Sep 17 00:00:00 2001 From: Josh Schwartz <52082483+jschwartz-cray@users.noreply.github.com> Date: Fri, 30 Aug 2019 15:31:23 -0600 Subject: [PATCH 151/206] Fix #179. -u (uniqueDir) will once again use the full file path specified by the client instead of truncating it. This was caused by a broken sprintf which was trying to read/write overlapping buffers. From the glibc sprintf() documentation: "The behavior of this function is undefined if copying takes place between objects that overlap" --- src/ior.c | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/src/ior.c b/src/ior.c index 4cd55718..e1738079 100755 --- a/src/ior.c +++ b/src/ior.c @@ -785,8 +785,7 @@ void GetTestFileName(char *testFileName, IOR_param_t * test) static char *PrependDir(IOR_param_t * test, char *rootDir) { char *dir; - char fname[MAX_STR + 1]; - char *p; + char *fname; int i; dir = (char *)malloc(MAX_STR + 1); @@ -806,18 +805,10 @@ static char *PrependDir(IOR_param_t * test, char *rootDir) } /* get file name */ - strcpy(fname, rootDir); - p = fname; - while (i > 0) { - if (fname[i] == '\0' || fname[i] == '/') { - p = fname + (i + 1); - break; - } - i--; - } + fname = rootDir + i + 1; /* create directory with rank as subdirectory */ - sprintf(dir, "%s%d", dir, (rank + rankOffset) % test->numTasks); + sprintf(dir + i + 1, "%d", (rank + rankOffset) % test->numTasks); /* dir doesn't exist, so create */ if (backend->access(dir, F_OK, test) != 0) { @@ -834,7 +825,7 @@ static char *PrependDir(IOR_param_t * test, char *rootDir) /* concatenate dir and file names */ strcat(dir, "/"); - strcat(dir, p); + strcat(dir, fname); return dir; } From 0e952f0f8cc56aad2967d61e3d8777131fa2a040 Mon Sep 17 00:00:00 2001 From: Josh Schwartz <52082483+jschwartz-cray@users.noreply.github.com> Date: Fri, 30 Aug 2019 16:45:03 -0600 Subject: [PATCH 152/206] Fix #181. On systems where numTasks is not evenly divisible by 'tasksPerNode' we were seeing some nodes reading multiple files while others read none after reordering. Commonly all nodes have the same number of tasks but there is nothing requiring that to be the case. Imagine having 64 tasks running against 4 nodes which can run 20 tasks each. Here you get three groups of 20 and one group of 4. On this sytem nodes running in the group of 4 were previously getting tasksPerNode of 4 which meant they reordered tasks differently than the nodes which got tasksPerNode of 20. The key to fixing this is ensuring that every node reorders tasks the same way, which means ensuring they all use the same input values. Obviously on systems where the number of tasks per node is inconsistent the reordering will also be inconsistent (some tasks may end up on the same node, or not as far separated as desired, etc.) but at least this way you'll always end up with a 1:1 reordering. - Renamed nodes/nodeCount to numNodes - Renamed tasksPerNode to numTasksOnNode0 - Ensured that numTasksOnNode0 will always have the same value regardless of which node you're on - Removed inconsistently used globals numTasksWorld and tasksPerNode and replaced with per-test params equivalents - Added utility functions for setting these values: - numNodes -> GetNumNodes - numTasks -> GetNumTasks - numTasksOnNode0 -> GetNumNodesOnTask0 - Improved MPI_VERSION < 3 logic for GetNumNodes so it works when numTasks is not evenly divisible by numTasksOnNode0 - Left 'nodes' and 'tasksPerNode' in output alone to not break compatibility - Allowed command-line params to override numTasks, numNodes, and numTasksOnNode0 but default to using the MPI-calculated values --- NEWS | 2 +- src/aiori-NCMPI.c | 4 +- src/ior-output.c | 11 +++-- src/ior.c | 79 ++++++++++++++++++-------------- src/ior.h | 4 +- src/mdtest.c | 13 +++--- src/parse_options.c | 12 +++-- src/utilities.c | 109 ++++++++++++++++++++++++++++++++++++-------- src/utilities.h | 6 +-- 9 files changed, 164 insertions(+), 76 deletions(-) diff --git a/NEWS b/NEWS index 1f2cc74d..93671124 100644 --- a/NEWS +++ b/NEWS @@ -120,7 +120,7 @@ Version 2.10.1 - Corrected IOR_GetFileSize() function to point to HDF5 and NCMPI versions of IOR_GetFileSize() calls - Changed the netcdf dataset from 1D array to 4D array, where the 4 dimensions - are: [segmentCount][numTasksWorld][numTransfers][transferSize] + are: [segmentCount][numTasks][numTransfers][transferSize] This patch from Wei-keng Liao allows for file sizes > 4GB (provided no single dimension is > 4GB). - Finalized random-capability release diff --git a/src/aiori-NCMPI.c b/src/aiori-NCMPI.c index 36074669..5fc13751 100755 --- a/src/aiori-NCMPI.c +++ b/src/aiori-NCMPI.c @@ -216,7 +216,7 @@ static IOR_offset_t NCMPI_Xfer(int access, void *fd, IOR_size_t * buffer, param->blockSize / param->transferSize; /* reshape 1D array to 3D array: - [segmentCount*numTasksWorld][numTransfers][transferSize] + [segmentCount*numTasks][numTransfers][transferSize] Requirement: none of these dimensions should be > 4G, */ NCMPI_CHECK(ncmpi_def_dim @@ -267,7 +267,7 @@ static IOR_offset_t NCMPI_Xfer(int access, void *fd, IOR_size_t * buffer, bufSize[1] = 1; bufSize[2] = param->transferSize; - offset[0] = segmentNum * numTasksWorld + rank; + offset[0] = segmentNum * param->numTasks + rank; offset[1] = transferNum; offset[2] = 0; diff --git a/src/ior-output.c b/src/ior-output.c index 7049a971..26600a37 100644 --- a/src/ior-output.c +++ b/src/ior-output.c @@ -339,10 +339,10 @@ void ShowTestStart(IOR_param_t *test) PrintKeyVal("options", test->options); PrintKeyValInt("dryRun", test->dryRun); - PrintKeyValInt("nodes", test->nodes); + PrintKeyValInt("nodes", test->numNodes); PrintKeyValInt("memoryPerTask", (unsigned long) test->memoryPerTask); PrintKeyValInt("memoryPerNode", (unsigned long) test->memoryPerNode); - PrintKeyValInt("tasksPerNode", tasksPerNode); + PrintKeyValInt("tasksPerNode", test->numTasksOnNode0); PrintKeyValInt("repetitions", test->repetitions); PrintKeyValInt("multiFile", test->multiFile); PrintKeyValInt("interTestDelay", test->interTestDelay); @@ -430,8 +430,9 @@ void ShowSetup(IOR_param_t *params) PrintKeyValInt("task offset", params->taskPerNodeOffset); PrintKeyValInt("reorder random seed", params->reorderTasksRandomSeed); } + PrintKeyValInt("nodes", params->numNodes); PrintKeyValInt("tasks", params->numTasks); - PrintKeyValInt("clients per node", params->tasksPerNode); + PrintKeyValInt("clients per node", params->numTasksOnNode0); if (params->memoryPerTask != 0){ PrintKeyVal("memoryPerTask", HumanReadable(params->memoryPerTask, BASE_TWO)); } @@ -571,7 +572,7 @@ static void PrintLongSummaryOneOperation(IOR_test_t *test, const int access) } fprintf(out_resultfile, "%5d ", params->id); fprintf(out_resultfile, "%6d ", params->numTasks); - fprintf(out_resultfile, "%3d ", params->tasksPerNode); + fprintf(out_resultfile, "%3d ", params->numTasksOnNode0); fprintf(out_resultfile, "%4d ", params->repetitions); fprintf(out_resultfile, "%3d ", params->filePerProc); fprintf(out_resultfile, "%5d ", params->reorderTasks); @@ -595,7 +596,7 @@ static void PrintLongSummaryOneOperation(IOR_test_t *test, const int access) PrintKeyValInt("blockSize", params->blockSize); PrintKeyValInt("transferSize", params->transferSize); PrintKeyValInt("numTasks", params->numTasks); - PrintKeyValInt("tasksPerNode", params->tasksPerNode); + PrintKeyValInt("tasksPerNode", params->numTasksOnNode0); PrintKeyValInt("repetitions", params->repetitions); PrintKeyValInt("filePerProc", params->filePerProc); PrintKeyValInt("reorderTasks", params->reorderTasks); diff --git a/src/ior.c b/src/ior.c index 4cd55718..b694c518 100755 --- a/src/ior.c +++ b/src/ior.c @@ -65,7 +65,6 @@ IOR_test_t * ior_run(int argc, char **argv, MPI_Comm world_com, FILE * world_out out_resultfile = world_out; mpi_comm_world = world_com; - MPI_CHECK(MPI_Comm_size(mpi_comm_world, &numTasksWorld), "cannot get number of tasks"); MPI_CHECK(MPI_Comm_rank(mpi_comm_world, &rank), "cannot get rank"); /* setup tests, and validate parameters */ @@ -113,8 +112,6 @@ int ior_main(int argc, char **argv) MPI_CHECK(MPI_Init(&argc, &argv), "cannot initialize MPI"); mpi_comm_world = MPI_COMM_WORLD; - MPI_CHECK(MPI_Comm_size(mpi_comm_world, &numTasksWorld), - "cannot get number of tasks"); MPI_CHECK(MPI_Comm_rank(mpi_comm_world, &rank), "cannot get rank"); /* set error-handling */ @@ -188,8 +185,14 @@ void init_IOR_Param_t(IOR_param_t * p) p->writeFile = p->readFile = FALSE; p->checkWrite = p->checkRead = FALSE; - p->nodes = 1; - p->tasksPerNode = 1; + /* + * These can be overridden from the command-line but otherwise will be + * set from MPI. + */ + p->numTasks = -1; + p->numNodes = -1; + p->numTasksOnNode0 = -1; + p->repetitions = 1; p->repCounter = -1; p->open = WRITE; @@ -919,12 +922,17 @@ static void RemoveFile(char *testFileName, int filePerProc, IOR_param_t * test) */ static void InitTests(IOR_test_t *tests, MPI_Comm com) { - int size; + int mpiNumNodes = 0; + int mpiNumTasks = 0; + int mpiNumTasksOnNode0 = 0; - MPI_CHECK(MPI_Comm_size(com, & size), "MPI_Comm_size() error"); - - /* count the tasks per node */ - tasksPerNode = CountTasksPerNode(com); + /* + * These default values are the same for every test and expensive to + * retrieve so just do it once. + */ + mpiNumNodes = GetNumNodes(com); + mpiNumTasks = GetNumTasks(com); + mpiNumTasksOnNode0 = GetNumTasksOnNode0(com); /* * Since there is no guarantee that anyone other than @@ -937,12 +945,28 @@ static void InitTests(IOR_test_t *tests, MPI_Comm com) while (tests != NULL) { IOR_param_t *params = & tests->params; params->testComm = com; - params->nodes = params->numTasks / tasksPerNode; - params->tasksPerNode = tasksPerNode; - params->tasksBlockMapping = QueryNodeMapping(com,false); - if (params->numTasks == 0) { - params->numTasks = size; + + /* use MPI values if not overridden on command-line */ + if (params->numNodes == -1) { + params->numNodes = mpiNumNodes; + } + if (params->numTasks == -1) { + params->numTasks = mpiNumTasks; + } else if (params->numTasks > mpiNumTasks) { + if (rank == 0) { + fprintf(out_logfile, + "WARNING: More tasks requested (%d) than available (%d),", + params->numTasks, mpiNumTasks); + fprintf(out_logfile, " running with %d tasks.\n", + mpiNumTasks); + } + params->numTasks = mpiNumTasks; + } + if (params->numTasksOnNode0 == -1) { + params->numTasksOnNode0 = mpiNumTasksOnNode0; } + + params->tasksBlockMapping = QueryNodeMapping(com,false); params->expectedAggFileSize = params->blockSize * params->segmentCount * params->numTasks; @@ -1090,7 +1114,7 @@ static void *HogMemory(IOR_param_t *params) if (verbose >= VERBOSE_3) fprintf(out_logfile, "This node hogging %ld bytes of memory\n", params->memoryPerNode); - size = params->memoryPerNode / params->tasksPerNode; + size = params->memoryPerNode / params->numTasksOnNode0; } else { return NULL; } @@ -1190,16 +1214,6 @@ static void TestIoSys(IOR_test_t *test) IOR_io_buffers ioBuffers; /* set up communicator for test */ - if (params->numTasks > numTasksWorld) { - if (rank == 0) { - fprintf(out_logfile, - "WARNING: More tasks requested (%d) than available (%d),", - params->numTasks, numTasksWorld); - fprintf(out_logfile, " running on %d tasks.\n", - numTasksWorld); - } - params->numTasks = numTasksWorld; - } MPI_CHECK(MPI_Comm_group(mpi_comm_world, &orig_group), "MPI_Comm_group() error"); range[0] = 0; /* first rank */ @@ -1226,7 +1240,6 @@ static void TestIoSys(IOR_test_t *test) "Using reorderTasks '-C' (useful to avoid read cache in client)\n"); fflush(out_logfile); } - params->tasksPerNode = CountTasksPerNode(testComm); backend = params->backend; /* show test setup */ if (rank == 0 && verbose >= VERBOSE_0) @@ -1363,7 +1376,7 @@ static void TestIoSys(IOR_test_t *test) /* move two nodes away from writing node */ int shift = 1; /* assume a by-node (round-robin) mapping of tasks to nodes */ if (params->tasksBlockMapping) { - shift = params->tasksPerNode; /* switch to by-slot (contiguous block) mapping */ + shift = params->numTasksOnNode0; /* switch to by-slot (contiguous block) mapping */ } rankOffset = (2 * shift) % params->numTasks; } @@ -1388,7 +1401,7 @@ static void TestIoSys(IOR_test_t *test) if(params->stoneWallingStatusFile){ params->stoneWallingWearOutIterations = ReadStoneWallingIterations(params->stoneWallingStatusFile); if(params->stoneWallingWearOutIterations == -1 && rank == 0){ - fprintf(out_logfile, "WARNING: Could not read back the stonewalling status from the file!"); + fprintf(out_logfile, "WARNING: Could not read back the stonewalling status from the file!\n"); params->stoneWallingWearOutIterations = 0; } } @@ -1403,7 +1416,7 @@ static void TestIoSys(IOR_test_t *test) /* move one node away from writing node */ int shift = 1; /* assume a by-node (round-robin) mapping of tasks to nodes */ if (params->tasksBlockMapping) { - shift=params->tasksPerNode; /* switch to a by-slot (contiguous block) mapping */ + shift=params->numTasksOnNode0; /* switch to a by-slot (contiguous block) mapping */ } rankOffset = (params->taskPerNodeOffset * shift) % params->numTasks; } @@ -1414,7 +1427,7 @@ static void TestIoSys(IOR_test_t *test) int nodeoffset; unsigned int iseed0; nodeoffset = params->taskPerNodeOffset; - nodeoffset = (nodeoffset < params->nodes) ? nodeoffset : params->nodes - 1; + nodeoffset = (nodeoffset < params->numNodes) ? nodeoffset : params->numNodes - 1; if (params->reorderTasksRandomSeed < 0) iseed0 = -1 * params->reorderTasksRandomSeed + rep; else @@ -1424,7 +1437,7 @@ static void TestIoSys(IOR_test_t *test) rankOffset = rand() % params->numTasks; } while (rankOffset < - (nodeoffset * params->tasksPerNode)) { + (nodeoffset * params->numTasksOnNode0)) { rankOffset = rand() % params->numTasks; } /* Get more detailed stats if requested by verbose level */ @@ -1454,7 +1467,7 @@ static void TestIoSys(IOR_test_t *test) "barrier error"); if (rank == 0 && verbose >= VERBOSE_1) { fprintf(out_logfile, - "Commencing read performance test: %s", + "Commencing read performance test: %s\n", CurrentTimeString()); } timer[2] = GetTimeStamp(); diff --git a/src/ior.h b/src/ior.h index e245b083..ccf47fa7 100755 --- a/src/ior.h +++ b/src/ior.h @@ -98,8 +98,8 @@ typedef struct // intermediate options int dryRun; /* do not perform any I/Os just run evtl. inputs print dummy output */ int numTasks; /* number of tasks for test */ - int nodes; /* number of nodes for test */ - int tasksPerNode; /* number of tasks per node */ + int numNodes; /* number of nodes for test */ + int numTasksOnNode0; /* number of tasks on node 0 (usually all the same, but don't have to be, use with caution) */ int tasksBlockMapping; /* are the tasks in contiguous blocks across nodes or round-robin */ int repetitions; /* number of repetitions of test */ int repCounter; /* rep counter */ diff --git a/src/mdtest.c b/src/mdtest.c index 77b27596..39edf8c0 100644 --- a/src/mdtest.c +++ b/src/mdtest.c @@ -1867,7 +1867,8 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * mdtest_init_args(); int i, j; - int nodeCount; + int numNodes; + int numTasksOnNode0 = 0; MPI_Group worldgroup, testgroup; struct { int first; @@ -1943,8 +1944,8 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * pid = getpid(); uid = getuid(); - tasksPerNode = CountTasksPerNode(testComm); - nodeCount = size / tasksPerNode; + numNodes = GetNumNodes(testComm); + numTasksOnNode0 = GetNumTasksOnNode0(testComm); char cmd_buffer[4096]; strncpy(cmd_buffer, argv[0], 4096); @@ -1953,7 +1954,7 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * } VERBOSE(0,-1,"-- started at %s --\n", PrintTimestamp()); - VERBOSE(0,-1,"mdtest-%s was launched with %d total task(s) on %d node(s)", RELEASE_VERS, size, nodeCount); + VERBOSE(0,-1,"mdtest-%s was launched with %d total task(s) on %d node(s)", RELEASE_VERS, size, numNodes); VERBOSE(0,-1,"Command line used: %s", cmd_buffer); /* adjust special variables */ @@ -2120,10 +2121,10 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * /* set the shift to mimic IOR and shift by procs per node */ if (nstride > 0) { - if ( nodeCount > 1 && tasksBlockMapping ) { + if ( numNodes > 1 && tasksBlockMapping ) { /* the user set the stride presumably to get the consumer tasks on a different node than the producer tasks however, if the mpirun scheduler placed the tasks by-slot (in a contiguous block) then we need to adjust the shift by ppn */ - nstride *= tasksPerNode; + nstride *= numTasksOnNode0; } VERBOSE(0,5,"Shifting ranks by %d for each phase.", nstride); } diff --git a/src/parse_options.c b/src/parse_options.c index af30c36e..ab9509de 100755 --- a/src/parse_options.c +++ b/src/parse_options.c @@ -151,8 +151,12 @@ void DecodeDirective(char *line, IOR_param_t *params, options_all_t * module_opt params->maxTimeDuration = atoi(value); } else if (strcasecmp(option, "outlierthreshold") == 0) { params->outlierThreshold = atoi(value); - } else if (strcasecmp(option, "nodes") == 0) { - params->nodes = atoi(value); + } else if (strcasecmp(option, "numnodes") == 0) { + params->numNodes = atoi(value); + } else if (strcasecmp(option, "numtasks") == 0) { + params->numTasks = atoi(value); + } else if (strcasecmp(option, "numtasksonnode0") == 0) { + params->numTasksOnNode0 = atoi(value); } else if (strcasecmp(option, "repetitions") == 0) { params->repetitions = atoi(value); } else if (strcasecmp(option, "intertestdelay") == 0) { @@ -286,8 +290,6 @@ void DecodeDirective(char *line, IOR_param_t *params, options_all_t * module_opt params->beegfs_chunkSize = string_to_bytes(value); if (!ISPOWEROFTWO(params->beegfs_chunkSize) || params->beegfs_chunkSize < (1<<16)) ERR("beegfsChunkSize must be a power of two and >64k"); - } else if (strcasecmp(option, "numtasks") == 0) { - params->numTasks = atoi(value); } else if (strcasecmp(option, "summaryalways") == 0) { params->summary_every_test = atoi(value); } else { @@ -498,7 +500,7 @@ option_help * createGlobalOptions(IOR_param_t * params){ {'m', NULL, "multiFile -- use number of reps (-i) for multiple file count", OPTION_FLAG, 'd', & params->multiFile}, {'M', NULL, "memoryPerNode -- hog memory on the node (e.g.: 2g, 75%)", OPTION_OPTIONAL_ARGUMENT, 's', & params->memoryPerNodeStr}, {'n', NULL, "noFill -- no fill in HDF5 file creation", OPTION_FLAG, 'd', & params->noFill}, - {'N', NULL, "numTasks -- number of tasks that should participate in the test", OPTION_OPTIONAL_ARGUMENT, 'd', & params->numTasks}, + {'N', NULL, "numTasks -- number of tasks that are participating in the test (overrides MPI)", OPTION_OPTIONAL_ARGUMENT, 'd', & params->numTasks}, {'o', NULL, "testFile -- full name for test", OPTION_OPTIONAL_ARGUMENT, 's', & params->testFileName}, {'O', NULL, "string of IOR directives (e.g. -O checkRead=1,lustreStripeCount=32)", OPTION_OPTIONAL_ARGUMENT, 'p', & decodeDirectiveWrapper}, {'p', NULL, "preallocate -- preallocate file size", OPTION_FLAG, 'd', & params->preallocate}, diff --git a/src/utilities.c b/src/utilities.c index cdb090ea..f7c073de 100755 --- a/src/utilities.c +++ b/src/utilities.c @@ -53,11 +53,9 @@ extern int errno; extern int numTasks; -/* globals used by other files, also defined "extern" in ior.h */ -int numTasksWorld = 0; +/* globals used by other files, also defined "extern" in utilities.h */ int rank = 0; int rankOffset = 0; -int tasksPerNode = 0; /* tasks per node */ int verbose = VERBOSE_0; /* verbose output */ MPI_Comm testComm; MPI_Comm mpi_comm_world; @@ -265,35 +263,108 @@ int QueryNodeMapping(MPI_Comm comm, int print_nodemap) { return ret; } +/* + * There is a more direct way to determine the node count in modern MPI + * versions so we use that if possible. + * + * For older versions we use a method which should still provide accurate + * results even if the total number of tasks is not evenly divisible by the + * tasks on node rank 0. + */ +int GetNumNodes(MPI_Comm comm) { #if MPI_VERSION >= 3 -int CountTasksPerNode(MPI_Comm comm) { - /* modern MPI provides a simple way to get the local process count */ - MPI_Comm shared_comm; - int count; + MPI_Comm shared_comm; + int shared_rank = 0; + int local_result = 0; + int numNodes = 0; + + MPI_CHECK(MPI_Comm_split_type(comm, MPI_COMM_TYPE_SHARED, 0, MPI_INFO_NULL, &shared_comm), + "MPI_Comm_split_type() error"); + MPI_CHECK(MPI_Comm_rank(shared_comm, &shared_rank), "MPI_Comm_rank() error"); + local_result = shared_rank == 0? 1 : 0; + MPI_CHECK(MPI_Allreduce(&local_result, &numNodes, 1, MPI_INT, MPI_SUM, comm), + "MPI_Allreduce() error"); + MPI_CHECK(MPI_Comm_free(&shared_comm), "MPI_Comm_free() error"); + + return numNodes; +#else + int numTasks = 0; + int numTasksOnNode0 = 0; + numTasks = GetNumTasks(comm); + numTasksOnNode0 = GetNumTasksOnNode0(comm); + + return ((numTasks - 1) / numTasksOnNode0) + 1; +#endif +} - MPI_Comm_split_type (comm, MPI_COMM_TYPE_SHARED, 0, MPI_INFO_NULL, &shared_comm); - MPI_Comm_size (shared_comm, &count); - MPI_Comm_free (&shared_comm); - return count; +int GetNumTasks(MPI_Comm comm) { + int numTasks = 0; + + MPI_CHECK(MPI_Comm_size(comm, &numTasks), "cannot get number of tasks"); + + return numTasks; } -#else + + /* - * Count the number of tasks that share a host. + * It's very important that this method provide the same result to every + * process as it's used for redistributing which jobs read from which files. + * It was renamed accordingly. + * + * If different nodes get different results from this method then jobs get + * redistributed unevenly and you no longer have a 1:1 relationship with some + * nodes reading multiple files while others read none. * - * This function employees the gethostname() call, rather than using + * In the common case the number of tasks on each node (MPI_Comm_size on an + * MPI_COMM_TYPE_SHARED communicator) will be the same. However, there is + * nothing which guarantees this. It's valid to have, for example, 64 jobs + * across 4 systems which can run 20 jobs each. In that scenario you end up + * with 3 MPI_COMM_TYPE_SHARED groups of 20, and one group of 4. + * + * In the (MPI_VERSION < 3) implementation of this method consistency is + * ensured by asking specifically about the number of tasks on the node with + * rank 0. In the original implementation for (MPI_VERSION >= 3) this was + * broken by using the LOCAL process count which differed depending on which + * node you were on. + * + * This was corrected below by first splitting the comm into groups by node + * (MPI_COMM_TYPE_SHARED) and then having only the node with world rank 0 and + * shared rank 0 return the MPI_Comm_size of its shared subgroup. This yields + * the original consistent behavior no matter which node asks. + * + * In the common case where every node has the same number of tasks this + * method will return the same value it always has. + */ +int GetNumTasksOnNode0(MPI_Comm comm) { +#if MPI_VERSION >= 3 + MPI_Comm shared_comm; + int shared_rank = 0; + int tasks_on_node_rank0 = 0; + int local_result = 0; + + MPI_CHECK(MPI_Comm_split_type(comm, MPI_COMM_TYPE_SHARED, 0, MPI_INFO_NULL, &shared_comm), + "MPI_Comm_split_type() error"); + MPI_CHECK(MPI_Comm_rank(shared_comm, &shared_rank), "MPI_Comm_rank() error"); + if (rank == 0 && shared_rank == 0) { + MPI_CHECK(MPI_Comm_size(shared_comm, &local_result), "MPI_Comm_size() error"); + } + MPI_CHECK(MPI_Allreduce(&local_result, &tasks_on_node_rank0, 1, MPI_INT, MPI_SUM, comm), + "MPI_Allreduce() error"); + MPI_CHECK(MPI_Comm_free(&shared_comm), "MPI_Comm_free() error"); + + return tasks_on_node_rank0; +#else +/* + * This version employs the gethostname() call, rather than using * MPI_Get_processor_name(). We are interested in knowing the number * of tasks that share a file system client (I/O node, compute node, * whatever that may be). However on machines like BlueGene/Q, * MPI_Get_processor_name() uniquely identifies a cpu in a compute node, * not the node where the I/O is function shipped to. gethostname() * is assumed to identify the shared filesystem client in more situations. - * - * NOTE: This also assumes that the task count on all nodes is equal - * to the task count on the host running MPI task 0. */ -int CountTasksPerNode(MPI_Comm comm) { int size; MPI_Comm_size(comm, & size); /* for debugging and testing */ @@ -336,8 +407,8 @@ int CountTasksPerNode(MPI_Comm comm) { MPI_Bcast(&count, 1, MPI_INT, 0, comm); return(count); -} #endif +} /* diff --git a/src/utilities.h b/src/utilities.h index d2c9962a..2a9abe36 100755 --- a/src/utilities.h +++ b/src/utilities.h @@ -18,10 +18,8 @@ #include #include "ior.h" -extern int numTasksWorld; extern int rank; extern int rankOffset; -extern int tasksPerNode; extern int verbose; extern MPI_Comm testComm; extern MPI_Comm mpi_comm_world; @@ -55,8 +53,10 @@ void SeedRandGen(MPI_Comm); void SetHints (MPI_Info *, char *); void ShowHints (MPI_Info *); char *HumanReadable(IOR_offset_t value, int base); -int CountTasksPerNode(MPI_Comm comm); int QueryNodeMapping(MPI_Comm comm, int print_nodemap); +int GetNumNodes(MPI_Comm); +int GetNumTasks(MPI_Comm); +int GetNumTasksOnNode0(MPI_Comm); void DelaySecs(int delay); void updateParsedOptions(IOR_param_t * options, options_all_t * global_options); size_t NodeMemoryStringToBytes(char *size_str); From 10d3db1dc829dd78ea8ac2da97f233e9dc01ec06 Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Sat, 31 Aug 2019 17:28:08 +0100 Subject: [PATCH 153/206] MDTest: fixing the memset() to account for the number of iterations. --- src/mdtest.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mdtest.c b/src/mdtest.c index 77b27596..96555d28 100644 --- a/src/mdtest.c +++ b/src/mdtest.c @@ -2148,7 +2148,7 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * /* setup summary table for recording results */ summary_table = (mdtest_results_t *) malloc(iterations * sizeof(mdtest_results_t)); - memset(summary_table, 0, sizeof(mdtest_results_t)); + memset(summary_table, 0, iterations * sizeof(mdtest_results_t)); for(int i=0; i < iterations; i++){ for(int j=0; j < MDTEST_LAST_NUM; j++){ summary_table[i].rate[j] = 0.0; From 60a641f911bb1b73685a10102b894f3a58341979 Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Sun, 1 Sep 2019 15:29:12 +0100 Subject: [PATCH 154/206] sync help corrected. --- src/parse_options.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/parse_options.c b/src/parse_options.c index af30c36e..47f9920b 100755 --- a/src/parse_options.c +++ b/src/parse_options.c @@ -477,7 +477,7 @@ option_help * createGlobalOptions(IOR_param_t * params){ {.help=" -O stoneWallingWearOut=1 -- once the stonewalling timout is over, all process finish to access the amount of data", .arg = OPTION_OPTIONAL_ARGUMENT}, {.help=" -O stoneWallingWearOutIterations=N -- stop after processing this number of iterations, needed for reading data back written with stoneWallingWearOut", .arg = OPTION_OPTIONAL_ARGUMENT}, {.help=" -O stoneWallingStatusFile=FILE -- this file keeps the number of iterations from stonewalling during write and allows to use them for read", .arg = OPTION_OPTIONAL_ARGUMENT}, - {'e', NULL, "fsync -- perform sync operation after each block write", OPTION_FLAG, 'd', & params->fsync}, + {'e', NULL, "fsync -- perform a fsync() operation at the end of each read/write phase", OPTION_FLAG, 'd', & params->fsync}, {'E', NULL, "useExistingTestFile -- do not remove test file before write access", OPTION_FLAG, 'd', & params->useExistingTestFile}, {'f', NULL, "scriptFile -- test script name", OPTION_OPTIONAL_ARGUMENT, 's', & params->testscripts}, {'F', NULL, "filePerProc -- file-per-process", OPTION_FLAG, 'd', & params->filePerProc}, From e3db1759b2736aaef826dae756283bfd7e58a1ee Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Sun, 1 Sep 2019 15:47:42 +0100 Subject: [PATCH 155/206] Moded sync() to aiori backend. --- src/aiori-POSIX.c | 12 ++++++++++++ src/aiori.h | 1 + src/mdtest.c | 5 ++++- src/utilities.c | 7 ------- src/utilities.h | 1 - 5 files changed, 17 insertions(+), 9 deletions(-) diff --git a/src/aiori-POSIX.c b/src/aiori-POSIX.c index 00e974c4..99a68f11 100755 --- a/src/aiori-POSIX.c +++ b/src/aiori-POSIX.c @@ -71,6 +71,7 @@ static IOR_offset_t POSIX_Xfer(int, void *, IOR_size_t *, IOR_offset_t, IOR_param_t *); static void POSIX_Fsync(void *, IOR_param_t *); +static void POSIX_Sync(IOR_param_t * ); /************************** O P T I O N S *****************************/ typedef struct{ @@ -122,6 +123,7 @@ ior_aiori_t posix_aiori = { .stat = aiori_posix_stat, .get_options = POSIX_options, .enable_mdtest = true, + .sync = POSIX_Sync }; /***************************** F U N C T I O N S ******************************/ @@ -588,6 +590,16 @@ static void POSIX_Fsync(void *fd, IOR_param_t * param) EWARNF("fsync(%d) failed", *(int *)fd); } + +static void POSIX_Sync(IOR_param_t * param) +{ + int ret = system("sync"); + if (ret != 0){ + FAIL("Error executing the sync command, ensure it exists."); + } +} + + /* * Close a file through the POSIX interface. */ diff --git a/src/aiori.h b/src/aiori.h index c2074c2f..8a5e207b 100755 --- a/src/aiori.h +++ b/src/aiori.h @@ -86,6 +86,7 @@ typedef struct ior_aiori { void (*finalize)(); /* called once per program after MPI is shutdown */ option_help * (*get_options)(void ** init_backend_options, void* init_values); /* initializes the backend options as well and returns the pointer to the option help structure */ bool enable_mdtest; + void (*sync)(IOR_param_t * ); /* synchronize every pending operation for this storage */ } ior_aiori_t; enum bench_type { diff --git a/src/mdtest.c b/src/mdtest.c index e34496a5..52efa347 100644 --- a/src/mdtest.c +++ b/src/mdtest.c @@ -266,7 +266,10 @@ static void prep_testdir(int j, int dir_iter){ static void phase_end(){ if (call_sync){ - call_sync_cmd(); + if(! backend->sync){ + FAIL("Error, backend does not provide the sync method, but your requested to use sync."); + } + backend->sync(& param); } if (barriers) { diff --git a/src/utilities.c b/src/utilities.c index a2e4b0dc..c7e1c8ce 100755 --- a/src/utilities.c +++ b/src/utilities.c @@ -809,10 +809,3 @@ char *HumanReadable(IOR_offset_t value, int base) } return valueStr; } - -void call_sync_cmd(){ - int ret = system("sync"); - if (ret != 0){ - FAIL("Error executing the sync command, ensure it exists."); - } -} diff --git a/src/utilities.h b/src/utilities.h index b85f9579..d2c9962a 100755 --- a/src/utilities.h +++ b/src/utilities.h @@ -60,7 +60,6 @@ int QueryNodeMapping(MPI_Comm comm, int print_nodemap); void DelaySecs(int delay); void updateParsedOptions(IOR_param_t * options, options_all_t * global_options); size_t NodeMemoryStringToBytes(char *size_str); -void call_sync_cmd(); /* Returns -1, if cannot be read */ int64_t ReadStoneWallingIterations(char * const filename); From c83edfe39b49481db7472dce85f1dbe972e6bfc6 Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Sun, 1 Sep 2019 15:59:52 +0100 Subject: [PATCH 156/206] Extracted check function into aiori. #24. #177 --- src/aiori-DUMMY.c | 5 +++++ src/aiori-S3.c | 21 ++++++++++++++++++++- src/aiori.h | 1 + src/ior.c | 15 ++++++--------- 4 files changed, 32 insertions(+), 10 deletions(-) diff --git a/src/aiori-DUMMY.c b/src/aiori-DUMMY.c index 0494cb4d..90fec9a2 100755 --- a/src/aiori-DUMMY.c +++ b/src/aiori-DUMMY.c @@ -143,6 +143,10 @@ static int DUMMY_stat (const char *path, struct stat *buf, IOR_param_t * param){ return 0; } +static int DUMMY_check_params(IOR_param_t * test){ + return 1; +} + ior_aiori_t dummy_aiori = { .name = "DUMMY", .name_legacy = NULL, @@ -163,4 +167,5 @@ ior_aiori_t dummy_aiori = { .finalize = NULL, .get_options = DUMMY_options, .enable_mdtest = true, + .check_params = DUMMY_check_params }; diff --git a/src/aiori-S3.c b/src/aiori-S3.c index 3fc12081..2c9a9afd 100755 --- a/src/aiori-S3.c +++ b/src/aiori-S3.c @@ -159,6 +159,8 @@ static void S3_Fsync(void*, IOR_param_t*); static IOR_offset_t S3_GetFileSize(IOR_param_t*, MPI_Comm, char*); static void S3_init(); static void S3_finalize(); +static int S3_check_params(IOR_param_t *); + /************************** D E C L A R A T I O N S ***************************/ @@ -177,7 +179,8 @@ ior_aiori_t s3_aiori = { .fsync = S3_Fsync, .get_file_size = S3_GetFileSize, .initialize = S3_init, - .finalize = S3_finalize + .finalize = S3_finalize, + .check_params = S3_check_params }; // "S3", plus EMC-extensions enabled @@ -228,6 +231,22 @@ static void S3_finalize(){ aws_cleanup(); } +static int S3_check_params(IOR_param_t * test){ + /* N:1 and N:N */ + IOR_offset_t NtoN = test->filePerProc; + IOR_offset_t Nto1 = ! NtoN; + IOR_offset_t s = test->segmentCount; + IOR_offset_t t = test->transferSize; + IOR_offset_t b = test->blockSize; + + if (Nto1 && (s != 1) && (b != t)) { + ERR("N:1 (strided) requires xfer-size == block-size"); + return 0; + } + + return 1; +} + /* modelled on similar macros in iordef.h */ #define CURL_ERR(MSG, CURL_ERRNO, PARAM) \ do { \ diff --git a/src/aiori.h b/src/aiori.h index 8a5e207b..06f9b315 100755 --- a/src/aiori.h +++ b/src/aiori.h @@ -86,6 +86,7 @@ typedef struct ior_aiori { void (*finalize)(); /* called once per program after MPI is shutdown */ option_help * (*get_options)(void ** init_backend_options, void* init_values); /* initializes the backend options as well and returns the pointer to the option help structure */ bool enable_mdtest; + int (*check_params)(IOR_param_t *); /* check if the provided parameters for the given test and the module options are correct, if they aren't print a message and exit(1) or return 1*/ void (*sync)(IOR_param_t * ); /* synchronize every pending operation for this storage */ } ior_aiori_t; diff --git a/src/ior.c b/src/ior.c index 1e66387a..123a4b52 100755 --- a/src/ior.c +++ b/src/ior.c @@ -1677,15 +1677,12 @@ static void ValidateTests(IOR_param_t * test) if (test->useExistingTestFile && test->lustre_set_striping) ERR("Lustre stripe options are incompatible with useExistingTestFile"); - /* N:1 and N:N */ - IOR_offset_t NtoN = test->filePerProc; - IOR_offset_t Nto1 = ! NtoN; - IOR_offset_t s = test->segmentCount; - IOR_offset_t t = test->transferSize; - IOR_offset_t b = test->blockSize; - - if (Nto1 && (s != 1) && (b != t)) { - ERR("N:1 (strided) requires xfer-size == block-size"); + /* allow the backend to validate the options */ + if(test->backend->check_params){ + int check = test->backend->check_params(test); + if (check == 0){ + ERR("The backend returned that the test parameters are invalid."); + } } } From 36d13d3b972766780e9d7e4c172bb85e756b7c88 Mon Sep 17 00:00:00 2001 From: Petros Koutoupis Date: Mon, 2 Sep 2019 22:26:12 -0700 Subject: [PATCH 157/206] Enable global default dir layout for subdirs in Lustre --- .gitignore | 1 + configure.ac | 13 ++++++++++++- doc/mdtest.1 | 3 +++ src/Makefile.am | 4 ++++ src/mdtest.c | 31 +++++++++++++++++++++++++++++++ 5 files changed, 51 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index eb59cf7a..5a61c7f0 100644 --- a/.gitignore +++ b/.gitignore @@ -28,6 +28,7 @@ src/Makefile.in src/config.h src/config.h.in src/stamp-h1 +*[-.]mod *~ NOTES.txt autom4te.cache diff --git a/configure.ac b/configure.ac index f6b958b4..537021b7 100755 --- a/configure.ac +++ b/configure.ac @@ -83,7 +83,7 @@ AC_ARG_WITH([lustre], [AS_HELP_STRING([--with-lustre], [support configurable Lustre striping values @<:@default=check@:>@])], [], [with_lustre=check]) -AS_IF([test "x$with_lustre" != xno], [ +AS_IF([test "x$with_lustre" = xyes ], [ AC_CHECK_HEADERS([linux/lustre/lustre_user.h lustre/lustre_user.h], break, [ if test "x$with_lustre" != xcheck -a \ "x$ac_cv_header_linux_lustre_lustre_user_h" = "xno" -a \ @@ -91,6 +91,17 @@ AS_IF([test "x$with_lustre" != xno], [ AC_MSG_FAILURE([--with-lustre was given, not found]) fi ]) + AC_CHECK_HEADERS([linux/lustre/lustreapi.h lustre/lustreapi.h], break, [ + if test "x$with_lustre" != xcheck -a \ + "x$ac_cv_header_linux_lustre_lustreapi_h" = "xno" -a \ + "x$ac_cv_header_lustre_lustreapi_h" = "xno" ; then + AC_MSG_FAILURE([--with-lustre was given, not found]) + fi + ]) +]) +AM_CONDITIONAL([WITH_LUSTRE], [test x$with_lustre = xyes]) +AM_COND_IF([WITH_LUSTRE],[ + AC_DEFINE([WITH_LUSTRE], [], [Build wth LUSTRE backend]) ]) # IME (DDN's Infinite Memory Engine) support diff --git a/doc/mdtest.1 b/doc/mdtest.1 index ba82d88a..3cfc082f 100644 --- a/doc/mdtest.1 +++ b/doc/mdtest.1 @@ -47,6 +47,9 @@ The first number of tasks on which the test will run .I "-F" Perform test on files only (no directories). .TP +.I "-g" +Use global default directory layout for test subdirectories (deletes inherited striping layout - Lustre only). +.TP .I "-h" Display help message. .TP diff --git a/src/Makefile.am b/src/Makefile.am index 0de3b4b4..56b4279e 100755 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -91,6 +91,10 @@ extraLDADD += -laws4c extraLDADD += -laws4c_extra endif +if WITH_LUSTRE +extraLDADD += -llustreapi +endif + ior_SOURCES += $(extraSOURCES) ior_LDFLAGS += $(extraLDFLAGS) ior_LDADD += $(extraLDADD) diff --git a/src/mdtest.c b/src/mdtest.c index 17f6e5cf..444f3916 100644 --- a/src/mdtest.c +++ b/src/mdtest.c @@ -76,6 +76,10 @@ #include +#ifdef HAVE_LUSTRE_LUSTREAPI_H +#include +#endif /* HAVE_LUSTRE_LUSTREAPI_H */ + #define FILEMODE S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH #define DIRMODE S_IRUSR|S_IWUSR|S_IXUSR|S_IRGRP|S_IWGRP|S_IXGRP|S_IROTH|S_IXOTH #define RELEASE_VERS META_VERSION @@ -152,6 +156,9 @@ static int call_sync; static int path_count; static int nstride; /* neighbor stride */ static int make_node = 0; +#ifdef HAVE_LUSTRE_LUSTREAPI_H +static int global_dir_layout; +#endif /* HAVE_LUSTRE_LUSTREAPI_H */ static mdtest_results_t * summary_table; static pid_t pid; @@ -1575,6 +1582,15 @@ void create_remove_directory_tree(int create, if (-1 == backend->mkdir (dir, DIRMODE, ¶m)) { fprintf(out_logfile, "error could not create directory '%s'\n", dir); } +#ifdef HAVE_LUSTRE_LUSTREAPI_H + /* internal node for branching, can be non-striped for children */ + if (global_dir_layout && \ + llapi_dir_set_default_lmv_stripe(dir, -1, 0, + LMV_HASH_TYPE_FNV_1A_64, + NULL) == -1) { + FAIL("Unable to reset to global default directory layout"); + } +#endif /* HAVE_LUSTRE_LUSTREAPI_H */ } create_remove_directory_tree(create, ++currDepth, dir, ++dirNum, progress); @@ -1641,6 +1657,12 @@ static void mdtest_iteration(int i, int j, MPI_Group testgroup, mdtest_results_t if (backend->mkdir(testdir, DIRMODE, ¶m) != 0) { FAIL("Unable to create test directory %s", testdir); } +#ifdef HAVE_LUSTRE_LUSTREAPI_H + /* internal node for branching, can be non-striped for children */ + if (global_dir_layout && unique_dir_per_task && llapi_dir_set_default_lmv_stripe(testdir, -1, 0, LMV_HASH_TYPE_FNV_1A_64, NULL) == -1) { + FAIL("Unable to reset to global default directory layout"); + } +#endif /* HAVE_LUSTRE_LUSTREAPI_H */ } } @@ -1859,6 +1881,9 @@ void mdtest_init_args(){ path_count = 0; nstride = 0; make_node = 0; +#ifdef HAVE_LUSTRE_LUSTREAPI_H + global_dir_layout = 0; +#endif /* HAVE_LUSTRE_LUSTREAPI_H */ } mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * world_out) { @@ -1905,6 +1930,9 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * {'e', NULL, "bytes to read from each file", OPTION_OPTIONAL_ARGUMENT, 'l', & read_bytes}, {'f', NULL, "first number of tasks on which the test will run", OPTION_OPTIONAL_ARGUMENT, 'd', & first}, {'F', NULL, "perform test on files only (no directories)", OPTION_FLAG, 'd', & files_only}, +#ifdef HAVE_LUSTRE_LUSTREAPI_H + {'g', NULL, "global default directory layout for test subdirectories (deletes inherited striping layout)", OPTION_FLAG, 'd', & global_dir_layout}, +#endif /* HAVE_LUSTRE_LUSTREAPI_H */ {'i', NULL, "number of iterations the test will run", OPTION_OPTIONAL_ARGUMENT, 'd', & iterations}, {'I', NULL, "number of items per directory in tree", OPTION_OPTIONAL_ARGUMENT, 'l', & items_per_dir}, {'k', NULL, "use mknod to create file", OPTION_FLAG, 'd', & make_node}, @@ -1998,6 +2026,9 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * VERBOSE(1,-1, "read_only : %s", ( read_only ? "True" : "False" )); VERBOSE(1,-1, "first : %d", first ); VERBOSE(1,-1, "files_only : %s", ( files_only ? "True" : "False" )); +#ifdef HAVE_LUSTRE_LUSTREAPI_H + VERBOSE(1,-1, "global_dir_layout : %s", ( global_dir_layout ? "True" : "False" )); +#endif /* HAVE_LUSTRE_LUSTREAPI_H */ VERBOSE(1,-1, "iterations : %d", iterations ); VERBOSE(1,-1, "items_per_dir : "LLU"", items_per_dir ); VERBOSE(1,-1, "last : %d", last ); From 57a16ddda855193e0c117aa4c00b84d3888959ad Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Mon, 9 Sep 2019 19:14:35 +0000 Subject: [PATCH 158/206] - update debugging error checking in DFS. - remove usleep() before daos_fini() - fix README Signed-off-by: Mohamad Chaarawi --- README_DAOS | 17 +++--- src/aiori-DAOS.c | 6 +-- src/aiori-DFS.c | 132 ++++++++++++++++++++++------------------------- 3 files changed, 73 insertions(+), 82 deletions(-) diff --git a/README_DAOS b/README_DAOS index b4e6dba0..ed98bd66 100644 --- a/README_DAOS +++ b/README_DAOS @@ -59,12 +59,15 @@ ior -a DFS [ior_options] [dfs_options] mdtest -a DFS [mdtest_options] [dfs_options] Required Options: ---daos.pool : pool uuid to connect to (has to be created beforehand) ---daos.svcl : pool svcl list (: separated) ---daos.cont : container uuid that will hold the encapsulated namespace +--dfs.pool : pool uuid to connect to (has to be created beforehand) +--dfs.svcl : pool svcl list (: separated) +--dfs.cont : container uuid that will hold the encapsulated namespace Optional Options: ---daos.group : group name of servers with the pool +--dfs.group : group name of servers with the pool +--dfs.chunk_size : Chunk size of the files +--dfs.destroy flag to destory the container on finalize +--dfs.oclass : specific object class for files In the IOR options, the file name should be specified on the root dir directly since ior does not create directories and the DFS container representing the @@ -72,9 +75,9 @@ encapsulated namespace is not the same as the system namespace the user is executing from. Examples that should work include: - - "ior -a DFS -w -W -o /test1 --daos.pool --daos.svcl --daos.cont " - - "ior -a DFS -w -W -r -R -o /test2 -b 1g -t 4m -C --daos.pool --daos.svcl --daos.cont " - - "ior -a DFS -w -r -o /test3 -b 8g -t 1m -C --daos.pool --daos.svcl --daos.cont " + - "ior -a DFS -w -W -o /test1 --dfs.pool --dfs.svcl --dfs.cont " + - "ior -a DFS -w -W -r -R -o /test2 -b 1g -t 4m -C --dfs.pool --dfs.svcl --dfs.cont " + - "ior -a DFS -w -r -o /test3 -b 8g -t 1m -C --dfs.pool --dfs.svcl --dfs.cont " Running mdtest, the user needs to specify a directory with -d where the test tree will be created. Some examples: diff --git a/src/aiori-DAOS.c b/src/aiori-DAOS.c index 18dd689e..a9297587 100644 --- a/src/aiori-DAOS.c +++ b/src/aiori-DAOS.c @@ -131,7 +131,7 @@ do { \ int _rc = (rc); \ \ if (_rc < 0) { \ - fprintf(stdout, "ior ERROR (%s:%d): %d: %d: " \ + fprintf(stderr, "ior ERROR (%s:%d): %d: %d: " \ format"\n", __FILE__, __LINE__, rank, _rc, \ ##__VA_ARGS__); \ fflush(stdout); \ @@ -148,7 +148,7 @@ do { \ /* For generic errors like invalid command line options. */ #define GERR(format, ...) \ do { \ - fprintf(stdout, format"\n", ##__VA_ARGS__); \ + fprintf(stderr, format"\n", ##__VA_ARGS__); \ MPI_CHECK(MPI_Abort(MPI_COMM_WORLD, -1), "MPI_Abort() error"); \ } while (0) @@ -331,8 +331,6 @@ DAOS_Fini() DCHECK(rc, "Failed to disconnect from pool %s", o.pool); MPI_CHECK(MPI_Barrier(MPI_COMM_WORLD), "barrier error"); - usleep(20000 * rank); - if (rank == 0) INFO(VERBOSE_1, "Finalizing DAOS.."); diff --git a/src/aiori-DFS.c b/src/aiori-DFS.c index df956e84..a49434ee 100755 --- a/src/aiori-DFS.c +++ b/src/aiori-DFS.c @@ -149,17 +149,16 @@ do { \ } \ } while (0) -#define DERR(rc, format, ...) \ +#define INFO(level, format, ...) \ do { \ - int _rc = (rc); \ - \ - if (_rc != 0) { \ - fprintf(stderr, "ERROR (%s:%d): %d: %d: " \ - format"\n", __FILE__, __LINE__, rank, _rc, \ - ##__VA_ARGS__); \ - fflush(stderr); \ - goto out; \ - } \ + if (verbose >= level) \ + printf("[%d] "format"\n", rank, ##__VA_ARGS__); \ +} while (0) + +#define GERR(format, ...) \ +do { \ + fprintf(stderr, format"\n", ##__VA_ARGS__); \ + MPI_CHECK(MPI_Abort(MPI_COMM_WORLD, -1), "MPI_Abort() error"); \ } while (0) static inline struct aiori_dir_hdl * @@ -351,21 +350,19 @@ lookup_insert_dir(const char *name) hdl = calloc(1, sizeof(struct aiori_dir_hdl)); if (hdl == NULL) - DERR(ENOMEM, "failed to alloc dir handle"); + GERR("failed to alloc dir handle"); strncpy(hdl->name, name, PATH_MAX-1); hdl->name[PATH_MAX-1] = '\0'; rc = dfs_lookup(dfs, name, O_RDWR, &hdl->oh, NULL, NULL); - DERR(rc, "dfs_lookup() of %s Failed", name); + DCHECK(rc, "dfs_lookup() of %s Failed", name); rc = d_hash_rec_insert(dir_hash, hdl->name, strlen(hdl->name), &hdl->entry, true); - DERR(rc, "Failed to insert dir handle in hashtable"); + DCHECK(rc, "Failed to insert dir handle in hashtable"); return hdl->oh; -out: - return NULL; } static option_help * DFS_options(){ @@ -382,7 +379,7 @@ DFS_Init() { if (o.oclass) { objectClass = daos_oclass_name2id(o.oclass); if (objectClass == OC_UNKNOWN) - DCHECK(-1, "Invalid DAOS Object class %s\n", o.oclass); + GERR("Invalid DAOS Object class %s\n", o.oclass); } rc = daos_init(); @@ -407,10 +404,8 @@ DFS_Init() { if (svcl == NULL) ERR("Failed to allocate svcl"); - if (verbose >= VERBOSE_1) { - printf("Pool uuid = %s, SVCL = %s\n", o.pool, o.svcl); - printf("DFS Container namespace uuid = %s\n", o.cont); - } + INFO(VERBOSE_1, "Pool uuid = %s, SVCL = %s\n", o.pool, o.svcl); + INFO(VERBOSE_1, "DFS Container namespace uuid = %s\n", o.cont); /** Connect to DAOS pool */ rc = daos_pool_connect(pool_uuid, o.group, svcl, DAOS_PC_RW, @@ -422,8 +417,7 @@ DFS_Init() { NULL); /* If NOEXIST we create it */ if (rc == -DER_NONEXIST) { - if (verbose >= VERBOSE_1) - printf("Creating DFS Container ...\n"); + INFO(VERBOSE_1, "Creating DFS Container ...\n"); rc = daos_cont_create(poh, co_uuid, NULL, NULL); if (rc == 0) { @@ -457,35 +451,38 @@ DFS_Finalize() DCHECK(rc, "Failed to close container %s (%d)", o.cont, rc); MPI_Barrier(MPI_COMM_WORLD); - if (rank == 0 && o.destroy) { - uuid_t uuid; - double t1, t2; - - if (verbose >= VERBOSE_1) - printf("Destorying DFS Container: %s\n", o.cont); - uuid_parse(o.cont, uuid); - t1 = MPI_Wtime(); - rc = daos_cont_destroy(poh, uuid, 1, NULL); - t2 = MPI_Wtime(); - if (rc == 0 && verbose >= VERBOSE_1) - printf("Container Destroy time = %f secs", t2-t1); - } + if (o.destroy) { + if (rank == 0) { + uuid_t uuid; + double t1, t2; + + INFO(VERBOSE_1, "Destorying DFS Container: %s\n", o.cont); + uuid_parse(o.cont, uuid); + t1 = MPI_Wtime(); + rc = daos_cont_destroy(poh, uuid, 1, NULL); + t2 = MPI_Wtime(); + if (rc == 0) + INFO(VERBOSE_1, "Container Destroy time = %f secs", t2-t1); + } - MPI_Bcast(&rc, 1, MPI_INT, 0, MPI_COMM_WORLD); - if (rc) - DCHECK(rc, "Failed to destroy container %s (%d)", o.cont, rc); + MPI_Bcast(&rc, 1, MPI_INT, 0, MPI_COMM_WORLD); + if (rc) { + if (rank == 0) + DCHECK(rc, "Failed to destroy container %s (%d)", o.cont, rc); + MPI_Abort(MPI_COMM_WORLD, -1); + } + } - if (rank == 0 && verbose >= VERBOSE_1) - printf("Disconnecting from DAOS POOL\n"); + if (rank == 0) + INFO(VERBOSE_1, "Disconnecting from DAOS POOL\n"); rc = daos_pool_disconnect(poh, NULL); DCHECK(rc, "Failed to disconnect from pool"); MPI_CHECK(MPI_Barrier(MPI_COMM_WORLD), "barrier error"); - usleep(20000 * rank); - if (rank == 0 && verbose >= VERBOSE_1) - printf("Finalizing DAOS..\n"); + if (rank == 0) + INFO(VERBOSE_1, "Finalizing DAOS..\n"); rc = daos_fini(); DCHECK(rc, "Failed to finalize DAOS"); @@ -506,13 +503,13 @@ DFS_Create(char *testFileName, IOR_param_t *param) assert(param); rc = parse_filename(testFileName, &name, &dir_name); - DERR(rc, "Failed to parse path %s", testFileName); + DCHECK(rc, "Failed to parse path %s", testFileName); assert(dir_name); assert(name); parent = lookup_insert_dir(dir_name); if (parent == NULL) - DERR(rc, "Failed to lookup parent dir"); + GERR("Failed to lookup parent dir"); mode = S_IFREG | param->mode; if (param->filePerProc || rank == 0) { @@ -520,7 +517,7 @@ DFS_Create(char *testFileName, IOR_param_t *param) rc = dfs_open(dfs, parent, name, mode, fd_oflag, objectClass, o.chunk_size, NULL, &obj); - DERR(rc, "dfs_open() of %s Failed", name); + DCHECK(rc, "dfs_open() of %s Failed", name); } if (!param->filePerProc) { MPI_Barrier(MPI_COMM_WORLD); @@ -528,11 +525,10 @@ DFS_Create(char *testFileName, IOR_param_t *param) fd_oflag |= O_RDWR; rc = dfs_open(dfs, parent, name, mode, fd_oflag, objectClass, o.chunk_size, NULL, &obj); - DERR(rc, "dfs_open() of %s Failed", name); + DCHECK(rc, "dfs_open() of %s Failed", name); } } -out: if (name) free(name); if (dir_name) @@ -557,20 +553,19 @@ DFS_Open(char *testFileName, IOR_param_t *param) mode = S_IFREG | param->mode; rc = parse_filename(testFileName, &name, &dir_name); - DERR(rc, "Failed to parse path %s", testFileName); + DCHECK(rc, "Failed to parse path %s", testFileName); assert(dir_name); assert(name); parent = lookup_insert_dir(dir_name); if (parent == NULL) - DERR(rc, "Failed to lookup parent dir"); + GERR("Failed to lookup parent dir"); rc = dfs_open(dfs, parent, name, mode, fd_oflag, objectClass, o.chunk_size, NULL, &obj); - DERR(rc, "dfs_open() of %s Failed", name); + DCHECK(rc, "dfs_open() of %s Failed", name); -out: if (name) free(name); if (dir_name) @@ -666,19 +661,18 @@ DFS_Delete(char *testFileName, IOR_param_t * param) int rc; rc = parse_filename(testFileName, &name, &dir_name); - DERR(rc, "Failed to parse path %s", testFileName); + DCHECK(rc, "Failed to parse path %s", testFileName); assert(dir_name); assert(name); parent = lookup_insert_dir(dir_name); if (parent == NULL) - DERR(rc, "Failed to lookup parent dir"); + GERR("Failed to lookup parent dir"); rc = dfs_remove(dfs, parent, name, false, NULL); - DERR(rc, "dfs_remove() of %s Failed", name); + DCHECK(rc, "dfs_remove() of %s Failed", name); -out: if (name) free(name); if (dir_name) @@ -753,7 +747,7 @@ DFS_Mkdir(const char *path, mode_t mode, IOR_param_t * param) int rc; rc = parse_filename(path, &name, &dir_name); - DERR(rc, "Failed to parse path %s", path); + DCHECK(rc, "Failed to parse path %s", path); assert(dir_name); if (!name) @@ -761,12 +755,11 @@ DFS_Mkdir(const char *path, mode_t mode, IOR_param_t * param) parent = lookup_insert_dir(dir_name); if (parent == NULL) - DERR(rc, "Failed to lookup parent dir"); + GERR("Failed to lookup parent dir"); rc = dfs_mkdir(dfs, parent, name, mode); - DERR(rc, "dfs_mkdir() of %s Failed", name); + DCHECK(rc, "dfs_mkdir() of %s Failed", name); -out: if (name) free(name); if (dir_name) @@ -784,19 +777,18 @@ DFS_Rmdir(const char *path, IOR_param_t * param) int rc; rc = parse_filename(path, &name, &dir_name); - DERR(rc, "Failed to parse path %s", path); + DCHECK(rc, "Failed to parse path %s", path); assert(dir_name); assert(name); parent = lookup_insert_dir(dir_name); if (parent == NULL) - DERR(rc, "Failed to lookup parent dir"); + GERR("Failed to lookup parent dir"); rc = dfs_remove(dfs, parent, name, false, NULL); - DERR(rc, "dfs_remove() of %s Failed", name); + DCHECK(rc, "dfs_remove() of %s Failed", name); -out: if (name) free(name); if (dir_name) @@ -815,13 +807,13 @@ DFS_Access(const char *path, int mode, IOR_param_t * param) int rc; rc = parse_filename(path, &name, &dir_name); - DERR(rc, "Failed to parse path %s", path); + DCHECK(rc, "Failed to parse path %s", path); assert(dir_name); parent = lookup_insert_dir(dir_name); if (parent == NULL) - DERR(rc, "Failed to lookup parent dir"); + GERR("Failed to lookup parent dir"); if (name && strcmp(name, ".") == 0) { free(name); @@ -829,7 +821,6 @@ DFS_Access(const char *path, int mode, IOR_param_t * param) } rc = dfs_stat(dfs, parent, name, &stbuf); -out: if (name) free(name); if (dir_name) @@ -847,19 +838,18 @@ DFS_Stat(const char *path, struct stat *buf, IOR_param_t * param) int rc; rc = parse_filename(path, &name, &dir_name); - DERR(rc, "Failed to parse path %s", path); + DCHECK(rc, "Failed to parse path %s", path); assert(dir_name); assert(name); parent = lookup_insert_dir(dir_name); if (parent == NULL) - DERR(rc, "Failed to lookup parent dir"); + GERR("Failed to lookup parent dir"); rc = dfs_stat(dfs, parent, name, buf); - DERR(rc, "dfs_stat() of %s Failed", name); + DCHECK(rc, "dfs_stat() of Failed (%d)", rc); -out: if (name) free(name); if (dir_name) From d332d586bfe2460e819bf169dfeef35a5128c174 Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Mon, 9 Sep 2019 20:07:55 +0000 Subject: [PATCH 159/206] remove uneeded code and doc Signed-off-by: Mohamad Chaarawi --- doc/USER_GUIDE | 37 ---- src/aiori-DAOS.c | 1 - src/list.h | 556 ----------------------------------------------- 3 files changed, 594 deletions(-) delete mode 100644 src/list.h diff --git a/doc/USER_GUIDE b/doc/USER_GUIDE index 7e0af0f1..b8888b9f 100755 --- a/doc/USER_GUIDE +++ b/doc/USER_GUIDE @@ -369,43 +369,6 @@ BeeGFS-SPECIFIC (POSIX only): * beegfsChunkSize - set the striping chunk size. Must be a power of two, and greater than 64kiB, (e.g.: 256k, 1M, ...) -DAOS-ONLY: -========== - * daosGroup - group name [NULL] - - * daosPool - UUID of the pool [] - - * daosPoolSvc - pool service replica ranks (e.g., 1:2:3:4:5) [] - - * daosRecordSize - size (in bytes) of an akey record [256k] - NOTE: must divide transferSize - - * daosStripeSize - size (in bytes) of a chunk in a stripe [512k] - NOTE: must be a multiple of transferSize - - * daosStripeCount - number of stripes [64 * number of targets] - NOTE: i.e., number of dkeys - - * daosStripeMax - max length of each stripe [0] - NOTE: must be a multiple of daosStripeSize - NOTE: for write testing with small storage - NOTE: offsets in a stripe larger than daosStripeMax - are mapped to offset % daosStripeMax - - * daosAios - max number of asychonous I/Os [1] - - * daosWriteOnly - skip flushing and committing [0=FALSE] - - * daosEpoch - epoch to read or write [0] - NOTE: 0 denotes reading GHCE or writing GHCE + 1 - - * daosWait - epoch to wait when opening the container [0] - - * daosKill - kill a target in the middle of the test [0] - NOTE: must also specify daosObjectClass=repl - - * daosObjectClass - object class (tiny, small, large, repl, repl_max) - [large] *********************** * 5. VERBOSITY LEVELS * diff --git a/src/aiori-DAOS.c b/src/aiori-DAOS.c index a91fe087..21df9aad 100644 --- a/src/aiori-DAOS.c +++ b/src/aiori-DAOS.c @@ -37,7 +37,6 @@ #include "ior.h" #include "aiori.h" #include "iordef.h" -#include "list.h" /************************** O P T I O N S *****************************/ struct daos_options{ diff --git a/src/list.h b/src/list.h deleted file mode 100644 index dbe052cd..00000000 --- a/src/list.h +++ /dev/null @@ -1,556 +0,0 @@ -/** - * GPL HEADER START - * - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 only, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License version 2 for more details (a copy is included - * in the LICENSE file that accompanied this code). - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - * - * GPL HEADER END - */ -#ifndef __DAOS_LIST_H__ -#define __DAOS_LIST_H__ - -/* - * Simple doubly linked list implementation. - * - * Some of the internal functions ("__xxx") are useful when - * manipulating whole lists rather than single entries, as - * sometimes we already know the next/prev entries and we can - * generate better code by using them directly rather than - * using the generic single-entry routines. - */ - -#define prefetch(a) ((void)a) - -struct cfs_list_head { - struct cfs_list_head *next, *prev; -}; - -typedef struct cfs_list_head cfs_list_t; - -#define CFS_LIST_HEAD_INIT(name) { &(name), &(name) } - -#define CFS_LIST_HEAD(name) \ - cfs_list_t name = CFS_LIST_HEAD_INIT(name) - -#define CFS_INIT_LIST_HEAD(ptr) do { \ - (ptr)->next = (ptr); (ptr)->prev = (ptr); \ -} while (0) - -/** - * Insert a new entry between two known consecutive entries. - * - * This is only for internal list manipulation where we know - * the prev/next entries already! - */ -static inline void __cfs_list_add(cfs_list_t * new, - cfs_list_t * prev, - cfs_list_t * next) -{ - next->prev = new; - new->next = next; - new->prev = prev; - prev->next = new; -} - -/** - * Insert an entry at the start of a list. - * \param new new entry to be inserted - * \param head list to add it to - * - * Insert a new entry after the specified head. - * This is good for implementing stacks. - */ -static inline void cfs_list_add(cfs_list_t *new, - cfs_list_t *head) -{ - __cfs_list_add(new, head, head->next); -} - -/** - * Insert an entry at the end of a list. - * \param new new entry to be inserted - * \param head list to add it to - * - * Insert a new entry before the specified head. - * This is useful for implementing queues. - */ -static inline void cfs_list_add_tail(cfs_list_t *new, - cfs_list_t *head) -{ - __cfs_list_add(new, head->prev, head); -} - -/* - * Delete a list entry by making the prev/next entries - * point to each other. - * - * This is only for internal list manipulation where we know - * the prev/next entries already! - */ -static inline void __cfs_list_del(cfs_list_t *prev, - cfs_list_t *next) -{ - next->prev = prev; - prev->next = next; -} - -/** - * Remove an entry from the list it is currently in. - * \param entry the entry to remove - * Note: list_empty(entry) does not return true after this, the entry is in an - * undefined state. - */ -static inline void cfs_list_del(cfs_list_t *entry) -{ - __cfs_list_del(entry->prev, entry->next); -} - -/** - * Remove an entry from the list it is currently in and reinitialize it. - * \param entry the entry to remove. - */ -static inline void cfs_list_del_init(cfs_list_t *entry) -{ - __cfs_list_del(entry->prev, entry->next); - CFS_INIT_LIST_HEAD(entry); -} - -/** - * Remove an entry from the list it is currently in and insert it at the start - * of another list. - * \param list the entry to move - * \param head the list to move it to - */ -static inline void cfs_list_move(cfs_list_t *list, - cfs_list_t *head) -{ - __cfs_list_del(list->prev, list->next); - cfs_list_add(list, head); -} - -/** - * Remove an entry from the list it is currently in and insert it at the end of - * another list. - * \param list the entry to move - * \param head the list to move it to - */ -static inline void cfs_list_move_tail(cfs_list_t *list, - cfs_list_t *head) -{ - __cfs_list_del(list->prev, list->next); - cfs_list_add_tail(list, head); -} - -/** - * Test whether a list is empty - * \param head the list to test. - */ -static inline int cfs_list_empty(cfs_list_t *head) -{ - return head->next == head; -} - -/** - * Test whether a list is empty and not being modified - * \param head the list to test - * - * Tests whether a list is empty _and_ checks that no other CPU might be - * in the process of modifying either member (next or prev) - * - * NOTE: using cfs_list_empty_careful() without synchronization - * can only be safe if the only activity that can happen - * to the list entry is cfs_list_del_init(). Eg. it cannot be used - * if another CPU could re-list_add() it. - */ -static inline int cfs_list_empty_careful(const cfs_list_t *head) -{ - cfs_list_t *next = head->next; - return (next == head) && (next == head->prev); -} - -static inline void __cfs_list_splice(cfs_list_t *list, - cfs_list_t *head) -{ - cfs_list_t *first = list->next; - cfs_list_t *last = list->prev; - cfs_list_t *at = head->next; - - first->prev = head; - head->next = first; - - last->next = at; - at->prev = last; -} - -/** - * Join two lists - * \param list the new list to add. - * \param head the place to add it in the first list. - * - * The contents of \a list are added at the start of \a head. \a list is in an - * undefined state on return. - */ -static inline void cfs_list_splice(cfs_list_t *list, - cfs_list_t *head) -{ - if (!cfs_list_empty(list)) - __cfs_list_splice(list, head); -} - -/** - * Join two lists and reinitialise the emptied list. - * \param list the new list to add. - * \param head the place to add it in the first list. - * - * The contents of \a list are added at the start of \a head. \a list is empty - * on return. - */ -static inline void cfs_list_splice_init(cfs_list_t *list, - cfs_list_t *head) -{ - if (!cfs_list_empty(list)) { - __cfs_list_splice(list, head); - CFS_INIT_LIST_HEAD(list); - } -} - -/** - * Get the container of a list - * \param ptr the embedded list. - * \param type the type of the struct this is embedded in. - * \param member the member name of the list within the struct. - */ -#define cfs_list_entry(ptr, type, member) \ - ((type *)((char *)(ptr)-(char *)(&((type *)0)->member))) - -/** - * Iterate over a list - * \param pos the iterator - * \param head the list to iterate over - * - * Behaviour is undefined if \a pos is removed from the list in the body of the - * loop. - */ -#define cfs_list_for_each(pos, head) \ - for (pos = (head)->next, prefetch(pos->next); pos != (head); \ - pos = pos->next, prefetch(pos->next)) - -/** - * Iterate over a list safely - * \param pos the iterator - * \param n temporary storage - * \param head the list to iterate over - * - * This is safe to use if \a pos could be removed from the list in the body of - * the loop. - */ -#define cfs_list_for_each_safe(pos, n, head) \ - for (pos = (head)->next, n = pos->next; pos != (head); \ - pos = n, n = pos->next) - -/** - * Iterate over a list continuing after existing point - * \param pos the type * to use as a loop counter - * \param head the list head - * \param member the name of the list_struct within the struct - */ -#define cfs_list_for_each_entry_continue(pos, head, member) \ - for (pos = cfs_list_entry(pos->member.next, typeof(*pos), member); \ - prefetch(pos->member.next), &pos->member != (head); \ - pos = cfs_list_entry(pos->member.next, typeof(*pos), member)) - -/** - * \defgroup hlist Hash List - * Double linked lists with a single pointer list head. - * Mostly useful for hash tables where the two pointer list head is too - * wasteful. You lose the ability to access the tail in O(1). - * @{ - */ - -typedef struct cfs_hlist_node { - struct cfs_hlist_node *next, **pprev; -} cfs_hlist_node_t; - -typedef struct cfs_hlist_head { - cfs_hlist_node_t *first; -} cfs_hlist_head_t; - -/* @} */ - -/* - * "NULL" might not be defined at this point - */ -#ifdef NULL -#define NULL_P NULL -#else -#define NULL_P ((void *)0) -#endif - -/** - * \addtogroup hlist - * @{ - */ - -#define CFS_HLIST_HEAD_INIT { NULL_P } -#define CFS_HLIST_HEAD(name) cfs_hlist_head_t name = { NULL_P } -#define CFS_INIT_HLIST_HEAD(ptr) ((ptr)->first = NULL_P) -#define CFS_INIT_HLIST_NODE(ptr) ((ptr)->next = NULL_P, (ptr)->pprev = NULL_P) - -static inline int cfs_hlist_unhashed(const cfs_hlist_node_t *h) -{ - return !h->pprev; -} - -static inline int cfs_hlist_empty(const cfs_hlist_head_t *h) -{ - return !h->first; -} - -static inline void __cfs_hlist_del(cfs_hlist_node_t *n) -{ - cfs_hlist_node_t *next = n->next; - cfs_hlist_node_t **pprev = n->pprev; - *pprev = next; - if (next) - next->pprev = pprev; -} - -static inline void cfs_hlist_del(cfs_hlist_node_t *n) -{ - __cfs_hlist_del(n); -} - -static inline void cfs_hlist_del_init(cfs_hlist_node_t *n) -{ - if (n->pprev) { - __cfs_hlist_del(n); - CFS_INIT_HLIST_NODE(n); - } -} - -static inline void cfs_hlist_add_head(cfs_hlist_node_t *n, - cfs_hlist_head_t *h) -{ - cfs_hlist_node_t *first = h->first; - n->next = first; - if (first) - first->pprev = &n->next; - h->first = n; - n->pprev = &h->first; -} - -/* next must be != NULL */ -static inline void cfs_hlist_add_before(cfs_hlist_node_t *n, - cfs_hlist_node_t *next) -{ - n->pprev = next->pprev; - n->next = next; - next->pprev = &n->next; - *(n->pprev) = n; -} - -static inline void cfs_hlist_add_after(cfs_hlist_node_t *n, - cfs_hlist_node_t *next) -{ - next->next = n->next; - n->next = next; - next->pprev = &n->next; - - if(next->next) - next->next->pprev = &next->next; -} - -#define cfs_hlist_entry(ptr, type, member) container_of(ptr,type,member) - -#define cfs_hlist_for_each(pos, head) \ - for (pos = (head)->first; pos && (prefetch(pos->next), 1); \ - pos = pos->next) - -#define cfs_hlist_for_each_safe(pos, n, head) \ - for (pos = (head)->first; pos && (n = pos->next, 1); \ - pos = n) - -/** - * Iterate over an hlist of given type - * \param tpos the type * to use as a loop counter. - * \param pos the &struct hlist_node to use as a loop counter. - * \param head the head for your list. - * \param member the name of the hlist_node within the struct. - */ -#define cfs_hlist_for_each_entry(tpos, pos, head, member) \ - for (pos = (head)->first; \ - pos && ({ prefetch(pos->next); 1;}) && \ - ({ tpos = cfs_hlist_entry(pos, typeof(*tpos), member); 1;}); \ - pos = pos->next) - -/** - * Iterate over an hlist continuing after existing point - * \param tpos the type * to use as a loop counter. - * \param pos the &struct hlist_node to use as a loop counter. - * \param member the name of the hlist_node within the struct. - */ -#define cfs_hlist_for_each_entry_continue(tpos, pos, member) \ - for (pos = (pos)->next; \ - pos && ({ prefetch(pos->next); 1;}) && \ - ({ tpos = cfs_hlist_entry(pos, typeof(*tpos), member); 1;}); \ - pos = pos->next) - -/** - * Iterate over an hlist continuing from an existing point - * \param tpos the type * to use as a loop counter. - * \param pos the &struct hlist_node to use as a loop counter. - * \param member the name of the hlist_node within the struct. - */ -#define cfs_hlist_for_each_entry_from(tpos, pos, member) \ - for (; pos && ({ prefetch(pos->next); 1;}) && \ - ({ tpos = cfs_hlist_entry(pos, typeof(*tpos), member); 1;}); \ - pos = pos->next) - -/** - * Iterate over an hlist of given type safe against removal of list entry - * \param tpos the type * to use as a loop counter. - * \param pos the &struct hlist_node to use as a loop counter. - * \param n another &struct hlist_node to use as temporary storage - * \param head the head for your list. - * \param member the name of the hlist_node within the struct. - */ -#define cfs_hlist_for_each_entry_safe(tpos, pos, n, head, member) \ - for (pos = (head)->first; \ - pos && ({ n = pos->next; 1; }) && \ - ({ tpos = cfs_hlist_entry(pos, typeof(*tpos), member); 1;}); \ - pos = n) - -/* @} */ - -#ifndef cfs_list_for_each_prev -/** - * Iterate over a list in reverse order - * \param pos the &struct list_head to use as a loop counter. - * \param head the head for your list. - */ -#define cfs_list_for_each_prev(pos, head) \ - for (pos = (head)->prev, prefetch(pos->prev); pos != (head); \ - pos = pos->prev, prefetch(pos->prev)) - -#endif /* cfs_list_for_each_prev */ - -#ifndef cfs_list_for_each_entry -/** - * Iterate over a list of given type - * \param pos the type * to use as a loop counter. - * \param head the head for your list. - * \param member the name of the list_struct within the struct. - */ -#define cfs_list_for_each_entry(pos, head, member) \ - for (pos = cfs_list_entry((head)->next, typeof(*pos), member), \ - prefetch(pos->member.next); \ - &pos->member != (head); \ - pos = cfs_list_entry(pos->member.next, typeof(*pos), member), \ - prefetch(pos->member.next)) -#endif /* cfs_list_for_each_entry */ - -#ifndef cfs_list_for_each_entry_rcu -#define cfs_list_for_each_entry_rcu(pos, head, member) \ - list_for_each_entry(pos, head, member) -#endif - -#ifndef cfs_list_for_each_entry_rcu -#define cfs_list_for_each_entry_rcu(pos, head, member) \ - list_for_each_entry(pos, head, member) -#endif - -#ifndef cfs_list_for_each_entry_reverse -/** - * Iterate backwards over a list of given type. - * \param pos the type * to use as a loop counter. - * \param head the head for your list. - * \param member the name of the list_struct within the struct. - */ -#define cfs_list_for_each_entry_reverse(pos, head, member) \ - for (pos = cfs_list_entry((head)->prev, typeof(*pos), member); \ - prefetch(pos->member.prev), &pos->member != (head); \ - pos = cfs_list_entry(pos->member.prev, typeof(*pos), member)) -#endif /* cfs_list_for_each_entry_reverse */ - -#ifndef cfs_list_for_each_entry_safe -/** - * Iterate over a list of given type safe against removal of list entry - * \param pos the type * to use as a loop counter. - * \param n another type * to use as temporary storage - * \param head the head for your list. - * \param member the name of the list_struct within the struct. - */ -#define cfs_list_for_each_entry_safe(pos, n, head, member) \ - for (pos = cfs_list_entry((head)->next, typeof(*pos), member), \ - n = cfs_list_entry(pos->member.next, typeof(*pos), member); \ - &pos->member != (head); \ - pos = n, n = cfs_list_entry(n->member.next, typeof(*n), member)) - -#endif /* cfs_list_for_each_entry_safe */ - -#ifndef cfs_list_for_each_entry_safe_from -/** - * Iterate over a list continuing from an existing point - * \param pos the type * to use as a loop cursor. - * \param n another type * to use as temporary storage - * \param head the head for your list. - * \param member the name of the list_struct within the struct. - * - * Iterate over list of given type from current point, safe against - * removal of list entry. - */ -#define cfs_list_for_each_entry_safe_from(pos, n, head, member) \ - for (n = cfs_list_entry(pos->member.next, typeof(*pos), member); \ - &pos->member != (head); \ - pos = n, n = cfs_list_entry(n->member.next, typeof(*n), member)) -#endif /* cfs_list_for_each_entry_safe_from */ - -#define cfs_list_for_each_entry_typed(pos, head, type, member) \ - for (pos = cfs_list_entry((head)->next, type, member), \ - prefetch(pos->member.next); \ - &pos->member != (head); \ - pos = cfs_list_entry(pos->member.next, type, member), \ - prefetch(pos->member.next)) - -#define cfs_list_for_each_entry_reverse_typed(pos, head, type, member) \ - for (pos = cfs_list_entry((head)->prev, type, member); \ - prefetch(pos->member.prev), &pos->member != (head); \ - pos = cfs_list_entry(pos->member.prev, type, member)) - -#define cfs_list_for_each_entry_safe_typed(pos, n, head, type, member) \ - for (pos = cfs_list_entry((head)->next, type, member), \ - n = cfs_list_entry(pos->member.next, type, member); \ - &pos->member != (head); \ - pos = n, n = cfs_list_entry(n->member.next, type, member)) - -#define cfs_list_for_each_entry_safe_from_typed(pos, n, head, type, member) \ - for (n = cfs_list_entry(pos->member.next, type, member); \ - &pos->member != (head); \ - pos = n, n = cfs_list_entry(n->member.next, type, member)) - -#define cfs_hlist_for_each_entry_typed(tpos, pos, head, type, member) \ - for (pos = (head)->first; \ - pos && (prefetch(pos->next), 1) && \ - (tpos = cfs_hlist_entry(pos, type, member), 1); \ - pos = pos->next) - -#define cfs_hlist_for_each_entry_safe_typed(tpos, pos, n, head, type, member) \ - for (pos = (head)->first; \ - pos && (n = pos->next, 1) && \ - (tpos = cfs_hlist_entry(pos, type, member), 1); \ - pos = n) - -#endif /* __DAOS_LIST_H__ */ From 5622aabf467a3ec7110a678426a06c2469f60d80 Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Mon, 9 Sep 2019 20:14:12 +0000 Subject: [PATCH 160/206] missed Makefile.am update Signed-off-by: Mohamad Chaarawi --- src/Makefile.am | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Makefile.am b/src/Makefile.am index 966508cf..0de3b4b4 100755 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -71,7 +71,7 @@ extraLDADD += -lrados endif if USE_DAOS_AIORI -extraSOURCES += aiori-DAOS.c aiori-DFS.c list.h +extraSOURCES += aiori-DAOS.c aiori-DFS.c endif if USE_GFARM_AIORI From 73dbda09c6e3376d309918da079af7104975bb5f Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Mon, 9 Sep 2019 20:19:42 +0000 Subject: [PATCH 161/206] remove printf added Signed-off-by: Mohamad Chaarawi --- src/option.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/option.c b/src/option.c index 096e5c0b..41ed09da 100644 --- a/src/option.c +++ b/src/option.c @@ -391,7 +391,6 @@ int option_parse(int argc, char ** argv, options_all_t * opt_all){ } if( requiredArgsSeen != requiredArgsNeeded ){ - printf("Seen = %d, needed = %d\n", requiredArgsSeen, requiredArgsNeeded); printf("Error: Missing some required arguments\n\n"); printhelp = 1; } From fe9d76ddf3a2662f53043b79e632bb175a392297 Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Mon, 9 Sep 2019 22:03:55 +0000 Subject: [PATCH 162/206] fix segfault when no API is specified to mdtest Signed-off-by: Mohamad Chaarawi --- src/mdtest.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mdtest.c b/src/mdtest.c index 3a48bc96..17f6e5cf 100644 --- a/src/mdtest.c +++ b/src/mdtest.c @@ -1548,7 +1548,7 @@ void display_freespace(char *testdirpath) strcpy(dirpath, "."); } - if (strcasecmp(param.api, "DFS") == 0) + if (param.api && strcasecmp(param.api, "DFS") == 0) return; VERBOSE(3,5,"Before show_file_system_size, dirpath is '%s'", dirpath ); From c58ba8ffb581384561533218c05162e18ecae2df Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Tue, 10 Sep 2019 14:11:55 +0000 Subject: [PATCH 163/206] add break that was accidentally removed Signed-off-by: Mohamad Chaarawi --- src/option.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/option.c b/src/option.c index 41ed09da..26c7b2db 100644 --- a/src/option.c +++ b/src/option.c @@ -316,6 +316,7 @@ static void option_parse_token(char ** argv, int * flag_parsed_next, int * requi if(strlen(arg) > 1){ printf("Error, ignoring remainder of string for option %c (%s).\n", o->shortVar, o->longVar); } + break; } case('l'):{ *(long long*) o->variable = string_to_bytes(arg); From 12284ae04af584e293dde105a4b2a564d551395b Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Tue, 10 Sep 2019 18:39:31 +0000 Subject: [PATCH 164/206] Add latency and iops numbers to each iteration. - Latency reported is computed by taking the average latency of all ops from a single task, then taking the minimum of that between all tasks. - IOPS is computed by taking the total number of ops across all tasks divided by the total access time to execute those ops. Signed-off-by: Mohamad Chaarawi --- src/ior-internal.h | 3 ++- src/ior-output.c | 11 +++++++---- src/ior.c | 30 +++++++++++++++++++++++------- 3 files changed, 32 insertions(+), 12 deletions(-) diff --git a/src/ior-internal.h b/src/ior-internal.h index 6b89af10..7daf8de6 100644 --- a/src/ior-internal.h +++ b/src/ior-internal.h @@ -20,7 +20,8 @@ void PrintLongSummaryOneTest(IOR_test_t *test); void DisplayFreespace(IOR_param_t * test); void GetTestFileName(char *, IOR_param_t *); void PrintRemoveTiming(double start, double finish, int rep); -void PrintReducedResult(IOR_test_t *test, int access, double bw, double *diff_subset, double totalTime, int rep); +void PrintReducedResult(IOR_test_t *test, int access, double bw, double iops, double latency, + double *diff_subset, double totalTime, int rep); void PrintTestEnds(); void PrintTableHeader(); /* End of ior-output */ diff --git a/src/ior-output.c b/src/ior-output.c index 560d995b..c3e0cb20 100644 --- a/src/ior-output.c +++ b/src/ior-output.c @@ -18,8 +18,8 @@ static void PrintNextToken(); void PrintTableHeader(){ if (outputFormat == OUTPUT_DEFAULT){ fprintf(out_resultfile, "\n"); - fprintf(out_resultfile, "access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter\n"); - fprintf(out_resultfile, "------ --------- ---------- --------- -------- -------- -------- -------- ----\n"); + fprintf(out_resultfile, "access bw(MiB/s) IOPS Latency(s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter\n"); + fprintf(out_resultfile, "------ --------- ---- ---------- ---------- --------- -------- -------- -------- -------- ----\n"); } } @@ -219,10 +219,13 @@ void PrintTestEnds(){ PrintEndSection(); } -void PrintReducedResult(IOR_test_t *test, int access, double bw, double *diff_subset, double totalTime, int rep){ +void PrintReducedResult(IOR_test_t *test, int access, double bw, double iops, double latency, + double *diff_subset, double totalTime, int rep){ if (outputFormat == OUTPUT_DEFAULT){ fprintf(out_resultfile, "%-10s", access == WRITE ? "write" : "read"); PPDouble(1, bw / MEBIBYTE, " "); + PPDouble(1, iops, " "); + PPDouble(1, latency, " "); PPDouble(1, (double)test->params.blockSize / KIBIBYTE, " "); PPDouble(1, (double)test->params.transferSize / KIBIBYTE, " "); PPDouble(1, diff_subset[0], " "); @@ -772,7 +775,7 @@ void PrintRemoveTiming(double start, double finish, int rep) return; if (outputFormat == OUTPUT_DEFAULT){ - fprintf(out_resultfile, "remove - - - - - - "); + fprintf(out_resultfile, "remove - - - - - - - - "); PPDouble(1, finish-start, " "); fprintf(out_resultfile, "%-4d\n", rep); }else if (outputFormat == OUTPUT_JSON){ diff --git a/src/ior.c b/src/ior.c index 2d082342..ef8b44c4 100755 --- a/src/ior.c +++ b/src/ior.c @@ -841,8 +841,9 @@ ReduceIterResults(IOR_test_t *test, double *timer, const int rep, const int acce { double reduced[IOR_NB_TIMERS] = { 0 }; double diff[IOR_NB_TIMERS / 2 + 1]; - double totalTime; - double bw; + double totalTime, accessTime; + IOR_param_t *params = &test->params; + double bw, iops, latency, minlatency; int i; MPI_Op op; @@ -856,15 +857,12 @@ ReduceIterResults(IOR_test_t *test, double *timer, const int rep, const int acce op, 0, testComm), "MPI_Reduce()"); } - /* Only rank 0 tallies and prints the results. */ - if (rank != 0) - return; - /* Calculate elapsed times and throughput numbers */ for (i = 0; i < IOR_NB_TIMERS / 2; i++) diff[i] = reduced[2 * i + 1] - reduced[2 * i]; totalTime = reduced[5] - reduced[0]; + accessTime = reduced[3] - reduced[2]; IOR_point_t *point = (access == WRITE) ? &test->results[rep].write : &test->results[rep].read; @@ -875,7 +873,25 @@ ReduceIterResults(IOR_test_t *test, double *timer, const int rep, const int acce return; bw = (double)point->aggFileSizeForBW / totalTime; - PrintReducedResult(test, access, bw, diff, totalTime, rep); + + /* For IOPS in this iteration, we divide the total amount of IOs from + * all ranks over the entire access time (first start -> last end). */ + iops = (point->aggFileSizeForBW / params->transferSize) / accessTime; + + /* For Latency, we divide the total access time for each task over the + * number of I/Os issued from that task; then reduce and display the + * minimum (best) latency achieved. So what is reported is the average + * latency of all ops from a single task, then taking the minimum of + * that between all tasks. */ + latency = (timer[3] - timer[2]) / (params->blockSize / params->transferSize); + MPI_CHECK(MPI_Reduce(&latency, &minlatency, 1, MPI_DOUBLE, + MPI_MIN, 0, testComm), "MPI_Reduce()"); + + /* Only rank 0 tallies and prints the results. */ + if (rank != 0) + return; + + PrintReducedResult(test, access, bw, iops, latency, diff, totalTime, rep); } /* From 0d0df855e689a9e494c165a2f956f3ce88d07a9f Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Wed, 18 Sep 2019 19:50:43 +0000 Subject: [PATCH 165/206] update user guide with IOPS and latency numbers for each iteration. Signed-off-by: Mohamad Chaarawi --- doc/USER_GUIDE | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/doc/USER_GUIDE b/doc/USER_GUIDE index b8888b9f..62a68025 100755 --- a/doc/USER_GUIDE +++ b/doc/USER_GUIDE @@ -550,6 +550,17 @@ HOW DOES IOR CALCULATE PERFORMANCE? operations (-g), the sum of the open, transfer, and close times may not equal the elapsed time from the first open to the last close. + After each iteration (-i) IOR reports performance for that iteration, and + those numbers include: + + - Bandwidth (described above) + + - IOPS: I/O rate (operations per second) achieved by all tasks given the total + time spent in reading and writing the data. + + - Latency: computed by taking the average latency of all I/O operations from a + single task. If ior is run with multiple tasks, then the latency reported is + the minimum that was computed between all tasks. HOW DO I ACCESS MULTIPLE FILE SYSTEMS IN IOR? From fca0a62ccba1d374a54d23fd6d95ee2b8ddb39a4 Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Sat, 21 Sep 2019 22:05:54 +0000 Subject: [PATCH 166/206] update to new DAOS API for creating dfs containers. Signed-off-by: Mohamad Chaarawi --- src/aiori-DFS.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/aiori-DFS.c b/src/aiori-DFS.c index 28384403..eb15b2ab 100755 --- a/src/aiori-DFS.c +++ b/src/aiori-DFS.c @@ -419,13 +419,12 @@ DFS_Init() { if (rc == -DER_NONEXIST) { INFO(VERBOSE_1, "Creating DFS Container ...\n"); - rc = daos_cont_create(poh, co_uuid, NULL, NULL); - if (rc == 0) { - rc = daos_cont_open(poh, co_uuid, DAOS_COO_RW, - &coh, &co_info, NULL); - } + rc = dfs_cont_create(poh, co_uuid, NULL, &coh, NULL); + if (rc) + DCHECK(rc, "Failed to create container"); + } else if (rc) { + DCHECK(rc, "Failed to create container"); } - DCHECK(rc, "Failed to create container"); } HandleDistribute(&poh, POOL_HANDLE); From 2aea04c8cb148f1a9ecf78a7668b59c3fcd92b9c Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Sat, 21 Sep 2019 22:43:20 +0000 Subject: [PATCH 167/206] fix some compiler warnings. Signed-off-by: Mohamad Chaarawi --- src/aiori-DAOS.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/aiori-DAOS.c b/src/aiori-DAOS.c index 21df9aad..19a9f648 100644 --- a/src/aiori-DAOS.c +++ b/src/aiori-DAOS.c @@ -241,7 +241,6 @@ DAOS_Init() if (rank == 0) { uuid_t uuid; d_rank_list_t *svcl = NULL; - d_rank_list_t ranks; static daos_pool_info_t po_info; static daos_cont_info_t co_info; @@ -368,6 +367,8 @@ DAOS_Create(char *testFileName, IOR_param_t *param) /** Distribute the array handle if not FPP */ if (!param->filePerProc) HandleDistribute(&aoh, ARRAY_HANDLE); + + return &aoh; } static int @@ -417,6 +418,8 @@ DAOS_Open(char *testFileName, IOR_param_t *param) /** Distribute the array handle if not FPP */ if (!param->filePerProc) HandleDistribute(&aoh, ARRAY_HANDLE); + + return &aoh; } static IOR_offset_t From b3ecd756cec6603bc04bc8bf2506fcf9655cb8f6 Mon Sep 17 00:00:00 2001 From: Petros Koutoupis Date: Mon, 30 Sep 2019 04:25:59 -0700 Subject: [PATCH 168/206] Implemented AC_CHECK_HEADERS fix in style of #190 --- configure.ac | 3 ++- src/mdtest.c | 28 ++++++++++++++-------------- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/configure.ac b/configure.ac index 537021b7..7ad26946 100755 --- a/configure.ac +++ b/configure.ac @@ -91,7 +91,8 @@ AS_IF([test "x$with_lustre" = xyes ], [ AC_MSG_FAILURE([--with-lustre was given, not found]) fi ]) - AC_CHECK_HEADERS([linux/lustre/lustreapi.h lustre/lustreapi.h], break, [ + AC_CHECK_HEADERS([linux/lustre/lustreapi.h lustre/lustreapi.h], + [AC_DEFINE([HAVE_LUSTRE_LUSTREAPI], [], [Lustre user API available in some shape or form])], [ if test "x$with_lustre" != xcheck -a \ "x$ac_cv_header_linux_lustre_lustreapi_h" = "xno" -a \ "x$ac_cv_header_lustre_lustreapi_h" = "xno" ; then diff --git a/src/mdtest.c b/src/mdtest.c index 444f3916..9bc08adb 100644 --- a/src/mdtest.c +++ b/src/mdtest.c @@ -76,9 +76,9 @@ #include -#ifdef HAVE_LUSTRE_LUSTREAPI_H +#ifdef HAVE_LUSTRE_LUSTREAPI #include -#endif /* HAVE_LUSTRE_LUSTREAPI_H */ +#endif /* HAVE_LUSTRE_LUSTREAPI */ #define FILEMODE S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH #define DIRMODE S_IRUSR|S_IWUSR|S_IXUSR|S_IRGRP|S_IWGRP|S_IXGRP|S_IROTH|S_IXOTH @@ -156,9 +156,9 @@ static int call_sync; static int path_count; static int nstride; /* neighbor stride */ static int make_node = 0; -#ifdef HAVE_LUSTRE_LUSTREAPI_H +#ifdef HAVE_LUSTRE_LUSTREAPI static int global_dir_layout; -#endif /* HAVE_LUSTRE_LUSTREAPI_H */ +#endif /* HAVE_LUSTRE_LUSTREAPI */ static mdtest_results_t * summary_table; static pid_t pid; @@ -1582,7 +1582,7 @@ void create_remove_directory_tree(int create, if (-1 == backend->mkdir (dir, DIRMODE, ¶m)) { fprintf(out_logfile, "error could not create directory '%s'\n", dir); } -#ifdef HAVE_LUSTRE_LUSTREAPI_H +#ifdef HAVE_LUSTRE_LUSTREAPI /* internal node for branching, can be non-striped for children */ if (global_dir_layout && \ llapi_dir_set_default_lmv_stripe(dir, -1, 0, @@ -1590,7 +1590,7 @@ void create_remove_directory_tree(int create, NULL) == -1) { FAIL("Unable to reset to global default directory layout"); } -#endif /* HAVE_LUSTRE_LUSTREAPI_H */ +#endif /* HAVE_LUSTRE_LUSTREAPI */ } create_remove_directory_tree(create, ++currDepth, dir, ++dirNum, progress); @@ -1657,12 +1657,12 @@ static void mdtest_iteration(int i, int j, MPI_Group testgroup, mdtest_results_t if (backend->mkdir(testdir, DIRMODE, ¶m) != 0) { FAIL("Unable to create test directory %s", testdir); } -#ifdef HAVE_LUSTRE_LUSTREAPI_H +#ifdef HAVE_LUSTRE_LUSTREAPI /* internal node for branching, can be non-striped for children */ if (global_dir_layout && unique_dir_per_task && llapi_dir_set_default_lmv_stripe(testdir, -1, 0, LMV_HASH_TYPE_FNV_1A_64, NULL) == -1) { FAIL("Unable to reset to global default directory layout"); } -#endif /* HAVE_LUSTRE_LUSTREAPI_H */ +#endif /* HAVE_LUSTRE_LUSTREAPI */ } } @@ -1881,9 +1881,9 @@ void mdtest_init_args(){ path_count = 0; nstride = 0; make_node = 0; -#ifdef HAVE_LUSTRE_LUSTREAPI_H +#ifdef HAVE_LUSTRE_LUSTREAPI global_dir_layout = 0; -#endif /* HAVE_LUSTRE_LUSTREAPI_H */ +#endif /* HAVE_LUSTRE_LUSTREAPI */ } mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * world_out) { @@ -1930,9 +1930,9 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * {'e', NULL, "bytes to read from each file", OPTION_OPTIONAL_ARGUMENT, 'l', & read_bytes}, {'f', NULL, "first number of tasks on which the test will run", OPTION_OPTIONAL_ARGUMENT, 'd', & first}, {'F', NULL, "perform test on files only (no directories)", OPTION_FLAG, 'd', & files_only}, -#ifdef HAVE_LUSTRE_LUSTREAPI_H +#ifdef HAVE_LUSTRE_LUSTREAPI {'g', NULL, "global default directory layout for test subdirectories (deletes inherited striping layout)", OPTION_FLAG, 'd', & global_dir_layout}, -#endif /* HAVE_LUSTRE_LUSTREAPI_H */ +#endif /* HAVE_LUSTRE_LUSTREAPI */ {'i', NULL, "number of iterations the test will run", OPTION_OPTIONAL_ARGUMENT, 'd', & iterations}, {'I', NULL, "number of items per directory in tree", OPTION_OPTIONAL_ARGUMENT, 'l', & items_per_dir}, {'k', NULL, "use mknod to create file", OPTION_FLAG, 'd', & make_node}, @@ -2026,9 +2026,9 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * VERBOSE(1,-1, "read_only : %s", ( read_only ? "True" : "False" )); VERBOSE(1,-1, "first : %d", first ); VERBOSE(1,-1, "files_only : %s", ( files_only ? "True" : "False" )); -#ifdef HAVE_LUSTRE_LUSTREAPI_H +#ifdef HAVE_LUSTRE_LUSTREAPI VERBOSE(1,-1, "global_dir_layout : %s", ( global_dir_layout ? "True" : "False" )); -#endif /* HAVE_LUSTRE_LUSTREAPI_H */ +#endif /* HAVE_LUSTRE_LUSTREAPI */ VERBOSE(1,-1, "iterations : %d", iterations ); VERBOSE(1,-1, "items_per_dir : "LLU"", items_per_dir ); VERBOSE(1,-1, "last : %d", last ); From a69a5916cf0a0572669ae5496fc85ca9038ef7de Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Wed, 16 Oct 2019 09:42:10 +0100 Subject: [PATCH 169/206] Hotfix for ior -F -u --- src/ior.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ior.c b/src/ior.c index d6b04182..30c3d287 100755 --- a/src/ior.c +++ b/src/ior.c @@ -130,7 +130,8 @@ int ior_main(int argc, char **argv) for (tptr = tests_head; tptr != NULL; tptr = tptr->next) { verbose = tptr->params.verbose; if (rank == 0 && verbose >= VERBOSE_0) { - ShowTestStart(&tptr->params); + backend = tptr->params.backend; + ShowTestStart(&tptr->params); } // This is useful for trapping a running MPI process. While From dc82a1bf2fb49203879af6f37527b1f86cbfd653 Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Wed, 16 Oct 2019 14:54:00 +0000 Subject: [PATCH 170/206] ior -R should memset the buffer being read, otherwise a read that does nothing will report success in the data verification phase. Signed-off-by: Mohamad Chaarawi --- src/ior.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/ior.c b/src/ior.c index ef8b44c4..658c3aed 100755 --- a/src/ior.c +++ b/src/ior.c @@ -1882,14 +1882,16 @@ static IOR_offset_t WriteOrReadSingle(IOR_offset_t pairCnt, IOR_offset_t *offset *transferCount, test, WRITECHECK); } else if (access == READCHECK) { - amtXferred = backend->xfer(access, fd, buffer, transfer, test); + memset(checkBuffer, 'a', transfer); + + amtXferred = backend->xfer(access, fd, checkBuffer, transfer, test); if (amtXferred != transfer){ ERR("cannot read from file"); } if (test->storeFileOffset == TRUE) { FillBuffer(readCheckBuffer, test, test->offset, pretendRank); } - *errors += CompareBuffers(readCheckBuffer, buffer, transfer, *transferCount, test, READCHECK); + *errors += CompareBuffers(readCheckBuffer, checkBuffer, transfer, *transferCount, test, READCHECK); } return amtXferred; } From 92180e022dbdd05ed9ebf3e75d87cbc969e4fcde Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Mon, 28 Oct 2019 16:42:44 +0000 Subject: [PATCH 171/206] update DFS read/write APIs with new DAOS API changes Signed-off-by: Mohamad Chaarawi --- src/aiori-DFS.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/aiori-DFS.c b/src/aiori-DFS.c index eb15b2ab..6b2f28ca 100755 --- a/src/aiori-DFS.c +++ b/src/aiori-DFS.c @@ -601,14 +601,14 @@ DFS_Xfer(int access, void *file, IOR_size_t *buffer, IOR_offset_t length, /* write/read file */ if (access == WRITE) { - rc = dfs_write(dfs, obj, sgl, param->offset); + rc = dfs_write(dfs, obj, &sgl, param->offset, NULL); if (rc) { fprintf(stderr, "dfs_write() failed (%d)", rc); return -1; } ret = remaining; } else { - rc = dfs_read(dfs, obj, sgl, param->offset, &ret); + rc = dfs_read(dfs, obj, &sgl, param->offset, &ret, NULL); if (rc || ret == 0) fprintf(stderr, "dfs_read() failed(%d)", rc); } From 847b8ad408c67294745136720c818eac74ba194f Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Mon, 4 Nov 2019 22:57:47 +0000 Subject: [PATCH 172/206] fix config error with older autocont/automake versions. Signed-off-by: Mohamad Chaarawi --- configure.ac | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/configure.ac b/configure.ac index f6b958b4..bfd88cf7 100755 --- a/configure.ac +++ b/configure.ac @@ -189,23 +189,22 @@ AM_COND_IF([USE_RADOS_AIORI],[ AC_ARG_WITH([cart], [AS_HELP_STRING([--with-cart], [support IO with DAOS backends @<:@default=no@:>@])], - [], - [with_daos=no]) + [], [with_daos=no]) -AS_IF([test "x$with_cart" != xno], +AS_IF([test "x$with_cart" != xno], [ CART="yes" LDFLAGS="$LDFLAGS -L$with_cart/lib" CPPFLAGS="$CPPFLAGS -I$with_cart/include/" AC_CHECK_HEADERS(gurt/common.h,, [unset CART]) - AC_CHECK_LIB([gurt], [d_hash_murmur64],, [unset CART])) + AC_CHECK_LIB([gurt], [d_hash_murmur64],, [unset CART]) +]) AC_ARG_WITH([daos], [AS_HELP_STRING([--with-daos], [support IO with DAOS backends @<:@default=no@:>@])], - [], - [with_daos=no]) + [], [with_daos=no]) -AS_IF([test "x$with_daos" != xno], +AS_IF([test "x$with_daos" != xno], [ DAOS="yes" LDFLAGS="$LDFLAGS -L$with_daos/lib" CPPFLAGS="$CPPFLAGS -I$with_daos/include" @@ -213,7 +212,8 @@ AS_IF([test "x$with_daos" != xno], AC_CHECK_LIB([uuid], [uuid_generate],, [unset DAOS]) AC_CHECK_LIB([daos_common], [daos_sgl_init],, [unset DAOS]) AC_CHECK_LIB([daos], [daos_init],, [unset DAOS]) - AC_CHECK_LIB([dfs], [dfs_mkdir],, [unset DAOS])) + AC_CHECK_LIB([dfs], [dfs_mkdir],, [unset DAOS]) +]) AM_CONDITIONAL([USE_DAOS_AIORI], [test x$DAOS = xyes]) AM_COND_IF([USE_DAOS_AIORI],[ From d926bf1c9062d83206e9973d4b2ffa1b080a824d Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Wed, 6 Nov 2019 23:10:45 +0000 Subject: [PATCH 173/206] update configure.ac to look under lib64 for DAOS libs Signed-off-by: Mohamad Chaarawi --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index bfd88cf7..acacc5d8 100755 --- a/configure.ac +++ b/configure.ac @@ -206,7 +206,7 @@ AC_ARG_WITH([daos], AS_IF([test "x$with_daos" != xno], [ DAOS="yes" - LDFLAGS="$LDFLAGS -L$with_daos/lib" + LDFLAGS="$LDFLAGS -L$with_daos/lib64 -L$with_daos/lib" CPPFLAGS="$CPPFLAGS -I$with_daos/include" AC_CHECK_HEADERS(daos_types.h,, [unset DAOS]) AC_CHECK_LIB([uuid], [uuid_generate],, [unset DAOS]) From fbff9b6fb23b2b24cbf233aa3902c45a1f61f297 Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Thu, 7 Nov 2019 18:01:07 +0000 Subject: [PATCH 174/206] add DFS prefix option Signed-off-by: Mohamad Chaarawi --- src/aiori-DFS.c | 40 ++++++++++++++-------------------------- 1 file changed, 14 insertions(+), 26 deletions(-) diff --git a/src/aiori-DFS.c b/src/aiori-DFS.c index 6b2f28ca..28c9096d 100755 --- a/src/aiori-DFS.c +++ b/src/aiori-DFS.c @@ -68,6 +68,7 @@ struct dfs_options{ char *cont; int chunk_size; char *oclass; + char *prefix; int destroy; }; @@ -78,6 +79,7 @@ static struct dfs_options o = { .cont = NULL, .chunk_size = 1048576, .oclass = NULL, + .prefix = NULL, .destroy = 0, }; @@ -88,6 +90,7 @@ static option_help options [] = { {0, "dfs.cont", "DFS container uuid", OPTION_OPTIONAL_ARGUMENT, 's', & o.cont}, {0, "dfs.chunk_size", "chunk size", OPTION_OPTIONAL_ARGUMENT, 'd', &o.chunk_size}, {0, "dfs.oclass", "object class", OPTION_OPTIONAL_ARGUMENT, 's', &o.oclass}, + {0, "dfs.prefix", "mount prefix", OPTION_OPTIONAL_ARGUMENT, 's', & o.prefix}, {0, "dfs.destroy", "Destroy DFS Container", OPTION_FLAG, 'd', &o.destroy}, LAST_OPTION }; @@ -356,7 +359,8 @@ lookup_insert_dir(const char *name) hdl->name[PATH_MAX-1] = '\0'; rc = dfs_lookup(dfs, name, O_RDWR, &hdl->oh, NULL, NULL); - DCHECK(rc, "dfs_lookup() of %s Failed", name); + if (rc) + return NULL; rc = d_hash_rec_insert(dir_hash, hdl->name, strlen(hdl->name), &hdl->entry, true); @@ -432,6 +436,11 @@ DFS_Init() { rc = dfs_mount(poh, coh, O_RDWR, &dfs); DCHECK(rc, "Failed to mount DFS namespace"); + + if (o.prefix) { + rc = dfs_set_prefix(dfs, o.prefix); + DCHECK(rc, "Failed to set DFS Prefix"); + } } static void @@ -800,33 +809,12 @@ DFS_Rmdir(const char *path, IOR_param_t * param) static int DFS_Access(const char *path, int mode, IOR_param_t * param) { - dfs_obj_t *parent = NULL; - char *name = NULL, *dir_name = NULL; - struct stat stbuf; - int rc; - - rc = parse_filename(path, &name, &dir_name); - DCHECK(rc, "Failed to parse path %s", path); - - assert(dir_name); - - parent = lookup_insert_dir(dir_name); - if (parent == NULL) - GERR("Failed to lookup parent dir"); + dfs_obj_t *obj = NULL; - if (name && strcmp(name, ".") == 0) { - free(name); - name = NULL; - } - rc = dfs_stat(dfs, parent, name, &stbuf); - - if (name) - free(name); - if (dir_name) - free(dir_name); - if (rc) + obj = lookup_insert_dir(path); + if (obj == NULL) return -1; - return rc; + return 0; } static int From 47199e28ddef6515aa1bb09d836bc4bbda58c238 Mon Sep 17 00:00:00 2001 From: Robert LeBlanc Date: Mon, 11 Nov 2019 23:16:01 +0000 Subject: [PATCH 175/206] Not sure why iops and latency are left out of the JSON job output. It is really useful information. Signed-off by: Robert LeBlanc --- src/ior-output.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/ior-output.c b/src/ior-output.c index 76daf6c1..99eeac6a 100644 --- a/src/ior-output.c +++ b/src/ior-output.c @@ -239,6 +239,8 @@ void PrintReducedResult(IOR_test_t *test, int access, double bw, double iops, do PrintKeyValDouble("bwMiB", bw / MEBIBYTE); PrintKeyValDouble("blockKiB", (double)test->params.blockSize / KIBIBYTE); PrintKeyValDouble("xferKiB", (double)test->params.transferSize / KIBIBYTE); + PrintKeyValDouble("iops", iops); + PrintKeyValDouble("latency", latency); PrintKeyValDouble("openTime", diff_subset[0]); PrintKeyValDouble("wrRdTime", diff_subset[1]); PrintKeyValDouble("closeTime", diff_subset[2]); From acfd4673ccd73eda42914135f64da5a8d8ee7fcc Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Wed, 13 Nov 2019 00:09:03 +0000 Subject: [PATCH 176/206] We only need to add dirs to the Hash in the DFS driver. Signed-off-by: Mohamad Chaarawi --- src/aiori-DFS.c | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/src/aiori-DFS.c b/src/aiori-DFS.c index 28c9096d..9b9859a5 100755 --- a/src/aiori-DFS.c +++ b/src/aiori-DFS.c @@ -339,7 +339,7 @@ parse_filename(const char *path, char **_obj_name, char **_cont_name) } static dfs_obj_t * -lookup_insert_dir(const char *name) +lookup_insert_dir(const char *name, mode_t *mode) { struct aiori_dir_hdl *hdl; d_list_t *rlink; @@ -358,9 +358,11 @@ lookup_insert_dir(const char *name) strncpy(hdl->name, name, PATH_MAX-1); hdl->name[PATH_MAX-1] = '\0'; - rc = dfs_lookup(dfs, name, O_RDWR, &hdl->oh, NULL, NULL); + rc = dfs_lookup(dfs, name, O_RDWR, &hdl->oh, mode, NULL); if (rc) return NULL; + if (mode && S_ISREG(*mode)) + return hdl->oh; rc = d_hash_rec_insert(dir_hash, hdl->name, strlen(hdl->name), &hdl->entry, true); @@ -515,7 +517,7 @@ DFS_Create(char *testFileName, IOR_param_t *param) assert(dir_name); assert(name); - parent = lookup_insert_dir(dir_name); + parent = lookup_insert_dir(dir_name, NULL); if (parent == NULL) GERR("Failed to lookup parent dir"); @@ -566,7 +568,7 @@ DFS_Open(char *testFileName, IOR_param_t *param) assert(dir_name); assert(name); - parent = lookup_insert_dir(dir_name); + parent = lookup_insert_dir(dir_name, NULL); if (parent == NULL) GERR("Failed to lookup parent dir"); @@ -674,7 +676,7 @@ DFS_Delete(char *testFileName, IOR_param_t * param) assert(dir_name); assert(name); - parent = lookup_insert_dir(dir_name); + parent = lookup_insert_dir(dir_name, NULL); if (parent == NULL) GERR("Failed to lookup parent dir"); @@ -761,7 +763,7 @@ DFS_Mkdir(const char *path, mode_t mode, IOR_param_t * param) if (!name) return 0; - parent = lookup_insert_dir(dir_name); + parent = lookup_insert_dir(dir_name, NULL); if (parent == NULL) GERR("Failed to lookup parent dir"); @@ -790,7 +792,7 @@ DFS_Rmdir(const char *path, IOR_param_t * param) assert(dir_name); assert(name); - parent = lookup_insert_dir(dir_name); + parent = lookup_insert_dir(dir_name, NULL); if (parent == NULL) GERR("Failed to lookup parent dir"); @@ -810,10 +812,16 @@ static int DFS_Access(const char *path, int mode, IOR_param_t * param) { dfs_obj_t *obj = NULL; + mode_t fmode; - obj = lookup_insert_dir(path); + obj = lookup_insert_dir(path, &fmode); if (obj == NULL) return -1; + + /** just close if it's a file */ + if (S_ISREG(fmode)) + dfs_release(obj); + return 0; } @@ -830,7 +838,7 @@ DFS_Stat(const char *path, struct stat *buf, IOR_param_t * param) assert(dir_name); assert(name); - parent = lookup_insert_dir(dir_name); + parent = lookup_insert_dir(dir_name, NULL); if (parent == NULL) GERR("Failed to lookup parent dir"); From be48206e5fba905bfd7d5c4e26b04fab67b3e4d0 Mon Sep 17 00:00:00 2001 From: Jean-Yves VET Date: Wed, 30 Oct 2019 15:43:43 +0100 Subject: [PATCH 177/206] Fix compilation warnings --- src/aiori-POSIX.c | 4 ++-- src/ior.c | 4 +--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/aiori-POSIX.c b/src/aiori-POSIX.c index 99a68f11..901eed5a 100755 --- a/src/aiori-POSIX.c +++ b/src/aiori-POSIX.c @@ -171,7 +171,7 @@ void gpfs_access_start(int fd, IOR_offset_t length, IOR_param_t *param, int acce rc = gpfs_fcntl(fd, &take_locks); if (verbose >= VERBOSE_2 && rc != 0) { - EWARNF("gpfs_fcntl(fd, ...) access range hint failed.", fd); + EWARNF("gpfs_fcntl(%d, ...) access range hint failed.", fd); } } @@ -195,7 +195,7 @@ void gpfs_access_end(int fd, IOR_offset_t length, IOR_param_t *param, int access rc = gpfs_fcntl(fd, &free_locks); if (verbose >= VERBOSE_2 && rc != 0) { - EWARNF("gpfs_fcntl(fd, ...) free range hint failed.", fd); + EWARNF("gpfs_fcntl(%d, ...) free range hint failed.", fd); } } diff --git a/src/ior.c b/src/ior.c index e811f83d..57f385d5 100755 --- a/src/ior.c +++ b/src/ior.c @@ -503,8 +503,6 @@ void AllocResults(IOR_test_t *test) if (test->results != NULL) return; - IOR_param_t * params = & test->params; - reps = test->params.repetitions; test->results = (IOR_results_t *) safeMalloc(sizeof(IOR_results_t) * reps); } @@ -886,7 +884,7 @@ ReduceIterResults(IOR_test_t *test, double *timer, const int rep, const int acce * number of I/Os issued from that task; then reduce and display the * minimum (best) latency achieved. So what is reported is the average * latency of all ops from a single task, then taking the minimum of - * that between all tasks. */ + * that between all tasks. */ latency = (timer[3] - timer[2]) / (params->blockSize / params->transferSize); MPI_CHECK(MPI_Reduce(&latency, &minlatency, 1, MPI_DOUBLE, MPI_MIN, 0, testComm), "MPI_Reduce()"); From 9da36abacacf8461ccc7601703d050c6de475317 Mon Sep 17 00:00:00 2001 From: Jean-Yves VET Date: Mon, 4 Nov 2019 17:10:36 +0100 Subject: [PATCH 178/206] Fix confusing output of mknod Context: mknod does not return a file descriptor. --- src/aiori-POSIX.c | 16 ++++++---------- src/aiori.h | 4 ++-- src/mdtest.c | 40 +++++++++++++++++++--------------------- 3 files changed, 27 insertions(+), 33 deletions(-) diff --git a/src/aiori-POSIX.c b/src/aiori-POSIX.c index 99a68f11..b57601ad 100755 --- a/src/aiori-POSIX.c +++ b/src/aiori-POSIX.c @@ -436,19 +436,15 @@ void *POSIX_Create(char *testFileName, IOR_param_t * param) /* * Creat a file through mknod interface. */ -void *POSIX_Mknod(char *testFileName) +int POSIX_Mknod(char *testFileName) { - int *fd; + int ret; - fd = (int *)malloc(sizeof(int)); - if (fd == NULL) - ERR("Unable to malloc file descriptor"); + ret = mknod(testFileName, S_IFREG | S_IRUSR, 0); + if (ret < 0) + ERR("mknod failed"); - *fd = mknod(testFileName, S_IFREG | S_IRUSR, 0); - if (*fd < 0) - ERR("mknod failed"); - - return ((void *)fd); + return ret; } /* diff --git a/src/aiori.h b/src/aiori.h index da93a1a0..3fc9f06b 100755 --- a/src/aiori.h +++ b/src/aiori.h @@ -68,7 +68,7 @@ typedef struct ior_aiori { char *name; char *name_legacy; void *(*create)(char *, IOR_param_t *); - void *(*mknod)(char *); + int (*mknod)(char *); void *(*open)(char *, IOR_param_t *); IOR_offset_t (*xfer)(int, void *, IOR_size_t *, IOR_offset_t, IOR_param_t *); @@ -131,7 +131,7 @@ int aiori_posix_access (const char *path, int mode, IOR_param_t * param); int aiori_posix_stat (const char *path, struct stat *buf, IOR_param_t * param); void *POSIX_Create(char *testFileName, IOR_param_t * param); -void *POSIX_Mknod(char *testFileName); +int POSIX_Mknod(char *testFileName); void *POSIX_Open(char *testFileName, IOR_param_t * param); IOR_offset_t POSIX_GetFileSize(IOR_param_t * test, MPI_Comm testComm, char *testFileName); void POSIX_Delete(char *testFileName, IOR_param_t * param); diff --git a/src/mdtest.c b/src/mdtest.c index fe9c1531..05e8a44e 100644 --- a/src/mdtest.c +++ b/src/mdtest.c @@ -345,7 +345,7 @@ static void remove_file (const char *path, uint64_t itemNum) { static void create_file (const char *path, uint64_t itemNum) { char curr_item[MAX_PATHLEN]; - void *aiori_fh; + void *aiori_fh = NULL; if ( (itemNum % ITEM_COUNT==0 && (itemNum != 0))) { VERBOSE(3,5,"create file: "LLU"", itemNum); @@ -355,36 +355,36 @@ static void create_file (const char *path, uint64_t itemNum) { sprintf(curr_item, "%s/file.%s"LLU"", path, mk_name, itemNum); VERBOSE(3,5,"create_remove_items_helper (non-dirs create): curr_item is '%s'", curr_item); - if (collective_creates) { - param.openFlags = IOR_WRONLY; + param.openFlags = IOR_WRONLY; + + if (make_node) { + int ret; + VERBOSE(3,5,"create_remove_items_helper : mknod..." ); + + ret = backend->mknod (curr_item); + if (ret != 0) + FAIL("unable to mknode file %s", curr_item); + return; + } else if (collective_creates) { VERBOSE(3,5,"create_remove_items_helper (collective): open..." ); - if (make_node) - aiori_fh = backend->mknod (curr_item); - else - aiori_fh = backend->open (curr_item, ¶m); - if (NULL == aiori_fh) { + aiori_fh = backend->open (curr_item, ¶m); + if (NULL == aiori_fh) FAIL("unable to open file %s", curr_item); - } /* * !collective_creates */ } else { - param.openFlags = IOR_CREAT | IOR_WRONLY; + param.openFlags |= IOR_CREAT; param.filePerProc = !shared_file; - param.mode = FILEMODE; - + param.mode = FILEMODE; VERBOSE(3,5,"create_remove_items_helper (non-collective, shared): open..." ); - if (make_node) - aiori_fh = backend->mknod (curr_item); - else - aiori_fh = backend->create (curr_item, ¶m); - if (NULL == aiori_fh) { + aiori_fh = backend->create (curr_item, ¶m); + if (NULL == aiori_fh) FAIL("unable to create file %s", curr_item); - } } if (write_bytes > 0) { @@ -402,9 +402,7 @@ static void create_file (const char *path, uint64_t itemNum) { } VERBOSE(3,5,"create_remove_items_helper: close..." ); - - if (!make_node) - backend->close (aiori_fh, ¶m); + backend->close (aiori_fh, ¶m); } /* helper for creating/removing items */ From 0666875df3c265137c9d4a7492cdb3e6d7574b34 Mon Sep 17 00:00:00 2001 From: Jean-Yves VET Date: Tue, 17 Dec 2019 14:45:24 +0100 Subject: [PATCH 179/206] aiori-IME: Add support of sync and mknod in IME backend This patch adds the support of sync (flush client page cache for all opened files) and mknod in the IME backend. --- src/aiori-IME.c | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/src/aiori-IME.c b/src/aiori-IME.c index b6cef34d..500f380f 100755 --- a/src/aiori-IME.c +++ b/src/aiori-IME.c @@ -51,6 +51,12 @@ static int IME_StatFS(const char *, ior_aiori_statfs_t *, static int IME_RmDir(const char *, IOR_param_t *); static int IME_MkDir(const char *, mode_t, IOR_param_t *); static int IME_Stat(const char *, struct stat *, IOR_param_t *); + +#if (IME_NATIVE_API_VERSION >= 132) +static int IME_Mknod(char *); +static void IME_Sync(IOR_param_t *); +#endif + static void IME_Initialize(); static void IME_Finalize(); @@ -107,6 +113,10 @@ ior_aiori_t ime_aiori = { .initialize = IME_Initialize, .finalize = IME_Finalize, .get_options = IME_options, +#if (IME_NATIVE_API_VERSION >= 132) + .sync = IME_Sync, + .mknod = IME_Mknod, +#endif .enable_mdtest = true, }; @@ -406,3 +416,27 @@ static IOR_offset_t IME_GetFileSize(IOR_param_t *test, MPI_Comm testComm, return(aggFileSizeFromStat); } + +#if (IME_NATIVE_API_VERSION >= 132) +/* + * Create a file through mknod interface. + */ +static int IME_Mknod(char *testFileName) +{ + int ret = ime_native_mknod(testFileName, S_IFREG | S_IRUSR, 0); + if (ret < 0) + ERR("mknod failed"); + + return ret; +} + +/* + * Use IME sync to flush page cache of all opened files. + */ +static void IME_Sync(IOR_param_t * param) +{ + int ret = ime_native_sync(0); + if (ret != 0) + FAIL("Error executing the sync command."); +} +#endif From 73c5c0efc20f91700a4c293a2f089e6d8b5bde67 Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Thu, 19 Dec 2019 13:42:34 +0000 Subject: [PATCH 180/206] Fix for collective operations by Wolfgang Szoecs #210. It does have performance implications but is only active if needed. --- src/ior.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/ior.c b/src/ior.c index 57f385d5..5e343d11 100755 --- a/src/ior.c +++ b/src/ior.c @@ -1946,6 +1946,13 @@ static IOR_offset_t WriteOrRead(IOR_param_t *test, IOR_results_t *results, hitStonewall = ((test->deadlineForStonewalling != 0 && (GetTimeStamp() - startForStonewall) > test->deadlineForStonewalling)) || (test->stoneWallingWearOutIterations != 0 && pairCnt == test->stoneWallingWearOutIterations) ; + + if ( test->collective && test->deadlineForStonewalling ) { + // if collective-mode, you'll get a HANG, if some rank 'accidentally' leave this loop + // it absolutely must be an 'all or none': + MPI_CHECK(MPI_Bcast(&hitStonewall, 1, MPI_INT, 0, MPI_COMM_WORLD), "hitStonewall broadcast failed"); + } + } if (test->stoneWallingWearOut){ if (verbose >= VERBOSE_1){ From 240d5fae7903acad748af456a6ce7d110f78484c Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Sat, 21 Dec 2019 11:19:02 +0000 Subject: [PATCH 181/206] Trivial fix to ensure all functions are prototypes. --- src/ior.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ior.h b/src/ior.h index ccf47fa7..745a2d36 100755 --- a/src/ior.h +++ b/src/ior.h @@ -241,7 +241,7 @@ typedef struct IOR_test_t { IOR_test_t *CreateTest(IOR_param_t *init_params, int test_num); void AllocResults(IOR_test_t *test); -char * GetPlatformName(); +char * GetPlatformName(void); void init_IOR_Param_t(IOR_param_t *p); /* From 6de5cdc6f9bd9fe5d4b7f92abcb8b9641caf7a50 Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Sat, 21 Dec 2019 14:14:27 +0000 Subject: [PATCH 182/206] Fixed wrong usage of platform, fixed printf output. --- src/ior-output.c | 4 ++-- src/ior.h | 2 +- src/parse_options.c | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ior-output.c b/src/ior-output.c index 99eeac6a..01136fbe 100644 --- a/src/ior-output.c +++ b/src/ior-output.c @@ -262,7 +262,7 @@ void PrintHeader(int argc, char **argv) if (outputFormat != OUTPUT_DEFAULT){ PrintKeyVal("Version", META_VERSION); }else{ - printf("IOR-" META_VERSION ": MPI Coordinated Test of Parallel I/O\n"); + fprintf(out_resultfile, "IOR-" META_VERSION ": MPI Coordinated Test of Parallel I/O\n"); } PrintKeyVal("Began", CurrentTimeString()); PrintKeyValStart("Command line"); @@ -323,7 +323,7 @@ void ShowTestStart(IOR_param_t *test) PrintKeyValInt("TestID", test->id); PrintKeyVal("StartTime", CurrentTimeString()); /* if pvfs2:, then skip */ - if (strcasecmp(test->api, "DFS") && + if (strcasecmp(test->api, "DFS") && Regex(test->testFileName, "^[a-z][a-z].*:") == 0) { DisplayFreespace(test); } diff --git a/src/ior.h b/src/ior.h index 745a2d36..6b9d1d75 100755 --- a/src/ior.h +++ b/src/ior.h @@ -100,7 +100,7 @@ typedef struct int numTasks; /* number of tasks for test */ int numNodes; /* number of nodes for test */ int numTasksOnNode0; /* number of tasks on node 0 (usually all the same, but don't have to be, use with caution) */ - int tasksBlockMapping; /* are the tasks in contiguous blocks across nodes or round-robin */ + int tasksBlockMapping; /* are the tasks in contiguous blocks across nodes or round-robin */ int repetitions; /* number of repetitions of test */ int repCounter; /* rep counter */ int multiFile; /* multiple files */ diff --git a/src/parse_options.c b/src/parse_options.c index 74a7b54a..607d0147 100755 --- a/src/parse_options.c +++ b/src/parse_options.c @@ -544,7 +544,7 @@ IOR_test_t *ParseCommandLine(int argc, char **argv) IOR_test_t *tests = NULL; - GetPlatformName(initialTestParams.platform); + initialTestParams.platform = GetPlatformName(); option_help * options = createGlobalOptions( & initialTestParams); parameters = & initialTestParams; From c702a98376a02841bbc5fe552bfe42028db183f4 Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Sun, 22 Dec 2019 11:21:40 +0000 Subject: [PATCH 183/206] Strict prototypes for AIORI --- src/aiori.h | 8 ++++---- src/ior.c | 4 ++-- src/mdtest.c | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/aiori.h b/src/aiori.h index 3fc9f06b..4be7bd54 100755 --- a/src/aiori.h +++ b/src/aiori.h @@ -74,7 +74,7 @@ typedef struct ior_aiori { IOR_offset_t, IOR_param_t *); void (*close)(void *, IOR_param_t *); void (*delete)(char *, IOR_param_t *); - char* (*get_version)(); + char* (*get_version)(void); void (*fsync)(void *, IOR_param_t *); IOR_offset_t (*get_file_size)(IOR_param_t *, MPI_Comm, char *); int (*statfs) (const char *, ior_aiori_statfs_t *, IOR_param_t * param); @@ -82,8 +82,8 @@ typedef struct ior_aiori { int (*rmdir) (const char *path, IOR_param_t * param); int (*access) (const char *path, int mode, IOR_param_t * param); int (*stat) (const char *path, struct stat *buf, IOR_param_t * param); - void (*initialize)(); /* called once per program before MPI is started */ - void (*finalize)(); /* called once per program after MPI is shutdown */ + void (*initialize)(void); /* called once per program before MPI is started */ + void (*finalize)(void); /* called once per program after MPI is shutdown */ option_help * (*get_options)(void ** init_backend_options, void* init_values); /* initializes the backend options as well and returns the pointer to the option help structure */ bool enable_mdtest; int (*check_params)(IOR_param_t *); /* check if the provided parameters for the given test and the module options are correct, if they aren't print a message and exit(1) or return 1*/ @@ -123,7 +123,7 @@ void * airoi_update_module_options(const ior_aiori_t * backend, options_all_t * const char *aiori_default (void); /* some generic POSIX-based backend calls */ -char * aiori_get_version(); +char * aiori_get_version (void); int aiori_posix_statfs (const char *path, ior_aiori_statfs_t *stat_buf, IOR_param_t * param); int aiori_posix_mkdir (const char *path, mode_t mode, IOR_param_t * param); int aiori_posix_rmdir (const char *path, IOR_param_t * param); diff --git a/src/ior.c b/src/ior.c index 5e343d11..c7c2c27d 100755 --- a/src/ior.c +++ b/src/ior.c @@ -1268,7 +1268,7 @@ static void TestIoSys(IOR_test_t *test) /* IO Buffer Setup */ if (params->setTimeStampSignature) { // initialize the buffer properly - params->timeStampSignatureValue = (unsigned int)params->setTimeStampSignature; + params->timeStampSignatureValue = (unsigned int) params->setTimeStampSignature; } XferBuffersSetup(&ioBuffers, params, pretendRank); reseed_incompressible_prng = TRUE; // reset pseudo random generator, necessary to guarantee the next call to FillBuffer produces the same value as it is right now @@ -1289,7 +1289,7 @@ static void TestIoSys(IOR_test_t *test) ERR("cannot get current time"); } params->timeStampSignatureValue = - (unsigned int)currentTime; + (unsigned int) currentTime; if (verbose >= VERBOSE_2) { fprintf(out_logfile, "Using Time Stamp %u (0x%x) for Data Signature\n", diff --git a/src/mdtest.c b/src/mdtest.c index 05e8a44e..3b478ca2 100644 --- a/src/mdtest.c +++ b/src/mdtest.c @@ -2264,7 +2264,7 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * } if (backend->finalize) - backend->finalize(NULL); + backend->finalize(); return summary_table; } From 2a4c21e4575ef303ea945a71b253695c1b9fc68f Mon Sep 17 00:00:00 2001 From: Jeff Olivier Date: Wed, 8 Jan 2020 16:45:50 -0700 Subject: [PATCH 184/206] Add runpath to configure script for daos and cart Enables running without setting LD_LIBRARY_PATH Uses DT_RUNPATH (--enable-new-dtags) so that it can be overwritten by setting LD_LIBRARY_PATH should libraries be moved. Signed-off-by: Jeff Olivier --- configure.ac | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index 26bee2f2..5bdd78b6 100755 --- a/configure.ac +++ b/configure.ac @@ -205,7 +205,7 @@ AC_ARG_WITH([cart], AS_IF([test "x$with_cart" != xno], [ CART="yes" - LDFLAGS="$LDFLAGS -L$with_cart/lib" + LDFLAGS="$LDFLAGS -L$with_cart/lib -Wl,--enable-new-dtags -Wl,-rpath=$with_cart/lib" CPPFLAGS="$CPPFLAGS -I$with_cart/include/" AC_CHECK_HEADERS(gurt/common.h,, [unset CART]) AC_CHECK_LIB([gurt], [d_hash_murmur64],, [unset CART]) @@ -218,7 +218,7 @@ AC_ARG_WITH([daos], AS_IF([test "x$with_daos" != xno], [ DAOS="yes" - LDFLAGS="$LDFLAGS -L$with_daos/lib64 -L$with_daos/lib" + LDFLAGS="$LDFLAGS -L$with_daos/lib64 -Wl,--enable-new-dtags -Wl,-rpath=$with_daos/lib64" CPPFLAGS="$CPPFLAGS -I$with_daos/include" AC_CHECK_HEADERS(daos_types.h,, [unset DAOS]) AC_CHECK_LIB([uuid], [uuid_generate],, [unset DAOS]) From c7974d194c6d479c537531c1c68c589574b29683 Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Sun, 19 Jan 2020 11:09:49 +0000 Subject: [PATCH 185/206] Updated user documentation for changed odirect --- doc/USER_GUIDE | 3 ++- doc/sphinx/userDoc/options.rst | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/doc/USER_GUIDE b/doc/USER_GUIDE index 62a68025..3d6b4e4f 100755 --- a/doc/USER_GUIDE +++ b/doc/USER_GUIDE @@ -65,7 +65,6 @@ These options are to be used on the command line. E.g., 'IOR -a POSIX -b 4K'. -a S api -- API for I/O, e.g., POSIX -A N refNum -- user reference number to include in long summary -b N blockSize -- contiguous bytes to write per task (e.g.: 8, 4k, 2m, 1g) - -B useO_DIRECT -- uses O_DIRECT for POSIX, bypassing I/O buffers -c collective -- collective I/O -C reorderTasksConstant -- changes task ordering to n+1 ordering for readback -d N interTestDelay -- delay between reps in seconds @@ -118,6 +117,8 @@ NOTES: * S is a string, N is an integer number. * For transfer and block sizes, the case-insensitive K, M, and G suffices are recognized. I.e., '4k' or '4K' is accepted as 4096. +Various options are only valid for specific modules, you can see details when running $ ./ior -h +These options are typically prefixed with the module name, an example is: --posix.odirect ********************* * 4. OPTION DETAILS * diff --git a/doc/sphinx/userDoc/options.rst b/doc/sphinx/userDoc/options.rst index 86fc97ee..31240f0e 100644 --- a/doc/sphinx/userDoc/options.rst +++ b/doc/sphinx/userDoc/options.rst @@ -22,7 +22,6 @@ These options are to be used on the command line (e.g., ``./ior -a POSIX -b 4K`` -a S api -- API for I/O [POSIX|MPIIO|HDF5|HDFS|S3|S3_EMC|NCMPI|RADOS] -A N refNum -- user reference number to include in long summary -b N blockSize -- contiguous bytes to write per task (e.g.: 8, 4k, 2m, 1g) - -B useO_DIRECT -- uses O_DIRECT for POSIX, bypassing I/O buffers -c collective -- collective I/O -C reorderTasksConstant -- changes task ordering to n+1 ordering for readback -d N interTestDelay -- delay between reps in seconds @@ -76,6 +75,9 @@ These options are to be used on the command line (e.g., ``./ior -a POSIX -b 4K`` * For transfer and block sizes, the case-insensitive K, M, and G suffices are recognized. I.e., '4k' or '4K' is accepted as 4096. +Various options are only valid for specific modules, you can see details when running $ ./ior -h +These options are typically prefixed with the module name, an example is: --posix.odirect + Directive Options ------------------ From e363713c07aab6defe340a360daf39afdbaadf33 Mon Sep 17 00:00:00 2001 From: Mark Nelson Date: Tue, 25 Feb 2020 18:13:29 -0600 Subject: [PATCH 186/206] src/aiori-RADOS: Remove Direct IO option check. Signed-off-by: Mark Nelson --- src/aiori-RADOS.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/aiori-RADOS.c b/src/aiori-RADOS.c index 090414d2..b8789d4b 100755 --- a/src/aiori-RADOS.c +++ b/src/aiori-RADOS.c @@ -137,9 +137,6 @@ static void *RADOS_Create_Or_Open(char *testFileName, IOR_param_t * param, int c RADOS_Cluster_Init(param); - if (param->useO_DIRECT == TRUE) - WARN("direct I/O mode is not implemented in RADOS\n"); - oid = strdup(testFileName); if (!oid) ERR("unable to allocate RADOS oid"); From 50a4ba07358347fda6ee2bbcf852455c5fc9f1a7 Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Mon, 24 Feb 2020 18:59:40 +0000 Subject: [PATCH 187/206] DAOS Driver: Update License & Align with 1.0 API changes - License should inherit from IOR License. - remove checksums from Array API - add DFS dir oclass option Signed-off-by: Mohamad Chaarawi --- src/aiori-DAOS.c | 17 +++++------------ src/aiori-DFS.c | 28 +++++++++++++++------------- 2 files changed, 20 insertions(+), 25 deletions(-) diff --git a/src/aiori-DAOS.c b/src/aiori-DAOS.c index 19a9f648..7f07f7c4 100644 --- a/src/aiori-DAOS.c +++ b/src/aiori-DAOS.c @@ -1,16 +1,9 @@ -/* - * -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- +/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- * vim:expandtab:shiftwidth=8:tabstop=8: */ /* - * Copyright (C) 2018-2019 Intel Corporation - * - * GOVERNMENT LICENSE RIGHTS-OPEN SOURCE SOFTWARE - * The Government's rights to use, modify, reproduce, release, perform, display, - * or disclose this software are subject to the terms of the Apache License as - * provided in Contract No. 8F-30005. - * Any reproduction of computer software, computer software documentation, or - * portions thereof marked with this legend must also reproduce the markings. + * Copyright (C) 2018-2020 Intel Corporation + * See the file COPYRIGHT for a complete copyright notice and license. */ /* @@ -444,10 +437,10 @@ DAOS_Xfer(int access, void *file, IOR_size_t *buffer, sgl.sg_iovs = &iov; if (access == WRITE) { - rc = daos_array_write(aoh, DAOS_TX_NONE, &iod, &sgl, NULL, NULL); + rc = daos_array_write(aoh, DAOS_TX_NONE, &iod, &sgl, NULL); DCHECK(rc, "daos_array_write() failed (%d).", rc); } else { - rc = daos_array_read(aoh, DAOS_TX_NONE, &iod, &sgl, NULL, NULL); + rc = daos_array_read(aoh, DAOS_TX_NONE, &iod, &sgl, NULL); DCHECK(rc, "daos_array_read() failed (%d).", rc); } diff --git a/src/aiori-DFS.c b/src/aiori-DFS.c index 9b9859a5..50996318 100755 --- a/src/aiori-DFS.c +++ b/src/aiori-DFS.c @@ -2,14 +2,8 @@ * vim:expandtab:shiftwidth=8:tabstop=8: */ /* - * Copyright (C) 2018-2019 Intel Corporation - * - * GOVERNMENT LICENSE RIGHTS-OPEN SOURCE SOFTWARE - * The Government's rights to use, modify, reproduce, release, perform, display, - * or disclose this software are subject to the terms of the Apache License as - * provided in Contract No. 8F-30005. - * Any reproduction of computer software, computer software documentation, or - * portions thereof marked with this legend must also reproduce the markings. + * Copyright (C) 2018-2020 Intel Corporation + * See the file COPYRIGHT for a complete copyright notice and license. */ /* @@ -46,6 +40,7 @@ dfs_t *dfs; static daos_handle_t poh, coh; static daos_oclass_id_t objectClass = OC_SX; +static daos_oclass_id_t dir_oclass = OC_SX; static struct d_hash_table *dir_hash; struct aiori_dir_hdl { @@ -68,6 +63,7 @@ struct dfs_options{ char *cont; int chunk_size; char *oclass; + char *dir_oclass; char *prefix; int destroy; }; @@ -79,6 +75,7 @@ static struct dfs_options o = { .cont = NULL, .chunk_size = 1048576, .oclass = NULL, + .dir_oclass = NULL, .prefix = NULL, .destroy = 0, }; @@ -90,6 +87,7 @@ static option_help options [] = { {0, "dfs.cont", "DFS container uuid", OPTION_OPTIONAL_ARGUMENT, 's', & o.cont}, {0, "dfs.chunk_size", "chunk size", OPTION_OPTIONAL_ARGUMENT, 'd', &o.chunk_size}, {0, "dfs.oclass", "object class", OPTION_OPTIONAL_ARGUMENT, 's', &o.oclass}, + {0, "dfs.dir_oclass", "directory object class", OPTION_OPTIONAL_ARGUMENT, 's', &o.dir_oclass}, {0, "dfs.prefix", "mount prefix", OPTION_OPTIONAL_ARGUMENT, 's', & o.prefix}, {0, "dfs.destroy", "Destroy DFS Container", OPTION_FLAG, 'd', &o.destroy}, LAST_OPTION @@ -385,7 +383,13 @@ DFS_Init() { if (o.oclass) { objectClass = daos_oclass_name2id(o.oclass); if (objectClass == OC_UNKNOWN) - GERR("Invalid DAOS Object class %s\n", o.oclass); + GERR("Invalid DAOS object class %s\n", o.oclass); + } + + if (o.dir_oclass) { + dir_oclass = daos_oclass_name2id(o.dir_oclass); + if (dir_oclass == OC_UNKNOWN) + GERR("Invalid DAOS directory object class %s\n", o.dir_oclass); } rc = daos_init(); @@ -506,7 +510,7 @@ DFS_Create(char *testFileName, IOR_param_t *param) { char *name = NULL, *dir_name = NULL; dfs_obj_t *obj = NULL, *parent = NULL; - mode_t mode; + mode_t mode = 0644; int fd_oflag = 0; int rc; @@ -767,15 +771,13 @@ DFS_Mkdir(const char *path, mode_t mode, IOR_param_t * param) if (parent == NULL) GERR("Failed to lookup parent dir"); - rc = dfs_mkdir(dfs, parent, name, mode); + rc = dfs_mkdir(dfs, parent, name, mode, dir_oclass); DCHECK(rc, "dfs_mkdir() of %s Failed", name); if (name) free(name); if (dir_name) free(dir_name); - if (rc) - return -1; return rc; } From 9726aa0ce2abdae4217c31a314c718182feceaf2 Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Sat, 7 Mar 2020 14:40:59 +0000 Subject: [PATCH 188/206] Bugfix API which doesn't set the AIORI backend. --- src/ior.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ior.c b/src/ior.c index c7c2c27d..4ec88ea4 100755 --- a/src/ior.c +++ b/src/ior.c @@ -78,6 +78,7 @@ IOR_test_t * ior_run(int argc, char **argv, MPI_Comm world_com, FILE * world_out for (tptr = tests_head; tptr != NULL; tptr = tptr->next) { totalErrorCount = 0; verbose = tptr->params.verbose; + backend = tptr->params.backend; if (rank == 0 && verbose >= VERBOSE_0) { ShowTestStart(&tptr->params); } @@ -129,6 +130,7 @@ int ior_main(int argc, char **argv) /* perform each test */ for (tptr = tests_head; tptr != NULL; tptr = tptr->next) { verbose = tptr->params.verbose; + backend = tptr->params.backend; if (rank == 0 && verbose >= VERBOSE_0) { backend = tptr->params.backend; ShowTestStart(&tptr->params); @@ -1256,7 +1258,6 @@ static void TestIoSys(IOR_test_t *test) "Using reorderTasks '-C' (useful to avoid read cache in client)\n"); fflush(out_logfile); } - backend = params->backend; /* show test setup */ if (rank == 0 && verbose >= VERBOSE_0) ShowSetup(params); From 9649a0c520a732cdae17cc774672e181525ed526 Mon Sep 17 00:00:00 2001 From: Mark Nelson Date: Thu, 5 Mar 2020 01:40:05 +0000 Subject: [PATCH 189/206] src/aiori-CEPHFS: New libcephfs backend Signed-off-by: Mark Nelson --- configure.ac | 14 ++ src/Makefile.am | 6 + src/aiori-CEPHFS.c | 361 +++++++++++++++++++++++++++++++++++++++++++++ src/aiori.c | 3 + src/aiori.h | 1 + src/ior.c | 3 +- src/ior.h | 1 - 7 files changed, 387 insertions(+), 2 deletions(-) create mode 100755 src/aiori-CEPHFS.c diff --git a/configure.ac b/configure.ac index 5bdd78b6..cc7ddbac 100755 --- a/configure.ac +++ b/configure.ac @@ -197,6 +197,20 @@ AM_COND_IF([USE_RADOS_AIORI],[ AC_DEFINE([USE_RADOS_AIORI], [], [Build RADOS backend AIORI]) ]) +# CEPHFS support +AC_ARG_WITH([cephfs], + [AS_HELP_STRING([--with-cephfs], + [support IO with libcephfs backend @<:@default=no@:>@])], + [], + [with_cephfs=no]) +AS_IF([test "x$with_cephfs" != xno], [ + CPPFLAGS="$CPPFLAGS -D_FILE_OFFSET_BITS=64 -std=gnu11" +]) +AM_CONDITIONAL([USE_CEPHFS_AIORI], [test x$with_cephfs = xyes]) +AM_COND_IF([USE_CEPHFS_AIORI],[ + AC_DEFINE([USE_CEPHFS_AIORI], [], [Build CEPHFS backend AIORI]) +]) + # DAOS Backends (DAOS and DFS) IO support require DAOS and CART/GURT AC_ARG_WITH([cart], [AS_HELP_STRING([--with-cart], diff --git a/src/Makefile.am b/src/Makefile.am index 56b4279e..1c37a6e3 100755 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -70,6 +70,12 @@ extraSOURCES += aiori-RADOS.c extraLDADD += -lrados endif +if USE_CEPHFS_AIORI +extraSOURCES += aiori-CEPHFS.c +extraLDADD += -lcephfs +endif + + if USE_DAOS_AIORI extraSOURCES += aiori-DAOS.c aiori-DFS.c endif diff --git a/src/aiori-CEPHFS.c b/src/aiori-CEPHFS.c new file mode 100755 index 00000000..d5c6154a --- /dev/null +++ b/src/aiori-CEPHFS.c @@ -0,0 +1,361 @@ +/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- + * vim:expandtab:shiftwidth=8:tabstop=8: + */ +/******************************************************************************\ +* * +* (C) 2015 The University of Chicago * +* (C) 2020 Red Hat, Inc. * +* * +* See COPYRIGHT in top-level directory. * +* * +******************************************************************************** +* +* Implement abstract I/O interface for CEPHFS. +* +\******************************************************************************/ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include +#include +#include +#include + +#include "ior.h" +#include "iordef.h" +#include "aiori.h" +#include "utilities.h" + +#define CEPH_O_RDONLY 00000000 +#define CEPH_O_WRONLY 00000001 +#define CEPH_O_RDWR 00000002 +#define CEPH_O_CREAT 00000100 +#define CEPH_O_EXCL 00000200 +#define CEPH_O_TRUNC 00001000 +#define CEPH_O_LAZY 00020000 +#define CEPH_O_DIRECTORY 00200000 +#define CEPH_O_NOFOLLOW 00400000 + +/************************** O P T I O N S *****************************/ +struct cephfs_options{ + char * user; + char * conf; + char * prefix; +}; + +static struct cephfs_options o = { + .user = NULL, + .conf = NULL, + .prefix = NULL, +}; + +static option_help options [] = { + {0, "cephfs.user", "Username for the ceph cluster", OPTION_REQUIRED_ARGUMENT, 's', & o.user}, + {0, "cephfs.conf", "Config file for the ceph cluster", OPTION_REQUIRED_ARGUMENT, 's', & o.conf}, + {0, "cephfs.prefix", "mount prefix", OPTION_OPTIONAL_ARGUMENT, 's', & o.prefix}, + LAST_OPTION +}; + +static struct ceph_mount_info *cmount; + +/**************************** P R O T O T Y P E S *****************************/ +static void CEPHFS_Init(); +static void CEPHFS_Final(); +static void *CEPHFS_Create(char *, IOR_param_t *); +static void *CEPHFS_Open(char *, IOR_param_t *); +static IOR_offset_t CEPHFS_Xfer(int, void *, IOR_size_t *, + IOR_offset_t, IOR_param_t *); +static void CEPHFS_Close(void *, IOR_param_t *); +static void CEPHFS_Delete(char *, IOR_param_t *); +static void CEPHFS_Fsync(void *, IOR_param_t *); +static IOR_offset_t CEPHFS_GetFileSize(IOR_param_t *, MPI_Comm, char *); +static int CEPHFS_StatFS(const char *, ior_aiori_statfs_t *, IOR_param_t *); +static int CEPHFS_MkDir(const char *, mode_t, IOR_param_t *); +static int CEPHFS_RmDir(const char *, IOR_param_t *); +static int CEPHFS_Access(const char *, int, IOR_param_t *); +static int CEPHFS_Stat(const char *, struct stat *, IOR_param_t *); +static option_help * CEPHFS_options(); + +/************************** D E C L A R A T I O N S ***************************/ +ior_aiori_t cephfs_aiori = { + .name = "CEPHFS", + .name_legacy = NULL, + .initialize = CEPHFS_Init, + .finalize = CEPHFS_Final, + .create = CEPHFS_Create, + .open = CEPHFS_Open, + .xfer = CEPHFS_Xfer, + .close = CEPHFS_Close, + .delete = CEPHFS_Delete, + .get_version = aiori_get_version, + .fsync = CEPHFS_Fsync, + .get_file_size = CEPHFS_GetFileSize, + .statfs = CEPHFS_StatFS, + .mkdir = CEPHFS_MkDir, + .rmdir = CEPHFS_RmDir, + .access = CEPHFS_Access, + .stat = CEPHFS_Stat, + .get_options = CEPHFS_options, +}; + +#define CEPHFS_ERR(__err_str, __ret) do { \ + errno = -__ret; \ + ERR(__err_str); \ +} while(0) + +/***************************** F U N C T I O N S ******************************/ +static const char* pfix(const char* path) { + const char* npath = path; + const char* prefix = o.prefix; + while (*prefix) { + if(*prefix++ != *npath++) { + return path; + } + } + return npath; +} + +static option_help * CEPHFS_options(){ + return options; +} + +static void CEPHFS_Init() +{ + /* Short circuit if the mount handle already exists */ + if (cmount) { + return; + } + + int ret; + /* create CEPHFS mount handle */ + ret = ceph_create(&cmount, o.user); + if (ret) { + CEPHFS_ERR("unable to create CEPHFS mount handle", ret); + } + + /* set the handle using the Ceph config */ + ret = ceph_conf_read_file(cmount, o.conf); + if (ret) { + CEPHFS_ERR("unable to read ceph config file", ret); + } + + /* mount the handle */ + ret = ceph_mount(cmount, "/"); + if (ret) { + CEPHFS_ERR("unable to mount cephfs", ret); + ceph_shutdown(cmount); + + } + + Inode *root; + + /* try retrieving the root cephfs inode */ + ret = ceph_ll_lookup_root(cmount, &root); + if (ret) { + CEPHFS_ERR("uanble to retrieve root cephfs inode", ret); + ceph_shutdown(cmount); + + } + + return; +} + +static void CEPHFS_Final() +{ + /* shutdown */ + ceph_shutdown(cmount); +} + +static void *CEPHFS_Create(char *testFileName, IOR_param_t * param) +{ + return CEPHFS_Open(testFileName, param); +} + +static void *CEPHFS_Open(char *testFileName, IOR_param_t * param) +{ + const char *file = pfix(testFileName); + int* fd; + fd = (int *)malloc(sizeof(int)); + + mode_t mode = 0664; + int flags = (int) 0; + + /* set IOR file flags to CephFS flags */ + /* -- file open flags -- */ + if (param->openFlags & IOR_RDONLY) { + flags |= CEPH_O_RDONLY; + } + if (param->openFlags & IOR_WRONLY) { + flags |= CEPH_O_WRONLY; + } + if (param->openFlags & IOR_RDWR) { + flags |= CEPH_O_RDWR; + } + if (param->openFlags & IOR_APPEND) { + fprintf(stdout, "File append not implemented in CephFS\n"); + } + if (param->openFlags & IOR_CREAT) { + flags |= CEPH_O_CREAT; + } + if (param->openFlags & IOR_EXCL) { + flags |= CEPH_O_EXCL; + } + if (param->openFlags & IOR_TRUNC) { + flags |= CEPH_O_TRUNC; + } + if (param->openFlags & IOR_DIRECT) { + fprintf(stdout, "O_DIRECT not implemented in CephFS\n"); + } + *fd = ceph_open(cmount, file, flags, mode); + if (*fd < 0) { + CEPHFS_ERR("ceph_open failed", *fd); + } + return (void *) fd; +} + +static IOR_offset_t CEPHFS_Xfer(int access, void *file, IOR_size_t * buffer, + IOR_offset_t length, IOR_param_t * param) +{ + uint64_t size = (uint64_t) length; + char *buf = (char *) buffer; + int fd = *(int *) file; + int ret; + + if (access == WRITE) + { + ret = ceph_write(cmount, fd, buf, size, param->offset); + if (ret < 0) { + CEPHFS_ERR("unable to write file to CephFS", ret); + } else if (ret < size) { + CEPHFS_ERR("short write to CephFS", ret); + } + if (param->fsyncPerWrite == TRUE) { + CEPHFS_Fsync(&fd, param); + } + } + else /* READ */ + { + ret = ceph_read(cmount, fd, buf, size, param->offset); + if (ret < 0) { + CEPHFS_ERR("unable to read file from CephFS", ret); + } else if (ret < size) { + CEPHFS_ERR("short read from CephFS", ret); + } + + } + return length; +} + +static void CEPHFS_Fsync(void *file, IOR_param_t * param) +{ + int fd = *(int *) file; + int ret = ceph_fsync(cmount, fd, 0); + if (ret < 0) { + CEPHFS_ERR("ceph_fsync failed", ret); + } +} + +static void CEPHFS_Close(void *file, IOR_param_t * param) +{ + int fd = *(int *) file; + int ret = ceph_close(cmount, fd); + if (ret < 0) { + CEPHFS_ERR("ceph_close failed", ret); + } + free(file); + + return; +} + +static void CEPHFS_Delete(char *testFileName, IOR_param_t * param) +{ + int ret = ceph_unlink(cmount, pfix(testFileName)); + if (ret < 0) { + CEPHFS_ERR("ceph_unlink failed", ret); + } + return; +} + +static IOR_offset_t CEPHFS_GetFileSize(IOR_param_t * param, MPI_Comm testComm, + char *testFileName) +{ + struct stat stat_buf; + IOR_offset_t aggFileSizeFromStat, tmpMin, tmpMax, tmpSum; + + int ret = ceph_stat(cmount, pfix(testFileName), &stat_buf); + if (ret < 0) { + CEPHFS_ERR("ceph_stat failed", ret); + } + aggFileSizeFromStat = stat_buf.st_size; + + if (param->filePerProc == TRUE) { + MPI_CHECK(MPI_Allreduce(&aggFileSizeFromStat, &tmpSum, 1, + MPI_LONG_LONG_INT, MPI_SUM, testComm), + "cannot total data moved"); + aggFileSizeFromStat = tmpSum; + } else { + MPI_CHECK(MPI_Allreduce(&aggFileSizeFromStat, &tmpMin, 1, + MPI_LONG_LONG_INT, MPI_MIN, testComm), + "cannot total data moved"); + MPI_CHECK(MPI_Allreduce(&aggFileSizeFromStat, &tmpMax, 1, + MPI_LONG_LONG_INT, MPI_MAX, testComm), + "cannot total data moved"); + if (tmpMin != tmpMax) { + if (rank == 0) { + WARN("inconsistent file size by different tasks"); + } + /* incorrect, but now consistent across tasks */ + aggFileSizeFromStat = tmpMin; + } + } + + return (aggFileSizeFromStat); + +} + +static int CEPHFS_StatFS(const char *path, ior_aiori_statfs_t *stat_buf, + IOR_param_t *param) +{ +#if defined(HAVE_STATVFS) + struct statvfs statfs_buf; + int ret = ceph_statfs(cmount, pfix(path), &statfs_buf); + if (ret < 0) { + CEPHFS_ERR("ceph_statfs failed", ret); + return -1; + } + + stat_buf->f_bsize = statfs_buf.f_bsize; + stat_buf->f_blocks = statfs_buf.f_blocks; + stat_buf->f_bfree = statfs_buf.f_bfree; + stat_buf->f_files = statfs_buf.f_files; + stat_buf->f_ffree = statfs_buf.f_ffree; + + return 0; +#else + WARN("ceph_statfs requires statvfs!"); + return -1; +#endif +} + +static int CEPHFS_MkDir(const char *path, mode_t mode, IOR_param_t *param) +{ + return ceph_mkdir(cmount, pfix(path), mode); +} + +static int CEPHFS_RmDir(const char *path, IOR_param_t *param) +{ + return ceph_rmdir(cmount, pfix(path)); +} + +static int CEPHFS_Access(const char *testFileName, int mode, IOR_param_t *param) +{ + struct stat buf; + return ceph_stat(cmount, pfix(testFileName), &buf); +} + +static int CEPHFS_Stat(const char *testFileName, struct stat *buf, IOR_param_t *param) +{ + return ceph_stat(cmount, pfix(testFileName), buf); +} diff --git a/src/aiori.c b/src/aiori.c index a72180d3..71f99d1c 100644 --- a/src/aiori.c +++ b/src/aiori.c @@ -73,6 +73,9 @@ ior_aiori_t *available_aiori[] = { #ifdef USE_RADOS_AIORI &rados_aiori, #endif +#ifdef USE_CEPHFS_AIORI + &cephfs_aiori, +#endif #ifdef USE_GFARM_AIORI &gfarm_aiori, #endif diff --git a/src/aiori.h b/src/aiori.h index 4be7bd54..4d416c21 100755 --- a/src/aiori.h +++ b/src/aiori.h @@ -109,6 +109,7 @@ extern ior_aiori_t s3_aiori; extern ior_aiori_t s3_plus_aiori; extern ior_aiori_t s3_emc_aiori; extern ior_aiori_t rados_aiori; +extern ior_aiori_t cephfs_aiori; extern ior_aiori_t gfarm_aiori; void aiori_initialize(IOR_test_t * tests); diff --git a/src/ior.c b/src/ior.c index c7c2c27d..13868a28 100755 --- a/src/ior.c +++ b/src/ior.c @@ -1618,7 +1618,8 @@ static void ValidateTests(IOR_param_t * test) && (strcasecmp(test->api, "MMAP") != 0) && (strcasecmp(test->api, "HDFS") != 0) && (strcasecmp(test->api, "Gfarm") != 0) - && (strcasecmp(test->api, "RADOS") != 0)) && test->fsync) + && (strcasecmp(test->api, "RADOS") != 0) + && (strcasecmp(test->api, "CEPHFS") != 0)) && test->fsync) WARN_RESET("fsync() not supported in selected backend", test, &defaults, fsync); if ((strcasecmp(test->api, "MPIIO") != 0) && test->preallocate) diff --git a/src/ior.h b/src/ior.h index 6b9d1d75..758b0486 100755 --- a/src/ior.h +++ b/src/ior.h @@ -35,7 +35,6 @@ typedef void *rados_t; typedef void *rados_ioctx_t; #endif - #include "option.h" #include "iordef.h" From a21137be3ecdd7e86cde6b7675308a804479d383 Mon Sep 17 00:00:00 2001 From: Jean-Yves VET Date: Tue, 17 Mar 2020 16:31:56 +0100 Subject: [PATCH 190/206] Display outlier host names Display host names when using the -j argument (outlierThreshold) so that IOR could help to better identify slow nodes: WARNING: for client02, task 6, write elapsed transfer time is 65.738326 --- src/ior.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/ior.c b/src/ior.c index 98711930..c831a84f 100755 --- a/src/ior.c +++ b/src/ior.c @@ -255,8 +255,13 @@ DisplayOutliers(int numTasks, strcpy(accessString, "read"); } if (fabs(timerVal - mean) > (double)outlierThreshold) { - fprintf(out_logfile, "WARNING: for task %d, %s %s is %f\n", - rank, accessString, timeString, timerVal); + char hostname[MAX_STR]; + int ret = gethostname(hostname, MAX_STR); + if (ret != 0) + strcpy(hostname, "unknown"); + + fprintf(out_logfile, "WARNING: for %s, task %d, %s %s is %f\n", + hostname, rank, accessString, timeString, timerVal); fprintf(out_logfile, " (mean=%f, stddev=%f)\n", mean, sd); fflush(out_logfile); } From eed22a4aaf7c4ba807e72fe1be3df6fe50e8c403 Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Mon, 6 Apr 2020 21:42:51 +0000 Subject: [PATCH 191/206] Look into lib64 for cart libraries too. Signed-off-by: Mohamad Chaarawi --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 5bdd78b6..5a2f8f94 100755 --- a/configure.ac +++ b/configure.ac @@ -205,7 +205,7 @@ AC_ARG_WITH([cart], AS_IF([test "x$with_cart" != xno], [ CART="yes" - LDFLAGS="$LDFLAGS -L$with_cart/lib -Wl,--enable-new-dtags -Wl,-rpath=$with_cart/lib" + LDFLAGS="$LDFLAGS -L$with_cart/lib -L$with_cart/lib64 -Wl,--enable-new-dtags -Wl,-rpath=$with_cart/lib64" CPPFLAGS="$CPPFLAGS -I$with_cart/include/" AC_CHECK_HEADERS(gurt/common.h,, [unset CART]) AC_CHECK_LIB([gurt], [d_hash_murmur64],, [unset CART]) From 1b8848ccbcda0ce04b6029698944e2b55a2da618 Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Mon, 13 Apr 2020 19:34:46 +0000 Subject: [PATCH 192/206] Use l2g-g2l to share the dfs mount instead of opening by each rank. Signed-off-by: Mohamad Chaarawi --- configure.ac | 3 ++- src/aiori-DFS.c | 35 +++++++++++++++++++++-------------- 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/configure.ac b/configure.ac index 5a2f8f94..5ee1530b 100755 --- a/configure.ac +++ b/configure.ac @@ -205,7 +205,8 @@ AC_ARG_WITH([cart], AS_IF([test "x$with_cart" != xno], [ CART="yes" - LDFLAGS="$LDFLAGS -L$with_cart/lib -L$with_cart/lib64 -Wl,--enable-new-dtags -Wl,-rpath=$with_cart/lib64" + LDFLAGS="$LDFLAGS -L$with_cart/lib64 -Wl,--enable-new-dtags -Wl,-rpath=$with_cart/lib64" + LDFLAGS="$LDFLAGS -L$with_cart/lib -Wl,--enable-new-dtags -Wl,-rpath=$with_cart/lib" CPPFLAGS="$CPPFLAGS -I$with_cart/include/" AC_CHECK_HEADERS(gurt/common.h,, [unset CART]) AC_CHECK_LIB([gurt], [d_hash_murmur64],, [unset CART]) diff --git a/src/aiori-DFS.c b/src/aiori-DFS.c index 50996318..f8c6ce95 100755 --- a/src/aiori-DFS.c +++ b/src/aiori-DFS.c @@ -52,7 +52,7 @@ struct aiori_dir_hdl { enum handleType { POOL_HANDLE, CONT_HANDLE, - ARRAY_HANDLE + DFS_HANDLE }; /************************** O P T I O N S *****************************/ @@ -194,7 +194,7 @@ static d_hash_table_ops_t hdl_hash_ops = { /* Distribute process 0's pool or container handle to others. */ static void -HandleDistribute(daos_handle_t *handle, enum handleType type) +HandleDistribute(enum handleType type) { d_iov_t global; int rc; @@ -203,13 +203,15 @@ HandleDistribute(daos_handle_t *handle, enum handleType type) global.iov_buf_len = 0; global.iov_len = 0; - assert(type == POOL_HANDLE || type == CONT_HANDLE); + assert(type == POOL_HANDLE || type == CONT_HANDLE || type == DFS_HANDLE); if (rank == 0) { /* Get the global handle size. */ if (type == POOL_HANDLE) - rc = daos_pool_local2global(*handle, &global); + rc = daos_pool_local2global(poh, &global); + else if (type == CONT_HANDLE) + rc = daos_cont_local2global(coh, &global); else - rc = daos_cont_local2global(*handle, &global); + rc = dfs_local2global(dfs, &global); DCHECK(rc, "Failed to get global handle size"); } @@ -224,9 +226,11 @@ HandleDistribute(daos_handle_t *handle, enum handleType type) if (rank == 0) { if (type == POOL_HANDLE) - rc = daos_pool_local2global(*handle, &global); + rc = daos_pool_local2global(poh, &global); + else if (type == CONT_HANDLE) + rc = daos_cont_local2global(coh, &global); else - rc = daos_cont_local2global(*handle, &global); + rc = dfs_local2global(dfs, &global); DCHECK(rc, "Failed to create global handle"); } @@ -236,9 +240,11 @@ HandleDistribute(daos_handle_t *handle, enum handleType type) if (rank != 0) { if (type == POOL_HANDLE) - rc = daos_pool_global2local(global, handle); + rc = daos_pool_global2local(global, &poh); + else if (type == CONT_HANDLE) + rc = daos_cont_global2local(poh, global, &coh); else - rc = daos_cont_global2local(poh, global, handle); + rc = dfs_global2local(poh, coh, 0, global, &dfs); DCHECK(rc, "Failed to get local handle"); } @@ -435,13 +441,14 @@ DFS_Init() { } else if (rc) { DCHECK(rc, "Failed to create container"); } - } - HandleDistribute(&poh, POOL_HANDLE); - HandleDistribute(&coh, CONT_HANDLE); + rc = dfs_mount(poh, coh, O_RDWR, &dfs); + DCHECK(rc, "Failed to mount DFS namespace"); + } - rc = dfs_mount(poh, coh, O_RDWR, &dfs); - DCHECK(rc, "Failed to mount DFS namespace"); + HandleDistribute(POOL_HANDLE); + HandleDistribute(CONT_HANDLE); + HandleDistribute(DFS_HANDLE); if (o.prefix) { rc = dfs_set_prefix(dfs, o.prefix); From 48eb1880e92d8580ad5cab24ecc690526698271e Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Tue, 21 Apr 2020 10:35:53 +0100 Subject: [PATCH 193/206] For better testing: Fixed fake tasks per node to work with MPI3, added environment variable to fake num nodes. --- src/utilities.c | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/src/utilities.c b/src/utilities.c index a657d9fa..34222e00 100755 --- a/src/utilities.c +++ b/src/utilities.c @@ -272,6 +272,15 @@ int QueryNodeMapping(MPI_Comm comm, int print_nodemap) { * tasks on node rank 0. */ int GetNumNodes(MPI_Comm comm) { + if (getenv("IOR_FAKE_NODES")){ + int numNodes = atoi(getenv("IOR_FAKE_NODES")); + int rank; + MPI_Comm_rank(comm, & rank); + if(rank == 0){ + printf("Fake number of node: using %d\n", numNodes); + } + return numNodes; + } #if MPI_VERSION >= 3 MPI_Comm shared_comm; int shared_rank = 0; @@ -338,6 +347,15 @@ int GetNumTasks(MPI_Comm comm) { * method will return the same value it always has. */ int GetNumTasksOnNode0(MPI_Comm comm) { + if (getenv("IOR_FAKE_TASK_PER_NODES")){ + int tasksPerNode = atoi(getenv("IOR_FAKE_TASK_PER_NODES")); + int rank; + MPI_Comm_rank(comm, & rank); + if(rank == 0){ + printf("Fake tasks per node: using %d\n", tasksPerNode); + } + return tasksPerNode; + } #if MPI_VERSION >= 3 MPI_Comm shared_comm; int shared_rank = 0; @@ -368,15 +386,6 @@ int GetNumTasksOnNode0(MPI_Comm comm) { int size; MPI_Comm_size(comm, & size); /* for debugging and testing */ - if (getenv("IOR_FAKE_TASK_PER_NODES")){ - int tasksPerNode = atoi(getenv("IOR_FAKE_TASK_PER_NODES")); - int rank; - MPI_Comm_rank(comm, & rank); - if(rank == 0){ - printf("Fake tasks per node: using %d\n", tasksPerNode); - } - return tasksPerNode; - } char localhost[MAX_PATHLEN], hostname[MAX_PATHLEN]; int count = 1, From a536649abb4335719dd503337778c64cfbdd0d5b Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Wed, 20 May 2020 17:46:35 +0100 Subject: [PATCH 194/206] Clarified some issues in the test framework and added example test. #219 --- configure.ac | 1 + src/Makefile.am | 10 +--------- src/ior-internal.h | 3 +++ src/ior.c | 6 ++---- src/test/Makefile.am | 8 ++++++++ src/test/example.c | 31 +++++++++++++++++++++++++++++++ 6 files changed, 46 insertions(+), 13 deletions(-) create mode 100755 src/test/Makefile.am create mode 100644 src/test/example.c diff --git a/configure.ac b/configure.ac index 0cb99c48..cd71b396 100755 --- a/configure.ac +++ b/configure.ac @@ -348,6 +348,7 @@ AM_CONDITIONAL([USE_CAPS], [test x$enable_caps = xyes]) AC_CONFIG_FILES([Makefile src/Makefile + src/test/Makefile contrib/Makefile doc/Makefile]) AC_OUTPUT diff --git a/src/Makefile.am b/src/Makefile.am index 1c37a6e3..3786560f 100755 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1,4 +1,4 @@ -SUBDIRS = . +SUBDIRS = . test bin_PROGRAMS = ior mdtest if USE_CAPS @@ -123,11 +123,3 @@ MDTEST_CPPFLAGS = $(mdtest_CPPFLAGS) libaiori_a_SOURCES += $(extraSOURCES) libaiori_a_CPPFLAGS = $(extraCPPFLAGS) - - -TESTS = testlib -bin_PROGRAMS += testlib - -testlib_SOURCES = ./test/lib.c -testlib_LDFLAGS = $(extraLDFLAGS) -testlib_LDADD = libaiori.a $(extraLDADD) diff --git a/src/ior-internal.h b/src/ior-internal.h index 7daf8de6..9cc8406b 100644 --- a/src/ior-internal.h +++ b/src/ior-internal.h @@ -26,6 +26,9 @@ void PrintTestEnds(); void PrintTableHeader(); /* End of ior-output */ +IOR_offset_t *GetOffsetArraySequential(IOR_param_t * test, int pretendRank); +IOR_offset_t *GetOffsetArrayRandom(IOR_param_t * test, int pretendRank, int access); + struct results { double min; double max; diff --git a/src/ior.c b/src/ior.c index c831a84f..2237cc3d 100755 --- a/src/ior.c +++ b/src/ior.c @@ -1731,8 +1731,7 @@ static void ValidateTests(IOR_param_t * test) * @param pretendRank int pretended Rank for shifting the offsest corectly * @return IOR_offset_t */ -static IOR_offset_t *GetOffsetArraySequential(IOR_param_t * test, - int pretendRank) +IOR_offset_t *GetOffsetArraySequential(IOR_param_t * test, int pretendRank) { IOR_offset_t i, j, k = 0; IOR_offset_t offsets; @@ -1781,8 +1780,7 @@ static IOR_offset_t *GetOffsetArraySequential(IOR_param_t * test, * @return IOR_offset_t * @return */ -static IOR_offset_t *GetOffsetArrayRandom(IOR_param_t * test, int pretendRank, - int access) +IOR_offset_t *GetOffsetArrayRandom(IOR_param_t * test, int pretendRank, int access) { int seed; IOR_offset_t i, value, tmp; diff --git a/src/test/Makefile.am b/src/test/Makefile.am new file mode 100755 index 00000000..1f2b1413 --- /dev/null +++ b/src/test/Makefile.am @@ -0,0 +1,8 @@ +LDFLAGS = $(extraLDFLAGS) +LDADD = ../libaiori.a $(extraLDADD) + +# Add test here +TESTS = testlib testexample +check_PROGRAMS = $(TESTS) +testexample_SOURCES = example.c +testlib_SOURCES = lib.c diff --git a/src/test/example.c b/src/test/example.c new file mode 100644 index 00000000..5bb4b2b5 --- /dev/null +++ b/src/test/example.c @@ -0,0 +1,31 @@ +#include + +#include +#include + +// build a single test via, e.g., mpicc example.c -I ../src/ ../src/libaiori.a -lm + +int main(){ + IOR_param_t test; + init_IOR_Param_t(& test); + test.blockSize = 10; + test.transferSize = 10; + test.segmentCount = 5; + test.numTasks = 2; + + // having an individual file + test.filePerProc = 1; + + IOR_offset_t * offsets; + offsets = GetOffsetArraySequential(& test, 0); + assert(offsets[0] == 0); + assert(offsets[1] == 10); + assert(offsets[2] == 20); + assert(offsets[3] == 30); + assert(offsets[4] == 40); + // for(int i = 0; i < test.segmentCount; i++){ + // printf("%lld\n", (long long int) offsets[i]); + // } + printf("OK\n"); + return 0; +} From 4b417ef1f3e3386164884ab3cab512f938fd4789 Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Thu, 28 May 2020 19:05:41 +0100 Subject: [PATCH 195/206] IOR shouldn't fail if it cannot determine file system sizes but warn. Functionality should presumably be moved into AIORI backends. --- src/mdtest.c | 2 +- src/utilities.c | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/mdtest.c b/src/mdtest.c index 3b478ca2..e0e7bbb5 100644 --- a/src/mdtest.c +++ b/src/mdtest.c @@ -1514,7 +1514,7 @@ void show_file_system_size(char *file_system) { * 100; if (realpath(file_system, real_path) == NULL) { - FAIL("unable to use realpath() on file system %s", file_system); + WARN("unable to use realpath() on file system"); } diff --git a/src/utilities.c b/src/utilities.c index 34222e00..bcb1e03c 100755 --- a/src/utilities.c +++ b/src/utilities.c @@ -565,7 +565,7 @@ IOR_offset_t StringToBytes(char *size_str) /* * Displays size of file system and percent of data blocks and inodes used. */ -void ShowFileSystemSize(char *fileSystem) +void ShowFileSystemSize(char *fileSystem) // this might be converted to an AIORI call { #ifndef _WIN32 /* FIXME */ char realPath[PATH_MAX]; @@ -585,11 +585,13 @@ void ShowFileSystemSize(char *fileSystem) #ifdef __sun if (statvfs(fileSystem, &statusBuffer) != 0) { - ERR("unable to statvfs() file system"); + WARN("unable to statvfs() file system"); + return; } #else /* !__sun */ if (statfs(fileSystem, &statusBuffer) != 0) { - ERR("unable to statfs() file system"); + WARN("unable to statfs() file system"); + return; } #endif /* __sun */ @@ -620,7 +622,8 @@ void ShowFileSystemSize(char *fileSystem) /* show results */ if (realpath(fileSystem, realPath) == NULL) { - ERR("unable to use realpath()"); + WARN("unable to use realpath()"); + return; } if(outputFormat == OUTPUT_DEFAULT){ From 28fae361f40e95899a1d027c0ec6ac58312c381a Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Thu, 28 May 2020 18:34:31 +0000 Subject: [PATCH 196/206] dfs: add DFS sync call Signed-off-by: Mohamad Chaarawi --- src/aiori-DFS.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/aiori-DFS.c b/src/aiori-DFS.c index f8c6ce95..64874887 100755 --- a/src/aiori-DFS.c +++ b/src/aiori-DFS.c @@ -102,6 +102,7 @@ static void DFS_Close(void *, IOR_param_t *); static void DFS_Delete(char *, IOR_param_t *); static char* DFS_GetVersion(); static void DFS_Fsync(void *, IOR_param_t *); +static void DFS_Sync(IOR_param_t *); static IOR_offset_t DFS_GetFileSize(IOR_param_t *, MPI_Comm, char *); static int DFS_Statfs (const char *, ior_aiori_statfs_t *, IOR_param_t *); static int DFS_Stat (const char *, struct stat *, IOR_param_t *); @@ -123,6 +124,7 @@ ior_aiori_t dfs_aiori = { .delete = DFS_Delete, .get_version = DFS_GetVersion, .fsync = DFS_Fsync, + .sync = DFS_Sync, .get_file_size = DFS_GetFileSize, .statfs = DFS_Statfs, .mkdir = DFS_Mkdir, @@ -132,6 +134,7 @@ ior_aiori_t dfs_aiori = { .initialize = DFS_Init, .finalize = DFS_Finalize, .get_options = DFS_options, + .enable_mdtest = true, }; /***************************** F U N C T I O N S ******************************/ @@ -658,6 +661,18 @@ DFS_Xfer(int access, void *file, IOR_size_t *buffer, IOR_offset_t length, static void DFS_Fsync(void *fd, IOR_param_t * param) { + /* no cache in DFS, so this is a no-op currently */ + dfs_sync(dfs); + return; +} + +/* + * Perform sync() on the dfs mount. + */ +static void +DFS_Sync(IOR_param_t * param) +{ + /* no cache in DFS, so this is a no-op currently */ dfs_sync(dfs); return; } From f97090a86bd3398c671364f5eb37fc9e3500bbae Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Thu, 28 May 2020 20:11:52 +0100 Subject: [PATCH 197/206] Typo fix. --- src/mdtest.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mdtest.c b/src/mdtest.c index e0e7bbb5..5e792a1d 100644 --- a/src/mdtest.c +++ b/src/mdtest.c @@ -274,7 +274,7 @@ static void prep_testdir(int j, int dir_iter){ static void phase_end(){ if (call_sync){ if(! backend->sync){ - FAIL("Error, backend does not provide the sync method, but your requested to use sync."); + FAIL("Error, backend does not provide the sync method, but you requested to use sync."); } backend->sync(& param); } From 4eb0ebec55d56194b0ef13cd3bc1f12ab1bbe4ee Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Thu, 28 May 2020 21:13:45 +0100 Subject: [PATCH 198/206] Add dummy sync to dummy backend. --- src/aiori-DUMMY.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/aiori-DUMMY.c b/src/aiori-DUMMY.c index 90fec9a2..f368c795 100755 --- a/src/aiori-DUMMY.c +++ b/src/aiori-DUMMY.c @@ -76,6 +76,11 @@ static void DUMMY_Fsync(void *fd, IOR_param_t * param) } } + +static void DUMMY_Sync(IOR_param_t * param) +{ +} + static void DUMMY_Close(void *fd, IOR_param_t * param) { if(verbose > 4){ @@ -143,7 +148,7 @@ static int DUMMY_stat (const char *path, struct stat *buf, IOR_param_t * param){ return 0; } -static int DUMMY_check_params(IOR_param_t * test){ +static int DUMMY_check_params(IOR_param_t * test){ return 1; } @@ -167,5 +172,7 @@ ior_aiori_t dummy_aiori = { .finalize = NULL, .get_options = DUMMY_options, .enable_mdtest = true, - .check_params = DUMMY_check_params + .check_params = DUMMY_check_params, + .sync = DUMMY_Sync, + .enable_mdtest = true }; From 153b7aa60fc5e0bc82d86b2487dc3760ce4407c0 Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Thu, 28 May 2020 21:18:44 +0100 Subject: [PATCH 199/206] Critical bugfix for ior API (used e.g. in IO500) TODO: Check potential issue when using IOR scripts with different backends. --- src/ior.c | 4 +++- src/mdtest.c | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/ior.c b/src/ior.c index 2237cc3d..05070b03 100755 --- a/src/ior.c +++ b/src/ior.c @@ -76,6 +76,7 @@ IOR_test_t * ior_run(int argc, char **argv, MPI_Comm world_com, FILE * world_out /* perform each test */ for (tptr = tests_head; tptr != NULL; tptr = tptr->next) { + aiori_initialize(tptr); totalErrorCount = 0; verbose = tptr->params.verbose; backend = tptr->params.backend; @@ -85,6 +86,7 @@ IOR_test_t * ior_run(int argc, char **argv, MPI_Comm world_com, FILE * world_out TestIoSys(tptr); tptr->results->errors = totalErrorCount; ShowTestEnd(tptr); + aiori_finalize(tptr); } PrintLongSummaryAllTests(tests_head); @@ -123,7 +125,7 @@ int ior_main(int argc, char **argv) InitTests(tests_head, mpi_comm_world); verbose = tests_head->params.verbose; - aiori_initialize(tests_head); + aiori_initialize(tests_head); // this is quite suspicious, likely an error when multiple tests need to be executed with different backends and options PrintHeader(argc, argv); diff --git a/src/mdtest.c b/src/mdtest.c index 5e792a1d..f82db1bf 100644 --- a/src/mdtest.c +++ b/src/mdtest.c @@ -274,7 +274,7 @@ static void prep_testdir(int j, int dir_iter){ static void phase_end(){ if (call_sync){ if(! backend->sync){ - FAIL("Error, backend does not provide the sync method, but you requested to use sync."); + FAIL("Error, backend does not provide the sync method, but you requested to use sync.\n"); } backend->sync(& param); } From 8abb7a62dcb1c30b05f071da3150d4cb07f5a04d Mon Sep 17 00:00:00 2001 From: Mark Nelson Date: Thu, 28 May 2020 22:28:16 +0000 Subject: [PATCH 200/206] src/aiori-CEPHFS: Add sync Signed-off-by: Mark Nelson --- src/aiori-CEPHFS.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/aiori-CEPHFS.c b/src/aiori-CEPHFS.c index d5c6154a..2c8d34b2 100755 --- a/src/aiori-CEPHFS.c +++ b/src/aiori-CEPHFS.c @@ -76,6 +76,7 @@ static int CEPHFS_MkDir(const char *, mode_t, IOR_param_t *); static int CEPHFS_RmDir(const char *, IOR_param_t *); static int CEPHFS_Access(const char *, int, IOR_param_t *); static int CEPHFS_Stat(const char *, struct stat *, IOR_param_t *); +static void CEPHFS_Sync(IOR_param_t *); static option_help * CEPHFS_options(); /************************** D E C L A R A T I O N S ***************************/ @@ -97,6 +98,7 @@ ior_aiori_t cephfs_aiori = { .rmdir = CEPHFS_RmDir, .access = CEPHFS_Access, .stat = CEPHFS_Stat, + .sync = CEPHFS_Sync, .get_options = CEPHFS_options, }; @@ -359,3 +361,12 @@ static int CEPHFS_Stat(const char *testFileName, struct stat *buf, IOR_param_t * { return ceph_stat(cmount, pfix(testFileName), buf); } + +static void CEPHFS_Sync(IOR_param_t *param) +{ + int ret = ceph_sync_fs(cmount); + if (ret < 0) { + CEPHFS_ERR("ceph_sync_fs failed", ret); + } + +} From efc6d4983245f2cea472a928428d9e236f68124f Mon Sep 17 00:00:00 2001 From: Mark Nelson Date: Thu, 28 May 2020 19:32:13 -0500 Subject: [PATCH 201/206] src/aiori-CEPHFS: Add option checks before init Signed-off-by: Mark Nelson --- src/aiori-CEPHFS.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/aiori-CEPHFS.c b/src/aiori-CEPHFS.c index 2c8d34b2..0d389019 100755 --- a/src/aiori-CEPHFS.c +++ b/src/aiori-CEPHFS.c @@ -125,6 +125,12 @@ static option_help * CEPHFS_options(){ static void CEPHFS_Init() { + /* Short circuit if the options haven't been filled yet. */ + if (!o.user || !o.conf || !o.prefix) { + WARN("CEPHFS_Init() called before options have been populated!"); + return; + } + /* Short circuit if the mount handle already exists */ if (cmount) { return; From edba70e99c7609a84943c0b2ae7317fbbd71b60e Mon Sep 17 00:00:00 2001 From: Mark Nelson Date: Thu, 28 May 2020 20:18:05 -0500 Subject: [PATCH 202/206] aiori-CEPHFS: don't just unmount on close, release cmount too. Signed-off-by: Mark Nelson --- src/aiori-CEPHFS.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/aiori-CEPHFS.c b/src/aiori-CEPHFS.c index 0d389019..27f12dbc 100755 --- a/src/aiori-CEPHFS.c +++ b/src/aiori-CEPHFS.c @@ -173,7 +173,15 @@ static void CEPHFS_Init() static void CEPHFS_Final() { /* shutdown */ - ceph_shutdown(cmount); + int ret = ceph_unmount(cmount); + if (ret < 0) { + CEPHFS_ERR("ceph_umount failed", ret); + } + ret = ceph_release(cmount); + if (ret < 0) { + CEPHFS_ERR("ceph_release failed", ret); + } + cmount = NULL; } static void *CEPHFS_Create(char *testFileName, IOR_param_t * param) @@ -273,7 +281,6 @@ static void CEPHFS_Close(void *file, IOR_param_t * param) CEPHFS_ERR("ceph_close failed", ret); } free(file); - return; } From 2de42103112ecdf44de28daf3e22185d239d5f34 Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Fri, 29 May 2020 16:05:42 +0100 Subject: [PATCH 203/206] Provide a string option parser function. --- src/option.c | 10 ++++++++++ src/option.h | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/option.c b/src/option.c index 26c7b2db..2c3e8ef3 100644 --- a/src/option.c +++ b/src/option.c @@ -351,6 +351,16 @@ static void option_parse_token(char ** argv, int * flag_parsed_next, int * requi } } +int option_parse_str(char*val, options_all_t * opt_all){ + int flag_parsed_next; + int error = 0; + int requiredArgsSeen = 0; + int print_help = 0; + char * argv[2] = {val, NULL}; + option_parse_token(argv, & flag_parsed_next, & requiredArgsSeen, opt_all, & error, & print_help); + return error; +} + int option_parse_key_value(char * key, char *val, options_all_t * opt_all){ int flag_parsed_next; int error = 0; diff --git a/src/option.h b/src/option.h index 72259216..624da510 100644 --- a/src/option.h +++ b/src/option.h @@ -38,9 +38,9 @@ typedef struct{ int64_t string_to_bytes(char *size_str); void option_print_current(option_help * args); - //@return the number of parsed arguments int option_parse(int argc, char ** argv, options_all_t * args); +int option_parse_str(char*val, options_all_t * opt_all); /* Parse a single line */ int option_parse_key_value(char * key, char * value, options_all_t * opt_all); From 3623f3b8bdf56d49f9307682c15fbbcffcd58a16 Mon Sep 17 00:00:00 2001 From: Mohamad Chaarawi Date: Sun, 31 May 2020 16:20:58 +0000 Subject: [PATCH 204/206] DFS: allow multiple IO phases when ior_run() is called - init/fini can be made multiple times - reset tunables on fini so they can change on a re-init Signed-off-by: Mohamad Chaarawi --- src/aiori-DAOS.c | 4 +--- src/aiori-DFS.c | 29 +++++++++++++++++++++++++---- src/ior.c | 2 ++ 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/src/aiori-DAOS.c b/src/aiori-DAOS.c index 7f07f7c4..8fa15786 100644 --- a/src/aiori-DAOS.c +++ b/src/aiori-DAOS.c @@ -216,10 +216,8 @@ DAOS_Init() if (daos_initialized) return; - if (o.pool == NULL || o.svcl == NULL || o.cont == NULL) { - GERR("Invalid DAOS pool/cont\n"); + if (o.pool == NULL || o.svcl == NULL || o.cont == NULL) return; - } if (o.oclass) { objectClass = daos_oclass_name2id(o.oclass); diff --git a/src/aiori-DFS.c b/src/aiori-DFS.c index 64874887..e7b1d6b9 100755 --- a/src/aiori-DFS.c +++ b/src/aiori-DFS.c @@ -42,6 +42,7 @@ static daos_handle_t poh, coh; static daos_oclass_id_t objectClass = OC_SX; static daos_oclass_id_t dir_oclass = OC_SX; static struct d_hash_table *dir_hash; +static bool dfs_init; struct aiori_dir_hdl { d_list_t entry; @@ -386,8 +387,16 @@ static void DFS_Init() { int rc; + /** in case we are already initialized, return */ + if (dfs_init) + return; + + /** shouldn't be fatal since it can be called with POSIX backend selection */ if (o.pool == NULL || o.svcl == NULL || o.cont == NULL) - ERR("Invalid pool or container options\n"); + return; + + rc = daos_init(); + DCHECK(rc, "Failed to initialize daos"); if (o.oclass) { objectClass = daos_oclass_name2id(o.oclass); @@ -401,9 +410,6 @@ DFS_Init() { GERR("Invalid DAOS directory object class %s\n", o.dir_oclass); } - rc = daos_init(); - DCHECK(rc, "Failed to initialize daos"); - rc = d_hash_table_create(0, 16, NULL, &hdl_hash_ops, &dir_hash); DCHECK(rc, "Failed to initialize dir hashtable"); @@ -457,6 +463,7 @@ DFS_Init() { rc = dfs_set_prefix(dfs, o.prefix); DCHECK(rc, "Failed to set DFS Prefix"); } + dfs_init = true; } static void @@ -510,6 +517,20 @@ DFS_Finalize() rc = daos_fini(); DCHECK(rc, "Failed to finalize DAOS"); + + /** reset tunables */ + o.pool = NULL; + o.svcl = NULL; + o.group = NULL; + o.cont = NULL; + o.chunk_size = 1048576; + o.oclass = NULL; + o.dir_oclass = NULL; + o.prefix = NULL; + o.destroy = 0; + objectClass = OC_SX; + dir_oclass = OC_SX; + dfs_init = false; } /* diff --git a/src/ior.c b/src/ior.c index 05070b03..361a9a4c 100755 --- a/src/ior.c +++ b/src/ior.c @@ -1625,6 +1625,8 @@ static void ValidateTests(IOR_param_t * test) && (strcasecmp(test->api, "MPIIO") != 0) && (strcasecmp(test->api, "MMAP") != 0) && (strcasecmp(test->api, "HDFS") != 0) + && (strcasecmp(test->api, "DFS") != 0) + && (strcasecmp(test->api, "DAOS") != 0) && (strcasecmp(test->api, "Gfarm") != 0) && (strcasecmp(test->api, "RADOS") != 0) && (strcasecmp(test->api, "CEPHFS") != 0)) && test->fsync) From 3eb488939ddb42434afad5e741e17b8564644582 Mon Sep 17 00:00:00 2001 From: Sven Breuner Date: Wed, 3 Jun 2020 00:30:38 +0300 Subject: [PATCH 205/206] mdtest: allocate aligned buffers to support DirectIO --- src/mdtest.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/mdtest.c b/src/mdtest.c index f82db1bf..c8a8b477 100644 --- a/src/mdtest.c +++ b/src/mdtest.c @@ -633,8 +633,8 @@ void mdtest_read(int random, int dirs, const long dir_iter, char *path) { /* allocate read buffer */ if (read_bytes > 0) { - read_buffer = (char *)malloc(read_bytes); - if (read_buffer == NULL) { + int alloc_res = posix_memalign((void**)&read_buffer, sysconf(_SC_PAGESIZE), write_bytes); + if (alloc_res) { FAIL("out of memory"); } @@ -2125,8 +2125,8 @@ mdtest_results_t * mdtest_run(int argc, char **argv, MPI_Comm world_com, FILE * /* allocate and initialize write buffer with # */ if (write_bytes > 0) { - write_buffer = (char *)malloc(write_bytes); - if (write_buffer == NULL) { + int alloc_res = posix_memalign((void**)&write_buffer, sysconf(_SC_PAGESIZE), write_bytes); + if (alloc_res) { FAIL("out of memory"); } generate_memory_pattern(write_buffer, write_bytes); From c828a0f32c50c8644832b6fab21974880860e415 Mon Sep 17 00:00:00 2001 From: Sven Breuner Date: Wed, 3 Jun 2020 01:22:44 +0300 Subject: [PATCH 206/206] mdtest: use correct number of bytes for posix_memalign of read buffer Fixes a typo in initial posix_memalign commit for DirectIO support. --- src/mdtest.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mdtest.c b/src/mdtest.c index c8a8b477..105837f3 100644 --- a/src/mdtest.c +++ b/src/mdtest.c @@ -633,7 +633,7 @@ void mdtest_read(int random, int dirs, const long dir_iter, char *path) { /* allocate read buffer */ if (read_bytes > 0) { - int alloc_res = posix_memalign((void**)&read_buffer, sysconf(_SC_PAGESIZE), write_bytes); + int alloc_res = posix_memalign((void**)&read_buffer, sysconf(_SC_PAGESIZE), read_bytes); if (alloc_res) { FAIL("out of memory"); }