test_dsync: add test_expectfail.sh, test_existence.sh, and stubs for two other tests

test/tests/test_dsync/test_data.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+##############################################################################
+# Description:
+#
+#   Verify dsync handles data
+#     - file data is identical after a copy
+#     - file with differing data is copied if --contents arg is used
+#     - file with differing data is not copied if --contents arg is not used and metadata match
+#
+# Notes:
+#   - does not test whether data copes are spread across nodes/tasks evenly
+#
+##############################################################################
+
+# Turn on verbose output
+#set -x
+
+DSYNC_TEST_BIN=${DSYNC_TEST_BIN:-${1}}
+DSYNC_SRC_DIR=${DSYNC_SRC_DIR:-${2}}
+DSYNC_DEST_DIR=${DSYNC_DEST_DIR:-${3}}
+DSYNC_TMP_FILE=${DSYNC_TMP_FILE:-${4}}
+
+echo "Using dsync binary at: $DSYNC_TEST_BIN"
+echo "Using src directory at: $DSYNC_SRC_DIR"
+echo "Using dest directory at: $DSYNC_DEST_DIR"
+echo "Using directory tree: $DSYNC_TMP_FILE"
+
+# tests not yet implemented
+
+exit 0

test/tests/test_dsync/test_existence.sh

@@ -0,0 +1,183 @@
+#!/bin/bash
+
+. utility/set_funcs.sh
+
+##############################################################################
+# Description:
+#
+#   Verify dsync handles presence or absence
+#     - empty source, but non-empty destination
+#     - non-empty source, but empty destination
+#     - directories on source, but not on destination, are copied
+#     - files on source, but not on destination, are copied
+#     - directories on destination, but not source, are undisturbed without --delete
+#     - files on destination, but not source, are undisturbed without --delete
+#     - directories on destination, but not source, are removed with --delete
+#     - files on destination, but not source, are removed with --delete
+#
+# Notes:
+#
+##############################################################################
+
+# Turn on verbose output
+#set -x
+
+DSYNC_TEST_BIN=${DSYNC_TEST_BIN:-${1}}
+DSYNC_SRC_BASE=${DSYNC_SRC_BASE:-${2}}
+DSYNC_DEST_BASE=${DSYNC_DEST_BASE:-${3}}
+DSYNC_TREE_NAME=${DSYNC_TREE_NAME:-${4}}
+
+DSYNC_TREE_DATA=/usr/include/c++
+
+mpirun=$(which mpirun 2>/dev/null)
+mpirun_opts=""
+if [[ -n $mpirun ]]; then
+	procs=$(( $(nproc ) / 8 ))
+	if [[ $procs -gt 16 ]]; then
+		procs=16
+	fi
+	mpirun_opts="-c $procs"
+
+	echo "Using mpirun: $mpirun $mpirun_opts"
+fi
+
+echo "Using dsync binary at: $DSYNC_TEST_BIN"
+echo "Using src parent directory at: $DSYNC_SRC_BASE"
+echo "Using dest parent directory at: $DSYNC_DEST_BASE"
+echo "Using test data from: $DSYNC_TREE_DATA"
+
+DSYNC_SRC_DIR=$(mktemp --directory ${DSYNC_SRC_BASE}/${DSYNC_TREE_NAME}.XXXXX)
+DSYNC_DEST_DIR=$(mktemp --directory ${DSYNC_DEST_BASE}/${DSYNC_TREE_NAME}.XXXXX)
+
+function fs_type()
+{
+	fname=$1
+	df -T ${fname} | awk '$1 != "Filesystem" {print $2}'
+}
+
+function list_all_files()
+{
+	find $1 -printf '%P\n' | sort | grep -v '^$'
+}
+
+function sync_and_verify()
+{
+	local srcdir=$1
+	local destdir=$2
+	local name=$3
+	local expectation=$4
+
+	local result=0
+	local dest_type=""
+
+	src_list=$(mktemp /tmp/sync_and_verify.src.XXXXX)
+	list_all_files $srcdir > $src_list
+
+	dest_list=$(mktemp /tmp/sync_and_verify.dest.XXXXX)
+
+	# dest may not exist, but its parent must exist
+	if [[ -d $destdir ]]; then
+		dest_type=$(fs_type $destdir)
+		list_all_files $destdir > $dest_list
+	else
+		parent=$(dirname $destdir)
+		if [[ ! -d $parent ]]; then
+			echo "sync_and_verify: destdir $destdir and its parent do not exist"
+			exit 1
+		fi
+		dest_type=$(fs_type $parent)
+	fi
+
+	quiet_opt="--quiet"
+	delete_opt=""
+	if [[ $name = "delete" ]]; then
+		delete_opt="--delete"
+	fi
+
+	if [[ -n $mpirun ]]; then
+		$mpirun $mpirun_opts $DSYNC_TEST_BIN $quiet_opt $delete_opt $srcdir $destdir
+	else
+		$mpirun $mpirun_opts $DSYNC_TEST_BIN $quiet_opt $delete_opt $srcdir $destdir
+	fi
+	rc=$?
+
+	if [[ $rc -ne 0 ]]; then
+		echo "dsync failed with rc $rc"
+		result=1
+	fi
+
+	if [[ $result -eq 0 ]]; then
+		after_list=$(mktemp /tmp/sync_and_verify.after.XXXXX)
+		list_all_files $destdir > $after_list
+
+		expected_list=$(mktemp /tmp/sync_and_verify.expected.XXXXX)
+
+		case $expectation in
+		  "union")
+			union $src_list $dest_list > $expected_list
+			sets_equal $after_list $expected_list
+			result=$?
+			;;
+		  "src_exactly")
+			cat $src_list > $expected_list
+			sets_equal $after_list $expected_list
+			result=$?
+			;;
+		esac
+	fi
+
+	if [ "$result" -eq 0 ]; then
+		echo "PASSED verify of option $name for $destdir type $dest_type"
+	else
+		echo "FAILED verify of option $name for $destdir type $dest_type"
+		echo =======================
+		echo "before: src_list"
+		cat $src_list
+		echo
+		echo "before: dest_list"
+		cat $dest_list
+		echo
+		echo "after: after_list:"
+		cat $after_list
+		echo
+		echo "expected:"
+		cat $expected_list
+		echo =======================
+	fi
+
+	rm $src_list $dest_list $after_list $expected_list
+
+	return $result
+}
+
+# empty source, but non-empty destination
+# directories on destination, but not source, are undisturbed without --delete
+# files on destination, but not source, are undisturbed without --delete
+rm -fr $DSYNC_SRC_DIR/stuff
+rm -fr $DSYNC_DEST_DIR/stuff
+mkdir $DSYNC_SRC_DIR/stuff
+mkdir $DSYNC_DEST_DIR/stuff
+cp -a $DSYNC_TREE_DATA $DSYNC_DEST_DIR/stuff
+sync_and_verify  $DSYNC_SRC_DIR/stuff $DSYNC_DEST_DIR/stuff empty_source union
+
+# non-empty source, but empty destination
+# files on source, but not on destination, are copied
+# directories on source, but not on destination, are copied
+rm -fr $DSYNC_SRC_DIR/stuff
+rm -fr $DSYNC_DEST_DIR/stuff
+mkdir $DSYNC_SRC_DIR/stuff
+cp -a $DSYNC_TREE_DATA $DSYNC_SRC_DIR/stuff
+sync_and_verify  $DSYNC_SRC_DIR/stuff $DSYNC_DEST_DIR/stuff empty_destination union
+
+# directories on destination, but not source, are removed with --delete
+# files on destination, but not source, are removed with --delete
+rm -fr $DSYNC_SRC_DIR/stuff
+rm -fr $DSYNC_DEST_DIR/stuff
+mkdir $DSYNC_SRC_DIR/stuff
+mkdir $DSYNC_DEST_DIR/stuff
+cp -a $DSYNC_TREE_DATA $DSYNC_SRC_DIR/stuff
+mkdir $DSYNC_DEST_DIR/stuff/destdir
+touch $DSYNC_DEST_DIR/stuff/destfile
+sync_and_verify  $DSYNC_SRC_DIR/stuff $DSYNC_DEST_DIR/stuff delete src_exactly
+
+exit 0

test/tests/test_dsync/test_expectfail.sh

@@ -0,0 +1,71 @@
+#!/bin/bash
+
+. utility/set_funcs.sh
+
+##############################################################################
+# Description:
+#
+#   Verify dsync fails when it should
+#      - source does not exist
+#      - parent of dest does not exist
+#
+# Notes:
+#
+##############################################################################
+
+# Turn on verbose output
+#set -x
+
+DSYNC_TEST_BIN=${DSYNC_TEST_BIN:-${1}}
+DSYNC_SRC_BASE=${DSYNC_SRC_BASE:-${2}}
+DSYNC_DEST_BASE=${DSYNC_DEST_BASE:-${3}}
+DSYNC_TREE_NAME=${DSYNC_TREE_NAME:-${4}}
+
+echo "Using dsync binary at: $DSYNC_TEST_BIN"
+echo "Using src parent directory at: $DSYNC_SRC_BASE"
+echo "Using dest parent directory at: $DSYNC_DEST_BASE"
+
+DSYNC_SRC_DIR=$(mktemp --directory ${DSYNC_SRC_BASE}/${DSYNC_TREE_NAME}.XXXXX)
+DSYNC_DEST_DIR=$(mktemp --directory ${DSYNC_DEST_BASE}/${DSYNC_TREE_NAME}.XXXXX)
+
+function sync_and_verify()
+{
+	local srcdir=$1
+	local destdir=$2
+	local name=$3
+	local expectation=$4
+
+	local result=0
+	local dest_type="unknown"
+
+	quiet_opt="--quiet"
+
+	$DSYNC_TEST_BIN $quiet_opt $delete_opt $srcdir $destdir
+	rc=$?
+	echo "dsync failed with rc $rc"
+
+	if [[ "$rc" -eq 0 ]]; then
+		echo "FAILED verify of option $name for $destdir type $dest_type"
+		result=1
+
+	else
+		echo "PASSED verify of option $name for $destdir type $dest_type"
+		result=0
+	fi
+
+	return $result
+}
+
+# source does not exist
+rm -fr $DSYNC_SRC_DIR/stuff
+rm -fr $DSYNC_DEST_DIR/stuff
+mkdir $DSYNC_DEST_DIR/stuff
+sync_and_verify  $DSYNC_SRC_DIR/stuff $DSYNC_DEST_DIR/stuff source_missing expect_dsync_fail
+
+# parent of dest does not exist
+rm -fr $DSYNC_SRC_DIR/stuff
+rm -fr $DSYNC_DEST_DIR/stuff
+mkdir $DSYNC_SRC_DIR/stuff
+sync_and_verify  $DSYNC_SRC_DIR/stuff $DSYNC_DEST_DIR/stuff/childdir dest_parent_missing expect_dsync_fail
+
+exit 0

test/tests/test_dsync/test_metadata.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+##############################################################################
+# Description:
+#
+#   Verify dsync handles metdata
+#     - file with differing metadata is copied
+#     - file metadata is identical after a copy
+#     - directory metadata is identical after a sync
+#
+# Notes:
+#
+##############################################################################
+
+# Turn on verbose output
+#set -x
+
+DSYNC_TEST_BIN=${DSYNC_TEST_BIN:-${1}}
+DSYNC_SRC_DIR=${DSYNC_SRC_DIR:-${2}}
+DSYNC_DEST_DIR=${DSYNC_DEST_DIR:-${3}}
+DSYNC_TMP_FILE=${DSYNC_TMP_FILE:-${4}}
+
+echo "Using dsync binary at: $DSYNC_TEST_BIN"
+echo "Using src directory at: $DSYNC_SRC_DIR"
+echo "Using dest directory at: $DSYNC_DEST_DIR"
+echo "Using directory tree: $DSYNC_TMP_FILE"
+
+# tests not yet implemented
+
+exit 0

test/tests/test_dsync/utility/one

@@ -0,0 +1,4 @@
+1
+2
+3
+4

test/tests/test_dsync/utility/set_funcs.sh

@@ -0,0 +1,31 @@
+#set -x
+
+function union() {
+	funcname=union
+	if [[ $# -lt 2 ]] || [[ ! -f $1 ]] || [[ ! -f $2 ]]; then
+		echo "$0: ${funcname}: requires 2 arguments, both files: $*"
+		exit
+	fi
+
+	sort -u <(sort $1) <(sort $2)
+}
+
+function intersection() {
+	funcname=intersection
+	if [[ $# -lt 2 ]] || [[ ! -f $1 ]] || [[ ! -f $2 ]]; then
+		echo "$0: ${funcname}: requires 2 arguments, both files: $*"
+		exit
+	fi
+
+	comm -12 <(sort $1) <(sort $2)
+}
+
+function sets_equal() {
+	funcname=sets_equal
+	if [[ $# -lt 2 ]] || [[ ! -f $1 ]] || [[ ! -f $2 ]]; then
+		echo "$0: ${funcname}: requires 2 arguments, both files: $*"
+		exit
+	fi
+
+	diff -q <(sort $1) <(sort $2) >/dev/null
+}

test/tests/test_dsync/utility/test_driver.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/bash
+
+. set_funcs.sh
+
+#echo test union with wrong arg count
+#union one || echo expected failure for wrong arg count
+#
+#echo test union with non-files
+#union notafile1 notafile2 || echo expected failure for non-files
+
+
+echo union: $(union one two)
+echo intersection: $(intersection one two)
+echo sets_equal 1 and 2: $(sets_equal one two; echo $?)
+echo sets_equal 1 and 1: $(sets_equal one one; echo $?)

test/tests/test_dsync/utility/two

@@ -0,0 +1,4 @@
+3
+4
+5
+6