Test bench

.gitignore

@@ -2,3 +2,11 @@
 #
 # See LICENCE.txt for license information
 /build
+
+.clangd
+
+.vscode
+
+*result*/
+*.xls
+*.out

README.md

@@ -59,6 +59,7 @@ All tests support the same set of arguments :
   * `-n,--iters <iteration count>` number of iterations. Default : 20.
   * `-w,--warmup_iters <warmup iteration count>` number of warmup iterations (not timed). Default : 5.
   * `-m,--agg_iters <aggregation count>` number of operations to aggregate together in each iteration. Default : 1.
+  * `-M,--multi_iters <multi seprate ncclComm iteration count>` number of operations with seprate ncclComm in each iteration. Default : 1.
   * `-a,--average <0/1/2/3>` Report performance as an average across all ranks (MPI=1 only). <0=Rank0,1=Avg,2=Min,3=Max>. Default : 1.
 * Test operation
   * `-p,--parallel_init <0/1>` use threads to initialize NCCL in parallel. Default : 0.

nccl_test.sh

@@ -0,0 +1,87 @@
+clear
+
+export MY_NUM_DEV=$1
+
+cd /home/panlichen/work2/nccl-tests
+export LD_LIBRARY_PATH=/home/panlichen/work2/ofccl/build/lib
+export NCCL_PROTO=Simple
+export NCCL_ALGO=Ring
+# export NCCL_MAX_NCHANNELS=1
+# export NCCL_MIN_NCHANNELS=1
+# export NCCL_NTHREADS=64
+
+if [ -z $BINARY ];then
+    BINARY="DEBUG"
+    # BINARY="MS"
+    # BINARY="PERF"
+fi
+
+FUNC=$2
+
+if [ "$FUNC" == "AR" ]; then
+    target="./build/all_reduce_perf"
+elif [ "$FUNC" == "AG" ]; then
+    target="./build/all_gather_perf"
+elif [ "$FUNC" == "RS" ]; then
+    target="./build/reduce_scatter_perf"
+elif [ "$FUNC" == "R" ]; then
+    target="./build/reduce_perf"
+elif [ "$FUNC" == "B" ]; then
+    target="./build/broadcast_perf"
+fi
+
+
+if [ "$BINARY" == "DEBUG" ];then
+    if [ $MY_NUM_DEV = 4 ]; then
+        export CUDA_VISIBLE_DEVICES=0,1,4,5
+    fi
+    export NITER=5
+    export NBYTES=64M
+    export WARMITER=2
+    export MITER=1
+    export CHECK=0
+elif [ "$BINARY" == "PERF" ];then
+    if [ $MY_NUM_DEV = 4 ]; then
+        export CUDA_VISIBLE_DEVICES=0,1,4,5
+    fi
+    export NITER=4
+    export NBYTES=8K
+    export WARMITER=2
+    export MITER=4
+    export CHECK=0
+elif [ "$BINARY" == "MS" ];then
+    if [ $MY_NUM_DEV = 4 ]; then
+        export CUDA_VISIBLE_DEVICES=0,1,4,5
+    fi
+    # export NITER=200
+    # export SHOW_ALL_PREPARED_COLL=1
+    # export WARMITER=0
+    # export NBYTES=8K
+    # export MITER=4
+fi
+
+export NSYS_FILE="nccl"
+export NCU_FILE="nccl"
+
+if [ -z $RUN_TYPE ];then
+    RUN_TYPE="PURE"
+    # RUN_TYPE="GDB"
+    # RUN_TYPE="NSYS"
+    # RUN_TYPE="NCU"
+fi
+
+if [ "$RUN_TYPE" == "PURE" ];then
+    cmd="$target -b $NBYTES -e $NBYTES -f 2 -t $MY_NUM_DEV -g 1 -n $NITER -w $WARMITER -c $CHECK -m $MITER"
+elif [ "$RUN_TYPE" == "GDB" ];then
+    cmd="cuda-gdb $target"
+    # set args -b 8M -e 8M -f 2 -t 2 -g 1 -n 1 -w 0 -c 0
+elif [ "$RUN_TYPE" == "NSYS" ];then
+    cmd="nsys profile -f true --trace=cuda,cudnn,cublas,osrt,nvtx -o /home/panlichen/work2/ofccl/log/nsys/$NSYS_FILE $target -b $NBYTES -e $NBYTES -f 2 -t $MY_NUM_DEV -g 1 -n $NITER -w $WARMITER -c $CHECK -m $MITER"
+elif [ "$RUN_TYPE" == "NCU" ];then
+    # cmd="ncu --nvtx -f -o /home/panlichen/work2/ofccl/log/nsys/$NCU_FILE $target -b $NBYTES -e $NBYTES -f 2 -t $MY_NUM_DEV -g 1 -n $NITER -w $WARMITER -c $CHECK -m $MITER"
+    cmd="ncu $target -b $NBYTES -e $NBYTES -f 2 -t $MY_NUM_DEV -g 1 -n $NITER -w $WARMITER -c $CHECK -m $MITER"
+fi
+
+echo cmd=$cmd
+$cmd #> /home/panlichen/work2/ofccl/log/ofccl-2ms-coll-master.log
+

ofccl_test.sh

@@ -0,0 +1,166 @@
+clear
+
+export MY_NUM_DEV=$1
+
+export DEBUG_CC=1
+export DEBUG_ENQ=1
+
+unset DEBUG_CC
+unset DEBUG_ENQ
+
+export DEBUG_NT=1
+unset DEBUG_NT
+
+cd /home/panlichen/work2/nccl-tests
+export LD_LIBRARY_PATH=/home/panlichen/work2/ofccl/build/lib
+export NCCL_PROTO=Simple
+export NCCL_ALGO=Ring
+# export NCCL_MAX_NCHANNELS=1
+# export NCCL_MIN_NCHANNELS=1
+# export NCCL_NTHREADS=64
+
+export CHECK=0
+export SHOW_ALL_PREPARED_COLL=0
+
+export TRAVERSE_TIMES=10
+export TOLERANT_UNPROGRESSED_CNT=10000
+export BASE_CTX_SWITCH_THRESHOLD=80
+export BOUNS_SWITCH_4_PROCESSED_COLL=0
+export DEV_TRY_ROUND=10
+export CHECK_REMAINING_SQE_INTERVAL=10000
+export DEBUG_FILE="/home/panlichen/work2/ofccl/log/oneflow_cpu_rank_"
+
+rm -rf /home/panlichen/work2/ofccl/log
+mkdir -p /home/panlichen/work2/ofccl/log
+
+# export ENABLE_VQ=1 # volunteer quit
+# export TOLERANT_FAIL_CHECK_SQ_CNT=5000
+# export CNT_BEFORE_QUIT=5
+
+echo TRAVERSE_TIMES=$TRAVERSE_TIMES
+echo TOLERANT_UNPROGRESSED_CNT=$TOLERANT_UNPROGRESSED_CNT
+echo BASE_CTX_SWITCH_THRESHOLD=$BASE_CTX_SWITCH_THRESHOLD
+echo BOUNS_SWITCH_4_PROCESSED_COLL=$BOUNS_SWITCH_4_PROCESSED_COLL
+echo DEV_TRY_ROUND=$DEV_TRY_ROUND
+echo CHECK_REMAINING_SQE_INTERVAL=$CHECK_REMAINING_SQE_INTERVAL
+echo DEBUG_FILE=$DEBUG_FILE
+
+if [ ! -z $ENABLE_VQ ];then
+    echo TOLERANT_FAIL_CHECK_SQ_CNT=$TOLERANT_FAIL_CHECK_SQ_CNT
+    echo CNT_BEFORE_QUIT=$CNT_BEFORE_QUIT
+fi
+
+FUNC=$2
+if [ -z $FUNC ]; then
+    FUNC="AR"
+fi
+
+if [ "$FUNC" == "AR" ]; then
+    target="./build/ofccl_all_reduce_perf"
+elif [ "$FUNC" == "AG" ]; then
+    target="./build/ofccl_all_gather_perf"
+elif [ "$FUNC" == "RS" ]; then
+    target="./build/ofccl_reduce_scatter_perf"
+elif [ "$FUNC" == "R" ]; then
+    target="./build/ofccl_reduce_perf"
+elif [ "$FUNC" == "B" ]; then
+    target="./build/ofccl_broadcast_perf"
+fi
+
+if [ -z $BINARY ];then
+    BINARY="DEBUG"
+    # BINARY="MS"
+    # BINARY="PERF"
+fi
+
+if [ "$BINARY" == "DEBUG" ];then
+    if [ $MY_NUM_DEV = 4 ]; then
+        export CUDA_VISIBLE_DEVICES=0,1,4,5
+    fi
+    if [ $MY_NUM_DEV = 2 ]; then
+        export CUDA_VISIBLE_DEVICES=4,5
+    fi
+    export NITER=5
+    export NBYTES=64M
+    export WARMITER=2
+    export MITER=1
+elif [ "$BINARY" == "PERF" ];then
+    if [ $MY_NUM_DEV = 4 ]; then
+        export CUDA_VISIBLE_DEVICES=0,1,4,5
+    fi
+    export NITER=8
+    export NBYTES=8K
+    export WARMITER=2
+    export MITER=1
+elif [ "$BINARY" == "MS" ];then
+    target="./build/ofccl_all_reduce_ms_perf"
+    if [ $MY_NUM_DEV = 4 ]; then
+        export CUDA_VISIBLE_DEVICES=0,1,4,5
+    fi
+    export NITER=200
+    export SHOW_ALL_PREPARED_COLL=1
+    export WARMITER=0
+    export NBYTES=8K
+    export MITER=4
+    export CHECK=0
+fi
+
+export NSYS_FILE="ofccl"
+export NCU_FILE="ofccl"
+
+if [ -z $RUN_TYPE ];then
+    RUN_TYPE="PURE"
+    # RUN_TYPE="GDB"
+    # RUN_TYPE="NSYS"
+    # RUN_TYPE="NCU"
+fi
+
+# typedef enum { ncclInt8       = 0, ncclChar       = 0,
+#                ncclUint8      = 1,
+#                ncclInt32      = 2, ncclInt        = 2,
+#                ncclUint32     = 3,
+#                ncclInt64      = 4,
+#                ncclUint64     = 5,
+#                ncclFloat16    = 6, ncclHalf       = 6,
+#                ncclFloat32    = 7, ncclFloat      = 7,
+#                ncclFloat64    = 8, ncclDouble     = 8,
+# #if defined(__CUDA_BF16_TYPES_EXIST__)
+#                ncclBfloat16   = 9,
+#                ncclNumTypes   = 10
+# #else
+#                ncclNumTypes   = 9
+# #endif
+# } ncclDataType_t;
+
+# 用这个：
+# const char *test_typenames[ncclNumTypes] = {"int8",
+#                                             "uint8",
+#                                             "int32",
+#                                             "uint32",
+#                                             "int64",
+#                                             "uint64",
+#                                             "half",
+#                                             "float",
+#                                             "double"
+# #if defined(__CUDA_BF16_TYPES_EXIST__) &&                                      \
+#     NCCL_VERSION_CODE >= NCCL_VERSION(2, 10, 0)
+#                                             ,
+#                                             "bfloat16"
+# #endif
+# };
+
+if [ "$RUN_TYPE" == "PURE" ];then
+    cmd="$target -b $NBYTES -e $NBYTES -f 2 -t $MY_NUM_DEV -g 1 -n $NITER -w $WARMITER -c $CHECK -M $MITER" #  -d half
+elif [ "$RUN_TYPE" == "GDB" ];then
+    cmd="cuda-gdb $target"
+    # set args -b 64 -e 64 -f 2 -t 2 -g 1 -n 1 -w 0 -c 0
+elif [ "$RUN_TYPE" == "NSYS" ];then
+    cmd="nsys profile -f true --trace=cuda,cudnn,cublas,osrt,nvtx -o /home/panlichen/work2/ofccl/log/nsys/$NSYS_FILE $target -b $NBYTES -e $NBYTES -f 2 -t $MY_NUM_DEV -g 1 -n $NITER -w $WARMITER -c $CHECK -M $MITER"
+elif [ "$RUN_TYPE" == "NCU" ];then
+    # cmd="ncu --nvtx -f -o /home/panlichen/work2/ofccl/log/nsys/$NCU_FILE $target -b $NBYTES -e $NBYTES -f 2 -t $MY_NUM_DEV -g 1 -n $NITER -w $WARMITER -c $CHECK -M $MITER"
+    cmd="ncu $target -b $NBYTES -e $NBYTES -f 2 -t $MY_NUM_DEV -g 1 -n $NITER -w $WARMITER -c $CHECK -M $MITER"
+fi
+
+echo cmd=$cmd
+$cmd #> /home/panlichen/work2/ofccl/log/ofccl.log
+

src/Makefile

@@ -7,7 +7,7 @@
 CUDA_HOME ?= /usr/local/cuda
 PREFIX ?= /usr/local
 VERBOSE ?= 0
-DEBUG ?= 0
+DEBUG_NT ?= 0
 
 CUDA_LIB ?= $(CUDA_HOME)/lib64
 CUDA_INC ?= $(CUDA_HOME)/include
@@ -19,27 +19,39 @@ CUDA_MAJOR = $(shell echo $(CUDA_VERSION) | cut -d "." -f 1)
 
 # Better define NVCC_GENCODE in your environment to the minimal set
 # of archs to reduce compile time.
-ifeq ($(shell test "0$(CUDA_MAJOR)" -ge 11; echo $$?),0)
-NVCC_GENCODE ?= -gencode=arch=compute_60,code=sm_60 \
-                -gencode=arch=compute_61,code=sm_61 \
-                -gencode=arch=compute_70,code=sm_70 \
-                -gencode=arch=compute_80,code=sm_80 \
-                -gencode=arch=compute_80,code=compute_80
+# ifeq ($(shell test "0$(CUDA_MAJOR)" -ge 11; echo $$?),0)
+# NVCC_GENCODE ?= -gencode=arch=compute_60,code=sm_60 \
+#                 -gencode=arch=compute_61,code=sm_61 \
+#                 -gencode=arch=compute_70,code=sm_70 \
+#                 -gencode=arch=compute_80,code=sm_80 \
+#                 -gencode=arch=compute_80,code=compute_80
+# else
+# NVCC_GENCODE ?= -gencode=arch=compute_35,code=sm_35 \
+#                 -gencode=arch=compute_50,code=sm_50 \
+#                 -gencode=arch=compute_60,code=sm_60 \
+#                 -gencode=arch=compute_61,code=sm_61 \
+#                 -gencode=arch=compute_70,code=sm_70 \
+#                 -gencode=arch=compute_70,code=compute_70
+# endif
+
+CUDA_GENCODE_3080   = -gencode=arch=compute_86,code=sm_86
+CUDA_GENCODE_2080   = -gencode=arch=compute_75,code=sm_75
+
+CARDNAME ?= 3080
+ifeq ($(CARDNAME), 3080)
+NVCC_GENCODE ?= $(CUDA_GENCODE_3080) $(CUDA_PTX_INUSE)
 else
-NVCC_GENCODE ?= -gencode=arch=compute_35,code=sm_35 \
-                -gencode=arch=compute_50,code=sm_50 \
-                -gencode=arch=compute_60,code=sm_60 \
-                -gencode=arch=compute_61,code=sm_61 \
-                -gencode=arch=compute_70,code=sm_70 \
-                -gencode=arch=compute_70,code=compute_70
+NVCC_GENCODE ?= $(CUDA_GENCODE_2080) $(CUDA_PTX_INUSE)
 endif
+$(info CARDNAME $(CARDNAME))
+$(info NVCC_GENCODE $(NVCC_GENCODE))
 
 NVCUFLAGS  := -ccbin $(CXX) $(NVCC_GENCODE) -std=c++11
 
 LDFLAGS    := -L${CUDA_LIB} -lcudart -lrt
 NVLDFLAGS  := -L${CUDA_LIB} -l${CUDARTLIB} -lrt
 
-ifeq ($(DEBUG), 0)
+ifeq ($(DEBUG_NT), 0)
 NVCUFLAGS += -O3 -g
 CXXFLAGS  += -O3 -g
 else
@@ -72,6 +84,8 @@ endif
 LIBRARIES += nccl
 NVLDFLAGS += $(LIBRARIES:%=-l%)
 
+$(info CARDNAME $(NVCUFLAGS))
+
 DST_DIR := $(BUILDDIR)
 SRC_FILES := $(wildcard *.cu)
 OBJ_FILES := $(SRC_FILES:%.cu=${DST_DIR}/%.o)