ADMIN_NODE=node-1
CLIENT_NODES=node-2, node-3
SERVER_NODES=node-4, node-5
ALL_NODES=$ADMIN_NODE,$CLIENT_NODES,$SERVER_NODES

module load gnu-mpich/3.4~a2

or 

export LD_LIBRARY_PATH=<mpich lib path>:$LD_LIBRARY_PATH
export PATH=<mpich bin path>:$PATH


# Download ior source 
git clone https://github.com/hpc/ior.git 

# Build IOR 
cd ior 
./bootstrap 
mkdir build;cd build 
MPICC=mpicc ../configure --with-daos=/usr --prefix=<your dir> 
make 
make install 

# Add IOR to paths  add <your dir>/lib to LD_LIBRARY_PATh and <your dir>/bin to PATH

mpirun -hosts <hosts> -np 16 --ppn 16 dcp --bufsize 64MB --chunksize 128MB /tmp/daos_dfuse/daos_test daos://$DAOS_POOL/$DAOS_CONT3


#Sample output

[2021-04-29T23:55:52] Walking /tmp/daos_dfuse/daos_test
[2021-04-29T23:55:52] Walked 11 items in 0.026 secs (417.452 items/sec) ...
[2021-04-29T23:55:52] Walked 11 items in 0.026 seconds (415.641 items/sec)
[2021-04-29T23:55:52] Copying to /
[2021-04-29T23:55:52] Items: 11
[2021-04-29T23:55:52]   Directories: 1
[2021-04-29T23:55:52]   Files: 10
[2021-04-29T23:55:52]   Links: 0
[2021-04-29T23:55:52] Data: 10.000 GiB (1.000 GiB per file)
[2021-04-29T23:55:52] Creating 1 directories
[2021-04-29T23:55:52] Creating 10 files.
[2021-04-29T23:55:52] Copying data.
[2021-04-29T23:56:53] Copied 1.312 GiB (13%) in 61.194 secs (21.963 MiB/s) 405 secs left ...
[2021-04-29T23:58:11] Copied 6.000 GiB (60%) in 139.322 secs (44.099 MiB/s) 93 secs left ...
[2021-04-29T23:58:11] Copied 10.000 GiB (100%) in 139.322 secs (73.499 MiB/s) done
[2021-04-29T23:58:11] Copy data: 10.000 GiB (10737418240 bytes)
[2021-04-29T23:58:11] Copy rate: 73.499 MiB/s (10737418240 bytes in 139.322 seconds)
[2021-04-29T23:58:11] Syncing data to disk.
[2021-04-29T23:58:11] Sync completed in 0.006 seconds.
[2021-04-29T23:58:11] Fixing permissions.
[2021-04-29T23:58:11] Updated 11 items in 0.002 seconds (4822.579 items/sec)
[2021-04-29T23:58:11] Syncing directory updates to disk.
[2021-04-29T23:58:11] Sync completed in 0.001 seconds.
[2021-04-29T23:58:11] Started: Apr-29-2021,23:55:52
[2021-04-29T23:58:11] Completed: Apr-29-2021,23:58:11
[2021-04-29T23:58:11] Seconds: 139.335
[2021-04-29T23:58:11] Items: 11
[2021-04-29T23:58:11]   Directories: 1
[2021-04-29T23:58:11]   Files: 10
[2021-04-29T23:58:11]   Links: 0
[2021-04-29T23:58:11] Data: 10.000 GiB (10737418240 bytes)
[2021-04-29T23:58:11] Rate: 73.492 MiB/s (10737418240 bytes in 139.335 seconds)


# Create directory
mkdir /tmp/datamover3

#RUN 
mpirun -hosts wolf-184<host> --ppn 16 -np 16 dcp --bufsize 64MB --chunksize 128MB daos://$DAOS_POOL/$DAOS_CONT3 /tmp/datamover3/

# Sample output
[2021-04-30T00:02:14] Walking /
[2021-04-30T00:02:15] Walked 12 items in 0.112 secs (107.354 items/sec) ...
[2021-04-30T00:02:15] Walked 12 items in 0.112 seconds (107.236 items/sec)
[2021-04-30T00:02:15] Copying to /tmp/datamover3
[2021-04-30T00:02:15] Items: 12
[2021-04-30T00:02:15]   Directories: 2
[2021-04-30T00:02:15]   Files: 10
[2021-04-30T00:02:15]   Links: 0
[2021-04-30T00:02:15] Data: 10.000 GiB (1.000 GiB per file)
[2021-04-30T00:02:15] Creating 2 directories
[2021-04-30T00:02:15] Original directory exists, skip the creation: `/tmp/datamover3/' (errno=17 File exists)
[2021-04-30T00:02:15] Creating 10 files.
[2021-04-30T00:02:15] Copying data.
[2021-04-30T00:03:15] Copied 1.938 GiB (19%) in 60.341 secs (32.880 MiB/s) 251 secs left ...
[2021-04-30T00:03:46] Copied 8.750 GiB (88%) in 91.953 secs (97.441 MiB/s) 13 secs left ...
[2021-04-30T00:03:46] Copied 10.000 GiB (100%) in 91.953 secs (111.361 MiB/s) done
[2021-04-30T00:03:46] Copy data: 10.000 GiB (10737418240 bytes)
[2021-04-30T00:03:46] Copy rate: 111.361 MiB/s (10737418240 bytes in 91.954 seconds)
[2021-04-30T00:03:46] Syncing data to disk.
[2021-04-30T00:03:47] Sync completed in 0.135 seconds.
[2021-04-30T00:03:47] Fixing permissions.
[2021-04-30T00:03:47] Updated 12 items in 0.000 seconds (71195.069 items/sec)
[2021-04-30T00:03:47] Syncing directory updates to disk.
[2021-04-30T00:03:47] Sync completed in 0.001 seconds.
[2021-04-30T00:03:47] Started: Apr-30-2021,00:02:15
[2021-04-30T00:03:47] Completed: Apr-30-2021,00:03:47
[2021-04-30T00:03:47] Seconds: 92.091
[2021-04-30T00:03:47] Items: 12
[2021-04-30T00:03:47]   Directories: 2
[2021-04-30T00:03:47]   Files: 10
[2021-04-30T00:03:47]   Links: 0
[2021-04-30T00:03:47] Data: 10.000 GiB (10737418240 bytes)
[2021-04-30T00:03:47] Rate: 111.194 MiB/s (10737418240 bytes in 92.091 seconds)


# Verify the two directories have the same content mjean@wolf-184:~/build> ls -la /tmp/datamover3/daos_test/
total 10485808
drwxr-xr-x 2 mjean mjean       4096 Apr 30 00:02 .
drwxr-xr-x 3 mjean mjean       4096 Apr 30 00:02 ..
-rw-r--r-- 1 mjean mjean 1073741824 Apr 30 00:03 testfile.00000000
-rw-r--r-- 1 mjean mjean 1073741824 Apr 30 00:03 testfile.00000001
-rw-r--r-- 1 mjean mjean 1073741824 Apr 30 00:03 testfile.00000002
-rw-r--r-- 1 mjean mjean 1073741824 Apr 30 00:03 testfile.00000003
-rw-r--r-- 1 mjean mjean 1073741824 Apr 30 00:03 testfile.00000004
-rw-r--r-- 1 mjean mjean 1073741824 Apr 30 00:03 testfile.00000005
-rw-r--r-- 1 mjean mjean 1073741824 Apr 30 00:03 testfile.00000006
-rw-r--r-- 1 mjean mjean 1073741824 Apr 30 00:03 testfile.00000007
-rw-r--r-- 1 mjean mjean 1073741824 Apr 30 00:03 testfile.00000008
-rw-r--r-- 1 mjean mjean 1073741824 Apr 30 00:03 testfile.00000009
mjean@wolf-184:~/build> ls -la /tmp/daos_dfuse/daos_test/
total 10485760
-rw-r--r-- 1 mjean mjean 1073741824 Apr 29 16:31 testfile.00000000
-rw-r--r-- 1 mjean mjean 1073741824 Apr 29 16:31 testfile.00000001
-rw-r--r-- 1 mjean mjean 1073741824 Apr 29 16:31 testfile.00000002
-rw-r--r-- 1 mjean mjean 1073741824 Apr 29 16:31 testfile.00000003
-rw-r--r-- 1 mjean mjean 1073741824 Apr 29 16:31 testfile.00000004
-rw-r--r-- 1 mjean mjean 1073741824 Apr 29 16:31 testfile.00000005
-rw-r--r-- 1 mjean mjean 1073741824 Apr 29 16:31 testfile.00000006
-rw-r--r-- 1 mjean mjean 1073741824 Apr 29 16:31 testfile.00000007
-rw-r--r-- 1 mjean mjean 1073741824 Apr 29 16:31 testfile.00000008
-rw-r--r-- 1 mjean mjean 1073741824 Apr 29 16:31 testfile.00000009