Skip to content

Ansys

/etc/hosts 

172.24.0.1   ansys-lic-srv
172.24.0.2   ansys-compute-01
172.24.0.3   ansys-compute-02


# /etc/hosts are prepared done
mkdir -p /mnt/ansys1
mkdir -p /mnt/ansys2
mkdir -p /mnt/ansys3
sudo mount -o loop ANSYS2023R1_LINX64_DISK1.iso /mnt/ansys1
sudo mount -o loop ANSYS2023R1_LINX64_DISK2.iso /mnt/ansys2
sudo mount -o loop ANSYS2023R1_LINX64_DISK3.iso /mnt/ansys3
cd /mnt/ansys1
./INSTALL -silent -install_dir /home/foo/ansys_inc -media_dir2 /mnt/ansys2 -media_dir3 /mnt/ansys3

license server configuration /home/foo/ansys_inc/shared_files/licensing/ansyslmd.ini

SERVER=1055@ansys-lic-srv
ANSYSLI_SERVERS=2325@ansys-lic-srv

Verify

/home/foo/ansys_inc/v231/licensingclient/linx64

./lmutil lmstat -a -c 1055@ansys-lic-srv 2>&1|egrep -i "cfx|fluent|ansys|dyna"

mpi test

foo@ansys-compute-01:~> /home/foo/ansys_inc/v231/ansys/bin/mpitest231 -mpi intelmpi2019 -machines ansys-compute-01:16:ansys-compute-02:16

latency =     42.8230 microseconds
bytes         bandwidth MB/s
-----         --------------
        8        0.1868
     1024       31.9512
     4096      118.1913
    16384      235.5924
    65536      359.5767
   262144      681.7651
  1048576      744.0079
  4194304      945.1639
  MPI Test has successfully completed!

fluent benchmark

prerequisite

RAM: 1TB

change the max locked memory

ulimit -a

edit the /etc/security/limit.conf 

* soft memlock unlimited
* hard memlock unlimited

operations

6,7 and 8 take effects

  1. I_MPI_FABRICS=shm:ofa => I_MPI_FABRICS=shm:ofi

  2. https://learn.microsoft.com/zh-tw/azure/virtual-machines/setup-mpi

  3. install the mft on ansys-4b

  4. https://www.ibm.com/support/pages/firmware-update-mellonox-adapter-fails-sles-15-sp3-using-mlnx-ofed-49-3-and-later-releases

  5. https://network.nvidia.com/products/adapter-software/firmware-tools/

  6. enlarge the ulimit and reboot for ansys-4b, ansys-9b

  7. stack
  8. file descriptor

/etc/security/limits.conf 

* soft stack unlimited
* hard stack unlimited
* soft nofile unlimited
* hard nofile unlimited

  1. enable mpi debug, I_MPI_DEBUG=5
I_MPI_DEBUG=5 USE_UCX=1 UCX_NET_DEVICES=mlx5_0:1 I_MPI_FABRICS=shm:ofi FI_PROVIDER=mlx /home/foo/ansys_inc/v231/ansys/bin/mpitest231 -mpi intelmpi2021 -machines ansys-4b:32:ansys-9b:32
  1. run aircraft_wing_14m
/home/foo/ansys_inc/v231/fluent/bin/fluentbench.pl aircraft_wing_14m -pib -t128 -mpi=intelmpi2021 -cnf=hostfile -mpiopt="-genv I_MPI_DEBUG=8 -genv USE_UCX=1 -genv UCX_NET_DEVICES=mlx5_0:1 -genv I_MPI_FABRICS=shm:ofi -genv FI_PROVIDER=mlx" -noloadchk  -nosyslog
  1. run exhaust_system_33m by tcp
/home/foo/ansys_inc/v231/fluent/bin/fluentbench.pl exhaust_system_33m -t128 -mpi=intelmpi2018 -cnf=hostfile  -mpiopt="-genv I_MPI_DEBUG=8 -genv USE_UCX=1 -genv FI_PROVIDER=tcp"
  1. run aircraft_wing_2m by tcp
/home/foo/ansys_inc/v231/fluent/bin/fluentbench.pl aircraft_wing_2m -t128 -mpi=intelmpi2018 -cnf=hostfile  -mpiopt="-genv I_MPI_DEBUG=8 -genv USE_UCX=1 -genv FI_PROVIDER=tcp" 
/home/foo/ansys_inc/v231/fluent/bin/fluentbench.pl aircraft_wing_2m -t128 -cnf=hostfile  -mpiopt="-genv I_MPI_DEBUG=8 -genv USE_UCX=1 -genv FI_PROVIDER=tcp"
  1. install the infiniband driver, MLNX_OFED_LINUX-23.10-1.1.9.0-sles15sp4-x86_64, on the ansys-4b

InfiniBand - Linux InfiniBand Drivers

sudo ethtool -i ib0
sudo zypper install createrepo_c insserv-compat tk
sudo ./mlnxofedinstall --force --add-kernel-support
sudo /etc/init.d/openibd restart
sudo ethtool -i ib0

foo@ansys-4b:~/ansys_inc/v231/fluent/bench/fluent/v6/aircraft_wing_2m> sudo ethtool -i ib0
[sudo] password for root:
driver: mlx5_core[ib_ipoib]
version: 23.10-1.1.9
firmware-version: 20.28.1002 (SM_1271000001000)
expansion-rom-version:
bus-info: 0000:81:00.0
supports-statistics: yes
supports-test: yes
supports-eeprom-access: no
supports-register-dump: no
supports-priv-flags: yes
foo@ansys-4b:~/ansys_inc/v231/fluent/bench/fluent/v6/aircraft_wing_2m>

/home/foo/ansys_inc/v231/fluent/bin/fluentbench.pl aircraft_wing_2m -t128 -mpi=intelmpi2018 -cnf=hostfile  -mpiopt="-genv I_MPI_DEBUG=8 -genv USE_UCX=1 -genv FI_PROVIDER=mlx -genv I_MPI_FABRICS=shm:ofi -genv UCX_NET_DEVICES=mlx5_0:1" 
/home/foo/ansys_inc/v231/fluent/bin/fluentbench.pl aircraft_wing_2m -t128 -cnf=hostfile  -mpiopt="-genv I_MPI_DEBUG=8 -genv USE_UCX=1 -genv FI_PROVIDER=mlx -genv I_MPI_FABRICS=shm:ofi -genv UCX_NET_DEVICES=mlx5_0:1"

uniform

with turbostat

turbostat --interval 1 -S --quiet --out freq_analysis_240.out & echo $! > /home/smci/tbs_log.pid ; /home/smci/ansys_inc/v231/fluent/bin/fluent 3d -g -nmon -ssh -peth -mpi=intel -platform=intel -t240 -cnf=/home/smci/hostfile -i run.jou |tee log.txt && pkill turbostat && cat /home/smci/tbs_log.pid; ps -ef |grep turbostat ; kill -9 `cat /home/smci/tbs_log.pid`