超算中心HPC集群监控

一，前言

高性能计算（HPC）是一个计算机集群系统，它通过各种互联技术将多个计算机系统连接在一起，利用所有被连接系统的综合计算能力来处理大型计算问题，所以又通常被称为高性能计算集群。

当服务器具备IPMI接口时，BMC/IPMI web后台可以看到硬件的运行情况，比如CPU温度、内存温度、风扇转速、主板电压等，这些数据默认只能通过登录web后台或者ipmitool等接口工具来实时查看，看不到历史曲线图，也做不到监控告警，当服务器出现问题时，除了日志之外，这些物理传感器指标绘制的趋势图更有利于我们判断服务器在历史异常时间点的硬件状态趋势变化情况，或者实时监测指标到达设定的阈值时自动告警，因此本文油然而生。

二，准备

该项目采用docker部署

1，基础环境准备，部署

a，离线安装docker环境，下载适合该项目的docker版本，离线安装包下载地址：https://download.docker.com/linux/static/stable/

b，上传docker安装包，解压docker安装包
root@h100-NF5280-M7-A0-R0-00:/opt# tar -zxvf docker-29.1.0.tgz
c，复制docker到/usr/bin/
root@h100-NF5280-M7-A0-R0-00:/opt# cp docker/* /usr/bin/
d，编辑docker启动服务
root@h100-NF5280-M7-A0-R0-00:/opt# vim /etc/systemd/system/docker.service

[Unit] Description=Docker Application Container Engine Documentation=https://docs.docker.com After=network-online.target firewalld.service Wants=network-online.target [Service] Type=notify ExecStart=/usr/bin/dockerd ExecReload=/bin/kill -s HUP $MAINPID LimitNOFILE=1048576 LimitNPROC=1048576 LimitCORE=infinity TimeoutStartSec=0 Delegate=yes KillMode=process Restart=on-failure StartLimitBurst=3 StartLimitInterval=60s [Install] WantedBy=multi-user.target

e，启动docker服务并验证
root@h100-NF5280-M7-A0-R0-00:/opt# systemctl start docker

root@h100-NF5280-M7-A0-R0-00:/opt# systemctl status docker

f，上传prometheus，grafana,antilax3_ipmi,自定义fc_ipmi镜像包，加载镜像

root@h100-NF5280-M7-A0-R0-00:/opt# docker load -i prometheus.tar
root@h100-NF5280-M7-A0-R0-00:/opt# docker load -i grafana.tar

#构建自定义ipmi镜像,主要修改项目服务器bmc信息，dockerfile文件配置

root@h100-NF5280-M7-A0-R0-00:/opt/ipmi# cat ipmi_remote3011.yml modules: default: user: "xxxx" pass: "xxxxxxxxxx" privilege: "admin" driver: "LAN_2_0" xx.xx.xx.xx: user: "xxxx" pass: "xxxxxxxxx" privilege: "admin" driver: "LAN_2_0" collectors: - bmc - ipmi - dcmi - chassis - sel custom_args: bmc: - "bmc-info" ipmi: - "ipmimonitoring" dcmi: - "ipmi-dcmi" chassis: - "ipmi-chassis" sel: - "ipmi-sel" root@h100-NF5280-M7-A0-R0-00:/opt/ipmi# cat Dockerfile FROM antilax3/ipmi-exporter # 本地安装软件 #RUN apt-get update && apt-get install -y curl # 设置工作目录 WORKDIR /app # 拷贝文件 COPY myscript.sh . COPY ipmi_remote3011.yml . # 设置执行权限 RUN chmod +x myscript.sh RUN touch /app/test.log # 入口点 ENTRYPOINT ["/app/myscript.sh"]

#构建自定义ipmi采集镜像
root@h100-NF5280-M7-A0-R0-00:/opt/ipmi# docker build -t fc_ipmi .

root@h100-NF5280-M7-A0-R0-00:/opt# docker images

g，起动ipmi采集服务，prometheus监控服务，grafana可视化服务

#创建每个服务持久化，配置文件目录,这里主要是prometheus，grafana

root@h100-NF5280-M7-A0-R0-00:/opt# mkdir -p -m 777 prometheus/config

#prometheus下主要是配置文件和告警规则

root@h100-NF5280-M7-A0-R0-00:/opt/prometheus/config# cat prometheus.yml # my global config global: scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute. evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute. # scrape_timeout is set to the global default (10s). # Alertmanager configuration alerting: alertmanagers: - static_configs: - targets: # - alertmanager:9093 # Load rules once and periodically evaluate them according to the global 'evaluation_interval'. rule_files: - "/etc/prometheus/rules/*.yml"" # - "second_rules.yml" # A scrape configuration containing exactly one endpoint to scrape: # Here it's Prometheus itself. scrape_configs: # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config. - job_name: "prometheus" # metrics_path defaults to '/metrics' # scheme defaults to 'http'. static_configs: - targets: ["xx.xx.xx.xx:9090"] - job_name: ipmi_exporter static_configs: - targets: ["xx.xx.xx.xx"] - targets: ["xx.xx.xx.xx"] params: module: ['default'] scrape_interval: 1m scrape_timeout: 30s metrics_path: /ipmi scheme: http file_sd_configs: - files: - /etc/ipmi_targets.yml refresh_interval: 5m relabel_configs: - source_labels: [__address__] separator: ; regex: (.*) target_label: __param_target replacement: ${1} action: replace - source_labels: [__param_target] separator: ; regex: (.*) target_label: instance replacement: ${1} action: replace - separator: ; regex: .* target_label: __address__ #该地址为ipmi服务地址 replacement: xx.xx.xx.xx:3011 action: replace #监控ipmi的告警规则 root@h100-NF5280-M7-A0-R0-00:/opt/prometheus/config/rules# cat Memory_hardware.yml groups: - name: Memory_hardware rules: - alert: Memory_hardware expr: ipmi_sensor_state{type="Memory"} == 1 for: 3m labels: user: caizh annotations: summary: "Instance {{ $labels.instance }} 内存硬件警告" description: "{{ $labels.instance }} of job {{$labels.job}} 内存硬件警告,当前状态[{{ $value }}]."

root@h100-NF5280-M7-A0-R0-00:/opt# mkdir -p -m 777 prometheus/data

root@h100-NF5280-M7-A0-R0-00:/opt# mkdir -p -m 777 grafana/data

#ipmic采集 docker run -d --restart=always -v /etc/localtime:/etc/localtime:ro -p 3011:3011 --name fc_ipmi fc_ipmi #prometheus监控 docker run -d --name prometheus --restart=always -v /etc/localtime:/etc/localtime:ro -p 9090:9090 -v /opt/prometheus/data:/prometheus -v /opt/prometheus/config/prometheus.yml:/etc/prometheus/prometheus.yml -v /opt/prometheus/config/rules:/etc/prometheus/rules prom/prometheus #grafana可视化 docker run -d --restart=always -v /etc/localtime:/etc/localtime:ro -p 3000:3000 --name=grafana -v /opt/grafana/data:/var/lib/grafana grafana/grafana-enterprise:11.3.0-ubuntu

h，服务验证

2，效果展示