Linux下Milvus向量数据库故障的系统化排查Shell脚本

#!/bin/bash# =============================================================================# 脚本名称: milvus_diagnose_native.sh# 描述: 哲学优化 Milvus 向量数据库故障排查脚本【Linux原生安装 无Docker版】# 适配：Systemd托管 / 二进制手动启动 Milvus 单机分布式组件# 特性：# 1. 代码分段可单独复制终端逐句执行# 2. 自动缓存进程PID、服务名、日志路径、配置路径，高亮可复制# 3. 完整故障现场自动打包归档# 4. 无任何Docker、K8s相关逻辑# 用法: chmod +x milvus_diagnose_native.sh && sudo ./milvus_diagnose_native.sh# 依赖: ps/top/vmstat/free/df/journalctl/grep/awk/systemctl# 哲学体系:# 本体论 - 先识别所有Milvus进程实体；奥卡姆剃刀 - 最少命令定位根因；# 辩证法 - 区分内存/IO/磁盘/网络核心矛盾；实用主义 - 过滤致命错误日志；# 整体论 - 校验etcd/MinIO存储依赖连通；道家无为 - 给出监控自愈方案。# =============================================================================set-opipefail# ======================【单段独立执行】全局颜色、工具函数、缓存初始化 ======================# 颜色定义RED='\033[0;31m'GREEN='\033[0;32m'YELLOW='\033[1;33m'CYAN='\033[0;36m'WHITE='\033[1;37m'NC='\033[0m'# 分割线打印print_section(){echo-e"\n${CYAN}==========$1==========${NC}"}print_ok(){echo-e"${GREEN}[✓]${NC}$1";}print_warn(){echo-e"${YELLOW}[⚠]${NC}$1";}print_error(){echo-e"${RED}[✗]${NC}$1";}# 手工提示manual_hint(){echo-e"${YELLOW}🔧 手工检验步骤:${NC}$1";}# 打印可复制变量并写入缓存print_copy(){locallabel="$1"localval="$2"echo-e"${WHITE}【可复制参数$label】${RED}$val${NC}"echo"$label=$val">>"$CACHE_FILE"}# 输出预填充快捷命令（自动带入缓存参数）quick_cmd(){echo-e"${YELLOW}[一键复制执行命令]$1${NC}"echo"快捷命令：$1">>"$CACHE_FILE"}# 采集日志持久化collect_file(){localcmd="$1"localsave_path="$2"echo"采集数据：$cmd">>"$DIAG_SUMMARY"eval"$cmd">"$save_path"2>&1}# 初始化取证缓存目录（所有分段共享）TIMESTAMP=$(date+%Y%m%d_%H%M%S)DIAG_ROOT="/tmp/milvus_native_diag_${TIMESTAMP}"CACHE_FILE="${DIAG_ROOT}/cache_param.txt"DIAG_SUMMARY="${DIAG_ROOT}/diagnose_record.log"mkdir-p"$DIAG_ROOT">"$CACHE_FILE">"$DIAG_SUMMARY"echo"Milvus故障取证时间：$(date)">>"$DIAG_SUMMARY"# 全局缓存变量MILVUS_ALL_PIDS=""MILVUS_PROXY_PID=""MILVUS_CONFIG_PATH=""MILVUS_LOG_BASE=""ETCD_ENDPOINT=""MINIO_DATA_PATH=""# ==========================================================================================# ======================【单段独立执行】前置环境校验 + 全局Milvus实体参数预采集 ======================print_section"前置校验 & Milvus全组件实体参数采集（全部存入缓存可复制）"# 1. 校验核心系统命令CMD_LIST=("ps""df""free""vmstat""systemctl""journalctl")forcmdin"${CMD_LIST[@]}";doif!command-v"$cmd"&>/dev/null;thenprint_warn"系统缺失命令$cmd，部分检测功能失效"fidone# 2. 抓取所有Milvus进程PID（区分所有分布式组件）MILVUS_ALL_PIDS=$(pgrep-fmilvus|tr'\n'' ')if[-z"$MILVUS_ALL_PIDS"];thenprint_error"未检测到任何Milvus运行进程！"manual_hint"手动查询全部相关进程：ps aux | grep milvus"manual_hint"查看systemd服务状态：systemctl list-units | grep milvus"elseprint_copy"Milvus全部进程PID""$MILVUS_ALL_PIDS"# 单独提取proxy进程（高频故障点）MILVUS_PROXY_PID=$(pgrep-fmilvus-proxy|head-1)print_copy"milvus-proxy进程PID""$MILVUS_PROXY_PID"fi# 3. 自动识别Milvus配置文件路径（通用默认路径）if[-f"/etc/milvus/milvus.yaml"];thenMILVUS_CONFIG_PATH="/etc/milvus/milvus.yaml"elif[-f"/usr/local/milvus/conf/milvus.yaml"];thenMILVUS_CONFIG_PATH="/usr/local/milvus/conf/milvus.yaml"elif[-f"$HOME/milvus/conf/milvus.yaml"];thenMILVUS_CONFIG_PATH="$HOME/milvus/conf/milvus.yaml"fiprint_copy"Milvus主配置文件路径""$MILVUS_CONFIG_PATH"# 4. 从配置提取日志根目录、etcd地址、minio存储路径if[-n"$MILVUS_CONFIG_PATH"]&&[-f"$MILVUS_CONFIG_PATH"];thenMILVUS_LOG_BASE=$(grep'path:'"$MILVUS_CONFIG_PATH"|greplog|awk'{print $2}'|head-1)ETCD_ENDPOINT=$(grep'endpoints:'"$MILVUS_CONFIG_PATH"|grepetcd|awk'{print $2}'|head-1)MINIO_DATA_PATH=$(grep'rootPath:'"$MILVUS_CONFIG_PATH"|grepminio|awk'{print $2}'|head-1)fiprint_copy"Milvus日志根目录""$MILVUS_LOG_BASE"print_copy"ETCD连接地址""$ETCD_ENDPOINT"print_copy"MinIO本地数据存储目录""$MINIO_DATA_PATH"# 预填充高频快捷命令quick_cmd"systemctl status milvus"quick_cmd"cat${MILVUS_CONFIG_PATH}"quick_cmd"tail -f${MILVUS_LOG_BASE}/milvus-proxy.log"quick_cmd"ps aux --sort=-%mem | grep milvus"quick_cmd"tar -czf${DIAG_ROOT}.tar.gz -C /tmp milvus_native_diag_${TIMESTAMP}"# ==========================================================================================# ======================【单段独立执行】1.本体论：识别Milvus所有分布式实体状态 ======================print_section"1. 本体论检查：Milvus全组件进程实体存活状态"echo"Milvus完整进程列表（组件名/PID/CPU/内存/启动命令）："collect_file"ps aux | grep -E 'milvus-(rootcoord|querycoord|datacoord|proxy|indexnode|querynode|datanode)' | grep -v grep""${DIAG_ROOT}/milvus_process_list.log"cat"${DIAG_ROOT}/milvus_process_list.log"# 统计异常终止进程（systemd托管场景）ifcommand-vsystemctl&>/dev/null;thenecho-e"\nSystemd Milvus服务运行状态："collect_file"systemctl list-units --all | grep milvus""${DIAG_ROOT}/milvus_systemd_service.log"cat"${DIAG_ROOT}/milvus_systemd_service.log"collect_file"systemctl status milvus --no-pager -l""${DIAG_ROOT}/milvus_service_status.log"fi# 判定实体完整性STOP_PROC_COUNT=$(journalctl-umilvus --no-pager|grep-iE'terminated|killed|exit'|wc-l)if["$STOP_PROC_COUNT"-gt0]||[-z"$MILVUS_ALL_PIDS"];thenprint_warn"检测到Milvus组件进程异常终止，实体缺失，优先排查崩溃根因"elseprint_ok"所有Milvus分布式组件进程正常运行，实体完整，排查业务卡顿问题"fiecho-e"\n${CYAN}💡 哲学提示(本体论)：${NC}Milvus由7类微服务进程组成，任一组件崩溃都会导致向量读写失败"manual_hint"单独查看单个组件日志：tail -200${MILVUS_LOG_BASE}/milvus-rootcoord.log"# ==========================================================================================# ======================【单段独立执行】2.奥卡姆剃刀三问（存在性/因果/环境内核） ======================print_section"2.1 奥卡姆之问一：进程存活校验（存在性）"if[-n"$MILVUS_ALL_PIDS"];thenprint_ok"Milvus进程PID：$MILVUS_ALL_PIDS处于运行状态，故障为业务不可用/超时"# 检查端口监听（19530默认proxy端口）echo"Milvus Proxy端口19530监听状态："ss-tlnp|grep19530||print_warn"19530端口无监听，客户端无法连接"elseprint_error"无Milvus进程运行，故障为服务完全崩溃"fimanual_hint"检查全部监听端口：ss -tlnp | grep milvus"print_section"2.2 奥卡姆之问二：崩溃致命日志提取（因果性，只抓崩溃关键词）"# 致命错误过滤规则LOG_FILTER="panic|fatal|OOM|out of memory|no space left|too many open files|segment fault|killed|disk full|etcd connect failed|minio io error|index build failed"echo"遍历所有Milvus日志，提取致命崩溃日志："if[-n"$MILVUS_LOG_BASE"]&&[-d"$MILVUS_LOG_BASE"];thencollect_file"grep -rniE '$LOG_FILTER'${MILVUS_LOG_BASE}--include='*.log' | tail -50""${DIAG_ROOT}/milvus_fatal_error.log"cat"${DIAG_ROOT}/milvus_fatal_error.log"elseprint_warn"未识别日志目录，读取systemd服务日志替代"collect_file"journalctl -u milvus --no-pager | grep -iE '$LOG_FILTER' | tail -50""${DIAG_ROOT}/milvus_journal_error.log"cat"${DIAG_ROOT}/milvus_journal_error.log"fimanual_hint"查看完整proxy日志：less${MILVUS_LOG_BASE}/milvus-proxy.log"print_section"2.3 奥卡姆之问三：内核裁决日志（OOM/IO崩溃）"echo"内核环形缓冲区，Milvus进程被杀死、磁盘IO异常记录："collect_file"dmesg -T | tail -40 | grep -iE 'out of memory|killed process|I/O error|ext4 read-only'""${DIAG_ROOT}/host_kernel_oom.log"cat"${DIAG_ROOT}/host_kernel_oom.log"manual_hint"完整内核日志检索OOM：dmesg -T | grep -i 'killed process'"# ==========================================================================================# ======================【单段独立执行】3.辩证法：识别系统资源核心矛盾 ======================print_section"3. 辩证法检查：CPU/内存/IO 瓶颈快速区分"echo"vmstat 2次采样，wa高=磁盘瓶颈，so高=内存OOM风险，r高=CPU耗尽"collect_file"vmstat 1 2""${DIAG_ROOT}/vmstat_snapshot.log"cat"${DIAG_ROOT}/vmstat_snapshot.log"# 资源矛盾解读提示print_warn"wa(IO等待)>20：向量存储/MinIO磁盘IO瓶颈"print_warn"so(交换输出)>0：内存不足，索引加载触发OOM杀死进程"print_warn"r运行队列>CPU核数：批量插入/向量检索耗尽CPU"manual_hint"持续监控资源：vmstat 1"# ==========================================================================================# ======================【单段独立执行】4.细分资源专项深度排查 ======================print_section"4.1 内存资源深度检测（Milvus向量索引内存占用重灾区）"collect_file"free -h; swapon --show""${DIAG_ROOT}/host_memory.log"cat"${DIAG_ROOT}/host_memory.log"# 排序Milvus进程内存占用echo"Milvus进程内存占用倒序："collect_file"ps aux --sort=-%mem | grep milvus | grep -v grep""${DIAG_ROOT}/milvus_mem_rank.log"cat"${DIAG_ROOT}/milvus_mem_rank.log"# Swap风险判定SWAP_USED=$(free|grepSwap|awk'{print $3}')SWAP_TOTAL=$(free|grepSwap|awk'{print $2}')if["$SWAP_TOTAL"-gt0]&&["$SWAP_USED"-gt$((SWAP_TOTAL/2))];thenprint_error"Swap占用超50%，内存严重不足，易触发OOM杀死Milvus"fiquick_cmd"cat /proc/${MILVUS_PROXY_PID}/status | grep VmRSS"print_section"4.2 磁盘空间 & Inode & 只读文件系统检测"collect_file"df -h""${DIAG_ROOT}/disk_usage.log"awk'{if(NR==1 || $5+0>80) print}'"${DIAG_ROOT}/disk_usage.log"echo-e"\nInode占用（向量分段文件多易耗尽inode）："collect_file"df -i""${DIAG_ROOT}/inode_usage.log"awk'{if(NR==1 || $5+0>80) print}'"${DIAG_ROOT}/inode_usage.log"# 检测只读磁盘mount|grep"ro,"&&print_error"存在只读文件系统，Milvus写入Segment/索引失败"manual_hint"统计MinIO数据目录大小：du -sh${MINIO_DATA_PATH}"print_section"4.3 文件句柄专项（Milvus海量Segment文件极易超限）"if[-n"$MILVUS_PROXY_PID"];thenecho"Proxy进程PID:${MILVUS_PROXY_PID}"collect_file"ls /proc/${MILVUS_PROXY_PID}/fd 2>/dev/null | wc -l""${DIAG_ROOT}/proxy_fd_count.log"FD_COUNT=$(cat"${DIAG_ROOT}/proxy_fd_count.log")echo"当前打开文件句柄数：$FD_COUNT"collect_file"cat /proc/${MILVUS_PROXY_PID}/limits | grep 'Max open files'""${DIAG_ROOT}/fd_limit.log"cat"${DIAG_ROOT}/fd_limit.log"# 句柄风险判定FD_LIMIT=$(cat"${DIAG_ROOT}/fd_limit.log"|awk'{print $4}')if["$FD_COUNT"-gt$((FD_LIMIT*8/10))];thenprint_error"文件句柄占用超过限制80%，即将触发too many open files崩溃"fifimanual_hint"全局调整ulimit：vim /etc/security/limits.conf"# ==========================================================================================# ======================【单段独立执行】5.整体论：依赖服务连通性全链路校验 ======================print_section"5. 整体论：ETCD元数据服务 + MinIO存储连通校验"# 5.1 ETCD连通健康检测if[-n"$ETCD_ENDPOINT"];thenecho"检测ETCD地址：$ETCD_ENDPOINT"collect_file"etcdctl --endpoints=$ETCD_ENDPOINTendpoint health""${DIAG_ROOT}/etcd_health.log"cat"${DIAG_ROOT}/etcd_health.log"ifgrep-q"unhealthy""${DIAG_ROOT}/etcd_health.log";thenprint_error"ETCD元数据服务异常，Milvus无法加载集合、索引元信息"fielseprint_warn"未自动识别ETCD地址，请手动核对milvus.yaml配置"fimanual_hint"手动测试etcd连通：etcdctl put test_key test_val"# 5.2 MinIO本地存储目录读写权限校验if[-n"$MINIO_DATA_PATH"]&&[-d"$MINIO_DATA_PATH"];thenecho"MinIO向量存储目录权限检查：$MINIO_DATA_PATH"ls-ld"$MINIO_DATA_PATH"# 测试写入权限touch"${MINIO_DATA_PATH}/.milvus_write_test"2>/dev/null&&rm-f"${MINIO_DATA_PATH}/.milvus_write_test"&&print_ok"存储目录读写正常"||print_error"MinIO存储目录无写入权限"fi# 5.3 全链路业务提示echo-e"\n${CYAN}💡 哲学提示(整体论)：${NC}"echo"1. ETCD宕机：所有集合无法加载，服务启动直接panic"echo"2. MinIO磁盘满/权限不足：插入向量、构建索引全部失败"echo"3. 网络延迟过高：跨节点分布式Coord组件超时阻塞"manual_hint"客户端连接全链路排查：使用pymilvus建立测试连接，捕获完整报错堆栈"# ==========================================================================================# ======================【单段独立执行】6.道家无为：长期监控自愈优化方案 ======================print_section"6. 道家无为：前置监控告警，被动故障转为主动预警"cat<<EOF # Milvus生产环境必备监控告警项（避免事后紧急排查） 1. 进程存活告警：7类milvus组件任一进程消失触发告警 2. 致命日志告警：监控panic/OOM/no space/etcd failed关键词 3. 资源阈值告警： - 磁盘使用率>80% / Inode>80% - 内存Swap占用>30% - Proxy文件句柄超过限制80% 4. 依赖健康告警：ETCD endpoint health失败、MinIO目录不可写 5. 业务指标告警：向量插入失败率>0、索引构建任务堆积 # 系统配置优化（规避高频故障） 1. 调大系统ulimit nofile 65535（Milvus海量Segment文件） 2. 分配独立磁盘给MinIO向量存储，隔离系统日志IO竞争 3. 设置milvus进程OOM优先级，防止内核优先杀死向量服务 4. 日志轮转配置，避免日志持续打满磁盘 EOF# ==========================================================================================# ======================【单段独立执行】收尾：故障现场打包归档、缓存查看指引 ======================print_section"诊断收尾：全量取证打包 + 可复制参数查看"# 打包所有采集的故障数据tar-czf"${DIAG_ROOT}.tar.gz"-C/tmp"milvus_native_diag_${TIMESTAMP}"2>/dev/null print_ok"✅ Milvus完整故障取证包生成完成"print_copy"取证压缩包完整路径""${DIAG_ROOT}.tar.gz"echo-e"\n${WHITE}【查看全部缓存PID/路径参数】执行命令：${RED}cat${CACHE_FILE}${NC}"echo-e"\n${GREEN}Milvus Linux原生故障系统化排查全部执行完毕${NC}"echo-e"${YELLOW}根因定位顺序建议：致命日志 > OOM内核记录 > 磁盘空间 > ETCD连通 > 内存/句柄限制${NC}"# ==========================================================================================