# # Read metrics from one or more commands that can output to stdout
[[inputs.exec]]
  ## Commands array
  commands = [
    "/opt/telegraf/scripts/collect_*"
  ]

  ## Timeout for each command to complete.
  timeout = "9s"
#
#   ## measurement name suffix (for separating different commands)
#   name_suffix = "_mycollector"
#
#   ## Data format to consume.
#   ## Each data format has its own unique set of configuration options, read
#   ## more about them here:
#   ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md
  data_format = "influx"

这里主要修改了commands、timeout、data_format三个参数，commands是脚本列表，支持通配符，这里我会把所有监控插件脚本统一以collect_作为文件名前缀，放在/opt/telegraf/scripts目录下，便于管理。

timeout默认是5s，这里改的长一点，改成9s，防止有些脚本执行时间太长。data_format支持很多种不同的格式，个人认为influx格式已经非常方便，其他的格式暂且不用了解。

之前的文章中讲到用promql来处理diskio相关的指标，这里给一种新方案，直接用脚本采集iostat输出，用exec这个插件来搞，脚本名称collect_iostat.sh，内容如下：

#!/bin/bash

while read -r device read_queued_sec write_queued_sec read_completed_sec write_completed_sec read_kbytes_sec write_kbytes_sec avg_sector_size avg_queue_len await_ms read_await_ms write_await_ms svctime_ms util_pct; do
    [ ! -z "$device" ] && echo "iostat,device=$device read_queued_sec=$read_queued_sec,write_queued_sec=$write_queued_sec,read_completed_sec=$read_completed_sec,write_completed_sec=$write_completed_sec,read_kbytes_sec=$read_kbytes_sec,write_kbytes_sec=$write_kbytes_sec,avg_sector_size=$avg_sector_size,avg_queue_len=$avg_queue_len,await_ms=$await_ms,read_await_ms=$read_await_ms,write_await_ms=$write_await_ms,svctime_ms=$svctime_ms,util_pct=$util_pct"
done < <(iostat -dxyk 5 1 |  tail -n +4)
exit 0

这个脚本非本人所写，没有经过生产验证，之前在查询diskio相关指标的处理的时候，看到了Telegraf的一个issue，有人提供了这个脚本，仅供参考，来学习exec插件。

net

net插件默认配置如下：

# # Read metrics about network interface usage
# [[inputs.net]]
#   ## By default, telegraf gathers stats from any up interface (excluding loopback)
#   ## Setting interfaces will tell it to gather these explicit interfaces,
#   ## regardless of status.
#   ##
#   # interfaces = ["eth0"]
#   ##
#   ## On linux systems telegraf also collects protocol stats.
#   ## Setting ignore_protocol_stats to true will skip reporting of protocol metrics.
#   ##
#   # ignore_protocol_stats = false
#   ##

表示Telegraf默认是不采集net相关指标的，打开这个section的配置：

# # Read metrics about network interface usage
[[inputs.net]]

执行一次测试，看看输出内容先：

[root@10-255-0-34 telegraf-1.20.2]# ./usr/bin/telegraf --config etc/telegraf/telegraf.conf --test --input-filter net
2021-11-06T15:24:24Z I! Starting Telegraf 1.20.2
> net,host=10-255-0-34,interface=eth0 bytes_recv=12632788068i,bytes_sent=42346381998i,drop_in=0i,drop_out=0i,err_in=0i,err_out=0i,packets_recv=89892475i,packets_sent=95639742i 1636212264000000000
> net,host=10-255-0-34,interface=all icmp_inaddrmaskreps=0i,icmp_inaddrmasks=0i,icmp_incsumerrors=0i,icmp_indestunreachs=1370i,icmp_inechoreps=21499523i,icmp_inechos=21939068i,icmp_inerrors=0i,icmp_inmsgs=43440008i,icmp_inparmprobs=0i,icmp_inredirects=0i,icmp_insrcquenchs=0i,icmp_intimeexcds=44i,icmp_intimestampreps=0i,icmp_intimestamps=3i,icmp_outaddrmaskreps=0i,icmp_outaddrmasks=0i,icmp_outdestunreachs=287i,icmp_outechoreps=21939068i,icmp_outechos=21500440i,icmp_outerrors=0i,icmp_outmsgs=43439798i,icmp_outparmprobs=0i,icmp_outredirects=0i,icmp_outsrcquenchs=0i,icmp_outtimeexcds=0i,icmp_outtimestampreps=3i,icmp_outtimestamps=0i,icmpmsg_intype0=21499523i,icmpmsg_intype11=44i,icmpmsg_intype13=3i,icmpmsg_intype3=1370i,icmpmsg_intype8=21939068i,icmpmsg_outtype0=21939068i,icmpmsg_outtype14=3i,icmpmsg_outtype3=287i,icmpmsg_outtype8=21500440i,ip_defaultttl=64i,ip_forwarding=2i,ip_forwdatagrams=0i,ip_fragcreates=0i,ip_fragfails=0i,ip_fragoks=0i,ip_inaddrerrors=0i,ip_indelivers=545644258i,ip_indiscards=0i,ip_inhdrerrors=0i,ip_inreceives=545644260i,ip_inunknownprotos=0i,ip_outdiscards=0i,ip_outnoroutes=40i,ip_outrequests=552615900i,ip_reasmfails=0i,ip_reasmoks=0i,ip_reasmreqds=0i,ip_reasmtimeout=0i,tcp_activeopens=10002318i,tcp_attemptfails=733002i,tcp_currestab=44i,tcp_estabresets=93230i,tcp_incsumerrors=0i,tcp_inerrs=31i,tcp_insegs=502036604i,tcp_maxconn=-1i,tcp_outrsts=843395i,tcp_outsegs=523956506i,tcp_passiveopens=10274186i,tcp_retranssegs=1122669i,tcp_rtoalgorithm=1i,tcp_rtomax=120000i,tcp_rtomin=200i,udp_incsumerrors=0i,udp_indatagrams=167642i,udp_inerrors=0i,udp_noports=3i,udp_outdatagrams=188422i,udp_rcvbuferrors=0i,udp_sndbuferrors=0i,udplite_incsumerrors=0i,udplite_indatagrams=0i,udplite_inerrors=0i,udplite_noports=0i,udplite_outdatagrams=0i,udplite_rcvbuferrors=0i,udplite_sndbuferrors=0i 1636212264000000000

eth0输出的这些field都挺有用的，interface=all这个输出的内容看起来用处不大（受限于本人知识水平…），可以把interface=all这部分不要采集，配置如下：

# # Read metrics about network interface usage
[[inputs.net]]
#   ## By default, telegraf gathers stats from any up interface (excluding loopback)
#   ## Setting interfaces will tell it to gather these explicit interfaces,
#   ## regardless of status.
#   ##
#   # interfaces = ["eth0"]
#   ##
#   ## On linux systems telegraf also collects protocol stats.
#   ## Setting ignore_protocol_stats to true will skip reporting of protocol metrics.
#   ##
  ignore_protocol_stats = true

即把ignore_protocol_stats设置为true即可。interfaces那个配置，注释里也说的比较明白了，如果不开启就是采集所有up的网卡（除了回环网卡），可以通过这个配置明确指定要采集的有限的网卡，支持通配符，比如：

[[inputs.net]]
  interfaces = ["eth*", "bond*"]
  ignore_protocol_stats = true

网卡相关的这些指标，都是counter类型的，即从OS启动就持续增长的，具体使用的时候，大都需要用rate计算之后再用。

netstat

netstat相关的配置，默认也是没有开启的，配置如下：

# # Read TCP metrics such as established, time wait and sockets counts.
# [[inputs.netstat]]
#   # no configuration

把注释打开，采集一下netstat相关的指标，我们看一下：

[root@10-255-0-34 telegraf-1.20.2]# ./usr/bin/telegraf --config etc/telegraf/telegraf.conf --test --input-filter netstat
2021-11-06T15:36:35Z I! Starting Telegraf 1.20.2
> netstat,host=10-255-0-34 tcp_close=0i,tcp_close_wait=1i,tcp_closing=0i,tcp_established=44i,tcp_fin_wait1=1i,tcp_fin_wait2=0i,tcp_last_ack=0i,tcp_listen=17i,tcp_none=8i,tcp_syn_recv=0i,tcp_syn_sent=0i,tcp_time_wait=40i,udp_socket=13i 1636212996000000000

还是比较全的，通常关注度比较高的是tcp_time_wait，如果连接回收的太慢，会占用较多文件句柄。

我看了一眼代码，在Linux环境下，统计网络连接状态Telegraf用的lsof命令，在网络连接很多的时候，可能会是个问题，大家要注意一下。只是统计一些简单summary信息的话，可以使用ss -s命令，连接很多也非常快。这里我们可以考虑注释掉netstat模块，自己写个简单脚本来采集网络连接信息，比如collect_ss.sh这个脚本：

#!/bin/bash
# TCP: 360 (estab 272, closed 71, orphaned 0, synrecv 0, timewait 71/0), ports 0

output=$(ss -s | grep TCP:)

estab=$(echo $output | grep -Po "estab (\d+)" | awk '{print $2}')
closed=$(echo $output | grep -Po "closed (\d+)" | awk '{print $2}')
orphaned=$(echo $output | grep -Po "orphaned (\d+)" | awk '{print $2}')
synrecv=$(echo $output | grep -Po "synrecv (\d+)" | awk '{print $2}')
timewait=$(echo $output | grep -Po "timewait (\d+)" | awk '{print $2}')

echo "ss estab=$estab,closed=$closed,orphaned=$orphaned,synrecv=$synrecv,timewait=$timewait"

这多条echo grep awk确实有点丑，好在能work，也能说明问题了。