diff --git a/roles/munin_client/files/nvme b/roles/munin_client/files/nvme new file mode 100644 index 0000000..b70e42f --- /dev/null +++ b/roles/munin_client/files/nvme @@ -0,0 +1,340 @@ +#! /usr/bin/perl +# -*- mode: perl; perl-indent-level: 4 -*- + +=head1 NAME + +nvme - Munin plugin to monitor the use of NVMe devices + +=head1 APPLICABLE SYSTEMS + +Linux systems with NVMe (Non-Volatile Memory storage attached via PCIe +bus). + +=head1 CONFIGURATION + +The plugin uses nvme(1) from the nvme-cli project to read status from +the NVMe devices. This requires root access. + + [nvme] + user root + +When setting alert levels per device, use graph and basename of device +name, e.g., 'nvme0n1', to make environment variable: + + env.nvme_usage_nvme0n1_warning 5: + env.nvme_usage_warning 8: + +If your device names change on reboot you can also use the labels +(based on serial numbers) to set the warning and critical labels + + env.nvme_usage_SN_1234567_warning 8:101 + env.nvme_usage_SN_1234567_critical 5:101 + +=head1 INTERPRETATION + +This is a multigraph plugin which makes three graphs. + +=head2 nvme_usage + +This reports how much of capacity is allocated in each NVMe +"namespace". The report is in percent. This number may not have much +relation to actual use, e.g., if deleted data areas have not been +trimmed/discarded. + +Default warning and critical: '95', '98' + +=head2 nvme_bytes + +This reports read and write activity on each NVMe device, in bytes per +second. Ideally there should be much more read than write. If they +are symmetrical, you are using your NVMe as a very expensive FIFO, and +if you write more than you read, you should probably look for archival +storage instead. + +It is a good idea to compare these numbers to I/O counters from +diskstats. If they are much higher, look into whether the write +amplification can be due to suboptimal I/O request sizes. + +This graph does not support alerting. + +=head2 nvme_writecycles + +This graphs is intended to give an indication of how much life there +is left in your NVMe. It calculates the number of bytes written +during each device's lifetime against the capacity of the device, +thereby getting an average number of write cycle each cell has +experienced. + +A prosumer NVMe will handle a few thousand writes to each cell before +the error rate gets out of hand. + +No default values for warning and critical. + +=head2 nvme_spare + +All NVMe has set a side reserve space to remap media errors. This +graphs how much is left in percent, taken directly from smart-log +output. + +Default warning and critical: '10:', '3:' + +=head1 MAGIC MARKERS + + #%# family=auto + #%# capabilities=autoconf + +=head1 BUGS + +None known. + +=head1 VERSION + + 1.1 + +=head1 AUTHOR + +Kjetil Torgrim Homme + +=head1 LICENSE + +GPLv2 + +=cut + +use strict; +use Munin::Plugin; +use IPC::Cmd qw(can_run); +use File::Basename; + +# Check that multigraph is supported +need_multigraph(); + +# Return undef if no problem, otherwise explanation +sub autoconf_problem { + return if can_run('nvme'); + if (open(my $mods, '/proc/modules')) { + while (<$mods>) { + return "missing nvme(1)" if /^nvme[^a-z]/; + } + close($mods); + } + return "missing nvme"; # vague message for non-Linux +} + +sub run_nvme { + my (@cmd) = @_; + my @lines; + if (can_run('nvme') && open(my $nvme, '-|', 'nvme', @cmd)) { + @lines = <$nvme>; + close($nvme); + warn "nvme: probably needs to run as user root\n" if $? && $> != 0; + } + @lines; +} + +sub human_to_bytes { + my ($str) = @_; + my %units = ( + kB => 1000, + MB => 1000_000, + GB => 1000_000_000, + TB => 1000_000_000_000, + PB => 1000_000_000_000_000, # I wish I had need for this + ); + $str =~ /(\d+(\.\d+)?)\s+(.B)/; + int($1 * $units{$3}); +} + +sub nvme_list { + # Node SN Model Namespace Usage Format FW Rev + # ---------------- -------------------- ---------------------------------------- --------- -------------------------- ---------------- -------- + # /dev/nvme1n1 S464NB0K601188N Samsung SSD 970 EVO 2TB 1 695.50 GB / 2.00 TB 512 B + 0 B 1B2QEXE7 + my %devices; + + my $recognised_output; + my $lineno = 0; + for (run_nvme('list')) { + ++$lineno; + if (m:^Node\s+SN\s+Model\s+Namespace Usage:) { + ++$recognised_output; + } elsif (m:^(/\S+)\s+(\S+)\s+(\S.*\S)\s{3,}(\d+)\s+(\S+\s+.B)\s+/\s+(\S+\s+.B):) { + $devices{'SN_'.$2} = { + device => $1, + sn => $2, + model => $3, + namespace => $4, + usage => human_to_bytes($5), + capacity => human_to_bytes($6), + }; + } elsif ($lineno > 2) { + # could not parse device information + $recognised_output = 0; + } + } + if ($lineno && !$recognised_output) { + warn "Could not recognise output from 'nvme list', please report\n"; + } + \%devices; +} + +sub smart_log { + my ($dev) = @_; + my %info; + for (run_nvme('smart-log', $dev)) { + next if /^Smart Log/; + if (/(.*?)\s+:\s+(.*)/) { + my ($var, $value) = ($1, $2); + $var =~ s/\s/_/g; + if ($value =~ /^\d+(,\d\d\d)+$/) { + $value =~ s/,//g; + } + $info{lc $var} = $value; + } + } + return \%info; +} + +sub my_print_thresholds { + my ($label, $graph, $device, $warn_default, $crit_default) = @_; + my $dev = basename($device); + my ($warn_label, $crit_label) = get_thresholds($graph, "${graph}_${label}_warning", "${graph}_${label}_critical", + $warn_default, $crit_default); + my ($warn, $crit) = get_thresholds($graph, "${graph}_${dev}_warning", "${graph}_${dev}_critical", + $warn_label, $crit_label); + print "${label}.warning $warn\n" if defined $warn; + print "${label}.critical $crit\n" if defined $crit; +} + +use Data::Dumper; + +my $mode = ($ARGV[0] or "print"); + +my $problem = autoconf_problem(); +my $list = nvme_list(); + +if ($mode eq 'autoconf') { + if (keys %{$list}) { + print "yes\n"; + } else { + printf("no (%s)\n", $problem || "no devices to monitor"); + } + exit 0; +} + +my @sn = sort keys %{$list}; + +if ($mode eq 'config') { + my $sn_list = join(' ', @sn); + + print <<'EOF'; +multigraph nvme_usage +graph_title NVME Namespace Usage +graph_order $sn_list +graph_vlabel Percent used +graph_scale no +graph_category disk +graph_info How much space is used +EOF + for (@sn) { + my $device = $list->{$_}->{device}; + print <<"EOF"; +$_.label $device used +$_.type GAUGE +$_.max 100 +$_.min 0 +EOF + my_print_thresholds($_, 'nvme_usage', $device, '95', '98'); + } + print <<'EOF'; +multigraph nvme_bytes +graph_title NVME Bytes Read / Written +graph_order $sn_list +graph_vlabel bytes read (-) / written (+) per ${graph_period}' +graph_category disk +graph_info How much data is read and written +graph_period second +EOF + for (@sn) { + print <<"EOF"; +${_}_r.label $list->{$_}->{device} +${_}_r.type DERIVE +${_}_r.min 0 +${_}_r.graph no +${_}_w.label $list->{$_}->{device} +${_}_w.type DERIVE +${_}_w.min 0 +${_}_w.negative ${_}_r +EOF + } + print <<'EOF'; +multigraph nvme_writecycles +graph_title NVME Write Cycles +graph_order $sn_list +graph_vlabel Cycles +graph_args --logarithmic +graph_category disk +graph_info How much data has been written in lifetime divided by capacity +EOF + for (@sn) { + my $device = $list->{$_}->{device}; + print <<"EOF"; +$_.label $device write cycles +$_.type GAUGE +$_.min 0 +EOF + my_print_thresholds($_, 'nvme_writecycles', $device); + } + print <<'EOF'; +multigraph nvme_spare +graph_title Available spare blocks +graph_order $sn_list +graph_vlabel Percent +graph_category disk +graph_info Spare capacity for replacing bad blocks +EOF + for (@sn) { + my $device = $list->{$_}->{device}; + print <<"EOF"; +$_.label $device spare capacity +$_.type GAUGE +$_.min 0 +$_.max 100 +EOF + my_print_thresholds($_, 'nvme_spare', $device, '10:', '3:'); + } +} else { + for (@sn) { + $list->{$_}->{smart} = smart_log($list->{$_}->{device}); + } + print "multigraph nvme_usage\n"; + for (@sn) { + my $info = $list->{$_}; + my $used = 100 * $info->{usage} / $info->{capacity}; + print "$_.value $used\n"; + } + print "multigraph nvme_bytes\n"; + for (@sn) { + my $info = $list->{$_}; + my $rbytes = $info->{smart}->{data_units_read}; + my $wbytes = $info->{smart}->{data_units_written}; + print "${_}_r.value $rbytes\n"; + print "${_}_w.value $wbytes\n"; + } + print "multigraph nvme_writecycles\n"; + for (@sn) { + my $info = $list->{$_}; + + # The unit size reported is 1000 blocks. + my $cycles = $info->{smart}->{data_units_written} * 512_000 / $info->{capacity}; + print "$_.value $cycles\n"; + } + print "multigraph nvme_spare\n"; + for (@sn) { + my $info = $list->{$_}; + + # The unit size reported is 1000 blocks. + my $spare = $info->{smart}->{available_spare}; + $spare =~ s/%//; + print "$_.value $spare\n"; + } +} diff --git a/roles/munin_client/tasks/garage.yml b/roles/munin_client/tasks/garage.yml index 593638a..7f999c7 100644 --- a/roles/munin_client/tasks/garage.yml +++ b/roles/munin_client/tasks/garage.yml @@ -15,6 +15,8 @@ owner: root group: root mode: '0640' + notify: + - Restart munin-node - name: Put garage scripts ansible.builtin.copy: diff --git a/roles/munin_client/tasks/hypervisors.yml b/roles/munin_client/tasks/hypervisors.yml index 00136a7..24a7a3f 100644 --- a/roles/munin_client/tasks/hypervisors.yml +++ b/roles/munin_client/tasks/hypervisors.yml @@ -1,5 +1,43 @@ --- +- name: Install prereq + ansible.builtin.package: + name: "{{ item }}" + state: present + loop: + - nvme-cli + - lm-sensors + +- name: Put nvme plugin configuration + ansible.builtin.template: + src: nvme.j2 + dest: /etc/munin/plugin-conf.d/nvme + owner: root + group: root + mode: '0640' + notify: + - Restart munin-node + +- name: Put nvme script + ansible.builtin.copy: + src: files/nvme + dest: /etc/munin/plugins/nvme + owner: root + group: root + mode: '0755' + notify: + - Restart munin-node + +- name: Configure specific munin plugin + ansible.builtin.file: + path: "/etc/munin/plugins/sensors_{{ item }}" + src: /usr/share/munin/plugins/sensors_ + state: link + notify: + - Restart munin-node + loop: + - temp + - name: Delete squid plugins ansible.builtin.shell: cmd: "rm -f /etc/munin/plugins/squid_*" diff --git a/roles/munin_client/tasks/main.yml b/roles/munin_client/tasks/main.yml index 3ee8c48..a4eee8b 100644 --- a/roles/munin_client/tasks/main.yml +++ b/roles/munin_client/tasks/main.yml @@ -87,11 +87,6 @@ - charge - voltages -# for physical servers -- name: Execute specific tasks for physical servers - ansible.builtin.include_tasks: physical_servers.yml - when: "'hypervisors' in group_names" - - name: Reconfigure munin-node ansible.builtin.shell: cmd: munin-node-configure --shell | sh # noqa: risky-shell-pipe diff --git a/roles/munin_client/tasks/physical_servers.yml b/roles/munin_client/tasks/physical_servers.yml deleted file mode 100644 index b4190fe..0000000 --- a/roles/munin_client/tasks/physical_servers.yml +++ /dev/null @@ -1,16 +0,0 @@ ---- -# for physical servers -- name: Install necessary packages for hypervisors - ansible.builtin.package: - name: lm-sensors - state: present - -- name: Configure specific munin plugin - ansible.builtin.file: - path: "/etc/munin/plugins/sensors_{{ item }}" - src: /usr/share/munin/plugins/sensors_ - state: link - notify: - - Restart munin-node - loop: - - temp diff --git a/roles/munin_client/templates/nvme.j2 b/roles/munin_client/templates/nvme.j2 new file mode 100644 index 0000000..31ac903 --- /dev/null +++ b/roles/munin_client/templates/nvme.j2 @@ -0,0 +1,2 @@ +[nvme] +user root