404Forbiden

✏️ Editing: cpu_usage.py

import agent_util
import time
import sys
import platform
import os
import socket
from agent_util import float

try:
    import psutil
except:
    psutil = None

try:
    import distro
except:
    distro = None

def search_esxtop(headers, search_string):
    for idx, column in enumerate(headers):
        if search_string in column:
            return idx
    return None

def get_cpu_metrics(cls):
    retcode, output = agent_util.execute_command("cat /proc/stat")
    cls.log.debug("cat /proc/stat output: %s" % str(output))
    output = output.splitlines()
    stat_fields = [
        "user",
        "nice",
        "system",
        "idle",
        "iowait",
        "irq",
        "softirq",
        "steal",
        "guest",
        "guest_nice",
    ]
    cpus = {}
    for line in output:
        if not line.startswith("cpu"):
            continue
        # python3 compatible lambda function
        if sys.version_info[0] == 3:
            parts = list(filter(lambda p: p, line.split(" ")))
        else:
            parts = filter(lambda p: p, line.split(" "))

core = parts[0]
        if core == "cpu":
            core = "Total"
        if len(parts) >= 11:
            user, nice, system, idle, iowait, irq, softirq, steal, guest, guest_nice = (
                map(int, parts[1:11])
            )
            cpus[core] = {
                "user": user,
                "nice": nice,
                "system": system,
                "idle": idle,
                "iowait": iowait,
                "irq": irq,
                "softirq": softirq,
                "steal": steal,
                "guest": guest,
                "guest_nice": guest_nice,
            }
        elif len(parts) > 8 and len(parts) < 11:
            user, nice, system, idle, iowait, irq, softirq = map(int, parts[1:8])
            cpus[core] = {
                "user": user,
                "nice": nice,
                "system": system,
                "idle": idle,
                "iowait": iowait,
                "irq": irq,
                "softirq": softirq,
            }
    return cpus

class CPUUsagePlugin(agent_util.Plugin):
    textkey = "cpu_usage"
    label = "CPU"

@classmethod
    def get_metadata(self, config):
        status = agent_util.SUPPORTED
        msg = None

if "aix" in sys.platform:
            status = agent_util.SUPPORTED
            data = {
                "load_average.1": {
                    "label": "1 minute CPU load average",
                    "options": None,
                    "status": status,
                    "error_message": msg,
                    "unit": "avg",
                },
                "load_average.5": {
                    "label": "5 minute CPU load average",
                    "options": None,
                    "status": status,
                    "error_message": msg,
                    "unit": "avg",
                },
                "load_average.15": {
                    "label": "15 minute CPU load average",
                    "options": None,
                    "status": status,
                    "error_message": msg,
                    "unit": "avg",
                },
                "usage_percentage": {
                    "label": "Usage percentage",
                    "options": sorted(get_cpu_metrics(self).keys()),
                    "status": status,
                    "error_message": msg,
                    "unit": "percent",
                },
                "user_usage_percentage": {
                    "label": "User usage percentage",
                    "options": None,
                    "status": status,
                    "error_message": msg,
                    "unit": "percent",
                },
                "system_usage_percentage": {
                    "label": "System usage percentage",
                    "options": None,
                    "status": status,
                    "error_message": msg,
                    "unit": "percent",
                },
                "idle_usage_percentage": {
                    "label": "Idle usage percentage",
                    "options": None,
                    "status": status,
                    "error_message": msg,
                    "unit": "percent",
                },
                "iowait_usage_percentage": {
                    "label": "I/O Wait usage percentage",
                    "options": None,
                    "status": status,
                    "error_message": msg,
                    "unit": "percent",
                },
                "cpu_entitlement_percentage": {
                    "label": "CPU entitlement percentage",
                    "options": None,
                    "status": status,
                    "error_message": msg,
                    "unit": "percent",
                },
            }
            return data

elif "sunos" in sys.platform:
            status = agent_util.SUPPORTED
            data = {
                "load_average.1": {
                    "label": "1 minute CPU load average",
                    "options": None,
                    "status": status,
                    "error_message": msg,
                    "unit": "avg",
                },
                "load_average.5": {
                    "label": "5 minute CPU load average",
                    "options": None,
                    "status": status,
                    "error_message": msg,
                    "unit": "avg",
                },
                "load_average.15": {
                    "label": "15 minute CPU load average",
                    "options": None,
                    "status": status,
                    "error_message": msg,
                    "unit": "avg",
                },
                "usage_percentage": {
                    "label": "Usage percentage",
                    "options": sorted(get_cpu_metrics(self).keys()),
                    "status": status,
                    "error_message": msg,
                    "unit": "percent",
                },
                "user_usage_percentage": {
                    "label": "User usage percentage",
                    "options": None,
                    "status": status,
                    "error_message": msg,
                    "unit": "percent",
                },
                "system_usage_percentage": {
                    "label": "System usage percentage",
                    "options": None,
                    "status": status,
                    "error_message": msg,
                    "unit": "percent",
                },
                "idle_usage_percentage": {
                    "label": "Idle usage percentage",
                    "options": None,
                    "status": status,
                    "error_message": msg,
                    "unit": "percent",
                },
                "iowait_usage_percentage": {
                    "label": "I/O Wait usage percentage",
                    "options": None,
                    "status": status,
                    "error_message": msg,
                    "unit": "percent",
                },
            }
            return data

elif "freebsd" in sys.platform or "darwin" in sys.platform:
            status = agent_util.SUPPORTED
            data = {
                "load_average.1": {
                    "label": "1 minute CPU load average",
                    "options": None,
                    "status": status,
                    "error_message": msg,
                    "unit": "avg",
                },
                "load_average.5": {
                    "label": "5 minute CPU load average",
                    "options": None,
                    "status": status,
                    "error_message": msg,
                    "unit": "avg",
                },
                "load_average.15": {
                    "label": "15 minute CPU load average",
                    "options": None,
                    "status": status,
                    "error_message": msg,
                    "unit": "avg",
                },
                "usage_percentage": {
                    "label": "Usage percentage",
                    "options": ["Total"],
                    "status": status,
                    "error_message": msg,
                    "unit": "percent",
                },
                "user_usage_percentage": {
                    "label": "User usage percentage",
                    "options": None,
                    "status": status,
                    "error_message": msg,
                    "unit": "percent",
                },
                "system_usage_percentage": {
                    "label": "System usage percentage",
                    "options": None,
                    "status": status,
                    "error_message": msg,
                    "unit": "percent",
                },
                "idle_usage_percentage": {
                    "label": "Idle usage percentage",
                    "options": None,
                    "status": status,
                    "error_message": msg,
                    "unit": "percent",
                },
            }
            return data
        elif "hp-ux" in sys.platform:
            status = agent_util.SUPPORTED
            metadata = {
                "load_average.1": {
                    "label": "1 minute CPU load average",
                    "options": None,
                    "status": status,
                    "error_message": msg,
                    "unit": "avg",
                },
                "load_average.5": {
                    "label": "5 minute CPU load average",
                    "options": None,
                    "status": status,
                    "error_message": msg,
                    "unit": "avg",
                },
                "load_average.15": {
                    "label": "15 minute CPU load average",
                    "options": None,
                    "status": status,
                    "error_message": msg,
                    "unit": "avg",
                },
                "usage_percentage": {
                    "label": "Total Usage percentage",
                    "options": ["Total"],
                    "status": status,
                    "error_message": msg,
                    "unit": "percent",
                },
                "user_usage_percentage": {
                    "label": "User usage percentage",
                    "options": None,
                    "status": status,
                    "error_message": msg,
                    "unit": "percent",
                },
                "system_usage_percentage": {
                    "label": "System usage percentage",
                    "options": None,
                    "status": status,
                    "error_message": msg,
                    "unit": "percent",
                },
                "idle_usage_percentage": {
                    "label": "Idle usage percentage",
                    "options": None,
                    "status": status,
                    "error_message": msg,
                    "unit": "percent",
                },
            }

return metadata

elif "vmware" in sys.platform:
            status = agent_util.SUPPORTED

# here we're gathering the CPU cores that we can monitor and adding in a Total aggregation
            cpus = []
            ret, out = agent_util.execute_command(
                'esxcli hardware cpu list | grep "CPU:"'
            )
            tmp_cpus = [x for x in out.split("\n") if x != ""]
            for c in tmp_cpus:
                cpu = "Cpu (%s)" % c.split(":")[1]
                cpus.append(cpu)
            cpus.append("Total")

data = {
                "load_average.1": {
                    "label": "1 minute CPU load average",
                    "options": None,
                    "status": status,
                    "error_message": msg,
                    "unit": "avg",
                },
                "load_average.5": {
                    "label": "5 minute CPU load average",
                    "options": None,
                    "status": status,
                    "error_message": msg,
                    "unit": "avg",
                },
                "load_average.15": {
                    "label": "15 minute CPU load average",
                    "options": None,
                    "status": status,
                    "error_message": msg,
                    "unit": "avg",
                },
                "usage_percentage": {
                    "label": "Usage percentage",
                    "options": cpus,
                    "status": status,
                    "error_message": msg,
                    "min_value": 0,
                    "max_value": 100,
                    "unit": "percent",
                },
                "idle_usage_percentage": {
                    "label": "Idle usage percentage",
                    "options": cpus,
                    "status": status,
                    "error_message": msg,
                    "min_value": 0,
                    "max_value": 100,
                    "unit": "percent",
                },
            }

return data

else:
            if psutil is None:
                # Unable to import psutil
                self.log.info(
                    "Unable to import psutil library, no process metrics available"
                )
                status = agent_util.UNSUPPORTED
                msg = "Unable to import psutil library, please install and rebuild metadata"
            # Core Linux
            if not agent_util.which("top", exc=False):
                self.log.info("top binary not found")
                status = agent_util.UNSUPPORTED
                msg = "top binary not found"

try:
                distro_info = platform.dist()
            except AttributeError:
                if distro:
                    distro_info = distro.linux_distribution()
                    distro_info = ". ".join(distro_info)
                else:
                    raise ValueError(
                        "Unable to grab distribution information. Please verify dependencies. Distro for Python3.8"
                    )

if (
                "centos" in distro_info
                or "redhat" in distro_info
                or "oracle" in distro_info
            ) and not agent_util.which("iostat", exc=False):
                self.log.info("Missing sysstat package.")
                status = agent_util.UNSUPPORTED
                msg = "iostat/sysstat binary not found. Please install"

metadata = {
                "load_average.1": {
                    "label": "1 minute CPU load average",
                    "options": None,
                    "status": status,
                    "error_message": msg,
                    "unit": "avg",
                },
                "load_average.5": {
                    "label": "5 minute CPU load average",
                    "options": None,
                    "status": status,
                    "error_message": msg,
                    "unit": "avg",
                },
                "load_average.15": {
                    "label": "15 minute CPU load average",
                    "options": None,
                    "status": status,
                    "error_message": msg,
                    "unit": "avg",
                },
                "usage_percentage": {
                    "label": "Usage percentage",
                    "options": sorted(get_cpu_metrics(self).keys()),
                    "status": status,
                    "error_message": msg,
                    "unit": "percent",
                },
                "user_usage_percentage": {
                    "label": "User usage percentage",
                    "options": None,
                    "status": status,
                    "error_message": msg,
                    "unit": "percent",
                },
                "system_usage_percentage": {
                    "label": "System usage percentage",
                    "options": None,
                    "status": status,
                    "error_message": msg,
                    "unit": "percent",
                },
                "idle_usage_percentage": {
                    "label": "Idle usage percentage",
                    "options": None,
                    "status": status,
                    "error_message": msg,
                    "unit": "percent",
                },
                "iowait_usage_percentage": {
                    "label": "I/O Wait usage percentage",
                    "options": None,
                    "status": status,
                    "error_message": msg,
                    "unit": "percent",
                },
                "irq_usage_percentage": {
                    "label": "Hardware IRQ usage percentage",
                    "options": None,
                    "status": status,
                    "error_message": msg,
                    "unit": "percent",
                },
                "softirg_usage_percentage": {
                    "label": "Software IRQ usage percentage",
                    "options": None,
                    "status": status,
                    "error_message": msg,
                    "unit": "percent",
                },
                "stealtime_usage_percentage": {
                    "label": "Steal Time usage percentage",
                    "options": None,
                    "status": status,
                    "error_message": msg,
                    "unit": "percent",
                },
                "nice_usage_percentage": {
                    "label": "Nice usage percentage",
                    "options": None,
                    "status": status,
                    "error_message": msg,
                    "unit": "percent",
                },
            }

return metadata

def check(self, textkey, data, config={}):
        # AIX-specific logic
        if (
            "aix" in sys.platform
            or "darwin" in sys.platform
            or "freebsd" in sys.platform
        ):
            if textkey.startswith("load_average"):
                retcode, load = agent_util.execute_command("uptime")
                fields = load.strip().split()
                if textkey == "load_average.1":
                    return float(fields[-3].strip(","))
                elif textkey == "load_average.5":
                    return float(fields[-2].strip(","))
                elif textkey == "load_average.15":
                    return float(fields[-1])
                else:
                    return None

else:
                iostat = str(agent_util.which("iostat"))
                if "aix" in sys.platform:
                    retcode, output = agent_util.execute_command(
                        iostat + " | grep -p tty"
                    )
                if "darwin" in sys.platform or "freebsd" in sys.platform:
                    retcode, output = agent_util.execute_command(
                        iostat + " -C -c 2 | tail -1"
                    )

output = output.strip().split("\n")
                self.log.debug("iostat output: %s" % output)
                iostatline = False
                enti = False
                entc = 0
                inuse = 0
                user = 0
                system = 0
                idle = 0
                iowait = 0
                for line in output:
                    if line.startswith("tty"):
                        iostatline = True
                        if "entc" in line.split()[-1]:
                            enti = True
                        continue
                    fields = line.split()
                    if "darwin" in sys.platform:
                        user = float(fields[-6])
                        system = float(fields[-5])
                        idle = float(fields[-4])
                    elif "freebsd" in sys.platform:
                        user = float(-5)
                        idle = float(fields[-1])
                        system = float(fields[-3])
                    else:
                        user = float(fields[2])
                        system = float(fields[3])
                        idle = float(fields[4])
                        iowait = float(fields[5])
                        if enti == True:
                            entc = float(fields[-1])
                    inuse = 100.0 - idle

if textkey == "usage_percentage":
                    return inuse
                elif textkey == "user_usage_percentage":
                    return user
                elif textkey == "system_usage_percentage":
                    return system
                elif textkey == "idle_usage_percentage":
                    return idle
                elif textkey == "iowait_usage_percentage":
                    return iowait
                elif textkey == "cpu_entitlement_percentage" and enti == True:
                    return entc

# If we got here, we don't know how to gather this metric
            # for AIX - return None
            return None

elif "sunos" in sys.platform:
            if textkey.startswith("load_average"):
                retcode, load = agent_util.execute_command("uptime")
                fields = load.strip().split()
                if textkey == "load_average.1":
                    return float(fields[-3].strip(","))
                elif textkey == "load_average.5":
                    return float(fields[-2].strip(","))
                elif textkey == "load_average.15":
                    return float(fields[-1])
                else:
                    return None

retcode, output = agent_util.execute_command("mpstat")
            output = output.split("\n")
            for line in output:
                if "CPU" in line or not line:
                    continue
                fields = line.split()
                if textkey == "usage_percentage":
                    return 100.0 - float(fields[-1])
                elif textkey == "user_usage_percentage":
                    return float(fields[-4])
                elif textkey == "system_usage_percentage":
                    return float(fields[-3])
                elif textkey == "idle_usage_percentage":
                    return float(fields[-1])
                elif textkey == "iowait_usage_percentage":
                    return float(fields[-2])

# If we got here we don't know how to gather this metric for Solaris
            return None

elif "vmware" in sys.platform:
            hostname = socket.gethostname()
            search_string = "\\\\%s\\Physical " % hostname
            metric_value = None

# actually gather the data to parse
            ret, out = agent_util.execute_command(
                "esxtop -b -n 2 -d 2", cache_timeout=agent_util.DEFAULT_CACHE_TIMEOUT
            )
            out_list = out.split("\n")
            headers = out_list[0].replace('"', "").split(",")
            esxtop_data = []
            for idx, val in enumerate(out_list[::1]):
                if not val or val == "":
                    continue
                esxtop_data = out_list[idx].replace('"', "").split(",")

# finish building search string
            if textkey.startswith("load_average"):
                search_string += (
                    "Cpu Load\\Cpu Load (%s Minute Avg)" % textkey.split(".")[-1]
                )
            elif data and (
                textkey == "usage_percentage" or textkey == "idle_usage_percentage"
            ):
                if data == "Total":
                    search_string += "Cpu(_Total)"
                else:
                    search_string += data

search_string += "\\% Processor Time"

# find index from headers and match to esxtop_data collected
            search_idx = search_esxtop(headers, search_string)
            if not search_idx:
                self.log.error("Unable to parse ESXTOP output for %s" % search_string)
                return None

if textkey == "idle_usage_percentage":
                metric_value = 100 - float(esxtop_data[search_idx])
            else:
                metric_value = float(esxtop_data[search_idx])

return metric_value

elif "hp-ux" in sys.platform:
            # add terminal specification for hpux
            os.environ["TERM"] = "xterm"
            # !!! applicable to HP-UX 11.31 !!!
            ret, out = agent_util.execute_command("top -s2 -d2", env=os.environ)
            top = out.strip().splitlines()
            self.log.debug(top)
            metric_mapping = {}
            cpu_str = ""
            load_str = ""
            for line in top:
                if line.lower().startswith("avg"):
                    cpu_str = line
                elif line.lower().startswith("load averages"):
                    load_str = line

cpu = cpu_str.replace("%", "").split()
            self.log.debug(cpu)
            metric_mapping["user_usage_percentage"] = float(cpu[2])
            metric_mapping["system_usage_percentage"] = float(cpu[4])
            metric_mapping["idle_usage_percentage"] = float(cpu[5])
            metric_mapping["usage_percentage"] = (
                100.0 - metric_mapping["idle_usage_percentage"]
            )

load = load_str.strip().replace(",", "").split()
            self.log.debug(load)
            self.log.debug("'%s'" % load[4][:4])
            metric_mapping["load_average.1"] = float(load[2])
            metric_mapping["load_average.5"] = float(load[3])
            metric_mapping["load_average.15"] = float(load[4][:4])

return float(metric_mapping.get(textkey, None))

else:
            if psutil is None:
                self.log.error("PSUTIL PACKAGE MISSING! UNABLE TO COLLECT CPU METRICS")
                return None
            # Default Linux/FreeBSD logic
            if textkey.startswith("load_average"):
                retcode, output = agent_util.execute_command("top -b -n 2 -d 0.5")
                if config.get("debug", False):
                    self.log.debug(
                        "#####################################################"
                    )
                    self.log.debug("CPU usage command 'top -b -n 2 -d 0.5:")
                    self.log.debug(str(output))
                    self.log.debug(
                        "#####################################################"
                    )
                self.log.debug("top -b -n 2 -d 0.5: %s" % str(output))
                output = output.splitlines()
                space_index = [0]

for var, item in enumerate(output):
                    if item == "":
                        space_index.append(var)

tmp_out = []
                for line in output[space_index[2] :]:
                    if line.strip():
                        tmp_out.append(line)
                output = tmp_out

if textkey.startswith("load_average"):
                    fields = output[0].split()
                    if textkey == "load_average.1":
                        index = -3
                    elif textkey == "load_average.5":
                        index = -2
                    elif textkey == "load_average.15":
                        index = -1
                    return float(fields[index].strip(","))

elif textkey.endswith("usage_percentage") and textkey != "usage_percentage":
                num_cores = psutil.cpu_count()
                usage_textkey_map = {
                    "user_usage_percentage": "user",
                    "system_usage_percentage": "system",
                    "idle_usage_percentage": "idle",
                    "iowait_usage_percentage": "iowait",
                    "irq_usage_percentage": "irq",
                    "softirg_usage_percentage": "softirq",
                    "stealtime_usage_percentage": "steal",
                    "nice_usage_percentage": "nice",
                }
                key_name = usage_textkey_map.get(textkey, None)
                if key_name is None:
                    self.log.error("Unknown resource textkey '%s'!" % textkey)
                    return None

c = self.get_cache_results("psutil", "detailed_cpu_usage")
                self.log.debug("Retrieved cached value:\n%s" % c)
                cur_cpu = psutil.cpu_times()
                self.log.debug(
                    "Retrieved instant value:\n%s" % getattr(cur_cpu, key_name)
                )
                last_cpu = c and c[0][1] or None
                self.cache_result("psutil", "detailed_cpu_usage", cur_cpu, replace=True)
                if last_cpu is None:
                    return None
                use_diff = (
                    getattr(cur_cpu, key_name) - getattr(last_cpu, key_name)
                ) / num_cores
                if use_diff < 0:
                    # The system was likely rebooted, and the cached
                    # CPU stats are no longer relevant.
                    # Cache new values and exit without reporting a value.
                    return None
                elapsed = c[0][0]
                usage_time = (use_diff / elapsed) * 100.0
                return usage_time

elif textkey == "usage_percentage" and data.lower() == "total":
                num_cores = psutil.cpu_count()
                c = self.get_cache_results("psutil", "total_cpu_usage")
                self.log.debug("Retrieved cached value:\n%s" % c)
                cur_cpu = psutil.cpu_times()
                self.log.debug("Retrieved instant value:\n%s" % cur_cpu.idle)
                last_cpu = c and c[0][1] or None
                self.cache_result("psutil", "total_cpu_usage", cur_cpu, replace=True)
                if last_cpu is None:
                    return None

idle_diff = (cur_cpu.idle - last_cpu.idle) / num_cores
                steal_diff = (cur_cpu.steal - last_cpu.steal) / num_cores
                if idle_diff < 0 or steal_diff < 0:
                    # The system was likely rebooted, and the cached
                    # CPU stats are no longer relevant.
                    # Cache new values and exit without reporting a value.
                    return None
                use_diff = idle_diff + steal_diff
                # Instead of using the time between cached calculate the exact time
                # between measures by substracting the sum of the current clock time vs
                # the previous clock time. This avoid issues where our usage was too small
                # and the seconds of the extra cache would give a negative result
                elapsed = (sum(cur_cpu) - sum(last_cpu)) / float(num_cores)
                usage_time = 100 - ((use_diff / elapsed) * 100.0)
                return usage_time

elif textkey == "usage_percentage" and data.lower() != "total":
                self.log.debug("Checking for core %s" % data)
                num_cores = psutil.cpu_count()
                c = self.get_cache_results("psutil", "%s_cpu_usage" % data)
                self.log.debug("Retrieved cached value:\n%s" % c)
                try:
                    cur_cpu = psutil.cpu_times(percpu=True)[int(str(data).strip("cpu"))]
                except IndexError:
                    self.log.critical("UNABLE TO FIND CPU #%s" % data)
                    return None
                self.log.debug("Retrieved instant value:\n%s" % cur_cpu.idle)
                last_cpu = c and c[0][1] or None
                self.cache_result(
                    "psutil", "%s_cpu_usage" % data, cur_cpu, replace=True
                )
                if last_cpu is None:
                    return None
                idle_diff = cur_cpu.idle - last_cpu.idle
                steal_diff = cur_cpu.steal - last_cpu.steal
                if idle_diff < 0 or steal_diff < 0:
                    # The system was likely rebooted, and the cached
                    # CPU stats are no longer relevant.
                    # Cache new values and exit without reporting a value.
                    return None
                use_diff = idle_diff + steal_diff
                elapsed = sum(cur_cpu) - sum(last_cpu)
                usage_time = 100 - ((use_diff / elapsed) * 100.0)
                return usage_time

return 0

Sid Gifari From Gifari Industries - BD Cyber Security Team