#!/usr/lib64/linuxfabrik-monitoring-plugins/venv/bin/python
# -*- coding: utf-8; py-indent-offset: 4 -*-
#
# Author:  Linuxfabrik GmbH, Zurich, Switzerland
# Contact: info (at) linuxfabrik (dot) ch
#          https://www.linuxfabrik.ch/
# License: The Unlicense, see LICENSE file.

# https://github.com/Linuxfabrik/monitoring-plugins/blob/main/CONTRIBUTING.md

"""See the check's README for more details."""

import argparse
import re
import sys
import urllib.parse

import lib.args
import lib.base
import lib.huawei_pacific
import lib.lftest
import lib.txt
from lib.globals import STATE_CRIT, STATE_OK, STATE_UNKNOWN, STATE_WARN

__author__ = 'Linuxfabrik GmbH, Zurich/Switzerland'
__version__ = '2026081201'

DESCRIPTION = """Checks the service processes of every node of a Huawei OceanStor Pacific storage
system via the REST API (/cluster_service/service_processes endpoint). Alerts when a process is not
running, and when a node of the cluster does not report its processes at all. Supports extended
reporting via --lengthy and shorter output via --brief."""

DEFAULT_CACHE_EXPIRE = 15  # minutes
DEFAULT_INSECURE = True
DEFAULT_NO_MATCH_SEVERITY = 'ok'
DEFAULT_NO_PROXY = False
DEFAULT_SCOPE = '0'
DEFAULT_SILENT_NODE_SEVERITY = 'warn'
DEFAULT_TIMEOUT = 30  # seconds; the vendor documents a timeout period of 30 s

# Fields `--match` and `--ignore` are applied to.
MATCH_FIELDS = ('node', 'process_name')

# The one process status both REST Interface References show, and the only one seen on
# real hardware. Neither of them documents an enumeration for the field, so every other
# code is reported as the appliance sent it rather than translated into an invented name.
PROCESS_STATUS_RUNNING = 0


def parse_args():
    """Parse command line arguments using argparse."""
    parser = argparse.ArgumentParser(
        description=DESCRIPTION,
        epilog=lib.args.epilog(__file__),
        formatter_class=lib.args.HelpFormatter,
    )

    parser.add_argument(
        '-V',
        '--version',
        action='version',
        version=f'%(prog)s: v{__version__} by {__author__}',
    )

    parser.add_argument(
        '--always-ok',
        help=lib.args.help('--always-ok'),
        dest='ALWAYS_OK',
        action='store_true',
        default=False,
    )

    parser.add_argument(
        '--brief',
        help='Hide table rows for processes that are running and show only those that '
        'are not. Perfdata and alerting are unaffected: every process still emits '
        'perfdata and still drives the overall check state. '
        'Default: %(default)s',
        dest='BRIEF',
        action='store_true',
        default=False,
    )

    parser.add_argument(
        '--cache-expire',
        help=lib.args.help('--cache-expire') + ' Default: %(default)s',
        dest='CACHE_EXPIRE',
        type=int,
        default=DEFAULT_CACHE_EXPIRE,
    )

    parser.add_argument(
        '--ignore',
        help='Skip processes. '
        + lib.args.help('--ignore-regex')
        + ' The regex is anchored at the start of the string (Python `re.match`) and '
        'is matched against the node name and the process name, so prefix with `.*` to '
        'match anywhere. Default: %(default)s',
        dest='IGNORE',
        action='append',
        default=None,
    )

    parser.add_argument(
        '--insecure',
        help=lib.args.help('--insecure'),
        dest='INSECURE',
        action='store_true',
        default=DEFAULT_INSECURE,
    )

    parser.add_argument(
        '--lengthy',
        help=lib.args.help('--lengthy'),
        dest='LENGTHY',
        action='store_true',
        default=False,
    )

    parser.add_argument(
        '--match',
        help='Limit to processes. '
        + lib.args.help('--match')
        + ' The regex is anchored at the start of the string (Python `re.match`) and '
        'is matched against the node name and the process name, so prefix with `.*` to '
        'match anywhere. Default: %(default)s',
        dest='MATCH',
        action='append',
        default=None,
    )

    parser.add_argument(
        '--no-insecure',
        help=lib.args.help('--no-insecure'),
        dest='INSECURE',
        action='store_false',
        default=DEFAULT_INSECURE,
    )

    parser.add_argument(
        '--no-match-severity',
        help=lib.args.help('--no-match-severity') + ' Default: %(default)s',
        dest='NO_MATCH_SEVERITY',
        choices=['ok', 'warn', 'crit', 'unknown'],
        default=DEFAULT_NO_MATCH_SEVERITY,
    )

    parser.add_argument(
        '--no-perfdata',
        help=lib.args.help('--no-perfdata'),
        dest='NO_PERFDATA',
        action='store_true',
        default=False,
    )

    parser.add_argument(
        '--no-proxy',
        help=lib.args.help('--no-proxy'),
        dest='NO_PROXY',
        action='store_true',
        default=DEFAULT_NO_PROXY,
    )

    parser.add_argument(
        '--password',
        help='Huawei OceanStor Pacific API password.',
        dest='PASSWORD',
    )

    parser.add_argument(
        '--password-file',
        help=lib.args.help('--password-file'),
        dest='PASSWORD_FILE',
    )

    parser.add_argument(
        '--scope',
        help='Huawei OceanStor Pacific API scope.',
        dest='SCOPE',
        default=DEFAULT_SCOPE,
    )

    parser.add_argument(
        '--silent-node-severity',
        help='State to report for a cluster node that the process query returns nothing '
        'for. Its processes are unmonitored for as long as that lasts, which is not the '
        'same as knowing they are down. '
        'Default: %(default)s',
        dest='SILENT_NODE_SEVERITY',
        choices=['ok', 'warn', 'crit', 'unknown'],
        default=DEFAULT_SILENT_NODE_SEVERITY,
    )

    parser.add_argument(
        '--test',
        help=lib.args.help('--test'),
        dest='TEST',
        type=lib.args.csv,
    )

    parser.add_argument(
        '--timeout',
        help=lib.args.help('--timeout') + ' Default: %(default)s (seconds)',
        dest='TIMEOUT',
        type=int,
        default=DEFAULT_TIMEOUT,
    )

    parser.add_argument(
        '-u',
        '--url',
        help='Huawei OceanStor Pacific API URL.',
        dest='URL',
        required=True,
    )

    parser.add_argument(
        '--username',
        help='Huawei OceanStor Pacific API username.',
        dest='USERNAME',
        required=True,
    )

    parser.add_argument(
        '-v',
        '--verbose',
        help=lib.args.help('--verbose')
        + ' Appends what every API request returned, so the appliance\'s own answers '
        'can be read while working out how it reports something. Session tokens are '
        'redacted. The output is as long as those answers are, so this is a debugging '
        'aid rather than something to leave switched on.',
        dest='VERBOSE',
        action='store_true',
        default=False,
    )

    args, _ = parser.parse_known_args()
    return args


def count_instances(process):
    """
    Return how many instances of a process the node reports.

    A process runs once on most nodes and many times on some: the OSD entry of a storage
    node carries one process ID per disk it serves. The appliance packs them into a single
    comma-separated string, so the count has to be taken from that.

    ### Parameters
    - **process** (`dict`): One process as the API returned it.

    ### Returns
    - **int**: The number of process IDs. `0` where the appliance reports none, which is
      what a process it tracks without a PID looks like.

    ### Example
    >>> count_instances({'process_id': '1000263,1003883'})
    2
    """
    pids = process.get('process_id')
    if not isinstance(pids, str):
        return 0
    return len([pid for pid in pids.split(',') if pid.strip()])


def get_processes_by_ip(result):
    """
    Turn the process response into a lookup from node management IP to its processes.

    The endpoint answers with one entry per node it was asked about, in no documented
    order, so a caller that reports per node has to index it first.

    ### Parameters
    - **result** (`dict`): The response as `get_data()` returns it.

    ### Returns
    - **dict**: `management_ip` to the list of processes reported for it.

    ### Example
    >>> get_processes_by_ip({'data': [{'management_ip': '192.0.2.11', 'processes': []}]})
    {'192.0.2.11': []}
    """
    by_ip = {}
    for entry in result.get('data') or []:
        if not isinstance(entry, dict):
            continue
        ip = entry.get('management_ip')
        if not ip:
            continue
        processes = entry.get('processes')
        by_ip[ip] = processes if isinstance(processes, list) else []
    return by_ip


def main():
    """The main function. This is where the magic happens."""

    # parse the command line
    try:
        args = parse_args()
    except SystemExit:
        sys.exit(STATE_UNKNOWN)

    if args.PASSWORD_FILE:
        args.PASSWORD = lib.args.load_secret(args.PASSWORD_FILE)
    if not args.PASSWORD:
        lib.base.cu('Provide the API password via --password or --password-file.')

    if not args.URL.startswith('http'):
        lib.base.cu('--url parameter has to start with "http://" or https://".')

    # set default values for append parameters that were not specified
    if args.IGNORE is None:
        args.IGNORE = []
    if args.MATCH is None:
        args.MATCH = []

    # fetch data
    if args.TEST is None:
        # The process endpoint is node-scoped, and takes the nodes as a comma-separated
        # query parameter rather than in a body, so the whole cluster fits into one
        # request. The addresses are quoted although they come from the appliance
        # itself: they are interpolated into a URL, and a value that is not an address
        # must not be able to add a parameter of its own.
        nodes = lib.huawei_pacific.get_cluster_nodes(args)
        ips = ','.join(node['management_ip'] for node in nodes)
        result = lib.huawei_pacific.get_data(
            'cluster_service/service_processes'
            f'?management_ips={urllib.parse.quote(ips, safe=",")}',
            args,
        )
    else:
        # do not call the API, put in test data. Each API call has its own fixture
        # suffix, so the fixture file names describe what they contain.
        test_base = args.TEST[0]
        nodes = (
            lib.lftest.test_json(args.TEST, f'{test_base}-servers').get('data') or []
        )
        result = lib.lftest.test_json(args.TEST, f'{test_base}-processes')

    # The process endpoint names a node by its management IP only, so the name an
    # operator would go and look at has to come from the cluster listing.
    node_names = lib.huawei_pacific.get_node_names_by_ip(nodes)

    # no valuable result?
    lib.huawei_pacific.assert_ok(result, 'the service processes')

    # A cluster node always runs service processes, so an empty answer is a query that
    # never reached them rather than an inventory that is genuinely empty.
    # Reporting OK here would hide the fault behind a green check.
    if not result.get('data'):
        lib.base.oao(
            f'{args.URL} reported no service processes.'
            ' Verify that the API user is allowed to query them.',
            STATE_UNKNOWN,
        )

    # init some vars
    msg = ''
    state = STATE_OK
    perfdata = ''
    table_data = []
    silent_nodes = []

    # compile user-supplied regex patterns
    compiled_match_regex = [
        lib.base.coe(item) for item in lib.txt.compile_regex(args.MATCH, '--match')
    ]
    compiled_ignore_regex = [
        lib.base.coe(item) for item in lib.txt.compile_regex(args.IGNORE, '--ignore')
    ]
    silent_node_state = lib.base.str2state(args.SILENT_NODE_SEVERITY)

    # analyze data
    processes_by_ip = get_processes_by_ip(result)
    for node in nodes:
        ip = node.get('management_ip')
        node_name = node_names.get(ip) or ip
        processes = processes_by_ip.get(ip)

        # The node was asked about and is not in the answer, or is in it without a single
        # process. Either way nothing is known about its processes, which is not the same
        # as knowing they are down, so it gets its own line rather than a fake row.
        if not processes:
            silent_nodes.append(node_name)
            state = lib.base.get_worst(state, silent_node_state)
            continue

        label = re.sub(r'\W+', '_', str(node_name)).strip('_')
        not_running = 0
        for process in processes:
            if not isinstance(process, dict):
                continue
            process_name = str(process.get('process_name') or '--')
            fields = {'node': node_name, 'process_name': process_name}

            if args.MATCH and not any(
                lib.base.coe(lib.txt.match_regex(pattern, str(fields[field])))
                for pattern in compiled_match_regex
                for field in MATCH_FIELDS
            ):
                continue

            if args.IGNORE and any(
                lib.base.coe(lib.txt.match_regex(pattern, str(fields[field])))
                for pattern in compiled_ignore_regex
                for field in MATCH_FIELDS
            ):
                continue

            status = lib.huawei_pacific.as_code(process.get('process_status'))
            process_state = (
                STATE_OK if status == PROCESS_STATUS_RUNNING else STATE_CRIT
            )
            if process_state != STATE_OK:
                not_running += 1
            state = lib.base.get_worst(state, process_state)

            table_data.append(
                {
                    'instances': count_instances(process),
                    'node': node_name,
                    'process': process_name,
                    'process_state': process_state,
                    'state': lib.base.state2str(process_state, empty_ok=False),
                    'status': (
                        'running (0)'
                        if status == PROCESS_STATUS_RUNNING
                        else f'not running ({process.get("process_status")})'
                    ),
                }
            )

        # Two gauges per node rather than one per process: a cluster runs the same two
        # dozen processes on every node, and a metric each would bury the graph. The
        # total is the one that catches a process which stops being reported at all,
        # because a process that is gone has no status to be non-zero.
        perfdata += lib.base.get_perfdata(
            f'{label}_processes_total',
            len(processes),
            uom=None,
            _min=0,
        )
        perfdata += lib.base.get_perfdata(
            f'{label}_processes_not_running',
            not_running,
            uom=None,
            _min=0,
        )

    # the cluster listed processes and the filter selected none of them
    if not table_data and not silent_nodes:
        lib.base.oao(
            f'No processes matched `{", ".join(args.MATCH)}`.',
            lib.base.str2state(args.NO_MATCH_SEVERITY),
            always_ok=args.ALWAYS_OK,
            no_perfdata=args.NO_PERFDATA,
        )

    # build the message
    down = [row for row in table_data if row['process_state'] != STATE_OK]
    if state == STATE_CRIT:
        msg += 'There are critical errors.'
    elif state == STATE_WARN:
        msg += 'There are warnings.'
    else:
        msg += 'Everything is ok.'
    # Said the positive way round where there is nothing wrong: "0 not running" makes the
    # reader resolve a double negative on every glance.
    answered = len(nodes) - len(silent_nodes)
    msg += (
        f' Checked {len(table_data)}'
        f' {lib.txt.pluralize("process", len(table_data), "es")}'
        f' on {answered} {lib.txt.pluralize("node", answered)},'
        f'{f" {len(down)} not running." if down else " all running."}'
    )
    if silent_nodes:
        msg += (
            f'\n\nThese cluster nodes report no processes at all, so nothing is known'
            f' about them: {", ".join(sorted(silent_nodes))}.'
        )

    # build table output
    display_rows = down if args.BRIEF else table_data
    if args.LENGTHY:
        keys = ['node', 'process', 'instances', 'status', 'state']
        headers = ['Node', 'Process', 'Instances', 'Status', 'State']
    else:
        keys = ['node', 'process', 'status', 'state']
        headers = ['Node', 'Process', 'Status', 'State']
    if display_rows:
        msg += '\n\n' + lib.base.get_table(
            display_rows, keys, header=headers, missing='--', hide_empty=True
        )

    if args.VERBOSE:
        msg += '\n\n' + lib.huawei_pacific.format_responses()

    # over and out
    lib.base.oao(
        msg, state, perfdata, always_ok=args.ALWAYS_OK, no_perfdata=args.NO_PERFDATA
    )


if __name__ == '__main__':
    try:
        main()
    except Exception:
        lib.base.cu()
