#!/usr/lib64/linuxfabrik-monitoring-plugins/venv/bin/python
# -*- coding: utf-8; py-indent-offset: 4 -*-
#
# Author:  Linuxfabrik GmbH, Zurich, Switzerland
# Contact: info (at) linuxfabrik (dot) ch
#          https://www.linuxfabrik.ch/
# License: The Unlicense, see LICENSE file.

# https://github.com/Linuxfabrik/monitoring-plugins/blob/main/CONTRIBUTING.md

"""See the check's README for more details."""

import argparse
import json
import re
import sys

import lib.args
import lib.base
import lib.huawei_pacific
import lib.human
import lib.lftest
import lib.time
import lib.txt
from lib.globals import STATE_CRIT, STATE_OK, STATE_UNKNOWN, STATE_WARN

__author__ = 'Linuxfabrik GmbH, Zurich/Switzerland'
__version__ = '2026081201'

DESCRIPTION = """Checks the remote replication pairs of a Huawei OceanStor Pacific storage system
via the REST API (/dsware/service/REPLICATIONPAIR endpoint). Alerts when a pair is faulty or has
stopped mirroring, and optionally when its last synchronization is older than the given thresholds.
Supports extended reporting via --lengthy and shorter output via --brief."""

DEFAULT_CACHE_EXPIRE = 15  # minutes
DEFAULT_CRIT = ''
DEFAULT_INSECURE = True
DEFAULT_NO_MATCH_SEVERITY = 'ok'
DEFAULT_NO_PROXY = False
DEFAULT_SCOPE = '0'
DEFAULT_TIMEOUT = 30  # seconds; the vendor documents a timeout period of 30 s
DEFAULT_WARN = ''

# Highest `HEALTHSTATUS` code both REST Interface References document for a replication
# pair: invalid (3). It bounds the metric so a graph scales to the whole enumeration; the
# thresholds stay out of the performance data, because which code is a fault is decided in
# `lib.huawei_pacific.get_replication_health_status_state()`.
HEALTH_STATUS_MAX = 3

# Fields `--match` and `--ignore` are applied to.
MATCH_FIELDS = ('ID', 'LOCALRESNAME', 'REMOTERESNAME')

# Objects the endpoint returns per request. The vendor documents the upper limit of the
# range as 40, so asking for more silently gets less.
PAGE_SIZE = 40

# The interface version in the path. Both REST Interface References use this one in their
# examples, and a cluster answers identically below v1.1 through v1.4.
VERSION = 'v1.3'


def parse_args():
    """Parse command line arguments using argparse."""
    parser = argparse.ArgumentParser(
        description=DESCRIPTION,
        epilog=lib.args.epilog(__file__),
        formatter_class=lib.args.HelpFormatter,
    )

    parser.add_argument(
        '-V',
        '--version',
        action='version',
        version=f'%(prog)s: v{__version__} by {__author__}',
    )

    parser.add_argument(
        '--always-ok',
        help=lib.args.help('--always-ok'),
        dest='ALWAYS_OK',
        action='store_true',
        default=False,
    )

    parser.add_argument(
        '--brief',
        help='Hide table rows for pairs that are mirroring and show only those that are '
        'not. Perfdata and alerting are unaffected: every pair still emits perfdata and '
        'still drives the overall check state. '
        'Default: %(default)s',
        dest='BRIEF',
        action='store_true',
        default=False,
    )

    parser.add_argument(
        '--cache-expire',
        help=lib.args.help('--cache-expire') + ' Default: %(default)s',
        dest='CACHE_EXPIRE',
        type=int,
        default=DEFAULT_CACHE_EXPIRE,
    )

    parser.add_argument(
        '-c',
        '--critical',
        help='CRIT threshold for how long a pair has been without a complete copy at the '
        'far end, as a Nagios range in seconds. That is the time since the pair last '
        'finished transferring, or, while a transfer is running that has not finished, '
        'the time since it started. A pair can report a healthy status and still have '
        'stopped moving data, which is what this catches. Set it above the pair\'s own '
        'synchronization interval; there is no useful default, because that interval is '
        'configured per pair on the appliance. Off by default. '
        'Example: `--critical=172800` for two days',
        dest='CRIT',
        default=DEFAULT_CRIT,
    )

    parser.add_argument(
        '--ignore',
        help='Skip replication pairs. '
        + lib.args.help('--ignore-regex')
        + ' The regex is anchored at the start of the string (Python `re.match`) and is '
        'matched against the pair identifier and the local and remote resource names, so '
        'prefix with `.*` to match anywhere. Default: %(default)s',
        dest='IGNORE',
        action='append',
        default=None,
    )

    parser.add_argument(
        '--insecure',
        help=lib.args.help('--insecure'),
        dest='INSECURE',
        action='store_true',
        default=DEFAULT_INSECURE,
    )

    parser.add_argument(
        '--lengthy',
        help=lib.args.help('--lengthy'),
        dest='LENGTHY',
        action='store_true',
        default=False,
    )

    parser.add_argument(
        '--match',
        help='Limit to replication pairs. '
        + lib.args.help('--match')
        + ' The regex is anchored at the start of the string (Python `re.match`) and is '
        'matched against the pair identifier and the local and remote resource names, so '
        'prefix with `.*` to match anywhere. Default: %(default)s',
        dest='MATCH',
        action='append',
        default=None,
    )

    parser.add_argument(
        '--no-insecure',
        help=lib.args.help('--no-insecure'),
        dest='INSECURE',
        action='store_false',
        default=DEFAULT_INSECURE,
    )

    parser.add_argument(
        '--no-match-severity',
        help=lib.args.help('--no-match-severity') + ' Default: %(default)s',
        dest='NO_MATCH_SEVERITY',
        choices=['ok', 'warn', 'crit', 'unknown'],
        default=DEFAULT_NO_MATCH_SEVERITY,
    )

    parser.add_argument(
        '--no-perfdata',
        help=lib.args.help('--no-perfdata'),
        dest='NO_PERFDATA',
        action='store_true',
        default=False,
    )

    parser.add_argument(
        '--no-proxy',
        help=lib.args.help('--no-proxy'),
        dest='NO_PROXY',
        action='store_true',
        default=DEFAULT_NO_PROXY,
    )

    parser.add_argument(
        '--password',
        help='Huawei OceanStor Pacific API password.',
        dest='PASSWORD',
    )

    parser.add_argument(
        '--password-file',
        help=lib.args.help('--password-file'),
        dest='PASSWORD_FILE',
    )

    parser.add_argument(
        '--scope',
        help='Huawei OceanStor Pacific API scope.',
        dest='SCOPE',
        default=DEFAULT_SCOPE,
    )

    parser.add_argument(
        '--test',
        help=lib.args.help('--test'),
        dest='TEST',
        type=lib.args.csv,
    )

    parser.add_argument(
        '--timeout',
        help=lib.args.help('--timeout') + ' Default: %(default)s (seconds)',
        dest='TIMEOUT',
        type=int,
        default=DEFAULT_TIMEOUT,
    )

    parser.add_argument(
        '-u',
        '--url',
        help='Huawei OceanStor Pacific API URL.',
        dest='URL',
        required=True,
    )

    parser.add_argument(
        '--username',
        help='Huawei OceanStor Pacific API username.',
        dest='USERNAME',
        required=True,
    )

    parser.add_argument(
        '-w',
        '--warning',
        help='WARN threshold for how long a pair has been without a complete copy at the '
        'far end, as a Nagios range in seconds. Off by default, see --critical. '
        'Example: `--warning=86400` for one day',
        dest='WARN',
        default=DEFAULT_WARN,
    )

    parser.add_argument(
        '-v',
        '--verbose',
        help=lib.args.help('--verbose')
        + ' Appends what every API request returned, so the appliance\'s own answers '
        'can be read while working out how it reports something. Session tokens are '
        'redacted. The output is as long as those answers are, so this is a debugging '
        'aid rather than something to leave switched on.',
        dest='VERBOSE',
        action='store_true',
        default=False,
    )

    args, _ = parser.parse_known_args()
    return args


def as_timestamp(value):
    """
    Return a synchronization timestamp, or `None` where the pair reports none.

    A pair with no timestamp to report answers with a hyphen, with `0`, or leaves the field
    out entirely, and none of those may reach `epoch2iso()` as if it were a date in 1970.

    ### Parameters
    - **value** (`any`): `STARTTIME` or `ENDTIME` as the API returned it.

    ### Returns
    - **int** or **None**: The timestamp, or `None` where there is none.

    ### Example
    >>> as_timestamp('1696907475')
    1696907475
    >>> as_timestamp('-') is None
    True
    >>> as_timestamp('0') is None
    True
    """
    code = lib.huawei_pacific.as_code(value)
    if code is None or code <= 0:
        return None
    return code


def get_schedule(pair):
    """
    Describe how often a pair synchronizes, as one cell for the table.

    The appliance sends the schedule as a JSON document inside the JSON response rather
    than as a value, so it has to be parsed out of the string.

    ### Parameters
    - **pair** (`dict`): One replication pair as the API returned it.

    ### Returns
    - **str**: The schedule as `interval:value` pairs, `'--'` where the pair carries none
      or the field does not hold the documented document.

    ### Example
    >>> get_schedule({'SYNCHRONIZESCHEDULE': '{"HOURLY":"00"}'})
    'HOURLY:00'
    """
    try:
        schedule = json.loads(pair.get('SYNCHRONIZESCHEDULE', ''))
    except (TypeError, ValueError):
        return '--'
    if not isinstance(schedule, dict) or not schedule:
        return '--'
    return ', '.join(f'{key}:{value}' for key, value in sorted(schedule.items()))


def main():
    """The main function. This is where the magic happens."""

    # parse the command line
    try:
        args = parse_args()
    except SystemExit:
        sys.exit(STATE_UNKNOWN)

    if args.PASSWORD_FILE:
        args.PASSWORD = lib.args.load_secret(args.PASSWORD_FILE)
    if not args.PASSWORD:
        lib.base.cu('Provide the API password via --password or --password-file.')

    if not args.URL.startswith('http'):
        lib.base.cu('--url parameter has to start with "http://" or https://".')

    # set default values for append parameters that were not specified
    if args.IGNORE is None:
        args.IGNORE = []
    if args.MATCH is None:
        args.MATCH = []

    # fetch data
    # Paged: the vendor caps the range of this endpoint at 40 objects, so a cluster with
    # more pairs than that would silently be reported as a smaller but healthy one.
    truncated = False
    if args.TEST is None:
        result, truncated = lib.huawei_pacific.get_all_data(
            f'{VERSION}/REPLICATIONPAIR',
            args,
            page_size=PAGE_SIZE,
            range_style='bracket',
            base_path='dsware/service',
        )
    else:
        # do not call the command, put in test data
        stdout, _stderr, _retc = lib.lftest.test(args.TEST)
        result = json.loads(stdout)

    # no valuable result?
    # This endpoint reports its outcome below `error` rather than below `result`, which
    # `lib.huawei_pacific` reads either way.
    lib.huawei_pacific.assert_ok(result, 'the replication pairs')

    # An empty list is a healthy answer here: a cluster that replicates nothing has no
    # pairs, unlike the hardware endpoints where nothing at all means the query never
    # arrived.
    pairs = result.get('data') or []
    if not pairs:
        lib.base.oao(
            'No replication pairs configured.',
            STATE_OK,
            always_ok=args.ALWAYS_OK,
            no_perfdata=args.NO_PERFDATA,
        )

    # init some vars
    msg = ''
    state = STATE_OK
    perfdata = ''
    table_data = []
    now = lib.time.now(as_type='epoch')

    # compile user-supplied regex patterns
    compiled_match_regex = [
        lib.base.coe(item) for item in lib.txt.compile_regex(args.MATCH, '--match')
    ]
    compiled_ignore_regex = [
        lib.base.coe(item) for item in lib.txt.compile_regex(args.IGNORE, '--ignore')
    ]

    # analyze data
    for pair in pairs:
        if args.MATCH and not any(
            lib.base.coe(lib.txt.match_regex(pattern, str(pair.get(field, ''))))
            for pattern in compiled_match_regex
            for field in MATCH_FIELDS
        ):
            continue

        if args.IGNORE and any(
            lib.base.coe(lib.txt.match_regex(pattern, str(pair.get(field, ''))))
            for pattern in compiled_ignore_regex
            for field in MATCH_FIELDS
        ):
            continue

        health_state = lib.huawei_pacific.get_replication_health_status_state(
            pair.get('HEALTHSTATUS')
        )
        state = lib.base.get_worst(state, health_state)

        running_state = lib.huawei_pacific.get_replication_running_status_state(
            pair.get('RUNNINGSTATUS')
        )
        state = lib.base.get_worst(state, running_state)

        # How long the far end has been without a complete copy. Normally that is the
        # time since the pair last finished transferring. A pair whose current transfer
        # has not finished reports no end time at all, and then the answer is how long
        # that transfer has been running: a synchronization that never completes leaves
        # the far end just as stale as one that never starts, and keying on the end time
        # alone would leave such a pair below every threshold forever.
        end_time = as_timestamp(pair.get('ENDTIME'))
        start_time = as_timestamp(pair.get('STARTTIME'))
        reference_time = end_time if end_time is not None else start_time
        age = None if reference_time is None else max(now - reference_time, 0)
        age_state = STATE_OK
        if age is not None and (args.WARN or args.CRIT):
            age_state = lib.base.get_state(
                age, args.WARN or None, args.CRIT or None, _operator='range'
            )
            state = lib.base.get_worst(state, age_state)

        # Perfdata labels carry the local resource name, the way every sibling Pacific
        # check labels its objects.
        label = re.sub(
            r'\W+', '_', str(pair.get('LOCALRESNAME') or pair.get('ID'))
        ).strip('_')
        perfdata += lib.base.get_perfdata(
            f'{label}_health_status',
            lib.huawei_pacific.as_code(pair.get('HEALTHSTATUS')),
            uom=None,
            _min=0,
            _max=HEALTH_STATUS_MAX,
        )
        perfdata += lib.base.get_perfdata(
            f'{label}_running_status',
            lib.huawei_pacific.as_code(pair.get('RUNNINGSTATUS')),
            uom=None,
            _min=0,
        )
        if age is not None:
            perfdata += lib.base.get_perfdata(
                f'{label}_last_sync_age',
                age,
                uom='s',
                warn=args.WARN or None,
                crit=args.CRIT or None,
                _min=0,
            )

        row_state = lib.base.get_worst(
            lib.base.get_worst(health_state, running_state), age_state
        )
        table_data.append(
            {
                'health': lib.huawei_pacific.get_replication_health_status(
                    pair.get('HEALTHSTATUS')
                ),
                'id': pair.get('ID'),
                'local': pair.get('LOCALRESNAME'),
                'remote': pair.get('REMOTERESNAME'),
                'remote_device': pair.get('REMOTEDEVICENAME'),
                # Which end of the pair this cluster is. On the disaster recovery side
                # every pair is secondary, which is what it is supposed to be.
                'role': (
                    'primary'
                    if str(pair.get('ISPRIMARY')).lower() == 'true'
                    else 'secondary'
                ),
                'row_state': row_state,
                'running': lib.huawei_pacific.get_replication_running_status(
                    pair.get('RUNNINGSTATUS')
                ),
                'schedule': get_schedule(pair),
                # One state per row, in the last column. IcingaWeb replaces a state with
                # an icon, which shifts everything to the right of it, so a second one
                # mid-row would break the table.
                'state': lib.base.state2str(row_state, empty_ok=False),
                'sync_duration': (
                    '--'
                    if end_time is None or start_time is None or end_time < start_time
                    else lib.human.seconds2human(end_time - start_time)
                ),
                # A pair whose transfer has not finished has no completed synchronization
                # to name, which is not the same as never having synchronized at all.
                'sync_last': (
                    f'{lib.time.epoch2iso(end_time)}'
                    f' ({lib.human.seconds2human(age)} ago)'
                    f'{lib.base.state2str(age_state, prefix=" ")}'
                    if end_time is not None
                    else 'never'
                    if start_time is None
                    else f'in progress since {lib.time.epoch2iso(start_time)}'
                    f' ({lib.human.seconds2human(age)})'
                    f'{lib.base.state2str(age_state, prefix=" ")}'
                ),
            }
        )

    # the appliance listed replication pairs and the filter selected none of them
    if not table_data:
        lib.base.oao(
            f'No replication pairs matched `{", ".join(args.MATCH)}`.',
            lib.base.str2state(args.NO_MATCH_SEVERITY),
            always_ok=args.ALWAYS_OK,
            no_perfdata=args.NO_PERFDATA,
        )

    # The truncated walk raises the state before the summary is written, so a partially
    # read appliance does not open with "Everything is ok." and then exit WARNING.
    if truncated:
        state = lib.base.get_worst(state, STATE_WARN)

    # build the message
    down = [row for row in table_data if row['row_state'] != STATE_OK]
    if state == STATE_CRIT:
        msg += 'There are critical errors.'
    elif state == STATE_WARN:
        msg += 'There are warnings.'
    else:
        msg += 'Everything is ok.'
    # Said the positive way round where there is nothing wrong: "0 not mirroring" makes
    # the reader resolve a double negative on every glance.
    msg += (
        f' Checked {len(table_data)}'
        f' {lib.txt.pluralize("replication pair", len(table_data))},'
        f'{f" {len(down)} not mirroring." if down else " all mirroring."}'
    )
    if truncated:
        # The walk hit its page cap, so pairs beyond it were never looked at.
        msg += (
            '\n\nThe appliance reports more replication pairs than this check reads in'
            ' one run; the list below is incomplete.'
        )

    # build table output
    display_rows = down if args.BRIEF else table_data
    if args.LENGTHY:
        keys = [
            'id',
            'local',
            'remote',
            'remote_device',
            'role',
            'schedule',
            'sync_last',
            'sync_duration',
            'running',
            'health',
            'state',
        ]
        headers = [
            'ID',
            'Local',
            'Remote',
            'Remote Device',
            'Role',
            'Schedule',
            'Last Sync',
            'Duration',
            'Running',
            'Health',
            'State',
        ]
    else:
        keys = ['local', 'remote_device', 'sync_last', 'running', 'health', 'state']
        headers = ['Local', 'Remote Device', 'Last Sync', 'Running', 'Health', 'State']
    if display_rows:
        msg += '\n\n' + lib.base.get_table(
            display_rows, keys, header=headers, missing='--', hide_empty=True
        )

    if args.VERBOSE:
        msg += '\n\n' + lib.huawei_pacific.format_responses()

    # over and out
    lib.base.oao(
        msg, state, perfdata, always_ok=args.ALWAYS_OK, no_perfdata=args.NO_PERFDATA
    )


if __name__ == '__main__':
    try:
        main()
    except Exception:
        lib.base.cu()
