#!/usr/lib64/linuxfabrik-monitoring-plugins/venv/bin/python
# -*- coding: utf-8; py-indent-offset: 4 -*-
#
# Author:  Linuxfabrik GmbH, Zurich, Switzerland
# Contact: info (at) linuxfabrik (dot) ch
#          https://www.linuxfabrik.ch/
# License: The Unlicense, see LICENSE file.

# https://github.com/Linuxfabrik/monitoring-plugins/blob/main/CONTRIBUTING.md

"""See the check's README for more details."""

import argparse
import base64
import configparser
import sys

import lib.args
import lib.base
import lib.db_sqlite
import lib.human
import lib.lftest
import lib.time
import lib.txt
import lib.url
from lib.globals import STATE_CRIT, STATE_OK, STATE_UNKNOWN, STATE_WARN

__author__ = 'Linuxfabrik GmbH, Zurich/Switzerland'
__version__ = '2026102402'

DESCRIPTION = """Detects fast-flapping Icinga services by counting state changes per service within a
configurable lookback interval. Queries the Icinga DB event history and alerts when
any service exceeds the configured number of state changes. State changes that happened
during a downtime of the service are not counted."""

DEFAULT_CRIT = '19'
DEFAULT_INSECURE = False
DEFAULT_LOOKBACK = 4 * 3600
DEFAULT_NO_PROXY = False
DEFAULT_PWFILE = '/var/spool/icinga2/.icingaweb'
DEFAULT_TIMEOUT = 8
DEFAULT_WARN = '7'


def parse_args():
    """Parse command line arguments using argparse."""
    parser = argparse.ArgumentParser(
        description=DESCRIPTION,
        epilog=lib.args.epilog(__file__),
        formatter_class=lib.args.HelpFormatter,
    )

    parser.add_argument(
        '-V',
        '--version',
        action='version',
        version=f'%(prog)s: v{__version__} by {__author__}',
    )

    parser.add_argument(
        '--always-ok',
        help=lib.args.help('--always-ok'),
        dest='ALWAYS_OK',
        action='store_true',
        default=False,
    )

    parser.add_argument(
        '-c',
        '--critical',
        help='CRIT threshold for the number of state changes per service within the lookback period. '
        'Supports Nagios ranges. '
        'Default: %(default)s',
        dest='CRIT',
        default=DEFAULT_CRIT,
    )

    parser.add_argument(
        '--insecure',
        help=lib.args.help('--insecure'),
        dest='INSECURE',
        action='store_true',
        default=DEFAULT_INSECURE,
    )

    parser.add_argument(
        '--lookback',
        help='State changes are counted within this window. '
        + lib.args.help('--lookback')
        + ' Example: `--lookback=7200`. '
        'Default: %(default)s (seconds)',
        dest='LOOKBACK',
        type=int,
        default=DEFAULT_LOOKBACK,
    )

    parser.add_argument(
        '--no-proxy',
        help=lib.args.help('--no-proxy'),
        dest='NO_PROXY',
        action='store_true',
        default=DEFAULT_NO_PROXY,
    )

    parser.add_argument(
        '--password',
        help='Icinga Web 2 password. Takes precedence over the value in `--pwfile`.',
        dest='PASSWORD',
    )

    parser.add_argument(
        '--proxy',
        help=lib.args.help('--proxy'),
        dest='PROXY',
        default=None,
    )

    parser.add_argument(
        '--pwfile',
        help='Path to a password file containing "url", "user" and "password" for Icinga Web 2. '
        'Example: `--pwfile=/var/spool/icinga2/.icingaweb`. '
        'Default: %(default)s',
        dest='PWFILE',
        default=DEFAULT_PWFILE,
    )

    parser.add_argument(
        '--test',
        help=lib.args.help('--test'),
        dest='TEST',
        type=lib.args.csv,
    )

    parser.add_argument(
        '--timeout',
        help=lib.args.help('--timeout') + ' Default: %(default)s (seconds)',
        dest='TIMEOUT',
        type=int,
        default=DEFAULT_TIMEOUT,
    )

    parser.add_argument(
        '--url',
        help='Icinga DB event history URL including filter parameters. '
        'Takes precedence over the value in `--pwfile`. '
        'Example: `--url https://icinga/icingaweb2/icingadb/history?limit=250`.',
        dest='URL',
    )

    parser.add_argument(
        '--username',
        help='Icinga Web 2 username. Takes precedence over the value in `--pwfile`.',
        dest='USERNAME',
    )

    parser.add_argument(
        '-w',
        '--warning',
        help='WARN threshold for the number of state changes per service within the lookback period. '
        'Supports Nagios ranges. '
        'Default: %(default)s',
        dest='WARN',
        default=DEFAULT_WARN,
    )

    args, _ = parser.parse_known_args()
    return args


def get_data(args):
    """Login to Icinga, call the URL and return JSON data."""
    header = {}
    auth = f'{args.USERNAME}:{args.PASSWORD}'
    encoded_auth = lib.txt.to_text(base64.b64encode(lib.txt.to_bytes(auth)))
    header['Authorization'] = f'Basic {encoded_auth}'
    header['Accept'] = 'application/json'
    return lib.url.fetch_json(
        args.URL,
        header=header,
        insecure=args.INSECURE,
        no_proxy=args.NO_PROXY,
        proxy=args.PROXY,
        timeout=args.TIMEOUT,
    )


def get_downtimes(events):
    """Collect the service downtime periods from the `downtime_start` and
    `downtime_end` events of the history.

    Every downtime event carries the whole downtime, so a downtime is known even if
    only one of its two events is part of the fetched history. `start_time` and
    `end_time` are the effective period (counted from the trigger for a flexible
    downtime), a downtime that never triggered has no events at all, and a cancelled
    one ends at its `cancel_time`, also on its `downtime_start` event. Verified
    against Icinga 2 (`lib/icingadb/icingadb-objects.cpp`) and Icinga DB
    (`pkg/icingadb/v1/history/downtime.go`, `downtime_history` is upserted by
    `downtime_id`).

    Host downtimes are left out: Icinga 2 does not put a service into a downtime
    because its host is in one (`Checkable::IsInDowntime()` only looks at the
    service's own downtimes). A host downtime scheduled with its services creates
    a downtime for each of them anyway.

    Returns a dict mapping the service id to a set of `(start, end)` epoch tuples.
    """
    downtimes = {}
    for event in events:
        if event.get('event_type') not in ('downtime_end', 'downtime_start'):
            continue
        downtime = event.get('downtime') or {}
        if downtime.get('object_type') != 'service':
            continue
        end_time = downtime.get('end_time')
        if downtime.get('has_been_cancelled') and downtime.get('cancel_time'):
            end_time = downtime.get('cancel_time')
        try:
            period = (
                lib.time.timestr2epoch(downtime.get('start_time'), pattern='iso8601'),
                lib.time.timestr2epoch(end_time, pattern='iso8601'),
            )
        except (AttributeError, TypeError, ValueError):
            continue
        downtimes.setdefault(downtime.get('service_id'), set()).add(period)
    return downtimes


def in_downtime(event_time, service_id, downtimes):
    """Return True if `event_time` falls into a downtime of the service."""
    # [start, end), like `Downtime::IsInEffect()` in Icinga 2
    return any(
        start <= event_time < end for start, end in downtimes.get(service_id, set())
    )


def check_credentials(args, pwfile_read):
    """Exit UNKNOWN if the URL, the username or the password is still missing after
    the command line and the password file were read.
    """
    if args.USERNAME and args.PASSWORD and args.URL:
        return
    if not args.PWFILE:
        lib.base.cu(
            'The three parameters `--url`, `--username` and `--password` are required. '
            'Alternatively, you can provide a password file for Icinga Web 2 to read '
            'the "url", "user" or "password" from, using `--pwfile`.'
        )
    msg = 'The Icinga Web 2 URL, username or password is missing.'
    if not pwfile_read:
        msg += (
            f' `{args.PWFILE}` does not exist or is not readable by the account'
            ' running this plugin. Create it, or pass `--url`, `--username` and'
            ' `--password`.'
        )
    else:
        msg += (
            f' Add the missing ones to `{args.PWFILE}`, or pass them with `--url`,'
            ' `--username` and `--password`.'
        )
    lib.base.cu(msg)


def read_pwfile(args):
    """Fill in URL, username and password from the password file where the command
    line left them out. Returns True if the file could be read.

    Nothing of the file's content goes into a message, since `--pwfile` may point
    at any file: configparser quotes the offending lines in its exceptions, so only
    the line number is passed on. Interpolation is off, otherwise a `%` in a
    password raises an error that quotes the password.
    """
    if not args.PWFILE:
        return False
    config = configparser.ConfigParser(interpolation=None)
    try:
        # configparser skips a file it cannot open without saying so, which is why
        # the result is kept to tell a missing file from an incomplete one later
        if not config.read(args.PWFILE, encoding='utf-8'):
            return False
    except (configparser.Error, UnicodeDecodeError) as e:
        lineno = getattr(e, 'lineno', None)
        if lineno is None and getattr(e, 'errors', None):
            lineno = e.errors[0][0]
        msg = f'`{args.PWFILE}` is not a valid INI file'
        if lineno:
            msg += f', see line {lineno}'
        # no traceback, it would carry the quoted line along
        lib.base.cu(f'{msg}.', traceback=False)

    # the command line takes precedence
    for section in config.sections():
        for key, value in config.items(section):
            if args.USERNAME is None and key.lower().startswith('user'):
                args.USERNAME = value
            if args.PASSWORD is None and key.lower() == 'password':
                args.PASSWORD = value
            if args.URL is None and key.lower() == 'url':
                args.URL = value
    return True


def main():
    """The main function. This is where the magic happens."""

    # parse the command line
    try:
        args = parse_args()
    except SystemExit:
        sys.exit(STATE_UNKNOWN)

    # fetch data
    pwfile_read = read_pwfile(args)
    if args.TEST is None:
        check_credentials(args, pwfile_read)
        events = lib.base.coe(get_data(args))
    else:
        # do not call the endpoint, put in test data
        events = lib.lftest.test_json(args.TEST)

    # init some vars
    msg = ''
    state = STATE_OK
    downtimes = get_downtimes(events)
    if args.TEST is None:
        now = lib.time.now()
    else:
        # a fixture does not age: its lookback ends at its newest event
        now = max(
            (
                lib.time.timestr2epoch(event['event_time'], pattern='iso8601')
                for event in events
                if event.get('event_time')
            ),
            default=0,
        )

    # create the db table
    definition = """
        host_id                 TEXT NOT NULL,
        host_display_name       TEXT NOT NULL,
        service_id              TEXT NOT NULL,
        service_display_name    TEXT NOT NULL,
        state_event_time        TEXT NOT NULL
    """
    # The database is private to this process: it holds nothing worth keeping
    # between runs, and a file under a fixed name is shared with every concurrent
    # run of this check, which then count each other's rows.
    conn = lib.base.coe(lib.db_sqlite.connect(in_memory=True))
    lib.base.coe(lib.db_sqlite.create_table(conn, definition))
    lib.base.coe(lib.db_sqlite.create_index(conn, 'host_id, service_id'))

    # analyze data
    for event in events:
        if event.get('event_type') != 'state_change':
            continue
        if event.get('object_type') != 'service':
            continue

        data = {}
        data['state_event_time'] = (event.get('state') or {}).get('event_time')
        if not data['state_event_time']:
            continue
        # Icinga DB Web returns every timestamp in UTC: 2025-02-26T13:45:15.207+00:00
        data['state_event_time'] = lib.time.timestr2epoch(
            data['state_event_time'], pattern='iso8601'
        )
        # ignore if event is outside lookback interval
        if abs(now - data['state_event_time']) > args.LOOKBACK:
            continue

        service = event.get('service') or {}
        host = event.get('host') or {}
        data['service_display_name'] = service.get('display_name')
        # ignore events with "Waiting for Icinga DB to synchronize the config."
        if data['service_display_name'] is None:
            continue

        # ignore myself
        if (
            'top' in data['service_display_name'].lower()
            and 'flap' in data['service_display_name'].lower()
        ):
            continue

        data['host_id'] = host.get('id')
        data['host_display_name'] = host.get('display_name')
        data['service_id'] = service.get('id')

        # A state change during a downtime is an expected one. `service.state` cannot
        # tell, it is the current state of the service, not the one at `event_time`.
        if in_downtime(data['state_event_time'], data['service_id'], downtimes):
            continue

        lib.base.coe(lib.db_sqlite.insert(conn, data))
    lib.base.coe(lib.db_sqlite.commit(conn))

    # from here on just working on the database
    data = lib.base.coe(
        lib.db_sqlite.select(
            conn,
            """
        select host_display_name, service_display_name, count(*) as cnt
        from perfdata
        where 1
        group by host_id, service_id
        order by cnt desc
        """,
        )
    )
    lib.db_sqlite.close(conn)

    for i, item in enumerate(data):
        data[i]['state'] = lib.base.get_state(
            item['cnt'],
            args.WARN,
            args.CRIT,
            _operator='range',
        )
        data[i]['state_hr'] = lib.base.state2str(data[i]['state'], empty_ok=False)
        state = lib.base.get_worst(state, data[i]['state'])

    # build the message
    msg = {
        STATE_CRIT: 'There are critical errors.',
        STATE_WARN: 'There are warnings.',
    }.get(state, 'Everything is ok.')
    msg += f' (lookback={lib.human.seconds2human(args.LOOKBACK)} warn={args.WARN} crit={args.CRIT})'
    if data:
        msg += '\n\n' + lib.base.get_table(
            data,
            [
                'host_display_name',
                'service_display_name',
                'cnt',
                'state_hr',
            ],
            header=[
                'Host',
                'Service',
                'Cnt',
                'State',
            ],
        )

    # over and out
    lib.base.oao(msg, state, always_ok=args.ALWAYS_OK)


if __name__ == '__main__':
    try:
        main()
    except Exception:
        lib.base.cu()
