#!/usr/lib64/linuxfabrik-monitoring-plugins/venv/bin/python
# -*- coding: utf-8; py-indent-offset: 4 -*-
#
# Author:  Linuxfabrik GmbH, Zurich, Switzerland
# Contact: info (at) linuxfabrik (dot) ch
#          https://www.linuxfabrik.ch/
# License: The Unlicense, see LICENSE file.

# https://github.com/Linuxfabrik/monitoring-plugins/blob/main/CONTRIBUTING.md

"""See the check's README for more details."""

import argparse
import datetime
import os
import re
import sys

import lib.args
import lib.base
import lib.db_sqlite
import lib.disk
import lib.human
import lib.lftest
import lib.logmatch
import lib.logsource
import lib.time
import lib.txt
from lib.globals import STATE_CRIT, STATE_OK, STATE_UNKNOWN, STATE_WARN

__author__ = 'Linuxfabrik GmbH, Zurich/Switzerland'
__version__ = '2026102401'

DESCRIPTION = """Scans the PHP-FPM error log for the events an administrator has to act
on: rejected configurations, worker crashes, workers killed with SIGKILL, requests that
ran into `request_terminate_timeout` or `request_slowlog_timeout`, workers that hang after
the SIGTERM such a timeout sent, workers that waited on a connection the web server never
sent a request over, pools that hit `pm.max_children`, PHP fatal errors, and the
emergency reload PHP-FPM performs after repeated worker failures. Startups, reloads and
shutdowns are counted alongside them, so a pool that keeps restarting is visible.
Alerts when one of those events shows up, when the lines a slow site provokes cross
the rates the thresholds set, and when a line arrives at a level `--critical-level`
or `--warning-level` covers.
Requests that ran long, a pool spawning workers in bursts and PHP fatal errors are
counted within `--lookback` and judged by how many of them arrived, not by the fact
that they did: one slow page is nobody's night, dozens within ten minutes say the pool
is mis-sized. Those
lines are counted there and nowhere else, so a site with one heavy report page does not
leave the check permanently yellow.
What is left is counted by the level PHP-FPM writes at the head of the line: `ALERT` and
`ERROR` return CRITICAL and `WARNING` returns WARNING, which `--critical-level` and
`--warning-level` move. The events named above carry their own state and are counted there
and nowhere else, because the level says nothing about what happened. A
message that merely contains the word "error" never counts.
The log is read either from a file, from a systemd unit (`systemd:`) or from a
container (`docker:`/`podman:`/`kubectl:`). `--server-log` may be given several times,
and everything named is then read as one window. Without it the file path is taken from
the `error_log` directive of the PHP-FPM configuration, with the common locations of the
distributions probed when that yields nothing, and the journal of the PHP-FPM unit is read
along with it, because a master that fails to start writes why to its standard error and
never reaches the error log. What both hold is counted once. The most recent
rotated file is read along with the live one, so the window does not end where
logrotate last ran. PHP's own messages go to the `error_log` file each pool names, which
is read along with it; only its fatal errors are counted there.
Requires root or sudo."""

# What `--critical-level` and `--warning-level` accept on top of the levels
# themselves: no level at all raises that state, which leaves the named events as
# the only way to reach it.
LEVEL_NONE = 'none'

DEFAULT_CRITICAL_LEVEL = 'ERROR'
DEFAULT_FATAL_ERRORS_CRITICAL = 100
DEFAULT_FATAL_ERRORS_WARNING = 10
DEFAULT_ICINGA_CALLBACK = False
DEFAULT_INSECURE = True
DEFAULT_LOOKBACK = 600  # seconds
DEFAULT_NO_MATCH_SEVERITY = 'ok'
DEFAULT_NO_PROXY = False
DEFAULT_REQUEST_TIMEOUTS_CRITICAL = 50
DEFAULT_REQUEST_TIMEOUTS_WARNING = 5
DEFAULT_SLOW_REQUESTS_CRITICAL = 200
DEFAULT_SLOW_REQUESTS_WARNING = 20
DEFAULT_SPAWN_PRESSURE_CRITICAL = 100
DEFAULT_SPAWN_PRESSURE_WARNING = 10
DEFAULT_TIMEOUT = 8
DEFAULT_WARNING_LEVEL = 'WARNING'

ACK_RETENTION_DAYS = 30
MAXLINES = 30000  # maximum log lines to consider from the end of the source
# Rotated predecessors of the log file to read along with it. The distributions
# rotate this log daily, so without one the window would end at the last
# rotation and an event from before it would stop being reported the moment
# logrotate runs. One covers a daily rotation with room to spare.
ROTATED_FILES = 1

# The unit the distributions run PHP-FPM as. Only the fixed locations systemd
# reads unit files from are looked at, and only for the names the distributions
# use, so nothing a log or a configuration file holds can point the check at
# another unit.
UNIT_CANDIDATES = ('php-fpm.service', 'php*-fpm.service')
UNIT_DIRECTORIES = (
    '/etc/systemd/system',
    '/usr/lib/systemd/system',
    '/lib/systemd/system',
)

# How long a path may be in the summary before it is abbreviated for display.
# The paths the distributions use fit within it untouched; a host serving a
# dozen sites names a dozen logs in one directory, and there the directory is
# the part that says nothing.
SOURCE_PATH_MAX_LEN = 32

# PHP-FPM configuration files that carry the global `error_log` directive. The
# directive is global-only, so the pool files in `php-fpm.d/` never override it
# and are not read. The Debian family installs one tree per PHP version, hence
# the glob; the last match wins, which is the highest version installed.
CONFIG_FILES = (
    '/etc/php-fpm.conf',
    '/etc/php/*/fpm/php-fpm.conf',
    '/usr/local/etc/php-fpm.conf',
)
ERROR_LOG_REGEX = re.compile(r'^\s*error_log\s*=\s*(\S+)', re.MULTILINE)

# The pool files, where every pool names the file PHP writes its own errors to
# (`php_admin_value[error_log]`, which the RHEL family sets by default). That
# file is where a PHP fatal error lands; the PHP-FPM error log above never sees
# it. `$pool` in the value is replaced by the name of the pool, as PHP-FPM does
# (`fpm_conf_expand_pool_name()` in php-src).
POOL_CONFIG_FILES = (
    '/etc/php-fpm.d/*.conf',
    '/etc/php/*/fpm/pool.d/*.conf',
    '/usr/local/etc/php-fpm.d/*.conf',
)
POOL_SECTION_REGEX = re.compile(r'^\s*\[([^\]]+)\]')
POOL_ERROR_LOG_REGEX = re.compile(r'^\s*php(?:_admin)?_value\[error_log\]\s*=\s*(\S+)')

# What PHP-FPM accepts instead of a path, matched case-insensitively the way
# PHP-FPM itself compares it.
SYSLOG_TARGET = 'syslog'

# Where the distributions keep the error log when the configuration does not say.
# The last entry is the path a source build compiles in (`log/php-fpm.log`
# relative to the local state directory).
LOG_FILE_CANDIDATES = (
    '/var/log/php-fpm/error.log',
    '/var/log/php-fpm.log',
    '/var/log/php*-fpm.log',
    '/usr/local/var/log/php-fpm.log',
)

# The master writes `[28-Aug-2026 15:08:23] NOTICE: message`. A worker that logs
# without going through the master writes the same line without the timestamp,
# because PHP-FPM prints the time only in the master process. Both shapes carry
# the level, so both are recognized (verified against php-fpm 8.0.30 on Rocky 9).
# What the journal calls PHP-FPM, for telling its own lines from the ones
# something else logged into the same place. The Debian family carries the
# version in the name (`php8.2-fpm`), the RHEL family does not.
IDENTIFIER_REGEX = re.compile(r'^php[\d.]*-?fpm$', re.IGNORECASE)

LEVEL_REGEX = re.compile(
    r'\b(ALERT|ERROR|WARNING|NOTICE|DEBUG):\s*(.*)$',
)

# The timestamp PHP-FPM writes in front of a master line, `[28-Aug-2026
# 15:21:20]`, in the local time of the host and in English regardless of the
# locale, because PHP-FPM formats it in the C locale. Python parses it in the C
# locale as well, as nothing here calls `setlocale()`. A worker line and a line
# a transport prefixed carry none; `lib.logsource.timestamp()` falls back to the
# transport's.
TIMESTAMP_REGEX = re.compile(r'\[(\d{2}-\w{3}-\d{4} \d{2}:\d{2}:\d{2})\]')
TIMESTAMP_FORMAT = '%d-%b-%Y %H:%M:%S'

# What PHP itself writes into the `error_log` of a pool: `[05-Oct-2026 04:03:05
# Europe/Zurich] PHP Fatal error:  Uncaught ...`, and the same prefix in front of
# what an application hands to `error_log()`. The time is in the zone of
# `date.timezone`, which PHP names behind it (`php_log_err_with_severity()`,
# format `d-M-Y H:i:s e`), so it is converted rather than read as local time.
# A message that spans several lines - the stack trace of an uncaught exception -
# continues on lines that carry no prefix at all.
PHP_LINE_REGEX = re.compile(r'^\[(\d{2}-\w{3}-\d{4} \d{2}:\d{2}:\d{2}) ([^\]\s]+)\] ')
# The error types PHP calls fatal (`php_error_cb()`): `Fatal error` covers
# E_ERROR, E_CORE_ERROR, E_COMPILE_ERROR and E_USER_ERROR, and an uncaught
# exception arrives as one too.
PHP_FATAL_REGEX = re.compile(
    r'\] PHP (?:Fatal error|Parse error|Recoverable fatal error):'
)

# A request timeout names the pool and the worker, and so does the exit of that
# worker. A worker that is reported as timed out a second time without having
# exited in between survived the SIGTERM `request_terminate_timeout` sent it.
TIMEOUT_WORKER_REGEX = re.compile(
    r'\[pool ([^\]]+)\] child (\d+), script .* execution timed out'
)
EXIT_WORKER_REGEX = re.compile(r'\[pool ([^\]]+)\] child (\d+) exited')

# The levels that say something is wrong, worst first. Which of them raise which
# state is left to `--critical-level` and `--warning-level`: a pool that logs one
# `WARNING` about a single slow request is not a reason to get anybody out of
# bed, and what genuinely is arrives as one of the named events below.
LEVELS = (
    {'level': 'ALERT', 'perfdata': 'php_fpm_alert_lines'},
    {'level': 'ERROR', 'perfdata': 'php_fpm_error_lines'},
    {'level': 'WARNING', 'perfdata': 'php_fpm_warning_lines'},
)
LEVEL_CHOICES = [item['level'] for item in LEVELS] + [LEVEL_NONE]

# The events worth naming on their own, because the level alone does not say what
# happened. Everything else PHP-FPM logs is covered by the per-level counts. The
# message texts are taken from php-src `sapi/fpm/fpm/` and reproduced on
# php-fpm 8.0.30; a pattern therefore matches the whole message, not a keyword.
EVENTS = (
    {
        'key': 'worker_crashes',
        'label': 'worker crash',
        'suffix': 'es',
        'perfdata': 'php_fpm_worker_crashes',
        # `SIGQUIT` is how PHP-FPM ends a worker gracefully and `SIGTERM` is how
        # it terminates one that ran over `request_terminate_timeout` (which logs
        # a line of its own), so both are ordinary. `SIGKILL` is counted as a
        # killed worker below, because PHP-FPM sends it itself. Every other
        # signal is one PHP-FPM never sends its workers, so the worker died on a
        # fault. The lookahead also lets the line through when PHP-FPM has no
        # name for the signal and prints an empty pair of brackets.
        'regex': re.compile(r'exited on signal \d+ \((?!SIGKILL|SIGQUIT|SIGTERM)'),
        'state': STATE_WARN,
        'recommendation': (
            'Workers died on a fault; look for a core dump or a faulty PHP extension'
        ),
    },
    {
        'key': 'worker_kills',
        'label': 'killed worker',
        'perfdata': 'php_fpm_worker_kills',
        # PHP-FPM sends `SIGKILL` itself to an idle worker that is still alive
        # one maintenance tick (a second) after the `SIGQUIT` that was to shrink
        # the pool (`fpm_pctl_kill_idle_child()` in php-src): the worker did not
        # end when told to, which is what a host short of memory looks like. The
        # `SIGKILL` that ends a reload or stop never shows up here, because
        # `fpm_children_bury()` logs every exit at `DEBUG` while the master is not
        # in its normal state. The kernel's OOM killer sends the same signal, and
        # the log line cannot tell the two apart (verified against php-8.3.31).
        'regex': re.compile(r'exited on signal 9 \(SIGKILL'),
        'state': STATE_WARN,
        'recommendation': (
            'Workers were killed with SIGKILL, either by PHP-FPM because they did'
            ' not end when the pool shrank or by the kernel when memory ran out;'
            ' look for `Out of memory` in the kernel log around the timestamps'
            ' above'
        ),
    },
    {
        'key': 'idle_connection_timeouts',
        'label': 'idle connection timeout',
        'perfdata': 'php_fpm_idle_connection_timeouts',
        # A worker that kept a connection open for the web server to reuse
        # (FastCGI `FCGI_KEEP_CONN`) does not go back to `accept()`; it marks
        # itself as reading headers and waits on that connection for the next
        # request (`fcgi_accept_request()` in php-src). If none arrives,
        # `request_terminate_timeout` ends it like a request that ran too long,
        # with no script and no request line. A real request always names its
        # script, so this is never load. Apache's `mod_proxy_fcgi` with
        # `enablereuse=on` produced them for weeks (php-fpm 8.3 and httpd 2.4.37
        # on Rocky 8): one every 12 minutes on a quiet host, below any rate
        # threshold, and up to 282 within ten minutes on a busy one. A pinned
        # worker is not idle to PHP-FPM either (`fpm_request_is_idle()`), so the
        # pool spawns more of them, runs into `pm.max_children`, and measures
        # the wait into the slowlog and the access log durations. The slowlog
        # itself never names the idle connection, because PHP-FPM only traces a
        # worker that is executing.
        'regex': re.compile(r'script \'\' \(request: " "\) execution timed out'),
        'state': STATE_WARN,
        'recommendation': (
            'Workers waited on a FastCGI connection the web server keeps open for'
            ' reuse (`enablereuse=on` in Apache `mod_proxy_fcgi`) until'
            ' `request_terminate_timeout` ended them; every such connection pins'
            ' a worker, which inflates busy workers, slow requests and timeouts and'
            ' can fill the pool, so remove `enablereuse=on` for this backend'
        ),
    },
    {
        'key': 'hung_workers',
        'label': 'hung worker',
        'perfdata': 'php_fpm_hung_workers',
        # PHP-FPM checks for timed-out requests on every heartbeat (a third of
        # the smallest timeout, `fpm_conf.c`) and sends SIGTERM again each time
        # (`fpm_request_check_timed_out()`). A worker keeps the default action
        # for SIGTERM (`fpm_signals_init_child()`), so the second report about
        # the same worker means the kernel could not deliver it: the process is
        # in uninterruptible sleep, which is what a hung NFS or storage mount
        # does to every process touching it. Seen with php-fpm 8.5 on Rocky 9
        # behind a hung NFS server: 50 workers reported every 21 minutes for 17
        # hours. Not matched by a pattern but by pairing lines, see main().
        'regex': None,
        'state': STATE_CRIT,
        'recommendation': (
            'Workers outlived the SIGTERM that `request_terminate_timeout` sent'
            ' and are stuck, almost always in uninterruptible I/O on a hung NFS'
            ' or storage mount; look for PHP-FPM workers in state `D` in'
            ' `ps -eo pid,stat,wchan:32,args` and for `nfs: server ... not'
            ' responding` in the kernel log'
        ),
    },
    {
        'key': 'pool_saturations',
        'label': 'pool saturation',
        'perfdata': 'php_fpm_pool_saturations',
        'regex': re.compile(
            r'server reached (?:pm\.)?max_children setting'
            r'|the maximum number of processes has been reached',
            re.IGNORECASE,
        ),
        # A pool that ran full once and recovered is a queue of a few seconds:
        # an editor's backend page firing a dozen parallel requests at a pool
        # of five does it. PHP-FPM logs the line once and only again after the
        # pool has spawned a worker in between (`warn_max_children` in
        # `fpm_process_ctl.c`), so a second one within `--lookback`, or one
        # next to requests that timed out or workers that hang, is a pool that
        # stays full, and that is CRITICAL (see main()).
        'state': STATE_WARN,
        # Raising the cap is the obvious lever and the wrong one to reach for
        # first. Workers that hang or wait on a stalled backend keep the pool
        # full however large it is, and a crawler or a scanner walking a few
        # hundred 404s fills it exactly the way real traffic does. A higher cap
        # then multiplies the memory the pool can claim and hands the next
        # burst more of it.
        'recommendation': (
            'A pool ran out of workers and clients waited in the listen queue;'
            ' before raising `pm.max_children` (or `process.max`), check whether'
            ' workers were stuck (request timeouts or hung workers above, a'
            ' stalled database or NFS mount) or a crawler drove the load (web'
            ' server access log at that time), because a higher cap only lets'
            ' either claim more memory'
        ),
    },
    {
        'key': 'emergency_restarts',
        'label': 'emergency restart',
        'perfdata': 'php_fpm_emergency_restarts',
        'regex': re.compile(
            r'failed processes threshold \(\d+ in \d+ sec\) is reached,'
            r' initiating reload',
        ),
        'state': STATE_CRIT,
        'recommendation': (
            'PHP-FPM reloaded itself because too many workers failed in a row;'
            ' treat this as a crashing application until proven otherwise'
        ),
    },
)

# What PHP-FPM writes once per request or once per maintenance tick rather than
# once per problem. One of these is a traffic peak or a single slow page and no
# reason to get anybody out of bed; dozens of them within ten minutes say the
# pool is mis-sized. They are therefore counted against a rate over `--lookback`,
# and they are kept out of the per-level counts above, so a site with one slow
# report page does not leave the check permanently yellow.
RATE_EVENTS = (
    {
        'key': 'request_timeouts',
        'label': 'request timeout',
        'perfdata': 'php_fpm_request_timeouts',
        # PHP-FPM kills the worker and the client gets nothing, so the threshold
        # is the lowest of the three: a handful of these is already a bad
        # afternoon for somebody.
        # A timeout without a script is no request at all and is named as an
        # idle connection timeout above, so the lookbehind leaves it out here.
        'regex': re.compile(
            r'(?<!script \'\' \(request: " "\) )'
            r'execution timed out \([\d.]+ sec\), terminating'
        ),
        'critical': 'REQUEST_TIMEOUTS_CRITICAL',
        'warning': 'REQUEST_TIMEOUTS_WARNING',
        'recommendation': (
            'Requests were killed after `request_terminate_timeout`; when many'
            ' different scripts timed out at once a backend (database, NFS,'
            ' remote API) stalled, otherwise profile the scripts listed above'
        ),
    },
    {
        'key': 'slow_requests',
        'label': 'slow request',
        'perfdata': 'php_fpm_slow_requests',
        # The request still finishes; PHP-FPM only writes a backtrace. A site
        # with one heavy report page produces these all day.
        'regex': re.compile(r'executing too slow \([\d.]+ sec\), logging'),
        'critical': 'SLOW_REQUESTS_CRITICAL',
        'warning': 'SLOW_REQUESTS_WARNING',
        'recommendation': (
            'Requests ran longer than `request_slowlog_timeout`; the backtrace of'
            ' each one is in the pool `slowlog`'
        ),
    },
    {
        'key': 'spawn_pressure',
        'label': 'spawn pressure warning',
        'perfdata': 'php_fpm_spawn_pressure',
        # Only `pm = dynamic` writes this, and only once the spawn rate has
        # doubled its way to 8, which takes several seconds of sustained
        # pressure. Unlike `server reached pm.max_children`, which PHP-FPM guards
        # with a flag and logs once, this one has no guard at all: it is written
        # on every maintenance tick, so roughly once a second, for as long as the
        # pressure lasts. A traffic peak therefore leaves a burst of them behind
        # and then silence, which is a rate and not an event (verified against
        # `fpm_pctl_perform_idle_server_maintenance()` in php-src).
        'regex': re.compile(r'seems busy \(you may need to increase'),
        'critical': 'SPAWN_PRESSURE_CRITICAL',
        'warning': 'SPAWN_PRESSURE_WARNING',
        'recommendation': (
            'A pool had to spawn workers in bursts; raise `pm.start_servers` and'
            ' `pm.min_spare_servers` so requests do not wait for a fork'
        ),
    },
    {
        'key': 'fatal_errors',
        'label': 'PHP fatal error',
        'perfdata': 'php_fpm_fatal_errors',
        # From the `error_log` files of the pools, not from PHP-FPM. One fatal
        # error is one broken page for one visitor and happens on every site
        # now and then; dozens within minutes are a deployment that went wrong
        # or a database that went away. Warnings, notices and deprecations
        # are left out: a busy application writes thousands of them a day.
        'regex': PHP_FATAL_REGEX,
        'critical': 'FATAL_ERRORS_CRITICAL',
        'warning': 'FATAL_ERRORS_WARNING',
        'recommendation': (
            'PHP scripts died with fatal errors, so pages failed for their'
            ' visitors; many of them at once follow a deployment or a backend'
            ' that went away, otherwise fix the code the lines above name'
        ),
    },
)

# A worker PHP-FPM ended with `SIGTERM` after `request_terminate_timeout`, which
# the `execution timed out` line just before it already counts. PHP-FPM writes
# every exit on a signal other than `SIGQUIT` at `WARNING` (`fpm_children_bury()`
# in php-src), so counting this line by its level as well turned every timeout
# into a second warning that no rate threshold holds back (seen with php-fpm 8.3
# on Rocky 8, one exit line per timeout line). The `SIGTERM` of a stop or reload
# does not get here: while the master is not in its normal state the same
# function logs the exit at `DEBUG`. What is left is somebody else's `kill`.
TERMINATED_EXIT_REGEX = re.compile(r'exited on signal 15 \(SIGTERM\)')

# PHP-FPM writes this once into the fresh file when logrotate has moved the old
# one away and signalled it with SIGUSR1, so the window of a just-rotated log
# holds exactly this line. Counted, and named next to the window size, which
# turns "no startup, no reload, no shutdown" from a puzzle into a sentence.
ROTATION_REGEX = re.compile(r'^error log file re-opened')

# Lifecycle markers, counted but never alerting on. A reload re-executes the
# master, so it shows up as a reload and as a startup. The reload is recognized
# by the line the exec itself writes, because the `Reloading in progress ...`
# that a `SIGUSR2` triggers is missing when PHP-FPM reloads itself after too many
# worker failures.
LIFECYCLE = (
    {
        'key': 'startups',
        'label': 'startup',
        'perfdata': 'php_fpm_startups',
        'regex': re.compile(r'^fpm is running, pid \d+'),
        'verb': 'detected',
    },
    {
        'key': 'reloads',
        'label': 'reload',
        'perfdata': 'php_fpm_reloads',
        'regex': re.compile(r'^reloading: execvp\('),
        'verb': 'detected',
    },
    {
        'key': 'shutdowns',
        'label': 'shutdown',
        'perfdata': 'php_fpm_shutdowns',
        'regex': re.compile(r'^exiting, bye-bye!'),
        'verb': 'detected',
    },
)


def parse_args():
    """Parse command line arguments using argparse."""
    parser = argparse.ArgumentParser(
        description=DESCRIPTION,
        epilog=lib.args.epilog(__file__),
        formatter_class=lib.args.HelpFormatter,
    )

    parser.add_argument(
        '-V',
        '--version',
        action='version',
        version=f'%(prog)s: v{__version__} by {__author__}',
    )

    parser.add_argument(
        '--always-ok',
        help=lib.args.help('--always-ok'),
        dest='ALWAYS_OK',
        action='store_true',
        default=False,
    )

    # Developer-only hook, like `--test` on the plugins that inject command
    # output: it prefixes every path the check opens, so the fixture tests can
    # stand a directory tree in for the host's file system. Deliberately not
    # named `--test-...`, because argparse would then resolve a bare `--test`
    # into it.
    parser.add_argument(
        '--config-root',
        help=argparse.SUPPRESS,
        dest='CONFIG_ROOT',
        default='',
    )

    parser.add_argument(
        '--critical-level',
        help='Least severe PHP-FPM log level that returns CRITICAL. '
        'Each level includes everything more severe than itself, so `WARNING` '
        'covers `ERROR` and `ALERT` as well. '
        'Case-sensitive. '
        '`none` lets no level return CRITICAL, which leaves the events this '
        'check names as the only way to reach it. '
        'Example: `--critical-level=WARNING`. '
        'Default: %(default)s',
        dest='CRITICAL_LEVEL',
        choices=LEVEL_CHOICES,
        default=DEFAULT_CRITICAL_LEVEL,
    )

    parser.add_argument(
        '--fatal-errors-critical',
        help='Number of PHP fatal errors in the `error_log` of the pools within '
        '`--lookback` that returns CRITICAL. '
        '0 turns the threshold off. '
        'Example: `--fatal-errors-critical=50`. '
        'Default: %(default)s',
        dest='FATAL_ERRORS_CRITICAL',
        type=int,
        default=DEFAULT_FATAL_ERRORS_CRITICAL,
    )

    parser.add_argument(
        '--fatal-errors-warning',
        help='Number of PHP fatal errors in the `error_log` of the pools within '
        '`--lookback` that returns WARNING. '
        '0 turns the threshold off. '
        'Example: `--fatal-errors-warning=1`. '
        'Default: %(default)s',
        dest='FATAL_ERRORS_WARNING',
        type=int,
        default=DEFAULT_FATAL_ERRORS_WARNING,
    )

    parser.add_argument(
        '--icinga-callback',
        help=lib.args.help('--icinga-callback'),
        dest='ICINGA_CALLBACK',
        action='store_true',
        default=DEFAULT_ICINGA_CALLBACK,
    )

    parser.add_argument(
        '--icinga-password',
        help=lib.args.help('--icinga-password'),
        dest='ICINGA_PASSWORD',
    )

    parser.add_argument(
        '--icinga-service-name',
        help=lib.args.help('--icinga-service-name'),
        dest='ICINGA_SERVICE_NAME',
    )

    parser.add_argument(
        '--icinga-url',
        help=lib.args.help('--icinga-url'),
        dest='ICINGA_URL',
    )

    parser.add_argument(
        '--icinga-username',
        help=lib.args.help('--icinga-username'),
        dest='ICINGA_USERNAME',
    )

    parser.add_argument(
        '--ignore',
        help='Ignore a log line matching this Python regular expression. '
        'The log line is lowercased before matching, so write the pattern in '
        'lowercase (or use the `(?i)` flag). '
        'Can be specified multiple times. '
        "Example: `--ignore='(?i)linuxfabrik'`.",
        action='append',
        default=None,
        dest='IGNORE',
    )

    parser.add_argument(
        '--insecure',
        help='Applies to the connection to the monitoring server that `--icinga-callback` makes, which is the only network connection this check opens. '
        + lib.args.help('--insecure'),
        dest='INSECURE',
        action='store_true',
        default=DEFAULT_INSECURE,
    )

    parser.add_argument(
        '--lookback',
        help='Request timeouts, slow requests and spawn pressure are counted '
        'within this window rather than reported one by one. '
        + lib.args.help('--lookback')
        + ' Example: `--lookback=3600`. '
        'Default: %(default)s (seconds)',
        dest='LOOKBACK',
        type=int,
        default=DEFAULT_LOOKBACK,
    )

    parser.add_argument(
        '--match',
        help='Only consider a log line matching this Python regular expression. '
        'The log line is lowercased before matching, so write the pattern in '
        'lowercase (or use the `(?i)` flag). '
        'Can be specified multiple times. '
        + lib.args.MATCH_IGNORE_PRECEDENCE
        + " Example: `--match='\\[pool www\\]'`.",
        action='append',
        default=None,
        dest='MATCH',
    )

    parser.add_argument(
        '--no-insecure',
        help='Applies to the connection to the monitoring server that `--icinga-callback` makes, which is the only network connection this check opens. '
        + lib.args.help('--no-insecure'),
        dest='INSECURE',
        action='store_false',
        default=DEFAULT_INSECURE,
    )

    parser.add_argument(
        '--no-match-severity',
        help=lib.args.help('--no-match-severity') + ' Default: %(default)s',
        dest='NO_MATCH_SEVERITY',
        choices=['ok', 'warn', 'crit', 'unknown'],
        default=DEFAULT_NO_MATCH_SEVERITY,
    )

    parser.add_argument(
        '--no-perfdata',
        help=lib.args.help('--no-perfdata'),
        dest='NO_PERFDATA',
        action='store_true',
        default=False,
    )

    parser.add_argument(
        '--no-proxy',
        help='Applies to the connection to the monitoring server that `--icinga-callback` makes, which is the only network connection this check opens. '
        + lib.args.help('--no-proxy'),
        dest='NO_PROXY',
        action='store_true',
        default=DEFAULT_NO_PROXY,
    )

    parser.add_argument(
        '--proxy',
        help='Applies to the connection to the monitoring server that `--icinga-callback` makes, which is the only network connection this check opens. '
        + lib.args.help('--proxy'),
        dest='PROXY',
        default=None,
    )

    parser.add_argument(
        '--request-timeouts-critical',
        help='Number of terminated requests within `--lookback` that returns '
        'CRITICAL. '
        '0 turns the threshold off. '
        'Example: `--request-timeouts-critical=20`. '
        'Default: %(default)s',
        dest='REQUEST_TIMEOUTS_CRITICAL',
        type=int,
        default=DEFAULT_REQUEST_TIMEOUTS_CRITICAL,
    )

    parser.add_argument(
        '--request-timeouts-warning',
        help='Number of terminated requests within `--lookback` that returns '
        'WARNING. '
        '0 turns the threshold off. '
        'Example: `--request-timeouts-warning=1`. '
        'Default: %(default)s',
        dest='REQUEST_TIMEOUTS_WARNING',
        type=int,
        default=DEFAULT_REQUEST_TIMEOUTS_WARNING,
    )

    parser.add_argument(
        '--server-log',
        help='Log source to read from. '
        'Accepts a file path, `docker:CONTAINER`, `podman:CONTAINER`, '
        '`kubectl:CONTAINER` or `systemd:UNITNAME`. '
        'Can be specified multiple times, and everything named is then read as '
        'one window; a source named twice is read once. '
        'If omitted, the check reads the `error_log` directive of the PHP-FPM '
        'configuration, falls back to the common locations of the '
        'distributions, and reads the journal of the PHP-FPM unit along with it; '
        'what the two share is counted once. It also reads the `error_log` file '
        'every pool names for the messages of PHP itself. '
        'Example: `--server-log=systemd:php-fpm.service`.',
        action='append',
        default=None,
        dest='SERVER_LOG',
    )

    parser.add_argument(
        '--slow-requests-critical',
        help='Number of slow requests within `--lookback` that returns CRITICAL. '
        '0 turns the threshold off. '
        'Example: `--slow-requests-critical=100`. '
        'Default: %(default)s',
        dest='SLOW_REQUESTS_CRITICAL',
        type=int,
        default=DEFAULT_SLOW_REQUESTS_CRITICAL,
    )

    parser.add_argument(
        '--slow-requests-warning',
        help='Number of slow requests within `--lookback` that returns WARNING. '
        '0 turns the threshold off. '
        'Example: `--slow-requests-warning=5`. '
        'Default: %(default)s',
        dest='SLOW_REQUESTS_WARNING',
        type=int,
        default=DEFAULT_SLOW_REQUESTS_WARNING,
    )

    parser.add_argument(
        '--spawn-pressure-critical',
        help='Number of spawn pressure warnings within `--lookback` that returns '
        'CRITICAL. '
        '0 turns the threshold off. '
        'Example: `--spawn-pressure-critical=50`. '
        'Default: %(default)s',
        dest='SPAWN_PRESSURE_CRITICAL',
        type=int,
        default=DEFAULT_SPAWN_PRESSURE_CRITICAL,
    )

    parser.add_argument(
        '--spawn-pressure-warning',
        help='Number of spawn pressure warnings within `--lookback` that returns '
        'WARNING. '
        'A pool logs one of these per second while it is spawning in bursts, so '
        'this counts how many arrived rather than that any did. '
        '0 turns the threshold off. '
        'Example: `--spawn-pressure-warning=30`. '
        'Default: %(default)s',
        dest='SPAWN_PRESSURE_WARNING',
        type=int,
        default=DEFAULT_SPAWN_PRESSURE_WARNING,
    )

    parser.add_argument(
        '--timeout',
        help=lib.args.help('--timeout') + ' Default: %(default)s (seconds)',
        dest='TIMEOUT',
        type=int,
        default=DEFAULT_TIMEOUT,
    )

    parser.add_argument(
        '--warning-level',
        help='Least severe PHP-FPM log level that returns WARNING. '
        'Each level includes everything more severe than itself, and a level '
        'that `--critical-level` already covers returns CRITICAL instead. '
        'Case-sensitive. '
        '`none` lets no level return WARNING, which leaves the events this '
        'check names as the only way to reach it. '
        'Example: `--warning-level=none`. '
        'Default: %(default)s',
        dest='WARNING_LEVEL',
        choices=LEVEL_CHOICES,
        default=DEFAULT_WARNING_LEVEL,
    )

    args, _ = parser.parse_known_args()
    return args


def get_configured_log_file(config_root=''):
    """Return the `error_log` the PHP-FPM configuration names, or None.

    The value `syslog` is handed back lowercased, because that is a value PHP-FPM
    understands (case-insensitively) and the caller has to report rather than
    open. Only the fixed configuration locations of the distributions are read,
    so nothing the log itself contains can steer the check to another file.
    """
    error_log = None
    for candidate in CONFIG_FILES:
        for filename in lib.disk.glob(f'{config_root}{candidate}'):
            success, content = lib.disk.read_file(filename)
            if not success or not content:
                continue
            match = ERROR_LOG_REGEX.search(content)
            if match:
                error_log = match.group(1)
    if error_log is None:
        return None
    if error_log.lower() == SYSLOG_TARGET:
        return SYSLOG_TARGET
    if not error_log.startswith('/'):
        # A relative path is resolved against a prefix that is compiled into
        # PHP-FPM and not readable from here, so probing the known locations
        # gets further than guessing that prefix.
        return None
    return f'{config_root}{error_log}'


def get_log_file_real_path(config_root=''):
    """Probe the locations the distributions keep the PHP-FPM error log in."""
    for candidate in LOG_FILE_CANDIDATES:
        for filename in lib.disk.glob(f'{config_root}{candidate}'):
            if lib.disk.file_exists(filename, allow_empty=True):
                return filename
    return None


def get_pool_error_logs(allowed_roots, config_root=''):
    """Return the existing `error_log` files the PHP-FPM pools name for PHP itself.

    PHP creates such a file on its first message only, so a pool that has not
    logged anything yet has none, and that is not worth a warning. A path that
    is not absolute is skipped for the same reason as the global `error_log`.
    A path outside `allowed_roots` is skipped before it is looked at, so a pool
    file an unprivileged user may edit cannot learn through this check which
    files exist elsewhere on the host.
    """
    found = []
    for candidate in POOL_CONFIG_FILES:
        for filename in sorted(lib.disk.glob(f'{config_root}{candidate}')):
            success, content = lib.disk.read_file(filename)
            if not success or not content:
                continue
            pool = None
            for line in content.splitlines():
                section = POOL_SECTION_REGEX.match(line)
                if section:
                    pool = section.group(1)
                    continue
                match = POOL_ERROR_LOG_REGEX.match(line)
                if not match or pool is None:
                    continue
                path = match.group(1).replace('$pool', pool)
                if not path.startswith('/'):
                    continue
                path = f'{config_root}{path}'
                if path in found or not lib.disk.is_within(path, allowed_roots):
                    continue
                if lib.disk.file_exists(path, allow_empty=True):
                    found.append(path)
    return found


def parse_timestamp(log_line):
    """Return the moment PHP-FPM stamped a line with, or None when it carries none.

    Only PHP-FPM's own timestamp; `lib.logsource.timestamp()` falls back to the
    one the transport prefixed. A line without either is not an error: PHP-FPM
    prints the time only in the master process, so a line a worker wrote on its
    own carries none.
    """
    match = TIMESTAMP_REGEX.search(log_line)
    if match:
        try:
            return datetime.datetime.strptime(match.group(1), TIMESTAMP_FORMAT)
        except ValueError:
            return None
    match = PHP_LINE_REGEX.match(log_line)
    if not match:
        return None
    try:
        written_at = datetime.datetime.strptime(match.group(1), TIMESTAMP_FORMAT)
    except ValueError:
        return None
    zone = match.group(2)
    if zone[:1] in '+-':
        try:
            offset = datetime.datetime.strptime(zone.replace(':', ''), '%z').tzinfo
        except ValueError:
            return written_at
        written_at = written_at.replace(tzinfo=offset)
    else:
        written_at = written_at.replace(tzinfo=lib.time.get_timezone(zone))
    try:
        return lib.time.datetime2local(written_at)
    except ValueError:
        # a moment local time cannot hold, written by whoever can write the log
        return None


def get_level_states(critical_level, warning_level):
    """Map every counted level to the state it raises.

    A level raises a state when it is at least as severe as the level the
    parameter names, and CRITICAL wins wherever both would apply.
    """
    ranks = {item['level']: rank for rank, item in enumerate(LEVELS)}
    states = {}
    for level, rank in ranks.items():
        if critical_level != LEVEL_NONE and rank <= ranks[critical_level]:
            states[level] = STATE_CRIT
        elif warning_level != LEVEL_NONE and rank <= ranks[warning_level]:
            states[level] = STATE_WARN
        else:
            states[level] = STATE_OK
    return states


def get_units(config_root=''):
    """Return the PHP-FPM units this host has a unit file for.

    A unit file two names point at is one unit: the MariaDB package ships
    `mysqld.service` and `mysql.service` as symlinks to `mariadb.service`, and
    reading that journal three times would count every line three times.
    """
    # A unit file says only that a package shipped one: hosts with another init
    # system carry them too, and have no journal to read ([#1544]).
    if not lib.logsource.systemd_booted(config_root):
        return []
    found = []
    seen = set()
    for candidate in UNIT_CANDIDATES:
        for directory in UNIT_DIRECTORIES:
            for filename in lib.disk.glob(f'{config_root}{directory}/{candidate}'):
                if not lib.disk.file_exists(filename, allow_empty=True):
                    continue
                identity = os.path.realpath(filename)
                if identity in seen:
                    continue
                seen.add(identity)
                found.append(os.path.basename(filename))
    return found


def dedup_key(line):
    """Identify the event in a line, whichever transport delivered it.

    The error log and the journal of the unit are both read, and PHP-FPM writes
    its own timestamp into either, so what follows the syslog prefix the journal
    adds is the line the file holds.
    """
    written_at = lib.logsource.timestamp(line, parse_timestamp)
    return (
        written_at.replace(microsecond=0) if written_at else None,
        lib.logsource.strip_syslog_prefix(line),
    )


def main():
    """The main function. This is where the magic happens."""

    # parse the command line
    try:
        args = parse_args()
    except SystemExit:
        sys.exit(STATE_UNKNOWN)
    # The hidden --config-root points the unit tests at a fixture tree. This check
    # runs as root, so only a tree inside its own unit-test/ directory counts: any
    # other would let a caller learn which files exist on the host.
    if args.CONFIG_ROOT:
        args.CONFIG_ROOT = lib.base.coe(lib.lftest.fixture_root(args.CONFIG_ROOT))

    # set default values for append parameters that were not specified
    if args.IGNORE is None:
        args.IGNORE = []
    if args.MATCH is None:
        args.MATCH = []
    if args.SERVER_LOG is None:
        args.SERVER_LOG = []

    if args.ICINGA_CALLBACK and not all(
        (
            args.ICINGA_URL,
            args.ICINGA_PASSWORD,
            args.ICINGA_USERNAME,
            args.ICINGA_SERVICE_NAME,
        )
    ):
        lib.base.cu(
            '`--icinga-callback` requires `--icinga-url`, `--icinga-password`, '
            '`--icinga-username` and `--icinga-service-name`.'
        )

    # fetch data
    # The plugin runs as root via sudo, so restrict an on-disk log to the system
    # log directory. The path the PHP-FPM configuration reports is deliberately
    # not trusted beyond that: a pool file an unprivileged user may edit could
    # otherwise point the check at any file on the host. The plugin's own
    # unit-test/ directory is allowed so the fixture tests work; it does not
    # exist next to the flat, root-owned plugin in production. Bind-mount a
    # custom log directory under /var/log to include it (see the README).
    allowed_roots = [
        '/var/log',
        os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), 'unit-test'),
    ]
    server_logs = args.SERVER_LOG
    if not server_logs:
        # The unit alongside the file, not instead of it: a master that failed to
        # start wrote why to its standard error and never reached the error log,
        # and a pool logging to syslog is in the journal only. What both carry is
        # counted once.
        found_log = get_configured_log_file(args.CONFIG_ROOT) or get_log_file_real_path(
            args.CONFIG_ROOT
        )
        server_logs = [found_log] if found_log else []
        server_logs += get_pool_error_logs(allowed_roots, args.CONFIG_ROOT)
        server_logs += [f'systemd:{unit}' for unit in get_units(args.CONFIG_ROOT)]
    if not server_logs:
        lib.base.cu(
            'Found no PHP-FPM error log. Set `error_log` in the PHP-FPM '
            "configuration, or name the log with the check's `--server-log` "
            'parameter.'
        )
    if SYSLOG_TARGET in server_logs:
        lib.base.cu(
            '`error_log` is set to syslog, so the log is not a file this check '
            'can open. Point `--server-log` at the systemd unit '
            '(`--server-log=systemd:php-fpm.service`) or at the file the syslog '
            'daemon writes.'
        )

    # No position is kept on purpose: this check reports on a window of the log,
    # counting startups, reloads and shutdowns next to the problems, so every run
    # has to see the whole window rather than only what is new.
    success, result = lib.logsource.read_many(
        server_logs,
        allowed_roots=allowed_roots,
        dedup_key=dedup_key,
        max_lines=MAXLINES,
        rotated=ROTATED_FILES,
        timeout=args.TIMEOUT,
    )
    if not success:
        # One file and nothing else: name the usual causes. Only for a path inside the
        # allowed roots; for any other the read refused it with the same answer whether
        # it exists or not, and that answer stands.
        kind, _, target = lib.base.coe(lib.logsource.parse(server_logs[0]))
        if (
            len(server_logs) == 1
            and kind == lib.logsource.KIND_FILE
            and lib.disk.is_within(target, allowed_roots)
        ):
            lib.base.oao(
                f'Logging seems to be configured, but `{target}` does not'
                f' seem to be an existing regular file. Check the path and'
                f' file permissions, or provide the `--server-log`'
                f' parameter.',
                STATE_WARN,
            )
        lib.base.cu(result)
    # A size is known for plain on-disk files only, so the size fact and its
    # perfdata series stay out of the output for a unit or a container. Where
    # several files are read, the series is what they add up to. The size comes from
    # the handle each file was read from, never from a second look at the path,
    # which could see another file by then, one outside the allowed roots included.
    sizes = {
        item['label']: item['size'] for item in result['sources'] if 'size' in item
    }
    if len(server_logs) == 1 and sizes.get(result['sources'][0]['label']) == 0:
        # An empty log file is a deterministic "no events observed" state,
        # not an unknown one - typical right after logrotate fires.
        lib.base.oao(
            f'Log file `{result["sources"][0]["label"]}` is empty.'
            ' Assuming log-rotation.',
            STATE_OK,
        )
    # What to call the whole of what was read, for the messages that have to
    # name it before the summary below spells every source out.
    # Abbreviated here, because this one shares a line with a sentence; the
    # listing further down gives the path in full.
    source_label = (
        '`'
        + lib.disk.shorten_path(
            result['sources'][0]['label'], max_len=SOURCE_PATH_MAX_LEN, truncate=False
        )
        + '`'
        if len(result['sources']) == 1
        else f'the {len(result["sources"])} sources read'
    )

    # init some vars
    state = STATE_OK
    sections = []
    facts = []
    # All recommendations from all WARN/CRIT paths land here and render once at
    # the end as a `Recommendations:\n* ...` bulleted block, regardless of which
    # combinations of paths fire.
    recommendations = []
    level_states = get_level_states(args.CRITICAL_LEVEL, args.WARNING_LEVEL)
    levels = {item['level']: [] for item in LEVELS}
    found = {item['key']: [] for item in EVENTS}
    found.update({item['key']: [] for item in RATE_EVENTS})
    found.update({item['key']: [] for item in LIFECYCLE})
    rate_since = datetime.datetime.now() - datetime.timedelta(seconds=args.LOOKBACK)
    rotations = []
    considered_cnt = 0
    recognized_cnt = 0
    unrecognized_cnt = 0
    suppressed_cnt = 0
    compiled_ignore = [
        lib.base.coe(item) for item in lib.txt.compile_regex(args.IGNORE, '--ignore')
    ]
    compiled_match = [
        lib.base.coe(item) for item in lib.txt.compile_regex(args.MATCH, '--match')
    ]

    # Persisted acknowledgements are only needed when the callback is in use.
    # Each unique combination of filters gets its own state database, so two
    # services watching the same error log for different things do not share what
    # has been acknowledged. Only the filters actually set take part in the
    # identifier, so adding a filter later does not orphan the state of a service
    # that does not use it.
    acked_fingerprints = set()
    ack_conn = None
    if args.ICINGA_CALLBACK:
        # Only what the operator set takes part in the identifier, so a host
        # that gains a second source - a unit next to the file - keeps the
        # acknowledgements it already has. `server_log` is in there when it was
        # named, because two services then deliberately watch different logs.
        instance_payload = {}
        for name, value in (
            ('ignore_regex', args.IGNORE),
            ('match', args.MATCH),
            ('server_log', args.SERVER_LOG),
        ):
            if value:
                instance_payload[name] = value
        ack_conn = lib.base.coe(
            lib.logmatch.connect(
                'php-fpm-logfile', lib.logmatch.instance_id(instance_payload)
            )
        )
        # Drop acknowledgements older than the retention: by that age the line has
        # left the window this check reads and can no longer re-appear.
        lib.base.coe(lib.logmatch.prune(ack_conn, retention=ACK_RETENTION_DAYS))
        acked_fingerprints = lib.base.coe(lib.logmatch.suppressed(ack_conn))

    # analyze data
    # Workers reported as timed out and not seen exiting since, per pool and
    # PID, and the ones already reported as hung, so a worker counts once.
    timed_out_workers = set()
    hung_workers = set()
    # Whether the previous line was one PHP itself wrote, whose message may go
    # on over the following lines without a prefix of their own.
    in_php_message = False
    for log_line in result['lines']:
        haystack = log_line.lower()
        if any(item.search(haystack) for item in compiled_ignore):
            continue
        # `--match` (include) is applied first, then `--ignore` (exclude), so a
        # line hit by `--ignore` is dropped even if it also matches `--match`.
        if compiled_match and not any(item.search(haystack) for item in compiled_match):
            continue
        considered_cnt += 1
        # Drop a line an operator has already taken on. The whole log line is
        # keyed, timestamp included, so the very same message logged again later
        # is a new event and alerts again.
        if acked_fingerprints and lib.logmatch.key(log_line) in acked_fingerprints:
            suppressed_cnt += 1
            continue
        identifier = lib.logsource.syslog_identifier(log_line)
        if identifier is not None and not IDENTIFIER_REGEX.match(identifier):
            # Somebody else writing into the same place: systemd's own
            # bookkeeping about the unit ("Starting ...", "Started ..."), and
            # the application's own messages, which PHP-FPM passes on under the
            # application's identifier - the journal of a PHP-FPM unit serving
            # Icinga Web is mostly `icingaweb2[1234]: PDOException ...`. Neither
            # is PHP-FPM's line, and counting them as lines without a level
            # would report the wrong source on a host where the source is right.
            continue
        # A line PHP itself wrote into the `error_log` of a pool. Only its
        # fatal errors are counted; the warnings and deprecations a busy
        # application writes all day are read and left alone.
        if PHP_LINE_REGEX.match(log_line):
            in_php_message = True
            recognized_cnt += 1
            if PHP_FATAL_REGEX.search(log_line):
                found['fatal_errors'].append(log_line)
            continue
        level_match = LEVEL_REGEX.search(log_line)
        if not level_match:
            if in_php_message and not log_line.startswith('['):
                # The stack trace of an uncaught exception, or another message
                # of PHP's that went on over several lines.
                continue
            # Not a line PHP-FPM wrote. Counted rather than dropped silently,
            # because a log full of them means the check is pointed at the wrong
            # file.
            unrecognized_cnt += 1
            continue
        in_php_message = False
        recognized_cnt += 1
        level, message = level_match.group(1), level_match.group(2)
        timed_out = TIMEOUT_WORKER_REGEX.search(message)
        if timed_out:
            worker = timed_out.groups()
            if worker in timed_out_workers and worker not in hung_workers:
                hung_workers.add(worker)
                found['hung_workers'].append(log_line)
            timed_out_workers.add(worker)
        exited = EXIT_WORKER_REGEX.search(message)
        if exited:
            timed_out_workers.discard(exited.groups())
        rate_hits = [event for event in RATE_EVENTS if event['regex'].search(message)]
        for event in rate_hits:
            found[event['key']].append(log_line)
        hits = [
            event
            for event in EVENTS
            if event['regex'] is not None and event['regex'].search(message)
        ]
        for event in hits:
            found[event['key']].append(log_line)
        # A line one of the catalogs owns is left out of the per-level counts.
        # A rate event is judged by how often it arrives, and counting it by its
        # level too would leave a site with one slow report page permanently
        # yellow. A named event carries its own state and says what happened,
        # which the level cannot; reporting the same lines once by level and
        # once by name says the same thing twice and doubles them in the graphs.
        if (
            not rate_hits
            and not hits
            and level in levels
            and not TERMINATED_EXIT_REGEX.search(message)
        ):
            levels[level].append(log_line)
        for item in LIFECYCLE:
            if item['regex'].search(message):
                found[item['key']].append(log_line)
        if ROTATION_REGEX.search(message):
            rotations.append(log_line)

    # Everything collected is put back into the order it was written in. The
    # sources are read one after the other, so a line from the file sits before
    # a newer one from the journal however late it was written, and "the last
    # one" would name something days old.
    for key, lines in found.items():
        found[key] = lib.logsource.sort_by_time(lines, parse_timestamp)
    for level, lines in levels.items():
        levels[level] = lib.logsource.sort_by_time(lines, parse_timestamp)

    # `--match` narrowed the run down to nothing, so this run looked at no line at
    # all rather than at a quiet log. Only reachable when the operator set a
    # filter, so a log that simply has nothing to report stays OK.
    if compiled_match and result['lines'] and not considered_cnt:
        if ack_conn is not None:
            lib.db_sqlite.close(ack_conn)
        lib.base.oao(
            f'Nothing checked: `--match` dropped all {len(result["lines"])} '
            f'{lib.txt.pluralize("line", len(result["lines"]))} of '
            f'{source_label}.',
            lib.base.str2state(args.NO_MATCH_SEVERITY),
            always_ok=args.ALWAYS_OK,
        )

    # Every line PHP-FPM writes carries its level, so a window without a single
    # one is not a quiet PHP-FPM but a source that holds something else, the pool
    # `slowlog` or the access log for example. Saying so beats reporting the
    # green state a check would otherwise keep on the wrong file forever.
    if unrecognized_cnt and not recognized_cnt:
        if ack_conn is not None:
            lib.db_sqlite.close(ack_conn)
        lib.base.oao(
            f'None of the {unrecognized_cnt} '
            f'{lib.txt.pluralize("line", unrecognized_cnt)} read from '
            f'{source_label} carries a PHP-FPM log level, so this does not '
            f'look like a PHP-FPM error log. Check `--server-log` and the '
            f'`error_log` directive of the PHP-FPM configuration.',
            STATE_UNKNOWN,
        )

    # build the message
    # What was read, and where from. The size stands next to the live file
    # rather than next to the line count, because it describes that one file and
    # nothing else - right after logrotate a 34 KiB file said to hold 30000
    # lines reads like a miscount, when almost all of them came from the rotated
    # predecessor. Naming the predecessor says where they did come from.
    # The count matters because everything below is counted within it: a run
    # that reports no startup at all is telling the truth about the window rather
    # than about the day.
    # And where the window stopped at the cap this check reads rather than at
    # the start of the log, it says so, because a busy host then reports on the
    # last few hours rather than on the day.
    # Which stretch of time those lines cover, because the count alone does not
    # say whether the window is an hour or a week: on a busy host the cap is
    # reached within hours, on a quiet one the same 30000 lines reach back
    # months, and every count below has to be read against that. Taken from the
    # ends of each source rather than from the ends of everything read: the
    # sources arrive one after the other, so the newest line of the first one
    # sits in the middle. A source that stamps no line simply leaves it out.
    covered = ''
    window_from, window_to = lib.logsource.covered_window(
        [item['lines'] for item in result['sources']], parse_timestamp
    )
    if window_from is not None and window_to is not None:
        span = int((window_to - window_from).total_seconds())
        covered = (
            f'{window_from:%Y-%m-%d %H:%M} .. {window_to:%Y-%m-%d %H:%M}'
            f' ({lib.human.seconds2human(span)}): '
        )
    line_cnt = len(result['lines'])
    lines_read = (
        f'{lib.human.number2human(line_cnt)} {lib.txt.pluralize("line", line_cnt)}'
    )
    if result['truncated']:
        lines_read = f'the most recent {lines_read}'
    # Every source by name, with its own size and its own rotated predecessor,
    # because a run that read three files and found nothing in two of them has
    # said something different from a run that read one.
    # The sources get a section of their own, the way the lines behind every
    # count do: a reverse proxy serving a dozen sites reads a dozen logs, and
    # naming them in the summary would bury the verdict under a paragraph of
    # paths - and the summary line is what a monitoring server shows in a list.
    # The paths are not abbreviated there: a bullet has the room, and that is
    # where an administrator copies them from.
    described = [
        lib.logsource.describe(item, sizes.get(item['label']))
        for item in result['sources']
    ]
    sources = f'{len(described)} {lib.txt.pluralize("source", len(described))}'

    if result['duplicates']:
        # Named rather than silently dropped: an administrator who sees two
        # sources read and one count has to be able to tell that the check knows
        # they hold the same events.
        shared = result['duplicates']
        source_fact_tail = (
            f', {lib.human.number2human(shared)} '
            f'{lib.txt.pluralize("line", shared)} they share counted once'
        )
    else:
        source_fact_tail = ''
    source_heading = f'Read {lines_read} from {sources}{source_fact_tail}:'

    # The counted events. Their state comes from how many of them arrived within
    # the rate window, not from the fact that they arrived at all. Their facts
    # are collected before the level counts are rendered, because a run that has
    # something to say about them is not the quiet one the literal below claims.
    rate_counts = {}
    rate_states = {}
    rate_facts = []
    for event in RATE_EVENTS:
        hits = found[event['key']]
        # No `key` here: PHP-FPM names the pool and the script of a request in
        # its error log but never the client, so there is no source to count per
        # and the number to judge by is everything that arrived.
        window = lib.logsource.count_within(
            hits, rate_since, parse_line=parse_timestamp
        )
        rate_counts[event['key']] = window['count']
        rate_states[event['key']] = STATE_OK
        if not hits:
            continue
        rate_state = lib.base.get_state(
            window['count'],
            getattr(args, event['warning']) or None,
            getattr(args, event['critical']) or None,
        )
        rate_states[event['key']] = rate_state
        state = lib.base.get_worst(state, rate_state)
        fact = (
            f'{window["count"]} '
            f'{lib.txt.pluralize(event["label"], window["count"])} in the last'
            f' {lib.human.seconds2human(args.LOOKBACK)}'
            f'{lib.base.state2str(rate_state, prefix=" ")}'
        )
        if len(hits) != window['count']:
            fact += f' ({len(hits)} in the window read)'
        if window['undated']:
            fact += (
                f', {window["undated"]} of them without a timestamp and therefore'
                f' not counted'
            )
        rate_facts.append(fact)
        if rate_state != STATE_OK:
            recommendations.append(event['recommendation'])

    counts = {}
    for item in LEVELS:
        lines = levels[item['level']]
        counts[item['level']] = len(lines)
        if not lines:
            continue
        level_state = level_states[item['level']]
        state = lib.base.get_worst(state, level_state)
        facts.append(
            f'{len(lines)} {item["level"]} '
            f'{lib.txt.pluralize("line", len(lines))} found'
            f'{lib.base.state2str(level_state, prefix=" ")}'
            f' (last: {lines[-1]})'
        )
    if (
        not any(counts.values())
        and not rate_facts
        and not any(found[event['key']] for event in EVENTS)
    ):
        # Naming all three levels one by one would bury the one sentence a
        # healthy host is supposed to be.
        facts.append('No errors or warnings found')

    # Name the events, so the summary says what happened and not only how bad it
    # was, and raise the state for those PHP-FPM logs too quietly. Quiet events
    # stay out of the line, which keeps a healthy host down to a single sentence.
    # A pool saturation is CRITICAL where the pool stays full: a second one
    # within the rate window, or one next to timed-out or hung workers there.
    event_states = {event['key']: event['state'] for event in EVENTS}
    recent_saturations = lib.logsource.count_within(
        found['pool_saturations'], rate_since, parse_line=parse_timestamp
    )['count']
    recent_hung = lib.logsource.count_within(
        found['hung_workers'], rate_since, parse_line=parse_timestamp
    )['count']
    if recent_saturations >= 2 or (
        recent_saturations and (rate_counts['request_timeouts'] or recent_hung)
    ):
        event_states['pool_saturations'] = STATE_CRIT
    named = []
    for event in EVENTS:
        hits = found[event['key']]
        if not hits:
            continue
        event_state = event_states[event['key']]
        state = lib.base.get_worst(state, event_state)
        named.append(
            f'{len(hits)} '
            f'{lib.txt.pluralize(event["label"], len(hits), event.get("suffix", "s"))}'
            f'{lib.base.state2str(event_state, prefix=" ")}'
        )
        recommendations.append(event['recommendation'])
    if named:
        facts.append('Found ' + ', '.join(named))

    facts.extend(rate_facts)

    for item in LIFECYCLE:
        lines = found[item['key']]
        # A counter at zero says nothing the performance data does not, and a
        # sentence about it would only push the verdict further right.
        if lines:
            facts.append(
                f'{len(lines)} {lib.txt.pluralize(item["label"], len(lines))}'
                f' {item["verb"]} (last: {lines[-1]})'
            )

    # A few lines without a level are normal where the application logs into the
    # same file; a majority of them means the source is mostly something else.
    if unrecognized_cnt:
        facts.append(
            f'{unrecognized_cnt} {lib.txt.pluralize("line", unrecognized_cnt)}'
            f' without a PHP-FPM log level'
        )
        if unrecognized_cnt > recognized_cnt:
            recommendations.append(
                'Most lines carry no PHP-FPM log level; check that the source is'
                ' the PHP-FPM error log and not the pool `slowlog`, the access'
                ' log or the log of the application itself'
            )

    if result['notice']:
        facts.append(result['notice'].rstrip('.'))

    # A source that could not be read at all leaves a hole in the window: with a
    # second source still delivering, the run goes on, but reporting the state of
    # what happened to work would hide that nothing is watching the rest.
    if result['failed']:
        state = lib.base.get_worst(state, STATE_WARN)
        facts.append(
            '; '.join(item.rstrip('.') for item in result['failed'])
            + lib.base.state2str(STATE_WARN, prefix=' ')
        )
        recommendations.append(
            'Check the log sources this check was told to read; '
            'one of them could not be read at all'
        )

    # The window the summary reports on comes first, because every count in it
    # is a count within that window and reads differently against an hour than
    # against a week.
    sections.append(covered + '. '.join(facts) + '.')

    for item in LEVELS:
        lines = levels[item['level']]
        if lines:
            sections.append(
                f'{item["level"].capitalize()} lines:\n'
                + '\n'.join(f'* {line}' for line in lib.txt.shorten_list(lines))
            )

    # The lines behind the named events. They are kept out of the per-level
    # counts above, so without this they are the one thing the check alerts on
    # that it never shows: "1 pool saturation" says that a pool ran dry, not
    # when, not which one, and not whether it was five minutes ago or a burst
    # from yesterday still sitting in the rotated file. The timestamp is what
    # lets the access log be read for the same minute, which is where the cause
    # of a saturation is.
    for event in EVENTS:
        lines = found[event['key']]
        if lines:
            label = lib.txt.pluralize(
                event['label'], len(lines), event.get('suffix', 's')
            )
            sections.append(
                f'{label.capitalize()}:\n'
                + '\n'.join(f'* {line}' for line in lib.txt.shorten_list(lines))
            )

    # The lines behind a counted event that raised a state, from within the rate
    # window. The recommendation points at them: the scripts that timed out say
    # whether one page is slow or a backend stalled under all of them.
    for event in RATE_EVENTS:
        if rate_states[event['key']] == STATE_OK:
            continue
        lines = [
            line
            for line in found[event['key']]
            if (lib.logsource.timestamp(line, parse_timestamp) or rate_since)
            >= rate_since
        ]
        if lines:
            label = lib.txt.pluralize(event['label'], len(lines))
            sections.append(
                f'{label[:1].upper()}{label[1:]} in the last'
                f' {lib.human.seconds2human(args.LOOKBACK)}:\n'
                + '\n'.join(f'* {line}' for line in lib.txt.shorten_list(lines))
            )

    # What was read, last of the sections: the verdict and the lines behind it
    # come first, the provenance after them.
    sections.append(
        source_heading + '\n' + '\n'.join(f'* {item}' for item in described)
    )

    if recommendations:
        sections.append(
            'Recommendations:\n' + '\n'.join(f'* {item}' for item in recommendations)
        )

    msg = '\n\n'.join(sections)

    perfdata = ''
    if sizes:
        perfdata += lib.base.get_perfdata(
            'php_fpm_logfile_size',
            sum(sizes.values()),
            uom='B',
            _min=0,
        )
    # The problem counters alert from their first hit on, hence `warn='0'`
    # ("outside 0..0") rather than `warn=1`.
    # A level counter carries the threshold of the state it actually raises, so
    # the graph shows the line the check follows and a level left quiet by
    # `--critical-level` and `--warning-level` draws none.
    for item in LEVELS:
        level_state = level_states[item['level']]
        perfdata += lib.base.get_perfdata(
            item['perfdata'],
            counts[item['level']],
            uom=None,
            warn='0' if level_state == STATE_WARN else None,
            crit='0' if level_state == STATE_CRIT else None,
            _min=0,
        )
    for event in EVENTS:
        perfdata += lib.base.get_perfdata(
            event['perfdata'],
            len(found[event['key']]),
            uom=None,
            warn='0' if event['state'] == STATE_WARN else None,
            crit='0' if event['state'] == STATE_CRIT else None,
            _min=0,
        )
    # The rate counters trend what the state follows: how many arrived within the
    # window, not how many the log holds.
    for event in RATE_EVENTS:
        perfdata += lib.base.get_perfdata(
            event['perfdata'],
            rate_counts[event['key']],
            uom=None,
            warn=getattr(args, event['warning']) or None,
            crit=getattr(args, event['critical']) or None,
            _min=0,
        )
    for item in LIFECYCLE:
        perfdata += lib.base.get_perfdata(
            item['perfdata'], len(found[item['key']]), uom=None, _min=0
        )
    perfdata += lib.base.get_perfdata(
        'php_fpm_log_rotations', len(rotations), uom=None, _min=0
    )

    # Ask the monitoring server about the acknowledgement. Where the service is
    # acknowledged, persist the lines currently being reported so an unchanged
    # error log does not raise them again on the following runs.
    if args.ICINGA_CALLBACK and state != STATE_OK:
        acknowledged, note = lib.base.coe(
            lib.logmatch.service_acknowledged(
                args.ICINGA_URL,
                args.ICINGA_USERNAME,
                args.ICINGA_PASSWORD,
                args.ICINGA_SERVICE_NAME,
                insecure=args.INSECURE,
                no_proxy=args.NO_PROXY,
                proxy=args.PROXY,
                timeout=args.TIMEOUT,
            )
        )
        if acknowledged:
            # Both the lines a level flagged and the lines an event flagged,
            # and the same line can be both, so they are deduplicated on the way
            # in. The counted events stay out: what alerts there is the arrival
            # rate and not one line, so silencing the lines already read would
            # not keep the next run from alerting on the ones arriving meanwhile.
            reported = {}
            for item in LEVELS:
                reported.update(dict.fromkeys(levels[item['level']]))
            for event in EVENTS:
                reported.update(dict.fromkeys(found[event['key']]))
            lib.base.coe(
                lib.logmatch.acknowledge(
                    ack_conn,
                    [
                        {'key': lib.logmatch.key(line), 'line': line}
                        for line in reported
                    ],
                )
            )
            state = STATE_OK
        if note:
            msg += f'\n\n{note}'
    if suppressed_cnt:
        msg += (
            f'\n\n{suppressed_cnt} acknowledged'
            f' {lib.txt.pluralize("line", suppressed_cnt)} suppressed.'
        )
    if ack_conn is not None:
        lib.db_sqlite.close(ack_conn)

    # over and out
    lib.base.oao(
        msg, state, perfdata, always_ok=args.ALWAYS_OK, no_perfdata=args.NO_PERFDATA
    )


if __name__ == '__main__':
    try:
        main()
    except Exception:
        lib.base.cu()
