Edits history of script submission #24215 for ' Device post-check (arista_eos)'

  • python3
    One script reply has been approved by the moderators
    Ap­pro­ved
    # py312
    # requirements:
    # pyeapi==1.0.4
    # wmill==1.851.0
    
    # Device post-check: one attempt, after an upgrade (or rollback) of one switch. Compares the switch with the snapshot
    # Device pre-check saved in this flow run, and raises a named error the workflow retries on:
    # - DeviceUnreachable while the switch is down or still runs the boot a reload of this run was sent on;
    # - NotStable while the state differs from the snapshot (the differences are in the message; ports stay errdisabled
    #   until the MLAG reload-delay has run, the message says how much of it is left);
    # - AuthFailure when the credential is refused (not worth retrying).
    # No snapshot in this run is a plain error. The version is reported, not compared with the target: the workflow does.
    
    import logging
    import os
    import re
    import ssl
    from typing import TypedDict
    from urllib.parse import urlsplit
    
    import pyeapi
    import wmill
    
    
    class arista_eos(TypedDict):
        host: str
        username: str
        password: str
        port: int
        verify_tls: bool
        allow_http: bool
    
    
    class NotStable(Exception):
        pass
    
    
    def main(auth: arista_eos):
        """
        Device post-check
        One post-upgrade check of one switch against the pre-check snapshot of this flow run: returns { running_version, status ok, checks_ok }. Raises DeviceUnreachable while the switch is down or still on the boot the reload was sent on, NotStable while interfaces/LLDP/MLAG differ from the snapshot (differences in the message), AuthFailure when the credential is refused. Retry it from the workflow; it does not wait.
        """
        state = load_state(state_keys(auth)) or {}
        # Before probing the switch: without a snapshot, retrying through a reboot would only delay the same error.
        if not state.get("snapshot"):
            raise RuntimeError(
                f"No pre-check snapshot for {auth['host']} in this flow run: run Device pre-check first, in the same run"
            )
        version = returned(auth, state)
        problems = diff(state["snapshot"], snapshot(auth))
        if problems:
            [mlag] = eapi(auth, ["show mlag"])
            left = round(reload_delay(mlag) - version["uptime"])
            hint = f" (MLAG reload-delay still running: {left} s left)" if left > 0 else ""
            raise NotStable(f"{auth['host']}: {'; '.join(problems)}{hint}")
        return {
            "running_version": short_version(version["version"]),
            "status": "ok",
            "checks_ok": ["interface"],
        }
    
    
    # Same function in Get Reload Delay and Device post-check. One attempt: show version, raising DeviceUnreachable
    # while the switch is down, and while it still runs the boot a reload of this run was sent on (reload now can answer
    # before EOS goes down, and the old boot must not pass for the new one). AuthFailure is raised as is.
    def returned(auth, state):
        try:
            # Short: a switch that is down holds the attempt for the whole timeout, and the workflow retries anyway.
            [v] = eapi(auth, ["show version"], timeout=15)
        except DeviceUnreachable as e:
            raise DeviceUnreachable(f"{e}: not back yet")
        for k in ("reload", "rollback"):
            m = state.get(k)
            if m and m["boot_timestamp"] == v["bootupTimestamp"]:
                raise DeviceUnreachable(
                    f"{auth['host']}: still on the boot the {k} sent at {m['sent_at']} was sent on"
                )
        return v
    
    
    # show mlag reports state disabled both when MLAG was never configured and when it is configured but shut down;
    # only the first has no domain-id, local interface or peer-link.
    def mlag_configured(mlag):
        return any(mlag.get(k) for k in ("domainId", "localInterface", "peerLink"))
    
    
    def snapshot(auth):
        status, lldp, mlag, mlag_intf = eapi(
            auth,
            [
                "show interfaces status",
                "show lldp neighbors",
                "show mlag",
                "show mlag interfaces",
            ],
        )
        return {
            "interfaces_up": sorted(
                n
                for n, i in status["interfaceStatuses"].items()
                if i.get("linkStatus") == "connected"
            ),
            "lldp": [
                {
                    "port": n["port"],
                    "neighbor": f"{n['neighborDevice']} {n['neighborPort']}",
                }
                for n in lldp["lldpNeighbors"]
            ],
            "mlag": {
                "state": mlag.get("state"),
                "negStatus": mlag.get("negStatus"),
                "peerLinkStatus": mlag.get("peerLinkStatus"),
                "ports": {
                    f"{i['localInterface']} (mlag {id})": i["status"]
                    for id, i in (mlag_intf.get("interfaces") or {}).items()
                },
            },
        }
    
    
    def diff(before, after):
        p = []
        for i in before["interfaces_up"]:
            if i not in after["interfaces_up"]:
                p.append(f"{i} down after the upgrade (up before)")
        for n in before["lldp"]:
            if not any(
                a["port"] == n["port"] and a["neighbor"] == n["neighbor"]
                for a in after["lldp"]
            ):
                p.append(f"LLDP neighbor {n['neighbor']} no longer seen on {n['port']}")
        for k in ("state", "negStatus", "peerLinkStatus"):
            if before["mlag"][k] != after["mlag"][k]:
                p.append(f"MLAG {k} {after['mlag'][k]} (was {before['mlag'][k]})")
        for port, s in before["mlag"]["ports"].items():
            if after["mlag"]["ports"].get(port) != s:
                p.append(f"{port} {after['mlag']['ports'].get(port, 'missing')} (was {s})")
        return p
    
    
    # Without MLAG configured, show mlag still reports the default reload-delay, and nothing waits on it.
    def reload_delay(mlag):
        if not mlag_configured(mlag):
            return 0
        return max(mlag.get("reloadDelay") or 0, mlag.get("reloadDelayNonMlag") or 0)
    
    
    # Errors the workflow branches and retries on (retry_if: result.error.name == "DeviceUnreachable").
    class DeviceUnreachable(Exception):
        pass
    
    
    class AuthFailure(Exception):
        pass
    
    
    # pyeapi logs a traceback for every socket error, and a rebooting switch gives many.
    logging.getLogger("pyeapi").addHandler(logging.NullHandler())
    
    
    # This script runs right after a reboot, when eAPI can answer before the EOS agents behind a show command have
    # started: such an error answer counts as not back yet, so the workflow retries it. It only sends show commands.
    def eapi(auth: arista_eos, cmds: list, fmt: str = "json", timeout: float = 60):
        eos = node(auth, timeout)
        try:
            return eos.run_commands(cmds, fmt)
        except pyeapi.eapilib.ConnectionError as e:
            raise connection_error(auth, eos, e)
        except pyeapi.eapilib.CommandError as e:
            raise DeviceUnreachable(f"{auth['host']}: not ready yet ({e.message})")
    
    
    # pyeapi 1.0.4 raises ConnectionError for a socket error (kept in socket_error), an answer that isn't JSON (eAPI
    # still starting, 5xx) and a 401, the only one with neither.
    def connection_error(auth, eos, e):
        if eos.connection.socket_error is None and e.message != "unable to connect to eAPI":
            return AuthFailure(f"{auth['host']}: eAPI refused the credential ({e.message})")
        return DeviceUnreachable(f"{auth['host']}: {e.message}")
    
    
    # host is an address or a URL (https://10.0.0.11:8443); a bare IPv6 address is bracketed. Only https, or http when the
    # resource ticks allow_http (Basic auth then crosses the network in cleartext). pyeapi skips certificate checks unless it
    # is given a context, so one is always passed: verifying, or not when the resource unticks verify_tls (EOS ships a
    # self-signed cert).
    def switch_url(auth: arista_eos):
        host = auth["host"]
        if "://" not in host:
            host = (
                f"https://[{host}]"
                if host.count(":") > 1 and "[" not in host
                else f"https://{host}"
            )
        url = urlsplit(host)
        if not url.hostname or (
            url.scheme != "https" and not (url.scheme == "http" and auth.get("allow_http"))
        ):
            raise ValueError(
                f"host must be an address or an https:// URL (http:// needs allow_http on the resource): {auth['host']}"
            )
        return url
    
    
    def node(auth: arista_eos, timeout: float = 60):
        url = switch_url(auth)
        context = ssl.create_default_context()
        if auth.get("verify_tls") is False:
            context.check_hostname = False
            context.verify_mode = ssl.CERT_NONE
        return pyeapi.connect(
            transport=url.scheme,
            host=url.hostname,
            port=url.port or auth.get("port") or None,
            username=auth["username"],
            password=auth["password"],
            context=context,
            timeout=timeout,
            return_node=True,
        )
    
    
    # Shared by the Device scripts: the root flow run's user state, keyed by switch, holds the pre-check snapshot, the
    # image activate replaced (rollback target), the image it booted and the reload markers. Every step of the run (nested
    # subflows and retried attempts included) reads the same entry. Outside a flow the Device scripts share one state
    # resource in the running user's home (a script's own state would keep the pre-check snapshot from the post-check).
    # The endpoint is called through the client so an unreachable state raises (wmill.get_flow_user_state returns None).
    # ponytail: the fallback is one map per user, so concurrent standalone runs can race; flows don't.
    # The key is the switch's eAPI address: arista_eos-<host>-<port> with non-word characters in the host as _, the port
    # always in (443 or 80 when unset). State written before the port was part of the key is read from arista_eos_<host>
    # as a fallback (all word characters, so never a new key); writes go to the new key only.
    def state_keys(auth: arista_eos):
        url = switch_url(auth)
        port = int(url.port or auth.get("port") or {"https": 443, "http": 80}[url.scheme])
        return [
            f"arista_eos-{word(url.hostname)}-{port}",
            f"arista_eos_{word(auth['host'])}",
        ]
    
    
    def word(s):
        return re.sub(r"\W", "_", s)
    
    
    def load_state(keys):
        for key in keys:
            if os.environ.get("WM_FLOW_JOB_ID"):
                c = wmill.Windmill()
                value = c.get(
                    f"/w/{c.workspace}/jobs/flow/user_states/{c.get_root_job_id()}/{key}"
                ).json()
            else:
                value = (wmill.get_state(standalone_state()) or {}).get(key)
            if value is not None:
                return value
    
    
    def save_state(keys, value):
        if os.environ.get("WM_FLOW_JOB_ID"):
            c = wmill.Windmill()
            c.post(
                f"/w/{c.workspace}/jobs/flow/user_states/{c.get_root_job_id()}/{keys[0]}",
                json=value,
            )
        else:
            path = standalone_state()
            wmill.set_state({**(wmill.get_state(path) or {}), keys[0]: value}, path)
    
    
    def standalone_state():
        return f"u/{os.environ['WM_USERNAME']}/arista_eos_state"
    
    
    # "4.34.9M-51132493.4349M (engineering build)" on cEOS, "4.34.9M" on hardware.
    def short_version(v):
        m = re.match(r"\d+\.\d+\.\d+(\.\d+)?[A-Z]*", v)
        return m.group(0) if m else v
    

    Submitted by hugo989 1 day ago