Edits history of script submission #23901 for ' All-in-one device check (deprecated) (arista_eos)'

  • python3
    One script reply has been approved by the moderators
    Ap­pro­ved
    # py312
    # requirements:
    # paramiko==5.1.0
    # pyeapi==1.0.4
    # wmill==1.851.0
    
    # Device check: pre- and post-upgrade checks of one switch.
    # - flash (pre): the target image is already on flash, or free space covers its size on the image server
    #   (image_url, or image_base_url + EOS-<target_version>.swi; required_free_mb without an image server or when its
    #   HEAD has no Content-Length). The HEAD checks the certificate and, if the worker doesn't trust it, sizes the image
    #   anyway: notes says when either happens, and when activate will need its sha512 input (untrusted certificate, or
    #   no <image>.sha512 reachable). A malformed image URL is flash ko.
    # - redundancy (pre): MLAG active and connected, peer-link and local interface up, config-sanity clean, no MLAG
    #   port active-partial, no port errdisabled. Left out when MLAG is not configured (notes says the reload is not
    #   hitless); configured but down is ko.
    # - interface: pre saves a snapshot (interfaces up, LLDP neighbor per port, MLAG state and per-port status) in the
    #   flow run's user state, on the root flow, under arista_eos-<host>-<port>; post compares the switch with it.
    # Post waits, in order: for eAPI to answer on a boot later than this run's reload (return_timeout_seconds), for the
    # MLAG reload-delay counted from boot when MLAG is configured, then up to stabilisation_seconds for the state to
    # match the snapshot. The three are returned in durations. The version is reported, not compared with the target.
    
    import base64
    import hashlib
    import json
    import logging
    import os
    import re
    import socket
    import ssl
    import time
    import urllib.error
    import urllib.request
    from typing import Literal, TypedDict
    from urllib.parse import urlsplit
    
    import paramiko
    import pyeapi
    import wmill
    
    
    class arista_eos(TypedDict):
        host: str
        username: str
        password: str
        port: int
        verify_tls: bool
        allow_http: bool
    
    
    def main(
        auth: arista_eos,
        phase: Literal["pre", "post"],
        host: str = "",
        target_version: str = "",
        image_base_url: str = "",
        required_free_mb: int = 1024,
        return_timeout_seconds: int = 900,
        stabilisation_seconds: int = 120,
        poll_interval_seconds: int = 15,
        image_url: str = "",
    ):
        """
        All-in-one device check (deprecated)
        Pre- or post-upgrade check of one switch: returns { running_version, status ok|ko, checks_ok, checks_ko[{check, message}], notes }. Pre: flash space for the target image, MLAG redundancy, and a snapshot of interfaces/LLDP/MLAG kept in the flow run's state. Post: waits for the switch to answer and for the MLAG reload-delay (when MLAG is configured), then compares with the snapshot over a bounded stabilisation window. Throws DeviceUnreachable or AuthFailure. Does not compare the version with the target: the workflow does. Deprecated: kept for flows built on it; new flows use the single-attempt Device actions, which return plain facts instead of this ok|ko result.
        """
        auth = with_host(auth, host)
        poll = poll_interval_seconds or 15
        durations = {}
        key = state_keys(auth)
        state = load_state(key) or {}
    
        if phase == "post":
            # Wait for the switch to answer on a boot later than the one each reload of this run was sent on: `reload now`
            # can answer before EOS goes down, and the old boot must not pass for the new one. AuthFailure stops at once.
            markers = [state[k] for k in ("reload", "rollback") if state.get(k)]
            timeout = return_timeout_seconds or 900
            start = time.time()
            while True:
                try:
                    [v] = eapi(auth, ["show version"])
                    m = next(
                        (m for m in markers if m["boot_timestamp"] == v["bootupTimestamp"]),
                        None,
                    )
                    if not m:
                        break
                    pending = (
                        f"still on the boot the reload sent at {m['sent_at']} was sent on"
                    )
                except DeviceUnreachable as e:
                    pending = str(e)
                if time.time() - start > timeout:
                    raise named(
                        DeviceUnreachable,
                        f"{auth['host']}: not back after {timeout} s ({pending})",
                    )
                time.sleep(poll)
            durations["return_timeout_seconds"] = timeout
            durations["returned_after_seconds"] = round(time.time() - start)
    
        version, mlag = eapi(auth, ["show version", "show mlag"])
        running_version = short_version(version["version"])
        # Without MLAG configured, show mlag still reports the default reload-delay, and nothing waits on it.
        mlag_on = mlag_configured(mlag)
        durations["reload_delay_seconds"] = (
            max(mlag.get("reloadDelay") or 0, mlag.get("reloadDelayNonMlag") or 0)
            if mlag_on
            else 0
        )
    
        ok, ko, notes = [], [], []
    
        def record(check, problems):
            if problems:
                ko.append({"check": check, "message": "; ".join(problems)})
            else:
                ok.append(check)
    
        if phase == "pre":
            record(
                "flash",
                flash_problems(
                    auth, target_version, image_base_url, image_url, required_free_mb, notes
                ),
            )
            if mlag_on:
                [sanity] = eapi(auth, ["show mlag config-sanity"])
                record("redundancy", redundancy_problems(mlag, sanity))
            else:
                notes.append("redundancy: MLAG not configured, the reload is not hitless")
            save_state(
                key,
                {
                    **state,
                    "snapshot": snapshot(auth),
                    "snapshot_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
                },
            )
            ok.append("interface")
        elif not state.get("snapshot"):
            ko.append(
                {
                    "check": "interface",
                    "message": f"No pre-check snapshot for {auth['host']} in this flow run: run the pre-check first, in the same run",
                }
            )
        else:
            # Ports stay errdisabled until the MLAG reload-delay has elapsed since boot: wait it out.
            wait = durations["reload_delay_seconds"] - version["uptime"]
            if wait > 0:
                time.sleep(wait)
            durations["reload_delay_waited_seconds"] = max(0, round(wait))
            stab = 120 if stabilisation_seconds is None else stabilisation_seconds
            start = time.time()
            problems = diff(state["snapshot"], snapshot(auth))
            while problems and time.time() - start < stab:
                time.sleep(poll)
                problems = diff(state["snapshot"], snapshot(auth))
            durations["stabilisation_seconds"] = stab
            durations["stabilised_after_seconds"] = round(time.time() - start)
            record("interface", problems)
    
        return {
            "running_version": running_version,
            "status": "ko" if ko else "ok",
            "checks_ok": ok,
            "checks_ko": ko,
            "durations": durations,
            "notes": notes,
        }
    
    
    # One resource can serve every switch that shares a login: the host input, when set, overrides the resource's host.
    def with_host(auth: arista_eos, host: str):
        host = (host or "").strip() or (auth.get("host") or "").strip()
        if not host:
            raise ValueError("host is required: set it on the resource or pass host")
        return {**auth, "host": host}
    
    
    def flash_problems(auth, target, base, image_url, required_mb, notes):
        [d] = eapi(auth, [{"cmd": "dir flash:/", "revision": 2}])
        flash = d["urls"]["flash:/"]
        try:
            image, url = image_source(target, base, image_url)
        except ValueError as e:
            return [str(e)]
        if image and image in flash["entries"]:
            if url:
                sidecar_note(url, image, notes)
            return []
        required = (required_mb or 1024) * 1e6
        if url:
            try:
                try:
                    size = head(url, ssl.create_default_context())
                    sidecar_note(url, image, notes)
                except urllib.error.URLError as e:
                    if not isinstance(e.reason, ssl.SSLCertVerificationError):
                        raise
                    # Only the size is read: the switch's copy doesn't check the certificate either, and activate
                    # verifies the image's sha512. So an untrusted (self-signed) server is sized anyway, with a note.
                    unverified = ssl.create_default_context()
                    unverified.check_hostname = False
                    unverified.verify_mode = ssl.CERT_NONE
                    size = head(url, unverified)
                    notes.append(
                        f"flash: the image server's certificate is not trusted on the worker ({e.reason.verify_message}), sized {image} without checking it; activate checks it when it fetches {image}.sha512, so pass it sha512"
                    )
                if size:
                    required = size
                else:
                    notes.append(
                        f"flash: the image server gave no size for {image}, assumed required_free_mb ({mb(required)})"
                    )
            except urllib.error.HTTPError as e:
                return [f"Target image {image} not found on the image server ({e.code})"]
            except OSError as e:
                return [
                    f"Image server unreachable from the worker: {getattr(e, 'reason', e)}"
                ]
        free = flash["filesystem"]["freeBytes"]
        if free < required:
            return [
                f"Insufficient flash space ({mb(free)} free, {mb(required)} required{f' for {image}' if image else ''})"
            ]
        return []
    
    
    def head(url, context):
        with urllib.request.urlopen(
            urllib.request.Request(url, method="HEAD"), timeout=30, context=context
        ) as r:
            return int(r.headers.get("content-length") or 0)
    
    
    # Activate reads the digest from the .sha512 file next to the image, certificate checked, unless given sha512.
    def sidecar_note(url, image, notes):
        try:
            head(f"{url}.sha512", ssl.create_default_context())
        except OSError as e:
            reason = getattr(e, "code", None) or getattr(e, "reason", e)
            notes.append(
                f"flash: {image}.sha512 not reachable from the worker ({reason}): pass sha512 to activate"
            )
    
    
    # Same function in Device check and Device upgrade.
    # image_url wins; else image_base_url + EOS-<target>.swi. Returns (file name on flash, URL or ""). The URL goes into
    # the switch's copy command, so it is an allowlist: http(s), then only A-Z a-z 0-9 . _ ~ : / % + - (no query, no
    # user:password@, nothing the CLI reads as a separator), ending in a plain .swi file name. Raises ValueError otherwise.
    def image_source(target, base, image_url):
        if image_url:
            image, url = urlsplit(image_url).path.rsplit("/", 1)[-1], image_url
        else:
            image = f"EOS-{target}.swi" if target else ""
            url = f"{base.rstrip('/')}/{image}" if image and base else ""
        if not url:
            return image, url
        # Not echoed: it would put the password in the steps and errors.
        if "@" in url:
            raise ValueError("the image URL must not contain @ (no user:password@ in it)")
        if not re.match(r"https?://", url):
            raise ValueError(
                f"the image URL must start with http:// or https://: got {url!r}"
            )
        if not urlsplit(url).hostname:
            raise ValueError(f"the image URL has no host: got {url!r}")
        bad = sorted(set(re.sub(r"[A-Za-z0-9._~:/%+-]", "", url.split("://", 1)[1])))
        if bad:
            raise ValueError(
                f"the image URL may only contain A-Z a-z 0-9 . _ ~ : / % + -, not {' '.join(map(repr, bad))}: got {url!r}"
            )
        if not re.fullmatch(r"[\w.+-]+\.swi", image):
            raise ValueError(f"the image URL must end in a .swi file name: got {url!r}")
        return image, url
    
    
    # show mlag reports state disabled both when MLAG was never configured and when it is configured but shut down;
    # only the first has no domain-id, local interface or peer-link.
    def mlag_configured(mlag):
        return any(mlag.get(k) for k in ("domainId", "localInterface", "peerLink"))
    
    
    def redundancy_problems(mlag, sanity):
        p = []
        if mlag.get("state") != "active":
            p.append(f"MLAG state {mlag.get('state')} (expected active)")
        if mlag.get("negStatus") != "connected":
            p.append(f"MLAG negotiation {mlag.get('negStatus')} (expected connected)")
        if mlag.get("peerLinkStatus") != "up":
            p.append(f"peer-link {mlag.get('peerLink')} {mlag.get('peerLinkStatus')}")
        if mlag.get("localIntfStatus") != "up":
            p.append(
                f"MLAG local interface {mlag.get('localInterface')} {mlag.get('localIntfStatus')}"
            )
        inconsistent = [
            *(sanity.get("globalConfiguration") or {}),
            *(sanity.get("interfaceConfiguration") or {}),
        ]
        if inconsistent:
            p.append(f"config-sanity inconsistent: {', '.join(inconsistent)}")
        partial = (mlag.get("mlagPorts") or {}).get("Active-partial")
        if partial:
            p.append(f"{partial} MLAG port(s) active-partial")
        if mlag.get("portsErrdisabled"):
            p.append("MLAG ports errdisabled")
        return p
    
    
    def snapshot(auth):
        status, lldp, mlag, mlag_intf = eapi(
            auth,
            [
                "show interfaces status",
                "show lldp neighbors",
                "show mlag",
                "show mlag interfaces",
            ],
        )
        return {
            "interfaces_up": sorted(
                n
                for n, i in status["interfaceStatuses"].items()
                if i.get("linkStatus") == "connected"
            ),
            "lldp": [
                {
                    "port": n["port"],
                    "neighbor": f"{n['neighborDevice']} {n['neighborPort']}",
                }
                for n in lldp["lldpNeighbors"]
            ],
            "mlag": {
                "state": mlag.get("state"),
                "negStatus": mlag.get("negStatus"),
                "peerLinkStatus": mlag.get("peerLinkStatus"),
                "ports": {
                    f"{i['localInterface']} (mlag {id})": i["status"]
                    for id, i in (mlag_intf.get("interfaces") or {}).items()
                },
            },
        }
    
    
    def diff(before, after):
        p = []
        for i in before["interfaces_up"]:
            if i not in after["interfaces_up"]:
                p.append(f"{i} down after the upgrade (up before)")
        for n in before["lldp"]:
            if not any(
                a["port"] == n["port"] and a["neighbor"] == n["neighbor"]
                for a in after["lldp"]
            ):
                p.append(f"LLDP neighbor {n['neighbor']} no longer seen on {n['port']}")
        for k in ("state", "negStatus", "peerLinkStatus"):
            if before["mlag"][k] != after["mlag"][k]:
                p.append(f"MLAG {k} {after['mlag'][k]} (was {before['mlag'][k]})")
        for port, s in before["mlag"]["ports"].items():
            if after["mlag"]["ports"].get(port) != s:
                p.append(f"{port} {after['mlag']['ports'].get(port, 'missing')} (was {s})")
        return p
    
    
    # The snapshot lives in the root flow run's user state, keyed by switch: pre- and post-check are different steps
    # (often in nested subflows) of the same run. Outside a flow, it falls back to this script's own state.
    # The endpoint is called through the client so an unreachable state raises (wmill.get_flow_user_state returns None).
    # ponytail: the fallback is one map per script, so concurrent standalone runs can race; flows don't.
    # The key is the switch's eAPI address: arista_eos-<host>-<port> with non-word characters in the host as _, the port
    # always in (443 or 80 when unset), so host 10.0.0.11 with port 8443 and host https://10.0.0.11:8443 share one key,
    # while two switches behind one address on different ports (or http and https) don't. State written before the port
    # was part of the key is read from arista_eos_<host> as a fallback (all word characters, so never a new key); writes
    # go to the new key only.
    def state_keys(auth: arista_eos):
        if auth.get("transport") == "ssh":
            host, port = ssh_addr(auth)
            return [f"arista_eos-{word(host)}-{port}"]
        url = switch_url(auth)
        port = int(url.port or auth.get("port") or {"https": 443, "http": 80}[url.scheme])
        return [
            f"arista_eos-{word(url.hostname)}-{port}",
            f"arista_eos_{word(auth['host'])}",
        ]
    
    
    def word(s):
        return re.sub(r"\W", "_", s)
    
    
    def load_state(keys):
        for key in keys:
            if os.environ.get("WM_FLOW_JOB_ID"):
                c = wmill.Windmill()
                value = c.get(
                    f"/w/{c.workspace}/jobs/flow/user_states/{c.get_root_job_id()}/{key}"
                ).json()
            else:
                value = (wmill.get_state() or {}).get(key)
            if value is not None:
                return value
    
    
    def save_state(keys, value):
        if os.environ.get("WM_FLOW_JOB_ID"):
            c = wmill.Windmill()
            c.post(
                f"/w/{c.workspace}/jobs/flow/user_states/{c.get_root_job_id()}/{keys[0]}",
                json=value,
            )
        else:
            wmill.set_state({**(wmill.get_state() or {}), keys[0]: value})
    
    
    # "4.34.9M-51132493.4349M (engineering build)" on cEOS, "4.34.9M" on hardware.
    def short_version(v):
        m = re.match(r"\d+\.\d+\.\d+(\.\d+)?[A-Z]*", v)
        return m.group(0) if m else v
    
    
    def mb(n):
        return f"{n / 1e9:.1f} GB" if n >= 1e9 else f"{round(n / 1e6)} MB"
    
    
    class DeviceUnreachable(Exception):
        pass
    
    
    class AuthFailure(Exception):
        pass
    
    
    def named(cls, message: str):
        return cls(f"{cls.__name__}: {message}")
    
    
    # pyeapi logs a traceback for every socket error, and the post-check polls a switch that is rebooting.
    logging.getLogger("pyeapi").addHandler(logging.NullHandler())
    
    
    def eapi(auth: arista_eos, cmds: list, fmt: str = "json", timeout: float = 60):
        eos = node(auth, timeout)
        try:
            return eos.run_commands(cmds, fmt)
        except pyeapi.eapilib.ConnectionError as e:
            raise connection_error(auth, eos, e)
    
    
    # pyeapi 1.0.4 raises ConnectionError for a socket error (kept in socket_error), an answer that isn't JSON (eAPI
    # still starting, 5xx) and a 401, the only one with neither.
    def connection_error(auth, eos, e):
        # A certificate the resource does not trust is a setting to fix (verify_tls, ca_certificate), not worth a retry.
        if isinstance(eos.connection.socket_error, ssl.SSLCertVerificationError):
            return RuntimeError(f"{auth['host']}: {eos.connection.socket_error}")
        if eos.connection.socket_error is None and e.message != "unable to connect to eAPI":
            return named(
                AuthFailure, f"{auth['host']}: {'SSH' if auth.get('transport') == 'ssh' else 'eAPI'} refused the credential ({e.message})"
            )
        return named(DeviceUnreachable, f"{auth['host']}: {e.message}")
    
    
    # host is an address or a URL (https://10.0.0.11:8443); a bare IPv6 address is bracketed. Only https, or http when the
    # resource ticks allow_http (Basic auth then crosses the network in cleartext). pyeapi skips certificate checks unless it
    # is given a context, so one is always passed: verifying, or not when the resource unticks verify_tls (EOS ships a
    # self-signed cert).
    def switch_url(auth: arista_eos):
        host = auth["host"]
        if "://" not in host:
            host = f"https://[{host}]" if host.count(":") > 1 and "[" not in host else f"https://{host}"
        url = urlsplit(host)
        if not url.hostname or (url.scheme != "https" and not (url.scheme == "http" and auth.get("allow_http"))):
            raise ValueError(f"host must be an address or an https:// URL (http:// needs allow_http on the resource): {auth['host']}")
        return url
    
    
    # SSH transport (transport: ssh): the EOS CLI over an SSH shell, behind the part of pyeapi's Node the scripts use
    # (run_commands, connection.socket_error, connection.transport.connect), raising pyeapi's own CommandError and
    # ConnectionError, so the checks and the error mapping are the same for both transports. Show commands in json get
    # "| json", which prints the eAPI JSON; other commands give { messages: [output] } like eAPI, or {} when silent.
    # A line starting with "% " is an error, raised as eAPI's CommandError. The host key is checked when the resource
    # has ssh_host_key (SHA256:... as ssh-keygen -lf prints it).
    class Ssh:
        def __init__(self, auth: arista_eos, timeout: float):
            self.auth, self.timeout = auth, timeout
            self.connection = self.transport = self
            self.socket_error = self.sock = self.chan = None
    
        # The TCP connection only, like eAPI's transport.connect(): if it fails, nothing was sent.
        def connect(self):
            self.sock = socket.create_connection(ssh_addr(self.auth), self.timeout)
    
        def lost(self, e):
            self.socket_error = e
            return pyeapi.eapilib.ConnectionError("ssh", f"SSH connection failed ({e})")
    
        def open(self):
            auth = self.auth
            try:
                if not self.sock:
                    self.connect()
                t = paramiko.Transport(self.sock)
                t.start_client(timeout=self.timeout)
                key = t.get_remote_server_key()
            except (OSError, EOFError, paramiko.SSHException) as e:
                raise self.lost(e)
            fp = "SHA256:" + base64.b64encode(hashlib.sha256(key.asbytes()).digest()).decode().rstrip("=")
            if auth.get("ssh_host_key") and auth["ssh_host_key"].strip() != fp:
                t.close()
                raise RuntimeError(f"{auth['host']}: the SSH host key is {fp}, not the resource's ssh_host_key")
            try:
                try:
                    t.auth_password(auth["username"], auth["password"])
                except paramiko.BadAuthenticationType:
                    t.auth_interactive(
                        auth["username"], lambda _t, _i, prompts: [auth["password"]] * len(prompts)
                    )
            except paramiko.AuthenticationException as e:
                t.close()
                raise pyeapi.eapilib.ConnectionError("ssh", str(e))
            except (OSError, EOFError, paramiko.SSHException) as e:
                raise self.lost(e)
            try:
                self.chan = t.open_session()
                self.chan.get_pty(width=4000)
                self.chan.invoke_shell()
                self.prompt = None
                out = self.read()
                self.prompt = re.escape(re.split(r"[(>#]", out.splitlines()[-1].strip())[0])
                if out.rstrip().endswith(">"):
                    self.cli("enable")
                self.cli("terminal length 0")
                self.cli("terminal width 32767")
            except (OSError, EOFError, paramiko.SSHException) as e:
                raise self.lost(e)
    
        # Reads until the prompt. Questions eAPI never asks are answered as eAPI behaves: confirm, don't save.
        def read(self):
            buf, end = "", time.monotonic() + self.timeout
            prompt = (self.prompt or r"[\w.\-]+") + r"(\([^)\r\n]*\))?[>#] ?$"
            while True:
                self.chan.settimeout(max(end - time.monotonic(), 0.01))
                chunk = self.chan.recv(65536)
                if not chunk:
                    raise self.lost("connection closed")
                buf += chunk.decode(errors="replace")
                tail = buf[-300:]
                if re.search(r"(^|\n)" + prompt, tail):
                    return buf
                for question, answer in (
                    # enable asks for a password on switches that land the user unprivileged: the login password.
                    (r"(^|\n)Password:\s*$", self.auth["password"]),
                    (r"\[confirm\]\s*$", ""),
                    (r"Save\?.*\[yes/no[^\]]*\]:?\s*$", "no"),
                    (r"\[(y|yes)/(n|no)\][:?]?\s*$", "y"),
                ):
                    if re.search(question, tail, re.I):
                        self.chan.send(answer + "\r")
                        buf += "\n"
    
        # One command: its output without the echoed command and the prompt; progress lines keep their last state.
        def cli(self, cmd):
            try:
                self.chan.send(cmd + "\r")
                out = self.read()
            except (OSError, EOFError, paramiko.SSHException) as e:
                raise self.lost(e)
            self.mode = out.rstrip().splitlines()[-1]
            lines = [l.rstrip("\r").split("\r")[-1] for l in out.split("\n")]
            return "\n".join(lines[1:-1]).strip("\n")
    
        def run_commands(self, cmds, encoding="json", **_):
            try:
                if not self.chan:
                    self.open()
                if "(config" in self.mode:
                    self.cli("end")
                results = []
                for i, c in enumerate(cmds):
                    # eAPI's { cmd, revision } form is the CLI's "| json revision N".
                    cmd, rev = (c["cmd"], c.get("revision")) if isinstance(c, dict) else (c, None)
                    show = encoding == "json" and (rev or re.match(r"\s*sh(ow)?\s", cmd)) and not re.search(r"\|\s*json", cmd)
                    # pyeapi sends "<cmd> MULTILINE: <text>" (banners) as cmd + input; the CLI reads the text up to EOF.
                    head, multi, text = cmd.partition("MULTILINE:")
                    # The shell would run what follows a line break as another command; eAPI refuses it too.
                    if re.search(r"[\r\n]", head):
                        raise ValueError(f"a command may not contain a line break: {head!r}")
                    line = "\r".join([head.strip(), *text.strip().splitlines(), "EOF"]) if multi else cmd
                    out = self.cli(f"{cmd} | json{f' revision {rev}' if rev else ''}" if show else line)
                    errors = [l[2:] for l in out.splitlines() if l.startswith("% ")]
                    code, kind = 1002, "invalid command"
                    if show and not errors:
                        code, kind = 1000, "could not run command"
                        try:
                            result = json.loads(out)
                        except ValueError:
                            errors = [out[:200]]
                        else:
                            errors = result.get("errors") or []
                    if errors:
                        raise pyeapi.eapilib.CommandError(
                            code,
                            f"CLI command {i + 2} of {len(cmds) + 1} '{cmd}' failed: {kind}",
                            command_error="; ".join(errors),
                            commands=cmds,
                            output=results,
                        )
                    if encoding != "json":
                        result = {"output": out + "\n" if out else ""}
                    elif not show:
                        # The CLI prints eAPI's warnings as "! " notices.
                        notices = [l[2:].strip() for l in out.splitlines() if l.startswith("! ")]
                        out = "\n".join(l for l in out.splitlines() if not l.startswith("! ")).strip()
                        result = {"messages": [out]} if out and not multi else {}
                        if notices:
                            result["warnings"] = notices
                    results.append(result)
                return results
            except pyeapi.eapilib.ConnectionError:
                if self.chan:
                    self.chan.close()
                self.sock = self.chan = None
                raise
    
    
    # paramiko logs a traceback when a rebooting switch drops the session.
    logging.getLogger("paramiko").addHandler(logging.NullHandler())
    
    
    # host, or host:port ([v6]:port), else port, else 22.
    def ssh_addr(auth: arista_eos):
        h = auth["host"]
        if "://" in h:
            raise ValueError(f"with transport ssh, host is an address or a hostname (with :port), not a URL: {h}")
        url = urlsplit(f"ssh://{h}" if h.count(":") <= 1 or h.startswith("[") else f"ssh://[{h}]")
        return url.hostname, url.port or int(auth.get("port") or 22)
    
    
    def node(auth: arista_eos, timeout: float = 60):
        if auth.get("transport") == "ssh":
            return Ssh(auth, timeout)
        url = switch_url(auth)
        # ca_certificate: the PEM of a private CA, trusted instead of the system's CAs.
        context = ssl.create_default_context(cadata=(auth.get("ca_certificate") or "").strip() or None)
        if auth.get("verify_tls") is False:
            context.check_hostname = False
            context.verify_mode = ssl.CERT_NONE
        return pyeapi.connect(
            transport=url.scheme,
            host=url.hostname,
            port=url.port or auth.get("port") or None,
            username=auth["username"],
            password=auth["password"],
            context=context,
            timeout=timeout,
            return_node=True,
        )
    

    Submitted by hugo989 1 day ago

  • python3
    # py312
    # requirements:
    # pyeapi==1.0.4
    # wmill==1.851.0
    
    # Device check: pre- and post-upgrade checks of one switch.
    # - flash (pre): the target image is already on flash, or free space covers its size on the image server
    #   (image_url, or image_base_url + EOS-<target_version>.swi; required_free_mb without an image server or when its
    #   HEAD has no Content-Length). The HEAD checks the certificate and, if the worker doesn't trust it, sizes the image
    #   anyway: notes says when either happens, and when activate will need its sha512 input (untrusted certificate, or
    #   no <image>.sha512 reachable). A malformed image URL is flash ko.
    # - redundancy (pre): MLAG active and connected, peer-link and local interface up, config-sanity clean, no MLAG
    #   port active-partial, no port errdisabled. Left out when MLAG is not configured (notes says the reload is not
    #   hitless); configured but down is ko.
    # - interface: pre saves a snapshot (interfaces up, LLDP neighbor per port, MLAG state and per-port status) in the
    #   flow run's user state, on the root flow, under arista_eos-<host>-<port>; post compares the switch with it.
    # Post waits, in order: for eAPI to answer on a boot later than this run's reload (return_timeout_seconds), for the
    # MLAG reload-delay counted from boot when MLAG is configured, then up to stabilisation_seconds for the state to
    # match the snapshot. The three are returned in durations. The version is reported, not compared with the target.
    
    import logging
    import os
    import re
    import ssl
    import time
    import urllib.error
    import urllib.request
    from typing import Literal, TypedDict
    from urllib.parse import urlsplit
    
    import pyeapi
    import wmill
    
    
    class arista_eos(TypedDict):
        host: str
        username: str
        password: str
        port: int
        verify_tls: bool
        allow_http: bool
    
    
    def main(
        auth: arista_eos,
        phase: Literal["pre", "post"],
        target_version: str = "",
        image_base_url: str = "",
        required_free_mb: int = 1024,
        return_timeout_seconds: int = 900,
        stabilisation_seconds: int = 120,
        poll_interval_seconds: int = 15,
        image_url: str = "",
    ):
        """
        All-in-one device check
        Pre- or post-upgrade check of one switch: returns { running_version, status ok|ko, checks_ok, checks_ko[{check, message}], notes }. Pre: flash space for the target image, MLAG redundancy, and a snapshot of interfaces/LLDP/MLAG kept in the flow run's state. Post: waits for the switch to answer and for the MLAG reload-delay (when MLAG is configured), then compares with the snapshot over a bounded stabilisation window. Throws DeviceUnreachable or AuthFailure. Does not compare the version with the target: the workflow does. For native step retry and sleep in the workflow, use the single-attempt Device actions (Device pre-check, Get Reload Delay, Device post-check, Device activate, Device reload, Device commit, Device rollback) instead.
        """
        poll = poll_interval_seconds or 15
        durations = {}
        key = state_keys(auth)
        state = load_state(key) or {}
    
        if phase == "post":
            # Wait for the switch to answer on a boot later than the one each reload of this run was sent on: `reload now`
            # can answer before EOS goes down, and the old boot must not pass for the new one. AuthFailure stops at once.
            markers = [state[k] for k in ("reload", "rollback") if state.get(k)]
            timeout = return_timeout_seconds or 900
            start = time.time()
            while True:
                try:
                    [v] = eapi(auth, ["show version"])
                    m = next(
                        (m for m in markers if m["boot_timestamp"] == v["bootupTimestamp"]),
                        None,
                    )
                    if not m:
                        break
                    pending = (
                        f"still on the boot the reload sent at {m['sent_at']} was sent on"
                    )
                except DeviceUnreachable as e:
                    pending = str(e)
                if time.time() - start > timeout:
                    raise named(
                        DeviceUnreachable,
                        f"{auth['host']}: not back after {timeout} s ({pending})",
                    )
                time.sleep(poll)
            durations["return_timeout_seconds"] = timeout
            durations["returned_after_seconds"] = round(time.time() - start)
    
        version, mlag = eapi(auth, ["show version", "show mlag"])
        running_version = short_version(version["version"])
        # Without MLAG configured, show mlag still reports the default reload-delay, and nothing waits on it.
        mlag_on = mlag_configured(mlag)
        durations["reload_delay_seconds"] = (
            max(mlag.get("reloadDelay") or 0, mlag.get("reloadDelayNonMlag") or 0)
            if mlag_on
            else 0
        )
    
        ok, ko, notes = [], [], []
    
        def record(check, problems):
            if problems:
                ko.append({"check": check, "message": "; ".join(problems)})
            else:
                ok.append(check)
    
        if phase == "pre":
            record(
                "flash",
                flash_problems(
                    auth, target_version, image_base_url, image_url, required_free_mb, notes
                ),
            )
            if mlag_on:
                [sanity] = eapi(auth, ["show mlag config-sanity"])
                record("redundancy", redundancy_problems(mlag, sanity))
            else:
                notes.append("redundancy: MLAG not configured, the reload is not hitless")
            save_state(
                key,
                {
                    **state,
                    "snapshot": snapshot(auth),
                    "snapshot_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
                },
            )
            ok.append("interface")
        elif not state.get("snapshot"):
            ko.append(
                {
                    "check": "interface",
                    "message": f"No pre-check snapshot for {auth['host']} in this flow run: run the pre-check first, in the same run",
                }
            )
        else:
            # Ports stay errdisabled until the MLAG reload-delay has elapsed since boot: wait it out.
            wait = durations["reload_delay_seconds"] - version["uptime"]
            if wait > 0:
                time.sleep(wait)
            durations["reload_delay_waited_seconds"] = max(0, round(wait))
            stab = 120 if stabilisation_seconds is None else stabilisation_seconds
            start = time.time()
            problems = diff(state["snapshot"], snapshot(auth))
            while problems and time.time() - start < stab:
                time.sleep(poll)
                problems = diff(state["snapshot"], snapshot(auth))
            durations["stabilisation_seconds"] = stab
            durations["stabilised_after_seconds"] = round(time.time() - start)
            record("interface", problems)
    
        return {
            "running_version": running_version,
            "status": "ko" if ko else "ok",
            "checks_ok": ok,
            "checks_ko": ko,
            "durations": durations,
            "notes": notes,
        }
    
    
    def flash_problems(auth, target, base, image_url, required_mb, notes):
        [d] = eapi(auth, [{"cmd": "dir flash:/", "revision": 2}])
        flash = d["urls"]["flash:/"]
        try:
            image, url = image_source(target, base, image_url)
        except ValueError as e:
            return [str(e)]
        if image and image in flash["entries"]:
            if url:
                sidecar_note(url, image, notes)
            return []
        required = (required_mb or 1024) * 1e6
        if url:
            try:
                try:
                    size = head(url, ssl.create_default_context())
                    sidecar_note(url, image, notes)
                except urllib.error.URLError as e:
                    if not isinstance(e.reason, ssl.SSLCertVerificationError):
                        raise
                    # Only the size is read: the switch's copy doesn't check the certificate either, and activate
                    # verifies the image's sha512. So an untrusted (self-signed) server is sized anyway, with a note.
                    unverified = ssl.create_default_context()
                    unverified.check_hostname = False
                    unverified.verify_mode = ssl.CERT_NONE
                    size = head(url, unverified)
                    notes.append(
                        f"flash: the image server's certificate is not trusted on the worker ({e.reason.verify_message}), sized {image} without checking it; activate checks it when it fetches {image}.sha512, so pass it sha512"
                    )
                if size:
                    required = size
                else:
                    notes.append(
                        f"flash: the image server gave no size for {image}, assumed required_free_mb ({mb(required)})"
                    )
            except urllib.error.HTTPError as e:
                return [f"Target image {image} not found on the image server ({e.code})"]
            except OSError as e:
                return [
                    f"Image server unreachable from the worker: {getattr(e, 'reason', e)}"
                ]
        free = flash["filesystem"]["freeBytes"]
        if free < required:
            return [
                f"Insufficient flash space ({mb(free)} free, {mb(required)} required{f' for {image}' if image else ''})"
            ]
        return []
    
    
    def head(url, context):
        with urllib.request.urlopen(
            urllib.request.Request(url, method="HEAD"), timeout=30, context=context
        ) as r:
            return int(r.headers.get("content-length") or 0)
    
    
    # Activate reads the digest from the .sha512 file next to the image, certificate checked, unless given sha512.
    def sidecar_note(url, image, notes):
        try:
            head(f"{url}.sha512", ssl.create_default_context())
        except OSError as e:
            reason = getattr(e, "code", None) or getattr(e, "reason", e)
            notes.append(
                f"flash: {image}.sha512 not reachable from the worker ({reason}): pass sha512 to activate"
            )
    
    
    # Same function in Device check and Device upgrade.
    # image_url wins; else image_base_url + EOS-<target>.swi. Returns (file name on flash, URL or ""). The URL goes into
    # the switch's copy command, so it is an allowlist: http(s), then only A-Z a-z 0-9 . _ ~ : / % + - (no query, no
    # user:password@, nothing the CLI reads as a separator), ending in a plain .swi file name. Raises ValueError otherwise.
    def image_source(target, base, image_url):
        if image_url:
            image, url = urlsplit(image_url).path.rsplit("/", 1)[-1], image_url
        else:
            image = f"EOS-{target}.swi" if target else ""
            url = f"{base.rstrip('/')}/{image}" if image and base else ""
        if not url:
            return image, url
        # Not echoed: it would put the password in the steps and errors.
        if "@" in url:
            raise ValueError("the image URL must not contain @ (no user:password@ in it)")
        if not re.match(r"https?://", url):
            raise ValueError(
                f"the image URL must start with http:// or https://: got {url!r}"
            )
        if not urlsplit(url).hostname:
            raise ValueError(f"the image URL has no host: got {url!r}")
        bad = sorted(set(re.sub(r"[A-Za-z0-9._~:/%+-]", "", url.split("://", 1)[1])))
        if bad:
            raise ValueError(
                f"the image URL may only contain A-Z a-z 0-9 . _ ~ : / % + -, not {' '.join(map(repr, bad))}: got {url!r}"
            )
        if not re.fullmatch(r"[\w.+-]+\.swi", image):
            raise ValueError(f"the image URL must end in a .swi file name: got {url!r}")
        return image, url
    
    
    # show mlag reports state disabled both when MLAG was never configured and when it is configured but shut down;
    # only the first has no domain-id, local interface or peer-link.
    def mlag_configured(mlag):
        return any(mlag.get(k) for k in ("domainId", "localInterface", "peerLink"))
    
    
    def redundancy_problems(mlag, sanity):
        p = []
        if mlag.get("state") != "active":
            p.append(f"MLAG state {mlag.get('state')} (expected active)")
        if mlag.get("negStatus") != "connected":
            p.append(f"MLAG negotiation {mlag.get('negStatus')} (expected connected)")
        if mlag.get("peerLinkStatus") != "up":
            p.append(f"peer-link {mlag.get('peerLink')} {mlag.get('peerLinkStatus')}")
        if mlag.get("localIntfStatus") != "up":
            p.append(
                f"MLAG local interface {mlag.get('localInterface')} {mlag.get('localIntfStatus')}"
            )
        inconsistent = [
            *(sanity.get("globalConfiguration") or {}),
            *(sanity.get("interfaceConfiguration") or {}),
        ]
        if inconsistent:
            p.append(f"config-sanity inconsistent: {', '.join(inconsistent)}")
        partial = (mlag.get("mlagPorts") or {}).get("Active-partial")
        if partial:
            p.append(f"{partial} MLAG port(s) active-partial")
        if mlag.get("portsErrdisabled"):
            p.append("MLAG ports errdisabled")
        return p
    
    
    def snapshot(auth):
        status, lldp, mlag, mlag_intf = eapi(
            auth,
            [
                "show interfaces status",
                "show lldp neighbors",
                "show mlag",
                "show mlag interfaces",
            ],
        )
        return {
            "interfaces_up": sorted(
                n
                for n, i in status["interfaceStatuses"].items()
                if i.get("linkStatus") == "connected"
            ),
            "lldp": [
                {
                    "port": n["port"],
                    "neighbor": f"{n['neighborDevice']} {n['neighborPort']}",
                }
                for n in lldp["lldpNeighbors"]
            ],
            "mlag": {
                "state": mlag.get("state"),
                "negStatus": mlag.get("negStatus"),
                "peerLinkStatus": mlag.get("peerLinkStatus"),
                "ports": {
                    f"{i['localInterface']} (mlag {id})": i["status"]
                    for id, i in (mlag_intf.get("interfaces") or {}).items()
                },
            },
        }
    
    
    def diff(before, after):
        p = []
        for i in before["interfaces_up"]:
            if i not in after["interfaces_up"]:
                p.append(f"{i} down after the upgrade (up before)")
        for n in before["lldp"]:
            if not any(
                a["port"] == n["port"] and a["neighbor"] == n["neighbor"]
                for a in after["lldp"]
            ):
                p.append(f"LLDP neighbor {n['neighbor']} no longer seen on {n['port']}")
        for k in ("state", "negStatus", "peerLinkStatus"):
            if before["mlag"][k] != after["mlag"][k]:
                p.append(f"MLAG {k} {after['mlag'][k]} (was {before['mlag'][k]})")
        for port, s in before["mlag"]["ports"].items():
            if after["mlag"]["ports"].get(port) != s:
                p.append(f"{port} {after['mlag']['ports'].get(port, 'missing')} (was {s})")
        return p
    
    
    # The snapshot lives in the root flow run's user state, keyed by switch: pre- and post-check are different steps
    # (often in nested subflows) of the same run. Outside a flow, it falls back to this script's own state.
    # The endpoint is called through the client so an unreachable state raises (wmill.get_flow_user_state returns None).
    # ponytail: the fallback is one map per script, so concurrent standalone runs can race; flows don't.
    # The key is the switch's eAPI address: arista_eos-<host>-<port> with non-word characters in the host as _, the port
    # always in (443 or 80 when unset), so host 10.0.0.11 with port 8443 and host https://10.0.0.11:8443 share one key,
    # while two switches behind one address on different ports (or http and https) don't. State written before the port
    # was part of the key is read from arista_eos_<host> as a fallback (all word characters, so never a new key); writes
    # go to the new key only.
    def state_keys(auth: arista_eos):
        url = switch_url(auth)
        port = int(url.port or auth.get("port") or {"https": 443, "http": 80}[url.scheme])
        return [
            f"arista_eos-{word(url.hostname)}-{port}",
            f"arista_eos_{word(auth['host'])}",
        ]
    
    
    def word(s):
        return re.sub(r"\W", "_", s)
    
    
    def load_state(keys):
        for key in keys:
            if os.environ.get("WM_FLOW_JOB_ID"):
                c = wmill.Windmill()
                value = c.get(
                    f"/w/{c.workspace}/jobs/flow/user_states/{c.get_root_job_id()}/{key}"
                ).json()
            else:
                value = (wmill.get_state() or {}).get(key)
            if value is not None:
                return value
    
    
    def save_state(keys, value):
        if os.environ.get("WM_FLOW_JOB_ID"):
            c = wmill.Windmill()
            c.post(
                f"/w/{c.workspace}/jobs/flow/user_states/{c.get_root_job_id()}/{keys[0]}",
                json=value,
            )
        else:
            wmill.set_state({**(wmill.get_state() or {}), keys[0]: value})
    
    
    # "4.34.9M-51132493.4349M (engineering build)" on cEOS, "4.34.9M" on hardware.
    def short_version(v):
        m = re.match(r"\d+\.\d+\.\d+(\.\d+)?[A-Z]*", v)
        return m.group(0) if m else v
    
    
    def mb(n):
        return f"{n / 1e9:.1f} GB" if n >= 1e9 else f"{round(n / 1e6)} MB"
    
    
    class DeviceUnreachable(Exception):
        pass
    
    
    class AuthFailure(Exception):
        pass
    
    
    def named(cls, message: str):
        return cls(f"{cls.__name__}: {message}")
    
    
    # pyeapi logs a traceback for every socket error, and the post-check polls a switch that is rebooting.
    logging.getLogger("pyeapi").addHandler(logging.NullHandler())
    
    
    def eapi(auth: arista_eos, cmds: list, fmt: str = "json", timeout: float = 60):
        eos = node(auth, timeout)
        try:
            return eos.run_commands(cmds, fmt)
        except pyeapi.eapilib.ConnectionError as e:
            raise connection_error(auth, eos, e)
    
    
    # pyeapi 1.0.4 raises ConnectionError for a socket error (kept in socket_error), an answer that isn't JSON (eAPI
    # still starting, 5xx) and a 401, the only one with neither.
    def connection_error(auth, eos, e):
        if eos.connection.socket_error is None and e.message != "unable to connect to eAPI":
            return named(
                AuthFailure, f"{auth['host']}: eAPI refused the credential ({e.message})"
            )
        return named(DeviceUnreachable, f"{auth['host']}: {e.message}")
    
    
    # host is an address or a URL (https://10.0.0.11:8443); a bare IPv6 address is bracketed. Only https, or http when the
    # resource ticks allow_http (Basic auth then crosses the network in cleartext). pyeapi skips certificate checks unless it
    # is given a context, so one is always passed: verifying, or not when the resource unticks verify_tls (EOS ships a
    # self-signed cert).
    def switch_url(auth: arista_eos):
        host = auth["host"]
        if "://" not in host:
            host = f"https://[{host}]" if host.count(":") > 1 and "[" not in host else f"https://{host}"
        url = urlsplit(host)
        if not url.hostname or (url.scheme != "https" and not (url.scheme == "http" and auth.get("allow_http"))):
            raise ValueError(f"host must be an address or an https:// URL (http:// needs allow_http on the resource): {auth['host']}")
        return url
    
    
    def node(auth: arista_eos, timeout: float = 60):
        url = switch_url(auth)
        context = ssl.create_default_context()
        if auth.get("verify_tls") is False:
            context.check_hostname = False
            context.verify_mode = ssl.CERT_NONE
        return pyeapi.connect(
            transport=url.scheme,
            host=url.hostname,
            port=url.port or auth.get("port") or None,
            username=auth["username"],
            password=auth["password"],
            context=context,
            timeout=timeout,
            return_node=True,
        )
    

    Submitted by hugo989 3 days ago

  • python3
    # py312
    # requirements:
    # pyeapi==1.0.4
    # wmill==1.851.0
    
    # Device check: pre- and post-upgrade checks of one switch.
    # - flash (pre): the target image is already on flash, or free space covers its size on the image server
    #   (image_url, or image_base_url + EOS-<target_version>.swi; required_free_mb without an image server or when its
    #   HEAD has no Content-Length). The HEAD checks the certificate and, if the worker doesn't trust it, sizes the image
    #   anyway: notes says when either happens, and when activate will need its sha512 input (untrusted certificate, or
    #   no <image>.sha512 reachable). A malformed image URL is flash ko.
    # - redundancy (pre): MLAG active and connected, peer-link and local interface up, config-sanity clean, no MLAG
    #   port active-partial, no port errdisabled. Left out when MLAG is not configured (notes says the reload is not
    #   hitless); configured but down is ko.
    # - interface: pre saves a snapshot (interfaces up, LLDP neighbor per port, MLAG state and per-port status) in the
    #   flow run's user state, on the root flow, under arista_eos-<host>-<port>; post compares the switch with it.
    # Post waits, in order: for eAPI to answer on a boot later than this run's reload (return_timeout_seconds), for the
    # MLAG reload-delay counted from boot when MLAG is configured, then up to stabilisation_seconds for the state to
    # match the snapshot. The three are returned in durations. The version is reported, not compared with the target.
    
    import logging
    import os
    import re
    import ssl
    import time
    import urllib.error
    import urllib.request
    from typing import Literal, TypedDict
    from urllib.parse import urlsplit
    
    import pyeapi
    import wmill
    
    
    class arista_eos(TypedDict):
        host: str
        username: str
        password: str
        port: int
        verify_tls: bool
        allow_http: bool
    
    
    def main(
        auth: arista_eos,
        phase: Literal["pre", "post"],
        target_version: str = "",
        image_base_url: str = "",
        required_free_mb: int = 1024,
        return_timeout_seconds: int = 900,
        stabilisation_seconds: int = 120,
        poll_interval_seconds: int = 15,
        image_url: str = "",
    ):
        """
        Device check
        Pre- or post-upgrade check of one switch: returns { running_version, status ok|ko, checks_ok, checks_ko[{check, message}], notes }. Pre: flash space for the target image, MLAG redundancy, and a snapshot of interfaces/LLDP/MLAG kept in the flow run's state. Post: waits for the switch to answer and for the MLAG reload-delay (when MLAG is configured), then compares with the snapshot over a bounded stabilisation window. Throws DeviceUnreachable or AuthFailure. Does not compare the version with the target: the workflow does.
        """
        poll = poll_interval_seconds or 15
        durations = {}
        key = state_keys(auth)
        state = load_state(key) or {}
    
        if phase == "post":
            # Wait for the switch to answer on a boot later than the one each reload of this run was sent on: `reload now`
            # can answer before EOS goes down, and the old boot must not pass for the new one. AuthFailure stops at once.
            markers = [state[k] for k in ("reload", "rollback") if state.get(k)]
            timeout = return_timeout_seconds or 900
            start = time.time()
            while True:
                try:
                    [v] = eapi(auth, ["show version"])
                    m = next(
                        (m for m in markers if m["boot_timestamp"] == v["bootupTimestamp"]),
                        None,
                    )
                    if not m:
                        break
                    pending = (
                        f"still on the boot the reload sent at {m['sent_at']} was sent on"
                    )
                except DeviceUnreachable as e:
                    pending = str(e)
                if time.time() - start > timeout:
                    raise named(
                        DeviceUnreachable,
                        f"{auth['host']}: not back after {timeout} s ({pending})",
                    )
                time.sleep(poll)
            durations["return_timeout_seconds"] = timeout
            durations["returned_after_seconds"] = round(time.time() - start)
    
        version, mlag = eapi(auth, ["show version", "show mlag"])
        running_version = short_version(version["version"])
        # Without MLAG configured, show mlag still reports the default reload-delay, and nothing waits on it.
        mlag_on = mlag_configured(mlag)
        durations["reload_delay_seconds"] = (
            max(mlag.get("reloadDelay") or 0, mlag.get("reloadDelayNonMlag") or 0)
            if mlag_on
            else 0
        )
    
        ok, ko, notes = [], [], []
    
        def record(check, problems):
            if problems:
                ko.append({"check": check, "message": "; ".join(problems)})
            else:
                ok.append(check)
    
        if phase == "pre":
            record(
                "flash",
                flash_problems(
                    auth, target_version, image_base_url, image_url, required_free_mb, notes
                ),
            )
            if mlag_on:
                [sanity] = eapi(auth, ["show mlag config-sanity"])
                record("redundancy", redundancy_problems(mlag, sanity))
            else:
                notes.append("redundancy: MLAG not configured, the reload is not hitless")
            save_state(
                key,
                {
                    **state,
                    "snapshot": snapshot(auth),
                    "snapshot_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
                },
            )
            ok.append("interface")
        elif not state.get("snapshot"):
            ko.append(
                {
                    "check": "interface",
                    "message": f"No pre-check snapshot for {auth['host']} in this flow run: run the pre-check first, in the same run",
                }
            )
        else:
            # Ports stay errdisabled until the MLAG reload-delay has elapsed since boot: wait it out.
            wait = durations["reload_delay_seconds"] - version["uptime"]
            if wait > 0:
                time.sleep(wait)
            durations["reload_delay_waited_seconds"] = max(0, round(wait))
            stab = 120 if stabilisation_seconds is None else stabilisation_seconds
            start = time.time()
            problems = diff(state["snapshot"], snapshot(auth))
            while problems and time.time() - start < stab:
                time.sleep(poll)
                problems = diff(state["snapshot"], snapshot(auth))
            durations["stabilisation_seconds"] = stab
            durations["stabilised_after_seconds"] = round(time.time() - start)
            record("interface", problems)
    
        return {
            "running_version": running_version,
            "status": "ko" if ko else "ok",
            "checks_ok": ok,
            "checks_ko": ko,
            "durations": durations,
            "notes": notes,
        }
    
    
    def flash_problems(auth, target, base, image_url, required_mb, notes):
        [d] = eapi(auth, [{"cmd": "dir flash:/", "revision": 2}])
        flash = d["urls"]["flash:/"]
        try:
            image, url = image_source(target, base, image_url)
        except ValueError as e:
            return [str(e)]
        if image and image in flash["entries"]:
            if url:
                sidecar_note(url, image, notes)
            return []
        required = (required_mb or 1024) * 1e6
        if url:
            try:
                try:
                    size = head(url, ssl.create_default_context())
                    sidecar_note(url, image, notes)
                except urllib.error.URLError as e:
                    if not isinstance(e.reason, ssl.SSLCertVerificationError):
                        raise
                    # Only the size is read: the switch's copy doesn't check the certificate either, and activate
                    # verifies the image's sha512. So an untrusted (self-signed) server is sized anyway, with a note.
                    unverified = ssl.create_default_context()
                    unverified.check_hostname = False
                    unverified.verify_mode = ssl.CERT_NONE
                    size = head(url, unverified)
                    notes.append(
                        f"flash: the image server's certificate is not trusted on the worker ({e.reason.verify_message}), sized {image} without checking it; activate checks it when it fetches {image}.sha512, so pass it sha512"
                    )
                if size:
                    required = size
                else:
                    notes.append(
                        f"flash: the image server gave no size for {image}, assumed required_free_mb ({mb(required)})"
                    )
            except urllib.error.HTTPError as e:
                return [f"Target image {image} not found on the image server ({e.code})"]
            except OSError as e:
                return [
                    f"Image server unreachable from the worker: {getattr(e, 'reason', e)}"
                ]
        free = flash["filesystem"]["freeBytes"]
        if free < required:
            return [
                f"Insufficient flash space ({mb(free)} free, {mb(required)} required{f' for {image}' if image else ''})"
            ]
        return []
    
    
    def head(url, context):
        with urllib.request.urlopen(
            urllib.request.Request(url, method="HEAD"), timeout=30, context=context
        ) as r:
            return int(r.headers.get("content-length") or 0)
    
    
    # Activate reads the digest from the .sha512 file next to the image, certificate checked, unless given sha512.
    def sidecar_note(url, image, notes):
        try:
            head(f"{url}.sha512", ssl.create_default_context())
        except OSError as e:
            reason = getattr(e, "code", None) or getattr(e, "reason", e)
            notes.append(
                f"flash: {image}.sha512 not reachable from the worker ({reason}): pass sha512 to activate"
            )
    
    
    # Same function in Device check and Device upgrade.
    # image_url wins; else image_base_url + EOS-<target>.swi. Returns (file name on flash, URL or ""). The URL goes into
    # the switch's copy command, so it is an allowlist: http(s), then only A-Z a-z 0-9 . _ ~ : / % + - (no query, no
    # user:password@, nothing the CLI reads as a separator), ending in a plain .swi file name. Raises ValueError otherwise.
    def image_source(target, base, image_url):
        if image_url:
            image, url = urlsplit(image_url).path.rsplit("/", 1)[-1], image_url
        else:
            image = f"EOS-{target}.swi" if target else ""
            url = f"{base.rstrip('/')}/{image}" if image and base else ""
        if not url:
            return image, url
        # Not echoed: it would put the password in the steps and errors.
        if "@" in url:
            raise ValueError("the image URL must not contain @ (no user:password@ in it)")
        if not re.match(r"https?://", url):
            raise ValueError(
                f"the image URL must start with http:// or https://: got {url!r}"
            )
        if not urlsplit(url).hostname:
            raise ValueError(f"the image URL has no host: got {url!r}")
        bad = sorted(set(re.sub(r"[A-Za-z0-9._~:/%+-]", "", url.split("://", 1)[1])))
        if bad:
            raise ValueError(
                f"the image URL may only contain A-Z a-z 0-9 . _ ~ : / % + -, not {' '.join(map(repr, bad))}: got {url!r}"
            )
        if not re.fullmatch(r"[\w.+-]+\.swi", image):
            raise ValueError(f"the image URL must end in a .swi file name: got {url!r}")
        return image, url
    
    
    # show mlag reports state disabled both when MLAG was never configured and when it is configured but shut down;
    # only the first has no domain-id, local interface or peer-link.
    def mlag_configured(mlag):
        return any(mlag.get(k) for k in ("domainId", "localInterface", "peerLink"))
    
    
    def redundancy_problems(mlag, sanity):
        p = []
        if mlag.get("state") != "active":
            p.append(f"MLAG state {mlag.get('state')} (expected active)")
        if mlag.get("negStatus") != "connected":
            p.append(f"MLAG negotiation {mlag.get('negStatus')} (expected connected)")
        if mlag.get("peerLinkStatus") != "up":
            p.append(f"peer-link {mlag.get('peerLink')} {mlag.get('peerLinkStatus')}")
        if mlag.get("localIntfStatus") != "up":
            p.append(
                f"MLAG local interface {mlag.get('localInterface')} {mlag.get('localIntfStatus')}"
            )
        inconsistent = [
            *(sanity.get("globalConfiguration") or {}),
            *(sanity.get("interfaceConfiguration") or {}),
        ]
        if inconsistent:
            p.append(f"config-sanity inconsistent: {', '.join(inconsistent)}")
        partial = (mlag.get("mlagPorts") or {}).get("Active-partial")
        if partial:
            p.append(f"{partial} MLAG port(s) active-partial")
        if mlag.get("portsErrdisabled"):
            p.append("MLAG ports errdisabled")
        return p
    
    
    def snapshot(auth):
        status, lldp, mlag, mlag_intf = eapi(
            auth,
            [
                "show interfaces status",
                "show lldp neighbors",
                "show mlag",
                "show mlag interfaces",
            ],
        )
        return {
            "interfaces_up": sorted(
                n
                for n, i in status["interfaceStatuses"].items()
                if i.get("linkStatus") == "connected"
            ),
            "lldp": [
                {
                    "port": n["port"],
                    "neighbor": f"{n['neighborDevice']} {n['neighborPort']}",
                }
                for n in lldp["lldpNeighbors"]
            ],
            "mlag": {
                "state": mlag.get("state"),
                "negStatus": mlag.get("negStatus"),
                "peerLinkStatus": mlag.get("peerLinkStatus"),
                "ports": {
                    f"{i['localInterface']} (mlag {id})": i["status"]
                    for id, i in (mlag_intf.get("interfaces") or {}).items()
                },
            },
        }
    
    
    def diff(before, after):
        p = []
        for i in before["interfaces_up"]:
            if i not in after["interfaces_up"]:
                p.append(f"{i} down after the upgrade (up before)")
        for n in before["lldp"]:
            if not any(
                a["port"] == n["port"] and a["neighbor"] == n["neighbor"]
                for a in after["lldp"]
            ):
                p.append(f"LLDP neighbor {n['neighbor']} no longer seen on {n['port']}")
        for k in ("state", "negStatus", "peerLinkStatus"):
            if before["mlag"][k] != after["mlag"][k]:
                p.append(f"MLAG {k} {after['mlag'][k]} (was {before['mlag'][k]})")
        for port, s in before["mlag"]["ports"].items():
            if after["mlag"]["ports"].get(port) != s:
                p.append(f"{port} {after['mlag']['ports'].get(port, 'missing')} (was {s})")
        return p
    
    
    # The snapshot lives in the root flow run's user state, keyed by switch: pre- and post-check are different steps
    # (often in nested subflows) of the same run. Outside a flow, it falls back to this script's own state.
    # The endpoint is called through the client so an unreachable state raises (wmill.get_flow_user_state returns None).
    # ponytail: the fallback is one map per script, so concurrent standalone runs can race; flows don't.
    # The key is the switch's eAPI address: arista_eos-<host>-<port> with non-word characters in the host as _, the port
    # always in (443 or 80 when unset), so host 10.0.0.11 with port 8443 and host https://10.0.0.11:8443 share one key,
    # while two switches behind one address on different ports (or http and https) don't. State written before the port
    # was part of the key is read from arista_eos_<host> as a fallback (all word characters, so never a new key); writes
    # go to the new key only.
    def state_keys(auth: arista_eos):
        url = switch_url(auth)
        port = int(url.port or auth.get("port") or {"https": 443, "http": 80}[url.scheme])
        return [
            f"arista_eos-{word(url.hostname)}-{port}",
            f"arista_eos_{word(auth['host'])}",
        ]
    
    
    def word(s):
        return re.sub(r"\W", "_", s)
    
    
    def load_state(keys):
        for key in keys:
            if os.environ.get("WM_FLOW_JOB_ID"):
                c = wmill.Windmill()
                value = c.get(
                    f"/w/{c.workspace}/jobs/flow/user_states/{c.get_root_job_id()}/{key}"
                ).json()
            else:
                value = (wmill.get_state() or {}).get(key)
            if value is not None:
                return value
    
    
    def save_state(keys, value):
        if os.environ.get("WM_FLOW_JOB_ID"):
            c = wmill.Windmill()
            c.post(
                f"/w/{c.workspace}/jobs/flow/user_states/{c.get_root_job_id()}/{keys[0]}",
                json=value,
            )
        else:
            wmill.set_state({**(wmill.get_state() or {}), keys[0]: value})
    
    
    # "4.34.9M-51132493.4349M (engineering build)" on cEOS, "4.34.9M" on hardware.
    def short_version(v):
        m = re.match(r"\d+\.\d+\.\d+(\.\d+)?[A-Z]*", v)
        return m.group(0) if m else v
    
    
    def mb(n):
        return f"{n / 1e9:.1f} GB" if n >= 1e9 else f"{round(n / 1e6)} MB"
    
    
    class DeviceUnreachable(Exception):
        pass
    
    
    class AuthFailure(Exception):
        pass
    
    
    def named(cls, message: str):
        return cls(f"{cls.__name__}: {message}")
    
    
    # pyeapi logs a traceback for every socket error, and the post-check polls a switch that is rebooting.
    logging.getLogger("pyeapi").addHandler(logging.NullHandler())
    
    
    def eapi(auth: arista_eos, cmds: list, fmt: str = "json", timeout: float = 60):
        eos = node(auth, timeout)
        try:
            return eos.run_commands(cmds, fmt)
        except pyeapi.eapilib.ConnectionError as e:
            raise connection_error(auth, eos, e)
    
    
    # pyeapi 1.0.4 raises ConnectionError for a socket error (kept in socket_error), an answer that isn't JSON (eAPI
    # still starting, 5xx) and a 401, the only one with neither.
    def connection_error(auth, eos, e):
        if eos.connection.socket_error is None and e.message != "unable to connect to eAPI":
            return named(
                AuthFailure, f"{auth['host']}: eAPI refused the credential ({e.message})"
            )
        return named(DeviceUnreachable, f"{auth['host']}: {e.message}")
    
    
    # host is an address or a URL (https://10.0.0.11:8443); a bare IPv6 address is bracketed. Only https, or http when the
    # resource ticks allow_http (Basic auth then crosses the network in cleartext). pyeapi skips certificate checks unless it
    # is given a context, so one is always passed: verifying, or not when the resource unticks verify_tls (EOS ships a
    # self-signed cert).
    def switch_url(auth: arista_eos):
        host = auth["host"]
        if "://" not in host:
            host = f"https://[{host}]" if host.count(":") > 1 and "[" not in host else f"https://{host}"
        url = urlsplit(host)
        if not url.hostname or (url.scheme != "https" and not (url.scheme == "http" and auth.get("allow_http"))):
            raise ValueError(f"host must be an address or an https:// URL (http:// needs allow_http on the resource): {auth['host']}")
        return url
    
    
    def node(auth: arista_eos, timeout: float = 60):
        url = switch_url(auth)
        context = ssl.create_default_context()
        if auth.get("verify_tls") is False:
            context.check_hostname = False
            context.verify_mode = ssl.CERT_NONE
        return pyeapi.connect(
            transport=url.scheme,
            host=url.hostname,
            port=url.port or auth.get("port") or None,
            username=auth["username"],
            password=auth["password"],
            context=context,
            timeout=timeout,
            return_node=True,
        )
    

    Submitted by hugo989 9 days ago

  • python3
    # py312
    # requirements:
    # pyeapi==1.0.4
    # wmill==1.851.0
    
    # Device check: pre- and post-upgrade checks of one switch.
    # - flash (pre): the target image is already on flash, or free space covers its size on the image server
    #   (image_url, or image_base_url + EOS-<target_version>.swi; required_free_mb without an image server or when its
    #   HEAD has no Content-Length). The HEAD checks the certificate and, if the worker doesn't trust it, sizes the image
    #   anyway: notes says when either happens, and when activate will need its sha512 input (untrusted certificate, or
    #   no <image>.sha512 reachable). A malformed image URL is flash ko.
    # - redundancy (pre): MLAG active and connected, peer-link and local interface up, config-sanity clean, no MLAG
    #   port active-partial, no port errdisabled. Left out when MLAG is not configured (notes says the reload is not
    #   hitless); configured but down is ko.
    # - interface: pre saves a snapshot (interfaces up, LLDP neighbor per port, MLAG state and per-port status) in the
    #   flow run's user state, on the root flow, under arista_eos_<host>; post compares the switch with it.
    # Post waits, in order: for eAPI to answer on a boot later than this run's reload (return_timeout_seconds), for the
    # MLAG reload-delay counted from boot when MLAG is configured, then up to stabilisation_seconds for the state to
    # match the snapshot. The three are returned in durations. The version is reported, not compared with the target.
    
    import logging
    import os
    import re
    import ssl
    import time
    import urllib.error
    import urllib.request
    from typing import Literal, TypedDict
    from urllib.parse import urlsplit
    
    import pyeapi
    import wmill
    
    
    class arista_eos(TypedDict):
        host: str
        username: str
        password: str
        port: int
        verify_tls: bool
        allow_http: bool
    
    
    def main(
        auth: arista_eos,
        phase: Literal["pre", "post"],
        target_version: str = "",
        image_base_url: str = "",
        required_free_mb: int = 1024,
        return_timeout_seconds: int = 900,
        stabilisation_seconds: int = 120,
        poll_interval_seconds: int = 15,
        image_url: str = "",
    ):
        """
        Device check
        Pre- or post-upgrade check of one switch: returns { running_version, status ok|ko, checks_ok, checks_ko[{check, message}], notes }. Pre: flash space for the target image, MLAG redundancy, and a snapshot of interfaces/LLDP/MLAG kept in the flow run's state. Post: waits for the switch to answer and for the MLAG reload-delay (when MLAG is configured), then compares with the snapshot over a bounded stabilisation window. Throws DeviceUnreachable or AuthFailure. Does not compare the version with the target: the workflow does.
        """
        poll = poll_interval_seconds or 15
        durations = {}
        key = "arista_eos_" + re.sub(r"\W", "_", auth["host"])
        state = load_state(key) or {}
    
        if phase == "post":
            # Wait for the switch to answer on a boot later than the one each reload of this run was sent on: `reload now`
            # can answer before EOS goes down, and the old boot must not pass for the new one. AuthFailure stops at once.
            markers = [state[k] for k in ("reload", "rollback") if state.get(k)]
            timeout = return_timeout_seconds or 900
            start = time.time()
            while True:
                try:
                    [v] = eapi(auth, ["show version"])
                    m = next(
                        (m for m in markers if m["boot_timestamp"] == v["bootupTimestamp"]),
                        None,
                    )
                    if not m:
                        break
                    pending = (
                        f"still on the boot the reload sent at {m['sent_at']} was sent on"
                    )
                except DeviceUnreachable as e:
                    pending = str(e)
                if time.time() - start > timeout:
                    raise named(
                        DeviceUnreachable,
                        f"{auth['host']}: not back after {timeout} s ({pending})",
                    )
                time.sleep(poll)
            durations["return_timeout_seconds"] = timeout
            durations["returned_after_seconds"] = round(time.time() - start)
    
        version, mlag = eapi(auth, ["show version", "show mlag"])
        running_version = short_version(version["version"])
        # Without MLAG configured, show mlag still reports the default reload-delay, and nothing waits on it.
        mlag_on = mlag_configured(mlag)
        durations["reload_delay_seconds"] = (
            max(mlag.get("reloadDelay") or 0, mlag.get("reloadDelayNonMlag") or 0)
            if mlag_on
            else 0
        )
    
        ok, ko, notes = [], [], []
    
        def record(check, problems):
            if problems:
                ko.append({"check": check, "message": "; ".join(problems)})
            else:
                ok.append(check)
    
        if phase == "pre":
            record(
                "flash",
                flash_problems(
                    auth, target_version, image_base_url, image_url, required_free_mb, notes
                ),
            )
            if mlag_on:
                [sanity] = eapi(auth, ["show mlag config-sanity"])
                record("redundancy", redundancy_problems(mlag, sanity))
            else:
                notes.append("redundancy: MLAG not configured, the reload is not hitless")
            save_state(
                key,
                {
                    **state,
                    "snapshot": snapshot(auth),
                    "snapshot_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
                },
            )
            ok.append("interface")
        elif not state.get("snapshot"):
            ko.append(
                {
                    "check": "interface",
                    "message": f"No pre-check snapshot for {auth['host']} in this flow run: run the pre-check first, in the same run",
                }
            )
        else:
            # Ports stay errdisabled until the MLAG reload-delay has elapsed since boot: wait it out.
            wait = durations["reload_delay_seconds"] - version["uptime"]
            if wait > 0:
                time.sleep(wait)
            durations["reload_delay_waited_seconds"] = max(0, round(wait))
            stab = 120 if stabilisation_seconds is None else stabilisation_seconds
            start = time.time()
            problems = diff(state["snapshot"], snapshot(auth))
            while problems and time.time() - start < stab:
                time.sleep(poll)
                problems = diff(state["snapshot"], snapshot(auth))
            durations["stabilisation_seconds"] = stab
            durations["stabilised_after_seconds"] = round(time.time() - start)
            record("interface", problems)
    
        return {
            "running_version": running_version,
            "status": "ko" if ko else "ok",
            "checks_ok": ok,
            "checks_ko": ko,
            "durations": durations,
            "notes": notes,
        }
    
    
    def flash_problems(auth, target, base, image_url, required_mb, notes):
        [d] = eapi(auth, [{"cmd": "dir flash:/", "revision": 2}])
        flash = d["urls"]["flash:/"]
        try:
            image, url = image_source(target, base, image_url)
        except ValueError as e:
            return [str(e)]
        if image and image in flash["entries"]:
            if url:
                sidecar_note(url, image, notes)
            return []
        required = (required_mb or 1024) * 1e6
        if url:
            try:
                try:
                    size = head(url, ssl.create_default_context())
                    sidecar_note(url, image, notes)
                except urllib.error.URLError as e:
                    if not isinstance(e.reason, ssl.SSLCertVerificationError):
                        raise
                    # Only the size is read: the switch's copy doesn't check the certificate either, and activate
                    # verifies the image's sha512. So an untrusted (self-signed) server is sized anyway, with a note.
                    unverified = ssl.create_default_context()
                    unverified.check_hostname = False
                    unverified.verify_mode = ssl.CERT_NONE
                    size = head(url, unverified)
                    notes.append(
                        f"flash: the image server's certificate is not trusted on the worker ({e.reason.verify_message}), sized {image} without checking it; activate checks it when it fetches {image}.sha512, so pass it sha512"
                    )
                if size:
                    required = size
                else:
                    notes.append(
                        f"flash: the image server gave no size for {image}, assumed required_free_mb ({mb(required)})"
                    )
            except urllib.error.HTTPError as e:
                return [f"Target image {image} not found on the image server ({e.code})"]
            except OSError as e:
                return [
                    f"Image server unreachable from the worker: {getattr(e, 'reason', e)}"
                ]
        free = flash["filesystem"]["freeBytes"]
        if free < required:
            return [
                f"Insufficient flash space ({mb(free)} free, {mb(required)} required{f' for {image}' if image else ''})"
            ]
        return []
    
    
    def head(url, context):
        with urllib.request.urlopen(
            urllib.request.Request(url, method="HEAD"), timeout=30, context=context
        ) as r:
            return int(r.headers.get("content-length") or 0)
    
    
    # Activate reads the digest from the .sha512 file next to the image, certificate checked, unless given sha512.
    def sidecar_note(url, image, notes):
        try:
            head(f"{url}.sha512", ssl.create_default_context())
        except OSError as e:
            reason = getattr(e, "code", None) or getattr(e, "reason", e)
            notes.append(
                f"flash: {image}.sha512 not reachable from the worker ({reason}): pass sha512 to activate"
            )
    
    
    # Same function in Device check and Device upgrade.
    # image_url wins; else image_base_url + EOS-<target>.swi. Returns (file name on flash, URL or ""). The URL goes into
    # the switch's copy command, so it is an allowlist: http(s), then only A-Z a-z 0-9 . _ ~ : / % + - (no query, no
    # user:password@, nothing the CLI reads as a separator), ending in a plain .swi file name. Raises ValueError otherwise.
    def image_source(target, base, image_url):
        if image_url:
            image, url = urlsplit(image_url).path.rsplit("/", 1)[-1], image_url
        else:
            image = f"EOS-{target}.swi" if target else ""
            url = f"{base.rstrip('/')}/{image}" if image and base else ""
        if not url:
            return image, url
        # Not echoed: it would put the password in the steps and errors.
        if "@" in url:
            raise ValueError("the image URL must not contain @ (no user:password@ in it)")
        if not re.match(r"https?://", url):
            raise ValueError(
                f"the image URL must start with http:// or https://: got {url!r}"
            )
        if not urlsplit(url).hostname:
            raise ValueError(f"the image URL has no host: got {url!r}")
        bad = sorted(set(re.sub(r"[A-Za-z0-9._~:/%+-]", "", url.split("://", 1)[1])))
        if bad:
            raise ValueError(
                f"the image URL may only contain A-Z a-z 0-9 . _ ~ : / % + -, not {' '.join(map(repr, bad))}: got {url!r}"
            )
        if not re.fullmatch(r"[\w.+-]+\.swi", image):
            raise ValueError(f"the image URL must end in a .swi file name: got {url!r}")
        return image, url
    
    
    # show mlag reports state disabled both when MLAG was never configured and when it is configured but shut down;
    # only the first has no domain-id, local interface or peer-link.
    def mlag_configured(mlag):
        return any(mlag.get(k) for k in ("domainId", "localInterface", "peerLink"))
    
    
    def redundancy_problems(mlag, sanity):
        p = []
        if mlag.get("state") != "active":
            p.append(f"MLAG state {mlag.get('state')} (expected active)")
        if mlag.get("negStatus") != "connected":
            p.append(f"MLAG negotiation {mlag.get('negStatus')} (expected connected)")
        if mlag.get("peerLinkStatus") != "up":
            p.append(f"peer-link {mlag.get('peerLink')} {mlag.get('peerLinkStatus')}")
        if mlag.get("localIntfStatus") != "up":
            p.append(
                f"MLAG local interface {mlag.get('localInterface')} {mlag.get('localIntfStatus')}"
            )
        inconsistent = [
            *(sanity.get("globalConfiguration") or {}),
            *(sanity.get("interfaceConfiguration") or {}),
        ]
        if inconsistent:
            p.append(f"config-sanity inconsistent: {', '.join(inconsistent)}")
        partial = (mlag.get("mlagPorts") or {}).get("Active-partial")
        if partial:
            p.append(f"{partial} MLAG port(s) active-partial")
        if mlag.get("portsErrdisabled"):
            p.append("MLAG ports errdisabled")
        return p
    
    
    def snapshot(auth):
        status, lldp, mlag, mlag_intf = eapi(
            auth,
            [
                "show interfaces status",
                "show lldp neighbors",
                "show mlag",
                "show mlag interfaces",
            ],
        )
        return {
            "interfaces_up": sorted(
                n
                for n, i in status["interfaceStatuses"].items()
                if i.get("linkStatus") == "connected"
            ),
            "lldp": [
                {
                    "port": n["port"],
                    "neighbor": f"{n['neighborDevice']} {n['neighborPort']}",
                }
                for n in lldp["lldpNeighbors"]
            ],
            "mlag": {
                "state": mlag.get("state"),
                "negStatus": mlag.get("negStatus"),
                "peerLinkStatus": mlag.get("peerLinkStatus"),
                "ports": {
                    f"{i['localInterface']} (mlag {id})": i["status"]
                    for id, i in (mlag_intf.get("interfaces") or {}).items()
                },
            },
        }
    
    
    def diff(before, after):
        p = []
        for i in before["interfaces_up"]:
            if i not in after["interfaces_up"]:
                p.append(f"{i} down after the upgrade (up before)")
        for n in before["lldp"]:
            if not any(
                a["port"] == n["port"] and a["neighbor"] == n["neighbor"]
                for a in after["lldp"]
            ):
                p.append(f"LLDP neighbor {n['neighbor']} no longer seen on {n['port']}")
        for k in ("state", "negStatus", "peerLinkStatus"):
            if before["mlag"][k] != after["mlag"][k]:
                p.append(f"MLAG {k} {after['mlag'][k]} (was {before['mlag'][k]})")
        for port, s in before["mlag"]["ports"].items():
            if after["mlag"]["ports"].get(port) != s:
                p.append(f"{port} {after['mlag']['ports'].get(port, 'missing')} (was {s})")
        return p
    
    
    # The snapshot lives in the root flow run's user state, keyed by switch: pre- and post-check are different steps
    # (often in nested subflows) of the same run. Outside a flow, it falls back to this script's own state.
    # The endpoint is called through the client so an unreachable state raises (wmill.get_flow_user_state returns None).
    # ponytail: the fallback is one map per script, so concurrent standalone runs can race; flows don't.
    def load_state(key):
        if os.environ.get("WM_FLOW_JOB_ID"):
            c = wmill.Windmill()
            return c.get(
                f"/w/{c.workspace}/jobs/flow/user_states/{c.get_root_job_id()}/{key}"
            ).json()
        return (wmill.get_state() or {}).get(key)
    
    
    def save_state(key, value):
        if os.environ.get("WM_FLOW_JOB_ID"):
            c = wmill.Windmill()
            c.post(
                f"/w/{c.workspace}/jobs/flow/user_states/{c.get_root_job_id()}/{key}",
                json=value,
            )
        else:
            wmill.set_state({**(wmill.get_state() or {}), key: value})
    
    
    # "4.34.9M-51132493.4349M (engineering build)" on cEOS, "4.34.9M" on hardware.
    def short_version(v):
        m = re.match(r"\d+\.\d+\.\d+(\.\d+)?[A-Z]*", v)
        return m.group(0) if m else v
    
    
    def mb(n):
        return f"{n / 1e9:.1f} GB" if n >= 1e9 else f"{round(n / 1e6)} MB"
    
    
    class DeviceUnreachable(Exception):
        pass
    
    
    class AuthFailure(Exception):
        pass
    
    
    def named(cls, message: str):
        return cls(f"{cls.__name__}: {message}")
    
    
    # pyeapi logs a traceback for every socket error, and the post-check polls a switch that is rebooting.
    logging.getLogger("pyeapi").addHandler(logging.NullHandler())
    
    
    def eapi(auth: arista_eos, cmds: list, fmt: str = "json", timeout: float = 60):
        eos = node(auth, timeout)
        try:
            return eos.run_commands(cmds, fmt)
        except pyeapi.eapilib.ConnectionError as e:
            raise connection_error(auth, eos, e)
    
    
    # pyeapi 1.0.4 raises ConnectionError for a socket error (kept in socket_error), an answer that isn't JSON (eAPI
    # still starting, 5xx) and a 401, the only one with neither.
    def connection_error(auth, eos, e):
        if eos.connection.socket_error is None and e.message != "unable to connect to eAPI":
            return named(
                AuthFailure, f"{auth['host']}: eAPI refused the credential ({e.message})"
            )
        return named(DeviceUnreachable, f"{auth['host']}: {e.message}")
    
    
    # host is an address or a URL (https://10.0.0.11:8443); a bare IPv6 address is bracketed. Only https, or http when the
    # resource ticks allow_http (Basic auth then crosses the network in cleartext). pyeapi skips certificate checks unless it
    # is given a context, so one is always passed: verifying, or not when the resource unticks verify_tls (EOS ships a
    # self-signed cert).
    def node(auth: arista_eos, timeout: float = 60):
        host = auth["host"]
        if "://" not in host:
            host = f"https://[{host}]" if host.count(":") > 1 and "[" not in host else f"https://{host}"
        url = urlsplit(host)
        if not url.hostname or (url.scheme != "https" and not (url.scheme == "http" and auth.get("allow_http"))):
            raise ValueError(f"host must be an address or an https:// URL (http:// needs allow_http on the resource): {auth['host']}")
        context = ssl.create_default_context()
        if auth.get("verify_tls") is False:
            context.check_hostname = False
            context.verify_mode = ssl.CERT_NONE
        return pyeapi.connect(
            transport=url.scheme,
            host=url.hostname,
            port=url.port or auth.get("port") or None,
            username=auth["username"],
            password=auth["password"],
            context=context,
            timeout=timeout,
            return_node=True,
        )
    

    Submitted by hugo989 17 days ago

  • python3
    # py312
    # requirements:
    # pyeapi==1.0.4
    # wmill==1.851.0
    
    # Device check: pre- and post-upgrade checks of one switch.
    # - flash (pre): the target image is already on flash, or free space covers its size on the image server
    #   (image_url, or image_base_url + EOS-<target_version>.swi; required_free_mb without an image server or when its
    #   HEAD has no Content-Length). The HEAD checks the certificate and, if the worker doesn't trust it, sizes the image
    #   anyway: notes says when either happens, and when activate will need its sha512 input (untrusted certificate, or
    #   no <image>.sha512 reachable). A malformed image URL is flash ko.
    # - redundancy (pre): MLAG active and connected, peer-link and local interface up, config-sanity clean, no MLAG
    #   port active-partial, no port errdisabled. Left out when MLAG is not configured (notes says the reload is not
    #   hitless); configured but down is ko.
    # - interface: pre saves a snapshot (interfaces up, LLDP neighbor per port, MLAG state and per-port status) in the
    #   flow run's user state, on the root flow, under arista_eos_<host>; post compares the switch with it.
    # Post waits, in order: for eAPI to answer on a boot later than this run's reload (return_timeout_seconds), for the
    # MLAG reload-delay counted from boot when MLAG is configured, then up to stabilisation_seconds for the state to
    # match the snapshot. The three are returned in durations. The version is reported, not compared with the target.
    
    import logging
    import os
    import re
    import ssl
    import time
    import urllib.error
    import urllib.request
    from typing import Literal, TypedDict
    from urllib.parse import urlsplit
    
    import pyeapi
    import wmill
    
    
    class arista_eos(TypedDict):
        host: str
        username: str
        password: str
        port: int
        verify_tls: bool
    
    
    def main(
        auth: arista_eos,
        phase: Literal["pre", "post"],
        target_version: str = "",
        image_base_url: str = "",
        required_free_mb: int = 1024,
        return_timeout_seconds: int = 900,
        stabilisation_seconds: int = 120,
        poll_interval_seconds: int = 15,
        image_url: str = "",
    ):
        """
        Device check
        Pre- or post-upgrade check of one switch: returns { running_version, status ok|ko, checks_ok, checks_ko[{check, message}], notes }. Pre: flash space for the target image, MLAG redundancy, and a snapshot of interfaces/LLDP/MLAG kept in the flow run's state. Post: waits for the switch to answer and for the MLAG reload-delay (when MLAG is configured), then compares with the snapshot over a bounded stabilisation window. Throws DeviceUnreachable or AuthFailure. Does not compare the version with the target: the workflow does.
        """
        poll = poll_interval_seconds or 15
        durations = {}
        key = "arista_eos_" + re.sub(r"\W", "_", auth["host"])
        state = load_state(key) or {}
    
        if phase == "post":
            # Wait for the switch to answer on a boot later than the one each reload of this run was sent on: `reload now`
            # can answer before EOS goes down, and the old boot must not pass for the new one. AuthFailure stops at once.
            markers = [state[k] for k in ("reload", "rollback") if state.get(k)]
            timeout = return_timeout_seconds or 900
            start = time.time()
            while True:
                try:
                    [v] = eapi(auth, ["show version"])
                    m = next(
                        (m for m in markers if m["boot_timestamp"] == v["bootupTimestamp"]),
                        None,
                    )
                    if not m:
                        break
                    pending = (
                        f"still on the boot the reload sent at {m['sent_at']} was sent on"
                    )
                except DeviceUnreachable as e:
                    pending = str(e)
                if time.time() - start > timeout:
                    raise named(
                        DeviceUnreachable,
                        f"{auth['host']}: not back after {timeout} s ({pending})",
                    )
                time.sleep(poll)
            durations["return_timeout_seconds"] = timeout
            durations["returned_after_seconds"] = round(time.time() - start)
    
        version, mlag = eapi(auth, ["show version", "show mlag"])
        running_version = short_version(version["version"])
        # Without MLAG configured, show mlag still reports the default reload-delay, and nothing waits on it.
        mlag_on = mlag_configured(mlag)
        durations["reload_delay_seconds"] = (
            max(mlag.get("reloadDelay") or 0, mlag.get("reloadDelayNonMlag") or 0)
            if mlag_on
            else 0
        )
    
        ok, ko, notes = [], [], []
    
        def record(check, problems):
            if problems:
                ko.append({"check": check, "message": "; ".join(problems)})
            else:
                ok.append(check)
    
        if phase == "pre":
            record(
                "flash",
                flash_problems(
                    auth, target_version, image_base_url, image_url, required_free_mb, notes
                ),
            )
            if mlag_on:
                [sanity] = eapi(auth, ["show mlag config-sanity"])
                record("redundancy", redundancy_problems(mlag, sanity))
            else:
                notes.append("redundancy: MLAG not configured, the reload is not hitless")
            save_state(
                key,
                {
                    **state,
                    "snapshot": snapshot(auth),
                    "snapshot_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
                },
            )
            ok.append("interface")
        elif not state.get("snapshot"):
            ko.append(
                {
                    "check": "interface",
                    "message": f"No pre-check snapshot for {auth['host']} in this flow run: run the pre-check first, in the same run",
                }
            )
        else:
            # Ports stay errdisabled until the MLAG reload-delay has elapsed since boot: wait it out.
            wait = durations["reload_delay_seconds"] - version["uptime"]
            if wait > 0:
                time.sleep(wait)
            durations["reload_delay_waited_seconds"] = max(0, round(wait))
            stab = 120 if stabilisation_seconds is None else stabilisation_seconds
            start = time.time()
            problems = diff(state["snapshot"], snapshot(auth))
            while problems and time.time() - start < stab:
                time.sleep(poll)
                problems = diff(state["snapshot"], snapshot(auth))
            durations["stabilisation_seconds"] = stab
            durations["stabilised_after_seconds"] = round(time.time() - start)
            record("interface", problems)
    
        return {
            "running_version": running_version,
            "status": "ko" if ko else "ok",
            "checks_ok": ok,
            "checks_ko": ko,
            "durations": durations,
            "notes": notes,
        }
    
    
    def flash_problems(auth, target, base, image_url, required_mb, notes):
        [d] = eapi(auth, [{"cmd": "dir flash:/", "revision": 2}])
        flash = d["urls"]["flash:/"]
        try:
            image, url = image_source(target, base, image_url)
        except ValueError as e:
            return [str(e)]
        if image and image in flash["entries"]:
            if url:
                sidecar_note(url, image, notes)
            return []
        required = (required_mb or 1024) * 1e6
        if url:
            try:
                try:
                    size = head(url, ssl.create_default_context())
                    sidecar_note(url, image, notes)
                except urllib.error.URLError as e:
                    if not isinstance(e.reason, ssl.SSLCertVerificationError):
                        raise
                    # Only the size is read: the switch's copy doesn't check the certificate either, and activate
                    # verifies the image's sha512. So an untrusted (self-signed) server is sized anyway, with a note.
                    unverified = ssl.create_default_context()
                    unverified.check_hostname = False
                    unverified.verify_mode = ssl.CERT_NONE
                    size = head(url, unverified)
                    notes.append(
                        f"flash: the image server's certificate is not trusted on the worker ({e.reason.verify_message}), sized {image} without checking it; activate checks it when it fetches {image}.sha512, so pass it sha512"
                    )
                if size:
                    required = size
                else:
                    notes.append(
                        f"flash: the image server gave no size for {image}, assumed required_free_mb ({mb(required)})"
                    )
            except urllib.error.HTTPError as e:
                return [f"Target image {image} not found on the image server ({e.code})"]
            except OSError as e:
                return [
                    f"Image server unreachable from the worker: {getattr(e, 'reason', e)}"
                ]
        free = flash["filesystem"]["freeBytes"]
        if free < required:
            return [
                f"Insufficient flash space ({mb(free)} free, {mb(required)} required{f' for {image}' if image else ''})"
            ]
        return []
    
    
    def head(url, context):
        with urllib.request.urlopen(
            urllib.request.Request(url, method="HEAD"), timeout=30, context=context
        ) as r:
            return int(r.headers.get("content-length") or 0)
    
    
    # Activate reads the digest from the .sha512 file next to the image, certificate checked, unless given sha512.
    def sidecar_note(url, image, notes):
        try:
            head(f"{url}.sha512", ssl.create_default_context())
        except OSError as e:
            reason = getattr(e, "code", None) or getattr(e, "reason", e)
            notes.append(
                f"flash: {image}.sha512 not reachable from the worker ({reason}): pass sha512 to activate"
            )
    
    
    # Same function in Device check and Device upgrade.
    # image_url wins; else image_base_url + EOS-<target>.swi. Returns (file name on flash, URL or ""). The URL goes into
    # the switch's copy command, so it is an allowlist: http(s), then only A-Z a-z 0-9 . _ ~ : / % + - (no query, no
    # user:password@, nothing the CLI reads as a separator), ending in a plain .swi file name. Raises ValueError otherwise.
    def image_source(target, base, image_url):
        if image_url:
            image, url = urlsplit(image_url).path.rsplit("/", 1)[-1], image_url
        else:
            image = f"EOS-{target}.swi" if target else ""
            url = f"{base.rstrip('/')}/{image}" if image and base else ""
        if not url:
            return image, url
        # Not echoed: it would put the password in the steps and errors.
        if "@" in url:
            raise ValueError("the image URL must not contain @ (no user:password@ in it)")
        if not re.match(r"https?://", url):
            raise ValueError(
                f"the image URL must start with http:// or https://: got {url!r}"
            )
        if not urlsplit(url).hostname:
            raise ValueError(f"the image URL has no host: got {url!r}")
        bad = sorted(set(re.sub(r"[A-Za-z0-9._~:/%+-]", "", url.split("://", 1)[1])))
        if bad:
            raise ValueError(
                f"the image URL may only contain A-Z a-z 0-9 . _ ~ : / % + -, not {' '.join(map(repr, bad))}: got {url!r}"
            )
        if not re.fullmatch(r"[\w.+-]+\.swi", image):
            raise ValueError(f"the image URL must end in a .swi file name: got {url!r}")
        return image, url
    
    
    # show mlag reports state disabled both when MLAG was never configured and when it is configured but shut down;
    # only the first has no domain-id, local interface or peer-link.
    def mlag_configured(mlag):
        return any(mlag.get(k) for k in ("domainId", "localInterface", "peerLink"))
    
    
    def redundancy_problems(mlag, sanity):
        p = []
        if mlag.get("state") != "active":
            p.append(f"MLAG state {mlag.get('state')} (expected active)")
        if mlag.get("negStatus") != "connected":
            p.append(f"MLAG negotiation {mlag.get('negStatus')} (expected connected)")
        if mlag.get("peerLinkStatus") != "up":
            p.append(f"peer-link {mlag.get('peerLink')} {mlag.get('peerLinkStatus')}")
        if mlag.get("localIntfStatus") != "up":
            p.append(
                f"MLAG local interface {mlag.get('localInterface')} {mlag.get('localIntfStatus')}"
            )
        inconsistent = [
            *(sanity.get("globalConfiguration") or {}),
            *(sanity.get("interfaceConfiguration") or {}),
        ]
        if inconsistent:
            p.append(f"config-sanity inconsistent: {', '.join(inconsistent)}")
        partial = (mlag.get("mlagPorts") or {}).get("Active-partial")
        if partial:
            p.append(f"{partial} MLAG port(s) active-partial")
        if mlag.get("portsErrdisabled"):
            p.append("MLAG ports errdisabled")
        return p
    
    
    def snapshot(auth):
        status, lldp, mlag, mlag_intf = eapi(
            auth,
            [
                "show interfaces status",
                "show lldp neighbors",
                "show mlag",
                "show mlag interfaces",
            ],
        )
        return {
            "interfaces_up": sorted(
                n
                for n, i in status["interfaceStatuses"].items()
                if i.get("linkStatus") == "connected"
            ),
            "lldp": [
                {
                    "port": n["port"],
                    "neighbor": f"{n['neighborDevice']} {n['neighborPort']}",
                }
                for n in lldp["lldpNeighbors"]
            ],
            "mlag": {
                "state": mlag.get("state"),
                "negStatus": mlag.get("negStatus"),
                "peerLinkStatus": mlag.get("peerLinkStatus"),
                "ports": {
                    f"{i['localInterface']} (mlag {id})": i["status"]
                    for id, i in (mlag_intf.get("interfaces") or {}).items()
                },
            },
        }
    
    
    def diff(before, after):
        p = []
        for i in before["interfaces_up"]:
            if i not in after["interfaces_up"]:
                p.append(f"{i} down after the upgrade (up before)")
        for n in before["lldp"]:
            if not any(
                a["port"] == n["port"] and a["neighbor"] == n["neighbor"]
                for a in after["lldp"]
            ):
                p.append(f"LLDP neighbor {n['neighbor']} no longer seen on {n['port']}")
        for k in ("state", "negStatus", "peerLinkStatus"):
            if before["mlag"][k] != after["mlag"][k]:
                p.append(f"MLAG {k} {after['mlag'][k]} (was {before['mlag'][k]})")
        for port, s in before["mlag"]["ports"].items():
            if after["mlag"]["ports"].get(port) != s:
                p.append(f"{port} {after['mlag']['ports'].get(port, 'missing')} (was {s})")
        return p
    
    
    # The snapshot lives in the root flow run's user state, keyed by switch: pre- and post-check are different steps
    # (often in nested subflows) of the same run. Outside a flow, it falls back to this script's own state.
    # The endpoint is called through the client so an unreachable state raises (wmill.get_flow_user_state returns None).
    # ponytail: the fallback is one map per script, so concurrent standalone runs can race; flows don't.
    def load_state(key):
        if os.environ.get("WM_FLOW_JOB_ID"):
            c = wmill.Windmill()
            return c.get(
                f"/w/{c.workspace}/jobs/flow/user_states/{c.get_root_job_id()}/{key}"
            ).json()
        return (wmill.get_state() or {}).get(key)
    
    
    def save_state(key, value):
        if os.environ.get("WM_FLOW_JOB_ID"):
            c = wmill.Windmill()
            c.post(
                f"/w/{c.workspace}/jobs/flow/user_states/{c.get_root_job_id()}/{key}",
                json=value,
            )
        else:
            wmill.set_state({**(wmill.get_state() or {}), key: value})
    
    
    # "4.34.9M-51132493.4349M (engineering build)" on cEOS, "4.34.9M" on hardware.
    def short_version(v):
        m = re.match(r"\d+\.\d+\.\d+(\.\d+)?[A-Z]*", v)
        return m.group(0) if m else v
    
    
    def mb(n):
        return f"{n / 1e9:.1f} GB" if n >= 1e9 else f"{round(n / 1e6)} MB"
    
    
    class DeviceUnreachable(Exception):
        pass
    
    
    class AuthFailure(Exception):
        pass
    
    
    def named(cls, message: str):
        return cls(f"{cls.__name__}: {message}")
    
    
    # pyeapi logs a traceback for every socket error, and the post-check polls a switch that is rebooting.
    logging.getLogger("pyeapi").addHandler(logging.NullHandler())
    
    
    def eapi(auth: arista_eos, cmds: list, fmt: str = "json", timeout: float = 60):
        eos = node(auth, timeout)
        try:
            return eos.run_commands(cmds, fmt)
        except pyeapi.eapilib.ConnectionError as e:
            raise connection_error(auth, eos, e)
    
    
    # pyeapi 1.0.4 raises ConnectionError for a socket error (kept in socket_error), an answer that isn't JSON (eAPI
    # still starting, 5xx) and a 401, the only one with neither.
    def connection_error(auth, eos, e):
        if eos.connection.socket_error is None and e.message != "unable to connect to eAPI":
            return named(
                AuthFailure, f"{auth['host']}: eAPI refused the credential ({e.message})"
            )
        return named(DeviceUnreachable, f"{auth['host']}: {e.message}")
    
    
    # host is an address or a URL (https://10.0.0.11:8443). pyeapi skips certificate checks unless it is given a context,
    # so one is always passed: verifying, or not when the resource unticks verify_tls (EOS ships a self-signed cert).
    def node(auth: arista_eos, timeout: float = 60):
        url = urlsplit(auth["host"] if "://" in auth["host"] else f"https://{auth['host']}")
        context = ssl.create_default_context()
        if auth.get("verify_tls") is False:
            context.check_hostname = False
            context.verify_mode = ssl.CERT_NONE
        return pyeapi.connect(
            transport=url.scheme,
            host=url.hostname,
            port=url.port or auth.get("port") or None,
            username=auth["username"],
            password=auth["password"],
            context=context,
            timeout=timeout,
            return_node=True,
        )
    

    Submitted by hugo989 17 days ago

  • python3
    # py312
    # requirements:
    # pyeapi==1.0.4
    # wmill==1.851.0
    
    import logging
    import os
    import re
    import ssl
    import time
    import urllib.error
    import urllib.request
    from typing import Literal, TypedDict
    from urllib.parse import urlsplit
    
    import pyeapi
    import wmill
    
    
    class arista_eos(TypedDict):
        host: str
        username: str
        password: str
        port: int
        verify_tls: bool
    
    
    def main(
        auth: arista_eos,
        phase: Literal["pre", "post"],
        target_version: str = "",
        image_base_url: str = "",
        required_free_mb: int = 1024,
        return_timeout_seconds: int = 900,
        stabilisation_seconds: int = 120,
        poll_interval_seconds: int = 15,
    ):
        """
        Device check
        Pre- or post-upgrade check of one switch: returns { running_version, status ok|ko, checks_ok, checks_ko[{check, message}] }. Pre: flash space for the target image, MLAG redundancy, and a snapshot of interfaces/LLDP/MLAG kept in the flow run's state. Post: waits for the switch to answer and for the MLAG reload-delay, then compares with the snapshot over a bounded stabilisation window. Throws DeviceUnreachable or AuthFailure. Does not compare the version with the target: the workflow does.
        """
        poll = poll_interval_seconds or 15
        durations = {}
        key = "arista_eos_" + re.sub(r"\W", "_", auth["host"])
        state = load_state(key) or {}
    
        if phase == "post":
            # Wait for the switch to answer on a boot later than the one each reload of this run was sent on: `reload now`
            # can answer before EOS goes down, and the old boot must not pass for the new one. AuthFailure stops at once.
            markers = [state[k] for k in ("reload", "rollback") if state.get(k)]
            timeout = return_timeout_seconds or 900
            start = time.time()
            while True:
                try:
                    [v] = eapi(auth, ["show version"])
                    m = next(
                        (m for m in markers if m["boot_timestamp"] == v["bootupTimestamp"]),
                        None,
                    )
                    if not m:
                        break
                    pending = (
                        f"still on the boot the reload sent at {m['sent_at']} was sent on"
                    )
                except DeviceUnreachable as e:
                    pending = str(e)
                if time.time() - start > timeout:
                    raise named(
                        DeviceUnreachable,
                        f"{auth['host']}: not back after {timeout} s ({pending})",
                    )
                time.sleep(poll)
            durations["return_timeout_seconds"] = timeout
            durations["returned_after_seconds"] = round(time.time() - start)
    
        version, mlag = eapi(auth, ["show version", "show mlag"])
        running_version = short_version(version["version"])
        durations["reload_delay_seconds"] = max(
            mlag.get("reloadDelay") or 0, mlag.get("reloadDelayNonMlag") or 0
        )
    
        ok, ko = [], []
    
        def record(check, problems):
            if problems:
                ko.append({"check": check, "message": "; ".join(problems)})
            else:
                ok.append(check)
    
        if phase == "pre":
            record(
                "flash",
                flash_problems(auth, target_version, image_base_url, required_free_mb),
            )
            [sanity] = eapi(auth, ["show mlag config-sanity"])
            record("redundancy", redundancy_problems(mlag, sanity))
            save_state(
                key,
                {
                    **state,
                    "snapshot": snapshot(auth),
                    "snapshot_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
                },
            )
            ok.append("interface")
        elif not state.get("snapshot"):
            ko.append(
                {
                    "check": "interface",
                    "message": f"No pre-check snapshot for {auth['host']} in this flow run: run the pre-check first, in the same run",
                }
            )
        else:
            # Ports stay errdisabled until the MLAG reload-delay has elapsed since boot: wait it out.
            wait = durations["reload_delay_seconds"] - version["uptime"]
            if wait > 0:
                time.sleep(wait)
            durations["reload_delay_waited_seconds"] = max(0, round(wait))
            stab = 120 if stabilisation_seconds is None else stabilisation_seconds
            start = time.time()
            problems = diff(state["snapshot"], snapshot(auth))
            while problems and time.time() - start < stab:
                time.sleep(poll)
                problems = diff(state["snapshot"], snapshot(auth))
            durations["stabilisation_seconds"] = stab
            durations["stabilised_after_seconds"] = round(time.time() - start)
            record("interface", problems)
    
        return {
            "running_version": running_version,
            "status": "ko" if ko else "ok",
            "checks_ok": ok,
            "checks_ko": ko,
            "durations": durations,
        }
    
    
    def flash_problems(auth, target, base, required_mb):
        [d] = eapi(auth, [{"cmd": "dir flash:/", "revision": 2}])
        flash = d["urls"]["flash:/"]
        image = f"EOS-{target}.swi" if target else ""
        if image and image in flash["entries"]:
            return []
        required = (required_mb or 1024) * 1e6
        if image and base:
            try:
                with urllib.request.urlopen(
                    urllib.request.Request(f"{base.rstrip('/')}/{image}", method="HEAD"),
                    timeout=30,
                ) as head:
                    required = int(head.headers.get("content-length") or 0) or required
            except urllib.error.HTTPError as e:
                return [f"Target image {image} not found on the image server ({e.code})"]
            except OSError as e:
                return [
                    f"Image server unreachable from the worker: {getattr(e, 'reason', e)}"
                ]
        free = flash["filesystem"]["freeBytes"]
        if free < required:
            return [
                f"Insufficient flash space ({mb(free)} free, {mb(required)} required{f' for {image}' if image else ''})"
            ]
        return []
    
    
    def redundancy_problems(mlag, sanity):
        if mlag.get("state") == "disabled":
            return [
                "MLAG is not configured: no peer to carry the traffic during the reload"
            ]
        p = []
        if mlag.get("state") != "active":
            p.append(f"MLAG state {mlag.get('state')} (expected active)")
        if mlag.get("negStatus") != "connected":
            p.append(f"MLAG negotiation {mlag.get('negStatus')} (expected connected)")
        if mlag.get("peerLinkStatus") != "up":
            p.append(f"peer-link {mlag.get('peerLink')} {mlag.get('peerLinkStatus')}")
        if mlag.get("localIntfStatus") != "up":
            p.append(
                f"MLAG local interface {mlag.get('localInterface')} {mlag.get('localIntfStatus')}"
            )
        inconsistent = [
            *(sanity.get("globalConfiguration") or {}),
            *(sanity.get("interfaceConfiguration") or {}),
        ]
        if inconsistent:
            p.append(f"config-sanity inconsistent: {', '.join(inconsistent)}")
        partial = (mlag.get("mlagPorts") or {}).get("Active-partial")
        if partial:
            p.append(f"{partial} MLAG port(s) active-partial")
        if mlag.get("portsErrdisabled"):
            p.append("MLAG ports errdisabled")
        return p
    
    
    def snapshot(auth):
        status, lldp, mlag, mlag_intf = eapi(
            auth,
            [
                "show interfaces status",
                "show lldp neighbors",
                "show mlag",
                "show mlag interfaces",
            ],
        )
        return {
            "interfaces_up": sorted(
                n
                for n, i in status["interfaceStatuses"].items()
                if i.get("linkStatus") == "connected"
            ),
            "lldp": [
                {
                    "port": n["port"],
                    "neighbor": f"{n['neighborDevice']} {n['neighborPort']}",
                }
                for n in lldp["lldpNeighbors"]
            ],
            "mlag": {
                "state": mlag.get("state"),
                "negStatus": mlag.get("negStatus"),
                "peerLinkStatus": mlag.get("peerLinkStatus"),
                "ports": {
                    f"{i['localInterface']} (mlag {id})": i["status"]
                    for id, i in (mlag_intf.get("interfaces") or {}).items()
                },
            },
        }
    
    
    def diff(before, after):
        p = []
        for i in before["interfaces_up"]:
            if i not in after["interfaces_up"]:
                p.append(f"{i} down after the upgrade (up before)")
        for n in before["lldp"]:
            if not any(
                a["port"] == n["port"] and a["neighbor"] == n["neighbor"]
                for a in after["lldp"]
            ):
                p.append(f"LLDP neighbor {n['neighbor']} no longer seen on {n['port']}")
        for k in ("state", "negStatus", "peerLinkStatus"):
            if before["mlag"][k] != after["mlag"][k]:
                p.append(f"MLAG {k} {after['mlag'][k]} (was {before['mlag'][k]})")
        for port, s in before["mlag"]["ports"].items():
            if after["mlag"]["ports"].get(port) != s:
                p.append(f"{port} {after['mlag']['ports'].get(port, 'missing')} (was {s})")
        return p
    
    
    # The snapshot lives in the root flow run's user state, keyed by switch: pre- and post-check are different steps
    # (often in nested subflows) of the same run. Outside a flow, it falls back to this script's own state.
    # The endpoint is called through the client so an unreachable state raises (wmill.get_flow_user_state returns None).
    # ponytail: the fallback is one map per script, so concurrent standalone runs can race; flows don't.
    def load_state(key):
        if os.environ.get("WM_FLOW_JOB_ID"):
            c = wmill.Windmill()
            return c.get(
                f"/w/{c.workspace}/jobs/flow/user_states/{c.get_root_job_id()}/{key}"
            ).json()
        return (wmill.get_state() or {}).get(key)
    
    
    def save_state(key, value):
        if os.environ.get("WM_FLOW_JOB_ID"):
            c = wmill.Windmill()
            c.post(
                f"/w/{c.workspace}/jobs/flow/user_states/{c.get_root_job_id()}/{key}",
                json=value,
            )
        else:
            wmill.set_state({**(wmill.get_state() or {}), key: value})
    
    
    # "4.34.9M-51132493.4349M (engineering build)" on cEOS, "4.34.9M" on hardware.
    def short_version(v):
        m = re.match(r"\d+\.\d+\.\d+(\.\d+)?[A-Z]*", v)
        return m.group(0) if m else v
    
    
    def mb(n):
        return f"{n / 1e9:.1f} GB" if n >= 1e9 else f"{round(n / 1e6)} MB"
    
    
    class DeviceUnreachable(Exception):
        pass
    
    
    class AuthFailure(Exception):
        pass
    
    
    def named(cls, message: str):
        return cls(f"{cls.__name__}: {message}")
    
    
    # pyeapi logs a traceback for every socket error, and the post-check polls a switch that is rebooting.
    logging.getLogger("pyeapi").addHandler(logging.NullHandler())
    
    
    def eapi(auth: arista_eos, cmds: list, fmt: str = "json", timeout: float = 60):
        eos = node(auth, timeout)
        try:
            return eos.run_commands(cmds, fmt)
        except pyeapi.eapilib.ConnectionError as e:
            raise connection_error(auth, eos, e)
    
    
    # pyeapi 1.0.4 raises ConnectionError for a socket error (kept in socket_error), an answer that isn't JSON (eAPI
    # still starting, 5xx) and a 401, the only one with neither.
    def connection_error(auth, eos, e):
        if eos.connection.socket_error is None and e.message != "unable to connect to eAPI":
            return named(
                AuthFailure, f"{auth['host']}: eAPI refused the credential ({e.message})"
            )
        return named(DeviceUnreachable, f"{auth['host']}: {e.message}")
    
    
    # host is an address or a URL (https://10.0.0.11:8443). pyeapi skips certificate checks unless it is given a context,
    # so one is always passed: verifying, or not when the resource unticks verify_tls (EOS ships a self-signed cert).
    def node(auth: arista_eos, timeout: float = 60):
        url = urlsplit(auth["host"] if "://" in auth["host"] else f"https://{auth['host']}")
        context = ssl.create_default_context()
        if auth.get("verify_tls") is False:
            context.check_hostname = False
            context.verify_mode = ssl.CERT_NONE
        return pyeapi.connect(
            transport=url.scheme,
            host=url.hostname,
            port=url.port or auth.get("port") or None,
            username=auth["username"],
            password=auth["password"],
            context=context,
            timeout=timeout,
            return_node=True,
        )
    

    Submitted by hugo989 18 days ago