Edits history of script submission #24219 for ' Get Reload Delay (arista_eos)'

  • python3
    One script reply has been approved by the moderators
    Ap­pro­ved
    # py312
    # requirements:
    # pyeapi==1.0.4
    # wmill==1.851.0
    
    # Get Reload Delay: one attempt, after a reload or rollback of one switch. Raises DeviceUnreachable while the switch
    # is down or still runs the boot a reload of this run was sent on, so the workflow retries this step until the switch
    # is back. Then returns the MLAG reload-delay (0 without MLAG) and the part of it still to run, counted from boot:
    # ports stay errdisabled until it ends, so the workflow sleeps remaining_seconds before Device post-check.
    
    import logging
    import os
    import re
    import ssl
    from typing import TypedDict
    from urllib.parse import urlsplit
    
    import pyeapi
    import wmill
    
    
    class arista_eos(TypedDict):
        host: str
        username: str
        password: str
        port: int
        verify_tls: bool
        allow_http: bool
    
    
    def main(auth: arista_eos):
        """
        Get Reload Delay
        After a reload, once the switch answers on a new boot: returns { running_version, uptime_seconds, reload_delay_seconds, remaining_seconds }, the MLAG reload-delay (0 without MLAG) and the seconds of it still to run; sleep remaining_seconds before Device post-check. Raises DeviceUnreachable while the switch is down or still on the boot the reload was sent on, AuthFailure when the credential is refused.
        """
        state = load_state(state_keys(auth)) or {}
        version = returned(auth, state)
        [mlag] = eapi(auth, ["show mlag"])
        delay = reload_delay(mlag)
        return {
            "running_version": short_version(version["version"]),
            "uptime_seconds": round(version["uptime"]),
            "reload_delay_seconds": delay,
            "remaining_seconds": max(0, round(delay - version["uptime"])),
        }
    
    
    # Same function in Get Reload Delay and Device post-check. One attempt: show version, raising DeviceUnreachable
    # while the switch is down, and while it still runs the boot a reload of this run was sent on (reload now can answer
    # before EOS goes down, and the old boot must not pass for the new one). AuthFailure is raised as is.
    def returned(auth, state):
        try:
            # Short: a switch that is down holds the attempt for the whole timeout, and the workflow retries anyway.
            [v] = eapi(auth, ["show version"], timeout=15)
        except DeviceUnreachable as e:
            raise DeviceUnreachable(f"{e}: not back yet")
        for k in ("reload", "rollback"):
            m = state.get(k)
            if m and m["boot_timestamp"] == v["bootupTimestamp"]:
                raise DeviceUnreachable(
                    f"{auth['host']}: still on the boot the {k} sent at {m['sent_at']} was sent on"
                )
        return v
    
    
    # show mlag reports state disabled both when MLAG was never configured and when it is configured but shut down;
    # only the first has no domain-id, local interface or peer-link.
    def mlag_configured(mlag):
        return any(mlag.get(k) for k in ("domainId", "localInterface", "peerLink"))
    
    
    # Without MLAG configured, show mlag still reports the default reload-delay, and nothing waits on it.
    def reload_delay(mlag):
        if not mlag_configured(mlag):
            return 0
        return max(mlag.get("reloadDelay") or 0, mlag.get("reloadDelayNonMlag") or 0)
    
    
    # Errors the workflow branches and retries on (retry_if: result.error.name == "DeviceUnreachable").
    class DeviceUnreachable(Exception):
        pass
    
    
    class AuthFailure(Exception):
        pass
    
    
    # pyeapi logs a traceback for every socket error, and a rebooting switch gives many.
    logging.getLogger("pyeapi").addHandler(logging.NullHandler())
    
    
    # This script runs right after a reboot, when eAPI can answer before the EOS agents behind a show command have
    # started: such an error answer counts as not back yet, so the workflow retries it. It only sends show commands.
    def eapi(auth: arista_eos, cmds: list, fmt: str = "json", timeout: float = 60):
        eos = node(auth, timeout)
        try:
            return eos.run_commands(cmds, fmt)
        except pyeapi.eapilib.ConnectionError as e:
            raise connection_error(auth, eos, e)
        except pyeapi.eapilib.CommandError as e:
            raise DeviceUnreachable(f"{auth['host']}: not ready yet ({e.message})")
    
    
    # pyeapi 1.0.4 raises ConnectionError for a socket error (kept in socket_error), an answer that isn't JSON (eAPI
    # still starting, 5xx) and a 401, the only one with neither.
    def connection_error(auth, eos, e):
        if eos.connection.socket_error is None and e.message != "unable to connect to eAPI":
            return AuthFailure(f"{auth['host']}: eAPI refused the credential ({e.message})")
        return DeviceUnreachable(f"{auth['host']}: {e.message}")
    
    
    # host is an address or a URL (https://10.0.0.11:8443); a bare IPv6 address is bracketed. Only https, or http when the
    # resource ticks allow_http (Basic auth then crosses the network in cleartext). pyeapi skips certificate checks unless it
    # is given a context, so one is always passed: verifying, or not when the resource unticks verify_tls (EOS ships a
    # self-signed cert).
    def switch_url(auth: arista_eos):
        host = auth["host"]
        if "://" not in host:
            host = (
                f"https://[{host}]"
                if host.count(":") > 1 and "[" not in host
                else f"https://{host}"
            )
        url = urlsplit(host)
        if not url.hostname or (
            url.scheme != "https" and not (url.scheme == "http" and auth.get("allow_http"))
        ):
            raise ValueError(
                f"host must be an address or an https:// URL (http:// needs allow_http on the resource): {auth['host']}"
            )
        return url
    
    
    def node(auth: arista_eos, timeout: float = 60):
        url = switch_url(auth)
        context = ssl.create_default_context()
        if auth.get("verify_tls") is False:
            context.check_hostname = False
            context.verify_mode = ssl.CERT_NONE
        return pyeapi.connect(
            transport=url.scheme,
            host=url.hostname,
            port=url.port or auth.get("port") or None,
            username=auth["username"],
            password=auth["password"],
            context=context,
            timeout=timeout,
            return_node=True,
        )
    
    
    # Shared by the Device scripts: the root flow run's user state, keyed by switch, holds the pre-check snapshot, the
    # image activate replaced (rollback target), the image it booted and the reload markers. Every step of the run (nested
    # subflows and retried attempts included) reads the same entry. Outside a flow the Device scripts share one state
    # resource in the running user's home (a script's own state would keep the pre-check snapshot from the post-check).
    # The endpoint is called through the client so an unreachable state raises (wmill.get_flow_user_state returns None).
    # ponytail: the fallback is one map per user, so concurrent standalone runs can race; flows don't.
    # The key is the switch's eAPI address: arista_eos-<host>-<port> with non-word characters in the host as _, the port
    # always in (443 or 80 when unset). State written before the port was part of the key is read from arista_eos_<host>
    # as a fallback (all word characters, so never a new key); writes go to the new key only.
    def state_keys(auth: arista_eos):
        url = switch_url(auth)
        port = int(url.port or auth.get("port") or {"https": 443, "http": 80}[url.scheme])
        return [
            f"arista_eos-{word(url.hostname)}-{port}",
            f"arista_eos_{word(auth['host'])}",
        ]
    
    
    def word(s):
        return re.sub(r"\W", "_", s)
    
    
    def load_state(keys):
        for key in keys:
            if os.environ.get("WM_FLOW_JOB_ID"):
                c = wmill.Windmill()
                value = c.get(
                    f"/w/{c.workspace}/jobs/flow/user_states/{c.get_root_job_id()}/{key}"
                ).json()
            else:
                value = (wmill.get_state(standalone_state()) or {}).get(key)
            if value is not None:
                return value
    
    
    def save_state(keys, value):
        if os.environ.get("WM_FLOW_JOB_ID"):
            c = wmill.Windmill()
            c.post(
                f"/w/{c.workspace}/jobs/flow/user_states/{c.get_root_job_id()}/{keys[0]}",
                json=value,
            )
        else:
            path = standalone_state()
            wmill.set_state({**(wmill.get_state(path) or {}), keys[0]: value}, path)
    
    
    def standalone_state():
        return f"u/{os.environ['WM_USERNAME']}/arista_eos_state"
    
    
    # "4.34.9M-51132493.4349M (engineering build)" on cEOS, "4.34.9M" on hardware.
    def short_version(v):
        m = re.match(r"\d+\.\d+\.\d+(\.\d+)?[A-Z]*", v)
        return m.group(0) if m else v
    

    Submitted by hugo989 1 day ago