Edits history of script submission #24218 for ' Device rollback (arista_eos)'

  • python3
    One script reply has been approved by the moderators
    Ap­pro­ved
    # py312
    # requirements:
    # pyeapi==1.0.4
    # wmill==1.851.0
    
    # Device rollback: boot system back to the image Device activate recorded as the previous one (or, in a new run, the
    # only other EOS image on flash), then reload now behind the same once-per-run guard as Device reload. It is the
    # recovery path: no unsaved-config check. A boot that this run's reload caused doesn't count against
    # min_uptime_seconds. A switch that doesn't answer raises DeviceUnreachable (retry it); anything else RollbackFailed.
    
    import logging
    import os
    import re
    import ssl
    import time
    from typing import TypedDict
    from urllib.parse import urlsplit
    
    import pyeapi
    import wmill
    
    
    class arista_eos(TypedDict):
        host: str
        username: str
        password: str
        port: int
        verify_tls: bool
        allow_http: bool
    
    
    class RollbackFailed(Exception):
        pass
    
    
    def main(auth: arista_eos, min_uptime_seconds: int = 300):
        """
        Device rollback
        Boot the switch back to its previous image and reload, exactly once per flow run (a retry skips it). Returns { status: done, reloaded, reason, boot_image, steps } (same shape when skipped). Raises RollbackFailed, DeviceUnreachable or AuthFailure. Follow it with Get Reload Delay.
        """
        key = state_keys(auth)
        state = load_state(key) or {}
        min_uptime = 300 if min_uptime_seconds is None else min_uptime_seconds
        sw = read_switch(auth, state, min_uptime)
        sent = sw["marker"]("rollback")
        if sw.get("unreachable"):
            if sent:
                return skip(
                    f"rollback sent at {sent['sent_at']}; the switch no longer answers: it is reloading",
                    state.get("previous_image"),
                )
            raise DeviceUnreachable(sw["unreachable"])
        if sent and (sent["confirmed"] or sent["boot_timestamp"] != sw["boot"]):
            return skip(guard_reason("rollback", sent, sw), state.get("previous_image"))
        try:
            d, boot = eapi(
                auth, [{"cmd": "dir flash:/", "revision": 2}, "show boot-config"]
            )
            files = list(d["urls"]["flash:/"]["entries"])
            current = boot["softwareImage"].replace("flash:/", "")
            # Without a recorded previous image (new run), the only other EOS image on flash is unambiguous.
            others = [f for f in files if re.match(r"^EOS.*\.swi$", f) and f != current]
            previous = (state.get("previous_image") or "").replace("flash:/", "") or (
                others[0] if len(others) == 1 else ""
            )
            if not previous:
                raise RollbackFailed(
                    f"previous image unknown (EOS images on flash: {', '.join(others) or 'none'})"
                )
            if previous not in files:
                raise RollbackFailed(f"previous image {previous} is no longer on flash")
            reload = sw["marker"]("reload")
            if reload and reload["confirmed"] and reload["boot_timestamp"] == sw["boot"]:
                raise RollbackFailed(
                    f"the reload sent at {reload['sent_at']} has not happened yet (same boot as when it was sent): rolling back now would race it, check the switch"
                )
            steps = boot_system(auth, previous)
            forget_activated(key, state)
            if sw["running"] == image_version(previous):
                return skip(
                    f"already running the previous image {previous}; boot-config set back to it",
                    f"flash:/{previous}",
                    steps,
                )
            # A recent boot that this run's reload caused doesn't block the rollback: the post-check that ran after it
            # can end well inside min_uptime_seconds.
            ours = reload and reload["boot_timestamp"] != sw["boot"]
            if not ours and sw["uptime"] < min_uptime:
                return skip(
                    f"rebooted {round(sw['uptime'])} s ago, not by this run",
                    f"flash:/{previous}",
                    steps,
                )
            fire_reload(auth, key, "rollback", sw["boot"])
        except (DeviceUnreachable, AuthFailure, RollbackFailed):
            raise
        except Exception as e:
            raise RollbackFailed(f"{auth['host']}: {e}")
        return {
            "status": "done",
            "reloaded": True,
            "reason": None,
            "boot_image": f"flash:/{previous}",
            "steps": steps,
        }
    
    
    def boot_system(auth, image):
        _, set_, _, after = eapi(
            auth, ["configure", f"boot system flash:{image}", "end", "show boot-config"]
        )
        if after["softwareImage"] != f"flash:/{image}":
            raise Exception(f"boot-config still points to {after['softwareImage']}")
        return [f"boot system flash:{image}", *(set_.get("warnings") or [])]
    
    
    def forget_activated(key, state):
        if state.pop("activated_image", None):
            save_state(key, state)
    
    
    # Every return has the same shape. boot_image is the image rollback boots (or set boot-config back to); before flash
    # is read it is the recorded previous image, which is null in a new run.
    def skip(reason, boot_image=None, steps=()):
        return {
            "status": "done",
            "reloaded": False,
            "reason": reason,
            "boot_image": boot_image,
            "steps": list(steps),
        }
    
    
    def guard_reason(kind, m, sw):
        if m["boot_timestamp"] == sw["boot"]:
            return f"{kind} sent at {m['sent_at']}; the switch is going down"
        return f"{kind} sent at {m['sent_at']} already rebooted the switch"
    
    
    # A reload is sent at most once per switch and flow run. Each one leaves a marker { sent_at, boot_timestamp,
    # confirmed }: boot_timestamp is the boot it was sent on, so a later boot means it happened, and confirmed means
    # the command reached the switch. An unconfirmed marker on the same boot (the worker died first) is sent again.
    # Outside a flow, a marker is dropped once the reboot it caused is min_uptime_seconds old.
    def read_switch(auth, state, min_uptime):
        try:
            version, boot = eapi(auth, ["show version", "show boot-config"])
        except DeviceUnreachable as e:
            return {"unreachable": str(e), "marker": lambda k: state.get(k)}
        sw = {
            "running": short_version(version["version"]),
            "image": boot["softwareImage"],
            "boot": version["bootupTimestamp"],
            "uptime": version["uptime"],
        }
        in_flow = bool(os.environ.get("WM_FLOW_JOB_ID"))
    
        def marker(k):
            m = state.get(k)
            return (
                m
                if m
                and (
                    in_flow
                    or m["boot_timestamp"] == sw["boot"]
                    or sw["uptime"] < min_uptime
                )
                else None
            )
    
        return {**sw, "marker": marker}
    
    
    # Fire and forget: the switch drops the connection while going down, which is not an error. A connection that
    # never opened, or a refusal from the switch, means it did not reload: the marker goes and the error is raised.
    def fire_reload(auth, key, kind, boot_ts):
        def mark(m):
            rest = {k: v for k, v in (load_state(key) or {}).items() if k != kind}
            save_state(key, {**rest, kind: m} if m else rest)
    
        m = {
            "sent_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
            "boot_timestamp": boot_ts,
            "confirmed": False,
        }
        mark(m)
        eos = node(auth, 15)
        try:
            # Opened before sending, so a failure here means the command never left.
            eos.connection.transport.connect()
        except OSError as e:
            mark(None)
            raise DeviceUnreachable(f"{auth['host']}: {e}")
        try:
            eos.run_commands(["reload now"])
        except pyeapi.eapilib.ConnectionError as e:
            # Sent, then the connection dropped or timed out as the switch went down. A 401 is a refusal.
            err = connection_error(auth, eos, e)
            if isinstance(err, AuthFailure):
                mark(None)
                raise err
        except Exception:
            mark(None)
            raise
        mark({**m, "confirmed": True})
    
    
    # "EOS64-4.34.9M.swi" -> "4.34.9M"
    def image_version(file):
        m = re.search(r"-(\d+\.\d+\.\d+(\.\d+)?[A-Z]*)\.swi$", file)
        return m.group(1) if m else None
    
    
    # Errors the workflow branches and retries on (retry_if: result.error.name == "DeviceUnreachable").
    class DeviceUnreachable(Exception):
        pass
    
    
    class AuthFailure(Exception):
        pass
    
    
    # pyeapi logs a traceback for every socket error, and a rebooting switch gives many.
    logging.getLogger("pyeapi").addHandler(logging.NullHandler())
    
    
    def eapi(auth: arista_eos, cmds: list, fmt: str = "json", timeout: float = 60):
        eos = node(auth, timeout)
        try:
            return eos.run_commands(cmds, fmt)
        except pyeapi.eapilib.ConnectionError as e:
            raise connection_error(auth, eos, e)
    
    
    # pyeapi 1.0.4 raises ConnectionError for a socket error (kept in socket_error), an answer that isn't JSON (eAPI
    # still starting, 5xx) and a 401, the only one with neither.
    def connection_error(auth, eos, e):
        if eos.connection.socket_error is None and e.message != "unable to connect to eAPI":
            return AuthFailure(f"{auth['host']}: eAPI refused the credential ({e.message})")
        return DeviceUnreachable(f"{auth['host']}: {e.message}")
    
    
    # host is an address or a URL (https://10.0.0.11:8443); a bare IPv6 address is bracketed. Only https, or http when the
    # resource ticks allow_http (Basic auth then crosses the network in cleartext). pyeapi skips certificate checks unless it
    # is given a context, so one is always passed: verifying, or not when the resource unticks verify_tls (EOS ships a
    # self-signed cert).
    def switch_url(auth: arista_eos):
        host = auth["host"]
        if "://" not in host:
            host = (
                f"https://[{host}]"
                if host.count(":") > 1 and "[" not in host
                else f"https://{host}"
            )
        url = urlsplit(host)
        if not url.hostname or (
            url.scheme != "https" and not (url.scheme == "http" and auth.get("allow_http"))
        ):
            raise ValueError(
                f"host must be an address or an https:// URL (http:// needs allow_http on the resource): {auth['host']}"
            )
        return url
    
    
    def node(auth: arista_eos, timeout: float = 60):
        url = switch_url(auth)
        context = ssl.create_default_context()
        if auth.get("verify_tls") is False:
            context.check_hostname = False
            context.verify_mode = ssl.CERT_NONE
        return pyeapi.connect(
            transport=url.scheme,
            host=url.hostname,
            port=url.port or auth.get("port") or None,
            username=auth["username"],
            password=auth["password"],
            context=context,
            timeout=timeout,
            return_node=True,
        )
    
    
    # Shared by the Device scripts: the root flow run's user state, keyed by switch, holds the pre-check snapshot, the
    # image activate replaced (rollback target), the image it booted and the reload markers. Every step of the run (nested
    # subflows and retried attempts included) reads the same entry. Outside a flow the Device scripts share one state
    # resource in the running user's home (a script's own state would keep the pre-check snapshot from the post-check).
    # The endpoint is called through the client so an unreachable state raises (wmill.get_flow_user_state returns None).
    # ponytail: the fallback is one map per user, so concurrent standalone runs can race; flows don't.
    # The key is the switch's eAPI address: arista_eos-<host>-<port> with non-word characters in the host as _, the port
    # always in (443 or 80 when unset). State written before the port was part of the key is read from arista_eos_<host>
    # as a fallback (all word characters, so never a new key); writes go to the new key only.
    def state_keys(auth: arista_eos):
        url = switch_url(auth)
        port = int(url.port or auth.get("port") or {"https": 443, "http": 80}[url.scheme])
        return [
            f"arista_eos-{word(url.hostname)}-{port}",
            f"arista_eos_{word(auth['host'])}",
        ]
    
    
    def word(s):
        return re.sub(r"\W", "_", s)
    
    
    def load_state(keys):
        for key in keys:
            if os.environ.get("WM_FLOW_JOB_ID"):
                c = wmill.Windmill()
                value = c.get(
                    f"/w/{c.workspace}/jobs/flow/user_states/{c.get_root_job_id()}/{key}"
                ).json()
            else:
                value = (wmill.get_state(standalone_state()) or {}).get(key)
            if value is not None:
                return value
    
    
    def save_state(keys, value):
        if os.environ.get("WM_FLOW_JOB_ID"):
            c = wmill.Windmill()
            c.post(
                f"/w/{c.workspace}/jobs/flow/user_states/{c.get_root_job_id()}/{keys[0]}",
                json=value,
            )
        else:
            path = standalone_state()
            wmill.set_state({**(wmill.get_state(path) or {}), keys[0]: value}, path)
    
    
    def standalone_state():
        return f"u/{os.environ['WM_USERNAME']}/arista_eos_state"
    
    
    # "4.34.9M-51132493.4349M (engineering build)" on cEOS, "4.34.9M" on hardware.
    def short_version(v):
        m = re.match(r"\d+\.\d+\.\d+(\.\d+)?[A-Z]*", v)
        return m.group(0) if m else v
    

    Submitted by hugo989 1 day ago