# py312
# requirements:
# pyeapi==1.0.4
# wmill==1.851.0
# Device rollback: boot system back to the image Device activate recorded as the previous one (or, in a new run, the
# only other EOS image on flash), then reload now behind the same once-per-run guard as Device reload. It is the
# recovery path: no unsaved-config check. A boot that this run's reload caused doesn't count against
# min_uptime_seconds. A switch that doesn't answer raises DeviceUnreachable (retry it); anything else RollbackFailed.
import logging
import os
import re
import ssl
import time
from typing import TypedDict
from urllib.parse import urlsplit
import pyeapi
import wmill
class arista_eos(TypedDict):
host: str
username: str
password: str
port: int
verify_tls: bool
allow_http: bool
class RollbackFailed(Exception):
pass
def main(auth: arista_eos, min_uptime_seconds: int = 300):
"""
Device rollback
Boot the switch back to its previous image and reload, exactly once per flow run (a retry skips it). Returns { status: done, reloaded, reason, boot_image, steps } (same shape when skipped). Raises RollbackFailed, DeviceUnreachable or AuthFailure. Follow it with Get Reload Delay.
"""
key = state_keys(auth)
state = load_state(key) or {}
min_uptime = 300 if min_uptime_seconds is None else min_uptime_seconds
sw = read_switch(auth, state, min_uptime)
sent = sw["marker"]("rollback")
if sw.get("unreachable"):
if sent:
return skip(
f"rollback sent at {sent['sent_at']}; the switch no longer answers: it is reloading",
state.get("previous_image"),
)
raise DeviceUnreachable(sw["unreachable"])
if sent and (sent["confirmed"] or sent["boot_timestamp"] != sw["boot"]):
return skip(guard_reason("rollback", sent, sw), state.get("previous_image"))
try:
d, boot = eapi(
auth, [{"cmd": "dir flash:/", "revision": 2}, "show boot-config"]
)
files = list(d["urls"]["flash:/"]["entries"])
current = boot["softwareImage"].replace("flash:/", "")
# Without a recorded previous image (new run), the only other EOS image on flash is unambiguous.
others = [f for f in files if re.match(r"^EOS.*\.swi$", f) and f != current]
previous = (state.get("previous_image") or "").replace("flash:/", "") or (
others[0] if len(others) == 1 else ""
)
if not previous:
raise RollbackFailed(
f"previous image unknown (EOS images on flash: {', '.join(others) or 'none'})"
)
if previous not in files:
raise RollbackFailed(f"previous image {previous} is no longer on flash")
reload = sw["marker"]("reload")
if reload and reload["confirmed"] and reload["boot_timestamp"] == sw["boot"]:
raise RollbackFailed(
f"the reload sent at {reload['sent_at']} has not happened yet (same boot as when it was sent): rolling back now would race it, check the switch"
)
steps = boot_system(auth, previous)
forget_activated(key, state)
if sw["running"] == image_version(previous):
return skip(
f"already running the previous image {previous}; boot-config set back to it",
f"flash:/{previous}",
steps,
)
# A recent boot that this run's reload caused doesn't block the rollback: the post-check that ran after it
# can end well inside min_uptime_seconds.
ours = reload and reload["boot_timestamp"] != sw["boot"]
if not ours and sw["uptime"] < min_uptime:
return skip(
f"rebooted {round(sw['uptime'])} s ago, not by this run",
f"flash:/{previous}",
steps,
)
fire_reload(auth, key, "rollback", sw["boot"])
except (DeviceUnreachable, AuthFailure, RollbackFailed):
raise
except Exception as e:
raise RollbackFailed(f"{auth['host']}: {e}")
return {
"status": "done",
"reloaded": True,
"reason": None,
"boot_image": f"flash:/{previous}",
"steps": steps,
}
def boot_system(auth, image):
_, set_, _, after = eapi(
auth, ["configure", f"boot system flash:{image}", "end", "show boot-config"]
)
if after["softwareImage"] != f"flash:/{image}":
raise Exception(f"boot-config still points to {after['softwareImage']}")
return [f"boot system flash:{image}", *(set_.get("warnings") or [])]
def forget_activated(key, state):
if state.pop("activated_image", None):
save_state(key, state)
# Every return has the same shape. boot_image is the image rollback boots (or set boot-config back to); before flash
# is read it is the recorded previous image, which is null in a new run.
def skip(reason, boot_image=None, steps=()):
return {
"status": "done",
"reloaded": False,
"reason": reason,
"boot_image": boot_image,
"steps": list(steps),
}
def guard_reason(kind, m, sw):
if m["boot_timestamp"] == sw["boot"]:
return f"{kind} sent at {m['sent_at']}; the switch is going down"
return f"{kind} sent at {m['sent_at']} already rebooted the switch"
# A reload is sent at most once per switch and flow run. Each one leaves a marker { sent_at, boot_timestamp,
# confirmed }: boot_timestamp is the boot it was sent on, so a later boot means it happened, and confirmed means
# the command reached the switch. An unconfirmed marker on the same boot (the worker died first) is sent again.
# Outside a flow, a marker is dropped once the reboot it caused is min_uptime_seconds old.
def read_switch(auth, state, min_uptime):
try:
version, boot = eapi(auth, ["show version", "show boot-config"])
except DeviceUnreachable as e:
return {"unreachable": str(e), "marker": lambda k: state.get(k)}
sw = {
"running": short_version(version["version"]),
"image": boot["softwareImage"],
"boot": version["bootupTimestamp"],
"uptime": version["uptime"],
}
in_flow = bool(os.environ.get("WM_FLOW_JOB_ID"))
def marker(k):
m = state.get(k)
return (
m
if m
and (
in_flow
or m["boot_timestamp"] == sw["boot"]
or sw["uptime"] < min_uptime
)
else None
)
return {**sw, "marker": marker}
# Fire and forget: the switch drops the connection while going down, which is not an error. A connection that
# never opened, or a refusal from the switch, means it did not reload: the marker goes and the error is raised.
def fire_reload(auth, key, kind, boot_ts):
def mark(m):
rest = {k: v for k, v in (load_state(key) or {}).items() if k != kind}
save_state(key, {**rest, kind: m} if m else rest)
m = {
"sent_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
"boot_timestamp": boot_ts,
"confirmed": False,
}
mark(m)
eos = node(auth, 15)
try:
# Opened before sending, so a failure here means the command never left.
eos.connection.transport.connect()
except OSError as e:
mark(None)
raise DeviceUnreachable(f"{auth['host']}: {e}")
try:
eos.run_commands(["reload now"])
except pyeapi.eapilib.ConnectionError as e:
# Sent, then the connection dropped or timed out as the switch went down. A 401 is a refusal.
err = connection_error(auth, eos, e)
if isinstance(err, AuthFailure):
mark(None)
raise err
except Exception:
mark(None)
raise
mark({**m, "confirmed": True})
# "EOS64-4.34.9M.swi" -> "4.34.9M"
def image_version(file):
m = re.search(r"-(\d+\.\d+\.\d+(\.\d+)?[A-Z]*)\.swi$", file)
return m.group(1) if m else None
# Errors the workflow branches and retries on (retry_if: result.error.name == "DeviceUnreachable").
class DeviceUnreachable(Exception):
pass
class AuthFailure(Exception):
pass
# pyeapi logs a traceback for every socket error, and a rebooting switch gives many.
logging.getLogger("pyeapi").addHandler(logging.NullHandler())
def eapi(auth: arista_eos, cmds: list, fmt: str = "json", timeout: float = 60):
eos = node(auth, timeout)
try:
return eos.run_commands(cmds, fmt)
except pyeapi.eapilib.ConnectionError as e:
raise connection_error(auth, eos, e)
# pyeapi 1.0.4 raises ConnectionError for a socket error (kept in socket_error), an answer that isn't JSON (eAPI
# still starting, 5xx) and a 401, the only one with neither.
def connection_error(auth, eos, e):
if eos.connection.socket_error is None and e.message != "unable to connect to eAPI":
return AuthFailure(f"{auth['host']}: eAPI refused the credential ({e.message})")
return DeviceUnreachable(f"{auth['host']}: {e.message}")
# host is an address or a URL (https://10.0.0.11:8443); a bare IPv6 address is bracketed. Only https, or http when the
# resource ticks allow_http (Basic auth then crosses the network in cleartext). pyeapi skips certificate checks unless it
# is given a context, so one is always passed: verifying, or not when the resource unticks verify_tls (EOS ships a
# self-signed cert).
def switch_url(auth: arista_eos):
host = auth["host"]
if "://" not in host:
host = (
f"https://[{host}]"
if host.count(":") > 1 and "[" not in host
else f"https://{host}"
)
url = urlsplit(host)
if not url.hostname or (
url.scheme != "https" and not (url.scheme == "http" and auth.get("allow_http"))
):
raise ValueError(
f"host must be an address or an https:// URL (http:// needs allow_http on the resource): {auth['host']}"
)
return url
def node(auth: arista_eos, timeout: float = 60):
url = switch_url(auth)
context = ssl.create_default_context()
if auth.get("verify_tls") is False:
context.check_hostname = False
context.verify_mode = ssl.CERT_NONE
return pyeapi.connect(
transport=url.scheme,
host=url.hostname,
port=url.port or auth.get("port") or None,
username=auth["username"],
password=auth["password"],
context=context,
timeout=timeout,
return_node=True,
)
# Shared by the Device scripts: the root flow run's user state, keyed by switch, holds the pre-check snapshot, the
# image activate replaced (rollback target), the image it booted and the reload markers. Every step of the run (nested
# subflows and retried attempts included) reads the same entry. Outside a flow the Device scripts share one state
# resource in the running user's home (a script's own state would keep the pre-check snapshot from the post-check).
# The endpoint is called through the client so an unreachable state raises (wmill.get_flow_user_state returns None).
# ponytail: the fallback is one map per user, so concurrent standalone runs can race; flows don't.
# The key is the switch's eAPI address: arista_eos-<host>-<port> with non-word characters in the host as _, the port
# always in (443 or 80 when unset). State written before the port was part of the key is read from arista_eos_<host>
# as a fallback (all word characters, so never a new key); writes go to the new key only.
def state_keys(auth: arista_eos):
url = switch_url(auth)
port = int(url.port or auth.get("port") or {"https": 443, "http": 80}[url.scheme])
return [
f"arista_eos-{word(url.hostname)}-{port}",
f"arista_eos_{word(auth['host'])}",
]
def word(s):
return re.sub(r"\W", "_", s)
def load_state(keys):
for key in keys:
if os.environ.get("WM_FLOW_JOB_ID"):
c = wmill.Windmill()
value = c.get(
f"/w/{c.workspace}/jobs/flow/user_states/{c.get_root_job_id()}/{key}"
).json()
else:
value = (wmill.get_state(standalone_state()) or {}).get(key)
if value is not None:
return value
def save_state(keys, value):
if os.environ.get("WM_FLOW_JOB_ID"):
c = wmill.Windmill()
c.post(
f"/w/{c.workspace}/jobs/flow/user_states/{c.get_root_job_id()}/{keys[0]}",
json=value,
)
else:
path = standalone_state()
wmill.set_state({**(wmill.get_state(path) or {}), keys[0]: value}, path)
def standalone_state():
return f"u/{os.environ['WM_USERNAME']}/arista_eos_state"
# "4.34.9M-51132493.4349M (engineering build)" on cEOS, "4.34.9M" on hardware.
def short_version(v):
m = re.match(r"\d+\.\d+\.\d+(\.\d+)?[A-Z]*", v)
return m.group(0) if m else v
Submitted by hugo989 1 day ago