# py312
# requirements:
# pyeapi==1.0.4
# wmill==1.851.0
# Device post-check: one attempt, after an upgrade (or rollback) of one switch. Compares the switch with the snapshot
# Device pre-check saved in this flow run, and raises a named error the workflow retries on:
# - DeviceUnreachable while the switch is down or still runs the boot a reload of this run was sent on;
# - NotStable while the state differs from the snapshot (the differences are in the message; ports stay errdisabled
# until the MLAG reload-delay has run, the message says how much of it is left);
# - AuthFailure when the credential is refused (not worth retrying).
# No snapshot in this run is a plain error. The version is reported, not compared with the target: the workflow does.
import logging
import os
import re
import ssl
from typing import TypedDict
from urllib.parse import urlsplit
import pyeapi
import wmill
class arista_eos(TypedDict):
host: str
username: str
password: str
port: int
verify_tls: bool
allow_http: bool
class NotStable(Exception):
pass
def main(auth: arista_eos):
"""
Device post-check
One post-upgrade check of one switch against the pre-check snapshot of this flow run: returns { running_version, status ok, checks_ok }. Raises DeviceUnreachable while the switch is down or still on the boot the reload was sent on, NotStable while interfaces/LLDP/MLAG differ from the snapshot (differences in the message), AuthFailure when the credential is refused. Retry it from the workflow; it does not wait.
"""
state = load_state(state_keys(auth)) or {}
# Before probing the switch: without a snapshot, retrying through a reboot would only delay the same error.
if not state.get("snapshot"):
raise RuntimeError(
f"No pre-check snapshot for {auth['host']} in this flow run: run Device pre-check first, in the same run"
)
version = returned(auth, state)
problems = diff(state["snapshot"], snapshot(auth))
if problems:
[mlag] = eapi(auth, ["show mlag"])
left = round(reload_delay(mlag) - version["uptime"])
hint = f" (MLAG reload-delay still running: {left} s left)" if left > 0 else ""
raise NotStable(f"{auth['host']}: {'; '.join(problems)}{hint}")
return {
"running_version": short_version(version["version"]),
"status": "ok",
"checks_ok": ["interface"],
}
# Same function in Get Reload Delay and Device post-check. One attempt: show version, raising DeviceUnreachable
# while the switch is down, and while it still runs the boot a reload of this run was sent on (reload now can answer
# before EOS goes down, and the old boot must not pass for the new one). AuthFailure is raised as is.
def returned(auth, state):
try:
# Short: a switch that is down holds the attempt for the whole timeout, and the workflow retries anyway.
[v] = eapi(auth, ["show version"], timeout=15)
except DeviceUnreachable as e:
raise DeviceUnreachable(f"{e}: not back yet")
for k in ("reload", "rollback"):
m = state.get(k)
if m and m["boot_timestamp"] == v["bootupTimestamp"]:
raise DeviceUnreachable(
f"{auth['host']}: still on the boot the {k} sent at {m['sent_at']} was sent on"
)
return v
# show mlag reports state disabled both when MLAG was never configured and when it is configured but shut down;
# only the first has no domain-id, local interface or peer-link.
def mlag_configured(mlag):
return any(mlag.get(k) for k in ("domainId", "localInterface", "peerLink"))
def snapshot(auth):
status, lldp, mlag, mlag_intf = eapi(
auth,
[
"show interfaces status",
"show lldp neighbors",
"show mlag",
"show mlag interfaces",
],
)
return {
"interfaces_up": sorted(
n
for n, i in status["interfaceStatuses"].items()
if i.get("linkStatus") == "connected"
),
"lldp": [
{
"port": n["port"],
"neighbor": f"{n['neighborDevice']} {n['neighborPort']}",
}
for n in lldp["lldpNeighbors"]
],
"mlag": {
"state": mlag.get("state"),
"negStatus": mlag.get("negStatus"),
"peerLinkStatus": mlag.get("peerLinkStatus"),
"ports": {
f"{i['localInterface']} (mlag {id})": i["status"]
for id, i in (mlag_intf.get("interfaces") or {}).items()
},
},
}
def diff(before, after):
p = []
for i in before["interfaces_up"]:
if i not in after["interfaces_up"]:
p.append(f"{i} down after the upgrade (up before)")
for n in before["lldp"]:
if not any(
a["port"] == n["port"] and a["neighbor"] == n["neighbor"]
for a in after["lldp"]
):
p.append(f"LLDP neighbor {n['neighbor']} no longer seen on {n['port']}")
for k in ("state", "negStatus", "peerLinkStatus"):
if before["mlag"][k] != after["mlag"][k]:
p.append(f"MLAG {k} {after['mlag'][k]} (was {before['mlag'][k]})")
for port, s in before["mlag"]["ports"].items():
if after["mlag"]["ports"].get(port) != s:
p.append(f"{port} {after['mlag']['ports'].get(port, 'missing')} (was {s})")
return p
# Without MLAG configured, show mlag still reports the default reload-delay, and nothing waits on it.
def reload_delay(mlag):
if not mlag_configured(mlag):
return 0
return max(mlag.get("reloadDelay") or 0, mlag.get("reloadDelayNonMlag") or 0)
# Errors the workflow branches and retries on (retry_if: result.error.name == "DeviceUnreachable").
class DeviceUnreachable(Exception):
pass
class AuthFailure(Exception):
pass
# pyeapi logs a traceback for every socket error, and a rebooting switch gives many.
logging.getLogger("pyeapi").addHandler(logging.NullHandler())
# This script runs right after a reboot, when eAPI can answer before the EOS agents behind a show command have
# started: such an error answer counts as not back yet, so the workflow retries it. It only sends show commands.
def eapi(auth: arista_eos, cmds: list, fmt: str = "json", timeout: float = 60):
eos = node(auth, timeout)
try:
return eos.run_commands(cmds, fmt)
except pyeapi.eapilib.ConnectionError as e:
raise connection_error(auth, eos, e)
except pyeapi.eapilib.CommandError as e:
raise DeviceUnreachable(f"{auth['host']}: not ready yet ({e.message})")
# pyeapi 1.0.4 raises ConnectionError for a socket error (kept in socket_error), an answer that isn't JSON (eAPI
# still starting, 5xx) and a 401, the only one with neither.
def connection_error(auth, eos, e):
if eos.connection.socket_error is None and e.message != "unable to connect to eAPI":
return AuthFailure(f"{auth['host']}: eAPI refused the credential ({e.message})")
return DeviceUnreachable(f"{auth['host']}: {e.message}")
# host is an address or a URL (https://10.0.0.11:8443); a bare IPv6 address is bracketed. Only https, or http when the
# resource ticks allow_http (Basic auth then crosses the network in cleartext). pyeapi skips certificate checks unless it
# is given a context, so one is always passed: verifying, or not when the resource unticks verify_tls (EOS ships a
# self-signed cert).
def switch_url(auth: arista_eos):
host = auth["host"]
if "://" not in host:
host = (
f"https://[{host}]"
if host.count(":") > 1 and "[" not in host
else f"https://{host}"
)
url = urlsplit(host)
if not url.hostname or (
url.scheme != "https" and not (url.scheme == "http" and auth.get("allow_http"))
):
raise ValueError(
f"host must be an address or an https:// URL (http:// needs allow_http on the resource): {auth['host']}"
)
return url
def node(auth: arista_eos, timeout: float = 60):
url = switch_url(auth)
context = ssl.create_default_context()
if auth.get("verify_tls") is False:
context.check_hostname = False
context.verify_mode = ssl.CERT_NONE
return pyeapi.connect(
transport=url.scheme,
host=url.hostname,
port=url.port or auth.get("port") or None,
username=auth["username"],
password=auth["password"],
context=context,
timeout=timeout,
return_node=True,
)
# Shared by the Device scripts: the root flow run's user state, keyed by switch, holds the pre-check snapshot, the
# image activate replaced (rollback target), the image it booted and the reload markers. Every step of the run (nested
# subflows and retried attempts included) reads the same entry. Outside a flow the Device scripts share one state
# resource in the running user's home (a script's own state would keep the pre-check snapshot from the post-check).
# The endpoint is called through the client so an unreachable state raises (wmill.get_flow_user_state returns None).
# ponytail: the fallback is one map per user, so concurrent standalone runs can race; flows don't.
# The key is the switch's eAPI address: arista_eos-<host>-<port> with non-word characters in the host as _, the port
# always in (443 or 80 when unset). State written before the port was part of the key is read from arista_eos_<host>
# as a fallback (all word characters, so never a new key); writes go to the new key only.
def state_keys(auth: arista_eos):
url = switch_url(auth)
port = int(url.port or auth.get("port") or {"https": 443, "http": 80}[url.scheme])
return [
f"arista_eos-{word(url.hostname)}-{port}",
f"arista_eos_{word(auth['host'])}",
]
def word(s):
return re.sub(r"\W", "_", s)
def load_state(keys):
for key in keys:
if os.environ.get("WM_FLOW_JOB_ID"):
c = wmill.Windmill()
value = c.get(
f"/w/{c.workspace}/jobs/flow/user_states/{c.get_root_job_id()}/{key}"
).json()
else:
value = (wmill.get_state(standalone_state()) or {}).get(key)
if value is not None:
return value
def save_state(keys, value):
if os.environ.get("WM_FLOW_JOB_ID"):
c = wmill.Windmill()
c.post(
f"/w/{c.workspace}/jobs/flow/user_states/{c.get_root_job_id()}/{keys[0]}",
json=value,
)
else:
path = standalone_state()
wmill.set_state({**(wmill.get_state(path) or {}), keys[0]: value}, path)
def standalone_state():
return f"u/{os.environ['WM_USERNAME']}/arista_eos_state"
# "4.34.9M-51132493.4349M (engineering build)" on cEOS, "4.34.9M" on hardware.
def short_version(v):
m = re.match(r"\d+\.\d+\.\d+(\.\d+)?[A-Z]*", v)
return m.group(0) if m else v
Submitted by hugo989 1 day ago