# py312
# requirements:
# pyeapi==1.0.4
# wmill==1.851.0
# Device reload: reload now, at most once per switch and flow run, so a retried step never reboots it twice.
# Skipped (reloaded false, with the reason) when this run already sent it, the switch already runs target_version, or
# it rebooted less than min_uptime_seconds ago. boot-config must be the image Device activate set in this run, or
# flash:/EOS-<target_version>.swi. ReloadRefused while show running-config diffs changes anything but its ! headers,
# or there is no startup-config (reload now doesn't prompt, so unsaved changes would be lost), unless allow_unsaved.
# Reload markers are the root flow run's user state under arista_eos-<host>-<port>; Get Reload Delay and Device
# post-check read them to tell the new boot from the old one.
import logging
import os
import re
import ssl
import time
from typing import TypedDict
from urllib.parse import urlsplit
import pyeapi
import wmill
class arista_eos(TypedDict):
host: str
username: str
password: str
port: int
verify_tls: bool
allow_http: bool
class ReloadRefused(Exception):
pass
def main(
auth: arista_eos,
target_version: str,
min_uptime_seconds: int = 300,
allow_unsaved: bool = False,
):
"""
Device reload
Reload the switch onto the image Device activate set: exactly once per flow run (a retry skips it), refused with unsaved config unless allow_unsaved. Returns { status: done, reloaded, reason }. Raises ReloadRefused (unsaved config, boot-config not the activated image), DeviceUnreachable or AuthFailure. Follow it with Get Reload Delay.
"""
key = state_keys(auth)
state = load_state(key) or {}
min_uptime = 300 if min_uptime_seconds is None else min_uptime_seconds
if not target_version:
raise ReloadRefused("target_version is required for the reload guard")
sw = read_switch(auth, state, min_uptime)
sent = sw["marker"]("reload")
if sw.get("unreachable"):
if sent:
return skip(
f"reload sent at {sent['sent_at']}; the switch no longer answers: it is reloading"
)
raise DeviceUnreachable(sw["unreachable"])
if sent and (sent["confirmed"] or sent["boot_timestamp"] != sw["boot"]):
return skip(guard_reason("reload", sent, sw))
if sw["running"] == target_version:
return skip(f"already running {target_version}")
# The image activate set in this run (state), or exactly EOS-<target_version>.swi.
activated = state.get("activated_image")
if sw["image"] not in (activated, f"flash:/EOS-{target_version}.swi"):
raise ReloadRefused(
f"boot-config points to {sw['image']}, neither the image activate set ({activated or 'none in this run'}) nor flash:/EOS-{target_version}.swi: run Device activate first"
)
if sw["uptime"] < min_uptime:
return skip(f"rebooted {round(sw['uptime'])} s ago")
if not allow_unsaved and unsaved(auth):
raise ReloadRefused(
"unsaved config: run Save Config first, or pass allow_unsaved: true"
)
fire_reload(auth, key, "reload", sw["boot"])
return {"status": "done", "reloaded": True}
# reload now doesn't prompt to save. show running-config diffs is a unified diff of startup-config against
# running-config, empty when they match. Its two file-header lines come before the first @@. Changed lines that
# start with a single "!" are headers EOS writes itself (boot system shows there, so every activate changes one);
# "!!" comments are config. With no startup-config every line shows as added: unsaved, it would boot into ZTP.
# A refused command (AAA authorization, an older EOS) is not unsaved config: saving wouldn't clear it.
def unsaved(auth):
try:
[d] = eapi(auth, ["show running-config diffs"], "text")
except pyeapi.eapilib.CommandError as e:
raise ReloadRefused(f"cannot compare running and startup config: {e}")
hunks, changed = False, []
for l in d["output"].splitlines():
hunks = hunks or l.startswith("@@")
if hunks and l[:1] in ("+", "-") and l[1:].strip():
changed.append(l[1:].strip())
return any(not re.match(r"!(?!!)", l) for l in changed)
def skip(reason):
return {"status": "done", "reloaded": False, "reason": reason}
def guard_reason(kind, m, sw):
if m["boot_timestamp"] == sw["boot"]:
return f"{kind} sent at {m['sent_at']}; the switch is going down"
return f"{kind} sent at {m['sent_at']} already rebooted the switch"
# A reload is sent at most once per switch and flow run. Each one leaves a marker { sent_at, boot_timestamp,
# confirmed }: boot_timestamp is the boot it was sent on, so a later boot means it happened, and confirmed means
# the command reached the switch. An unconfirmed marker on the same boot (the worker died first) is sent again.
# Outside a flow, a marker is dropped once the reboot it caused is min_uptime_seconds old.
def read_switch(auth, state, min_uptime):
try:
version, boot = eapi(auth, ["show version", "show boot-config"])
except DeviceUnreachable as e:
return {"unreachable": str(e), "marker": lambda k: state.get(k)}
sw = {
"running": short_version(version["version"]),
"image": boot["softwareImage"],
"boot": version["bootupTimestamp"],
"uptime": version["uptime"],
}
in_flow = bool(os.environ.get("WM_FLOW_JOB_ID"))
def marker(k):
m = state.get(k)
return (
m
if m
and (
in_flow
or m["boot_timestamp"] == sw["boot"]
or sw["uptime"] < min_uptime
)
else None
)
return {**sw, "marker": marker}
# Fire and forget: the switch drops the connection while going down, which is not an error. A connection that
# never opened, or a refusal from the switch, means it did not reload: the marker goes and the error is raised.
def fire_reload(auth, key, kind, boot_ts):
def mark(m):
rest = {k: v for k, v in (load_state(key) or {}).items() if k != kind}
save_state(key, {**rest, kind: m} if m else rest)
m = {
"sent_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
"boot_timestamp": boot_ts,
"confirmed": False,
}
mark(m)
eos = node(auth, 15)
try:
# Opened before sending, so a failure here means the command never left.
eos.connection.transport.connect()
except OSError as e:
mark(None)
raise DeviceUnreachable(f"{auth['host']}: {e}")
try:
eos.run_commands(["reload now"])
except pyeapi.eapilib.ConnectionError as e:
# Sent, then the connection dropped or timed out as the switch went down. A 401 is a refusal.
err = connection_error(auth, eos, e)
if isinstance(err, AuthFailure):
mark(None)
raise err
except Exception:
mark(None)
raise
mark({**m, "confirmed": True})
# Errors the workflow branches and retries on (retry_if: result.error.name == "DeviceUnreachable").
class DeviceUnreachable(Exception):
pass
class AuthFailure(Exception):
pass
# pyeapi logs a traceback for every socket error, and a rebooting switch gives many.
logging.getLogger("pyeapi").addHandler(logging.NullHandler())
def eapi(auth: arista_eos, cmds: list, fmt: str = "json", timeout: float = 60):
eos = node(auth, timeout)
try:
return eos.run_commands(cmds, fmt)
except pyeapi.eapilib.ConnectionError as e:
raise connection_error(auth, eos, e)
# pyeapi 1.0.4 raises ConnectionError for a socket error (kept in socket_error), an answer that isn't JSON (eAPI
# still starting, 5xx) and a 401, the only one with neither.
def connection_error(auth, eos, e):
if eos.connection.socket_error is None and e.message != "unable to connect to eAPI":
return AuthFailure(f"{auth['host']}: eAPI refused the credential ({e.message})")
return DeviceUnreachable(f"{auth['host']}: {e.message}")
# host is an address or a URL (https://10.0.0.11:8443); a bare IPv6 address is bracketed. Only https, or http when the
# resource ticks allow_http (Basic auth then crosses the network in cleartext). pyeapi skips certificate checks unless it
# is given a context, so one is always passed: verifying, or not when the resource unticks verify_tls (EOS ships a
# self-signed cert).
def switch_url(auth: arista_eos):
host = auth["host"]
if "://" not in host:
host = (
f"https://[{host}]"
if host.count(":") > 1 and "[" not in host
else f"https://{host}"
)
url = urlsplit(host)
if not url.hostname or (
url.scheme != "https" and not (url.scheme == "http" and auth.get("allow_http"))
):
raise ValueError(
f"host must be an address or an https:// URL (http:// needs allow_http on the resource): {auth['host']}"
)
return url
def node(auth: arista_eos, timeout: float = 60):
url = switch_url(auth)
context = ssl.create_default_context()
if auth.get("verify_tls") is False:
context.check_hostname = False
context.verify_mode = ssl.CERT_NONE
return pyeapi.connect(
transport=url.scheme,
host=url.hostname,
port=url.port or auth.get("port") or None,
username=auth["username"],
password=auth["password"],
context=context,
timeout=timeout,
return_node=True,
)
# Shared by the Device scripts: the root flow run's user state, keyed by switch, holds the pre-check snapshot, the
# image activate replaced (rollback target), the image it booted and the reload markers. Every step of the run (nested
# subflows and retried attempts included) reads the same entry. Outside a flow the Device scripts share one state
# resource in the running user's home (a script's own state would keep the pre-check snapshot from the post-check).
# The endpoint is called through the client so an unreachable state raises (wmill.get_flow_user_state returns None).
# ponytail: the fallback is one map per user, so concurrent standalone runs can race; flows don't.
# The key is the switch's eAPI address: arista_eos-<host>-<port> with non-word characters in the host as _, the port
# always in (443 or 80 when unset). State written before the port was part of the key is read from arista_eos_<host>
# as a fallback (all word characters, so never a new key); writes go to the new key only.
def state_keys(auth: arista_eos):
url = switch_url(auth)
port = int(url.port or auth.get("port") or {"https": 443, "http": 80}[url.scheme])
return [
f"arista_eos-{word(url.hostname)}-{port}",
f"arista_eos_{word(auth['host'])}",
]
def word(s):
return re.sub(r"\W", "_", s)
def load_state(keys):
for key in keys:
if os.environ.get("WM_FLOW_JOB_ID"):
c = wmill.Windmill()
value = c.get(
f"/w/{c.workspace}/jobs/flow/user_states/{c.get_root_job_id()}/{key}"
).json()
else:
value = (wmill.get_state(standalone_state()) or {}).get(key)
if value is not None:
return value
def save_state(keys, value):
if os.environ.get("WM_FLOW_JOB_ID"):
c = wmill.Windmill()
c.post(
f"/w/{c.workspace}/jobs/flow/user_states/{c.get_root_job_id()}/{keys[0]}",
json=value,
)
else:
path = standalone_state()
wmill.set_state({**(wmill.get_state(path) or {}), keys[0]: value}, path)
def standalone_state():
return f"u/{os.environ['WM_USERNAME']}/arista_eos_state"
# "4.34.9M-51132493.4349M (engineering build)" on cEOS, "4.34.9M" on hardware.
def short_version(v):
m = re.match(r"\d+\.\d+\.\d+(\.\d+)?[A-Z]*", v)
return m.group(0) if m else v
Submitted by hugo989 1 day ago