# py312
# requirements:
# pyeapi==1.0.4
# wmill==1.851.0
# Get Reload Delay: one attempt, after a reload or rollback of one switch. Raises DeviceUnreachable while the switch
# is down or still runs the boot a reload of this run was sent on, so the workflow retries this step until the switch
# is back. Then returns the MLAG reload-delay (0 without MLAG) and the part of it still to run, counted from boot:
# ports stay errdisabled until it ends, so the workflow sleeps remaining_seconds before Device post-check.
import logging
import os
import re
import ssl
from typing import TypedDict
from urllib.parse import urlsplit
import pyeapi
import wmill
class arista_eos(TypedDict):
host: str
username: str
password: str
port: int
verify_tls: bool
allow_http: bool
def main(auth: arista_eos):
"""
Get Reload Delay
After a reload, once the switch answers on a new boot: returns { running_version, uptime_seconds, reload_delay_seconds, remaining_seconds }, the MLAG reload-delay (0 without MLAG) and the seconds of it still to run; sleep remaining_seconds before Device post-check. Raises DeviceUnreachable while the switch is down or still on the boot the reload was sent on, AuthFailure when the credential is refused.
"""
state = load_state(state_keys(auth)) or {}
version = returned(auth, state)
[mlag] = eapi(auth, ["show mlag"])
delay = reload_delay(mlag)
return {
"running_version": short_version(version["version"]),
"uptime_seconds": round(version["uptime"]),
"reload_delay_seconds": delay,
"remaining_seconds": max(0, round(delay - version["uptime"])),
}
# Same function in Get Reload Delay and Device post-check. One attempt: show version, raising DeviceUnreachable
# while the switch is down, and while it still runs the boot a reload of this run was sent on (reload now can answer
# before EOS goes down, and the old boot must not pass for the new one). AuthFailure is raised as is.
def returned(auth, state):
try:
# Short: a switch that is down holds the attempt for the whole timeout, and the workflow retries anyway.
[v] = eapi(auth, ["show version"], timeout=15)
except DeviceUnreachable as e:
raise DeviceUnreachable(f"{e}: not back yet")
for k in ("reload", "rollback"):
m = state.get(k)
if m and m["boot_timestamp"] == v["bootupTimestamp"]:
raise DeviceUnreachable(
f"{auth['host']}: still on the boot the {k} sent at {m['sent_at']} was sent on"
)
return v
# show mlag reports state disabled both when MLAG was never configured and when it is configured but shut down;
# only the first has no domain-id, local interface or peer-link.
def mlag_configured(mlag):
return any(mlag.get(k) for k in ("domainId", "localInterface", "peerLink"))
# Without MLAG configured, show mlag still reports the default reload-delay, and nothing waits on it.
def reload_delay(mlag):
if not mlag_configured(mlag):
return 0
return max(mlag.get("reloadDelay") or 0, mlag.get("reloadDelayNonMlag") or 0)
# Errors the workflow branches and retries on (retry_if: result.error.name == "DeviceUnreachable").
class DeviceUnreachable(Exception):
pass
class AuthFailure(Exception):
pass
# pyeapi logs a traceback for every socket error, and a rebooting switch gives many.
logging.getLogger("pyeapi").addHandler(logging.NullHandler())
# This script runs right after a reboot, when eAPI can answer before the EOS agents behind a show command have
# started: such an error answer counts as not back yet, so the workflow retries it. It only sends show commands.
def eapi(auth: arista_eos, cmds: list, fmt: str = "json", timeout: float = 60):
eos = node(auth, timeout)
try:
return eos.run_commands(cmds, fmt)
except pyeapi.eapilib.ConnectionError as e:
raise connection_error(auth, eos, e)
except pyeapi.eapilib.CommandError as e:
raise DeviceUnreachable(f"{auth['host']}: not ready yet ({e.message})")
# pyeapi 1.0.4 raises ConnectionError for a socket error (kept in socket_error), an answer that isn't JSON (eAPI
# still starting, 5xx) and a 401, the only one with neither.
def connection_error(auth, eos, e):
if eos.connection.socket_error is None and e.message != "unable to connect to eAPI":
return AuthFailure(f"{auth['host']}: eAPI refused the credential ({e.message})")
return DeviceUnreachable(f"{auth['host']}: {e.message}")
# host is an address or a URL (https://10.0.0.11:8443); a bare IPv6 address is bracketed. Only https, or http when the
# resource ticks allow_http (Basic auth then crosses the network in cleartext). pyeapi skips certificate checks unless it
# is given a context, so one is always passed: verifying, or not when the resource unticks verify_tls (EOS ships a
# self-signed cert).
def switch_url(auth: arista_eos):
host = auth["host"]
if "://" not in host:
host = (
f"https://[{host}]"
if host.count(":") > 1 and "[" not in host
else f"https://{host}"
)
url = urlsplit(host)
if not url.hostname or (
url.scheme != "https" and not (url.scheme == "http" and auth.get("allow_http"))
):
raise ValueError(
f"host must be an address or an https:// URL (http:// needs allow_http on the resource): {auth['host']}"
)
return url
def node(auth: arista_eos, timeout: float = 60):
url = switch_url(auth)
context = ssl.create_default_context()
if auth.get("verify_tls") is False:
context.check_hostname = False
context.verify_mode = ssl.CERT_NONE
return pyeapi.connect(
transport=url.scheme,
host=url.hostname,
port=url.port or auth.get("port") or None,
username=auth["username"],
password=auth["password"],
context=context,
timeout=timeout,
return_node=True,
)
# Shared by the Device scripts: the root flow run's user state, keyed by switch, holds the pre-check snapshot, the
# image activate replaced (rollback target), the image it booted and the reload markers. Every step of the run (nested
# subflows and retried attempts included) reads the same entry. Outside a flow the Device scripts share one state
# resource in the running user's home (a script's own state would keep the pre-check snapshot from the post-check).
# The endpoint is called through the client so an unreachable state raises (wmill.get_flow_user_state returns None).
# ponytail: the fallback is one map per user, so concurrent standalone runs can race; flows don't.
# The key is the switch's eAPI address: arista_eos-<host>-<port> with non-word characters in the host as _, the port
# always in (443 or 80 when unset). State written before the port was part of the key is read from arista_eos_<host>
# as a fallback (all word characters, so never a new key); writes go to the new key only.
def state_keys(auth: arista_eos):
url = switch_url(auth)
port = int(url.port or auth.get("port") or {"https": 443, "http": 80}[url.scheme])
return [
f"arista_eos-{word(url.hostname)}-{port}",
f"arista_eos_{word(auth['host'])}",
]
def word(s):
return re.sub(r"\W", "_", s)
def load_state(keys):
for key in keys:
if os.environ.get("WM_FLOW_JOB_ID"):
c = wmill.Windmill()
value = c.get(
f"/w/{c.workspace}/jobs/flow/user_states/{c.get_root_job_id()}/{key}"
).json()
else:
value = (wmill.get_state(standalone_state()) or {}).get(key)
if value is not None:
return value
def save_state(keys, value):
if os.environ.get("WM_FLOW_JOB_ID"):
c = wmill.Windmill()
c.post(
f"/w/{c.workspace}/jobs/flow/user_states/{c.get_root_job_id()}/{keys[0]}",
json=value,
)
else:
path = standalone_state()
wmill.set_state({**(wmill.get_state(path) or {}), keys[0]: value}, path)
def standalone_state():
return f"u/{os.environ['WM_USERNAME']}/arista_eos_state"
# "4.34.9M-51132493.4349M (engineering build)" on cEOS, "4.34.9M" on hardware.
def short_version(v):
m = re.match(r"\d+\.\d+\.\d+(\.\d+)?[A-Z]*", v)
return m.group(0) if m else v
Submitted by hugo989 1 day ago