# py312
# requirements:
# pyeapi==1.0.4
# wmill==1.851.0
# Device pre-check: one attempt, before an upgrade of one switch.
# - flash: the target image is already on flash, or free space covers its size on the image server (image_url, or
# image_base_url + EOS-<target_version>.swi; required_free_mb without an image server or when its HEAD has no
# Content-Length). The HEAD checks the certificate and, if the worker doesn't trust it, sizes the image anyway:
# notes says when either happens, and when activate will need its sha512 input. A malformed image URL is flash ko.
# - redundancy: MLAG active and connected, peer-link and local interface up, config-sanity clean, no MLAG port
# active-partial, no port errdisabled. Left out when MLAG is not configured (notes says the reload is not
# hitless); configured but down is ko.
# - interface: saves a snapshot (interfaces up, LLDP neighbor per port, MLAG state and per-port status) in the root
# flow run's user state under arista_eos-<host>-<port>, for Device post-check to compare with.
# A ko check is returned, not raised: nothing to retry, the workflow decides. Raises DeviceUnreachable or AuthFailure.
import logging
import os
import re
import ssl
import time
import urllib.error
import urllib.request
from typing import TypedDict
from urllib.parse import urlsplit
import pyeapi
import wmill
class arista_eos(TypedDict):
host: str
username: str
password: str
port: int
verify_tls: bool
allow_http: bool
def main(
auth: arista_eos,
target_version: str = "",
image_base_url: str = "",
image_url: str = "",
required_free_mb: int = 1024,
):
"""
Device pre-check
One pre-upgrade check of one switch: returns { running_version, status ok|ko, checks_ok, checks_ko[{check, message}], notes, reload_delay_seconds }. Checks flash space for the target image and MLAG redundancy, and saves a snapshot of interfaces/LLDP/MLAG in the flow run's state for Device post-check. Raises DeviceUnreachable or AuthFailure.
"""
key = state_keys(auth)
state = load_state(key) or {}
version, mlag = eapi(auth, ["show version", "show mlag"])
ok, ko, notes = [], [], []
def record(check, problems):
if problems:
ko.append({"check": check, "message": "; ".join(problems)})
else:
ok.append(check)
record(
"flash",
flash_problems(
auth, target_version, image_base_url, image_url, required_free_mb, notes
),
)
if mlag_configured(mlag):
[sanity] = eapi(auth, ["show mlag config-sanity"])
record("redundancy", redundancy_problems(mlag, sanity))
else:
notes.append("redundancy: MLAG not configured, the reload is not hitless")
save_state(
key,
{
**state,
"snapshot": snapshot(auth),
"snapshot_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
},
)
ok.append("interface")
return {
"running_version": short_version(version["version"]),
"status": "ko" if ko else "ok",
"checks_ok": ok,
"checks_ko": ko,
"notes": notes,
"reload_delay_seconds": reload_delay(mlag),
}
def flash_problems(auth, target, base, image_url, required_mb, notes):
[d] = eapi(auth, [{"cmd": "dir flash:/", "revision": 2}])
flash = d["urls"]["flash:/"]
try:
image, url = image_source(target, base, image_url)
except ValueError as e:
return [str(e)]
if image and image in flash["entries"]:
if url:
sidecar_note(url, image, notes)
return []
required = (required_mb or 1024) * 1e6
if url:
try:
try:
size = head(url, ssl.create_default_context())
sidecar_note(url, image, notes)
except urllib.error.URLError as e:
if not isinstance(e.reason, ssl.SSLCertVerificationError):
raise
# Only the size is read: the switch's copy doesn't check the certificate either, and activate
# verifies the image's sha512. So an untrusted (self-signed) server is sized anyway, with a note.
unverified = ssl.create_default_context()
unverified.check_hostname = False
unverified.verify_mode = ssl.CERT_NONE
size = head(url, unverified)
notes.append(
f"flash: the image server's certificate is not trusted on the worker ({e.reason.verify_message}), sized {image} without checking it; activate checks it when it fetches {image}.sha512, so pass it sha512"
)
if size:
required = size
else:
notes.append(
f"flash: the image server gave no size for {image}, assumed required_free_mb ({mb(required)})"
)
except urllib.error.HTTPError as e:
return [f"Target image {image} not found on the image server ({e.code})"]
except OSError as e:
return [
f"Image server unreachable from the worker: {getattr(e, 'reason', e)}"
]
free = flash["filesystem"]["freeBytes"]
if free < required:
return [
f"Insufficient flash space ({mb(free)} free, {mb(required)} required{f' for {image}' if image else ''})"
]
return []
def head(url, context):
with urllib.request.urlopen(
urllib.request.Request(url, method="HEAD"), timeout=30, context=context
) as r:
return int(r.headers.get("content-length") or 0)
# Activate reads the digest from the .sha512 file next to the image, certificate checked, unless given sha512.
def sidecar_note(url, image, notes):
try:
head(f"{url}.sha512", ssl.create_default_context())
except OSError as e:
reason = getattr(e, "code", None) or getattr(e, "reason", e)
notes.append(
f"flash: {image}.sha512 not reachable from the worker ({reason}): pass sha512 to activate"
)
# Same function in Device pre-check and Device activate.
# image_url wins; else image_base_url + EOS-<target>.swi. Returns (file name on flash, URL or ""). The URL goes into
# the switch's copy command, so it is an allowlist: http(s), then only A-Z a-z 0-9 . _ ~ : / % + - (no query, no
# user:password@, nothing the CLI reads as a separator), ending in a plain .swi file name. Raises ValueError otherwise.
def image_source(target, base, image_url):
if image_url:
image, url = urlsplit(image_url).path.rsplit("/", 1)[-1], image_url
else:
image = f"EOS-{target}.swi" if target else ""
url = f"{base.rstrip('/')}/{image}" if image and base else ""
if not url:
return image, url
# Not echoed: it would put the password in the steps and errors.
if "@" in url:
raise ValueError("the image URL must not contain @ (no user:password@ in it)")
if not re.match(r"https?://", url):
raise ValueError(
f"the image URL must start with http:// or https://: got {url!r}"
)
if not urlsplit(url).hostname:
raise ValueError(f"the image URL has no host: got {url!r}")
bad = sorted(set(re.sub(r"[A-Za-z0-9._~:/%+-]", "", url.split("://", 1)[1])))
if bad:
raise ValueError(
f"the image URL may only contain A-Z a-z 0-9 . _ ~ : / % + -, not {' '.join(map(repr, bad))}: got {url!r}"
)
if not re.fullmatch(r"[\w.+-]+\.swi", image):
raise ValueError(f"the image URL must end in a .swi file name: got {url!r}")
return image, url
def redundancy_problems(mlag, sanity):
p = []
if mlag.get("state") != "active":
p.append(f"MLAG state {mlag.get('state')} (expected active)")
if mlag.get("negStatus") != "connected":
p.append(f"MLAG negotiation {mlag.get('negStatus')} (expected connected)")
if mlag.get("peerLinkStatus") != "up":
p.append(f"peer-link {mlag.get('peerLink')} {mlag.get('peerLinkStatus')}")
if mlag.get("localIntfStatus") != "up":
p.append(
f"MLAG local interface {mlag.get('localInterface')} {mlag.get('localIntfStatus')}"
)
inconsistent = [
*(sanity.get("globalConfiguration") or {}),
*(sanity.get("interfaceConfiguration") or {}),
]
if inconsistent:
p.append(f"config-sanity inconsistent: {', '.join(inconsistent)}")
partial = (mlag.get("mlagPorts") or {}).get("Active-partial")
if partial:
p.append(f"{partial} MLAG port(s) active-partial")
if mlag.get("portsErrdisabled"):
p.append("MLAG ports errdisabled")
return p
# show mlag reports state disabled both when MLAG was never configured and when it is configured but shut down;
# only the first has no domain-id, local interface or peer-link.
def mlag_configured(mlag):
return any(mlag.get(k) for k in ("domainId", "localInterface", "peerLink"))
def snapshot(auth):
status, lldp, mlag, mlag_intf = eapi(
auth,
[
"show interfaces status",
"show lldp neighbors",
"show mlag",
"show mlag interfaces",
],
)
return {
"interfaces_up": sorted(
n
for n, i in status["interfaceStatuses"].items()
if i.get("linkStatus") == "connected"
),
"lldp": [
{
"port": n["port"],
"neighbor": f"{n['neighborDevice']} {n['neighborPort']}",
}
for n in lldp["lldpNeighbors"]
],
"mlag": {
"state": mlag.get("state"),
"negStatus": mlag.get("negStatus"),
"peerLinkStatus": mlag.get("peerLinkStatus"),
"ports": {
f"{i['localInterface']} (mlag {id})": i["status"]
for id, i in (mlag_intf.get("interfaces") or {}).items()
},
},
}
# Without MLAG configured, show mlag still reports the default reload-delay, and nothing waits on it.
def reload_delay(mlag):
if not mlag_configured(mlag):
return 0
return max(mlag.get("reloadDelay") or 0, mlag.get("reloadDelayNonMlag") or 0)
def mb(n):
return f"{n / 1e9:.1f} GB" if n >= 1e9 else f"{round(n / 1e6)} MB"
# Errors the workflow branches and retries on (retry_if: result.error.name == "DeviceUnreachable").
class DeviceUnreachable(Exception):
pass
class AuthFailure(Exception):
pass
# pyeapi logs a traceback for every socket error, and a rebooting switch gives many.
logging.getLogger("pyeapi").addHandler(logging.NullHandler())
def eapi(auth: arista_eos, cmds: list, fmt: str = "json", timeout: float = 60):
eos = node(auth, timeout)
try:
return eos.run_commands(cmds, fmt)
except pyeapi.eapilib.ConnectionError as e:
raise connection_error(auth, eos, e)
# pyeapi 1.0.4 raises ConnectionError for a socket error (kept in socket_error), an answer that isn't JSON (eAPI
# still starting, 5xx) and a 401, the only one with neither.
def connection_error(auth, eos, e):
if eos.connection.socket_error is None and e.message != "unable to connect to eAPI":
return AuthFailure(f"{auth['host']}: eAPI refused the credential ({e.message})")
return DeviceUnreachable(f"{auth['host']}: {e.message}")
# host is an address or a URL (https://10.0.0.11:8443); a bare IPv6 address is bracketed. Only https, or http when the
# resource ticks allow_http (Basic auth then crosses the network in cleartext). pyeapi skips certificate checks unless it
# is given a context, so one is always passed: verifying, or not when the resource unticks verify_tls (EOS ships a
# self-signed cert).
def switch_url(auth: arista_eos):
host = auth["host"]
if "://" not in host:
host = (
f"https://[{host}]"
if host.count(":") > 1 and "[" not in host
else f"https://{host}"
)
url = urlsplit(host)
if not url.hostname or (
url.scheme != "https" and not (url.scheme == "http" and auth.get("allow_http"))
):
raise ValueError(
f"host must be an address or an https:// URL (http:// needs allow_http on the resource): {auth['host']}"
)
return url
def node(auth: arista_eos, timeout: float = 60):
url = switch_url(auth)
context = ssl.create_default_context()
if auth.get("verify_tls") is False:
context.check_hostname = False
context.verify_mode = ssl.CERT_NONE
return pyeapi.connect(
transport=url.scheme,
host=url.hostname,
port=url.port or auth.get("port") or None,
username=auth["username"],
password=auth["password"],
context=context,
timeout=timeout,
return_node=True,
)
# Shared by the Device scripts: the root flow run's user state, keyed by switch, holds the pre-check snapshot, the
# image activate replaced (rollback target), the image it booted and the reload markers. Every step of the run (nested
# subflows and retried attempts included) reads the same entry. Outside a flow the Device scripts share one state
# resource in the running user's home (a script's own state would keep the pre-check snapshot from the post-check).
# The endpoint is called through the client so an unreachable state raises (wmill.get_flow_user_state returns None).
# ponytail: the fallback is one map per user, so concurrent standalone runs can race; flows don't.
# The key is the switch's eAPI address: arista_eos-<host>-<port> with non-word characters in the host as _, the port
# always in (443 or 80 when unset). State written before the port was part of the key is read from arista_eos_<host>
# as a fallback (all word characters, so never a new key); writes go to the new key only.
def state_keys(auth: arista_eos):
url = switch_url(auth)
port = int(url.port or auth.get("port") or {"https": 443, "http": 80}[url.scheme])
return [
f"arista_eos-{word(url.hostname)}-{port}",
f"arista_eos_{word(auth['host'])}",
]
def word(s):
return re.sub(r"\W", "_", s)
def load_state(keys):
for key in keys:
if os.environ.get("WM_FLOW_JOB_ID"):
c = wmill.Windmill()
value = c.get(
f"/w/{c.workspace}/jobs/flow/user_states/{c.get_root_job_id()}/{key}"
).json()
else:
value = (wmill.get_state(standalone_state()) or {}).get(key)
if value is not None:
return value
def save_state(keys, value):
if os.environ.get("WM_FLOW_JOB_ID"):
c = wmill.Windmill()
c.post(
f"/w/{c.workspace}/jobs/flow/user_states/{c.get_root_job_id()}/{keys[0]}",
json=value,
)
else:
path = standalone_state()
wmill.set_state({**(wmill.get_state(path) or {}), keys[0]: value}, path)
def standalone_state():
return f"u/{os.environ['WM_USERNAME']}/arista_eos_state"
# "4.34.9M-51132493.4349M (engineering build)" on cEOS, "4.34.9M" on hardware.
def short_version(v):
m = re.match(r"\d+\.\d+\.\d+(\.\d+)?[A-Z]*", v)
return m.group(0) if m else v
Submitted by hugo989 1 day ago