1 | # py312 |
2 | # requirements: |
3 | # pyeapi==1.0.4 |
4 | # wmill==1.851.0 |
5 |
|
6 | # Device check: pre- and post-upgrade checks of one switch. |
7 | # - flash (pre): the target image is already on flash, or free space covers its size on the image server |
8 | # (image_url, or image_base_url + EOS-<target_version>.swi; required_free_mb without an image server or when its |
9 | # HEAD has no Content-Length). The HEAD checks the certificate and, if the worker doesn't trust it, sizes the image |
10 | # anyway: notes says when either happens, and when activate will need its sha512 input (untrusted certificate, or |
11 | # no <image>.sha512 reachable). A malformed image URL is flash ko. |
12 | # - redundancy (pre): MLAG active and connected, peer-link and local interface up, config-sanity clean, no MLAG |
13 | # port active-partial, no port errdisabled. Left out when MLAG is not configured (notes says the reload is not |
14 | # hitless); configured but down is ko. |
15 | # - interface: pre saves a snapshot (interfaces up, LLDP neighbor per port, MLAG state and per-port status) in the |
16 | # flow run's user state, on the root flow, under arista_eos-<host>-<port>; post compares the switch with it. |
17 | # Post waits, in order: for eAPI to answer on a boot later than this run's reload (return_timeout_seconds), for the |
18 | # MLAG reload-delay counted from boot when MLAG is configured, then up to stabilisation_seconds for the state to |
19 | # match the snapshot. The three are returned in durations. The version is reported, not compared with the target. |
20 |
|
21 | import logging |
22 | import os |
23 | import re |
24 | import ssl |
25 | import time |
26 | import urllib.error |
27 | import urllib.request |
28 | from typing import Literal, TypedDict |
29 | from urllib.parse import urlsplit |
30 |
|
31 | import pyeapi |
32 | import wmill |
33 |
|
34 |
|
35 | class arista_eos(TypedDict): |
36 | host: str |
37 | username: str |
38 | password: str |
39 | port: int |
40 | verify_tls: bool |
41 | allow_http: bool |
42 |
|
43 |
|
44 | def main( |
45 | auth: arista_eos, |
46 | phase: Literal["pre", "post"], |
47 | target_version: str = "", |
48 | image_base_url: str = "", |
49 | required_free_mb: int = 1024, |
50 | return_timeout_seconds: int = 900, |
51 | stabilisation_seconds: int = 120, |
52 | poll_interval_seconds: int = 15, |
53 | image_url: str = "", |
54 | ): |
55 | """ |
56 | All-in-one device check |
57 | Pre- or post-upgrade check of one switch: returns { running_version, status ok|ko, checks_ok, checks_ko[{check, message}], notes }. Pre: flash space for the target image, MLAG redundancy, and a snapshot of interfaces/LLDP/MLAG kept in the flow run's state. Post: waits for the switch to answer and for the MLAG reload-delay (when MLAG is configured), then compares with the snapshot over a bounded stabilisation window. Throws DeviceUnreachable or AuthFailure. Does not compare the version with the target: the workflow does. For native step retry and sleep in the workflow, use the single-attempt Device actions (Device pre-check, Get Reload Delay, Device post-check, Device activate, Device reload, Device commit, Device rollback) instead. |
58 | """ |
59 | poll = poll_interval_seconds or 15 |
60 | durations = {} |
61 | key = state_keys(auth) |
62 | state = load_state(key) or {} |
63 |
|
64 | if phase == "post": |
65 | # Wait for the switch to answer on a boot later than the one each reload of this run was sent on: `reload now` |
66 | # can answer before EOS goes down, and the old boot must not pass for the new one. AuthFailure stops at once. |
67 | markers = [state[k] for k in ("reload", "rollback") if state.get(k)] |
68 | timeout = return_timeout_seconds or 900 |
69 | start = time.time() |
70 | while True: |
71 | try: |
72 | [v] = eapi(auth, ["show version"]) |
73 | m = next( |
74 | (m for m in markers if m["boot_timestamp"] == v["bootupTimestamp"]), |
75 | None, |
76 | ) |
77 | if not m: |
78 | break |
79 | pending = ( |
80 | f"still on the boot the reload sent at {m['sent_at']} was sent on" |
81 | ) |
82 | except DeviceUnreachable as e: |
83 | pending = str(e) |
84 | if time.time() - start > timeout: |
85 | raise named( |
86 | DeviceUnreachable, |
87 | f"{auth['host']}: not back after {timeout} s ({pending})", |
88 | ) |
89 | time.sleep(poll) |
90 | durations["return_timeout_seconds"] = timeout |
91 | durations["returned_after_seconds"] = round(time.time() - start) |
92 |
|
93 | version, mlag = eapi(auth, ["show version", "show mlag"]) |
94 | running_version = short_version(version["version"]) |
95 | # Without MLAG configured, show mlag still reports the default reload-delay, and nothing waits on it. |
96 | mlag_on = mlag_configured(mlag) |
97 | durations["reload_delay_seconds"] = ( |
98 | max(mlag.get("reloadDelay") or 0, mlag.get("reloadDelayNonMlag") or 0) |
99 | if mlag_on |
100 | else 0 |
101 | ) |
102 |
|
103 | ok, ko, notes = [], [], [] |
104 |
|
105 | def record(check, problems): |
106 | if problems: |
107 | ko.append({"check": check, "message": "; ".join(problems)}) |
108 | else: |
109 | ok.append(check) |
110 |
|
111 | if phase == "pre": |
112 | record( |
113 | "flash", |
114 | flash_problems( |
115 | auth, target_version, image_base_url, image_url, required_free_mb, notes |
116 | ), |
117 | ) |
118 | if mlag_on: |
119 | [sanity] = eapi(auth, ["show mlag config-sanity"]) |
120 | record("redundancy", redundancy_problems(mlag, sanity)) |
121 | else: |
122 | notes.append("redundancy: MLAG not configured, the reload is not hitless") |
123 | save_state( |
124 | key, |
125 | { |
126 | **state, |
127 | "snapshot": snapshot(auth), |
128 | "snapshot_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), |
129 | }, |
130 | ) |
131 | ok.append("interface") |
132 | elif not state.get("snapshot"): |
133 | ko.append( |
134 | { |
135 | "check": "interface", |
136 | "message": f"No pre-check snapshot for {auth['host']} in this flow run: run the pre-check first, in the same run", |
137 | } |
138 | ) |
139 | else: |
140 | # Ports stay errdisabled until the MLAG reload-delay has elapsed since boot: wait it out. |
141 | wait = durations["reload_delay_seconds"] - version["uptime"] |
142 | if wait > 0: |
143 | time.sleep(wait) |
144 | durations["reload_delay_waited_seconds"] = max(0, round(wait)) |
145 | stab = 120 if stabilisation_seconds is None else stabilisation_seconds |
146 | start = time.time() |
147 | problems = diff(state["snapshot"], snapshot(auth)) |
148 | while problems and time.time() - start < stab: |
149 | time.sleep(poll) |
150 | problems = diff(state["snapshot"], snapshot(auth)) |
151 | durations["stabilisation_seconds"] = stab |
152 | durations["stabilised_after_seconds"] = round(time.time() - start) |
153 | record("interface", problems) |
154 |
|
155 | return { |
156 | "running_version": running_version, |
157 | "status": "ko" if ko else "ok", |
158 | "checks_ok": ok, |
159 | "checks_ko": ko, |
160 | "durations": durations, |
161 | "notes": notes, |
162 | } |
163 |
|
164 |
|
165 | def flash_problems(auth, target, base, image_url, required_mb, notes): |
166 | [d] = eapi(auth, [{"cmd": "dir flash:/", "revision": 2}]) |
167 | flash = d["urls"]["flash:/"] |
168 | try: |
169 | image, url = image_source(target, base, image_url) |
170 | except ValueError as e: |
171 | return [str(e)] |
172 | if image and image in flash["entries"]: |
173 | if url: |
174 | sidecar_note(url, image, notes) |
175 | return [] |
176 | required = (required_mb or 1024) * 1e6 |
177 | if url: |
178 | try: |
179 | try: |
180 | size = head(url, ssl.create_default_context()) |
181 | sidecar_note(url, image, notes) |
182 | except urllib.error.URLError as e: |
183 | if not isinstance(e.reason, ssl.SSLCertVerificationError): |
184 | raise |
185 | # Only the size is read: the switch's copy doesn't check the certificate either, and activate |
186 | # verifies the image's sha512. So an untrusted (self-signed) server is sized anyway, with a note. |
187 | unverified = ssl.create_default_context() |
188 | unverified.check_hostname = False |
189 | unverified.verify_mode = ssl.CERT_NONE |
190 | size = head(url, unverified) |
191 | notes.append( |
192 | f"flash: the image server's certificate is not trusted on the worker ({e.reason.verify_message}), sized {image} without checking it; activate checks it when it fetches {image}.sha512, so pass it sha512" |
193 | ) |
194 | if size: |
195 | required = size |
196 | else: |
197 | notes.append( |
198 | f"flash: the image server gave no size for {image}, assumed required_free_mb ({mb(required)})" |
199 | ) |
200 | except urllib.error.HTTPError as e: |
201 | return [f"Target image {image} not found on the image server ({e.code})"] |
202 | except OSError as e: |
203 | return [ |
204 | f"Image server unreachable from the worker: {getattr(e, 'reason', e)}" |
205 | ] |
206 | free = flash["filesystem"]["freeBytes"] |
207 | if free < required: |
208 | return [ |
209 | f"Insufficient flash space ({mb(free)} free, {mb(required)} required{f' for {image}' if image else ''})" |
210 | ] |
211 | return [] |
212 |
|
213 |
|
214 | def head(url, context): |
215 | with urllib.request.urlopen( |
216 | urllib.request.Request(url, method="HEAD"), timeout=30, context=context |
217 | ) as r: |
218 | return int(r.headers.get("content-length") or 0) |
219 |
|
220 |
|
221 | # Activate reads the digest from the .sha512 file next to the image, certificate checked, unless given sha512. |
222 | def sidecar_note(url, image, notes): |
223 | try: |
224 | head(f"{url}.sha512", ssl.create_default_context()) |
225 | except OSError as e: |
226 | reason = getattr(e, "code", None) or getattr(e, "reason", e) |
227 | notes.append( |
228 | f"flash: {image}.sha512 not reachable from the worker ({reason}): pass sha512 to activate" |
229 | ) |
230 |
|
231 |
|
232 | # Same function in Device check and Device upgrade. |
233 | # image_url wins; else image_base_url + EOS-<target>.swi. Returns (file name on flash, URL or ""). The URL goes into |
234 | # the switch's copy command, so it is an allowlist: http(s), then only A-Z a-z 0-9 . _ ~ : / % + - (no query, no |
235 | # user:password@, nothing the CLI reads as a separator), ending in a plain .swi file name. Raises ValueError otherwise. |
236 | def image_source(target, base, image_url): |
237 | if image_url: |
238 | image, url = urlsplit(image_url).path.rsplit("/", 1)[-1], image_url |
239 | else: |
240 | image = f"EOS-{target}.swi" if target else "" |
241 | url = f"{base.rstrip('/')}/{image}" if image and base else "" |
242 | if not url: |
243 | return image, url |
244 | # Not echoed: it would put the password in the steps and errors. |
245 | if "@" in url: |
246 | raise ValueError("the image URL must not contain @ (no user:password@ in it)") |
247 | if not re.match(r"https?: |
248 | raise ValueError( |
249 | f"the image URL must start with http:// or https://: got {url!r}" |
250 | ) |
251 | if not urlsplit(url).hostname: |
252 | raise ValueError(f"the image URL has no host: got {url!r}") |
253 | bad = sorted(set(re.sub(r"[A-Za-z0-9._~:/%+-]", "", url.split("://", 1)[1]))) |
254 | if bad: |
255 | raise ValueError( |
256 | f"the image URL may only contain A-Z a-z 0-9 . _ ~ : / % + -, not {' '.join(map(repr, bad))}: got {url!r}" |
257 | ) |
258 | if not re.fullmatch(r"[\w.+-]+\.swi", image): |
259 | raise ValueError(f"the image URL must end in a .swi file name: got {url!r}") |
260 | return image, url |
261 |
|
262 |
|
263 | # show mlag reports state disabled both when MLAG was never configured and when it is configured but shut down; |
264 | # only the first has no domain-id, local interface or peer-link. |
265 | def mlag_configured(mlag): |
266 | return any(mlag.get(k) for k in ("domainId", "localInterface", "peerLink")) |
267 |
|
268 |
|
269 | def redundancy_problems(mlag, sanity): |
270 | p = [] |
271 | if mlag.get("state") != "active": |
272 | p.append(f"MLAG state {mlag.get('state')} (expected active)") |
273 | if mlag.get("negStatus") != "connected": |
274 | p.append(f"MLAG negotiation {mlag.get('negStatus')} (expected connected)") |
275 | if mlag.get("peerLinkStatus") != "up": |
276 | p.append(f"peer-link {mlag.get('peerLink')} {mlag.get('peerLinkStatus')}") |
277 | if mlag.get("localIntfStatus") != "up": |
278 | p.append( |
279 | f"MLAG local interface {mlag.get('localInterface')} {mlag.get('localIntfStatus')}" |
280 | ) |
281 | inconsistent = [ |
282 | *(sanity.get("globalConfiguration") or {}), |
283 | *(sanity.get("interfaceConfiguration") or {}), |
284 | ] |
285 | if inconsistent: |
286 | p.append(f"config-sanity inconsistent: {', '.join(inconsistent)}") |
287 | partial = (mlag.get("mlagPorts") or {}).get("Active-partial") |
288 | if partial: |
289 | p.append(f"{partial} MLAG port(s) active-partial") |
290 | if mlag.get("portsErrdisabled"): |
291 | p.append("MLAG ports errdisabled") |
292 | return p |
293 |
|
294 |
|
295 | def snapshot(auth): |
296 | status, lldp, mlag, mlag_intf = eapi( |
297 | auth, |
298 | [ |
299 | "show interfaces status", |
300 | "show lldp neighbors", |
301 | "show mlag", |
302 | "show mlag interfaces", |
303 | ], |
304 | ) |
305 | return { |
306 | "interfaces_up": sorted( |
307 | n |
308 | for n, i in status["interfaceStatuses"].items() |
309 | if i.get("linkStatus") == "connected" |
310 | ), |
311 | "lldp": [ |
312 | { |
313 | "port": n["port"], |
314 | "neighbor": f"{n['neighborDevice']} {n['neighborPort']}", |
315 | } |
316 | for n in lldp["lldpNeighbors"] |
317 | ], |
318 | "mlag": { |
319 | "state": mlag.get("state"), |
320 | "negStatus": mlag.get("negStatus"), |
321 | "peerLinkStatus": mlag.get("peerLinkStatus"), |
322 | "ports": { |
323 | f"{i['localInterface']} (mlag {id})": i["status"] |
324 | for id, i in (mlag_intf.get("interfaces") or {}).items() |
325 | }, |
326 | }, |
327 | } |
328 |
|
329 |
|
330 | def diff(before, after): |
331 | p = [] |
332 | for i in before["interfaces_up"]: |
333 | if i not in after["interfaces_up"]: |
334 | p.append(f"{i} down after the upgrade (up before)") |
335 | for n in before["lldp"]: |
336 | if not any( |
337 | a["port"] == n["port"] and a["neighbor"] == n["neighbor"] |
338 | for a in after["lldp"] |
339 | ): |
340 | p.append(f"LLDP neighbor {n['neighbor']} no longer seen on {n['port']}") |
341 | for k in ("state", "negStatus", "peerLinkStatus"): |
342 | if before["mlag"][k] != after["mlag"][k]: |
343 | p.append(f"MLAG {k} {after['mlag'][k]} (was {before['mlag'][k]})") |
344 | for port, s in before["mlag"]["ports"].items(): |
345 | if after["mlag"]["ports"].get(port) != s: |
346 | p.append(f"{port} {after['mlag']['ports'].get(port, 'missing')} (was {s})") |
347 | return p |
348 |
|
349 |
|
350 | # The snapshot lives in the root flow run's user state, keyed by switch: pre- and post-check are different steps |
351 | # (often in nested subflows) of the same run. Outside a flow, it falls back to this script's own state. |
352 | # The endpoint is called through the client so an unreachable state raises (wmill.get_flow_user_state returns None). |
353 | # ponytail: the fallback is one map per script, so concurrent standalone runs can race; flows don't. |
354 | # The key is the switch's eAPI address: arista_eos-<host>-<port> with non-word characters in the host as _, the port |
355 | # always in (443 or 80 when unset), so host 10.0.0.11 with port 8443 and host https://10.0.0.11:8443 share one key, |
356 | # while two switches behind one address on different ports (or http and https) don't. State written before the port |
357 | # was part of the key is read from arista_eos_<host> as a fallback (all word characters, so never a new key); writes |
358 | # go to the new key only. |
359 | def state_keys(auth: arista_eos): |
360 | url = switch_url(auth) |
361 | port = int(url.port or auth.get("port") or {"https": 443, "http": 80}[url.scheme]) |
362 | return [ |
363 | f"arista_eos-{word(url.hostname)}-{port}", |
364 | f"arista_eos_{word(auth['host'])}", |
365 | ] |
366 |
|
367 |
|
368 | def word(s): |
369 | return re.sub(r"\W", "_", s) |
370 |
|
371 |
|
372 | def load_state(keys): |
373 | for key in keys: |
374 | if os.environ.get("WM_FLOW_JOB_ID"): |
375 | c = wmill.Windmill() |
376 | value = c.get( |
377 | f"/w/{c.workspace}/jobs/flow/user_states/{c.get_root_job_id()}/{key}" |
378 | ).json() |
379 | else: |
380 | value = (wmill.get_state() or {}).get(key) |
381 | if value is not None: |
382 | return value |
383 |
|
384 |
|
385 | def save_state(keys, value): |
386 | if os.environ.get("WM_FLOW_JOB_ID"): |
387 | c = wmill.Windmill() |
388 | c.post( |
389 | f"/w/{c.workspace}/jobs/flow/user_states/{c.get_root_job_id()}/{keys[0]}", |
390 | json=value, |
391 | ) |
392 | else: |
393 | wmill.set_state({**(wmill.get_state() or {}), keys[0]: value}) |
394 |
|
395 |
|
396 | # "4.34.9M-51132493.4349M (engineering build)" on cEOS, "4.34.9M" on hardware. |
397 | def short_version(v): |
398 | m = re.match(r"\d+\.\d+\.\d+(\.\d+)?[A-Z]*", v) |
399 | return m.group(0) if m else v |
400 |
|
401 |
|
402 | def mb(n): |
403 | return f"{n / 1e9:.1f} GB" if n >= 1e9 else f"{round(n / 1e6)} MB" |
404 |
|
405 |
|
406 | class DeviceUnreachable(Exception): |
407 | pass |
408 |
|
409 |
|
410 | class AuthFailure(Exception): |
411 | pass |
412 |
|
413 |
|
414 | def named(cls, message: str): |
415 | return cls(f"{cls.__name__}: {message}") |
416 |
|
417 |
|
418 | # pyeapi logs a traceback for every socket error, and the post-check polls a switch that is rebooting. |
419 | logging.getLogger("pyeapi").addHandler(logging.NullHandler()) |
420 |
|
421 |
|
422 | def eapi(auth: arista_eos, cmds: list, fmt: str = "json", timeout: float = 60): |
423 | eos = node(auth, timeout) |
424 | try: |
425 | return eos.run_commands(cmds, fmt) |
426 | except pyeapi.eapilib.ConnectionError as e: |
427 | raise connection_error(auth, eos, e) |
428 |
|
429 |
|
430 | # pyeapi 1.0.4 raises ConnectionError for a socket error (kept in socket_error), an answer that isn't JSON (eAPI |
431 | # still starting, 5xx) and a 401, the only one with neither. |
432 | def connection_error(auth, eos, e): |
433 | if eos.connection.socket_error is None and e.message != "unable to connect to eAPI": |
434 | return named( |
435 | AuthFailure, f"{auth['host']}: eAPI refused the credential ({e.message})" |
436 | ) |
437 | return named(DeviceUnreachable, f"{auth['host']}: {e.message}") |
438 |
|
439 |
|
440 | # host is an address or a URL (https://10.0.0.11:8443); a bare IPv6 address is bracketed. Only https, or http when the |
441 | # resource ticks allow_http (Basic auth then crosses the network in cleartext). pyeapi skips certificate checks unless it |
442 | # is given a context, so one is always passed: verifying, or not when the resource unticks verify_tls (EOS ships a |
443 | # self-signed cert). |
444 | def switch_url(auth: arista_eos): |
445 | host = auth["host"] |
446 | if ": |
447 | host = f"https://[{host}]" if host.count(":") > 1 and "[" not in host else f"https://{host}" |
448 | url = urlsplit(host) |
449 | if not url.hostname or (url.scheme != "https" and not (url.scheme == "http" and auth.get("allow_http"))): |
450 | raise ValueError(f"host must be an address or an https:// URL (http:// needs allow_http on the resource): {auth['host']}") |
451 | return url |
452 |
|
453 |
|
454 | def node(auth: arista_eos, timeout: float = 60): |
455 | url = switch_url(auth) |
456 | context = ssl.create_default_context() |
457 | if auth.get("verify_tls") is False: |
458 | context.check_hostname = False |
459 | context.verify_mode = ssl.CERT_NONE |
460 | return pyeapi.connect( |
461 | transport=url.scheme, |
462 | host=url.hostname, |
463 | port=url.port or auth.get("port") or None, |
464 | username=auth["username"], |
465 | password=auth["password"], |
466 | context=context, |
467 | timeout=timeout, |
468 | return_node=True, |
469 | ) |
470 |
|