1 | # py312 |
2 | # requirements: |
3 | # pyeapi==1.0.4 |
4 | # wmill==1.851.0 |
5 |
|
6 | # Device pre-check: one attempt, before an upgrade of one switch. |
7 | # - flash: the target image is already on flash, or free space covers its size on the image server (image_url, or |
8 | # image_base_url + EOS-<target_version>.swi; required_free_mb without an image server or when its HEAD has no |
9 | # Content-Length). The HEAD checks the certificate and, if the worker doesn't trust it, sizes the image anyway: |
10 | # notes says when either happens, and when activate will need its sha512 input. A malformed image URL is flash ko. |
11 | # - redundancy: MLAG active and connected, peer-link and local interface up, config-sanity clean, no MLAG port |
12 | # active-partial, no port errdisabled. Left out when MLAG is not configured (notes says the reload is not |
13 | # hitless); configured but down is ko. |
14 | # - interface: saves a snapshot (interfaces up, LLDP neighbor per port, MLAG state and per-port status) in the root |
15 | # flow run's user state under arista_eos-<host>-<port>, for Device post-check to compare with. |
16 | # A ko check is returned, not raised: nothing to retry, the workflow decides. Raises DeviceUnreachable or AuthFailure. |
17 |
|
18 |
|
19 | import logging |
20 | import os |
21 | import re |
22 | import ssl |
23 | import time |
24 | import urllib.error |
25 | import urllib.request |
26 | from typing import TypedDict |
27 | from urllib.parse import urlsplit |
28 |
|
29 | import pyeapi |
30 | import wmill |
31 |
|
32 |
|
33 | class arista_eos(TypedDict): |
34 | host: str |
35 | username: str |
36 | password: str |
37 | port: int |
38 | verify_tls: bool |
39 | allow_http: bool |
40 |
|
41 |
|
42 | def main( |
43 | auth: arista_eos, |
44 | target_version: str = "", |
45 | image_base_url: str = "", |
46 | image_url: str = "", |
47 | required_free_mb: int = 1024, |
48 | ): |
49 | """ |
50 | Device pre-check |
51 | One pre-upgrade check of one switch: returns { running_version, status ok|ko, checks_ok, checks_ko[{check, message}], notes, reload_delay_seconds }. Checks flash space for the target image and MLAG redundancy, and saves a snapshot of interfaces/LLDP/MLAG in the flow run's state for Device post-check. Raises DeviceUnreachable or AuthFailure. |
52 | """ |
53 | key = state_keys(auth) |
54 | state = load_state(key) or {} |
55 | version, mlag = eapi(auth, ["show version", "show mlag"]) |
56 | ok, ko, notes = [], [], [] |
57 |
|
58 | def record(check, problems): |
59 | if problems: |
60 | ko.append({"check": check, "message": "; ".join(problems)}) |
61 | else: |
62 | ok.append(check) |
63 |
|
64 | record( |
65 | "flash", |
66 | flash_problems( |
67 | auth, target_version, image_base_url, image_url, required_free_mb, notes |
68 | ), |
69 | ) |
70 | if mlag_configured(mlag): |
71 | [sanity] = eapi(auth, ["show mlag config-sanity"]) |
72 | record("redundancy", redundancy_problems(mlag, sanity)) |
73 | else: |
74 | notes.append("redundancy: MLAG not configured, the reload is not hitless") |
75 | save_state( |
76 | key, |
77 | { |
78 | **state, |
79 | "snapshot": snapshot(auth), |
80 | "snapshot_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), |
81 | }, |
82 | ) |
83 | ok.append("interface") |
84 | return { |
85 | "running_version": short_version(version["version"]), |
86 | "status": "ko" if ko else "ok", |
87 | "checks_ok": ok, |
88 | "checks_ko": ko, |
89 | "notes": notes, |
90 | "reload_delay_seconds": reload_delay(mlag), |
91 | } |
92 |
|
93 |
|
94 | def flash_problems(auth, target, base, image_url, required_mb, notes): |
95 | [d] = eapi(auth, [{"cmd": "dir flash:/", "revision": 2}]) |
96 | flash = d["urls"]["flash:/"] |
97 | try: |
98 | image, url = image_source(target, base, image_url) |
99 | except ValueError as e: |
100 | return [str(e)] |
101 | if image and image in flash["entries"]: |
102 | if url: |
103 | sidecar_note(url, image, notes) |
104 | return [] |
105 | required = (required_mb or 1024) * 1e6 |
106 | if url: |
107 | try: |
108 | try: |
109 | size = head(url, ssl.create_default_context()) |
110 | sidecar_note(url, image, notes) |
111 | except urllib.error.URLError as e: |
112 | if not isinstance(e.reason, ssl.SSLCertVerificationError): |
113 | raise |
114 | # Only the size is read: the switch's copy doesn't check the certificate either, and activate |
115 | # verifies the image's sha512. So an untrusted (self-signed) server is sized anyway, with a note. |
116 | unverified = ssl.create_default_context() |
117 | unverified.check_hostname = False |
118 | unverified.verify_mode = ssl.CERT_NONE |
119 | size = head(url, unverified) |
120 | notes.append( |
121 | f"flash: the image server's certificate is not trusted on the worker ({e.reason.verify_message}), sized {image} without checking it; activate checks it when it fetches {image}.sha512, so pass it sha512" |
122 | ) |
123 | if size: |
124 | required = size |
125 | else: |
126 | notes.append( |
127 | f"flash: the image server gave no size for {image}, assumed required_free_mb ({mb(required)})" |
128 | ) |
129 | except urllib.error.HTTPError as e: |
130 | return [f"Target image {image} not found on the image server ({e.code})"] |
131 | except OSError as e: |
132 | return [ |
133 | f"Image server unreachable from the worker: {getattr(e, 'reason', e)}" |
134 | ] |
135 | free = flash["filesystem"]["freeBytes"] |
136 | if free < required: |
137 | return [ |
138 | f"Insufficient flash space ({mb(free)} free, {mb(required)} required{f' for {image}' if image else ''})" |
139 | ] |
140 | return [] |
141 |
|
142 |
|
143 | def head(url, context): |
144 | with urllib.request.urlopen( |
145 | urllib.request.Request(url, method="HEAD"), timeout=30, context=context |
146 | ) as r: |
147 | return int(r.headers.get("content-length") or 0) |
148 |
|
149 |
|
150 | # Activate reads the digest from the .sha512 file next to the image, certificate checked, unless given sha512. |
151 | def sidecar_note(url, image, notes): |
152 | try: |
153 | head(f"{url}.sha512", ssl.create_default_context()) |
154 | except OSError as e: |
155 | reason = getattr(e, "code", None) or getattr(e, "reason", e) |
156 | notes.append( |
157 | f"flash: {image}.sha512 not reachable from the worker ({reason}): pass sha512 to activate" |
158 | ) |
159 |
|
160 |
|
161 | # Same function in Device pre-check and Device activate. |
162 | # image_url wins; else image_base_url + EOS-<target>.swi. Returns (file name on flash, URL or ""). The URL goes into |
163 | # the switch's copy command, so it is an allowlist: http(s), then only A-Z a-z 0-9 . _ ~ : / % + - (no query, no |
164 | # user:password@, nothing the CLI reads as a separator), ending in a plain .swi file name. Raises ValueError otherwise. |
165 | def image_source(target, base, image_url): |
166 | if image_url: |
167 | image, url = urlsplit(image_url).path.rsplit("/", 1)[-1], image_url |
168 | else: |
169 | image = f"EOS-{target}.swi" if target else "" |
170 | url = f"{base.rstrip('/')}/{image}" if image and base else "" |
171 | if not url: |
172 | return image, url |
173 | # Not echoed: it would put the password in the steps and errors. |
174 | if "@" in url: |
175 | raise ValueError("the image URL must not contain @ (no user:password@ in it)") |
176 | if not re.match(r"https?: |
177 | raise ValueError( |
178 | f"the image URL must start with http:// or https://: got {url!r}" |
179 | ) |
180 | if not urlsplit(url).hostname: |
181 | raise ValueError(f"the image URL has no host: got {url!r}") |
182 | bad = sorted(set(re.sub(r"[A-Za-z0-9._~:/%+-]", "", url.split("://", 1)[1]))) |
183 | if bad: |
184 | raise ValueError( |
185 | f"the image URL may only contain A-Z a-z 0-9 . _ ~ : / % + -, not {' '.join(map(repr, bad))}: got {url!r}" |
186 | ) |
187 | if not re.fullmatch(r"[\w.+-]+\.swi", image): |
188 | raise ValueError(f"the image URL must end in a .swi file name: got {url!r}") |
189 | return image, url |
190 |
|
191 |
|
192 | def redundancy_problems(mlag, sanity): |
193 | p = [] |
194 | if mlag.get("state") != "active": |
195 | p.append(f"MLAG state {mlag.get('state')} (expected active)") |
196 | if mlag.get("negStatus") != "connected": |
197 | p.append(f"MLAG negotiation {mlag.get('negStatus')} (expected connected)") |
198 | if mlag.get("peerLinkStatus") != "up": |
199 | p.append(f"peer-link {mlag.get('peerLink')} {mlag.get('peerLinkStatus')}") |
200 | if mlag.get("localIntfStatus") != "up": |
201 | p.append( |
202 | f"MLAG local interface {mlag.get('localInterface')} {mlag.get('localIntfStatus')}" |
203 | ) |
204 | inconsistent = [ |
205 | *(sanity.get("globalConfiguration") or {}), |
206 | *(sanity.get("interfaceConfiguration") or {}), |
207 | ] |
208 | if inconsistent: |
209 | p.append(f"config-sanity inconsistent: {', '.join(inconsistent)}") |
210 | partial = (mlag.get("mlagPorts") or {}).get("Active-partial") |
211 | if partial: |
212 | p.append(f"{partial} MLAG port(s) active-partial") |
213 | if mlag.get("portsErrdisabled"): |
214 | p.append("MLAG ports errdisabled") |
215 | return p |
216 |
|
217 |
|
218 | # show mlag reports state disabled both when MLAG was never configured and when it is configured but shut down; |
219 | # only the first has no domain-id, local interface or peer-link. |
220 | def mlag_configured(mlag): |
221 | return any(mlag.get(k) for k in ("domainId", "localInterface", "peerLink")) |
222 |
|
223 |
|
224 | def snapshot(auth): |
225 | status, lldp, mlag, mlag_intf = eapi( |
226 | auth, |
227 | [ |
228 | "show interfaces status", |
229 | "show lldp neighbors", |
230 | "show mlag", |
231 | "show mlag interfaces", |
232 | ], |
233 | ) |
234 | return { |
235 | "interfaces_up": sorted( |
236 | n |
237 | for n, i in status["interfaceStatuses"].items() |
238 | if i.get("linkStatus") == "connected" |
239 | ), |
240 | "lldp": [ |
241 | { |
242 | "port": n["port"], |
243 | "neighbor": f"{n['neighborDevice']} {n['neighborPort']}", |
244 | } |
245 | for n in lldp["lldpNeighbors"] |
246 | ], |
247 | "mlag": { |
248 | "state": mlag.get("state"), |
249 | "negStatus": mlag.get("negStatus"), |
250 | "peerLinkStatus": mlag.get("peerLinkStatus"), |
251 | "ports": { |
252 | f"{i['localInterface']} (mlag {id})": i["status"] |
253 | for id, i in (mlag_intf.get("interfaces") or {}).items() |
254 | }, |
255 | }, |
256 | } |
257 |
|
258 |
|
259 | # Without MLAG configured, show mlag still reports the default reload-delay, and nothing waits on it. |
260 | def reload_delay(mlag): |
261 | if not mlag_configured(mlag): |
262 | return 0 |
263 | return max(mlag.get("reloadDelay") or 0, mlag.get("reloadDelayNonMlag") or 0) |
264 |
|
265 |
|
266 | def mb(n): |
267 | return f"{n / 1e9:.1f} GB" if n >= 1e9 else f"{round(n / 1e6)} MB" |
268 |
|
269 |
|
270 | # Errors the workflow branches and retries on (retry_if: result.error.name == "DeviceUnreachable"). |
271 | class DeviceUnreachable(Exception): |
272 | pass |
273 |
|
274 |
|
275 | class AuthFailure(Exception): |
276 | pass |
277 |
|
278 |
|
279 | # pyeapi logs a traceback for every socket error, and a rebooting switch gives many. |
280 | logging.getLogger("pyeapi").addHandler(logging.NullHandler()) |
281 |
|
282 |
|
283 | def eapi(auth: arista_eos, cmds: list, fmt: str = "json", timeout: float = 60): |
284 | eos = node(auth, timeout) |
285 | try: |
286 | return eos.run_commands(cmds, fmt) |
287 | except pyeapi.eapilib.ConnectionError as e: |
288 | raise connection_error(auth, eos, e) |
289 |
|
290 |
|
291 | # pyeapi 1.0.4 raises ConnectionError for a socket error (kept in socket_error), an answer that isn't JSON (eAPI |
292 | # still starting, 5xx) and a 401, the only one with neither. |
293 | def connection_error(auth, eos, e): |
294 | if eos.connection.socket_error is None and e.message != "unable to connect to eAPI": |
295 | return AuthFailure(f"{auth['host']}: eAPI refused the credential ({e.message})") |
296 | return DeviceUnreachable(f"{auth['host']}: {e.message}") |
297 |
|
298 |
|
299 | # host is an address or a URL (https://10.0.0.11:8443); a bare IPv6 address is bracketed. Only https, or http when the |
300 | # resource ticks allow_http (Basic auth then crosses the network in cleartext). pyeapi skips certificate checks unless it |
301 | # is given a context, so one is always passed: verifying, or not when the resource unticks verify_tls (EOS ships a |
302 | # self-signed cert). |
303 | def switch_url(auth: arista_eos): |
304 | host = auth["host"] |
305 | if "://" not in host: |
306 | host = ( |
307 | f"https://[{host}]" |
308 | if host.count(":") > 1 and "[" not in host |
309 | else f"https://{host}" |
310 | ) |
311 | url = urlsplit(host) |
312 | if not url.hostname or ( |
313 | url.scheme != "https" and not (url.scheme == "http" and auth.get("allow_http")) |
314 | ): |
315 | raise ValueError( |
316 | f"host must be an address or an https:// URL (http:// needs allow_http on the resource): {auth['host']}" |
317 | ) |
318 | return url |
319 |
|
320 |
|
321 | def node(auth: arista_eos, timeout: float = 60): |
322 | url = switch_url(auth) |
323 | context = ssl.create_default_context() |
324 | if auth.get("verify_tls") is False: |
325 | context.check_hostname = False |
326 | context.verify_mode = ssl.CERT_NONE |
327 | return pyeapi.connect( |
328 | transport=url.scheme, |
329 | host=url.hostname, |
330 | port=url.port or auth.get("port") or None, |
331 | username=auth["username"], |
332 | password=auth["password"], |
333 | context=context, |
334 | timeout=timeout, |
335 | return_node=True, |
336 | ) |
337 |
|
338 |
|
339 | # Shared by the Device scripts: the root flow run's user state, keyed by switch, holds the pre-check snapshot, the |
340 | # image activate replaced (rollback target), the image it booted and the reload markers. Every step of the run (nested |
341 | # subflows and retried attempts included) reads the same entry. Outside a flow the Device scripts share one state |
342 | # resource in the running user's home (a script's own state would keep the pre-check snapshot from the post-check). |
343 | # The endpoint is called through the client so an unreachable state raises (wmill.get_flow_user_state returns None). |
344 | # ponytail: the fallback is one map per user, so concurrent standalone runs can race; flows don't. |
345 | # The key is the switch's eAPI address: arista_eos-<host>-<port> with non-word characters in the host as _, the port |
346 | # always in (443 or 80 when unset). State written before the port was part of the key is read from arista_eos_<host> |
347 | # as a fallback (all word characters, so never a new key); writes go to the new key only. |
348 | def state_keys(auth: arista_eos): |
349 | url = switch_url(auth) |
350 | port = int(url.port or auth.get("port") or {"https": 443, "http": 80}[url.scheme]) |
351 | return [ |
352 | f"arista_eos-{word(url.hostname)}-{port}", |
353 | f"arista_eos_{word(auth['host'])}", |
354 | ] |
355 |
|
356 |
|
357 | def word(s): |
358 | return re.sub(r"\W", "_", s) |
359 |
|
360 |
|
361 | def load_state(keys): |
362 | for key in keys: |
363 | if os.environ.get("WM_FLOW_JOB_ID"): |
364 | c = wmill.Windmill() |
365 | value = c.get( |
366 | f"/w/{c.workspace}/jobs/flow/user_states/{c.get_root_job_id()}/{key}" |
367 | ).json() |
368 | else: |
369 | value = (wmill.get_state(standalone_state()) or {}).get(key) |
370 | if value is not None: |
371 | return value |
372 |
|
373 |
|
374 | def save_state(keys, value): |
375 | if os.environ.get("WM_FLOW_JOB_ID"): |
376 | c = wmill.Windmill() |
377 | c.post( |
378 | f"/w/{c.workspace}/jobs/flow/user_states/{c.get_root_job_id()}/{keys[0]}", |
379 | json=value, |
380 | ) |
381 | else: |
382 | path = standalone_state() |
383 | wmill.set_state({**(wmill.get_state(path) or {}), keys[0]: value}, path) |
384 |
|
385 |
|
386 | def standalone_state(): |
387 | return f"u/{os.environ['WM_USERNAME']}/arista_eos_state" |
388 |
|
389 |
|
390 | # "4.34.9M-51132493.4349M (engineering build)" on cEOS, "4.34.9M" on hardware. |
391 | def short_version(v): |
392 | m = re.match(r"\d+\.\d+\.\d+(\.\d+)?[A-Z]*", v) |
393 | return m.group(0) if m else v |
394 |
|