1 | # py312 |
2 | # requirements: |
3 | # paramiko==5.1.0 |
4 | # pyeapi==1.0.4 |
5 | # wmill==1.851.0 |
6 |
|
7 | # Device upgrade: one upgrade step on one switch, picked by action. |
8 | # - activate: records the image of the running version as the rollback target (once per run, before any change), |
9 | # copies the image to flash (skipped when already there), compares verify /sha512 with the expected digest, then |
10 | # boot system. No reload. The image is image_url, or image_base_url + EOS-<target_version>.swi; the digest is |
11 | # sha512, or the .sha512 file next to the image. A file that doesn't match is copied once more, then deleted. |
12 | # An image file name that carries a version must carry target_version. |
13 | # - reload: reload now, at most once per switch and flow run. Skipped (reloaded false, with the reason) when this run |
14 | # already sent it, the switch already runs target_version, or it rebooted less than min_uptime_seconds ago. |
15 | # boot-config must be the image activate set in this run, or flash:/EOS-<target_version>.swi. It refuses while |
16 | # show running-config diffs changes anything but its ! headers, or there is no startup-config (reload now doesn't |
17 | # prompt, so unsaved changes would be lost), unless allow_unsaved. Rollback has no such check: it is the recovery path. |
18 | # - commit: write memory (refused while a config session's commit timer is pending, so a change about to roll back |
19 | # isn't saved); the previous image stays on flash (delete_old_images removes only the others). |
20 | # - rollback: boot system back to the recorded previous image, then reload now behind the same guard. A boot that |
21 | # this run's reload caused doesn't count against min_uptime_seconds. |
22 | # State (rollback target, activated image, reload markers) is the root flow run's user state under arista_eos-<host>-<port>. |
23 |
|
24 | import base64 |
25 | import hashlib |
26 | import json |
27 | import logging |
28 | import os |
29 | import re |
30 | import socket |
31 | import ssl |
32 | import time |
33 | import urllib.error |
34 | import urllib.request |
35 | from typing import Literal, TypedDict |
36 | from urllib.parse import urlsplit |
37 |
|
38 | import paramiko |
39 | import pyeapi |
40 | import wmill |
41 |
|
42 |
|
43 | class arista_eos(TypedDict): |
44 | host: str |
45 | username: str |
46 | password: str |
47 | port: int |
48 | verify_tls: bool |
49 | allow_http: bool |
50 |
|
51 |
|
52 | class ActivateFailed(Exception): |
53 | pass |
54 |
|
55 |
|
56 | class RollbackFailed(Exception): |
57 | pass |
58 |
|
59 |
|
60 | def main( |
61 | auth: arista_eos, |
62 | action: Literal["activate", "reload", "commit", "rollback"], |
63 | host: str = "", |
64 | target_version: str = "", |
65 | image_base_url: str = "", |
66 | vrf: str = "", |
67 | min_uptime_seconds: int = 300, |
68 | delete_old_images: bool = False, |
69 | image_url: str = "", |
70 | sha512: str = "", |
71 | allow_unsaved: bool = False, |
72 | ): |
73 | """ |
74 | All-in-one device upgrade (deprecated) |
75 | One upgrade step on one switch: activate (copy the image from image_url or EOS-<target>.swi from image_base_url, verify its sha512, boot system it, no reload), reload (exactly once, guarded, refused with unsaved config unless allow_unsaved), commit (write memory, refused while a config-session commit timer is pending; keep the previous image), rollback (boot the previous image and reload, same guard). Returns { status: "done" } or throws ActivateFailed, RollbackFailed, DeviceUnreachable or AuthFailure as the first token of the error. Deprecated: kept for flows built on it; new flows use the single-attempt Device actions, which return plain facts instead of this ok|ko result. |
76 | """ |
77 | auth = with_host(auth, host) |
78 | key = state_keys(auth) |
79 | state = load_state(key) or {} |
80 | min_uptime = 300 if min_uptime_seconds is None else min_uptime_seconds |
81 |
|
82 | if action == "activate": |
83 | # Set again only once this activate has booted the image: a stale one would let reload through. |
84 | forget_activated(key, state) |
85 | if not target_version or not (image_base_url or image_url): |
86 | raise named( |
87 | ActivateFailed, |
88 | "target_version and image_base_url (or image_url) are required", |
89 | ) |
90 | try: |
91 | image, url = image_source(target_version, image_base_url, image_url) |
92 | except ValueError as e: |
93 | raise named(ActivateFailed, str(e)) |
94 | # A file name that carries a version must carry this one; one without (a lab image) is taken as is. |
95 | if image_version(image) not in (None, target_version): |
96 | raise named( |
97 | ActivateFailed, |
98 | f"{image} is named for {image_version(image)}, not target_version {target_version}", |
99 | ) |
100 | steps = [] |
101 | previous = state.get("previous_image") or "" |
102 | try: |
103 | if sha512: |
104 | expected = digest(sha512) |
105 | if not expected: |
106 | raise Exception("sha512 must be the 128-hex digest of the image") |
107 | else: |
108 | try: |
109 | with urllib.request.urlopen(f"{url}.sha512", timeout=60) as r: |
110 | expected = digest(r.read().decode(errors="replace")) |
111 | except urllib.error.HTTPError as e: |
112 | raise Exception(f"cannot read {url}.sha512 ({e.code}): pass sha512") |
113 | except OSError as e: |
114 | reason = getattr(e, "reason", e) |
115 | # Unlike the switch's copy, this fetch checks the certificate: the digest is what activate trusts. |
116 | trust = ( |
117 | ", or trust the image server's CA on the worker (SSL_CERT_FILE)" |
118 | if isinstance(reason, ssl.SSLCertVerificationError) |
119 | else "" |
120 | ) |
121 | raise Exception( |
122 | f"cannot read {url}.sha512 ({reason}): pass sha512{trust}" |
123 | ) |
124 | if not expected: |
125 | raise Exception(f"no sha512 hash in {url}.sha512") |
126 |
|
127 | d, boot, version = eapi( |
128 | auth, |
129 | [ |
130 | {"cmd": "dir flash:/", "revision": 2}, |
131 | "show boot-config", |
132 | "show version", |
133 | ], |
134 | ) |
135 | files = list(d["urls"]["flash:/"]["entries"]) |
136 | running = short_version(version["version"]) |
137 | booted = boot["softwareImage"].replace("flash:/", "") |
138 | # boot-config is the running image (first run) or the target (an earlier run got past boot system). |
139 | # Anything else is a stale boot-config this step won't guess around. |
140 | if image_version(booted) != running and booted != image: |
141 | raise Exception( |
142 | f"boot-config points to {boot['softwareImage']}, neither the running version {running} nor {image}: fix boot-config first" |
143 | ) |
144 | # The rollback target is the image of the running version, recorded once per run before anything changes. |
145 | if not previous and running != target_version: |
146 | candidates = ( |
147 | [booted] |
148 | if booted != image |
149 | else [ |
150 | f |
151 | for f in files |
152 | if f.endswith(".swi") and image_version(f) == running |
153 | ] |
154 | ) |
155 | if len(candidates) != 1 or candidates[0] not in files: |
156 | raise Exception( |
157 | f"cannot tell which image on flash runs {running} (candidates: {', '.join(candidates) or 'none'}): no rollback target" |
158 | ) |
159 | previous = f"flash:/{candidates[0]}" |
160 | save_state(key, {**state, "previous_image": previous}) |
161 |
|
162 | def copy(): |
163 | # The copy runs on the switch and can take minutes for a ~1 GB image. |
164 | eapi( |
165 | auth, |
166 | [*([f"cli vrf {vrf}"] if vrf else []), f"copy {url} flash:{image}"], |
167 | "json", |
168 | 3600, |
169 | ) |
170 | steps.append(f"copied {url}") |
171 |
|
172 | def verify(): |
173 | [v] = eapi(auth, [f"verify /sha512 flash:{image}"]) |
174 | return digest((v.get("messages") or [""])[0]) |
175 |
|
176 | copied = image not in files |
177 | if copied: |
178 | copy() |
179 | actual = verify() |
180 | if actual != expected and not copied: |
181 | # Most likely a partial copy left by a killed run: copy it once more. |
182 | eapi(auth, [f"delete flash:{image}"]) |
183 | copy() |
184 | actual = verify() |
185 | if actual != expected: |
186 | # An earlier run may have booted it already: never leave boot-config on a deleted image. |
187 | restored = booted == image and previous |
188 | if restored: |
189 | boot_system(auth, previous.replace("flash:/", "")) |
190 | eapi(auth, [f"delete flash:{image}"]) |
191 | raise Exception( |
192 | f"sha512 mismatch for {image} (switch {actual or 'none'}, expected {expected}), deleted it" |
193 | + (f", boot-config back on {previous}" if restored else "") |
194 | ) |
195 | steps.append("sha512 verified") |
196 | steps.extend(boot_system(auth, image)) |
197 | # reload checks boot-config against this, so it needs no image_url of its own. |
198 | save_state( |
199 | key, |
200 | { |
201 | **state, |
202 | "previous_image": previous, |
203 | "activated_image": f"flash:/{image}", |
204 | }, |
205 | ) |
206 | except (DeviceUnreachable, AuthFailure): |
207 | raise |
208 | except Exception as e: |
209 | raise named(ActivateFailed, str(e)) |
210 | return {"status": "done", "steps": steps, "previous_image": previous} |
211 |
|
212 | if action == "commit": |
213 | [sessions] = eapi(auth, ["show configuration sessions detail"]) |
214 | if sessions.get("commitTimerSessionName"): |
215 | raise RuntimeError( |
216 | f"config session {sessions['commitTimerSessionName']} is pending under a commit timer: confirm or abort it before commit (write memory)" |
217 | ) |
218 | _, boot, d = eapi( |
219 | auth, |
220 | ["write memory", "show boot-config", {"cmd": "dir flash:/", "revision": 2}], |
221 | ) |
222 | deleted = [] |
223 | # Rollback needs the previous image: only images that are neither booted nor the recorded previous one go. |
224 | if delete_old_images and state.get("previous_image"): |
225 | keep = [ |
226 | i.replace("flash:/", "") |
227 | for i in (boot["softwareImage"], state["previous_image"]) |
228 | ] |
229 | for f in list(d["urls"]["flash:/"]["entries"]): |
230 | if re.match(r"^EOS.*\.swi$", f) and f not in keep: |
231 | eapi(auth, [f"delete flash:{f}"]) |
232 | deleted.append(f) |
233 | return {"status": "done", "saved": True, "deleted": deleted} |
234 |
|
235 | if action == "reload": |
236 | if not target_version: |
237 | raise Exception("target_version is required for the reload guard") |
238 | sw = read_switch(auth, state, min_uptime) |
239 | sent = sw["marker"]("reload") |
240 | if sw.get("unreachable"): |
241 | if sent: |
242 | return skip( |
243 | f"reload sent at {sent['sent_at']}; the switch no longer answers: it is reloading" |
244 | ) |
245 | raise named(DeviceUnreachable, sw["unreachable"]) |
246 | if sent and (sent["confirmed"] or sent["boot_timestamp"] != sw["boot"]): |
247 | return skip(guard_reason("reload", sent, sw)) |
248 | if sw["running"] == target_version: |
249 | return skip(f"already running {target_version}") |
250 | # The image activate set in this run (state), or exactly EOS-<target_version>.swi. |
251 | activated = state.get("activated_image") |
252 | if sw["image"] not in (activated, f"flash:/EOS-{target_version}.swi"): |
253 | raise Exception( |
254 | f"boot-config points to {sw['image']}, neither the image activate set ({activated or 'none in this run'}) nor flash:/EOS-{target_version}.swi: run activate first" |
255 | ) |
256 | if sw["uptime"] < min_uptime: |
257 | return skip(f"rebooted {round(sw['uptime'])} s ago") |
258 | if not allow_unsaved and unsaved(auth): |
259 | raise Exception( |
260 | "unsaved config: run Save Config first, or pass allow_unsaved: true" |
261 | ) |
262 | fire_reload(auth, key, "reload", sw["boot"]) |
263 | return {"status": "done", "reloaded": True} |
264 |
|
265 | # rollback: boot the previous image (still on flash) and reload, with the same once-only guard. |
266 | try: |
267 | sw = read_switch(auth, state, min_uptime) |
268 | sent = sw["marker"]("rollback") |
269 | if sw.get("unreachable"): |
270 | if sent: |
271 | return skip( |
272 | f"rollback sent at {sent['sent_at']}; the switch no longer answers: it is reloading" |
273 | ) |
274 | raise Exception(sw["unreachable"]) |
275 | if sent and (sent["confirmed"] or sent["boot_timestamp"] != sw["boot"]): |
276 | return skip(guard_reason("rollback", sent, sw)) |
277 | d, boot = eapi( |
278 | auth, [{"cmd": "dir flash:/", "revision": 2}, "show boot-config"] |
279 | ) |
280 | files = list(d["urls"]["flash:/"]["entries"]) |
281 | current = boot["softwareImage"].replace("flash:/", "") |
282 | # Without a recorded previous image (new run), the only other EOS image on flash is unambiguous. |
283 | others = [f for f in files if re.match(r"^EOS.*\.swi$", f) and f != current] |
284 | previous = (state.get("previous_image") or "").replace("flash:/", "") or ( |
285 | others[0] if len(others) == 1 else "" |
286 | ) |
287 | if not previous: |
288 | raise Exception( |
289 | f"previous image unknown (EOS images on flash: {', '.join(others) or 'none'})" |
290 | ) |
291 | if previous not in files: |
292 | raise Exception(f"previous image {previous} is no longer on flash") |
293 | reload = sw["marker"]("reload") |
294 | if reload and reload["confirmed"] and reload["boot_timestamp"] == sw["boot"]: |
295 | raise Exception( |
296 | f"the reload sent at {reload['sent_at']} has not happened yet (same boot as when it was sent): rolling back now would race it, check the switch" |
297 | ) |
298 | steps = boot_system(auth, previous) |
299 | forget_activated(key, state) |
300 | if sw["running"] == image_version(previous): |
301 | return skip( |
302 | f"already running the previous image {previous}; boot-config set back to it" |
303 | ) |
304 | # A recent boot that this run's reload caused doesn't block the rollback: the post-check that ran after it |
305 | # can end well inside min_uptime_seconds. |
306 | ours = reload and reload["boot_timestamp"] != sw["boot"] |
307 | if not ours and sw["uptime"] < min_uptime: |
308 | return skip(f"rebooted {round(sw['uptime'])} s ago, not by this run") |
309 | fire_reload(auth, key, "rollback", sw["boot"]) |
310 | return { |
311 | "status": "done", |
312 | "reloaded": True, |
313 | "boot_image": f"flash:/{previous}", |
314 | "steps": steps, |
315 | } |
316 | except AuthFailure: |
317 | raise |
318 | except Exception as e: |
319 | raise named( |
320 | RollbackFailed, f"{auth['host']}: {re.sub(r'^[A-Za-z]+: ', '', str(e))}" |
321 | ) |
322 |
|
323 |
|
324 | # One resource can serve every switch that shares a login: the host input, when set, overrides the resource's host. |
325 | def with_host(auth: arista_eos, host: str): |
326 | host = (host or "").strip() or (auth.get("host") or "").strip() |
327 | if not host: |
328 | raise ValueError("host is required: set it on the resource or pass host") |
329 | return {**auth, "host": host} |
330 |
|
331 |
|
332 | # Same function in Device check and Device upgrade. |
333 | # image_url wins; else image_base_url + EOS-<target>.swi. Returns (file name on flash, URL or ""). The URL goes into |
334 | # the switch's copy command, so it is an allowlist: http(s), then only A-Z a-z 0-9 . _ ~ : / % + - (no query, no |
335 | # user:password@, nothing the CLI reads as a separator), ending in a plain .swi file name. Raises ValueError otherwise. |
336 | def image_source(target, base, image_url): |
337 | if image_url: |
338 | image, url = urlsplit(image_url).path.rsplit("/", 1)[-1], image_url |
339 | else: |
340 | image = f"EOS-{target}.swi" if target else "" |
341 | url = f"{base.rstrip('/')}/{image}" if image and base else "" |
342 | if not url: |
343 | return image, url |
344 | # Not echoed: it would put the password in the steps and errors. |
345 | if "@" in url: |
346 | raise ValueError("the image URL must not contain @ (no user:password@ in it)") |
347 | if not re.match(r"https?://", url): |
348 | raise ValueError( |
349 | f"the image URL must start with http:// or https://: got {url!r}" |
350 | ) |
351 | if not urlsplit(url).hostname: |
352 | raise ValueError(f"the image URL has no host: got {url!r}") |
353 | bad = sorted(set(re.sub(r"[A-Za-z0-9._~:/%+-]", "", url.split("://", 1)[1]))) |
354 | if bad: |
355 | raise ValueError( |
356 | f"the image URL may only contain A-Z a-z 0-9 . _ ~ : / % + -, not {' '.join(map(repr, bad))}: got {url!r}" |
357 | ) |
358 | if not re.fullmatch(r"[\w.+-]+\.swi", image): |
359 | raise ValueError(f"the image URL must end in a .swi file name: got {url!r}") |
360 | return image, url |
361 |
|
362 |
|
363 | def boot_system(auth, image): |
364 | _, set_, _, after = eapi( |
365 | auth, ["configure", f"boot system flash:{image}", "end", "show boot-config"] |
366 | ) |
367 | if after["softwareImage"] != f"flash:/{image}": |
368 | raise Exception(f"boot-config still points to {after['softwareImage']}") |
369 | return [f"boot system flash:{image}", *(set_.get("warnings") or [])] |
370 |
|
371 |
|
372 | # reload now doesn't prompt to save. show running-config diffs is a unified diff of startup-config against |
373 | # running-config, empty when they match. Its two file-header lines come before the first @@. Changed lines that |
374 | # start with a single "!" are headers EOS writes itself (boot system shows there, so every activate changes one); |
375 | # "!!" comments are config. With no startup-config every line shows as added: unsaved, it would boot into ZTP. |
376 | # A refused command (AAA authorization, an older EOS) is not unsaved config: saving wouldn't clear it. |
377 | def unsaved(auth): |
378 | try: |
379 | [d] = eapi(auth, ["show running-config diffs"], "text") |
380 | except pyeapi.eapilib.CommandError as e: |
381 | raise Exception(f"cannot compare running and startup config: {e}") |
382 | hunks, changed = False, [] |
383 | for l in d["output"].splitlines(): |
384 | hunks = hunks or l.startswith("@@") |
385 | if hunks and l[:1] in ("+", "-") and l[1:].strip(): |
386 | changed.append(l[1:].strip()) |
387 | return any(not re.match(r"!(?!!)", l) for l in changed) |
388 |
|
389 |
|
390 | def forget_activated(key, state): |
391 | if state.pop("activated_image", None): |
392 | save_state(key, state) |
393 |
|
394 |
|
395 | def skip(reason): |
396 | return {"status": "done", "reloaded": False, "reason": reason} |
397 |
|
398 |
|
399 | def guard_reason(kind, m, sw): |
400 | if m["boot_timestamp"] == sw["boot"]: |
401 | return f"{kind} sent at {m['sent_at']}; the switch is going down" |
402 | return f"{kind} sent at {m['sent_at']} already rebooted the switch" |
403 |
|
404 |
|
405 | # A reload is sent at most once per switch and flow run. Each one leaves a marker { sent_at, boot_timestamp, |
406 | # confirmed }: boot_timestamp is the boot it was sent on, so a later boot means it happened, and confirmed means |
407 | # the command reached the switch. An unconfirmed marker on the same boot (the worker died first) is sent again. |
408 | # Outside a flow, a marker is dropped once the reboot it caused is min_uptime_seconds old. |
409 | def read_switch(auth, state, min_uptime): |
410 | try: |
411 | version, boot = eapi(auth, ["show version", "show boot-config"]) |
412 | except DeviceUnreachable as e: |
413 | return {"unreachable": str(e), "marker": lambda k: state.get(k)} |
414 | sw = { |
415 | "running": short_version(version["version"]), |
416 | "image": boot["softwareImage"], |
417 | "boot": version["bootupTimestamp"], |
418 | "uptime": version["uptime"], |
419 | } |
420 | in_flow = bool(os.environ.get("WM_FLOW_JOB_ID")) |
421 |
|
422 | def marker(k): |
423 | m = state.get(k) |
424 | return ( |
425 | m |
426 | if m |
427 | and ( |
428 | in_flow |
429 | or m["boot_timestamp"] == sw["boot"] |
430 | or sw["uptime"] < min_uptime |
431 | ) |
432 | else None |
433 | ) |
434 |
|
435 | return {**sw, "marker": marker} |
436 |
|
437 |
|
438 | # Fire and forget: the switch drops the connection while going down, which is not an error. A connection that |
439 | # never opened, or a refusal from the switch, means it did not reload: the marker goes and the error is raised. |
440 | def fire_reload(auth, key, kind, boot_ts): |
441 | def mark(m): |
442 | rest = {k: v for k, v in (load_state(key) or {}).items() if k != kind} |
443 | save_state(key, {**rest, kind: m} if m else rest) |
444 |
|
445 | m = { |
446 | "sent_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), |
447 | "boot_timestamp": boot_ts, |
448 | "confirmed": False, |
449 | } |
450 | mark(m) |
451 | eos = node(auth, 15) |
452 | try: |
453 | # Opened before sending, so a failure here means the command never left. |
454 | eos.connection.transport.connect() |
455 | except OSError as e: |
456 | mark(None) |
457 | raise named(DeviceUnreachable, f"{auth['host']}: {e}") |
458 | try: |
459 | eos.run_commands(["reload now"]) |
460 | except pyeapi.eapilib.ConnectionError as e: |
461 | # Sent, then the connection dropped or timed out as the switch went down. A 401 is a refusal. |
462 | err = connection_error(auth, eos, e) |
463 | if isinstance(err, AuthFailure): |
464 | mark(None) |
465 | raise err |
466 | except Exception: |
467 | mark(None) |
468 | raise |
469 | mark({**m, "confirmed": True}) |
470 |
|
471 |
|
472 | # Shared with Device check: the flow run's user state, keyed by switch, holds the pre-check snapshot, the image |
473 | # activate replaced (rollback target) and the reload markers. Outside a flow, it falls back to this script's own state. |
474 | # The endpoint is called through the client so an unreachable state raises (wmill.get_flow_user_state returns None). |
475 | # ponytail: the fallback is one map per script, so concurrent standalone runs can race; flows don't. |
476 | # The key is the switch's eAPI address: arista_eos-<host>-<port> with non-word characters in the host as _, the port |
477 | # always in (443 or 80 when unset), so host 10.0.0.11 with port 8443 and host https://10.0.0.11:8443 share one key, |
478 | # while two switches behind one address on different ports (or http and https) don't. State written before the port |
479 | # was part of the key is read from arista_eos_<host> as a fallback (all word characters, so never a new key); writes |
480 | # go to the new key only. |
481 | def state_keys(auth: arista_eos): |
482 | if auth.get("transport") == "ssh": |
483 | host, port = ssh_addr(auth) |
484 | return [f"arista_eos-{word(host)}-{port}"] |
485 | url = switch_url(auth) |
486 | port = int(url.port or auth.get("port") or {"https": 443, "http": 80}[url.scheme]) |
487 | return [ |
488 | f"arista_eos-{word(url.hostname)}-{port}", |
489 | f"arista_eos_{word(auth['host'])}", |
490 | ] |
491 |
|
492 |
|
493 | def word(s): |
494 | return re.sub(r"\W", "_", s) |
495 |
|
496 |
|
497 | def load_state(keys): |
498 | for key in keys: |
499 | if os.environ.get("WM_FLOW_JOB_ID"): |
500 | c = wmill.Windmill() |
501 | value = c.get( |
502 | f"/w/{c.workspace}/jobs/flow/user_states/{c.get_root_job_id()}/{key}" |
503 | ).json() |
504 | else: |
505 | value = (wmill.get_state() or {}).get(key) |
506 | if value is not None: |
507 | return value |
508 |
|
509 |
|
510 | def save_state(keys, value): |
511 | if os.environ.get("WM_FLOW_JOB_ID"): |
512 | c = wmill.Windmill() |
513 | c.post( |
514 | f"/w/{c.workspace}/jobs/flow/user_states/{c.get_root_job_id()}/{keys[0]}", |
515 | json=value, |
516 | ) |
517 | else: |
518 | wmill.set_state({**(wmill.get_state() or {}), keys[0]: value}) |
519 |
|
520 |
|
521 | # "4.34.9M-51132493.4349M (engineering build)" on cEOS, "4.34.9M" on hardware. |
522 | def short_version(v): |
523 | m = re.match(r"\d+\.\d+\.\d+(\.\d+)?[A-Z]*", v) |
524 | return m.group(0) if m else v |
525 |
|
526 |
|
527 | # "EOS64-4.34.9M.swi" -> "4.34.9M" |
528 | def image_version(file): |
529 | m = re.search(r"-(\d+\.\d+\.\d+(\.\d+)?[A-Z]*)\.swi$", file) |
530 | return m.group(1) if m else None |
531 |
|
532 |
|
533 | # The 128-hex digest in a sha512sum line, a BSD "SHA512 (f) = <hash>" line or verify /sha512 output. |
534 | def digest(text): |
535 | m = re.search(r"\b[0-9a-fA-F]{128}\b", text) |
536 | return m.group(0).lower() if m else "" |
537 |
|
538 |
|
539 | class DeviceUnreachable(Exception): |
540 | pass |
541 |
|
542 |
|
543 | class AuthFailure(Exception): |
544 | pass |
545 |
|
546 |
|
547 | def named(cls, message: str): |
548 | return cls(f"{cls.__name__}: {message}") |
549 |
|
550 |
|
551 | # pyeapi logs a traceback for every socket error, and the post-check polls a switch that is rebooting. |
552 | logging.getLogger("pyeapi").addHandler(logging.NullHandler()) |
553 |
|
554 |
|
555 | def eapi(auth: arista_eos, cmds: list, fmt: str = "json", timeout: float = 60): |
556 | eos = node(auth, timeout) |
557 | try: |
558 | return eos.run_commands(cmds, fmt) |
559 | except pyeapi.eapilib.ConnectionError as e: |
560 | raise connection_error(auth, eos, e) |
561 |
|
562 |
|
563 | # pyeapi 1.0.4 raises ConnectionError for a socket error (kept in socket_error), an answer that isn't JSON (eAPI |
564 | # still starting, 5xx) and a 401, the only one with neither. |
565 | def connection_error(auth, eos, e): |
566 | # A certificate the resource does not trust is a setting to fix (verify_tls, ca_certificate), not worth a retry. |
567 | if isinstance(eos.connection.socket_error, ssl.SSLCertVerificationError): |
568 | return RuntimeError(f"{auth['host']}: {eos.connection.socket_error}") |
569 | if eos.connection.socket_error is None and e.message != "unable to connect to eAPI": |
570 | return named( |
571 | AuthFailure, f"{auth['host']}: {'SSH' if auth.get('transport') == 'ssh' else 'eAPI'} refused the credential ({e.message})" |
572 | ) |
573 | return named(DeviceUnreachable, f"{auth['host']}: {e.message}") |
574 |
|
575 |
|
576 | # host is an address or a URL (https://10.0.0.11:8443); a bare IPv6 address is bracketed. Only https, or http when the |
577 | # resource ticks allow_http (Basic auth then crosses the network in cleartext). pyeapi skips certificate checks unless it |
578 | # is given a context, so one is always passed: verifying, or not when the resource unticks verify_tls (EOS ships a |
579 | # self-signed cert). |
580 | def switch_url(auth: arista_eos): |
581 | host = auth["host"] |
582 | if "://" not in host: |
583 | host = f"https://[{host}]" if host.count(":") > 1 and "[" not in host else f"https://{host}" |
584 | url = urlsplit(host) |
585 | if not url.hostname or (url.scheme != "https" and not (url.scheme == "http" and auth.get("allow_http"))): |
586 | raise ValueError(f"host must be an address or an https:// URL (http:// needs allow_http on the resource): {auth['host']}") |
587 | return url |
588 |
|
589 |
|
590 | # SSH transport (transport: ssh): the EOS CLI over an SSH shell, behind the part of pyeapi's Node the scripts use |
591 | # (run_commands, connection.socket_error, connection.transport.connect), raising pyeapi's own CommandError and |
592 | # ConnectionError, so the checks and the error mapping are the same for both transports. Show commands in json get |
593 | # "| json", which prints the eAPI JSON; other commands give { messages: [output] } like eAPI, or {} when silent. |
594 | # A line starting with "% " is an error, raised as eAPI's CommandError. The host key is checked when the resource |
595 | # has ssh_host_key (SHA256:... as ssh-keygen -lf prints it). |
596 | class Ssh: |
597 | def __init__(self, auth: arista_eos, timeout: float): |
598 | self.auth, self.timeout = auth, timeout |
599 | self.connection = self.transport = self |
600 | self.socket_error = self.sock = self.chan = None |
601 |
|
602 | # The TCP connection only, like eAPI's transport.connect(): if it fails, nothing was sent. |
603 | def connect(self): |
604 | self.sock = socket.create_connection(ssh_addr(self.auth), self.timeout) |
605 |
|
606 | def lost(self, e): |
607 | self.socket_error = e |
608 | return pyeapi.eapilib.ConnectionError("ssh", f"SSH connection failed ({e})") |
609 |
|
610 | def open(self): |
611 | auth = self.auth |
612 | try: |
613 | if not self.sock: |
614 | self.connect() |
615 | t = paramiko.Transport(self.sock) |
616 | t.start_client(timeout=self.timeout) |
617 | key = t.get_remote_server_key() |
618 | except (OSError, EOFError, paramiko.SSHException) as e: |
619 | raise self.lost(e) |
620 | fp = "SHA256:" + base64.b64encode(hashlib.sha256(key.asbytes()).digest()).decode().rstrip("=") |
621 | if auth.get("ssh_host_key") and auth["ssh_host_key"].strip() != fp: |
622 | t.close() |
623 | raise RuntimeError(f"{auth['host']}: the SSH host key is {fp}, not the resource's ssh_host_key") |
624 | try: |
625 | try: |
626 | t.auth_password(auth["username"], auth["password"]) |
627 | except paramiko.BadAuthenticationType: |
628 | t.auth_interactive( |
629 | auth["username"], lambda _t, _i, prompts: [auth["password"]] * len(prompts) |
630 | ) |
631 | except paramiko.AuthenticationException as e: |
632 | t.close() |
633 | raise pyeapi.eapilib.ConnectionError("ssh", str(e)) |
634 | except (OSError, EOFError, paramiko.SSHException) as e: |
635 | raise self.lost(e) |
636 | try: |
637 | self.chan = t.open_session() |
638 | self.chan.get_pty(width=4000) |
639 | self.chan.invoke_shell() |
640 | self.prompt = None |
641 | out = self.read() |
642 | self.prompt = re.escape(re.split(r"[(>#]", out.splitlines()[-1].strip())[0]) |
643 | if out.rstrip().endswith(">"): |
644 | self.cli("enable") |
645 | self.cli("terminal length 0") |
646 | self.cli("terminal width 32767") |
647 | except (OSError, EOFError, paramiko.SSHException) as e: |
648 | raise self.lost(e) |
649 |
|
650 | # Reads until the prompt. Questions eAPI never asks are answered as eAPI behaves: confirm, don't save. |
651 | def read(self): |
652 | buf, end = "", time.monotonic() + self.timeout |
653 | prompt = (self.prompt or r"[\w.\-]+") + r"(\([^)\r\n]*\))?[>#] ?$" |
654 | while True: |
655 | self.chan.settimeout(max(end - time.monotonic(), 0.01)) |
656 | chunk = self.chan.recv(65536) |
657 | if not chunk: |
658 | raise self.lost("connection closed") |
659 | buf += chunk.decode(errors="replace") |
660 | tail = buf[-300:] |
661 | if re.search(r"(^|\n)" + prompt, tail): |
662 | return buf |
663 | for question, answer in ( |
664 | # enable asks for a password on switches that land the user unprivileged: the login password. |
665 | (r"(^|\n)Password:\s*$", self.auth["password"]), |
666 | (r"\[confirm\]\s*$", ""), |
667 | (r"Save\?.*\[yes/no[^\]]*\]:?\s*$", "no"), |
668 | (r"\[(y|yes)/(n|no)\][:?]?\s*$", "y"), |
669 | ): |
670 | if re.search(question, tail, re.I): |
671 | self.chan.send(answer + "\r") |
672 | buf += "\n" |
673 |
|
674 | # One command: its output without the echoed command and the prompt; progress lines keep their last state. |
675 | def cli(self, cmd): |
676 | try: |
677 | self.chan.send(cmd + "\r") |
678 | out = self.read() |
679 | except (OSError, EOFError, paramiko.SSHException) as e: |
680 | raise self.lost(e) |
681 | self.mode = out.rstrip().splitlines()[-1] |
682 | lines = [l.rstrip("\r").split("\r")[-1] for l in out.split("\n")] |
683 | return "\n".join(lines[1:-1]).strip("\n") |
684 |
|
685 | def run_commands(self, cmds, encoding="json", **_): |
686 | try: |
687 | if not self.chan: |
688 | self.open() |
689 | if "(config" in self.mode: |
690 | self.cli("end") |
691 | results = [] |
692 | for i, c in enumerate(cmds): |
693 | # eAPI's { cmd, revision } form is the CLI's "| json revision N". |
694 | cmd, rev = (c["cmd"], c.get("revision")) if isinstance(c, dict) else (c, None) |
695 | show = encoding == "json" and (rev or re.match(r"\s*sh(ow)?\s", cmd)) and not re.search(r"\|\s*json", cmd) |
696 | # pyeapi sends "<cmd> MULTILINE: <text>" (banners) as cmd + input; the CLI reads the text up to EOF. |
697 | head, multi, text = cmd.partition("MULTILINE:") |
698 | # The shell would run what follows a line break as another command; eAPI refuses it too. |
699 | if re.search(r"[\r\n]", head): |
700 | raise ValueError(f"a command may not contain a line break: {head!r}") |
701 | line = "\r".join([head.strip(), *text.strip().splitlines(), "EOF"]) if multi else cmd |
702 | out = self.cli(f"{cmd} | json{f' revision {rev}' if rev else ''}" if show else line) |
703 | errors = [l[2:] for l in out.splitlines() if l.startswith("% ")] |
704 | code, kind = 1002, "invalid command" |
705 | if show and not errors: |
706 | code, kind = 1000, "could not run command" |
707 | try: |
708 | result = json.loads(out) |
709 | except ValueError: |
710 | errors = [out[:200]] |
711 | else: |
712 | errors = result.get("errors") or [] |
713 | if errors: |
714 | raise pyeapi.eapilib.CommandError( |
715 | code, |
716 | f"CLI command {i + 2} of {len(cmds) + 1} '{cmd}' failed: {kind}", |
717 | command_error="; ".join(errors), |
718 | commands=cmds, |
719 | output=results, |
720 | ) |
721 | if encoding != "json": |
722 | result = {"output": out + "\n" if out else ""} |
723 | elif not show: |
724 | # The CLI prints eAPI's warnings as "! " notices. |
725 | notices = [l[2:].strip() for l in out.splitlines() if l.startswith("! ")] |
726 | out = "\n".join(l for l in out.splitlines() if not l.startswith("! ")).strip() |
727 | result = {"messages": [out]} if out and not multi else {} |
728 | if notices: |
729 | result["warnings"] = notices |
730 | results.append(result) |
731 | return results |
732 | except pyeapi.eapilib.ConnectionError: |
733 | if self.chan: |
734 | self.chan.close() |
735 | self.sock = self.chan = None |
736 | raise |
737 |
|
738 |
|
739 | # paramiko logs a traceback when a rebooting switch drops the session. |
740 | logging.getLogger("paramiko").addHandler(logging.NullHandler()) |
741 |
|
742 |
|
743 | # host, or host:port ([v6]:port), else port, else 22. |
744 | def ssh_addr(auth: arista_eos): |
745 | h = auth["host"] |
746 | if ": |
747 | raise ValueError(f"with transport ssh, host is an address or a hostname (with :port), not a URL: {h}") |
748 | url = urlsplit(f"ssh://{h}" if h.count(":") <= 1 or h.startswith("[") else f"ssh://[{h}]") |
749 | return url.hostname, url.port or int(auth.get("port") or 22) |
750 |
|
751 |
|
752 | def node(auth: arista_eos, timeout: float = 60): |
753 | if auth.get("transport") == "ssh": |
754 | return Ssh(auth, timeout) |
755 | url = switch_url(auth) |
756 | # ca_certificate: the PEM of a private CA, trusted instead of the system's CAs. |
757 | context = ssl.create_default_context(cadata=(auth.get("ca_certificate") or "").strip() or None) |
758 | if auth.get("verify_tls") is False: |
759 | context.check_hostname = False |
760 | context.verify_mode = ssl.CERT_NONE |
761 | return pyeapi.connect( |
762 | transport=url.scheme, |
763 | host=url.hostname, |
764 | port=url.port or auth.get("port") or None, |
765 | username=auth["username"], |
766 | password=auth["password"], |
767 | context=context, |
768 | timeout=timeout, |
769 | return_node=True, |
770 | ) |
771 | |