1 | # py312 |
2 | # requirements: |
3 | # paramiko==5.1.0 |
4 | # pyeapi==1.0.4 |
5 | # wmill==1.851.0 |
6 |
|
7 | # Device check: pre- and post-upgrade checks of one switch. |
8 | # - flash (pre): the target image is already on flash, or free space covers its size on the image server |
9 | # (image_url, or image_base_url + EOS-<target_version>.swi; required_free_mb without an image server or when its |
10 | # HEAD has no Content-Length). The HEAD checks the certificate and, if the worker doesn't trust it, sizes the image |
11 | # anyway: notes says when either happens, and when activate will need its sha512 input (untrusted certificate, or |
12 | # no <image>.sha512 reachable). A malformed image URL is flash ko. |
13 | # - redundancy (pre): MLAG active and connected, peer-link and local interface up, config-sanity clean, no MLAG |
14 | # port active-partial, no port errdisabled. Left out when MLAG is not configured (notes says the reload is not |
15 | # hitless); configured but down is ko. |
16 | # - interface: pre saves a snapshot (interfaces up, LLDP neighbor per port, MLAG state and per-port status) in the |
17 | # flow run's user state, on the root flow, under arista_eos-<host>-<port>; post compares the switch with it. |
18 | # Post waits, in order: for eAPI to answer on a boot later than this run's reload (return_timeout_seconds), for the |
19 | # MLAG reload-delay counted from boot when MLAG is configured, then up to stabilisation_seconds for the state to |
20 | # match the snapshot. The three are returned in durations. The version is reported, not compared with the target. |
21 |
|
22 | import base64 |
23 | import hashlib |
24 | import json |
25 | import logging |
26 | import os |
27 | import re |
28 | import socket |
29 | import ssl |
30 | import time |
31 | import urllib.error |
32 | import urllib.request |
33 | from typing import Literal, TypedDict |
34 | from urllib.parse import urlsplit |
35 |
|
36 | import paramiko |
37 | import pyeapi |
38 | import wmill |
39 |
|
40 |
|
41 | class arista_eos(TypedDict): |
42 | host: str |
43 | username: str |
44 | password: str |
45 | port: int |
46 | verify_tls: bool |
47 | allow_http: bool |
48 |
|
49 |
|
50 | def main( |
51 | auth: arista_eos, |
52 | phase: Literal["pre", "post"], |
53 | host: str = "", |
54 | target_version: str = "", |
55 | image_base_url: str = "", |
56 | required_free_mb: int = 1024, |
57 | return_timeout_seconds: int = 900, |
58 | stabilisation_seconds: int = 120, |
59 | poll_interval_seconds: int = 15, |
60 | image_url: str = "", |
61 | ): |
62 | """ |
63 | All-in-one device check (deprecated) |
64 | Pre- or post-upgrade check of one switch: returns { running_version, status ok|ko, checks_ok, checks_ko[{check, message}], notes }. Pre: flash space for the target image, MLAG redundancy, and a snapshot of interfaces/LLDP/MLAG kept in the flow run's state. Post: waits for the switch to answer and for the MLAG reload-delay (when MLAG is configured), then compares with the snapshot over a bounded stabilisation window. Throws DeviceUnreachable or AuthFailure. Does not compare the version with the target: the workflow does. Deprecated: kept for flows built on it; new flows use the single-attempt Device actions, which return plain facts instead of this ok|ko result. |
65 | """ |
66 | auth = with_host(auth, host) |
67 | poll = poll_interval_seconds or 15 |
68 | durations = {} |
69 | key = state_keys(auth) |
70 | state = load_state(key) or {} |
71 |
|
72 | if phase == "post": |
73 | # Wait for the switch to answer on a boot later than the one each reload of this run was sent on: `reload now` |
74 | # can answer before EOS goes down, and the old boot must not pass for the new one. AuthFailure stops at once. |
75 | markers = [state[k] for k in ("reload", "rollback") if state.get(k)] |
76 | timeout = return_timeout_seconds or 900 |
77 | start = time.time() |
78 | while True: |
79 | try: |
80 | [v] = eapi(auth, ["show version"]) |
81 | m = next( |
82 | (m for m in markers if m["boot_timestamp"] == v["bootupTimestamp"]), |
83 | None, |
84 | ) |
85 | if not m: |
86 | break |
87 | pending = ( |
88 | f"still on the boot the reload sent at {m['sent_at']} was sent on" |
89 | ) |
90 | except DeviceUnreachable as e: |
91 | pending = str(e) |
92 | if time.time() - start > timeout: |
93 | raise named( |
94 | DeviceUnreachable, |
95 | f"{auth['host']}: not back after {timeout} s ({pending})", |
96 | ) |
97 | time.sleep(poll) |
98 | durations["return_timeout_seconds"] = timeout |
99 | durations["returned_after_seconds"] = round(time.time() - start) |
100 |
|
101 | version, mlag = eapi(auth, ["show version", "show mlag"]) |
102 | running_version = short_version(version["version"]) |
103 | # Without MLAG configured, show mlag still reports the default reload-delay, and nothing waits on it. |
104 | mlag_on = mlag_configured(mlag) |
105 | durations["reload_delay_seconds"] = ( |
106 | max(mlag.get("reloadDelay") or 0, mlag.get("reloadDelayNonMlag") or 0) |
107 | if mlag_on |
108 | else 0 |
109 | ) |
110 |
|
111 | ok, ko, notes = [], [], [] |
112 |
|
113 | def record(check, problems): |
114 | if problems: |
115 | ko.append({"check": check, "message": "; ".join(problems)}) |
116 | else: |
117 | ok.append(check) |
118 |
|
119 | if phase == "pre": |
120 | record( |
121 | "flash", |
122 | flash_problems( |
123 | auth, target_version, image_base_url, image_url, required_free_mb, notes |
124 | ), |
125 | ) |
126 | if mlag_on: |
127 | [sanity] = eapi(auth, ["show mlag config-sanity"]) |
128 | record("redundancy", redundancy_problems(mlag, sanity)) |
129 | else: |
130 | notes.append("redundancy: MLAG not configured, the reload is not hitless") |
131 | save_state( |
132 | key, |
133 | { |
134 | **state, |
135 | "snapshot": snapshot(auth), |
136 | "snapshot_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), |
137 | }, |
138 | ) |
139 | ok.append("interface") |
140 | elif not state.get("snapshot"): |
141 | ko.append( |
142 | { |
143 | "check": "interface", |
144 | "message": f"No pre-check snapshot for {auth['host']} in this flow run: run the pre-check first, in the same run", |
145 | } |
146 | ) |
147 | else: |
148 | # Ports stay errdisabled until the MLAG reload-delay has elapsed since boot: wait it out. |
149 | wait = durations["reload_delay_seconds"] - version["uptime"] |
150 | if wait > 0: |
151 | time.sleep(wait) |
152 | durations["reload_delay_waited_seconds"] = max(0, round(wait)) |
153 | stab = 120 if stabilisation_seconds is None else stabilisation_seconds |
154 | start = time.time() |
155 | problems = diff(state["snapshot"], snapshot(auth)) |
156 | while problems and time.time() - start < stab: |
157 | time.sleep(poll) |
158 | problems = diff(state["snapshot"], snapshot(auth)) |
159 | durations["stabilisation_seconds"] = stab |
160 | durations["stabilised_after_seconds"] = round(time.time() - start) |
161 | record("interface", problems) |
162 |
|
163 | return { |
164 | "running_version": running_version, |
165 | "status": "ko" if ko else "ok", |
166 | "checks_ok": ok, |
167 | "checks_ko": ko, |
168 | "durations": durations, |
169 | "notes": notes, |
170 | } |
171 |
|
172 |
|
173 | # One resource can serve every switch that shares a login: the host input, when set, overrides the resource's host. |
174 | def with_host(auth: arista_eos, host: str): |
175 | host = (host or "").strip() or (auth.get("host") or "").strip() |
176 | if not host: |
177 | raise ValueError("host is required: set it on the resource or pass host") |
178 | return {**auth, "host": host} |
179 |
|
180 |
|
181 | def flash_problems(auth, target, base, image_url, required_mb, notes): |
182 | [d] = eapi(auth, [{"cmd": "dir flash:/", "revision": 2}]) |
183 | flash = d["urls"]["flash:/"] |
184 | try: |
185 | image, url = image_source(target, base, image_url) |
186 | except ValueError as e: |
187 | return [str(e)] |
188 | if image and image in flash["entries"]: |
189 | if url: |
190 | sidecar_note(url, image, notes) |
191 | return [] |
192 | required = (required_mb or 1024) * 1e6 |
193 | if url: |
194 | try: |
195 | try: |
196 | size = head(url, ssl.create_default_context()) |
197 | sidecar_note(url, image, notes) |
198 | except urllib.error.URLError as e: |
199 | if not isinstance(e.reason, ssl.SSLCertVerificationError): |
200 | raise |
201 | # Only the size is read: the switch's copy doesn't check the certificate either, and activate |
202 | # verifies the image's sha512. So an untrusted (self-signed) server is sized anyway, with a note. |
203 | unverified = ssl.create_default_context() |
204 | unverified.check_hostname = False |
205 | unverified.verify_mode = ssl.CERT_NONE |
206 | size = head(url, unverified) |
207 | notes.append( |
208 | f"flash: the image server's certificate is not trusted on the worker ({e.reason.verify_message}), sized {image} without checking it; activate checks it when it fetches {image}.sha512, so pass it sha512" |
209 | ) |
210 | if size: |
211 | required = size |
212 | else: |
213 | notes.append( |
214 | f"flash: the image server gave no size for {image}, assumed required_free_mb ({mb(required)})" |
215 | ) |
216 | except urllib.error.HTTPError as e: |
217 | return [f"Target image {image} not found on the image server ({e.code})"] |
218 | except OSError as e: |
219 | return [ |
220 | f"Image server unreachable from the worker: {getattr(e, 'reason', e)}" |
221 | ] |
222 | free = flash["filesystem"]["freeBytes"] |
223 | if free < required: |
224 | return [ |
225 | f"Insufficient flash space ({mb(free)} free, {mb(required)} required{f' for {image}' if image else ''})" |
226 | ] |
227 | return [] |
228 |
|
229 |
|
230 | def head(url, context): |
231 | with urllib.request.urlopen( |
232 | urllib.request.Request(url, method="HEAD"), timeout=30, context=context |
233 | ) as r: |
234 | return int(r.headers.get("content-length") or 0) |
235 |
|
236 |
|
237 | # Activate reads the digest from the .sha512 file next to the image, certificate checked, unless given sha512. |
238 | def sidecar_note(url, image, notes): |
239 | try: |
240 | head(f"{url}.sha512", ssl.create_default_context()) |
241 | except OSError as e: |
242 | reason = getattr(e, "code", None) or getattr(e, "reason", e) |
243 | notes.append( |
244 | f"flash: {image}.sha512 not reachable from the worker ({reason}): pass sha512 to activate" |
245 | ) |
246 |
|
247 |
|
248 | # Same function in Device check and Device upgrade. |
249 | # image_url wins; else image_base_url + EOS-<target>.swi. Returns (file name on flash, URL or ""). The URL goes into |
250 | # the switch's copy command, so it is an allowlist: http(s), then only A-Z a-z 0-9 . _ ~ : / % + - (no query, no |
251 | # user:password@, nothing the CLI reads as a separator), ending in a plain .swi file name. Raises ValueError otherwise. |
252 | def image_source(target, base, image_url): |
253 | if image_url: |
254 | image, url = urlsplit(image_url).path.rsplit("/", 1)[-1], image_url |
255 | else: |
256 | image = f"EOS-{target}.swi" if target else "" |
257 | url = f"{base.rstrip('/')}/{image}" if image and base else "" |
258 | if not url: |
259 | return image, url |
260 | # Not echoed: it would put the password in the steps and errors. |
261 | if "@" in url: |
262 | raise ValueError("the image URL must not contain @ (no user:password@ in it)") |
263 | if not re.match(r"https?://", url): |
264 | raise ValueError( |
265 | f"the image URL must start with http:// or https://: got {url!r}" |
266 | ) |
267 | if not urlsplit(url).hostname: |
268 | raise ValueError(f"the image URL has no host: got {url!r}") |
269 | bad = sorted(set(re.sub(r"[A-Za-z0-9._~:/%+-]", "", url.split("://", 1)[1]))) |
270 | if bad: |
271 | raise ValueError( |
272 | f"the image URL may only contain A-Z a-z 0-9 . _ ~ : / % + -, not {' '.join(map(repr, bad))}: got {url!r}" |
273 | ) |
274 | if not re.fullmatch(r"[\w.+-]+\.swi", image): |
275 | raise ValueError(f"the image URL must end in a .swi file name: got {url!r}") |
276 | return image, url |
277 |
|
278 |
|
279 | # show mlag reports state disabled both when MLAG was never configured and when it is configured but shut down; |
280 | # only the first has no domain-id, local interface or peer-link. |
281 | def mlag_configured(mlag): |
282 | return any(mlag.get(k) for k in ("domainId", "localInterface", "peerLink")) |
283 |
|
284 |
|
285 | def redundancy_problems(mlag, sanity): |
286 | p = [] |
287 | if mlag.get("state") != "active": |
288 | p.append(f"MLAG state {mlag.get('state')} (expected active)") |
289 | if mlag.get("negStatus") != "connected": |
290 | p.append(f"MLAG negotiation {mlag.get('negStatus')} (expected connected)") |
291 | if mlag.get("peerLinkStatus") != "up": |
292 | p.append(f"peer-link {mlag.get('peerLink')} {mlag.get('peerLinkStatus')}") |
293 | if mlag.get("localIntfStatus") != "up": |
294 | p.append( |
295 | f"MLAG local interface {mlag.get('localInterface')} {mlag.get('localIntfStatus')}" |
296 | ) |
297 | inconsistent = [ |
298 | *(sanity.get("globalConfiguration") or {}), |
299 | *(sanity.get("interfaceConfiguration") or {}), |
300 | ] |
301 | if inconsistent: |
302 | p.append(f"config-sanity inconsistent: {', '.join(inconsistent)}") |
303 | partial = (mlag.get("mlagPorts") or {}).get("Active-partial") |
304 | if partial: |
305 | p.append(f"{partial} MLAG port(s) active-partial") |
306 | if mlag.get("portsErrdisabled"): |
307 | p.append("MLAG ports errdisabled") |
308 | return p |
309 |
|
310 |
|
311 | def snapshot(auth): |
312 | status, lldp, mlag, mlag_intf = eapi( |
313 | auth, |
314 | [ |
315 | "show interfaces status", |
316 | "show lldp neighbors", |
317 | "show mlag", |
318 | "show mlag interfaces", |
319 | ], |
320 | ) |
321 | return { |
322 | "interfaces_up": sorted( |
323 | n |
324 | for n, i in status["interfaceStatuses"].items() |
325 | if i.get("linkStatus") == "connected" |
326 | ), |
327 | "lldp": [ |
328 | { |
329 | "port": n["port"], |
330 | "neighbor": f"{n['neighborDevice']} {n['neighborPort']}", |
331 | } |
332 | for n in lldp["lldpNeighbors"] |
333 | ], |
334 | "mlag": { |
335 | "state": mlag.get("state"), |
336 | "negStatus": mlag.get("negStatus"), |
337 | "peerLinkStatus": mlag.get("peerLinkStatus"), |
338 | "ports": { |
339 | f"{i['localInterface']} (mlag {id})": i["status"] |
340 | for id, i in (mlag_intf.get("interfaces") or {}).items() |
341 | }, |
342 | }, |
343 | } |
344 |
|
345 |
|
346 | def diff(before, after): |
347 | p = [] |
348 | for i in before["interfaces_up"]: |
349 | if i not in after["interfaces_up"]: |
350 | p.append(f"{i} down after the upgrade (up before)") |
351 | for n in before["lldp"]: |
352 | if not any( |
353 | a["port"] == n["port"] and a["neighbor"] == n["neighbor"] |
354 | for a in after["lldp"] |
355 | ): |
356 | p.append(f"LLDP neighbor {n['neighbor']} no longer seen on {n['port']}") |
357 | for k in ("state", "negStatus", "peerLinkStatus"): |
358 | if before["mlag"][k] != after["mlag"][k]: |
359 | p.append(f"MLAG {k} {after['mlag'][k]} (was {before['mlag'][k]})") |
360 | for port, s in before["mlag"]["ports"].items(): |
361 | if after["mlag"]["ports"].get(port) != s: |
362 | p.append(f"{port} {after['mlag']['ports'].get(port, 'missing')} (was {s})") |
363 | return p |
364 |
|
365 |
|
366 | # The snapshot lives in the root flow run's user state, keyed by switch: pre- and post-check are different steps |
367 | # (often in nested subflows) of the same run. Outside a flow, it falls back to this script's own state. |
368 | # The endpoint is called through the client so an unreachable state raises (wmill.get_flow_user_state returns None). |
369 | # ponytail: the fallback is one map per script, so concurrent standalone runs can race; flows don't. |
370 | # The key is the switch's eAPI address: arista_eos-<host>-<port> with non-word characters in the host as _, the port |
371 | # always in (443 or 80 when unset), so host 10.0.0.11 with port 8443 and host https://10.0.0.11:8443 share one key, |
372 | # while two switches behind one address on different ports (or http and https) don't. State written before the port |
373 | # was part of the key is read from arista_eos_<host> as a fallback (all word characters, so never a new key); writes |
374 | # go to the new key only. |
375 | def state_keys(auth: arista_eos): |
376 | if auth.get("transport") == "ssh": |
377 | host, port = ssh_addr(auth) |
378 | return [f"arista_eos-{word(host)}-{port}"] |
379 | url = switch_url(auth) |
380 | port = int(url.port or auth.get("port") or {"https": 443, "http": 80}[url.scheme]) |
381 | return [ |
382 | f"arista_eos-{word(url.hostname)}-{port}", |
383 | f"arista_eos_{word(auth['host'])}", |
384 | ] |
385 |
|
386 |
|
387 | def word(s): |
388 | return re.sub(r"\W", "_", s) |
389 |
|
390 |
|
391 | def load_state(keys): |
392 | for key in keys: |
393 | if os.environ.get("WM_FLOW_JOB_ID"): |
394 | c = wmill.Windmill() |
395 | value = c.get( |
396 | f"/w/{c.workspace}/jobs/flow/user_states/{c.get_root_job_id()}/{key}" |
397 | ).json() |
398 | else: |
399 | value = (wmill.get_state() or {}).get(key) |
400 | if value is not None: |
401 | return value |
402 |
|
403 |
|
404 | def save_state(keys, value): |
405 | if os.environ.get("WM_FLOW_JOB_ID"): |
406 | c = wmill.Windmill() |
407 | c.post( |
408 | f"/w/{c.workspace}/jobs/flow/user_states/{c.get_root_job_id()}/{keys[0]}", |
409 | json=value, |
410 | ) |
411 | else: |
412 | wmill.set_state({**(wmill.get_state() or {}), keys[0]: value}) |
413 |
|
414 |
|
415 | # "4.34.9M-51132493.4349M (engineering build)" on cEOS, "4.34.9M" on hardware. |
416 | def short_version(v): |
417 | m = re.match(r"\d+\.\d+\.\d+(\.\d+)?[A-Z]*", v) |
418 | return m.group(0) if m else v |
419 |
|
420 |
|
421 | def mb(n): |
422 | return f"{n / 1e9:.1f} GB" if n >= 1e9 else f"{round(n / 1e6)} MB" |
423 |
|
424 |
|
425 | class DeviceUnreachable(Exception): |
426 | pass |
427 |
|
428 |
|
429 | class AuthFailure(Exception): |
430 | pass |
431 |
|
432 |
|
433 | def named(cls, message: str): |
434 | return cls(f"{cls.__name__}: {message}") |
435 |
|
436 |
|
437 | # pyeapi logs a traceback for every socket error, and the post-check polls a switch that is rebooting. |
438 | logging.getLogger("pyeapi").addHandler(logging.NullHandler()) |
439 |
|
440 |
|
441 | def eapi(auth: arista_eos, cmds: list, fmt: str = "json", timeout: float = 60): |
442 | eos = node(auth, timeout) |
443 | try: |
444 | return eos.run_commands(cmds, fmt) |
445 | except pyeapi.eapilib.ConnectionError as e: |
446 | raise connection_error(auth, eos, e) |
447 |
|
448 |
|
449 | # pyeapi 1.0.4 raises ConnectionError for a socket error (kept in socket_error), an answer that isn't JSON (eAPI |
450 | # still starting, 5xx) and a 401, the only one with neither. |
451 | def connection_error(auth, eos, e): |
452 | # A certificate the resource does not trust is a setting to fix (verify_tls, ca_certificate), not worth a retry. |
453 | if isinstance(eos.connection.socket_error, ssl.SSLCertVerificationError): |
454 | return RuntimeError(f"{auth['host']}: {eos.connection.socket_error}") |
455 | if eos.connection.socket_error is None and e.message != "unable to connect to eAPI": |
456 | return named( |
457 | AuthFailure, f"{auth['host']}: {'SSH' if auth.get('transport') == 'ssh' else 'eAPI'} refused the credential ({e.message})" |
458 | ) |
459 | return named(DeviceUnreachable, f"{auth['host']}: {e.message}") |
460 |
|
461 |
|
462 | # host is an address or a URL (https://10.0.0.11:8443); a bare IPv6 address is bracketed. Only https, or http when the |
463 | # resource ticks allow_http (Basic auth then crosses the network in cleartext). pyeapi skips certificate checks unless it |
464 | # is given a context, so one is always passed: verifying, or not when the resource unticks verify_tls (EOS ships a |
465 | # self-signed cert). |
466 | def switch_url(auth: arista_eos): |
467 | host = auth["host"] |
468 | if "://" not in host: |
469 | host = f"https://[{host}]" if host.count(":") > 1 and "[" not in host else f"https://{host}" |
470 | url = urlsplit(host) |
471 | if not url.hostname or (url.scheme != "https" and not (url.scheme == "http" and auth.get("allow_http"))): |
472 | raise ValueError(f"host must be an address or an https:// URL (http:// needs allow_http on the resource): {auth['host']}") |
473 | return url |
474 |
|
475 |
|
476 | # SSH transport (transport: ssh): the EOS CLI over an SSH shell, behind the part of pyeapi's Node the scripts use |
477 | # (run_commands, connection.socket_error, connection.transport.connect), raising pyeapi's own CommandError and |
478 | # ConnectionError, so the checks and the error mapping are the same for both transports. Show commands in json get |
479 | # "| json", which prints the eAPI JSON; other commands give { messages: [output] } like eAPI, or {} when silent. |
480 | # A line starting with "% " is an error, raised as eAPI's CommandError. The host key is checked when the resource |
481 | # has ssh_host_key (SHA256:... as ssh-keygen -lf prints it). |
482 | class Ssh: |
483 | def __init__(self, auth: arista_eos, timeout: float): |
484 | self.auth, self.timeout = auth, timeout |
485 | self.connection = self.transport = self |
486 | self.socket_error = self.sock = self.chan = None |
487 |
|
488 | # The TCP connection only, like eAPI's transport.connect(): if it fails, nothing was sent. |
489 | def connect(self): |
490 | self.sock = socket.create_connection(ssh_addr(self.auth), self.timeout) |
491 |
|
492 | def lost(self, e): |
493 | self.socket_error = e |
494 | return pyeapi.eapilib.ConnectionError("ssh", f"SSH connection failed ({e})") |
495 |
|
496 | def open(self): |
497 | auth = self.auth |
498 | try: |
499 | if not self.sock: |
500 | self.connect() |
501 | t = paramiko.Transport(self.sock) |
502 | t.start_client(timeout=self.timeout) |
503 | key = t.get_remote_server_key() |
504 | except (OSError, EOFError, paramiko.SSHException) as e: |
505 | raise self.lost(e) |
506 | fp = "SHA256:" + base64.b64encode(hashlib.sha256(key.asbytes()).digest()).decode().rstrip("=") |
507 | if auth.get("ssh_host_key") and auth["ssh_host_key"].strip() != fp: |
508 | t.close() |
509 | raise RuntimeError(f"{auth['host']}: the SSH host key is {fp}, not the resource's ssh_host_key") |
510 | try: |
511 | try: |
512 | t.auth_password(auth["username"], auth["password"]) |
513 | except paramiko.BadAuthenticationType: |
514 | t.auth_interactive( |
515 | auth["username"], lambda _t, _i, prompts: [auth["password"]] * len(prompts) |
516 | ) |
517 | except paramiko.AuthenticationException as e: |
518 | t.close() |
519 | raise pyeapi.eapilib.ConnectionError("ssh", str(e)) |
520 | except (OSError, EOFError, paramiko.SSHException) as e: |
521 | raise self.lost(e) |
522 | try: |
523 | self.chan = t.open_session() |
524 | self.chan.get_pty(width=4000) |
525 | self.chan.invoke_shell() |
526 | self.prompt = None |
527 | out = self.read() |
528 | self.prompt = re.escape(re.split(r"[(>#]", out.splitlines()[-1].strip())[0]) |
529 | if out.rstrip().endswith(">"): |
530 | self.cli("enable") |
531 | self.cli("terminal length 0") |
532 | self.cli("terminal width 32767") |
533 | except (OSError, EOFError, paramiko.SSHException) as e: |
534 | raise self.lost(e) |
535 |
|
536 | # Reads until the prompt. Questions eAPI never asks are answered as eAPI behaves: confirm, don't save. |
537 | def read(self): |
538 | buf, end = "", time.monotonic() + self.timeout |
539 | prompt = (self.prompt or r"[\w.\-]+") + r"(\([^)\r\n]*\))?[>#] ?$" |
540 | while True: |
541 | self.chan.settimeout(max(end - time.monotonic(), 0.01)) |
542 | chunk = self.chan.recv(65536) |
543 | if not chunk: |
544 | raise self.lost("connection closed") |
545 | buf += chunk.decode(errors="replace") |
546 | tail = buf[-300:] |
547 | if re.search(r"(^|\n)" + prompt, tail): |
548 | return buf |
549 | for question, answer in ( |
550 | # enable asks for a password on switches that land the user unprivileged: the login password. |
551 | (r"(^|\n)Password:\s*$", self.auth["password"]), |
552 | (r"\[confirm\]\s*$", ""), |
553 | (r"Save\?.*\[yes/no[^\]]*\]:?\s*$", "no"), |
554 | (r"\[(y|yes)/(n|no)\][:?]?\s*$", "y"), |
555 | ): |
556 | if re.search(question, tail, re.I): |
557 | self.chan.send(answer + "\r") |
558 | buf += "\n" |
559 |
|
560 | # One command: its output without the echoed command and the prompt; progress lines keep their last state. |
561 | def cli(self, cmd): |
562 | try: |
563 | self.chan.send(cmd + "\r") |
564 | out = self.read() |
565 | except (OSError, EOFError, paramiko.SSHException) as e: |
566 | raise self.lost(e) |
567 | self.mode = out.rstrip().splitlines()[-1] |
568 | lines = [l.rstrip("\r").split("\r")[-1] for l in out.split("\n")] |
569 | return "\n".join(lines[1:-1]).strip("\n") |
570 |
|
571 | def run_commands(self, cmds, encoding="json", **_): |
572 | try: |
573 | if not self.chan: |
574 | self.open() |
575 | if "(config" in self.mode: |
576 | self.cli("end") |
577 | results = [] |
578 | for i, c in enumerate(cmds): |
579 | # eAPI's { cmd, revision } form is the CLI's "| json revision N". |
580 | cmd, rev = (c["cmd"], c.get("revision")) if isinstance(c, dict) else (c, None) |
581 | show = encoding == "json" and (rev or re.match(r"\s*sh(ow)?\s", cmd)) and not re.search(r"\|\s*json", cmd) |
582 | # pyeapi sends "<cmd> MULTILINE: <text>" (banners) as cmd + input; the CLI reads the text up to EOF. |
583 | head, multi, text = cmd.partition("MULTILINE:") |
584 | # The shell would run what follows a line break as another command; eAPI refuses it too. |
585 | if re.search(r"[\r\n]", head): |
586 | raise ValueError(f"a command may not contain a line break: {head!r}") |
587 | line = "\r".join([head.strip(), *text.strip().splitlines(), "EOF"]) if multi else cmd |
588 | out = self.cli(f"{cmd} | json{f' revision {rev}' if rev else ''}" if show else line) |
589 | errors = [l[2:] for l in out.splitlines() if l.startswith("% ")] |
590 | code, kind = 1002, "invalid command" |
591 | if show and not errors: |
592 | code, kind = 1000, "could not run command" |
593 | try: |
594 | result = json.loads(out) |
595 | except ValueError: |
596 | errors = [out[:200]] |
597 | else: |
598 | errors = result.get("errors") or [] |
599 | if errors: |
600 | raise pyeapi.eapilib.CommandError( |
601 | code, |
602 | f"CLI command {i + 2} of {len(cmds) + 1} '{cmd}' failed: {kind}", |
603 | command_error="; ".join(errors), |
604 | commands=cmds, |
605 | output=results, |
606 | ) |
607 | if encoding != "json": |
608 | result = {"output": out + "\n" if out else ""} |
609 | elif not show: |
610 | # The CLI prints eAPI's warnings as "! " notices. |
611 | notices = [l[2:].strip() for l in out.splitlines() if l.startswith("! ")] |
612 | out = "\n".join(l for l in out.splitlines() if not l.startswith("! ")).strip() |
613 | result = {"messages": [out]} if out and not multi else {} |
614 | if notices: |
615 | result["warnings"] = notices |
616 | results.append(result) |
617 | return results |
618 | except pyeapi.eapilib.ConnectionError: |
619 | if self.chan: |
620 | self.chan.close() |
621 | self.sock = self.chan = None |
622 | raise |
623 |
|
624 |
|
625 | # paramiko logs a traceback when a rebooting switch drops the session. |
626 | logging.getLogger("paramiko").addHandler(logging.NullHandler()) |
627 |
|
628 |
|
629 | # host, or host:port ([v6]:port), else port, else 22. |
630 | def ssh_addr(auth: arista_eos): |
631 | h = auth["host"] |
632 | if ": |
633 | raise ValueError(f"with transport ssh, host is an address or a hostname (with :port), not a URL: {h}") |
634 | url = urlsplit(f"ssh://{h}" if h.count(":") <= 1 or h.startswith("[") else f"ssh://[{h}]") |
635 | return url.hostname, url.port or int(auth.get("port") or 22) |
636 |
|
637 |
|
638 | def node(auth: arista_eos, timeout: float = 60): |
639 | if auth.get("transport") == "ssh": |
640 | return Ssh(auth, timeout) |
641 | url = switch_url(auth) |
642 | # ca_certificate: the PEM of a private CA, trusted instead of the system's CAs. |
643 | context = ssl.create_default_context(cadata=(auth.get("ca_certificate") or "").strip() or None) |
644 | if auth.get("verify_tls") is False: |
645 | context.check_hostname = False |
646 | context.verify_mode = ssl.CERT_NONE |
647 | return pyeapi.connect( |
648 | transport=url.scheme, |
649 | host=url.hostname, |
650 | port=url.port or auth.get("port") or None, |
651 | username=auth["username"], |
652 | password=auth["password"], |
653 | context=context, |
654 | timeout=timeout, |
655 | return_node=True, |
656 | ) |
657 | |