0

All-in-one device check (deprecated)

Published 17 days ago

Pre- or post-upgrade check of one switch: returns { running_version, status ok|ko, checks_ok, checks_ko[{check, message}], notes }. Pre: flash space for the target image, MLAG redundancy, and a snapshot of interfaces/LLDP/MLAG kept in the flow run's state. Post: waits for the switch to answer and for the MLAG reload-delay (when MLAG is configured), then compares with the snapshot over a bounded stabilisation window. Throws DeviceUnreachable or AuthFailure. Does not compare the version with the target: the workflow does. Deprecated: kept for flows built on it; new flows use the single-attempt Device actions, which return plain facts instead of this ok|ko result.

Script arista_eos Verified

The script

Submitted by hugo989 Python3
Verified 1 day ago
1
# py312
2
# requirements:
3
# paramiko==5.1.0
4
# pyeapi==1.0.4
5
# wmill==1.851.0
6

7
# Device check: pre- and post-upgrade checks of one switch.
8
# - flash (pre): the target image is already on flash, or free space covers its size on the image server
9
#   (image_url, or image_base_url + EOS-<target_version>.swi; required_free_mb without an image server or when its
10
#   HEAD has no Content-Length). The HEAD checks the certificate and, if the worker doesn't trust it, sizes the image
11
#   anyway: notes says when either happens, and when activate will need its sha512 input (untrusted certificate, or
12
#   no <image>.sha512 reachable). A malformed image URL is flash ko.
13
# - redundancy (pre): MLAG active and connected, peer-link and local interface up, config-sanity clean, no MLAG
14
#   port active-partial, no port errdisabled. Left out when MLAG is not configured (notes says the reload is not
15
#   hitless); configured but down is ko.
16
# - interface: pre saves a snapshot (interfaces up, LLDP neighbor per port, MLAG state and per-port status) in the
17
#   flow run's user state, on the root flow, under arista_eos-<host>-<port>; post compares the switch with it.
18
# Post waits, in order: for eAPI to answer on a boot later than this run's reload (return_timeout_seconds), for the
19
# MLAG reload-delay counted from boot when MLAG is configured, then up to stabilisation_seconds for the state to
20
# match the snapshot. The three are returned in durations. The version is reported, not compared with the target.
21

22
import base64
23
import hashlib
24
import json
25
import logging
26
import os
27
import re
28
import socket
29
import ssl
30
import time
31
import urllib.error
32
import urllib.request
33
from typing import Literal, TypedDict
34
from urllib.parse import urlsplit
35

36
import paramiko
37
import pyeapi
38
import wmill
39

40

41
class arista_eos(TypedDict):
42
    host: str
43
    username: str
44
    password: str
45
    port: int
46
    verify_tls: bool
47
    allow_http: bool
48

49

50
def main(
51
    auth: arista_eos,
52
    phase: Literal["pre", "post"],
53
    host: str = "",
54
    target_version: str = "",
55
    image_base_url: str = "",
56
    required_free_mb: int = 1024,
57
    return_timeout_seconds: int = 900,
58
    stabilisation_seconds: int = 120,
59
    poll_interval_seconds: int = 15,
60
    image_url: str = "",
61
):
62
    """
63
    All-in-one device check (deprecated)
64
    Pre- or post-upgrade check of one switch: returns { running_version, status ok|ko, checks_ok, checks_ko[{check, message}], notes }. Pre: flash space for the target image, MLAG redundancy, and a snapshot of interfaces/LLDP/MLAG kept in the flow run's state. Post: waits for the switch to answer and for the MLAG reload-delay (when MLAG is configured), then compares with the snapshot over a bounded stabilisation window. Throws DeviceUnreachable or AuthFailure. Does not compare the version with the target: the workflow does. Deprecated: kept for flows built on it; new flows use the single-attempt Device actions, which return plain facts instead of this ok|ko result.
65
    """
66
    auth = with_host(auth, host)
67
    poll = poll_interval_seconds or 15
68
    durations = {}
69
    key = state_keys(auth)
70
    state = load_state(key) or {}
71

72
    if phase == "post":
73
        # Wait for the switch to answer on a boot later than the one each reload of this run was sent on: `reload now`
74
        # can answer before EOS goes down, and the old boot must not pass for the new one. AuthFailure stops at once.
75
        markers = [state[k] for k in ("reload", "rollback") if state.get(k)]
76
        timeout = return_timeout_seconds or 900
77
        start = time.time()
78
        while True:
79
            try:
80
                [v] = eapi(auth, ["show version"])
81
                m = next(
82
                    (m for m in markers if m["boot_timestamp"] == v["bootupTimestamp"]),
83
                    None,
84
                )
85
                if not m:
86
                    break
87
                pending = (
88
                    f"still on the boot the reload sent at {m['sent_at']} was sent on"
89
                )
90
            except DeviceUnreachable as e:
91
                pending = str(e)
92
            if time.time() - start > timeout:
93
                raise named(
94
                    DeviceUnreachable,
95
                    f"{auth['host']}: not back after {timeout} s ({pending})",
96
                )
97
            time.sleep(poll)
98
        durations["return_timeout_seconds"] = timeout
99
        durations["returned_after_seconds"] = round(time.time() - start)
100

101
    version, mlag = eapi(auth, ["show version", "show mlag"])
102
    running_version = short_version(version["version"])
103
    # Without MLAG configured, show mlag still reports the default reload-delay, and nothing waits on it.
104
    mlag_on = mlag_configured(mlag)
105
    durations["reload_delay_seconds"] = (
106
        max(mlag.get("reloadDelay") or 0, mlag.get("reloadDelayNonMlag") or 0)
107
        if mlag_on
108
        else 0
109
    )
110

111
    ok, ko, notes = [], [], []
112

113
    def record(check, problems):
114
        if problems:
115
            ko.append({"check": check, "message": "; ".join(problems)})
116
        else:
117
            ok.append(check)
118

119
    if phase == "pre":
120
        record(
121
            "flash",
122
            flash_problems(
123
                auth, target_version, image_base_url, image_url, required_free_mb, notes
124
            ),
125
        )
126
        if mlag_on:
127
            [sanity] = eapi(auth, ["show mlag config-sanity"])
128
            record("redundancy", redundancy_problems(mlag, sanity))
129
        else:
130
            notes.append("redundancy: MLAG not configured, the reload is not hitless")
131
        save_state(
132
            key,
133
            {
134
                **state,
135
                "snapshot": snapshot(auth),
136
                "snapshot_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
137
            },
138
        )
139
        ok.append("interface")
140
    elif not state.get("snapshot"):
141
        ko.append(
142
            {
143
                "check": "interface",
144
                "message": f"No pre-check snapshot for {auth['host']} in this flow run: run the pre-check first, in the same run",
145
            }
146
        )
147
    else:
148
        # Ports stay errdisabled until the MLAG reload-delay has elapsed since boot: wait it out.
149
        wait = durations["reload_delay_seconds"] - version["uptime"]
150
        if wait > 0:
151
            time.sleep(wait)
152
        durations["reload_delay_waited_seconds"] = max(0, round(wait))
153
        stab = 120 if stabilisation_seconds is None else stabilisation_seconds
154
        start = time.time()
155
        problems = diff(state["snapshot"], snapshot(auth))
156
        while problems and time.time() - start < stab:
157
            time.sleep(poll)
158
            problems = diff(state["snapshot"], snapshot(auth))
159
        durations["stabilisation_seconds"] = stab
160
        durations["stabilised_after_seconds"] = round(time.time() - start)
161
        record("interface", problems)
162

163
    return {
164
        "running_version": running_version,
165
        "status": "ko" if ko else "ok",
166
        "checks_ok": ok,
167
        "checks_ko": ko,
168
        "durations": durations,
169
        "notes": notes,
170
    }
171

172

173
# One resource can serve every switch that shares a login: the host input, when set, overrides the resource's host.
174
def with_host(auth: arista_eos, host: str):
175
    host = (host or "").strip() or (auth.get("host") or "").strip()
176
    if not host:
177
        raise ValueError("host is required: set it on the resource or pass host")
178
    return {**auth, "host": host}
179

180

181
def flash_problems(auth, target, base, image_url, required_mb, notes):
182
    [d] = eapi(auth, [{"cmd": "dir flash:/", "revision": 2}])
183
    flash = d["urls"]["flash:/"]
184
    try:
185
        image, url = image_source(target, base, image_url)
186
    except ValueError as e:
187
        return [str(e)]
188
    if image and image in flash["entries"]:
189
        if url:
190
            sidecar_note(url, image, notes)
191
        return []
192
    required = (required_mb or 1024) * 1e6
193
    if url:
194
        try:
195
            try:
196
                size = head(url, ssl.create_default_context())
197
                sidecar_note(url, image, notes)
198
            except urllib.error.URLError as e:
199
                if not isinstance(e.reason, ssl.SSLCertVerificationError):
200
                    raise
201
                # Only the size is read: the switch's copy doesn't check the certificate either, and activate
202
                # verifies the image's sha512. So an untrusted (self-signed) server is sized anyway, with a note.
203
                unverified = ssl.create_default_context()
204
                unverified.check_hostname = False
205
                unverified.verify_mode = ssl.CERT_NONE
206
                size = head(url, unverified)
207
                notes.append(
208
                    f"flash: the image server's certificate is not trusted on the worker ({e.reason.verify_message}), sized {image} without checking it; activate checks it when it fetches {image}.sha512, so pass it sha512"
209
                )
210
            if size:
211
                required = size
212
            else:
213
                notes.append(
214
                    f"flash: the image server gave no size for {image}, assumed required_free_mb ({mb(required)})"
215
                )
216
        except urllib.error.HTTPError as e:
217
            return [f"Target image {image} not found on the image server ({e.code})"]
218
        except OSError as e:
219
            return [
220
                f"Image server unreachable from the worker: {getattr(e, 'reason', e)}"
221
            ]
222
    free = flash["filesystem"]["freeBytes"]
223
    if free < required:
224
        return [
225
            f"Insufficient flash space ({mb(free)} free, {mb(required)} required{f' for {image}' if image else ''})"
226
        ]
227
    return []
228

229

230
def head(url, context):
231
    with urllib.request.urlopen(
232
        urllib.request.Request(url, method="HEAD"), timeout=30, context=context
233
    ) as r:
234
        return int(r.headers.get("content-length") or 0)
235

236

237
# Activate reads the digest from the .sha512 file next to the image, certificate checked, unless given sha512.
238
def sidecar_note(url, image, notes):
239
    try:
240
        head(f"{url}.sha512", ssl.create_default_context())
241
    except OSError as e:
242
        reason = getattr(e, "code", None) or getattr(e, "reason", e)
243
        notes.append(
244
            f"flash: {image}.sha512 not reachable from the worker ({reason}): pass sha512 to activate"
245
        )
246

247

248
# Same function in Device check and Device upgrade.
249
# image_url wins; else image_base_url + EOS-<target>.swi. Returns (file name on flash, URL or ""). The URL goes into
250
# the switch's copy command, so it is an allowlist: http(s), then only A-Z a-z 0-9 . _ ~ : / % + - (no query, no
251
# user:password@, nothing the CLI reads as a separator), ending in a plain .swi file name. Raises ValueError otherwise.
252
def image_source(target, base, image_url):
253
    if image_url:
254
        image, url = urlsplit(image_url).path.rsplit("/", 1)[-1], image_url
255
    else:
256
        image = f"EOS-{target}.swi" if target else ""
257
        url = f"{base.rstrip('/')}/{image}" if image and base else ""
258
    if not url:
259
        return image, url
260
    # Not echoed: it would put the password in the steps and errors.
261
    if "@" in url:
262
        raise ValueError("the image URL must not contain @ (no user:password@ in it)")
263
    if not re.match(r"https?://", url):
264
        raise ValueError(
265
            f"the image URL must start with http:// or https://: got {url!r}"
266
        )
267
    if not urlsplit(url).hostname:
268
        raise ValueError(f"the image URL has no host: got {url!r}")
269
    bad = sorted(set(re.sub(r"[A-Za-z0-9._~:/%+-]", "", url.split("://", 1)[1])))
270
    if bad:
271
        raise ValueError(
272
            f"the image URL may only contain A-Z a-z 0-9 . _ ~ : / % + -, not {' '.join(map(repr, bad))}: got {url!r}"
273
        )
274
    if not re.fullmatch(r"[\w.+-]+\.swi", image):
275
        raise ValueError(f"the image URL must end in a .swi file name: got {url!r}")
276
    return image, url
277

278

279
# show mlag reports state disabled both when MLAG was never configured and when it is configured but shut down;
280
# only the first has no domain-id, local interface or peer-link.
281
def mlag_configured(mlag):
282
    return any(mlag.get(k) for k in ("domainId", "localInterface", "peerLink"))
283

284

285
def redundancy_problems(mlag, sanity):
286
    p = []
287
    if mlag.get("state") != "active":
288
        p.append(f"MLAG state {mlag.get('state')} (expected active)")
289
    if mlag.get("negStatus") != "connected":
290
        p.append(f"MLAG negotiation {mlag.get('negStatus')} (expected connected)")
291
    if mlag.get("peerLinkStatus") != "up":
292
        p.append(f"peer-link {mlag.get('peerLink')} {mlag.get('peerLinkStatus')}")
293
    if mlag.get("localIntfStatus") != "up":
294
        p.append(
295
            f"MLAG local interface {mlag.get('localInterface')} {mlag.get('localIntfStatus')}"
296
        )
297
    inconsistent = [
298
        *(sanity.get("globalConfiguration") or {}),
299
        *(sanity.get("interfaceConfiguration") or {}),
300
    ]
301
    if inconsistent:
302
        p.append(f"config-sanity inconsistent: {', '.join(inconsistent)}")
303
    partial = (mlag.get("mlagPorts") or {}).get("Active-partial")
304
    if partial:
305
        p.append(f"{partial} MLAG port(s) active-partial")
306
    if mlag.get("portsErrdisabled"):
307
        p.append("MLAG ports errdisabled")
308
    return p
309

310

311
def snapshot(auth):
312
    status, lldp, mlag, mlag_intf = eapi(
313
        auth,
314
        [
315
            "show interfaces status",
316
            "show lldp neighbors",
317
            "show mlag",
318
            "show mlag interfaces",
319
        ],
320
    )
321
    return {
322
        "interfaces_up": sorted(
323
            n
324
            for n, i in status["interfaceStatuses"].items()
325
            if i.get("linkStatus") == "connected"
326
        ),
327
        "lldp": [
328
            {
329
                "port": n["port"],
330
                "neighbor": f"{n['neighborDevice']} {n['neighborPort']}",
331
            }
332
            for n in lldp["lldpNeighbors"]
333
        ],
334
        "mlag": {
335
            "state": mlag.get("state"),
336
            "negStatus": mlag.get("negStatus"),
337
            "peerLinkStatus": mlag.get("peerLinkStatus"),
338
            "ports": {
339
                f"{i['localInterface']} (mlag {id})": i["status"]
340
                for id, i in (mlag_intf.get("interfaces") or {}).items()
341
            },
342
        },
343
    }
344

345

346
def diff(before, after):
347
    p = []
348
    for i in before["interfaces_up"]:
349
        if i not in after["interfaces_up"]:
350
            p.append(f"{i} down after the upgrade (up before)")
351
    for n in before["lldp"]:
352
        if not any(
353
            a["port"] == n["port"] and a["neighbor"] == n["neighbor"]
354
            for a in after["lldp"]
355
        ):
356
            p.append(f"LLDP neighbor {n['neighbor']} no longer seen on {n['port']}")
357
    for k in ("state", "negStatus", "peerLinkStatus"):
358
        if before["mlag"][k] != after["mlag"][k]:
359
            p.append(f"MLAG {k} {after['mlag'][k]} (was {before['mlag'][k]})")
360
    for port, s in before["mlag"]["ports"].items():
361
        if after["mlag"]["ports"].get(port) != s:
362
            p.append(f"{port} {after['mlag']['ports'].get(port, 'missing')} (was {s})")
363
    return p
364

365

366
# The snapshot lives in the root flow run's user state, keyed by switch: pre- and post-check are different steps
367
# (often in nested subflows) of the same run. Outside a flow, it falls back to this script's own state.
368
# The endpoint is called through the client so an unreachable state raises (wmill.get_flow_user_state returns None).
369
# ponytail: the fallback is one map per script, so concurrent standalone runs can race; flows don't.
370
# The key is the switch's eAPI address: arista_eos-<host>-<port> with non-word characters in the host as _, the port
371
# always in (443 or 80 when unset), so host 10.0.0.11 with port 8443 and host https://10.0.0.11:8443 share one key,
372
# while two switches behind one address on different ports (or http and https) don't. State written before the port
373
# was part of the key is read from arista_eos_<host> as a fallback (all word characters, so never a new key); writes
374
# go to the new key only.
375
def state_keys(auth: arista_eos):
376
    if auth.get("transport") == "ssh":
377
        host, port = ssh_addr(auth)
378
        return [f"arista_eos-{word(host)}-{port}"]
379
    url = switch_url(auth)
380
    port = int(url.port or auth.get("port") or {"https": 443, "http": 80}[url.scheme])
381
    return [
382
        f"arista_eos-{word(url.hostname)}-{port}",
383
        f"arista_eos_{word(auth['host'])}",
384
    ]
385

386

387
def word(s):
388
    return re.sub(r"\W", "_", s)
389

390

391
def load_state(keys):
392
    for key in keys:
393
        if os.environ.get("WM_FLOW_JOB_ID"):
394
            c = wmill.Windmill()
395
            value = c.get(
396
                f"/w/{c.workspace}/jobs/flow/user_states/{c.get_root_job_id()}/{key}"
397
            ).json()
398
        else:
399
            value = (wmill.get_state() or {}).get(key)
400
        if value is not None:
401
            return value
402

403

404
def save_state(keys, value):
405
    if os.environ.get("WM_FLOW_JOB_ID"):
406
        c = wmill.Windmill()
407
        c.post(
408
            f"/w/{c.workspace}/jobs/flow/user_states/{c.get_root_job_id()}/{keys[0]}",
409
            json=value,
410
        )
411
    else:
412
        wmill.set_state({**(wmill.get_state() or {}), keys[0]: value})
413

414

415
# "4.34.9M-51132493.4349M (engineering build)" on cEOS, "4.34.9M" on hardware.
416
def short_version(v):
417
    m = re.match(r"\d+\.\d+\.\d+(\.\d+)?[A-Z]*", v)
418
    return m.group(0) if m else v
419

420

421
def mb(n):
422
    return f"{n / 1e9:.1f} GB" if n >= 1e9 else f"{round(n / 1e6)} MB"
423

424

425
class DeviceUnreachable(Exception):
426
    pass
427

428

429
class AuthFailure(Exception):
430
    pass
431

432

433
def named(cls, message: str):
434
    return cls(f"{cls.__name__}: {message}")
435

436

437
# pyeapi logs a traceback for every socket error, and the post-check polls a switch that is rebooting.
438
logging.getLogger("pyeapi").addHandler(logging.NullHandler())
439

440

441
def eapi(auth: arista_eos, cmds: list, fmt: str = "json", timeout: float = 60):
442
    eos = node(auth, timeout)
443
    try:
444
        return eos.run_commands(cmds, fmt)
445
    except pyeapi.eapilib.ConnectionError as e:
446
        raise connection_error(auth, eos, e)
447

448

449
# pyeapi 1.0.4 raises ConnectionError for a socket error (kept in socket_error), an answer that isn't JSON (eAPI
450
# still starting, 5xx) and a 401, the only one with neither.
451
def connection_error(auth, eos, e):
452
    # A certificate the resource does not trust is a setting to fix (verify_tls, ca_certificate), not worth a retry.
453
    if isinstance(eos.connection.socket_error, ssl.SSLCertVerificationError):
454
        return RuntimeError(f"{auth['host']}: {eos.connection.socket_error}")
455
    if eos.connection.socket_error is None and e.message != "unable to connect to eAPI":
456
        return named(
457
            AuthFailure, f"{auth['host']}: {'SSH' if auth.get('transport') == 'ssh' else 'eAPI'} refused the credential ({e.message})"
458
        )
459
    return named(DeviceUnreachable, f"{auth['host']}: {e.message}")
460

461

462
# host is an address or a URL (https://10.0.0.11:8443); a bare IPv6 address is bracketed. Only https, or http when the
463
# resource ticks allow_http (Basic auth then crosses the network in cleartext). pyeapi skips certificate checks unless it
464
# is given a context, so one is always passed: verifying, or not when the resource unticks verify_tls (EOS ships a
465
# self-signed cert).
466
def switch_url(auth: arista_eos):
467
    host = auth["host"]
468
    if "://" not in host:
469
        host = f"https://[{host}]" if host.count(":") > 1 and "[" not in host else f"https://{host}"
470
    url = urlsplit(host)
471
    if not url.hostname or (url.scheme != "https" and not (url.scheme == "http" and auth.get("allow_http"))):
472
        raise ValueError(f"host must be an address or an https:// URL (http:// needs allow_http on the resource): {auth['host']}")
473
    return url
474

475

476
# SSH transport (transport: ssh): the EOS CLI over an SSH shell, behind the part of pyeapi's Node the scripts use
477
# (run_commands, connection.socket_error, connection.transport.connect), raising pyeapi's own CommandError and
478
# ConnectionError, so the checks and the error mapping are the same for both transports. Show commands in json get
479
# "| json", which prints the eAPI JSON; other commands give { messages: [output] } like eAPI, or {} when silent.
480
# A line starting with "% " is an error, raised as eAPI's CommandError. The host key is checked when the resource
481
# has ssh_host_key (SHA256:... as ssh-keygen -lf prints it).
482
class Ssh:
483
    def __init__(self, auth: arista_eos, timeout: float):
484
        self.auth, self.timeout = auth, timeout
485
        self.connection = self.transport = self
486
        self.socket_error = self.sock = self.chan = None
487

488
    # The TCP connection only, like eAPI's transport.connect(): if it fails, nothing was sent.
489
    def connect(self):
490
        self.sock = socket.create_connection(ssh_addr(self.auth), self.timeout)
491

492
    def lost(self, e):
493
        self.socket_error = e
494
        return pyeapi.eapilib.ConnectionError("ssh", f"SSH connection failed ({e})")
495

496
    def open(self):
497
        auth = self.auth
498
        try:
499
            if not self.sock:
500
                self.connect()
501
            t = paramiko.Transport(self.sock)
502
            t.start_client(timeout=self.timeout)
503
            key = t.get_remote_server_key()
504
        except (OSError, EOFError, paramiko.SSHException) as e:
505
            raise self.lost(e)
506
        fp = "SHA256:" + base64.b64encode(hashlib.sha256(key.asbytes()).digest()).decode().rstrip("=")
507
        if auth.get("ssh_host_key") and auth["ssh_host_key"].strip() != fp:
508
            t.close()
509
            raise RuntimeError(f"{auth['host']}: the SSH host key is {fp}, not the resource's ssh_host_key")
510
        try:
511
            try:
512
                t.auth_password(auth["username"], auth["password"])
513
            except paramiko.BadAuthenticationType:
514
                t.auth_interactive(
515
                    auth["username"], lambda _t, _i, prompts: [auth["password"]] * len(prompts)
516
                )
517
        except paramiko.AuthenticationException as e:
518
            t.close()
519
            raise pyeapi.eapilib.ConnectionError("ssh", str(e))
520
        except (OSError, EOFError, paramiko.SSHException) as e:
521
            raise self.lost(e)
522
        try:
523
            self.chan = t.open_session()
524
            self.chan.get_pty(width=4000)
525
            self.chan.invoke_shell()
526
            self.prompt = None
527
            out = self.read()
528
            self.prompt = re.escape(re.split(r"[(>#]", out.splitlines()[-1].strip())[0])
529
            if out.rstrip().endswith(">"):
530
                self.cli("enable")
531
            self.cli("terminal length 0")
532
            self.cli("terminal width 32767")
533
        except (OSError, EOFError, paramiko.SSHException) as e:
534
            raise self.lost(e)
535

536
    # Reads until the prompt. Questions eAPI never asks are answered as eAPI behaves: confirm, don't save.
537
    def read(self):
538
        buf, end = "", time.monotonic() + self.timeout
539
        prompt = (self.prompt or r"[\w.\-]+") + r"(\([^)\r\n]*\))?[>#] ?$"
540
        while True:
541
            self.chan.settimeout(max(end - time.monotonic(), 0.01))
542
            chunk = self.chan.recv(65536)
543
            if not chunk:
544
                raise self.lost("connection closed")
545
            buf += chunk.decode(errors="replace")
546
            tail = buf[-300:]
547
            if re.search(r"(^|\n)" + prompt, tail):
548
                return buf
549
            for question, answer in (
550
                # enable asks for a password on switches that land the user unprivileged: the login password.
551
                (r"(^|\n)Password:\s*$", self.auth["password"]),
552
                (r"\[confirm\]\s*$", ""),
553
                (r"Save\?.*\[yes/no[^\]]*\]:?\s*$", "no"),
554
                (r"\[(y|yes)/(n|no)\][:?]?\s*$", "y"),
555
            ):
556
                if re.search(question, tail, re.I):
557
                    self.chan.send(answer + "\r")
558
                    buf += "\n"
559

560
    # One command: its output without the echoed command and the prompt; progress lines keep their last state.
561
    def cli(self, cmd):
562
        try:
563
            self.chan.send(cmd + "\r")
564
            out = self.read()
565
        except (OSError, EOFError, paramiko.SSHException) as e:
566
            raise self.lost(e)
567
        self.mode = out.rstrip().splitlines()[-1]
568
        lines = [l.rstrip("\r").split("\r")[-1] for l in out.split("\n")]
569
        return "\n".join(lines[1:-1]).strip("\n")
570

571
    def run_commands(self, cmds, encoding="json", **_):
572
        try:
573
            if not self.chan:
574
                self.open()
575
            if "(config" in self.mode:
576
                self.cli("end")
577
            results = []
578
            for i, c in enumerate(cmds):
579
                # eAPI's { cmd, revision } form is the CLI's "| json revision N".
580
                cmd, rev = (c["cmd"], c.get("revision")) if isinstance(c, dict) else (c, None)
581
                show = encoding == "json" and (rev or re.match(r"\s*sh(ow)?\s", cmd)) and not re.search(r"\|\s*json", cmd)
582
                # pyeapi sends "<cmd> MULTILINE: <text>" (banners) as cmd + input; the CLI reads the text up to EOF.
583
                head, multi, text = cmd.partition("MULTILINE:")
584
                # The shell would run what follows a line break as another command; eAPI refuses it too.
585
                if re.search(r"[\r\n]", head):
586
                    raise ValueError(f"a command may not contain a line break: {head!r}")
587
                line = "\r".join([head.strip(), *text.strip().splitlines(), "EOF"]) if multi else cmd
588
                out = self.cli(f"{cmd} | json{f' revision {rev}' if rev else ''}" if show else line)
589
                errors = [l[2:] for l in out.splitlines() if l.startswith("% ")]
590
                code, kind = 1002, "invalid command"
591
                if show and not errors:
592
                    code, kind = 1000, "could not run command"
593
                    try:
594
                        result = json.loads(out)
595
                    except ValueError:
596
                        errors = [out[:200]]
597
                    else:
598
                        errors = result.get("errors") or []
599
                if errors:
600
                    raise pyeapi.eapilib.CommandError(
601
                        code,
602
                        f"CLI command {i + 2} of {len(cmds) + 1} '{cmd}' failed: {kind}",
603
                        command_error="; ".join(errors),
604
                        commands=cmds,
605
                        output=results,
606
                    )
607
                if encoding != "json":
608
                    result = {"output": out + "\n" if out else ""}
609
                elif not show:
610
                    # The CLI prints eAPI's warnings as "! " notices.
611
                    notices = [l[2:].strip() for l in out.splitlines() if l.startswith("! ")]
612
                    out = "\n".join(l for l in out.splitlines() if not l.startswith("! ")).strip()
613
                    result = {"messages": [out]} if out and not multi else {}
614
                    if notices:
615
                        result["warnings"] = notices
616
                results.append(result)
617
            return results
618
        except pyeapi.eapilib.ConnectionError:
619
            if self.chan:
620
                self.chan.close()
621
            self.sock = self.chan = None
622
            raise
623

624

625
# paramiko logs a traceback when a rebooting switch drops the session.
626
logging.getLogger("paramiko").addHandler(logging.NullHandler())
627

628

629
# host, or host:port ([v6]:port), else port, else 22.
630
def ssh_addr(auth: arista_eos):
631
    h = auth["host"]
632
    if "://" in h:
633
        raise ValueError(f"with transport ssh, host is an address or a hostname (with :port), not a URL: {h}")
634
    url = urlsplit(f"ssh://{h}" if h.count(":") <= 1 or h.startswith("[") else f"ssh://[{h}]")
635
    return url.hostname, url.port or int(auth.get("port") or 22)
636

637

638
def node(auth: arista_eos, timeout: float = 60):
639
    if auth.get("transport") == "ssh":
640
        return Ssh(auth, timeout)
641
    url = switch_url(auth)
642
    # ca_certificate: the PEM of a private CA, trusted instead of the system's CAs.
643
    context = ssl.create_default_context(cadata=(auth.get("ca_certificate") or "").strip() or None)
644
    if auth.get("verify_tls") is False:
645
        context.check_hostname = False
646
        context.verify_mode = ssl.CERT_NONE
647
    return pyeapi.connect(
648
        transport=url.scheme,
649
        host=url.hostname,
650
        port=url.port or auth.get("port") or None,
651
        username=auth["username"],
652
        password=auth["password"],
653
        context=context,
654
        timeout=timeout,
655
        return_node=True,
656
    )
657