0

All-in-one device upgrade (deprecated)

Published 17 days ago

One upgrade step on one switch: activate (copy the image from image_url or EOS-<target>.swi from image_base_url, verify its sha512, boot system it, no reload), reload (exactly once, guarded, refused with unsaved config unless allow_unsaved), commit (write memory, refused while a config-session commit timer is pending; keep the previous image), rollback (boot the previous image and reload, same guard). Returns { status: "done" } or throws ActivateFailed, RollbackFailed, DeviceUnreachable or AuthFailure as the first token of the error. Deprecated: kept for flows built on it; new flows use the single-attempt Device actions, which return plain facts instead of this ok|ko result.

Script arista_eos Verified

The script

Submitted by hugo989 Python3
Verified 1 day ago
1
# py312
2
# requirements:
3
# paramiko==5.1.0
4
# pyeapi==1.0.4
5
# wmill==1.851.0
6

7
# Device upgrade: one upgrade step on one switch, picked by action.
8
# - activate: records the image of the running version as the rollback target (once per run, before any change),
9
#   copies the image to flash (skipped when already there), compares verify /sha512 with the expected digest, then
10
#   boot system. No reload. The image is image_url, or image_base_url + EOS-<target_version>.swi; the digest is
11
#   sha512, or the .sha512 file next to the image. A file that doesn't match is copied once more, then deleted.
12
#   An image file name that carries a version must carry target_version.
13
# - reload: reload now, at most once per switch and flow run. Skipped (reloaded false, with the reason) when this run
14
#   already sent it, the switch already runs target_version, or it rebooted less than min_uptime_seconds ago.
15
#   boot-config must be the image activate set in this run, or flash:/EOS-<target_version>.swi. It refuses while
16
#   show running-config diffs changes anything but its ! headers, or there is no startup-config (reload now doesn't
17
#   prompt, so unsaved changes would be lost), unless allow_unsaved. Rollback has no such check: it is the recovery path.
18
# - commit: write memory (refused while a config session's commit timer is pending, so a change about to roll back
19
#   isn't saved); the previous image stays on flash (delete_old_images removes only the others).
20
# - rollback: boot system back to the recorded previous image, then reload now behind the same guard. A boot that
21
#   this run's reload caused doesn't count against min_uptime_seconds.
22
# State (rollback target, activated image, reload markers) is the root flow run's user state under arista_eos-<host>-<port>.
23

24
import base64
25
import hashlib
26
import json
27
import logging
28
import os
29
import re
30
import socket
31
import ssl
32
import time
33
import urllib.error
34
import urllib.request
35
from typing import Literal, TypedDict
36
from urllib.parse import urlsplit
37

38
import paramiko
39
import pyeapi
40
import wmill
41

42

43
class arista_eos(TypedDict):
44
    host: str
45
    username: str
46
    password: str
47
    port: int
48
    verify_tls: bool
49
    allow_http: bool
50

51

52
class ActivateFailed(Exception):
53
    pass
54

55

56
class RollbackFailed(Exception):
57
    pass
58

59

60
def main(
61
    auth: arista_eos,
62
    action: Literal["activate", "reload", "commit", "rollback"],
63
    host: str = "",
64
    target_version: str = "",
65
    image_base_url: str = "",
66
    vrf: str = "",
67
    min_uptime_seconds: int = 300,
68
    delete_old_images: bool = False,
69
    image_url: str = "",
70
    sha512: str = "",
71
    allow_unsaved: bool = False,
72
):
73
    """
74
    All-in-one device upgrade (deprecated)
75
    One upgrade step on one switch: activate (copy the image from image_url or EOS-<target>.swi from image_base_url, verify its sha512, boot system it, no reload), reload (exactly once, guarded, refused with unsaved config unless allow_unsaved), commit (write memory, refused while a config-session commit timer is pending; keep the previous image), rollback (boot the previous image and reload, same guard). Returns { status: "done" } or throws ActivateFailed, RollbackFailed, DeviceUnreachable or AuthFailure as the first token of the error. Deprecated: kept for flows built on it; new flows use the single-attempt Device actions, which return plain facts instead of this ok|ko result.
76
    """
77
    auth = with_host(auth, host)
78
    key = state_keys(auth)
79
    state = load_state(key) or {}
80
    min_uptime = 300 if min_uptime_seconds is None else min_uptime_seconds
81

82
    if action == "activate":
83
        # Set again only once this activate has booted the image: a stale one would let reload through.
84
        forget_activated(key, state)
85
        if not target_version or not (image_base_url or image_url):
86
            raise named(
87
                ActivateFailed,
88
                "target_version and image_base_url (or image_url) are required",
89
            )
90
        try:
91
            image, url = image_source(target_version, image_base_url, image_url)
92
        except ValueError as e:
93
            raise named(ActivateFailed, str(e))
94
        # A file name that carries a version must carry this one; one without (a lab image) is taken as is.
95
        if image_version(image) not in (None, target_version):
96
            raise named(
97
                ActivateFailed,
98
                f"{image} is named for {image_version(image)}, not target_version {target_version}",
99
            )
100
        steps = []
101
        previous = state.get("previous_image") or ""
102
        try:
103
            if sha512:
104
                expected = digest(sha512)
105
                if not expected:
106
                    raise Exception("sha512 must be the 128-hex digest of the image")
107
            else:
108
                try:
109
                    with urllib.request.urlopen(f"{url}.sha512", timeout=60) as r:
110
                        expected = digest(r.read().decode(errors="replace"))
111
                except urllib.error.HTTPError as e:
112
                    raise Exception(f"cannot read {url}.sha512 ({e.code}): pass sha512")
113
                except OSError as e:
114
                    reason = getattr(e, "reason", e)
115
                    # Unlike the switch's copy, this fetch checks the certificate: the digest is what activate trusts.
116
                    trust = (
117
                        ", or trust the image server's CA on the worker (SSL_CERT_FILE)"
118
                        if isinstance(reason, ssl.SSLCertVerificationError)
119
                        else ""
120
                    )
121
                    raise Exception(
122
                        f"cannot read {url}.sha512 ({reason}): pass sha512{trust}"
123
                    )
124
                if not expected:
125
                    raise Exception(f"no sha512 hash in {url}.sha512")
126

127
            d, boot, version = eapi(
128
                auth,
129
                [
130
                    {"cmd": "dir flash:/", "revision": 2},
131
                    "show boot-config",
132
                    "show version",
133
                ],
134
            )
135
            files = list(d["urls"]["flash:/"]["entries"])
136
            running = short_version(version["version"])
137
            booted = boot["softwareImage"].replace("flash:/", "")
138
            # boot-config is the running image (first run) or the target (an earlier run got past boot system).
139
            # Anything else is a stale boot-config this step won't guess around.
140
            if image_version(booted) != running and booted != image:
141
                raise Exception(
142
                    f"boot-config points to {boot['softwareImage']}, neither the running version {running} nor {image}: fix boot-config first"
143
                )
144
            # The rollback target is the image of the running version, recorded once per run before anything changes.
145
            if not previous and running != target_version:
146
                candidates = (
147
                    [booted]
148
                    if booted != image
149
                    else [
150
                        f
151
                        for f in files
152
                        if f.endswith(".swi") and image_version(f) == running
153
                    ]
154
                )
155
                if len(candidates) != 1 or candidates[0] not in files:
156
                    raise Exception(
157
                        f"cannot tell which image on flash runs {running} (candidates: {', '.join(candidates) or 'none'}): no rollback target"
158
                    )
159
                previous = f"flash:/{candidates[0]}"
160
                save_state(key, {**state, "previous_image": previous})
161

162
            def copy():
163
                # The copy runs on the switch and can take minutes for a ~1 GB image.
164
                eapi(
165
                    auth,
166
                    [*([f"cli vrf {vrf}"] if vrf else []), f"copy {url} flash:{image}"],
167
                    "json",
168
                    3600,
169
                )
170
                steps.append(f"copied {url}")
171

172
            def verify():
173
                [v] = eapi(auth, [f"verify /sha512 flash:{image}"])
174
                return digest((v.get("messages") or [""])[0])
175

176
            copied = image not in files
177
            if copied:
178
                copy()
179
            actual = verify()
180
            if actual != expected and not copied:
181
                # Most likely a partial copy left by a killed run: copy it once more.
182
                eapi(auth, [f"delete flash:{image}"])
183
                copy()
184
                actual = verify()
185
            if actual != expected:
186
                # An earlier run may have booted it already: never leave boot-config on a deleted image.
187
                restored = booted == image and previous
188
                if restored:
189
                    boot_system(auth, previous.replace("flash:/", ""))
190
                eapi(auth, [f"delete flash:{image}"])
191
                raise Exception(
192
                    f"sha512 mismatch for {image} (switch {actual or 'none'}, expected {expected}), deleted it"
193
                    + (f", boot-config back on {previous}" if restored else "")
194
                )
195
            steps.append("sha512 verified")
196
            steps.extend(boot_system(auth, image))
197
            # reload checks boot-config against this, so it needs no image_url of its own.
198
            save_state(
199
                key,
200
                {
201
                    **state,
202
                    "previous_image": previous,
203
                    "activated_image": f"flash:/{image}",
204
                },
205
            )
206
        except (DeviceUnreachable, AuthFailure):
207
            raise
208
        except Exception as e:
209
            raise named(ActivateFailed, str(e))
210
        return {"status": "done", "steps": steps, "previous_image": previous}
211

212
    if action == "commit":
213
        [sessions] = eapi(auth, ["show configuration sessions detail"])
214
        if sessions.get("commitTimerSessionName"):
215
            raise RuntimeError(
216
                f"config session {sessions['commitTimerSessionName']} is pending under a commit timer: confirm or abort it before commit (write memory)"
217
            )
218
        _, boot, d = eapi(
219
            auth,
220
            ["write memory", "show boot-config", {"cmd": "dir flash:/", "revision": 2}],
221
        )
222
        deleted = []
223
        # Rollback needs the previous image: only images that are neither booted nor the recorded previous one go.
224
        if delete_old_images and state.get("previous_image"):
225
            keep = [
226
                i.replace("flash:/", "")
227
                for i in (boot["softwareImage"], state["previous_image"])
228
            ]
229
            for f in list(d["urls"]["flash:/"]["entries"]):
230
                if re.match(r"^EOS.*\.swi$", f) and f not in keep:
231
                    eapi(auth, [f"delete flash:{f}"])
232
                    deleted.append(f)
233
        return {"status": "done", "saved": True, "deleted": deleted}
234

235
    if action == "reload":
236
        if not target_version:
237
            raise Exception("target_version is required for the reload guard")
238
        sw = read_switch(auth, state, min_uptime)
239
        sent = sw["marker"]("reload")
240
        if sw.get("unreachable"):
241
            if sent:
242
                return skip(
243
                    f"reload sent at {sent['sent_at']}; the switch no longer answers: it is reloading"
244
                )
245
            raise named(DeviceUnreachable, sw["unreachable"])
246
        if sent and (sent["confirmed"] or sent["boot_timestamp"] != sw["boot"]):
247
            return skip(guard_reason("reload", sent, sw))
248
        if sw["running"] == target_version:
249
            return skip(f"already running {target_version}")
250
        # The image activate set in this run (state), or exactly EOS-<target_version>.swi.
251
        activated = state.get("activated_image")
252
        if sw["image"] not in (activated, f"flash:/EOS-{target_version}.swi"):
253
            raise Exception(
254
                f"boot-config points to {sw['image']}, neither the image activate set ({activated or 'none in this run'}) nor flash:/EOS-{target_version}.swi: run activate first"
255
            )
256
        if sw["uptime"] < min_uptime:
257
            return skip(f"rebooted {round(sw['uptime'])} s ago")
258
        if not allow_unsaved and unsaved(auth):
259
            raise Exception(
260
                "unsaved config: run Save Config first, or pass allow_unsaved: true"
261
            )
262
        fire_reload(auth, key, "reload", sw["boot"])
263
        return {"status": "done", "reloaded": True}
264

265
    # rollback: boot the previous image (still on flash) and reload, with the same once-only guard.
266
    try:
267
        sw = read_switch(auth, state, min_uptime)
268
        sent = sw["marker"]("rollback")
269
        if sw.get("unreachable"):
270
            if sent:
271
                return skip(
272
                    f"rollback sent at {sent['sent_at']}; the switch no longer answers: it is reloading"
273
                )
274
            raise Exception(sw["unreachable"])
275
        if sent and (sent["confirmed"] or sent["boot_timestamp"] != sw["boot"]):
276
            return skip(guard_reason("rollback", sent, sw))
277
        d, boot = eapi(
278
            auth, [{"cmd": "dir flash:/", "revision": 2}, "show boot-config"]
279
        )
280
        files = list(d["urls"]["flash:/"]["entries"])
281
        current = boot["softwareImage"].replace("flash:/", "")
282
        # Without a recorded previous image (new run), the only other EOS image on flash is unambiguous.
283
        others = [f for f in files if re.match(r"^EOS.*\.swi$", f) and f != current]
284
        previous = (state.get("previous_image") or "").replace("flash:/", "") or (
285
            others[0] if len(others) == 1 else ""
286
        )
287
        if not previous:
288
            raise Exception(
289
                f"previous image unknown (EOS images on flash: {', '.join(others) or 'none'})"
290
            )
291
        if previous not in files:
292
            raise Exception(f"previous image {previous} is no longer on flash")
293
        reload = sw["marker"]("reload")
294
        if reload and reload["confirmed"] and reload["boot_timestamp"] == sw["boot"]:
295
            raise Exception(
296
                f"the reload sent at {reload['sent_at']} has not happened yet (same boot as when it was sent): rolling back now would race it, check the switch"
297
            )
298
        steps = boot_system(auth, previous)
299
        forget_activated(key, state)
300
        if sw["running"] == image_version(previous):
301
            return skip(
302
                f"already running the previous image {previous}; boot-config set back to it"
303
            )
304
        # A recent boot that this run's reload caused doesn't block the rollback: the post-check that ran after it
305
        # can end well inside min_uptime_seconds.
306
        ours = reload and reload["boot_timestamp"] != sw["boot"]
307
        if not ours and sw["uptime"] < min_uptime:
308
            return skip(f"rebooted {round(sw['uptime'])} s ago, not by this run")
309
        fire_reload(auth, key, "rollback", sw["boot"])
310
        return {
311
            "status": "done",
312
            "reloaded": True,
313
            "boot_image": f"flash:/{previous}",
314
            "steps": steps,
315
        }
316
    except AuthFailure:
317
        raise
318
    except Exception as e:
319
        raise named(
320
            RollbackFailed, f"{auth['host']}: {re.sub(r'^[A-Za-z]+: ', '', str(e))}"
321
        )
322

323

324
# One resource can serve every switch that shares a login: the host input, when set, overrides the resource's host.
325
def with_host(auth: arista_eos, host: str):
326
    host = (host or "").strip() or (auth.get("host") or "").strip()
327
    if not host:
328
        raise ValueError("host is required: set it on the resource or pass host")
329
    return {**auth, "host": host}
330

331

332
# Same function in Device check and Device upgrade.
333
# image_url wins; else image_base_url + EOS-<target>.swi. Returns (file name on flash, URL or ""). The URL goes into
334
# the switch's copy command, so it is an allowlist: http(s), then only A-Z a-z 0-9 . _ ~ : / % + - (no query, no
335
# user:password@, nothing the CLI reads as a separator), ending in a plain .swi file name. Raises ValueError otherwise.
336
def image_source(target, base, image_url):
337
    if image_url:
338
        image, url = urlsplit(image_url).path.rsplit("/", 1)[-1], image_url
339
    else:
340
        image = f"EOS-{target}.swi" if target else ""
341
        url = f"{base.rstrip('/')}/{image}" if image and base else ""
342
    if not url:
343
        return image, url
344
    # Not echoed: it would put the password in the steps and errors.
345
    if "@" in url:
346
        raise ValueError("the image URL must not contain @ (no user:password@ in it)")
347
    if not re.match(r"https?://", url):
348
        raise ValueError(
349
            f"the image URL must start with http:// or https://: got {url!r}"
350
        )
351
    if not urlsplit(url).hostname:
352
        raise ValueError(f"the image URL has no host: got {url!r}")
353
    bad = sorted(set(re.sub(r"[A-Za-z0-9._~:/%+-]", "", url.split("://", 1)[1])))
354
    if bad:
355
        raise ValueError(
356
            f"the image URL may only contain A-Z a-z 0-9 . _ ~ : / % + -, not {' '.join(map(repr, bad))}: got {url!r}"
357
        )
358
    if not re.fullmatch(r"[\w.+-]+\.swi", image):
359
        raise ValueError(f"the image URL must end in a .swi file name: got {url!r}")
360
    return image, url
361

362

363
def boot_system(auth, image):
364
    _, set_, _, after = eapi(
365
        auth, ["configure", f"boot system flash:{image}", "end", "show boot-config"]
366
    )
367
    if after["softwareImage"] != f"flash:/{image}":
368
        raise Exception(f"boot-config still points to {after['softwareImage']}")
369
    return [f"boot system flash:{image}", *(set_.get("warnings") or [])]
370

371

372
# reload now doesn't prompt to save. show running-config diffs is a unified diff of startup-config against
373
# running-config, empty when they match. Its two file-header lines come before the first @@. Changed lines that
374
# start with a single "!" are headers EOS writes itself (boot system shows there, so every activate changes one);
375
# "!!" comments are config. With no startup-config every line shows as added: unsaved, it would boot into ZTP.
376
# A refused command (AAA authorization, an older EOS) is not unsaved config: saving wouldn't clear it.
377
def unsaved(auth):
378
    try:
379
        [d] = eapi(auth, ["show running-config diffs"], "text")
380
    except pyeapi.eapilib.CommandError as e:
381
        raise Exception(f"cannot compare running and startup config: {e}")
382
    hunks, changed = False, []
383
    for l in d["output"].splitlines():
384
        hunks = hunks or l.startswith("@@")
385
        if hunks and l[:1] in ("+", "-") and l[1:].strip():
386
            changed.append(l[1:].strip())
387
    return any(not re.match(r"!(?!!)", l) for l in changed)
388

389

390
def forget_activated(key, state):
391
    if state.pop("activated_image", None):
392
        save_state(key, state)
393

394

395
def skip(reason):
396
    return {"status": "done", "reloaded": False, "reason": reason}
397

398

399
def guard_reason(kind, m, sw):
400
    if m["boot_timestamp"] == sw["boot"]:
401
        return f"{kind} sent at {m['sent_at']}; the switch is going down"
402
    return f"{kind} sent at {m['sent_at']} already rebooted the switch"
403

404

405
# A reload is sent at most once per switch and flow run. Each one leaves a marker { sent_at, boot_timestamp,
406
# confirmed }: boot_timestamp is the boot it was sent on, so a later boot means it happened, and confirmed means
407
# the command reached the switch. An unconfirmed marker on the same boot (the worker died first) is sent again.
408
# Outside a flow, a marker is dropped once the reboot it caused is min_uptime_seconds old.
409
def read_switch(auth, state, min_uptime):
410
    try:
411
        version, boot = eapi(auth, ["show version", "show boot-config"])
412
    except DeviceUnreachable as e:
413
        return {"unreachable": str(e), "marker": lambda k: state.get(k)}
414
    sw = {
415
        "running": short_version(version["version"]),
416
        "image": boot["softwareImage"],
417
        "boot": version["bootupTimestamp"],
418
        "uptime": version["uptime"],
419
    }
420
    in_flow = bool(os.environ.get("WM_FLOW_JOB_ID"))
421

422
    def marker(k):
423
        m = state.get(k)
424
        return (
425
            m
426
            if m
427
            and (
428
                in_flow
429
                or m["boot_timestamp"] == sw["boot"]
430
                or sw["uptime"] < min_uptime
431
            )
432
            else None
433
        )
434

435
    return {**sw, "marker": marker}
436

437

438
# Fire and forget: the switch drops the connection while going down, which is not an error. A connection that
439
# never opened, or a refusal from the switch, means it did not reload: the marker goes and the error is raised.
440
def fire_reload(auth, key, kind, boot_ts):
441
    def mark(m):
442
        rest = {k: v for k, v in (load_state(key) or {}).items() if k != kind}
443
        save_state(key, {**rest, kind: m} if m else rest)
444

445
    m = {
446
        "sent_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
447
        "boot_timestamp": boot_ts,
448
        "confirmed": False,
449
    }
450
    mark(m)
451
    eos = node(auth, 15)
452
    try:
453
        # Opened before sending, so a failure here means the command never left.
454
        eos.connection.transport.connect()
455
    except OSError as e:
456
        mark(None)
457
        raise named(DeviceUnreachable, f"{auth['host']}: {e}")
458
    try:
459
        eos.run_commands(["reload now"])
460
    except pyeapi.eapilib.ConnectionError as e:
461
        # Sent, then the connection dropped or timed out as the switch went down. A 401 is a refusal.
462
        err = connection_error(auth, eos, e)
463
        if isinstance(err, AuthFailure):
464
            mark(None)
465
            raise err
466
    except Exception:
467
        mark(None)
468
        raise
469
    mark({**m, "confirmed": True})
470

471

472
# Shared with Device check: the flow run's user state, keyed by switch, holds the pre-check snapshot, the image
473
# activate replaced (rollback target) and the reload markers. Outside a flow, it falls back to this script's own state.
474
# The endpoint is called through the client so an unreachable state raises (wmill.get_flow_user_state returns None).
475
# ponytail: the fallback is one map per script, so concurrent standalone runs can race; flows don't.
476
# The key is the switch's eAPI address: arista_eos-<host>-<port> with non-word characters in the host as _, the port
477
# always in (443 or 80 when unset), so host 10.0.0.11 with port 8443 and host https://10.0.0.11:8443 share one key,
478
# while two switches behind one address on different ports (or http and https) don't. State written before the port
479
# was part of the key is read from arista_eos_<host> as a fallback (all word characters, so never a new key); writes
480
# go to the new key only.
481
def state_keys(auth: arista_eos):
482
    if auth.get("transport") == "ssh":
483
        host, port = ssh_addr(auth)
484
        return [f"arista_eos-{word(host)}-{port}"]
485
    url = switch_url(auth)
486
    port = int(url.port or auth.get("port") or {"https": 443, "http": 80}[url.scheme])
487
    return [
488
        f"arista_eos-{word(url.hostname)}-{port}",
489
        f"arista_eos_{word(auth['host'])}",
490
    ]
491

492

493
def word(s):
494
    return re.sub(r"\W", "_", s)
495

496

497
def load_state(keys):
498
    for key in keys:
499
        if os.environ.get("WM_FLOW_JOB_ID"):
500
            c = wmill.Windmill()
501
            value = c.get(
502
                f"/w/{c.workspace}/jobs/flow/user_states/{c.get_root_job_id()}/{key}"
503
            ).json()
504
        else:
505
            value = (wmill.get_state() or {}).get(key)
506
        if value is not None:
507
            return value
508

509

510
def save_state(keys, value):
511
    if os.environ.get("WM_FLOW_JOB_ID"):
512
        c = wmill.Windmill()
513
        c.post(
514
            f"/w/{c.workspace}/jobs/flow/user_states/{c.get_root_job_id()}/{keys[0]}",
515
            json=value,
516
        )
517
    else:
518
        wmill.set_state({**(wmill.get_state() or {}), keys[0]: value})
519

520

521
# "4.34.9M-51132493.4349M (engineering build)" on cEOS, "4.34.9M" on hardware.
522
def short_version(v):
523
    m = re.match(r"\d+\.\d+\.\d+(\.\d+)?[A-Z]*", v)
524
    return m.group(0) if m else v
525

526

527
# "EOS64-4.34.9M.swi" -> "4.34.9M"
528
def image_version(file):
529
    m = re.search(r"-(\d+\.\d+\.\d+(\.\d+)?[A-Z]*)\.swi$", file)
530
    return m.group(1) if m else None
531

532

533
# The 128-hex digest in a sha512sum line, a BSD "SHA512 (f) = <hash>" line or verify /sha512 output.
534
def digest(text):
535
    m = re.search(r"\b[0-9a-fA-F]{128}\b", text)
536
    return m.group(0).lower() if m else ""
537

538

539
class DeviceUnreachable(Exception):
540
    pass
541

542

543
class AuthFailure(Exception):
544
    pass
545

546

547
def named(cls, message: str):
548
    return cls(f"{cls.__name__}: {message}")
549

550

551
# pyeapi logs a traceback for every socket error, and the post-check polls a switch that is rebooting.
552
logging.getLogger("pyeapi").addHandler(logging.NullHandler())
553

554

555
def eapi(auth: arista_eos, cmds: list, fmt: str = "json", timeout: float = 60):
556
    eos = node(auth, timeout)
557
    try:
558
        return eos.run_commands(cmds, fmt)
559
    except pyeapi.eapilib.ConnectionError as e:
560
        raise connection_error(auth, eos, e)
561

562

563
# pyeapi 1.0.4 raises ConnectionError for a socket error (kept in socket_error), an answer that isn't JSON (eAPI
564
# still starting, 5xx) and a 401, the only one with neither.
565
def connection_error(auth, eos, e):
566
    # A certificate the resource does not trust is a setting to fix (verify_tls, ca_certificate), not worth a retry.
567
    if isinstance(eos.connection.socket_error, ssl.SSLCertVerificationError):
568
        return RuntimeError(f"{auth['host']}: {eos.connection.socket_error}")
569
    if eos.connection.socket_error is None and e.message != "unable to connect to eAPI":
570
        return named(
571
            AuthFailure, f"{auth['host']}: {'SSH' if auth.get('transport') == 'ssh' else 'eAPI'} refused the credential ({e.message})"
572
        )
573
    return named(DeviceUnreachable, f"{auth['host']}: {e.message}")
574

575

576
# host is an address or a URL (https://10.0.0.11:8443); a bare IPv6 address is bracketed. Only https, or http when the
577
# resource ticks allow_http (Basic auth then crosses the network in cleartext). pyeapi skips certificate checks unless it
578
# is given a context, so one is always passed: verifying, or not when the resource unticks verify_tls (EOS ships a
579
# self-signed cert).
580
def switch_url(auth: arista_eos):
581
    host = auth["host"]
582
    if "://" not in host:
583
        host = f"https://[{host}]" if host.count(":") > 1 and "[" not in host else f"https://{host}"
584
    url = urlsplit(host)
585
    if not url.hostname or (url.scheme != "https" and not (url.scheme == "http" and auth.get("allow_http"))):
586
        raise ValueError(f"host must be an address or an https:// URL (http:// needs allow_http on the resource): {auth['host']}")
587
    return url
588

589

590
# SSH transport (transport: ssh): the EOS CLI over an SSH shell, behind the part of pyeapi's Node the scripts use
591
# (run_commands, connection.socket_error, connection.transport.connect), raising pyeapi's own CommandError and
592
# ConnectionError, so the checks and the error mapping are the same for both transports. Show commands in json get
593
# "| json", which prints the eAPI JSON; other commands give { messages: [output] } like eAPI, or {} when silent.
594
# A line starting with "% " is an error, raised as eAPI's CommandError. The host key is checked when the resource
595
# has ssh_host_key (SHA256:... as ssh-keygen -lf prints it).
596
class Ssh:
597
    def __init__(self, auth: arista_eos, timeout: float):
598
        self.auth, self.timeout = auth, timeout
599
        self.connection = self.transport = self
600
        self.socket_error = self.sock = self.chan = None
601

602
    # The TCP connection only, like eAPI's transport.connect(): if it fails, nothing was sent.
603
    def connect(self):
604
        self.sock = socket.create_connection(ssh_addr(self.auth), self.timeout)
605

606
    def lost(self, e):
607
        self.socket_error = e
608
        return pyeapi.eapilib.ConnectionError("ssh", f"SSH connection failed ({e})")
609

610
    def open(self):
611
        auth = self.auth
612
        try:
613
            if not self.sock:
614
                self.connect()
615
            t = paramiko.Transport(self.sock)
616
            t.start_client(timeout=self.timeout)
617
            key = t.get_remote_server_key()
618
        except (OSError, EOFError, paramiko.SSHException) as e:
619
            raise self.lost(e)
620
        fp = "SHA256:" + base64.b64encode(hashlib.sha256(key.asbytes()).digest()).decode().rstrip("=")
621
        if auth.get("ssh_host_key") and auth["ssh_host_key"].strip() != fp:
622
            t.close()
623
            raise RuntimeError(f"{auth['host']}: the SSH host key is {fp}, not the resource's ssh_host_key")
624
        try:
625
            try:
626
                t.auth_password(auth["username"], auth["password"])
627
            except paramiko.BadAuthenticationType:
628
                t.auth_interactive(
629
                    auth["username"], lambda _t, _i, prompts: [auth["password"]] * len(prompts)
630
                )
631
        except paramiko.AuthenticationException as e:
632
            t.close()
633
            raise pyeapi.eapilib.ConnectionError("ssh", str(e))
634
        except (OSError, EOFError, paramiko.SSHException) as e:
635
            raise self.lost(e)
636
        try:
637
            self.chan = t.open_session()
638
            self.chan.get_pty(width=4000)
639
            self.chan.invoke_shell()
640
            self.prompt = None
641
            out = self.read()
642
            self.prompt = re.escape(re.split(r"[(>#]", out.splitlines()[-1].strip())[0])
643
            if out.rstrip().endswith(">"):
644
                self.cli("enable")
645
            self.cli("terminal length 0")
646
            self.cli("terminal width 32767")
647
        except (OSError, EOFError, paramiko.SSHException) as e:
648
            raise self.lost(e)
649

650
    # Reads until the prompt. Questions eAPI never asks are answered as eAPI behaves: confirm, don't save.
651
    def read(self):
652
        buf, end = "", time.monotonic() + self.timeout
653
        prompt = (self.prompt or r"[\w.\-]+") + r"(\([^)\r\n]*\))?[>#] ?$"
654
        while True:
655
            self.chan.settimeout(max(end - time.monotonic(), 0.01))
656
            chunk = self.chan.recv(65536)
657
            if not chunk:
658
                raise self.lost("connection closed")
659
            buf += chunk.decode(errors="replace")
660
            tail = buf[-300:]
661
            if re.search(r"(^|\n)" + prompt, tail):
662
                return buf
663
            for question, answer in (
664
                # enable asks for a password on switches that land the user unprivileged: the login password.
665
                (r"(^|\n)Password:\s*$", self.auth["password"]),
666
                (r"\[confirm\]\s*$", ""),
667
                (r"Save\?.*\[yes/no[^\]]*\]:?\s*$", "no"),
668
                (r"\[(y|yes)/(n|no)\][:?]?\s*$", "y"),
669
            ):
670
                if re.search(question, tail, re.I):
671
                    self.chan.send(answer + "\r")
672
                    buf += "\n"
673

674
    # One command: its output without the echoed command and the prompt; progress lines keep their last state.
675
    def cli(self, cmd):
676
        try:
677
            self.chan.send(cmd + "\r")
678
            out = self.read()
679
        except (OSError, EOFError, paramiko.SSHException) as e:
680
            raise self.lost(e)
681
        self.mode = out.rstrip().splitlines()[-1]
682
        lines = [l.rstrip("\r").split("\r")[-1] for l in out.split("\n")]
683
        return "\n".join(lines[1:-1]).strip("\n")
684

685
    def run_commands(self, cmds, encoding="json", **_):
686
        try:
687
            if not self.chan:
688
                self.open()
689
            if "(config" in self.mode:
690
                self.cli("end")
691
            results = []
692
            for i, c in enumerate(cmds):
693
                # eAPI's { cmd, revision } form is the CLI's "| json revision N".
694
                cmd, rev = (c["cmd"], c.get("revision")) if isinstance(c, dict) else (c, None)
695
                show = encoding == "json" and (rev or re.match(r"\s*sh(ow)?\s", cmd)) and not re.search(r"\|\s*json", cmd)
696
                # pyeapi sends "<cmd> MULTILINE: <text>" (banners) as cmd + input; the CLI reads the text up to EOF.
697
                head, multi, text = cmd.partition("MULTILINE:")
698
                # The shell would run what follows a line break as another command; eAPI refuses it too.
699
                if re.search(r"[\r\n]", head):
700
                    raise ValueError(f"a command may not contain a line break: {head!r}")
701
                line = "\r".join([head.strip(), *text.strip().splitlines(), "EOF"]) if multi else cmd
702
                out = self.cli(f"{cmd} | json{f' revision {rev}' if rev else ''}" if show else line)
703
                errors = [l[2:] for l in out.splitlines() if l.startswith("% ")]
704
                code, kind = 1002, "invalid command"
705
                if show and not errors:
706
                    code, kind = 1000, "could not run command"
707
                    try:
708
                        result = json.loads(out)
709
                    except ValueError:
710
                        errors = [out[:200]]
711
                    else:
712
                        errors = result.get("errors") or []
713
                if errors:
714
                    raise pyeapi.eapilib.CommandError(
715
                        code,
716
                        f"CLI command {i + 2} of {len(cmds) + 1} '{cmd}' failed: {kind}",
717
                        command_error="; ".join(errors),
718
                        commands=cmds,
719
                        output=results,
720
                    )
721
                if encoding != "json":
722
                    result = {"output": out + "\n" if out else ""}
723
                elif not show:
724
                    # The CLI prints eAPI's warnings as "! " notices.
725
                    notices = [l[2:].strip() for l in out.splitlines() if l.startswith("! ")]
726
                    out = "\n".join(l for l in out.splitlines() if not l.startswith("! ")).strip()
727
                    result = {"messages": [out]} if out and not multi else {}
728
                    if notices:
729
                        result["warnings"] = notices
730
                results.append(result)
731
            return results
732
        except pyeapi.eapilib.ConnectionError:
733
            if self.chan:
734
                self.chan.close()
735
            self.sock = self.chan = None
736
            raise
737

738

739
# paramiko logs a traceback when a rebooting switch drops the session.
740
logging.getLogger("paramiko").addHandler(logging.NullHandler())
741

742

743
# host, or host:port ([v6]:port), else port, else 22.
744
def ssh_addr(auth: arista_eos):
745
    h = auth["host"]
746
    if "://" in h:
747
        raise ValueError(f"with transport ssh, host is an address or a hostname (with :port), not a URL: {h}")
748
    url = urlsplit(f"ssh://{h}" if h.count(":") <= 1 or h.startswith("[") else f"ssh://[{h}]")
749
    return url.hostname, url.port or int(auth.get("port") or 22)
750

751

752
def node(auth: arista_eos, timeout: float = 60):
753
    if auth.get("transport") == "ssh":
754
        return Ssh(auth, timeout)
755
    url = switch_url(auth)
756
    # ca_certificate: the PEM of a private CA, trusted instead of the system's CAs.
757
    context = ssl.create_default_context(cadata=(auth.get("ca_certificate") or "").strip() or None)
758
    if auth.get("verify_tls") is False:
759
        context.check_hostname = False
760
        context.verify_mode = ssl.CERT_NONE
761
    return pyeapi.connect(
762
        transport=url.scheme,
763
        host=url.hostname,
764
        port=url.port or auth.get("port") or None,
765
        username=auth["username"],
766
        password=auth["password"],
767
        context=context,
768
        timeout=timeout,
769
        return_node=True,
770
    )
771