0

All-in-one device check

Published 15 days ago

Pre- or post-upgrade check of one switch: running version, ok|ko, checks passed and failed (flash, redundancy, interface). Throws DeviceUnreachable or AuthFailure. For native step retry and sleep in the workflow, use the single-attempt Device actions (Device pre-check, Get Reload Delay, Device post-check, Device activate, Device reload, Device commit, Device rollback) instead.

Script arista_eos Verified

The script

Submitted by hugo989 Python3
Verified 1 day ago
1
# py312
2
# requirements:
3
# pyeapi==1.0.4
4
# wmill==1.851.0
5

6
# Device check: pre- and post-upgrade checks of one switch.
7
# - flash (pre): the target image is already on flash, or free space covers its size on the image server
8
#   (image_url, or image_base_url + EOS-<target_version>.swi; required_free_mb without an image server or when its
9
#   HEAD has no Content-Length). The HEAD checks the certificate and, if the worker doesn't trust it, sizes the image
10
#   anyway: notes says when either happens, and when activate will need its sha512 input (untrusted certificate, or
11
#   no <image>.sha512 reachable). A malformed image URL is flash ko.
12
# - redundancy (pre): MLAG active and connected, peer-link and local interface up, config-sanity clean, no MLAG
13
#   port active-partial, no port errdisabled. Left out when MLAG is not configured (notes says the reload is not
14
#   hitless); configured but down is ko.
15
# - interface: pre saves a snapshot (interfaces up, LLDP neighbor per port, MLAG state and per-port status) in the
16
#   flow run's user state, on the root flow, under arista_eos-<host>-<port>; post compares the switch with it.
17
# Post waits, in order: for eAPI to answer on a boot later than this run's reload (return_timeout_seconds), for the
18
# MLAG reload-delay counted from boot when MLAG is configured, then up to stabilisation_seconds for the state to
19
# match the snapshot. The three are returned in durations. The version is reported, not compared with the target.
20

21
import logging
22
import os
23
import re
24
import ssl
25
import time
26
import urllib.error
27
import urllib.request
28
from typing import Literal, TypedDict
29
from urllib.parse import urlsplit
30

31
import pyeapi
32
import wmill
33

34

35
class arista_eos(TypedDict):
36
    host: str
37
    username: str
38
    password: str
39
    port: int
40
    verify_tls: bool
41
    allow_http: bool
42

43

44
def main(
45
    auth: arista_eos,
46
    phase: Literal["pre", "post"],
47
    target_version: str = "",
48
    image_base_url: str = "",
49
    required_free_mb: int = 1024,
50
    return_timeout_seconds: int = 900,
51
    stabilisation_seconds: int = 120,
52
    poll_interval_seconds: int = 15,
53
    image_url: str = "",
54
):
55
    """
56
    All-in-one device check
57
    Pre- or post-upgrade check of one switch: returns { running_version, status ok|ko, checks_ok, checks_ko[{check, message}], notes }. Pre: flash space for the target image, MLAG redundancy, and a snapshot of interfaces/LLDP/MLAG kept in the flow run's state. Post: waits for the switch to answer and for the MLAG reload-delay (when MLAG is configured), then compares with the snapshot over a bounded stabilisation window. Throws DeviceUnreachable or AuthFailure. Does not compare the version with the target: the workflow does. For native step retry and sleep in the workflow, use the single-attempt Device actions (Device pre-check, Get Reload Delay, Device post-check, Device activate, Device reload, Device commit, Device rollback) instead.
58
    """
59
    poll = poll_interval_seconds or 15
60
    durations = {}
61
    key = state_keys(auth)
62
    state = load_state(key) or {}
63

64
    if phase == "post":
65
        # Wait for the switch to answer on a boot later than the one each reload of this run was sent on: `reload now`
66
        # can answer before EOS goes down, and the old boot must not pass for the new one. AuthFailure stops at once.
67
        markers = [state[k] for k in ("reload", "rollback") if state.get(k)]
68
        timeout = return_timeout_seconds or 900
69
        start = time.time()
70
        while True:
71
            try:
72
                [v] = eapi(auth, ["show version"])
73
                m = next(
74
                    (m for m in markers if m["boot_timestamp"] == v["bootupTimestamp"]),
75
                    None,
76
                )
77
                if not m:
78
                    break
79
                pending = (
80
                    f"still on the boot the reload sent at {m['sent_at']} was sent on"
81
                )
82
            except DeviceUnreachable as e:
83
                pending = str(e)
84
            if time.time() - start > timeout:
85
                raise named(
86
                    DeviceUnreachable,
87
                    f"{auth['host']}: not back after {timeout} s ({pending})",
88
                )
89
            time.sleep(poll)
90
        durations["return_timeout_seconds"] = timeout
91
        durations["returned_after_seconds"] = round(time.time() - start)
92

93
    version, mlag = eapi(auth, ["show version", "show mlag"])
94
    running_version = short_version(version["version"])
95
    # Without MLAG configured, show mlag still reports the default reload-delay, and nothing waits on it.
96
    mlag_on = mlag_configured(mlag)
97
    durations["reload_delay_seconds"] = (
98
        max(mlag.get("reloadDelay") or 0, mlag.get("reloadDelayNonMlag") or 0)
99
        if mlag_on
100
        else 0
101
    )
102

103
    ok, ko, notes = [], [], []
104

105
    def record(check, problems):
106
        if problems:
107
            ko.append({"check": check, "message": "; ".join(problems)})
108
        else:
109
            ok.append(check)
110

111
    if phase == "pre":
112
        record(
113
            "flash",
114
            flash_problems(
115
                auth, target_version, image_base_url, image_url, required_free_mb, notes
116
            ),
117
        )
118
        if mlag_on:
119
            [sanity] = eapi(auth, ["show mlag config-sanity"])
120
            record("redundancy", redundancy_problems(mlag, sanity))
121
        else:
122
            notes.append("redundancy: MLAG not configured, the reload is not hitless")
123
        save_state(
124
            key,
125
            {
126
                **state,
127
                "snapshot": snapshot(auth),
128
                "snapshot_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
129
            },
130
        )
131
        ok.append("interface")
132
    elif not state.get("snapshot"):
133
        ko.append(
134
            {
135
                "check": "interface",
136
                "message": f"No pre-check snapshot for {auth['host']} in this flow run: run the pre-check first, in the same run",
137
            }
138
        )
139
    else:
140
        # Ports stay errdisabled until the MLAG reload-delay has elapsed since boot: wait it out.
141
        wait = durations["reload_delay_seconds"] - version["uptime"]
142
        if wait > 0:
143
            time.sleep(wait)
144
        durations["reload_delay_waited_seconds"] = max(0, round(wait))
145
        stab = 120 if stabilisation_seconds is None else stabilisation_seconds
146
        start = time.time()
147
        problems = diff(state["snapshot"], snapshot(auth))
148
        while problems and time.time() - start < stab:
149
            time.sleep(poll)
150
            problems = diff(state["snapshot"], snapshot(auth))
151
        durations["stabilisation_seconds"] = stab
152
        durations["stabilised_after_seconds"] = round(time.time() - start)
153
        record("interface", problems)
154

155
    return {
156
        "running_version": running_version,
157
        "status": "ko" if ko else "ok",
158
        "checks_ok": ok,
159
        "checks_ko": ko,
160
        "durations": durations,
161
        "notes": notes,
162
    }
163

164

165
def flash_problems(auth, target, base, image_url, required_mb, notes):
166
    [d] = eapi(auth, [{"cmd": "dir flash:/", "revision": 2}])
167
    flash = d["urls"]["flash:/"]
168
    try:
169
        image, url = image_source(target, base, image_url)
170
    except ValueError as e:
171
        return [str(e)]
172
    if image and image in flash["entries"]:
173
        if url:
174
            sidecar_note(url, image, notes)
175
        return []
176
    required = (required_mb or 1024) * 1e6
177
    if url:
178
        try:
179
            try:
180
                size = head(url, ssl.create_default_context())
181
                sidecar_note(url, image, notes)
182
            except urllib.error.URLError as e:
183
                if not isinstance(e.reason, ssl.SSLCertVerificationError):
184
                    raise
185
                # Only the size is read: the switch's copy doesn't check the certificate either, and activate
186
                # verifies the image's sha512. So an untrusted (self-signed) server is sized anyway, with a note.
187
                unverified = ssl.create_default_context()
188
                unverified.check_hostname = False
189
                unverified.verify_mode = ssl.CERT_NONE
190
                size = head(url, unverified)
191
                notes.append(
192
                    f"flash: the image server's certificate is not trusted on the worker ({e.reason.verify_message}), sized {image} without checking it; activate checks it when it fetches {image}.sha512, so pass it sha512"
193
                )
194
            if size:
195
                required = size
196
            else:
197
                notes.append(
198
                    f"flash: the image server gave no size for {image}, assumed required_free_mb ({mb(required)})"
199
                )
200
        except urllib.error.HTTPError as e:
201
            return [f"Target image {image} not found on the image server ({e.code})"]
202
        except OSError as e:
203
            return [
204
                f"Image server unreachable from the worker: {getattr(e, 'reason', e)}"
205
            ]
206
    free = flash["filesystem"]["freeBytes"]
207
    if free < required:
208
        return [
209
            f"Insufficient flash space ({mb(free)} free, {mb(required)} required{f' for {image}' if image else ''})"
210
        ]
211
    return []
212

213

214
def head(url, context):
215
    with urllib.request.urlopen(
216
        urllib.request.Request(url, method="HEAD"), timeout=30, context=context
217
    ) as r:
218
        return int(r.headers.get("content-length") or 0)
219

220

221
# Activate reads the digest from the .sha512 file next to the image, certificate checked, unless given sha512.
222
def sidecar_note(url, image, notes):
223
    try:
224
        head(f"{url}.sha512", ssl.create_default_context())
225
    except OSError as e:
226
        reason = getattr(e, "code", None) or getattr(e, "reason", e)
227
        notes.append(
228
            f"flash: {image}.sha512 not reachable from the worker ({reason}): pass sha512 to activate"
229
        )
230

231

232
# Same function in Device check and Device upgrade.
233
# image_url wins; else image_base_url + EOS-<target>.swi. Returns (file name on flash, URL or ""). The URL goes into
234
# the switch's copy command, so it is an allowlist: http(s), then only A-Z a-z 0-9 . _ ~ : / % + - (no query, no
235
# user:password@, nothing the CLI reads as a separator), ending in a plain .swi file name. Raises ValueError otherwise.
236
def image_source(target, base, image_url):
237
    if image_url:
238
        image, url = urlsplit(image_url).path.rsplit("/", 1)[-1], image_url
239
    else:
240
        image = f"EOS-{target}.swi" if target else ""
241
        url = f"{base.rstrip('/')}/{image}" if image and base else ""
242
    if not url:
243
        return image, url
244
    # Not echoed: it would put the password in the steps and errors.
245
    if "@" in url:
246
        raise ValueError("the image URL must not contain @ (no user:password@ in it)")
247
    if not re.match(r"https?://", url):
248
        raise ValueError(
249
            f"the image URL must start with http:// or https://: got {url!r}"
250
        )
251
    if not urlsplit(url).hostname:
252
        raise ValueError(f"the image URL has no host: got {url!r}")
253
    bad = sorted(set(re.sub(r"[A-Za-z0-9._~:/%+-]", "", url.split("://", 1)[1])))
254
    if bad:
255
        raise ValueError(
256
            f"the image URL may only contain A-Z a-z 0-9 . _ ~ : / % + -, not {' '.join(map(repr, bad))}: got {url!r}"
257
        )
258
    if not re.fullmatch(r"[\w.+-]+\.swi", image):
259
        raise ValueError(f"the image URL must end in a .swi file name: got {url!r}")
260
    return image, url
261

262

263
# show mlag reports state disabled both when MLAG was never configured and when it is configured but shut down;
264
# only the first has no domain-id, local interface or peer-link.
265
def mlag_configured(mlag):
266
    return any(mlag.get(k) for k in ("domainId", "localInterface", "peerLink"))
267

268

269
def redundancy_problems(mlag, sanity):
270
    p = []
271
    if mlag.get("state") != "active":
272
        p.append(f"MLAG state {mlag.get('state')} (expected active)")
273
    if mlag.get("negStatus") != "connected":
274
        p.append(f"MLAG negotiation {mlag.get('negStatus')} (expected connected)")
275
    if mlag.get("peerLinkStatus") != "up":
276
        p.append(f"peer-link {mlag.get('peerLink')} {mlag.get('peerLinkStatus')}")
277
    if mlag.get("localIntfStatus") != "up":
278
        p.append(
279
            f"MLAG local interface {mlag.get('localInterface')} {mlag.get('localIntfStatus')}"
280
        )
281
    inconsistent = [
282
        *(sanity.get("globalConfiguration") or {}),
283
        *(sanity.get("interfaceConfiguration") or {}),
284
    ]
285
    if inconsistent:
286
        p.append(f"config-sanity inconsistent: {', '.join(inconsistent)}")
287
    partial = (mlag.get("mlagPorts") or {}).get("Active-partial")
288
    if partial:
289
        p.append(f"{partial} MLAG port(s) active-partial")
290
    if mlag.get("portsErrdisabled"):
291
        p.append("MLAG ports errdisabled")
292
    return p
293

294

295
def snapshot(auth):
296
    status, lldp, mlag, mlag_intf = eapi(
297
        auth,
298
        [
299
            "show interfaces status",
300
            "show lldp neighbors",
301
            "show mlag",
302
            "show mlag interfaces",
303
        ],
304
    )
305
    return {
306
        "interfaces_up": sorted(
307
            n
308
            for n, i in status["interfaceStatuses"].items()
309
            if i.get("linkStatus") == "connected"
310
        ),
311
        "lldp": [
312
            {
313
                "port": n["port"],
314
                "neighbor": f"{n['neighborDevice']} {n['neighborPort']}",
315
            }
316
            for n in lldp["lldpNeighbors"]
317
        ],
318
        "mlag": {
319
            "state": mlag.get("state"),
320
            "negStatus": mlag.get("negStatus"),
321
            "peerLinkStatus": mlag.get("peerLinkStatus"),
322
            "ports": {
323
                f"{i['localInterface']} (mlag {id})": i["status"]
324
                for id, i in (mlag_intf.get("interfaces") or {}).items()
325
            },
326
        },
327
    }
328

329

330
def diff(before, after):
331
    p = []
332
    for i in before["interfaces_up"]:
333
        if i not in after["interfaces_up"]:
334
            p.append(f"{i} down after the upgrade (up before)")
335
    for n in before["lldp"]:
336
        if not any(
337
            a["port"] == n["port"] and a["neighbor"] == n["neighbor"]
338
            for a in after["lldp"]
339
        ):
340
            p.append(f"LLDP neighbor {n['neighbor']} no longer seen on {n['port']}")
341
    for k in ("state", "negStatus", "peerLinkStatus"):
342
        if before["mlag"][k] != after["mlag"][k]:
343
            p.append(f"MLAG {k} {after['mlag'][k]} (was {before['mlag'][k]})")
344
    for port, s in before["mlag"]["ports"].items():
345
        if after["mlag"]["ports"].get(port) != s:
346
            p.append(f"{port} {after['mlag']['ports'].get(port, 'missing')} (was {s})")
347
    return p
348

349

350
# The snapshot lives in the root flow run's user state, keyed by switch: pre- and post-check are different steps
351
# (often in nested subflows) of the same run. Outside a flow, it falls back to this script's own state.
352
# The endpoint is called through the client so an unreachable state raises (wmill.get_flow_user_state returns None).
353
# ponytail: the fallback is one map per script, so concurrent standalone runs can race; flows don't.
354
# The key is the switch's eAPI address: arista_eos-<host>-<port> with non-word characters in the host as _, the port
355
# always in (443 or 80 when unset), so host 10.0.0.11 with port 8443 and host https://10.0.0.11:8443 share one key,
356
# while two switches behind one address on different ports (or http and https) don't. State written before the port
357
# was part of the key is read from arista_eos_<host> as a fallback (all word characters, so never a new key); writes
358
# go to the new key only.
359
def state_keys(auth: arista_eos):
360
    url = switch_url(auth)
361
    port = int(url.port or auth.get("port") or {"https": 443, "http": 80}[url.scheme])
362
    return [
363
        f"arista_eos-{word(url.hostname)}-{port}",
364
        f"arista_eos_{word(auth['host'])}",
365
    ]
366

367

368
def word(s):
369
    return re.sub(r"\W", "_", s)
370

371

372
def load_state(keys):
373
    for key in keys:
374
        if os.environ.get("WM_FLOW_JOB_ID"):
375
            c = wmill.Windmill()
376
            value = c.get(
377
                f"/w/{c.workspace}/jobs/flow/user_states/{c.get_root_job_id()}/{key}"
378
            ).json()
379
        else:
380
            value = (wmill.get_state() or {}).get(key)
381
        if value is not None:
382
            return value
383

384

385
def save_state(keys, value):
386
    if os.environ.get("WM_FLOW_JOB_ID"):
387
        c = wmill.Windmill()
388
        c.post(
389
            f"/w/{c.workspace}/jobs/flow/user_states/{c.get_root_job_id()}/{keys[0]}",
390
            json=value,
391
        )
392
    else:
393
        wmill.set_state({**(wmill.get_state() or {}), keys[0]: value})
394

395

396
# "4.34.9M-51132493.4349M (engineering build)" on cEOS, "4.34.9M" on hardware.
397
def short_version(v):
398
    m = re.match(r"\d+\.\d+\.\d+(\.\d+)?[A-Z]*", v)
399
    return m.group(0) if m else v
400

401

402
def mb(n):
403
    return f"{n / 1e9:.1f} GB" if n >= 1e9 else f"{round(n / 1e6)} MB"
404

405

406
class DeviceUnreachable(Exception):
407
    pass
408

409

410
class AuthFailure(Exception):
411
    pass
412

413

414
def named(cls, message: str):
415
    return cls(f"{cls.__name__}: {message}")
416

417

418
# pyeapi logs a traceback for every socket error, and the post-check polls a switch that is rebooting.
419
logging.getLogger("pyeapi").addHandler(logging.NullHandler())
420

421

422
def eapi(auth: arista_eos, cmds: list, fmt: str = "json", timeout: float = 60):
423
    eos = node(auth, timeout)
424
    try:
425
        return eos.run_commands(cmds, fmt)
426
    except pyeapi.eapilib.ConnectionError as e:
427
        raise connection_error(auth, eos, e)
428

429

430
# pyeapi 1.0.4 raises ConnectionError for a socket error (kept in socket_error), an answer that isn't JSON (eAPI
431
# still starting, 5xx) and a 401, the only one with neither.
432
def connection_error(auth, eos, e):
433
    if eos.connection.socket_error is None and e.message != "unable to connect to eAPI":
434
        return named(
435
            AuthFailure, f"{auth['host']}: eAPI refused the credential ({e.message})"
436
        )
437
    return named(DeviceUnreachable, f"{auth['host']}: {e.message}")
438

439

440
# host is an address or a URL (https://10.0.0.11:8443); a bare IPv6 address is bracketed. Only https, or http when the
441
# resource ticks allow_http (Basic auth then crosses the network in cleartext). pyeapi skips certificate checks unless it
442
# is given a context, so one is always passed: verifying, or not when the resource unticks verify_tls (EOS ships a
443
# self-signed cert).
444
def switch_url(auth: arista_eos):
445
    host = auth["host"]
446
    if "://" not in host:
447
        host = f"https://[{host}]" if host.count(":") > 1 and "[" not in host else f"https://{host}"
448
    url = urlsplit(host)
449
    if not url.hostname or (url.scheme != "https" and not (url.scheme == "http" and auth.get("allow_http"))):
450
        raise ValueError(f"host must be an address or an https:// URL (http:// needs allow_http on the resource): {auth['host']}")
451
    return url
452

453

454
def node(auth: arista_eos, timeout: float = 60):
455
    url = switch_url(auth)
456
    context = ssl.create_default_context()
457
    if auth.get("verify_tls") is False:
458
        context.check_hostname = False
459
        context.verify_mode = ssl.CERT_NONE
460
    return pyeapi.connect(
461
        transport=url.scheme,
462
        host=url.hostname,
463
        port=url.port or auth.get("port") or None,
464
        username=auth["username"],
465
        password=auth["password"],
466
        context=context,
467
        timeout=timeout,
468
        return_node=True,
469
    )
470