0

Device post-check

Published today

One post-upgrade check of one switch against the pre-check snapshot of this flow run: returns { running_version, status ok, checks_ok }. Raises DeviceUnreachable while the switch is down or still on the boot the reload was sent on, NotStable while interfaces/LLDP/MLAG differ from the snapshot (differences in the message), AuthFailure when the credential is refused. Retry it from the workflow; it does not wait.

Script arista_eos Verified

The script

Submitted by hugo989 Python3
Verified 1 day ago
1
# py312
2
# requirements:
3
# pyeapi==1.0.4
4
# wmill==1.851.0
5

6
# Device post-check: one attempt, after an upgrade (or rollback) of one switch. Compares the switch with the snapshot
7
# Device pre-check saved in this flow run, and raises a named error the workflow retries on:
8
# - DeviceUnreachable while the switch is down or still runs the boot a reload of this run was sent on;
9
# - NotStable while the state differs from the snapshot (the differences are in the message; ports stay errdisabled
10
#   until the MLAG reload-delay has run, the message says how much of it is left);
11
# - AuthFailure when the credential is refused (not worth retrying).
12
# No snapshot in this run is a plain error. The version is reported, not compared with the target: the workflow does.
13

14
import logging
15
import os
16
import re
17
import ssl
18
from typing import TypedDict
19
from urllib.parse import urlsplit
20

21
import pyeapi
22
import wmill
23

24

25
class arista_eos(TypedDict):
26
    host: str
27
    username: str
28
    password: str
29
    port: int
30
    verify_tls: bool
31
    allow_http: bool
32

33

34
class NotStable(Exception):
35
    pass
36

37

38
def main(auth: arista_eos):
39
    """
40
    Device post-check
41
    One post-upgrade check of one switch against the pre-check snapshot of this flow run: returns { running_version, status ok, checks_ok }. Raises DeviceUnreachable while the switch is down or still on the boot the reload was sent on, NotStable while interfaces/LLDP/MLAG differ from the snapshot (differences in the message), AuthFailure when the credential is refused. Retry it from the workflow; it does not wait.
42
    """
43
    state = load_state(state_keys(auth)) or {}
44
    # Before probing the switch: without a snapshot, retrying through a reboot would only delay the same error.
45
    if not state.get("snapshot"):
46
        raise RuntimeError(
47
            f"No pre-check snapshot for {auth['host']} in this flow run: run Device pre-check first, in the same run"
48
        )
49
    version = returned(auth, state)
50
    problems = diff(state["snapshot"], snapshot(auth))
51
    if problems:
52
        [mlag] = eapi(auth, ["show mlag"])
53
        left = round(reload_delay(mlag) - version["uptime"])
54
        hint = f" (MLAG reload-delay still running: {left} s left)" if left > 0 else ""
55
        raise NotStable(f"{auth['host']}: {'; '.join(problems)}{hint}")
56
    return {
57
        "running_version": short_version(version["version"]),
58
        "status": "ok",
59
        "checks_ok": ["interface"],
60
    }
61

62

63
# Same function in Get Reload Delay and Device post-check. One attempt: show version, raising DeviceUnreachable
64
# while the switch is down, and while it still runs the boot a reload of this run was sent on (reload now can answer
65
# before EOS goes down, and the old boot must not pass for the new one). AuthFailure is raised as is.
66
def returned(auth, state):
67
    try:
68
        # Short: a switch that is down holds the attempt for the whole timeout, and the workflow retries anyway.
69
        [v] = eapi(auth, ["show version"], timeout=15)
70
    except DeviceUnreachable as e:
71
        raise DeviceUnreachable(f"{e}: not back yet")
72
    for k in ("reload", "rollback"):
73
        m = state.get(k)
74
        if m and m["boot_timestamp"] == v["bootupTimestamp"]:
75
            raise DeviceUnreachable(
76
                f"{auth['host']}: still on the boot the {k} sent at {m['sent_at']} was sent on"
77
            )
78
    return v
79

80

81
# show mlag reports state disabled both when MLAG was never configured and when it is configured but shut down;
82
# only the first has no domain-id, local interface or peer-link.
83
def mlag_configured(mlag):
84
    return any(mlag.get(k) for k in ("domainId", "localInterface", "peerLink"))
85

86

87
def snapshot(auth):
88
    status, lldp, mlag, mlag_intf = eapi(
89
        auth,
90
        [
91
            "show interfaces status",
92
            "show lldp neighbors",
93
            "show mlag",
94
            "show mlag interfaces",
95
        ],
96
    )
97
    return {
98
        "interfaces_up": sorted(
99
            n
100
            for n, i in status["interfaceStatuses"].items()
101
            if i.get("linkStatus") == "connected"
102
        ),
103
        "lldp": [
104
            {
105
                "port": n["port"],
106
                "neighbor": f"{n['neighborDevice']} {n['neighborPort']}",
107
            }
108
            for n in lldp["lldpNeighbors"]
109
        ],
110
        "mlag": {
111
            "state": mlag.get("state"),
112
            "negStatus": mlag.get("negStatus"),
113
            "peerLinkStatus": mlag.get("peerLinkStatus"),
114
            "ports": {
115
                f"{i['localInterface']} (mlag {id})": i["status"]
116
                for id, i in (mlag_intf.get("interfaces") or {}).items()
117
            },
118
        },
119
    }
120

121

122
def diff(before, after):
123
    p = []
124
    for i in before["interfaces_up"]:
125
        if i not in after["interfaces_up"]:
126
            p.append(f"{i} down after the upgrade (up before)")
127
    for n in before["lldp"]:
128
        if not any(
129
            a["port"] == n["port"] and a["neighbor"] == n["neighbor"]
130
            for a in after["lldp"]
131
        ):
132
            p.append(f"LLDP neighbor {n['neighbor']} no longer seen on {n['port']}")
133
    for k in ("state", "negStatus", "peerLinkStatus"):
134
        if before["mlag"][k] != after["mlag"][k]:
135
            p.append(f"MLAG {k} {after['mlag'][k]} (was {before['mlag'][k]})")
136
    for port, s in before["mlag"]["ports"].items():
137
        if after["mlag"]["ports"].get(port) != s:
138
            p.append(f"{port} {after['mlag']['ports'].get(port, 'missing')} (was {s})")
139
    return p
140

141

142
# Without MLAG configured, show mlag still reports the default reload-delay, and nothing waits on it.
143
def reload_delay(mlag):
144
    if not mlag_configured(mlag):
145
        return 0
146
    return max(mlag.get("reloadDelay") or 0, mlag.get("reloadDelayNonMlag") or 0)
147

148

149
# Errors the workflow branches and retries on (retry_if: result.error.name == "DeviceUnreachable").
150
class DeviceUnreachable(Exception):
151
    pass
152

153

154
class AuthFailure(Exception):
155
    pass
156

157

158
# pyeapi logs a traceback for every socket error, and a rebooting switch gives many.
159
logging.getLogger("pyeapi").addHandler(logging.NullHandler())
160

161

162
# This script runs right after a reboot, when eAPI can answer before the EOS agents behind a show command have
163
# started: such an error answer counts as not back yet, so the workflow retries it. It only sends show commands.
164
def eapi(auth: arista_eos, cmds: list, fmt: str = "json", timeout: float = 60):
165
    eos = node(auth, timeout)
166
    try:
167
        return eos.run_commands(cmds, fmt)
168
    except pyeapi.eapilib.ConnectionError as e:
169
        raise connection_error(auth, eos, e)
170
    except pyeapi.eapilib.CommandError as e:
171
        raise DeviceUnreachable(f"{auth['host']}: not ready yet ({e.message})")
172

173

174
# pyeapi 1.0.4 raises ConnectionError for a socket error (kept in socket_error), an answer that isn't JSON (eAPI
175
# still starting, 5xx) and a 401, the only one with neither.
176
def connection_error(auth, eos, e):
177
    if eos.connection.socket_error is None and e.message != "unable to connect to eAPI":
178
        return AuthFailure(f"{auth['host']}: eAPI refused the credential ({e.message})")
179
    return DeviceUnreachable(f"{auth['host']}: {e.message}")
180

181

182
# host is an address or a URL (https://10.0.0.11:8443); a bare IPv6 address is bracketed. Only https, or http when the
183
# resource ticks allow_http (Basic auth then crosses the network in cleartext). pyeapi skips certificate checks unless it
184
# is given a context, so one is always passed: verifying, or not when the resource unticks verify_tls (EOS ships a
185
# self-signed cert).
186
def switch_url(auth: arista_eos):
187
    host = auth["host"]
188
    if "://" not in host:
189
        host = (
190
            f"https://[{host}]"
191
            if host.count(":") > 1 and "[" not in host
192
            else f"https://{host}"
193
        )
194
    url = urlsplit(host)
195
    if not url.hostname or (
196
        url.scheme != "https" and not (url.scheme == "http" and auth.get("allow_http"))
197
    ):
198
        raise ValueError(
199
            f"host must be an address or an https:// URL (http:// needs allow_http on the resource): {auth['host']}"
200
        )
201
    return url
202

203

204
def node(auth: arista_eos, timeout: float = 60):
205
    url = switch_url(auth)
206
    context = ssl.create_default_context()
207
    if auth.get("verify_tls") is False:
208
        context.check_hostname = False
209
        context.verify_mode = ssl.CERT_NONE
210
    return pyeapi.connect(
211
        transport=url.scheme,
212
        host=url.hostname,
213
        port=url.port or auth.get("port") or None,
214
        username=auth["username"],
215
        password=auth["password"],
216
        context=context,
217
        timeout=timeout,
218
        return_node=True,
219
    )
220

221

222
# Shared by the Device scripts: the root flow run's user state, keyed by switch, holds the pre-check snapshot, the
223
# image activate replaced (rollback target), the image it booted and the reload markers. Every step of the run (nested
224
# subflows and retried attempts included) reads the same entry. Outside a flow the Device scripts share one state
225
# resource in the running user's home (a script's own state would keep the pre-check snapshot from the post-check).
226
# The endpoint is called through the client so an unreachable state raises (wmill.get_flow_user_state returns None).
227
# ponytail: the fallback is one map per user, so concurrent standalone runs can race; flows don't.
228
# The key is the switch's eAPI address: arista_eos-<host>-<port> with non-word characters in the host as _, the port
229
# always in (443 or 80 when unset). State written before the port was part of the key is read from arista_eos_<host>
230
# as a fallback (all word characters, so never a new key); writes go to the new key only.
231
def state_keys(auth: arista_eos):
232
    url = switch_url(auth)
233
    port = int(url.port or auth.get("port") or {"https": 443, "http": 80}[url.scheme])
234
    return [
235
        f"arista_eos-{word(url.hostname)}-{port}",
236
        f"arista_eos_{word(auth['host'])}",
237
    ]
238

239

240
def word(s):
241
    return re.sub(r"\W", "_", s)
242

243

244
def load_state(keys):
245
    for key in keys:
246
        if os.environ.get("WM_FLOW_JOB_ID"):
247
            c = wmill.Windmill()
248
            value = c.get(
249
                f"/w/{c.workspace}/jobs/flow/user_states/{c.get_root_job_id()}/{key}"
250
            ).json()
251
        else:
252
            value = (wmill.get_state(standalone_state()) or {}).get(key)
253
        if value is not None:
254
            return value
255

256

257
def save_state(keys, value):
258
    if os.environ.get("WM_FLOW_JOB_ID"):
259
        c = wmill.Windmill()
260
        c.post(
261
            f"/w/{c.workspace}/jobs/flow/user_states/{c.get_root_job_id()}/{keys[0]}",
262
            json=value,
263
        )
264
    else:
265
        path = standalone_state()
266
        wmill.set_state({**(wmill.get_state(path) or {}), keys[0]: value}, path)
267

268

269
def standalone_state():
270
    return f"u/{os.environ['WM_USERNAME']}/arista_eos_state"
271

272

273
# "4.34.9M-51132493.4349M (engineering build)" on cEOS, "4.34.9M" on hardware.
274
def short_version(v):
275
    m = re.match(r"\d+\.\d+\.\d+(\.\d+)?[A-Z]*", v)
276
    return m.group(0) if m else v
277