0

Device reload

Published today

Reload the switch onto the image Device activate set: exactly once per flow run (a retry skips it), refused with unsaved config unless allow_unsaved. Returns { status: done, reloaded, reason }. Raises ReloadRefused (unsaved config, boot-config not the activated image), DeviceUnreachable or AuthFailure. Follow it with Get Reload Delay.

Script arista_eos Verified

The script

Submitted by hugo989 Python3
Verified 1 day ago
1
# py312
2
# requirements:
3
# pyeapi==1.0.4
4
# wmill==1.851.0
5

6
# Device reload: reload now, at most once per switch and flow run, so a retried step never reboots it twice.
7
# Skipped (reloaded false, with the reason) when this run already sent it, the switch already runs target_version, or
8
# it rebooted less than min_uptime_seconds ago. boot-config must be the image Device activate set in this run, or
9
# flash:/EOS-<target_version>.swi. ReloadRefused while show running-config diffs changes anything but its ! headers,
10
# or there is no startup-config (reload now doesn't prompt, so unsaved changes would be lost), unless allow_unsaved.
11
# Reload markers are the root flow run's user state under arista_eos-<host>-<port>; Get Reload Delay and Device
12
# post-check read them to tell the new boot from the old one.
13

14
import logging
15
import os
16
import re
17
import ssl
18
import time
19
from typing import TypedDict
20
from urllib.parse import urlsplit
21

22
import pyeapi
23
import wmill
24

25

26
class arista_eos(TypedDict):
27
    host: str
28
    username: str
29
    password: str
30
    port: int
31
    verify_tls: bool
32
    allow_http: bool
33

34

35
class ReloadRefused(Exception):
36
    pass
37

38

39
def main(
40
    auth: arista_eos,
41
    target_version: str,
42
    min_uptime_seconds: int = 300,
43
    allow_unsaved: bool = False,
44
):
45
    """
46
    Device reload
47
    Reload the switch onto the image Device activate set: exactly once per flow run (a retry skips it), refused with unsaved config unless allow_unsaved. Returns { status: done, reloaded, reason }. Raises ReloadRefused (unsaved config, boot-config not the activated image), DeviceUnreachable or AuthFailure. Follow it with Get Reload Delay.
48
    """
49
    key = state_keys(auth)
50
    state = load_state(key) or {}
51
    min_uptime = 300 if min_uptime_seconds is None else min_uptime_seconds
52
    if not target_version:
53
        raise ReloadRefused("target_version is required for the reload guard")
54
    sw = read_switch(auth, state, min_uptime)
55
    sent = sw["marker"]("reload")
56
    if sw.get("unreachable"):
57
        if sent:
58
            return skip(
59
                f"reload sent at {sent['sent_at']}; the switch no longer answers: it is reloading"
60
            )
61
        raise DeviceUnreachable(sw["unreachable"])
62
    if sent and (sent["confirmed"] or sent["boot_timestamp"] != sw["boot"]):
63
        return skip(guard_reason("reload", sent, sw))
64
    if sw["running"] == target_version:
65
        return skip(f"already running {target_version}")
66
    # The image activate set in this run (state), or exactly EOS-<target_version>.swi.
67
    activated = state.get("activated_image")
68
    if sw["image"] not in (activated, f"flash:/EOS-{target_version}.swi"):
69
        raise ReloadRefused(
70
            f"boot-config points to {sw['image']}, neither the image activate set ({activated or 'none in this run'}) nor flash:/EOS-{target_version}.swi: run Device activate first"
71
        )
72
    if sw["uptime"] < min_uptime:
73
        return skip(f"rebooted {round(sw['uptime'])} s ago")
74
    if not allow_unsaved and unsaved(auth):
75
        raise ReloadRefused(
76
            "unsaved config: run Save Config first, or pass allow_unsaved: true"
77
        )
78
    fire_reload(auth, key, "reload", sw["boot"])
79
    return {"status": "done", "reloaded": True}
80

81

82
# reload now doesn't prompt to save. show running-config diffs is a unified diff of startup-config against
83
# running-config, empty when they match. Its two file-header lines come before the first @@. Changed lines that
84
# start with a single "!" are headers EOS writes itself (boot system shows there, so every activate changes one);
85
# "!!" comments are config. With no startup-config every line shows as added: unsaved, it would boot into ZTP.
86
# A refused command (AAA authorization, an older EOS) is not unsaved config: saving wouldn't clear it.
87
def unsaved(auth):
88
    try:
89
        [d] = eapi(auth, ["show running-config diffs"], "text")
90
    except pyeapi.eapilib.CommandError as e:
91
        raise ReloadRefused(f"cannot compare running and startup config: {e}")
92
    hunks, changed = False, []
93
    for l in d["output"].splitlines():
94
        hunks = hunks or l.startswith("@@")
95
        if hunks and l[:1] in ("+", "-") and l[1:].strip():
96
            changed.append(l[1:].strip())
97
    return any(not re.match(r"!(?!!)", l) for l in changed)
98

99

100
def skip(reason):
101
    return {"status": "done", "reloaded": False, "reason": reason}
102

103

104
def guard_reason(kind, m, sw):
105
    if m["boot_timestamp"] == sw["boot"]:
106
        return f"{kind} sent at {m['sent_at']}; the switch is going down"
107
    return f"{kind} sent at {m['sent_at']} already rebooted the switch"
108

109

110
# A reload is sent at most once per switch and flow run. Each one leaves a marker { sent_at, boot_timestamp,
111
# confirmed }: boot_timestamp is the boot it was sent on, so a later boot means it happened, and confirmed means
112
# the command reached the switch. An unconfirmed marker on the same boot (the worker died first) is sent again.
113
# Outside a flow, a marker is dropped once the reboot it caused is min_uptime_seconds old.
114
def read_switch(auth, state, min_uptime):
115
    try:
116
        version, boot = eapi(auth, ["show version", "show boot-config"])
117
    except DeviceUnreachable as e:
118
        return {"unreachable": str(e), "marker": lambda k: state.get(k)}
119
    sw = {
120
        "running": short_version(version["version"]),
121
        "image": boot["softwareImage"],
122
        "boot": version["bootupTimestamp"],
123
        "uptime": version["uptime"],
124
    }
125
    in_flow = bool(os.environ.get("WM_FLOW_JOB_ID"))
126

127
    def marker(k):
128
        m = state.get(k)
129
        return (
130
            m
131
            if m
132
            and (
133
                in_flow
134
                or m["boot_timestamp"] == sw["boot"]
135
                or sw["uptime"] < min_uptime
136
            )
137
            else None
138
        )
139

140
    return {**sw, "marker": marker}
141

142

143
# Fire and forget: the switch drops the connection while going down, which is not an error. A connection that
144
# never opened, or a refusal from the switch, means it did not reload: the marker goes and the error is raised.
145
def fire_reload(auth, key, kind, boot_ts):
146
    def mark(m):
147
        rest = {k: v for k, v in (load_state(key) or {}).items() if k != kind}
148
        save_state(key, {**rest, kind: m} if m else rest)
149

150
    m = {
151
        "sent_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
152
        "boot_timestamp": boot_ts,
153
        "confirmed": False,
154
    }
155
    mark(m)
156
    eos = node(auth, 15)
157
    try:
158
        # Opened before sending, so a failure here means the command never left.
159
        eos.connection.transport.connect()
160
    except OSError as e:
161
        mark(None)
162
        raise DeviceUnreachable(f"{auth['host']}: {e}")
163
    try:
164
        eos.run_commands(["reload now"])
165
    except pyeapi.eapilib.ConnectionError as e:
166
        # Sent, then the connection dropped or timed out as the switch went down. A 401 is a refusal.
167
        err = connection_error(auth, eos, e)
168
        if isinstance(err, AuthFailure):
169
            mark(None)
170
            raise err
171
    except Exception:
172
        mark(None)
173
        raise
174
    mark({**m, "confirmed": True})
175

176

177
# Errors the workflow branches and retries on (retry_if: result.error.name == "DeviceUnreachable").
178
class DeviceUnreachable(Exception):
179
    pass
180

181

182
class AuthFailure(Exception):
183
    pass
184

185

186
# pyeapi logs a traceback for every socket error, and a rebooting switch gives many.
187
logging.getLogger("pyeapi").addHandler(logging.NullHandler())
188

189

190
def eapi(auth: arista_eos, cmds: list, fmt: str = "json", timeout: float = 60):
191
    eos = node(auth, timeout)
192
    try:
193
        return eos.run_commands(cmds, fmt)
194
    except pyeapi.eapilib.ConnectionError as e:
195
        raise connection_error(auth, eos, e)
196

197

198
# pyeapi 1.0.4 raises ConnectionError for a socket error (kept in socket_error), an answer that isn't JSON (eAPI
199
# still starting, 5xx) and a 401, the only one with neither.
200
def connection_error(auth, eos, e):
201
    if eos.connection.socket_error is None and e.message != "unable to connect to eAPI":
202
        return AuthFailure(f"{auth['host']}: eAPI refused the credential ({e.message})")
203
    return DeviceUnreachable(f"{auth['host']}: {e.message}")
204

205

206
# host is an address or a URL (https://10.0.0.11:8443); a bare IPv6 address is bracketed. Only https, or http when the
207
# resource ticks allow_http (Basic auth then crosses the network in cleartext). pyeapi skips certificate checks unless it
208
# is given a context, so one is always passed: verifying, or not when the resource unticks verify_tls (EOS ships a
209
# self-signed cert).
210
def switch_url(auth: arista_eos):
211
    host = auth["host"]
212
    if "://" not in host:
213
        host = (
214
            f"https://[{host}]"
215
            if host.count(":") > 1 and "[" not in host
216
            else f"https://{host}"
217
        )
218
    url = urlsplit(host)
219
    if not url.hostname or (
220
        url.scheme != "https" and not (url.scheme == "http" and auth.get("allow_http"))
221
    ):
222
        raise ValueError(
223
            f"host must be an address or an https:// URL (http:// needs allow_http on the resource): {auth['host']}"
224
        )
225
    return url
226

227

228
def node(auth: arista_eos, timeout: float = 60):
229
    url = switch_url(auth)
230
    context = ssl.create_default_context()
231
    if auth.get("verify_tls") is False:
232
        context.check_hostname = False
233
        context.verify_mode = ssl.CERT_NONE
234
    return pyeapi.connect(
235
        transport=url.scheme,
236
        host=url.hostname,
237
        port=url.port or auth.get("port") or None,
238
        username=auth["username"],
239
        password=auth["password"],
240
        context=context,
241
        timeout=timeout,
242
        return_node=True,
243
    )
244

245

246
# Shared by the Device scripts: the root flow run's user state, keyed by switch, holds the pre-check snapshot, the
247
# image activate replaced (rollback target), the image it booted and the reload markers. Every step of the run (nested
248
# subflows and retried attempts included) reads the same entry. Outside a flow the Device scripts share one state
249
# resource in the running user's home (a script's own state would keep the pre-check snapshot from the post-check).
250
# The endpoint is called through the client so an unreachable state raises (wmill.get_flow_user_state returns None).
251
# ponytail: the fallback is one map per user, so concurrent standalone runs can race; flows don't.
252
# The key is the switch's eAPI address: arista_eos-<host>-<port> with non-word characters in the host as _, the port
253
# always in (443 or 80 when unset). State written before the port was part of the key is read from arista_eos_<host>
254
# as a fallback (all word characters, so never a new key); writes go to the new key only.
255
def state_keys(auth: arista_eos):
256
    url = switch_url(auth)
257
    port = int(url.port or auth.get("port") or {"https": 443, "http": 80}[url.scheme])
258
    return [
259
        f"arista_eos-{word(url.hostname)}-{port}",
260
        f"arista_eos_{word(auth['host'])}",
261
    ]
262

263

264
def word(s):
265
    return re.sub(r"\W", "_", s)
266

267

268
def load_state(keys):
269
    for key in keys:
270
        if os.environ.get("WM_FLOW_JOB_ID"):
271
            c = wmill.Windmill()
272
            value = c.get(
273
                f"/w/{c.workspace}/jobs/flow/user_states/{c.get_root_job_id()}/{key}"
274
            ).json()
275
        else:
276
            value = (wmill.get_state(standalone_state()) or {}).get(key)
277
        if value is not None:
278
            return value
279

280

281
def save_state(keys, value):
282
    if os.environ.get("WM_FLOW_JOB_ID"):
283
        c = wmill.Windmill()
284
        c.post(
285
            f"/w/{c.workspace}/jobs/flow/user_states/{c.get_root_job_id()}/{keys[0]}",
286
            json=value,
287
        )
288
    else:
289
        path = standalone_state()
290
        wmill.set_state({**(wmill.get_state(path) or {}), keys[0]: value}, path)
291

292

293
def standalone_state():
294
    return f"u/{os.environ['WM_USERNAME']}/arista_eos_state"
295

296

297
# "4.34.9M-51132493.4349M (engineering build)" on cEOS, "4.34.9M" on hardware.
298
def short_version(v):
299
    m = re.match(r"\d+\.\d+\.\d+(\.\d+)?[A-Z]*", v)
300
    return m.group(0) if m else v
301