0

Device rollback

Published today

Boot the switch back to its previous image and reload, exactly once per flow run (a retry skips it). Returns { status: done, reloaded, reason, boot_image, steps } (same shape when skipped). Raises RollbackFailed, DeviceUnreachable or AuthFailure. Follow it with Get Reload Delay.

Script arista_eos Verified

The script

Submitted by hugo989 Python3
Verified 1 day ago
1
# py312
2
# requirements:
3
# pyeapi==1.0.4
4
# wmill==1.851.0
5

6
# Device rollback: boot system back to the image Device activate recorded as the previous one (or, in a new run, the
7
# only other EOS image on flash), then reload now behind the same once-per-run guard as Device reload. It is the
8
# recovery path: no unsaved-config check. A boot that this run's reload caused doesn't count against
9
# min_uptime_seconds. A switch that doesn't answer raises DeviceUnreachable (retry it); anything else RollbackFailed.
10

11
import logging
12
import os
13
import re
14
import ssl
15
import time
16
from typing import TypedDict
17
from urllib.parse import urlsplit
18

19
import pyeapi
20
import wmill
21

22

23
class arista_eos(TypedDict):
24
    host: str
25
    username: str
26
    password: str
27
    port: int
28
    verify_tls: bool
29
    allow_http: bool
30

31

32
class RollbackFailed(Exception):
33
    pass
34

35

36
def main(auth: arista_eos, min_uptime_seconds: int = 300):
37
    """
38
    Device rollback
39
    Boot the switch back to its previous image and reload, exactly once per flow run (a retry skips it). Returns { status: done, reloaded, reason, boot_image, steps } (same shape when skipped). Raises RollbackFailed, DeviceUnreachable or AuthFailure. Follow it with Get Reload Delay.
40
    """
41
    key = state_keys(auth)
42
    state = load_state(key) or {}
43
    min_uptime = 300 if min_uptime_seconds is None else min_uptime_seconds
44
    sw = read_switch(auth, state, min_uptime)
45
    sent = sw["marker"]("rollback")
46
    if sw.get("unreachable"):
47
        if sent:
48
            return skip(
49
                f"rollback sent at {sent['sent_at']}; the switch no longer answers: it is reloading",
50
                state.get("previous_image"),
51
            )
52
        raise DeviceUnreachable(sw["unreachable"])
53
    if sent and (sent["confirmed"] or sent["boot_timestamp"] != sw["boot"]):
54
        return skip(guard_reason("rollback", sent, sw), state.get("previous_image"))
55
    try:
56
        d, boot = eapi(
57
            auth, [{"cmd": "dir flash:/", "revision": 2}, "show boot-config"]
58
        )
59
        files = list(d["urls"]["flash:/"]["entries"])
60
        current = boot["softwareImage"].replace("flash:/", "")
61
        # Without a recorded previous image (new run), the only other EOS image on flash is unambiguous.
62
        others = [f for f in files if re.match(r"^EOS.*\.swi$", f) and f != current]
63
        previous = (state.get("previous_image") or "").replace("flash:/", "") or (
64
            others[0] if len(others) == 1 else ""
65
        )
66
        if not previous:
67
            raise RollbackFailed(
68
                f"previous image unknown (EOS images on flash: {', '.join(others) or 'none'})"
69
            )
70
        if previous not in files:
71
            raise RollbackFailed(f"previous image {previous} is no longer on flash")
72
        reload = sw["marker"]("reload")
73
        if reload and reload["confirmed"] and reload["boot_timestamp"] == sw["boot"]:
74
            raise RollbackFailed(
75
                f"the reload sent at {reload['sent_at']} has not happened yet (same boot as when it was sent): rolling back now would race it, check the switch"
76
            )
77
        steps = boot_system(auth, previous)
78
        forget_activated(key, state)
79
        if sw["running"] == image_version(previous):
80
            return skip(
81
                f"already running the previous image {previous}; boot-config set back to it",
82
                f"flash:/{previous}",
83
                steps,
84
            )
85
        # A recent boot that this run's reload caused doesn't block the rollback: the post-check that ran after it
86
        # can end well inside min_uptime_seconds.
87
        ours = reload and reload["boot_timestamp"] != sw["boot"]
88
        if not ours and sw["uptime"] < min_uptime:
89
            return skip(
90
                f"rebooted {round(sw['uptime'])} s ago, not by this run",
91
                f"flash:/{previous}",
92
                steps,
93
            )
94
        fire_reload(auth, key, "rollback", sw["boot"])
95
    except (DeviceUnreachable, AuthFailure, RollbackFailed):
96
        raise
97
    except Exception as e:
98
        raise RollbackFailed(f"{auth['host']}: {e}")
99
    return {
100
        "status": "done",
101
        "reloaded": True,
102
        "reason": None,
103
        "boot_image": f"flash:/{previous}",
104
        "steps": steps,
105
    }
106

107

108
def boot_system(auth, image):
109
    _, set_, _, after = eapi(
110
        auth, ["configure", f"boot system flash:{image}", "end", "show boot-config"]
111
    )
112
    if after["softwareImage"] != f"flash:/{image}":
113
        raise Exception(f"boot-config still points to {after['softwareImage']}")
114
    return [f"boot system flash:{image}", *(set_.get("warnings") or [])]
115

116

117
def forget_activated(key, state):
118
    if state.pop("activated_image", None):
119
        save_state(key, state)
120

121

122
# Every return has the same shape. boot_image is the image rollback boots (or set boot-config back to); before flash
123
# is read it is the recorded previous image, which is null in a new run.
124
def skip(reason, boot_image=None, steps=()):
125
    return {
126
        "status": "done",
127
        "reloaded": False,
128
        "reason": reason,
129
        "boot_image": boot_image,
130
        "steps": list(steps),
131
    }
132

133

134
def guard_reason(kind, m, sw):
135
    if m["boot_timestamp"] == sw["boot"]:
136
        return f"{kind} sent at {m['sent_at']}; the switch is going down"
137
    return f"{kind} sent at {m['sent_at']} already rebooted the switch"
138

139

140
# A reload is sent at most once per switch and flow run. Each one leaves a marker { sent_at, boot_timestamp,
141
# confirmed }: boot_timestamp is the boot it was sent on, so a later boot means it happened, and confirmed means
142
# the command reached the switch. An unconfirmed marker on the same boot (the worker died first) is sent again.
143
# Outside a flow, a marker is dropped once the reboot it caused is min_uptime_seconds old.
144
def read_switch(auth, state, min_uptime):
145
    try:
146
        version, boot = eapi(auth, ["show version", "show boot-config"])
147
    except DeviceUnreachable as e:
148
        return {"unreachable": str(e), "marker": lambda k: state.get(k)}
149
    sw = {
150
        "running": short_version(version["version"]),
151
        "image": boot["softwareImage"],
152
        "boot": version["bootupTimestamp"],
153
        "uptime": version["uptime"],
154
    }
155
    in_flow = bool(os.environ.get("WM_FLOW_JOB_ID"))
156

157
    def marker(k):
158
        m = state.get(k)
159
        return (
160
            m
161
            if m
162
            and (
163
                in_flow
164
                or m["boot_timestamp"] == sw["boot"]
165
                or sw["uptime"] < min_uptime
166
            )
167
            else None
168
        )
169

170
    return {**sw, "marker": marker}
171

172

173
# Fire and forget: the switch drops the connection while going down, which is not an error. A connection that
174
# never opened, or a refusal from the switch, means it did not reload: the marker goes and the error is raised.
175
def fire_reload(auth, key, kind, boot_ts):
176
    def mark(m):
177
        rest = {k: v for k, v in (load_state(key) or {}).items() if k != kind}
178
        save_state(key, {**rest, kind: m} if m else rest)
179

180
    m = {
181
        "sent_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
182
        "boot_timestamp": boot_ts,
183
        "confirmed": False,
184
    }
185
    mark(m)
186
    eos = node(auth, 15)
187
    try:
188
        # Opened before sending, so a failure here means the command never left.
189
        eos.connection.transport.connect()
190
    except OSError as e:
191
        mark(None)
192
        raise DeviceUnreachable(f"{auth['host']}: {e}")
193
    try:
194
        eos.run_commands(["reload now"])
195
    except pyeapi.eapilib.ConnectionError as e:
196
        # Sent, then the connection dropped or timed out as the switch went down. A 401 is a refusal.
197
        err = connection_error(auth, eos, e)
198
        if isinstance(err, AuthFailure):
199
            mark(None)
200
            raise err
201
    except Exception:
202
        mark(None)
203
        raise
204
    mark({**m, "confirmed": True})
205

206

207
# "EOS64-4.34.9M.swi" -> "4.34.9M"
208
def image_version(file):
209
    m = re.search(r"-(\d+\.\d+\.\d+(\.\d+)?[A-Z]*)\.swi$", file)
210
    return m.group(1) if m else None
211

212

213
# Errors the workflow branches and retries on (retry_if: result.error.name == "DeviceUnreachable").
214
class DeviceUnreachable(Exception):
215
    pass
216

217

218
class AuthFailure(Exception):
219
    pass
220

221

222
# pyeapi logs a traceback for every socket error, and a rebooting switch gives many.
223
logging.getLogger("pyeapi").addHandler(logging.NullHandler())
224

225

226
def eapi(auth: arista_eos, cmds: list, fmt: str = "json", timeout: float = 60):
227
    eos = node(auth, timeout)
228
    try:
229
        return eos.run_commands(cmds, fmt)
230
    except pyeapi.eapilib.ConnectionError as e:
231
        raise connection_error(auth, eos, e)
232

233

234
# pyeapi 1.0.4 raises ConnectionError for a socket error (kept in socket_error), an answer that isn't JSON (eAPI
235
# still starting, 5xx) and a 401, the only one with neither.
236
def connection_error(auth, eos, e):
237
    if eos.connection.socket_error is None and e.message != "unable to connect to eAPI":
238
        return AuthFailure(f"{auth['host']}: eAPI refused the credential ({e.message})")
239
    return DeviceUnreachable(f"{auth['host']}: {e.message}")
240

241

242
# host is an address or a URL (https://10.0.0.11:8443); a bare IPv6 address is bracketed. Only https, or http when the
243
# resource ticks allow_http (Basic auth then crosses the network in cleartext). pyeapi skips certificate checks unless it
244
# is given a context, so one is always passed: verifying, or not when the resource unticks verify_tls (EOS ships a
245
# self-signed cert).
246
def switch_url(auth: arista_eos):
247
    host = auth["host"]
248
    if "://" not in host:
249
        host = (
250
            f"https://[{host}]"
251
            if host.count(":") > 1 and "[" not in host
252
            else f"https://{host}"
253
        )
254
    url = urlsplit(host)
255
    if not url.hostname or (
256
        url.scheme != "https" and not (url.scheme == "http" and auth.get("allow_http"))
257
    ):
258
        raise ValueError(
259
            f"host must be an address or an https:// URL (http:// needs allow_http on the resource): {auth['host']}"
260
        )
261
    return url
262

263

264
def node(auth: arista_eos, timeout: float = 60):
265
    url = switch_url(auth)
266
    context = ssl.create_default_context()
267
    if auth.get("verify_tls") is False:
268
        context.check_hostname = False
269
        context.verify_mode = ssl.CERT_NONE
270
    return pyeapi.connect(
271
        transport=url.scheme,
272
        host=url.hostname,
273
        port=url.port or auth.get("port") or None,
274
        username=auth["username"],
275
        password=auth["password"],
276
        context=context,
277
        timeout=timeout,
278
        return_node=True,
279
    )
280

281

282
# Shared by the Device scripts: the root flow run's user state, keyed by switch, holds the pre-check snapshot, the
283
# image activate replaced (rollback target), the image it booted and the reload markers. Every step of the run (nested
284
# subflows and retried attempts included) reads the same entry. Outside a flow the Device scripts share one state
285
# resource in the running user's home (a script's own state would keep the pre-check snapshot from the post-check).
286
# The endpoint is called through the client so an unreachable state raises (wmill.get_flow_user_state returns None).
287
# ponytail: the fallback is one map per user, so concurrent standalone runs can race; flows don't.
288
# The key is the switch's eAPI address: arista_eos-<host>-<port> with non-word characters in the host as _, the port
289
# always in (443 or 80 when unset). State written before the port was part of the key is read from arista_eos_<host>
290
# as a fallback (all word characters, so never a new key); writes go to the new key only.
291
def state_keys(auth: arista_eos):
292
    url = switch_url(auth)
293
    port = int(url.port or auth.get("port") or {"https": 443, "http": 80}[url.scheme])
294
    return [
295
        f"arista_eos-{word(url.hostname)}-{port}",
296
        f"arista_eos_{word(auth['host'])}",
297
    ]
298

299

300
def word(s):
301
    return re.sub(r"\W", "_", s)
302

303

304
def load_state(keys):
305
    for key in keys:
306
        if os.environ.get("WM_FLOW_JOB_ID"):
307
            c = wmill.Windmill()
308
            value = c.get(
309
                f"/w/{c.workspace}/jobs/flow/user_states/{c.get_root_job_id()}/{key}"
310
            ).json()
311
        else:
312
            value = (wmill.get_state(standalone_state()) or {}).get(key)
313
        if value is not None:
314
            return value
315

316

317
def save_state(keys, value):
318
    if os.environ.get("WM_FLOW_JOB_ID"):
319
        c = wmill.Windmill()
320
        c.post(
321
            f"/w/{c.workspace}/jobs/flow/user_states/{c.get_root_job_id()}/{keys[0]}",
322
            json=value,
323
        )
324
    else:
325
        path = standalone_state()
326
        wmill.set_state({**(wmill.get_state(path) or {}), keys[0]: value}, path)
327

328

329
def standalone_state():
330
    return f"u/{os.environ['WM_USERNAME']}/arista_eos_state"
331

332

333
# "4.34.9M-51132493.4349M (engineering build)" on cEOS, "4.34.9M" on hardware.
334
def short_version(v):
335
    m = re.match(r"\d+\.\d+\.\d+(\.\d+)?[A-Z]*", v)
336
    return m.group(0) if m else v
337