1 | # py312 |
2 | # requirements: |
3 | # pyeapi==1.0.4 |
4 | # wmill==1.851.0 |
5 |
|
6 | # Device rollback: boot system back to the image Device activate recorded as the previous one (or, in a new run, the |
7 | # only other EOS image on flash), then reload now behind the same once-per-run guard as Device reload. It is the |
8 | # recovery path: no unsaved-config check. A boot that this run's reload caused doesn't count against |
9 | # min_uptime_seconds. A switch that doesn't answer raises DeviceUnreachable (retry it); anything else RollbackFailed. |
10 |
|
11 | import logging |
12 | import os |
13 | import re |
14 | import ssl |
15 | import time |
16 | from typing import TypedDict |
17 | from urllib.parse import urlsplit |
18 |
|
19 | import pyeapi |
20 | import wmill |
21 |
|
22 |
|
23 | class arista_eos(TypedDict): |
24 | host: str |
25 | username: str |
26 | password: str |
27 | port: int |
28 | verify_tls: bool |
29 | allow_http: bool |
30 |
|
31 |
|
32 | class RollbackFailed(Exception): |
33 | pass |
34 |
|
35 |
|
36 | def main(auth: arista_eos, min_uptime_seconds: int = 300): |
37 | """ |
38 | Device rollback |
39 | Boot the switch back to its previous image and reload, exactly once per flow run (a retry skips it). Returns { status: done, reloaded, reason, boot_image, steps } (same shape when skipped). Raises RollbackFailed, DeviceUnreachable or AuthFailure. Follow it with Get Reload Delay. |
40 | """ |
41 | key = state_keys(auth) |
42 | state = load_state(key) or {} |
43 | min_uptime = 300 if min_uptime_seconds is None else min_uptime_seconds |
44 | sw = read_switch(auth, state, min_uptime) |
45 | sent = sw["marker"]("rollback") |
46 | if sw.get("unreachable"): |
47 | if sent: |
48 | return skip( |
49 | f"rollback sent at {sent['sent_at']}; the switch no longer answers: it is reloading", |
50 | state.get("previous_image"), |
51 | ) |
52 | raise DeviceUnreachable(sw["unreachable"]) |
53 | if sent and (sent["confirmed"] or sent["boot_timestamp"] != sw["boot"]): |
54 | return skip(guard_reason("rollback", sent, sw), state.get("previous_image")) |
55 | try: |
56 | d, boot = eapi( |
57 | auth, [{"cmd": "dir flash:/", "revision": 2}, "show boot-config"] |
58 | ) |
59 | files = list(d["urls"]["flash:/"]["entries"]) |
60 | current = boot["softwareImage"].replace("flash:/", "") |
61 | # Without a recorded previous image (new run), the only other EOS image on flash is unambiguous. |
62 | others = [f for f in files if re.match(r"^EOS.*\.swi$", f) and f != current] |
63 | previous = (state.get("previous_image") or "").replace("flash:/", "") or ( |
64 | others[0] if len(others) == 1 else "" |
65 | ) |
66 | if not previous: |
67 | raise RollbackFailed( |
68 | f"previous image unknown (EOS images on flash: {', '.join(others) or 'none'})" |
69 | ) |
70 | if previous not in files: |
71 | raise RollbackFailed(f"previous image {previous} is no longer on flash") |
72 | reload = sw["marker"]("reload") |
73 | if reload and reload["confirmed"] and reload["boot_timestamp"] == sw["boot"]: |
74 | raise RollbackFailed( |
75 | f"the reload sent at {reload['sent_at']} has not happened yet (same boot as when it was sent): rolling back now would race it, check the switch" |
76 | ) |
77 | steps = boot_system(auth, previous) |
78 | forget_activated(key, state) |
79 | if sw["running"] == image_version(previous): |
80 | return skip( |
81 | f"already running the previous image {previous}; boot-config set back to it", |
82 | f"flash:/{previous}", |
83 | steps, |
84 | ) |
85 | # A recent boot that this run's reload caused doesn't block the rollback: the post-check that ran after it |
86 | # can end well inside min_uptime_seconds. |
87 | ours = reload and reload["boot_timestamp"] != sw["boot"] |
88 | if not ours and sw["uptime"] < min_uptime: |
89 | return skip( |
90 | f"rebooted {round(sw['uptime'])} s ago, not by this run", |
91 | f"flash:/{previous}", |
92 | steps, |
93 | ) |
94 | fire_reload(auth, key, "rollback", sw["boot"]) |
95 | except (DeviceUnreachable, AuthFailure, RollbackFailed): |
96 | raise |
97 | except Exception as e: |
98 | raise RollbackFailed(f"{auth['host']}: {e}") |
99 | return { |
100 | "status": "done", |
101 | "reloaded": True, |
102 | "reason": None, |
103 | "boot_image": f"flash:/{previous}", |
104 | "steps": steps, |
105 | } |
106 |
|
107 |
|
108 | def boot_system(auth, image): |
109 | _, set_, _, after = eapi( |
110 | auth, ["configure", f"boot system flash:{image}", "end", "show boot-config"] |
111 | ) |
112 | if after["softwareImage"] != f"flash:/{image}": |
113 | raise Exception(f"boot-config still points to {after['softwareImage']}") |
114 | return [f"boot system flash:{image}", *(set_.get("warnings") or [])] |
115 |
|
116 |
|
117 | def forget_activated(key, state): |
118 | if state.pop("activated_image", None): |
119 | save_state(key, state) |
120 |
|
121 |
|
122 | # Every return has the same shape. boot_image is the image rollback boots (or set boot-config back to); before flash |
123 | # is read it is the recorded previous image, which is null in a new run. |
124 | def skip(reason, boot_image=None, steps=()): |
125 | return { |
126 | "status": "done", |
127 | "reloaded": False, |
128 | "reason": reason, |
129 | "boot_image": boot_image, |
130 | "steps": list(steps), |
131 | } |
132 |
|
133 |
|
134 | def guard_reason(kind, m, sw): |
135 | if m["boot_timestamp"] == sw["boot"]: |
136 | return f"{kind} sent at {m['sent_at']}; the switch is going down" |
137 | return f"{kind} sent at {m['sent_at']} already rebooted the switch" |
138 |
|
139 |
|
140 | # A reload is sent at most once per switch and flow run. Each one leaves a marker { sent_at, boot_timestamp, |
141 | # confirmed }: boot_timestamp is the boot it was sent on, so a later boot means it happened, and confirmed means |
142 | # the command reached the switch. An unconfirmed marker on the same boot (the worker died first) is sent again. |
143 | # Outside a flow, a marker is dropped once the reboot it caused is min_uptime_seconds old. |
144 | def read_switch(auth, state, min_uptime): |
145 | try: |
146 | version, boot = eapi(auth, ["show version", "show boot-config"]) |
147 | except DeviceUnreachable as e: |
148 | return {"unreachable": str(e), "marker": lambda k: state.get(k)} |
149 | sw = { |
150 | "running": short_version(version["version"]), |
151 | "image": boot["softwareImage"], |
152 | "boot": version["bootupTimestamp"], |
153 | "uptime": version["uptime"], |
154 | } |
155 | in_flow = bool(os.environ.get("WM_FLOW_JOB_ID")) |
156 |
|
157 | def marker(k): |
158 | m = state.get(k) |
159 | return ( |
160 | m |
161 | if m |
162 | and ( |
163 | in_flow |
164 | or m["boot_timestamp"] == sw["boot"] |
165 | or sw["uptime"] < min_uptime |
166 | ) |
167 | else None |
168 | ) |
169 |
|
170 | return {**sw, "marker": marker} |
171 |
|
172 |
|
173 | # Fire and forget: the switch drops the connection while going down, which is not an error. A connection that |
174 | # never opened, or a refusal from the switch, means it did not reload: the marker goes and the error is raised. |
175 | def fire_reload(auth, key, kind, boot_ts): |
176 | def mark(m): |
177 | rest = {k: v for k, v in (load_state(key) or {}).items() if k != kind} |
178 | save_state(key, {**rest, kind: m} if m else rest) |
179 |
|
180 | m = { |
181 | "sent_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), |
182 | "boot_timestamp": boot_ts, |
183 | "confirmed": False, |
184 | } |
185 | mark(m) |
186 | eos = node(auth, 15) |
187 | try: |
188 | # Opened before sending, so a failure here means the command never left. |
189 | eos.connection.transport.connect() |
190 | except OSError as e: |
191 | mark(None) |
192 | raise DeviceUnreachable(f"{auth['host']}: {e}") |
193 | try: |
194 | eos.run_commands(["reload now"]) |
195 | except pyeapi.eapilib.ConnectionError as e: |
196 | # Sent, then the connection dropped or timed out as the switch went down. A 401 is a refusal. |
197 | err = connection_error(auth, eos, e) |
198 | if isinstance(err, AuthFailure): |
199 | mark(None) |
200 | raise err |
201 | except Exception: |
202 | mark(None) |
203 | raise |
204 | mark({**m, "confirmed": True}) |
205 |
|
206 |
|
207 | # "EOS64-4.34.9M.swi" -> "4.34.9M" |
208 | def image_version(file): |
209 | m = re.search(r"-(\d+\.\d+\.\d+(\.\d+)?[A-Z]*)\.swi$", file) |
210 | return m.group(1) if m else None |
211 |
|
212 |
|
213 | # Errors the workflow branches and retries on (retry_if: result.error.name == "DeviceUnreachable"). |
214 | class DeviceUnreachable(Exception): |
215 | pass |
216 |
|
217 |
|
218 | class AuthFailure(Exception): |
219 | pass |
220 |
|
221 |
|
222 | # pyeapi logs a traceback for every socket error, and a rebooting switch gives many. |
223 | logging.getLogger("pyeapi").addHandler(logging.NullHandler()) |
224 |
|
225 |
|
226 | def eapi(auth: arista_eos, cmds: list, fmt: str = "json", timeout: float = 60): |
227 | eos = node(auth, timeout) |
228 | try: |
229 | return eos.run_commands(cmds, fmt) |
230 | except pyeapi.eapilib.ConnectionError as e: |
231 | raise connection_error(auth, eos, e) |
232 |
|
233 |
|
234 | # pyeapi 1.0.4 raises ConnectionError for a socket error (kept in socket_error), an answer that isn't JSON (eAPI |
235 | # still starting, 5xx) and a 401, the only one with neither. |
236 | def connection_error(auth, eos, e): |
237 | if eos.connection.socket_error is None and e.message != "unable to connect to eAPI": |
238 | return AuthFailure(f"{auth['host']}: eAPI refused the credential ({e.message})") |
239 | return DeviceUnreachable(f"{auth['host']}: {e.message}") |
240 |
|
241 |
|
242 | # host is an address or a URL (https: |
243 | # resource ticks allow_http (Basic auth then crosses the network in cleartext). pyeapi skips certificate checks unless it |
244 | # is given a context, so one is always passed: verifying, or not when the resource unticks verify_tls (EOS ships a |
245 | # self-signed cert). |
246 | def switch_url(auth: arista_eos): |
247 | host = auth["host"] |
248 | if "://" not in host: |
249 | host = ( |
250 | f"https://[{host}]" |
251 | if host.count(":") > 1 and "[" not in host |
252 | else f"https://{host}" |
253 | ) |
254 | url = urlsplit(host) |
255 | if not url.hostname or ( |
256 | url.scheme != "https" and not (url.scheme == "http" and auth.get("allow_http")) |
257 | ): |
258 | raise ValueError( |
259 | f"host must be an address or an https:// URL (http:// needs allow_http on the resource): {auth['host']}" |
260 | ) |
261 | return url |
262 |
|
263 |
|
264 | def node(auth: arista_eos, timeout: float = 60): |
265 | url = switch_url(auth) |
266 | context = ssl.create_default_context() |
267 | if auth.get("verify_tls") is False: |
268 | context.check_hostname = False |
269 | context.verify_mode = ssl.CERT_NONE |
270 | return pyeapi.connect( |
271 | transport=url.scheme, |
272 | host=url.hostname, |
273 | port=url.port or auth.get("port") or None, |
274 | username=auth["username"], |
275 | password=auth["password"], |
276 | context=context, |
277 | timeout=timeout, |
278 | return_node=True, |
279 | ) |
280 |
|
281 |
|
282 | # Shared by the Device scripts: the root flow run's user state, keyed by switch, holds the pre-check snapshot, the |
283 | # image activate replaced (rollback target), the image it booted and the reload markers. Every step of the run (nested |
284 | # subflows and retried attempts included) reads the same entry. Outside a flow the Device scripts share one state |
285 | # resource in the running user's home (a script's own state would keep the pre-check snapshot from the post-check). |
286 | # The endpoint is called through the client so an unreachable state raises (wmill.get_flow_user_state returns None). |
287 | # ponytail: the fallback is one map per user, so concurrent standalone runs can race; flows don't. |
288 | # The key is the switch's eAPI address: arista_eos-<host>-<port> with non-word characters in the host as _, the port |
289 | # always in (443 or 80 when unset). State written before the port was part of the key is read from arista_eos_<host> |
290 | # as a fallback (all word characters, so never a new key); writes go to the new key only. |
291 | def state_keys(auth: arista_eos): |
292 | url = switch_url(auth) |
293 | port = int(url.port or auth.get("port") or {"https": 443, "http": 80}[url.scheme]) |
294 | return [ |
295 | f"arista_eos-{word(url.hostname)}-{port}", |
296 | f"arista_eos_{word(auth['host'])}", |
297 | ] |
298 |
|
299 |
|
300 | def word(s): |
301 | return re.sub(r"\W", "_", s) |
302 |
|
303 |
|
304 | def load_state(keys): |
305 | for key in keys: |
306 | if os.environ.get("WM_FLOW_JOB_ID"): |
307 | c = wmill.Windmill() |
308 | value = c.get( |
309 | f"/w/{c.workspace}/jobs/flow/user_states/{c.get_root_job_id()}/{key}" |
310 | ).json() |
311 | else: |
312 | value = (wmill.get_state(standalone_state()) or {}).get(key) |
313 | if value is not None: |
314 | return value |
315 |
|
316 |
|
317 | def save_state(keys, value): |
318 | if os.environ.get("WM_FLOW_JOB_ID"): |
319 | c = wmill.Windmill() |
320 | c.post( |
321 | f"/w/{c.workspace}/jobs/flow/user_states/{c.get_root_job_id()}/{keys[0]}", |
322 | json=value, |
323 | ) |
324 | else: |
325 | path = standalone_state() |
326 | wmill.set_state({**(wmill.get_state(path) or {}), keys[0]: value}, path) |
327 |
|
328 |
|
329 | def standalone_state(): |
330 | return f"u/{os.environ['WM_USERNAME']}/arista_eos_state" |
331 |
|
332 |
|
333 | # "4.34.9M-51132493.4349M (engineering build)" on cEOS, "4.34.9M" on hardware. |
334 | def short_version(v): |
335 | m = re.match(r"\d+\.\d+\.\d+(\.\d+)?[A-Z]*", v) |
336 | return m.group(0) if m else v |
337 | |