1 | # py312 |
2 | # requirements: |
3 | # pyeapi==1.0.4 |
4 | # wmill==1.851.0 |
5 |
|
6 | # Device reload: reload now, at most once per switch and flow run, so a retried step never reboots it twice. |
7 | # Skipped (reloaded false, with the reason) when this run already sent it, the switch already runs target_version, or |
8 | # it rebooted less than min_uptime_seconds ago. boot-config must be the image Device activate set in this run, or |
9 | # flash:/EOS-<target_version>.swi. ReloadRefused while show running-config diffs changes anything but its ! headers, |
10 | # or there is no startup-config (reload now doesn't prompt, so unsaved changes would be lost), unless allow_unsaved. |
11 | # Reload markers are the root flow run's user state under arista_eos-<host>-<port>; Get Reload Delay and Device |
12 | # post-check read them to tell the new boot from the old one. |
13 |
|
14 | import logging |
15 | import os |
16 | import re |
17 | import ssl |
18 | import time |
19 | from typing import TypedDict |
20 | from urllib.parse import urlsplit |
21 |
|
22 | import pyeapi |
23 | import wmill |
24 |
|
25 |
|
26 | class arista_eos(TypedDict): |
27 | host: str |
28 | username: str |
29 | password: str |
30 | port: int |
31 | verify_tls: bool |
32 | allow_http: bool |
33 |
|
34 |
|
35 | class ReloadRefused(Exception): |
36 | pass |
37 |
|
38 |
|
39 | def main( |
40 | auth: arista_eos, |
41 | target_version: str, |
42 | min_uptime_seconds: int = 300, |
43 | allow_unsaved: bool = False, |
44 | ): |
45 | """ |
46 | Device reload |
47 | Reload the switch onto the image Device activate set: exactly once per flow run (a retry skips it), refused with unsaved config unless allow_unsaved. Returns { status: done, reloaded, reason }. Raises ReloadRefused (unsaved config, boot-config not the activated image), DeviceUnreachable or AuthFailure. Follow it with Get Reload Delay. |
48 | """ |
49 | key = state_keys(auth) |
50 | state = load_state(key) or {} |
51 | min_uptime = 300 if min_uptime_seconds is None else min_uptime_seconds |
52 | if not target_version: |
53 | raise ReloadRefused("target_version is required for the reload guard") |
54 | sw = read_switch(auth, state, min_uptime) |
55 | sent = sw["marker"]("reload") |
56 | if sw.get("unreachable"): |
57 | if sent: |
58 | return skip( |
59 | f"reload sent at {sent['sent_at']}; the switch no longer answers: it is reloading" |
60 | ) |
61 | raise DeviceUnreachable(sw["unreachable"]) |
62 | if sent and (sent["confirmed"] or sent["boot_timestamp"] != sw["boot"]): |
63 | return skip(guard_reason("reload", sent, sw)) |
64 | if sw["running"] == target_version: |
65 | return skip(f"already running {target_version}") |
66 | # The image activate set in this run (state), or exactly EOS-<target_version>.swi. |
67 | activated = state.get("activated_image") |
68 | if sw["image"] not in (activated, f"flash:/EOS-{target_version}.swi"): |
69 | raise ReloadRefused( |
70 | f"boot-config points to {sw['image']}, neither the image activate set ({activated or 'none in this run'}) nor flash:/EOS-{target_version}.swi: run Device activate first" |
71 | ) |
72 | if sw["uptime"] < min_uptime: |
73 | return skip(f"rebooted {round(sw['uptime'])} s ago") |
74 | if not allow_unsaved and unsaved(auth): |
75 | raise ReloadRefused( |
76 | "unsaved config: run Save Config first, or pass allow_unsaved: true" |
77 | ) |
78 | fire_reload(auth, key, "reload", sw["boot"]) |
79 | return {"status": "done", "reloaded": True} |
80 |
|
81 |
|
82 | # reload now doesn't prompt to save. show running-config diffs is a unified diff of startup-config against |
83 | # running-config, empty when they match. Its two file-header lines come before the first @@. Changed lines that |
84 | # start with a single "!" are headers EOS writes itself (boot system shows there, so every activate changes one); |
85 | # "!!" comments are config. With no startup-config every line shows as added: unsaved, it would boot into ZTP. |
86 | # A refused command (AAA authorization, an older EOS) is not unsaved config: saving wouldn't clear it. |
87 | def unsaved(auth): |
88 | try: |
89 | [d] = eapi(auth, ["show running-config diffs"], "text") |
90 | except pyeapi.eapilib.CommandError as e: |
91 | raise ReloadRefused(f"cannot compare running and startup config: {e}") |
92 | hunks, changed = False, [] |
93 | for l in d["output"].splitlines(): |
94 | hunks = hunks or l.startswith("@@") |
95 | if hunks and l[:1] in ("+", "-") and l[1:].strip(): |
96 | changed.append(l[1:].strip()) |
97 | return any(not re.match(r"!(?!!)", l) for l in changed) |
98 |
|
99 |
|
100 | def skip(reason): |
101 | return {"status": "done", "reloaded": False, "reason": reason} |
102 |
|
103 |
|
104 | def guard_reason(kind, m, sw): |
105 | if m["boot_timestamp"] == sw["boot"]: |
106 | return f"{kind} sent at {m['sent_at']}; the switch is going down" |
107 | return f"{kind} sent at {m['sent_at']} already rebooted the switch" |
108 |
|
109 |
|
110 | # A reload is sent at most once per switch and flow run. Each one leaves a marker { sent_at, boot_timestamp, |
111 | # confirmed }: boot_timestamp is the boot it was sent on, so a later boot means it happened, and confirmed means |
112 | # the command reached the switch. An unconfirmed marker on the same boot (the worker died first) is sent again. |
113 | # Outside a flow, a marker is dropped once the reboot it caused is min_uptime_seconds old. |
114 | def read_switch(auth, state, min_uptime): |
115 | try: |
116 | version, boot = eapi(auth, ["show version", "show boot-config"]) |
117 | except DeviceUnreachable as e: |
118 | return {"unreachable": str(e), "marker": lambda k: state.get(k)} |
119 | sw = { |
120 | "running": short_version(version["version"]), |
121 | "image": boot["softwareImage"], |
122 | "boot": version["bootupTimestamp"], |
123 | "uptime": version["uptime"], |
124 | } |
125 | in_flow = bool(os.environ.get("WM_FLOW_JOB_ID")) |
126 |
|
127 | def marker(k): |
128 | m = state.get(k) |
129 | return ( |
130 | m |
131 | if m |
132 | and ( |
133 | in_flow |
134 | or m["boot_timestamp"] == sw["boot"] |
135 | or sw["uptime"] < min_uptime |
136 | ) |
137 | else None |
138 | ) |
139 |
|
140 | return {**sw, "marker": marker} |
141 |
|
142 |
|
143 | # Fire and forget: the switch drops the connection while going down, which is not an error. A connection that |
144 | # never opened, or a refusal from the switch, means it did not reload: the marker goes and the error is raised. |
145 | def fire_reload(auth, key, kind, boot_ts): |
146 | def mark(m): |
147 | rest = {k: v for k, v in (load_state(key) or {}).items() if k != kind} |
148 | save_state(key, {**rest, kind: m} if m else rest) |
149 |
|
150 | m = { |
151 | "sent_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), |
152 | "boot_timestamp": boot_ts, |
153 | "confirmed": False, |
154 | } |
155 | mark(m) |
156 | eos = node(auth, 15) |
157 | try: |
158 | # Opened before sending, so a failure here means the command never left. |
159 | eos.connection.transport.connect() |
160 | except OSError as e: |
161 | mark(None) |
162 | raise DeviceUnreachable(f"{auth['host']}: {e}") |
163 | try: |
164 | eos.run_commands(["reload now"]) |
165 | except pyeapi.eapilib.ConnectionError as e: |
166 | # Sent, then the connection dropped or timed out as the switch went down. A 401 is a refusal. |
167 | err = connection_error(auth, eos, e) |
168 | if isinstance(err, AuthFailure): |
169 | mark(None) |
170 | raise err |
171 | except Exception: |
172 | mark(None) |
173 | raise |
174 | mark({**m, "confirmed": True}) |
175 |
|
176 |
|
177 | # Errors the workflow branches and retries on (retry_if: result.error.name == "DeviceUnreachable"). |
178 | class DeviceUnreachable(Exception): |
179 | pass |
180 |
|
181 |
|
182 | class AuthFailure(Exception): |
183 | pass |
184 |
|
185 |
|
186 | # pyeapi logs a traceback for every socket error, and a rebooting switch gives many. |
187 | logging.getLogger("pyeapi").addHandler(logging.NullHandler()) |
188 |
|
189 |
|
190 | def eapi(auth: arista_eos, cmds: list, fmt: str = "json", timeout: float = 60): |
191 | eos = node(auth, timeout) |
192 | try: |
193 | return eos.run_commands(cmds, fmt) |
194 | except pyeapi.eapilib.ConnectionError as e: |
195 | raise connection_error(auth, eos, e) |
196 |
|
197 |
|
198 | # pyeapi 1.0.4 raises ConnectionError for a socket error (kept in socket_error), an answer that isn't JSON (eAPI |
199 | # still starting, 5xx) and a 401, the only one with neither. |
200 | def connection_error(auth, eos, e): |
201 | if eos.connection.socket_error is None and e.message != "unable to connect to eAPI": |
202 | return AuthFailure(f"{auth['host']}: eAPI refused the credential ({e.message})") |
203 | return DeviceUnreachable(f"{auth['host']}: {e.message}") |
204 |
|
205 |
|
206 | # host is an address or a URL (https://10.0.0.11:8443); a bare IPv6 address is bracketed. Only https, or http when the |
207 | # resource ticks allow_http (Basic auth then crosses the network in cleartext). pyeapi skips certificate checks unless it |
208 | # is given a context, so one is always passed: verifying, or not when the resource unticks verify_tls (EOS ships a |
209 | # self-signed cert). |
210 | def switch_url(auth: arista_eos): |
211 | host = auth["host"] |
212 | if "://" not in host: |
213 | host = ( |
214 | f"https://[{host}]" |
215 | if host.count(":") > 1 and "[" not in host |
216 | else f"https://{host}" |
217 | ) |
218 | url = urlsplit(host) |
219 | if not url.hostname or ( |
220 | url.scheme != "https" and not (url.scheme == "http" and auth.get("allow_http")) |
221 | ): |
222 | raise ValueError( |
223 | f"host must be an address or an https:// URL (http:// needs allow_http on the resource): {auth['host']}" |
224 | ) |
225 | return url |
226 |
|
227 |
|
228 | def node(auth: arista_eos, timeout: float = 60): |
229 | url = switch_url(auth) |
230 | context = ssl.create_default_context() |
231 | if auth.get("verify_tls") is False: |
232 | context.check_hostname = False |
233 | context.verify_mode = ssl.CERT_NONE |
234 | return pyeapi.connect( |
235 | transport=url.scheme, |
236 | host=url.hostname, |
237 | port=url.port or auth.get("port") or None, |
238 | username=auth["username"], |
239 | password=auth["password"], |
240 | context=context, |
241 | timeout=timeout, |
242 | return_node=True, |
243 | ) |
244 |
|
245 |
|
246 | # Shared by the Device scripts: the root flow run's user state, keyed by switch, holds the pre-check snapshot, the |
247 | # image activate replaced (rollback target), the image it booted and the reload markers. Every step of the run (nested |
248 | # subflows and retried attempts included) reads the same entry. Outside a flow the Device scripts share one state |
249 | # resource in the running user's home (a script's own state would keep the pre-check snapshot from the post-check). |
250 | # The endpoint is called through the client so an unreachable state raises (wmill.get_flow_user_state returns None). |
251 | # ponytail: the fallback is one map per user, so concurrent standalone runs can race; flows don't. |
252 | # The key is the switch's eAPI address: arista_eos-<host>-<port> with non-word characters in the host as _, the port |
253 | # always in (443 or 80 when unset). State written before the port was part of the key is read from arista_eos_<host> |
254 | # as a fallback (all word characters, so never a new key); writes go to the new key only. |
255 | def state_keys(auth: arista_eos): |
256 | url = switch_url(auth) |
257 | port = int(url.port or auth.get("port") or {"https": 443, "http": 80}[url.scheme]) |
258 | return [ |
259 | f"arista_eos-{word(url.hostname)}-{port}", |
260 | f"arista_eos_{word(auth['host'])}", |
261 | ] |
262 |
|
263 |
|
264 | def word(s): |
265 | return re.sub(r"\W", "_", s) |
266 |
|
267 |
|
268 | def load_state(keys): |
269 | for key in keys: |
270 | if os.environ.get("WM_FLOW_JOB_ID"): |
271 | c = wmill.Windmill() |
272 | value = c.get( |
273 | f"/w/{c.workspace}/jobs/flow/user_states/{c.get_root_job_id()}/{key}" |
274 | ).json() |
275 | else: |
276 | value = (wmill.get_state(standalone_state()) or {}).get(key) |
277 | if value is not None: |
278 | return value |
279 |
|
280 |
|
281 | def save_state(keys, value): |
282 | if os.environ.get("WM_FLOW_JOB_ID"): |
283 | c = wmill.Windmill() |
284 | c.post( |
285 | f"/w/{c.workspace}/jobs/flow/user_states/{c.get_root_job_id()}/{keys[0]}", |
286 | json=value, |
287 | ) |
288 | else: |
289 | path = standalone_state() |
290 | wmill.set_state({**(wmill.get_state(path) or {}), keys[0]: value}, path) |
291 |
|
292 |
|
293 | def standalone_state(): |
294 | return f"u/{os.environ['WM_USERNAME']}/arista_eos_state" |
295 |
|
296 |
|
297 | # "4.34.9M-51132493.4349M (engineering build)" on cEOS, "4.34.9M" on hardware. |
298 | def short_version(v): |
299 | m = re.match(r"\d+\.\d+\.\d+(\.\d+)?[A-Z]*", v) |
300 | return m.group(0) if m else v |
301 |
|