1 | # py312 |
2 | # requirements: |
3 | # pyeapi==1.0.4 |
4 | # wmill==1.851.0 |
5 |
|
6 | # Device post-check: one attempt, after an upgrade (or rollback) of one switch. Compares the switch with the snapshot |
7 | # Device pre-check saved in this flow run, and raises a named error the workflow retries on: |
8 | # - DeviceUnreachable while the switch is down or still runs the boot a reload of this run was sent on; |
9 | # - NotStable while the state differs from the snapshot (the differences are in the message; ports stay errdisabled |
10 | # until the MLAG reload-delay has run, the message says how much of it is left); |
11 | # - AuthFailure when the credential is refused (not worth retrying). |
12 | # No snapshot in this run is a plain error. The version is reported, not compared with the target: the workflow does. |
13 |
|
14 | import logging |
15 | import os |
16 | import re |
17 | import ssl |
18 | from typing import TypedDict |
19 | from urllib.parse import urlsplit |
20 |
|
21 | import pyeapi |
22 | import wmill |
23 |
|
24 |
|
25 | class arista_eos(TypedDict): |
26 | host: str |
27 | username: str |
28 | password: str |
29 | port: int |
30 | verify_tls: bool |
31 | allow_http: bool |
32 |
|
33 |
|
34 | class NotStable(Exception): |
35 | pass |
36 |
|
37 |
|
38 | def main(auth: arista_eos): |
39 | """ |
40 | Device post-check |
41 | One post-upgrade check of one switch against the pre-check snapshot of this flow run: returns { running_version, status ok, checks_ok }. Raises DeviceUnreachable while the switch is down or still on the boot the reload was sent on, NotStable while interfaces/LLDP/MLAG differ from the snapshot (differences in the message), AuthFailure when the credential is refused. Retry it from the workflow; it does not wait. |
42 | """ |
43 | state = load_state(state_keys(auth)) or {} |
44 | # Before probing the switch: without a snapshot, retrying through a reboot would only delay the same error. |
45 | if not state.get("snapshot"): |
46 | raise RuntimeError( |
47 | f"No pre-check snapshot for {auth['host']} in this flow run: run Device pre-check first, in the same run" |
48 | ) |
49 | version = returned(auth, state) |
50 | problems = diff(state["snapshot"], snapshot(auth)) |
51 | if problems: |
52 | [mlag] = eapi(auth, ["show mlag"]) |
53 | left = round(reload_delay(mlag) - version["uptime"]) |
54 | hint = f" (MLAG reload-delay still running: {left} s left)" if left > 0 else "" |
55 | raise NotStable(f"{auth['host']}: {'; '.join(problems)}{hint}") |
56 | return { |
57 | "running_version": short_version(version["version"]), |
58 | "status": "ok", |
59 | "checks_ok": ["interface"], |
60 | } |
61 |
|
62 |
|
63 | # Same function in Get Reload Delay and Device post-check. One attempt: show version, raising DeviceUnreachable |
64 | # while the switch is down, and while it still runs the boot a reload of this run was sent on (reload now can answer |
65 | # before EOS goes down, and the old boot must not pass for the new one). AuthFailure is raised as is. |
66 | def returned(auth, state): |
67 | try: |
68 | # Short: a switch that is down holds the attempt for the whole timeout, and the workflow retries anyway. |
69 | [v] = eapi(auth, ["show version"], timeout=15) |
70 | except DeviceUnreachable as e: |
71 | raise DeviceUnreachable(f"{e}: not back yet") |
72 | for k in ("reload", "rollback"): |
73 | m = state.get(k) |
74 | if m and m["boot_timestamp"] == v["bootupTimestamp"]: |
75 | raise DeviceUnreachable( |
76 | f"{auth['host']}: still on the boot the {k} sent at {m['sent_at']} was sent on" |
77 | ) |
78 | return v |
79 |
|
80 |
|
81 | # show mlag reports state disabled both when MLAG was never configured and when it is configured but shut down; |
82 | # only the first has no domain-id, local interface or peer-link. |
83 | def mlag_configured(mlag): |
84 | return any(mlag.get(k) for k in ("domainId", "localInterface", "peerLink")) |
85 |
|
86 |
|
87 | def snapshot(auth): |
88 | status, lldp, mlag, mlag_intf = eapi( |
89 | auth, |
90 | [ |
91 | "show interfaces status", |
92 | "show lldp neighbors", |
93 | "show mlag", |
94 | "show mlag interfaces", |
95 | ], |
96 | ) |
97 | return { |
98 | "interfaces_up": sorted( |
99 | n |
100 | for n, i in status["interfaceStatuses"].items() |
101 | if i.get("linkStatus") == "connected" |
102 | ), |
103 | "lldp": [ |
104 | { |
105 | "port": n["port"], |
106 | "neighbor": f"{n['neighborDevice']} {n['neighborPort']}", |
107 | } |
108 | for n in lldp["lldpNeighbors"] |
109 | ], |
110 | "mlag": { |
111 | "state": mlag.get("state"), |
112 | "negStatus": mlag.get("negStatus"), |
113 | "peerLinkStatus": mlag.get("peerLinkStatus"), |
114 | "ports": { |
115 | f"{i['localInterface']} (mlag {id})": i["status"] |
116 | for id, i in (mlag_intf.get("interfaces") or {}).items() |
117 | }, |
118 | }, |
119 | } |
120 |
|
121 |
|
122 | def diff(before, after): |
123 | p = [] |
124 | for i in before["interfaces_up"]: |
125 | if i not in after["interfaces_up"]: |
126 | p.append(f"{i} down after the upgrade (up before)") |
127 | for n in before["lldp"]: |
128 | if not any( |
129 | a["port"] == n["port"] and a["neighbor"] == n["neighbor"] |
130 | for a in after["lldp"] |
131 | ): |
132 | p.append(f"LLDP neighbor {n['neighbor']} no longer seen on {n['port']}") |
133 | for k in ("state", "negStatus", "peerLinkStatus"): |
134 | if before["mlag"][k] != after["mlag"][k]: |
135 | p.append(f"MLAG {k} {after['mlag'][k]} (was {before['mlag'][k]})") |
136 | for port, s in before["mlag"]["ports"].items(): |
137 | if after["mlag"]["ports"].get(port) != s: |
138 | p.append(f"{port} {after['mlag']['ports'].get(port, 'missing')} (was {s})") |
139 | return p |
140 |
|
141 |
|
142 | # Without MLAG configured, show mlag still reports the default reload-delay, and nothing waits on it. |
143 | def reload_delay(mlag): |
144 | if not mlag_configured(mlag): |
145 | return 0 |
146 | return max(mlag.get("reloadDelay") or 0, mlag.get("reloadDelayNonMlag") or 0) |
147 |
|
148 |
|
149 | # Errors the workflow branches and retries on (retry_if: result.error.name == "DeviceUnreachable"). |
150 | class DeviceUnreachable(Exception): |
151 | pass |
152 |
|
153 |
|
154 | class AuthFailure(Exception): |
155 | pass |
156 |
|
157 |
|
158 | # pyeapi logs a traceback for every socket error, and a rebooting switch gives many. |
159 | logging.getLogger("pyeapi").addHandler(logging.NullHandler()) |
160 |
|
161 |
|
162 | # This script runs right after a reboot, when eAPI can answer before the EOS agents behind a show command have |
163 | # started: such an error answer counts as not back yet, so the workflow retries it. It only sends show commands. |
164 | def eapi(auth: arista_eos, cmds: list, fmt: str = "json", timeout: float = 60): |
165 | eos = node(auth, timeout) |
166 | try: |
167 | return eos.run_commands(cmds, fmt) |
168 | except pyeapi.eapilib.ConnectionError as e: |
169 | raise connection_error(auth, eos, e) |
170 | except pyeapi.eapilib.CommandError as e: |
171 | raise DeviceUnreachable(f"{auth['host']}: not ready yet ({e.message})") |
172 |
|
173 |
|
174 | # pyeapi 1.0.4 raises ConnectionError for a socket error (kept in socket_error), an answer that isn't JSON (eAPI |
175 | # still starting, 5xx) and a 401, the only one with neither. |
176 | def connection_error(auth, eos, e): |
177 | if eos.connection.socket_error is None and e.message != "unable to connect to eAPI": |
178 | return AuthFailure(f"{auth['host']}: eAPI refused the credential ({e.message})") |
179 | return DeviceUnreachable(f"{auth['host']}: {e.message}") |
180 |
|
181 |
|
182 | # host is an address or a URL (https://10.0.0.11:8443); a bare IPv6 address is bracketed. Only https, or http when the |
183 | # resource ticks allow_http (Basic auth then crosses the network in cleartext). pyeapi skips certificate checks unless it |
184 | # is given a context, so one is always passed: verifying, or not when the resource unticks verify_tls (EOS ships a |
185 | # self-signed cert). |
186 | def switch_url(auth: arista_eos): |
187 | host = auth["host"] |
188 | if "://" not in host: |
189 | host = ( |
190 | f"https://[{host}]" |
191 | if host.count(":") > 1 and "[" not in host |
192 | else f"https://{host}" |
193 | ) |
194 | url = urlsplit(host) |
195 | if not url.hostname or ( |
196 | url.scheme != "https" and not (url.scheme == "http" and auth.get("allow_http")) |
197 | ): |
198 | raise ValueError( |
199 | f"host must be an address or an https:// URL (http:// needs allow_http on the resource): {auth['host']}" |
200 | ) |
201 | return url |
202 |
|
203 |
|
204 | def node(auth: arista_eos, timeout: float = 60): |
205 | url = switch_url(auth) |
206 | context = ssl.create_default_context() |
207 | if auth.get("verify_tls") is False: |
208 | context.check_hostname = False |
209 | context.verify_mode = ssl.CERT_NONE |
210 | return pyeapi.connect( |
211 | transport=url.scheme, |
212 | host=url.hostname, |
213 | port=url.port or auth.get("port") or None, |
214 | username=auth["username"], |
215 | password=auth["password"], |
216 | context=context, |
217 | timeout=timeout, |
218 | return_node=True, |
219 | ) |
220 |
|
221 |
|
222 | # Shared by the Device scripts: the root flow run's user state, keyed by switch, holds the pre-check snapshot, the |
223 | # image activate replaced (rollback target), the image it booted and the reload markers. Every step of the run (nested |
224 | # subflows and retried attempts included) reads the same entry. Outside a flow the Device scripts share one state |
225 | # resource in the running user's home (a script's own state would keep the pre-check snapshot from the post-check). |
226 | # The endpoint is called through the client so an unreachable state raises (wmill.get_flow_user_state returns None). |
227 | # ponytail: the fallback is one map per user, so concurrent standalone runs can race; flows don't. |
228 | # The key is the switch's eAPI address: arista_eos-<host>-<port> with non-word characters in the host as _, the port |
229 | # always in (443 or 80 when unset). State written before the port was part of the key is read from arista_eos_<host> |
230 | # as a fallback (all word characters, so never a new key); writes go to the new key only. |
231 | def state_keys(auth: arista_eos): |
232 | url = switch_url(auth) |
233 | port = int(url.port or auth.get("port") or {"https": 443, "http": 80}[url.scheme]) |
234 | return [ |
235 | f"arista_eos-{word(url.hostname)}-{port}", |
236 | f"arista_eos_{word(auth['host'])}", |
237 | ] |
238 |
|
239 |
|
240 | def word(s): |
241 | return re.sub(r"\W", "_", s) |
242 |
|
243 |
|
244 | def load_state(keys): |
245 | for key in keys: |
246 | if os.environ.get("WM_FLOW_JOB_ID"): |
247 | c = wmill.Windmill() |
248 | value = c.get( |
249 | f"/w/{c.workspace}/jobs/flow/user_states/{c.get_root_job_id()}/{key}" |
250 | ).json() |
251 | else: |
252 | value = (wmill.get_state(standalone_state()) or {}).get(key) |
253 | if value is not None: |
254 | return value |
255 |
|
256 |
|
257 | def save_state(keys, value): |
258 | if os.environ.get("WM_FLOW_JOB_ID"): |
259 | c = wmill.Windmill() |
260 | c.post( |
261 | f"/w/{c.workspace}/jobs/flow/user_states/{c.get_root_job_id()}/{keys[0]}", |
262 | json=value, |
263 | ) |
264 | else: |
265 | path = standalone_state() |
266 | wmill.set_state({**(wmill.get_state(path) or {}), keys[0]: value}, path) |
267 |
|
268 |
|
269 | def standalone_state(): |
270 | return f"u/{os.environ['WM_USERNAME']}/arista_eos_state" |
271 |
|
272 |
|
273 | # "4.34.9M-51132493.4349M (engineering build)" on cEOS, "4.34.9M" on hardware. |
274 | def short_version(v): |
275 | m = re.match(r"\d+\.\d+\.\d+(\.\d+)?[A-Z]*", v) |
276 | return m.group(0) if m else v |
277 |
|