1 | # py312 |
2 | # requirements: |
3 | # pyeapi==1.0.4 |
4 | # wmill==1.851.0 |
5 |
|
6 | # Get Reload Delay: one attempt, after a reload or rollback of one switch. Raises DeviceUnreachable while the switch |
7 | # is down or still runs the boot a reload of this run was sent on, so the workflow retries this step until the switch |
8 | # is back. Then returns the MLAG reload-delay (0 without MLAG) and the part of it still to run, counted from boot: |
9 | # ports stay errdisabled until it ends, so the workflow sleeps remaining_seconds before Device post-check. |
10 |
|
11 | import logging |
12 | import os |
13 | import re |
14 | import ssl |
15 | from typing import TypedDict |
16 | from urllib.parse import urlsplit |
17 |
|
18 | import pyeapi |
19 | import wmill |
20 |
|
21 |
|
22 | class arista_eos(TypedDict): |
23 | host: str |
24 | username: str |
25 | password: str |
26 | port: int |
27 | verify_tls: bool |
28 | allow_http: bool |
29 |
|
30 |
|
31 | def main(auth: arista_eos): |
32 | """ |
33 | Get Reload Delay |
34 | After a reload, once the switch answers on a new boot: returns { running_version, uptime_seconds, reload_delay_seconds, remaining_seconds }, the MLAG reload-delay (0 without MLAG) and the seconds of it still to run; sleep remaining_seconds before Device post-check. Raises DeviceUnreachable while the switch is down or still on the boot the reload was sent on, AuthFailure when the credential is refused. |
35 | """ |
36 | state = load_state(state_keys(auth)) or {} |
37 | version = returned(auth, state) |
38 | [mlag] = eapi(auth, ["show mlag"]) |
39 | delay = reload_delay(mlag) |
40 | return { |
41 | "running_version": short_version(version["version"]), |
42 | "uptime_seconds": round(version["uptime"]), |
43 | "reload_delay_seconds": delay, |
44 | "remaining_seconds": max(0, round(delay - version["uptime"])), |
45 | } |
46 |
|
47 |
|
48 | # Same function in Get Reload Delay and Device post-check. One attempt: show version, raising DeviceUnreachable |
49 | # while the switch is down, and while it still runs the boot a reload of this run was sent on (reload now can answer |
50 | # before EOS goes down, and the old boot must not pass for the new one). AuthFailure is raised as is. |
51 | def returned(auth, state): |
52 | try: |
53 | # Short: a switch that is down holds the attempt for the whole timeout, and the workflow retries anyway. |
54 | [v] = eapi(auth, ["show version"], timeout=15) |
55 | except DeviceUnreachable as e: |
56 | raise DeviceUnreachable(f"{e}: not back yet") |
57 | for k in ("reload", "rollback"): |
58 | m = state.get(k) |
59 | if m and m["boot_timestamp"] == v["bootupTimestamp"]: |
60 | raise DeviceUnreachable( |
61 | f"{auth['host']}: still on the boot the {k} sent at {m['sent_at']} was sent on" |
62 | ) |
63 | return v |
64 |
|
65 |
|
66 | # show mlag reports state disabled both when MLAG was never configured and when it is configured but shut down; |
67 | # only the first has no domain-id, local interface or peer-link. |
68 | def mlag_configured(mlag): |
69 | return any(mlag.get(k) for k in ("domainId", "localInterface", "peerLink")) |
70 |
|
71 |
|
72 | # Without MLAG configured, show mlag still reports the default reload-delay, and nothing waits on it. |
73 | def reload_delay(mlag): |
74 | if not mlag_configured(mlag): |
75 | return 0 |
76 | return max(mlag.get("reloadDelay") or 0, mlag.get("reloadDelayNonMlag") or 0) |
77 |
|
78 |
|
79 | # Errors the workflow branches and retries on (retry_if: result.error.name == "DeviceUnreachable"). |
80 | class DeviceUnreachable(Exception): |
81 | pass |
82 |
|
83 |
|
84 | class AuthFailure(Exception): |
85 | pass |
86 |
|
87 |
|
88 | # pyeapi logs a traceback for every socket error, and a rebooting switch gives many. |
89 | logging.getLogger("pyeapi").addHandler(logging.NullHandler()) |
90 |
|
91 |
|
92 | # This script runs right after a reboot, when eAPI can answer before the EOS agents behind a show command have |
93 | # started: such an error answer counts as not back yet, so the workflow retries it. It only sends show commands. |
94 | def eapi(auth: arista_eos, cmds: list, fmt: str = "json", timeout: float = 60): |
95 | eos = node(auth, timeout) |
96 | try: |
97 | return eos.run_commands(cmds, fmt) |
98 | except pyeapi.eapilib.ConnectionError as e: |
99 | raise connection_error(auth, eos, e) |
100 | except pyeapi.eapilib.CommandError as e: |
101 | raise DeviceUnreachable(f"{auth['host']}: not ready yet ({e.message})") |
102 |
|
103 |
|
104 | # pyeapi 1.0.4 raises ConnectionError for a socket error (kept in socket_error), an answer that isn't JSON (eAPI |
105 | # still starting, 5xx) and a 401, the only one with neither. |
106 | def connection_error(auth, eos, e): |
107 | if eos.connection.socket_error is None and e.message != "unable to connect to eAPI": |
108 | return AuthFailure(f"{auth['host']}: eAPI refused the credential ({e.message})") |
109 | return DeviceUnreachable(f"{auth['host']}: {e.message}") |
110 |
|
111 |
|
112 | # host is an address or a URL (https://10.0.0.11:8443); a bare IPv6 address is bracketed. Only https, or http when the |
113 | # resource ticks allow_http (Basic auth then crosses the network in cleartext). pyeapi skips certificate checks unless it |
114 | # is given a context, so one is always passed: verifying, or not when the resource unticks verify_tls (EOS ships a |
115 | # self-signed cert). |
116 | def switch_url(auth: arista_eos): |
117 | host = auth["host"] |
118 | if ": |
119 | host = ( |
120 | f"https://[{host}]" |
121 | if host.count(":") > 1 and "[" not in host |
122 | else f"https://{host}" |
123 | ) |
124 | url = urlsplit(host) |
125 | if not url.hostname or ( |
126 | url.scheme != "https" and not (url.scheme == "http" and auth.get("allow_http")) |
127 | ): |
128 | raise ValueError( |
129 | f"host must be an address or an https:// URL (http:// needs allow_http on the resource): {auth['host']}" |
130 | ) |
131 | return url |
132 |
|
133 |
|
134 | def node(auth: arista_eos, timeout: float = 60): |
135 | url = switch_url(auth) |
136 | context = ssl.create_default_context() |
137 | if auth.get("verify_tls") is False: |
138 | context.check_hostname = False |
139 | context.verify_mode = ssl.CERT_NONE |
140 | return pyeapi.connect( |
141 | transport=url.scheme, |
142 | host=url.hostname, |
143 | port=url.port or auth.get("port") or None, |
144 | username=auth["username"], |
145 | password=auth["password"], |
146 | context=context, |
147 | timeout=timeout, |
148 | return_node=True, |
149 | ) |
150 |
|
151 |
|
152 | # Shared by the Device scripts: the root flow run's user state, keyed by switch, holds the pre-check snapshot, the |
153 | # image activate replaced (rollback target), the image it booted and the reload markers. Every step of the run (nested |
154 | # subflows and retried attempts included) reads the same entry. Outside a flow the Device scripts share one state |
155 | # resource in the running user's home (a script's own state would keep the pre-check snapshot from the post-check). |
156 | # The endpoint is called through the client so an unreachable state raises (wmill.get_flow_user_state returns None). |
157 | # ponytail: the fallback is one map per user, so concurrent standalone runs can race; flows don't. |
158 | # The key is the switch's eAPI address: arista_eos-<host>-<port> with non-word characters in the host as _, the port |
159 | # always in (443 or 80 when unset). State written before the port was part of the key is read from arista_eos_<host> |
160 | # as a fallback (all word characters, so never a new key); writes go to the new key only. |
161 | def state_keys(auth: arista_eos): |
162 | url = switch_url(auth) |
163 | port = int(url.port or auth.get("port") or {"https": 443, "http": 80}[url.scheme]) |
164 | return [ |
165 | f"arista_eos-{word(url.hostname)}-{port}", |
166 | f"arista_eos_{word(auth['host'])}", |
167 | ] |
168 |
|
169 |
|
170 | def word(s): |
171 | return re.sub(r"\W", "_", s) |
172 |
|
173 |
|
174 | def load_state(keys): |
175 | for key in keys: |
176 | if os.environ.get("WM_FLOW_JOB_ID"): |
177 | c = wmill.Windmill() |
178 | value = c.get( |
179 | f"/w/{c.workspace}/jobs/flow/user_states/{c.get_root_job_id()}/{key}" |
180 | ).json() |
181 | else: |
182 | value = (wmill.get_state(standalone_state()) or {}).get(key) |
183 | if value is not None: |
184 | return value |
185 |
|
186 |
|
187 | def save_state(keys, value): |
188 | if os.environ.get("WM_FLOW_JOB_ID"): |
189 | c = wmill.Windmill() |
190 | c.post( |
191 | f"/w/{c.workspace}/jobs/flow/user_states/{c.get_root_job_id()}/{keys[0]}", |
192 | json=value, |
193 | ) |
194 | else: |
195 | path = standalone_state() |
196 | wmill.set_state({**(wmill.get_state(path) or {}), keys[0]: value}, path) |
197 |
|
198 |
|
199 | def standalone_state(): |
200 | return f"u/{os.environ['WM_USERNAME']}/arista_eos_state" |
201 |
|
202 |
|
203 | # "4.34.9M-51132493.4349M (engineering build)" on cEOS, "4.34.9M" on hardware. |
204 | def short_version(v): |
205 | m = re.match(r"\d+\.\d+\.\d+(\.\d+)?[A-Z]*", v) |
206 | return m.group(0) if m else v |
207 | |