1 | # py312 |
2 | # requirements: |
3 | # pyeapi==1.0.4 |
4 | # wmill==1.851.0 |
5 |
|
6 | # Device activate: one attempt at staging the target image on one switch, no reload. |
7 | # Records the image of the running version as the rollback target (once per run, before any change), copies the |
8 | # image to flash (skipped when already there), compares verify /sha512 with the expected digest, then boot system. |
9 | # The image is image_url, or image_base_url + EOS-<target_version>.swi; the digest is sha512, or the .sha512 file next |
10 | # to the image. A file already on flash that doesn't match (a partial copy left by a killed run) is replaced by one |
11 | # copy; a freshly copied file that doesn't match is deleted and ActivateFailed raised, so a retry copies it again. |
12 | # An image file name that carries a version must carry target_version. Device reload checks boot-config against the |
13 | # image recorded here. State is the root flow run's user state under arista_eos-<host>-<port>. |
14 |
|
15 | import logging |
16 | import os |
17 | import re |
18 | import ssl |
19 | import urllib.error |
20 | import urllib.request |
21 | from typing import TypedDict |
22 | from urllib.parse import urlsplit |
23 |
|
24 | import pyeapi |
25 | import wmill |
26 |
|
27 |
|
28 | class arista_eos(TypedDict): |
29 | host: str |
30 | username: str |
31 | password: str |
32 | port: int |
33 | verify_tls: bool |
34 | allow_http: bool |
35 |
|
36 |
|
37 | class ActivateFailed(Exception): |
38 | pass |
39 |
|
40 |
|
41 | def main( |
42 | auth: arista_eos, |
43 | target_version: str, |
44 | image_base_url: str = "", |
45 | image_url: str = "", |
46 | sha512: str = "", |
47 | vrf: str = "", |
48 | ): |
49 | """ |
50 | Device activate |
51 | Copy the target image to the switch's flash (image_url, or EOS-<target_version>.swi from image_base_url), verify its sha512 (the sha512 input or the .sha512 file next to the image), and boot system it. No reload. Returns { status: done, steps, previous_image }. Raises ActivateFailed, DeviceUnreachable or AuthFailure. |
52 | """ |
53 | key = state_keys(auth) |
54 | state = load_state(key) or {} |
55 | # Set again only once this activate has booted the image: a stale one would let reload through. |
56 | forget_activated(key, state) |
57 | if not target_version or not (image_base_url or image_url): |
58 | raise ActivateFailed( |
59 | "target_version and image_base_url (or image_url) are required" |
60 | ) |
61 | try: |
62 | image, url = image_source(target_version, image_base_url, image_url) |
63 | except ValueError as e: |
64 | raise ActivateFailed(str(e)) |
65 | # A file name that carries a version must carry this one; one without (a lab image) is taken as is. |
66 | if image_version(image) not in (None, target_version): |
67 | raise ActivateFailed( |
68 | f"{image} is named for {image_version(image)}, not target_version {target_version}" |
69 | ) |
70 | steps = [] |
71 | previous = state.get("previous_image") or "" |
72 | try: |
73 | if sha512: |
74 | expected = digest(sha512) |
75 | if not expected: |
76 | raise ActivateFailed("sha512 must be the 128-hex digest of the image") |
77 | else: |
78 | try: |
79 | with urllib.request.urlopen(f"{url}.sha512", timeout=60) as r: |
80 | expected = digest(r.read().decode(errors="replace")) |
81 | except urllib.error.HTTPError as e: |
82 | raise ActivateFailed( |
83 | f"cannot read {url}.sha512 ({e.code}): pass sha512" |
84 | ) |
85 | except OSError as e: |
86 | reason = getattr(e, "reason", e) |
87 | # Unlike the switch's copy, this fetch checks the certificate: the digest is what activate trusts. |
88 | trust = ( |
89 | ", or trust the image server's CA on the worker (SSL_CERT_FILE)" |
90 | if isinstance(reason, ssl.SSLCertVerificationError) |
91 | else "" |
92 | ) |
93 | raise ActivateFailed( |
94 | f"cannot read {url}.sha512 ({reason}): pass sha512{trust}" |
95 | ) |
96 | if not expected: |
97 | raise ActivateFailed(f"no sha512 hash in {url}.sha512") |
98 |
|
99 | d, boot, version = eapi( |
100 | auth, |
101 | [ |
102 | {"cmd": "dir flash:/", "revision": 2}, |
103 | "show boot-config", |
104 | "show version", |
105 | ], |
106 | ) |
107 | files = list(d["urls"]["flash:/"]["entries"]) |
108 | running = short_version(version["version"]) |
109 | booted = boot["softwareImage"].replace("flash:/", "") |
110 | # boot-config is the running image (first run) or the target (an earlier run got past boot system). |
111 | # Anything else is a stale boot-config this step won't guess around. |
112 | if image_version(booted) != running and booted != image: |
113 | raise ActivateFailed( |
114 | f"boot-config points to {boot['softwareImage']}, neither the running version {running} nor {image}: fix boot-config first" |
115 | ) |
116 | # The rollback target is the image of the running version, recorded once per run before anything changes. |
117 | if not previous and running != target_version: |
118 | candidates = ( |
119 | [booted] |
120 | if booted != image |
121 | else [ |
122 | f |
123 | for f in files |
124 | if f.endswith(".swi") and image_version(f) == running |
125 | ] |
126 | ) |
127 | if len(candidates) != 1 or candidates[0] not in files: |
128 | raise ActivateFailed( |
129 | f"cannot tell which image on flash runs {running} (candidates: {', '.join(candidates) or 'none'}): no rollback target" |
130 | ) |
131 | previous = f"flash:/{candidates[0]}" |
132 | save_state(key, {**state, "previous_image": previous}) |
133 |
|
134 | def copy(): |
135 | # The copy runs on the switch and can take minutes for a ~1 GB image. |
136 | eapi( |
137 | auth, |
138 | [*([f"cli vrf {vrf}"] if vrf else []), f"copy {url} flash:{image}"], |
139 | "json", |
140 | 3600, |
141 | ) |
142 | steps.append(f"copied {url}") |
143 |
|
144 | def verify(): |
145 | [v] = eapi(auth, [f"verify /sha512 flash:{image}"]) |
146 | return digest((v.get("messages") or [""])[0]) |
147 |
|
148 | copied = image not in files |
149 | if copied: |
150 | copy() |
151 | actual = verify() |
152 | if actual != expected and not copied: |
153 | # Most likely a partial copy left by a killed run: replace it. |
154 | eapi(auth, [f"delete flash:{image}"]) |
155 | copy() |
156 | actual = verify() |
157 | if actual != expected: |
158 | # An earlier run may have booted it already: never leave boot-config on a deleted image. |
159 | restored = booted == image and previous |
160 | if restored: |
161 | boot_system(auth, previous.replace("flash:/", "")) |
162 | eapi(auth, [f"delete flash:{image}"]) |
163 | raise ActivateFailed( |
164 | f"sha512 mismatch for {image} (switch {actual or 'none'}, expected {expected}), deleted it" |
165 | + (f", boot-config back on {previous}" if restored else "") |
166 | ) |
167 | steps.append("sha512 verified") |
168 | steps.extend(boot_system(auth, image)) |
169 | except (DeviceUnreachable, AuthFailure, ActivateFailed): |
170 | raise |
171 | except Exception as e: |
172 | raise ActivateFailed(str(e)) |
173 | # Device reload checks boot-config against this, so it needs no image_url of its own. |
174 | save_state( |
175 | key, |
176 | {**state, "previous_image": previous, "activated_image": f"flash:/{image}"}, |
177 | ) |
178 | return {"status": "done", "steps": steps, "previous_image": previous} |
179 |
|
180 |
|
181 | # Same function in Device pre-check and Device activate. |
182 | # image_url wins; else image_base_url + EOS-<target>.swi. Returns (file name on flash, URL or ""). The URL goes into |
183 | # the switch's copy command, so it is an allowlist: http(s), then only A-Z a-z 0-9 . _ ~ : / % + - (no query, no |
184 | # user:password@, nothing the CLI reads as a separator), ending in a plain .swi file name. Raises ValueError otherwise. |
185 | def image_source(target, base, image_url): |
186 | if image_url: |
187 | image, url = urlsplit(image_url).path.rsplit("/", 1)[-1], image_url |
188 | else: |
189 | image = f"EOS-{target}.swi" if target else "" |
190 | url = f"{base.rstrip('/')}/{image}" if image and base else "" |
191 | if not url: |
192 | return image, url |
193 | # Not echoed: it would put the password in the steps and errors. |
194 | if "@" in url: |
195 | raise ValueError("the image URL must not contain @ (no user:password@ in it)") |
196 | if not re.match(r"https?://", url): |
197 | raise ValueError( |
198 | f"the image URL must start with http:// or https://: got {url!r}" |
199 | ) |
200 | if not urlsplit(url).hostname: |
201 | raise ValueError(f"the image URL has no host: got {url!r}") |
202 | bad = sorted(set(re.sub(r"[A-Za-z0-9._~:/%+-]", "", url.split("://", 1)[1]))) |
203 | if bad: |
204 | raise ValueError( |
205 | f"the image URL may only contain A-Z a-z 0-9 . _ ~ : / % + -, not {' '.join(map(repr, bad))}: got {url!r}" |
206 | ) |
207 | if not re.fullmatch(r"[\w.+-]+\.swi", image): |
208 | raise ValueError(f"the image URL must end in a .swi file name: got {url!r}") |
209 | return image, url |
210 |
|
211 |
|
212 | def boot_system(auth, image): |
213 | _, set_, _, after = eapi( |
214 | auth, ["configure", f"boot system flash:{image}", "end", "show boot-config"] |
215 | ) |
216 | if after["softwareImage"] != f"flash:/{image}": |
217 | raise Exception(f"boot-config still points to {after['softwareImage']}") |
218 | return [f"boot system flash:{image}", *(set_.get("warnings") or [])] |
219 |
|
220 |
|
221 | def forget_activated(key, state): |
222 | if state.pop("activated_image", None): |
223 | save_state(key, state) |
224 |
|
225 |
|
226 | # "EOS64-4.34.9M.swi" -> "4.34.9M" |
227 | def image_version(file): |
228 | m = re.search(r"-(\d+\.\d+\.\d+(\.\d+)?[A-Z]*)\.swi$", file) |
229 | return m.group(1) if m else None |
230 |
|
231 |
|
232 | # The 128-hex digest in a sha512sum line, a BSD "SHA512 (f) = <hash>" line or verify /sha512 output. |
233 | def digest(text): |
234 | m = re.search(r"\b[0-9a-fA-F]{128}\b", text) |
235 | return m.group(0).lower() if m else "" |
236 |
|
237 |
|
238 | # Errors the workflow branches and retries on (retry_if: result.error.name == "DeviceUnreachable"). |
239 | class DeviceUnreachable(Exception): |
240 | pass |
241 |
|
242 |
|
243 | class AuthFailure(Exception): |
244 | pass |
245 |
|
246 |
|
247 | # pyeapi logs a traceback for every socket error, and a rebooting switch gives many. |
248 | logging.getLogger("pyeapi").addHandler(logging.NullHandler()) |
249 |
|
250 |
|
251 | def eapi(auth: arista_eos, cmds: list, fmt: str = "json", timeout: float = 60): |
252 | eos = node(auth, timeout) |
253 | try: |
254 | return eos.run_commands(cmds, fmt) |
255 | except pyeapi.eapilib.ConnectionError as e: |
256 | raise connection_error(auth, eos, e) |
257 |
|
258 |
|
259 | # pyeapi 1.0.4 raises ConnectionError for a socket error (kept in socket_error), an answer that isn't JSON (eAPI |
260 | # still starting, 5xx) and a 401, the only one with neither. |
261 | def connection_error(auth, eos, e): |
262 | if eos.connection.socket_error is None and e.message != "unable to connect to eAPI": |
263 | return AuthFailure(f"{auth['host']}: eAPI refused the credential ({e.message})") |
264 | return DeviceUnreachable(f"{auth['host']}: {e.message}") |
265 |
|
266 |
|
267 | # host is an address or a URL (https: |
268 | # resource ticks allow_http (Basic auth then crosses the network in cleartext). pyeapi skips certificate checks unless it |
269 | # is given a context, so one is always passed: verifying, or not when the resource unticks verify_tls (EOS ships a |
270 | # self-signed cert). |
271 | def switch_url(auth: arista_eos): |
272 | host = auth["host"] |
273 | if "://" not in host: |
274 | host = ( |
275 | f"https://[{host}]" |
276 | if host.count(":") > 1 and "[" not in host |
277 | else f"https://{host}" |
278 | ) |
279 | url = urlsplit(host) |
280 | if not url.hostname or ( |
281 | url.scheme != "https" and not (url.scheme == "http" and auth.get("allow_http")) |
282 | ): |
283 | raise ValueError( |
284 | f"host must be an address or an https:// URL (http:// needs allow_http on the resource): {auth['host']}" |
285 | ) |
286 | return url |
287 |
|
288 |
|
289 | def node(auth: arista_eos, timeout: float = 60): |
290 | url = switch_url(auth) |
291 | context = ssl.create_default_context() |
292 | if auth.get("verify_tls") is False: |
293 | context.check_hostname = False |
294 | context.verify_mode = ssl.CERT_NONE |
295 | return pyeapi.connect( |
296 | transport=url.scheme, |
297 | host=url.hostname, |
298 | port=url.port or auth.get("port") or None, |
299 | username=auth["username"], |
300 | password=auth["password"], |
301 | context=context, |
302 | timeout=timeout, |
303 | return_node=True, |
304 | ) |
305 |
|
306 |
|
307 | # Shared by the Device scripts: the root flow run's user state, keyed by switch, holds the pre-check snapshot, the |
308 | # image activate replaced (rollback target), the image it booted and the reload markers. Every step of the run (nested |
309 | # subflows and retried attempts included) reads the same entry. Outside a flow the Device scripts share one state |
310 | # resource in the running user's home (a script's own state would keep the pre-check snapshot from the post-check). |
311 | # The endpoint is called through the client so an unreachable state raises (wmill.get_flow_user_state returns None). |
312 | # ponytail: the fallback is one map per user, so concurrent standalone runs can race; flows don't. |
313 | # The key is the switch's eAPI address: arista_eos-<host>-<port> with non-word characters in the host as _, the port |
314 | # always in (443 or 80 when unset). State written before the port was part of the key is read from arista_eos_<host> |
315 | # as a fallback (all word characters, so never a new key); writes go to the new key only. |
316 | def state_keys(auth: arista_eos): |
317 | url = switch_url(auth) |
318 | port = int(url.port or auth.get("port") or {"https": 443, "http": 80}[url.scheme]) |
319 | return [ |
320 | f"arista_eos-{word(url.hostname)}-{port}", |
321 | f"arista_eos_{word(auth['host'])}", |
322 | ] |
323 |
|
324 |
|
325 | def word(s): |
326 | return re.sub(r"\W", "_", s) |
327 |
|
328 |
|
329 | def load_state(keys): |
330 | for key in keys: |
331 | if os.environ.get("WM_FLOW_JOB_ID"): |
332 | c = wmill.Windmill() |
333 | value = c.get( |
334 | f"/w/{c.workspace}/jobs/flow/user_states/{c.get_root_job_id()}/{key}" |
335 | ).json() |
336 | else: |
337 | value = (wmill.get_state(standalone_state()) or {}).get(key) |
338 | if value is not None: |
339 | return value |
340 |
|
341 |
|
342 | def save_state(keys, value): |
343 | if os.environ.get("WM_FLOW_JOB_ID"): |
344 | c = wmill.Windmill() |
345 | c.post( |
346 | f"/w/{c.workspace}/jobs/flow/user_states/{c.get_root_job_id()}/{keys[0]}", |
347 | json=value, |
348 | ) |
349 | else: |
350 | path = standalone_state() |
351 | wmill.set_state({**(wmill.get_state(path) or {}), keys[0]: value}, path) |
352 |
|
353 |
|
354 | def standalone_state(): |
355 | return f"u/{os.environ['WM_USERNAME']}/arista_eos_state" |
356 |
|
357 |
|
358 | # "4.34.9M-51132493.4349M (engineering build)" on cEOS, "4.34.9M" on hardware. |
359 | def short_version(v): |
360 | m = re.match(r"\d+\.\d+\.\d+(\.\d+)?[A-Z]*", v) |
361 | return m.group(0) if m else v |
362 | |