0

Get Reload Delay

Published today

After a reload, once the switch answers on a new boot: returns { running_version, uptime_seconds, reload_delay_seconds, remaining_seconds }, the MLAG reload-delay (0 without MLAG) and the seconds of it still to run; sleep remaining_seconds before Device post-check. Raises DeviceUnreachable while the switch is down or still on the boot the reload was sent on, AuthFailure when the credential is refused.

Script arista_eos Verified

The script

Submitted by hugo989 Python3
Verified 1 day ago
1
# py312
2
# requirements:
3
# pyeapi==1.0.4
4
# wmill==1.851.0
5

6
# Get Reload Delay: one attempt, after a reload or rollback of one switch. Raises DeviceUnreachable while the switch
7
# is down or still runs the boot a reload of this run was sent on, so the workflow retries this step until the switch
8
# is back. Then returns the MLAG reload-delay (0 without MLAG) and the part of it still to run, counted from boot:
9
# ports stay errdisabled until it ends, so the workflow sleeps remaining_seconds before Device post-check.
10

11
import logging
12
import os
13
import re
14
import ssl
15
from typing import TypedDict
16
from urllib.parse import urlsplit
17

18
import pyeapi
19
import wmill
20

21

22
class arista_eos(TypedDict):
23
    host: str
24
    username: str
25
    password: str
26
    port: int
27
    verify_tls: bool
28
    allow_http: bool
29

30

31
def main(auth: arista_eos):
32
    """
33
    Get Reload Delay
34
    After a reload, once the switch answers on a new boot: returns { running_version, uptime_seconds, reload_delay_seconds, remaining_seconds }, the MLAG reload-delay (0 without MLAG) and the seconds of it still to run; sleep remaining_seconds before Device post-check. Raises DeviceUnreachable while the switch is down or still on the boot the reload was sent on, AuthFailure when the credential is refused.
35
    """
36
    state = load_state(state_keys(auth)) or {}
37
    version = returned(auth, state)
38
    [mlag] = eapi(auth, ["show mlag"])
39
    delay = reload_delay(mlag)
40
    return {
41
        "running_version": short_version(version["version"]),
42
        "uptime_seconds": round(version["uptime"]),
43
        "reload_delay_seconds": delay,
44
        "remaining_seconds": max(0, round(delay - version["uptime"])),
45
    }
46

47

48
# Same function in Get Reload Delay and Device post-check. One attempt: show version, raising DeviceUnreachable
49
# while the switch is down, and while it still runs the boot a reload of this run was sent on (reload now can answer
50
# before EOS goes down, and the old boot must not pass for the new one). AuthFailure is raised as is.
51
def returned(auth, state):
52
    try:
53
        # Short: a switch that is down holds the attempt for the whole timeout, and the workflow retries anyway.
54
        [v] = eapi(auth, ["show version"], timeout=15)
55
    except DeviceUnreachable as e:
56
        raise DeviceUnreachable(f"{e}: not back yet")
57
    for k in ("reload", "rollback"):
58
        m = state.get(k)
59
        if m and m["boot_timestamp"] == v["bootupTimestamp"]:
60
            raise DeviceUnreachable(
61
                f"{auth['host']}: still on the boot the {k} sent at {m['sent_at']} was sent on"
62
            )
63
    return v
64

65

66
# show mlag reports state disabled both when MLAG was never configured and when it is configured but shut down;
67
# only the first has no domain-id, local interface or peer-link.
68
def mlag_configured(mlag):
69
    return any(mlag.get(k) for k in ("domainId", "localInterface", "peerLink"))
70

71

72
# Without MLAG configured, show mlag still reports the default reload-delay, and nothing waits on it.
73
def reload_delay(mlag):
74
    if not mlag_configured(mlag):
75
        return 0
76
    return max(mlag.get("reloadDelay") or 0, mlag.get("reloadDelayNonMlag") or 0)
77

78

79
# Errors the workflow branches and retries on (retry_if: result.error.name == "DeviceUnreachable").
80
class DeviceUnreachable(Exception):
81
    pass
82

83

84
class AuthFailure(Exception):
85
    pass
86

87

88
# pyeapi logs a traceback for every socket error, and a rebooting switch gives many.
89
logging.getLogger("pyeapi").addHandler(logging.NullHandler())
90

91

92
# This script runs right after a reboot, when eAPI can answer before the EOS agents behind a show command have
93
# started: such an error answer counts as not back yet, so the workflow retries it. It only sends show commands.
94
def eapi(auth: arista_eos, cmds: list, fmt: str = "json", timeout: float = 60):
95
    eos = node(auth, timeout)
96
    try:
97
        return eos.run_commands(cmds, fmt)
98
    except pyeapi.eapilib.ConnectionError as e:
99
        raise connection_error(auth, eos, e)
100
    except pyeapi.eapilib.CommandError as e:
101
        raise DeviceUnreachable(f"{auth['host']}: not ready yet ({e.message})")
102

103

104
# pyeapi 1.0.4 raises ConnectionError for a socket error (kept in socket_error), an answer that isn't JSON (eAPI
105
# still starting, 5xx) and a 401, the only one with neither.
106
def connection_error(auth, eos, e):
107
    if eos.connection.socket_error is None and e.message != "unable to connect to eAPI":
108
        return AuthFailure(f"{auth['host']}: eAPI refused the credential ({e.message})")
109
    return DeviceUnreachable(f"{auth['host']}: {e.message}")
110

111

112
# host is an address or a URL (https://10.0.0.11:8443); a bare IPv6 address is bracketed. Only https, or http when the
113
# resource ticks allow_http (Basic auth then crosses the network in cleartext). pyeapi skips certificate checks unless it
114
# is given a context, so one is always passed: verifying, or not when the resource unticks verify_tls (EOS ships a
115
# self-signed cert).
116
def switch_url(auth: arista_eos):
117
    host = auth["host"]
118
    if "://" not in host:
119
        host = (
120
            f"https://[{host}]"
121
            if host.count(":") > 1 and "[" not in host
122
            else f"https://{host}"
123
        )
124
    url = urlsplit(host)
125
    if not url.hostname or (
126
        url.scheme != "https" and not (url.scheme == "http" and auth.get("allow_http"))
127
    ):
128
        raise ValueError(
129
            f"host must be an address or an https:// URL (http:// needs allow_http on the resource): {auth['host']}"
130
        )
131
    return url
132

133

134
def node(auth: arista_eos, timeout: float = 60):
135
    url = switch_url(auth)
136
    context = ssl.create_default_context()
137
    if auth.get("verify_tls") is False:
138
        context.check_hostname = False
139
        context.verify_mode = ssl.CERT_NONE
140
    return pyeapi.connect(
141
        transport=url.scheme,
142
        host=url.hostname,
143
        port=url.port or auth.get("port") or None,
144
        username=auth["username"],
145
        password=auth["password"],
146
        context=context,
147
        timeout=timeout,
148
        return_node=True,
149
    )
150

151

152
# Shared by the Device scripts: the root flow run's user state, keyed by switch, holds the pre-check snapshot, the
153
# image activate replaced (rollback target), the image it booted and the reload markers. Every step of the run (nested
154
# subflows and retried attempts included) reads the same entry. Outside a flow the Device scripts share one state
155
# resource in the running user's home (a script's own state would keep the pre-check snapshot from the post-check).
156
# The endpoint is called through the client so an unreachable state raises (wmill.get_flow_user_state returns None).
157
# ponytail: the fallback is one map per user, so concurrent standalone runs can race; flows don't.
158
# The key is the switch's eAPI address: arista_eos-<host>-<port> with non-word characters in the host as _, the port
159
# always in (443 or 80 when unset). State written before the port was part of the key is read from arista_eos_<host>
160
# as a fallback (all word characters, so never a new key); writes go to the new key only.
161
def state_keys(auth: arista_eos):
162
    url = switch_url(auth)
163
    port = int(url.port or auth.get("port") or {"https": 443, "http": 80}[url.scheme])
164
    return [
165
        f"arista_eos-{word(url.hostname)}-{port}",
166
        f"arista_eos_{word(auth['host'])}",
167
    ]
168

169

170
def word(s):
171
    return re.sub(r"\W", "_", s)
172

173

174
def load_state(keys):
175
    for key in keys:
176
        if os.environ.get("WM_FLOW_JOB_ID"):
177
            c = wmill.Windmill()
178
            value = c.get(
179
                f"/w/{c.workspace}/jobs/flow/user_states/{c.get_root_job_id()}/{key}"
180
            ).json()
181
        else:
182
            value = (wmill.get_state(standalone_state()) or {}).get(key)
183
        if value is not None:
184
            return value
185

186

187
def save_state(keys, value):
188
    if os.environ.get("WM_FLOW_JOB_ID"):
189
        c = wmill.Windmill()
190
        c.post(
191
            f"/w/{c.workspace}/jobs/flow/user_states/{c.get_root_job_id()}/{keys[0]}",
192
            json=value,
193
        )
194
    else:
195
        path = standalone_state()
196
        wmill.set_state({**(wmill.get_state(path) or {}), keys[0]: value}, path)
197

198

199
def standalone_state():
200
    return f"u/{os.environ['WM_USERNAME']}/arista_eos_state"
201

202

203
# "4.34.9M-51132493.4349M (engineering build)" on cEOS, "4.34.9M" on hardware.
204
def short_version(v):
205
    m = re.match(r"\d+\.\d+\.\d+(\.\d+)?[A-Z]*", v)
206
    return m.group(0) if m else v
207