Edits history of script submission #24213 for ' Device activate (arista_eos)'

  • python3
    One script reply has been approved by the moderators
    Ap­pro­ved
    # py312
    # requirements:
    # pyeapi==1.0.4
    # wmill==1.851.0
    
    # Device activate: one attempt at staging the target image on one switch, no reload.
    # Records the image of the running version as the rollback target (once per run, before any change), copies the
    # image to flash (skipped when already there), compares verify /sha512 with the expected digest, then boot system.
    # The image is image_url, or image_base_url + EOS-<target_version>.swi; the digest is sha512, or the .sha512 file next
    # to the image. A file already on flash that doesn't match (a partial copy left by a killed run) is replaced by one
    # copy; a freshly copied file that doesn't match is deleted and ActivateFailed raised, so a retry copies it again.
    # An image file name that carries a version must carry target_version. Device reload checks boot-config against the
    # image recorded here. State is the root flow run's user state under arista_eos-<host>-<port>.
    
    import logging
    import os
    import re
    import ssl
    import urllib.error
    import urllib.request
    from typing import TypedDict
    from urllib.parse import urlsplit
    
    import pyeapi
    import wmill
    
    
    class arista_eos(TypedDict):
        host: str
        username: str
        password: str
        port: int
        verify_tls: bool
        allow_http: bool
    
    
    class ActivateFailed(Exception):
        pass
    
    
    def main(
        auth: arista_eos,
        target_version: str,
        image_base_url: str = "",
        image_url: str = "",
        sha512: str = "",
        vrf: str = "",
    ):
        """
        Device activate
        Copy the target image to the switch's flash (image_url, or EOS-<target_version>.swi from image_base_url), verify its sha512 (the sha512 input or the .sha512 file next to the image), and boot system it. No reload. Returns { status: done, steps, previous_image }. Raises ActivateFailed, DeviceUnreachable or AuthFailure.
        """
        key = state_keys(auth)
        state = load_state(key) or {}
        # Set again only once this activate has booted the image: a stale one would let reload through.
        forget_activated(key, state)
        if not target_version or not (image_base_url or image_url):
            raise ActivateFailed(
                "target_version and image_base_url (or image_url) are required"
            )
        try:
            image, url = image_source(target_version, image_base_url, image_url)
        except ValueError as e:
            raise ActivateFailed(str(e))
        # A file name that carries a version must carry this one; one without (a lab image) is taken as is.
        if image_version(image) not in (None, target_version):
            raise ActivateFailed(
                f"{image} is named for {image_version(image)}, not target_version {target_version}"
            )
        steps = []
        previous = state.get("previous_image") or ""
        try:
            if sha512:
                expected = digest(sha512)
                if not expected:
                    raise ActivateFailed("sha512 must be the 128-hex digest of the image")
            else:
                try:
                    with urllib.request.urlopen(f"{url}.sha512", timeout=60) as r:
                        expected = digest(r.read().decode(errors="replace"))
                except urllib.error.HTTPError as e:
                    raise ActivateFailed(
                        f"cannot read {url}.sha512 ({e.code}): pass sha512"
                    )
                except OSError as e:
                    reason = getattr(e, "reason", e)
                    # Unlike the switch's copy, this fetch checks the certificate: the digest is what activate trusts.
                    trust = (
                        ", or trust the image server's CA on the worker (SSL_CERT_FILE)"
                        if isinstance(reason, ssl.SSLCertVerificationError)
                        else ""
                    )
                    raise ActivateFailed(
                        f"cannot read {url}.sha512 ({reason}): pass sha512{trust}"
                    )
                if not expected:
                    raise ActivateFailed(f"no sha512 hash in {url}.sha512")
    
            d, boot, version = eapi(
                auth,
                [
                    {"cmd": "dir flash:/", "revision": 2},
                    "show boot-config",
                    "show version",
                ],
            )
            files = list(d["urls"]["flash:/"]["entries"])
            running = short_version(version["version"])
            booted = boot["softwareImage"].replace("flash:/", "")
            # boot-config is the running image (first run) or the target (an earlier run got past boot system).
            # Anything else is a stale boot-config this step won't guess around.
            if image_version(booted) != running and booted != image:
                raise ActivateFailed(
                    f"boot-config points to {boot['softwareImage']}, neither the running version {running} nor {image}: fix boot-config first"
                )
            # The rollback target is the image of the running version, recorded once per run before anything changes.
            if not previous and running != target_version:
                candidates = (
                    [booted]
                    if booted != image
                    else [
                        f
                        for f in files
                        if f.endswith(".swi") and image_version(f) == running
                    ]
                )
                if len(candidates) != 1 or candidates[0] not in files:
                    raise ActivateFailed(
                        f"cannot tell which image on flash runs {running} (candidates: {', '.join(candidates) or 'none'}): no rollback target"
                    )
                previous = f"flash:/{candidates[0]}"
                save_state(key, {**state, "previous_image": previous})
    
            def copy():
                # The copy runs on the switch and can take minutes for a ~1 GB image.
                eapi(
                    auth,
                    [*([f"cli vrf {vrf}"] if vrf else []), f"copy {url} flash:{image}"],
                    "json",
                    3600,
                )
                steps.append(f"copied {url}")
    
            def verify():
                [v] = eapi(auth, [f"verify /sha512 flash:{image}"])
                return digest((v.get("messages") or [""])[0])
    
            copied = image not in files
            if copied:
                copy()
            actual = verify()
            if actual != expected and not copied:
                # Most likely a partial copy left by a killed run: replace it.
                eapi(auth, [f"delete flash:{image}"])
                copy()
                actual = verify()
            if actual != expected:
                # An earlier run may have booted it already: never leave boot-config on a deleted image.
                restored = booted == image and previous
                if restored:
                    boot_system(auth, previous.replace("flash:/", ""))
                eapi(auth, [f"delete flash:{image}"])
                raise ActivateFailed(
                    f"sha512 mismatch for {image} (switch {actual or 'none'}, expected {expected}), deleted it"
                    + (f", boot-config back on {previous}" if restored else "")
                )
            steps.append("sha512 verified")
            steps.extend(boot_system(auth, image))
        except (DeviceUnreachable, AuthFailure, ActivateFailed):
            raise
        except Exception as e:
            raise ActivateFailed(str(e))
        # Device reload checks boot-config against this, so it needs no image_url of its own.
        save_state(
            key,
            {**state, "previous_image": previous, "activated_image": f"flash:/{image}"},
        )
        return {"status": "done", "steps": steps, "previous_image": previous}
    
    
    # Same function in Device pre-check and Device activate.
    # image_url wins; else image_base_url + EOS-<target>.swi. Returns (file name on flash, URL or ""). The URL goes into
    # the switch's copy command, so it is an allowlist: http(s), then only A-Z a-z 0-9 . _ ~ : / % + - (no query, no
    # user:password@, nothing the CLI reads as a separator), ending in a plain .swi file name. Raises ValueError otherwise.
    def image_source(target, base, image_url):
        if image_url:
            image, url = urlsplit(image_url).path.rsplit("/", 1)[-1], image_url
        else:
            image = f"EOS-{target}.swi" if target else ""
            url = f"{base.rstrip('/')}/{image}" if image and base else ""
        if not url:
            return image, url
        # Not echoed: it would put the password in the steps and errors.
        if "@" in url:
            raise ValueError("the image URL must not contain @ (no user:password@ in it)")
        if not re.match(r"https?://", url):
            raise ValueError(
                f"the image URL must start with http:// or https://: got {url!r}"
            )
        if not urlsplit(url).hostname:
            raise ValueError(f"the image URL has no host: got {url!r}")
        bad = sorted(set(re.sub(r"[A-Za-z0-9._~:/%+-]", "", url.split("://", 1)[1])))
        if bad:
            raise ValueError(
                f"the image URL may only contain A-Z a-z 0-9 . _ ~ : / % + -, not {' '.join(map(repr, bad))}: got {url!r}"
            )
        if not re.fullmatch(r"[\w.+-]+\.swi", image):
            raise ValueError(f"the image URL must end in a .swi file name: got {url!r}")
        return image, url
    
    
    def boot_system(auth, image):
        _, set_, _, after = eapi(
            auth, ["configure", f"boot system flash:{image}", "end", "show boot-config"]
        )
        if after["softwareImage"] != f"flash:/{image}":
            raise Exception(f"boot-config still points to {after['softwareImage']}")
        return [f"boot system flash:{image}", *(set_.get("warnings") or [])]
    
    
    def forget_activated(key, state):
        if state.pop("activated_image", None):
            save_state(key, state)
    
    
    # "EOS64-4.34.9M.swi" -> "4.34.9M"
    def image_version(file):
        m = re.search(r"-(\d+\.\d+\.\d+(\.\d+)?[A-Z]*)\.swi$", file)
        return m.group(1) if m else None
    
    
    # The 128-hex digest in a sha512sum line, a BSD "SHA512 (f) = <hash>" line or verify /sha512 output.
    def digest(text):
        m = re.search(r"\b[0-9a-fA-F]{128}\b", text)
        return m.group(0).lower() if m else ""
    
    
    # Errors the workflow branches and retries on (retry_if: result.error.name == "DeviceUnreachable").
    class DeviceUnreachable(Exception):
        pass
    
    
    class AuthFailure(Exception):
        pass
    
    
    # pyeapi logs a traceback for every socket error, and a rebooting switch gives many.
    logging.getLogger("pyeapi").addHandler(logging.NullHandler())
    
    
    def eapi(auth: arista_eos, cmds: list, fmt: str = "json", timeout: float = 60):
        eos = node(auth, timeout)
        try:
            return eos.run_commands(cmds, fmt)
        except pyeapi.eapilib.ConnectionError as e:
            raise connection_error(auth, eos, e)
    
    
    # pyeapi 1.0.4 raises ConnectionError for a socket error (kept in socket_error), an answer that isn't JSON (eAPI
    # still starting, 5xx) and a 401, the only one with neither.
    def connection_error(auth, eos, e):
        if eos.connection.socket_error is None and e.message != "unable to connect to eAPI":
            return AuthFailure(f"{auth['host']}: eAPI refused the credential ({e.message})")
        return DeviceUnreachable(f"{auth['host']}: {e.message}")
    
    
    # host is an address or a URL (https://10.0.0.11:8443); a bare IPv6 address is bracketed. Only https, or http when the
    # resource ticks allow_http (Basic auth then crosses the network in cleartext). pyeapi skips certificate checks unless it
    # is given a context, so one is always passed: verifying, or not when the resource unticks verify_tls (EOS ships a
    # self-signed cert).
    def switch_url(auth: arista_eos):
        host = auth["host"]
        if "://" not in host:
            host = (
                f"https://[{host}]"
                if host.count(":") > 1 and "[" not in host
                else f"https://{host}"
            )
        url = urlsplit(host)
        if not url.hostname or (
            url.scheme != "https" and not (url.scheme == "http" and auth.get("allow_http"))
        ):
            raise ValueError(
                f"host must be an address or an https:// URL (http:// needs allow_http on the resource): {auth['host']}"
            )
        return url
    
    
    def node(auth: arista_eos, timeout: float = 60):
        url = switch_url(auth)
        context = ssl.create_default_context()
        if auth.get("verify_tls") is False:
            context.check_hostname = False
            context.verify_mode = ssl.CERT_NONE
        return pyeapi.connect(
            transport=url.scheme,
            host=url.hostname,
            port=url.port or auth.get("port") or None,
            username=auth["username"],
            password=auth["password"],
            context=context,
            timeout=timeout,
            return_node=True,
        )
    
    
    # Shared by the Device scripts: the root flow run's user state, keyed by switch, holds the pre-check snapshot, the
    # image activate replaced (rollback target), the image it booted and the reload markers. Every step of the run (nested
    # subflows and retried attempts included) reads the same entry. Outside a flow the Device scripts share one state
    # resource in the running user's home (a script's own state would keep the pre-check snapshot from the post-check).
    # The endpoint is called through the client so an unreachable state raises (wmill.get_flow_user_state returns None).
    # ponytail: the fallback is one map per user, so concurrent standalone runs can race; flows don't.
    # The key is the switch's eAPI address: arista_eos-<host>-<port> with non-word characters in the host as _, the port
    # always in (443 or 80 when unset). State written before the port was part of the key is read from arista_eos_<host>
    # as a fallback (all word characters, so never a new key); writes go to the new key only.
    def state_keys(auth: arista_eos):
        url = switch_url(auth)
        port = int(url.port or auth.get("port") or {"https": 443, "http": 80}[url.scheme])
        return [
            f"arista_eos-{word(url.hostname)}-{port}",
            f"arista_eos_{word(auth['host'])}",
        ]
    
    
    def word(s):
        return re.sub(r"\W", "_", s)
    
    
    def load_state(keys):
        for key in keys:
            if os.environ.get("WM_FLOW_JOB_ID"):
                c = wmill.Windmill()
                value = c.get(
                    f"/w/{c.workspace}/jobs/flow/user_states/{c.get_root_job_id()}/{key}"
                ).json()
            else:
                value = (wmill.get_state(standalone_state()) or {}).get(key)
            if value is not None:
                return value
    
    
    def save_state(keys, value):
        if os.environ.get("WM_FLOW_JOB_ID"):
            c = wmill.Windmill()
            c.post(
                f"/w/{c.workspace}/jobs/flow/user_states/{c.get_root_job_id()}/{keys[0]}",
                json=value,
            )
        else:
            path = standalone_state()
            wmill.set_state({**(wmill.get_state(path) or {}), keys[0]: value}, path)
    
    
    def standalone_state():
        return f"u/{os.environ['WM_USERNAME']}/arista_eos_state"
    
    
    # "4.34.9M-51132493.4349M (engineering build)" on cEOS, "4.34.9M" on hardware.
    def short_version(v):
        m = re.match(r"\d+\.\d+\.\d+(\.\d+)?[A-Z]*", v)
        return m.group(0) if m else v
    

    Submitted by hugo989 1 day ago