Blame SOURCES/gen_updates2.py

9d39cf
#! /usr/bin/python
9d39cf
# SPDX-License-Identifier: CC0-1.0
9d39cf
9d39cf
import argparse
9d39cf
import errno
cc944f
import fnmatch
9d39cf
import io
9d39cf
import itertools
9d39cf
import os
9d39cf
import re
9d39cf
import shutil
9d39cf
import struct
9d39cf
import sys
cc944f
import tarfile
9d39cf
import tempfile
9d39cf
from subprocess import PIPE, Popen, STDOUT
9d39cf
9d39cf
# Python 3 shims
9d39cf
try:
9d39cf
    from functools import reduce
9d39cf
except:
9d39cf
    pass
9d39cf
try:
9d39cf
    from itertools import zip_longest as izip_longest
9d39cf
except:
9d39cf
    from itertools import izip_longest
9d39cf
9d39cf
# revs:
9d39cf
# [ { "path", "cpuid", "pf", "rev", "date" } ]
9d39cf
9d39cf
# artifacts:
9d39cf
#  * content summary (per-file)
9d39cf
#   * overlay summary (per-fms/pf)
9d39cf
#  * changelog (per-file?)
9d39cf
#  * discrepancies (per-fms/pf)
9d39cf
9d39cf
log_level = 0
9d39cf
print_date = False
cc944f
file_glob = ["*??-??-??", "*microcode*.dat"]
9d39cf
9d39cf
9d39cf
def log_status(msg, level=0):
9d39cf
    global log_level
9d39cf
9d39cf
    if log_level >= level:
9d39cf
        sys.stderr.write(msg + "\n")
9d39cf
9d39cf
9d39cf
def log_info(msg, level=2):
9d39cf
    global log_level
9d39cf
9d39cf
    if log_level >= level:
9d39cf
        sys.stderr.write("INFO: " + msg + "\n")
9d39cf
9d39cf
9d39cf
def log_warn(msg, level=1):
9d39cf
    global log_level
9d39cf
9d39cf
    if log_level >= level:
9d39cf
        sys.stderr.write("WARNING: " + msg + "\n")
9d39cf
9d39cf
9d39cf
def log_error(msg, level=-1):
9d39cf
    global log_level
9d39cf
9d39cf
    if log_level >= level:
9d39cf
        sys.stderr.write("ERROR: " + msg + "\n")
9d39cf
9d39cf
9d39cf
def remove_prefix(text, prefix):
9d39cf
    if isinstance(prefix, str):
9d39cf
        prefix = [prefix, ]
9d39cf
9d39cf
    for p in prefix:
9d39cf
        pfx = p if p.endswith(os.sep) else p + os.sep
9d39cf
        if text.startswith(pfx):
9d39cf
            return text[len(pfx):]
9d39cf
9d39cf
    return text
9d39cf
9d39cf
9d39cf
def file_walk(args, yield_dirs=False):
9d39cf
    for content in args:
9d39cf
        if os.path.isdir(content):
9d39cf
            if yield_dirs:
9d39cf
                yield ("", content)
9d39cf
            for root, dirs, files in os.walk(content):
9d39cf
                if yield_dirs:
9d39cf
                    for f in dirs:
9d39cf
                        p = os.path.join(root, f)
9d39cf
                        yield (remove_prefix(p, content), p)
9d39cf
                for f in files:
9d39cf
                    p = os.path.join(root, f)
9d39cf
                    yield (remove_prefix(p, content), p)
9d39cf
        elif os.path.exists(content):
9d39cf
            yield ("", content)
9d39cf
        else:
9d39cf
            raise IOError(errno.ENOENT, os.strerror(errno.ENOENT), content)
9d39cf
9d39cf
9d39cf
def cpuid_fname(c):
cc944f
    # Note that the Extended Family is summed up with the Family,
cc944f
    # while the Extended Model is concatenated with the Model.
9d39cf
    return "%02x-%02x-%02x" % (
cc944f
        ((c >> 20) & 0xff) + ((c >> 8) & 0xf),
9d39cf
        ((c >> 12) & 0xf0) + ((c >> 4) & 0xf),
9d39cf
        c & 0xf)
9d39cf
9d39cf
cc944f
def read_revs_dir(path, args, src=None, ret=None):
9d39cf
    if ret is None:
9d39cf
        ret = []
9d39cf
9d39cf
    ucode_re = re.compile('[0-9a-f]{2}-[0-9a-f]{2}-0[0-9a-f]$')
9d39cf
    ucode_dat_re = re.compile('microcode.*\.dat$')
9d39cf
9d39cf
    for rp, ap in file_walk([path, ]):
9d39cf
        rp_fname = os.path.basename(rp)
9d39cf
        if not ucode_re.match(rp_fname) and not ucode_dat_re.match(rp_fname):
9d39cf
            continue
9d39cf
9d39cf
        # Text-based format
9d39cf
        data = None
9d39cf
        if ucode_dat_re.match(rp_fname):
9d39cf
            data = io.BytesIO()
9d39cf
            with open(ap, "r") as f:
9d39cf
                for line in f:
9d39cf
                    if line.startswith("/"):
9d39cf
                        continue
9d39cf
                    vals = line.split(",")
9d39cf
                    for val in vals:
9d39cf
                        val = val.strip()
9d39cf
                        if not val:
9d39cf
                            continue
9d39cf
                        data.write(struct.pack("
9d39cf
            sz = data.seek(0, os.SEEK_CUR)
9d39cf
            data.seek(0, os.SEEK_SET)
9d39cf
        else:
9d39cf
            sz = os.stat(ap).st_size
9d39cf
9d39cf
        try:
9d39cf
            with data or open(ap, "rb") as f:
9d39cf
                log_info("Processing %s" % ap)
9d39cf
                offs = 0
9d39cf
                while offs < sz:
9d39cf
                    f.seek(offs, os.SEEK_SET)
9d39cf
                    hdr = struct.unpack("IiIIIIIIIIII", f.read(48))
9d39cf
                    ret.append({"path": rp, "src": src or path,
9d39cf
                                "cpuid": hdr[3], "pf": hdr[6], "rev": hdr[1],
9d39cf
                                "date": hdr[2], "offs": offs, "cksum": hdr[4],
9d39cf
                                "data_size": hdr[7], "total_size": hdr[8]})
9d39cf
9d39cf
                    if hdr[8] and hdr[8] - hdr[7] > 48:
9d39cf
                        f.seek(hdr[7], os.SEEK_CUR)
9d39cf
                        ext_tbl = struct.unpack("IIIII", f.read(20))
9d39cf
                        log_status("Found %u extended signatures for %s:%#x" %
9d39cf
                                   (ext_tbl[0], rp, offs), level=1)
9d39cf
9d39cf
                        cur_offs = offs + hdr[7] + 48 + 20
9d39cf
                        ext_sig_cnt = 0
9d39cf
                        while cur_offs < offs + hdr[8] \
9d39cf
                                and ext_sig_cnt <= ext_tbl[0]:
9d39cf
                            ext_sig = struct.unpack("III", f.read(12))
cc944f
                            ignore = args.ignore_ext_dups and \
cc944f
                                (ext_sig[0] == hdr[3])
cc944f
                            if not ignore:
cc944f
                                ret.append({"path": rp, "src": src or path,
cc944f
                                            "cpuid": ext_sig[0],
cc944f
                                            "pf": ext_sig[1],
cc944f
                                            "rev": hdr[1], "date": hdr[2],
cc944f
                                            "offs": offs, "ext_offs": cur_offs,
cc944f
                                            "cksum": hdr[4],
cc944f
                                            "ext_cksum": ext_sig[2],
cc944f
                                            "data_size": hdr[7],
cc944f
                                            "total_size": hdr[8]})
9d39cf
                            log_status(("Got ext sig %#x/%#x for " +
cc944f
                                        "%s:%#x:%#x/%#x%s") %
cc944f
                                       (ext_sig[0], ext_sig[1],
cc944f
                                        rp, offs, hdr[3], hdr[6],
cc944f
                                        " (ignored)" if ignore else ""),
cc944f
                                       level=2)
9d39cf
9d39cf
                            cur_offs += 12
9d39cf
                            ext_sig_cnt += 1
9d39cf
9d39cf
                    offs += hdr[8] or 2048
9d39cf
        except Exception as e:
9d39cf
            log_error("a problem occurred while processing %s: %s" % (ap, e),
9d39cf
                      level=1)
9d39cf
9d39cf
    return ret
9d39cf
9d39cf
cc944f
def read_revs_rpm(path, args, ret=None):
9d39cf
    if ret is None:
9d39cf
        ret = []
9d39cf
9d39cf
    dir_tmp = tempfile.mkdtemp()
9d39cf
9d39cf
    log_status("Trying to extract files from RPM \"%s\"..." % path,
9d39cf
               level=1)
9d39cf
9d39cf
    rpm2cpio = Popen(args=["rpm2cpio", path], stdout=PIPE, stderr=PIPE,
9d39cf
                     close_fds=True)
cc944f
    cpio = Popen(args=["cpio", "-idmv"] + file_glob,
9d39cf
                 cwd=dir_tmp, stdin=rpm2cpio.stdout,
9d39cf
                 stdout=PIPE, stderr=STDOUT)
9d39cf
    out, cpio_stderr = cpio.communicate()
9d39cf
    rpm2cpio_out, rpm2cpio_err = rpm2cpio.communicate()
9d39cf
9d39cf
    rpm2cpio_ret = rpm2cpio.returncode
9d39cf
    cpio_ret = cpio.returncode
9d39cf
9d39cf
    log_info("rpm2cpio exit code: %d, cpio exit code: %d" %
9d39cf
             (rpm2cpio_ret, cpio_ret))
9d39cf
    if rpm2cpio_err:
9d39cf
        log_info("rpm2cpio stderr:\n%s" % rpm2cpio_err, level=3)
9d39cf
    if out:
9d39cf
        log_info("cpio output:\n%s" % out, level=3)
9d39cf
    if cpio_stderr:
9d39cf
        log_info("cpio stderr:\n%s" % cpio_stderr, level=3)
9d39cf
9d39cf
    if rpm2cpio_ret == 0 and cpio_ret == 0:
cc944f
        ret = read_revs_dir(dir_tmp, args, path)
9d39cf
9d39cf
    shutil.rmtree(dir_tmp)
9d39cf
9d39cf
    return ret
9d39cf
9d39cf
cc944f
def read_revs_tar(path, args, ret=None):
cc944f
    if ret is None:
cc944f
        ret = []
cc944f
cc944f
    dir_tmp = tempfile.mkdtemp()
cc944f
cc944f
    log_status("Trying to extract files from tarball \"%s\"..." % path,
cc944f
               level=1)
cc944f
cc944f
    try:
cc944f
        with tarfile.open(path, "r:*") as tar:
cc944f
            for ti in tar:
cc944f
                if any(fnmatch.fnmatchcase(ti.name, p) for p in file_glob):
cc944f
                    d = os.path.normpath(os.path.join("/",
cc944f
                                         os.path.dirname(ti.name)))
cc944f
                    # For now, strip exactl one level
cc944f
                    d = os.path.join(*(d.split(os.path.sep)[2:]))
cc944f
                    n = os.path.join(d, os.path.basename(ti.name))
cc944f
cc944f
                    if not os.path.exists(d):
cc944f
                        os.makedirs(d)
cc944f
                    t = tar.extractfile(ti)
cc944f
                    with open(n, "wb") as f:
cc944f
                        shutil.copyfileobj(t, f)
cc944f
                    t.close()
cc944f
cc944f
        ret = read_revs_dir(dir_tmp, args, path)
cc944f
    except Exception as err:
cc944f
        log_error("Error while reading \"%s\" as a tarball: \"%s\"" %
cc944f
                  (path, str(err)))
cc944f
cc944f
    shutil.rmtree(dir_tmp)
cc944f
cc944f
    return ret
cc944f
cc944f
cc944f
def read_revs(path, args, ret=None):
9d39cf
    if ret is None:
9d39cf
        ret = []
9d39cf
    if os.path.isdir(path):
cc944f
        return read_revs_dir(path, args, ret)
cc944f
    elif tarfile.is_tarfile(path):
cc944f
        return read_revs_tar(path, args, ret)
9d39cf
    else:
cc944f
        return read_revs_rpm(path, args, ret)
9d39cf
9d39cf
9d39cf
def gen_mc_map(mc_data, merge=False, merge_path=False):
9d39cf
    """
9d39cf
    Converts an array of microcode file information to a map with path/sig/pf
9d39cf
    as a key.
9d39cf
9d39cf
    merge: whether to leave only the newest mc variant in the map or leave all
9d39cf
           possible variants.
9d39cf
    """
9d39cf
    res = dict()
9d39cf
9d39cf
    for mc in mc_data:
9d39cf
        key = (None if merge_path else mc["path"], mc["cpuid"], mc["pf"])
9d39cf
9d39cf
        if key not in res:
9d39cf
            res[key] = dict()
9d39cf
9d39cf
        cpuid = mc["cpuid"]
9d39cf
        cur_pf = mc["pf"]
9d39cf
        pid = 1
9d39cf
        while cur_pf > 0:
9d39cf
            if cur_pf & 1 and not (merge and pid in res[key]
9d39cf
                                   and res[key][pid]["rev"][0] >= mc["rev"]):
9d39cf
                if pid not in res[cpuid] or merge:
9d39cf
                    res[cpuid][pid] = []
9d39cf
                res[cpuid][pid].append(mc)
9d39cf
9d39cf
            cur_pf = cur_pf / 2
9d39cf
            pid = pid * 2
9d39cf
9d39cf
    return res
9d39cf
9d39cf
9d39cf
def gen_fn_map(mc_data, merge=False, merge_path=False):
9d39cf
    res = dict()
9d39cf
9d39cf
    for mc in mc_data:
9d39cf
        key = (None if merge_path else mc["path"], mc["cpuid"], mc["pf"])
9d39cf
        if key in res:
9d39cf
            log_warn("Duplicate path/cpuid/pf: %s/%#x/%#x" % key)
9d39cf
        else:
9d39cf
            res[key] = []
9d39cf
        if merge and len(res[key]):
9d39cf
            if mc["rev"] > res[key][0]["rev"]:
9d39cf
                res[key][0] = mc
9d39cf
        else:
9d39cf
            res[key].append(mc)
9d39cf
9d39cf
    return res
9d39cf
9d39cf
9d39cf
def revcmp(a, b):
9d39cf
    return b["rev"] - a["rev"]
9d39cf
9d39cf
9d39cf
class ChangeLogEntry:
9d39cf
    ADDED = 0
9d39cf
    REMOVED = 1
9d39cf
    UPDATED = 2
9d39cf
    DOWNGRADED = 3
9d39cf
    OTHER = 4
9d39cf
9d39cf
9d39cf
def mc_stripped_path(mc):
9d39cf
    paths = ("usr/share/microcode_ctl/ucode_with_caveats/intel",
9d39cf
             "usr/share/microcode_ctl/ucode_with_caveats",
9d39cf
             "usr/share/microcode_ctl",
9d39cf
             "lib/firmware",
9d39cf
             "etc/firmware",
9d39cf
             )
9d39cf
9d39cf
    return remove_prefix(mc["path"], paths)
9d39cf
9d39cf
9d39cf
class mcnm:
9d39cf
    MCNM_ABBREV = 0
9d39cf
    MCNM_FAMILIES = 1
9d39cf
    MCNM_MODELS = 2
9d39cf
    MCNM_FAMILIES_MODELS = 3
9d39cf
    MCNM_CODENAME = 4
9d39cf
9d39cf
cc944f
def get_mc_cnames(mc, cmap, mode=mcnm.MCNM_ABBREV, stringify=True,
cc944f
                  segment=False):
9d39cf
    if not isinstance(mc, dict):
9d39cf
        mc = mc_from_mc_key(mc)
9d39cf
    sig = mc["cpuid"]
9d39cf
    pf = mc["pf"]
9d39cf
    res = []
9d39cf
9d39cf
    if not cmap:
9d39cf
        return None
9d39cf
    if sig not in cmap:
9d39cf
        log_info("No codename information for sig %#x" % sig)
9d39cf
        return None
9d39cf
9d39cf
    cnames = cmap[sig]
9d39cf
9d39cf
    if mode in (mcnm.MCNM_FAMILIES, mcnm.MCNM_MODELS,
9d39cf
                mcnm.MCNM_FAMILIES_MODELS):
9d39cf
        for c in cnames:
9d39cf
            if not (pf & c["pf_mask"]):
9d39cf
                continue
9d39cf
            for m, f in ((mcnm.MCNM_FAMILIES, "families"),
9d39cf
                         (mcnm.MCNM_MODELS, "models")):
9d39cf
                if m & mode == 0:
9d39cf
                    continue
9d39cf
                if f not in c or not c[f]:
9d39cf
                    log_info("No %s for sig %#x in %r" % (f, sig, c))
9d39cf
                    continue
9d39cf
9d39cf
                res.append(c[f])
9d39cf
9d39cf
        return ", ".join(res) or None
9d39cf
9d39cf
    steppings = dict()
9d39cf
    suffices = dict()
9d39cf
    for c in cnames:
9d39cf
        if pf and not (pf & c["pf_mask"]):
9d39cf
            continue
9d39cf
9d39cf
        if mode == mcnm.MCNM_ABBREV and "abbrev" in c and c["abbrev"]:
9d39cf
            cname = c["abbrev"]
9d39cf
        else:
9d39cf
            cname = c["codename"]
9d39cf
cc944f
        if segment:
cc944f
            cname = c["segment"] + " " + cname
cc944f
9d39cf
        if cname not in suffices:
9d39cf
            suffices[cname] = set()
9d39cf
        if "variant" in c and c["variant"]:
9d39cf
            suffices[cname] |= set(c["variant"])
9d39cf
9d39cf
        if cname not in steppings:
9d39cf
            steppings[cname] = set()
9d39cf
        if c["stepping"]:
9d39cf
            steppings[cname] |= set(c["stepping"])
9d39cf
9d39cf
    for cname in sorted(steppings.keys()):
cc944f
        cname_res = [cname]
9d39cf
        if len(suffices[cname]):
cc944f
            cname_res[0] += "-" + "/".join(sorted(suffices[cname]))
9d39cf
        if len(steppings[cname]):
cc944f
            cname_res.append("/".join(sorted(steppings[cname])))
cc944f
        res.append(" ".join(cname_res) if stringify else cname_res)
9d39cf
cc944f
    return (", ".join(res) or None) if stringify else res
9d39cf
9d39cf
9d39cf
def mc_from_mc_key(k):
9d39cf
    return dict(zip(("path", "cpuid", "pf"), k))
9d39cf
9d39cf
cc944f
def mc_path(mc, pf_sfx=True, midword=None, cmap=None, cname_segment=False):
9d39cf
    if not isinstance(mc, dict):
9d39cf
        mc = mc_from_mc_key(mc)
9d39cf
    path = mc_stripped_path(mc) if mc["path"] is not None else None
9d39cf
    cpuid_fn = cpuid_fname(mc["cpuid"])
9d39cf
    fname = os.path.basename(mc["path"] or cpuid_fn)
9d39cf
    midword = "" if midword is None else " " + midword
cc944f
    cname = get_mc_cnames(mc, cmap, segment=cname_segment)
9d39cf
    cname_str = " (" + cname + ")" if cname else ""
9d39cf
9d39cf
    if pf_sfx:
9d39cf
        sfx = "/0x%02x" % mc["pf"]
9d39cf
    else:
9d39cf
        sfx = ""
9d39cf
9d39cf
    if not path or path == os.path.join("intel-ucode", cpuid_fn):
9d39cf
        return "%s%s%s%s" % (fname, sfx, cname_str, midword)
9d39cf
    else:
9d39cf
        return "%s%s%s%s (in %s)" % (cpuid_fn, sfx, cname_str, midword, path)
9d39cf
9d39cf
9d39cf
def gen_changelog_file(old, new):
9d39cf
    pass
9d39cf
9d39cf
9d39cf
def mc_cmp(old_mc, new_mc):
9d39cf
    res = []
9d39cf
9d39cf
    old_mc_revs = [x["rev"] for x in old_mc]
9d39cf
    new_mc_revs = [x["rev"] for x in new_mc]
9d39cf
    common = set(old_mc_revs) & set(new_mc_revs)
9d39cf
    old_rev_list = [x for x in sorted(old_mc_revs) if x not in common]
9d39cf
    new_rev_list = [x for x in sorted(new_mc_revs) if x not in common]
9d39cf
9d39cf
    if len(old_rev_list) != 1 or len(new_rev_list) != 1:
9d39cf
        for i in new_mc:
9d39cf
            if i["rev"] in new_rev_list:
9d39cf
                res.append((ChangeLogEntry.ADDED, None, i))
9d39cf
        for i in old_mc:
9d39cf
            if i["rev"] in old_rev_list:
9d39cf
                res.append((ChangeLogEntry.REMOVED, i, None))
9d39cf
    else:
9d39cf
        for old in old_mc:
9d39cf
            if old["rev"] == old_rev_list[0]:
9d39cf
                break
9d39cf
        for new in new_mc:
9d39cf
            if new["rev"] == new_rev_list[0]:
9d39cf
                break
9d39cf
        if new["rev"] > old["rev"]:
9d39cf
            res.append((ChangeLogEntry.UPDATED, old, new))
9d39cf
        elif new["rev"] < old["rev"]:
9d39cf
            res.append((ChangeLogEntry.DOWNGRADED, old, new))
9d39cf
9d39cf
    return res
9d39cf
9d39cf
9d39cf
def gen_changelog(old, new):
9d39cf
    res = []
9d39cf
9d39cf
    old_map = gen_fn_map(old)
9d39cf
    new_map = gen_fn_map(new)
9d39cf
9d39cf
    old_files = set(old_map.keys())
9d39cf
    new_files = set(new_map.keys())
9d39cf
9d39cf
    both = old_files & new_files
9d39cf
    added = new_files - old_files
9d39cf
    removed = old_files - new_files
9d39cf
9d39cf
    for f in sorted(added):
9d39cf
        p = mc_path(new_map[f][0])
9d39cf
        for old_f in sorted(removed):
9d39cf
            old_p = mc_path(old_map[old_f][0])
9d39cf
            if p == old_p and f[1] == old_f[1] and f[2] == old_f[2]:
9d39cf
                log_info("Matched %s (%s and %s)" %
9d39cf
                         (p, old_map[old_f][0]["path"], new_map[f][0]["path"]))
9d39cf
                added.remove(f)
9d39cf
                removed.remove(old_f)
9d39cf
9d39cf
                res += mc_cmp(old_map[old_f], new_map[f])
9d39cf
9d39cf
    for f in sorted(added):
9d39cf
        for i in new_map[f]:
9d39cf
            res.append((ChangeLogEntry.ADDED, None, i))
9d39cf
    for f in sorted(removed):
9d39cf
        for i in old_map[f]:
9d39cf
            res.append((ChangeLogEntry.REMOVED, i, None))
9d39cf
    for f in sorted(both):
9d39cf
        res += mc_cmp(old_map[f], new_map[f])
9d39cf
9d39cf
    return res
9d39cf
9d39cf
9d39cf
def mc_date(mc):
9d39cf
    if isinstance(mc, dict):
9d39cf
        mc = mc["date"]
9d39cf
    return "%04x-%02x-%02x" % (mc & 0xffff, mc >> 24, (mc >> 16) & 0xff)
9d39cf
9d39cf
9d39cf
def mc_rev(mc, date=None):
9d39cf
    '''
9d39cf
    While revision is signed for comparison purposes, historically
9d39cf
    it is printed as unsigned,  Oh well.
9d39cf
    '''
9d39cf
    global print_date
9d39cf
9d39cf
    if mc["rev"] < 0:
9d39cf
        rev = 2**32 + mc["rev"]
9d39cf
    else:
9d39cf
        rev = mc["rev"]
9d39cf
9d39cf
    if date if date is not None else print_date:
9d39cf
        return "%#x (%s)" % (rev, mc_date(mc))
9d39cf
    else:
9d39cf
        return "%#x" % rev
9d39cf
9d39cf
cc944f
def print_changelog_rpm(clog, cmap, args):
cc944f
    for e, old, new in clog:
cc944f
        mc_str = mc_path(new if e == ChangeLogEntry.ADDED else old,
cc944f
                         midword="microcode",
cc944f
                         cmap=cmap, cname_segment=args.segment)
cc944f
9d39cf
        if e == ChangeLogEntry.ADDED:
cc944f
            print("Addition of %s at revision %s" % (mc_str, mc_rev(new)))
9d39cf
        elif e == ChangeLogEntry.REMOVED:
cc944f
            print("Removal of %s at revision %s" % (mc_str, mc_rev(old)))
9d39cf
        elif e == ChangeLogEntry.UPDATED:
9d39cf
            print("Update of %s from revision %s up to %s" %
cc944f
                  (mc_str, mc_rev(old), mc_rev(new)))
9d39cf
        elif e == ChangeLogEntry.DOWNGRADED:
9d39cf
                print("Downgrade of %s from revision %s down to %s" %
cc944f
                      (mc_str, mc_rev(old), mc_rev(new)))
9d39cf
        elif e == ChangeLogEntry.OTHER:
9d39cf
            print("Other change in %s:" % old["path"])
9d39cf
            print("  old: %#x/%#x: rev %s (offs %#x)" %
9d39cf
                  (old["cpuid"], old["pf"], mc_rev(old), old["offs"]))
9d39cf
            print("  new: %#x/%#x: rev %s (offs %#x)" %
9d39cf
                  (new["cpuid"], new["pf"], mc_rev(new), new["offs"]))
9d39cf
9d39cf
cc944f
def print_changelog_intel(clog, cmap, args):
cc944f
    def clog_sort_key(x):
cc944f
        res = str(x[0])
cc944f
cc944f
        if x[0] != ChangeLogEntry.ADDED:
cc944f
            res += "%08x%02x" % (x[1]["cpuid"], x[1]["pf"])
cc944f
        else:
cc944f
            res += "0" * 10
cc944f
cc944f
        if x[0] != ChangeLogEntry.REMOVED:
cc944f
            res += "%08x%02x" % (x[2]["cpuid"], x[2]["pf"])
cc944f
        else:
cc944f
            res += "0" * 10
cc944f
cc944f
        return res
cc944f
cc944f
    sorted_clog = sorted(clog, key=clog_sort_key)
cc944f
    sections = (("New Platforms",     (ChangeLogEntry.ADDED, )),
cc944f
                ("Updated Platforms", (ChangeLogEntry.UPDATED,
cc944f
                                       ChangeLogEntry.DOWNGRADED)),
cc944f
                ("Removed Platforms", (ChangeLogEntry.REMOVED, )))
cc944f
cc944f
    def print_line(e, old, new, types):
cc944f
        if e not in types:
cc944f
            return
cc944f
cc944f
        if not print_line.hdr:
cc944f
            print("""
cc944f
| Processor      | Stepping | F-M-S/PI    | Old Ver  | New Ver  | Products
cc944f
|:---------------|:---------|:------------|:---------|:---------|:---------""")
cc944f
            print_line.hdr = True
cc944f
cc944f
        mc = new if e == ChangeLogEntry.ADDED else old
cc944f
        cnames = get_mc_cnames(mc, cmap, stringify=False,
cc944f
                               segment=args.segment) or (("???", ""), )
cc944f
        for cn in cnames:
cc944f
            cname = cn[0]
cc944f
            stepping = cn[1] if len(cn) > 1 else ""
cc944f
        print("| %-14s | %-8s | %8s/%02x | %8s | %8s | %s" %
cc944f
              (cname,
cc944f
               stepping,
cc944f
               cpuid_fname(mc["cpuid"]), mc["pf"],
cc944f
               ("%08x" % old["rev"]) if e != ChangeLogEntry.ADDED else "",
cc944f
               ("%08x" % new["rev"]) if e != ChangeLogEntry.REMOVED else "",
cc944f
               get_mc_cnames(mc, cmap, mode=mcnm.MCNM_FAMILIES,
cc944f
                             segment=args.segment) or ""))
cc944f
cc944f
    for h, types in sections:
cc944f
        print("\n### %s" % h)
cc944f
        print_line.hdr = False
cc944f
        for e, old, new in sorted_clog:
cc944f
            print_line(e, old, new, types)
cc944f
cc944f
cc944f
def print_changelog(clog, cmap, args):
cc944f
    if args.format == "rpm":
cc944f
        print_changelog_rpm(clog, cmap, args)
cc944f
    elif args.format == "intel":
cc944f
        print_changelog_intel(clog, cmap, args)
cc944f
    else:
cc944f
        log_error(("unknown changelog format: \"%s\". " +
cc944f
                   "Supported formats are: rpm, intel.") % args.format)
cc944f
cc944f
9d39cf
class TableStyles:
9d39cf
    TS_CSV = 0
9d39cf
    TS_FANCY = 1
9d39cf
9d39cf
9d39cf
def print_line(line, column_sz):
9d39cf
    print(" | ".join([str(x).ljust(column_sz[i])
9d39cf
                      for i, x in zip(itertools.count(),
9d39cf
                                      itertools.chain(line,
9d39cf
                                      [""] * (len(column_sz) -
9d39cf
                                              len(line))))]).rstrip())
9d39cf
9d39cf
9d39cf
def print_table(items, header=[], style=TableStyles.TS_CSV):
9d39cf
    if style == TableStyles.TS_CSV:
9d39cf
        for i in items:
9d39cf
            print(";".join(i))
9d39cf
    elif style == TableStyles.TS_FANCY:
9d39cf
        column_sz = list(reduce(lambda x, y:
9d39cf
                                map(max, izip_longest(x, y, fillvalue=0)),
9d39cf
                                [[len(x) for x in i]
9d39cf
                                 for i in itertools.chain(header, items)]))
9d39cf
        for i in header:
9d39cf
            print_line(i, column_sz)
9d39cf
        if header:
9d39cf
            print("-+-".join(["-" * x for x in column_sz]))
9d39cf
        for i in items:
9d39cf
            print_line(i, column_sz)
9d39cf
9d39cf
9d39cf
def print_summary(revs, cmap, args):
9d39cf
    m = gen_fn_map(revs)
9d39cf
    cnames_mode = mcnm.MCNM_ABBREV if args.abbrev else mcnm.MCNM_CODENAME
9d39cf
9d39cf
    header = []
9d39cf
    if args.header:
cc944f
        header.append(["Path", "Offset", "Ext. Offset", "Data Size",
cc944f
                       "Total Size", "CPUID", "Platform ID Mask", "Revision",
cc944f
                       "Date", "Checksum", "Codenames"] +
9d39cf
                      (["Models"] if args.models else []))
9d39cf
    tbl = []
9d39cf
    for k in sorted(m.keys()):
9d39cf
        for mc in m[k]:
9d39cf
            tbl.append([mc_stripped_path(mc),
9d39cf
                        "0x%x" % mc["offs"],
9d39cf
                        "0x%x" % mc["ext_offs"] if "ext_offs" in mc else "-",
cc944f
                        "0x%05x" % mc["data_size"],
cc944f
                        "0x%05x" % mc["total_size"],
9d39cf
                        "0x%05x" % mc["cpuid"],
9d39cf
                        "0x%02x" % mc["pf"],
9d39cf
                        mc_rev(mc, date=False),
9d39cf
                        mc_date(mc),
cc944f
                        "0x%08x" % (mc["ext_cksum"]
cc944f
                                    if "ext_cksum" in mc else mc["cksum"]),
cc944f
                        get_mc_cnames(mc, cmap, cnames_mode,
cc944f
                                      segment=args.segment) or ""] +
9d39cf
                       ([get_mc_cnames(mc, cmap,
cc944f
                                       mcnm.MCNM_FAMILIES_MODELS,
cc944f
                                       segment=args.segment)]
9d39cf
                        if args.models else []))
9d39cf
9d39cf
    print_table(tbl, header, style=TableStyles.TS_FANCY)
9d39cf
9d39cf
9d39cf
def read_codenames_file(path):
9d39cf
    '''
9d39cf
    Supports two formats: new and old
9d39cf
     * old: tab-separated. Field order:
9d39cf
       Segment, (unused), Codename, (dash-separated) Stepping,
9d39cf
       Platform ID mask, CPUID, (unused) Update link, (unused) Specs link
9d39cf
     * new: semicolon-separated; support comments.  Distinguished
9d39cf
       by the first line that starts with octothorp.  Field order:
9d39cf
       Segment, Unused, Codename, Stepping, Platform ID mask, CPUID,
9d39cf
       Abbreviation, Variant(s), Families, Models
9d39cf
    '''
9d39cf
    old_fields = ["segment", "_", "codename", "stepping", "pf_mask", "sig",
9d39cf
                  "_update", "_specs"]
9d39cf
    new_fields = ["segment", "_", "codename", "stepping", "pf_mask", "sig",
9d39cf
                  "abbrev", "variant", "families", "models"]
9d39cf
    new_fmt = False
9d39cf
    field_names = old_fields
9d39cf
9d39cf
    res = dict()
9d39cf
9d39cf
    try:
9d39cf
        with open(path, "r") as f:
9d39cf
            for line in f:
9d39cf
                line = line.strip()
9d39cf
                if len(line) == 0:
9d39cf
                    continue
9d39cf
                if line[0] == '#':
9d39cf
                    new_fmt = True
9d39cf
                    field_names = new_fields
9d39cf
                    continue
9d39cf
9d39cf
                fields = line.split(";" if new_fmt else "\t",
9d39cf
                                    1 + len(field_names))
9d39cf
                fields = dict(zip(field_names, fields))
9d39cf
                if "sig" not in fields:
9d39cf
                    log_warn("Skipping %r (from \"%s\")" % (fields, line))
9d39cf
                    continue
9d39cf
9d39cf
                sig = fields["sig"] = int(fields["sig"], 16)
9d39cf
                fields["pf_mask"] = int(fields["pf_mask"], 16)
9d39cf
                fields["stepping"] = fields["stepping"].split(",")
9d39cf
                if "variant" in fields:
9d39cf
                    if fields["variant"]:
9d39cf
                        fields["variant"] = fields["variant"].split(",")
9d39cf
                    else:
9d39cf
                        fields["variant"] = []
9d39cf
9d39cf
                if sig not in res:
9d39cf
                    res[sig] = list()
9d39cf
                res[sig].append(fields)
9d39cf
    except Exception as e:
9d39cf
        log_error("a problem occurred while reading code names: %s" % e)
9d39cf
9d39cf
    return res
9d39cf
9d39cf
9d39cf
def print_discrepancies(rev_map, deps, cmap, args):
9d39cf
    """
9d39cf
    rev_map: dict "name": revs
9d39cf
    deps: list of tuples (name, parent/None)
9d39cf
    """
9d39cf
    sigs = set()
9d39cf
9d39cf
    for p, r in rev_map.items():
9d39cf
        sigs |= set(r.keys())
9d39cf
9d39cf
    if args.header:
9d39cf
        header1 = ["sig"]
9d39cf
        if args.print_vs:
9d39cf
            header2 = [""]
9d39cf
        for p, n, d in deps:
9d39cf
            header1.append(n)
9d39cf
            if args.print_vs:
9d39cf
                add = ""
9d39cf
                if d:
9d39cf
                    for pd, nd, dd in deps:
9d39cf
                        if pd == d:
9d39cf
                            add = "(vs. %s)" % nd
9d39cf
                            break
9d39cf
                header2.append(add)
9d39cf
        if args.models:
9d39cf
            header1.append("Model names")
9d39cf
            if args.print_vs:
9d39cf
                header2.append("")
9d39cf
    header = [header1] + ([header2] if args.print_vs else [])
9d39cf
9d39cf
    tbl = []
9d39cf
    for s in sorted(sigs):
9d39cf
        out = [mc_path(s)]
9d39cf
        print_out = not args.print_filter
9d39cf
        print_date = args.min_date is None
9d39cf
9d39cf
        for p, n, d in deps:
9d39cf
            cur = dict([(x["rev"], x) for x in rev_map[p][s]]) \
9d39cf
                  if s in rev_map[p] else []
9d39cf
            v = "/".join([mc_rev(y) for x, y in sorted(cur.items())]) \
9d39cf
                if cur else "-"
9d39cf
            if d is not None:
9d39cf
                prev = [x["rev"] for x in rev_map[d][s]] if s in rev_map[d] \
9d39cf
                        else []
9d39cf
                if [x for x in cur if x not in prev]:
9d39cf
                    v += " (*)"
9d39cf
                    print_out = True
9d39cf
            if args.min_date is not None and s in rev_map[p]:
9d39cf
                for x in rev_map[p][s]:
9d39cf
                    print_date |= mc_date(x) > args.min_date
9d39cf
            out.append(v)
9d39cf
9d39cf
        if print_out and print_date:
9d39cf
            if args.models:
cc944f
                out.append(get_mc_cnames(s, cmap, segment=args.segment) or "")
9d39cf
            tbl.append(out)
9d39cf
9d39cf
    print_table(tbl, header, style=TableStyles.TS_FANCY)
9d39cf
9d39cf
9d39cf
def cmd_summary(args):
9d39cf
    revs = []
9d39cf
    for p in args.filelist:
cc944f
        revs = read_revs(p, args, ret=revs)
9d39cf
9d39cf
    codenames_map = read_codenames_file(args.codenames)
9d39cf
9d39cf
    print_summary(revs, codenames_map, args)
9d39cf
9d39cf
    return 0
9d39cf
9d39cf
9d39cf
def cmd_changelog(args):
9d39cf
    codenames_map = read_codenames_file(args.codenames)
9d39cf
    base_path = args.filelist[0]
9d39cf
    upd_path = args.filelist[1]
9d39cf
cc944f
    base = read_revs(base_path, args)
cc944f
    upd = read_revs(upd_path, args)
9d39cf
9d39cf
    print_changelog(gen_changelog(base, upd), codenames_map, args)
9d39cf
9d39cf
    return 0
9d39cf
9d39cf
9d39cf
def cmd_discrepancies(args):
9d39cf
    """
9d39cf
    filenames:
9d39cf
     * "<" prefix (possibly multiple times) to refer to a previous entry
9d39cf
       to compare against
9d39cf
     * "[name]" prefix is a name reference
9d39cf
    """
9d39cf
    codenames_map = read_codenames_file(args.codenames)
9d39cf
    rev_map = dict()
9d39cf
    deps = list()
9d39cf
    cur = -1
9d39cf
9d39cf
    for path in args.filelist:
9d39cf
        orig_path = path
9d39cf
        name = None
9d39cf
        cur += 1
9d39cf
        dep = None
9d39cf
        while True:
9d39cf
            if path[0] == '<':
9d39cf
                path = path[1:]
9d39cf
                dep = cur - 1 if dep is None else dep - 1
9d39cf
            elif path[0] == '[' and path.find(']') > 0:
9d39cf
                pos = path.find(']')
9d39cf
                name = path[1:pos]
9d39cf
                path = path[pos + 1:]
9d39cf
            else:
9d39cf
                break
9d39cf
        if name is None:
9d39cf
            name = path
9d39cf
        if dep is not None and dep < 0:
9d39cf
            log_error("Incorrect dep reference for '%s' (points to index %d)" %
9d39cf
                      (orig_path, dep))
9d39cf
            return 1
9d39cf
        deps.append((path, name, deps[dep][0] if dep is not None else None))
cc944f
        rev_map[path] = gen_fn_map(read_revs(path, args), merge=args.merge,
9d39cf
                                   merge_path=True)
9d39cf
9d39cf
    print_discrepancies(rev_map, deps, codenames_map, args)
9d39cf
9d39cf
    return 0
9d39cf
9d39cf
9d39cf
def parse_cli():
9d39cf
    root_parser = argparse.ArgumentParser(prog="gen_updates",
9d39cf
                                          description="Intel CPU Microcode " +
9d39cf
                                          "parser")
9d39cf
    root_parser.add_argument("-C", "--codenames", default='codenames',
9d39cf
                             help="Code names file")
9d39cf
    root_parser.add_argument("-v", "--verbose", action="count", default=0,
9d39cf
                             help="Increase output verbosity")
cc944f
    root_parser.add_argument("-E", "--no-ignore-ext-duplicates",
cc944f
                             action="store_const", dest="ignore_ext_dups",
cc944f
                             default=False, const=False,
cc944f
                             help="Do not ignore duplicates of the main " +
cc944f
                                  "signature in the extended signature header")
cc944f
    root_parser.add_argument("-e", "--ignore-ext-duplicates",
cc944f
                             action="store_const", dest="ignore_ext_dups",
cc944f
                             const=True,
cc944f
                             help="Ignore duplicates of the main signature " +
cc944f
                                   "in the extended signature header")
cc944f
    root_parser.add_argument("-t", "--print-segment", action="store_const",
cc944f
                             dest="segment", const=True,
cc944f
                             help="Print model segment")
cc944f
    root_parser.add_argument("-T", "--no-print-segment", action="store_const",
cc944f
                             dest="segment", const=False, default=False,
cc944f
                             help="Do not print model segment")
9d39cf
9d39cf
    cmdparsers = root_parser.add_subparsers(title="Commands",
9d39cf
                                            help="main gen_updates commands")
9d39cf
9d39cf
    parser_s = cmdparsers.add_parser("summary",
9d39cf
                                     help="Generate microcode summary")
9d39cf
    parser_s.add_argument("-a", "--abbreviate", action="store_const",
9d39cf
                          dest="abbrev", const=True, default=True,
9d39cf
                          help="Abbreviate code names")
9d39cf
    parser_s.add_argument("-A", "--no-abbreviate", action="store_const",
9d39cf
                          dest="abbrev", const=False,
9d39cf
                          help="Do not abbreviate code names")
9d39cf
    parser_s.add_argument("-m", "--print-models", action="store_const",
9d39cf
                          dest="models", const=True, default=False,
9d39cf
                          help="Print models")
9d39cf
    parser_s.add_argument("-M", "--no-print-models",
9d39cf
                          action="store_const", dest="models",
9d39cf
                          const=False, help="Do not print models")
9d39cf
    parser_s.add_argument("-H", "--no-print-header",
9d39cf
                          action="store_const", dest="header",
9d39cf
                          const=False, default=True,
9d39cf
                          help="Do not print hader")
9d39cf
    parser_s.add_argument("filelist", nargs="*", default=[],
9d39cf
                          help="List or RPMs/directories to process")
9d39cf
    parser_s.set_defaults(func=cmd_summary)
9d39cf
9d39cf
    parser_c = cmdparsers.add_parser("changelog",
9d39cf
                                     help="Generate changelog")
cc944f
    parser_c.add_argument("-F", "--format", choices=["rpm", "intel"],
cc944f
                          default="rpm", help="Changelog format")
9d39cf
    parser_c.add_argument("filelist", nargs=2,
9d39cf
                          help="RPMs/directories to compare")
9d39cf
    parser_c.set_defaults(func=cmd_changelog)
9d39cf
9d39cf
    parser_d = cmdparsers.add_parser("discrepancies",
9d39cf
                                     help="Generate discrepancies")
9d39cf
    parser_d.add_argument("-s", "--merge-revs", action="store_const",
9d39cf
                          dest="merge", const=True, default=False,
9d39cf
                          help="Merge revisions that come" +
9d39cf
                               " from different files")
9d39cf
    parser_d.add_argument("-S", "--no-merge-revs", action="store_const",
9d39cf
                          dest="merge", const=False,
9d39cf
                          help="Do not Merge revisions that come" +
9d39cf
                               " from different files")
9d39cf
    parser_d.add_argument("-v", "--print-vs", action="store_const",
9d39cf
                          dest="print_vs", const=True, default=False,
9d39cf
                          help="Print base version ")
9d39cf
    parser_d.add_argument("-V", "--no-print-vs", action="store_const",
9d39cf
                          dest="print_vs", const=False,
9d39cf
                          help="Do not Merge revisions that come" +
9d39cf
                               " from different files")
9d39cf
    parser_d.add_argument("-m", "--print-models", action="store_const",
9d39cf
                          dest="models", const=True, default=True,
9d39cf
                          help="Print model names")
9d39cf
    parser_d.add_argument("-M", "--no-print-models", action="store_const",
9d39cf
                          dest="models", const=False,
9d39cf
                          help="Do not print model names")
9d39cf
    parser_d.add_argument("-H", "--no-print-header", action="store_const",
9d39cf
                          dest="header", const=False, default=True,
9d39cf
                          help="Do not print hader")
9d39cf
    parser_d.add_argument("-a", "--print-all-files", action="store_const",
9d39cf
                          dest="print_filter", const=False, default=True,
9d39cf
                          help="Print all files")
9d39cf
    parser_d.add_argument("-c", "--print-changed-files", action="store_const",
9d39cf
                          dest="print_filter", const=True,
9d39cf
                          help="Print only changed files")
9d39cf
    parser_d.add_argument("-d", "--min-date", action="store",
9d39cf
                          help="Minimum date filter")
9d39cf
    parser_d.add_argument("filelist", nargs='*',
9d39cf
                          help="RPMs/directories to compare")
9d39cf
    parser_d.set_defaults(func=cmd_discrepancies)
9d39cf
9d39cf
    args = root_parser.parse_args()
9d39cf
    if not hasattr(args, "func"):
9d39cf
        root_parser.print_help()
9d39cf
        return None
cc944f
cc944f
    global log_level
cc944f
    log_level = args.verbose
cc944f
9d39cf
    return args
9d39cf
9d39cf
9d39cf
def main():
9d39cf
    args = parse_cli()
9d39cf
    if args is None:
9d39cf
        return 1
9d39cf
9d39cf
    return args.func(args)
9d39cf
9d39cf
9d39cf
if __name__ == "__main__":
9d39cf
    sys.exit(main())