Blame SOURCES/gen_updates2.py

36ce7f
#! /usr/bin/python
36ce7f
# SPDX-License-Identifier: CC0-1.0
36ce7f
36ce7f
import argparse
36ce7f
import errno
36ce7f
import io
36ce7f
import itertools
36ce7f
import os
36ce7f
import re
36ce7f
import shutil
36ce7f
import struct
36ce7f
import sys
36ce7f
import tempfile
36ce7f
from subprocess import PIPE, Popen, STDOUT
36ce7f
36ce7f
# Python 3 shims
36ce7f
try:
36ce7f
    from functools import reduce
36ce7f
except:
36ce7f
    pass
36ce7f
try:
36ce7f
    from itertools import zip_longest as izip_longest
36ce7f
except:
36ce7f
    from itertools import izip_longest
36ce7f
36ce7f
# revs:
36ce7f
# [ { "path", "cpuid", "pf", "rev", "date" } ]
36ce7f
36ce7f
# artifacts:
36ce7f
#  * content summary (per-file)
36ce7f
#   * overlay summary (per-fms/pf)
36ce7f
#  * changelog (per-file?)
36ce7f
#  * discrepancies (per-fms/pf)
36ce7f
36ce7f
log_level = 0
36ce7f
print_date = False
36ce7f
36ce7f
36ce7f
def log_status(msg, level=0):
36ce7f
    global log_level
36ce7f
36ce7f
    if log_level >= level:
36ce7f
        sys.stderr.write(msg + "\n")
36ce7f
36ce7f
36ce7f
def log_info(msg, level=2):
36ce7f
    global log_level
36ce7f
36ce7f
    if log_level >= level:
36ce7f
        sys.stderr.write("INFO: " + msg + "\n")
36ce7f
36ce7f
36ce7f
def log_warn(msg, level=1):
36ce7f
    global log_level
36ce7f
36ce7f
    if log_level >= level:
36ce7f
        sys.stderr.write("WARNING: " + msg + "\n")
36ce7f
36ce7f
36ce7f
def log_error(msg, level=-1):
36ce7f
    global log_level
36ce7f
36ce7f
    if log_level >= level:
36ce7f
        sys.stderr.write("ERROR: " + msg + "\n")
36ce7f
36ce7f
36ce7f
def remove_prefix(text, prefix):
36ce7f
    if isinstance(prefix, str):
36ce7f
        prefix = [prefix, ]
36ce7f
36ce7f
    for p in prefix:
36ce7f
        pfx = p if p.endswith(os.sep) else p + os.sep
36ce7f
        if text.startswith(pfx):
36ce7f
            return text[len(pfx):]
36ce7f
36ce7f
    return text
36ce7f
36ce7f
36ce7f
def file_walk(args, yield_dirs=False):
36ce7f
    for content in args:
36ce7f
        if os.path.isdir(content):
36ce7f
            if yield_dirs:
36ce7f
                yield ("", content)
36ce7f
            for root, dirs, files in os.walk(content):
36ce7f
                if yield_dirs:
36ce7f
                    for f in dirs:
36ce7f
                        p = os.path.join(root, f)
36ce7f
                        yield (remove_prefix(p, content), p)
36ce7f
                for f in files:
36ce7f
                    p = os.path.join(root, f)
36ce7f
                    yield (remove_prefix(p, content), p)
36ce7f
        elif os.path.exists(content):
36ce7f
            yield ("", content)
36ce7f
        else:
36ce7f
            raise IOError(errno.ENOENT, os.strerror(errno.ENOENT), content)
36ce7f
36ce7f
36ce7f
def cpuid_fname(c):
36ce7f
    return "%02x-%02x-%02x" % (
36ce7f
        ((c >> 16) & 0xff0) + ((c >> 8) & 0xf),
36ce7f
        ((c >> 12) & 0xf0) + ((c >> 4) & 0xf),
36ce7f
        c & 0xf)
36ce7f
36ce7f
36ce7f
def read_revs_dir(path, src=None, ret=None):
36ce7f
    if ret is None:
36ce7f
        ret = []
36ce7f
36ce7f
    ucode_re = re.compile('[0-9a-f]{2}-[0-9a-f]{2}-0[0-9a-f]$')
36ce7f
    ucode_dat_re = re.compile('microcode.*\.dat$')
36ce7f
36ce7f
    for rp, ap in file_walk([path, ]):
36ce7f
        rp_fname = os.path.basename(rp)
36ce7f
        if not ucode_re.match(rp_fname) and not ucode_dat_re.match(rp_fname):
36ce7f
            continue
36ce7f
36ce7f
        # Text-based format
36ce7f
        data = None
36ce7f
        if ucode_dat_re.match(rp_fname):
36ce7f
            data = io.BytesIO()
36ce7f
            with open(ap, "r") as f:
36ce7f
                for line in f:
36ce7f
                    if line.startswith("/"):
36ce7f
                        continue
36ce7f
                    vals = line.split(",")
36ce7f
                    for val in vals:
36ce7f
                        val = val.strip()
36ce7f
                        if not val:
36ce7f
                            continue
36ce7f
                        data.write(struct.pack("
36ce7f
            sz = data.seek(0, os.SEEK_CUR)
36ce7f
            data.seek(0, os.SEEK_SET)
36ce7f
        else:
36ce7f
            sz = os.stat(ap).st_size
36ce7f
36ce7f
        try:
36ce7f
            with data or open(ap, "rb") as f:
36ce7f
                log_info("Processing %s" % ap)
36ce7f
                offs = 0
36ce7f
                while offs < sz:
36ce7f
                    f.seek(offs, os.SEEK_SET)
36ce7f
                    hdr = struct.unpack("IiIIIIIIIIII", f.read(48))
36ce7f
                    ret.append({"path": rp, "src": src or path,
36ce7f
                                "cpuid": hdr[3], "pf": hdr[6], "rev": hdr[1],
36ce7f
                                "date": hdr[2], "offs": offs, "cksum": hdr[4],
36ce7f
                                "data_size": hdr[7], "total_size": hdr[8]})
36ce7f
36ce7f
                    if hdr[8] and hdr[8] - hdr[7] > 48:
36ce7f
                        f.seek(hdr[7], os.SEEK_CUR)
36ce7f
                        ext_tbl = struct.unpack("IIIII", f.read(20))
36ce7f
                        log_status("Found %u extended signatures for %s:%#x" %
36ce7f
                                   (ext_tbl[0], rp, offs), level=1)
36ce7f
36ce7f
                        cur_offs = offs + hdr[7] + 48 + 20
36ce7f
                        ext_sig_cnt = 0
36ce7f
                        while cur_offs < offs + hdr[8] \
36ce7f
                                and ext_sig_cnt <= ext_tbl[0]:
36ce7f
                            ext_sig = struct.unpack("III", f.read(12))
36ce7f
                            ret.append({"path": rp, "src": src or path,
36ce7f
                                        "cpuid": ext_sig[0], "pf": ext_sig[1],
36ce7f
                                        "rev": hdr[1], "date": hdr[2],
36ce7f
                                        "offs": offs, "ext_offs": cur_offs,
36ce7f
                                        "cksum": hdr[4],
36ce7f
                                        "ext_cksum": ext_sig[2],
36ce7f
                                        "data_size": hdr[7],
36ce7f
                                        "total_size": hdr[8]})
36ce7f
                            log_status(("Got ext sig %#x/%#x for " +
36ce7f
                                        "%s:%#x:%#x/%#x") %
36ce7f
                                       (ext_sig[0], ext_sig[1], rp, offs,
36ce7f
                                        hdr[3], hdr[6]), level=2)
36ce7f
36ce7f
                            cur_offs += 12
36ce7f
                            ext_sig_cnt += 1
36ce7f
36ce7f
                    offs += hdr[8] or 2048
36ce7f
        except Exception as e:
36ce7f
            log_error("a problem occurred while processing %s: %s" % (ap, e),
36ce7f
                      level=1)
36ce7f
36ce7f
    return ret
36ce7f
36ce7f
36ce7f
def read_revs_rpm(path, ret=None):
36ce7f
    if ret is None:
36ce7f
        ret = []
36ce7f
36ce7f
    dir_tmp = tempfile.mkdtemp()
36ce7f
36ce7f
    log_status("Trying to extract files from RPM \"%s\"..." % path,
36ce7f
               level=1)
36ce7f
36ce7f
    rpm2cpio = Popen(args=["rpm2cpio", path], stdout=PIPE, stderr=PIPE,
36ce7f
                     close_fds=True)
36ce7f
    cpio = Popen(args=["cpio", "-idmv", "*??-??-??", "*microcode*.dat"],
36ce7f
                 cwd=dir_tmp, stdin=rpm2cpio.stdout,
36ce7f
                 stdout=PIPE, stderr=STDOUT)
36ce7f
    out, cpio_stderr = cpio.communicate()
36ce7f
    rpm2cpio_out, rpm2cpio_err = rpm2cpio.communicate()
36ce7f
36ce7f
    rpm2cpio_ret = rpm2cpio.returncode
36ce7f
    cpio_ret = cpio.returncode
36ce7f
36ce7f
    log_info("rpm2cpio exit code: %d, cpio exit code: %d" %
36ce7f
             (rpm2cpio_ret, cpio_ret))
36ce7f
    if rpm2cpio_err:
36ce7f
        log_info("rpm2cpio stderr:\n%s" % rpm2cpio_err, level=3)
36ce7f
    if out:
36ce7f
        log_info("cpio output:\n%s" % out, level=3)
36ce7f
    if cpio_stderr:
36ce7f
        log_info("cpio stderr:\n%s" % cpio_stderr, level=3)
36ce7f
36ce7f
    if rpm2cpio_ret == 0 and cpio_ret == 0:
36ce7f
        ret = read_revs_dir(dir_tmp, path)
36ce7f
36ce7f
    shutil.rmtree(dir_tmp)
36ce7f
36ce7f
    return ret
36ce7f
36ce7f
36ce7f
def read_revs(path, ret=None):
36ce7f
    if ret is None:
36ce7f
        ret = []
36ce7f
    if os.path.isdir(path):
36ce7f
        return read_revs_dir(path, ret)
36ce7f
    else:
36ce7f
        return read_revs_rpm(path, ret)
36ce7f
36ce7f
36ce7f
def gen_mc_map(mc_data, merge=False, merge_path=False):
36ce7f
    """
36ce7f
    Converts an array of microcode file information to a map with path/sig/pf
36ce7f
    as a key.
36ce7f
36ce7f
    merge: whether to leave only the newest mc variant in the map or leave all
36ce7f
           possible variants.
36ce7f
    """
36ce7f
    res = dict()
36ce7f
36ce7f
    for mc in mc_data:
36ce7f
        key = (None if merge_path else mc["path"], mc["cpuid"], mc["pf"])
36ce7f
36ce7f
        if key not in res:
36ce7f
            res[key] = dict()
36ce7f
36ce7f
        cpuid = mc["cpuid"]
36ce7f
        cur_pf = mc["pf"]
36ce7f
        pid = 1
36ce7f
        while cur_pf > 0:
36ce7f
            if cur_pf & 1 and not (merge and pid in res[key]
36ce7f
                                   and res[key][pid]["rev"][0] >= mc["rev"]):
36ce7f
                if pid not in res[cpuid] or merge:
36ce7f
                    res[cpuid][pid] = []
36ce7f
                res[cpuid][pid].append(mc)
36ce7f
36ce7f
            cur_pf = cur_pf / 2
36ce7f
            pid = pid * 2
36ce7f
36ce7f
    return res
36ce7f
36ce7f
36ce7f
def gen_fn_map(mc_data, merge=False, merge_path=False):
36ce7f
    res = dict()
36ce7f
36ce7f
    for mc in mc_data:
36ce7f
        key = (None if merge_path else mc["path"], mc["cpuid"], mc["pf"])
36ce7f
        if key in res:
36ce7f
            log_warn("Duplicate path/cpuid/pf: %s/%#x/%#x" % key)
36ce7f
        else:
36ce7f
            res[key] = []
36ce7f
        if merge and len(res[key]):
36ce7f
            if mc["rev"] > res[key][0]["rev"]:
36ce7f
                res[key][0] = mc
36ce7f
        else:
36ce7f
            res[key].append(mc)
36ce7f
36ce7f
    return res
36ce7f
36ce7f
36ce7f
def revcmp(a, b):
36ce7f
    return b["rev"] - a["rev"]
36ce7f
36ce7f
36ce7f
class ChangeLogEntry:
36ce7f
    ADDED = 0
36ce7f
    REMOVED = 1
36ce7f
    UPDATED = 2
36ce7f
    DOWNGRADED = 3
36ce7f
    OTHER = 4
36ce7f
36ce7f
36ce7f
def mc_stripped_path(mc):
36ce7f
    paths = ("usr/share/microcode_ctl/ucode_with_caveats/intel",
36ce7f
             "usr/share/microcode_ctl/ucode_with_caveats",
36ce7f
             "usr/share/microcode_ctl",
36ce7f
             "lib/firmware",
36ce7f
             "etc/firmware",
36ce7f
             )
36ce7f
36ce7f
    return remove_prefix(mc["path"], paths)
36ce7f
36ce7f
36ce7f
class mcnm:
36ce7f
    MCNM_ABBREV = 0
36ce7f
    MCNM_FAMILIES = 1
36ce7f
    MCNM_MODELS = 2
36ce7f
    MCNM_FAMILIES_MODELS = 3
36ce7f
    MCNM_CODENAME = 4
36ce7f
36ce7f
36ce7f
def get_mc_cnames(mc, cmap, mode=mcnm.MCNM_ABBREV):
36ce7f
    if not isinstance(mc, dict):
36ce7f
        mc = mc_from_mc_key(mc)
36ce7f
    sig = mc["cpuid"]
36ce7f
    pf = mc["pf"]
36ce7f
    res = []
36ce7f
36ce7f
    if not cmap:
36ce7f
        return None
36ce7f
    if sig not in cmap:
36ce7f
        log_info("No codename information for sig %#x" % sig)
36ce7f
        return None
36ce7f
36ce7f
    cnames = cmap[sig]
36ce7f
36ce7f
    if mode in (mcnm.MCNM_FAMILIES, mcnm.MCNM_MODELS,
36ce7f
                mcnm.MCNM_FAMILIES_MODELS):
36ce7f
        for c in cnames:
36ce7f
            if not (pf & c["pf_mask"]):
36ce7f
                continue
36ce7f
            for m, f in ((mcnm.MCNM_FAMILIES, "families"),
36ce7f
                         (mcnm.MCNM_MODELS, "models")):
36ce7f
                if m & mode == 0:
36ce7f
                    continue
36ce7f
                if f not in c or not c[f]:
36ce7f
                    log_info("No %s for sig %#x in %r" % (f, sig, c))
36ce7f
                    continue
36ce7f
36ce7f
                res.append(c[f])
36ce7f
36ce7f
        return ", ".join(res) or None
36ce7f
36ce7f
    steppings = dict()
36ce7f
    suffices = dict()
36ce7f
    for c in cnames:
36ce7f
        if pf and not (pf & c["pf_mask"]):
36ce7f
            continue
36ce7f
36ce7f
        if mode == mcnm.MCNM_ABBREV and "abbrev" in c and c["abbrev"]:
36ce7f
            cname = c["abbrev"]
36ce7f
        else:
36ce7f
            cname = c["codename"]
36ce7f
36ce7f
        if cname not in suffices:
36ce7f
            suffices[cname] = set()
36ce7f
        if "variant" in c and c["variant"]:
36ce7f
            suffices[cname] |= set(c["variant"])
36ce7f
36ce7f
        if cname not in steppings:
36ce7f
            steppings[cname] = set()
36ce7f
        if c["stepping"]:
36ce7f
            steppings[cname] |= set(c["stepping"])
36ce7f
36ce7f
    for cname in sorted(steppings.keys()):
36ce7f
        cname_str = cname
36ce7f
        if len(suffices[cname]):
36ce7f
            cname_str += "-" + "/".join(sorted(suffices[cname]))
36ce7f
        if len(steppings[cname]):
36ce7f
            cname_str += " " + "/".join(sorted(steppings[cname]))
36ce7f
        res.append(cname_str)
36ce7f
36ce7f
    return ", ".join(res) or None
36ce7f
36ce7f
36ce7f
def mc_from_mc_key(k):
36ce7f
    return dict(zip(("path", "cpuid", "pf"), k))
36ce7f
36ce7f
36ce7f
def mc_path(mc, pf_sfx=True, midword=None, cmap=None):
36ce7f
    if not isinstance(mc, dict):
36ce7f
        mc = mc_from_mc_key(mc)
36ce7f
    path = mc_stripped_path(mc) if mc["path"] is not None else None
36ce7f
    cpuid_fn = cpuid_fname(mc["cpuid"])
36ce7f
    fname = os.path.basename(mc["path"] or cpuid_fn)
36ce7f
    midword = "" if midword is None else " " + midword
36ce7f
    cname = get_mc_cnames(mc, cmap)
36ce7f
    cname_str = " (" + cname + ")" if cname else ""
36ce7f
36ce7f
    if pf_sfx:
36ce7f
        sfx = "/0x%02x" % mc["pf"]
36ce7f
    else:
36ce7f
        sfx = ""
36ce7f
36ce7f
    if not path or path == os.path.join("intel-ucode", cpuid_fn):
36ce7f
        return "%s%s%s%s" % (fname, sfx, cname_str, midword)
36ce7f
    else:
36ce7f
        return "%s%s%s%s (in %s)" % (cpuid_fn, sfx, cname_str, midword, path)
36ce7f
36ce7f
36ce7f
def gen_changelog_file(old, new):
36ce7f
    pass
36ce7f
36ce7f
36ce7f
def mc_cmp(old_mc, new_mc):
36ce7f
    res = []
36ce7f
36ce7f
    old_mc_revs = [x["rev"] for x in old_mc]
36ce7f
    new_mc_revs = [x["rev"] for x in new_mc]
36ce7f
    common = set(old_mc_revs) & set(new_mc_revs)
36ce7f
    old_rev_list = [x for x in sorted(old_mc_revs) if x not in common]
36ce7f
    new_rev_list = [x for x in sorted(new_mc_revs) if x not in common]
36ce7f
36ce7f
    if len(old_rev_list) != 1 or len(new_rev_list) != 1:
36ce7f
        for i in new_mc:
36ce7f
            if i["rev"] in new_rev_list:
36ce7f
                res.append((ChangeLogEntry.ADDED, None, i))
36ce7f
        for i in old_mc:
36ce7f
            if i["rev"] in old_rev_list:
36ce7f
                res.append((ChangeLogEntry.REMOVED, i, None))
36ce7f
    else:
36ce7f
        for old in old_mc:
36ce7f
            if old["rev"] == old_rev_list[0]:
36ce7f
                break
36ce7f
        for new in new_mc:
36ce7f
            if new["rev"] == new_rev_list[0]:
36ce7f
                break
36ce7f
        if new["rev"] > old["rev"]:
36ce7f
            res.append((ChangeLogEntry.UPDATED, old, new))
36ce7f
        elif new["rev"] < old["rev"]:
36ce7f
            res.append((ChangeLogEntry.DOWNGRADED, old, new))
36ce7f
36ce7f
    return res
36ce7f
36ce7f
36ce7f
def gen_changelog(old, new):
36ce7f
    res = []
36ce7f
36ce7f
    old_map = gen_fn_map(old)
36ce7f
    new_map = gen_fn_map(new)
36ce7f
36ce7f
    old_files = set(old_map.keys())
36ce7f
    new_files = set(new_map.keys())
36ce7f
36ce7f
    both = old_files & new_files
36ce7f
    added = new_files - old_files
36ce7f
    removed = old_files - new_files
36ce7f
36ce7f
    for f in sorted(added):
36ce7f
        p = mc_path(new_map[f][0])
36ce7f
        for old_f in sorted(removed):
36ce7f
            old_p = mc_path(old_map[old_f][0])
36ce7f
            if p == old_p and f[1] == old_f[1] and f[2] == old_f[2]:
36ce7f
                log_info("Matched %s (%s and %s)" %
36ce7f
                         (p, old_map[old_f][0]["path"], new_map[f][0]["path"]))
36ce7f
                added.remove(f)
36ce7f
                removed.remove(old_f)
36ce7f
36ce7f
                res += mc_cmp(old_map[old_f], new_map[f])
36ce7f
36ce7f
    for f in sorted(added):
36ce7f
        for i in new_map[f]:
36ce7f
            res.append((ChangeLogEntry.ADDED, None, i))
36ce7f
    for f in sorted(removed):
36ce7f
        for i in old_map[f]:
36ce7f
            res.append((ChangeLogEntry.REMOVED, i, None))
36ce7f
    for f in sorted(both):
36ce7f
        res += mc_cmp(old_map[f], new_map[f])
36ce7f
36ce7f
    return res
36ce7f
36ce7f
36ce7f
def mc_date(mc):
36ce7f
    if isinstance(mc, dict):
36ce7f
        mc = mc["date"]
36ce7f
    return "%04x-%02x-%02x" % (mc & 0xffff, mc >> 24, (mc >> 16) & 0xff)
36ce7f
36ce7f
36ce7f
def mc_rev(mc, date=None):
36ce7f
    '''
36ce7f
    While revision is signed for comparison purposes, historically
36ce7f
    it is printed as unsigned,  Oh well.
36ce7f
    '''
36ce7f
    global print_date
36ce7f
36ce7f
    if mc["rev"] < 0:
36ce7f
        rev = 2**32 + mc["rev"]
36ce7f
    else:
36ce7f
        rev = mc["rev"]
36ce7f
36ce7f
    if date if date is not None else print_date:
36ce7f
        return "%#x (%s)" % (rev, mc_date(mc))
36ce7f
    else:
36ce7f
        return "%#x" % rev
36ce7f
36ce7f
36ce7f
def print_changelog(clog, cmap, args):
36ce7f
    for e, old, new in sorted(clog):
36ce7f
        if e == ChangeLogEntry.ADDED:
36ce7f
            print("Addition of %s at revision %s" %
36ce7f
                  (mc_path(new, midword="microcode", cmap=cmap), mc_rev(new)))
36ce7f
        elif e == ChangeLogEntry.REMOVED:
36ce7f
            print("Removal of %s at revision %s" %
36ce7f
                  (mc_path(old, midword="microcode", cmap=cmap), mc_rev(old)))
36ce7f
        elif e == ChangeLogEntry.UPDATED:
36ce7f
            print("Update of %s from revision %s up to %s" %
36ce7f
                  (mc_path(old, midword="microcode", cmap=cmap),
36ce7f
                   mc_rev(old), mc_rev(new)))
36ce7f
        elif e == ChangeLogEntry.DOWNGRADED:
36ce7f
                print("Downgrade of %s from revision %s down to %s" %
36ce7f
                      (mc_path(old, midword="microcode", cmap=cmap),
36ce7f
                       mc_rev(old), mc_rev(new)))
36ce7f
        elif e == ChangeLogEntry.OTHER:
36ce7f
            print("Other change in %s:" % old["path"])
36ce7f
            print("  old: %#x/%#x: rev %s (offs %#x)" %
36ce7f
                  (old["cpuid"], old["pf"], mc_rev(old), old["offs"]))
36ce7f
            print("  new: %#x/%#x: rev %s (offs %#x)" %
36ce7f
                  (new["cpuid"], new["pf"], mc_rev(new), new["offs"]))
36ce7f
36ce7f
36ce7f
class TableStyles:
36ce7f
    TS_CSV = 0
36ce7f
    TS_FANCY = 1
36ce7f
36ce7f
36ce7f
def print_line(line, column_sz):
36ce7f
    print(" | ".join([str(x).ljust(column_sz[i])
36ce7f
                      for i, x in zip(itertools.count(),
36ce7f
                                      itertools.chain(line,
36ce7f
                                      [""] * (len(column_sz) -
36ce7f
                                              len(line))))]).rstrip())
36ce7f
36ce7f
36ce7f
def print_table(items, header=[], style=TableStyles.TS_CSV):
36ce7f
    if style == TableStyles.TS_CSV:
36ce7f
        for i in items:
36ce7f
            print(";".join(i))
36ce7f
    elif style == TableStyles.TS_FANCY:
36ce7f
        column_sz = list(reduce(lambda x, y:
36ce7f
                                map(max, izip_longest(x, y, fillvalue=0)),
36ce7f
                                [[len(x) for x in i]
36ce7f
                                 for i in itertools.chain(header, items)]))
36ce7f
        for i in header:
36ce7f
            print_line(i, column_sz)
36ce7f
        if header:
36ce7f
            print("-+-".join(["-" * x for x in column_sz]))
36ce7f
        for i in items:
36ce7f
            print_line(i, column_sz)
36ce7f
36ce7f
36ce7f
def print_summary(revs, cmap, args):
36ce7f
    m = gen_fn_map(revs)
36ce7f
    cnames_mode = mcnm.MCNM_ABBREV if args.abbrev else mcnm.MCNM_CODENAME
36ce7f
36ce7f
    header = []
36ce7f
    if args.header:
36ce7f
        header.append(["Path", "Offset", "Ext. Offset", "CPUID",
36ce7f
                       "Platform ID Mask", "Revision", "Date", "Checksum",
36ce7f
                       "Codenames"] +
36ce7f
                      (["Models"] if args.models else []))
36ce7f
    tbl = []
36ce7f
    for k in sorted(m.keys()):
36ce7f
        for mc in m[k]:
36ce7f
            tbl.append([mc_stripped_path(mc),
36ce7f
                        "0x%x" % mc["offs"],
36ce7f
                        "0x%x" % mc["ext_offs"] if "ext_offs" in mc else "-",
36ce7f
                        "0x%05x" % mc["cpuid"],
36ce7f
                        "0x%02x" % mc["pf"],
36ce7f
                        mc_rev(mc, date=False),
36ce7f
                        mc_date(mc),
36ce7f
                        "0x%08x" % mc["cksum"],
36ce7f
                        get_mc_cnames(mc, cmap, cnames_mode) or ""] +
36ce7f
                       ([get_mc_cnames(mc, cmap,
36ce7f
                                       mcnm.MCNM_FAMILIES_MODELS)]
36ce7f
                        if args.models else []))
36ce7f
36ce7f
    print_table(tbl, header, style=TableStyles.TS_FANCY)
36ce7f
36ce7f
36ce7f
def read_codenames_file(path):
36ce7f
    '''
36ce7f
    Supports two formats: new and old
36ce7f
     * old: tab-separated. Field order:
36ce7f
       Segment, (unused), Codename, (dash-separated) Stepping,
36ce7f
       Platform ID mask, CPUID, (unused) Update link, (unused) Specs link
36ce7f
     * new: semicolon-separated; support comments.  Distinguished
36ce7f
       by the first line that starts with octothorp.  Field order:
36ce7f
       Segment, Unused, Codename, Stepping, Platform ID mask, CPUID,
36ce7f
       Abbreviation, Variant(s), Families, Models
36ce7f
    '''
36ce7f
    old_fields = ["segment", "_", "codename", "stepping", "pf_mask", "sig",
36ce7f
                  "_update", "_specs"]
36ce7f
    new_fields = ["segment", "_", "codename", "stepping", "pf_mask", "sig",
36ce7f
                  "abbrev", "variant", "families", "models"]
36ce7f
    new_fmt = False
36ce7f
    field_names = old_fields
36ce7f
36ce7f
    res = dict()
36ce7f
36ce7f
    try:
36ce7f
        with open(path, "r") as f:
36ce7f
            for line in f:
36ce7f
                line = line.strip()
36ce7f
                if len(line) == 0:
36ce7f
                    continue
36ce7f
                if line[0] == '#':
36ce7f
                    new_fmt = True
36ce7f
                    field_names = new_fields
36ce7f
                    continue
36ce7f
36ce7f
                fields = line.split(";" if new_fmt else "\t",
36ce7f
                                    1 + len(field_names))
36ce7f
                fields = dict(zip(field_names, fields))
36ce7f
                if "sig" not in fields:
36ce7f
                    log_warn("Skipping %r (from \"%s\")" % (fields, line))
36ce7f
                    continue
36ce7f
36ce7f
                sig = fields["sig"] = int(fields["sig"], 16)
36ce7f
                fields["pf_mask"] = int(fields["pf_mask"], 16)
36ce7f
                fields["stepping"] = fields["stepping"].split(",")
36ce7f
                if "variant" in fields:
36ce7f
                    if fields["variant"]:
36ce7f
                        fields["variant"] = fields["variant"].split(",")
36ce7f
                    else:
36ce7f
                        fields["variant"] = []
36ce7f
36ce7f
                if sig not in res:
36ce7f
                    res[sig] = list()
36ce7f
                res[sig].append(fields)
36ce7f
    except Exception as e:
36ce7f
        log_error("a problem occurred while reading code names: %s" % e)
36ce7f
36ce7f
    return res
36ce7f
36ce7f
36ce7f
def print_discrepancies(rev_map, deps, cmap, args):
36ce7f
    """
36ce7f
    rev_map: dict "name": revs
36ce7f
    deps: list of tuples (name, parent/None)
36ce7f
    """
36ce7f
    sigs = set()
36ce7f
36ce7f
    for p, r in rev_map.items():
36ce7f
        sigs |= set(r.keys())
36ce7f
36ce7f
    if args.header:
36ce7f
        header1 = ["sig"]
36ce7f
        if args.print_vs:
36ce7f
            header2 = [""]
36ce7f
        for p, n, d in deps:
36ce7f
            header1.append(n)
36ce7f
            if args.print_vs:
36ce7f
                add = ""
36ce7f
                if d:
36ce7f
                    for pd, nd, dd in deps:
36ce7f
                        if pd == d:
36ce7f
                            add = "(vs. %s)" % nd
36ce7f
                            break
36ce7f
                header2.append(add)
36ce7f
        if args.models:
36ce7f
            header1.append("Model names")
36ce7f
            if args.print_vs:
36ce7f
                header2.append("")
36ce7f
    header = [header1] + ([header2] if args.print_vs else [])
36ce7f
36ce7f
    tbl = []
36ce7f
    for s in sorted(sigs):
36ce7f
        out = [mc_path(s)]
36ce7f
        print_out = not args.print_filter
36ce7f
        print_date = args.min_date is None
36ce7f
36ce7f
        for p, n, d in deps:
36ce7f
            cur = dict([(x["rev"], x) for x in rev_map[p][s]]) \
36ce7f
                  if s in rev_map[p] else []
36ce7f
            v = "/".join([mc_rev(y) for x, y in sorted(cur.items())]) \
36ce7f
                if cur else "-"
36ce7f
            if d is not None:
36ce7f
                prev = [x["rev"] for x in rev_map[d][s]] if s in rev_map[d] \
36ce7f
                        else []
36ce7f
                if [x for x in cur if x not in prev]:
36ce7f
                    v += " (*)"
36ce7f
                    print_out = True
36ce7f
            if args.min_date is not None and s in rev_map[p]:
36ce7f
                for x in rev_map[p][s]:
36ce7f
                    print_date |= mc_date(x) > args.min_date
36ce7f
            out.append(v)
36ce7f
36ce7f
        if print_out and print_date:
36ce7f
            if args.models:
36ce7f
                out.append(get_mc_cnames(s, cmap) or "")
36ce7f
            tbl.append(out)
36ce7f
36ce7f
    print_table(tbl, header, style=TableStyles.TS_FANCY)
36ce7f
36ce7f
36ce7f
def cmd_summary(args):
36ce7f
    revs = []
36ce7f
    for p in args.filelist:
36ce7f
        revs = read_revs(p, ret=revs)
36ce7f
36ce7f
    codenames_map = read_codenames_file(args.codenames)
36ce7f
36ce7f
    print_summary(revs, codenames_map, args)
36ce7f
36ce7f
    return 0
36ce7f
36ce7f
36ce7f
def cmd_changelog(args):
36ce7f
    codenames_map = read_codenames_file(args.codenames)
36ce7f
    base_path = args.filelist[0]
36ce7f
    upd_path = args.filelist[1]
36ce7f
36ce7f
    base = read_revs(base_path)
36ce7f
    upd = read_revs(upd_path)
36ce7f
36ce7f
    print_changelog(gen_changelog(base, upd), codenames_map, args)
36ce7f
36ce7f
    return 0
36ce7f
36ce7f
36ce7f
def cmd_discrepancies(args):
36ce7f
    """
36ce7f
    filenames:
36ce7f
     * "<" prefix (possibly multiple times) to refer to a previous entry
36ce7f
       to compare against
36ce7f
     * "[name]" prefix is a name reference
36ce7f
    """
36ce7f
    codenames_map = read_codenames_file(args.codenames)
36ce7f
    rev_map = dict()
36ce7f
    deps = list()
36ce7f
    cur = -1
36ce7f
36ce7f
    for path in args.filelist:
36ce7f
        orig_path = path
36ce7f
        name = None
36ce7f
        cur += 1
36ce7f
        dep = None
36ce7f
        while True:
36ce7f
            if path[0] == '<':
36ce7f
                path = path[1:]
36ce7f
                dep = cur - 1 if dep is None else dep - 1
36ce7f
            elif path[0] == '[' and path.find(']') > 0:
36ce7f
                pos = path.find(']')
36ce7f
                name = path[1:pos]
36ce7f
                path = path[pos + 1:]
36ce7f
            else:
36ce7f
                break
36ce7f
        if name is None:
36ce7f
            name = path
36ce7f
        if dep is not None and dep < 0:
36ce7f
            log_error("Incorrect dep reference for '%s' (points to index %d)" %
36ce7f
                      (orig_path, dep))
36ce7f
            return 1
36ce7f
        deps.append((path, name, deps[dep][0] if dep is not None else None))
36ce7f
        rev_map[path] = gen_fn_map(read_revs(path), merge=args.merge,
36ce7f
                                   merge_path=True)
36ce7f
36ce7f
    print_discrepancies(rev_map, deps, codenames_map, args)
36ce7f
36ce7f
    return 0
36ce7f
36ce7f
36ce7f
def parse_cli():
36ce7f
    root_parser = argparse.ArgumentParser(prog="gen_updates",
36ce7f
                                          description="Intel CPU Microcode " +
36ce7f
                                          "parser")
36ce7f
    root_parser.add_argument("-C", "--codenames", default='codenames',
36ce7f
                             help="Code names file")
36ce7f
    root_parser.add_argument("-v", "--verbose", action="count", default=0,
36ce7f
                             help="Increase output verbosity")
36ce7f
36ce7f
    cmdparsers = root_parser.add_subparsers(title="Commands",
36ce7f
                                            help="main gen_updates commands")
36ce7f
36ce7f
    parser_s = cmdparsers.add_parser("summary",
36ce7f
                                     help="Generate microcode summary")
36ce7f
    parser_s.add_argument("-a", "--abbreviate", action="store_const",
36ce7f
                          dest="abbrev", const=True, default=True,
36ce7f
                          help="Abbreviate code names")
36ce7f
    parser_s.add_argument("-A", "--no-abbreviate", action="store_const",
36ce7f
                          dest="abbrev", const=False,
36ce7f
                          help="Do not abbreviate code names")
36ce7f
    parser_s.add_argument("-m", "--print-models", action="store_const",
36ce7f
                          dest="models", const=True, default=False,
36ce7f
                          help="Print models")
36ce7f
    parser_s.add_argument("-M", "--no-print-models",
36ce7f
                          action="store_const", dest="models",
36ce7f
                          const=False, help="Do not print models")
36ce7f
    parser_s.add_argument("-H", "--no-print-header",
36ce7f
                          action="store_const", dest="header",
36ce7f
                          const=False, default=True,
36ce7f
                          help="Do not print hader")
36ce7f
    parser_s.add_argument("filelist", nargs="*", default=[],
36ce7f
                          help="List or RPMs/directories to process")
36ce7f
    parser_s.set_defaults(func=cmd_summary)
36ce7f
36ce7f
    parser_c = cmdparsers.add_parser("changelog",
36ce7f
                                     help="Generate changelog")
36ce7f
    parser_c.add_argument("filelist", nargs=2,
36ce7f
                          help="RPMs/directories to compare")
36ce7f
    parser_c.set_defaults(func=cmd_changelog)
36ce7f
36ce7f
    parser_d = cmdparsers.add_parser("discrepancies",
36ce7f
                                     help="Generate discrepancies")
36ce7f
    parser_d.add_argument("-s", "--merge-revs", action="store_const",
36ce7f
                          dest="merge", const=True, default=False,
36ce7f
                          help="Merge revisions that come" +
36ce7f
                               " from different files")
36ce7f
    parser_d.add_argument("-S", "--no-merge-revs", action="store_const",
36ce7f
                          dest="merge", const=False,
36ce7f
                          help="Do not Merge revisions that come" +
36ce7f
                               " from different files")
36ce7f
    parser_d.add_argument("-v", "--print-vs", action="store_const",
36ce7f
                          dest="print_vs", const=True, default=False,
36ce7f
                          help="Print base version ")
36ce7f
    parser_d.add_argument("-V", "--no-print-vs", action="store_const",
36ce7f
                          dest="print_vs", const=False,
36ce7f
                          help="Do not Merge revisions that come" +
36ce7f
                               " from different files")
36ce7f
    parser_d.add_argument("-m", "--print-models", action="store_const",
36ce7f
                          dest="models", const=True, default=True,
36ce7f
                          help="Print model names")
36ce7f
    parser_d.add_argument("-M", "--no-print-models", action="store_const",
36ce7f
                          dest="models", const=False,
36ce7f
                          help="Do not print model names")
36ce7f
    parser_d.add_argument("-H", "--no-print-header", action="store_const",
36ce7f
                          dest="header", const=False, default=True,
36ce7f
                          help="Do not print hader")
36ce7f
    parser_d.add_argument("-a", "--print-all-files", action="store_const",
36ce7f
                          dest="print_filter", const=False, default=True,
36ce7f
                          help="Print all files")
36ce7f
    parser_d.add_argument("-c", "--print-changed-files", action="store_const",
36ce7f
                          dest="print_filter", const=True,
36ce7f
                          help="Print only changed files")
36ce7f
    parser_d.add_argument("-d", "--min-date", action="store",
36ce7f
                          help="Minimum date filter")
36ce7f
    parser_d.add_argument("filelist", nargs='*',
36ce7f
                          help="RPMs/directories to compare")
36ce7f
    parser_d.set_defaults(func=cmd_discrepancies)
36ce7f
36ce7f
    args = root_parser.parse_args()
36ce7f
    if not hasattr(args, "func"):
36ce7f
        root_parser.print_help()
36ce7f
        return None
36ce7f
    return args
36ce7f
36ce7f
36ce7f
def main():
36ce7f
    args = parse_cli()
36ce7f
    if args is None:
36ce7f
        return 1
36ce7f
36ce7f
    return args.func(args)
36ce7f
36ce7f
36ce7f
if __name__ == "__main__":
36ce7f
    sys.exit(main())