#!/usr/bin/python
#
# ddiskit - tool for Red Hat Enterprise Linux Driver Update Disk creation
#
# Author: Petr Oros <poros@redhat.com>
# Copyright (C) 2016-2017 Red Hat, Inc.
#
# This software may be freely redistributed under the terms of the GNU
# General Public License version 3 (GPLv3).
from __future__ import print_function
from __future__ import unicode_literals

import argparse
import codecs
import errno
import functools
import os
import re
import shlex
import shutil
import sys
import tarfile
import tempfile
import time
from datetime import datetime
from subprocess import PIPE, Popen, STDOUT
try:
    import configparser
except ImportError:
    import ConfigParser as configparser

if (sys.version_info > (3, 0)):
    unicode = str
    long = int

class ErrCode:
    """
    List of possible error codes. Can be converted to enumeration at some
    point.

    These are intended to be used as exit codes, so please keep them in the
    0..63 range.

    For now, exit code space is divided as follows:
     * 1..15 - generic/logic error
     * 32..63 - I/O errors
    """

    SUCCESS = 0

    GENERIC_ERROR = 1
    ARGS_PARSE_ERROR = 2
    CONFIG_CHECK_ERROR = 3
    QUILT_DEAPPLY_ERROR = 4
    QUILT_APPLY_ERROR = 5
    GIT_SRC_CHECK_ERROR = 6
    GPG_SIGNATURE_CHECK_ERROR = 7
    GIT_CHECKOUT_ERROR = 8
    SPEC_CHECK_ERROR = 9
    KABI_SYMBOL_CONFLICT_CHECK_ERROR = 10
    GIT_ADD_ERROR = 11
    GIT_COMMIT_ERROR = 12

    GENERIC_IO_ERROR = 32
    CONFIG_READ_ERROR = 34
    CONFIG_WRITE_ERROR = 35
    SPEC_TEMPLATE_READ_ERROR = 36
    SPEC_READ_ERROR = 38
    SPEC_WRITE_ERROR = 39
    SOURCE_ARCHIVE_WRITE_ERROR = 41
    MAKEFILE_NOT_FOUND = 42
    DD_ID_WRITE_ERROR = 45
    CONFIG_DUMP_WRITE_ERROR = 47
    DIRECTORY_CREATION_ERROR = 49


class QuiltCmd:
    APPLY = 0
    DEAPPLY = 1


RES_DIR = "/usr/share/ddiskit/"
TEMPLATE_DIR = "{res_dir}/templates"
PROFILE_DIR = "{res_dir}/profiles"

CONFIG_TEMPLATE = "config"
SPEC_TEMPLATE = "spec"

DEFAULT_CFG = "ddiskit.config"
SYSTEM_CFG = "/etc/ddiskit.config"
USER_CFG = "~/.ddiskitrc"

SRC_PATTERNS = "^Kbuild$|^Kconfig$|^Makefile$|^.*\.[ch]$"

KERNEL_NVR_RE = r"(?P<version>[0-9])\." + \
                r"(?P<patchlevel>[0-9]{1,2})\." + \
                r"(?P<sublevel>[0-9]{1,2})-(?P<rpm_release>[0-9]{1,4})"
KERNEL_Z_PART_RE = r"(?P<rpm_release_add>(\.[0-9]{1,3})+)"
KERNEL_DIST_RE = r"(?P<rpm_dist>\.el([6-9]|[1-9][0-9]))"

# Default configuration, put here values which can be overwritten by anything,
# but should be defined somewhere.
#
# How to determine, what should go here and what should be defined in
# package-wise ddiskit.config: if ddiskit itself breaks in case it can't find
# the key, then the default value should be placed there. If something else
# breaks (for example, rpm, in case of improperly generated spec), then
# the sane default should be provided in ddiskit.config.
default_config = {
    "defaults": {
        "res_dir": RES_DIR,
        "template_dir": TEMPLATE_DIR,
        "profile_dir": PROFILE_DIR,
        "config_template": CONFIG_TEMPLATE,
        "quilt_support": True,
        "spec_template": SPEC_TEMPLATE,
        "src_patterns": SRC_PATTERNS,
        "mock_config": "default",

        "kernel_nvr_re": KERNEL_NVR_RE,
        "kernel_fixed_re": KERNEL_Z_PART_RE,
        "kernel_dist_re": KERNEL_DIST_RE,
        "kernel_flex_version_re": "{kernel_nvr_re}{kernel_dist_re}",
        "kernel_fixed_version_re": "{kernel_nvr_re}{kernel_fixed_re}" +
                                   "{kernel_dist_re}",
        },
    "global": {
        "include_srpm": True,
        "module_vendor": "ENTER_MODULE_VENDOR",
        "module_author": "ENTER_MODULE_AUTHOR",
        "module_author_email": "ENTER_MODULE_AUTHOR_EMAIL",
        },
    "spec_file": {
        "module_name": "ENTER_MODULE_NAME",
        "module_version": "ENTER_MODULE_VERSION",
        "module_rpm_release": 1,
        "rpm_dist": "ENTER_RPM_DIST",
        "kernel_version": "",
        "kernel_arch": "x86_64 ppc64 ppc64le",
        "firmware_include": False,
        "firmware_version": "ENTER_FIRMWARE_VERSION",
        "devel_package": 0,
        },
    }

config_var_re = re.compile(r"{([^{}]*)}")
spec_var_re = re.compile(r"%{([^{}]*)}")


class DDiskitConfig(object):
    def __init__(self, cfg={}):
        # Some sanity checks
        for k, v in cfg.items():
            if not isinstance(k, u"".__class__) or not isinstance(v, dict):
                raise TypeError("cfg must be a dict of dicts with strings " +
                                "as keys")
            if "." in k:
                raise ValueError("dict keys in cfg shouldn't contain dots")

            for k, v in v.items():
                if not isinstance(k, u"".__class__):
                    raise TypeError("cfg dicts' keys must be strings")
                if "." in k:
                    raise ValueError("dict keys in cfg shouldn't contain dots")

        self._cfg = cfg

    def __str__(self):
        res = []

        for sk, sv in self._cfg.items():
            res.append("[%s]" % sk)

            for ik, iv in sv.items():
                res.append("\t%s = %r" % (ik, iv))

        return "\n".join(res)

    def __repr__(self):
        def get_class_name(i):
            """ Returns class name for an object instance. """
            if hasattr(i, "__class__"):
                c = i.__class__
            else:
                c = type(i)

            if hasattr(c, "__qualname__"):
                return c.__qualname__
            else:
                return c.__name__

        return "%s(cfg=%r)" % (get_class_name(self), self._cfg)

    def has_section(self, section):
        return section in self._cfg

    def iterkeys(self, section):
        return self._cfg[section].keys() if section else []

    def get(self, key, section="defaults", default=None, max_subst_depth=8,
            overrides=None):
        """
        Retrieve a value from configuration dict and perform substitution on
        it.

        So, let's reimplement ConfigParser's interpolation, in a better(tm)
        way.

        There are three main differences:
         * Syntax. ConfigParser uses %(name)s and ddiskit historically sticks
           with {name}.
         * Support for cross-section references, {section.name}.
         * It doesn't fail on exceeding of substitution depth

        :param key:             Key to return value for. Can be in dot-less
                                form (in this case value of "section" parameter
                                is used) or in the form "section.key".
        :param section:         Section where key is resides. By default it has
                                value "defaults", which makes retrieving
                                arguments/defaults less verbose:
                                configs.get("arg_name")
        :param default:         Default value which is used in case section/key
                                is absent.
        :param max_subst_depth: Maximum number of substitution rounds
                                performed. Providing value of 0 in this
                                argument allows retrieving raw value.
        :param overrides:       Dict (which is expected to be small) of local
                                overrides for configuration values. Keys
                                can be dotless strings (in this case they are
                                assumed to belong to the section provided in
                                section argument) or contain dot (which is
                                interpreted as "section.key"); with the former
                                having priority over the latter during the
                                matching process. Note that keys in this dict
                                are not normalised (converted to lower case)
                                and compared against normalised key/section
                                values.
        :return:                Value of the requested key (or default), with
                                substitution performed no more than
                                max_subst_depth times.
        """
        def check_overrides(s, k):
            """
            Check whether key is overridden.

            :param s: Configuration option section.
            :param k: Configuration option key name.
            :return:  2-value tuple; first element is a boolean value which
                      signalises whether the key is overridden; second value is
                      the overriding value.
            """
            if overrides is None:
                return (False, None)

            if s == section and k in overrides:
                return (True, overrides[k])

            full_key = "%s.%s" % (s, k)
            if full_key in overrides:
                return (True, overrides[full_key])

            return (False, None)

        def config_subst(m):
            """
            Callback for performing substitution in the configuration value.

            :param m: Match object, containing information about substring
                      matching variable substitution regex.
            :return:  Value to substitute it with: configuration value in case
                      it exists or unchanged value in case it does not.
            """
            key = list(map(lambda s: s.lower(), m.group(1).split('.', 1)))

            if len(key) == 1:
                key.insert(0, section)

            override, val = check_overrides(key[0], key[1])

            if override:
                return val

            if key[0] in self._cfg and key[1] in self._cfg[key[0]]:
                return self._cfg[key[0]][key[1]]

            return m.group(0)

        if key.find(".") >= 0:
            section, key = key.split(".", 1)

        val = default
        section = section.lower()
        key = key.lower()

        override, override_val = check_overrides(section, key)

        if override:
            val = override_val
        elif section in self._cfg and key in self._cfg[section]:
            val = self._cfg[section][key]

        if not isinstance(val, (str, unicode)):
            return val

        while max_subst_depth > 0:
            max_subst_depth -= 1
            new_val, cnt = config_var_re.subn(config_subst, val)

            if cnt == 0 or new_val == val:
                break

            val = new_val

        return val

    def get_bool(self, key, section="defaults", default=None,
                 max_subst_depth=8, overrides=None):
        """
        Simple wrapper for val2bool(self.get()). Parameters are the same as
        for get() method, return value is the same as for val2bool() method.
        """
        return val2bool(self.get(key, section, default, max_subst_depth,
                                 overrides))

    def set(self, key, val, section="defaults"):
        """
        Set value in configuration dict.

        :param key:     Key to change. Can be in dot-less form (in this case
                        value of "section" parameter is used) or in the
                        "section.key" form.
        :param val:     New value.
        :param section: Section where key is resides. By default it has value
                        "defaults", which makes updating arguments/defaults
                        less verbose: configs.set("arg_name", val)
        :return:        Updated configuration dict.
        """
        if key.find(".") >= 0:
            section, key = key.split(".", 1)

        section = section.lower()
        key = key.lower()

        if section not in self._cfg:
            self._cfg[section] = {}

        self._cfg[section][key] = val

        return self

    def humble_set(self, key, val, section="spec_file"):
        """
        Set value in case it is not set already. Used primarily for "spec_file"
        section.

        Implemented as a wrapper around get()/set(), which makes
        it somewhat less efficient, but not too much.

        :param key:     Key to change. Can be in dot-less form (in this case
                        value of "section" parameter is used) or in the
                        "section.key" form.
        :param val:     New value.
        :param section: Section where key is resides. By default it has value
                        "spec_file", which makes providing fallback values for
                        spec file less verbose:
                        configs.humble_set("arg_name", val)
        :return:        Updated configuration dict.
        """
        if self.get(key, section, max_subst_depth=0) is None:
            self.set(key, val, section)

        return self

    def apply_args(self, args, encoding=sys.stdin.encoding or 'utf-8'):
        """
        Puts command-line args as a values of "defaults" section of
        configuration dict, so it could be used for providing defaults when
        specific argument is not provided.

        :param args:    Namespace instance containing command line arguments,
                        as returned by argparse.ArgumentParser.parse_args()
        :return:        Updated config instance.
        """
        def unicodify(v):
            if isinstance(v, str):
                if (sys.version_info < (3, 0)):
                    return unicode(v, encoding=encoding)

            return v

        discarded_args = []

        if "defaults" not in self._cfg:
            self._cfg["defaults"] = {}
        self._cfg["defaults"].update([(unicodify(k), unicodify(v))
                                      for k, v in args.__dict__.items()
                                      if v is not None])

        if args.config_option:
            for arg in args.config_option:
                if isinstance(arg, str):
                    arg = unicode(arg, encoding=encoding)

                kv = arg.split("=", 1)
                if len(kv) != 2:
                    discarded_args.append(arg)
                    continue
                self.set(kv[0].strip(), kv[1])

        return (self, discarded_args)

    def dump_config(self, cfgparser=None):
        """
        Dump contents of config object into configparser.RawConfigParser
        object.

        :param cfgparser: Existing configparser.RawConfigParser object in case
                          the creation of new one is not needed.
        :returns:         configparser.RawConfigParser object filled with
                          configuration values.
        """
        if cfgparser is None:
            cfgparser = configparser.RawConfigParser()

        for s_name, section in self._cfg.items():
            cfgparser.add_section(s_name)

            for i_name, item in section.items():
                if s_name == "defaults" and i_name == "func":
                    continue
                cfgparser.set(s_name, i_name, item)

        return cfgparser


def val2bool(val):
    """
    Convert (string) value to some boolean one or None in case it doesn't look
    true-ish of false-ish enough.

    True-ish values: True, non-zero integer, "t", "y", "true", "yes", "1"
    False-ish values: False, 0, "f", "n", "false", "no", "0"

    String match for true-ish/false-ish values is case insensitive.

    Of course we are in desperate need of our own implementation of str2bool
    because ours is much better than anyone's else.

    :param val: Value which should be converted to boolean.
    :return:    True in case value is True-isch, False in case value is
                False-ish, None in all other cases (some weird invalid input
                on which we do not want to decide).
    """
    if isinstance(val, bool):
        return val
    if isinstance(val, (int, long)):
        return val != 0
    if isinstance(val, u"".__class__):
        val = val.lower()

        if val in ("t", "y", "true", "yes", "1"):
            return True
        if val in ("f", "n", "false", "no", "0"):
            return False

    return None


def command(cmd, configs, cwd=None, cmd_print_lvl=1, res_print_lvl=2,
            capture_output=True, capture_stderr=False):
    """
    Execute shell command and return stdout string
    :param cmd: Command
    :return: Tuple with command exit code as a first element and command output
             as a second. In case OSError occurred during the execution, value
             of -256 - errno is returned as the return code.
    """
    log_status("Executing command: %r" % cmd, configs, level=cmd_print_lvl)
    try:
        process = Popen(
            args=cmd,
            cwd=cwd,
            close_fds=True,
            stdout=PIPE if capture_output else None,
            stderr=PIPE if capture_stderr else None
        )
        result, reserr = process.communicate()
        ret = process.returncode
        if capture_output:
            result = result.decode(sys.stdin.encoding or 'utf-8', "replace")
        if capture_stderr:
            reserr = reserr.decode(sys.stdin.encoding or 'utf-8', "replace")
    except OSError as err:
        log_error("Got OSError when tried to execute %r (errno %d): %s" %
                  (cmd, err.errno, err.strerror), configs,
                  level=max(1, min(cmd_print_lvl, res_print_lvl)))
        result = ""
        reserr = ""
        ret = -256 - err.errno

    if capture_output:
        log_status(result, configs, level=res_print_lvl)
    if capture_stderr and reserr:
        log_status("--- stderr: ---\n" + reserr, configs, level=res_print_lvl)

    log_status("  Return code for %r: %d" % (cmd, ret), configs,
               level=cmd_print_lvl)

    return (ret, result, reserr) if capture_stderr else (ret, result)


# XXX global variable for log_* functions
need_newline = False


def log_status(msg, configs, level=0, newline=True):
    """
    Log execution progress information.

    :param msg:     Message to log.
    :param configs: Configuration object.
    :param level:   Message verbosity level.
    """
    global need_newline

    if configs.get("verbosity", default=0) >= level:
        print(msg, end="\n" if newline else "")
        need_newline = not newline


def log_info(msg, configs, level=2):
    """
    Log informational message.

    :param msg:     Message to log.
    :param configs: Configuration object.
    :param level:   Message verbosity level.
    """
    global need_newline

    if configs.get("verbosity", default=0) >= level:
        if need_newline:
            print("")
        print("INFO:", msg)
        need_newline = False


def log_warn(msg, configs, level=1):
    """
    Log warning message.

    :param msg:     Message to log.
    :param configs: Configuration object.
    :param level:   Message verbosity level.
    """
    global need_newline

    if configs.get("verbosity", default=0) >= level:
        if need_newline:
            print("")
        print("WARNING:", msg)
        need_newline = False


def log_error(msg, configs, level=-128):
    """
    Log error message.

    :param msg:     Message to log.
    :param configs: Configuration object.
    :param level:   Message verbosity level.
    """
    global need_newline

    if configs.get("verbosity", default=0) >= level:
        if need_newline:
            print("")
        print("ERROR:", msg)
        need_newline = False


def get_kernel_version(ver, configs):
    """
    Matches kernel version string and returns parsed kernel version.

    :param ver:     Version string.
    :param configs: Configuration object.
    :return:        Tuple. First element is dict with keys version, patchlevel,
                    sublevel, rpm_release, rpm_release_add, rpm_dist, or None,
                    if kernel version string wasn't matched. Second element
                    indicates whether flexible dependencies can be generated
                    based on this kernel version.
    """
    kernel_flex_re = configs.get("kernel_flex_version_re", default="%s%s" %
                                 (KERNEL_NVR_RE, KERNEL_DIST_RE))
    kernel_fixed_re = configs.get("kernel_fixed_version_re", default="%s%s%s" %
                                  (KERNEL_NVR_RE, KERNEL_Z_PART_RE,
                                   KERNEL_DIST_RE))

    keys = ("version", "patchlevel", "sublevel", "rpm_release", "rpm_dist")
    optional_keys = ("rpm_release_add", )

    ret = None
    matched_re = None

    for r in (kernel_flex_re, kernel_fixed_re):
        try:
            m = re.match(r, ver)
            groups = m.groupdict()

            ret = dict()

            for i in keys:
                ret[i] = groups[i]
            for i in optional_keys:
                ret[i] = groups.get(i, "")
        except:
            log_info("Kernel version \"%s\" doesn't match \"%s\"" % (ver, r),
                     configs)
            ret = None

        if ret:
            matched_re = r
            break

    return (ret, matched_re == kernel_flex_re)


def get_mock_kernel_version(configs):
    pkg = "kernel-devel"
    mock_args = get_mock_args(configs)
    if not configs.get_bool("mock_offline"):
        ret, out = command(["mock", "-r", mock_args[1]["config"], "--install",
                            pkg], configs)
        if ret:
            log_warn(("Mock returned non-zero status %d when tried " +
                      "to install %s, output:\n%s") % (ret, pkg, out),
                      configs)
            return ""

    ret, out = command(["mock", "-r", mock_args[1]["config"], "--dnf-cmd", "--",
                        "repoquery", pkg, "--latest-limit", "1",
                        "--qf", "[DNF] %{NAME} %{VERSION}-%{RELEASE}"], configs,
                        cmd_print_lvl=2)

    if ret:
        log_warn(("DNF returned non-zero status %d when queried version " +
                  "of the \"%s\" package, output:\n%s") % (ret, pkg, out),
                 configs)
        return ""

    for l in out.splitlines():
        if not l.startswith("[DNF]"):
            continue

        pkg = l.split()
        if len(pkg) != 3:
            log_warn("Unexpected output line: \"%s\", skipping" % l, configs)
        if pkg[1] != pkg:
            log_warn(("Unexpected output package \"%s\" in line \"%s\", " +
                      "skipping") % (pkg[1], l), configs)

        return pkg[2]

    return ""


def process_configs_for_spec(configs):
    """
    Process configuration before spec file generation.

    This is a set of hacks which make applying configuration to spec file
    template a uniform process. More specifically, it does two sets of
    configuration changes:
     * Ones that always overwrite existing values. Currently, this is done only
       for spec_file.firmware_begin/spec_file.firmware end configuration
       variables, which are set based on the value of the
       spec_file.firmware_include configuration variable, which determines
       whether firmware should be packaged.
     * Ones that do not change existing values: generation of spec_file.date,
       spec_file.kernel_requires, spec_file.module_requires.

    The fact that most variables are overwritten only in case they are not have
    been set previously allows performing reproducible builds based on
    pre-created configuration dump file (changes in spec template not
    withstanding).

    :param configs: Configuration object.
    :return:        Updated configuration dict.
    """
    # no firmware? -> remove all firmware definitions from spec file
    have_fw = configs.get_bool("firmware_include", "spec_file", False)
    configs.set("spec_file.firmware_begin", "%%if %d" % int(have_fw))
    configs.set("spec_file.firmware_end", "%endif")

    # generic keys code
    # date of creation
    configs.humble_set("date",
                       datetime.__format__(datetime.now(), "%a %b %d %Y"))

    # kernel_requires
    kernel_ver_string = configs.get("spec_file.kernel_version", default="")

    if not kernel_ver_string:
        log_info("Kernel version hasn't been specified, trying to get from " +
                 "the build environment", configs)

        kernel_ver_string = get_mock_kernel_version(configs) \
                            if configs.get_bool("mock") \
                            else get_rpm_kernel_version(configs)

        if kernel_ver_string:
            log_status("Got kernel version from the mock environment: %s",
                       configs)
            configs.set("spec_file.kernel_version", kernel_ver_string)
        else:
            log_error("Failed to get kernel version form build environment, " +
                      "please provide spec_file.kernel_version configuration " +
                      "parameter manually", configs)
            return ErrCode.CONFIG_CHECK_ERROR

    kernel_ver, flex_deps = get_kernel_version(kernel_ver_string, configs)

    kernel_ver_dep_string = configs.get("spec_file.kernel_version_dep")
    kernel_ver_min_string = configs.get("spec_file.kernel_version_min")

    if kernel_ver_min_string:
        kernel_requires = "Requires:	kernel >= " + kernel_ver_min_string
    elif kernel_ver_dep_string:
        kernel_requires = "Requires:	kernel = " + kernel_ver_dep_string
    elif flex_deps:
        kver = "%s.%s.%s" % (kernel_ver["version"],
                             kernel_ver["patchlevel"],
                             kernel_ver["sublevel"])
        rpm_release = int(kernel_ver["rpm_release"])
        rpm_release_add = kernel_ver["rpm_release_add"]
        rpm_dist = kernel_ver["rpm_dist"]

        kernel_requires = "Requires:	kernel >= %s-%s%s%s\n" % \
            (kver, rpm_release, rpm_release_add, rpm_dist)
        kernel_requires += "Requires:	kernel < %s-%s%s%s" % \
            (kver, rpm_release + 1, rpm_release_add, rpm_dist)
    elif kernel_ver:
        kernel_requires = "Requires:	kernel = " + kernel_ver_string

    configs.humble_set("kernel_requires", kernel_requires)

    # module_requires - deprecated
    module_dep_string = configs.get("spec_file.dependencies", default="")
    if module_dep_string != "":
        module_requires = "Requires:	" + module_dep_string
    else:
        module_requires = ""

    configs.humble_set("module_requires", module_requires)

    return ErrCode.SUCCESS


def apply_config(data, configs, empty_is_nil=True):
    """
    Perform spec template substitution from configuration dict.

    Process spec file template by replacing existing tags by strings from
    configuration dict. Tags which exist end evaluate to empty string are
    replaced with %{nil}, unless empty_is_nil argument is provided with False
    value.

    :param data:         Input content with tags to replace.
    :param configs:      Configuration object.
    :param empty_is_nil: Whether to replace tags with corresponding
                         configuration values evaluated to empty string with
                         %{nil}.
    :return:             Replaced content.
    """
    def stringify(v):
        if not isinstance(v, (str, unicode)):
            return str(v)

        return v

    return spec_var_re.sub(lambda m: stringify(configs.get(m.group(1),
                                                           "spec_file",
                                                           m.group(0))) or
                           ("%{nil}" if empty_is_nil else ""), data)


def check_config(configs):
    """
    Check config and repair non-critical mistakes.
    :param configs: Configuration object to check.
    :return: Fixed config or None
    """
    rpmname_char_wl = ".-_+%{}"
    rpmver_char_wl  = "._+%{}~"

    rpm_char_wl = {
        "module_name":        rpmname_char_wl,
        "module_version":     rpmver_char_wl,
        "module_rpm_release": rpmver_char_wl,
    }

    config_critic = False
    log_status("Checking config ... ", configs)
    for section in ["global", "spec_file"]:
        if not configs.has_section(section):
            log_error("required section \"%s\" hasn't been found!", configs)
            config_critic = True
            continue

        for key in configs.iterkeys(section):
            val = configs.get(key, section)

            if val == "ENTER_" + key.upper():
                if section == "spec_file" and key == "firmware_version" and \
                        not configs.get_bool("spec_file.firmware_include"):
                    continue
                else:
                    log_error("key: %s.%s value: %s is a default value" %
                              (section, key, val), configs)
                    config_critic = True
            elif section == "spec_file" and key == "kernel_version":
                kver, y_ver = get_kernel_version(val, configs)
                if y_ver:
                    continue
                elif kver:
                    kernel_y_re = configs.get("kernel_flex_version_re")
                    log_warn("You are using z-stream kernel version! " +
                             "You shouldn't use it. If you don't have good " +
                             "reason for it, please use kernel version that" +
                             "matches regular expression \"%s\"" % kernel_y_re,
                             configs, level=0)
                    continue
                elif not val:
                    log_warn("Kernel version is not specified, an attempt " +
                             "will be made to get it from the build " +
                             "environment", configs)
                else:
                    kernel_y_re = configs.get("kernel_flex_version_re")
                    log_error(("Invalid kernel version in config file: " +
                               "\"%s\". Valid version is, for example, " +
                               "one that matches \"%s\"") % (val, kernel_y_re),
                              configs)
                    config_critic = True
            elif section == "spec_file" and key == "module_build_dir":
                if val[0] == "/":
                    val = val[1:]
                    configs.set(key, val, section)
                    log_warn("Leading \"/\" in module_build_dir, fixing.",
                             configs)
                if val[-1] == "/":
                    val = val[:-1]
                    configs.set(key, val, section)
                    log_warn("Trailing \"/\" in module_build_dir, fixing.",
                             configs)
            elif section == "spec_file" and key in rpm_char_wl:
                if not all([c.isalnum() or c in rpm_char_wl[key] for c in val]):
                    log_error(("Incorrect characters in %s.%s: \"%s\" may " +
                               "contain only alphanumeric characters and " +
                               "\"%s\"") %
                              (section, key, val, rpm_char_wl[key]), configs)
                    config_critic = True

    if config_critic:
        log_error("Unrecoverable failure, please fix the aforementioned " +
                  "issues and run ddiskit again.", configs)
        return None
    log_status("Config check ... OK", configs)
    return configs


def get_config_name(cfg, extension=".cfg"):
    """
    Get config name based on provided config option. Uses the following
    heuristic (based on the one used in mock): if config options ends with
    config file extension, this is path to file (and then dirname and extension
    should be stripped), otherwise it is config name. In order to be cautious,
    it leaves only base name first in any case.

    :param cfg: Config name passed as the command-line argument
    :param extension: Configuration file extension
    """
    cfg = os.path.basename(cfg)
    if extension != "" and cfg.endswith(extension):
        cfg = cfg[:-len(extension)]

    return cfg


def get_config_path(cfg, default_dir=".", rel_dir=".", extension=".cfg"):
    """
    Get path to config based on provided configuration "name".

    It uses the following heuristic: if configuration name does not have
    slashes and does not end with expected extension (or this extension is
    empty) then it is considered that this name refers to the file in "default
    directory", otherwise it is interpreted as a path to file (relative to some
    path provided in rel_dir).

    :param cfg:         Configuration file name.
    :param default_dir: Default directory for these configuration files.
    :param rel_dir:     Path configuration file should be relative to in case
                        path provided instead of name.
    :param extension:   Expected configuration file extension.
    :return:            Path to the configuration file.
    """
    if "/" not in cfg and (extension == "" or not cfg.endswith(extension)):
        cfg = os.path.basename(cfg)
        cfg = os.path.join(default_dir, cfg + extension)
    else:
        cfg = os.path.join(rel_dir, cfg)

    cfg = os.path.normpath(cfg)

    return cfg


def do_quilt(action, configs, restore_patch=None):
    """
    Perform quilt-related operation.

    :param action:        Action to perform. Currently supported operations:
     * QuiltCmd.DEAPPLY - de-apply all currently applied patches and store
                          the list of patches originally applied inside
                          function object.
     * QuiltCmd.APPLY - restore patches provided in restore_patch argument,
                        or, in case it is None, retrieve previously stored
                        patches the from the function object.
    :param configs:       Configuration object.
    :param restore_patch: List of patches to restore (used by QuiltCmd.APPLY
                          action).
    :return:              0 in case of success, non-zero return code in case of
                          errors.
    """
    series_dir = configs.get("quilt_series_dir", default="src")
    quilt_enabled = configs.get_bool("quilt_support")

    if not quilt_enabled:
        log_warn(("Quilt support is not enabled (got \"%s\" instead), " +
                  "skipping quilt actions") % quilt_enabled, configs, level=2)

        return 0

    if action == QuiltCmd.APPLY:
        patches = restore_patch if restore_patch is not None else \
            getattr(do_quilt, "saved_patches", None)

        if patches is None:
            log_warn("No quilt patches were applied, not trying to restore " +
                     "anything", configs, level=2)

            return 0

        ret = 0
        for p in patches.split('\n'):
            if not p:
                continue

            ret = command(["quilt", "push", p], configs, cwd=series_dir,
                          cmd_print_lvl=2)[0] or ret

        return ret
    elif action == QuiltCmd.DEAPPLY:
        ret, out = command(["quilt", "applied"], configs, cwd=series_dir,
                           cmd_print_lvl=2)

        do_quilt.saved_patches = out if ret == 0 else None

        if ret != 0:
            log_warn(("Quilt returned non-zero exit code (%d), do not try " +
                     "to de-apply patches") % ret, configs, level=2)

            return 0

        return command(["quilt", "pop", "-a"], configs, cwd=series_dir,
                       cmd_print_lvl=2)[0]
    else:
        log_warn("Unknown quilt action %d" % action, configs)

    return 1


def do_git_src_check(configs):
    """
    Check whether sources put in src correspond to the ones present in the
    repository at the specified commit.

    Configuration options used:
        defaults.git_src_check - whether to perform check.
                                 0 - no, 1 - issue warning, 2 - issue error.
        defaults.git_dir - patch to the (kernel) git repo (.git)
        spec_file.git_hash - commit ID to check against
        spec_file.module_build_dir - directory inside repo/src which should be
                                     checked

    :param configs: Configuration object.
    :return:        Tuple of two value, the first one signalises check level,
                    the second is the check result, that is decoded as follows:
                    0 in case of successful check, positive value in case there
                    is some difference, negative value in case there are some
                    issues during the verification attempt, specifically:
                     * -1 - defaults.git_repo is missing
                     * -2 - spec_file.git_hash is missing
                     * -3 - spec_file. module_build_dir is missing
                     * -1024...-255 - issues with repo check (increase the
                                      value by 512 in order to get command exit
                                      code / OS error code)
                     * -2048...-1025 - issue with git diff call (increase the
                                       value by 1536 in order to get command
                                       exit code / OS error code)
    """
    check_enabled = int(configs.get("check_git_src", default=0))

    if check_enabled < 1:
        log_info("Source verification against Git repository " +
                 "(defaults.check_git_src) is not enabled,skipping.", configs)
        return (0, 0)

    git_repo = configs.get("defaults.git_dir")
    git_hash = configs.get("spec_file.git_hash")
    subdir = configs.get("spec_file.module_build_dir")

    if git_repo is None:
        log_warn("Path to git repository (defaults.git_dir) is not provided," +
                 " can't verify integrity of sources against it.", configs)
        return (check_enabled, -1)
    if git_hash is None:
        log_warn("Git commit hash (spec_file.git_hash) is not provided," +
                 " can't verify integrity of sources against it.", configs)
        return (check_enabled, -2)
    if subdir is None:
        log_warn("Directory inside git repository " +
                 "(spec_file.module_build_dir) is not provided, " +
                 "can't verify integrity of sources against it.", configs)
        return (check_enabled, -3)

    # Check that git is working and the repo is present
    ret = command(["git", "--git-dir=%s" % git_repo, "--work-tree=src/",
                   "rev-parse", "--revs-only", "--verify", git_hash],
                  configs, cmd_print_lvl=2)

    if ret[0]:
        log_warn(("Git repo check failed (return code %d, output \"%s\"), " +
                 "can't verify authentity of sources") % ret, configs)
        return (check_enabled, ret[0] - 512)

    diff_fmt = "--stat" if configs.get("verbosity") < 2 else "--patch"
    ret = command(["git", "--git-dir=%s" % git_repo, "--work-tree=src/",
                   "diff", diff_fmt, "--exit-code", git_hash, "--", subdir],
                  configs, capture_output=False)[0]

    return (check_enabled, ret if ret <= 1 else ret - 1536)


def get_mock_args(configs):
    """
    Constructs list of mock arguments based on the configuration provided.

    Arguments currently supported:
     * "mock_config" (default "defult.cfg") - "-r CONFIG"
     * "verbosity" (default 0) - "-q" for verbosity 0, no additional arguments
                                 for verbosity 1, "-v" for verbosity 2 and
                                 more.
     * "mock_offline" (default False) - adds "--offline" in case it is True

    :param configs: Configuration object.
    :return:        Tuple containing array of command line arguments as first
                    value and retrieved configuration values in second.
    """
    res = []
    mock_cfg = configs.get("mock_config", default="default.cfg")
    mock_offline = configs.get_bool("mock_offline")
    verbosity = configs.get("verbosity", default=0)
    opts = configs.get("mock_opts", default=None)

    res += ["-r", mock_cfg]

    if verbosity > 1:
        res.append("-v")
    elif not verbosity:
        res.append("-q")
    if mock_offline:
        res.append("--offline")
    if opts is not None:
        opts = shlex.split(opts)
        res += opts

    return (res, {"config": mock_cfg, "offline": mock_offline, "opts": opts})


def do_build_rpm(configs, arch):
    """
    Binary RPM building routine.

    :param configs: Configuration object.
    :param arch:    Architecture to build RPM for.
    """
    use_mock = configs.get_bool("mock")

    log_status("Start RPM build for %s %s... " %
               (arch, "using mock " if use_mock else ""), configs)
    spec_path = get_spec_path(configs)
    if use_mock:
        mock_args = get_mock_args(configs)

        # We build RPM out of SRPM and we should build SRPM inside target
        # config
        if do_build_srpm(configs) != 0:
            return 1

        log_status("Start binary RPM build for %s using mock... " % arch,
                   configs)

        ret, dist = command(["mock", "-q", "-r", mock_args[1]["config"],
                             "--chroot", "rpm --eval %{dist}"], configs)
        if ret != 0:
            return 1

        if dist.endswith('\n'):
            dist = dist[:-1]

        srpm_name = "%s%s.src.rpm" % \
            (configs.get("", "spec_file",
                         "{rpm_name}-{module_version}-{module_rpm_release}"),
             dist)

        cmd = ["mock", "--no-cleanup-after", "--rebuild"]
        cmd += mock_args[0]
        cmd += ["--arch", arch, "--resultdir", "rpm/RPMS/",
                "rpm/SRPMS/%s" % srpm_name]
    else:
        cmd = ["rpmbuild", "--target", arch,
               "--define", "_topdir " + os.getcwd() + "/rpm",
               "-ba", spec_path]

    return command(cmd, configs, capture_output=False)[0]


def do_build_srpm(configs):
    """
    Source RPM building routine.

    :param configs: Configuration object.
    """
    use_mock = configs.get_bool("mock")

    log_status("Start SRPM build %s... " % ("using mock " if use_mock else ""),
               configs)
    spec_path = get_spec_path(configs)
    if use_mock:
        cmd = ["mock", "--buildsrpm"]
        cmd += get_mock_args(configs)[0]
        cmd += ["--spec", spec_path,
                "--sources", "rpm/SOURCES/", "--resultdir", "rpm/SRPMS/"]
    else:
        cmd = ["rpmbuild", "--define", "_topdir " + os.getcwd() + "/rpm",
               "-bs", spec_path]

    return command(cmd, configs, capture_output=False)[0]


def do_check_rpm_build(configs):
    """
    Check whether RPM can be built on the host.
    :param configs: Configuration object.
    """
    spec_path = get_spec_path(configs)
    cmd = ["rpmbuild", "--define", "_topdir " + os.getcwd() + "/rpm",
           "--nobuild", "-bc", spec_path]

    return command(cmd, configs, capture_output=False)[0] == 0


def create_dirs(dir_list, configs, caption=None):
    """
    Try to create supplied list of directories, return information whether it
    was successful or there were errors.

    :param dir_list: List of directories to create.
    :param configs:  Configuration object.
    :param caption:  Caption to print during the directory creation process.
                     If None, nothing is printed.
    :return:         True if there were no errors, otherwise False.
    """
    if caption is not None:
        log_status("%s ... " % caption, configs, newline=False)

    dir_creation_ok = True
    for dirs in dir_list:
        try:
            if not os.path.exists(dirs):
                os.makedirs(dirs)
        except OSError as err:
            log_error(str(err), configs)
            dir_creation_ok = False

    if caption is not None:
        log_status("OK", configs)

    return dir_creation_ok


def cmd_prepare_sources(configs):
    """
    CMD prepare_sources callback
    :param configs: Configuration object.
    """
    cfgfile = configs.get("config", default="module.config")
    result = ErrCode.SUCCESS

    ret = create_dirs(["rpm", "rpm/BUILD", "rpm/BUILDROOT", "rpm/RPMS",
                       "rpm/SOURCES", "rpm/SPECS", "rpm/SRPMS"], configs,
                      "Creating directory structure for RPM build")
    ret &= create_dirs(["src", "src/patches", "src/firmware"], configs,
                       "Creating directory structure for source code")

    if not ret:
        log_error("Directory structure creation failed", configs)
        result = ErrCode.DIRECTORY_CREATION_ERROR

    src_dirs = configs.get("git_src_directory")
    revspec = ""
    src_copied = False

    if not src_dirs:
        src_dirs = configs.get("spec_file.module_build_dir")

        if src_dirs == "ENTER_MODULE_BUILD_DIR":
            src_dirs = ""

    if src_dirs:
        src_dirs = src_dirs.split()
        repo = configs.get("git_dir")
        revspec = configs.get("git_revision", default="HEAD")

        ret, revision = command(["git", "--git-dir=%s" % repo,
                                 "--work-tree=src/", "rev-parse",
                                 "--revs-only", "--verify", revspec],
                                configs, cmd_print_lvl=2)

        if ret:
            log_error(("Failed parsing revision specification \"%s\" for " +
                       "repository \"%s\", can't copy sources") %
                      (revspec, repo), configs)
            result = ErrCode.GIT_SRC_CHECK_ERROR
        else:
            revspec = revision.split()[0]
            ret, out = command(["git", "--git-dir=%s" % repo,
                                "--work-tree=src/", "checkout",
                                revspec, "--"] + src_dirs, configs)

            if ret:
                log_error(("Checkout of directories %r, from repository " +
                           "\"%s\" at revision \"%s\" failed (exit code " +
                           "%d): %s") % (src_dirs, repo, revspec, ret, out),
                          configs)
                result = ErrCode.GIT_CHECKOUT_ERROR
            else:
                log_info("Successfully checked out %r" % src_dirs, configs,
                         level=0)
                src_copied = True
    else:
        log_info("No source directories to check out, skipping", configs)
        src_dirs = ["ENTER_MODULE_BUILD_DIR"]

    configs.set("spec_file.module_build_dir", src_dirs[0])
    configs.set("spec_file.git_hash", revspec)

    try:
        log_status("Writing new config file (%s) ... " % cfgfile,
                   configs, newline=False)
        if os.path.isfile(cfgfile):
            log_status("File exists, skipping", configs)
        else:
            template_dir = configs.get("template_dir")
            config_template = configs.get("config_template")
            with codecs.open(get_config_path(config_template, extension="",
                             default_dir=template_dir), 'r',
                             encoding='utf-8') as fin:
                read_data = apply_config(fin.read(), configs,
                                         empty_is_nil=False)
                with codecs.open(cfgfile, 'w+', encoding='utf-8') as fout:
                    fout.write(read_data)
            log_status("OK", configs)
    except IOError as err:
        log_error(str(err), configs)
        result = ErrCode.CONFIG_WRITE_ERROR

    if result == ErrCode.SUCCESS and not src_copied:
        log_status("Put your module source code in src directory.", configs)

    return result


def get_spec_path(configs):
    """
    Generate path to the RPM spec file (relative to current work directory,
    i.e. module configuration directory).

    :param configs: Configuration object.
    :return:        String with patch to the spec file.
    """
    return "rpm/SPECS/%s.spec" % configs.get("spec_file.module_name")


def check_patches_presence(configs, src_root):
    if configs.get_bool("quilt_support"):
        if os.path.isfile(os.path.join(src_root, "patches/series")):
            return True
        else:
            log_info("Quilt support enabled, but series file is not present," +
                     " falling back to default patch discovery method",
                     configs)

    if os.path.isdir(src_root + "patches") and \
            os.listdir(src_root + "patches"):
        return True

    log_status("Patch directory is empty or nonexistent, skipping", configs)

    return False


def iterate_patches(configs, src_root):
    if configs.get_bool("quilt_support") and \
            os.path.isfile(os.path.join(src_root, "patches/series")):
        patches_yielded = False

        try:
            with codecs.open(os.path.join(src_root, "patches/series"), 'r',
                             encoding='utf-8') as series:
                for patch in series.readlines():
                    if patch == "":
                        return
                    if patch.lstrip().startswith('#'):
                        continue
                    if patch[-1] == '\n':
                        patch = patch[:-1]
                    patch = patch.rsplit(" #", 1)[0]

                    if not os.path.isfile(os.path.join(src_root, "patches",
                                          patch)):
                        log_warn("Non-existing file is listed in series " +
                                 "file, skipping: %s" % patch, configs)

                        continue

                    yield patch

                    patches_yielded = True

                return

        except IOError as err:
            log_error("Error while parsing series file: " + str(err), configs)
            if patches_yielded:
                return
            else:
                log_warn("Falling back to default patch enumeration method",
                         configs, level=0)

    for file in sorted(os.listdir(os.path.join(src_root, "patches"))):
        # Hack for series file inside patches directory
        if file == "series" and configs.get_bool("quilt_support"):
            log_status("  Skipping %s" % file, configs, level=2)

            continue

        yield file


def cmd_generate_spec(configs, spec_path=None):
    """
    CMD generate_spec callback
    :param configs: Configuration object.
    """
    cfgfile = configs.get("config", default="module.config")

    if configs is None or not os.path.isfile(cfgfile):
        log_error(("Configuration file \"%(f)s\" was not found, use " +
                   "'ddiskit prepare_sources -c \"%(f)s\"' to create it") %
                  {"f": cfgfile}, configs)
        return ErrCode.CONFIG_READ_ERROR

    if not spec_path:
        spec_path = get_spec_path(configs)
        # NB: Current assumption is that when spec_file is provided in
        #     arguments, caller knows what it is doing in regard of
        #     already existing file.
        if os.path.isfile(spec_path):
            log_warn("RPM spec file \"%s\" exists!" % spec_path, configs,
                     level=0)
    try:
        template_dir = configs.get("template_dir")
        spec_template = configs.get("spec_template")
        with codecs.open(get_config_path(spec_template, extension="",
                         default_dir=template_dir), 'r',
                         encoding='utf-8') as fin:
            read_data = fin.read()
    except IOError as err:
        log_error(str(err), configs)
        return ErrCode.SPEC_TEMPLATE_READ_ERROR

    cwd = os.getcwd()

    src_root = "src/"
    patches = ""
    patches_do = ""
    if check_patches_presence(configs, src_root):
        log_status("Patches found, adding to the spec file:", configs)
        index = 0
        patches = "# Source code patches"
        for files in iterate_patches(configs, src_root):
            log_status("  Patch%d: %s" % (index, files), configs)
            patches += "\nPatch%d:\t%s" % (index, files)
            patches_do += "\n%%patch%d -p1" % index
            index = index + 1

    configs.set("spec_file.source_patches", patches)
    configs.set("spec_file.source_patches_do", patches_do)

    os.chdir(cwd)

    if os.path.isdir(src_root + "firmware") and \
            os.listdir(src_root + "firmware"):
        if not configs.get_bool("spec_file.firmware_include"):
            log_warn("Firmware directory contains files, but firmware " +
                     "package is disabled in config!", configs, level=0)
        else:
            fw_files = ""
            fw_install = ""
            log_status("Firmware found, adding to the spec file:", configs)
            for root, dirs, files in os.walk(src_root + "firmware/"):
                for file in files:
                    fpath = os.path.join(root[len(src_root + "firmware/"):],
                                         file)
                    log_status("  Firmware: %s" % fpath, configs)
                    fw_files += "/lib/firmware/%s\n" % fpath
                    fw_install += \
                        ("install -m 644 -D source/firmware/%(f)s " +
                         "$RPM_BUILD_ROOT/lib/firmware/%(f)s\n") % {"f": fpath}

            configs.set("spec_file.firmware_files", fw_files)
            configs.set("spec_file.firmware_files_install", fw_install)
    else:
        log_status("Firmware directory is empty or nonexistent, skipping",
                   configs)

    ret = process_configs_for_spec(configs)
    if ret:
        return ret

    source_fail = False
    for arch in configs.get("spec_file.kernel_arch").split():
        kernel_dir = "/usr/src/kernels/%s.%s" % \
            (configs.get("spec_file.kernel_version"), arch)
        if not os.path.isdir(kernel_dir):
            log_warn("Kernel source code is not found: \"%s\"" % kernel_dir,
                     configs, level=0)
            source_fail = True

    if source_fail:
        log_warn(("It probably will not be possible to build all RPMs " +
                  "on this system. You can try to install kernel-devel-%s " +
                  "package in order to fix this") %
                 configs.get("spec_file.kernel_version"), configs, level=0)

    read_data = apply_config(read_data, configs)
    log_status("Writing spec into %s ... " % spec_path, configs, newline=False)
    try:
        with codecs.open(spec_path, 'w', encoding='utf-8') as fout:
            fout.write(read_data)
    except IOError as err:
        log_error(str(err), configs)
        return ErrCode.SPEC_WRITE_ERROR

    log_status("OK", configs)

    return ErrCode.SUCCESS


def filter_tar_info(configs, nvv):
    def filter_tar_info_args(ti, configs, nvv):
        ti.mode = 0o755 if ti.isdir() else 0o644
        ti.uname = "nobody"
        ti.gname = "nobody"
        ti.uid = 0
        ti.gid = 0
        ti.mtime = time.time()

        if not filter_tar_info_args.tar_all and \
                any([x.startswith(".") for x in ti.name.split("/")]):
            log_status("  Hidden file, skipping: %s" % ti.name, configs,
                       level=1)

            return None

        fn = os.path.basename(ti.name)

        if ti.isfile() and ti.name.split("/")[0] != "firmware" and \
                not filter_tar_info_args.src_patterns.match(fn):
            log_status("  Unexpected file%s: %s" %
                       (", skipping"
                        if filter_tar_info_args.tar_strict else "", ti.name),
                       configs)

            if filter_tar_info_args.tar_strict:
                return None

        log_status("  Adding: %s" % ti.name, configs, level=2)

        ti.name = os.path.join(nvv, ti.name)

        return ti

    filter_tar_info_args.tar_all = configs.get_bool("tar_all")
    filter_tar_info_args.tar_strict = configs.get_bool("tar_strict")
    filter_tar_info_args.src_patterns = \
        re.compile(configs.get("src_patterns", default=SRC_PATTERNS))

    return functools.partial(filter_tar_info_args, configs=configs, nvv=nvv)


def file_walk(args, yield_dirs=False):
    for content in args:
        if os.path.isdir(content):
            if yield_dirs:
                yield content
            for root, dirs, files in os.walk(content):
                if yield_dirs:
                    for f in dirs:
                        yield os.path.join(root, f)
                for f in files:
                    yield os.path.join(root, f)
        elif os.path.exists(content):
            yield content
        else:
            raise IOError(errno.ENOENT, os.strerror(errno.ENOENT), content)


# pre-python 2.7 compat for TarFile.add(..., filter=)
def tar_add(tar, path, filter=None):
    for f in file_walk([path, ], yield_dirs=True):
        ti = tar.gettarinfo(f)

        if filter is not None:
            ti = filter(ti)

        if ti is None:
            continue

        if ti.isreg():
            with open(f, "rb") as f:
                tar.addfile(ti, f)
        else:
            tar.addfile(ti)


def cmd_build_rpm(configs):
    """
    CMD build_rpm callback
    :param configs: Configuration object.
    """
    warning = False
    cfgfile = configs.get("config", default="module.config")

    if configs is None or not os.path.isfile(cfgfile):
        log_error(("Configuration file \"%(f)s\" was not found, use " +
                   "'ddiskit prepare_sources -c \"%(f)s\"' to create it") %
                  {"f": cfgfile}, configs)
        return ErrCode.CONFIG_READ_ERROR

    ret = ErrCode.SUCCESS

    # Regenerate/check spec
    if configs.get_bool("generate_spec_on_build"):
        ret = cmd_generate_spec(configs)
        if ret:
            log_error("spec file generation failed, aborting", configs)
    else:
        check_spec = int(configs.get("check_spec_on_build", default=0))

        if check_spec:
            temp_file = tempfile.mkstemp(".spec")[1]
            ret = cmd_generate_spec(configs, temp_file)

            if ret:
                log_warn("Generation of temporary spec file failed",
                         configs, level=0)
            else:
                spec_path = get_spec_path(configs)
                verbosity = configs.get("verbosity")

                if verbosity >= 1:
                    diff_fmt = "--patch"
                else:
                    diff_fmt = "--quiet"

                # We use git diff because it's pretty and we've used it already
                # in do_git_src_check()
                diff_ret = command(["git", "diff", diff_fmt, "--no-index",
                                    "--exit-code", "--", spec_path, temp_file],
                                   configs, cmd_print_lvl=2,
                                   capture_output=False)[0]

                if diff_ret:
                    log_warn("Generated and existing spec files differ"
                             if diff_ret > 0 else
                             "Error occurred during git diff call",
                             configs, level=0)
                    ret = ErrCode.SPEC_CHECK_ERROR

            if check_spec == 2 and ret:
                proceed = val2bool(raw_input("Do you want to proceed (y/n)? "))

                if proceed:
                    ret = ErrCode.SUCCESS

            if check_spec < 2 or not ret:
                # We assume that if check_spec level is low, no one wants to
                # inspect temporary spec file
                os.unlink(temp_file)

                ret = ErrCode.SUCCESS
            elif ret:
                log_error("spec file check failed, aborting", configs)

    if ret:
        return ret

    # check Makefile
    saved_root = ""
    makefile_found = False
    src_root = "src/"
    for root, dirs, files in os.walk(src_root):
        if len(root) > len(saved_root):
            saved_root = root
        if "Makefile" in files:
            makefile_found = True
    saved_root = saved_root.replace(src_root, "")

    if not makefile_found:
        log_error("Makefile not found -> Please create one in %s" %
                  (src_root + saved_root), configs)
        return ErrCode.MAKEFILE_NOT_FOUND
    else:
        log_status("Checking makefile ... OK", configs)

    if do_quilt(QuiltCmd.DEAPPLY, configs):
        log_error("Quilt de-applying returned error, aborting", configs)
        return ErrCode.QUILT_DEAPPLY_ERROR

    src_check_lvl, src_check_ret = do_git_src_check(configs)
    if (src_check_lvl >= 2) and src_check_ret:
        log_error(("Source verification is critical (%d >= 2) and failed " +
                   "(return code %d), aborting") %
                  (src_check_lvl, src_check_ret), configs)
        return ErrCode.GIT_SRC_CHECK_ERROR
    elif src_check_lvl == 1:
        log_warn("Sources differ from the contents of the repository",
                 configs, level=0)

    nvv = "%s-%s-%s" % \
        (configs.get("spec_file.module_name"),
         configs.get("global.module_vendor"),
         configs.get("spec_file.module_version"))
    archive = "rpm/SOURCES/" + nvv + ".tar.bz2"
    log_status("Writing archive \"%s\" ..." % archive, configs)

    cwd = os.getcwd()

    try:
        tar = tarfile.open(archive, "w:bz2")
        os.chdir(src_root)
        for files in os.listdir("."):
            if "patches" == files or files.endswith(".rpm"):
                log_status("  Skipping: %s" % files, configs, level=1)
                continue
            if "firmware" == files:
                if os.path.isdir("firmware") and os.listdir("firmware"):
                    if not configs.get_bool("spec_file.firmware_include"):
                        warning = True
                        log_warn("Firmware directory contains files, but " +
                                 "firmware package is disabled in config!",
                                 configs, level=0)
                        continue
                else:
                    log_status("  Skipping: %s" % files, configs, level=1)
                    continue
            tar_add(tar, os.path.normpath(files),
                    filter=filter_tar_info(configs, nvv))
        tar.close()
    except Exception as err:
        log_error(str(err), configs)
        return ErrCode.SOURCE_ARCHIVE_WRITE_ERROR
    else:
        if not warning:
            log_status("Finished writing the archive.", configs)

    os.chdir(cwd)

    if os.path.isdir(src_root + "patches") and \
            os.listdir(src_root + "patches"):
        log_status("Copying patches into rpm/SOURCES:", configs)
        for files in iterate_patches(configs, src_root):
            shutil.copyfile(os.path.join(src_root, "patches", files),
                            os.path.join("rpm", "SOURCES", files))
            log_status("  Copying: %s" % files, configs)
    else:
        log_warn("Patch directory not found or empty -> skipping",
                 configs, level=0)

    build_arch = os.uname()[4]
    if not configs.get_bool("srpm") and \
            build_arch in configs.get("spec_file.kernel_arch").split():
        if not configs.get_bool("mock") and \
                not do_check_rpm_build(configs):
            log_warn("Binary RPM build check failed, building SRPM only",
                     configs, level=0)
            ret = do_build_srpm(configs)
        else:
            ret = do_build_rpm(configs, build_arch)
    else:
        if not configs.get_bool("srpm"):
            log_warn("Because you are not on the target architecture, " +
                     "building SRPM only", configs, level=0)
        ret = do_build_srpm(configs)

    ret_quilt = do_quilt(QuiltCmd.APPLY, configs)

    # Return ret in case it is non-zero or QUILT_APPLY_ERROR in case of quilt
    # errors or 0 in case everything is fine.
    return ret or ret_quilt and ErrCode.QUILT_APPLY_ERROR


def rpm_verbosity_opts(configs):
    if not hasattr(rpm_verbosity_opts, "verbosity"):
        rpm_verbosity_opts.verbosity = int(configs.get("verbosity", default=0))

    if rpm_verbosity_opts.verbosity <= 1:
        return []

    if rpm_verbosity_opts.verbosity == 2:
        return ['-v']

    if rpm_verbosity_opts.verbosity >= 3:
        return ['-vv']


def rpm_is_src(pkg, configs):
    ret, out = command(["rpm"] + rpm_verbosity_opts(configs) + ["-qp", "--qf",
                       "%|SOURCERPM?{0}:{1}|", "--nosignature", pkg],
                       configs, cmd_print_lvl=2)

    if ret:
        return None

    return out == "1"


def rpm_is_debuginfo(pkg, configs):
    ret, out = command(["rpm"] + rpm_verbosity_opts(configs) + ["-qp", "--qf",
                       "%{GROUP}", "--nosignature", pkg],
                       configs, cmd_print_lvl=2)

    if ret:
        return None

    return out == "Development/Debug"


def rpm_get_arch(pkg, configs):
    ret, arch = command(['rpm'] + rpm_verbosity_opts(configs) + ['-q', '--qf',
                        '%{ARCH}', '--nosignature', '-p', pkg], configs,
                        cmd_print_lvl=2)

    if ret:
        return None

    return arch


def rpm_get_requires(pkg, configs):
    ret, deps = command(['rpm'] + rpm_verbosity_opts(configs) + ['-qRp', pkg],
                        configs, cmd_print_lvl=2)

    if ret:
        return None

    return deps


def rpm_check_gpg(pkg, configs):
    """
    Performs checking of RPM's GPG signature.

    If rpm_gpg_check.use_keyring boolean option is not enabled, simple sanity
    check against GPG keys stored in RPM DB is assumed.  Otherwise it is
    assumed that RPMs should be signed with keys for which public part is
    present in one of the *.key files in rpm_gpg_check.keyring_dir.

    :param pkg:     RPM package file name.
    :param configs: Configuration object.
    :return:        Tuple, first item is boolean value that signifies whether
                    GPG signature check has been passed and the second item is
                    string containing error/informational message.
    """
    use_keyring_dir = configs.get_bool('rpm_gpg_check.use_keyring')

    if use_keyring_dir:
        keyring = configs.get('rpm_gpg_check.keyring_dir')

        if not keyring:
            return (False, "Check against keyring directory " +
                    "(rpm_gpg_check.use_keyring) is requested and " +
                    "rpm_gpg_check.keyring_dir is not set (%s)" % keyring)

        keyring = os.path.abspath(str(keyring))

        if not os.path.isdir(keyring):
            return (False, "\"%s\" is not a directory" % keyring)

        if not [x for x in os.listdir(keyring) if x.endswith(".key") and
                os.path.isfile(os.path.join(keyring, x)) and
                os.access(os.path.join(keyring, x), os.R_OK)]:
            return (False,
                    "\"%s\" doesn't contain accessible *.key files" % keyring)

        keyring_opts = ['-D', '%%_keyringpath "%s"' % keyring]
    else:
        keyring_opts = []

    ret, out = command(['rpm'] + rpm_verbosity_opts(configs) + keyring_opts +
                       ['-q', '--qf', '%|SIGPGP?{1}:{0}| %|SIGGPG?{1}:{0}|',
                        '-p', pkg], configs, cmd_print_lvl=2)
    if ret:
        return (False, "Can't check presence of GPG signature")
    if out == "0 0":
        return (False, "GPG signature is not present")

    ret, out = command(['rpm'] + rpm_verbosity_opts(configs) + keyring_opts +
                       ['-K', pkg], configs, cmd_print_lvl=2)

    return (not ret, out)


def cmd_build_iso(configs):
    """
    CMD build_iso callback
    :param configs: Configuration object.
    """
    rpm_sign_check = int(configs.get("rpm_gpg_check.check_level", default=0))
    error = 0

    src_rpms = []
    bin_rpms = {}  # Binary RPMs are stored per-arch
    for content in file_walk(configs.get("filelist")):
        try:
            if not content.endswith(".rpm"):
                log_warn(("File name \"%s\" does not end with .rpm " +
                          "extension, skipping") % content, configs)
                continue

            arch = rpm_get_arch(content, configs)
            if arch is None or not arch:
                log_warn("Failed to get arch for \"%s\", skipping" % content,
                         configs)
                continue

            is_src = rpm_is_src(content, configs)
            if is_src is None:
                log_warn("Failed to get whether RPM is source RPM for " +
                         "\"%s\", skipping" % content, configs)
                continue

            if not configs.get_bool("global.include_srpm") and is_src:
                log_warn("Source RPMs are disabled in configuration, " +
                         "skipping \"%s\"" % content, configs)
                continue

            if rpm_is_debuginfo(content, configs):
                log_warn("Debuginfo packages are not supported, skipping " +
                         "\"%s\"" % content, configs)
                continue

            if rpm_sign_check:
                check_ok, check_msg = rpm_check_gpg(content, configs)

                if not check_ok:
                    log_warn(("Failed to check GPG signature for \"%s\" " +
                              "(%s)%s") %
                             (content, check_msg,
                              ", aborting" if rpm_sign_check > 2 else
                              ", skipping" if rpm_sign_check > 1 else ""),
                             configs, level=0)

                    if rpm_sign_check > 2:
                        error = ErrCode.GPG_SIGNATURE_CHECK_ERROR
                    if rpm_sign_check > 1:
                        continue

            arch = re.sub(re.compile(r'i[0-9]86', re.IGNORECASE), 'i386', arch)

            if is_src:
                src_rpms.append(content)
            else:
                if arch not in bin_rpms:
                    bin_rpms[arch] = []
                bin_rpms[arch].append(content)

            log_status("Including \"%s\" (%s)" %
                       (content, "source" if is_src else "binary, %s" % arch),
                       configs)
        except OSError as err:
            log_warn(str(err), configs, level=0)

    if error:
        log_error("Errors occurred during the preparation of RPM list, " +
                  "aborting ISO creation", configs)

        return error

    dir_tmp = tempfile.mkdtemp()
    saved_umask = os.umask(0o077)
    cwd = os.getcwd()
    os.chdir(dir_tmp)
    create_dirs(["disk", "disk/rpms", "disk/src"] +
                [os.path.join("disk/rpms", a) for a in bin_rpms],
                configs, "Creating ISO directory structure")

    os.chdir(cwd)

    dir = os.path.join(dir_tmp, "disk/src")
    for file in src_rpms:
        shutil.copy(file, dir)

    for arch, rpms in bin_rpms.items():
        dir = os.path.join(dir_tmp, "disk/rpms", arch)
        for file in rpms:
            shutil.copy(file, dir)

        command(['createrepo', '--pretty', dir],
                configs, res_print_lvl=0, capture_output=False)

    try:
        with open(dir_tmp + "/disk/rhdd3", 'w') as fout:
            fout.write("Driver Update Disk version 3")
    except IOError as err:
        log_error(str(err), configs)
        return ErrCode.DD_ID_WRITE_ERROR

    isofile = configs.get("isofile")
    if isofile is None:
        # Try to use info from config for constructing file name
        try:
            isofile = "dd-" + \
                configs.get("spec_file.module_name") + "-" + \
                configs.get("spec_file.module_version") + "-" + \
                configs.get("spec_file.module_rpm_release") + "." + \
                configs.get("spec_file.rpm_dist") + ".iso"
        except TypeError:
            isofile = "dd.iso"

    configs.set("isofile", isofile)

    ret, _ = command(['mkisofs', '-V', 'OEMDRV', '-input-charset', 'UTF-8',
                      '-R', '-uid', '0', '-gid', '0', '-dir-mode', '0555',
                      '-file-mode', '0444', '-o',
                      isofile, dir_tmp + '/disk'],
                     configs, res_print_lvl=0, capture_output=False)
    os.umask(saved_umask)

    iso_mode = configs.get("isofile_mode")
    if not ret and iso_mode is not None:
        try:
            iso_mode_numeric = int(iso_mode, 0)
            log_info("Applying file mode %#o to \"%s\"..." %
                     (iso_mode_numeric, isofile), configs)
            os.chmod(isofile, iso_mode_numeric)
        except ValueError:
            log_warn("Failed to apply numeric file mode \"%s\"" % iso_mode,
                     configs, level=0)
    else:
        log_info(("mkisofs exit code (%d) is not zero or no ISO file mode " +
                  "provided in the defaults.iso_file mode, skipping ISO " +
                  "file mode setting") % ret, configs)

    shutil.rmtree(dir_tmp)

    log_status("ISO creation (file name \"%s\")... %s" %
               (isofile, "Failed" if ret else "OK"), configs,
               level=0 if ret else 1)

    return ret


def apply_config_file(filename, configs):
    """
    Read configuration file and apply it to a configuration dict.

    Ignores sections and keys containing dot as it breaks configs get/set
    (and we do not use such section/key names anyway).

    :param filename: Path to configuration file.
    :param configs:  Configuration object.
    :return:         Tuple containing updated configs value (useful in case no
                     starting configuration has been provided) and reading
                     result (None in case of configparser errors and result of
                     cfgparser.read() otherwise).
    """
    cfgparser = configparser.RawConfigParser()

    res = cfgparser.read(filename)

    try:
        for section in cfgparser.sections():
            if "." in section:
                log_warn(("section \"%s\" (config file \"%s\") contains" +
                          " dot in its name, ignored.") % (section, filename),
                         configs, level=0)
                continue
            for key, val in cfgparser.items(section):
                if "." in key:
                    log_warn(("key \"%s\" in section \"%s\" (config " +
                              "file \"%s\") contains dot in its name, " +
                              "ignored.") % (key, section, filename),
                             configs, level=0)
                    continue
                configs.set(key, val, section)
    except configparser.Error as err:
        log_error(str(err), configs)
        return (configs, None)

    return (configs, res)


def parse_config(filename, args, configs):
    """
    Parse configuration file.

    Returns configuration dict based on the supplied config filename, command
    line arguments, and default configuration dict.

    It merges default, system, user, profile, and module configs, and provide
    resulting configuration dictionary.

    :param filename: Path to input file.
    :param args:     Namespace instance containing command line arguments, as
                     returned by argparse.ArgumentParser.parse_args()
    :param configs:  Configuration object.
    :return:         Resulting configuration dict.
    """
    _, discarded_args = configs.apply_args(args)

    for a in discarded_args:
        log_warn("Configuration option override arguments should be " +
                 "provided in the <key>=<value> format, skipping \"%s\"" % a,
                 configs, level=0)

    implicit_configs = [
        os.path.join(configs.get("res_dir", default=RES_DIR), DEFAULT_CFG),
        SYSTEM_CFG,
        os.path.expanduser(USER_CFG),
        ]

    for cfg in implicit_configs:
        apply_config_file(cfg, configs)

    # Apply args here in order to derive profile to use
    configs.apply_args(args)

    profile_dir = configs.get("profile_dir")
    profile = configs.get("profile")
    if profile is not None and profile_dir is not None:
        profile_path = get_config_path(profile, profile_dir, extension=""),
        ret = apply_config_file(profile_path, configs)
        if ret is None or len(ret) == 0:
            log_error("Profile \"" + profile_path + "\" not found", configs)
            return None

    if filename is not None:
        ret = apply_config_file(os.path.basename(filename), configs)[1]
        if ret is None or len(ret) == 0:
            log_error("Configuration file \"%s\" not found" % filename,
                      configs)
            return None

    configs.apply_args(args)

    return configs


def cmd_dump_config(configs):
    filename = configs.get("dump_config_name", default="{config}.generated")

    log_status("Dumping config ... ", configs, newline=False)

    cfgparser = configs.dump_config()

    try:
        with codecs.open(filename, "w", encoding="utf-8") as f:
            cfgparser.write(f)
            log_status("OK", configs)
    except IOError as err:
        log_error(str(err), configs)
        return ErrCode.CONFIG_DUMP_WRITE_ERROR

    return ErrCode.SUCCESS


# So far, we only check machine and endianness
elf_machines = {
    ("AArch64", "2's complement, little endian"): "aarch64",
    ("Advanced Micro Devices X86-64", "2's complement, little endian"):
        "x86_64",
    ("Intel 80386", "2's complement, little endian"): "i386",
    ("PowerPC64", "2's complement, big endian"): "ppc64",
    ("PowerPC64", "2's complement, little endian"): "ppc64le",
    ("IBM S/390", "2's complement, big endian"): "s390x",
}
elf_machine_re = re.compile('^  Machine:\s+(?P<machine>.*)$', re.MULTILINE)
elf_endianness_re = re.compile('^  Data:\s+(?P<data>.*)$', re.MULTILINE)


def get_kmod_modvers(configs, path, rpm=None):
    log_status("    Processing \"%s\"%s as a kmod..." %
               (path, " (inside \"%s\")" % rpm if rpm else ""),
               configs, level=1)

    # Checking that the file is a kmod using modinfo tool.  Apparently,
    # modprobe --dump-modversions doesn't return erorr on *.ko.debug,
    # but produces garbage.
    ret, _, _ = command(["/sbin/modinfo", path],
                        configs, cmd_print_lvl=2, capture_stderr=True)
    if ret:
        return None

    ret, out = command(["readelf", "-h", path], configs, cmd_print_lvl=2)
    if ret:
        return None

    machine = elf_machine_re.search(out)
    endianness = elf_endianness_re.search(out)
    if machine is None or endianness is None:
        log_warn("Can't find \"Machine:\" and/or \"Data:\" tags in " +
                 "readelf -h output:\n%s" % out, configs)

        return None

    machine = machine.group("machine")
    endianness = endianness.group("data")
    arch = elf_machines.get((machine, endianness))
    if arch is None:
        log_warn(("Unknown machine/data combination, can't detect kmod arch:" +
                  " machine \"%s\", data \"%s\"") % (machine, endianness),
                 configs)

        return None

    ret, out = command(["/sbin/modprobe", "--dump-modversions", path],
                       configs, cmd_print_lvl=2)
    if ret:
        log_warn("modprobe failed to dump modversions for \"%s\"" % path,
                 configs)

        return None

    ret = []
    for line in out.split('\n'):
        if not line:
            continue

        parts = line.split('\t', 1)

        if len(parts) != 2:
            log_warn(("Unexpected string in modprobe --dump-modversions " +
                      "output: \"%s\"") % line, configs)
            continue

        ret.append((parts[1], {"arch": arch, "ver": parts[0],
                               "files": [os.path.basename(path)]
                               if rpm is None else
                               ["%s:%s" % (os.path.basename(rpm),
                                           os.path.basename(path))]}))

    return ret


def extract_kmods(configs, path):
    ret = None

    dir_tmp = tempfile.mkdtemp()

    log_status("    Trying to extract files from RPM \"%s\"..." % path,
               configs, level=1)

    rpm2cpio = Popen(args=["rpm2cpio", path], stdout=PIPE, stderr=PIPE,
                     close_fds=True)
    cpio = Popen(args=["cpio", "-idmv", "*.ko*"], cwd=dir_tmp,
                 stdin=rpm2cpio.stdout, stdout=PIPE, stderr=STDOUT)
    out, cpio_stderr = cpio.communicate()
    rpm2cpio_out, rpm2cpio_err = rpm2cpio.communicate()

    rpm2cpio_ret = rpm2cpio.returncode
    cpio_ret = cpio.returncode

    log_info("rpm2cpio exit code: %d, cpio exit code: %d" %
             (rpm2cpio_ret, cpio_ret), configs)
    if rpm2cpio_err:
        log_info("rpm2cpio stderr:\n%s" % rpm2cpio_err, configs, level=3)
    if out:
        log_info("cpio output:\n%s" % out, configs, level=3)
    if cpio_stderr:
        log_info("cpio stderr:\n%s" % cpio_stderr, configs, level=3)

    if rpm2cpio_ret == 0 and cpio_ret == 0:
        ret = []

        for f in file_walk([dir_tmp]):
            kmod_modvers = get_kmod_modvers(configs, f, rpm=path)

            if kmod_modvers is not None:
                ret += kmod_modvers

    shutil.rmtree(dir_tmp)

    return ret


sym_dep_re = re.compile(r'^([\w,]+: )?(kernel|ksym)\((?P<sym>[^)]+)\)' +
                        r'\s+=\s+(?P<ver>.*)$')


def get_rpm_modvers(configs, path):
    log_status("    Processing \"%s\" as an RPM..." % path, configs, level=1)

    extract_kmod = configs.get_bool("kabi_use_rpm_ko", default=False)

    if extract_kmod:
        return extract_kmods(configs, path)

    ret = []

    arch = rpm_get_arch(path, configs)
    deps = rpm_get_requires(path, configs)

    if arch is None or deps is None:
        return None

    for dep in deps.split('\n'):
        m = sym_dep_re.match(dep)
        if m is None:
            continue

        ret.append((m.group("sym"),
                   {"arch": arch,  "ver": m.group("ver"),
                    "files": [os.path.basename(path)]}))

    return ret


def tristate(configs, val, error, question, warning):
    """
    Helper for handling abort/ask/continue tri-state configuration options.
    """

    if val == 2:
        log_error(error, configs)
        return False
    elif val == 1:
        return val2bool(raw_input(question))
    elif val == 0:
        log_warn(warning, configs)
        return True


def cmd_update_kabi(configs):
    git_repo = configs.get("git_dir")
    arch_set = set()

    symver_pattern_str = configs.get("symvers_symbol_re")
    symver_pattern = re.compile(symver_pattern_str)

    overwrite = int(configs.get("kabi_files_overwrite", default=1))
    check_conflicts = int(configs.get("kabi_check_symvers_conflicts",
                                      default=1))

    commit = configs.get_bool("kabi_commit", default=False)
    commit_log = []
    commit_list = []

    if git_repo:
        # We presume that this is not a bare repo
        if os.path.basename(git_repo) == ".git":
            # Formally, we have to check core,worktree here
            parent_dir = os.path.dirname(git_repo)
        else:
            parent_dir = git_repo
    else:
        parent_dir = ""

        if commit:
            log_warn(("Git repository setting (%s) is not correct, "
                     "commit to it is not possible") % git_repo, configs)
            commit = 0

    result_kabi = {}

    log_status("Scanning files for kenel symbol dependencies...", configs)

    files_total = 0
    files_added = 0
    symbols_found = 0
    # Parse input files
    for f in file_walk(configs.get("filelist")):
        files_total += 1
        log_status("  Scanning \"%s\" for kernel symbol dependencies" % f,
                   configs, level=1)
        modvers = get_rpm_modvers(configs, f)

        if modvers is None:
            (log_warn if f.endswith(".rpm") else log_info)(
                    "Failed to parse file \"%s\" as an RPM" % f, configs)
            modvers = get_kmod_modvers(configs, f)

        if modvers is None:
            (log_warn if f.endswith((".ko", ".ko.gz", ".ko.xz")) else
             log_info)(
                    "Failed to parse file \"%s\" as a kmod" % f, configs)

            continue

        files_added += 1
        for sym, val in modvers:
            arch = val["arch"]
            if arch in result_kabi and sym in result_kabi[arch]:
                if val["ver"] != result_kabi[arch][sym]["ver"]:
                    msg = ("Conflicting versions for symbol \"%s\" (%s): " +
                           "version is %s in %s, but \"%s\" in %s") % \
                              (sym, arch, val["ver"], ", ".join(val["files"]),
                               result_kabi[sym]["ver"],
                               result_kabi[sym]["file_list"])
                    continue
                else:
                    result_kabi[arch][sym]["files"] += val["files"]
                    result_kabi[arch][sym]["file_list"] = \
                        ", ".join(result_kabi[sym]["files"])

            log_status(("    Adding symbol \"%s\" on architecture %s with " +
                        "version \"%s\" from file \"%s\"") %
                       (sym, arch, val["ver"], f), configs, level=2)

            symbols_found += 1
            if arch not in result_kabi:
                result_kabi[arch] = {}
            result_kabi[arch][sym] = val
            result_kabi[arch][sym]["file_list"] = \
                ", ".join(result_kabi[arch][sym]["files"])
            arch_set.add(arch)

    log_status("%d files scanned, added %d symbols (%s) from %d files" %
               (files_total, symbols_found,
                ", ".join(["%d unique on %s" % (len(s), a) for a, s in
                           result_kabi.items()]), files_added),
               configs)

    log_status("Scanning files for kenel symbol dependencies...", configs)

    # Used for both Module.symvers and kABI whitelist
    def parse_symvers_file(param, desc):
        filenames = {}
        ret = {}

        for arch in arch_set:
            filenames[arch] = configs.get(param, overrides={"arch": arch})

            with open(filenames[arch], 'r') as f:
                log_status("  Processing %s file \"%s\"..." %
                           (desc, filenames[arch]), configs, level=2)
                symvers = ret[arch] = {}

                line_count = 0
                for l in f.readlines():
                    line_count += 1
                    if l == "":
                        break
                    if l.endswith("\n"):
                        l = l[:-1]
                    m = symver_pattern.match(l)
                    if not m:
                        log_error(("Malformed record in Module.symvers (%s):" +
                                   " \"%s\" doesn't match \"%s\"") %
                                  (filenames[arch], symver_pattern_str, l),
                                  configs)
                        continue

                    # ver, symbol, file, export
                    parts = m.groupdict()

                    if parts["symbol"] in symvers:
                        log_warn(("Duplicate Module.symvers record (in " +
                                  "\"%s\"): existing \"%s\", new \"%s\". " +
                                  "Skipping.") % (filenames[arch],
                                 symvers[parts["symbol"]]["str"], l))
                        continue

                    symvers[parts["symbol"]] = parts
                    symvers[parts["symbol"]]["str"] = l
                    symvers[parts["symbol"]]["line"] = line_count

        return (filenames, ret)

    symvers_path, kernel_symvers = parse_symvers_file("symvers_path",
                                                      "module symvers")

    log_status("%s module symvers files loaded, %s symbols total" %
               (len(symvers_path),
                sum([len(x) for x in kernel_symvers.itervalues()])),
               configs)

    whitelist_path = {}
    whitelist = {}
    if configs.get("kabi_whitelist") is not None:
        whitelist_path, whitelist = parse_symvers_file("kabi_whitelist",
                                                       "kABI whitelist")

        log_status("%s kABI whitelist files loaded, %s symbols total" %
                   (len(whitelist_path),
                    sum([len(x) for x in whitelist.itervalues()])),
                   configs)

    log_status("Generating output files...", configs)

    def map_map_iterator(m):
        for sm in m.itervalues():
            for k, v in sm.items():
                yield k, v

    # Generate resulting files
    # Mandatory symbol fields: ver, files, arch
    # Optional symbol fields: kmod
    for sym, val in map_map_iterator(result_kabi):
        file_list = val["file_list"]
        ver = val["ver"]
        arch = val["arch"]

        if sym not in kernel_symvers[arch]:
            log_warn(("Symbol \"%s\" (%s, from %s) is not present in " +
                      "Module.symvers (%s), skipping") % (sym, arch,
                     file_list, symvers_path[arch]), configs)
            continue

        if arch in whitelist and sym in whitelist[arch]:
            wl_sym = whitelist[arch][sym]

            if ver != wl_sym["ver"]:
                msg = ("Discovered a discrepancy between version of symbol " +
                       "\"%s\" in files %s (%s) and kABI whitelist (%s, from" +
                       " %s, line %d)") % (sym, file_list, ver, wl_sym["ver"],
                                           whitelist_path[arch],
                                           wl_sym["line"])

                if not tristate(check_conflicts, msg + ", aborting",
                                msg + ", continue (y/n)? ", msg):
                    return ErrCode.KABI_SYMBOL_CONFLICT_CHECK_ERROR
            else:
                log_info(("Symbol \"%s\" (%s, from %s) is already present in" +
                          " kABI whitelist (%s, line %d), skipping") %
                         (sym, arch, file_list, whitelist_path[arch],
                         wl_sym["line"]), configs)
            continue

        kernel_sym = kernel_symvers[arch][sym]

        if ver != kernel_sym["ver"]:
            msg = ("Discovered a discrepancy between version of symbol "
                   "\"%s\" in files %s (%s) and Module.symvers (%s, from %s," +
                   " line %d)") % (sym, file_list, ver, kernel_sym["ver"],
                                   symvers_path[arch], kernel_sym["line"])

            if not tristate(check_conflicts, msg + ", aborting",
                            msg + ", continue (y/n)? ", msg):
                return ErrCode.KABI_SYMBOL_CONFLICT_CHECK_ERROR

        overrides = {
            "sym": sym,
            "arch": arch,
            "ver": ver,
            "kmod_file": file_list,
            "kernel_file": kernel_sym["file"],
            "kernel_export": kernel_sym["export"],
        }

        out_path = os.path.join(parent_dir,
                                configs.get("kabi_dest_dir",
                                            overrides=overrides),
                                configs.get("kabi_file_name_pattern",
                                            default=sym, overrides=overrides))
        write_path = out_path

        if not os.path.exists(os.path.dirname(out_path)):
            os.makedirs(os.path.dirname(out_path))

        if os.path.exists(out_path):
            if overwrite == 0:
                log_warn("kABI file overwrite disabled, skipping \"%s\"" %
                         out_path, configs)
                continue
            elif overwrite == 1:
                write_path = tempfile.mkstemp(".kabi", sym)[1]
            else:
                log_info("kABI file \"%s\" already exists, overwriting" %
                         out_path)

        with open(write_path, 'w') as f:
            kabi_record = configs.get("kabi_file_template", default="",
                                      overrides=overrides)

            # Files should have \n at the end
            if not kabi_record.endswith('\n'):
                kabi_record += '\n'

            f.write(kabi_record)

        if write_path != out_path:
            command(["git", "diff", "--patch", "--no-index", "--",
                     out_path, write_path],
                    configs, cmd_print_lvl=2, capture_output=False)

            if val2bool(raw_input(
                    "Do you want to overwrite the file (y/n)? ")):
                shutil.move(write_path, out_path)
            else:
                os.unlink(write_path)
                continue

        commit_log.append(configs.get("kabi_commit_log_template",
                                      overrides=overrides))
        commit_list.append(out_path)

    log_status("%d files created." % len(commit_list), configs)

    commit_message = configs.get("kabi_commit_message", default="", overrides={
            "kabi_commit_log": "\n".join(commit_log)})

    if commit:
        if len(commit_list) == 0:
            log_status("Nothing to commit, skipping.", configs)
            return ErrCode.SUCCESS

        log_status("Adding %d files to git..." % len(commit_list), configs)
        ret = command(["git", "--git-dir=%s" % git_repo,
                       "--work-tree=%s" % parent_dir, "add", ] + commit_list,
                      configs, capture_output=False, cmd_print_lvl=2)[0]

        if ret:
            log_error("Got error when tried to add kABI files, aborting.")
            return ErrCode.GIT_ADD_ERROR

        log_status("Commit...", configs)
        ret = command(["git", "--git-dir=%s" % git_repo,
                       "--work-tree=%s" % parent_dir, "commit",
                       "-m", commit_message, "--", ] + commit_list,
                      configs, capture_output=False, cmd_print_lvl=2)[0]

        if ret:
            log_error("Got an error when tried to make a git commit, " +
                      "aborting.", configs)
            return ErrCode.GIT_COMMIT_ERROR

    log_status("Done.", configs)

    return ErrCode.SUCCESS


def parse_cli():
    """
    Commandline argument parser
    :return: commandline arguments
    """
    root_parser = argparse.ArgumentParser(prog='ddiskit',
                                          description='Red Hat tool for ' +
                                          'create Driver Update Disk')
    root_parser.add_argument("-v", "--verbosity", action="count", default=0,
                             help="Increase output verbosity")
    root_parser.add_argument("-p", "--profile",
                             help="Configuration profile to use")
    root_parser.add_argument("-R", "--res-dir",
                             help="Resources dir (%s by default)" % RES_DIR)
    root_parser.add_argument("-T", "--template-dir",
                             help="Templates dir (%s by default)" %
                             TEMPLATE_DIR)
    root_parser.add_argument("-P", "--profile-dir",
                             help="Profiles dir (%s by default)" % PROFILE_DIR)
    root_parser.add_argument("-d", "--dump-config", action='store_true',
                             default=None,
                             help="Dump derived configuration after command " +
                                  "execution")
    root_parser.add_argument("-o", "--dump-config-name",
                             help="Name of config dump file")
    root_parser.add_argument("-q", "--quilt-enable", action="store_const",
                             dest="quilt_support", const=True,
                             help="Enable quilt integration")
    root_parser.add_argument("-Q", "--quilt-disable", action="store_const",
                             dest="quilt_support", const=False,
                             help="Disable quilt integration")
    root_parser.add_argument("-C", "--config-option", action="append",
                             help="Override configuration options")

    cmdparsers = root_parser.add_subparsers(title='Commands',
                                            help='main ddiskit commands')

    # parser for the "prepare_sources" command
    parser_prepare_sources = cmdparsers.add_parser('prepare_sources',
                                                   help='Prepare sources')
    parser_prepare_sources.add_argument("-c", "--config",
                                        default='module.config',
                                        help="Config file")
    parser_prepare_sources.add_argument("-t", "--config-template",
                                        help="Config file template")
    parser_prepare_sources.add_argument("-g", "--git-dir",
                                        help="Directory containing source " +
                                             "repository")
    parser_prepare_sources.add_argument("-r", "--git-revision",
                                        help="Git commit to source from")
    parser_prepare_sources.add_argument("-d", "--git-src-directory",
                                        help="Whitespace-separated list of " +
                                             "directories in the git " +
                                             "repository that contains " +
                                             "module source files")
    parser_prepare_sources.add_argument("-M", "--major",
                                        help="Major distribution version")
    parser_prepare_sources.add_argument("-m", "--minor",
                                        help="Minor distribution version")
    parser_prepare_sources.set_defaults(func=cmd_prepare_sources)

    # parser for the "generate_spec" command
    parser_generate_spec = cmdparsers.add_parser('generate_spec',
                                                 help='Generate spec file')
    parser_generate_spec.add_argument("-c", "--config",
                                      default='module.config',
                                      help="Config file")
    parser_generate_spec.add_argument("-t", "--spec-template",
                                      help="RPM spec file template")
    parser_generate_spec.set_defaults(func=cmd_generate_spec)

    # parser for the "build_rpm" command
    parser_build_rpm = cmdparsers.add_parser('build_rpm', help='Build rpm')
    parser_build_rpm.add_argument("-c", "--config", default='module.config',
                                  help="Config file")
    parser_build_rpm.add_argument("-a", "--tar-all", action='store_true',
                                  default=None,
                                  help="Tar all files, including hidden ones")
    parser_build_rpm.add_argument("-e", "--tar-strict", action='store_true',
                                  default=None,
                                  help="Tar only expected files")
    parser_build_rpm.add_argument("-s", "--srpm", action='store_true',
                                  default=None, help="Build src RPM")
    parser_build_rpm.add_argument("-m", "--mock", action="store_true",
                                  default=None, help="Build using mock")
    parser_build_rpm.add_argument("-r", "--mock-config", default=None,
                                  help="Mock config")
    parser_build_rpm.add_argument("-l", "--mock-offline", action="store_true",
                                  default=None,
                                  help="Run mock in offline mode")
    parser_build_rpm.add_argument("-O", "--mock-opts",
                                  default=None,
                                  help="Mock options, shell quoting " + \
                                       "is supported")
    parser_build_rpm.add_argument("-A", "--mock-opts-append",
                                  default=None, action="append",
                                  dest="mock_opts",
                                  help="Additional mock options (avoid " + \
                                       "overriding options already set), " + \
                                       "shell quoting is supported")
    parser_build_rpm.add_argument("-g", "--check-git-src", default=None,
                                  help="Whether to perform verification of " +
                                       "sources against git repository. " +
                                       "0 - do not perform, " +
                                       "1 - perform check, issue warning, " +
                                       "2 - perform check, abort on failure")
    parser_build_rpm.add_argument("-G", "--generate-spec-on-build",
                                  action='store_true', default=None,
                                  help="Regenerate spec faile at the " +
                                       "beginning of RPM build")
    parser_build_rpm.set_defaults(func=cmd_build_rpm)

    # parser for the "build_iso" command
    parser_build_iso = cmdparsers.add_parser('build_iso', help='Build iso')
    parser_build_iso.add_argument("-c", "--config", default='module.config',
                                  help="Config file")
    parser_build_iso.add_argument("-i", "--isofile", default=None,
                                  help="Output file name")
    parser_build_iso.add_argument("filelist", nargs="*",
                                  default=["rpm/RPMS/", "rpm/SRPMS/"],
                                  help="RPM list, separated by space and " +
                                  "can use directory path")
    parser_build_iso.set_defaults(func=cmd_build_iso)

    # parser for the "dump_config" command
    parser_dump_config = cmdparsers.add_parser('dump_config',
                                               help='Dump derived ' +
                                                    'configuration')
    parser_dump_config.add_argument("-c", "--config", default='module.config',
                                    help="(Input) config file")
    parser_dump_config.add_argument("-o", "--dump-config-name",
                                    help="Name of config dump file")
    parser_dump_config.set_defaults(func=cmd_dump_config)

    # parser for the "update_kabi" command
    parser_update_kabi = cmdparsers.add_parser('update_kabi',
                                               help='Update RHEL kABI' +
                                                    'whitelist')
    parser_update_kabi.add_argument("-c", "--config", default='module.config',
                                    help="Config file")
    parser_update_kabi.add_argument("-g", "--git-dir",
                                    help="Directory containing source " +
                                         "repository")
    parser_update_kabi.add_argument("-d", "--kabi-dest-dir",
                                    help="Destination directory where new " +
                                         "whitelist entry files should be " +
                                         "created")
    parser_update_kabi.add_argument("-e", "--extract-kmod",
                                    action="store_const",
                                    dest="kabi_use_rpm_ko", const=True,
                                    help="Use modvers of *.ko inside RPM " +
                                         "instead of RPM requires")
    parser_update_kabi.add_argument("-E", "--no-extract-kmod",
                                    action="store_const",
                                    dest="kabi_use_rpm_ko", const=False,
                                    help="Use RPM Requires: field instead of" +
                                         " modvers of the included kmods")
    parser_update_kabi.add_argument("-o", "--overwrite",
                                    action="store_const",
                                    dest="kabi_files_overwrite", const=2,
                                    help="Overwrite existing kABI files")
    parser_update_kabi.add_argument("-O", "--no-overwrite",
                                    action="store_const",
                                    dest="kabi_files_overwrite", const=0,
                                    help="Do not overwrite existing kABI " +
                                         "files ")
    parser_update_kabi.add_argument("-i", "--overwrite-interactive",
                                    action="store_const",
                                    dest="kabi_files_overwrite", const=1,
                                    help="Ask for kABI file overwrite")
    parser_update_kabi.add_argument("-t", "--commit", action="store_const",
                                    dest="kabi_commit", const=True,
                                    help="Commit changes on success")
    parser_update_kabi.add_argument("-n", "--no-commit", action="store_const",
                                    dest="kabi_commit", const=False,
                                    help="Commit changes on success")
    parser_update_kabi.add_argument("-m", "--kabi-commit-message",
                                    help="Commit message")
    parser_update_kabi.add_argument("-M", "--symvers-path",
                                    help="Path to a relevant Module.symvers " +
                                         "file")
    parser_update_kabi.add_argument("-b", "--break-on-errors",
                                    dest="kabi_check_symvers_conflicts",
                                    action="store_const", const=2,
                                    help="Abort processing if a symbol " +
                                         "version conflict is discovered")
    parser_update_kabi.add_argument("-B", "--no-break-on-version-conflicts",
                                    dest="kabi_check_symvers_conflicts",
                                    action="store_const", const=0,
                                    help="Continue processing even if " +
                                         "a symbol version conflict is " +
                                         "discovered")
    parser_update_kabi.add_argument("-a", "--ask-on-version-conflicts",
                                    dest="kabi_check_symvers_conflicts",
                                    action="store_const", const=1,
                                    help="Ask whether to continue processing" +
                                         " if a symbol version conflict " +
                                         "is discovered")
    parser_update_kabi.add_argument("-w", "--kabi-whitelist",
                                    help="Whitelist file, for excluding kABI" +
                                         " symbols that are added already")
    parser_update_kabi.add_argument("filelist", nargs="*",
                                    default=["rpm/RPMS/", "rpm/SRPMS/"],
                                    help="File list (*.rpm/*.ko), whitespace" +
                                         " separated")
    parser_update_kabi.set_defaults(func=cmd_update_kabi)

    args = root_parser.parse_args()
    if not hasattr(args, "func"):
        root_parser.print_help()
        return None
    return args


def main():
    args = parse_cli()
    if args is None:
        return ErrCode.ARGS_PARSE_ERROR
    if os.path.isfile(args.config):
        config_dir = os.path.dirname(args.config)
        if config_dir:
            os.chdir(config_dir)

        configs = parse_config(args.config, args,
                               DDiskitConfig(default_config))
        if configs is None:
            return ErrCode.CONFIG_READ_ERROR
        configs = check_config(configs)
        if configs is None:
            return ErrCode.CONFIG_CHECK_ERROR
    else:
        configs = parse_config(None, args,
                               DDiskitConfig(default_config))

    log_info("Config:\n%s" % configs, configs)

    ret = args.func(configs)

    if configs.get_bool("dump_config"):
        dump_ret = cmd_dump_config(configs)
        ret = ret or dump_ret

    log_info("Command \"%s\" returned \"%r\"" % (args.func.__name__, ret),
             configs, level=1)

    return ret


if __name__ == "__main__":
    sys.exit(main())
