Blame SOURCES/edk2-build.py

8ff9b3
#!/usr/bin/python3
8ff9b3
import os
8ff9b3
import sys
8ff9b3
import glob
8ff9b3
import shutil
8ff9b3
import optparse
8ff9b3
import subprocess
8ff9b3
import configparser
8ff9b3
8ff9b3
rebase_prefix    = ""
8ff9b3
version_override = None
8ff9b3
release_date     = None
8ff9b3
8ff9b3
def check_rebase():
8ff9b3
    """ detect 'git rebase -x edk2-build.py master' testbuilds """
8ff9b3
    global rebase_prefix
8ff9b3
    global version_override
8ff9b3
8ff9b3
    if not os.path.exists('.git/rebase-merge/msgnum'):
8ff9b3
        return ""
8ff9b3
    with open('.git/rebase-merge/msgnum', 'r') as f:
8ff9b3
        msgnum = int(f.read())
8ff9b3
    with open('.git/rebase-merge/end', 'r') as f:
8ff9b3
        end = int(f.read())
8ff9b3
    with open('.git/rebase-merge/head-name', 'r') as f:
8ff9b3
        head = f.read().strip().split('/')
8ff9b3
8ff9b3
    rebase_prefix = f'[ {int(msgnum/2)} / {int(end/2)} - {head[-1]} ] '
8ff9b3
    if msgnum != end and not version_override:
8ff9b3
        # fixed version speeds up builds
8ff9b3
        version_override = "test-build-patch-series"
8ff9b3
8ff9b3
def get_coredir(cfg):
8ff9b3
    if cfg.has_option('global', 'core'):
8ff9b3
        return os.path.abspath(cfg['global']['core'])
8ff9b3
    else:
8ff9b3
        return os.getcwd()
8ff9b3
8ff9b3
def get_version(cfg):
8ff9b3
    coredir = get_coredir(cfg)
8ff9b3
    if version_override:
8ff9b3
        version = version_override
8ff9b3
        print('')
8ff9b3
        print(f'### version [override]: {version}')
8ff9b3
        return version
8ff9b3
    if os.environ.get('RPM_PACKAGE_NAME'):
8ff9b3
        version = os.environ.get('RPM_PACKAGE_NAME');
8ff9b3
        version += '-' + os.environ.get('RPM_PACKAGE_VERSION');
8ff9b3
        version += '-' + os.environ.get('RPM_PACKAGE_RELEASE');
8ff9b3
        print('')
8ff9b3
        print(f'### version [rpmbuild]: {version}')
8ff9b3
        return version
8ff9b3
    if os.path.exists(coredir + '/.git'):
8ff9b3
        cmdline = [ 'git', 'describe', '--tags', '--abbrev=8', '--match=edk2-stable*' ]
8ff9b3
        result = subprocess.run(cmdline, stdout = subprocess.PIPE, cwd = coredir)
8ff9b3
        version = result.stdout.decode().strip()
8ff9b3
        print('')
8ff9b3
        print(f'### version [git]: {version}')
8ff9b3
        return version
8ff9b3
    return None
8ff9b3
8ff9b3
def pcd_string(name, value):
8ff9b3
    return f'{name}=L{value}\\0'
8ff9b3
8ff9b3
def pcd_version(cfg):
8ff9b3
    version = get_version(cfg)
8ff9b3
    if version is None:
8ff9b3
        return []
8ff9b3
    return [ '--pcd', pcd_string('PcdFirmwareVersionString', version) ]
8ff9b3
8ff9b3
def pcd_release_date(cfg):
8ff9b3
    if release_date is None:
8ff9b3
        return []
8ff9b3
    return [ '--pcd', pcd_string('PcdFirmwareReleaseDateString', release_date) ]
8ff9b3
8ff9b3
def build_message(line, line2 = None):
8ff9b3
    if os.environ.get('TERM') in [ 'xterm', 'xterm-256color' ]:
8ff9b3
        # setxterm  title
8ff9b3
        start  = '\x1b]2;'
8ff9b3
        end    = '\x07'
8ff9b3
        print(f'{start}{rebase_prefix}{line}{end}', end = '')
8ff9b3
8ff9b3
    print('')
8ff9b3
    print('###')
8ff9b3
    print(f'### {rebase_prefix}{line}')
8ff9b3
    if line2:
8ff9b3
        print(f'### {line2}')
8ff9b3
    print('###')
8ff9b3
8ff9b3
def build_run(cmdline, name, section, silent = False):
8ff9b3
    print(cmdline)
8ff9b3
    if silent:
8ff9b3
        print('### building in silent mode ...', flush = True)
8ff9b3
        result = subprocess.run(cmdline,
8ff9b3
                                stdout = subprocess.PIPE,
8ff9b3
                                stderr = subprocess.STDOUT)
8ff9b3
8ff9b3
        logfile = f'{section}.log'
8ff9b3
        print(f'### writing log to {logfile} ...')
8ff9b3
        with open(logfile, 'wb') as f:
8ff9b3
            f.write(result.stdout)
8ff9b3
8ff9b3
        if result.returncode:
8ff9b3
            print('### BUILD FAILURE')
8ff9b3
            print('### output')
8ff9b3
            print(result.stdout.decode())
8ff9b3
            print(f'### exit code: {result.returncode}')
8ff9b3
        else:
8ff9b3
            print('### OK')
8ff9b3
    else:
8ff9b3
        result = subprocess.run(cmdline)
8ff9b3
    if result.returncode:
8ff9b3
        print(f'ERROR: {cmdline[0]} exited with {result.returncode} while building {name}')
8ff9b3
        sys.exit(result.returncode)
8ff9b3
8ff9b3
def build_copy(plat, tgt, dstdir, copy):
8ff9b3
    srcdir = f'Build/{plat}/{tgt}_GCC5'
8ff9b3
    names = copy.split()
8ff9b3
    srcfile = names[0]
8ff9b3
    if len(names) > 1:
8ff9b3
        dstfile = names[1]
8ff9b3
    else:
8ff9b3
        dstfile = os.path.basename(srcfile)
8ff9b3
    print(f'# copy: {srcdir} / {srcfile}  =>  {dstdir} / {dstfile}')
8ff9b3
8ff9b3
    src = srcdir + '/' + srcfile
8ff9b3
    dst = dstdir + '/' + dstfile
8ff9b3
    os.makedirs(os.path.dirname(dst), exist_ok = True)
8ff9b3
    shutil.copy(src, dst)
8ff9b3
8ff9b3
def pad_file(dstdir, pad):
8ff9b3
    args = pad.split()
8ff9b3
    if len(args) < 2:
8ff9b3
        raise RuntimeError(f'missing arg for pad ({args})')
8ff9b3
    name = args[0]
8ff9b3
    size = args[1]
8ff9b3
    cmdline = [
8ff9b3
        'truncate',
8ff9b3
        '--size', size,
8ff9b3
        dstdir + '/' + name,
8ff9b3
    ]
8ff9b3
    print(f'# padding: {dstdir} / {name}  =>  {size}')
8ff9b3
    subprocess.run(cmdline)
8ff9b3
8ff9b3
def build_one(cfg, build, jobs = None, silent = False):
8ff9b3
    cmdline  = [ 'build' ]
8ff9b3
    cmdline += [ '-t', 'GCC5' ]
8ff9b3
    cmdline += [ '-p', cfg[build]['conf'] ]
8ff9b3
8ff9b3
    if (cfg[build]['conf'].startswith('OvmfPkg/') or
8ff9b3
        cfg[build]['conf'].startswith('ArmVirtPkg/')):
8ff9b3
        cmdline += pcd_version(cfg)
8ff9b3
        cmdline += pcd_release_date(cfg)
8ff9b3
8ff9b3
    if jobs:
8ff9b3
        cmdline += [ '-n', jobs ]
8ff9b3
    for arch in cfg[build]['arch'].split():
8ff9b3
        cmdline += [ '-a', arch ]
8ff9b3
    if 'opts' in cfg[build]:
8ff9b3
        for name in cfg[build]['opts'].split():
8ff9b3
            section = 'opts.' + name
8ff9b3
            for opt in cfg[section]:
8ff9b3
                cmdline += [ '-D', opt + '=' + cfg[section][opt] ]
8ff9b3
    if 'pcds' in cfg[build]:
8ff9b3
        for name in cfg[build]['pcds'].split():
8ff9b3
            section = 'pcds.' + name
8ff9b3
            for pcd in cfg[section]:
8ff9b3
                cmdline += [ '--pcd', pcd + '=' + cfg[section][pcd] ]
8ff9b3
    if 'tgts' in cfg[build]:
8ff9b3
        tgts = cfg[build]['tgts'].split()
8ff9b3
    else:
8ff9b3
        tgts = [ 'DEBUG' ]
8ff9b3
    for tgt in tgts:
8ff9b3
        desc = None
8ff9b3
        if 'desc' in cfg[build]:
8ff9b3
            desc = cfg[build]['desc']
8ff9b3
        build_message(f'building: {cfg[build]["conf"]} ({cfg[build]["arch"]}, {tgt})',
8ff9b3
                      f'description: {desc}')
8ff9b3
        build_run(cmdline + [ '-b', tgt ],
8ff9b3
                  cfg[build]['conf'],
8ff9b3
                  build + '.' + tgt,
8ff9b3
                  silent)
8ff9b3
8ff9b3
        if 'plat' in cfg[build]:
8ff9b3
            # copy files
8ff9b3
            for cpy in cfg[build]:
8ff9b3
                if not cpy.startswith('cpy'):
8ff9b3
                    continue
8ff9b3
                build_copy(cfg[build]['plat'],
8ff9b3
                           tgt,
8ff9b3
                           cfg[build]['dest'],
8ff9b3
                           cfg[build][cpy])
8ff9b3
            # pad builds
8ff9b3
            for pad in cfg[build]:
8ff9b3
                if not pad.startswith('pad'):
8ff9b3
                    continue
8ff9b3
                pad_file(cfg[build]['dest'],
8ff9b3
                         cfg[build][pad])
8ff9b3
8ff9b3
def build_basetools(silent = False):
8ff9b3
    build_message(f'building: BaseTools')
8ff9b3
    basedir = os.environ['EDK_TOOLS_PATH']
8ff9b3
    cmdline = [ 'make', '-C', basedir ]
8ff9b3
    build_run(cmdline, 'BaseTools', 'build.basetools', silent)
8ff9b3
8ff9b3
def binary_exists(name):
8ff9b3
    for dir in os.environ['PATH'].split(':'):
8ff9b3
        if os.path.exists(dir + '/' + name):
8ff9b3
            return True
8ff9b3
    return False
8ff9b3
8ff9b3
def prepare_env(cfg):
8ff9b3
    """ mimic Conf/BuildEnv.sh """
8ff9b3
    workspace = os.getcwd()
8ff9b3
    packages = [ workspace, ]
8ff9b3
    path = os.environ['PATH'].split(':')
8ff9b3
    dirs = [
8ff9b3
        'BaseTools/Bin/Linux-x86_64',
8ff9b3
        'BaseTools/BinWrappers/PosixLike'
8ff9b3
    ]
8ff9b3
8ff9b3
    if cfg.has_option('global', 'pkgs'):
8ff9b3
        for pkgdir in cfg['global']['pkgs'].split():
8ff9b3
            packages.append(os.path.abspath(pkgdir))
8ff9b3
    coredir = get_coredir(cfg)
8ff9b3
    if coredir != workspace:
8ff9b3
        packages.append(coredir)
8ff9b3
8ff9b3
    # add basetools to path
8ff9b3
    for dir in dirs:
8ff9b3
        p = coredir + '/' + dir
8ff9b3
        if not os.path.exists(p):
8ff9b3
            continue
8ff9b3
        if p in path:
8ff9b3
            continue
8ff9b3
        path.insert(0, p)
8ff9b3
8ff9b3
    # run edksetup if needed
8ff9b3
    toolsdef = coredir + '/Conf/tools_def.txt';
8ff9b3
    if not os.path.exists(toolsdef):
8ff9b3
        os.makedirs(os.path.dirname(toolsdef), exist_ok = True)
8ff9b3
        build_message('running BaseTools/BuildEnv')
8ff9b3
        cmdline = [ 'sh', 'BaseTools/BuildEnv' ]
8ff9b3
        subprocess.run(cmdline, cwd = coredir)
8ff9b3
8ff9b3
    # set variables
8ff9b3
    os.environ['PATH'] = ':'.join(path)
8ff9b3
    os.environ['PACKAGES_PATH'] = ':'.join(packages)
8ff9b3
    os.environ['WORKSPACE'] = workspace
8ff9b3
    os.environ['EDK_TOOLS_PATH'] = coredir + '/BaseTools'
8ff9b3
    os.environ['CONF_PATH'] = coredir + '/Conf'
8ff9b3
    os.environ['PYTHON_COMMAND'] = '/usr/bin/python3'
8ff9b3
    os.environ['PYTHONHASHSEED'] = '1'
8ff9b3
8ff9b3
    # for cross builds
8ff9b3
    if binary_exists('arm-linux-gnu-gcc'):
8ff9b3
        os.environ['GCC5_ARM_PREFIX'] = 'arm-linux-gnu-'
8ff9b3
    if binary_exists('aarch64-linux-gnu-gcc'):
8ff9b3
        os.environ['GCC5_AARCH64_PREFIX'] = 'aarch64-linux-gnu-'
8ff9b3
    if binary_exists('riscv64-linux-gnu-gcc'):
8ff9b3
        os.environ['GCC5_RISCV64_PREFIX'] = 'riscv64-linux-gnu-'
8ff9b3
    if binary_exists('x86_64-linux-gnu-gcc'):
8ff9b3
        os.environ['GCC5_IA32_PREFIX'] = 'x86_64-linux-gnu-'
8ff9b3
        os.environ['GCC5_X64_PREFIX'] = 'x86_64-linux-gnu-'
8ff9b3
        os.environ['GCC5_BIN'] = 'x86_64-linux-gnu-'
8ff9b3
8ff9b3
def build_list(cfg):
8ff9b3
    for build in cfg.sections():
8ff9b3
        if not build.startswith('build.'):
8ff9b3
            continue
8ff9b3
        name = build.lstrip('build.')
8ff9b3
        desc = 'no description'
8ff9b3
        if 'desc' in cfg[build]:
8ff9b3
            desc = cfg[build]['desc']
8ff9b3
        print(f'# {name:20s} - {desc}')
8ff9b3
    
8ff9b3
def main():
8ff9b3
    parser = optparse.OptionParser()
8ff9b3
    parser.add_option('-c', '--config', dest = 'configfile',
8ff9b3
                      type = 'string', default = '.edk2.builds')
8ff9b3
    parser.add_option('-C', '--directory', dest = 'directory', type = 'string')
8ff9b3
    parser.add_option('-j', '--jobs', dest = 'jobs', type = 'string')
8ff9b3
    parser.add_option('-m', '--match', dest = 'match', type = 'string')
8ff9b3
    parser.add_option('-l', '--list', dest = 'list', action = 'store_true', default = False)
8ff9b3
    parser.add_option('--silent', dest = 'silent', action = 'store_true', default = False)
8ff9b3
    parser.add_option('--core', dest = 'core', type = 'string')
8ff9b3
    parser.add_option('--pkg', '--package', dest = 'pkgs', type = 'string', action = 'append')
8ff9b3
    parser.add_option('--version-override', dest = 'version_override', type = 'string')
8ff9b3
    parser.add_option('--release-date', dest = 'release_date', type = 'string')
8ff9b3
    (options, args) = parser.parse_args()
8ff9b3
8ff9b3
    if options.directory:
8ff9b3
        os.chdir(options.directory)
8ff9b3
8ff9b3
    cfg = configparser.ConfigParser()
8ff9b3
    cfg.optionxform = str
8ff9b3
    cfg.read(options.configfile)
8ff9b3
8ff9b3
    if options.list:
8ff9b3
        build_list(cfg)
8ff9b3
        return
8ff9b3
8ff9b3
    if not cfg.has_section('global'):
8ff9b3
        cfg.add_section('global')
8ff9b3
    if options.core:
8ff9b3
        cfg.set('global', 'core', options.core)
8ff9b3
    if options.pkgs:
8ff9b3
        cfg.set('global', 'pkgs', ' '.join(options.pkgs))
8ff9b3
8ff9b3
    global version_override
8ff9b3
    global release_date
8ff9b3
    check_rebase()
8ff9b3
    if options.version_override:
8ff9b3
        version_override = options.version_override
8ff9b3
    if options.release_date:
8ff9b3
        release_date = options.release_date
8ff9b3
8ff9b3
    prepare_env(cfg)
8ff9b3
    build_basetools(options.silent)
8ff9b3
    for build in cfg.sections():
8ff9b3
        if not build.startswith('build.'):
8ff9b3
            continue
8ff9b3
        if options.match and options.match not in build:
8ff9b3
            print(f'# skipping "{build}" (not matching "{options.match}")')
8ff9b3
            continue
8ff9b3
        build_one(cfg, build, options.jobs, options.silent)
8ff9b3
8ff9b3
if __name__ == '__main__':
8ff9b3
    sys.exit(main())