From 9ed232b8cdd8b565477889dcb23eefffb5bbf5d8 Mon Sep 17 00:00:00 2001 From: Howard Johnson Date: Dec 13 2014 14:07:55 +0000 Subject: Initial version of the lookaside upload script --- diff --git a/lookaside/README b/lookaside/README new file mode 100644 index 0000000..168f71b --- /dev/null +++ b/lookaside/README @@ -0,0 +1,113 @@ +CentOS Infra lookaside upload script + +This upload script is a fork of Fedora's. The original version of the script was taken from https://git.fedorahosted.org/cgit/fedora-infrastructure.git/tree/scripts/upload.cgi (although I think that version might be a bit on the old side, but it does the job). + +The script has been modified somewhat (ok, quite a lot) to fit CentOS' requirements. + + +Requirements: + +Basic requirements from kbsingh et al: + +- Users must be authenticated to be able to upload +- Authentication must be done using client SSL certificates from a private CA +- Files must be uploaded in the centos // scheme, rather than the fedora scheme +- The upload process must be able to be driven from centpkg (so ideally similar script/parameters to fedora) +- Upload permissions must be controlled from the gitblit config + +Some more requirements of mine: + +- The upload system must check for revocation of client certs +- Client cert revocation should be done quickly (immediately if possible) + + +Assumptions: + +- Every user who can upload has a unique username. +- Every user has an account in git.centos.org's gitblit. +- Every user has a client SSL certificate, issued by the CBS CA. +- The client SSL certificate has a CN of their username as part of the certificate subject. +- A users git username matches their koji username. + + +Access Control: + +The script requires a user to authenticate with their client SSL cert. All users can run the script in check mode. If a user tries to upload, the script checks the gitblit config to ensure that the user has permissions to the package. + +(This access control can be disabled using the script config file, if desired) + + +Apache SSL Configuration: + +The following config is sufficient for the SSL client auth: + + # This needs to point to the CA cert that issued the client certs + SSLCACertificateFile /etc/pki/tls/certs/cbs-ca.crt + + # Don't verify client certs on the server by default + SSLVerifyClient none + + # Tell Apache the upload script is CGI + ScriptAlias /lookaside/upload.cgi /var/www/cgi-bin/upload.cgi + + # Enable client cert verification for the upload script URL + + SSLVerifyClient require + SSLVerifyDepth 1 + + +Ideally we want to check for client cert revocation. We can check a local CRL file with the following config. Note that if the CRL is updated, Apache needs to be restarted to pick up the changes. + + # check revocation of client certs against the CRL + SSLCARevocationCheck leaf + # specify the CRL file location (must be in PEM format) + SSLCARevocationFile /etc/pki/tls/certs/ipa.crl + +If at some point we switch to a CA with an OCSP responder, we can use the following config (Apache 2.4 required) to do a live OCSP client cert revocation check on upload: + + # turn on OCSP checking of client certs + SSLOCSPEnable on + # set the URL for the OCSP responder + SSLOCSPDefaultResponder http://my.ca.server/ca/ocsp + # ignore the OCSP URL in client certs and use the one we configured + SSLOCSPOverrideResponder on + + +Calling The Script: + +Obviously, you need a valid client cert. You then need to call the script with the right parameters. Here's the parameters: + +- name +- branch +- sha1sum +- file + +name, branch, and sha1sum are mandatory. name is the package name. branch is the name of the branch. sha1sum is a (lowercase) hex SHA1 checksum for the file. If only these three parameters are provided, the script checks if there is a matching file uploaded. If the file exists, the script returns the string "Available". If the file does not exist, the script returns the string "Missing". + +If the parameter file is passed, this parameter must be the contents of the file. The uploaded file will be written to a temporary file, and the checksum of the file compared to the value of the sha1sum parameter. If the sums match, the file is moved into position. + +The script can be called using curl commands similar to the following. The file mycert.pem contains the client's cert (and private key). + +To upload a file (389-ds-base-1.3.1.6.tar.bz2 in the current directory): + +curl --cert ./mycert.pem https://git.centos.org/lookaside/upload.cgi --form "name=389-ds-base" --form "branch=c7" --form "sha1sum=ce4e6293a996e1045bc8f75533418f3172b391ee" --form "file=@389-ds-base-1.3.1.6.tar.bz2" +File 389-ds-base-1.3.1.6.tar.bz2 size 3070988 SHA1 ce4e6293a996e1045bc8f75533418f3172b391ee stored OK + +To check if a file exists: + +curl --cert ./mycert.pem https://git.centos.org/lookaside/upload.cgi --form "name=389-ds-base" --form "branch=c7" --form "sha1sum=ce4e6293a996e1045bc8f75533418f3172b391ee" +Available + +(on a normal end client, this would be handled by centpkg) + +(centos' curl defaults to looking in a NSS db for a client cert; force the --cert arg to be a path - even a relative one like ./mycert.pem - to stop this behaviour) + + +Email Notification: + +The script sends an email to a configured email address when a file is uploaded. The script tries to send the mail via a configured mail relay. By default, SELinux will block the script from connecting to port 25 on the configured mail server. Set the httpd_can_network_connect boolean on to allow it. + + +Script Config File: + +The upload script config file is /etc/lookaside.cfg. There are config options for all the interesting values. The script doesn't check the config syntax is correct, so missing / malformed values are likely to case python tracebacks. diff --git a/lookaside/lookaside.cfg b/lookaside/lookaside.cfg new file mode 100644 index 0000000..e7a53f7 --- /dev/null +++ b/lookaside/lookaside.cfg @@ -0,0 +1,13 @@ +[lookaside] +cache_dir = /var/www/html/lookaside + +[acl] +do_acl = yes +gitblit_config = /etc/gitblit.conf + +[mail] +send_mail = yes +sender_name = CentOS source uploads +sender_email = nobody@centos.org +recipient = centos-build-reports@centos.org +smtp_server = localhost diff --git a/lookaside/upload.cgi b/lookaside/upload.cgi new file mode 100755 index 0000000..014b7fb --- /dev/null +++ b/lookaside/upload.cgi @@ -0,0 +1,236 @@ +#!/usr/bin/python +# +# CGI script to handle file updates for the rpms CVS repository. There +# is nothing really complex here other than tedious checking of our +# every step along the way... +# +# Written for Fedora, modified to suit CentOS Infrastructure. +# Modified by Howard Johnson 2014 +# +# License: GPL + +# +# centos' lookaside is a bit differently laid out to fedora's. +# centos uses a // scheme. +# +# The upload.cgi gets called with the following arguments: +# name - package (git repo) name +# branch - branch name +# sha1sum - SHA1 checksum of the file +# file - the file to upload (optional) +# +# With only the first three args, the script runs in check mode. +# With the fourth too, it operates in upload mode. +# + +import os +import sys +import cgi +import tempfile +import syslog +import smtplib +import re +from ConfigParser import SafeConfigParser + +from email import Header, Utils +try: + from email.mime.text import MIMEText +except ImportError: + from email.MIMEText import MIMEText + +import hashlib +sha1_constructor = hashlib.sha1 + +# Config file with all our settings +CONFIG = '/etc/lookaside.cfg' +conf = SafeConfigParser() +conf.read(CONFIG) + +# Reading buffer size +BUFFER_SIZE = 4096 + +# Gitblit config file regexes +SECTION = re.compile(r'\[(.+)\]') +OPTION = re.compile(r'(.+)=(.+)') +REPO = re.compile(r'([^/]+).git$') + +def merge_gitblit_section(repoacl, repos, users): + for repo in repos: + if repo not in repoacl: + repoacl[repo] = [] + for user in users: + if user not in repoacl[repo]: + repoacl[repo].append(user) + +# turns the gitblit config file into a dict of package name to permitted users +def parse_gitblit_config(filename): + insection = False + repoacl = {} + sectrepos = [] + sectusers = [] + + f = open(filename) + for line in f: + if line.strip() == '' or line[0] == '#': + continue + secth = SECTION.match(line) + if secth: + if insection: + merge_gitblit_section(repoacl, sectrepos, sectusers) + + sectrepos = [] + sectusers = [] + else: + insection = True + continue + opt = OPTION.match(line) + if opt: + if not insection: + continue + key = opt.group(1).strip() + value = opt.group(2).strip() + + if key == "repository": + pack = REPO.search(value) + if pack: + sectrepos.append(pack.group(1)) + elif key == "user": + sectusers.append(value) + if insection: + merge_gitblit_section(repoacl, sectrepos, sectusers) + f.close() + return repoacl + +def send_error(text): + print text + sys.exit(1) + +def check_form(form, var): + ret = form.getvalue(var, None) + if ret is None: + send_error('Required field "%s" is not present.' % var) + if isinstance(ret, list): + send_error('Multiple values given for "%s". Aborting.' % var) + return ret + +def send_email(pkg, sha1, filename, username): + text = """A file has been added to the lookaside cache for %(pkg)s: + +%(sha1)s %(filename)s""" % locals() + msg = MIMEText(text) + sender_name = conf.get('mail', 'sender_name') + sender_email = conf.get('mail', 'sender_email') + sender = Utils.formataddr((sender_name, sender_email)) + recipient = conf.get('mail', 'recipient') + msg['Subject'] = 'File %s uploaded to lookaside cache by %s' % ( + filename, username) + msg['From'] = sender + msg['To'] = recipient + try: + s = smtplib.SMTP(conf.get('mail', 'smtp_server')) + s.sendmail(sender, recipient, msg.as_string()) + except: + errstr = 'sending mail for upload of %s failed!' % filename + print >> sys.stderr, errstr + syslog.syslog(errstr) + +def main(): + os.umask(002) + + username = os.environ.get('SSL_CLIENT_S_DN_CN', None) + + print 'Content-Type: text/plain' + print + + assert os.environ['REQUEST_URI'].split('/')[1] == 'lookaside' + + form = cgi.FieldStorage() + name = check_form(form, 'name') + branch = check_form(form, 'branch') + sha1sum = check_form(form, 'sha1sum') + + action = None + upload_file = None + filename = None + + # Is this a submission or a test? + # in a test, we don't get a file. + if not form.has_key('file'): + action = 'check' + print >> sys.stderr, '[username=%s] Checking file status: NAME=%s BRANCH=%s SHA1SUM=%s' % (username, name, branch, sha1sum) + else: + action = 'upload' + upload_file = form['file'] + if not upload_file.file: + send_error('No file given for upload. Aborting.') + filename = os.path.basename(upload_file.filename) + print >> sys.stderr, '[username=%s] Processing upload request: NAME=%s BRANCH=%s SHA1SUM=%s' % (username, name, branch, sha1sum) + + module_dir = os.path.join(conf.get('lookaside', 'cache_dir'), name, branch) + dest_file = os.path.join(module_dir, sha1sum) + + # try to see if we already have this file... + if os.path.exists(dest_file): + if action == 'check': + print 'Available' + else: + upload_file.file.close() + dest_file_stat = os.stat(dest_file) + print 'File %s already exists' % filename + print 'File: %s Size: %d' % (dest_file, dest_file_stat.st_size) + sys.exit(0) + elif action == 'check': + print 'Missing' + sys.exit(0) + + # if desired, make sure the user has permission to write to this branch + if conf.getboolean('acl', 'do_acl'): + # load in the gitblit config + repoacl = parse_gitblit_config(conf.get('acl', 'gitblit_config')) + + # if the package isn't in the gitblit config, we can't give upload perms + if name not in repoacl: + send_error("Unknown package %s" % name) + + # now check the perms + if username not in repoacl[name]: + send_error("Write access package %s rejected for user %s" % (name, username)) + + # check that all directories are in place + if not os.path.isdir(module_dir): + os.makedirs(module_dir, 02775) + + # grab a temporary filename and dump our file in there + tempfile.tempdir = module_dir + tmpfile = tempfile.mkstemp(sha1sum)[1] + tmpfd = open(tmpfile, 'w') + + # now read the whole file in + m = sha1_constructor() + filesize = 0 + while True: + data = upload_file.file.read(BUFFER_SIZE) + if not data: + break + tmpfd.write(data) + m.update(data) + filesize += len(data) + + # now we're done reading, check the MD5 sum of what we got + tmpfd.close() + check_sha1sum = m.hexdigest() + if sha1sum != check_sha1sum: + os.unlink(tmpfile) + send_error("SHA1 check failed. Received %s instead of %s." % (check_sha1sum, sha1sum)) + + # rename it its final name + os.rename(tmpfile, dest_file) + os.chmod(dest_file, 0644) + + print >> sys.stderr, '[username=%s] Stored %s (%d bytes)' % (username, dest_file, filesize) + print 'File %s size %d SHA1 %s stored OK' % (filename, filesize, sha1sum) + if conf.getboolean('mail', 'send_mail'): + send_email(name, sha1sum, filename, username) + +if __name__ == '__main__': + main()