Blame centos.git.repolist.py

bonniek 7a56e5
#!/usr/bin/env python
059392
#pylint: disable=line-too-long
a6a2bc
#
a6a2bc
#  License: GPLv3
a6a2bc
#
a6a2bc
#  Initial Author: Bonnie King <bonniek@fnal.gov>
a6a2bc
#         Updates:
a6a2bc
#                  Pat Riehecky <riehecky@fnal.gov>
a6a2bc
#
059392
'''Get list of repos from pagure, to grab CentOS sources'''
bonniek 7a56e5
059392
# for python3 compat
059392
from __future__ import unicode_literals
059392
from __future__ import absolute_import
059392
from __future__ import print_function
059392
059392
import logging
bonniek 7a56e5
import sys
059392
import json
059392
import textwrap
059392
import time
bonniek 7a56e5
059392
sys.setrecursionlimit(500)
bonniek 7a56e5
059392
try:
059392
    from argparse import ArgumentParser
059392
except ImportError:  # pragma: no cover
059392
    print("Please install argparse - rpm: python-argparse", file=sys.stderr)
059392
    raise
bonniek 7a56e5
059392
try:
059392
    import requests
059392
except ImportError:  # pragma: no cover
059392
    print("Please install requests - rpm: python-requests", file=sys.stderr)
059392
    raise
bonniek 7a56e5
bonniek 7a56e5
059392
def setup_args():
059392
    '''
059392
        Setup the argparse object.
a6a2bc
059392
        Make sure all fields have defaults so we could use this as an object
059392
    '''
059392
    parser = ArgumentParser(description=textwrap.dedent(__doc__))
059392
059392
    parser.add_argument('--debug', action='store_true', default=False,
059392
                        help='Print debugging information')
059392
    parser.add_argument('--hostname', default='git.centos.org',
059392
                        type=str, help='What host should we query?')
059392
    parser.add_argument('--apiver', default='0',
059392
                        type=str, help='What api version is the host?')
059392
    parser.add_argument('--namespace', default='rpms',
059392
                        type=str, help='What project namespace?')
059392
    parser.add_argument('--show-forks', action='store_true', default=False,
059392
                        help='Should we also show project forks?')
059392
059392
    return parser
059392
059392
def run_query(hostname, api, namespace, forks):
059392
    '''
059392
        Actually call the API version
059392
    '''
059392
    list_of_urls = []
059392
    if str(api) == '0':
059392
        query = 'https://{hostname}/api/0/projects?per_page=50&namespace={namespace}'.format(hostname=hostname, namespace=namespace)
059392
        if forks:
059392
            query = query + '&forks=1'
059392
        else:
059392
            query = query + '&forks=0'
bonniek 7a56e5
059392
        fetch_prefix = 'https://{hostname}/'.format(hostname=hostname)
bonniek 7a56e5
059392
        fetch_next_v0(query, fetch_prefix, list_of_urls)
059392
    else:
059392
        raise NotImplementedError("Unknown API version %s", api)
059392
059392
    list_of_urls.sort()
059392
    return list_of_urls
059392
059392
def fetch_next_v0(page, fetch_prefix, list_of_urls):
059392
    '''
059392
        Recursively fetch the page until we are done
059392
    '''
059392
    logging.debug('Trying to fetch %s', page)
bonniek 7a56e5
    try:
059392
        req = requests.get(page)
a6a2bc
    except requests.exceptions.RequestException as err_msg:
059392
        print(err_msg, file=sys.stderr)
059392
        raise
059392
059392
    try:
059392
        message = json.loads(req.text)
059392
    except ValueError as err_msg:
059392
        print(page, file=sys.stderr)
059392
        print(req.text, file=sys.stderr)
059392
        print(err_msg, file=sys.stderr)
059392
        raise
bonniek 7a56e5
059392
    for project in message['projects']:
059392
        list_of_urls.append(fetch_prefix + project['fullname'])
4ec6d6
059392
    if 'next' in message['pagination']:
059392
        if message['pagination']['next']:
059392
            time.sleep(0.25) # Add a smallish delay to help with load
059392
            fetch_next_v0(message['pagination']['next'], fetch_prefix, list_of_urls)
4ec6d6
059392
if __name__ == '__main__':
bonniek 7a56e5
059392
    PARSER = setup_args()
059392
    ARGS = PARSER.parse_args()
bonniek 7a56e5
059392
    if ARGS.debug:
059392
        logging.basicConfig(level=logging.DEBUG)
bonniek 7a56e5
059392
    URLS = run_query(ARGS.hostname, ARGS.apiver, ARGS.namespace, ARGS.show_forks)
4ec6d6
059392
    for URL in URLS:
059392
        print(URL)