|
bonniek |
7a56e5 |
#!/usr/bin/env python
|
|
|
059392 |
#pylint: disable=line-too-long
|
|
|
a6a2bc |
#
|
|
|
a6a2bc |
# License: GPLv3
|
|
|
a6a2bc |
#
|
|
|
a6a2bc |
# Initial Author: Bonnie King <bonniek@fnal.gov>
|
|
|
a6a2bc |
# Updates:
|
|
|
a6a2bc |
# Pat Riehecky <riehecky@fnal.gov>
|
|
|
a6a2bc |
#
|
|
|
059392 |
'''Get list of repos from pagure, to grab CentOS sources'''
|
|
bonniek |
7a56e5 |
|
|
|
059392 |
# for python3 compat
|
|
|
059392 |
from __future__ import unicode_literals
|
|
|
059392 |
from __future__ import absolute_import
|
|
|
059392 |
from __future__ import print_function
|
|
|
059392 |
|
|
|
059392 |
import logging
|
|
bonniek |
7a56e5 |
import sys
|
|
|
059392 |
import json
|
|
|
059392 |
import textwrap
|
|
|
059392 |
import time
|
|
bonniek |
7a56e5 |
|
|
|
059392 |
sys.setrecursionlimit(500)
|
|
bonniek |
7a56e5 |
|
|
|
059392 |
try:
|
|
|
059392 |
from argparse import ArgumentParser
|
|
|
059392 |
except ImportError: # pragma: no cover
|
|
|
059392 |
print("Please install argparse - rpm: python-argparse", file=sys.stderr)
|
|
|
059392 |
raise
|
|
bonniek |
7a56e5 |
|
|
|
059392 |
try:
|
|
|
059392 |
import requests
|
|
|
059392 |
except ImportError: # pragma: no cover
|
|
|
059392 |
print("Please install requests - rpm: python-requests", file=sys.stderr)
|
|
|
059392 |
raise
|
|
bonniek |
7a56e5 |
|
|
bonniek |
7a56e5 |
|
|
|
059392 |
def setup_args():
|
|
|
059392 |
'''
|
|
|
059392 |
Setup the argparse object.
|
|
|
a6a2bc |
|
|
|
059392 |
Make sure all fields have defaults so we could use this as an object
|
|
|
059392 |
'''
|
|
|
059392 |
parser = ArgumentParser(description=textwrap.dedent(__doc__))
|
|
|
059392 |
|
|
|
059392 |
parser.add_argument('--debug', action='store_true', default=False,
|
|
|
059392 |
help='Print debugging information')
|
|
|
059392 |
parser.add_argument('--hostname', default='git.centos.org',
|
|
|
059392 |
type=str, help='What host should we query?')
|
|
|
059392 |
parser.add_argument('--apiver', default='0',
|
|
|
059392 |
type=str, help='What api version is the host?')
|
|
|
059392 |
parser.add_argument('--namespace', default='rpms',
|
|
|
059392 |
type=str, help='What project namespace?')
|
|
|
059392 |
parser.add_argument('--show-forks', action='store_true', default=False,
|
|
|
059392 |
help='Should we also show project forks?')
|
|
|
059392 |
|
|
|
059392 |
return parser
|
|
|
059392 |
|
|
|
059392 |
def run_query(hostname, api, namespace, forks):
|
|
|
059392 |
'''
|
|
|
059392 |
Actually call the API version
|
|
|
059392 |
'''
|
|
|
059392 |
list_of_urls = []
|
|
|
059392 |
if str(api) == '0':
|
|
|
059392 |
query = 'https://{hostname}/api/0/projects?per_page=50&namespace={namespace}'.format(hostname=hostname, namespace=namespace)
|
|
|
059392 |
if forks:
|
|
|
059392 |
query = query + '&forks=1'
|
|
|
059392 |
else:
|
|
|
059392 |
query = query + '&forks=0'
|
|
bonniek |
7a56e5 |
|
|
|
059392 |
fetch_prefix = 'https://{hostname}/'.format(hostname=hostname)
|
|
bonniek |
7a56e5 |
|
|
|
059392 |
fetch_next_v0(query, fetch_prefix, list_of_urls)
|
|
|
059392 |
else:
|
|
|
059392 |
raise NotImplementedError("Unknown API version %s", api)
|
|
|
059392 |
|
|
|
059392 |
list_of_urls.sort()
|
|
|
059392 |
return list_of_urls
|
|
|
059392 |
|
|
|
059392 |
def fetch_next_v0(page, fetch_prefix, list_of_urls):
|
|
|
059392 |
'''
|
|
|
059392 |
Recursively fetch the page until we are done
|
|
|
059392 |
'''
|
|
|
059392 |
logging.debug('Trying to fetch %s', page)
|
|
bonniek |
7a56e5 |
try:
|
|
|
059392 |
req = requests.get(page)
|
|
|
a6a2bc |
except requests.exceptions.RequestException as err_msg:
|
|
|
059392 |
print(err_msg, file=sys.stderr)
|
|
|
059392 |
raise
|
|
|
059392 |
|
|
|
059392 |
try:
|
|
|
059392 |
message = json.loads(req.text)
|
|
|
059392 |
except ValueError as err_msg:
|
|
|
059392 |
print(page, file=sys.stderr)
|
|
|
059392 |
print(req.text, file=sys.stderr)
|
|
|
059392 |
print(err_msg, file=sys.stderr)
|
|
|
059392 |
raise
|
|
bonniek |
7a56e5 |
|
|
|
059392 |
for project in message['projects']:
|
|
|
059392 |
list_of_urls.append(fetch_prefix + project['fullname'])
|
|
|
4ec6d6 |
|
|
|
059392 |
if 'next' in message['pagination']:
|
|
|
059392 |
if message['pagination']['next']:
|
|
|
059392 |
time.sleep(0.25) # Add a smallish delay to help with load
|
|
|
059392 |
fetch_next_v0(message['pagination']['next'], fetch_prefix, list_of_urls)
|
|
|
4ec6d6 |
|
|
|
059392 |
if __name__ == '__main__':
|
|
bonniek |
7a56e5 |
|
|
|
059392 |
PARSER = setup_args()
|
|
|
059392 |
ARGS = PARSER.parse_args()
|
|
bonniek |
7a56e5 |
|
|
|
059392 |
if ARGS.debug:
|
|
|
059392 |
logging.basicConfig(level=logging.DEBUG)
|
|
bonniek |
7a56e5 |
|
|
|
059392 |
URLS = run_query(ARGS.hostname, ARGS.apiver, ARGS.namespace, ARGS.show_forks)
|
|
|
4ec6d6 |
|
|
|
059392 |
for URL in URLS:
|
|
|
059392 |
print(URL)
|