| |
@@ -1,4 +1,5 @@
|
| |
#!/usr/bin/env python
|
| |
+ #pylint: disable=line-too-long
|
| |
#
|
| |
# License: GPLv3
|
| |
#
|
| |
@@ -6,73 +7,112 @@
|
| |
# Updates:
|
| |
# Pat Riehecky <riehecky@fnal.gov>
|
| |
#
|
| |
- '''Get list of repos from gitblit RPC, to grab CentOS sources'''
|
| |
+ '''Get list of repos from pagure, to grab CentOS sources'''
|
| |
|
| |
- import optparse
|
| |
- import requests
|
| |
- import simplejson as json
|
| |
+ # for python3 compat
|
| |
+ from __future__ import unicode_literals
|
| |
+ from __future__ import absolute_import
|
| |
+ from __future__ import print_function
|
| |
+
|
| |
+ import logging
|
| |
import sys
|
| |
+ import json
|
| |
+ import textwrap
|
| |
+ import time
|
| |
|
| |
- RPCURL = "https://git.centos.org/rpc/?req=LIST_REPOSITORIES"
|
| |
+ sys.setrecursionlimit(500)
|
| |
|
| |
- def read_args():
|
| |
- '''
|
| |
- read in the command line args and set things up
|
| |
- '''
|
| |
+ try:
|
| |
+ from argparse import ArgumentParser
|
| |
+ except ImportError: # pragma: no cover
|
| |
+ print("Please install argparse - rpm: python-argparse", file=sys.stderr)
|
| |
+ raise
|
| |
|
| |
- desc = '''Get list of git repositories from the GitBlit json RPC
|
| |
- '''
|
| |
+ try:
|
| |
+ import requests
|
| |
+ except ImportError: # pragma: no cover
|
| |
+ print("Please install requests - rpm: python-requests", file=sys.stderr)
|
| |
+ raise
|
| |
|
| |
- usage = "usage: %prog [options] "
|
| |
- parser = optparse.OptionParser(usage=usage, description=desc)
|
| |
- parser.add_option('-p', '--project', metavar="<PROJECTS>",
|
| |
- help='''project path (default 'rpms', could be 'all', 'core-sig'...)''',
|
| |
- default='rpms')
|
| |
|
| |
- parser.add_option('-b', '--branch', metavar="<branch name>",
|
| |
- help='Only list repos with this branch (default master)',
|
| |
- default = 'master')
|
| |
+ def setup_args():
|
| |
+ '''
|
| |
+ Setup the argparse object.
|
| |
|
| |
- parser.add_option('-u', '--url', metavar="<URL>",
|
| |
- help='URL to check (default %s)' % (RPCURL),
|
| |
- default = RPCURL)
|
| |
+ Make sure all fields have defaults so we could use this as an object
|
| |
+ '''
|
| |
+ parser = ArgumentParser(description=textwrap.dedent(__doc__))
|
| |
+
|
| |
+ parser.add_argument('--debug', action='store_true', default=False,
|
| |
+ help='Print debugging information')
|
| |
+ parser.add_argument('--hostname', default='git.centos.org',
|
| |
+ type=str, help='What host should we query?')
|
| |
+ parser.add_argument('--apiver', default='0',
|
| |
+ type=str, help='What api version is the host?')
|
| |
+ parser.add_argument('--namespace', default='rpms',
|
| |
+ type=str, help='What project namespace?')
|
| |
+ parser.add_argument('--show-forks', action='store_true', default=False,
|
| |
+ help='Should we also show project forks?')
|
| |
+
|
| |
+ return parser
|
| |
+
|
| |
+ def run_query(hostname, api, namespace, forks):
|
| |
+ '''
|
| |
+ Actually call the API version
|
| |
+ '''
|
| |
+ list_of_urls = []
|
| |
+ if str(api) == '0':
|
| |
+ query = 'https://{hostname}/api/0/projects?per_page=50&namespace={namespace}'.format(hostname=hostname, namespace=namespace)
|
| |
+ if forks:
|
| |
+ query = query + '&forks=1'
|
| |
+ else:
|
| |
+ query = query + '&forks=0'
|
| |
|
| |
- (options, args) = parser.parse_args()
|
| |
- return options
|
| |
+ fetch_prefix = 'https://{hostname}/'.format(hostname=hostname)
|
| |
|
| |
- def get_repo_list(url, branch, projectpath):
|
| |
- '''return a list of repo URLs'''
|
| |
+ fetch_next_v0(query, fetch_prefix, list_of_urls)
|
| |
+ else:
|
| |
+ raise NotImplementedError("Unknown API version %s", api)
|
| |
+
|
| |
+ list_of_urls.sort()
|
| |
+ return list_of_urls
|
| |
+
|
| |
+ def fetch_next_v0(page, fetch_prefix, list_of_urls):
|
| |
+ '''
|
| |
+ Recursively fetch the page until we are done
|
| |
+ '''
|
| |
+ logging.debug('Trying to fetch %s', page)
|
| |
try:
|
| |
- req = requests.get(url)
|
| |
+ req = requests.get(page)
|
| |
except requests.exceptions.RequestException as err_msg:
|
| |
- print err_msg
|
| |
- sys.exit(1)
|
| |
+ print(err_msg, file=sys.stderr)
|
| |
+ raise
|
| |
+
|
| |
+ try:
|
| |
+ message = json.loads(req.text)
|
| |
+ except ValueError as err_msg:
|
| |
+ print(page, file=sys.stderr)
|
| |
+ print(req.text, file=sys.stderr)
|
| |
+ print(err_msg, file=sys.stderr)
|
| |
+ raise
|
| |
|
| |
- if req.status_code != 200:
|
| |
- print "Unable to access gitblit api at " + url
|
| |
- sys.exit(1)
|
| |
+ for project in message['projects']:
|
| |
+ list_of_urls.append(fetch_prefix + project['fullname'])
|
| |
|
| |
+ if 'next' in message['pagination']:
|
| |
+ if message['pagination']['next']:
|
| |
+ time.sleep(0.25) # Add a smallish delay to help with load
|
| |
+ fetch_next_v0(message['pagination']['next'], fetch_prefix, list_of_urls)
|
| |
|
| |
- payload = req.text
|
| |
- repos = json.loads(payload)
|
| |
- branchname = 'refs/heads/' + branch
|
| |
+ if __name__ == '__main__':
|
| |
|
| |
- for repo in repos.keys():
|
| |
- if projectpath != 'all':
|
| |
- if repos[repo]['projectPath'] != projectpath:
|
| |
- del repos[repo]
|
| |
- continue
|
| |
- if branchname not in repos[repo]['availableRefs']:
|
| |
- del repos[repo]
|
| |
+ PARSER = setup_args()
|
| |
+ ARGS = PARSER.parse_args()
|
| |
|
| |
- return repos.keys()
|
| |
+ if ARGS.debug:
|
| |
+ logging.basicConfig(level=logging.DEBUG)
|
| |
|
| |
- def main():
|
| |
- '''Broken out so it can be inherited if someone wants'''
|
| |
- options = read_args()
|
| |
- repos = get_repo_list(url=options.url, branch=options.branch, projectpath=options.project)
|
| |
- if repos:
|
| |
- print '\n'.join(repos)
|
| |
+ URLS = run_query(ARGS.hostname, ARGS.apiver, ARGS.namespace, ARGS.show_forks)
|
| |
|
| |
- if __name__ == "__main__":
|
| |
- main()
|
| |
+ for URL in URLS:
|
| |
+ print(URL)
|
| |
jcpunk
commented 5 years ago
This swaps out the gitblit api for pagure