From 7d53729c55049c4cc96bf3931dcacc933f57ee43 Mon Sep 17 00:00:00 2001 From: Ricardo Guilherme Schmidt <3esmit@gmail.com> Date: Fri, 28 Apr 2017 05:37:47 +0000 Subject: [PATCH] script is now object oriented --- scripts/github-oracle/github_oracle.py | 500 ++++++++++++++----------- scripts/test.sh | 10 +- 2 files changed, 293 insertions(+), 217 deletions(-) diff --git a/scripts/github-oracle/github_oracle.py b/scripts/github-oracle/github_oracle.py index 0ea2b0e..f3bba14 100644 --- a/scripts/github-oracle/github_oracle.py +++ b/scripts/github-oracle/github_oracle.py @@ -4,7 +4,7 @@ from collections import defaultdict start = '' def logmsg(msg): - sys.stderr.write("[GitHubAPI] "+msg+" \n") + sys.stderr.write("[GitHubOracle] "+msg+" \n") try: @@ -18,188 +18,253 @@ if argn > 3: logmsg("Started " + os.environ['ARG0'] + "(" + os.environ['ARG1']+")") -def oAuth(client, secret, code): - # POST https://github.com/login/oauth/access_token - # { "Accept": "application/json", "client_id" : client, "client_secret" : secret, "code": code } - # RESPONSE {"access_token":"e72e16c7e42f292c6912e7710c838347ae178b4a", "scope":"repo,gist", "token_type":"bearer"} - sys.exit("501 Not implemented") - return "" + #global -points = defaultdict(int) + claimed = defaultdict(bool) count = 0 repo_link = "" -#load credentials -auth = 0 -if argn > 2: - auth = [x.strip() for x in os.environ['ARG2'].split(',')] - if len(auth) == 3: #is token - logmsg("Using OAuth") - client = auth[0] - secret = auth[1] - code = auth[2] - token = oAuth(client, secret, code) - auth = 2 - elif len(auth) == 2: #is secret - logmsg("Using Secret Mode") - client = auth[0] - secret = auth[1] - auth = 1 - elif len(auth) == 1 and len(auth[0]) > 0: - logmsg("Using Token") - token = auth[0] - auth = 2 - else: - logmsg("Anonymous API") - auth = 0 - args = os.environ['ARG1'] -def requestAPI(api_link, arguments_get=None, arguments_post=None): - if arguments_get is None: - arguments_get = [] - if auth == 1: - arguments_get += [["client_id", client], ["client_secret", secret]] - if(auth == 2): - arguments_post += [["Access-Token",token]] +class GitHubAPI: - if len(arguments_get) > 0: - api_link += "?" - for argument in arguments_get: - api_link += argument[0]+"="+argument[1]+"&" - api_link = api_link[0:-1] - req = urllib2.Request(api_link) - if arguments_post is not None and len(arguments_post) > 0: - for argument in arguments_post: - req.add_header(argument[0], argument[1]) - return urllib2.urlopen(req) + auth = 0 -def getRepositoryURL(repository, name=True): - global repo_link - if name: - repo_link = "https://api.github.com/repos/" - else: - repo_link = "https://api.github.com/repositories/" - return repo_link + repository + def oAuth(self): + # POST https://github.com/login/oauth/access_token + # { "Accept": "application/json", "client_id" : client, "client_secret" : secret, "code": code } + # RESPONSE {"access_token":"e72e16c7e42f292c6912e7710c838347ae178b4a", "scope":"repo,gist", "token_type":"bearer"} + sys.exit("501 Not implemented") + return "" -def repositoryAdd(full_name): - repository = json.load(requestAPI(getRepositoryURL(full_name))) - print "["+json.dumps(repository['id'])+",", - print json.dumps(repository['full_name'])+",", - print json.dumps(repository['watchers_count'])+",", - print json.dumps(repository['stargazers_count'])+"]" - sys.exit() - -def updateCommits(full_name, branch_name, head, tail): - global repo_link - logmsg("Update commits " + full_name) - head_end = head - claim_head = True - repo_link = getRepositoryURL(full_name) - repository = json.load(requestAPI(repo_link)) - if branch_name is None: - branch_name = repository['default_branch'] - logmsg("No branch provided, using default: " + branch_name) - if head is None and tail is not None: # error - logmsg("Invalid call") - sys.exit("400 Error") - elif head is None and tail is None: #(new) = from latest head to reachable tail - branches_link = repo_link + "/branches/" + branch_name - branch = json.load(requestAPI(branches_link)) - head_start = branch['commit']['sha'] - head_end="" - head_out = head_start - elif head is not None and tail is None: #head only (sync) = from latestHead to head - branches_link = repo_link + "/branches/" + branch_name - branch = json.load(requestAPI(branches_link)) - head_start = branch['commit']['sha'] - logmsg (branch_name + " head is "+head_start+".") - head_end = head - head_out = head_start - elif tail is not None and head is not None: #head and tail (continue) = continue from tail - head_start = tail - head_end="" - claim_head=False - head_out = head - - - tail_out = loadPoints(head_start,head_end,claim_head) - - print "["+json.dumps(repository['id'])+",", - print json.dumps(repository['full_name']) + "," + json.dumps(branch_name) + ",", - print json.dumps(head_out) + "," + json.dumps(tail_out) + ",", - print str(len(points)) + ",", - print json.dumps(points.items()), - print "]" - -def __parseLinkHeader(headers): - links = {} - if "Link" in headers: - linkHeaders = headers["Link"].split(", ") - for linkHeader in linkHeaders: - (url, rel) = linkHeader.split("; ") - url = url[1:-1] - rel = rel[5:-1] - links[rel] = url - return links - -def loadPoints(head, old_head="", claim_head=True): - global repo_link - global count - global claimed - global points - logmsg("Loading from "+head+("" if claim_head else " parent") + (" up to "+old_head if len(old_head) > 0 else "") + ".") - page = '1' - while True: - response = requestAPI(repo_link + "/commits", [['per_page', '100'], ['sha', head], ['page', page]]) - links = __parseLinkHeader(response.headers) - commits = json.load(response) - logmsg(" page "+page+" contains " + str(len(commits)) +" commits.") - for commit in commits: - if(int(response.headers.get("X-RateLimit-Remaining")) < 1): - logmsg("X-RateLimit reached. Try again in "+response.headers.get("X-RateLimit-Reset")+".") - return tail - if commit['sha'] == old_head: - logmsg(commit['sha']+": ") - return commit['sha'] + def __init__(self): + + argn = int(os.environ['ARGN']) + if argn > 2: + autharg = [x.strip() for x in os.environ['ARG2'].split(',')] + if len(autharg) == 3: #is token + logmsg("Using OAuth") + self.client = autharg[0] + self.secret = autharg[1] + self.code = autharg[2] + self.token = self.oAuth() + self.auth = 2 + elif len(autharg) == 2: #is secret + logmsg("Using Secret Mode") + self.client = autharg[0] + self.secret = autharg[1] + self.auth = 1 + elif len(autharg) == 1 and len(autharg[0]) > 0: + logmsg("Using Token") + token = autharg[0] + self.auth = 2 else: - count += 1 - if len(commit['parents']) < 2 and commit['author'] is not None and (claim_head or commit['sha'] != head): - if len(points) > 10 and points[author] == 0: - logmsg("reached limit of 10 authors.") - break - commit = json.load(requestAPI(commit['url'])) - author = commit['author']['login'] - points[author] += int(commit['stats']['additions']) - if(len(commit['parents']) == 0): - parent = "" + logmsg("Anonymous API") + self.auth = 0 + api_link = "https://api.github.com/rate_limit" + if self.auth == 1: + req = urllib2.Request(api_link+"?client_id="+self.client+"&client_secret="+self.secret) + elif(self.auth == 2): + req = urllib2.Request(api_link) + req.add_header("Access-Token", self.token) + else: + req = urllib2.Request(api_link) + + res = json.load(urllib2.urlopen(req)) + self.api = defaultdict(int) + self.api['rate_limit'] = int(res['rate']['limit']) + self.api['rate_remaining'] = int(res['rate']['remaining']) + self.api['rate_reset'] = int(res['rate']['reset']) + + def checkLimit(self, more_than = 0): + if(self.api['rate_remaining'] > more_than): + return True + else: + logmsg("X-RateLimit reached. Try again in "+self.api['rate_reset']+".") + return False + + def request(self, api_link, arguments_get=None, arguments_post=None): + if arguments_get is None: + arguments_get = [] + if self.auth == 1: + arguments_get += [["client_id", self.client], ["client_secret", self.secret]] + if(self.auth == 2): + arguments_post += [["Access-Token",self.token]] + + if len(arguments_get) > 0: + api_link += "?" + for argument in arguments_get: + api_link += argument[0]+"="+argument[1]+"&" + api_link = api_link[0:-1] + req = urllib2.Request(api_link) + if arguments_post is not None and len(arguments_post) > 0: + for argument in arguments_post: + req.add_header(argument[0], argument[1]) + response = urllib2.urlopen(req) + self.api['rate_limit'] = int(response.headers.get("X-RateLimit-Limit")) + self.api['rate_remaining'] = int(response.headers.get("X-RateLimit-Remaining")) + self.api['rate_reset'] = int(response.headers.get("X-RateLimit-Reset")) + return response + +class GitRepository: + branch = None + head = "" + tail = "" + repo_link="" + points = defaultdict(int) + count = 0 + + def __init__(self, api, repository, name=True): + self.api = api + if name: + self.repo_link = "https://api.github.com/repos/" + else: + self.repo_link = "https://api.github.com/repositories/" + self.repo_link += repository + self.data = json.load(api.request(self.repo_link)) + self.branch_name = self.data['default_branch'] + + def setBranch(self, branch_name): + branch = None + self.branch_name = branch_name + + def setHead(self, head): + self.head = head + + def setTail(self, tail): + self.tail = tail + + def getBranch(self): + logmsg("Loaded branch " + self.branch_name) + if(self.branch is None): + branches_link = self.repo_link + "/branches/" + self.branch_name + self.branch = json.load(api.request(branches_link)) + return self.branch + + def __parseLinkHeader(self, headers): + links = {} + if "Link" in headers: + linkHeaders = headers["Link"].split(", ") + for linkHeader in linkHeaders: + (url, rel) = linkHeader.split("; ") + url = url[1:-1] + rel = rel[5:-1] + links[rel] = url + return links + + def updateCommits(self): + branch_head = self.getBranch()['commit']['sha'] + logmsg("Loading from "+branch_head+ (" up to "+self.head if len(self.head) > 0 else "") + ".") + page = '1' + while self.api.checkLimit(): + response = self.api.request(self.repo_link + "/commits", [['per_page', '100'], ['sha', branch_head], ['page', page]]) + commits = json.load(response) + logmsg("page "+page+" contains " + str(len(commits)) +" commits.") + for commit in commits: + if commit['sha'] != self.head: + if self.api.checkLimit() and not (len(self.points) > 10 and self.points[author] == 0): + self.tail = self.__claimCommit(commit)['sha'] else: - parent = "<"+commit['parents'][0]['sha']+">" - logmsg(commit['sha']+": "+ author + " +" + str(commit['stats']['additions']) + " -"+ str(commit['stats']['deletions']) + " |= " + str(commit['stats']['total']) + " " + parent) + self.head = branch_head + return self.tail else: - if(len(commit['parents']) >= 2): - parents = "" - for parent in commit['parents']: - parents += parent['sha']+", " - logmsg(commit['sha'] +": ") - elif(commit['author'] is None): - logmsg(commit['sha'] +": ") - else: - logmsg(commit['sha'] +": ") - tail = commit['sha'] - try: - page = links['next'].split('&page=')[1].split('&')[0] - except KeyError: - logmsg("Reached end of pagination.") - break - - return tail - + logmsg(commit['sha']+": ") + self.tail = commit['sha'] + self.head = branch_head + return self.tail + try: + links = self.__parseLinkHeader(response.headers) + page = links['next'].split('&page=')[1].split('&')[0] + except KeyError: + logmsg("Reached end of pagination.") + break + self.head = branch_head + return self.tail + + def continueLoading(self, old_tail, limit=""): + logmsg("Continuing from "+self.head+ (" up to "+limit if len(limit) > 0 else "") +".") + page = '1' + claim = False + while self.api.checkLimit(): + response = self.api.request(self.repo_link + "/commits", [['per_page', '100'], ['sha', self.head], ['page', page]]) + commits = json.load(response) + logmsg("page "+page+" contains " + str(len(commits)) +" commits.") + for commit in commits: + if commit['sha'] != limit: + if(commit['sha'] == old_tail): + logmsg(commit['sha']+": ") + claim = True; + elif(claim): + if self.api.checkLimit() and not (len(self.points) > 10 and self.points[author] == 0): + self.tail = self.__claimCommit(commit)['sha'] + else: + return self.tail + else: + logmsg(commit['sha']+": ") + self.tail = limit + return self.tail + try: + links = self.__parseLinkHeader(response.headers) + page = links['next'].split('&page=')[1].split('&')[0] + except KeyError: + logmsg("Reached end of pagination.") + break + return self.tail + + def __claimCommit(self, commit): + self.count += 1 + if len(commit['parents']) < 2 and commit['author'] is not None: + commit = json.load(self.api.request(commit['url'])) + author = commit['author']['login'] + self.points[author] += int(commit['stats']['additions']) + if(len(commit['parents']) == 0): + parent = "" + else: + parent = "<"+commit['parents'][0]['sha']+">" + logmsg(commit['sha']+": "+ author + " +" + str(commit['stats']['additions']) + " -"+ str(commit['stats']['deletions']) + " |= " + str(commit['stats']['total']) + " " + parent) + else: + if(len(commit['parents']) >= 2): + parents = "" + for parent in commit['parents']: + parents += parent['sha']+", " + logmsg(commit['sha'] +": ") + elif(commit['author'] is None): + logmsg(commit['sha'] +": ") + else: + logmsg(commit['sha'] +": ") + return commit + + def issuePoints(self, issueid): + link_issue = self.repo_link + "/issues/" + issueid + issue = json.load(api.request(link_issue)) + + link_issue = self.repo_link + "/issues/" + issueid + "/timeline" + api.request(link_issue, None, ["Accept","application/vnd.github.mockingbird-preview"]) + + issue_timeline = json.load(urllib2.urlopen(req)) + + for elem in issue_timeline: + if(elem["event"] == "cross-referenced"): + if(elem["source"]["type"] == "issue"): + pr = str(elem["source"]["issue"]["number"]) + #print pr + link_pull = repo_link + "/pulls/" + pr + pull = json.load(requestAPI(link_pull)) + if(pull['merged_at']): + link_pulls_commits = repo_link + "/pulls/" + pr + "/commits" + commits = json.load(api.request(link_pulls_commits)) + for commit in commits: + if(commit['url']): + _commit = json.load(requestAPI(commit['url'])) + author = _commit['author']['login'] + self.points[author] += int(json.dumps(_commit['stats']['total'])) + return issue + def userRegister(github_user,gistid): logmsg("Reading Gist "+gistid+" from "+github_user+".") - value = json.load(requestAPI("https://api.github.com/gists/" + gistid)) + value = json.load(api.request("https://api.github.com/gists/" + gistid)) login = value['owner']['login'] logmsg("Gist owner is "+login+".") if login == github_user: @@ -211,54 +276,61 @@ def userRegister(github_user,gistid): logmsg("Wrong condition: "+github_user+" != "+login); sys.exit("403 Forbidden") -def updateIssue(repository,issueid): - global repo_link - link_issue = repo_link + "/issues/" + issueid - issue = json.load(requestAPI(link_issue)) - print issue['state'] + ", " + datetime.datetime.strptime( json.dumps(issue['closed_at'])[1:-1], "%Y-%m-%dT%H:%M:%SZ" ).strftime('%s') + ", ", - - link_issue = repo_link + "/issues/" + issueid + "/timeline" - requestAPI(link_issue, None, ["Accept","application/vnd.github.mockingbird-preview"]) - - issue = json.load(urllib2.urlopen(req)) - - for elem in issue: - if(elem["event"] == "cross-referenced"): - if(elem["source"]["type"] == "issue"): - pr = str(elem["source"]["issue"]["number"]) - #print pr - link_pull = repo_link + "/pulls/" + pr - pull = json.load(requestAPI(link_pull)) - if(pull['merged_at']): - link_pulls_commits = repo_link + "/pulls/" + pr + "/commits" - commits = json.load(requestAPI(link_pulls_commits)) - for commit in commits: - if(commit['url']): - _commit = json.load(requestAPI(commit['url'])) - author = _commit['author']['login'] - points[author] += int(json.dumps(_commit['stats']['total'])) - print points.items() - script = os.environ['ARG0'] args = [x.strip() for x in os.environ['ARG1'].split(',')] +api = GitHubAPI() -if script == 'repo-update': - full_name = args[0] - branch_name = None - head = None - tail = None - try: - branch_name = args[1] - head = args[2] - tail = args[3] - except IndexError: - print '', - updateCommits(full_name, branch_name, head, tail) -elif script == 'repo-add': - repositoryAdd(args[0]) -elif script == "user-add": - userRegister(args[0],args[1]) -elif script == "issue-update": - issueUpdate(args[0],args[1]) +if api.checkLimit(5): + if script == 'update-new': + repository = GitRepository(api, args[0]) + try: + repository.setBranch(args[1]) + repository.setHead(args[2]) + except IndexError: + a = None + + repository.updateCommits() + + print "["+json.dumps(repository.data['id'])+",", + print json.dumps(repository.data['full_name']) + "," + json.dumps(repository.branch['name']) + ",", + print json.dumps(repository.head) + "," + json.dumps(repository.tail) + ",", + print str(len(repository.points)) + ",", + print json.dumps(repository.points.items()), + print "]" + + elif script == 'update-old': + repository = GitRepository(api, args[0]) + repository.setBranch(args[1]) + repository.setHead(args[2]) + try: + repository.continueLoading(args[3],args[4]) + except IndexError: + repository.continueLoading(args[3]) + print "["+json.dumps(repository.data['id'])+",", + print json.dumps(repository.data['full_name']) + "," + json.dumps(repository.getBranch()['name']) + ",", + print json.dumps(repository.head) + "," + json.dumps(repository.tail) + ",", + print str(len(repository.points)) + ",", + print json.dumps(repository.points.items()), + print "]" + + elif script == 'repository-add': + repository = GitRepository(api, args[0]) + print "["+json.dumps(repository.data['id'])+",", + print json.dumps(repository.data['full_name'])+",", + print json.dumps(repository.data['watchers_count'])+",", + print json.dumps(repository.data['stargazers_count'])+"]" + elif script == "user-add": + userRegister(args[0],args[1]) + elif script == "issue-update": + repository = GitRepository(api, args[0]) + issue = repository.issuePoints(args[1]) + print "["+json.dumps(repository.data['id'])+",", + print json.dumps(repository.data['full_name']) + ",", + print issue['state'] + ", " + datetime.datetime.strptime( json.dumps(issue['closed_at'])[1:-1], "%Y-%m-%dT%H:%M:%SZ" ).strftime('%s') + ", ", + print str(len(repository.points)) + ",", + print json.dumps(repository.points.items()), + print "]" + else: + sys.exit("501 Not implemented") else: - sys.exit("501 Not implemented") + sys.exit("503 Service Unavailable") \ No newline at end of file diff --git a/scripts/test.sh b/scripts/test.sh index cbb68b4..67d759b 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -13,8 +13,12 @@ export ARG2 python github-oracle/github_oracle.py -ARG0="repo-update" -ARG1="ethereans/TheEtherian,master,b2ca0a249c6614d2be14d7d77694f52cd83acfd9,3dcfc9e50a10828017b780cd6793c29ad6f77bdb" +ARG0="update-new" +ARG1="ethereans/TheEtherian,master,f6a4474822a1193f81b45a571e521218a48ca8e3" -python github-oracle/github_oracle.py +#python github-oracle/github_oracle.py +ARG0="update-old" +ARG1="ethereans/TheEtherian,master,b2ca0a249c6614d2be14d7d77694f52cd83acfd9,7d69d0e065d983cacb39f88ea1500c9224fe622f,e2c8c7a091ce3893635ce47169c972c6d5434ffa" + +python github-oracle/github_oracle.py \ No newline at end of file