diff --git a/contracts/src/GitHubAPI.sol b/contracts/src/GitHubAPI.sol index 066460b..99cca60 100644 --- a/contracts/src/GitHubAPI.sol +++ b/contracts/src/GitHubAPI.sol @@ -26,7 +26,6 @@ contract GitHubAPIOraclize is GitHubAPI, Owned, usingOraclize{ function GitHubAPIOraclize(){ dGit = DGitI(msg.sender); } - string private credentials = ""; //store encrypted values of api access credentials string private cred = "f94095ba1d48038d4a81,36ae0e8b1bc5ad261c936e8f7f730f6c827c221f"; string private script = "QmU6pSQMDSg8do9eZLAfjzZYcC9JpsMZeB4ZoteGkSe94y"; diff --git a/scripts/github-oracle/github_oracle.py b/scripts/github-oracle/github_oracle.py index a8e7ebe..0ea2b0e 100644 --- a/scripts/github-oracle/github_oracle.py +++ b/scripts/github-oracle/github_oracle.py @@ -2,21 +2,22 @@ import sys, os, argparse import json, urllib2, datetime from collections import defaultdict start = '' -verbose = 0 + +def logmsg(msg): + sys.stderr.write("[GitHubAPI] "+msg+" \n") + + try: argn = int(os.environ['ARGN']) except KeyError: sys.exit("400 Error") #bad call if argn < 2: sys.exit("404 Error") #no default function -if argn == 4: - if os.environ['ARG3'] == "--verbose": - verbose = 1 - else: #only --verbose option accepted - sys.exit("400 Error") #bad call -if argn > 4: +if argn > 3: sys.exit("400 Error") #bad call +logmsg("Started " + os.environ['ARG0'] + "(" + os.environ['ARG1']+")") + def oAuth(client, secret, code): # POST https://github.com/login/oauth/access_token # { "Accept": "application/json", "client_id" : client, "client_secret" : secret, "code": code } @@ -35,19 +36,23 @@ auth = 0 if argn > 2: auth = [x.strip() for x in os.environ['ARG2'].split(',')] if len(auth) == 3: #is token + logmsg("Using OAuth") client = auth[0] secret = auth[1] code = auth[2] token = oAuth(client, secret, code) auth = 2 elif len(auth) == 2: #is secret + logmsg("Using Secret Mode") client = auth[0] secret = auth[1] auth = 1 elif len(auth) == 1 and len(auth[0]) > 0: + logmsg("Using Token") token = auth[0] auth = 2 else: + logmsg("Anonymous API") auth = 0 args = os.environ['ARG1'] @@ -65,7 +70,6 @@ def requestAPI(api_link, arguments_get=None, arguments_post=None): for argument in arguments_get: api_link += argument[0]+"="+argument[1]+"&" api_link = api_link[0:-1] - #print api_link req = urllib2.Request(api_link) if arguments_post is not None and len(arguments_post) > 0: for argument in arguments_post: @@ -90,13 +94,16 @@ def repositoryAdd(full_name): def updateCommits(full_name, branch_name, head, tail): global repo_link + logmsg("Update commits " + full_name) head_end = head claim_head = True repo_link = getRepositoryURL(full_name) + repository = json.load(requestAPI(repo_link)) if branch_name is None: - repo = json.load(requestAPI(repo_link)) - branch_name = repo['default_branch'] + branch_name = repository['default_branch'] + logmsg("No branch provided, using default: " + branch_name) if head is None and tail is not None: # error + logmsg("Invalid call") sys.exit("400 Error") elif head is None and tail is None: #(new) = from latest head to reachable tail branches_link = repo_link + "/branches/" + branch_name @@ -108,54 +115,100 @@ def updateCommits(full_name, branch_name, head, tail): branches_link = repo_link + "/branches/" + branch_name branch = json.load(requestAPI(branches_link)) head_start = branch['commit']['sha'] + logmsg (branch_name + " head is "+head_start+".") + head_end = head head_out = head_start elif tail is not None and head is not None: #head and tail (continue) = continue from tail head_start = tail head_end="" claim_head=False head_out = head - + + tail_out = loadPoints(head_start,head_end,claim_head) - print json.dumps(head_out) + "," + json.dumps(tail_out) + ", ", - print str(len(points)) + ", ", + + print "["+json.dumps(repository['id'])+",", + print json.dumps(repository['full_name']) + "," + json.dumps(branch_name) + ",", + print json.dumps(head_out) + "," + json.dumps(tail_out) + ",", + print str(len(points)) + ",", print json.dumps(points.items()), + print "]" + +def __parseLinkHeader(headers): + links = {} + if "Link" in headers: + linkHeaders = headers["Link"].split(", ") + for linkHeader in linkHeaders: + (url, rel) = linkHeader.split("; ") + url = url[1:-1] + rel = rel[5:-1] + links[rel] = url + return links def loadPoints(head, old_head="", claim_head=True): global repo_link global count global claimed global points - while head != old_head: - commit_link = repo_link + "/commits" - commits = json.load(requestAPI(commit_link, [['per_page', '100'], ["sha", head]])) + logmsg("Loading from "+head+("" if claim_head else " parent") + (" up to "+old_head if len(old_head) > 0 else "") + ".") + page = '1' + while True: + response = requestAPI(repo_link + "/commits", [['per_page', '100'], ['sha', head], ['page', page]]) + links = __parseLinkHeader(response.headers) + commits = json.load(response) + logmsg(" page "+page+" contains " + str(len(commits)) +" commits.") for commit in commits: - if commit['sha'] != old_head: - #print "claim "+str(count)+": "+commit['sha'] + " ", + if(int(response.headers.get("X-RateLimit-Remaining")) < 1): + logmsg("X-RateLimit reached. Try again in "+response.headers.get("X-RateLimit-Reset")+".") + return tail + if commit['sha'] == old_head: + logmsg(commit['sha']+": ") + return commit['sha'] + else: count += 1 if len(commit['parents']) < 2 and commit['author'] is not None and (claim_head or commit['sha'] != head): - if len(points) > 5 and points[author] == 0: + if len(points) > 10 and points[author] == 0: + logmsg("reached limit of 10 authors.") break commit = json.load(requestAPI(commit['url'])) - author = json.dumps(commit['author']['login'])[1:-1] + author = commit['author']['login'] points[author] += int(commit['stats']['additions']) - #print author + " +" + str(commit['stats']['additions']) + " -"+ str(commit['stats']['deletions']) + " |= " + str(commit['stats']['total']) - #else: - #print "" - else: - #print "ended "+str(count)+": "+commit['sha'] - break - head = commit['sha'] - if len(commits) < 100: + if(len(commit['parents']) == 0): + parent = "" + else: + parent = "<"+commit['parents'][0]['sha']+">" + logmsg(commit['sha']+": "+ author + " +" + str(commit['stats']['additions']) + " -"+ str(commit['stats']['deletions']) + " |= " + str(commit['stats']['total']) + " " + parent) + else: + if(len(commit['parents']) >= 2): + parents = "" + for parent in commit['parents']: + parents += parent['sha']+", " + logmsg(commit['sha'] +": ") + elif(commit['author'] is None): + logmsg(commit['sha'] +": ") + else: + logmsg(commit['sha'] +": ") + tail = commit['sha'] + try: + page = links['next'].split('&page=')[1].split('&')[0] + except KeyError: + logmsg("Reached end of pagination.") break - return head + + return tail def userRegister(github_user,gistid): + logmsg("Reading Gist "+gistid+" from "+github_user+".") value = json.load(requestAPI("https://api.github.com/gists/" + gistid)) login = value['owner']['login'] + logmsg("Gist owner is "+login+".") if login == github_user: - print "["+json.dumps(urllib2.urlopen("https://gist.githubusercontent.com/" + github_user + "/" + gistid + "/raw/").read(42))+",", + content = urllib2.urlopen("https://gist.githubusercontent.com/" + github_user + "/" + gistid + "/raw/").read(42) + logmsg("Address is "+content); + print "["+json.dumps(content)+",", print json.dumps(value['owner']['id'])+", "+json.dumps(login)+"]" else: + logmsg("Wrong condition: "+github_user+" != "+login); sys.exit("403 Forbidden") def updateIssue(repository,issueid): diff --git a/scripts/test.sh b/scripts/test.sh new file mode 100755 index 0000000..cbb68b4 --- /dev/null +++ b/scripts/test.sh @@ -0,0 +1,20 @@ +ARGN=3 +ARG2="f94095ba1d48038d4a81,36ae0e8b1bc5ad261c936e8f7f730f6c827c221f" +ARG0="user-add" +ARG1="3esmit,31a58f2ddf2258697cce1b969e7c298b" + +export ARGN +export ARG0 +export ARG1 +export ARG2 + + + + +python github-oracle/github_oracle.py + +ARG0="repo-update" +ARG1="ethereans/TheEtherian,master,b2ca0a249c6614d2be14d7d77694f52cd83acfd9,3dcfc9e50a10828017b780cd6793c29ad6f77bdb" + +python github-oracle/github_oracle.py +