status-im
/
research
mirror of https://github.com/status-im/research.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity
research/price_analysis/spread.py

461 lines
19 KiB
Python
Raw Blame History

import copy, re
from sets import Set
# NOTE THAT ALL FUNCTIONS HERE WORK BY RETURNING A NEW ARRAY, THERE ARE NO IN-PLACE MODIFICATION METHODS
# ALSO KEEP IN MIND THAT COLUMNS AND ROW INDICES START FROM ZERO (ie. column A -> 0, E -> 4, Z -> 25, etc)
# Scans a CSV line, keeping track of separators and quotes
def scanline(ln,sp=','):
arr = []
inq = False
ind = 0
buff = ""
while ind < len(ln):
if ln[ind] == '"':
inq = not inq
ind += 1
elif ln[ind] == '\\':
buff += ln[ind+1]
ind += 2
elif ln[ind] == sp and not inq:
arr.append(buff)
buff = ""
ind += 1
else:
buff += ln[ind]
ind += 1
arr.append(buff)
return arr
# Load a CSV into a 2D array, automatically filling in unevenly wide rows to make the array square
def load(f,sp=','):
array = [scanline(x,sp) for x in open(f,'r').readlines()]
maxlen = 0
for i in range(len(array)):
if len(array[i]) > maxlen: maxlen = len(array[i])
for i in range(len(array)):
if len(array[i]) < maxlen: array[i] += [''] * (maxlen - len(array[i]))
return array
# Apply a Porter stemmer to every cell in a given range of cols in an array (calling stem with just a list and no cols argument stems _every_ cell)
# Example outputs: manage, management, manager, managing -> manag; pony, ponies -> poni; reincarnate, reincarnated, reincarnation -> reincarn
def stem(li,cols=0):
if cols == 0: cols = range(len(li[0]))
import porter
pstemmer = porter.PorterStemmer()
newlist = copy.deepcopy(li)
for i in range(len(li)):
for j in cols:
string = str(li[i][j])
for ch in "'"+'"+[]?!\n': string = string.replace(ch,'')
words = string.split(' ')
newlist[i][j] = ' '.join([pstemmer.stem(x.strip().lower(),0,len(x.strip())-1) for x in words])
return newlist
# Is a string a number?
def isnumber(s):
t = re.findall('^-?[0-9,]*\.[0-9,]*$',s)
return len(t) > 0
# Declutters (removes special characters, numerifies numbers) every cell, rules same as those for stem(li,cols=0)
def declutter(li,cols=0):
if cols == 0: cols = range(len(li[0]))
newlist = copy.deepcopy(li)
for i in range(len(li)):
for j in cols:
string = str(li[i][j])
for ch in "'"+'"+[]?!\n': string = string.replace(ch,'')
words = string.split(' ')
newlist[i][j] = ' '.join([x.strip().lower() for x in words])
if isnumber(newlist[i][j]):
newlist[i][j] = float(newlist[i][j])
return newlist
# Generate a list of individual words occurring in a given column in a given array; useful for generating source lists to do n-grams from
def wordlist(li,col):
wlist = []
for i in range(len(li)):
words = li[i][col].split(' ')
for w in words:
if w not in wlist: wlist.append(w)
return wlist
# Generates a list of phrases (complete cell entries)
def phraselist(li,col):
wlist = []
for i in range(len(li)):
phrase= li[i][col]
if phrase not in wlist: wlist.append(phrase)
return wlist
# Retrieve just a few columns from a given array to make a smaller (narrower) array
def cols(li,cols):
result = []
for i in range(len(li)):
newline = []
for c in cols:
if c >= 0: newline.append(li[i][c])
else: newline.append(1)
result.append(newline)
return result
# Combine two possibly unsorted arrays matching rows by heading in headingcol1 in li1 and headingcol2 in li2
# setting linclusive = True makes sure every row in li1 makes it into the output, same with rinclusive and li2
# Recommended to do some kind of sort after splice is done
def splice(li1,li2,headingcol1,headingcol2,linclusive=False,rinclusive=False):
s1 = sorted(li1,key=lambda x:x[headingcol1],reverse=True)
s2 = sorted(li2,key=lambda x:x[headingcol2],reverse=True)
l1 = len(s1[0])
l2 = len(s2[0])
ind1 = 0
ind2 = 0
output = []
while ind1 < len(s1) and ind2 < len(s2):
if cmp(s2[ind2][headingcol2],s1[ind1][headingcol1]) == 1:
if rinclusive: output.append([s2[ind2][headingcol2]] + [''] * (l1-1) + s2[ind2][:headingcol2] + s2[ind2][headingcol2 + 1:])
ind2 += 1
elif cmp(s2[ind2][headingcol2],s1[ind1][headingcol1]) == -1:
if linclusive: output.append([s1[ind1][headingcol1]] + s1[ind1][:headingcol1] + s1[ind1][headingcol1 + 1:] + [''] * (l2-1))
ind1 += 1
else:
output.append([s1[ind1][headingcol1]] + s1[ind1][:headingcol1] + s1[ind1][headingcol1 + 1:] + s2[ind2][:headingcol2] + s2[ind2][headingcol2 + 1:])
ind1, ind2 = ind1 + 1, ind2 + 1
while ind1 < len(s1) and linclusive:
output.append([s1[ind1][headingcol1]] + s1[ind1][:headingcol1] + s1[ind1][headingcol1 + 1:] + [''] * l2)
ind1 += 1
while ind2 < len(s2) and rinclusive:
output.append([s2[ind2][headingcol2]] + [''] * l1 + s2[ind2][:headingcol2] + s2[ind2][headingcol2 + 1:])
ind2 += 1
return output
# Creates a wordlist sorted according to function f taken of an array with the results in the addcols in order
# eg. sorted_wordlist with addcols = [2,4,6], row is 1 2 4 8 16 32 64, f=lambda x:x[2]+x[1]+1.01*x[0] returns sorting key 84.04
def sorted_wordlist(li,wcol,addcols,f=lambda x:x[1],rev=True):
return [x[0] for x in sorted(onegrams(li,wcol,addcols),key=f,reverse=rev)]
# Utility function, used by twograms, threegrams and fourgrams
def compose(arg):
return ' '.join(sorted(list(Set(arg))))
# Calculate a total sum for every desired column for different exact matches in wcol, column -1 is implied to be 1 for every row
# for example, consider the array
# dog 20 3
# dog house 15 28
# cat 25 31
# cat 10 7
# dog 40 0
# house 10 14
# Doing pivot(li,0,[1,-1]) gives you the list:
# dog 60 2
# dog house 15 1
# cat 35 2
# house 10 1
# wlist allows you to restrict the table to a given wordlist
def pivot(li, wcol, addcols,wlist=0,sortkey=lambda x:1):
if wlist == 0: wlist = phraselist(li,wcol)
result = {}
for i in range(len(wlist)):
result[wlist[i]] = [0] * len(addcols)
for i in range(len(li)):
nums = []
for ac in addcols:
if ac >= 0:
num = str(li[i][ac]).replace(',','').replace(' ','')
if num == '': num = 0
elif num[-1] == '%': num = float(num[:-1] * 0.01)
else: num = float(num)
else: num = 1
nums.append(num)
if li[i][wcol] in result: result[li[i][wcol]] = [pair[0] + pair[1] for pair in zip(result[li[i][wcol]],nums)]
array = []
for word in result.keys():
array.append([word] + result[word])
return sorted(array,key=sortkey,reverse=True)
# Similar to a pivot table but looks at individual keywords. The example list above will return with onegrams(li,0,[1,2]):
# dog 75 3
# cat 35 2
# house 25 2
def onegrams(li, wcol, addcols,wlist=0,sortkey=lambda x: 1):
if wlist == 0: wlist = wordlist(li,wcol)
result = {}
for i in range(len(wlist)):
result[wlist[i]] = [0] * len(addcols)
for i in range(len(li)):
words = [x.strip() for x in li[i][wcol].split(' ')]
nums = []
for ac in addcols:
if ac >= 0:
num = str(li[i][ac]).replace(',','').replace(' ','')
if num == '': num = 0
elif num[-1] == '%': num = float(num[:-1] * 0.01)
else: num = float(num)
else: num = 1
nums.append(num)
for i in range(len(words)):
if words[i] in result: result[words[i]] = [pair[0] + pair[1] for pair in zip(result[words[i]],nums)]
array = []
for word in result.keys():
array.append([word] + result[word])
return sorted(array,key=sortkey,reverse=True)
# Calculate a total sum for every column in addcols and for every word pair in wcol
# words do not need to be beside each other or in any particular order, so "buy a dog house", "good house for dog owners", "dog in my house" all go under "dog house"
def twograms(li,wcol,addcols,wlist=0,sortkey=lambda x:1,allindices=False):
if wlist == 0: wlist = wordlist(li,wcol)
result = {}
if allindices:
for i in range(len(wlist)):
for j in range(len(wlist)):
if i != j: result[compose([wlist[i],wlist[j]])] = [0] * len(addcols)
for i in range(len(li)):
if i % int(len(li)/10) == (int(len(li)/10) - 1): print "Two grams: " + str(i) + " / " + str(len(li))
words = [x.strip() for x in li[i][wcol].split(' ')]
nums = []
for ac in addcols:
if ac >= 0:
num = str(li[i][ac]).replace(',','').replace(' ','')
if num == '': num = 0
elif num[-1] == '%': num = float(num[:-1]) * 0.01
else: num = float(num)
else: num = 1
nums.append(num)
for i in range(len(words)):
if words[i] in wlist:
for j in range(i+1,len(words)):
if words[j] in wlist:
comb = compose([words[i],words[j]])
if comb in result: result[comb] = [pair[0] + pair[1] for pair in zip(result[comb],nums)]
elif allindices == False: result[comb] = nums
array = []
for words in result.keys():
array.append([words] + result[words])
return sorted(array,key=sortkey,reverse=True)
# Calculate a total sum for every column in addcols and for every word triplet in wcol (do not need to be beside each other or in any particular order)
# setting allindices to True slows down the calculation a lot but gives you a CSV with all possible combinations of words, making it convenient for
# working with the same word list on different data
def threegrams(li,wcol,addcols,wlist=0,sortkey=lambda x:1,allindices=False):
if wlist == 0: wlist = wordlist(li,wcol)
result = {}
if allindices:
for i in range(len(wlist)):
for j in range(len(wlist)):
for k in range(len(wlist)):
if i != j and i != k and j != k: result[compose([wlist[i],wlist[j],wlist[k]])] = [0] * len(addcols)
for i in range(len(li)):
if i % int(len(li)/10) == (int(len(li)/10) - 1): print "Three grams: " + str(i) + " / " + str(len(li))
words = [x.strip() for x in li[i][wcol].split(' ')]
nums = []
for ac in addcols:
if ac >= 0:
num = str(li[i][ac]).replace(',','').replace(' ','')
if num == '': num = 0
elif num[-1] == '%': num = float(num[:-1]) * 0.01
else: num = float(num)
else: num = 1
nums.append(num)
for i in range(len(words)):
if words[i] in wlist:
for j in range(i+1,len(words)):
if words[j] in wlist:
for k in range(j+1,len(words)):
if words[k] in wlist:
comb = compose([words[i],words[j],words[k]])
if comb in result:
result[comb] = [pair[0] + pair[1] for pair in zip(result[comb],nums)]
elif allindices == False: result[comb] = nums
array = []
for words in result.keys():
array.append([words] + result[words])
return sorted(array,key=sortkey,reverse=True)
# Calculate a total sum for every column in addcols and for every word quadruplet in wcol
def fourgrams(li,wcol,addcols,wlist=0,sortkey=lambda x:1):
if wlist == 0: wlist = wordlist(li,wcol)
result = {}
for i in range(len(li)):
if i % int(len(li)/10) == (int(len(li)/10) - 1): print "Four grams: " + str(i) + " / " + str(len(li))
words = [x.strip() for x in li[i][wcol].split(' ')]
nums = []
for ac in addcols:
if ac >= 0:
num = str(li[i][ac]).replace(',','').replace(' ','')
if num == '': num = 0
elif num[-1] == '%': num = float(num[:-1]) * 0.01
else: num = float(num)
else: num = 1
nums.append(num)
for i in range(len(words)):
if words[i] in wlist:
for j in range(i+1,len(words)):
if words[j] in wlist:
for k in range(j+1,len(words)):
if words[j] in wlist:
for l in range(k+1,len(words)):
if words[l] in wlist:
comb = compose([words[i],words[j],words[k],words[l]])
if comb in result:
result[comb] = [pair[0] + pair[1] for pair in zip(result[comb],nums)]
else: result[comb] = nums
array = []
for words in result.keys():
array.append([words] + result[words])
return sorted(array,key=sortkey,reverse=True)
# Filters array, returning only the rows where column wcol of that row contains the query keywords (keywords can appear in any order)
# This and the other filters are useful for taking a list of entries and creating a list of only valid entries according to some validity characteristic
# eg:
# dog house, 15
# cat, 18
# dog, 33
# filter(li,0,'dog'):
# dog house, 15
# dog, 33
def filter(li,wcol,query):
result = []
for i in range(len(li)):
words = [x.strip() for x in li[i][wcol].split(' ')]
inlist = True
queryarray = query.split(' ')
if queryarray == ['']: queryarray = []
for w in queryarray:
if w not in words: inlist = False
if queryarray == ['*']: inlist = len(li[i][wcol]) > 0
if inlist: result.append(li[i])
return result
# Filters array, requiring column wcol to exactly match query
def phrasefilter(li,wcol,query):
result = []
for i in range(len(li)):
if li[i][wcol] == query: result.append(li[i])
return result
# Filters array, requiring function func taken of the row to return True (or 1)
def funcfilter(li,func):
result = []
for i in range(len(li)):
if func(li[i]): result.append(li[i])
return result
# Adds up columns in addcols for a query matching keyfilter(li,wcol,query); can also be thought of as doing a single n-keyword match
# eg:
# dog, 25
# cat, 15
# dog, 75
# dog, 10
# horse, 55
# cat, 7
# search(li,0,[1],'dog') gives ['dog',110]
def search(li,wcol,addcols,query):
result = [0] * len(addcols)
for i in range(len(li)):
words = [x.strip() for x in li[i][wcol].split(' ')]
nums = []
for ac in addcols:
if ac >= 0:
num = str(li[i][ac]).replace(',','').replace(' ','')
if num == '': num = 0
elif num[-1] == '%': num = float(num[:-1] * 0.01)
else: num = float(num)
else: num = 1
nums.append(num)
inlist = True
queryarray = query.split(' ')
if queryarray == ['']: queryarray = []
for w in queryarray:
if w not in words: inlist = False
if queryarray == ['*']: inlist = len(li[i][wcol]) > 0
if inlist:
result = [pair[0] + pair[1] for pair in zip(result,nums)]
return [query] + result
# Print a CSV from an array to stdout
def tochars(array,sp=','):
string = ""
for line in array: string += sp.join([str(x) for x in line]) + '\n'
return string[:-1]
# Save an array to CSV
def save(f,array,sp=','):
writeto = open(f,'w')
writeto.write(tochars(array,sp))
writeto.close()
# Compares keywords by two different parameters from two different lists. For example, li1 can be a list of how much money is spent (on addcol1) on a particular combination of keywords (on keycol1) and li2 can be a list of upgraded accounts with the search query they came from on keycol2, and addcol 2 can be left blank to default to -1 (each row is worth one point). Fourth column is statistical significance.
# Remember that you may have to filter the list yourself first
# Arguments:
# grams = 1 for single keywords, 2 for pairs, 3 for triplets and 4 for quadruplets
# li1, li2 = your two lists
# keycol1, keycol2 = where the keywords are located in those two lists
# addcol1, addcol2 = the columns of what you want to add up, eg. cost (set to -1 or leave blank to make it add 1 for each row)
# sortkey = function to sort results by (highest first)
# usestem = stem keywords
# sigtable = add ratio and significance to table
# invertratio = set ratio column to col1/col2 instead of col2/col1
# preformatted = li1 and li2 are already properly formatted
# justpreformat = convert li1 and li2 into twocolumns for comparison but don't go all the way
# wordlimit = limit search to some more common keywords for speedup purposes
# Example: list of customers, some upgraded, with originating keywords, and a list of how much you're paying for each search phrase
#
# customers.csv:
# Name, Keyword, Status
# Bob Jones, spreadsheet csv software, upgraded
# Matt Bones, csv python utils, free
# Army Drones, free spreadsheet, free
# Glenn Mitt, csv software, upgraded
# Pat Submitt, python utils software, upgraded
# Shawn Wit, python spreadsheet program, upgraded
#
# costs.csv:
# csv software, useless, and, irrelevant, data, 5.00, blah, blah
# python spreadsheet, useless, and, irrelevant, data, 2.50, blah, blah
# spreadsheet utils, useless, and, irrelevant, data, 10.00, blah, blah
# csv utils, useless, and, irrelevant, data, 1.50, blah, blah
#
# Steps:
# 1. import spread (if not imported already)
# 2. upgrades = spread.filter(spread.load('customers.csv'),2,'upgraded')
# 3. costs = spread.load('costs.csv')
# 4. res = compare(1,costs,upgrades,0,1,5,invertratio=True)
# 5. spread.save('saved.csv',res)
#
# Res should look like:
#
# Keyword, Column 1, Column 2, Ratio, Significance
# spreadsheet, 12.50, 2, 6.25, -0.389
# utils, 11.50, 1, 11.50, -0.913
# csv, 7.50, 2, 3.75, 0.335
# python, 2.50, 2, 1.25, 2.031
#
# Or, if desired, you can:
# i1,i2 = compare(1,costs,upgrades,0,1,5,justpreformat=True)
# res1 = compare(1,i1,i2,0,1,5,invertratio=True,preformatted=True)
# res2 = compare(2,i1,i2,0,1,5,invertratio=True,preformatted=True)
# res3 = compare(3,i1,i2,0,1,5,invertratio=True,preformatted=True)
# res4 = compare(4,i1,i2,0,1,5,invertratio=True,preformatted=True)
#
# Note that significance is calculated based on col2/col1 regardless of invertratio, since getting 0 upgrades when you should have gotten 2 is not that unlikely, but calculating significance based on col1/col2 would give you infinity as infinity is infinitely far away from 0.5.
def compare(grams,li1,li2,keycol1,keycol2,addcol1=-1,addcol2=-1,sortkey=lambda x:x[1],usestem=True,sigtable=True,invertratio=False,preformatted=False,justpreformat=False,wordlimit=0):
gramfuncs = [0,onegrams,twograms,threegrams,fourgrams]
if preformatted == False:
s1 = declutter(cols(li1,[keycol1,addcol1]),[1])
print "Done decluttering/stemming: 1/4"
s2 = declutter(cols(li2,[keycol2,addcol2]),[1])
print "Done decluttering/stemming: 2/4"
s1 = stem(s1,[0]) if usestem else declutter(s1,[0])
print "Done decluttering/stemming: 3/4"
s2 = stem(s2,[0]) if usestem else declutter(s2,[0])
print "Done decluttering/stemming: 4/4"
else: s1,s2 = li1,li2
print "Printing sample of list 1"
print s1[:10]
print "Printing sample of list 2"
print s2[:10]
if justpreformat: return s1,s2
while type(s1[0][1]) is str: s1.pop(0)
while type(s2[0][1]) is str: s2.pop(0)
print "Cleaned invalid rows"
wl = sorted_wordlist(s1,0,[1])
if wl.count('') > 0: blank = wl.pop(wl.index(''))
print "Base wordlist length: " + str(len(wl)) + " ; Top ten: " + str(wl[:10])
if wordlimit > 0 and wordlimit < len(wl):
print "Shortening to " + str(wordlimit)
wl = wl[:wordlimit]
res1 = gramfuncs[grams](s1,0,[1],wl)
print "Done search: 1/2"
res2 = gramfuncs[grams](s2,0,[1],wl)
print "Done search: 2/2"
comb = sorted(splice(res1,res2,0,0),key=sortkey,reverse=True)
if sigtable:
tot1 = search(s1,0,[1],'')
tot2 = search(s2,0,[1],'')
ev = tot2[1]*1.0/tot1[1]
print "Totals: " + str(tot1[1]) + ", " + str(tot2[1])
for i in range(len(comb)):
comb[i].append(comb[i][2 - invertratio]*1.0/(comb[i][1 + invertratio] + 0.000001))
comb[i].append((comb[i][2] - ev * comb[i][1])*1.0/(ev * comb[i][1] + 0.000001) ** 0.5)
comb = [['Keyword','Column 1','Column 2','Ratio','Significance']] + comb
else: comb = [['Keyword','Column 1','Column 2']] + comb
print "Done"
return comb
Reference in New Issue View Git Blame Copy Permalink