import copy, re
from sets import Set
# NOTE THAT ALL FUNCTIONS HERE WORK BY RETURNING A NEW ARRAY, THERE ARE NO IN-PLACE MODIFICATION METHODS
# ALSO KEEP IN MIND THAT COLUMNS AND ROW INDICES START FROM ZERO (ie. column A -> 0, E -> 4, Z -> 25, etc)
# Scans a CSV line, keeping track of separators and quotes
def scanline(ln,sp=','):
  arr = []
  inq = False
  ind = 0
  buff = ""
  while ind < len(ln):
    if ln[ind] == '"':
       inq = not inq
       ind += 1
    elif ln[ind] == '\\':
       buff += ln[ind+1]
       ind += 2
    elif ln[ind] == sp and not inq:
       arr.append(buff)
       buff = ""
       ind += 1
    else:
       buff += ln[ind]   
       ind += 1
  arr.append(buff)
  return arr
# Load a CSV into a 2D array, automatically filling in unevenly wide rows to make the array square
def load(f,sp=','):
  array = [scanline(x,sp) for x in open(f,'r').readlines()]
  maxlen = 0
  for i in range(len(array)):
    if len(array[i]) > maxlen: maxlen = len(array[i])
  for i in range(len(array)):
    if len(array[i]) < maxlen: array[i] += [''] * (maxlen - len(array[i]))
  return array
# Apply a Porter stemmer to every cell in a given range of cols in an array (calling stem with just a list and no cols argument stems _every_ cell)
# Example outputs: manage, management, manager, managing -> manag; pony, ponies -> poni; reincarnate, reincarnated, reincarnation -> reincarn
def stem(li,cols=0):
  if cols == 0: cols = range(len(li[0]))
  import porter
  pstemmer = porter.PorterStemmer()
  newlist = copy.deepcopy(li)
  for i in range(len(li)):
    for j in cols:
      string = str(li[i][j])
      for ch in "'"+'"+[]?!\n': string = string.replace(ch,'')
      words = string.split(' ')
      newlist[i][j] = ' '.join([pstemmer.stem(x.strip().lower(),0,len(x.strip())-1) for x in words])
  return newlist
# Is a string a number?
def isnumber(s):
    t = re.findall('^-?[0-9,]*\.[0-9,]*$',s)
    return len(t) > 0
            
# Declutters (removes special characters, numerifies numbers) every cell, rules same as those for stem(li,cols=0)
def declutter(li,cols=0):
  if cols == 0: cols = range(len(li[0]))
  newlist = copy.deepcopy(li)
  for i in range(len(li)):
    for j in cols:
      string = str(li[i][j])
      for ch in "'"+'"+[]?!\n': string = string.replace(ch,'')
      words = string.split(' ')
      newlist[i][j] = ' '.join([x.strip().lower() for x in words])
      if isnumber(newlist[i][j]):
        newlist[i][j] = float(newlist[i][j])
  return newlist
# Generate a list of individual words occurring in a given column in a given array; useful for generating source lists to do n-grams from
def wordlist(li,col):
  wlist = []
  for i in range(len(li)):
      words = li[i][col].split(' ')
      for w in words:
        if w not in wlist: wlist.append(w)
  return wlist
# Generates a list of phrases (complete cell entries)
def phraselist(li,col):
  wlist = []
  for i in range(len(li)):
     phrase= li[i][col]
     if phrase not in wlist: wlist.append(phrase)
  return wlist
# Retrieve just a few columns from a given array to make a smaller (narrower) array
def cols(li,cols):
  result = []
  for i in range(len(li)):
    newline = []
    for c in cols: 
      if c >= 0: newline.append(li[i][c])
      else: newline.append(1)
    result.append(newline)
  return result
# Combine two possibly unsorted arrays matching rows by heading in headingcol1 in li1 and headingcol2 in li2
# setting linclusive = True makes sure every row in li1 makes it into the output, same with rinclusive and li2
# Recommended to do some kind of sort after splice is done
def splice(li1,li2,headingcol1,headingcol2,linclusive=False,rinclusive=False):
  s1 = sorted(li1,key=lambda x:x[headingcol1],reverse=True)
  s2 = sorted(li2,key=lambda x:x[headingcol2],reverse=True)
  l1 = len(s1[0])
  l2 = len(s2[0])
  ind1 = 0
  ind2 = 0
  output = []
  while ind1 < len(s1) and ind2 < len(s2):
    if cmp(s2[ind2][headingcol2],s1[ind1][headingcol1]) == 1:
      if rinclusive: output.append([s2[ind2][headingcol2]] + [''] * (l1-1) + s2[ind2][:headingcol2] + s2[ind2][headingcol2 + 1:])
      ind2 += 1
    elif cmp(s2[ind2][headingcol2],s1[ind1][headingcol1]) == -1:
      if linclusive: output.append([s1[ind1][headingcol1]] + s1[ind1][:headingcol1] + s1[ind1][headingcol1 + 1:] + [''] * (l2-1))
      ind1 += 1
    else:
      output.append([s1[ind1][headingcol1]] + s1[ind1][:headingcol1] + s1[ind1][headingcol1 + 1:] + s2[ind2][:headingcol2] + s2[ind2][headingcol2 + 1:])
      ind1, ind2 = ind1 + 1, ind2 + 1
  while ind1 < len(s1) and linclusive:
    output.append([s1[ind1][headingcol1]] + s1[ind1][:headingcol1] + s1[ind1][headingcol1 + 1:] + [''] * l2)
    ind1 += 1
  while ind2 < len(s2) and rinclusive:
    output.append([s2[ind2][headingcol2]] + [''] * l1 + s2[ind2][:headingcol2] + s2[ind2][headingcol2 + 1:])
    ind2 += 1
  return output
# Creates a wordlist sorted according to function f taken of an array with the results in the addcols in order
# eg. sorted_wordlist with addcols = [2,4,6], row is 1 2 4 8 16 32 64, f=lambda x:x[2]+x[1]+1.01*x[0] returns sorting key 84.04
def sorted_wordlist(li,wcol,addcols,f=lambda x:x[1],rev=True):
  return [x[0] for x in sorted(onegrams(li,wcol,addcols),key=f,reverse=rev)]
# Utility function, used by twograms, threegrams and fourgrams
def compose(arg):
  return ' '.join(sorted(list(Set(arg))))
# Calculate a total sum for every desired column for different exact matches in wcol, column -1 is implied to be 1 for every row
# for example, consider the array
# dog        20   3
# dog house  15   28
# cat        25   31
# cat        10   7
# dog        40   0
# house      10   14
# Doing pivot(li,0,[1,-1]) gives you the list:
# dog        60   2
# dog house  15   1
# cat        35   2
# house      10   1
# wlist allows you to restrict the table to a given wordlist
def pivot(li, wcol, addcols,wlist=0,sortkey=lambda x:1):
  if wlist == 0: wlist = phraselist(li,wcol)
  result = {}
  for i in range(len(wlist)):
    result[wlist[i]] = [0] * len(addcols)
  for i in range(len(li)):
    nums = []
    for ac in addcols:
      if ac >= 0:
        num = str(li[i][ac]).replace(',','').replace(' ','')
        if num == '': num = 0
        elif num[-1] == '%': num = float(num[:-1] * 0.01)
        else: num = float(num)
      else: num = 1
      nums.append(num)
    if li[i][wcol] in result: result[li[i][wcol]] = [pair[0] + pair[1] for pair in zip(result[li[i][wcol]],nums)]
  array = []
  for word in result.keys():
    array.append([word] + result[word])
  return sorted(array,key=sortkey,reverse=True)
# Similar to a pivot table but looks at individual keywords. The example list above will return with onegrams(li,0,[1,2]):
# dog        75   3
# cat        35   2
# house      25   2
def onegrams(li, wcol, addcols,wlist=0,sortkey=lambda x: 1):
  if wlist == 0: wlist = wordlist(li,wcol) 
  result = {}
  for i in range(len(wlist)):
    result[wlist[i]] = [0] * len(addcols)
  for i in range(len(li)):
    words = [x.strip() for x in li[i][wcol].split(' ')]
    nums = []
    for ac in addcols:
      if ac >= 0: 
        num = str(li[i][ac]).replace(',','').replace(' ','')
        if num == '': num = 0
        elif num[-1] == '%': num = float(num[:-1] * 0.01)
        else: num = float(num)
      else: num = 1
      nums.append(num)
    for i in range(len(words)):
      if words[i] in result: result[words[i]] = [pair[0] + pair[1] for pair in zip(result[words[i]],nums)]
  array = []
  for word in result.keys():
    array.append([word] + result[word])
  return sorted(array,key=sortkey,reverse=True)
# Calculate a total sum for every column in addcols and for every word pair in wcol
# words do not need to be beside each other or in any particular order, so "buy a dog house", "good house for dog owners", "dog in my house" all go under "dog house"
def twograms(li,wcol,addcols,wlist=0,sortkey=lambda x:1,allindices=False):
  if wlist == 0: wlist = wordlist(li,wcol) 
  result = {}
  if allindices:
    for i in range(len(wlist)):
      for j in range(len(wlist)):
        if i != j: result[compose([wlist[i],wlist[j]])] = [0] * len(addcols)
  for i in range(len(li)):
    if i % int(len(li)/10) == (int(len(li)/10) - 1): print "Two grams: " + str(i) + " / " + str(len(li))
    words = [x.strip() for x in li[i][wcol].split(' ')]
    nums = []
    for ac in addcols:
      if ac >= 0: 
        num = str(li[i][ac]).replace(',','').replace(' ','')
        if num == '': num = 0
        elif num[-1] == '%': num = float(num[:-1]) * 0.01
        else: num = float(num)
      else: num = 1
      nums.append(num)
    for i in range(len(words)):
      if words[i] in wlist:
        for j in range(i+1,len(words)):
          if words[j] in wlist:
            comb = compose([words[i],words[j]])
            if comb in result: result[comb] = [pair[0] + pair[1] for pair in zip(result[comb],nums)]
            elif allindices == False: result[comb] = nums
  array = []
  for words in result.keys():
    array.append([words] + result[words])
  return sorted(array,key=sortkey,reverse=True)
# Calculate a total sum for every column in addcols and for every word triplet in wcol (do not need to be beside each other or in any particular order)
# setting allindices to True slows down the calculation a lot but gives you a CSV with all possible combinations of words, making it convenient for
# working with the same word list on different data
def threegrams(li,wcol,addcols,wlist=0,sortkey=lambda x:1,allindices=False):
  if wlist == 0: wlist = wordlist(li,wcol) 
  result = {}
  if allindices:
    for i in range(len(wlist)):
        for j in range(len(wlist)):
          for k in range(len(wlist)):
              if i != j and i != k and j != k: result[compose([wlist[i],wlist[j],wlist[k]])] = [0] * len(addcols)
  for i in range(len(li)):
    if i % int(len(li)/10) == (int(len(li)/10) - 1): print "Three grams: " + str(i) + " / " + str(len(li))
    words = [x.strip() for x in li[i][wcol].split(' ')]
    nums = []
    for ac in addcols:
      if ac >= 0:
        num = str(li[i][ac]).replace(',','').replace(' ','')
        if num == '': num = 0
        elif num[-1] == '%': num = float(num[:-1]) * 0.01
        else: num = float(num)
      else: num = 1
      nums.append(num)
    for i in range(len(words)):
      if words[i] in wlist:
        for j in range(i+1,len(words)):
          if words[j] in wlist:
            for k in range(j+1,len(words)):
              if words[k] in wlist:
                comb = compose([words[i],words[j],words[k]])
                if comb in result: 
                  result[comb] = [pair[0] + pair[1] for pair in zip(result[comb],nums)]
                elif allindices == False: result[comb] = nums
  array = []
  for words in result.keys():
    array.append([words] + result[words])
  return sorted(array,key=sortkey,reverse=True)
# Calculate a total sum for every column in addcols and for every word quadruplet in wcol
def fourgrams(li,wcol,addcols,wlist=0,sortkey=lambda x:1):
  if wlist == 0: wlist = wordlist(li,wcol) 
  result = {}
  for i in range(len(li)):
    if i % int(len(li)/10) == (int(len(li)/10) - 1): print "Four grams: " + str(i) + " / " + str(len(li))
    words = [x.strip() for x in li[i][wcol].split(' ')]
    nums = []
    for ac in addcols:
      if ac >= 0:
        num = str(li[i][ac]).replace(',','').replace(' ','')
        if num == '': num = 0
        elif num[-1] == '%': num = float(num[:-1]) * 0.01
        else: num = float(num)
      else: num = 1
      nums.append(num)
    for i in range(len(words)):
      if words[i] in wlist:
        for j in range(i+1,len(words)):
          if words[j] in wlist:
            for k in range(j+1,len(words)):
              if words[j] in wlist:
                for l in range(k+1,len(words)):
                  if words[l] in wlist:
                    comb = compose([words[i],words[j],words[k],words[l]])
                    if comb in result:
                      result[comb] = [pair[0] + pair[1] for pair in zip(result[comb],nums)]
                    else: result[comb] = nums
                  
  array = []
  for words in result.keys():
    array.append([words] + result[words])
  return sorted(array,key=sortkey,reverse=True)
# Filters array, returning only the rows where column wcol of that row contains the query keywords (keywords can appear in any order)
# This and the other filters are useful for taking a list of entries and creating a list of only valid entries according to some validity characteristic
# eg:
# dog house, 15
# cat, 18
# dog, 33
# filter(li,0,'dog'):
# dog house, 15
# dog, 33
def filter(li,wcol,query):
  result = []
  for i in range(len(li)):
    words = [x.strip() for x in li[i][wcol].split(' ')]
    inlist = True
    queryarray = query.split(' ')
    if queryarray == ['']: queryarray = []
    for w in queryarray:
      if w not in words: inlist = False
    if queryarray == ['*']: inlist = len(li[i][wcol]) > 0
    if inlist: result.append(li[i])
  return result
# Filters array, requiring column wcol to exactly match query
def phrasefilter(li,wcol,query):
  result = []
  for i in range(len(li)):
    if li[i][wcol] == query: result.append(li[i])
  return result
# Filters array, requiring function func taken of the row to return True (or 1)
def funcfilter(li,func):
  result = []
  for i in range(len(li)):
    if func(li[i]): result.append(li[i])
  return result
# Adds up columns in addcols for a query matching keyfilter(li,wcol,query); can also be thought of as doing a single n-keyword match
# eg:
# dog, 25
# cat, 15
# dog, 75
# dog, 10
# horse, 55
# cat, 7
# search(li,0,[1],'dog') gives ['dog',110]
def search(li,wcol,addcols,query):
  result = [0] * len(addcols)
  for i in range(len(li)):
    words = [x.strip() for x in li[i][wcol].split(' ')]
    nums = []
    for ac in addcols:
      if ac >= 0:
        num = str(li[i][ac]).replace(',','').replace(' ','')
        if num == '': num = 0
        elif num[-1] == '%': num = float(num[:-1] * 0.01)
        else: num = float(num)
      else: num = 1
      nums.append(num)
    inlist = True
    queryarray = query.split(' ')
    if queryarray == ['']: queryarray = []
    for w in queryarray:
      if w not in words: inlist = False
    if queryarray == ['*']: inlist = len(li[i][wcol]) > 0
    if inlist:
      result = [pair[0] + pair[1] for pair in zip(result,nums)]
  return [query] + result
# Print a CSV from an array to stdout
def tochars(array,sp=','):
  string = ""
  for line in array: string += sp.join([str(x) for x in line]) + '\n'
  return string[:-1]
# Save an array to CSV
def save(f,array,sp=','):
  writeto = open(f,'w')
  writeto.write(tochars(array,sp))
  writeto.close()
# Compares keywords by two different parameters from two different lists. For example, li1 can be a list of how much money is spent (on addcol1) on a particular combination of keywords (on keycol1) and li2 can be a list of upgraded accounts with the search query they came from on keycol2, and addcol 2 can be left blank to default to -1 (each row is worth one point). Fourth column is statistical significance.
# Remember that you may have to filter the list yourself first
# Arguments:
# grams = 1 for single keywords, 2 for pairs, 3 for triplets and 4 for quadruplets
# li1, li2 = your two lists
# keycol1, keycol2 = where the keywords are located in those two lists
# addcol1, addcol2 = the columns of what you want to add up, eg. cost (set to -1 or leave blank to make it add 1 for each row)
# sortkey = function to sort results by (highest first)
# usestem = stem keywords
# sigtable = add ratio and significance to table
# invertratio = set ratio column to col1/col2 instead of col2/col1
# preformatted = li1 and li2 are already properly formatted
# justpreformat = convert li1 and li2 into twocolumns for comparison but don't go all the way
# wordlimit = limit search to some more common keywords for speedup purposes
# Example: list of customers, some upgraded, with originating keywords, and a list of how much you're paying for each search phrase
#
# customers.csv:
# Name, Keyword, Status
# Bob Jones, spreadsheet csv software, upgraded
# Matt Bones, csv python utils, free
# Army Drones, free spreadsheet, free
# Glenn Mitt, csv software, upgraded
# Pat Submitt, python utils software, upgraded
# Shawn Wit, python spreadsheet program, upgraded
#
# costs.csv:
# csv software, useless, and, irrelevant, data, 5.00, blah, blah
# python spreadsheet, useless, and, irrelevant, data, 2.50, blah, blah
# spreadsheet utils, useless, and, irrelevant, data, 10.00, blah, blah
# csv utils, useless, and, irrelevant, data, 1.50, blah, blah
# 
# Steps:
# 1. import spread (if not imported already)
# 2. upgrades = spread.filter(spread.load('customers.csv'),2,'upgraded')
# 3. costs = spread.load('costs.csv')
# 4. res = compare(1,costs,upgrades,0,1,5,invertratio=True)
# 5. spread.save('saved.csv',res)
#
# Res should look like:
#
# Keyword, Column 1, Column 2, Ratio, Significance
# spreadsheet, 12.50, 2, 6.25, -0.389
# utils, 11.50, 1, 11.50, -0.913
# csv, 7.50, 2, 3.75, 0.335
# python, 2.50, 2, 1.25, 2.031
#
# Or, if desired, you can:
# i1,i2 = compare(1,costs,upgrades,0,1,5,justpreformat=True)
# res1 = compare(1,i1,i2,0,1,5,invertratio=True,preformatted=True)
# res2 = compare(2,i1,i2,0,1,5,invertratio=True,preformatted=True)
# res3 = compare(3,i1,i2,0,1,5,invertratio=True,preformatted=True)
# res4 = compare(4,i1,i2,0,1,5,invertratio=True,preformatted=True)
#
# Note that significance is calculated based on col2/col1 regardless of invertratio, since getting 0 upgrades when you should have gotten 2 is not that unlikely, but calculating significance based on col1/col2 would give you infinity as infinity is infinitely far away from 0.5.
def compare(grams,li1,li2,keycol1,keycol2,addcol1=-1,addcol2=-1,sortkey=lambda x:x[1],usestem=True,sigtable=True,invertratio=False,preformatted=False,justpreformat=False,wordlimit=0):
  gramfuncs = [0,onegrams,twograms,threegrams,fourgrams]
  if preformatted == False:
    s1 = declutter(cols(li1,[keycol1,addcol1]),[1])
    print "Done decluttering/stemming: 1/4"
    s2 = declutter(cols(li2,[keycol2,addcol2]),[1])
    print "Done decluttering/stemming: 2/4"
    s1 = stem(s1,[0]) if usestem else declutter(s1,[0])
    print "Done decluttering/stemming: 3/4"
    s2 = stem(s2,[0]) if usestem else declutter(s2,[0])
    print "Done decluttering/stemming: 4/4"
  else: s1,s2 = li1,li2
  print "Printing sample of list 1"
  print s1[:10]
  print "Printing sample of list 2"
  print s2[:10]
  if justpreformat: return s1,s2
  while type(s1[0][1]) is str: s1.pop(0)
  while type(s2[0][1]) is str: s2.pop(0)
  print "Cleaned invalid rows"
  wl = sorted_wordlist(s1,0,[1])
  if wl.count('') > 0: blank = wl.pop(wl.index(''))
  print "Base wordlist length: " + str(len(wl)) + " ; Top ten: " + str(wl[:10])
  if wordlimit > 0 and wordlimit < len(wl):
    print "Shortening to " + str(wordlimit)
    wl = wl[:wordlimit]
  res1 = gramfuncs[grams](s1,0,[1],wl)
  print "Done search: 1/2"
  res2 = gramfuncs[grams](s2,0,[1],wl)
  print "Done search: 2/2"
  comb = sorted(splice(res1,res2,0,0),key=sortkey,reverse=True)
  if sigtable:
    tot1 = search(s1,0,[1],'')
    tot2 = search(s2,0,[1],'')
    ev = tot2[1]*1.0/tot1[1]
    print "Totals: " + str(tot1[1]) + ", " + str(tot2[1])
    for i in range(len(comb)): 
      comb[i].append(comb[i][2 - invertratio]*1.0/(comb[i][1 + invertratio] + 0.000001))
      comb[i].append((comb[i][2] - ev * comb[i][1])*1.0/(ev * comb[i][1] + 0.000001) ** 0.5)
    comb = [['Keyword','Column 1','Column 2','Ratio','Significance']] + comb
  else: comb = [['Keyword','Column 1','Column 2']] + comb
  print "Done"
  return comb